dsh-file-convert 0.4.0

package/lib/core/binaries/cache.js

@@ -0,0 +1,137 @@
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+/** Persistent per-user cache for downloaded external binaries. */
+export function cacheDir() {
+    return path.join(os.homedir(), '.dsh-file-convert', 'bin');
+}
+export function cachedBinaryPath(name) {
+    return path.join(cacheDir(), name + (process.platform === 'win32' ? '.exe' : ''));
+}
+async function exists(p) {
+    try {
+        await fs.access(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function platformKey() {
+    return process.platform === 'win32' ? 'win32' : process.platform === 'darwin' ? 'darwin' : 'linux';
+}
+/** Probe results are memoized per (dependency, path) for the process lifetime. */
+const probeCache = new Map();
+function probeDependency(dep, resolved) {
+    if (!dep.probe)
+        return Promise.resolve(true);
+    const key = `${dep.name}:${resolved}`;
+    let pending = probeCache.get(key);
+    if (!pending) {
+        pending = dep.probe(resolved).catch(() => false);
+        probeCache.set(key, pending);
+    }
+    return pending;
+}
+/**
+ * Locate the first existing entry. Patterns may carry `*` in ANY segment
+ * (e.g. `C:\Program Files\gs\gs*\bin\gswin64c.exe`); wildcard segments match
+ * one path level, preferring the highest-sorted match so newer versions win.
+ */
+async function firstExisting(paths) {
+    for (const candidate of paths) {
+        if (!candidate.includes('*')) {
+            if (await exists(candidate))
+                return candidate;
+            continue;
+        }
+        const expanded = await expandPattern(candidate);
+        for (const full of expanded) {
+            if (await exists(full))
+                return full;
+        }
+    }
+    return null;
+}
+function wildcardRegex(segment) {
+    return new RegExp('^' + segment.replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '[^\\\\/:]*') + '$', 'i');
+}
+/** Expand a path pattern segment by segment; [] when nothing matches. */
+export async function expandPattern(pattern) {
+    const absolute = /^[\\/]/.test(pattern);
+    const segments = pattern.split(/[\\/]+/).filter((seg, index) => !(index === 0 && seg === ''));
+    let current = [absolute ? path.sep : ''];
+    for (const seg of segments) {
+        if (!seg.includes('*')) {
+            current = current.map((prefix) => {
+                if (prefix === '')
+                    return seg;
+                // path.join('C:', 'x') drops the drive separator on Windows
+                if (/^[A-Za-z]:$/.test(prefix))
+                    return prefix + path.sep + seg;
+                return path.join(prefix, seg);
+            });
+            continue;
+        }
+        const regex = wildcardRegex(seg);
+        const next = [];
+        for (const prefix of current) {
+            let entries = [];
+            try {
+                entries = await fs.readdir(prefix === '' ? '.' : prefix);
+            }
+            catch {
+                continue;
+            }
+            const matches = entries
+                .filter((entry) => regex.test(entry))
+                .sort((a, b) => b.localeCompare(a, undefined, { numeric: true }));
+            for (const entry of matches)
+                next.push(path.join(prefix, entry));
+        }
+        if (next.length === 0)
+            return [];
+        current = next;
+    }
+    return current;
+}
+/**
+ * Resolution order: explicit config override → system PATH → known install
+ * locations (Windows soffice is rarely on PATH) → plugin cache. Every
+ * candidate must pass the dependency's probe when it declares one — a config
+ * override pointing at a python without pdf2docx counts as missing.
+ */
+export async function resolveBinaryCached(dep, overrides, which, logger) {
+    const accept = async (resolved) => {
+        if (!(await probeDependency(dep, resolved))) {
+            logger.debug(`binary ${dep.name}: ${resolved} resolved but failed its probe`);
+            return null;
+        }
+        logger.debug(`binary ${dep.name}: using ${resolved}`);
+        return resolved;
+    };
+    const override = dep.configKey ? overrides[dep.configKey] : undefined;
+    if (override && (await exists(override)))
+        return accept(override);
+    for (const command of dep.commands) {
+        const found = await which(command);
+        if (found) {
+            const accepted = await accept(found);
+            if (accepted)
+                return accepted;
+        }
+    }
+    const located = await firstExisting(dep.extraPaths?.[platformKey()] ?? []);
+    if (located) {
+        const accepted = await accept(located);
+        if (accepted)
+            return accepted;
+    }
+    const cached = cachedBinaryPath(dep.name);
+    if (await exists(cached)) {
+        const accepted = await accept(cached);
+        if (accepted)
+            return accepted;
+    }
+    return null;
+}

package/lib/core/binaries/download.d.ts

@@ -0,0 +1,21 @@
+import type { BinaryDependency } from '../types.js';
+export declare class DownloadError extends Error {
+    readonly code: 'unsupported_platform' | 'integrity' | 'network' | 'verify_failed';
+    constructor(code: 'unsupported_platform' | 'integrity' | 'network' | 'verify_failed', message: string);
+}
+export interface DownloadOutcome {
+    path: string;
+    bytes: number;
+    /** First line of `<binary> -version`, e.g. "ffmpeg version 6.1.1 ...". */
+    versionLine: string;
+}
+/**
+ * Download one external binary into the plugin cache. Mirrors are tried in
+ * order; the gzip is sha256-verified before decompression, and the binary is
+ * proven by actually running `-version` once before success is reported.
+ */
+export declare function downloadBinary(dep: BinaryDependency, opts: {
+    timeoutMs: number;
+    signal?: AbortSignal;
+    force?: boolean;
+}): Promise<DownloadOutcome>;

package/lib/core/binaries/download.js

@@ -0,0 +1,140 @@
+import { createHash, timingSafeEqual } from 'node:crypto';
+import { gunzipSync } from 'node:zlib';
+import fs from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { createWriteStream } from 'node:fs';
+import { execTool } from '../utils/exec.js';
+import { cachedBinaryPath } from './cache.js';
+/**
+ * Where each external binary comes from when the machine has none: pinned
+ * single-binary FFmpeg 6.1.1 builds from the ffmpeg-static project, served by
+ * the npmmirror binary CDN first (fast and reachable where GitHub-style hosts
+ * may not be) with the GitHub release as fallback. Both mirrors serve
+ * byte-identical files validated against the pinned sha256 below. Bump the
+ * version deliberately and re-pin the hashes; never resolve to "latest".
+ */
+const FFMPEG_STATIC_VERSION = 'b6.1.1'; // FFmpeg 6.1.1
+const FFMPEG_STATIC_MIRRORS = [
+    (asset) => `https://registry.npmmirror.com/-/binary/ffmpeg-static/${FFMPEG_STATIC_VERSION}/${asset}.gz`,
+    (asset) => `https://github.com/eugeneware/ffmpeg-static/releases/download/${FFMPEG_STATIC_VERSION}/${asset}.gz`,
+];
+/** asset name (without .gz) and its pinned sha256, per binary per platform.
+ * Hashes were computed from the npmmirror-mirrored b6.1.1 assets; the GitHub
+ * release fallback serves byte-identical files. */
+const STATIC_BINS = {
+    'ffmpeg:win32-x64': {
+        asset: 'ffmpeg-win32-x64',
+        sha256: '8883a3dffbd0a16cf4ef95206ea05283f78908dbfb118f73c83f4951dcc06d77',
+    },
+    'ffprobe:win32-x64': {
+        asset: 'ffprobe-win32-x64',
+        sha256: 'f309e6223ad89d2fe54bccd420a7709b66fd27540674e92309578ed491a43c8d',
+    },
+    'ffmpeg:darwin-arm64': {
+        asset: 'ffmpeg-darwin-arm64',
+        sha256: '8923876afa8db5585022d7860ec7e589af192f441c56793971276d450ed3bbfa',
+    },
+    'ffprobe:darwin-arm64': {
+        asset: 'ffprobe-darwin-arm64',
+        sha256: 'd986a8ec7b030899fe66a8a288ed809a3543338705a3ce178cfb85869c5d80be',
+    },
+    'ffmpeg:darwin-x64': {
+        asset: 'ffmpeg-darwin-x64',
+        sha256: '929b375c1182d956c51f7ac25e0b2b0411fb01f6f407aa15c9758efeb4242106',
+    },
+    'ffprobe:darwin-x64': {
+        asset: 'ffprobe-darwin-x64',
+        sha256: 'd4da574d6e2e197bd259b47d69cf262df9e312af24ad960444f6d806d3d4c186',
+    },
+    'ffmpeg:linux-x64': {
+        asset: 'ffmpeg-linux-x64',
+        sha256: 'bfe8a8fc511530457b528c48d77b5737527b504a3797a9bc4866aeca69c2dffa',
+    },
+    'ffprobe:linux-x64': {
+        asset: 'ffprobe-linux-x64',
+        sha256: '25d9b6ccb05e3d9de9e04e31e2506d8dd7f9f0418981965ac6df12e8d3afd067',
+    },
+};
+export class DownloadError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.code = code;
+    }
+}
+/**
+ * Download one external binary into the plugin cache. Mirrors are tried in
+ * order; the gzip is sha256-verified before decompression, and the binary is
+ * proven by actually running `-version` once before success is reported.
+ */
+export async function downloadBinary(dep, opts) {
+    const target = cachedBinaryPath(dep.name);
+    if (!opts.force && (await fileExists(target))) {
+        return { path: target, bytes: (await fs.stat(target)).size, versionLine: 'already cached' };
+    }
+    const platformKey = `${process.platform}-${process.arch}`;
+    const entry = STATIC_BINS[`${dep.name}:${platformKey}`];
+    if (!entry) {
+        throw new DownloadError('unsupported_platform', `No prebuilt ${dep.name} for ${platformKey}; install it manually.`);
+    }
+    const errors = [];
+    for (const mirror of FFMPEG_STATIC_MIRRORS) {
+        const url = mirror(entry.asset);
+        const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dsh-file-convert-dl-'));
+        try {
+            const gzPath = path.join(tmpDir, 'bin.gz');
+            const bytes = await downloadTo(url, gzPath, opts.timeoutMs, opts.signal);
+            verifyDigest(await fs.readFile(gzPath), entry.sha256);
+            await fs.mkdir(path.dirname(target), { recursive: true });
+            await fs.writeFile(target, gunzipSync(await fs.readFile(gzPath)));
+            if (process.platform !== 'win32')
+                await fs.chmod(target, 0o755);
+            const { stdout, stderr } = await execTool(target, ['-version'], { timeoutMs: 15_000, signal: opts.signal });
+            const versionLine = (stdout || stderr).split(/\r?\n/, 1)[0]?.trim() ?? '';
+            return { path: target, bytes, versionLine };
+        }
+        catch (err) {
+            errors.push(`${new URL(url).host}: ${err instanceof Error ? err.message : String(err)}`);
+            // either mirror may be unreachable; the pinned digest protects us either way
+        }
+        finally {
+            await fs.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
+        }
+    }
+    throw new DownloadError('network', `All mirrors failed for ${dep.name}: ${errors.join(' | ')}`);
+}
+async function downloadTo(url, target, timeoutMs, signal) {
+    let res;
+    try {
+        res = await fetch(url, { redirect: 'follow', signal: AbortSignal.any([AbortSignal.timeout(timeoutMs), ...(signal ? [signal] : [])]) });
+    }
+    catch (err) {
+        throw new DownloadError('network', `Download failed (${url}): ${err instanceof Error ? err.message : String(err)}`);
+    }
+    if (!res.ok || !res.body) {
+        throw new DownloadError('network', `Download returned HTTP ${res.status} for ${url}`);
+    }
+    await pipeline(Readable.fromWeb(res.body), createWriteStream(target));
+    return (await fs.stat(target)).size;
+}
+function verifyDigest(data, expectedSha256Hex) {
+    const actual = createHash('sha256').update(data).digest('hex').toLowerCase();
+    const wanted = expectedSha256Hex.toLowerCase();
+    const a = Buffer.from(actual);
+    const b = Buffer.from(wanted);
+    if (a.length !== b.length || !timingSafeEqual(a, b)) {
+        throw new DownloadError('integrity', `Checksum mismatch: downloaded binary does not match its pinned sha256.`);
+    }
+}
+async function fileExists(p) {
+    try {
+        await fs.access(p);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/lib/core/binary.d.ts

@@ -0,0 +1,7 @@
+import type { BinaryDependency, Logger } from './types.js';
+/**
+ * Resolve an external binary: plugin config override → system PATH → plugin
+ * cache (populated by install_media_dependencies). No auto-install happens
+ * here — downloading is an explicit, user-approved tool call.
+ */
+export declare function resolveBinary(dep: BinaryDependency, overrides: Record<string, string | undefined>, logger: Logger): Promise<string | null>;

package/lib/core/binary.js

@@ -0,0 +1,29 @@
+import { resolveBinaryCached } from './binaries/cache.js';
+/**
+ * Resolve an external binary: plugin config override → system PATH → plugin
+ * cache (populated by install_media_dependencies). No auto-install happens
+ * here — downloading is an explicit, user-approved tool call.
+ */
+export async function resolveBinary(dep, overrides, logger) {
+    return resolveBinaryCached(dep, overrides, which, logger);
+}
+const whichCache = new Map();
+async function which(command) {
+    const cached = whichCache.get(command);
+    if (cached !== undefined)
+        return cached;
+    let result = null;
+    try {
+        const { execFile } = await import('node:child_process');
+        const { promisify } = await import('node:util');
+        const run = promisify(execFile);
+        const cmd = process.platform === 'win32' ? 'where' : 'which';
+        const { stdout } = await run(cmd, [command], { timeout: 5000 });
+        result = stdout.split(/\r?\n/)[0]?.trim() || null;
+    }
+    catch {
+        result = null;
+    }
+    whichCache.set(command, result);
+    return result;
+}

package/lib/core/converters/data.d.ts

@@ -0,0 +1,14 @@
+import type { ConvertContext, ConvertRequest, ConvertResult, Converter, ConversionCapability } from '../types.js';
+/**
+ * JSON / YAML / CSV share one intermediate representation (a plain JS value):
+ * three readers + three writers cover every pair, including YAML↔CSV.
+ */
+export declare class DataConverter implements Converter {
+    readonly id = "data";
+    readonly concurrency = 4;
+    readonly binaryDeps: never[];
+    readonly capabilities: ConversionCapability[];
+    convert(req: ConvertRequest, ctx: ConvertContext): Promise<ConvertResult>;
+    private read;
+    private write;
+}

package/lib/core/converters/data.js

@@ -0,0 +1,145 @@
+import fs from 'node:fs/promises';
+import { parse as csvParse } from 'csv-parse/sync';
+import { stringify as csvStringify } from 'csv-stringify/sync';
+import yaml from 'js-yaml';
+import { convertError } from '../errors.js';
+/**
+ * JSON / YAML / CSV share one intermediate representation (a plain JS value):
+ * three readers + three writers cover every pair, including YAML↔CSV.
+ */
+export class DataConverter {
+    id = 'data';
+    concurrency = 4;
+    binaryDeps = [];
+    capabilities = [
+        { from: 'json', to: 'yaml' },
+        { from: 'yaml', to: 'json' },
+        { from: 'json', to: 'csv' },
+        { from: 'csv', to: 'json' },
+        { from: 'yaml', to: 'csv' },
+        { from: 'csv', to: 'yaml' },
+    ];
+    async convert(req, ctx) {
+        const started = Date.now();
+        const from = req.from;
+        const to = req.to;
+        const readable = from === 'json' || from === 'yaml' || from === 'csv';
+        const writable = to === 'json' || to === 'yaml' || to === 'csv';
+        if (!readable || !writable) {
+            return {
+                ok: false,
+                input: req.input,
+                from,
+                to,
+                error: convertError('unsupported_conversion', `DataConverter handles json/yaml/csv only, got ${from} -> ${to}.`),
+            };
+        }
+        try {
+            const raw = await fs.readFile(req.input, 'utf8');
+            const bytesIn = Buffer.byteLength(raw);
+            const value = this.read(from, raw, req.options.delimiter);
+            const text = this.write(to, value, req.options.indent);
+            await fs.writeFile(req.output, text, 'utf8');
+            return {
+                ok: true,
+                input: req.input,
+                output: req.output,
+                from: req.from,
+                to: req.to,
+                bytesIn,
+                bytesOut: Buffer.byteLength(text),
+                durationMs: Date.now() - started,
+                warnings: [],
+            };
+        }
+        catch (err) {
+            // Reader/parser problems are invalid_input; everything else is backend failure.
+            const code = err instanceof ParseError ? 'invalid_input' : 'conversion_failed';
+            return {
+                ok: false,
+                input: req.input,
+                from: req.from,
+                to: req.to,
+                error: convertError(code, `Failed to convert ${req.from} → ${req.to}`, {
+                    detail: err instanceof Error ? err.message : String(err),
+                }),
+            };
+        }
+    }
+    read(from, raw, delimiter) {
+        const text = raw.replace(/^\uFEFF/, ''); // strip UTF-8 BOM (Excel exports)
+        try {
+            if (from === 'json')
+                return JSON.parse(text);
+            if (from === 'yaml')
+                return yaml.load(text);
+            return csvParse(text, {
+                bom: true,
+                columns: true,
+                skip_empty_lines: true,
+                delimiter: delimiter ?? sniffDelimiter(text),
+            });
+        }
+        catch (err) {
+            throw new ParseError(err instanceof Error ? err.message : String(err));
+        }
+    }
+    write(to, value, indent) {
+        if (to === 'json')
+            return JSON.stringify(value, null, indent ?? 2) + '\n';
+        if (to === 'yaml') {
+            return yaml.dump(value, { indent: indent ?? 2, lineWidth: 1000, noRefs: true });
+        }
+        const records = toRecords(value);
+        return csvStringify(records, { header: true });
+    }
+}
+class ParseError extends Error {
+}
+/** CSV needs an array of flat objects; anything else is normalized into one. */
+function toRecords(value) {
+    if (Array.isArray(value)) {
+        return value.map((item) => (isPlainItem(item) ? flattenRecord(item) : { value: item }));
+    }
+    if (value !== null && typeof value === 'object') {
+        return [flattenRecord(value)];
+    }
+    return [{ value }];
+}
+function isPlainItem(item) {
+    return item !== null && typeof item === 'object' && !Array.isArray(item);
+}
+/** Nested objects become dot-notation columns; arrays are JSON-encoded cells. */
+function flattenRecord(record, prefix = '') {
+    const out = {};
+    for (const [key, val] of Object.entries(record)) {
+        const column = prefix ? `${prefix}.${key}` : key;
+        if (isPlainItem(val)) {
+            Object.assign(out, flattenRecord(val, column));
+        }
+        else if (Array.isArray(val)) {
+            out[column] = JSON.stringify(val);
+        }
+        else {
+            out[column] = val;
+        }
+    }
+    return out;
+}
+/**
+ * Pick the dominant delimiter among , ; \t from the first two lines.
+ * Semicolon-separated exports are common in European Excel locales.
+ */
+function sniffDelimiter(text) {
+    const head = text.split(/\r?\n/, 2).join('\n');
+    let best = ',';
+    let bestCount = 0;
+    for (const candidate of [',', ';', '\t']) {
+        const count = head.split(candidate).length - 1;
+        if (count > bestCount) {
+            best = candidate;
+            bestCount = count;
+        }
+    }
+    return best;
+}

package/lib/core/converters/image.d.ts

@@ -0,0 +1,12 @@
+import type { ConvertContext, ConvertRequest, ConvertResult, Converter, ConversionCapability } from '../types.js';
+/**
+ * All raster↔raster pairs plus SVG rasterization, backed by sharp/libvips
+ * (prebuilt npm binaries - no system dependencies).
+ */
+export declare class ImageConverter implements Converter {
+    readonly id = "image";
+    readonly concurrency = 4;
+    readonly binaryDeps: never[];
+    readonly capabilities: ConversionCapability[];
+    convert(req: ConvertRequest, ctx: ConvertContext): Promise<ConvertResult>;
+}

package/lib/core/converters/image.js

@@ -0,0 +1,64 @@
+import fs from 'node:fs/promises';
+import sharp from 'sharp';
+import { convertError } from '../errors.js';
+const IMAGE_INPUTS = ['png', 'jpg', 'webp', 'svg'];
+const IMAGE_OUTPUTS = ['png', 'jpg', 'webp'];
+/**
+ * All raster↔raster pairs plus SVG rasterization, backed by sharp/libvips
+ * (prebuilt npm binaries - no system dependencies).
+ */
+export class ImageConverter {
+    id = 'image';
+    concurrency = 4;
+    binaryDeps = [];
+    capabilities = IMAGE_INPUTS.flatMap((from) => IMAGE_OUTPUTS.filter((to) => to !== from).map((to) => ({ from, to })));
+    async convert(req, ctx) {
+        const started = Date.now();
+        if (ctx.signal?.aborted) {
+            return { ok: false, input: req.input, from: req.from, to: req.to, error: convertError('cancelled', 'Conversion cancelled.') };
+        }
+        try {
+            const bytesIn = (await fs.stat(req.input)).size;
+            // density only affects vector input (SVG): 72 renders at the SVG's own
+            // pixel size; a higher dpi option upscales. Rasters ignore it.
+            const pipeline = sharp(req.input, { density: req.options.dpi ?? 72 });
+            const { quality, background } = req.options;
+            let out;
+            if (req.to === 'jpg') {
+                out = pipeline
+                    .flatten({ background: background ?? '#ffffff' }) // jpg has no alpha channel
+                    .jpeg({ quality: quality ?? 85 });
+            }
+            else if (req.to === 'webp') {
+                out = pipeline.webp({ quality: quality ?? 85 });
+            }
+            else {
+                out = pipeline.png();
+            }
+            await out.toFile(req.output);
+            const bytesOut = (await fs.stat(req.output)).size;
+            return {
+                ok: true,
+                input: req.input,
+                output: req.output,
+                from: req.from,
+                to: req.to,
+                bytesIn,
+                bytesOut,
+                durationMs: Date.now() - started,
+                warnings: [],
+            };
+        }
+        catch (err) {
+            return {
+                ok: false,
+                input: req.input,
+                from: req.from,
+                to: req.to,
+                error: convertError('conversion_failed', `Failed to convert ${req.from} → ${req.to}`, {
+                    detail: err instanceof Error ? err.message : String(err),
+                }),
+            };
+        }
+    }
+}

package/lib/core/converters/media.d.ts

@@ -0,0 +1,21 @@
+import type { BinaryDependency, ConvertContext, ConvertRequest, ConvertResult, Converter, ConversionCapability } from '../types.js';
+export declare const FFMPEG: BinaryDependency;
+export declare const FFPROBE: BinaryDependency;
+/**
+ * Audio/video conversions, delegated to a locally installed FFmpeg.
+ * The converter declares its dependencies; the router refuses to run (and
+ * list_conversions reports what is missing) until the binaries resolve.
+ */
+export declare class MediaConverter implements Converter {
+    private readonly resolve;
+    readonly id = "media";
+    readonly concurrency = 2;
+    readonly binaryDeps: BinaryDependency[];
+    readonly capabilities: ConversionCapability[];
+    constructor(resolve: (dep: BinaryDependency) => Promise<string | null>);
+    convert(req: ConvertRequest, ctx: ConvertContext): Promise<ConvertResult>;
+    /** MOV→MP4: try a lossless container swap first, fall back to re-encoding. */
+    private movToMp4;
+    private required;
+    private execOpts;
+}