dsh-file-convert 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +225 -0
- package/README.zh-CN.md +203 -0
- package/cordis.patch.yml +3 -0
- package/lib/config.d.ts +36 -0
- package/lib/config.js +17 -0
- package/lib/core/binaries/cache.d.ts +13 -0
- package/lib/core/binaries/cache.js +137 -0
- package/lib/core/binaries/download.d.ts +21 -0
- package/lib/core/binaries/download.js +140 -0
- package/lib/core/binary.d.ts +7 -0
- package/lib/core/binary.js +29 -0
- package/lib/core/converters/data.d.ts +14 -0
- package/lib/core/converters/data.js +145 -0
- package/lib/core/converters/image.d.ts +12 -0
- package/lib/core/converters/image.js +64 -0
- package/lib/core/converters/media.d.ts +21 -0
- package/lib/core/converters/media.js +135 -0
- package/lib/core/converters/office.d.ts +33 -0
- package/lib/core/converters/office.js +206 -0
- package/lib/core/converters/pdf-env.d.ts +1 -0
- package/lib/core/converters/pdf-env.js +8 -0
- package/lib/core/converters/pdf.d.ts +22 -0
- package/lib/core/converters/pdf.js +316 -0
- package/lib/core/detect.d.ts +16 -0
- package/lib/core/detect.js +121 -0
- package/lib/core/errors.d.ts +5 -0
- package/lib/core/errors.js +16 -0
- package/lib/core/formats.d.ts +11 -0
- package/lib/core/formats.js +60 -0
- package/lib/core/index.d.ts +25 -0
- package/lib/core/index.js +52 -0
- package/lib/core/inspect.d.ts +13 -0
- package/lib/core/inspect.js +134 -0
- package/lib/core/ocr.d.ts +38 -0
- package/lib/core/ocr.js +152 -0
- package/lib/core/optimizers.d.ts +29 -0
- package/lib/core/optimizers.js +275 -0
- package/lib/core/paths.d.ts +11 -0
- package/lib/core/paths.js +19 -0
- package/lib/core/router.d.ts +64 -0
- package/lib/core/router.js +288 -0
- package/lib/core/types.d.ts +199 -0
- package/lib/core/types.js +8 -0
- package/lib/core/utils/exec.d.ts +38 -0
- package/lib/core/utils/exec.js +89 -0
- package/lib/core/utils/pages.d.ts +12 -0
- package/lib/core/utils/pages.js +46 -0
- package/lib/format.d.ts +23 -0
- package/lib/format.js +84 -0
- package/lib/index.d.ts +6 -0
- package/lib/index.js +51 -0
- package/lib/tools/batch-convert.d.ts +5 -0
- package/lib/tools/batch-convert.js +155 -0
- package/lib/tools/convert-file.d.ts +3 -0
- package/lib/tools/convert-file.js +57 -0
- package/lib/tools/inspect-file.d.ts +3 -0
- package/lib/tools/inspect-file.js +29 -0
- package/lib/tools/install-media.d.ts +9 -0
- package/lib/tools/install-media.js +55 -0
- package/lib/tools/install-ocr.d.ts +7 -0
- package/lib/tools/install-ocr.js +43 -0
- package/lib/tools/list-conversions.d.ts +2 -0
- package/lib/tools/list-conversions.js +18 -0
- package/lib/tools/optimize-file.d.ts +3 -0
- package/lib/tools/optimize-file.js +91 -0
- package/package.json +66 -0
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { pathToFileURL } from 'node:url';
|
|
5
|
+
import { createRequire } from 'node:module';
|
|
6
|
+
import './pdf-env.js';
|
|
7
|
+
import { createCanvas } from '@napi-rs/canvas';
|
|
8
|
+
import { getDocument } from 'pdfjs-dist/legacy/build/pdf.mjs';
|
|
9
|
+
import { convertError } from '../errors.js';
|
|
10
|
+
import { PageRangeError, parsePageRange } from '../utils/pages.js';
|
|
11
|
+
import { OcrLanguageMissingError, OCR_LANGUAGE_DATA, TESSERACT, ocrLanguagesCached, resolveOcrEngine } from '../ocr.js';
|
|
12
|
+
const require = createRequire(import.meta.url);
|
|
13
|
+
/**
|
|
14
|
+
* Locate pdfjs-dist's standard_fonts directory so PDFs using the base-14
|
|
15
|
+
* (and similar) fonts render with correct glyphs. Best effort: rendering
|
|
16
|
+
* still works without it, some glyph substitutions may occur.
|
|
17
|
+
*/
|
|
18
|
+
function standardFontDataUrl() {
|
|
19
|
+
for (const target of ['pdfjs-dist/package.json', 'pdfjs-dist/build/pdf.mjs', 'pdfjs-dist']) {
|
|
20
|
+
let resolved;
|
|
21
|
+
try {
|
|
22
|
+
resolved = require.resolve(target);
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
let dir = path.dirname(resolved);
|
|
28
|
+
for (let i = 0; i < 5; i++) {
|
|
29
|
+
try {
|
|
30
|
+
const pkg = JSON.parse(readFileSync(path.join(dir, 'package.json'), 'utf8'));
|
|
31
|
+
if (pkg?.name === 'pdfjs-dist') {
|
|
32
|
+
return pathToFileURL(path.join(dir, 'standard_fonts')).href + '/';
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
/* keep walking up */
|
|
37
|
+
}
|
|
38
|
+
const parent = path.dirname(dir);
|
|
39
|
+
if (parent === dir)
|
|
40
|
+
break;
|
|
41
|
+
dir = parent;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* PDF conversions, fully local: rasterization via pdfjs-dist + @napi-rs/canvas
|
|
48
|
+
* (prebuilt npm binaries), text extraction via pdfjs-dist. No Poppler needed.
|
|
49
|
+
*/
|
|
50
|
+
export class PdfConverter {
|
|
51
|
+
resolve;
|
|
52
|
+
id = 'pdf';
|
|
53
|
+
concurrency = 2;
|
|
54
|
+
binaryDeps = [];
|
|
55
|
+
capabilities = [
|
|
56
|
+
{ from: 'pdf', to: 'png' },
|
|
57
|
+
{ from: 'pdf', to: 'jpg' },
|
|
58
|
+
{ from: 'pdf', to: 'txt' },
|
|
59
|
+
];
|
|
60
|
+
/** Optional binary resolver, needed only for OCR (Tesseract). */
|
|
61
|
+
constructor(resolve) {
|
|
62
|
+
this.resolve = resolve;
|
|
63
|
+
}
|
|
64
|
+
async convert(req, ctx) {
|
|
65
|
+
const started = Date.now();
|
|
66
|
+
try {
|
|
67
|
+
const bytesIn = (await fs.stat(req.input)).size;
|
|
68
|
+
const data = new Uint8Array(await fs.readFile(req.input));
|
|
69
|
+
const doc = (await getDocument({
|
|
70
|
+
data,
|
|
71
|
+
standardFontDataUrl: standardFontDataUrl(),
|
|
72
|
+
verbosity: 0,
|
|
73
|
+
}).promise);
|
|
74
|
+
try {
|
|
75
|
+
return req.to === 'txt'
|
|
76
|
+
? await this.toText(doc, req, ctx, bytesIn, started)
|
|
77
|
+
: await this.toImages(doc, req, ctx, bytesIn, started);
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
await doc.cleanup().catch(() => undefined);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
catch (err) {
|
|
84
|
+
if (err instanceof PageRangeError) {
|
|
85
|
+
return { ok: false, input: req.input, from: req.from, to: req.to, error: err.error };
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
ok: false,
|
|
89
|
+
input: req.input,
|
|
90
|
+
from: req.from,
|
|
91
|
+
to: req.to,
|
|
92
|
+
error: convertError('conversion_failed', `Failed to convert ${req.from} → ${req.to}`, {
|
|
93
|
+
detail: err instanceof Error ? err.message : String(err),
|
|
94
|
+
}),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Multi-page PDFs produce `<base>-<n>.<ext>` files named after the REAL page
|
|
100
|
+
* number; converting a single page (or a one-page PDF) writes exactly `output`.
|
|
101
|
+
*/
|
|
102
|
+
async toImages(doc, req, ctx, bytesIn, started) {
|
|
103
|
+
const totalPages = doc.numPages;
|
|
104
|
+
const explicitSelection = Boolean(req.options.pages);
|
|
105
|
+
const selected = req.options.pages ? parsePageRange(req.options.pages, totalPages) : pageRange(totalPages);
|
|
106
|
+
const maxPdfPages = ctx.limits?.maxPdfPages;
|
|
107
|
+
if (!explicitSelection && maxPdfPages && totalPages > maxPdfPages) {
|
|
108
|
+
return failErr(req, convertError('invalid_input', `PDF has ${totalPages} pages, above the ${maxPdfPages}-page rasterization limit.`, {
|
|
109
|
+
hint: `Use pages (e.g. '1-${maxPdfPages}') to select, or raise 'maxPdfPages' in the plugin config.`,
|
|
110
|
+
}));
|
|
111
|
+
}
|
|
112
|
+
const scale = (req.options.dpi ?? 150) / 72; // PDFs have no intrinsic pixel size; 150 is a sane default.
|
|
113
|
+
const maxOutputPixels = ctx.limits?.maxOutputPixels;
|
|
114
|
+
let scaleReduced = false;
|
|
115
|
+
const pad = String(totalPages).length;
|
|
116
|
+
const outputs = [];
|
|
117
|
+
const warnings = [];
|
|
118
|
+
let bytesOut = 0;
|
|
119
|
+
for (const n of selected) {
|
|
120
|
+
if (ctx.signal?.aborted) {
|
|
121
|
+
return fail(req, 'cancelled', 'Conversion cancelled.');
|
|
122
|
+
}
|
|
123
|
+
const page = await doc.getPage(n);
|
|
124
|
+
try {
|
|
125
|
+
let renderScale = scale;
|
|
126
|
+
if (maxOutputPixels) {
|
|
127
|
+
const viewport = page.getViewport({ scale });
|
|
128
|
+
const pixels = viewport.width * viewport.height;
|
|
129
|
+
if (pixels > maxOutputPixels) {
|
|
130
|
+
renderScale = scale * Math.sqrt(maxOutputPixels / pixels);
|
|
131
|
+
// ceil() on the viewport dims can nudge us just over the budget
|
|
132
|
+
const cw = Math.ceil(viewport.width * (renderScale / scale));
|
|
133
|
+
const ch = Math.ceil(viewport.height * (renderScale / scale));
|
|
134
|
+
if (cw * ch > maxOutputPixels)
|
|
135
|
+
renderScale *= maxOutputPixels / (cw * ch);
|
|
136
|
+
scaleReduced = true;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
const buffer = await renderPage(page, renderScale, {
|
|
140
|
+
background: req.to === 'jpg' ? req.options.background ?? '#ffffff' : undefined,
|
|
141
|
+
quality: req.options.quality ?? 85,
|
|
142
|
+
to: req.to,
|
|
143
|
+
});
|
|
144
|
+
const out = selected.length === 1 ? req.output : withPageNumber(req.output, n, pad);
|
|
145
|
+
await fs.writeFile(out, buffer);
|
|
146
|
+
bytesOut += buffer.byteLength;
|
|
147
|
+
outputs.push(out);
|
|
148
|
+
}
|
|
149
|
+
finally {
|
|
150
|
+
page.cleanup();
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
if (selected.length < totalPages) {
|
|
154
|
+
warnings.push(`Converted page(s) ${selected.join(', ')} of ${totalPages} (pages option).`);
|
|
155
|
+
}
|
|
156
|
+
else if (totalPages > 1) {
|
|
157
|
+
warnings.push(`PDF has ${totalPages} pages; wrote ${totalPages} files named <name>-<page>.${req.to}.`);
|
|
158
|
+
}
|
|
159
|
+
if (scaleReduced) {
|
|
160
|
+
warnings.push(`Raster scale was reduced on some pages to fit the pixel budget (maxOutputPixels).`);
|
|
161
|
+
}
|
|
162
|
+
return {
|
|
163
|
+
ok: true,
|
|
164
|
+
input: req.input,
|
|
165
|
+
output: outputs[0] ?? req.output,
|
|
166
|
+
outputs: outputs.length > 1 ? outputs : undefined,
|
|
167
|
+
from: req.from,
|
|
168
|
+
to: req.to,
|
|
169
|
+
bytesIn,
|
|
170
|
+
bytesOut,
|
|
171
|
+
durationMs: Date.now() - started,
|
|
172
|
+
warnings,
|
|
173
|
+
};
|
|
174
|
+
}
|
|
175
|
+
async toText(doc, req, ctx, bytesIn, started) {
|
|
176
|
+
const totalPages = doc.numPages;
|
|
177
|
+
const selected = req.options.pages ? parsePageRange(req.options.pages, totalPages) : pageRange(totalPages);
|
|
178
|
+
const warnings = [];
|
|
179
|
+
const pageTexts = [];
|
|
180
|
+
for (const n of selected) {
|
|
181
|
+
if (ctx.signal?.aborted) {
|
|
182
|
+
return fail(req, 'cancelled', 'Conversion cancelled.');
|
|
183
|
+
}
|
|
184
|
+
const page = await doc.getPage(n);
|
|
185
|
+
try {
|
|
186
|
+
const content = await page.getTextContent();
|
|
187
|
+
pageTexts.push(itemsToText(content.items));
|
|
188
|
+
}
|
|
189
|
+
finally {
|
|
190
|
+
page.cleanup();
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
if (req.options.ocr === true) {
|
|
194
|
+
if (!this.resolve) {
|
|
195
|
+
return failErr(req, convertError('conversion_failed', 'OCR is unavailable in this build (no binary resolver).'));
|
|
196
|
+
}
|
|
197
|
+
const engine = await resolveOcrEngine(this.resolve, ctx.logger);
|
|
198
|
+
if (!engine) {
|
|
199
|
+
return failErr(req, convertError('missing_dependency', 'OCR was requested but no OCR engine is available.', {
|
|
200
|
+
missing: [TESSERACT],
|
|
201
|
+
hint: `Install hint (${process.platform}): ${TESSERACT.installHint[platformKey()]} - or reinstall the plugin so its bundled tesseract.js fallback is present.`,
|
|
202
|
+
}));
|
|
203
|
+
}
|
|
204
|
+
const ocrLang = req.options.ocrLang ?? 'chi_sim+eng';
|
|
205
|
+
// The bundled engine needs language data that is NOT downloaded
|
|
206
|
+
// implicitly: without a cached pack we fail with guidance instead.
|
|
207
|
+
if (engine.name === 'tesseract.js' && !(await ocrLanguagesCached(ocrLang))) {
|
|
208
|
+
return failErr(req, convertError('missing_dependency', `OCR language data for '${ocrLang}' is not cached yet.`, {
|
|
209
|
+
missing: [OCR_LANGUAGE_DATA],
|
|
210
|
+
hint: 'Ask the agent to run install_ocr_dependencies (downloads about 10-30 MB per language into the plugin cache), or install a local Tesseract CLI and set tesseractPath if needed.',
|
|
211
|
+
}));
|
|
212
|
+
}
|
|
213
|
+
if (pageTexts.some((t) => t.trim().length > 0)) {
|
|
214
|
+
warnings.push('A text layer was detected; OCR was used anyway because ocr: true.');
|
|
215
|
+
}
|
|
216
|
+
// OCR needs legible pixels: default to a higher density than rasterization.
|
|
217
|
+
const scale = Math.max((req.options.dpi ?? 200) / 72, 200 / 72);
|
|
218
|
+
const ocrTexts = [];
|
|
219
|
+
for (const n of selected) {
|
|
220
|
+
if (ctx.signal?.aborted) {
|
|
221
|
+
return fail(req, 'cancelled', 'Conversion cancelled.');
|
|
222
|
+
}
|
|
223
|
+
const page = await doc.getPage(n);
|
|
224
|
+
try {
|
|
225
|
+
const png = await renderPage(page, scale, { to: 'png', quality: 100 });
|
|
226
|
+
ocrTexts.push((await engine.recognizePng(png, ocrLang, ctx)).trim());
|
|
227
|
+
}
|
|
228
|
+
catch (err) {
|
|
229
|
+
if (err instanceof OcrLanguageMissingError) {
|
|
230
|
+
return failErr(req, convertError('missing_dependency', `The local Tesseract is missing language data for '${ocrLang}'.`, {
|
|
231
|
+
hint: `Install the '${ocrLang}' traineddata for your Tesseract, or pick an ocr_lang it provides.`,
|
|
232
|
+
}));
|
|
233
|
+
}
|
|
234
|
+
throw err;
|
|
235
|
+
}
|
|
236
|
+
finally {
|
|
237
|
+
page.cleanup();
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
warnings.push(`OCR via ${engine.name} (${ocrLang}); quality depends on scan quality.`);
|
|
241
|
+
pageTexts.length = 0;
|
|
242
|
+
pageTexts.push(...ocrTexts);
|
|
243
|
+
}
|
|
244
|
+
else if (pageTexts.every((t) => t.trim().length === 0)) {
|
|
245
|
+
warnings.push('No extractable text found - this may be a scanned PDF. Re-run with ocr: true.');
|
|
246
|
+
}
|
|
247
|
+
const text = pageTexts.join('\n\n').replace(/\n{4,}/g, '\n\n\n') + '\n';
|
|
248
|
+
await fs.writeFile(req.output, text, 'utf8');
|
|
249
|
+
if (selected.length < totalPages) {
|
|
250
|
+
warnings.push(`Converted page(s) ${selected.join(', ')} of ${totalPages} (pages option).`);
|
|
251
|
+
}
|
|
252
|
+
return {
|
|
253
|
+
ok: true,
|
|
254
|
+
input: req.input,
|
|
255
|
+
output: req.output,
|
|
256
|
+
from: req.from,
|
|
257
|
+
to: req.to,
|
|
258
|
+
bytesIn,
|
|
259
|
+
bytesOut: Buffer.byteLength(text),
|
|
260
|
+
durationMs: Date.now() - started,
|
|
261
|
+
warnings,
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function pageRange(totalPages) {
|
|
266
|
+
return Array.from({ length: totalPages }, (_, i) => i + 1);
|
|
267
|
+
}
|
|
268
|
+
/** Rasterize one page at a scale; white background for JPEG, transparent PNG otherwise. */
|
|
269
|
+
async function renderPage(page, scale, opts) {
|
|
270
|
+
const viewport = page.getViewport({ scale });
|
|
271
|
+
const width = Math.max(1, Math.ceil(viewport.width));
|
|
272
|
+
const height = Math.max(1, Math.ceil(viewport.height));
|
|
273
|
+
const canvas = createCanvas(width, height);
|
|
274
|
+
const cctx = canvas.getContext('2d');
|
|
275
|
+
if (opts.background) {
|
|
276
|
+
cctx.fillStyle = opts.background;
|
|
277
|
+
cctx.fillRect(0, 0, width, height);
|
|
278
|
+
}
|
|
279
|
+
await page.render({ canvasContext: cctx, viewport }).promise;
|
|
280
|
+
return opts.to === 'png' ? canvas.encode('png') : canvas.encode('jpeg', opts.quality);
|
|
281
|
+
}
|
|
282
|
+
function fail(req, code, message) {
|
|
283
|
+
return { ok: false, input: req.input, from: req.from, to: req.to, error: convertError(code, message) };
|
|
284
|
+
}
|
|
285
|
+
function failErr(req, error) {
|
|
286
|
+
return { ok: false, input: req.input, from: req.from, to: req.to, error };
|
|
287
|
+
}
|
|
288
|
+
function platformKey() {
|
|
289
|
+
return process.platform === 'win32' ? 'win32' : process.platform === 'darwin' ? 'darwin' : 'linux';
|
|
290
|
+
}
|
|
291
|
+
/** Minimal per-page text assembly: keep line breaks, avoid word-gluing. */
|
|
292
|
+
function itemsToText(items) {
|
|
293
|
+
const lines = [];
|
|
294
|
+
let line = '';
|
|
295
|
+
for (const item of items) {
|
|
296
|
+
const s = item.str;
|
|
297
|
+
if (s === undefined)
|
|
298
|
+
continue;
|
|
299
|
+
if (s.length > 0) {
|
|
300
|
+
if (line.length > 0 && !line.endsWith(' ') && !s.startsWith(' '))
|
|
301
|
+
line += ' ';
|
|
302
|
+
line += s;
|
|
303
|
+
}
|
|
304
|
+
if (item.hasEOL) {
|
|
305
|
+
lines.push(line.trimEnd());
|
|
306
|
+
line = '';
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
if (line.trimEnd().length > 0)
|
|
310
|
+
lines.push(line.trimEnd());
|
|
311
|
+
return lines.join('\n').trimEnd();
|
|
312
|
+
}
|
|
313
|
+
function withPageNumber(output, page, pad) {
|
|
314
|
+
const { dir, name, ext } = path.parse(output);
|
|
315
|
+
return path.join(dir, `${name}-${String(page).padStart(pad, '0')}${ext}`);
|
|
316
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { Detection } from './types.js';
|
|
2
|
+
export interface DetectOutcome {
|
|
3
|
+
detection: Detection;
|
|
4
|
+
warnings: string[];
|
|
5
|
+
}
|
|
6
|
+
export declare class DetectError extends Error {
|
|
7
|
+
readonly error: import('./types.js').ConvertError;
|
|
8
|
+
constructor(error: import('./types.js').ConvertError);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Detect the format of a file. Priority: binary magic bytes (file-type) →
|
|
12
|
+
* SVG content → JSON content → extension → YAML document marker. When content
|
|
13
|
+
* and extension disagree, content wins and a warning is returned — users
|
|
14
|
+
* rename files wrongly all the time.
|
|
15
|
+
*/
|
|
16
|
+
export declare function detectFile(input: string): Promise<DetectOutcome>;
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { fileTypeFromFile } from 'file-type';
|
|
4
|
+
import { convertError } from './errors.js';
|
|
5
|
+
import { formatFromExtension } from './formats.js';
|
|
6
|
+
/** MIME types file-type can report that map 1:1 onto a supported format. */
|
|
7
|
+
const MIME_TO_FORMAT = {
|
|
8
|
+
'image/png': 'png',
|
|
9
|
+
'image/jpeg': 'jpg',
|
|
10
|
+
'image/webp': 'webp',
|
|
11
|
+
'image/gif': 'gif',
|
|
12
|
+
'application/pdf': 'pdf',
|
|
13
|
+
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': 'docx',
|
|
14
|
+
'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'pptx',
|
|
15
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'xlsx',
|
|
16
|
+
'video/mp4': 'mp4',
|
|
17
|
+
'video/quicktime': 'mov',
|
|
18
|
+
'audio/mpeg': 'mp3',
|
|
19
|
+
'audio/wav': 'wav',
|
|
20
|
+
'audio/vnd.wave': 'wav',
|
|
21
|
+
'audio/x-wav': 'wav',
|
|
22
|
+
};
|
|
23
|
+
const TEXT_SNIFF_BYTES = 512;
|
|
24
|
+
/** Whole-file JSON.parse is only attempted up to this size; larger files fall back to the extension. */
|
|
25
|
+
const JSON_SNIFF_LIMIT = 1_000_000;
|
|
26
|
+
const SVG_MIME = 'image/svg+xml';
|
|
27
|
+
export class DetectError extends Error {
|
|
28
|
+
error;
|
|
29
|
+
constructor(error) {
|
|
30
|
+
super(error.message);
|
|
31
|
+
this.error = error;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Detect the format of a file. Priority: binary magic bytes (file-type) →
|
|
36
|
+
* SVG content → JSON content → extension → YAML document marker. When content
|
|
37
|
+
* and extension disagree, content wins and a warning is returned — users
|
|
38
|
+
* rename files wrongly all the time.
|
|
39
|
+
*/
|
|
40
|
+
export async function detectFile(input) {
|
|
41
|
+
const stat = await fs.stat(input).catch(() => null);
|
|
42
|
+
if (!stat) {
|
|
43
|
+
throw new DetectError(convertError('input_not_found', `Input file not found: ${input}`));
|
|
44
|
+
}
|
|
45
|
+
if (stat.isDirectory()) {
|
|
46
|
+
throw new DetectError(convertError('invalid_input', `Input is a directory, not a file: ${input}`));
|
|
47
|
+
}
|
|
48
|
+
const warnings = [];
|
|
49
|
+
const ext = path.extname(input).replace(/^\./, '');
|
|
50
|
+
const extFormat = ext ? formatFromExtension(ext) : null;
|
|
51
|
+
// 1. Binary formats via magic bytes.
|
|
52
|
+
const magic = await fileTypeFromFile(input).catch(() => undefined);
|
|
53
|
+
const magicFormat = magic ? MIME_TO_FORMAT[magic.mime] : undefined;
|
|
54
|
+
// 2. SVG is text-based; file-type does not report it.
|
|
55
|
+
const svgLike = magicFormat ? false : await looksLikeSvg(input);
|
|
56
|
+
const detected = magicFormat ?? (svgLike ? 'svg' : null);
|
|
57
|
+
if (detected) {
|
|
58
|
+
if (extFormat && extFormat !== detected) {
|
|
59
|
+
warnings.push(`File extension suggests ${extFormat} but content is ${detected}; using ${detected}.`);
|
|
60
|
+
}
|
|
61
|
+
return {
|
|
62
|
+
detection: { format: detected, confidence: 'magic', mime: magic?.mime ?? SVG_MIME },
|
|
63
|
+
warnings,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
// 3. JSON parses successfully — essentially zero false positives when the
|
|
67
|
+
// content starts with { or [ (scalars like `123` stay ambiguous).
|
|
68
|
+
if (await isJsonContent(input)) {
|
|
69
|
+
if (extFormat && extFormat !== 'json') {
|
|
70
|
+
warnings.push(`File extension suggests ${extFormat} but content parses as JSON; using json.`);
|
|
71
|
+
}
|
|
72
|
+
return { detection: { format: 'json', confidence: 'magic', mime: 'application/json' }, warnings };
|
|
73
|
+
}
|
|
74
|
+
// 4. Extension mapping (csv/txt cannot be told apart from plain text by content).
|
|
75
|
+
if (extFormat) {
|
|
76
|
+
return { detection: { format: extFormat, confidence: 'extension', mime: undefined }, warnings };
|
|
77
|
+
}
|
|
78
|
+
// 5. YAML document marker, for extension-less files only — a leading `---`
|
|
79
|
+
// line is a strong YAML signal but not proof (markdown frontmatter, rules).
|
|
80
|
+
if (await startsWithYamlMarker(input)) {
|
|
81
|
+
return { detection: { format: 'yaml', confidence: 'guess', mime: 'application/yaml' }, warnings };
|
|
82
|
+
}
|
|
83
|
+
throw new DetectError(convertError('unknown_format', `Cannot determine the format of ${input}`, {
|
|
84
|
+
hint: 'inspect_file it first, or rename it with a known extension (png, jpg, webp, svg, pdf, json, yaml, csv, txt).',
|
|
85
|
+
}));
|
|
86
|
+
}
|
|
87
|
+
async function readHead(input, bytes) {
|
|
88
|
+
return fs.open(input, 'r').then(async (handle) => {
|
|
89
|
+
try {
|
|
90
|
+
const buffer = Buffer.alloc(bytes);
|
|
91
|
+
const { bytesRead } = await handle.read(buffer, 0, bytes, 0);
|
|
92
|
+
return buffer.subarray(0, bytesRead).toString('utf8').replace(/^\uFEFF/, '');
|
|
93
|
+
}
|
|
94
|
+
finally {
|
|
95
|
+
await handle.close();
|
|
96
|
+
}
|
|
97
|
+
}).catch(() => '');
|
|
98
|
+
}
|
|
99
|
+
async function looksLikeSvg(input) {
|
|
100
|
+
return (await readHead(input, TEXT_SNIFF_BYTES)).includes('<svg');
|
|
101
|
+
}
|
|
102
|
+
/** True when the whole file JSON-parses and starts with an unambiguous container. */
|
|
103
|
+
async function isJsonContent(input) {
|
|
104
|
+
const stat = await fs.stat(input).catch(() => null);
|
|
105
|
+
if (!stat || stat.size === 0 || stat.size > JSON_SNIFF_LIMIT)
|
|
106
|
+
return false;
|
|
107
|
+
const text = await fs.readFile(input, 'utf8').catch(() => '');
|
|
108
|
+
const trimmed = text.trim();
|
|
109
|
+
if (!trimmed.startsWith('{') && !trimmed.startsWith('['))
|
|
110
|
+
return false;
|
|
111
|
+
try {
|
|
112
|
+
JSON.parse(trimmed);
|
|
113
|
+
return true;
|
|
114
|
+
}
|
|
115
|
+
catch {
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
async function startsWithYamlMarker(input) {
|
|
120
|
+
return /^---\s*(\r?\n|$)/.test((await readHead(input, 64)).trimStart());
|
|
121
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ConvertError, ConvertErrorCode } from './types.js';
|
|
2
|
+
export declare function convertError(code: ConvertErrorCode, message: string, extra?: Partial<Omit<ConvertError, 'code' | 'message'>>): ConvertError;
|
|
3
|
+
export declare function truncate(text: string, max?: number): string;
|
|
4
|
+
/** Normalize an unknown thrown value into a ConvertError. */
|
|
5
|
+
export declare function toConvertError(err: unknown, fallbackMessage: string): ConvertError;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/** Detail output longer than this is truncated into ConvertError.detail. */
|
|
2
|
+
const MAX_DETAIL = 2048;
|
|
3
|
+
export function convertError(code, message, extra = {}) {
|
|
4
|
+
return { code, message, ...extra };
|
|
5
|
+
}
|
|
6
|
+
export function truncate(text, max = MAX_DETAIL) {
|
|
7
|
+
const clean = text.replace(/\s+/g, ' ').trim();
|
|
8
|
+
return clean.length <= max ? clean : clean.slice(0, max) + `… (+${clean.length - max} chars)`;
|
|
9
|
+
}
|
|
10
|
+
/** Normalize an unknown thrown value into a ConvertError. */
|
|
11
|
+
export function toConvertError(err, fallbackMessage) {
|
|
12
|
+
if (err instanceof Error) {
|
|
13
|
+
return convertError('conversion_failed', fallbackMessage, { detail: truncate(err.stack ?? err.message) });
|
|
14
|
+
}
|
|
15
|
+
return convertError('conversion_failed', fallbackMessage, { detail: truncate(String(err)) });
|
|
16
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { FormatCategory, FormatId, FormatMeta } from './types.js';
|
|
2
|
+
/** Single source of truth for format metadata. */
|
|
3
|
+
export declare const FORMATS: Record<FormatId, FormatMeta>;
|
|
4
|
+
export declare const FORMAT_IDS: FormatId[];
|
|
5
|
+
/** Resolve a free-text format name ('JPEG', '.yml', 'webp') to a FormatId. */
|
|
6
|
+
export declare function parseFormatArg(value: string): FormatId | null;
|
|
7
|
+
/** Extension (without dot) → FormatId, e.g. 'jpeg' → 'jpg'. */
|
|
8
|
+
export declare function formatFromExtension(ext: string): FormatId | null;
|
|
9
|
+
/** Canonical output extension for a format, e.g. 'jpg' → '.jpg'. */
|
|
10
|
+
export declare function canonicalExtension(format: FormatId): string;
|
|
11
|
+
export declare function formatCategory(format: FormatId): FormatCategory;
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/** Single source of truth for format metadata. */
|
|
2
|
+
export const FORMATS = {
|
|
3
|
+
pdf: { category: 'document', extensions: ['.pdf'], mime: 'application/pdf' },
|
|
4
|
+
docx: {
|
|
5
|
+
category: 'document',
|
|
6
|
+
extensions: ['.docx'],
|
|
7
|
+
mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
|
|
8
|
+
},
|
|
9
|
+
pptx: {
|
|
10
|
+
category: 'document',
|
|
11
|
+
extensions: ['.pptx'],
|
|
12
|
+
mime: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
|
|
13
|
+
},
|
|
14
|
+
xlsx: {
|
|
15
|
+
category: 'document',
|
|
16
|
+
extensions: ['.xlsx'],
|
|
17
|
+
mime: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
|
|
18
|
+
},
|
|
19
|
+
png: { category: 'image', extensions: ['.png'], mime: 'image/png' },
|
|
20
|
+
jpg: { category: 'image', extensions: ['.jpg', '.jpeg'], mime: 'image/jpeg' },
|
|
21
|
+
webp: { category: 'image', extensions: ['.webp'], mime: 'image/webp' },
|
|
22
|
+
svg: { category: 'image', extensions: ['.svg'], mime: 'image/svg+xml' },
|
|
23
|
+
gif: { category: 'image', extensions: ['.gif'], mime: 'image/gif' },
|
|
24
|
+
mp4: { category: 'video', extensions: ['.mp4'], mime: 'video/mp4' },
|
|
25
|
+
mov: { category: 'video', extensions: ['.mov'], mime: 'video/quicktime' },
|
|
26
|
+
mp3: { category: 'audio', extensions: ['.mp3'], mime: 'audio/mpeg' },
|
|
27
|
+
wav: { category: 'audio', extensions: ['.wav'], mime: 'audio/wav' },
|
|
28
|
+
json: { category: 'data', extensions: ['.json'], mime: 'application/json' },
|
|
29
|
+
yaml: { category: 'data', extensions: ['.yaml', '.yml'], mime: 'application/yaml' },
|
|
30
|
+
csv: { category: 'data', extensions: ['.csv'], mime: 'text/csv' },
|
|
31
|
+
txt: { category: 'text', extensions: ['.txt'], mime: 'text/plain' },
|
|
32
|
+
};
|
|
33
|
+
export const FORMAT_IDS = Object.keys(FORMATS);
|
|
34
|
+
const EXT_TO_FORMAT = new Map();
|
|
35
|
+
for (const [id, meta] of Object.entries(FORMATS)) {
|
|
36
|
+
for (const ext of meta.extensions)
|
|
37
|
+
EXT_TO_FORMAT.set(ext.slice(1).toLowerCase(), id);
|
|
38
|
+
}
|
|
39
|
+
/** Input aliases users (and agents) type instead of canonical ids. */
|
|
40
|
+
const ALIASES = {
|
|
41
|
+
jpeg: 'jpg',
|
|
42
|
+
jpe: 'jpg',
|
|
43
|
+
yml: 'yaml',
|
|
44
|
+
};
|
|
45
|
+
/** Resolve a free-text format name ('JPEG', '.yml', 'webp') to a FormatId. */
|
|
46
|
+
export function parseFormatArg(value) {
|
|
47
|
+
const key = value.trim().toLowerCase().replace(/^\./, '');
|
|
48
|
+
return ALIASES[key] ?? (key in FORMATS ? key : null);
|
|
49
|
+
}
|
|
50
|
+
/** Extension (without dot) → FormatId, e.g. 'jpeg' → 'jpg'. */
|
|
51
|
+
export function formatFromExtension(ext) {
|
|
52
|
+
return EXT_TO_FORMAT.get(ext.toLowerCase()) ?? (ALIASES[ext.toLowerCase()] ?? null);
|
|
53
|
+
}
|
|
54
|
+
/** Canonical output extension for a format, e.g. 'jpg' → '.jpg'. */
|
|
55
|
+
export function canonicalExtension(format) {
|
|
56
|
+
return FORMATS[format].extensions[0];
|
|
57
|
+
}
|
|
58
|
+
export function formatCategory(format) {
|
|
59
|
+
return FORMATS[format].category;
|
|
60
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-file-convert core: a DSH-independent conversion library.
|
|
3
|
+
* Usable directly from tests, a CLI, or an MCP server - no harness required.
|
|
4
|
+
*/
|
|
5
|
+
export * from './types.js';
|
|
6
|
+
export { FORMATS, FORMAT_IDS, formatFromExtension, parseFormatArg, canonicalExtension, formatCategory } from './formats.js';
|
|
7
|
+
export { ConversionRouter, type RouterDefaults, type ConvertFileRequest, type ConvertRunContext } from './router.js';
|
|
8
|
+
export { detectFile, DetectError, type DetectOutcome } from './detect.js';
|
|
9
|
+
export { resolveBinary } from './binary.js';
|
|
10
|
+
export { defaultOutputPath, batchOutputPath } from './paths.js';
|
|
11
|
+
export { ImageConverter } from './converters/image.js';
|
|
12
|
+
export { PdfConverter } from './converters/pdf.js';
|
|
13
|
+
export { DataConverter } from './converters/data.js';
|
|
14
|
+
export { MediaConverter, FFMPEG, FFPROBE } from './converters/media.js';
|
|
15
|
+
export { OfficeConverter, PdfToDocxConverter, SOFFICE, PYTHON_PDF2DOCX } from './converters/office.js';
|
|
16
|
+
export { GHOSTSCRIPT } from './optimizers.js';
|
|
17
|
+
export { TESSERACT, OCR_LANGUAGE_DATA, ocrLanguagesCached, installOcrLanguages, tessdataDir, resolveOcrEngine, type OcrEngine } from './ocr.js';
|
|
18
|
+
export { parsePageRange, PageRangeError } from './utils/pages.js';
|
|
19
|
+
export { optimizeFile, type OptimizeResult } from './optimizers.js';
|
|
20
|
+
import { ConversionRouter, type RouterDefaults } from './router.js';
|
|
21
|
+
/**
|
|
22
|
+
* Assemble the registry: image (sharp), pdf (pdfjs), data (yaml/csv),
|
|
23
|
+
* media (ffmpeg), office (LibreOffice), pdf→docx (python pdf2docx).
|
|
24
|
+
*/
|
|
25
|
+
export declare function createRouter(defaults?: Partial<RouterDefaults>): ConversionRouter;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-file-convert core: a DSH-independent conversion library.
|
|
3
|
+
* Usable directly from tests, a CLI, or an MCP server - no harness required.
|
|
4
|
+
*/
|
|
5
|
+
export * from './types.js';
|
|
6
|
+
export { FORMATS, FORMAT_IDS, formatFromExtension, parseFormatArg, canonicalExtension, formatCategory } from './formats.js';
|
|
7
|
+
export { ConversionRouter } from './router.js';
|
|
8
|
+
export { detectFile, DetectError } from './detect.js';
|
|
9
|
+
export { resolveBinary } from './binary.js';
|
|
10
|
+
export { defaultOutputPath, batchOutputPath } from './paths.js';
|
|
11
|
+
export { ImageConverter } from './converters/image.js';
|
|
12
|
+
export { PdfConverter } from './converters/pdf.js';
|
|
13
|
+
export { DataConverter } from './converters/data.js';
|
|
14
|
+
export { MediaConverter, FFMPEG, FFPROBE } from './converters/media.js';
|
|
15
|
+
export { OfficeConverter, PdfToDocxConverter, SOFFICE, PYTHON_PDF2DOCX } from './converters/office.js';
|
|
16
|
+
export { GHOSTSCRIPT } from './optimizers.js';
|
|
17
|
+
export { TESSERACT, OCR_LANGUAGE_DATA, ocrLanguagesCached, installOcrLanguages, tessdataDir, resolveOcrEngine } from './ocr.js';
|
|
18
|
+
export { parsePageRange, PageRangeError } from './utils/pages.js';
|
|
19
|
+
export { optimizeFile } from './optimizers.js';
|
|
20
|
+
import { ConversionRouter } from './router.js';
|
|
21
|
+
import { ImageConverter } from './converters/image.js';
|
|
22
|
+
import { PdfConverter } from './converters/pdf.js';
|
|
23
|
+
import { DataConverter } from './converters/data.js';
|
|
24
|
+
import { MediaConverter } from './converters/media.js';
|
|
25
|
+
import { OfficeConverter, PdfToDocxConverter } from './converters/office.js';
|
|
26
|
+
import { resolveBinary } from './binary.js';
|
|
27
|
+
const SILENT_LOGGER = { debug() { }, info() { }, warn() { }, error() { } };
|
|
28
|
+
/**
|
|
29
|
+
* Assemble the registry: image (sharp), pdf (pdfjs), data (yaml/csv),
|
|
30
|
+
* media (ffmpeg), office (LibreOffice), pdf→docx (python pdf2docx).
|
|
31
|
+
*/
|
|
32
|
+
export function createRouter(defaults) {
|
|
33
|
+
const overrides = defaults?.binaryOverrides ?? {};
|
|
34
|
+
const resolve = (dep) => resolveBinary(dep, overrides, SILENT_LOGGER);
|
|
35
|
+
const router = new ConversionRouter({
|
|
36
|
+
quality: defaults?.quality ?? 85,
|
|
37
|
+
dpi: defaults?.dpi ?? 150,
|
|
38
|
+
timeoutMs: defaults?.timeoutMs ?? 120_000,
|
|
39
|
+
outputRoots: defaults?.outputRoots,
|
|
40
|
+
binaryOverrides: overrides,
|
|
41
|
+
maxInputBytes: defaults?.maxInputBytes,
|
|
42
|
+
maxPdfPages: defaults?.maxPdfPages,
|
|
43
|
+
maxOutputPixels: defaults?.maxOutputPixels,
|
|
44
|
+
});
|
|
45
|
+
router.register(new ImageConverter());
|
|
46
|
+
router.register(new PdfConverter(resolve));
|
|
47
|
+
router.register(new DataConverter());
|
|
48
|
+
router.register(new MediaConverter(resolve));
|
|
49
|
+
router.register(new OfficeConverter(resolve));
|
|
50
|
+
router.register(new PdfToDocxConverter(resolve));
|
|
51
|
+
return router;
|
|
52
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import './converters/pdf-env.js';
|
|
2
|
+
import type { Detection, InspectResult } from './types.js';
|
|
3
|
+
export interface MediaProbeContext {
|
|
4
|
+
ffprobePath: string;
|
|
5
|
+
timeoutMs: number;
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Structured file facts so an agent can decide before converting.
|
|
10
|
+
* Inspect is informational: parse problems degrade to `kind: 'unknown'`
|
|
11
|
+
* (or `probeUnavailable` for media) instead of failing.
|
|
12
|
+
*/
|
|
13
|
+
export declare function inspectFile(input: string, detection: Detection, bytes: number, media?: MediaProbeContext): Promise<InspectResult>;
|