@scrymore/scry-deployer 0.11.0-next.20261001044855 → 0.11.0

package/lib/importBundle.js

@@ -11,7 +11,7 @@ const fs = require('fs');
 const path = require('path');
 const { version: PACKAGE_VERSION } = require('../package.json');
 const { SCF_SCHEMA_URL } = require('./scf.js');
-const { convertFile, measure, MAX_IMAGE_BYTES, MAX_IMAGE_DIMENSION } = require('./importConvert.js');
+const { convertFile, measure, MAX_RASTER_BYTES, MAX_NATIVE_DIMENSION } = require('./importConvert.js');
 const { stripMetadata } = require('./importStrip.js');
 const { readEmbeddedPacket, parseAllowListed, mergeXmp, mergeDuplicateMeta } = require('./importXmp.js');
 const { readSidecarPacket } = require('./importScan.js');
@@ -63,10 +63,10 @@ async function prepareNative(file, size, deps) {
     const dims = clean ? measure(clean) : null;
     if (!clean || !dims) return { skip: { reason: 'error', detail: `not a readable ${file.ext} image` } };
     const idHex = sha256Hex(clean);
-    const fits = clean.length <= MAX_IMAGE_BYTES && dims.width <= MAX_IMAGE_DIMENSION && dims.height <= MAX_IMAGE_DIMENSION;
+    const fits = clean.length <= MAX_RASTER_BYTES && dims.width <= MAX_NATIVE_DIMENSION && dims.height <= MAX_NATIVE_DIMENSION;
     if (fits) return { idHex, picture: { bytes: clean, ...dims }, converted: false };
     const result = await deps.convert(file.abs, file.ext, { tools: deps.tools });
-    if (!result.ok) return { idHex, skip: { reason: result.reason, detail: result.detail } };
+    if (!result.ok) return { idHex, skip: { reason: result.reason, detail: `over the ${MAX_RASTER_BYTES / 1048576} MB or ${MAX_NATIVE_DIMENSION} px the AI services accept, and could not be reduced: ${result.detail}` } };
     return { idHex, picture: result, converted: true };
 }
 

package/lib/importConvert.js

@@ -12,10 +12,21 @@ const fs = require('fs');
 const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
+const zlib = require('zlib');
 const { stripMetadata } = require('./importStrip.js');
 
+// SCF limits (what the validator accepts) ...
 const MAX_IMAGE_BYTES = 20 * 1024 * 1024;
 const MAX_IMAGE_DIMENSION = 16384;
+// ... and the smaller bounds every picture is held to. Stage proved the SCF limits are not what the models behind
+// captioning and embeddings accept: a 16384 x 16384 PNG passes the validator and is then refused by OpenAI with
+// `image_parse_error`, and an image over 5 MB is refused by Jina (422); either fails the whole chunk it is in.
+// Anything we convert is rendered at or below these.
+const MAX_RASTER_DIMENSION = 2048;
+const MAX_RASTER_BYTES = 4 * 1024 * 1024; // Jina refuses an image over 5,242,880 bytes (422), so keep clear of it
+// A picture that is already PNG/JPEG/WebP is used as it is only when it is within the bytes above and this edge
+// (stage indexed 4096 px; 16384 px was refused). Larger ones are re-encoded down to MAX_RASTER_DIMENSION.
+const MAX_NATIVE_DIMENSION = 4096;
 const TOOL_TIMEOUT_MS = 120_000;
 
 // ImageMagick resource limits on every call: a hostile or corrupt PSD/TIFF/HEIC/PDF must not exhaust
@@ -42,12 +53,13 @@ const CONVERT_CODERS = Object.freeze({
 const NATIVE_EXTENSIONS = new Set(Object.keys(NATIVE_CODERS));
 const CONVERTIBLE_EXTENSIONS = new Set(Object.keys(CONVERT_CODERS));
 
-// Largest first: lossless PNG, then JPEG at shrinking sizes until the file is under the limit.
+// Largest first: lossless PNG, then JPEG at shrinking sizes until the file is under the byte cap.
+// No attempt is ever larger than MAX_RASTER_DIMENSION.
 const ATTEMPTS = Object.freeze([
-    { format: 'png', max: MAX_IMAGE_DIMENSION, quality: 0 },
-    { format: 'jpeg', max: MAX_IMAGE_DIMENSION, quality: 90 },
-    { format: 'jpeg', max: 8192, quality: 85 },
-    { format: 'jpeg', max: 4096, quality: 80 },
+    { format: 'png', max: MAX_RASTER_DIMENSION, quality: 0 },
+    { format: 'jpeg', max: MAX_RASTER_DIMENSION, quality: 90 },
+    { format: 'jpeg', max: Math.round(MAX_RASTER_DIMENSION * 0.75), quality: 85 },
+    { format: 'jpeg', max: Math.round(MAX_RASTER_DIMENSION * 0.5), quality: 80 },
 ]);
 
 const JPEG_SOF = new Set([0xc0, 0xc1, 0xc2, 0xc3, 0xc5, 0xc6, 0xc7, 0xc9, 0xca, 0xcb, 0xcd, 0xce, 0xcf]);
@@ -136,15 +148,17 @@ function toolCommand(tool, { input, coder, output, attempt }) {
         const options = attempt.quality ? ['-s', 'formatOptions', String(attempt.quality)] : [];
         return { file: 'sips', args: ['-s', 'format', attempt.format, ...options, '-Z', String(attempt.max), input, '--out', output] };
     }
-    // pdftoppm writes <prefix>.png or <prefix>.jpg itself.
+    // pdftoppm writes <prefix>.png or <prefix>.jpg itself. -scale-to N makes the LONGEST edge exactly N (it
+    // also enlarges a small page, which is how a 624 pt page became 16384 px), so N is always the bound.
     const flag = attempt.format === 'png' ? '-png' : '-jpeg';
-    return { file: 'pdftoppm', args: [flag, '-singlefile', '-r', '150', '-scale-to', String(attempt.max), input, output.replace(/\.(png|jpg)$/, '')] };
+    return { file: 'pdftoppm', args: [flag, '-singlefile', '-scale-to', String(attempt.max), input, output.replace(/\.(png|jpg)$/, '')] };
 }
 
+/** Converters for one file type, in the order they are tried. PDF/AI: pdftoppm first, it renders straight at the bound. */
 function toolsFor(ext, tools) {
-    const coder = CONVERT_CODERS[ext] || NATIVE_CODERS[ext];
-    const isPdf = coder === 'pdf';
-    return tools.filter((t) => (t === 'pdftoppm' ? isPdf : true));
+    const isPdf = (CONVERT_CODERS[ext] || NATIVE_CODERS[ext]) === 'pdf';
+    const usable = tools.filter((t) => (t === 'pdftoppm' ? isPdf : true));
+    return isPdf ? [...usable.filter((t) => t === 'pdftoppm'), ...usable.filter((t) => t !== 'pdftoppm')] : usable;
 }
 
 function runAttempt(tool, request, run) {
@@ -153,19 +167,62 @@ function runAttempt(tool, request, run) {
     return !result.error && result.status === 0 && fs.existsSync(request.output);
 }
 
-/** Read, strip and size-check one converter output. Returns {bytes, family, width, height} or null. */
+const PNG_CHANNELS = Object.freeze({ 0: 1, 2: 3, 3: 1, 4: 2, 6: 4 });
+
+/** Why the PNG pixel data cannot be read (truncated, corrupt or short), or null when it inflates to the full size. */
+function pngDecodeProblem(buf) {
+    const idat = [];
+    let header = null;
+    let ended = false;
+    for (let offset = 8; offset + 12 <= buf.length && !ended; ) {
+        const length = buf.readUInt32BE(offset);
+        const type = buf.toString('latin1', offset + 4, offset + 8);
+        if (offset + 12 + length > buf.length) return 'PNG is truncated';
+        const data = buf.subarray(offset + 8, offset + 8 + length);
+        if (type === 'IHDR') header = { width: data.readUInt32BE(0), height: data.readUInt32BE(4), depth: data[8], colour: data[9], interlace: data[12] };
+        if (type === 'IDAT') idat.push(data);
+        ended = type === 'IEND';
+        offset += 12 + length;
+    }
+    if (!header || !ended || idat.length === 0) return 'PNG is truncated';
+    let pixels;
+    try {
+        pixels = zlib.inflateSync(Buffer.concat(idat));
+    } catch {
+        return 'PNG pixel data is corrupt';
+    }
+    const channels = PNG_CHANNELS[header.colour];
+    if (header.interlace !== 0 || !channels) return null;
+    const expected = header.height * (Math.ceil((header.width * channels * header.depth) / 8) + 1);
+    return pixels.length === expected ? null : 'PNG pixel data is incomplete';
+}
+
+/** Why the converted picture must not be uploaded (does not decode, too big in pixels or bytes), or null when it is fine. */
+function outputProblem(buf, dims) {
+    if (dims.width > MAX_RASTER_DIMENSION || dims.height > MAX_RASTER_DIMENSION) {
+        return `output was ${dims.width}x${dims.height}, over ${MAX_RASTER_DIMENSION} px`;
+    }
+    if (buf.length > MAX_RASTER_BYTES) return `output was ${(buf.length / 1048576).toFixed(1)} MB, over ${MAX_RASTER_BYTES / 1048576} MB`;
+    if (dims.family === 'png') return pngDecodeProblem(buf);
+    if (dims.family === 'jpeg' && !(buf[buf.length - 2] === 0xff && buf[buf.length - 1] === 0xd9)) return 'JPEG is truncated';
+    return null;
+}
+
+/** Read, strip and check one converter output. Returns {bytes, family, width, height} or {problem}. */
 function acceptOutput(outputPath) {
     const raw = fs.readFileSync(outputPath);
     const family = measure(raw)?.family;
     const clean = family ? stripMetadata(raw, family) : null;
     const dims = clean ? measure(clean) : null;
-    if (!clean || !dims) return null;
-    const fits = clean.length <= MAX_IMAGE_BYTES && dims.width <= MAX_IMAGE_DIMENSION && dims.height <= MAX_IMAGE_DIMENSION;
-    return fits ? { bytes: clean, ...dims } : null;
+    if (!clean || !dims) return { problem: 'output is not a readable PNG or JPEG' };
+    const problem = outputProblem(clean, dims);
+    return problem ? { problem } : { bytes: clean, ...dims };
 }
 
 /**
- * Convert (or re-encode a too-large native image) with the first tool that works.
+ * Convert (or re-encode a native image that is over the bound) with the first tool that works.
+ * Every output is held to MAX_RASTER_DIMENSION and MAX_RASTER_BYTES and checked to decode; a file that
+ * cannot be made valid comes back as a named skip, never as an image that is known to be unreadable.
  *
  * @param {string} file absolute path of the source file
  * @param {string} ext lower-case extension with the dot
@@ -181,16 +238,19 @@ async function convertFile(file, ext, opts = {}) {
     }
     const coder = CONVERT_CODERS[ext] || NATIVE_CODERS[ext];
     const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'scry-import-conv-'));
+    let lastProblem = 'the converter failed';
     try {
         for (const attempt of ATTEMPTS) {
             const output = path.join(dir, `out-${attempt.format}-${attempt.max}.${EXT_OF[attempt.format]}`);
             for (const tool of tools) {
                 if (!runAttempt(tool, { input: file, coder, output, attempt }, run)) continue;
                 const accepted = acceptOutput(output);
-                if (accepted) return { ok: true, ...accepted };
+                if (!accepted.problem) return { ok: true, ...accepted };
+                lastProblem = accepted.problem;
             }
         }
-        return { ok: false, reason: 'error', detail: `${ext} could not be converted to a PNG or JPEG under 20 MB and 16384 px` };
+        const limits = `${MAX_RASTER_DIMENSION} px and ${MAX_RASTER_BYTES / 1048576} MB`;
+        return { ok: false, reason: 'error', detail: `${ext} could not be converted to a valid PNG or JPEG within ${limits} (${lastProblem})` };
     } finally {
         fs.rmSync(dir, { recursive: true, force: true });
     }
@@ -200,6 +260,9 @@ module.exports = {
     MAGICK_LIMITS,
     MAX_IMAGE_BYTES,
     MAX_IMAGE_DIMENSION,
+    MAX_NATIVE_DIMENSION,
+    MAX_RASTER_BYTES,
+    MAX_RASTER_DIMENSION,
     NATIVE_EXTENSIONS,
     CONVERTIBLE_EXTENSIONS,
     detectTools,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@scrymore/scry-deployer",
-  "version": "0.11.0-next.20261001044855",
+  "version": "0.11.0",
   "description": "A CLI to automate the deployment of Storybook static builds.",
   "main": "index.js",
   "bin": {