@byline/core 6.10.1 → 6.12.0

package/dist/storage/stored-files-server.js

@@ -0,0 +1,292 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+// Server-only stored-file identity helpers, published as
+// `@byline/core/stored-files`. They hash streams incrementally with
+// `node:crypto`, so they must stay out of the browser-safe main entry.
+import { createHash } from 'node:crypto';
+import { Transform } from 'node:stream';
+import { ERR_STORAGE, ERR_VALIDATION } from '../lib/errors.js';
+/** SHA-256 (lower-case hex) and byte count of a stream, read to its end. */
+export async function hashStream(stream, options) {
+    const hash = createHash('sha256');
+    let size = 0;
+    const signal = options?.signal;
+    if (signal?.aborted) {
+        // An already-aborted signal never fires 'abort'; release the stream here.
+        // Destroy without an error: nothing listens to this stream yet, so an
+        // error argument would surface as an unhandled 'error' event.
+        destroyStream(stream);
+        signal.throwIfAborted();
+    }
+    const onAbort = () => destroyStream(stream, signal?.reason);
+    signal?.addEventListener('abort', onAbort, { once: true });
+    try {
+        for await (const chunk of stream) {
+            const bytes = typeof chunk === 'string' ? Buffer.from(chunk) : chunk;
+            hash.update(bytes);
+            size += bytes.length;
+        }
+    }
+    finally {
+        signal?.removeEventListener('abort', onAbort);
+    }
+    return { fileHash: hash.digest('hex'), size };
+}
+/**
+ * Upload an original object and compute its content hash in the same pass.
+ *
+ * - A `Buffer` is hashed in memory; a length that differs from `options.size`
+ *   fails before the provider is called.
+ * - A stream is piped through a hashing pass-through as the provider consumes
+ *   it. If it ends short, runs long, or is not fully consumed when the provider
+ *   resolves, the upload fails, the input is destroyed, and no hash is
+ *   returned. The object is deleted when the provider generated its path; a
+ *   `targetStoragePath` write may have replaced an existing object, which
+ *   cannot be undone, and the error says so.
+ */
+export async function uploadWithContentHash(storage, input, options) {
+    if (Buffer.isBuffer(input)) {
+        if (input.length !== options.size) {
+            throw ERR_VALIDATION({
+                message: `Upload is ${input.length} bytes but declared ${options.size} bytes.`,
+                details: { reason: 'size-mismatch', actualSize: input.length, declaredSize: options.size },
+            });
+        }
+        const fileHash = createHash('sha256').update(input).digest('hex');
+        const location = await storage.upload(input, options);
+        return { location, fileHash };
+    }
+    const hash = createHash('sha256');
+    let size = 0;
+    let ended = false;
+    // The provider reads this transform. Its readable side emits 'end' only
+    // after the provider has consumed every byte, so `ended` is "fully read by
+    // the provider", not merely "fully written by the source".
+    const hashing = new Transform({
+        transform(chunk, _encoding, callback) {
+            hash.update(chunk);
+            size += chunk.length;
+            callback(null, chunk);
+        },
+    });
+    hashing.on('end', () => {
+        ended = true;
+    });
+    // A provider that rejects may never listen to the transform; keep a late
+    // error from becoming an unhandled 'error' event. Errors still reach any
+    // listener the provider attached.
+    const swallow = () => { };
+    hashing.on('error', swallow);
+    // Forward a source failure to the provider through the transform.
+    const onInputError = (err) => hashing.destroy(err);
+    input.on('error', onInputError);
+    input.pipe(hashing);
+    /** Detach and destroy both sides. The input's own later errors are absorbed. */
+    const release = () => {
+        input.unpipe(hashing);
+        input.removeListener('error', onInputError);
+        input.on('error', swallow);
+        destroyStream(input);
+        hashing.destroy();
+    };
+    let location;
+    try {
+        location = await storage.upload(hashing, options);
+    }
+    catch (err) {
+        // The provider's error is the primary failure; cleanup must not mask it.
+        release();
+        throw err;
+    }
+    if (!ended || size !== options.size) {
+        release();
+        const providerGeneratedPath = options.targetStoragePath === undefined;
+        let objectDeleted = false;
+        if (providerGeneratedPath) {
+            try {
+                await storage.delete(location.storagePath);
+                objectDeleted = true;
+            }
+            catch {
+                // Reported through `objectDeleted: false`; the size failure is primary.
+            }
+        }
+        throw ERR_STORAGE({
+            message: ended
+                ? `Upload stream was ${size} bytes but declared ${options.size} bytes.`
+                : 'The storage provider resolved before consuming the whole upload stream.',
+            details: {
+                reason: ended ? 'size-mismatch' : 'stream-not-consumed',
+                actualSize: size,
+                declaredSize: options.size,
+                storagePath: location.storagePath,
+                objectDeleted,
+                // A verbatim `targetStoragePath` write may have replaced an existing
+                // object; Byline cannot restore it.
+                replacementUndone: false,
+            },
+        });
+    }
+    input.removeListener('error', onInputError);
+    return { location, fileHash: hash.digest('hex') };
+}
+function destroyStream(stream, reason) {
+    const destroyable = stream;
+    destroyable.destroy?.(reason instanceof Error ? reason : undefined);
+}
+/** Maximum entries kept in each summary path list. Counts are never capped. */
+const BACKFILL_REPORT_LIMIT = 1000;
+/**
+ * Fill or verify stored-file content identity for existing `byline_store_file`
+ * rows (spec 2026-10-31 §4.8).
+ *
+ * - Selects only rows written by `storage.providerName`, optionally one
+ *   collection's rows, paging by distinct storage path. Each path is read once
+ *   per run with a bound `download`, so the recorded validator always belongs
+ *   to the bytes that were hashed.
+ * - Fill never overwrites a recorded hash, so re-running is idempotent and an
+ *   interrupted run resumes cleanly. It hashes the bytes present now: where an
+ *   older version's object was later replaced at the same path, the recorded
+ *   hash describes the current bytes, which the validator identifies.
+ * - Legacy-null policy: a missing object, or a read or hash failure, leaves the
+ *   rows without identity and is reported. A provider without `download`, or
+ *   an adapter without stored-file maintenance, refuses to start.
+ *
+ * This is a maintenance primitive: it uses direct `db.queries` / `db.commands`
+ * calls and bypasses collection auth and hooks.
+ */
+export async function backfillStoredFileIdentity(options) {
+    const { db, storage, signal, logger } = options;
+    const mode = options.mode ?? 'fill';
+    const dryRun = mode === 'fill' && options.dryRun === true;
+    const batchSize = options.batchSize ?? 100;
+    const limit = options.limit ?? Number.POSITIVE_INFINITY;
+    if (!Number.isInteger(batchSize) || batchSize < 1) {
+        throw ERR_VALIDATION({
+            message: 'batchSize must be a positive integer.',
+            details: { batchSize },
+        });
+    }
+    const queries = db.queries.storedFiles;
+    const commands = db.commands.storedFiles;
+    if (!queries || (mode === 'fill' && !commands)) {
+        throw ERR_VALIDATION({
+            message: 'The database adapter does not support stored-file identity maintenance.',
+            details: { reason: 'adapter-unsupported' },
+        });
+    }
+    if (!storage.download) {
+        throw ERR_STORAGE({
+            message: `Storage provider '${storage.providerName}' has no download(); stored-file ` +
+                'identity cannot be backfilled or verified. Rows stay without identity.',
+            details: { reason: 'download-unavailable', storageProvider: storage.providerName },
+        });
+    }
+    let collectionId;
+    if (options.collectionPath !== undefined) {
+        const collection = await db.queries.collections.getCollectionByPath(options.collectionPath);
+        if (!collection?.id) {
+            throw ERR_VALIDATION({
+                message: `Unknown collection '${options.collectionPath}'.`,
+                details: { reason: 'unknown-collection', collectionPath: options.collectionPath },
+            });
+        }
+        collectionId = collection.id;
+    }
+    const summary = {
+        mode,
+        dryRun,
+        scanned: 0,
+        hashed: 0,
+        rowsUpdated: 0,
+        missing: 0,
+        failed: 0,
+        matched: 0,
+        mismatched: 0,
+        missingPaths: [],
+        failures: [],
+        mismatches: [],
+    };
+    const keep = (list, entry) => {
+        if (list.length < BACKFILL_REPORT_LIMIT)
+            list.push(entry);
+    };
+    let afterPath;
+    while (summary.scanned < limit) {
+        signal?.throwIfAborted();
+        const pageLimit = Math.min(batchSize, limit - summary.scanned);
+        const page = await queries.listStoredFilePaths({
+            storageProvider: storage.providerName,
+            ...(collectionId !== undefined && { collectionId }),
+            identity: mode === 'fill' ? 'missing' : 'present',
+            ...(afterPath !== undefined && { afterPath }),
+            limit: pageLimit,
+        });
+        for (const group of page) {
+            signal?.throwIfAborted();
+            summary.scanned += 1;
+            afterPath = group.storagePath;
+            let current;
+            try {
+                const read = await storage.download(group.storagePath, signal ? { signal } : undefined);
+                if (!read) {
+                    current = null;
+                }
+                else {
+                    const { fileHash } = await hashStream(read.body, signal ? { signal } : undefined);
+                    current = { fileHash, validator: read.validator };
+                }
+            }
+            catch (err) {
+                if (signal?.aborted)
+                    throw err;
+                summary.failed += 1;
+                const message = err instanceof Error ? err.message : String(err);
+                keep(summary.failures, { storagePath: group.storagePath, error: message });
+                logger?.warn({ storagePath: group.storagePath, err }, 'stored-file identity read failed');
+                continue;
+            }
+            if (!current) {
+                summary.missing += 1;
+                keep(summary.missingPaths, group.storagePath);
+                continue;
+            }
+            summary.hashed += 1;
+            if (mode === 'verify') {
+                for (const recorded of group.hashes) {
+                    if (recorded.fileHash === current.fileHash) {
+                        summary.matched += 1;
+                    }
+                    else {
+                        summary.mismatched += 1;
+                        keep(summary.mismatches, {
+                            storagePath: group.storagePath,
+                            recordedHash: recorded.fileHash,
+                            currentHash: current.fileHash,
+                            rowCount: recorded.rowCount,
+                        });
+                    }
+                }
+                continue;
+            }
+            if (!dryRun && commands) {
+                summary.rowsUpdated += await commands.setIdentityForPath({
+                    storageProvider: storage.providerName,
+                    storagePath: group.storagePath,
+                    ...(collectionId !== undefined && { collectionId }),
+                    fileHash: current.fileHash,
+                    storageValidator: current.validator,
+                });
+            }
+        }
+        if (page.length < pageLimit)
+            break;
+    }
+    logger?.info({ ...summary, missingPaths: undefined, failures: undefined, mismatches: undefined }, 'stored-file identity backfill finished');
+    return summary;
+}

package/dist/storage/stored-files-server.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/storage/stored-files-server.test.node.js

@@ -0,0 +1,216 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { createHash } from 'node:crypto';
+import { Readable } from 'node:stream';
+import { describe, expect, it, vi } from 'vitest';
+import { ErrorCodes } from '../lib/errors.js';
+import { hashStream, uploadWithContentHash } from './stored-files-server.js';
+const sha = (bytes) => createHash('sha256').update(bytes).digest('hex');
+/** A source that never ends on its own. */
+function endless() {
+    return new Readable({
+        read() {
+            this.push(Buffer.alloc(1024, 1));
+        },
+    });
+}
+/** A provider that drains the stream (or takes the buffer) and records what it stored. */
+function drainingProvider(options) {
+    const stored = new Map();
+    const upload = vi.fn(async (input, opts) => {
+        const storagePath = opts.targetStoragePath ?? 'media/generated-abc123.bin';
+        if (Buffer.isBuffer(input)) {
+            stored.set(storagePath, input);
+        }
+        else if (options?.consume !== 'none') {
+            const chunks = [];
+            for await (const chunk of input)
+                chunks.push(chunk);
+            stored.set(storagePath, Buffer.concat(chunks));
+        }
+        else {
+            stored.set(storagePath, Buffer.alloc(0));
+        }
+        return {
+            storageProvider: 'test',
+            storagePath,
+            storageUrl: null,
+            storageValidator: { kind: 'test', value: '1' },
+        };
+    });
+    const del = vi.fn(async (storagePath) => {
+        stored.delete(storagePath);
+    });
+    const storage = {
+        providerName: 'test',
+        upload,
+        delete: del,
+        getUrl: (p) => `/files/${p}`,
+    };
+    return { storage, upload, del, stored };
+}
+const opts = (size, extra) => ({
+    filename: 'a.bin',
+    mimeType: 'application/octet-stream',
+    size,
+    ...extra,
+});
+describe('hashStream', () => {
+    it('hashes a stream to its end and counts bytes', async () => {
+        const bytes = Buffer.from('hello world');
+        await expect(hashStream(Readable.from([bytes.subarray(0, 5), bytes.subarray(5)]))).resolves.toEqual({
+            fileHash: sha(bytes),
+            size: bytes.length,
+        });
+    });
+    it('rejects an already-aborted signal and destroys the stream', async () => {
+        const controller = new AbortController();
+        controller.abort(new Error('stop'));
+        const stream = Readable.from([Buffer.from('x')]);
+        await expect(hashStream(stream, { signal: controller.signal })).rejects.toThrow('stop');
+        expect(stream.destroyed).toBe(true);
+    });
+    it('destroys the stream when aborted mid-read', async () => {
+        const controller = new AbortController();
+        let chunks = 0;
+        // Aborts from inside the source after a few chunks, so the test does not
+        // depend on a timer firing between synchronous pushes.
+        const stream = new Readable({
+            read() {
+                chunks += 1;
+                if (chunks === 4)
+                    controller.abort(new Error('later'));
+                this.push(Buffer.alloc(1024, 1));
+            },
+        });
+        await expect(hashStream(stream, { signal: controller.signal })).rejects.toThrow();
+        expect(stream.destroyed).toBe(true);
+    });
+});
+describe('uploadWithContentHash', () => {
+    it('hashes a buffer and returns the provider location', async () => {
+        const { storage, stored } = drainingProvider();
+        const bytes = Buffer.from('buffered bytes');
+        const result = await uploadWithContentHash(storage, bytes, opts(bytes.length));
+        expect(result.fileHash).toBe(sha(bytes));
+        expect(result.location.storageValidator).toEqual({ kind: 'test', value: '1' });
+        expect(stored.get(result.location.storagePath)).toEqual(bytes);
+    });
+    it('rejects a buffer of the wrong length before calling the provider', async () => {
+        const { storage, upload } = drainingProvider();
+        await expect(uploadWithContentHash(storage, Buffer.from('abc'), opts(4))).rejects.toMatchObject({
+            code: ErrorCodes.VALIDATION,
+            details: expect.objectContaining({ reason: 'size-mismatch' }),
+        });
+        expect(upload).not.toHaveBeenCalled();
+    });
+    it('hashes a stream in the same pass the provider consumes it', async () => {
+        const { storage, stored } = drainingProvider();
+        const parts = [Buffer.from('chunk-one;'), Buffer.from('chunk-two;'), Buffer.from('three')];
+        const whole = Buffer.concat(parts);
+        const result = await uploadWithContentHash(storage, Readable.from(parts), opts(whole.length));
+        expect(result.fileHash).toBe(sha(whole));
+        expect(stored.get(result.location.storagePath)).toEqual(whole);
+    });
+    it('fails a short stream and deletes the provider-generated object', async () => {
+        const { storage, del, stored } = drainingProvider();
+        await expect(uploadWithContentHash(storage, Readable.from([Buffer.from('abc')]), opts(10))).rejects.toMatchObject({
+            code: ErrorCodes.STORAGE,
+            details: expect.objectContaining({
+                reason: 'size-mismatch',
+                actualSize: 3,
+                declaredSize: 10,
+                objectDeleted: true,
+                replacementUndone: false,
+            }),
+        });
+        expect(del).toHaveBeenCalledWith('media/generated-abc123.bin');
+        expect(stored.size).toBe(0);
+    });
+    it('fails a long stream', async () => {
+        const { storage } = drainingProvider();
+        await expect(uploadWithContentHash(storage, Readable.from([Buffer.from('abcdef')]), opts(3))).rejects.toMatchObject({ details: expect.objectContaining({ reason: 'size-mismatch' }) });
+    });
+    it('does not delete a verbatim targetStoragePath write it cannot undo', async () => {
+        const { storage, del } = drainingProvider();
+        await expect(uploadWithContentHash(storage, Readable.from([Buffer.from('abc')]), opts(10, { targetStoragePath: 'media/fixed.bin' }))).rejects.toMatchObject({
+            details: expect.objectContaining({ objectDeleted: false, replacementUndone: false }),
+        });
+        expect(del).not.toHaveBeenCalled();
+    });
+    it('fails when the provider resolves without consuming the stream', async () => {
+        const { storage, del } = drainingProvider({ consume: 'none' });
+        const input = Readable.from([Buffer.from('abc')]);
+        await expect(uploadWithContentHash(storage, input, opts(3))).rejects.toMatchObject({
+            code: ErrorCodes.STORAGE,
+            details: expect.objectContaining({ reason: 'stream-not-consumed' }),
+        });
+        expect(del).toHaveBeenCalledOnce();
+        expect(input.destroyed).toBe(true);
+    });
+    describe('cleanup when the provider or source fails', () => {
+        function rejectingProvider(readBeforeFailing) {
+            let received;
+            const storage = {
+                providerName: 'test',
+                upload: async (input) => {
+                    received = input;
+                    let read = 0;
+                    if (readBeforeFailing > 0) {
+                        for await (const chunk of received) {
+                            read += chunk.length;
+                            if (read >= readBeforeFailing)
+                                break;
+                        }
+                    }
+                    throw new Error('provider refused');
+                },
+                delete: vi.fn(async () => { }),
+                getUrl: (p) => p,
+            };
+            return { storage, received: () => received };
+        }
+        it('destroys both sides when the provider rejects before reading', async () => {
+            const { storage, received } = rejectingProvider(0);
+            const input = endless();
+            await expect(uploadWithContentHash(storage, input, opts(10))).rejects.toThrow('provider refused');
+            expect(input.destroyed).toBe(true);
+            expect(received()?.destroyed).toBe(true);
+        });
+        it('destroys both sides when the provider rejects part-way', async () => {
+            const { storage, received } = rejectingProvider(4096);
+            const input = endless();
+            await expect(uploadWithContentHash(storage, input, opts(1_000_000))).rejects.toThrow('provider refused');
+            expect(input.destroyed).toBe(true);
+            expect(received()?.destroyed).toBe(true);
+        });
+        it('surfaces a source failure through the provider and releases the input', async () => {
+            const { storage } = drainingProvider();
+            const input = new Readable({ read() { } });
+            const pending = uploadWithContentHash(storage, input, opts(100));
+            input.push(Buffer.from('partial'));
+            setTimeout(() => input.destroy(new Error('source broke')), 5);
+            await expect(pending).rejects.toThrow('source broke');
+            expect(input.destroyed).toBe(true);
+        });
+        it('absorbs a source error raised after the provider has rejected', async () => {
+            const { storage } = rejectingProvider(0);
+            const input = new Readable({ read() { } });
+            await expect(uploadWithContentHash(storage, input, opts(10))).rejects.toThrow('provider refused');
+            // Would be an unhandled 'error' event (and fail the run) without cleanup.
+            input.emit('error', new Error('late'));
+        });
+        it('reports objectDeleted: false when deleting the partial object fails', async () => {
+            const { storage, del } = drainingProvider();
+            del.mockRejectedValueOnce(new Error('delete failed'));
+            await expect(uploadWithContentHash(storage, Readable.from([Buffer.from('abc')]), opts(10))).rejects.toMatchObject({
+                details: expect.objectContaining({ reason: 'size-mismatch', objectDeleted: false }),
+            });
+        });
+    });
+});

package/package.json

@@ -2,7 +2,7 @@
   "name": "@byline/core",
   "private": false,
   "license": "MPL-2.0",
-  "version": "6.10.1",
+  "version": "6.12.0",
   "engines": {
     "node": ">=22.22.2"
   },
@@ -51,6 +51,11 @@
       "import": "./dist/workflow/index.js",
       "require": "./dist/workflow/index.js"
     },
+    "./stored-files": {
+      "types": "./dist/storage/stored-files-server.d.ts",
+      "import": "./dist/storage/stored-files-server.js",
+      "require": "./dist/storage/stored-files-server.js"
+    },
     "./scheduler": {
       "types": "./dist/scheduler/index.d.ts",
       "import": "./dist/scheduler/index.js",
@@ -82,7 +87,7 @@
     "dist"
   ],
   "dependencies": {
-    "@byline/auth": "6.10.1",
+    "@byline/auth": "6.12.0",
     "dotenv": "^18.0.8",
     "pino": "^10.4.0",
     "sharp": "^0.35.5",