@byline/core 6.11.0 → 6.12.0

package/dist/storage/storage-flatten-reconstruct.test.node.js

@@ -406,3 +406,62 @@ describe('virtual fields', () => {
         expect(items[0]?.filesGroup.thumbnailPage).toBeUndefined();
     });
 });
+describe('stored-file content identity round trip', () => {
+    const FilesCollection = defineCollection({
+        path: 'identity-files',
+        labels: { singular: 'File', plural: 'Files' },
+        fields: [{ name: 'attachment', label: 'Attachment', type: 'file' }],
+    });
+    const baseFile = {
+        fileId: 'file-1',
+        filename: 'report.pdf',
+        originalFilename: 'Report.pdf',
+        mimeType: 'application/pdf',
+        fileSize: 3,
+        storageProvider: 'local',
+        storagePath: 'media/report.pdf',
+        processingStatus: 'complete',
+    };
+    const HASH = 'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad';
+    it('round-trips fileHash and storageValidator through the flattened row', () => {
+        const flattened = flattenFieldSetData(FilesCollection.fields, {
+            attachment: {
+                ...baseFile,
+                fileHash: HASH,
+                storageValidator: { kind: 'local-stat', value: '1:2:3:3' },
+            },
+        }, 'all');
+        const row = flattened[0];
+        expect(row.file_hash).toBe(HASH);
+        expect(row.storage_validator_kind).toBe('local-stat');
+        expect(row.storage_validator_value).toBe('1:2:3:3');
+        const { data } = restoreFieldSetData(FilesCollection.fields, flattened);
+        expect(data.attachment).toMatchObject({
+            fileHash: HASH,
+            storageValidator: { kind: 'local-stat', value: '1:2:3:3' },
+        });
+    });
+    it('omits unknown identity instead of restoring null', () => {
+        const flattened = flattenFieldSetData(FilesCollection.fields, { attachment: baseFile }, 'all').map((row) => ({
+            ...row,
+            file_hash: null,
+            storage_validator_kind: null,
+            storage_validator_value: null,
+        }));
+        const { data } = restoreFieldSetData(FilesCollection.fields, flattened);
+        const attachment = data.attachment;
+        expect('fileHash' in attachment).toBe(false);
+        expect('storageValidator' in attachment).toBe(false);
+    });
+    it('restores a validator only as a complete pair', () => {
+        const flattened = flattenFieldSetData(FilesCollection.fields, { attachment: { ...baseFile, fileHash: HASH } }, 'all').map((row) => ({
+            ...row,
+            storage_validator_kind: 'local-stat',
+            storage_validator_value: null,
+        }));
+        const { data } = restoreFieldSetData(FilesCollection.fields, flattened);
+        const attachment = data.attachment;
+        expect(attachment.fileHash).toBe(HASH);
+        expect('storageValidator' in attachment).toBe(false);
+    });
+});

package/dist/storage/storage-flatten.js

@@ -269,6 +269,8 @@ const flattenValueFieldData = (field, field_path, value, locale) => {
                 storage_path: v.storagePath,
                 storage_url: v.storageUrl,
                 file_hash: v.fileHash,
+                storage_validator_kind: v.storageValidator?.kind,
+                storage_validator_value: v.storageValidator?.value,
                 image_width: v.imageWidth,
                 image_height: v.imageHeight,
                 image_format: v.imageFormat,

package/dist/storage/storage-restore.js

@@ -300,7 +300,17 @@ const extractValueFieldData = (data) => {
                 storageProvider: data.storage_provider,
                 storagePath: data.storage_path,
                 storageUrl: data.storage_url,
-                fileHash: data.file_hash,
+                // Content identity is omitted, never null or '', when unknown. The
+                // validator is restored only as a complete pair.
+                ...(data.file_hash ? { fileHash: data.file_hash } : {}),
+                ...(data.storage_validator_kind && data.storage_validator_value
+                    ? {
+                        storageValidator: {
+                            kind: data.storage_validator_kind,
+                            value: data.storage_validator_value,
+                        },
+                    }
+                    : {}),
                 imageWidth: data.image_width,
                 imageHeight: data.image_height,
                 imageFormat: data.image_format,
@@ -379,6 +389,8 @@ export const extractFlattenedFieldValue = (unifiedValue) => {
                 storage_path: unifiedValue.storage_path,
                 storage_url: orUndefined(unifiedValue.storage_url),
                 file_hash: orUndefined(unifiedValue.file_hash),
+                storage_validator_kind: orUndefined(unifiedValue.storage_validator_kind),
+                storage_validator_value: orUndefined(unifiedValue.storage_validator_value),
                 image_width: orUndefined(unifiedValue.image_width),
                 image_height: orUndefined(unifiedValue.image_height),
                 image_format: orUndefined(unifiedValue.image_format),

package/dist/storage/storage-row-types.d.ts

@@ -43,6 +43,8 @@ interface FlattenedFileFieldValue extends BaseFlattenedFieldData {
     storage_path: string;
     storage_url?: string;
     file_hash?: string;
+    storage_validator_kind?: string;
+    storage_validator_value?: string;
     image_width?: number;
     image_height?: number;
     image_format?: string;
@@ -93,6 +95,8 @@ export interface UnifiedFieldValue {
     storage_path: string | null;
     storage_url: string | null;
     file_hash: string | null;
+    storage_validator_kind: string | null;
+    storage_validator_value: string | null;
     image_width: number | null;
     image_height: number | null;
     image_format: string | null;

package/dist/storage/store-manifest.js

@@ -129,6 +129,16 @@ export const storeColumnManifest = [
         nullCast: 'varchar',
         sources: { file: 'file_hash' },
     },
+    {
+        name: 'storage_validator_kind',
+        nullCast: 'varchar',
+        sources: { file: 'storage_validator_kind' },
+    },
+    {
+        name: 'storage_validator_value',
+        nullCast: 'varchar',
+        sources: { file: 'storage_validator_value' },
+    },
     {
         name: 'image_width',
         nullCast: 'integer',

package/dist/storage/store-manifest.test.node.js

@@ -16,7 +16,7 @@ import { storeColumnManifest, storeTableNames } from './store-manifest.js';
  */
 describe('store-manifest', () => {
     it('pins the manifest column count', () => {
-        expect(storeColumnManifest.length).toBe(39);
+        expect(storeColumnManifest.length).toBe(41);
     });
     it('pins the manifest column order', () => {
         expect(storeColumnManifest.map((col) => col.name)).toEqual([
@@ -43,6 +43,8 @@ describe('store-manifest', () => {
             'storage_path',
             'storage_url',
             'file_hash',
+            'storage_validator_kind',
+            'storage_validator_value',
             'image_width',
             'image_height',
             'image_format',
@@ -105,6 +107,8 @@ describe('store-manifest', () => {
             'storage_path',
             'storage_url',
             'file_hash',
+            'storage_validator_kind',
+            'storage_validator_value',
             'image_width',
             'image_height',
             'image_format',

package/dist/storage/stored-files-backfill.test.node.d.ts

@@ -0,0 +1,8 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+export {};

package/dist/storage/stored-files-backfill.test.node.js

@@ -0,0 +1,251 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import { createHash } from 'node:crypto';
+import { Readable } from 'node:stream';
+import { describe, expect, it, vi } from 'vitest';
+import { ErrorCodes } from '../lib/errors.js';
+import { backfillStoredFileIdentity } from './stored-files-server.js';
+const sha = (text) => createHash('sha256').update(Buffer.from(text)).digest('hex');
+/** In-memory adapter with the documented paging and update semantics. */
+function fakeDb(rows, collections = { media: 'col-media' }) {
+    const matches = (row, p) => row.storageProvider === p.storageProvider &&
+        (p.collectionId === undefined || row.collectionId === p.collectionId);
+    const queries = {
+        async listStoredFilePaths(p) {
+            const selected = rows.filter((row) => matches(row, p) &&
+                (p.identity === 'missing' ? row.fileHash === null : row.fileHash !== null) &&
+                (p.afterPath === undefined || row.storagePath > p.afterPath));
+            const paths = [...new Set(selected.map((row) => row.storagePath))].sort().slice(0, p.limit);
+            return paths.map((storagePath) => {
+                if (p.identity === 'missing')
+                    return { storagePath, hashes: [] };
+                const counts = new Map();
+                for (const row of selected) {
+                    if (row.storagePath === storagePath && row.fileHash) {
+                        counts.set(row.fileHash, (counts.get(row.fileHash) ?? 0) + 1);
+                    }
+                }
+                return {
+                    storagePath,
+                    hashes: [...counts].sort().map(([fileHash, rowCount]) => ({ fileHash, rowCount })),
+                };
+            });
+        },
+    };
+    const setIdentityForPath = vi.fn(async (p) => {
+        let updated = 0;
+        for (const row of rows) {
+            if (matches(row, p) && row.storagePath === p.storagePath && row.fileHash === null) {
+                row.fileHash = p.fileHash;
+                row.validator = p.storageValidator;
+                updated += 1;
+            }
+        }
+        return updated;
+    });
+    const db = {
+        commands: { storedFiles: { setIdentityForPath } },
+        queries: {
+            storedFiles: queries,
+            collections: {
+                getCollectionByPath: async (path) => collections[path] ? { id: collections[path], path } : null,
+            },
+        },
+    };
+    return { db, setIdentityForPath };
+}
+/** A provider whose objects are in-memory strings; `download` is a bound read. */
+function fakeStorage(objects, options) {
+    const download = vi.fn(async (storagePath) => {
+        if (options?.failOn?.includes(storagePath))
+            throw new Error(`read failed: ${storagePath}`);
+        const content = objects[storagePath];
+        if (content === undefined)
+            return null;
+        return {
+            body: Readable.from([Buffer.from(content)]),
+            validator: { kind: 'test', value: `v:${storagePath}:${content.length}` },
+            size: content.length,
+        };
+    });
+    const storage = {
+        providerName: 'local',
+        upload: vi.fn(),
+        delete: vi.fn(),
+        getUrl: (p) => p,
+        download,
+    };
+    return { storage, download };
+}
+const row = (storagePath, extra) => ({
+    storageProvider: 'local',
+    storagePath,
+    collectionId: 'col-media',
+    fileHash: null,
+    validator: null,
+    ...extra,
+});
+describe('backfillStoredFileIdentity — fill', () => {
+    it('reads each path once and records hash and validator on every null row', async () => {
+        const rows = [row('media/a.pdf'), row('media/a.pdf'), row('media/b.pdf')];
+        const { db } = fakeDb(rows);
+        const { storage, download } = fakeStorage({ 'media/a.pdf': 'AAA', 'media/b.pdf': 'BB' });
+        const summary = await backfillStoredFileIdentity({ db, storage });
+        expect(summary).toMatchObject({ mode: 'fill', scanned: 2, hashed: 2, rowsUpdated: 3 });
+        expect(download).toHaveBeenCalledTimes(2);
+        expect(rows.map((r) => r.fileHash)).toEqual([sha('AAA'), sha('AAA'), sha('BB')]);
+        expect(rows[0]?.validator).toEqual({ kind: 'test', value: 'v:media/a.pdf:3' });
+    });
+    it('makes no changes on a second run', async () => {
+        const rows = [row('media/a.pdf')];
+        const { db, setIdentityForPath } = fakeDb(rows);
+        const { storage } = fakeStorage({ 'media/a.pdf': 'AAA' });
+        await backfillStoredFileIdentity({ db, storage });
+        setIdentityForPath.mockClear();
+        const again = await backfillStoredFileIdentity({ db, storage });
+        expect(again).toMatchObject({ scanned: 0, rowsUpdated: 0 });
+        expect(setIdentityForPath).not.toHaveBeenCalled();
+    });
+    it('never overwrites a recorded hash', async () => {
+        const rows = [row('media/a.pdf', { fileHash: sha('OLD') }), row('media/a.pdf')];
+        const { db } = fakeDb(rows);
+        const { storage } = fakeStorage({ 'media/a.pdf': 'NEW' });
+        await backfillStoredFileIdentity({ db, storage });
+        expect(rows.map((r) => r.fileHash)).toEqual([sha('OLD'), sha('NEW')]);
+    });
+    it('resumes cleanly after a run stopped by its limit', async () => {
+        const rows = [row('media/a.pdf'), row('media/b.pdf'), row('media/c.pdf')];
+        const { db } = fakeDb(rows);
+        const { storage } = fakeStorage({ 'media/a.pdf': 'A', 'media/b.pdf': 'B', 'media/c.pdf': 'C' });
+        const first = await backfillStoredFileIdentity({ db, storage, limit: 1, batchSize: 1 });
+        const rest = await backfillStoredFileIdentity({ db, storage, batchSize: 1 });
+        expect(first.scanned).toBe(1);
+        expect(rest.scanned).toBe(2);
+        expect(rows.every((r) => r.fileHash !== null)).toBe(true);
+    });
+    it('leaves missing and unreadable objects without identity and reports them', async () => {
+        const rows = [row('media/gone.pdf'), row('media/bad.pdf'), row('media/ok.pdf')];
+        const { db } = fakeDb(rows);
+        const { storage } = fakeStorage({ 'media/bad.pdf': 'x', 'media/ok.pdf': 'ok' }, { failOn: ['media/bad.pdf'] });
+        const summary = await backfillStoredFileIdentity({ db, storage });
+        expect(summary).toMatchObject({ scanned: 3, hashed: 1, missing: 1, failed: 1, rowsUpdated: 1 });
+        expect(summary.missingPaths).toEqual(['media/gone.pdf']);
+        expect(summary.failures).toEqual([
+            { storagePath: 'media/bad.pdf', error: 'read failed: media/bad.pdf' },
+        ]);
+        expect(rows.map((r) => r.fileHash === null)).toEqual([true, true, false]);
+    });
+    it('writes nothing in a dry run', async () => {
+        const rows = [row('media/a.pdf')];
+        const { db, setIdentityForPath } = fakeDb(rows);
+        const { storage } = fakeStorage({ 'media/a.pdf': 'AAA' });
+        const summary = await backfillStoredFileIdentity({ db, storage, dryRun: true });
+        expect(summary).toMatchObject({ dryRun: true, scanned: 1, hashed: 1, rowsUpdated: 0 });
+        expect(setIdentityForPath).not.toHaveBeenCalled();
+        expect(rows[0]?.fileHash).toBeNull();
+    });
+    it('selects only rows written by the configured provider', async () => {
+        const rows = [row('media/a.pdf'), row('media/a.pdf', { storageProvider: 's3' })];
+        const { db } = fakeDb(rows);
+        const { storage } = fakeStorage({ 'media/a.pdf': 'AAA' });
+        await backfillStoredFileIdentity({ db, storage });
+        expect(rows.map((r) => r.fileHash)).toEqual([sha('AAA'), null]);
+    });
+    it('confines a collection-scoped run to that collection', async () => {
+        const rows = [row('shared/a.pdf'), row('shared/a.pdf', { collectionId: 'col-docs' })];
+        const { db } = fakeDb(rows, { media: 'col-media', docs: 'col-docs' });
+        const { storage } = fakeStorage({ 'shared/a.pdf': 'AAA' });
+        const summary = await backfillStoredFileIdentity({ db, storage, collectionPath: 'media' });
+        expect(summary.rowsUpdated).toBe(1);
+        expect(rows.map((r) => r.fileHash)).toEqual([sha('AAA'), null]);
+    });
+});
+describe('backfillStoredFileIdentity — refusals', () => {
+    it('refuses a provider without download', async () => {
+        const { db } = fakeDb([row('media/a.pdf')]);
+        const { storage } = fakeStorage({});
+        delete storage.download;
+        await expect(backfillStoredFileIdentity({ db, storage })).rejects.toMatchObject({
+            code: ErrorCodes.STORAGE,
+            details: expect.objectContaining({ reason: 'download-unavailable' }),
+        });
+    });
+    it('refuses an adapter without stored-file maintenance', async () => {
+        const { storage } = fakeStorage({});
+        const db = { commands: {}, queries: {} };
+        await expect(backfillStoredFileIdentity({ db, storage })).rejects.toMatchObject({
+            code: ErrorCodes.VALIDATION,
+            details: expect.objectContaining({ reason: 'adapter-unsupported' }),
+        });
+    });
+    it('rejects an unknown collection path', async () => {
+        const { db } = fakeDb([row('media/a.pdf')]);
+        const { storage } = fakeStorage({});
+        await expect(backfillStoredFileIdentity({ db, storage, collectionPath: 'nope' })).rejects.toMatchObject({
+            code: ErrorCodes.VALIDATION,
+            details: expect.objectContaining({ reason: 'unknown-collection' }),
+        });
+    });
+});
+describe('backfillStoredFileIdentity — verify', () => {
+    it('reports matched and mismatched hash groups and writes nothing', async () => {
+        const rows = [
+            row('media/a.pdf', { fileHash: sha('AAA') }),
+            row('media/b.pdf', { fileHash: sha('STALE') }),
+            row('media/c.pdf'),
+        ];
+        const { db, setIdentityForPath } = fakeDb(rows);
+        const { storage } = fakeStorage({ 'media/a.pdf': 'AAA', 'media/b.pdf': 'CURRENT' });
+        const summary = await backfillStoredFileIdentity({ db, storage, mode: 'verify' });
+        expect(summary).toMatchObject({ mode: 'verify', scanned: 2, matched: 1, mismatched: 1 });
+        expect(summary.mismatches).toEqual([
+            {
+                storagePath: 'media/b.pdf',
+                recordedHash: sha('STALE'),
+                currentHash: sha('CURRENT'),
+                rowCount: 1,
+            },
+        ]);
+        expect(setIdentityForPath).not.toHaveBeenCalled();
+        expect(rows[2]?.fileHash).toBeNull();
+    });
+    it('pages by whole paths: batchSize 1 reports every hash group on a multi-hash path', async () => {
+        // The first path carries two different recorded hashes (an older version
+        // recorded bytes later replaced at the same path); another path follows.
+        // A row of the same object in another collection must not be touched.
+        const rows = [
+            row('media/a.pdf', { fileHash: sha('OLD') }),
+            row('media/a.pdf', { fileHash: sha('NEW') }),
+            row('media/a.pdf', { fileHash: sha('NEW') }),
+            row('media/b.pdf', { fileHash: sha('B') }),
+            row('media/a.pdf', { collectionId: 'col-docs' }),
+        ];
+        const before = structuredClone(rows);
+        const { db, setIdentityForPath } = fakeDb(rows, { media: 'col-media', docs: 'col-docs' });
+        const { storage, download } = fakeStorage({ 'media/a.pdf': 'NEW', 'media/b.pdf': 'B' });
+        const summary = await backfillStoredFileIdentity({
+            db,
+            storage,
+            mode: 'verify',
+            batchSize: 1,
+            collectionPath: 'media',
+        });
+        expect(summary).toMatchObject({ scanned: 2, hashed: 2, matched: 2, mismatched: 1 });
+        expect(summary.mismatches).toEqual([
+            {
+                storagePath: 'media/a.pdf',
+                recordedHash: sha('OLD'),
+                currentHash: sha('NEW'),
+                rowCount: 1,
+            },
+        ]);
+        expect(download.mock.calls.map((call) => call[0])).toEqual(['media/a.pdf', 'media/b.pdf']);
+        expect(setIdentityForPath).not.toHaveBeenCalled();
+        expect(rows).toEqual(before);
+    });
+});

package/dist/storage/stored-files-server.d.ts

@@ -0,0 +1,104 @@
+/**
+ * This Source Code is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ *
+ * Copyright (c) Infonomic Company Limited
+ */
+import type { IDbAdapter } from '../@types/db-types.js';
+import type { IStorageProvider, StoredFileLocation, UploadFileOptions } from '../@types/storage-types.js';
+import type { BylineLogger } from '../lib/logger.js';
+/** SHA-256 (lower-case hex) and byte count of a stream, read to its end. */
+export declare function hashStream(stream: NodeJS.ReadableStream, options?: {
+    signal?: AbortSignal;
+}): Promise<{
+    fileHash: string;
+    size: number;
+}>;
+/**
+ * Upload an original object and compute its content hash in the same pass.
+ *
+ * - A `Buffer` is hashed in memory; a length that differs from `options.size`
+ *   fails before the provider is called.
+ * - A stream is piped through a hashing pass-through as the provider consumes
+ *   it. If it ends short, runs long, or is not fully consumed when the provider
+ *   resolves, the upload fails, the input is destroyed, and no hash is
+ *   returned. The object is deleted when the provider generated its path; a
+ *   `targetStoragePath` write may have replaced an existing object, which
+ *   cannot be undone, and the error says so.
+ */
+export declare function uploadWithContentHash(storage: IStorageProvider, input: NodeJS.ReadableStream | Buffer, options: UploadFileOptions): Promise<{
+    location: StoredFileLocation;
+    fileHash: string;
+}>;
+/** Options for {@link backfillStoredFileIdentity}. */
+export interface BackfillStoredFileIdentityOptions {
+    db: IDbAdapter;
+    storage: IStorageProvider;
+    /**
+     * `'fill'` (default): record `fileHash` and `storageValidator` on rows
+     * without a hash. `'verify'`: re-hash paths that already have a recorded
+     * hash and report matches and mismatches; writes nothing.
+     */
+    mode?: 'fill' | 'verify';
+    /** Restrict selection and updates to one collection's rows. */
+    collectionPath?: string;
+    /** Distinct paths per page. Default 100. */
+    batchSize?: number;
+    /** Maximum distinct paths to process in this run. Default: no limit. */
+    limit?: number;
+    /** Fill mode only: read and hash, but write nothing. */
+    dryRun?: boolean;
+    signal?: AbortSignal;
+    logger?: Pick<BylineLogger, 'info' | 'warn'>;
+}
+/** Per-run summary. Each path list keeps at most 1000 entries; counts are never capped. */
+export interface BackfillStoredFileIdentitySummary {
+    mode: 'fill' | 'verify';
+    dryRun: boolean;
+    /** Distinct paths selected. */
+    scanned: number;
+    /** Paths read and hashed. */
+    hashed: number;
+    /** Fill mode: rows updated (0 in a dry run). */
+    rowsUpdated: number;
+    /** Paths whose object no longer exists. */
+    missing: number;
+    /** Paths that could not be read or hashed. */
+    failed: number;
+    /** Verify mode: recorded-hash groups equal to the current bytes. */
+    matched: number;
+    /** Verify mode: recorded-hash groups that differ from the current bytes. */
+    mismatched: number;
+    missingPaths: string[];
+    failures: Array<{
+        storagePath: string;
+        error: string;
+    }>;
+    mismatches: Array<{
+        storagePath: string;
+        recordedHash: string;
+        currentHash: string;
+        rowCount: number;
+    }>;
+}
+/**
+ * Fill or verify stored-file content identity for existing `byline_store_file`
+ * rows (spec 2026-10-31 §4.8).
+ *
+ * - Selects only rows written by `storage.providerName`, optionally one
+ *   collection's rows, paging by distinct storage path. Each path is read once
+ *   per run with a bound `download`, so the recorded validator always belongs
+ *   to the bytes that were hashed.
+ * - Fill never overwrites a recorded hash, so re-running is idempotent and an
+ *   interrupted run resumes cleanly. It hashes the bytes present now: where an
+ *   older version's object was later replaced at the same path, the recorded
+ *   hash describes the current bytes, which the validator identifies.
+ * - Legacy-null policy: a missing object, or a read or hash failure, leaves the
+ *   rows without identity and is reported. A provider without `download`, or
+ *   an adapter without stored-file maintenance, refuses to start.
+ *
+ * This is a maintenance primitive: it uses direct `db.queries` / `db.commands`
+ * calls and bypasses collection auth and hooks.
+ */
+export declare function backfillStoredFileIdentity(options: BackfillStoredFileIdentityOptions): Promise<BackfillStoredFileIdentitySummary>;