@evomap/evolver-core 2.0.0-beta.9 → 2.0.0

package/dist/assetstore/assetSidecarRecovery.js

@@ -5,7 +5,7 @@ import { TextDecoder } from 'node:util';
 import { canonicalize } from '../wire/index.js';
 import { LockReleaseError } from '../util/fileLock.js';
 import { AssetStoreReadLimitError, assertAssetStoreDirectory, createBufferDurableExclusive, fsyncDirectoryBestEffort, readRegularBuffer, regularFileFingerprint, replaceUtf8Durable, withAssetStoreLock, } from './assetStoreStorage.js';
-import { parseAssetSyncRecord, parseProvenanceRecord, parseReviewRecord, } from './assetSidecarRecords.js';
+import { parseAssetSyncSidecarRecord, parseProvenanceRecord, parseReviewRecord, } from './assetSidecarRecords.js';
 export class AssetSidecarRecoveryError extends Error {
     reason;
     code = 'ASSET_SIDECAR_RECOVERY_FAILED';
@@ -29,7 +29,7 @@ const SIDECAR_FILES = {
 const RECORD_PARSERS = {
     provenance: parseProvenanceRecord,
     review: parseReviewRecord,
-    'asset-sync': parseAssetSyncRecord,
+    'asset-sync': parseAssetSyncSidecarRecord,
 };
 function isErrno(error, code) {
     return typeof error === 'object' && error !== null && error.code === code;

package/dist/assetstore/assetStoreHealth.d.ts

@@ -15,6 +15,11 @@ export interface AssetStoreFileHealth {
     duplicateRows: number;
     corruptRows: number;
     hashMismatchRows: number;
+    /**
+     * Rows that fail content-hash verification but carry an active, content-bound unverified provenance waiver.
+     * They remain untrusted and are counted separately instead of being misclassified as store corruption.
+     */
+    unverifiedRows: number;
     schemaInvalidRows: number;
     unterminated: boolean;
     reason?: UnsafeAssetStorePathReason | 'base_directory' | 'lock_unavailable' | 'read_unavailable' | 'scan_limit_exceeded';
@@ -31,6 +36,7 @@ export interface AssetStoreHealthTotals {
     duplicateRows: number;
     corruptRows: number;
     hashMismatchRows: number;
+    unverifiedRows: number;
     schemaInvalidRows: number;
     unterminatedFiles: number;
 }

package/dist/assetstore/assetStoreHealth.js

@@ -1,15 +1,16 @@
 import { lstatSync } from 'node:fs';
 import { join } from 'node:path';
 import { acquireLock, releaseLock } from '../util/fileLock.js';
-import { validateWire, verifyAssetId } from '../wire/index.js';
+import { computeAssetId, validateWire, verifyAssetId } from '../wire/index.js';
 import { LOCAL_ASSET_FILES } from './assetStoreLayout.js';
 import { assertOptionalRegularFile, isReliableAssetStoreLockRelease, readUtf8Regular, UnsafeAssetStorePathError, } from './assetStoreStorage.js';
-import { parseAssetSyncRecord, parseProvenanceRecord, parseReviewRecord, parseSidecarJsonl, } from './assetSidecarRecords.js';
+import { isActiveUnverifiedProvenance } from './provenance.js';
+import { parseAssetSyncSidecarRecord, parseProvenanceRecord, parseReviewRecord, parseSidecarJsonl, } from './assetSidecarRecords.js';
 export const DEFAULT_ASSET_HEALTH_MAX_FILE_BYTES = 64 * 1024 * 1024;
 const LOCAL_ASSET_SIDECARS = [
     { kind: 'provenance', file: 'provenance.jsonl', parseRecord: parseProvenanceRecord },
     { kind: 'review', file: 'review.jsonl', parseRecord: parseReviewRecord },
-    { kind: 'asset-sync', file: 'asset-sync.jsonl', parseRecord: parseAssetSyncRecord },
+    { kind: 'asset-sync', file: 'asset-sync.jsonl', parseRecord: parseAssetSyncSidecarRecord },
 ];
 function healthScanLimit(value) {
     if (value === undefined || !Number.isFinite(value))
@@ -31,6 +32,7 @@ function emptyFile(kind, file, status, reason) {
         duplicateRows: 0,
         corruptRows: 0,
         hashMismatchRows: 0,
+        unverifiedRows: 0,
         schemaInvalidRows: 0,
         unterminated: false,
         ...(reason ? { reason } : {}),
@@ -69,6 +71,7 @@ function summarize(files, sidecars) {
         duplicateRows: files.reduce((sum, file) => sum + file.duplicateRows, 0),
         corruptRows: files.reduce((sum, file) => sum + file.corruptRows, 0),
         hashMismatchRows: files.reduce((sum, file) => sum + file.hashMismatchRows, 0),
+        unverifiedRows: files.reduce((sum, file) => sum + file.unverifiedRows, 0),
         schemaInvalidRows: files.reduce((sum, file) => sum + file.schemaInvalidRows, 0),
         unterminatedFiles: files.filter((file) => file.unterminated).length,
     };
@@ -105,7 +108,7 @@ function summarize(files, sidecars) {
         sidecars: [...sidecars],
     };
 }
-function inspectFile(baseDir, kind, file, maxFileBytes) {
+function inspectFile(baseDir, kind, file, maxFileBytes, unverifiedContentIds) {
     try {
         const path = join(baseDir, file);
         const stat = assertOptionalRegularFile(path);
@@ -114,7 +117,7 @@ function inspectFile(baseDir, kind, file, maxFileBytes) {
         if (stat.size > maxFileBytes) {
             return { ...emptyFile(kind, file, 'unavailable', 'scan_limit_exceeded'), bytes: stat.size };
         }
-        const raw = readUtf8Regular(path);
+        const raw = readUtf8Regular(path, maxFileBytes);
         if (raw === null)
             return emptyFile(kind, file, 'unavailable', 'read_unavailable');
         const rows = raw.split('\n').filter((line) => line.trim().length > 0);
@@ -123,6 +126,7 @@ function inspectFile(baseDir, kind, file, maxFileBytes) {
         let duplicateRows = 0;
         let corruptRows = 0;
         let hashMismatchRows = 0;
+        let unverifiedRows = 0;
         let schemaInvalidRows = 0;
         for (const line of rows) {
             try {
@@ -141,15 +145,22 @@ function inspectFile(baseDir, kind, file, maxFileBytes) {
                     duplicateRows += 1;
                 else
                     seen.add(assetId);
-                if (!verifyAssetId(record)) {
-                    hashMismatchRows += 1;
-                    continue;
+                const hashMatches = verifyAssetId(record);
+                if (!hashMatches) {
+                    // A hash mismatch is corruption UNLESS provenance says this id is an unverified hub reuse: the hub
+                    // rewrote the delivered bytes and reuse froze them untrusted on purpose (#570). That is expected, not
+                    // store rot, so it lands in the benign unverifiedRows bucket and never degrades the store.
+                    if (unverifiedContentIds.get(assetId) === computeAssetId(record))
+                        unverifiedRows += 1;
+                    else
+                        hashMismatchRows += 1;
                 }
-                if (kind !== 'AntiGene' && !validateWire(record).ok) {
+                const schemaValid = kind === 'AntiGene' || validateWire(record).ok;
+                if (!schemaValid) {
                     schemaInvalidRows += 1;
-                    continue;
                 }
-                validRows += 1;
+                if (hashMatches && schemaValid)
+                    validRows += 1;
             }
             catch {
                 corruptRows += 1;
@@ -174,6 +185,7 @@ function inspectFile(baseDir, kind, file, maxFileBytes) {
             duplicateRows,
             corruptRows,
             hashMismatchRows,
+            unverifiedRows,
             schemaInvalidRows,
             unterminated,
         };
@@ -184,6 +196,33 @@ function inspectFile(baseDir, kind, file, maxFileBytes) {
         return emptyFile(kind, file, 'unavailable', 'read_unavailable');
     }
 }
+/**
+ * Asset ids whose latest provenance decision marks a supported unverified frozen ingest.
+ * Read directly from `<baseDir>/provenance.jsonl` — NOT via ProvenanceStore — because inspectLocalAssetStore
+ * already holds the shared `.assetstore.lock`, and ProvenanceStore would try to re-acquire it. A missing or
+ * unreadable sidecar yields an empty map: without provenance every mismatch stays classified as corruption.
+ */
+function readUnverifiedReuseContentIds(baseDir, maxFileBytes) {
+    let raw;
+    try {
+        raw = readUtf8Regular(join(baseDir, 'provenance.jsonl'), maxFileBytes);
+    }
+    catch {
+        return new Map();
+    }
+    if (raw === null)
+        return new Map();
+    const ids = new Map();
+    for (const record of parseSidecarJsonl(raw, parseProvenanceRecord).records) {
+        const frozenContentId = record.frozenContentId;
+        if (frozenContentId && isActiveUnverifiedProvenance(record, frozenContentId)) {
+            ids.set(record.assetId, frozenContentId);
+        }
+        else
+            ids.delete(record.assetId);
+    }
+    return ids;
+}
 function inspectSidecar(baseDir, definition, maxFileBytes) {
     const { kind, file, parseRecord } = definition;
     try {
@@ -194,7 +233,7 @@ function inspectSidecar(baseDir, definition, maxFileBytes) {
         if (stat.size > maxFileBytes) {
             return { ...emptySidecar(kind, file, 'unavailable', 'scan_limit_exceeded'), bytes: stat.size };
         }
-        const raw = readUtf8Regular(path);
+        const raw = readUtf8Regular(path, maxFileBytes);
         if (raw === null)
             return emptySidecar(kind, file, 'unavailable', 'read_unavailable');
         const parsed = parseSidecarJsonl(raw, parseRecord);
@@ -255,11 +294,14 @@ export function inspectLocalAssetStore(baseDir, opts = {}, deps = {}) {
     catch {
         return summarize(allFiles('unavailable', 'lock_unavailable'), allSidecars('unavailable', 'lock_unavailable'));
     }
-    const maxFileBytes = healthScanLimit(opts.maxFileBytes);
     let report;
     try {
+        const maxFileBytes = healthScanLimit(opts.maxFileBytes);
+        // Read the unverified-reuse set once, under the lock we already hold, so every asset file classifies its
+        // hash mismatches consistently against the same bounded provenance snapshot.
+        const unverifiedContentIds = readUnverifiedReuseContentIds(baseDir, maxFileBytes);
         report = summarize(Object.entries(LOCAL_ASSET_FILES)
-            .map(([kind, file]) => inspectFile(baseDir, kind, file, maxFileBytes)), LOCAL_ASSET_SIDECARS.map((definition) => inspectSidecar(baseDir, definition, maxFileBytes)));
+            .map(([kind, file]) => inspectFile(baseDir, kind, file, maxFileBytes, unverifiedContentIds)), LOCAL_ASSET_SIDECARS.map((definition) => inspectSidecar(baseDir, definition, maxFileBytes)));
     }
     catch {
         report = summarize(allFiles('unavailable', 'read_unavailable'), allSidecars('unavailable', 'read_unavailable'));

package/dist/assetstore/assetStoreStorage.d.ts

@@ -33,7 +33,7 @@ export declare function withAssetStoreLock<T>(lockPath: string, operation: () =>
 export declare function assertOptionalRegularFile(path: string, role?: AssetStorePathRole): Stats | null;
 export declare function regularFileFingerprint(path: string): string;
 export declare function readRegularBuffer(path: string, maxBytes?: number): Buffer | null;
-export declare function readUtf8Regular(path: string): string | null;
+export declare function readUtf8Regular(path: string, maxBytes?: number): string | null;
 export declare function createBufferDurableExclusive(path: string, value: Buffer, opts?: DurableWriteOptions): void;
 export declare function fsyncDirectoryBestEffort(path: string): void;
 export declare function appendUtf8Durable(path: string, value: string, opts?: DurableWriteOptions): void;

package/dist/assetstore/assetStoreStorage.js

@@ -1,5 +1,5 @@
 import { randomUUID } from 'node:crypto';
-import { closeSync, constants, fstatSync, fsyncSync, ftruncateSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, renameSync, unlinkSync, writeSync, } from 'node:fs';
+import { closeSync, constants, fstatSync, fsyncSync, ftruncateSync, lstatSync, mkdirSync, openSync, readSync, renameSync, unlinkSync, writeSync, } from 'node:fs';
 import { basename, dirname, join } from 'node:path';
 import { acquireLock, LockReleaseError, releaseLock, } from '../util/fileLock.js';
 export class UnsafeAssetStorePathError extends Error {
@@ -152,17 +152,26 @@ export function readRegularBuffer(path, maxBytes = Number.MAX_SAFE_INTEGER) {
         assertOpenedPathMatches(fd, path, 'asset_file');
         if (fstatSync(fd).size > maxBytes)
             throw new AssetStoreReadLimitError();
-        const value = readFileSync(fd);
-        if (value.byteLength > maxBytes)
-            throw new AssetStoreReadLimitError();
-        return value;
+        const chunks = [];
+        let total = 0;
+        while (total <= maxBytes) {
+            const chunk = Buffer.allocUnsafe(Math.min(64 * 1024, maxBytes - total + 1));
+            const bytesRead = readSync(fd, chunk, 0, chunk.byteLength, null);
+            if (bytesRead === 0)
+                break;
+            total += bytesRead;
+            if (total > maxBytes)
+                throw new AssetStoreReadLimitError();
+            chunks.push(bytesRead === chunk.byteLength ? chunk : chunk.subarray(0, bytesRead));
+        }
+        return Buffer.concat(chunks, total);
     }
     finally {
         closeSync(fd);
     }
 }
-export function readUtf8Regular(path) {
-    return readRegularBuffer(path)?.toString('utf8') ?? null;
+export function readUtf8Regular(path, maxBytes = Number.MAX_SAFE_INTEGER) {
+    return readRegularBuffer(path, maxBytes)?.toString('utf8') ?? null;
 }
 function writeAll(fd, value) {
     const bytes = typeof value === 'string' ? Buffer.from(value, 'utf8') : value;
@@ -237,10 +246,19 @@ export function appendUtf8Durable(path, value, opts = {}) {
     const parent = dirname(path);
     assertAssetStoreDirectory(parent);
     const existed = assertOptionalRegularFile(path) !== null;
-    const fd = openNoFollow(path, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT, 0o600);
+    const fd = openNoFollow(path, constants.O_RDWR | constants.O_APPEND | constants.O_CREAT, 0o600);
     try {
         assertOpenedPathMatches(fd, path, 'asset_file');
-        writeAll(fd, value);
+        const stat = fstatSync(fd);
+        let needsLineSeparator = false;
+        if (stat.size > 0 && value.length > 0) {
+            const tail = Buffer.allocUnsafe(1);
+            const bytesRead = readSync(fd, tail, 0, 1, stat.size - 1);
+            if (bytesRead !== 1)
+                throw new Error('asset store tail read made no progress');
+            needsLineSeparator = tail[0] !== 0x0a;
+        }
+        writeAll(fd, needsLineSeparator ? `\n${value}` : value);
         (opts.syncFile ?? fsyncSync)(fd);
     }
     finally {

package/dist/assetstore/assetSyncLedger.d.ts

@@ -1,4 +1,8 @@
 import type { AssetKind } from './provider.js';
+export declare const ASSET_SYNC_INVENTORY_MAX_SEGMENTS = 24;
+export declare const ASSET_SYNC_INVENTORY_MAX_ITEMS_PER_SEGMENT = 10000;
+export declare const ASSET_SYNC_INVENTORY_MAX_UNIQUE_ITEMS = 240000;
+export declare const ASSET_SYNC_INVENTORY_MAX_TOTAL_BYTES: number;
 export type AssetSyncSource = 'hub';
 export type AssetSyncScope = 'purchased' | 'published';
 export interface AssetSyncRecord {
@@ -8,25 +12,107 @@ export interface AssetSyncRecord {
     scope: AssetSyncScope;
     syncedAt: string;
     remoteAssetId: string;
+    /** Opaque sync identity hash. Legacy records omit it and are not used for identity-scoped reconciliation. */
+    runKey?: string;
+    /** Parameter-independent remote inventory identity used for reconciliation across sync runs. */
+    inventoryKey?: string;
     logicalId?: string;
     status?: string;
     forced?: true;
     collisionWithAssetId?: string;
 }
+export type AssetSyncRunState = 'started' | 'progress' | 'completed';
+export type AssetSyncRunOutcome = 'imported' | 'already_local' | 'failed' | 'remote_missing';
+export interface AssetSyncRunRecord {
+    recordType: 'run';
+    runId: string;
+    runKey: string;
+    state: AssetSyncRunState;
+    remoteAssetId?: string;
+    outcome?: AssetSyncRunOutcome;
+    reason?: string;
+    /** Ordered candidate IDs selected when this run was started. */
+    plan?: readonly string[];
+    syncedAt: string;
+}
+export interface AssetSyncRunSnapshot extends AssetSyncRunRecord {
+    readonly processed: ReadonlyMap<string, AssetSyncRunRecord>;
+}
+export type AssetSyncInventoryOutcome = 'imported' | 'already_local' | 'blocked' | 'failed' | 'pending';
+export interface AssetSyncInventoryCursorFingerprints {
+    readonly purchased: string | null;
+    readonly published: string | null;
+}
+export interface AssetSyncInventoryItem {
+    readonly remoteAssetId: string;
+    readonly outcome: AssetSyncInventoryOutcome;
+}
+export interface AssetSyncInventorySegmentRecord {
+    readonly recordType: 'inventory_scan';
+    readonly scanId: string;
+    readonly inventoryKey: string;
+    readonly scope: 'purchased' | 'published' | 'all';
+    readonly index: number;
+    readonly inputCursorFingerprints: AssetSyncInventoryCursorFingerprints;
+    readonly nextCursorFingerprints: AssetSyncInventoryCursorFingerprints;
+    readonly items: readonly AssetSyncInventoryItem[];
+    readonly anonymousBlocked: number;
+    /** This segment must be replayed from its input cursor before the scan may advance. */
+    readonly cursorHeld?: true;
+    /** Multi-segment batches are applied only after every physical segment is present. */
+    readonly batchId?: string;
+    readonly batchIndex?: number;
+    readonly batchSize?: number;
+    readonly syncedAt: string;
+}
+export type AssetSyncInventoryBatch = Omit<AssetSyncInventorySegmentRecord, 'recordType' | 'syncedAt' | 'batchId' | 'batchIndex' | 'batchSize'> & {
+    syncedAt?: string;
+};
+export interface AssetSyncInventorySnapshot {
+    readonly scanId: string;
+    readonly inventoryKey: string;
+    readonly scope: 'purchased' | 'published' | 'all';
+    readonly segmentCount: number;
+    readonly nextCursorFingerprints: AssetSyncInventoryCursorFingerprints;
+    readonly outcomes: ReadonlyMap<string, AssetSyncInventoryOutcome>;
+    readonly anonymousBlocked: number;
+    readonly complete: boolean;
+    /** Physical segment index whose input cursor must be replayed before this scan can advance. */
+    readonly retryIndex?: number;
+}
 export declare class AssetSyncLedger {
     private readonly now;
     private readonly path;
     private readonly lockPath;
     private readonly index;
+    private readonly runKeyIndex;
+    private readonly inventoryKeyIndex;
     private fileState;
     private loaded;
     constructor(baseDir: string, now?: () => number);
     append(rec: Omit<AssetSyncRecord, 'syncedAt'> & {
         syncedAt?: string;
     }): AssetSyncRecord;
+    appendRun(rec: Omit<AssetSyncRunRecord, 'recordType' | 'syncedAt'> & {
+        syncedAt?: string;
+    }): AssetSyncRunRecord;
+    appendInventorySegment(rec: Omit<AssetSyncInventorySegmentRecord, 'recordType' | 'syncedAt'> & {
+        syncedAt?: string;
+    }): AssetSyncInventorySegmentRecord | null;
+    appendInventoryBatch(rec: AssetSyncInventoryBatch): readonly AssetSyncInventorySegmentRecord[] | null;
+    replaceInventoryRetryBatch(rec: AssetSyncInventoryBatch): readonly AssetSyncInventorySegmentRecord[] | null;
+    clearInventoryScan(inventoryKey: string): void;
+    runRecords(runId: string): AssetSyncRunRecord[];
+    latestIncompleteRun(runKey: string): AssetSyncRunSnapshot | undefined;
+    latestInventoryScan(inventoryKey: string): AssetSyncInventorySnapshot | undefined;
     get(assetId: string): AssetSyncRecord | null;
+    getForRunKey(runKey: string, assetId: string): AssetSyncRecord | null;
     list(): AssetSyncRecord[];
+    listForRunKey(runKey: string): AssetSyncRecord[];
+    listForInventoryKey(inventoryKey: string): AssetSyncRecord[];
     private rebuildIndex;
     private refreshUnderLock;
     private withFreshRead;
+    private readRunRecordsUnderLock;
+    private latestInventorySnapshotUnderLock;
 }