@murphai/murph 1.2.0 → 1.2.3

package/node_modules/@murphai/core/dist/integration-ingests.js

@@ -1,7 +1,11 @@
 import { createHash } from "node:crypto";
 import { createReadStream } from "node:fs";
+import { open as openFile, readFile, stat } from "node:fs/promises";
 import { createInterface } from "node:readline";
+import { Readable } from "node:stream";
+import { createGunzip, crc32, deflateRawSync, gzipSync, inflateRawSync } from "node:zlib";
 import { integrationIngestReceiptSchema, integrationIngestRecordSchema, } from "@murphai/contracts";
+import { writeFileAtomic } from "./atomic-write.js";
 import { VAULT_LAYOUT } from "./constants.js";
 import { VaultError } from "./errors.js";
 import { pathExists, walkVaultFiles } from "./fs.js";
@@ -11,6 +15,20 @@ export const MAX_INTEGRATION_INGEST_PARTS = 10_000;
 export const MAX_INTEGRATION_EVIDENCE_PART_BYTES = 100 * 1024 * 1024;
 export const MAX_INTEGRATION_INGEST_BYTES = 100 * 1024 * 1024;
 export const MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES = 128 * 1024 * 1024;
+export const MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES = 128 * 1024 * 1024;
+export const MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES = 256 * 1024 * 1024;
+const INTEGRATION_INGEST_NOVELTY_MAX_SCAN_BYTES = 8 * 1024 * 1024;
+const INTEGRATION_INGEST_NOVELTY_MAX_SCAN_ROWS = 64;
+const INTEGRATION_INGEST_NOVELTY_SCAN_CHUNK_BYTES = 64 * 1024;
+const INTEGRATION_INGEST_APPEND_PLAN_AUTHORITY = Symbol("integration-ingest-append-plan-authority");
+const INTEGRATION_INGEST_ID_INSPECTION_AUTHORITY = Symbol("integration-ingest-id-inspection-authority");
+const INTEGRATION_INGEST_ARCHIVE_SUFFIXES = [".gz", ".zip"];
+const ZIP_EOCD_SIGNATURE = 0x06054b50;
+const ZIP_CENTRAL_DIRECTORY_SIGNATURE = 0x02014b50;
+const ZIP_LOCAL_FILE_HEADER_SIGNATURE = 0x04034b50;
+const ZIP_MAX_EOCD_SEARCH_BYTES = 65_557;
+const ZIP64_MARKER_16 = 0xffff;
+const ZIP64_MARKER_32 = 0xffffffff;
 export function integrationIngestShardPath(importedAt) {
     return toMonthlyShardRelativePath(VAULT_LAYOUT.integrationIngestLedgerDirectory, importedAt, "importedAt");
 }
@@ -119,16 +137,537 @@ export function stableSerializeIntegrationIngest(value) {
     return JSON.stringify(stableSortJson(value));
 }
 export async function readIntegrationIngestEntries(vaultRoot) {
-    const paths = await walkVaultFiles(vaultRoot, VAULT_LAYOUT.integrationIngestLedgerDirectory, {
-        extension: ".jsonl",
+    return readIntegrationIngestEntriesFromSources(vaultRoot, await listIntegrationIngestRowSources(vaultRoot));
+}
+function integrationEvidenceFingerprint(part) {
+    return JSON.stringify([
+        part.role,
+        part.sha256,
+        part.mediaType,
+        stableSortJson(part.metadata ?? null),
+    ]);
+}
+function integrationIngestAccountMatches(record, accountId) {
+    return (record.accountId ?? null) === (accountId ?? null);
+}
+function integrationIngestReceiptFingerprint(receipt, includePayloadIdentity) {
+    return JSON.stringify([
+        ...(includePayloadIdentity ? [receipt.id, receipt.payloadHash] : []),
+        receipt.sourceKind,
+        receipt.deliveryMode,
+        receipt.resourceType ?? null,
+        receipt.resourceId ?? null,
+        receipt.providerEventId ?? null,
+        receipt.eventType ?? null,
+        receipt.occurredAt ?? null,
+        receipt.signatureVerified ?? null,
+    ]);
+}
+function rawIntegrationEvidenceFingerprint(value) {
+    if (!isRecord(value)
+        || typeof value.role !== "string"
+        || typeof value.sha256 !== "string"
+        || typeof value.mediaType !== "string") {
+        return null;
+    }
+    return JSON.stringify([
+        value.role,
+        value.sha256,
+        value.mediaType,
+        stableSortJson(value.metadata ?? null),
+    ]);
+}
+function rawIntegrationIngestReceiptFingerprint(value, includePayloadIdentity) {
+    if (!isRecord(value)
+        || typeof value.sourceKind !== "string"
+        || typeof value.deliveryMode !== "string"
+        || (includePayloadIdentity
+            && (typeof value.id !== "string" || typeof value.payloadHash !== "string"))) {
+        return null;
+    }
+    return JSON.stringify([
+        ...(includePayloadIdentity ? [value.id, value.payloadHash] : []),
+        value.sourceKind,
+        value.deliveryMode,
+        value.resourceType ?? null,
+        value.resourceId ?? null,
+        value.providerEventId ?? null,
+        value.eventType ?? null,
+        value.occurredAt ?? null,
+        value.signatureVerified ?? null,
+    ]);
+}
+function rawIntegrationIngestCouldProveNovelty(input) {
+    if (Array.isArray(input.raw.parts)
+        && input.raw.parts.some((part) => {
+            const fingerprint = rawIntegrationEvidenceFingerprint(part);
+            return fingerprint !== null && input.requestedFingerprints.has(fingerprint);
+        })) {
+        return true;
+    }
+    return input.requestedReceiptFingerprint !== null
+        && rawIntegrationIngestReceiptFingerprint(input.raw.receipt, input.receiptRequiresPayloadIdentity) === input.requestedReceiptFingerprint;
+}
+async function scanIntegrationIngestNoveltyTail(absolutePath, visitLine) {
+    const handle = await openFile(absolutePath, "r");
+    try {
+        const fileStat = await handle.stat();
+        if (fileStat.size === 0) {
+            return {
+                historyComplete: true,
+                selectionComplete: false,
+                unsafe: false,
+            };
+        }
+        let bytesReadTotal = 0;
+        let lineSuffixChunks = [];
+        let position = fileStat.size;
+        let rowsVisited = 0;
+        let scanByteLimit = INTEGRATION_INGEST_NOVELTY_MAX_SCAN_BYTES;
+        let completingOversizedTailRow = false;
+        let strippedTrailingNewline = false;
+        const visit = (lineBytes) => {
+            rowsVisited += 1;
+            if (lineBytes.length === 0) {
+                return null;
+            }
+            const decision = visitLine(lineBytes.toString("utf8"));
+            if (decision === "complete") {
+                return {
+                    historyComplete: false,
+                    selectionComplete: true,
+                    unsafe: false,
+                };
+            }
+            if (decision === "unsafe") {
+                return {
+                    historyComplete: false,
+                    selectionComplete: false,
+                    unsafe: true,
+                };
+            }
+            return null;
+        };
+        while (position > 0
+            && bytesReadTotal < scanByteLimit
+            && rowsVisited < INTEGRATION_INGEST_NOVELTY_MAX_SCAN_ROWS) {
+            const readSize = Math.min(position, INTEGRATION_INGEST_NOVELTY_SCAN_CHUNK_BYTES, scanByteLimit - bytesReadTotal);
+            const chunk = Buffer.allocUnsafe(readSize);
+            const readPosition = position - readSize;
+            const result = await handle.read(chunk, 0, readSize, readPosition);
+            if (result.bytesRead !== readSize) {
+                return {
+                    historyComplete: false,
+                    selectionComplete: false,
+                    unsafe: true,
+                };
+            }
+            position = readPosition;
+            bytesReadTotal += result.bytesRead;
+            let segmentEnd = chunk.length;
+            if (!strippedTrailingNewline) {
+                if (chunk[chunk.length - 1] !== 0x0a) {
+                    return {
+                        historyComplete: false,
+                        selectionComplete: false,
+                        unsafe: true,
+                    };
+                }
+                segmentEnd -= 1;
+                strippedTrailingNewline = true;
+            }
+            while (segmentEnd > 0
+                && rowsVisited < INTEGRATION_INGEST_NOVELTY_MAX_SCAN_ROWS) {
+                const newlineIndex = chunk.lastIndexOf(0x0a, segmentEnd - 1);
+                if (newlineIndex < 0) {
+                    break;
+                }
+                const leadingSegment = chunk.subarray(newlineIndex + 1, segmentEnd);
+                const lineChunks = leadingSegment.length > 0
+                    ? [leadingSegment, ...lineSuffixChunks]
+                    : lineSuffixChunks;
+                const firstLineChunk = lineChunks[0];
+                const lineBytes = lineChunks.length === 1 && firstLineChunk
+                    ? firstLineChunk
+                    : Buffer.concat(lineChunks);
+                const visitResult = visit(lineBytes);
+                if (visitResult) {
+                    return visitResult;
+                }
+                if (completingOversizedTailRow) {
+                    return {
+                        historyComplete: false,
+                        selectionComplete: false,
+                        unsafe: false,
+                    };
+                }
+                lineSuffixChunks = [];
+                segmentEnd = newlineIndex;
+            }
+            if (rowsVisited >= INTEGRATION_INGEST_NOVELTY_MAX_SCAN_ROWS) {
+                return {
+                    historyComplete: position === 0 && segmentEnd === 0 && lineSuffixChunks.length === 0,
+                    selectionComplete: false,
+                    unsafe: false,
+                };
+            }
+            if (segmentEnd > 0) {
+                lineSuffixChunks.unshift(chunk.subarray(0, segmentEnd));
+            }
+            if (position > 0
+                && rowsVisited === 0
+                && bytesReadTotal >= scanByteLimit
+                && scanByteLimit === INTEGRATION_INGEST_NOVELTY_MAX_SCAN_BYTES) {
+                // The ordinary tail budget ended inside the newest row. Finish only
+                // that row so a bounded fail-open copy can prove the next replay, but
+                // never use the extension to traverse older out-of-budget history.
+                completingOversizedTailRow = true;
+                scanByteLimit = MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES + 1;
+            }
+        }
+        if (position === 0
+            && lineSuffixChunks.length > 0
+            && rowsVisited < INTEGRATION_INGEST_NOVELTY_MAX_SCAN_ROWS) {
+            const firstLineChunk = lineSuffixChunks[0];
+            const lineBytes = lineSuffixChunks.length === 1 && firstLineChunk
+                ? firstLineChunk
+                : Buffer.concat(lineSuffixChunks);
+            const visitResult = visit(lineBytes);
+            if (visitResult) {
+                return visitResult;
+            }
+            if (completingOversizedTailRow) {
+                return {
+                    historyComplete: false,
+                    selectionComplete: false,
+                    unsafe: false,
+                };
+            }
+            lineSuffixChunks = [];
+        }
+        return {
+            historyComplete: position === 0 && lineSuffixChunks.length === 0,
+            selectionComplete: false,
+            unsafe: false,
+        };
+    }
+    finally {
+        await handle.close();
+    }
+}
+export async function inspectIntegrationIngestIdsForImportedAt(vaultRoot, importedAt, requestedIds, options = {}) {
+    const logicalPath = integrationIngestShardPath(importedAt);
+    const sources = await listIntegrationIngestRowSourcesForLogicalPaths(vaultRoot, [logicalPath]);
+    const source = sources.length === 1 ? sources[0] : undefined;
+    const archivedLogicalPaths = new Set(sources.filter((candidate) => candidate.kind !== "jsonl").map((candidate) => candidate.logicalPath));
+    const entriesById = new Map();
+    const invalidIds = new Set();
+    let unsafe = sources.length > 1;
+    const visitRaw = (raw, sourcePath) => {
+        if (!isRecord(raw) || typeof raw.id !== "string" || !requestedIds.has(raw.id)) {
+            return;
+        }
+        try {
+            const record = parseIntegrationIngestRecord(raw, sourcePath);
+            assertIntegrationIngestShard(record.id, record.importedAt, logicalPath);
+            assertIntegrationIngestRecordIntegrity(record);
+            if (entriesById.has(record.id)) {
+                unsafe = true;
+                return;
+            }
+            entriesById.set(record.id, record);
+        }
+        catch {
+            invalidIds.add(raw.id);
+        }
+    };
+    if (!source) {
+        return {
+            [INTEGRATION_INGEST_ID_INSPECTION_AUTHORITY]: true,
+            archivedLogicalPaths,
+            entriesById,
+            failOpenAppendAllowed: false,
+            historyComplete: !unsafe,
+            invalidIds,
+            logicalPath,
+            requestedIds: new Set(requestedIds),
+            unsafe,
+        };
+    }
+    if (options.fullScan && source.kind === "jsonl") {
+        let malformedHistory = false;
+        let readFailed = false;
+        let structurallyUnsafe = false;
+        try {
+            const lines = createInterface({
+                input: await openIntegrationIngestLineStream(vaultRoot, source),
+                crlfDelay: Infinity,
+            });
+            for await (const line of lines) {
+                if (line.length === 0) {
+                    continue;
+                }
+                if (Buffer.byteLength(line, "utf8") > MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES) {
+                    structurallyUnsafe = true;
+                    break;
+                }
+                let raw;
+                try {
+                    raw = JSON.parse(line);
+                }
+                catch {
+                    malformedHistory = true;
+                    for (const requestedId of requestedIds) {
+                        if (line.includes(JSON.stringify(requestedId))) {
+                            invalidIds.add(requestedId);
+                        }
+                    }
+                    continue;
+                }
+                visitRaw(raw, source.sourcePath);
+                if (unsafe) {
+                    break;
+                }
+            }
+        }
+        catch {
+            readFailed = true;
+        }
+        const failOpenAppendAllowed = malformedHistory
+            && !unsafe
+            && !readFailed
+            && !structurallyUnsafe
+            && invalidIds.size === 0;
+        unsafe = unsafe || malformedHistory || readFailed || structurallyUnsafe;
+        return {
+            [INTEGRATION_INGEST_ID_INSPECTION_AUTHORITY]: true,
+            archivedLogicalPaths,
+            entriesById,
+            failOpenAppendAllowed,
+            historyComplete: !unsafe,
+            invalidIds,
+            logicalPath,
+            requestedIds: new Set(requestedIds),
+            unsafe,
+        };
+    }
+    if (options.fullScan || source.kind !== "jsonl") {
+        try {
+            for await (const row of readIntegrationIngestJsonlRows(vaultRoot, [source])) {
+                visitRaw(row.raw, row.sourcePath);
+            }
+        }
+        catch {
+            unsafe = true;
+        }
+        return {
+            [INTEGRATION_INGEST_ID_INSPECTION_AUTHORITY]: true,
+            archivedLogicalPaths,
+            entriesById,
+            failOpenAppendAllowed: false,
+            historyComplete: !unsafe,
+            invalidIds,
+            logicalPath,
+            requestedIds: new Set(requestedIds),
+            unsafe,
+        };
+    }
+    const absolutePath = resolveVaultPath(vaultRoot, source.sourcePath).absolutePath;
+    let scan;
+    try {
+        scan = await scanIntegrationIngestNoveltyTail(absolutePath, (line) => {
+            let raw;
+            try {
+                raw = JSON.parse(line);
+            }
+            catch {
+                return "unsafe";
+            }
+            visitRaw(raw, source.sourcePath);
+            if (unsafe) {
+                return "unsafe";
+            }
+            return entriesById.size > 0 ? "complete" : "continue";
+        });
+    }
+    catch {
+        scan = { historyComplete: false, selectionComplete: false, unsafe: true };
+    }
+    return {
+        [INTEGRATION_INGEST_ID_INSPECTION_AUTHORITY]: true,
+        archivedLogicalPaths,
+        entriesById,
+        failOpenAppendAllowed: false,
+        historyComplete: scan.historyComplete,
+        invalidIds,
+        logicalPath,
+        requestedIds: new Set(requestedIds),
+        unsafe: scan.unsafe || unsafe,
+    };
+}
+/**
+ * Selects evidence whose provider/account/role/content identity has not already
+ * been retained. This is a storage optimization, so damaged historical rows
+ * fail open: only a valid, integrity-checked row may prove that incoming
+ * evidence is redundant.
+ */
+export async function selectNovelIntegrationIngestEvidence(input) {
+    if (!input.accountId) {
+        return {
+            parts: [...input.parts],
+            receiptIsNovel: input.receipt !== undefined,
+        };
+    }
+    const requestedFingerprints = new Set(input.parts.map(integrationEvidenceFingerprint));
+    const existingFingerprints = new Set();
+    const requestedEventIdsByFingerprint = new Map(input.parts.map((part) => [
+        integrationEvidenceFingerprint(part),
+        input.eventIdsByRole?.get(part.role) ?? new Set(),
+    ]));
+    const existingEventLinks = new Set();
+    const existingSampleLinks = new Set();
+    const receiptRequiresPayloadIdentity = input.parts.length === 0;
+    const requestedReceiptFingerprint = input.receipt
+        ? integrationIngestReceiptFingerprint(input.receipt, receiptRequiresPayloadIdentity)
+        : null;
+    let receiptIsNovel = requestedReceiptFingerprint !== null;
+    const failOpenSelection = () => ({
+        parts: [...input.parts],
+        receiptIsNovel: requestedReceiptFingerprint !== null,
     });
-    return readIntegrationIngestEntriesFromPaths(vaultRoot, paths.sort());
+    const buildSelection = () => ({
+        parts: input.parts.filter((part) => !existingFingerprints.has(integrationEvidenceFingerprint(part))
+            || [...(input.eventIdsByRole?.get(part.role) ?? [])].some((eventId) => !existingEventLinks.has(`${integrationEvidenceFingerprint(part)}\u0000${eventId}`))
+            || [...(input.sampleIds ?? [])].some((sampleId) => !existingSampleLinks.has(`${integrationEvidenceFingerprint(part)}\u0000${sampleId}`))),
+        receiptIsNovel,
+    });
+    const selectionIsComplete = () => existingFingerprints.size === requestedFingerprints.size
+        && [...requestedEventIdsByFingerprint.entries()].every(([fingerprint, eventIds]) => [...eventIds].every((eventId) => existingEventLinks.has(`${fingerprint}\u0000${eventId}`)))
+        && [...requestedFingerprints].every((fingerprint) => [...(input.sampleIds ?? [])].every((sampleId) => existingSampleLinks.has(`${fingerprint}\u0000${sampleId}`)))
+        && !receiptIsNovel;
+    if (requestedFingerprints.size === 0 && !receiptIsNovel) {
+        return { parts: [], receiptIsNovel: false };
+    }
+    let sources;
+    try {
+        sources = await listIntegrationIngestRowSourcesForLogicalPaths(input.vaultRoot, [integrationIngestShardPath(input.importedAt)]);
+    }
+    catch {
+        return failOpenSelection();
+    }
+    const source = sources.length === 1 ? sources[0] : undefined;
+    if (!source) {
+        return failOpenSelection();
+    }
+    const visitRaw = (raw, sourcePath, logicalPath) => {
+        if (!isRecord(raw) || raw.provider !== input.provider) {
+            return "continue";
+        }
+        const rawAccountId = typeof raw.accountId === "string" ? raw.accountId : undefined;
+        if ((rawAccountId ?? null) !== (input.accountId ?? null)) {
+            return "continue";
+        }
+        if (!rawIntegrationIngestCouldProveNovelty({
+            raw,
+            receiptRequiresPayloadIdentity,
+            requestedFingerprints,
+            requestedReceiptFingerprint,
+        })) {
+            return "continue";
+        }
+        let record;
+        try {
+            record = parseIntegrationIngestRecord(raw, sourcePath);
+            assertIntegrationIngestShard(record.id, record.importedAt, logicalPath);
+            assertIntegrationIngestRecordIntegrity(record);
+        }
+        catch {
+            return "unsafe";
+        }
+        if (!integrationIngestAccountMatches(record, input.accountId)) {
+            return "continue";
+        }
+        for (const part of record.parts) {
+            const fingerprint = integrationEvidenceFingerprint(part);
+            if (!requestedFingerprints.has(fingerprint)) {
+                continue;
+            }
+            existingFingerprints.add(fingerprint);
+            const requestedEventIds = requestedEventIdsByFingerprint.get(fingerprint);
+            if (requestedEventIds && requestedEventIds.size > 0) {
+                for (const output of record.outputs.events) {
+                    if (requestedEventIds.has(output.id)
+                        && output.roles.includes(part.role)) {
+                        existingEventLinks.add(`${fingerprint}\u0000${output.id}`);
+                    }
+                }
+            }
+            for (const sampleId of record.outputs.sampleIds) {
+                if (input.sampleIds?.has(sampleId)) {
+                    existingSampleLinks.add(`${fingerprint}\u0000${sampleId}`);
+                }
+            }
+        }
+        if (requestedReceiptFingerprint
+            && record.receipt
+            && integrationIngestReceiptFingerprint(record.receipt, receiptRequiresPayloadIdentity) === requestedReceiptFingerprint) {
+            receiptIsNovel = false;
+        }
+        return selectionIsComplete() ? "complete" : "continue";
+    };
+    if (source.kind !== "jsonl") {
+        try {
+            for await (const row of readIntegrationIngestJsonlRows(input.vaultRoot, [source])) {
+                const decision = visitRaw(row.raw, row.sourcePath, row.relativePath);
+                if (decision === "unsafe") {
+                    return failOpenSelection();
+                }
+                if (decision === "complete") {
+                    return { parts: [], receiptIsNovel: false };
+                }
+            }
+        }
+        catch {
+            return failOpenSelection();
+        }
+        return buildSelection();
+    }
+    // Live shards use a bounded reverse-tail scan. Archived shards above use
+    // their already-bounded representation reader because an amendment remains
+    // archived and cannot create a future live-tail proof.
+    const absolutePath = resolveVaultPath(input.vaultRoot, source.sourcePath).absolutePath;
+    if (!(await pathExists(absolutePath))) {
+        return failOpenSelection();
+    }
+    let scan;
+    try {
+        scan = await scanIntegrationIngestNoveltyTail(absolutePath, (line) => {
+            let raw;
+            try {
+                raw = JSON.parse(line);
+            }
+            catch {
+                return "unsafe";
+            }
+            return visitRaw(raw, source.sourcePath, source.logicalPath);
+        });
+    }
+    catch {
+        return failOpenSelection();
+    }
+    if (scan.selectionComplete) {
+        return { parts: [], receiptIsNovel: false };
+    }
+    if (scan.unsafe || !scan.historyComplete) {
+        return failOpenSelection();
+    }
+    return buildSelection();
 }
-async function readIntegrationIngestEntriesFromPaths(vaultRoot, paths) {
+async function readIntegrationIngestEntriesFromSources(vaultRoot, sources) {
     const entries = [];
     const seen = new Map();
-    for await (const { raw, relativePath } of readIntegrationIngestJsonlRows(vaultRoot, paths)) {
-        const record = parseIntegrationIngestRecord(raw, relativePath);
+    for await (const { raw, relativePath, sourcePath } of readIntegrationIngestJsonlRows(vaultRoot, sources)) {
+        const record = parseIntegrationIngestRecord(raw, sourcePath);
         assertIntegrationIngestShard(record.id, record.importedAt, relativePath);
         assertIntegrationIngestRecordIntegrity(record);
         assertUniqueIntegrationIngestId(seen, record.id, relativePath);
@@ -137,62 +676,84 @@ async function readIntegrationIngestEntriesFromPaths(vaultRoot, paths) {
     return entries;
 }
 async function readIntegrationIngestEntriesByIdFromPaths(vaultRoot, paths, ids) {
+    return readIntegrationIngestEntriesByIdFromSources(vaultRoot, await listIntegrationIngestRowSourcesForLogicalPaths(vaultRoot, paths), ids);
+}
+async function readIntegrationIngestEntriesByIdFromSources(vaultRoot, sources, ids) {
     if (ids.size === 0)
         return [];
     const entries = [];
     const seen = new Map();
-    for (const relativePath of [...paths].sort()) {
-        const absolutePath = resolveVaultPath(vaultRoot, relativePath).absolutePath;
-        if (!(await pathExists(absolutePath))) {
+    for await (const { raw, relativePath, sourcePath, lineNumber } of readIntegrationIngestJsonlRows(vaultRoot, sources)) {
+        if (!isRecord(raw) || typeof raw.id !== "string") {
             continue;
         }
-        const lines = createInterface({
-            input: createReadStream(absolutePath, { encoding: "utf8" }),
-            crlfDelay: Infinity,
-        });
-        let lineNumber = 0;
-        for await (const line of lines) {
-            lineNumber += 1;
-            if (line.length === 0)
-                continue;
-            let raw;
-            try {
-                raw = JSON.parse(line);
-            }
-            catch (error) {
-                throw new VaultError("VAULT_INVALID_JSONL", `Invalid JSON on line ${lineNumber}.`, {
-                    relativePath,
-                    lineNumber,
-                    cause: error instanceof Error ? error.message : String(error),
-                });
-            }
-            if (!isRecord(raw) || typeof raw.id !== "string" || typeof raw.importedAt !== "string") {
-                throw new VaultError("INTEGRATION_INGEST_INVALID", `Integration ingest record in "${relativePath}" is missing id or importedAt.`, { relativePath, lineNumber });
-            }
-            assertIntegrationIngestShard(raw.id, raw.importedAt, relativePath);
-            if (!ids.has(raw.id)) {
-                continue;
-            }
-            const record = parseIntegrationIngestRecord(raw, relativePath);
-            assertIntegrationIngestRecordIntegrity(record);
-            assertUniqueIntegrationIngestId(seen, record.id, relativePath);
-            entries.push({ relativePath, record });
+        if (!ids.has(raw.id)) {
+            continue;
+        }
+        if (typeof raw.importedAt !== "string") {
+            throw new VaultError("INTEGRATION_INGEST_INVALID", `Integration ingest record in "${sourcePath}" is missing id or importedAt.`, { relativePath: sourcePath, lineNumber });
         }
+        assertIntegrationIngestShard(raw.id, raw.importedAt, relativePath);
+        const record = parseIntegrationIngestRecord(raw, sourcePath);
+        assertIntegrationIngestRecordIntegrity(record);
+        assertUniqueIntegrationIngestId(seen, record.id, relativePath);
+        entries.push({ relativePath, record });
     }
     return entries;
 }
-export async function buildIntegrationIngestAppendPlan(vaultRoot, records) {
+export async function buildIntegrationIngestAppendPlan(vaultRoot, records, options = {}) {
     const targetShardPaths = [
         ...new Set(records.map((record) => integrationIngestShardPath(record.importedAt))),
     ].sort();
     const requestedIds = new Set(records.map((record) => record.id));
-    const existingById = new Map((await readIntegrationIngestEntriesByIdFromPaths(vaultRoot, targetShardPaths, requestedIds)).map((entry) => [entry.record.id, entry.record]));
+    const targetSources = await listIntegrationIngestRowSourcesForLogicalPaths(vaultRoot, targetShardPaths);
+    const archivedTargets = new Set(targetSources
+        .filter((source) => source.kind !== "jsonl")
+        .map((source) => source.logicalPath));
+    const targetExistingEntries = await readIntegrationIngestEntriesByIdFromSources(vaultRoot, targetSources, requestedIds);
+    const existingById = new Map(targetExistingEntries.map((entry) => [entry.record.id, entry.record]));
+    return buildIntegrationIngestAppendPlanFromExisting({
+        archivedTargets,
+        existingById,
+        options,
+        records,
+        targetShardPaths,
+    });
+}
+export function buildIntegrationIngestAppendPlanFromInspection(records, inspection, options = {}) {
+    if (inspection[INTEGRATION_INGEST_ID_INSPECTION_AUTHORITY] !== true) {
+        throw new VaultError("INTEGRATION_INGEST_INVALID", "Integration ingest inspection authority is invalid.");
+    }
+    const targetShardPaths = [
+        ...new Set(records.map((record) => integrationIngestShardPath(record.importedAt))),
+    ].sort();
+    if ((inspection.unsafe && !inspection.failOpenAppendAllowed)
+        || targetShardPaths.length !== 1
+        || targetShardPaths[0] !== inspection.logicalPath
+        || records.some((record) => !inspection.requestedIds.has(record.id))
+        || records.some((record) => inspection.invalidIds.has(record.id))
+        || records.some((record) => !inspection.historyComplete
+            && !inspection.failOpenAppendAllowed
+            && !inspection.entriesById.has(record.id))) {
+        throw new VaultError("INTEGRATION_INGEST_INVALID", "Integration ingest inspection does not authorize this append plan.");
+    }
+    const existingById = new Map([...inspection.entriesById].filter(([id]) => inspection.requestedIds.has(id)));
+    return buildIntegrationIngestAppendPlanFromExisting({
+        archivedTargets: new Set(inspection.archivedLogicalPaths),
+        existingById,
+        options,
+        records,
+        targetShardPaths,
+    });
+}
+function buildIntegrationIngestAppendPlanFromExisting(input) {
     const pendingById = new Map();
+    const archivedAmendmentShardPaths = new Set();
     const payloads = new Map();
     const appendedIds = [];
-    for (const record of records) {
+    for (const record of input.records) {
         assertIntegrationIngestRecordIntegrity(record);
-        const existing = existingById.get(record.id) ?? pendingById.get(record.id);
+        const existing = input.existingById.get(record.id) ?? pendingById.get(record.id);
         if (existing) {
             if (stableSerializeIntegrationIngest(existing) !== stableSerializeIntegrationIngest(record)) {
                 throw new VaultError("INTEGRATION_INGEST_ID_CONFLICT", `Integration ingest id "${record.id}" already exists with different content.`, { ingestId: record.id });
@@ -202,6 +763,12 @@ export async function buildIntegrationIngestAppendPlan(vaultRoot, records) {
         pendingById.set(record.id, record);
         appendedIds.push(record.id);
         const relativePath = integrationIngestShardPath(record.importedAt);
+        if (input.archivedTargets.has(relativePath)) {
+            if (!input.options.allowArchivedShardAmendments) {
+                throw new VaultError("INTEGRATION_INGEST_SHARD_ARCHIVED", `Integration ingest shard "${relativePath}" is archived and cannot be appended.`, { ingestId: record.id, relativePath });
+            }
+            archivedAmendmentShardPaths.add(relativePath);
+        }
         const rowPayload = `${JSON.stringify(record)}\n`;
         const rowPayloadBytes = Buffer.byteLength(rowPayload, "utf8");
         if (rowPayloadBytes > MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES) {
@@ -209,24 +776,199 @@ export async function buildIntegrationIngestAppendPlan(vaultRoot, records) {
         }
         payloads.set(relativePath, `${payloads.get(relativePath) ?? ""}${rowPayload}`);
     }
-    return { appendedIds, payloads, targetShardPaths };
+    return {
+        [INTEGRATION_INGEST_APPEND_PLAN_AUTHORITY]: true,
+        archivedAmendmentShardPaths: [...archivedAmendmentShardPaths].sort(),
+        appendedIds,
+        payloads,
+        targetShardPaths: [...input.targetShardPaths],
+    };
+}
+export function assertAuthorizedIntegrationIngestAppendPlan(plan) {
+    if (typeof plan !== "object"
+        || plan === null
+        || Reflect.get(plan, INTEGRATION_INGEST_APPEND_PLAN_AUTHORITY) !== true) {
+        throw new VaultError("INTEGRATION_INGEST_INVALID", "Integration ingest append plans must be created by the integration ingest planner.");
+    }
+}
+export async function parseIntegrationIngestAppendPayload(payload, logicalPath) {
+    const records = [];
+    const lines = createInterface({
+        input: Readable.from([payload]),
+        crlfDelay: Infinity,
+    });
+    const source = {
+        kind: "jsonl",
+        logicalPath,
+        sourcePath: logicalPath,
+    };
+    for await (const { raw, sourcePath } of parseIntegrationIngestJsonlLines(lines, source)) {
+        const record = parseIntegrationIngestRecord(raw, sourcePath);
+        assertIntegrationIngestShard(record.id, record.importedAt, logicalPath);
+        assertIntegrationIngestRecordIntegrity(record);
+        records.push(record);
+    }
+    return records;
 }
 export async function stageIntegrationIngestAppendPlan(batch, plan) {
-    for (const relativePath of [...plan.payloads.keys()].sort()) {
-        const payload = plan.payloads.get(relativePath);
-        if (payload) {
-            await batch.stageJsonlAppend(relativePath, payload);
+    await batch.stageIntegrationIngestAppendPlan(plan);
+}
+export async function prepareLiveIntegrationIngestAppendPayload(vaultRoot, targetRelativePath, payload) {
+    const target = resolveVaultPath(vaultRoot, targetRelativePath);
+    if (!(await pathExists(target.absolutePath))) {
+        return payload;
+    }
+    const targetStat = await stat(target.absolutePath);
+    if (targetStat.size === 0) {
+        return payload;
+    }
+    const handle = await openFile(target.absolutePath, "r");
+    const chunks = [];
+    let position = targetStat.size;
+    let finalRowByteLength = 0;
+    try {
+        while (position > 0) {
+            const readLength = Math.min(INTEGRATION_INGEST_NOVELTY_SCAN_CHUNK_BYTES, position);
+            position -= readLength;
+            const chunk = Buffer.allocUnsafe(readLength);
+            const { bytesRead } = await handle.read(chunk, 0, readLength, position);
+            const bytes = chunk.subarray(0, bytesRead);
+            const newlineIndex = bytes.lastIndexOf(0x0a);
+            const rowChunk = newlineIndex >= 0 ? bytes.subarray(newlineIndex + 1) : bytes;
+            chunks.push(rowChunk);
+            finalRowByteLength += rowChunk.byteLength;
+            if (finalRowByteLength > MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES) {
+                throw new VaultError("INTEGRATION_INGEST_ROW_TOO_LARGE", `Final integration ingest row in "${targetRelativePath}" exceeds the journal row limit.`, { relativePath: targetRelativePath, rowPayloadBytes: finalRowByteLength });
+            }
+            if (newlineIndex >= 0) {
+                break;
+            }
         }
     }
+    finally {
+        await handle.close();
+    }
+    const finalRow = Buffer.concat(chunks.reverse()).toString("utf8");
+    if (finalRow.length === 0) {
+        return payload;
+    }
+    try {
+        const record = parseIntegrationIngestRecord(JSON.parse(finalRow), targetRelativePath);
+        assertIntegrationIngestShard(record.id, record.importedAt, targetRelativePath);
+        assertIntegrationIngestRecordIntegrity(record);
+    }
+    catch (error) {
+        throw new VaultError("VAULT_INVALID_JSONL", `Integration ingest shard "${targetRelativePath}" has an incomplete final row; append was rejected.`, {
+            relativePath: targetRelativePath,
+            cause: error instanceof Error ? error.message : String(error),
+        });
+    }
+    return `\n${payload}`;
 }
-export async function listIntegrationIngestsForEvent(vaultRoot, eventId) {
-    const paths = await walkVaultFiles(vaultRoot, VAULT_LAYOUT.integrationIngestLedgerDirectory, {
-        extension: ".jsonl",
+export async function readArchivedIntegrationIngestShardText(vaultRoot, logicalPath) {
+    const [source] = await listIntegrationIngestRowSourcesForLogicalPaths(vaultRoot, [logicalPath]);
+    if (!source || source.kind === "jsonl") {
+        return null;
+    }
+    return {
+        content: await readIntegrationIngestSourceText(vaultRoot, source),
+    };
+}
+export async function appendArchivedIntegrationIngestShard({ expectedBaseByteLength, expectedBaseSha256, payload, targetRelativePath, vaultRoot, }) {
+    const source = await readArchivedIntegrationIngestShardSource(vaultRoot, targetRelativePath);
+    const baseContent = await readIntegrationIngestSourceText(vaultRoot, source);
+    const baseBytes = Buffer.from(baseContent, "utf8");
+    const payloadBytes = Buffer.from(payload, "utf8");
+    const records = await parseIntegrationIngestAppendPayload(payload, targetRelativePath);
+    if (typeof expectedBaseByteLength !== "number" || typeof expectedBaseSha256 !== "string") {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_BASE_MISMATCH", `Integration ingest archive "${source.sourcePath}" append receipt base is required.`, { relativePath: source.sourcePath });
+    }
+    const baseSlice = baseBytes.subarray(0, expectedBaseByteLength);
+    const baseSliceSha256 = createHash("sha256").update(baseSlice).digest("hex");
+    if (baseSlice.byteLength !== expectedBaseByteLength || baseSliceSha256 !== expectedBaseSha256) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_BASE_MISMATCH", `Integration ingest archive "${source.sourcePath}" base content does not match the append receipt.`, { relativePath: source.sourcePath });
+    }
+    if (baseBytes.byteLength > expectedBaseByteLength) {
+        const appendedEnd = expectedBaseByteLength + payloadBytes.byteLength;
+        if (baseBytes.byteLength >= appendedEnd &&
+            baseBytes.subarray(expectedBaseByteLength, appendedEnd).equals(payloadBytes)) {
+            return {
+                originalSize: expectedBaseByteLength,
+            };
+        }
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_BASE_MISMATCH", `Integration ingest archive "${source.sourcePath}" changed after the append receipt base.`, { relativePath: source.sourcePath });
+    }
+    if (baseContent.length > 0 && !baseContent.endsWith("\n")) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" is not newline-terminated.`, { relativePath: source.sourcePath });
+    }
+    validateIntegrationIngestArchiveBaseContent(source, baseContent);
+    const appendPlan = await buildIntegrationIngestAppendPlan(vaultRoot, records, {
+        allowArchivedShardAmendments: true,
     });
+    const appendPayload = appendPlan.payloads.get(targetRelativePath);
+    const originalSize = expectedBaseByteLength;
+    if (!appendPayload) {
+        return {
+            originalSize,
+        };
+    }
+    await writeIntegrationIngestArchiveText(vaultRoot, source, `${baseContent}${appendPayload}`);
+    return {
+        originalSize,
+    };
+}
+function validateIntegrationIngestArchiveBaseContent(source, baseContent) {
+    const seen = new Map();
+    const lines = baseContent.split("\n");
+    for (let index = 0; index < lines.length; index += 1) {
+        const line = lines[index];
+        if (line.length === 0) {
+            continue;
+        }
+        const lineNumber = index + 1;
+        const lineBytes = Buffer.byteLength(line, "utf8");
+        if (lineBytes > MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES) {
+            throw new VaultError("INTEGRATION_INGEST_ROW_TOO_LARGE", `Integration ingest row in "${source.sourcePath}" exceeds the ${MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES}-byte journal limit.`, { lineNumber, relativePath: source.sourcePath, rowPayloadBytes: lineBytes });
+        }
+        let raw;
+        try {
+            raw = JSON.parse(line);
+        }
+        catch (error) {
+            throw new VaultError("VAULT_INVALID_JSONL", `Invalid JSON on line ${lineNumber}.`, {
+                relativePath: source.sourcePath,
+                lineNumber,
+                cause: error instanceof Error ? error.message : String(error),
+            });
+        }
+        const record = parseIntegrationIngestRecord(raw, source.sourcePath);
+        assertIntegrationIngestShard(record.id, record.importedAt, source.logicalPath);
+        assertIntegrationIngestRecordIntegrity(record);
+        assertUniqueIntegrationIngestId(seen, record.id, source.logicalPath);
+    }
+}
+export async function truncateArchivedIntegrationIngestShard({ expectedBaseByteLength, expectedBaseSha256, targetRelativePath, vaultRoot, }) {
+    const source = await readArchivedIntegrationIngestShardSource(vaultRoot, targetRelativePath);
+    const content = await readIntegrationIngestSourceText(vaultRoot, source);
+    const contentBytes = Buffer.from(content, "utf8");
+    if (contentBytes.byteLength < expectedBaseByteLength) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_BASE_MISMATCH", `Integration ingest archive "${source.sourcePath}" is smaller than the rollback base.`, { relativePath: source.sourcePath });
+    }
+    const baseBytes = contentBytes.subarray(0, expectedBaseByteLength);
+    const baseSha256 = createHash("sha256").update(baseBytes).digest("hex");
+    if (baseSha256 !== expectedBaseSha256) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_BASE_MISMATCH", `Integration ingest archive "${source.sourcePath}" base content does not match the rollback receipt.`, { relativePath: source.sourcePath });
+    }
+    if (contentBytes.byteLength === expectedBaseByteLength) {
+        return;
+    }
+    await writeIntegrationIngestArchiveText(vaultRoot, source, baseBytes.toString("utf8"));
+}
+export async function listIntegrationIngestsForEvent(vaultRoot, eventId) {
     const entries = [];
     const seen = new Map();
-    for await (const { raw, relativePath } of readIntegrationIngestJsonlRows(vaultRoot, paths.sort())) {
-        const record = parseIntegrationIngestRecord(raw, relativePath);
+    for await (const { raw, relativePath, sourcePath } of readIntegrationIngestJsonlRows(vaultRoot, await listIntegrationIngestRowSources(vaultRoot))) {
+        const record = parseIntegrationIngestRecord(raw, sourcePath);
         assertIntegrationIngestShard(record.id, record.importedAt, relativePath);
         if (!record.outputs.events.some((output) => output.id === eventId)) {
             continue;
@@ -238,10 +980,7 @@ export async function listIntegrationIngestsForEvent(vaultRoot, eventId) {
     return entries;
 }
 export async function readIntegrationIngestById(vaultRoot, ingestId) {
-    const paths = await walkVaultFiles(vaultRoot, VAULT_LAYOUT.integrationIngestLedgerDirectory, {
-        extension: ".jsonl",
-    });
-    return (await readIntegrationIngestEntriesByIdFromPaths(vaultRoot, paths.sort(), new Set([ingestId])))[0] ?? null;
+    return (await readIntegrationIngestEntriesByIdFromSources(vaultRoot, await listIntegrationIngestRowSources(vaultRoot), new Set([ingestId])))[0] ?? null;
 }
 export function readIntegrationEvidencePart(record, role) {
     assertIntegrationIngestRecordIntegrity(record);
@@ -258,35 +997,415 @@ function stableSortJson(value) {
     }
     return value;
 }
-async function* readIntegrationIngestJsonlRows(vaultRoot, paths) {
-    for (const relativePath of [...paths].sort()) {
-        const absolutePath = resolveVaultPath(vaultRoot, relativePath).absolutePath;
+async function* readIntegrationIngestJsonlRows(vaultRoot, sources) {
+    for (const source of sortIntegrationIngestRowSources(sources)) {
+        const absolutePath = resolveVaultPath(vaultRoot, source.sourcePath).absolutePath;
         if (!(await pathExists(absolutePath))) {
             continue;
         }
         const lines = createInterface({
-            input: createReadStream(absolutePath, { encoding: "utf8" }),
+            input: await openIntegrationIngestLineStream(vaultRoot, source),
             crlfDelay: Infinity,
         });
-        let lineNumber = 0;
-        for await (const line of lines) {
-            lineNumber += 1;
-            if (line.length === 0)
-                continue;
-            let raw;
-            try {
-                raw = JSON.parse(line);
+        yield* parseIntegrationIngestJsonlLines(lines, source);
+    }
+}
+async function* parseIntegrationIngestJsonlLines(lines, source) {
+    let lineNumber = 0;
+    for await (const line of lines) {
+        lineNumber += 1;
+        if (line.length === 0)
+            continue;
+        const lineBytes = Buffer.byteLength(line, "utf8");
+        if (lineBytes > MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES) {
+            throw new VaultError("INTEGRATION_INGEST_ROW_TOO_LARGE", `Integration ingest row in "${source.sourcePath}" exceeds the ${MAX_INTEGRATION_INGEST_JOURNAL_ROW_BYTES}-byte journal limit.`, { lineNumber, relativePath: source.sourcePath, rowPayloadBytes: lineBytes });
+        }
+        let raw;
+        try {
+            raw = JSON.parse(line);
+        }
+        catch (error) {
+            throw new VaultError("VAULT_INVALID_JSONL", `Invalid JSON on line ${lineNumber}.`, {
+                relativePath: source.sourcePath,
+                lineNumber,
+                cause: error instanceof Error ? error.message : String(error),
+            });
+        }
+        yield {
+            lineNumber,
+            raw,
+            relativePath: source.logicalPath,
+            sourcePath: source.sourcePath,
+        };
+    }
+}
+async function openIntegrationIngestLineStream(vaultRoot, source) {
+    if (source.kind === "jsonl") {
+        const absolutePath = resolveVaultPath(vaultRoot, source.sourcePath).absolutePath;
+        return createReadStream(absolutePath, { encoding: "utf8" });
+    }
+    return Readable.from([await readIntegrationIngestSourceText(vaultRoot, source)]);
+}
+async function listIntegrationIngestRowSources(vaultRoot) {
+    const sources = new Map();
+    for (const extension of [".jsonl", ".jsonl.gz", ".jsonl.zip"]) {
+        const paths = await walkVaultFiles(vaultRoot, VAULT_LAYOUT.integrationIngestLedgerDirectory, {
+            extension,
+        });
+        for (const sourcePath of paths) {
+            const source = integrationIngestRowSourceFromPath(sourcePath);
+            sources.set(source.sourcePath, source);
+        }
+    }
+    return assertSingleIntegrationIngestShardRepresentation(sortIntegrationIngestRowSources([...sources.values()]));
+}
+async function listIntegrationIngestRowSourcesForLogicalPaths(vaultRoot, logicalPaths) {
+    const sources = [];
+    for (const logicalPath of [...new Set(logicalPaths)].sort()) {
+        for (const sourcePath of [logicalPath, ...INTEGRATION_INGEST_ARCHIVE_SUFFIXES.map((suffix) => `${logicalPath}${suffix}`)]) {
+            const resolved = resolveVaultPath(vaultRoot, sourcePath);
+            if (await pathExists(resolved.absolutePath)) {
+                sources.push(integrationIngestRowSourceFromPath(sourcePath));
             }
-            catch (error) {
-                throw new VaultError("VAULT_INVALID_JSONL", `Invalid JSON on line ${lineNumber}.`, {
-                    relativePath,
-                    lineNumber,
-                    cause: error instanceof Error ? error.message : String(error),
-                });
+        }
+    }
+    return assertSingleIntegrationIngestShardRepresentation(sortIntegrationIngestRowSources(sources));
+}
+function integrationIngestRowSourceFromPath(sourcePath) {
+    if (sourcePath.endsWith(".jsonl.gz")) {
+        return {
+            kind: "gzip",
+            logicalPath: sourcePath.slice(0, -".gz".length),
+            sourcePath,
+        };
+    }
+    if (sourcePath.endsWith(".jsonl.zip")) {
+        return {
+            kind: "zip",
+            logicalPath: sourcePath.slice(0, -".zip".length),
+            sourcePath,
+        };
+    }
+    return {
+        kind: "jsonl",
+        logicalPath: sourcePath,
+        sourcePath,
+    };
+}
+async function readArchivedIntegrationIngestShardSource(vaultRoot, logicalPath) {
+    const [source] = await listIntegrationIngestRowSourcesForLogicalPaths(vaultRoot, [logicalPath]);
+    if (!source || source.kind === "jsonl") {
+        throw new VaultError("INTEGRATION_INGEST_SHARD_ARCHIVED", `Integration ingest shard "${logicalPath}" is not archived.`, { relativePath: logicalPath });
+    }
+    return source;
+}
+async function readIntegrationIngestSourceText(vaultRoot, source) {
+    const absolutePath = resolveVaultPath(vaultRoot, source.sourcePath).absolutePath;
+    if (source.kind === "jsonl") {
+        return readFile(absolutePath, "utf8");
+    }
+    if (source.kind === "gzip") {
+        return readGzippedIntegrationIngestJsonlText(absolutePath, source.sourcePath);
+    }
+    return readZippedIntegrationIngestJsonlText(vaultRoot, source);
+}
+async function writeIntegrationIngestArchiveText(vaultRoot, source, content) {
+    if (source.kind === "jsonl") {
+        throw new VaultError("INTEGRATION_INGEST_SHARD_ARCHIVED", `Integration ingest shard "${source.logicalPath}" is not archived.`, { relativePath: source.logicalPath });
+    }
+    const archivePath = resolveVaultPath(vaultRoot, source.sourcePath).absolutePath;
+    if (source.kind === "gzip") {
+        const archive = gzipSync(content);
+        assertIntegrationIngestArchiveReplacementSize(content, archive, source.sourcePath);
+        await writeFileAtomic(archivePath, archive);
+        return;
+    }
+    const archive = createSingleEntryIntegrationIngestZipArchive(source.logicalPath.split("/").at(-1) ?? "", content);
+    assertIntegrationIngestArchiveReplacementSize(content, archive, source.sourcePath);
+    await writeFileAtomic(archivePath, archive);
+}
+function sortIntegrationIngestRowSources(sources) {
+    return [...sources].sort((left, right) => {
+        const logicalComparison = left.logicalPath.localeCompare(right.logicalPath);
+        if (logicalComparison !== 0)
+            return logicalComparison;
+        return integrationIngestSourceKindOrder(left.kind) - integrationIngestSourceKindOrder(right.kind)
+            || left.sourcePath.localeCompare(right.sourcePath);
+    });
+}
+function integrationIngestSourceKindOrder(kind) {
+    if (kind === "jsonl")
+        return 0;
+    if (kind === "gzip")
+        return 1;
+    return 2;
+}
+async function readGzippedIntegrationIngestJsonlText(absolutePath, relativePath) {
+    const archiveStat = await stat(absolutePath);
+    if (archiveStat.size > MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${relativePath}" exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES}-byte compressed size limit.`, { byteSize: archiveStat.size, relativePath });
+    }
+    return readBoundedIntegrationIngestArchiveText(createReadStream(absolutePath).pipe(createGunzip()), relativePath, "gzip");
+}
+async function readBoundedIntegrationIngestArchiveText(chunks, relativePath, archiveKind) {
+    const buffers = [];
+    let byteSize = 0;
+    try {
+        for await (const chunk of chunks) {
+            const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk, "utf8");
+            byteSize += buffer.byteLength;
+            if (byteSize > MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES) {
+                throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${relativePath}" exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES}-byte uncompressed size limit.`, { byteSize, relativePath });
             }
-            yield { lineNumber, raw, relativePath };
+            buffers.push(buffer);
+        }
+    }
+    catch (error) {
+        if (error instanceof VaultError) {
+            throw error;
         }
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" contains invalid ${archiveKind} data.`, { relativePath });
+    }
+    return Buffer.concat(buffers, byteSize).toString("utf8");
+}
+async function readZippedIntegrationIngestJsonlText(vaultRoot, source) {
+    const archivePath = resolveVaultPath(vaultRoot, source.sourcePath).absolutePath;
+    const archiveStat = await stat(archivePath);
+    if (archiveStat.size > MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${source.sourcePath}" exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES}-byte compressed size limit.`, { byteSize: archiveStat.size, relativePath: source.sourcePath });
+    }
+    const archive = await readFile(archivePath);
+    const entry = selectZippedIntegrationIngestJsonlEntry(archive, source);
+    assertZippedIntegrationIngestEntrySize(entry, source.sourcePath);
+    const localHeaderOffset = entry.localHeaderOffset;
+    if (localHeaderOffset !== 0) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" has hidden bytes before its JSONL entry.`, { relativePath: source.sourcePath });
     }
+    assertZipReadableRange(archive, localHeaderOffset, 30, source.sourcePath);
+    if (archive.readUInt32LE(localHeaderOffset) !== ZIP_LOCAL_FILE_HEADER_SIGNATURE) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" has an invalid local file header.`, { relativePath: source.sourcePath });
+    }
+    if ((entry.flags & 0x1) !== 0) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_UNSUPPORTED", `Integration ingest archive "${source.sourcePath}" contains an encrypted entry.`, { relativePath: source.sourcePath });
+    }
+    const fileNameLength = archive.readUInt16LE(localHeaderOffset + 26);
+    const extraLength = archive.readUInt16LE(localHeaderOffset + 28);
+    if (extraLength !== 0) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" has hidden local header metadata.`, { relativePath: source.sourcePath });
+    }
+    const localNameStart = localHeaderOffset + 30;
+    const localNameEnd = localNameStart + fileNameLength;
+    assertZipReadableRange(archive, localNameStart, fileNameLength, source.sourcePath);
+    const localName = archive.subarray(localNameStart, localNameEnd).toString("utf8").replaceAll("\\", "/");
+    if (localName !== entry.name) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" local file header does not match its central directory.`, { relativePath: source.sourcePath });
+    }
+    const contentOffset = localHeaderOffset + 30 + fileNameLength + extraLength;
+    assertZipReadableRange(archive, contentOffset, entry.compressedSize, source.sourcePath);
+    if (contentOffset + entry.compressedSize !== entry.centralDirectoryOffset) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" has hidden bytes before its central directory.`, { relativePath: source.sourcePath });
+    }
+    const compressed = archive.subarray(contentOffset, contentOffset + entry.compressedSize);
+    const content = unzipIntegrationIngestEntry(compressed, entry, source.sourcePath);
+    if (content.byteLength !== entry.uncompressedSize) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" has an invalid uncompressed size.`, { relativePath: source.sourcePath });
+    }
+    if ((crc32(content) >>> 0) !== entry.crc32) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" failed CRC-32 verification.`, { relativePath: source.sourcePath });
+    }
+    return content.toString("utf8");
+}
+function selectZippedIntegrationIngestJsonlEntry(archive, source) {
+    const entries = readZipCentralDirectory(archive, source.sourcePath);
+    const expectedName = source.logicalPath.split("/").at(-1) ?? "";
+    if (entries.length === 1 && entries[0]?.name === expectedName) {
+        return entries[0];
+    }
+    throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${source.sourcePath}" must contain exactly one entry named "${expectedName}".`, { relativePath: source.sourcePath, entryCount: entries.length });
+}
+function readZipCentralDirectory(archive, relativePath) {
+    const eocdOffset = findZipEndOfCentralDirectory(archive);
+    if (eocdOffset < 0) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" is missing a ZIP central directory.`, { relativePath });
+    }
+    assertZipReadableRange(archive, eocdOffset, 22, relativePath);
+    const diskNumber = archive.readUInt16LE(eocdOffset + 4);
+    const centralDirectoryDisk = archive.readUInt16LE(eocdOffset + 6);
+    const diskEntryCount = archive.readUInt16LE(eocdOffset + 8);
+    const entryCount = archive.readUInt16LE(eocdOffset + 10);
+    const centralDirectorySize = archive.readUInt32LE(eocdOffset + 12);
+    const centralDirectoryOffset = archive.readUInt32LE(eocdOffset + 16);
+    const commentLength = archive.readUInt16LE(eocdOffset + 20);
+    if (diskNumber !== 0
+        || centralDirectoryDisk !== 0
+        || diskEntryCount !== entryCount
+        || entryCount === ZIP64_MARKER_16
+        || centralDirectorySize === ZIP64_MARKER_32
+        || centralDirectoryOffset === ZIP64_MARKER_32) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_UNSUPPORTED", `Integration ingest archive "${relativePath}" uses an unsupported ZIP variant.`, { relativePath });
+    }
+    if (commentLength !== 0 || eocdOffset + 22 !== archive.byteLength) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" has an invalid ZIP comment length.`, { relativePath });
+    }
+    const entries = [];
+    let offset = centralDirectoryOffset;
+    const centralDirectoryEnd = centralDirectoryOffset + centralDirectorySize;
+    if (centralDirectoryEnd !== eocdOffset) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" has hidden central directory padding.`, { relativePath });
+    }
+    assertZipReadableRange(archive, centralDirectoryOffset, centralDirectorySize, relativePath);
+    for (let index = 0; index < entryCount; index += 1) {
+        assertZipReadableRange(archive, offset, 46, relativePath);
+        if (archive.readUInt32LE(offset) !== ZIP_CENTRAL_DIRECTORY_SIGNATURE) {
+            throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" has an invalid central directory entry.`, { relativePath });
+        }
+        const flags = archive.readUInt16LE(offset + 8);
+        const compressionMethod = archive.readUInt16LE(offset + 10);
+        const entryCrc32 = archive.readUInt32LE(offset + 16);
+        const compressedSize = archive.readUInt32LE(offset + 20);
+        const uncompressedSize = archive.readUInt32LE(offset + 24);
+        const fileNameLength = archive.readUInt16LE(offset + 28);
+        const extraLength = archive.readUInt16LE(offset + 30);
+        const commentLength = archive.readUInt16LE(offset + 32);
+        const localHeaderOffset = archive.readUInt32LE(offset + 42);
+        if (extraLength !== 0 || commentLength !== 0) {
+            throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" has hidden central directory metadata.`, { relativePath });
+        }
+        if (compressedSize === ZIP64_MARKER_32
+            || uncompressedSize === ZIP64_MARKER_32
+            || localHeaderOffset === ZIP64_MARKER_32) {
+            throw new VaultError("INTEGRATION_INGEST_ARCHIVE_UNSUPPORTED", `Integration ingest archive "${relativePath}" uses an unsupported ZIP64 entry.`, { relativePath });
+        }
+        const nameStart = offset + 46;
+        const nameEnd = nameStart + fileNameLength;
+        assertZipReadableRange(archive, nameStart, fileNameLength, relativePath);
+        entries.push({
+            centralDirectoryOffset,
+            compressedSize,
+            compressionMethod,
+            crc32: entryCrc32,
+            flags,
+            localHeaderOffset,
+            name: archive.subarray(nameStart, nameEnd).toString("utf8").replaceAll("\\", "/"),
+            uncompressedSize,
+        });
+        offset = nameEnd + extraLength + commentLength;
+    }
+    if (offset !== centralDirectoryEnd) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" has an invalid central directory size.`, { relativePath });
+    }
+    return entries;
+}
+function findZipEndOfCentralDirectory(archive) {
+    const firstOffset = Math.max(0, archive.byteLength - ZIP_MAX_EOCD_SEARCH_BYTES);
+    for (let offset = archive.byteLength - 22; offset >= firstOffset; offset -= 1) {
+        if (archive.readUInt32LE(offset) === ZIP_EOCD_SIGNATURE) {
+            return offset;
+        }
+    }
+    return -1;
+}
+function unzipIntegrationIngestEntry(compressed, entry, relativePath) {
+    if (entry.compressionMethod === 0) {
+        return Buffer.from(compressed);
+    }
+    if (entry.compressionMethod === 8) {
+        try {
+            return inflateRawSync(compressed, {
+                maxOutputLength: MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES,
+            });
+        }
+        catch (error) {
+            if (isZlibMaxOutputLengthError(error)) {
+                throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${relativePath}" exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES}-byte uncompressed size limit.`, { relativePath });
+            }
+            throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" contains invalid deflated data.`, { relativePath });
+        }
+    }
+    throw new VaultError("INTEGRATION_INGEST_ARCHIVE_UNSUPPORTED", `Integration ingest archive "${relativePath}" uses an unsupported compression method.`, { compressionMethod: entry.compressionMethod, relativePath });
+}
+function assertSingleIntegrationIngestShardRepresentation(sources) {
+    const byLogicalPath = new Map();
+    for (const source of sources) {
+        const siblings = byLogicalPath.get(source.logicalPath) ?? [];
+        siblings.push(source);
+        byLogicalPath.set(source.logicalPath, siblings);
+    }
+    for (const [logicalPath, siblings] of byLogicalPath.entries()) {
+        if (siblings.length <= 1) {
+            continue;
+        }
+        throw new VaultError("INTEGRATION_INGEST_SHARD_REPRESENTATION_CONFLICT", `Integration ingest shard "${logicalPath}" has multiple physical representations.`, {
+            relativePath: logicalPath,
+            sourcePaths: siblings.map((source) => source.sourcePath).sort(),
+        });
+    }
+    return [...sources];
+}
+function assertZippedIntegrationIngestEntrySize(entry, relativePath) {
+    if (entry.compressedSize > MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${relativePath}" contains an entry that exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES}-byte compressed size limit.`, { byteSize: entry.compressedSize, relativePath });
+    }
+    if (entry.uncompressedSize > MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${relativePath}" contains an entry that exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES}-byte uncompressed size limit.`, { byteSize: entry.uncompressedSize, relativePath });
+    }
+}
+function assertZipReadableRange(archive, offset, length, relativePath) {
+    if (offset < 0 || length < 0 || offset + length > archive.byteLength) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_INVALID", `Integration ingest archive "${relativePath}" has an invalid ZIP offset.`, { relativePath });
+    }
+}
+function assertIntegrationIngestArchiveReplacementSize(content, archive, relativePath) {
+    const entryBytes = Buffer.byteLength(content, "utf8");
+    if (entryBytes > MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${relativePath}" exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ENTRY_BYTES}-byte uncompressed size limit.`, { byteSize: entryBytes, relativePath });
+    }
+    if (archive.byteLength > MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_TOO_LARGE", `Integration ingest archive "${relativePath}" exceeds the ${MAX_INTEGRATION_INGEST_ZIP_ARCHIVE_BYTES}-byte compressed size limit.`, { byteSize: archive.byteLength, relativePath });
+    }
+}
+function createSingleEntryIntegrationIngestZipArchive(fileName, content) {
+    const fileNameBytes = Buffer.from(fileName, "utf8");
+    if (fileNameBytes.byteLength === 0 || fileNameBytes.byteLength > 0xffff) {
+        throw new VaultError("INTEGRATION_INGEST_ARCHIVE_UNSUPPORTED", "Integration ingest ZIP archive entry name is unsupported.");
+    }
+    const contentBytes = Buffer.from(content, "utf8");
+    const compressed = deflateRawSync(contentBytes);
+    const contentCrc32 = crc32(contentBytes) >>> 0;
+    const localHeader = Buffer.alloc(30 + fileNameBytes.byteLength);
+    localHeader.writeUInt32LE(ZIP_LOCAL_FILE_HEADER_SIGNATURE, 0);
+    localHeader.writeUInt16LE(20, 4);
+    localHeader.writeUInt16LE(0, 6);
+    localHeader.writeUInt16LE(8, 8);
+    localHeader.writeUInt32LE(contentCrc32, 14);
+    localHeader.writeUInt32LE(compressed.byteLength, 18);
+    localHeader.writeUInt32LE(contentBytes.byteLength, 22);
+    localHeader.writeUInt16LE(fileNameBytes.byteLength, 26);
+    fileNameBytes.copy(localHeader, 30);
+    const centralDirectoryOffset = localHeader.byteLength + compressed.byteLength;
+    const centralDirectory = Buffer.alloc(46 + fileNameBytes.byteLength);
+    centralDirectory.writeUInt32LE(ZIP_CENTRAL_DIRECTORY_SIGNATURE, 0);
+    centralDirectory.writeUInt16LE(20, 4);
+    centralDirectory.writeUInt16LE(20, 6);
+    centralDirectory.writeUInt16LE(0, 8);
+    centralDirectory.writeUInt16LE(8, 10);
+    centralDirectory.writeUInt32LE(contentCrc32, 16);
+    centralDirectory.writeUInt32LE(compressed.byteLength, 20);
+    centralDirectory.writeUInt32LE(contentBytes.byteLength, 24);
+    centralDirectory.writeUInt16LE(fileNameBytes.byteLength, 28);
+    centralDirectory.writeUInt32LE(0, 42);
+    fileNameBytes.copy(centralDirectory, 46);
+    const eocd = Buffer.alloc(22);
+    eocd.writeUInt32LE(ZIP_EOCD_SIGNATURE, 0);
+    eocd.writeUInt16LE(1, 8);
+    eocd.writeUInt16LE(1, 10);
+    eocd.writeUInt32LE(centralDirectory.byteLength, 12);
+    eocd.writeUInt32LE(centralDirectoryOffset, 16);
+    return Buffer.concat([localHeader, compressed, centralDirectory, eocd]);
+}
+function isZlibMaxOutputLengthError(error) {
+    return error instanceof RangeError
+        && error.code === "ERR_BUFFER_TOO_LARGE";
 }
 function parseIntegrationIngestRecord(raw, relativePath) {
     const parsed = integrationIngestRecordSchema.safeParse(raw);