@jcode.labs/ragmir 2.17.2 → 2.18.0

package/dist/ingestion-state.js

@@ -1,15 +1,24 @@
 import { randomUUID } from "node:crypto";
-import { readFile, rename, rm, writeFile } from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import { open as openFile, readdir, readFile, rm } from "node:fs/promises";
 import path from "node:path";
+import { syncDirectory, writePrivateFileAtomic } from "./durable-file.js";
 import { isRecord } from "./guards.js";
 import { ensurePrivateDirectory, hardenPrivateFile } from "./permissions.js";
-const INGESTION_STATE_VERSION = 1;
+const INGESTION_STATE_VERSION = 3;
+const LEGACY_INGESTION_STATE_VERSION = 2;
 const INGESTION_STATE_FILENAME = "ingestion-state.json";
+const INGESTION_FILES_PREFIX = "ingestion-state.files.";
+const INGESTION_FILES_SUFFIX = ".jsonl";
+const INGESTION_JOURNAL_VERSION = 1;
+const INGESTION_JOURNAL_FILENAME = "ingestion-state.journal.jsonl";
+const STREAM_WRITE_BYTES = 64 * 1_024;
 const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
 const GENERATION_ID_PATTERN = /^[0-9a-f]{32}$/iu;
+const ingestionRuntimes = new WeakMap();
 export function createIngestionRunState(options) {
     const now = new Date().toISOString();
-    return {
+    const state = {
         version: INGESTION_STATE_VERSION,
         runId: randomUUID(),
         mode: options.mode,
@@ -24,6 +33,9 @@ export function createIngestionRunState(options) {
         resumed: false,
         files: options.files.map((file) => {
             const reused = options.reusablePaths.has(file.relativePath);
+            const previous = options.previousFiles?.get(file.relativePath);
+            const lastGoodChecksum = previous?.checksum ?? (reused ? file.checksum : null);
+            const lastGoodChunkCount = previous?.chunkCount ?? options.reusableChunkCounts.get(file.relativePath) ?? 0;
             return {
                 relativePath: file.relativePath,
                 checksum: file.checksum,
@@ -31,7 +43,12 @@ export function createIngestionRunState(options) {
                 mtimeMs: file.mtimeMs,
                 policyFingerprint: options.policyFingerprint,
                 state: reused ? "indexed" : "pending",
-                chunkCount: options.reusableChunkCounts.get(file.relativePath) ?? 0,
+                chunkCount: lastGoodChunkCount,
+                lastGoodChecksum,
+                lastGoodChunkCount,
+                lastGoodBytes: previous?.bytes ?? (reused ? file.bytes : null),
+                lastGoodMtimeMs: previous?.mtimeMs ?? (reused ? file.mtimeMs : null),
+                staleLastKnownGood: lastGoodChecksum !== null && lastGoodChecksum !== file.checksum,
                 redactions: 0,
                 error: null,
                 reused,
@@ -39,6 +56,8 @@ export function createIngestionRunState(options) {
             };
         }),
     };
+    ingestionRuntimes.set(state, createRuntime(state, false));
+    return state;
 }
 export function canResumeIngestion(state, files, policyFingerprint, rebuildRequested) {
     if (!["running", "interrupted", "failed"].includes(state.status) ||
@@ -54,38 +73,91 @@ export function canResumeIngestion(state, files, policyFingerprint, rebuildReque
     });
 }
 export function resumeIngestionState(state) {
-    const now = new Date().toISOString();
-    return {
-        ...state,
-        status: "running",
-        resumed: true,
-        updatedAt: now,
-        lastActivityAt: now,
-        files: state.files.map((file) => file.state === "indexed"
-            ? file
-            : { ...file, state: "pending", chunkCount: 0, error: null, updatedAt: now }),
-    };
+    const now = nextIsoTimestamp(state.updatedAt);
+    const runtime = runtimeFor(state);
+    for (const file of state.files) {
+        if (file.state === "indexed") {
+            continue;
+        }
+        const staleLastKnownGood = file.lastGoodChecksum !== null && file.lastGoodChecksum !== file.checksum;
+        if (file.state === "pending" &&
+            file.chunkCount === file.lastGoodChunkCount &&
+            file.error === null &&
+            file.staleLastKnownGood === staleLastKnownGood) {
+            continue;
+        }
+        applyIngestionFileUpdate(runtime, file, {
+            state: "pending",
+            chunkCount: file.lastGoodChunkCount,
+            error: null,
+            staleLastKnownGood,
+        }, now);
+    }
+    state.status = "running";
+    state.resumed = true;
+    state.updatedAt = now;
+    state.lastActivityAt = now;
+    runtime.headerDirty = true;
+    return state;
 }
 export function updateIngestionFile(state, relativePath, update) {
-    const now = new Date().toISOString();
-    return {
-        ...state,
-        updatedAt: now,
-        lastActivityAt: now,
-        files: state.files.map((file) => file.relativePath === relativePath ? { ...file, ...update, updatedAt: now } : file),
-    };
+    const now = nextIsoTimestamp(state.updatedAt);
+    const runtime = runtimeFor(state);
+    const file = runtime.filesByPath.get(relativePath);
+    if (!file) {
+        return state;
+    }
+    applyIngestionFileUpdate(runtime, file, update, now);
+    state.updatedAt = now;
+    state.lastActivityAt = now;
+    runtime.headerDirty = true;
+    return state;
+}
+export function reconcileIngestionFile(state, relativePath, update) {
+    const now = nextIsoTimestamp(state.updatedAt);
+    const runtime = runtimeFor(state);
+    const file = runtime.filesByPath.get(relativePath);
+    if (!file) {
+        return state;
+    }
+    applyIngestionFileUpdate(runtime, file, update, file.updatedAt);
+    state.updatedAt = now;
+    state.lastActivityAt = now;
+    runtime.headerDirty = true;
+    return state;
 }
 export function finishIngestionState(state, status) {
-    const now = new Date().toISOString();
-    return {
-        ...state,
-        status,
-        updatedAt: now,
-        lastActivityAt: now,
-    };
+    const now = nextIsoTimestamp(state.updatedAt);
+    state.status = status;
+    state.updatedAt = now;
+    state.lastActivityAt = now;
+    runtimeFor(state).headerDirty = true;
+    return state;
 }
 export async function writeIngestionState(state, config) {
-    await writePrivateJsonAtomic(path.join(config.storageDir, INGESTION_STATE_FILENAME), state, config.storageDir);
+    const runtime = runtimeFor(state);
+    if (!runtime.snapshotPersisted) {
+        await compactIngestionState(state, config);
+        return;
+    }
+    if (!runtime.headerDirty && runtime.dirtyFiles.size === 0) {
+        return;
+    }
+    await appendPrivateJournal(path.join(config.storageDir, INGESTION_JOURNAL_FILENAME), ingestionJournalRecords(state, runtime), config.storageDir);
+    runtime.headerDirty = false;
+    runtime.dirtyFiles.clear();
+}
+export async function compactIngestionState(state, config) {
+    state.version = INGESTION_STATE_VERSION;
+    const fileSnapshot = ingestionFileSnapshotFilename(state.runId);
+    await writePrivateJsonLinesAtomic(path.join(config.storageDir, fileSnapshot), state.files, config.storageDir);
+    await writePrivateJsonAtomic(path.join(config.storageDir, INGESTION_STATE_FILENAME), ingestionRunHeader(state, fileSnapshot), config.storageDir);
+    await rm(path.join(config.storageDir, INGESTION_JOURNAL_FILENAME), { force: true });
+    await removeStaleIngestionFileSnapshots(fileSnapshot, config);
+    const runtime = runtimeFor(state);
+    runtime.snapshotPersisted = true;
+    runtime.headerDirty = false;
+    runtime.dirtyFiles.clear();
 }
 export async function writeStagedIndexManifest(manifest, runId, config) {
     await writePrivateJsonAtomic(stagedManifestPath(runId, config), manifest, config.storageDir);
@@ -96,7 +168,19 @@ export async function removeStagedIndexManifest(runId, config) {
 export async function readIngestionState(config) {
     try {
         const value = JSON.parse(await readFile(path.join(config.storageDir, INGESTION_STATE_FILENAME), "utf8"));
-        return isIngestionRunState(value, config) ? value : null;
+        const hydrated = await hydrateIngestionRunState(value, config);
+        if (!hydrated) {
+            return null;
+        }
+        const { state, filesByPath } = hydrated;
+        const runtime = createRuntime(state, true, filesByPath);
+        ingestionRuntimes.set(state, runtime);
+        if (!(await applyIngestionJournal(state, config, runtime, state.updatedAt))) {
+            return null;
+        }
+        runtime.headerDirty = false;
+        runtime.dirtyFiles.clear();
+        return state;
     }
     catch (error) {
         if (error instanceof SyntaxError || (isNodeError(error) && error.code === "ENOENT")) {
@@ -110,28 +194,241 @@ export async function getIngestionProgress(config) {
     return state ? ingestionProgress(state) : null;
 }
 export function ingestionProgress(state) {
-    const count = (fileState) => state.files.filter((file) => file.state === fileState).length;
+    const runtime = runtimeFor(state);
     return {
         runId: state.runId,
         mode: state.mode,
         status: state.status,
         resumed: state.resumed,
         batchSize: state.batchSize,
-        totalFiles: state.files.length,
-        pendingFiles: count("pending"),
-        parsedFiles: count("parsed"),
-        embeddedFiles: count("embedded"),
-        indexedFiles: count("indexed"),
-        errorFiles: count("error"),
-        chunksIndexed: state.files
-            .filter((file) => file.state === "indexed")
-            .reduce((sum, file) => sum + file.chunkCount, 0),
+        totalFiles: runtime.filesByPath.size,
+        pendingFiles: runtime.stageCounts.pending,
+        parsedFiles: runtime.stageCounts.parsed,
+        embeddedFiles: runtime.stageCounts.embedded,
+        indexedFiles: runtime.stageCounts.indexed,
+        errorFiles: runtime.stageCounts.error,
+        staleFiles: runtime.staleFiles,
+        chunksIndexed: runtime.chunksIndexed,
         lastActivityAt: state.lastActivityAt,
     };
 }
+function runtimeFor(state) {
+    const existing = ingestionRuntimes.get(state);
+    if (existing) {
+        return existing;
+    }
+    const runtime = createRuntime(state, false);
+    ingestionRuntimes.set(state, runtime);
+    return runtime;
+}
+function createRuntime(state, snapshotPersisted, existingFilesByPath) {
+    const stageCounts = {
+        pending: 0,
+        parsed: 0,
+        embedded: 0,
+        indexed: 0,
+        error: 0,
+    };
+    const filesByPath = existingFilesByPath ?? new Map();
+    let staleFiles = 0;
+    let chunksIndexed = 0;
+    for (const file of state.files) {
+        if (!existingFilesByPath) {
+            filesByPath.set(file.relativePath, file);
+        }
+        stageCounts[file.state] += 1;
+        staleFiles += file.staleLastKnownGood ? 1 : 0;
+        chunksIndexed += indexedChunkContribution(file);
+    }
+    return {
+        filesByPath,
+        stageCounts,
+        staleFiles,
+        chunksIndexed,
+        dirtyFiles: new Set(),
+        headerDirty: false,
+        snapshotPersisted,
+    };
+}
+function applyIngestionFileUpdate(runtime, file, update, now) {
+    const previousStage = file.state;
+    const previousStale = file.staleLastKnownGood;
+    const previousChunks = indexedChunkContribution(file);
+    Object.assign(file, update, { updatedAt: now });
+    if (file.state !== previousStage) {
+        runtime.stageCounts[previousStage] -= 1;
+        runtime.stageCounts[file.state] += 1;
+    }
+    runtime.staleFiles += Number(file.staleLastKnownGood) - Number(previousStale);
+    runtime.chunksIndexed += indexedChunkContribution(file) - previousChunks;
+    runtime.dirtyFiles.add(file.relativePath);
+    runtime.filesByPath.set(file.relativePath, file);
+}
+function indexedChunkContribution(file) {
+    if (file.staleLastKnownGood) {
+        return file.lastGoodChunkCount;
+    }
+    return file.state === "indexed" ? file.chunkCount : 0;
+}
+async function applyIngestionJournal(state, config, runtime, snapshotUpdatedAt) {
+    const stream = createReadStream(path.join(config.storageDir, INGESTION_JOURNAL_FILENAME));
+    stream.setEncoding("utf8");
+    let buffered = "";
+    try {
+        for await (const chunk of stream) {
+            buffered += typeof chunk === "string" ? chunk : chunk.toString("utf8");
+            let lineEnd = buffered.indexOf("\n");
+            while (lineEnd >= 0) {
+                const line = buffered.slice(0, lineEnd);
+                buffered = buffered.slice(lineEnd + 1);
+                if (line && !applyIngestionJournalLine(state, runtime, snapshotUpdatedAt, line)) {
+                    return false;
+                }
+                lineEnd = buffered.indexOf("\n");
+            }
+        }
+        return true;
+    }
+    catch (error) {
+        if (isNodeError(error) && error.code === "ENOENT") {
+            return true;
+        }
+        throw error;
+    }
+}
+function applyIngestionJournalLine(state, runtime, snapshotUpdatedAt, line) {
+    let value;
+    try {
+        value = JSON.parse(line);
+    }
+    catch {
+        return false;
+    }
+    if (!isRecord(value) ||
+        value.version !== INGESTION_JOURNAL_VERSION ||
+        typeof value.runId !== "string") {
+        return false;
+    }
+    if (value.runId !== state.runId) {
+        return true;
+    }
+    if (value.type === "run") {
+        if (!isIngestionRunStatus(value.status) ||
+            typeof value.resumed !== "boolean" ||
+            !isIsoTimestamp(value.updatedAt) ||
+            !isIsoTimestamp(value.lastActivityAt)) {
+            return false;
+        }
+        if (value.updatedAt > snapshotUpdatedAt && value.updatedAt > state.updatedAt) {
+            state.status = value.status;
+            state.resumed = value.resumed;
+            state.updatedAt = value.updatedAt;
+            state.lastActivityAt = value.lastActivityAt;
+        }
+        return true;
+    }
+    if (value.type !== "file" ||
+        !isIsoTimestamp(value.writtenAt) ||
+        !isIngestionFileState(value.file, state.policyFingerprint)) {
+        return false;
+    }
+    const current = runtime.filesByPath.get(value.file.relativePath);
+    if (!current ||
+        current.checksum !== value.file.checksum ||
+        current.bytes !== value.file.bytes ||
+        current.mtimeMs !== value.file.mtimeMs) {
+        return false;
+    }
+    if (value.writtenAt > snapshotUpdatedAt) {
+        applyIngestionFileUpdate(runtime, current, value.file, value.file.updatedAt);
+    }
+    return true;
+}
+function nextIsoTimestamp(previous) {
+    const now = Date.now();
+    const previousTime = Date.parse(previous);
+    return new Date(Math.max(now, previousTime + 1)).toISOString();
+}
+async function hydrateIngestionRunState(value, config) {
+    if (isIngestionRunState(value, config)) {
+        return { state: value };
+    }
+    if (!isPersistedIngestionRunHeader(value, config)) {
+        return null;
+    }
+    const files = await readIngestionFileSnapshot(value, config);
+    if (!files) {
+        return null;
+    }
+    return {
+        state: {
+            version: value.version,
+            runId: value.runId,
+            mode: value.mode,
+            status: value.status,
+            tableName: value.tableName,
+            previousTableName: value.previousTableName,
+            policyFingerprint: value.policyFingerprint,
+            batchSize: value.batchSize,
+            createdAt: value.createdAt,
+            updatedAt: value.updatedAt,
+            lastActivityAt: value.lastActivityAt,
+            resumed: value.resumed,
+            files: files.values,
+        },
+        filesByPath: files.byPath,
+    };
+}
+async function readIngestionFileSnapshot(header, config) {
+    const values = [];
+    const byPath = new Map();
+    const stream = createReadStream(path.join(config.storageDir, header.fileSnapshot));
+    stream.setEncoding("utf8");
+    let buffered = "";
+    const applyLine = (line) => {
+        let value;
+        try {
+            value = JSON.parse(line);
+        }
+        catch {
+            return false;
+        }
+        if (!isIngestionFileState(value, header.policyFingerprint) || byPath.has(value.relativePath)) {
+            return false;
+        }
+        values.push(value);
+        byPath.set(value.relativePath, value);
+        return true;
+    };
+    try {
+        for await (const chunk of stream) {
+            buffered += typeof chunk === "string" ? chunk : chunk.toString("utf8");
+            let lineEnd = buffered.indexOf("\n");
+            while (lineEnd >= 0) {
+                const line = buffered.slice(0, lineEnd);
+                buffered = buffered.slice(lineEnd + 1);
+                if (!line || !applyLine(line)) {
+                    return null;
+                }
+                lineEnd = buffered.indexOf("\n");
+            }
+        }
+        if (buffered && !applyLine(buffered)) {
+            return null;
+        }
+        return values.length === header.fileCount ? { values, byPath } : null;
+    }
+    catch (error) {
+        if (isNodeError(error) && error.code === "ENOENT") {
+            return null;
+        }
+        throw error;
+    }
+}
 function isIngestionRunState(value, config) {
     if (!isRecord(value) ||
-        value.version !== INGESTION_STATE_VERSION ||
+        (value.version !== LEGACY_INGESTION_STATE_VERSION &&
+            value.version !== INGESTION_STATE_VERSION) ||
         !isUuidV4(value.runId) ||
         (value.mode !== "incremental" && value.mode !== "rebuild") ||
         !isIngestionRunStatus(value.status) ||
@@ -163,6 +460,37 @@ function isIngestionRunState(value, config) {
         value.previousTableName !== value.tableName &&
         isManagedTableName(value.previousTableName, config));
 }
+function isPersistedIngestionRunHeader(value, config) {
+    return (isRecord(value) &&
+        value.version === INGESTION_STATE_VERSION &&
+        isIngestionRunMetadata(value, config) &&
+        value.fileSnapshot === ingestionFileSnapshotFilename(value.runId) &&
+        isNonNegativeSafeInteger(value.fileCount));
+}
+function isIngestionRunMetadata(value, config) {
+    if (!isUuidV4(value.runId) ||
+        (value.mode !== "incremental" && value.mode !== "rebuild") ||
+        !isIngestionRunStatus(value.status) ||
+        typeof value.tableName !== "string" ||
+        (value.previousTableName !== null && typeof value.previousTableName !== "string") ||
+        typeof value.policyFingerprint !== "string" ||
+        value.policyFingerprint.length === 0 ||
+        !isPositiveSafeInteger(value.batchSize) ||
+        !isIsoTimestamp(value.createdAt) ||
+        !isIsoTimestamp(value.updatedAt) ||
+        !isIsoTimestamp(value.lastActivityAt) ||
+        typeof value.resumed !== "boolean" ||
+        !isManagedTableName(value.tableName, config)) {
+        return false;
+    }
+    if (value.mode === "incremental") {
+        return value.previousTableName === null;
+    }
+    return (value.tableName === generationTableName(config.tableName, value.runId) &&
+        value.previousTableName !== null &&
+        value.previousTableName !== value.tableName &&
+        isManagedTableName(value.previousTableName, config));
+}
 function isIngestionRunStatus(value) {
     return (value === "running" ||
         value === "interrupted" ||
@@ -183,7 +511,7 @@ function stagedManifestPath(runId, config) {
     return targetPath;
 }
 function isIngestionFileState(value, policyFingerprint) {
-    return (isRecord(value) &&
+    if (!(isRecord(value) &&
         typeof value.relativePath === "string" &&
         value.relativePath.length > 0 &&
         !value.relativePath.includes("\0") &&
@@ -194,11 +522,34 @@ function isIngestionFileState(value, policyFingerprint) {
         value.policyFingerprint === policyFingerprint &&
         isIngestionFileStage(value.state) &&
         isNonNegativeSafeInteger(value.chunkCount) &&
+        (value.lastGoodChecksum === null ||
+            (typeof value.lastGoodChecksum === "string" && value.lastGoodChecksum.length > 0)) &&
+        isNonNegativeSafeInteger(value.lastGoodChunkCount) &&
+        (value.lastGoodBytes === null || isNonNegativeSafeInteger(value.lastGoodBytes)) &&
+        (value.lastGoodMtimeMs === null || isNonNegativeFiniteNumber(value.lastGoodMtimeMs)) &&
+        typeof value.staleLastKnownGood === "boolean" &&
         isNonNegativeSafeInteger(value.redactions) &&
         (value.error === null || typeof value.error === "string") &&
         typeof value.reused === "boolean" &&
         isIsoTimestamp(value.updatedAt) &&
-        (!value.reused || value.state === "indexed"));
+        (!value.reused || value.state === "indexed"))) {
+        return false;
+    }
+    const hasLastGood = value.lastGoodChecksum !== null;
+    if ((!hasLastGood &&
+        (value.lastGoodChunkCount !== 0 ||
+            value.lastGoodBytes !== null ||
+            value.lastGoodMtimeMs !== null)) ||
+        (value.staleLastKnownGood &&
+            (!hasLastGood || value.lastGoodChecksum === value.checksum || value.state === "indexed"))) {
+        return false;
+    }
+    return (value.state !== "indexed" ||
+        (value.lastGoodChecksum === value.checksum &&
+            value.lastGoodChunkCount === value.chunkCount &&
+            value.lastGoodBytes === value.bytes &&
+            value.lastGoodMtimeMs === value.mtimeMs &&
+            !value.staleLastKnownGood));
 }
 export function generationTableName(baseName, runId) {
     if (!isUuidV4(runId)) {
@@ -242,16 +593,96 @@ function isIngestionFileStage(value) {
 function isNodeError(error) {
     return error instanceof Error && "code" in error;
 }
+function ingestionRunHeader(state, fileSnapshot) {
+    return {
+        version: INGESTION_STATE_VERSION,
+        runId: state.runId,
+        mode: state.mode,
+        status: state.status,
+        tableName: state.tableName,
+        previousTableName: state.previousTableName,
+        policyFingerprint: state.policyFingerprint,
+        batchSize: state.batchSize,
+        createdAt: state.createdAt,
+        updatedAt: state.updatedAt,
+        lastActivityAt: state.lastActivityAt,
+        resumed: state.resumed,
+        fileSnapshot,
+        fileCount: state.files.length,
+    };
+}
+function ingestionFileSnapshotFilename(runId) {
+    if (!isUuidV4(runId)) {
+        throw new Error("Ingestion runId must be a valid UUID v4.");
+    }
+    return `${INGESTION_FILES_PREFIX}${runId}${INGESTION_FILES_SUFFIX}`;
+}
+function* ingestionJournalRecords(state, runtime) {
+    if (runtime.headerDirty) {
+        yield {
+            version: INGESTION_JOURNAL_VERSION,
+            runId: state.runId,
+            type: "run",
+            status: state.status,
+            resumed: state.resumed,
+            updatedAt: state.updatedAt,
+            lastActivityAt: state.lastActivityAt,
+        };
+    }
+    for (const relativePath of runtime.dirtyFiles) {
+        const file = runtime.filesByPath.get(relativePath);
+        if (file) {
+            yield {
+                version: INGESTION_JOURNAL_VERSION,
+                runId: state.runId,
+                type: "file",
+                writtenAt: state.updatedAt,
+                file,
+            };
+        }
+    }
+}
+async function removeStaleIngestionFileSnapshots(currentFilename, config) {
+    const entries = await readdir(config.storageDir);
+    await Promise.all(entries
+        .filter((entry) => entry !== currentFilename &&
+        entry.startsWith(INGESTION_FILES_PREFIX) &&
+        entry.endsWith(INGESTION_FILES_SUFFIX) &&
+        isUuidV4(entry.slice(INGESTION_FILES_PREFIX.length, -INGESTION_FILES_SUFFIX.length)))
+        .map((entry) => rm(path.join(config.storageDir, entry), { force: true })));
+}
 async function writePrivateJsonAtomic(targetPath, value, directory) {
+    await writePrivateFileAtomic(targetPath, directory, async (handle) => {
+        await handle.writeFile(JSON.stringify(value, null, 2), "utf8");
+    });
+}
+async function writePrivateJsonLinesAtomic(targetPath, values, directory) {
+    await writePrivateFileAtomic(targetPath, directory, (handle) => writeJsonLines(handle, values));
+}
+async function appendPrivateJournal(targetPath, records, directory) {
     await ensurePrivateDirectory(directory);
-    const temporaryPath = `${targetPath}.${process.pid}.${randomUUID()}.tmp`;
+    const handle = await openFile(targetPath, "a", 0o600);
     try {
-        await writeFile(temporaryPath, JSON.stringify(value, null, 2), "utf8");
-        await hardenPrivateFile(temporaryPath);
-        await rename(temporaryPath, targetPath);
+        await hardenPrivateFile(targetPath);
+        await writeJsonLines(handle, records);
+        await handle.sync();
     }
     finally {
-        await rm(temporaryPath, { force: true });
+        await handle.close();
+    }
+    await syncDirectory(directory);
+}
+async function writeJsonLines(handle, values) {
+    let buffered = "";
+    for (const value of values) {
+        buffered += `${JSON.stringify(value)}\n`;
+        if (Buffer.byteLength(buffered) >= STREAM_WRITE_BYTES) {
+            await handle.writeFile(buffered, "utf8");
+            buffered = "";
+        }
+    }
+    if (buffered) {
+        await handle.writeFile(buffered, "utf8");
     }
 }
 //# sourceMappingURL=ingestion-state.js.map