@untangle-code/jobs 0.0.0-stage → 0.1.1

package/dist/runner.js

@@ -0,0 +1,686 @@
+import { UntangleError, assertGenerationComplete, buildCorpusRevisionFingerprint, isUntangleError, validateGenerationCompleteness, } from "@untangle-code/domain";
+import { configInvalidError, indexIncompleteError } from "@untangle-code/observability";
+import { isJobCancelledError, throwIfAborted } from "./abort.js";
+import { BoundedQueue } from "./bounded-queue.js";
+import { AsyncEventChannel } from "./events.js";
+import { planResume } from "./resume.js";
+import { JobStateMachine } from "./state-machine.js";
+const DEFAULT_MEMORY_BUDGET_BYTES = 64 * 1024 * 1024;
+const DEFAULT_EMBED_BATCH_SIZE = 8;
+const DEFAULT_ANALYSIS_CAPACITY = 16;
+const DEFAULT_UPSERT_CAPACITY = 4;
+const DEFAULT_LOCK_LEASE_MS = 30_000;
+const MIN_LOCK_LEASE_MS = 3;
+/**
+ * Kleinstes zulaessiges Gesamtbudget. Das Budget wird haelftig auf beide
+ * Queues verteilt (`Math.floor(budget / 2)`), also muss es mindestens 2
+ * Bytes betragen, damit beide Haelften positiv sind. Bewusst KEINE hoehere,
+ * gegriffene Schwelle: ein knappes Budget ist eine legitime Testkonfiguration
+ * — dass es die Byte-Grenze nicht mehr abschaltet, sichert seit WP-116 die
+ * `BoundedQueue` selbst, nicht eine Mindestzahl hier.
+ */
+const MIN_MEMORY_BUDGET_BYTES = 2;
+function errorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function lockRenewalError(lock, error) {
+    return new UntangleError("LOCK_HELD", `Index lock "${lock.key}" could not be renewed; exclusive ownership is no longer proven: ${errorMessage(error)}`, {
+        retryable: true,
+        operation: "index_lock_renew",
+        context: {
+            key: lock.key,
+            processInstanceId: lock.record.processInstanceId,
+        },
+        cause: error,
+    });
+}
+/**
+ * Keeps one lease alive without ever issuing overlapping `renew()` calls.
+ * A failed renewal aborts the job immediately; `assertOwnership()` repeats
+ * the ownership check after the heartbeat has stopped. A runner whose lease
+ * loss was observed does not invoke the terminal-generation observer.
+ */
+function startIndexLockHeartbeat(lock, leaseMs, internal, onFailure) {
+    const intervalMs = Math.max(1, Math.floor(leaseMs / 3));
+    let stopped = false;
+    let timer = null;
+    let inFlight = null;
+    let failure = null;
+    const fail = (error) => {
+        if (failure === null) {
+            failure = lockRenewalError(lock, error);
+            onFailure(failure);
+            internal.abort();
+        }
+        return failure;
+    };
+    const schedule = () => {
+        if (stopped || failure !== null)
+            return;
+        timer = setTimeout(() => {
+            timer = null;
+            if (stopped || failure !== null)
+                return;
+            inFlight = lock
+                .renew()
+                .catch((error) => {
+                fail(error);
+            })
+                .finally(() => {
+                inFlight = null;
+                schedule();
+            });
+        }, intervalMs);
+    };
+    const stop = async () => {
+        stopped = true;
+        if (timer !== null) {
+            clearTimeout(timer);
+            timer = null;
+        }
+        await inFlight;
+    };
+    schedule();
+    return {
+        get failure() {
+            return failure;
+        },
+        stop,
+        assertOwnership: async () => {
+            await stop();
+            if (failure !== null)
+                throw failure;
+            try {
+                await lock.renew();
+            }
+            catch (error) {
+                throw fail(error);
+            }
+        },
+    };
+}
+export class IndexJobRunner {
+    options;
+    machine = new JobStateMachine("queued");
+    jobId;
+    generationId = null;
+    completedFiles = new Set();
+    storedPointIdsByFile = {};
+    manifestHashes = {};
+    corpusManifestHashes = null;
+    filesPlanned = 0;
+    queues = null;
+    lastQueueStats = null;
+    started = false;
+    constructor(options) {
+        IndexJobRunner.validateResourceOptions(options);
+        this.options = options;
+        this.jobId = options.jobId ?? options.random.uuid();
+    }
+    /**
+     * Prueft die Ressourcen-Optionen am EINGANG des Jobs — dort, wo sie ein
+     * Aufrufer setzt, nicht erst tief in der Pipeline.
+     *
+     * WP-116: `queueCapacities` und `memoryBudgetBytes` sind frei
+     * konfigurierbar und entscheiden zusammen darueber, ob die Byte-Grenze der
+     * Queues ueberhaupt greifen kann. Die Haertung aus WP-79 sitzt im
+     * `BoundedQueue`-Konstruktor und damit erst in `runPipeline()` — ein
+     * fehlerhafter Wert faellt dort mitten im Lauf auf, nachdem der Job
+     * bereits Lock, Generation und Checkpoints angefasst hat. Hier scheitert
+     * er, bevor irgendetwas passiert ist.
+     */
+    static validateResourceOptions(options) {
+        const capacities = [
+            ["queueCapacities.analysis", options.queueCapacities?.analysis],
+            ["queueCapacities.upsert", options.queueCapacities?.upsert],
+            ["embedBatchSize", options.embedBatchSize],
+            ["lockLeaseMs", options.lockLeaseMs],
+        ];
+        for (const [field, value] of capacities) {
+            if (value === undefined) {
+                continue;
+            }
+            const minimum = field === "lockLeaseMs" ? MIN_LOCK_LEASE_MS : 1;
+            if (!Number.isSafeInteger(value) || value < minimum) {
+                throw configInvalidError(`IndexJobRunner ${field} must be a safe integer >= ${String(minimum)}, got ${String(value)}`, {
+                    retryable: false,
+                    operation: "index_job_options",
+                    context: { field, value, minimum },
+                });
+            }
+        }
+        const budget = options.memoryBudgetBytes;
+        if (budget === undefined) {
+            return;
+        }
+        if (!Number.isSafeInteger(budget) || budget < MIN_MEMORY_BUDGET_BYTES) {
+            throw configInvalidError(`IndexJobRunner memoryBudgetBytes must be a safe integer >= ${String(MIN_MEMORY_BUDGET_BYTES)}, got ${String(budget)} — the budget is split evenly across the two pipeline queues ` +
+                "and a non-integer/non-finite budget would disable their byte bound " +
+                "(architecture.md §13.2)", {
+                retryable: false,
+                operation: "index_job_options",
+                context: { field: "memoryBudgetBytes", value: budget, minimum: MIN_MEMORY_BUDGET_BYTES },
+            });
+        }
+    }
+    status() {
+        const queues = this.queues !== null
+            ? { analysis: this.queues.analysis.stats(), upsert: this.queues.upsert.stats() }
+            : this.lastQueueStats;
+        return {
+            jobId: this.jobId,
+            generationId: this.generationId,
+            stage: this.machine.stage,
+            filesPlanned: this.filesPlanned,
+            filesCompleted: this.completedFiles.size,
+            memoryBudgetBytes: this.options.memoryBudgetBytes ?? DEFAULT_MEMORY_BUDGET_BYTES,
+            queuedBytes: queues === null ? 0 : queues.analysis.bytes + queues.upsert.bytes,
+            queues,
+        };
+    }
+    /**
+     * Runs the job once. Cold: work starts when iteration starts. The stream
+     * always ends with exactly one terminal event — `job_completed`,
+     * `job_failed` or `job_cancelled` — and never throws for job-level
+     * failures (they ARE the `job_failed` event).
+     */
+    run(options = {}) {
+        if (this.started) {
+            throw new Error("IndexJobRunner.run() may only be called once per runner");
+        }
+        this.started = true;
+        const channel = new AsyncEventChannel();
+        const internal = new AbortController();
+        const external = options.signal;
+        if (external !== undefined) {
+            if (external.aborted) {
+                internal.abort();
+            }
+            else {
+                external.addEventListener("abort", () => internal.abort(), {
+                    once: true,
+                    signal: internal.signal,
+                });
+            }
+        }
+        let driver = null;
+        const start = () => this.drive(channel, external, internal).then(() => channel.close(), (error) => channel.fail(error));
+        return {
+            [Symbol.asyncIterator]() {
+                driver ??= start();
+                const iterator = channel[Symbol.asyncIterator]();
+                const startedDriver = driver;
+                return {
+                    next: () => iterator.next(),
+                    async return(value) {
+                        // Consumer abandoned the stream: stop the pipeline, await the
+                        // driver so no orphaned worker keeps running.
+                        internal.abort();
+                        await startedDriver.catch(() => undefined);
+                        return { done: true, value: value };
+                    },
+                };
+            },
+        };
+    }
+    // -- driver ---------------------------------------------------------------
+    async drive(channel, external, internal) {
+        const { clock, stages, identity } = this.options;
+        const startedAtMs = clock.nowEpochMs();
+        const signal = internal.signal;
+        const emit = (event) => channel.emit(event);
+        const base = () => ({
+            jobId: this.jobId,
+            workspaceId: identity.workspaceId,
+            occurredAt: clock.nowIso(),
+            ...(this.generationId !== null ? { generationId: this.generationId } : {}),
+        });
+        let lock = null;
+        let lockHeartbeat = null;
+        const notifyGenerationFinished = async (outcome) => {
+            if (this.generationId === null || this.options.onGenerationFinished === undefined)
+                return;
+            try {
+                const corpusRevisionFingerprint = outcome === "completed" && this.corpusManifestHashes !== null
+                    ? buildCorpusRevisionFingerprint(this.corpusManifestHashes)
+                    : undefined;
+                await this.options.onGenerationFinished(this.generationId, outcome, corpusRevisionFingerprint);
+            }
+            catch (error) {
+                // Terminal state publication is an adapter concern. The index outcome
+                // itself is already decided, and the preceding running marker remains
+                // conservative if this observer fails.
+                this.options.logger?.log({
+                    timestamp: clock.nowIso(),
+                    level: "warn",
+                    component: "jobs.index",
+                    event: "generation_terminal_observer_failed",
+                    jobId: this.jobId,
+                    context: {
+                        generationId: this.generationId,
+                        outcome,
+                        message: errorMessage(error),
+                    },
+                });
+            }
+        };
+        try {
+            emit({ kind: "job_queued", operation: "index", ...base() });
+            if (this.options.lockProvider !== undefined) {
+                lock = await this.options.lockProvider.acquire(`repo:${identity.repoId}:${identity.refId}:index`, { signal, leaseMs: this.options.lockLeaseMs ?? DEFAULT_LOCK_LEASE_MS });
+                lockHeartbeat = startIndexLockHeartbeat(lock, this.options.lockLeaseMs ?? DEFAULT_LOCK_LEASE_MS, internal, (error) => {
+                    this.options.logger?.log({
+                        timestamp: clock.nowIso(),
+                        level: "error",
+                        component: "jobs.index",
+                        event: "index_lock_renew_failed",
+                        jobId: this.jobId,
+                        context: { key: lock?.key ?? "", message: error.message },
+                    });
+                });
+            }
+            throwIfAborted(external, "index_job");
+            this.machine.transition("preparing_infrastructure");
+            const infrastructure = await stages.infrastructure.ensureReady({ signal });
+            emit({ kind: "infrastructure_ready", endpoints: infrastructure.endpoints, ...base() });
+            throwIfAborted(external, "index_job");
+            this.machine.transition("discovering_files");
+            const discovery = await stages.discovery.discover({ signal });
+            this.corpusManifestHashes = { ...discovery.corpusManifestHashes };
+            const checkpoint = this.options.resume === false
+                ? null
+                : await this.options.checkpointStore.loadLatest(identity.repoId, identity.refId, {
+                    signal,
+                });
+            const plan = planResume(discovery.planned, checkpoint, identity);
+            this.generationId =
+                plan.resumed && plan.generationId !== null
+                    ? plan.generationId
+                    : (this.options.generationId ?? this.options.random.uuid());
+            await this.options.onGenerationStarted?.(this.generationId);
+            for (const fileId of plan.completedFiles) {
+                this.completedFiles.add(fileId);
+            }
+            this.storedPointIdsByFile = { ...plan.storedPointIdsByFile };
+            this.manifestHashes = { ...plan.manifestHashes };
+            this.filesPlanned = discovery.planned.length;
+            emit({
+                kind: "files_discovered",
+                repoId: identity.repoId,
+                refId: identity.refId,
+                totalFiles: discovery.totalFiles,
+                changedFiles: plan.remaining.length,
+                unchangedFiles: discovery.unchangedFiles + plan.skippedCompleted,
+                ignoredFiles: discovery.ignoredFiles,
+                ...base(),
+            });
+            throwIfAborted(external, "index_job");
+            this.machine.transition("parsing_and_chunking");
+            await this.runPipeline(plan.remaining, emit, base, signal);
+            throwIfAborted(external, "index_job");
+            this.machine.advanceTo("building_graphs");
+            const graphResults = await stages.graphs.buildGraphs({ signal });
+            for (const result of graphResults) {
+                emit({
+                    kind: "graph_updated",
+                    graph: result.graph,
+                    nodeCount: result.nodeCount,
+                    edgeCount: result.edgeCount,
+                    ...base(),
+                });
+            }
+            throwIfAborted(external, "index_job");
+            this.machine.advanceTo("indexing_context");
+            if (stages.context !== undefined) {
+                await stages.context.indexContext({ signal });
+            }
+            throwIfAborted(external, "index_job");
+            this.machine.advanceTo("finalizing");
+            await this.saveCheckpoint({ signal });
+            emit({
+                kind: "checkpoint_saved",
+                stage: this.machine.stage,
+                completedFileCount: this.completedFiles.size,
+                ...base(),
+            });
+            // Generation commit (architecture.md §13.4). Pre-check everything the
+            // runner can prove BEFORE asking for the metadata commit, so an
+            // incomplete generation is never even submitted for finalization.
+            const plannedFileIds = discovery.planned.map((file) => file.fileId);
+            const confirmedFileIds = [...this.completedFiles].sort();
+            const storedGraphs = graphResults.map((result) => result.graph);
+            const requiredGraphs = this.options.requiredGraphs ?? ["file", "symbol"];
+            const preCheck = validateGenerationCompleteness({
+                generationId: this.generationId,
+                plannedFileIds,
+                confirmedFileIds,
+                checkpointsSaved: true,
+                requiredGraphs,
+                storedGraphs,
+                metadataStatus: "completed",
+            });
+            if (!preCheck.complete) {
+                throw indexIncompleteError(`generation "${this.generationId}" cannot be committed: ${preCheck.reasons.join("; ")}`, {
+                    retryable: false,
+                    operation: "generation_commit",
+                    context: {
+                        generationId: this.generationId,
+                        missingFileIds: preCheck.missingFileIds,
+                        missingGraphs: preCheck.missingGraphs,
+                    },
+                });
+            }
+            const commitResult = await stages.commit.commit({
+                jobId: this.jobId,
+                generationId: this.generationId,
+                workspaceId: identity.workspaceId,
+                repoId: identity.repoId,
+                refId: identity.refId,
+                effectiveProfileId: identity.effectiveProfileId,
+                plannedFileIds,
+                confirmedFileIds,
+                storedGraphs,
+            }, { signal });
+            // Full §13.4 gate including the FINAL metadata status the commit
+            // stage actually reached — `completed` is only reported after this.
+            assertGenerationComplete({
+                generationId: this.generationId,
+                plannedFileIds,
+                confirmedFileIds,
+                checkpointsSaved: true,
+                requiredGraphs,
+                storedGraphs,
+                metadataStatus: commitResult.metadataStatus,
+            });
+            // Stop the heartbeat and perform one last owner-checked renewal before
+            // publishing anything terminal. If another process took the lease, this
+            // throws LOCK_HELD and the stale runner never calls the observer.
+            await lockHeartbeat?.assertOwnership();
+            await notifyGenerationFinished("completed");
+            this.machine.transition("completed");
+            emit({
+                kind: "job_completed",
+                durationMs: clock.nowEpochMs() - startedAtMs,
+                ...base(),
+            });
+        }
+        catch (error) {
+            let effectiveError = lockHeartbeat?.failure ?? error;
+            const externallyCancelled = (lockHeartbeat === null || lockHeartbeat.failure === null) &&
+                external?.aborted === true &&
+                isJobCancelledError(error);
+            if (externallyCancelled) {
+                const checkpointSaved = await this.prepareCancelled(emit, base);
+                try {
+                    await lockHeartbeat?.assertOwnership();
+                }
+                catch (ownershipError) {
+                    // Cancellation cleanup can outlive a lease. Losing it changes the
+                    // truthful terminal outcome from cancelled to a lock failure.
+                    effectiveError = ownershipError;
+                    this.finishFailed(effectiveError, emit, base);
+                    return;
+                }
+                await notifyGenerationFinished("cancelled");
+                this.emitCancelled(checkpointSaved, emit, base);
+                return;
+            }
+            let ownsLock = true;
+            try {
+                await lockHeartbeat?.assertOwnership();
+            }
+            catch (ownershipError) {
+                effectiveError = ownershipError;
+                ownsLock = false;
+            }
+            if (ownsLock)
+                await notifyGenerationFinished("failed");
+            this.finishFailed(effectiveError, emit, base);
+        }
+        finally {
+            await lockHeartbeat?.stop();
+            internal.abort();
+            if (lock !== null) {
+                try {
+                    await lock.release();
+                }
+                catch {
+                    // Releasing a lock after the job already reached a terminal state
+                    // must never mask the job's own outcome.
+                }
+            }
+        }
+    }
+    /** Cancellation path part one (architecture.md §21.3): save the checkpoint. */
+    async prepareCancelled(emit, base) {
+        this.machine.transition("cancel_requested");
+        let checkpointSaved = false;
+        if (this.generationId !== null) {
+            try {
+                // Only CONFIRMED work is in the runner state, so this checkpoint is
+                // valid by construction; per-batch checkpoints were already saved.
+                await this.saveCheckpoint({});
+                checkpointSaved = true;
+                emit({
+                    kind: "checkpoint_saved",
+                    stage: this.machine.stage,
+                    completedFileCount: this.completedFiles.size,
+                    ...base(),
+                });
+            }
+            catch {
+                checkpointSaved = false;
+            }
+        }
+        return checkpointSaved;
+    }
+    /** Cancellation path part two: emit only after terminal lock ownership was proven. */
+    emitCancelled(checkpointSaved, emit, base) {
+        emit({ kind: "job_cancelled", checkpointSaved, ...base() });
+        this.machine.transition("cancelled");
+    }
+    finishFailed(error, emit, base) {
+        const failure = isUntangleError(error)
+            ? error
+            : new UntangleError(
+            // A stage that throws something untyped still leaves the index
+            // incomplete — that is the truthful classification. Never guess a
+            // more specific infrastructure code (spec.md §25.2).
+            "INDEX_INCOMPLETE", error instanceof Error ? error.message : String(error), { retryable: false, operation: "index_job", cause: error });
+        this.machine.transition("failed");
+        emit({
+            kind: "job_failed",
+            errorCode: failure.code,
+            message: failure.message,
+            retryable: failure.retryable,
+            ...base(),
+        });
+    }
+    /**
+     * Meldet eine Budgetverletzung der Queues. Eine Ueberschreitung des
+     * Speicherbudgets darf nie stillschweigend passieren (E4) — auch dann
+     * nicht, wenn sie unvermeidbar ist, weil ein einzelnes Element groesser
+     * als das ganze Budget ist.
+     */
+    reportOverBudget(queue, admission) {
+        this.options.logger?.log({
+            timestamp: this.options.clock.nowIso(),
+            level: "warn",
+            component: "jobs.index",
+            event: "queue_over_budget_admission",
+            jobId: this.jobId,
+            context: {
+                queue,
+                itemBytes: admission.itemBytes,
+                maxBytes: admission.maxBytes,
+                ordinal: admission.ordinal,
+                allowance: admission.allowance,
+            },
+        });
+    }
+    // -- pipeline -------------------------------------------------------------
+    async runPipeline(remaining, emit, base, signal) {
+        if (remaining.length === 0) {
+            return;
+        }
+        const { stages, identity } = this.options;
+        const memoryBudgetBytes = this.options.memoryBudgetBytes ?? DEFAULT_MEMORY_BUDGET_BYTES;
+        const embedBatchSize = this.options.embedBatchSize ?? DEFAULT_EMBED_BATCH_SIZE;
+        const analysisQueue = new BoundedQueue({
+            capacity: this.options.queueCapacities?.analysis ?? DEFAULT_ANALYSIS_CAPACITY,
+            maxBytes: Math.floor(memoryBudgetBytes / 2),
+            sizeOf: (item) => item.estimatedBytes,
+            onOverBudgetAdmission: (admission) => this.reportOverBudget("analysis", admission),
+        });
+        const upsertQueue = new BoundedQueue({
+            capacity: this.options.queueCapacities?.upsert ?? DEFAULT_UPSERT_CAPACITY,
+            maxBytes: Math.floor(memoryBudgetBytes / 2),
+            sizeOf: (item) => item.estimatedBytes,
+            onOverBudgetAdmission: (admission) => this.reportOverBudget("upsert", admission),
+        });
+        this.queues = { analysis: analysisQueue, upsert: upsertQueue };
+        const failAll = (error) => {
+            analysisQueue.fail(error);
+            upsertQueue.fail(error);
+        };
+        const analysisWorker = async () => {
+            for (const file of remaining) {
+                // Safe batch boundary: between files (architecture.md §21.3).
+                throwIfAborted(signal, "index_analysis");
+                const analyzed = await stages.analysis.analyze(file, { signal });
+                this.machine.advanceTo("computing_metrics");
+                emit({
+                    kind: "file_analyzed",
+                    repoId: identity.repoId,
+                    refId: identity.refId,
+                    fileId: file.fileId,
+                    filePath: file.filePath,
+                    parseQuality: analyzed.parseQuality,
+                    chunkCount: analyzed.chunkCount,
+                    symbolCount: analyzed.symbolCount,
+                    ...base(),
+                });
+                await analysisQueue.push(analyzed, { signal });
+            }
+            analysisQueue.close();
+        };
+        const embedWorker = async () => {
+            let batchIndex = 0;
+            while (true) {
+                const first = await analysisQueue.pull({ signal });
+                if (first.done) {
+                    break;
+                }
+                const batch = [first.value];
+                while (batch.length < embedBatchSize) {
+                    const next = analysisQueue.tryPull();
+                    if (next === null || next.done) {
+                        break;
+                    }
+                    batch.push(next.value);
+                }
+                // Safe batch boundary: before embedding the next batch.
+                throwIfAborted(signal, "index_embedding");
+                this.machine.advanceTo("embedding");
+                const embedded = await stages.embedding.embedBatch(batch, { signal });
+                emit({
+                    kind: "embedding_batch_completed",
+                    batchIndex,
+                    batchSize: batch.length,
+                    cacheHits: embedded.cacheHits,
+                    truncatedCount: embedded.truncatedCount,
+                    ...base(),
+                });
+                await upsertQueue.push(embedded, { signal });
+                batchIndex += 1;
+            }
+            upsertQueue.close();
+        };
+        const storeWorker = async () => {
+            while (true) {
+                const item = await upsertQueue.pull({ signal });
+                if (item.done) {
+                    break;
+                }
+                // Safe batch boundary: before starting the next upsert.
+                throwIfAborted(signal, "index_storing");
+                this.machine.advanceTo("storing");
+                const confirmation = await stages.storage.upsertBatch(item.value, { signal });
+                emit({
+                    kind: "points_stored",
+                    collection: confirmation.collection,
+                    pointCount: confirmation.pointCount,
+                    serializedBytes: confirmation.serializedBytes,
+                    ...base(),
+                });
+                // Record confirmed work ONLY after the store acknowledged the
+                // upsert, and persist the checkpoint ONLY after that (spec.md §10.5).
+                for (const file of item.value.files) {
+                    this.completedFiles.add(file.fileId);
+                    this.manifestHashes[file.fileId] = file.contentHash;
+                }
+                for (const [fileId, pointIds] of Object.entries(confirmation.storedPointIdsByFile)) {
+                    this.storedPointIdsByFile[fileId] = [...pointIds];
+                }
+                this.machine.advanceTo("checkpointing");
+                await this.saveCheckpoint({ signal });
+                emit({
+                    kind: "checkpoint_saved",
+                    stage: this.machine.stage,
+                    completedFileCount: this.completedFiles.size,
+                    ...base(),
+                });
+            }
+        };
+        const guard = async (worker) => {
+            try {
+                await worker();
+            }
+            catch (error) {
+                failAll(error);
+                throw error;
+            }
+        };
+        const settled = await Promise.allSettled([
+            guard(analysisWorker),
+            guard(embedWorker),
+            guard(storeWorker),
+        ]);
+        this.lastQueueStats = { analysis: analysisQueue.stats(), upsert: upsertQueue.stats() };
+        this.queues = null;
+        const rejections = settled
+            .filter((result) => result.status === "rejected")
+            .map((result) => result.reason);
+        const firstRealFailure = rejections.find((error) => !isJobCancelledError(error));
+        if (firstRealFailure !== undefined) {
+            throw firstRealFailure;
+        }
+        const firstCancellation = rejections[0];
+        if (firstCancellation !== undefined) {
+            throw firstCancellation;
+        }
+    }
+    async saveCheckpoint(options) {
+        if (this.generationId === null) {
+            throw indexIncompleteError("cannot save a checkpoint before the generation is known", {
+                retryable: false,
+                operation: "checkpoint_save",
+            });
+        }
+        const checkpoint = {
+            jobId: this.jobId,
+            generationId: this.generationId,
+            workspaceId: this.options.identity.workspaceId,
+            repoId: this.options.identity.repoId,
+            refId: this.options.identity.refId,
+            effectiveProfileId: this.options.identity.effectiveProfileId,
+            completedFiles: [...this.completedFiles].sort(),
+            storedPointIdsByFile: Object.fromEntries(Object.entries(this.storedPointIdsByFile).map(([fileId, ids]) => [fileId, [...ids]])),
+            manifestHashes: { ...this.manifestHashes },
+            stage: this.machine.stage,
+            savedAt: this.options.clock.nowIso(),
+        };
+        await this.options.checkpointStore.save(checkpoint, options.signal !== undefined ? { signal: options.signal } : undefined);
+    }
+}
+//# sourceMappingURL=runner.js.map