@kici-dev/orchestrator 0.1.26 → 0.2.0

package/dist/reporting/agent-failure-category.d.ts

@@ -13,7 +13,10 @@ export interface FailureSignals {
     hasInitFailure: boolean;
     /** The init-failure category (`InitFailureCategory`), if any. */
     initFailureCategory: string | null;
-    /** Any job ended in `timed_out_stale` (or a workflow/job timeout fired). */
+    /**
+     * Any job ended in an infra-class verdict that means it never really ran —
+     * `timed_out_stale` or `unroutable` — or a workflow/job timeout fired.
+     */
     timedOut: boolean;
     /** Any step recorded a non-zero exit code. */
     anyStepNonZeroExit: boolean;

package/dist/reporting/check-run-reporter.d.ts

@@ -30,7 +30,7 @@ import type { ProviderRegistry } from '../provider-registry.js';
 import type { StepLogBuffer } from './step-log-buffer.js';
 import { type SourceLocationData } from './check-run-summary.js';
 import type { CheckRunTrackingStore } from './check-run-tracking-store.js';
-import { ExecutionJobStatus } from '@kici-dev/engine';
+import { type TerminalJobStatus } from '@kici-dev/engine';
 /**
  * Dependencies for the CheckRunReporter.
  */
@@ -122,7 +122,7 @@ interface SetBuildCompleteOptions {
     repo: string;
     sha: string;
     workflowName: string;
-    status: Extract<ExecutionJobStatus, 'success' | 'failed' | 'cancelled' | 'timed_out_stale'>;
+    status: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
     routingKey?: string;
@@ -142,7 +142,7 @@ interface UpdateJobStatusOptions {
     sha: string;
     workflowName: string;
     jobName: string;
-    state: Extract<ExecutionJobStatus, 'success' | 'failed' | 'cancelled' | 'timed_out_stale'>;
+    state: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
     routingKey?: string;
@@ -167,7 +167,7 @@ interface UpdateWorkflowStatusOptions {
     repo: string;
     sha: string;
     workflowName: string;
-    overallStatus: Extract<ExecutionJobStatus, 'success' | 'failed' | 'cancelled' | 'timed_out_stale'>;
+    overallStatus: TerminalJobStatus;
     installationId?: number;
     /** Routing key for per-app credential lookup (e.g., "github:12345"). */
     routingKey?: string;
@@ -238,6 +238,19 @@ export declare class CheckRunReporter {
     private readonly progressTimers;
     /** L1 cache: first in-progress sent flag (synced to `check_run_tracking.in_progress_sent_at`). */
     private readonly inProgressSent;
+    /**
+     * Keys whose job check run has already been completed.
+     *
+     * Makes the check run's status monotonic: once a `completed` update is
+     * issued, no later step-progress update may push it back to `in_progress`.
+     * Cancelling the pending debounce timer at completion time is not sufficient
+     * on its own — a step status that arrives after the completion schedules a
+     * FRESH timer, which then fires and leaves the check run showing
+     * `status: in_progress` with a terminal `conclusion` already attached. That
+     * is the permanently-unresolved state check-run completion exists to prevent,
+     * so the guard is on the write itself rather than on the timer.
+     */
+    private readonly terminalSent;
     /**
      * L1 cache: runId → set of check-run composite keys. Synced to the
      * indexed `check_run_tracking.run_id` column so a replacement coord can
@@ -334,21 +347,28 @@ export declare class CheckRunReporter {
      */
     setBuildComplete(opts: SetBuildCompleteOptions): void;
     /**
-     * Clean up step-progress entries, debounce timers, and DB rows for a
-     * completed run. Called when the execution tracker prunes the run.
+     * Clean up step-progress entries and debounce timers for a completed run.
+     * Called when the execution tracker prunes the run.
      *
-     * In-memory cleanup is synchronous; the DB cleanup is fire-and-forget
-     * because the caller (run-pruning hook) is on the response-shaping path
-     * and shouldn't block on a network round-trip. Failure logs but does
-     * not propagate.
+     * In-memory only. Database rows are owned by the retention sweep in
+     * `queue/cleanup.ts`, which deletes on inactivity age rather than on run
+     * completion. Deleting here would strand a late terminal update: a check-run
+     * status PATCH that arrives after the prune resolves its check-run ID by
+     * loading through to this row, and a deleted row makes that lookup fail.
      */
     cleanupRun(runId: string): void;
     /**
-     * Hydrate the L1 caches from the DB after a leader switch (or any
-     * boot-time recovery). Called once on coord become-leader so the
-     * runIdToKeys reverse map is populated for any future cleanupRun calls
-     * without requiring a DB round-trip per cleanup. If no store is wired,
+     * Mark the reporter as DB-backed after a leader switch (or any boot-time
+     * recovery). Called once on coord become-leader. If no store is wired,
      * this is a no-op.
+     *
+     * Nothing is hydrated up front — the table can be large across many shas —
+     * so check-run IDs load through on demand inside `resolveCheckRunId`. The
+     * `runIdToKeys` reverse map is rebuilt only from this coord's own writes,
+     * so a run whose keys all predate the switch leaves `cleanupRun` nothing to
+     * evict. That is harmless: the L1 caches it clears are equally empty on a
+     * fresh coord, and the DB rows belong to the retention sweep rather than to
+     * run prune.
      */
     recoverState(): Promise<void>;
     /** Track a check run key associated with a runId for later cleanup. */

package/dist/reporting/check-run-tracking-store.d.ts

@@ -29,7 +29,16 @@ export interface CheckRunTrackingState {
     stepProgress: StepProgressEntry[];
     /** Timestamp the first running-step transition was sent to GitHub. */
     inProgressSentAt?: Date;
-    /** KiCI run this check-run belongs to. Used by `cleanupRun`. */
+    /**
+     * Timestamp the terminal (`completed`) update was accepted by the provider.
+     * Undefined means we have no record of sending it — not proof it failed.
+     */
+    terminalSentAt?: Date;
+    /**
+     * KiCI run this check-run belongs to. Written at create time so the row
+     * records its owning run; it is an attribution/lookup key, not a retention
+     * key — the sweep prunes on `updated_at` age alone.
+     */
     runId?: string;
     /** Last persisted update time; powers debounce-after-failover recovery. */
     updatedAt?: Date;
@@ -64,8 +73,13 @@ export declare class CheckRunTrackingStore {
      * Performed as an upsert so a re-issued setPending after a coord
      * failover replaces the prior ID rather than silently leaving a row
      * mismatched with the GitHub-side state.
+     *
+     * `runId` is written alongside so the row records which run owns it from
+     * the moment it is created. A row created without it carries a NULL
+     * `run_id`, which makes `listKeysByRunId` under-report and leaves the
+     * column useless to an operator asking which run posted a given check.
      */
-    setCheckRunId(key: CheckRunTrackingKey, checkRunId: number): Promise<void>;
+    setCheckRunId(key: CheckRunTrackingKey, checkRunId: number, runId?: string): Promise<void>;
     /**
      * Lookup the check-run ID for a key. Returns undefined if no row exists
      * yet OR the row exists but the GitHub create has not finished
@@ -93,6 +107,14 @@ export declare class CheckRunTrackingStore {
      * cluster-wide.
      */
     markInProgressSent(key: CheckRunTrackingKey, runId?: string): Promise<void>;
+    /**
+     * Stamp that the terminal (`completed`) provider update was sent.
+     *
+     * Mirrors `markInProgressSent`. Best-effort like every write on this table —
+     * the caller swallows failures, because a tracking write must never break
+     * check-run reporting — so a null column is "no record", not "never sent".
+     */
+    markTerminalSent(key: CheckRunTrackingKey, runId?: string): Promise<void>;
     /**
      * Get the full state snapshot for a key. Used by the L1 cache to
      * hydrate on miss and by tests to verify the on-disk layout. Returns
@@ -104,17 +126,56 @@ export declare class CheckRunTrackingStore {
      */
     deleteRow(key: CheckRunTrackingKey): Promise<boolean>;
     /**
-     * List every key currently tracked for a runId. Used by `cleanupRun`
-     * to reproduce the runId → keys reverse index that the in-memory map
-     * provided. Index `idx_check_run_tracking_run_id` keeps this O(matches).
+     * List every key currently tracked for a runId — the runId → keys reverse
+     * index that `CheckRunReporter` otherwise holds only in memory, so a
+     * replacement coord that never saw the run can still recover it. Index
+     * `idx_check_run_tracking_run_id` keeps this O(matches).
+     *
+     * No caller in tree: run-prune stopped consulting the store once row
+     * lifetime moved to {@link pruneStale}. Kept as the read half of the
+     * `run_id` column, beside {@link deleteByRunId}.
      */
     listKeysByRunId(runId: string): Promise<CheckRunTrackingKey[]>;
     /**
-     * Delete every row for a runId. Mirrors the bulk-cleanup semantics of
-     * `cleanupRun` so a single call from execution-tracker prune releases
-     * all rows for the run.
+     * Delete every row for a runId.
+     *
+     * Not on the run-prune path: `cleanupRun` evicts only its in-memory state,
+     * because a row deleted at prune time strands a late terminal PATCH that
+     * still has to resolve its check-run ID. Routine reclamation is
+     * {@link pruneStale}. This remains as the targeted escape hatch for
+     * discarding one run's rows deliberately.
      */
     deleteByRunId(runId: string): Promise<number>;
+    /**
+     * Delete every row untouched for longer than `retentionDays`, and return
+     * how many were removed. `retentionDays <= 0` disables the sweep, matching
+     * the `JobQueue.pruneTerminalDispatchRows` convention.
+     *
+     * `updated_at` is the staleness signal because the table carries no status
+     * column, and the rows that most need reaping are precisely the ones with no
+     * `run_id` to join on — so the predicate is age alone, deliberately
+     * unfiltered by `run_id`. Every upsert bumps `updated_at`, so a run that is
+     * still emitting check-run traffic keeps its rows fresh; a workflow-level
+     * row, written once at create, ages from creation.
+     *
+     * Inactivity age rather than run completion is what keeps a reader safe: a
+     * row a diagnostic is polling for cannot disappear underneath it, because
+     * the window is days and the row is only removed once nothing has touched
+     * it for that long.
+     *
+     * The bound this trades for that safety: age is measured on check-run
+     * writes, not on run liveness, so a run that stays live while emitting no
+     * check-run traffic for longer than the window — a run parked on a manual
+     * approval gate, whose hold statuses go through `CheckStatusPoster` and
+     * never touch this table — has its rows swept while it is still running.
+     * The reporter's L1 cache still holds the ID, so its terminal update lands
+     * anyway on a coord that stayed up; it takes a restart or failover across
+     * that window to leave the update with no row to load through and no
+     * check-run ID to resolve.
+     *
+     * Called from the hourly cleanup tick.
+     */
+    pruneStale(retentionDays: number): Promise<number>;
     private selectRow;
     private upsertRow;
 }
@@ -127,6 +188,7 @@ export declare function rowToState(row: {
     build_creation_state: string | null;
     step_progress_json: unknown;
     in_progress_sent_at: Date | null;
+    terminal_sent_at: Date | null;
     run_id: string | null;
     updated_at: Date;
 }): CheckRunTrackingState;

package/dist/reporting/execution-tracker.d.ts

@@ -13,7 +13,7 @@
  */
 import { type Kysely } from 'kysely';
 import type { Database } from '../db/types.js';
-import { ExecutionRunStatus, type InitFailure, ScalerEventType } from '@kici-dev/engine';
+import { ExecutionRunStatus, type InitFailure, RunFailureClass, ScalerEventType, OrchLogPhase } from '@kici-dev/engine';
 import type { ObserverRegistry } from '../ws/observer-registry.js';
 import type { LogStorage } from './log-storage.js';
 import type { JobQueue } from '../queue/job-queue.js';
@@ -50,6 +50,12 @@ export interface ExecutionContext {
     triggerActorUsername?: string | null;
     /** Immutable provider user id of the triggering actor (preferred for resolution). */
     triggerActorUserId?: string | null;
+    /**
+     * Why a terminal run failed (`RunFailureClass`). Set on terminal-failed /
+     * cancelled runs; forwarded to the Platform on `execution.status` so managed
+     * subscriptions can match on it. Null/undefined for success or non-terminal.
+     */
+    failureClass?: RunFailureClass | null;
     /** Workflow-level concurrency config from the lock file. */
     concurrency?: {
         cancelInProgress?: boolean;
@@ -68,6 +74,8 @@ interface WorkflowCompleteCallbackData {
     }>;
     routingKey?: string;
     repo: string;
+    /** Why the run failed (`RunFailureClass`); carried onto the `__workflow_complete` event. */
+    failureClass?: string;
 }
 /** Data passed to the onJobComplete callback. */
 interface JobCompleteCallbackData {
@@ -78,6 +86,11 @@ interface JobCompleteCallbackData {
     routingKey?: string;
     repo: string;
     workflowName: string;
+    /**
+     * Job payload carried by the status update (step results, error, duration).
+     * Feeds the enriched check-run summary and the failure description.
+     */
+    data?: Record<string, unknown>;
 }
 export interface ExecutionTrackerDeps {
     db: Kysely<Database>;
@@ -125,8 +138,8 @@ export interface ExecutionTrackerDeps {
      * jobs that never started. Persisted in execution_jobs.init_failure.
      */
     initFailure?: InitFailure, 
-    /** Ordered bound deployment-environment names for this job (multi-env jobs). */
-    environments?: string[]) => void;
+    /** Ordered bound deployment-context names for this job (multi-env jobs). */
+    contexts?: string[]) => void;
     /**
      * Optional callback to emit run.event messages to Platform.
      * Fires at orchestrator lifecycle points (dispatch, agent assignment, job start/complete).
@@ -145,12 +158,37 @@ export interface ExecutionTrackerDeps {
         metadata?: Record<string, unknown>;
         durationMs?: number | null;
     }) => void;
+    /**
+     * Optional live forward of orchestration/provisioning log lines to Platform,
+     * fired in addition to persisting them to `logStorage`. Platform fans these
+     * out to run-detail browser subscribers as `orch-log.lines`. Best-effort:
+     * a push failure must never break dispatch.
+     *
+     * SECURITY: like `onRunEventEmit`, this payload MUST NOT carry an `orgId`.
+     * The Platform attributes the tenant from `authState.orgId`.
+     */
+    onOrchLog?: (chunk: {
+        runId: string;
+        jobId: string;
+        phase: OrchLogPhase;
+        lines: string[];
+        ts: number;
+    }) => void;
     /** Optional log storage for writing per-job orchestration logs (JSONL). */
     logStorage?: LogStorage;
     /** Org ID for this orchestrator instance (used in run.event emission). */
     orgId?: string;
     /** Optional job queue for cascading run failures to dispatch_queue entries. */
     jobQueue?: JobQueue;
+    /**
+     * Resolves a routing key to its owning org (customer_id), used to populate
+     * `execution_runs.customer_id` at insert time so the concurrency-gate running
+     * count is scoped per tenant. Injected (bound to `(rk) => resolveOrgId(db, rk)`
+     * at construction) to keep the tracker free of a pipeline import cycle. When
+     * absent — or when a run has no routing key — the row falls back to the
+     * `'__default__'` column default (the no-source fallback org).
+     */
+    resolveOrgId?: (routingKey: string) => Promise<string>;
 }
 export declare class ExecutionTracker {
     private readonly db;
@@ -163,9 +201,11 @@ export declare class ExecutionTracker {
     private readonly onExecutionStatusChange?;
     private readonly onJobStatusChange?;
     private readonly onRunEventEmit?;
+    private readonly onOrchLog?;
     private readonly logStorage?;
     private readonly orgId?;
     private readonly jobQueue?;
+    private readonly resolveOrgIdFn?;
     private readonly runs;
     /**
      * Per-run async-mutex chain. `onJobStatus` and `addJobsToRun` mutate the same
@@ -200,6 +240,12 @@ export declare class ExecutionTracker {
     /** Callback fired when a job's needs are satisfied and it's ready for dispatch. */
     onJobReadyCallback?: (runId: string, jobName: string) => Promise<void>;
     constructor(deps: ExecutionTrackerDeps);
+    /**
+     * Resolve the owning org for an `execution_runs` row from its routing key.
+     * Falls back to the `'__default__'` org (the column default) when the run has
+     * no routing key or no resolver is injected (single-tenant / test contexts).
+     */
+    private resolveCustomerId;
     /**
      * Register a callback fired when a job's needs become satisfied.
      * The processor sets this to dispatch newly-ready jobs to agents.
@@ -222,8 +268,8 @@ export declare class ExecutionTracker {
         waveGated?: boolean;
         waveMaxParallel?: number;
         waveFailFast?: boolean;
-        environments?: string[];
-        skippedEnvironments?: string[];
+        contexts?: string[];
+        skippedContexts?: string[];
         envWarning?: string;
     }>, routingKey?: string, 
     /** Secret context names dispatched with jobs (for context-disable job lookup). */
@@ -254,7 +300,9 @@ export declare class ExecutionTracker {
     /** Immutable provider user id of the triggering actor. */
     triggerActorUserId?: string | null, 
     /** Agent provenance label when triggered through an agent credential. */
-    triggeredByAgentLabel?: string | null): Promise<void>;
+    triggeredByAgentLabel?: string | null, 
+    /** Pull-request number for PR-triggered runs; null/omitted for non-PR runs. */
+    prNumber?: number | null): Promise<void>;
     /**
      * Upsert one execution_jobs row per dispatched job (idempotent on
      * (run_id, job_id) to tolerate a race with an early `onJobStatus`). The
@@ -311,8 +359,8 @@ export declare class ExecutionTracker {
         baseJobName?: string;
         variantKind?: string;
         variantLabel?: string;
-        environments?: string[];
-        skippedEnvironments?: string[];
+        contexts?: string[];
+        skippedContexts?: string[];
         envWarning?: string;
     }>, dispatchedContexts?: string[], 
     /** Synthetic job ID to replace (e.g. needs-pending-deploy-{uuid}). */
@@ -438,12 +486,17 @@ export declare class ExecutionTracker {
      */
     onBuildFailedBeforeTracking(runId: string, workflowName: string, provider: string, repoIdentifier: string, ref: string, sha: string, deliveryId: string | null, providerContext: Record<string, unknown>, routingKey: string, triggerEvent?: string, commitMessage?: string, failureReason?: string, initFailure?: InitFailure): Promise<void>;
     /**
-     * Insert a `failed` execution_runs row directly for an init failure that
-     * occurred BEFORE onExecutionStarted ran (so no in-memory state exists
-     * and no jobs were dispatched). Also writes the structured init_failure
-     * signal and fires onExecutionStatusChange so Platform's projection picks
-     * it up via the normal forward path. Idempotent: if a row already exists
-     * for this runId, the insert is a no-op (ON CONFLICT DO NOTHING).
+     * Write a `failed` execution_runs row directly for an init failure, and drop
+     * any in-memory run so the recorded failure is the run's final word. Also
+     * writes the structured init_failure signal and fires onExecutionStatusChange
+     * so Platform's projection picks it up via the normal forward path.
+     *
+     * An existing row for this runId is overwritten while it is still live —
+     * dispatch may already have registered the run before init failed — and left
+     * alone once it is terminal, so a run that genuinely finished is never
+     * rewritten as failed. When the guard leaves the row alone, the failure
+     * metric and the Platform forward are suppressed with it: this orchestrator
+     * did not record the init failure, so it must not report one.
      *
      * Closes the silent pre-run-failure gap — without this helper, secret /
      * install-secret / all-jobs-rejected early-exits in dispatch-matched-workflow
@@ -469,7 +522,11 @@ export declare class ExecutionTracker {
      * `execution_runs` row in the `held` state — alive and resumable — so the
      * dashboard run list surfaces the paused workflow. No jobs are tracked: the
      * workflow-scoped held_runs row + pending workflow context (written by the
-     * caller) keep the run from being counted complete. Idempotent on runId.
+     * caller) keep the run from being counted complete, and any in-memory run is
+     * dropped so jobs registered before the gate cannot roll it up.
+     *
+     * An existing row for this runId is flipped to `held` while it is still live
+     * and left alone once it is terminal — a finished run is never re-held.
      */
     recordRunHeld(args: {
         runId: string;
@@ -481,10 +538,17 @@ export declare class ExecutionTracker {
         deliveryId: string | null;
         providerContext: Record<string, unknown>;
         routingKey: string;
-        environmentName?: string;
+        contextName?: string;
         reason: string;
         triggerEvent?: string;
         commitMessage?: string;
+        /**
+         * Pull-request number for PR-triggered holds; null/omitted for non-PR runs.
+         * Stamped so PR-scoped `/kici approve|reject` (which joins `execution_runs`
+         * on `pr_number`) can attribute the held run to its PR — a NULL leaves a
+         * security hold fail-closed unreachable by the comment path.
+         */
+        prNumber?: number | null;
     }): Promise<void>;
     /**
      * Flip a `held` run back to `pending` so the resumed dispatch can proceed
@@ -524,6 +588,54 @@ export declare class ExecutionTracker {
      * Check if all tracked jobs in a run have reached terminal state.
      */
     isRunComplete(runId: string): boolean;
+    /**
+     * Take a token holding a run open while some of its jobs are still to be
+     * registered. Returns true when a token was taken, false for an unknown or
+     * already-completed run (nothing to hold).
+     *
+     * Three registration windows need this. A run whose source-pack `__build__`
+     * job is dispatched first is registered with that job ALONE, and its real
+     * jobs are only dispatched once the build finishes. A deferred init job and a
+     * deferred dynamic entry each register their jobs from a fire-and-forget task
+     * that outlives the dispatch call. In every case, without a token the already
+     * -registered jobs reaching a terminal state satisfies {@link isRunComplete},
+     * so the run is finalized early: a terminal run status is written, the
+     * provider check is posted, and the status is forwarded to the Platform — all
+     * before a single real job has run.
+     *
+     * The hold is a counter on the in-memory run, read only by
+     * {@link isRunComplete} — deliberately NOT a synthetic entry in the run's job
+     * map. The job map is what every outward projection enumerates (the Platform
+     * `state.replay` snapshot and its `execution_jobs` mirror, the reported job
+     * count, the workflow-complete job results, the active-run summary), so a
+     * marker parked there would surface as a phantom pending job on the dashboard
+     * and an off-by-one job count for the length of the window. A counter holds
+     * the completion check open without being visible to any of them, and it
+     * writes no `execution_jobs` row, so it never surfaces as a phantom job.
+     * A token whose holder never settles keeps the run `running` until the stale
+     * detector reaps it — the same backstop that covers a job that never
+     * reports.
+     *
+     * Each token must be paired with exactly one {@link releasePendingJobsHold}.
+     */
+    holdRunForPendingJobs(runId: string): boolean;
+    /**
+     * Drop one token taken by {@link holdRunForPendingJobs}, and finalize the run
+     * if dropping the LAST one left every remaining job terminal.
+     *
+     * The re-evaluation matters for the same reason it does in `addJobsToRun`:
+     * once the last token is gone nothing else drives a completion check (no
+     * further `job.status` messages are coming for already-finished jobs), so a
+     * run whose jobs all finished while a token was held would otherwise hang in
+     * `running` forever. Only the last release may finalize — while any other
+     * token is outstanding more jobs are still on their way.
+     *
+     * Idempotent — a no-op when no token is outstanding. The count clamps at
+     * zero so a stray extra release cannot drive it negative, which would
+     * permanently un-hold the run and stop a genuinely outstanding token from
+     * holding it open.
+     */
+    releasePendingJobsHold(runId: string): Promise<void>;
     /**
      * Get the current overall status of a run.
      */
@@ -533,6 +645,16 @@ export declare class ExecutionTracker {
      * Used by LogWriter to construct log paths.
      */
     getJobName(runId: string, jobId: string): string | undefined;
+    /**
+     * Find the runId of the in-memory run that owns a given dispatched jobId.
+     *
+     * The agent's `event.emit` message is job-scoped (it carries only the
+     * emitting jobId), but run context is keyed by runId. This maps job → run so
+     * `getExecutionContext(runId)` resolves for an agent-emitted custom event.
+     * In-memory only (matches `getExecutionContext`'s semantics); emit happens
+     * mid-execution, so the owning run is always live in `this.runs`.
+     */
+    getRunIdForJob(jobId: string): string | undefined;
     /**
      * Get execution context for a run from in-memory state.
      * Used by commit status reporting to access provider/repo/sha info.
@@ -584,6 +706,7 @@ export declare class ExecutionTracker {
         triggeredBy?: string | null;
         triggeredByAgentLabel?: string | null;
         failureReason?: string;
+        failureClass?: RunFailureClass;
         jobCount: number;
         startedAt: number;
         completedAt?: number;
@@ -659,13 +782,35 @@ export declare class ExecutionTracker {
      * stale detector when memRun is still tracked locally — writes the
      * execution_runs terminal row, fires Platform-forwarding +
      * workflow-complete callbacks, and schedules in-memory pruning.
+     *
+     * The terminal write is clobber-guarded. If the guard rejects it (the run
+     * already finished through the normal path), only the pruning happens: the
+     * recomputed status is not this orchestrator's record of the run, so neither
+     * the callbacks nor the completion metrics may claim it.
      */
     private completeRunFromMemoryState;
+    /**
+     * Drop a finished run's in-memory state after a grace period, so a late
+     * status/heartbeat arriving just after completion still finds its run.
+     *
+     * This is the single owner of run-lifetime state teardown, so every
+     * run-lifetime map is cleared here — including `testRunIds`: a run finished by
+     * the stale detector rather than the normal completion path would otherwise
+     * leave its observer-broadcast marker behind for the process's lifetime.
+     * (`runLockTails` is deliberately not touched: it is keyed by runId but scoped
+     * to a lock's holders, and `withRunLock` drops its own entry on release.)
+     */
+    private scheduleRunPrune;
     /**
      * Path B helper: complete a run using DB rows only (crash recovery /
      * pruned runs). Loads jobs + run row from execution_jobs/execution_runs,
      * checks the all-terminal predicate + run-state guard, then writes the
      * terminal row and fires Platform-forwarding callbacks.
+     *
+     * The run-state read and the terminal write are separate statements, so the
+     * write carries its own clobber guard. If that guard rejects it — a
+     * concurrent normal completion landed in between — the callbacks and the
+     * completion metrics are suppressed with it.
      */
     private completeRunFromDbFallback;
     /**
@@ -704,10 +849,21 @@ export declare class ExecutionTracker {
      *
      * Per locked decision:
      * - success ONLY if ALL jobs pass (skipped jobs count as success)
-     * - failed if ANY job failed or timed_out_stale
+     * - failed if ANY job ended in a failure status (see `STATUS_FAILURE_CLASS`)
      * - cancelled if ANY job cancelled (and none failed)
      */
     private computeRunStatus;
+    /**
+     * Derive the failure class from a terminal run's job statuses. Null for
+     * success (and any non-terminal status). `timed_out_stale` / `drift_dropped`
+     * / `unroutable` jobs — a job that never dispatched, whose `runsOn` matched no
+     * agent, whose agent went silent, or that was dropped by a topology reroute —
+     * are infra-class and collapse to `timed_out`; any other `failed` run is a
+     * `step_failure` (a job actually ran and failed). `never_started` (init
+     * failure) and `dead_orchestrator` (Platform-detected) are stamped by their
+     * own paths, not here.
+     */
+    private computeFailureClass;
 }
 export {};
 //# sourceMappingURL=execution-tracker.d.ts.map

package/dist/reporting/fs-log-storage.d.ts

@@ -1,8 +1,9 @@
 /**
  * Filesystem-backed log storage.
  *
- * Stores execution logs as files on disk. No TTL -- logs persist indefinitely.
- * Supports append-only writes (JSONL accumulation) and cursor-based pagination.
+ * Stores execution logs as files on disk. Lifecycle is operator-managed (the
+ * cleanup retention sweep runs on the S3 backend only). Supports append-only
+ * writes (JSONL accumulation) and cursor-based pagination.
  *
  * File layout:
  *   {basePath}/executions/{runId}/job-{name}/step-{index}.log
@@ -14,6 +15,27 @@ interface FilesystemLogStorageOptions {
 export declare class FilesystemLogStorage implements LogStorage {
     private readonly basePath;
     constructor(options: FilesystemLogStorageOptions);
+    /**
+     * Resolve a logical log path to an absolute filesystem path, refusing
+     * anything that would address a file outside the storage root.
+     *
+     * Two checks, deliberately both. `assertSafeLogPath` is the effective gate on
+     * POSIX: once a path is relative and carries no `..` segment, `resolve()`
+     * cannot produce anything outside the root, so the comparison below is
+     * unreachable there. It is kept as a second, independent statement of the
+     * same invariant — it is what catches a resolver with extra escape hatches
+     * (win32 resolves a drive-relative `C:foo` against that drive's own cwd, not
+     * against the root), and it means relaxing the logical check cannot silently
+     * reopen an escape.
+     *
+     * Both checks are lexical — neither resolves symlinks. A symlink planted
+     * inside the log root still redirects a logically-clean path outside it;
+     * containing that would need a `realpath` on every call, and the log root is
+     * the orchestrator user's own directory.
+     *
+     * Every verb on this class funnels through here, so a new caller is
+     * contained on arrival.
+     */
     private fullPath;
     /**
      * Ensure the directory for a given file path exists.
@@ -33,9 +55,16 @@ export declare class FilesystemLogStorage implements LogStorage {
      */
     private listRecursive;
     append(path: string, data: string): Promise<void>;
+    appendStreaming(path: string, data: string): Promise<void>;
+    finalize(_path: string): Promise<void>;
     read(path: string, options?: LogReadOptions): Promise<LogReadResult>;
     exists(path: string): Promise<boolean>;
     list(prefix: string): Promise<string[]>;
+    listWithMetadata(prefix: string): Promise<Array<{
+        path: string;
+        lastModified: Date;
+    }>>;
+    deleteMany(paths: string[]): Promise<number>;
 }
 export {};
 //# sourceMappingURL=fs-log-storage.d.ts.map