@hyperdrive.bot/fleet-server 0.3.99

package/dist/server/server/ingestion/cohort-resume.js

@@ -0,0 +1,171 @@
+// FINISH A HALF-DISPATCHED COHORT after a restart.
+//
+// A cohort of six that crashed with three groups still `pending` has three
+// sessions it owes the user and no process left to run them. This module is what
+// the next boot uses to make good on them - and, just as importantly, what stops
+// it making good on them twice.
+//
+// Three properties, each of which is the reason for a specific line below:
+//
+//   1. EXACTLY ONCE per group. Only `pending` groups are resumed. A group that
+//      was `running` when the process died is NOT here: its session died with the
+//      process, and Epic 3's `releaseStaleClaims()` is what returns its items to
+//      eligible. Resuming it as well would dispatch the same group twice for one
+//      gesture.
+//   2. THE SAME PATH. Every resumed group goes through `dispatchGroup` - the same
+//      function `applySelection` calls - so it gets the same claim, the same
+//      prompt and the same single-flight. A restart is not a shortcut.
+//   3. THE SAME CEILING. Every resumed group is admitted through the same
+//      `CapLimiter`. A restart is not permission to exceed the day's cap; a group
+//      the cap refuses stays `pending` and is offered again next boot.
+import { applyInFlightFilterIds, dispatchGroup, } from "./apply.js";
+/**
+ * Cancel a group that can never run, so the next boot does not offer it again.
+ *
+ * `pending -> cancelled` is the legal transition for "did not start and now never
+ * will", and it is the right one here: the filter was deleted, or the items left
+ * the source's window. Leaving the row `pending` instead would make the daemon
+ * re-scan a source on every single boot, forever, for a group that has nothing
+ * left to dispatch - and would keep the cohort non-terminal, so the progress row
+ * would poll forever too.
+ */
+function abandon(deps, group, reason) {
+    try {
+        deps.store.markGroup({
+            cohortId: group.cohortId,
+            groupKey: group.groupKey,
+            status: "cancelled",
+        });
+        deps.logger.info({ cohortId: group.cohortId, groupKey: group.groupKey, reason }, "Cohort group cancelled on resume: it can no longer be dispatched");
+    }
+    catch (error) {
+        deps.logger.warn({ err: error, cohortId: group.cohortId, groupKey: group.groupKey }, "Failed to cancel an undispatchable cohort group on resume");
+    }
+}
+/**
+ * Dispatch every resumable group, one at a time.
+ *
+ * SEQUENTIAL, never a parallel fan-out: a boot that found thirty pending groups
+ * must not open thirty provider sessions at once in one working directory. This
+ * is the same bound `applySelection`'s loop keeps, for the same reason.
+ *
+ * Best-effort as a whole: a throw anywhere is logged and the pass moves on. This
+ * runs inside `ScheduleService.start()`, and a resume that could take the daemon
+ * down at boot would turn one bad cohort into a host that never comes back.
+ */
+/**
+ * Resume ONE group: resolve its filter, its items and a unit of budget, then
+ * dispatch it down the same path a fresh apply takes.
+ *
+ * Extracted from the loop rather than inlined so the nesting stays inside the
+ * repo's limit, and so each of the four ways a group can decline to run has a
+ * named exit rather than a `continue` five levels deep.
+ */
+async function resumeOneGroup(deps, group, filters, claimedFilters) {
+    if (!filters.has(group.filterId)) {
+        filters.set(group.filterId, await deps.filterResolver(group.filterId));
+    }
+    const filter = filters.get(group.filterId) ?? null;
+    if (!filter) {
+        abandon(deps, group, "filter-deleted");
+        return;
+    }
+    // The SAME single-flight `applySelection` uses. A client that pressed Apply
+    // while the daemon was still booting owns the filter; its groups wait for the
+    // next boot rather than racing a live batch.
+    if (!claimedFilters.has(filter.id)) {
+        if (applyInFlightFilterIds.has(filter.id)) {
+            deps.logger.info({ filterId: filter.id, cohortId: group.cohortId, groupKey: group.groupKey }, "Skipping cohort resume: an apply is already running for this filter");
+            return;
+        }
+        applyInFlightFilterIds.add(filter.id);
+        claimedFilters.add(filter.id);
+    }
+    const items = await deps.resolveItems(filter, group.itemKeys);
+    if (items.length === 0) {
+        // Not an error and not a silent skip: the items left the source's window
+        // between the crash and the boot, so there is nothing left for this group to
+        // act on and saying so is the honest record.
+        abandon(deps, group, "items-unresolvable");
+        return;
+    }
+    // ONE unit of the day's budget for ONE group, through the same limiter the
+    // tick and the manual apply spend. A restart is not permission to exceed the cap.
+    const admission = deps.limiter.admit(filter.id, [group.groupKey], filter.cap.runsPerDay);
+    if (admission.admitted.length === 0) {
+        deps.logger.info({ filterId: filter.id, cohortId: group.cohortId, groupKey: group.groupKey }, "Cohort group left pending on resume: the filter's daily cap is spent");
+        // Left `pending` deliberately. Tomorrow's boot offers it again; a cancel here
+        // would silently drop work the cap only meant to defer.
+        return;
+    }
+    await dispatchGroup({
+        filter,
+        group: { key: group.groupKey, items },
+        runId: deps.runIdFactory(),
+        ledger: deps.ledger,
+        schedule: deps.schedule,
+        cohorts: deps.cohorts,
+        cohortId: group.cohortId,
+        limiter: deps.limiter,
+    });
+}
+export function resumeCohorts(deps) {
+    let resumable;
+    try {
+        // FIRST, and only ever here: a group still `running` had its session die with
+        // the previous process. Left alone it would keep its cohort non-terminal
+        // forever - the progress row would poll until the tab closed and the pruner
+        // could never collect it.
+        const interrupted = deps.store.failInterrupted();
+        if (interrupted > 0) {
+            deps.logger.info({ groups: interrupted }, "Cohort groups left running by a previous process recorded as failed");
+        }
+        resumable = deps.store.resumePending();
+    }
+    catch (error) {
+        deps.logger.warn({ err: error }, "Could not read resumable cohort groups; skipping resume");
+        return Promise.resolve();
+    }
+    if (resumable.length === 0) {
+        return Promise.resolve();
+    }
+    deps.logger.info({
+        groups: resumable.length,
+        cohorts: new Set(resumable.map((g) => g.cohortId)).size,
+    }, "Resuming cohort groups left pending by a previous run");
+    // Cached per filter so thirty groups of one cohort read the filter once.
+    const filters = new Map();
+    // Filters this pass reserved the single-flight guard for, released at the end.
+    const claimedFilters = new Set();
+    // RESOLVES EARLY, exactly as `applySelection` does and for the same reason.
+    // This is awaited inside `ScheduleService.start()`, so awaiting the dispatch
+    // loop here would block daemon boot until every resumed session came back - a
+    // forty-group cohort would hold the host down for minutes, and a provider that
+    // hangs would hold it down forever. The reconciliation above IS synchronous and
+    // has already happened by the time this returns; only the dispatching is
+    // detached.
+    //
+    // `void` plus a `.catch` that cannot itself throw: an unhandled rejection at
+    // boot would take the daemon down long after `start()` returned.
+    void (async () => {
+        try {
+            for (const group of resumable) {
+                try {
+                    await resumeOneGroup(deps, group, filters, claimedFilters);
+                }
+                catch (error) {
+                    deps.logger.warn({ err: error, cohortId: group.cohortId, groupKey: group.groupKey }, "Failed to resume a cohort group");
+                }
+            }
+        }
+        finally {
+            for (const filterId of claimedFilters) {
+                applyInFlightFilterIds.delete(filterId);
+            }
+        }
+    })().catch((error) => {
+        deps.logger.warn({ err: error }, "Cohort resume loop failed");
+    });
+    return Promise.resolve();
+}
+//# sourceMappingURL=cohort-resume.js.map

package/dist/server/server/ingestion/cohort-store.d.ts

@@ -0,0 +1,227 @@
+import type { DatabaseSync } from "node:sqlite";
+import { type Cohort, type CohortGroupStatus } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+/**
+ * How long a FINISHED cohort is kept. This story's decision; the epic left it
+ * unspecified.
+ *
+ * `30`, matching `LEDGER_PRUNE_DAYS`, on purpose: a cohort and the ledger rows
+ * it produced are two halves of one account of an apply, and letting them expire
+ * on different clocks would leave a month where the items say "handled" and
+ * nothing on disk says by what.
+ */
+export declare const COHORT_RETENTION_DAYS = 30;
+/**
+ * Which statuses keep a cohort IN FLIGHT, named one by one.
+ *
+ * This is "enumerate what you silence, not what you want" made structural. A
+ * query that asked for `status IN ('pending','running')` would make a sixth
+ * status vanish from the progress line silently - the cohort would collapse
+ * early and read as finished while groups in the unnamed status were still
+ * moving. Here every member is named, so:
+ *
+ *   - adding a sixth `CohortGroupStatus` without adding it below is a TypeScript
+ *     compile error, not a runtime surprise, and
+ *   - a status added as `counted` makes a cohort stay non-terminal (noisy, and
+ *     recoverable) rather than disappear from the count (invisible, and not).
+ *
+ * `counted` = still in flight, holds the cohort open.
+ * `excluded` = terminal, deliberately silenced from the in-flight tally. It is
+ * still returned in `counts`; what it is excluded from is the "is there anything
+ * left to wait for" question.
+ *
+ * IF YOU ADD A STATUS: add it here too, or this file will not compile.
+ */
+export declare const COHORT_STATUS_DISPOSITION: Record<CohortGroupStatus, "counted" | "excluded">;
+/**
+ * A group was asked to make a transition the state machine does not allow.
+ *
+ * Thrown rather than swallowed, and carrying `from`/`to` structurally so a
+ * caller never has to regex the message. Follows `ScheduleTargetGoneError` in
+ * `schedule/service.ts`.
+ */
+export declare class CohortTransitionError extends Error {
+    readonly cohortId: string;
+    readonly groupKey: string;
+    readonly from: CohortGroupStatus;
+    readonly to: CohortGroupStatus;
+    constructor(params: {
+        cohortId: string;
+        groupKey: string;
+        from: CohortGroupStatus;
+        to: CohortGroupStatus;
+    });
+}
+/**
+ * A patch arrived for a group nobody recorded.
+ *
+ * A wiring bug, not a data condition, and named separately from
+ * `CohortTransitionError` so the two are never confused: one means the caller
+ * asked for something illegal, the other means the caller is talking about a row
+ * that does not exist. Silently ignoring it would leave a cohort reporting
+ * `pending` forever with no way to tell that from a run that never started.
+ */
+export declare class CohortGroupMissingError extends Error {
+    readonly cohortId: string;
+    readonly groupKey: string;
+    constructor(cohortId: string, groupKey: string);
+}
+/**
+ * A cohort RPC refusal, carrying a closed-set code.
+ *
+ * Its own class rather than an `IngestionError` code, for the reason `ApplyError`
+ * states one file over: `IngestionError`'s set is consumed by `health.ts` to
+ * decide whether a SOURCE is `revoked` or `degraded`, and neither is true of "no
+ * such cohort" or "this daemon has no ledger". Adding these codes there would
+ * force a health verdict about an aggregator that is working perfectly.
+ *
+ * `code` is what reaches the client; the message may quote a local id, so it does
+ * not.
+ */
+export declare class CohortRpcError extends Error {
+    readonly code: "COHORT_MISSING" | "COHORT_STORE_UNAVAILABLE";
+    constructor(code: CohortRpcError["code"], message: string);
+}
+export interface CohortStoreOptions {
+    /**
+     * The ledger's handle, injected. This class NEVER calls `new DatabaseSync(...)`:
+     * a second connection to the same file would be a second WAL reader and would
+     * make a cohort write invisible to a ledger read in the same transaction.
+     */
+    db: DatabaseSync;
+    /** Epoch milliseconds. Injected so a test can pin the clock. */
+    now: () => number;
+}
+export interface CohortCreateParams {
+    cohortId: string;
+    filterId: string;
+    mode: "dry-run" | "apply";
+    requested: number;
+    groupsTotal: number;
+    truncated: number;
+    capPerDay: number | null;
+    groups: ReadonlyArray<{
+        key: string;
+        itemKeys: readonly string[];
+    }>;
+}
+export interface MarkGroupParams {
+    cohortId: string;
+    groupKey: string;
+    status: CohortGroupStatus;
+    /** Written on dispatch; what makes "dispatched exactly once" checkable. */
+    runId?: string | null;
+    agentId?: string | null;
+}
+export interface CancelRemainingResult {
+    /** Groups flipped from `pending` to `cancelled` by this call. */
+    cancelled: number;
+    /** Groups left alone on purpose. They still reach `succeeded` or `failed`. */
+    stillRunning: number;
+}
+export interface ResumableCohortGroup {
+    cohortId: string;
+    filterId: string;
+    groupKey: string;
+    itemKeys: string[];
+}
+export declare class CohortStore {
+    private readonly db;
+    private readonly now;
+    private readonly insertCohortStmt;
+    private readonly insertGroupStmt;
+    private readonly selectCohortStmt;
+    private readonly selectGroupStmt;
+    private readonly countsStmt;
+    private readonly updateGroupStmt;
+    private readonly touchCohortStmt;
+    private readonly cancelPendingStmt;
+    private readonly countRunningStmt;
+    private readonly finishCancelStmt;
+    private readonly resumePendingStmt;
+    private readonly selectInterruptedStmt;
+    private readonly failInterruptedStmt;
+    private readonly addFailedStmt;
+    private readonly selectPrunableStmt;
+    private readonly deleteGroupsStmt;
+    private readonly deleteCohortStmt;
+    constructor(options: CohortStoreOptions);
+    /**
+     * Record a cohort and one `pending` group per group, in ONE transaction.
+     *
+     * Half a cohort on disk - the header row without its groups - would report
+     * `groupsTotal: 6` above zero group rows, which `inspect` would collapse to
+     * `terminal: true` and a client would render as a finished batch that never
+     * ran. The transaction is what makes that state unreachable.
+     */
+    create(params: CohortCreateParams): void;
+    /**
+     * Move one group, and the parent's counters, together or not at all.
+     *
+     * The group row and the denormalized counter are updated inside ONE
+     * transaction precisely so the two can be compared later: `inspect` derives
+     * its counts from the rows, and a divergence from the columns is a real
+     * transaction bug rather than a test artifact.
+     */
+    markGroup(params: MarkGroupParams): void;
+    /**
+     * One cohort, or `null` when no such cohort exists.
+     *
+     * `null` here means THIS COHORT DOES NOT EXIST and nothing else. A read that
+     * fails throws, so the caller can tell the two apart - an empty answer for
+     * both would render a vanished sqlite file as a batch that was never created.
+     */
+    inspect(cohortId: string): Cohort | null;
+    /**
+     * Cancel what has not started, and say how many were deliberately left alone.
+     *
+     * `stillRunning` is not decoration. Without it the UI cannot tell the user that
+     * two sessions are still writing, and a cancel that stopped 3 of 5 groups reads
+     * identically to a cancel that stopped everything.
+     */
+    cancelRemaining(cohortId: string): CancelRemainingResult;
+    /**
+     * Every group a restart still owes work for.
+     *
+     * `pending` only, and only on a cohort nobody cancelled. A `running` group is
+     * NOT here: its session died with the process, and Epic 3's
+     * `releaseStaleClaims()` is what turns its claimed-but-unmarked items back
+     * into eligible ones. Resuming it here as well would dispatch the same group
+     * twice for one gesture.
+     */
+    resumePending(): ResumableCohortGroup[];
+    /**
+     * At BOOT ONLY: a group still marked `running` had its session die with the
+     * previous process. Record that as `failed`.
+     *
+     * This is the cohort-level counterpart to Epic 3's `releaseStaleClaims()`,
+     * which returns the ITEMS of such a group to eligible but knows nothing about
+     * cohorts. Without this, a crash leaves the group `running` forever: the cohort
+     * never reaches `terminal`, so the progress row polls until the tab is closed
+     * and the pruner is forbidden from ever collecting it.
+     *
+     * The safety of "every running group is dead" rests entirely on WHEN this
+     * runs. At `ScheduleService.start()` nothing in this process has dispatched
+     * anything yet, so a `running` row can only have been written by a process
+     * that is gone. Calling this at any other moment would mark a live session
+     * failed while it was mid-write, which is the exact thing `cancelRemaining`
+     * refuses to do. It is called from `resumeCohorts` and from nowhere else.
+     *
+     * `failed`, not `cancelled`: the session started, so a group that was mid-write
+     * may well have produced a partial artifact. Recording it as cancelled would
+     * claim nothing happened, and something may have.
+     */
+    failInterrupted(): number;
+    /**
+     * Drop finished cohorts older than `COHORT_RETENTION_DAYS`.
+     *
+     * NEVER a non-terminal cohort, regardless of age: a cohort with a pending group
+     * is work this daemon still owes, and age is not evidence it is finished. A
+     * long-stalled cohort is a bug to see, not a row to delete.
+     *
+     * Groups are deleted explicitly. There is no foreign key between the two
+     * tables, so there is no cascade to rely on, and an orphaned group row would
+     * keep answering `resumePending` forever with no cohort to attach to.
+     */
+    prune(nowMs?: number): number;
+}
+//# sourceMappingURL=cohort-store.d.ts.map