@hyperdrive.bot/fleet-server 0.3.99

package/dist/server/server/ingestion/cap-limiter.js

@@ -0,0 +1,264 @@
+// ONE runs/day budget, spent by both the tick path and the manual "Apply now"
+// path, held on disk rather than in a field.
+//
+// The two things this file exists to guarantee:
+//
+//   1. A filter capped at 10/day mints at most 10 sessions, no matter which
+//      path asked and no matter how many times the daemon restarted. The budget
+//      is a row in `ingestion_dispatch_budget`, so a restart resumes the day's
+//      spend instead of resetting it.
+//   2. A SHORTENED batch never looks like a completed one. `admit()` returns a
+//      `CapReport` carrying the number it dropped, so the truncation is a value
+//      in the caller's hands and not a log line nobody reads.
+//
+// This module is deliberately storage-blind: it holds a `CapBudgetStore` and
+// nothing else. It opens no database connection, imports no sqlite binding, and
+// has no idea where the daemon's home directory is - a grep in the story's AC #1
+// fails the build if any of the three ever appears here. All persistence, and
+// the clamp that makes it race-safe, live in the ledger's `reserveDispatches`
+// transaction.
+//
+// It also does not decide WHEN to dispatch. `admit` is a reservation, spent at
+// admit time, and `release` is the only way back (decision D-3.4-B). Nothing
+// here infers "it probably did not run" on the caller's behalf, because that
+// would be a second place recording intent as fact.
+import { nextUtcMidnight, utcDateKey } from "./utc-day.js";
+/**
+ * Reject a cap that cannot mean anything, rather than silently uncapping.
+ *
+ * `0` throws instead of admitting everything (the uncapped reading) or nothing
+ * (the frozen reading): both are defensible, which is exactly why guessing is
+ * worse than refusing. A caller that means "no cap" has to say so somewhere
+ * other than here.
+ */
+function assertValidCap(capPerDay) {
+    if (!Number.isInteger(capPerDay) || capPerDay < 1) {
+        throw new RangeError(`capPerDay must be a positive integer, received ${capPerDay}`);
+    }
+}
+function assertValidFilterId(filterId) {
+    if (typeof filterId !== "string" || filterId.trim().length === 0) {
+        throw new RangeError(`filterId must be a non-empty string, received ${JSON.stringify(filterId)}`);
+    }
+}
+/**
+ * ONE instance per daemon, holding THREE budget verbs over the same filter id.
+ *
+ *   `admit`      the APPLY budget. Ledger-backed, on disk, survives a restart,
+ *                CONSUMES on every call. Bounds outward artifacts - merge
+ *                requests, Gmail drafts, reviewer pings.
+ *   `admitDryRun` the DRY-RUN tuning budget (Story 5.2). In memory on this
+ *                object, reset by a restart, CONSUMES on every call. Bounds
+ *                token spend and nothing else.
+ *   `preview`    read-only (Story 5.3). Reads the SAME on-disk counter `admit`
+ *                spends and returns the identical `CapReport`, CONSUMING
+ *                NOTHING. It exists because the confirmation screen has to show
+ *                a number before the user has agreed to it, and computing that
+ *                number with `admit` would spend the day's real budget on a
+ *                plan the user is still free to cancel.
+ *
+ * Three verbs, two counters, one object. Adding a fourth means saying which of
+ * the two substrates it spends and why.
+ */
+export class CapLimiter {
+    constructor(options) {
+        // The SECOND counter: the day's dry-run budget, per filter, in memory.
+        //
+        // This counter is in memory and resets on daemon restart; that is acceptable
+        // for the tuning budget and would not be acceptable for `admit`, whose budget
+        // bounds outward artifacts and therefore lives on disk.
+        //
+        // It is not a shortcut, it is the only option, and the reason is decision D2 of
+        // the epic: the dry-run path writes NO ledger row, so there is nothing on disk
+        // to count. Overspending this budget costs tokens. Overspending `admit`'s costs
+        // merge requests, Gmail drafts and messages other people read. Do not "fix"
+        // this by moving it into sqlite - that would hand the dry run a write path it
+        // exists not to have, and AC #7 (the ledger is byte-identical across a dry run)
+        // would stop being true.
+        this.dryRunBudget = new Map();
+        this.store = options.store;
+        this.now = options.now ?? (() => new Date());
+    }
+    /**
+     * Reserve budget for `items` and hand back the ones that fit.
+     *
+     * Generic so the caller's item type survives the trip: the limiter never looks
+     * inside a `T`, and persists a COUNT and nothing derived from item content -
+     * which is what makes it structurally impossible for a payload carrying a
+     * credential to reach the ledger through this surface.
+     *
+     * `admitted` is `items.slice(0, granted)`, preserving input order, so the
+     * caller's own prioritisation decides which items make the cut.
+     */
+    admit(filterId, items, capPerDay) {
+        assertValidFilterId(filterId);
+        assertValidCap(capPerDay);
+        // The clock is called ONCE. Reading it twice would let the day key and
+        // `resetsAt` straddle a midnight - a batch charged to yesterday but told to
+        // reset tomorrow night, once a day, forever.
+        const now = this.now();
+        const requested = items.length;
+        // An empty ask short-circuits BEFORE the store, so it cannot create a
+        // `dispatched = 0` row. An absent row and a zero row must stay
+        // indistinguishable to every reader (AC #7).
+        if (requested === 0) {
+            return {
+                admitted: [],
+                report: {
+                    requested: 0,
+                    admitted: 0,
+                    truncated: 0,
+                    capPerDay,
+                    resetsAt: nextUtcMidnight(now),
+                },
+            };
+        }
+        const granted = this.store.reserveDispatches(filterId, utcDateKey(now), requested, capPerDay);
+        return {
+            admitted: items.slice(0, granted),
+            report: {
+                requested,
+                admitted: granted,
+                // `requested === admitted + truncated` by construction, not by
+                // convention: the third number is derived from the other two and can
+                // never disagree with them.
+                truncated: requested - granted,
+                capPerDay,
+                resetsAt: nextUtcMidnight(now),
+            },
+        };
+    }
+    /**
+     * What `admit` WOULD grant right now, without granting it.
+     *
+     * Same validation, same arithmetic, same five-field `CapReport`, same order
+     * preservation, same single clock read - deliberately, so a reader comparing
+     * this with `admit` is comparing whether it WRITES and nothing else. The only
+     * difference a caller can observe is that calling this twice leaves the day's
+     * remaining budget exactly where one call left it, and where zero calls left
+     * it.
+     *
+     * It reads `readDispatched` rather than `reserveDispatches`, which is the
+     * whole implementation: `reserveDispatches` is the clamp that makes admission
+     * race-safe, and a preview has nothing to make safe because it decides
+     * nothing.
+     *
+     * A CAVEAT the caller owns, not this method: a preview is a statement about
+     * the instant it was taken. Between a preview and the admit that follows it,
+     * a tick can spend the budget, so the two reports can differ. Story 5.3's
+     * confirmation gate handles exactly that by recomputing on the confirming
+     * round trip and refusing on a mismatch (`stale-plan`) rather than by
+     * pretending the preview is a reservation.
+     */
+    preview(filterId, items, capPerDay) {
+        assertValidFilterId(filterId);
+        assertValidCap(capPerDay);
+        // ONE clock read, for the same reason `admit` and `admitDryRun` take one:
+        // reading it twice would let the day key and `resetsAt` straddle a midnight.
+        const now = this.now();
+        const requested = items.length;
+        // Read, never reserve. `readDispatched` on an absent row returns 0, so an
+        // empty ask needs no short-circuit here the way `admit` does - there is no
+        // row for this call to create.
+        const used = this.store.readDispatched(filterId, utcDateKey(now));
+        const granted = Math.max(0, Math.min(requested, capPerDay - used));
+        return {
+            admitted: items.slice(0, granted),
+            report: {
+                requested,
+                admitted: granted,
+                // Derived from the other two, never asserted alongside them.
+                truncated: requested - granted,
+                capPerDay,
+                resetsAt: nextUtcMidnight(now),
+            },
+        };
+    }
+    /**
+     * Give `count` of today's reservations back, for dispatches the caller
+     * decided not to make.
+     *
+     * Explicit and caller-stated on purpose. Story 3.3's hard-failure abort stops
+     * a tick after item 3 of 10 admitted; without this the other 7 would stay
+     * burned for the rest of the UTC day and the operator would see "cap reached"
+     * with three artifacts to show for it. `admit` still never refunds on its own.
+     */
+    release(filterId, count) {
+        assertValidFilterId(filterId);
+        this.store.releaseDispatches(filterId, utcDateKey(this.now()), count);
+    }
+    /**
+     * Today's spend, read-only.
+     *
+     * The `day` comes back with the number because a caller rendering "7 of 10
+     * used today" needs to know WHICH day it counted - a UI that read the number
+     * at 23:59:59 and rendered it at 00:00:01 would otherwise show yesterday's
+     * spend against today's cap with nothing to reveal the mismatch.
+     *
+     * This is what keeps `CapReport` at five fields: the report carries this
+     * batch's numbers, and the day's prior spend is a separate, indexed lookup.
+     */
+    spent(filterId) {
+        assertValidFilterId(filterId);
+        const day = utcDateKey(this.now());
+        return { used: this.store.readDispatched(filterId, day), day };
+    }
+    /**
+     * The dry-run counterpart of `admit`, spending the in-memory budget instead of
+     * the on-disk one.
+     *
+     * Same validation, same arithmetic, same `CapReport` shape, same order
+     * preservation - deliberately, so a reader comparing the two verbs is comparing
+     * WHERE the number is kept and nothing else. The only difference visible to a
+     * caller is that this one writes no row: a filter that has only ever been dry
+     * run has no `ingestion_dispatch_budget` row at all.
+     *
+     * The limiter admits GROUPS, not items, on both verbs, because `cap.runsPerDay`
+     * counts sessions (decision D-5.2-E). The generic never looks inside a `T`.
+     */
+    admitDryRun(filterId, items, capPerDay) {
+        assertValidFilterId(filterId);
+        assertValidCap(capPerDay);
+        // One clock read, for the same reason `admit` takes one: reading it twice
+        // would let the day key and `resetsAt` straddle a midnight.
+        const now = this.now();
+        const day = utcDateKey(now);
+        const requested = items.length;
+        const used = this.readDryRunBudget(filterId, day);
+        const granted = Math.max(0, Math.min(requested, capPerDay - used));
+        // A zero-item ask writes nothing, mirroring `admit`'s short-circuit: an absent
+        // entry and a `used: 0` entry must stay indistinguishable to every reader.
+        if (granted > 0) {
+            this.dryRunBudget.set(filterId, { day, used: used + granted });
+        }
+        return {
+            admitted: items.slice(0, granted),
+            report: {
+                requested,
+                admitted: granted,
+                // Derived from the other two, never asserted alongside them.
+                truncated: requested - granted,
+                capPerDay,
+                resetsAt: nextUtcMidnight(now),
+            },
+        };
+    }
+    /** Today's dry-run spend, read-only. Writes nothing, not even a day rollover. */
+    spentDryRun(filterId) {
+        assertValidFilterId(filterId);
+        const day = utcDateKey(this.now());
+        return { used: this.readDryRunBudget(filterId, day), day };
+    }
+    /**
+     * Today's dry-run count for `filterId`, treating a stale day as zero.
+     *
+     * The rollover happens on READ rather than on a timer: nothing wakes up at UTC
+     * midnight to sweep this map, so a filter last dry-run yesterday reads as `0`
+     * today and only overwrites its entry when something is actually granted.
+     */
+    readDryRunBudget(filterId, day) {
+        const entry = this.dryRunBudget.get(filterId);
+        return entry && entry.day === day ? entry.used : 0;
+    }
+}
+//# sourceMappingURL=cap-limiter.js.map

package/dist/server/server/ingestion/cohort-recorder-memory.d.ts

@@ -0,0 +1,25 @@
+import type { CohortCreateInput, CohortGroupPatch, CohortGroupStatus, CohortRecorder } from "./apply.js";
+export interface RecordedCohortGroup {
+    key: string;
+    itemKeys: string[];
+    status: CohortGroupStatus;
+    agentId: string | null;
+    error: string | null;
+}
+export interface RecordedCohort extends Omit<CohortCreateInput, "groups"> {
+    id: string;
+    groups: RecordedCohortGroup[];
+}
+export declare class InMemoryCohortRecorder implements CohortRecorder {
+    private readonly cohorts;
+    private readonly newId;
+    constructor(options?: {
+        idFactory?: () => string;
+    });
+    create(input: CohortCreateInput): Promise<string>;
+    markGroup(cohortId: string, groupKey: string, patch: CohortGroupPatch): Promise<void>;
+    /** Read one cohort back. Returns a deep copy: callers must not mutate the record. */
+    get(cohortId: string): RecordedCohort | null;
+    list(): RecordedCohort[];
+}
+//# sourceMappingURL=cohort-recorder-memory.d.ts.map

package/dist/server/server/ingestion/cohort-recorder-memory.js

@@ -0,0 +1,78 @@
+// An in-memory `CohortRecorder`: the apply tests' double, and the fallback for a
+// daemon with no sqlite.
+//
+// Story 5.4 landed `cohort-store.ts` and `cohort-recorder-store.ts`, so this is
+// no longer the production path: `bootstrap.ts` builds a `CohortStoreRecorder`
+// over the ledger's own handle whenever a ledger exists, and a cohort now
+// survives a restart. That swap was one constructor at one wiring site, which is
+// the entire reason `applySelection` depends on the narrow port (D6) rather than
+// on a store.
+//
+// It is kept, rather than deleted, for two live reasons:
+//
+//   - `apply.test.ts` and `chat-ingestion-session.test.ts` record through it, and
+//     a fake that satisfies the real port beats a `vi.mock` of the store.
+//   - A daemon whose sqlite could not be opened still has to record what an apply
+//     dispatched somewhere. This is honest about what it cannot do - a cohort in
+//     this map is gone after a restart - and that is strictly better than an apply
+//     that mints sessions with nowhere to record having made them.
+//
+// What it DOES record honestly: which groups a cohort dispatched, and which of
+// them reached `succeeded`, `failed` or `skipped`. `dispatched` is stored as its
+// own number and is never derived from, or conflated with, the count of
+// succeeded groups.
+import { randomUUID } from "node:crypto";
+export class InMemoryCohortRecorder {
+    constructor(options) {
+        this.cohorts = new Map();
+        this.newId = options?.idFactory ?? (() => randomUUID());
+    }
+    create(input) {
+        const id = this.newId();
+        this.cohorts.set(id, {
+            ...input,
+            id,
+            // Every group starts `pending`. It is a real state - dispatched to the
+            // cohort, not yet handed to a provider - and collapsing it into `running`
+            // would make a cohort that crashed before its first session look like one
+            // whose first session was already under way.
+            groups: input.groups.map((group) => ({
+                key: group.key,
+                itemKeys: [...group.itemKeys],
+                status: "pending",
+                agentId: null,
+                error: null,
+            })),
+        });
+        return Promise.resolve(id);
+    }
+    markGroup(cohortId, groupKey, patch) {
+        const cohort = this.cohorts.get(cohortId);
+        const group = cohort?.groups.find((candidate) => candidate.key === groupKey);
+        if (!group) {
+            // A patch for a group nobody recorded is a wiring bug, not a data
+            // condition. Silently ignoring it would let a cohort report `pending`
+            // forever with no way to tell that from a run that never started.
+            return Promise.reject(new Error(`No group ${groupKey} on cohort ${cohortId}`));
+        }
+        if (patch.status !== undefined) {
+            group.status = patch.status;
+        }
+        if (patch.agentId !== undefined) {
+            group.agentId = patch.agentId;
+        }
+        if (patch.error !== undefined) {
+            group.error = patch.error;
+        }
+        return Promise.resolve();
+    }
+    /** Read one cohort back. Returns a deep copy: callers must not mutate the record. */
+    get(cohortId) {
+        const cohort = this.cohorts.get(cohortId);
+        return cohort ? structuredClone(cohort) : null;
+    }
+    list() {
+        return [...this.cohorts.values()].map((cohort) => structuredClone(cohort));
+    }
+}
+//# sourceMappingURL=cohort-recorder-memory.js.map

package/dist/server/server/ingestion/cohort-recorder-store.d.ts

@@ -0,0 +1,12 @@
+import type { CohortCreateInput, CohortGroupPatch, CohortRecorder } from "./apply.js";
+import type { CohortStore } from "./cohort-store.js";
+export declare class CohortStoreRecorder implements CohortRecorder {
+    private readonly store;
+    private readonly newId;
+    constructor(store: CohortStore, options?: {
+        idFactory?: () => string;
+    });
+    create(input: CohortCreateInput): Promise<string>;
+    markGroup(cohortId: string, groupKey: string, patch: CohortGroupPatch): Promise<void>;
+}
+//# sourceMappingURL=cohort-recorder-store.d.ts.map

package/dist/server/server/ingestion/cohort-recorder-store.js

@@ -0,0 +1,99 @@
+// The sqlite `CohortStore` seen through Story 5.3's narrow `CohortRecorder` port.
+//
+// 5.3 depends on a two-method port (D6) precisely so this swap is possible
+// without touching `applySelection`. The two shapes do not line up field for
+// field, and this file is where that is absorbed rather than by bending either
+// side:
+//
+//   - the port's `create` MINTS the cohort id and returns it; the store is TOLD
+//     an id, because a restart has to be able to name a cohort it did not create.
+//   - the port's status vocabulary has `skipped`; the store's has `cancelled`.
+//   - the port's `create` carries `dispatched` up front; the store refuses to
+//     take that number on trust (see below).
+//
+// Everything this adapter does is a translation. It contains no dispatch logic,
+// no transition rules of its own, and no second cohort store.
+import { randomUUID } from "node:crypto";
+/**
+ * `applySelection`'s five statuses onto the cohort's five.
+ *
+ * Four are identical. `skipped` becomes `cancelled` because the two describe the
+ * same fact from two sides: `applySelection` says "every item in this group was
+ * already claimed, so I did not dispatch it", and the cohort vocabulary's word
+ * for a group that did not start and now never will is `cancelled`. Mapping it
+ * to `failed` instead would put a red count on a progress line for a group where
+ * nothing went wrong, and inventing a sixth cohort status for it would break the
+ * containment AC 11 exists to protect.
+ *
+ * Named exhaustively so a status added on either side is a compile error here.
+ */
+const STATUS_MAP = {
+    pending: "pending",
+    running: "running",
+    succeeded: "succeeded",
+    failed: "failed",
+    skipped: "cancelled",
+};
+export class CohortStoreRecorder {
+    constructor(store, options) {
+        this.store = store;
+        this.newId = options?.idFactory ?? (() => randomUUID());
+    }
+    create(input) {
+        const cohortId = this.newId();
+        this.store.create({
+            cohortId,
+            filterId: input.filterId,
+            mode: input.mode,
+            requested: input.report.requested,
+            // The number of groups this cohort HAS, which is the number of rows about
+            // to be written. `input.dispatched` is the caller's forecast of how many it
+            // is about to hand to a provider; the store's `dispatched` column counts
+            // the ones that actually left, and it starts at zero for exactly that
+            // reason. Seeding it from the forecast would record intent as fact.
+            groupsTotal: input.groups.length,
+            truncated: input.report.truncated,
+            capPerDay: input.report.capPerDay,
+            groups: input.groups.map((group) => ({ key: group.key, itemKeys: group.itemKeys })),
+        });
+        // `input.createdAt` is deliberately not forwarded: the store stamps its own
+        // injected clock, so a caller cannot backdate a cohort out of the pruner's
+        // reach or forward past the collapse rule.
+        return Promise.resolve(cohortId);
+    }
+    markGroup(cohortId, groupKey, patch) {
+        if (patch.status === undefined) {
+            // Unreachable from `applySelection`, which names a status on every call.
+            // A throw rather than a silent skip so a future caller finds out on its
+            // first run instead of losing an `agentId` into a no-op: the store has no
+            // partial-update path, and inventing one here would be a second place
+            // where a group row can move.
+            return Promise.reject(new Error(`A cohort group patch with no status has nowhere to go (cohort ${cohortId}, group ${groupKey}).`));
+        }
+        try {
+            this.store.markGroup({
+                cohortId,
+                groupKey,
+                status: STATUS_MAP[patch.status],
+                // `error` is NOT persisted. The cohort tables have no text column for it
+                // by design - a provider's error text can echo request context - so the
+                // failure count is the durable record and the message stays in the log.
+                runId: patch.runId ?? null,
+                agentId: patch.agentId ?? null,
+            });
+            return Promise.resolve();
+        }
+        catch (error) {
+            // Note for whoever reads a stuck cohort: a group that throws BEFORE
+            // reaching `running` (a ledger claim that itself failed) is marked
+            // `failed` by `applySelection`'s catch, which the store refuses because
+            // `pending -> failed` is not a legal transition. The row stays `pending`,
+            // so the next boot's `resumePending()` retries it - which is right, since
+            // nothing was ever dispatched for it. The cost is that such a cohort stays
+            // non-terminal until that boot. Widening the transition table to hide this
+            // would let a never-dispatched group be counted as a failure.
+            return Promise.reject(error);
+        }
+    }
+}
+//# sourceMappingURL=cohort-recorder-store.js.map

package/dist/server/server/ingestion/cohort-resume.d.ts

@@ -0,0 +1,22 @@
+import type { Logger } from "pino";
+import type { StoredFilter } from "@hyperdrive.bot/fleet-protocol/ingestion/filter-types";
+import { type ApplyLedger, type CohortRecorder } from "./apply.js";
+import type { CapLimiterAdmit } from "./cap-limiter.js";
+import type { CohortStore } from "./cohort-store.js";
+import type { DryRunDispatcher } from "./dry-run.js";
+import type { MatchedItem } from "./types.js";
+export interface CohortResumeDeps {
+    store: CohortStore;
+    /** Resolves a filter id, or null when the filter has since been deleted. */
+    filterResolver: (filterId: string) => Promise<StoredFilter | null>;
+    /** The shared resolution (`resolve-items.ts`), not a second one. */
+    resolveItems: (filter: StoredFilter, itemKeys: readonly string[]) => Promise<MatchedItem[]>;
+    ledger: ApplyLedger;
+    schedule: DryRunDispatcher;
+    cohorts: CohortRecorder;
+    limiter: CapLimiterAdmit;
+    runIdFactory: () => string;
+    logger: Logger;
+}
+export declare function resumeCohorts(deps: CohortResumeDeps): Promise<void>;
+//# sourceMappingURL=cohort-resume.d.ts.map