@hyperdrive.bot/fleet-server 0.3.99

package/dist/server/server/ingestion/backfill.d.ts

@@ -0,0 +1,123 @@
+import { BACKFILL_DEFAULT_WINDOW_DAYS, BACKFILL_WINDOW_OPTIONS_DAYS, type BackfillItem, type BackfillResult } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+import type { StoredFilter } from "@hyperdrive.bot/fleet-protocol/ingestion/filter-types";
+import type { SourceGateway, SourceItem } from "./types.js";
+/**
+ * The backfill preview: read a window of a source's history, run the filter's
+ * match over it, and report every survivor.
+ *
+ * This module WRITES NOTHING. It resolves no MCP target, calls no
+ * ScheduleService, and holds no ledger handle that could write one (see
+ * `BackfillLedgerReader`). That is the whole guarantee of the feature - the act
+ * of looking at a backlog must not act on it - and it is asserted byte-for-byte
+ * against the ledger in `backfill.no-side-effects.test.ts`.
+ */
+/**
+ * The window constants live in the protocol package, not here: the picker in
+ * the app and the wire schema need the same list, and a second copy would let
+ * the UI offer a window the daemon rejects. Re-exported so every backfill
+ * constant is reachable from this module.
+ */
+export { BACKFILL_DEFAULT_WINDOW_DAYS, BACKFILL_WINDOW_OPTIONS_DAYS };
+/**
+ * The hard ceiling on one preview.
+ *
+ * Reached rather than exceeded: hitting it sets `truncated` and surfaces the
+ * scanned count to the user. A cap that is not rendered is a silent cap, and a
+ * silent cap turns "nothing else matched" into a claim nobody checked.
+ */
+export declare const BACKFILL_MAX_ITEMS = 500;
+/** Items requested per `listItems` call. Five full pages reach the cap. */
+export declare const BACKFILL_PAGE_SIZE = 100;
+export type { BackfillItem, BackfillResult };
+/**
+ * The ONE ledger method a backfill may reach.
+ *
+ * A one-method port rather than the ledger itself, because a read path holding
+ * a write handle is a write path waiting for a bug. `runBackfill` cannot call
+ * `mark`, `claim`, `touch` or `recordFailure` here for the same structural
+ * reason it cannot call `ScheduleService`: the method is not on the object it
+ * was given. The adaptation from `IngestionLedger.get` (synchronous, keyed by
+ * `(filterId, itemKey)`, field named `handledAt`) happens at the wiring site in
+ * `chat-ingestion-session.ts`, so this signature stays the narrow one.
+ */
+export interface BackfillLedgerReader {
+    get(itemKey: string): Promise<{
+        contentHash: string;
+        markedAt: number | null;
+    } | null>;
+}
+export interface RunBackfillInput {
+    filter: StoredFilter;
+    gateway: SourceGateway;
+    /** The aggregator-side isolation key for this daemon's user. */
+    externalUserId: string;
+    /** The connected account, from `StoredSource.externalAccountId`. */
+    accountId: string;
+    ledger: BackfillLedgerReader;
+    /** Epoch ms, inclusive. The oldest item worth returning. */
+    sinceMs: number;
+    /** Ceiling for this run. Clamped to BACKFILL_MAX_ITEMS. */
+    limit?: number;
+    /**
+     * Injected, matching the store convention, so a window boundary is testable
+     * without waiting for one. `match.ts` stays clock-free; the clock enters here
+     * and nowhere deeper.
+     */
+    now: () => number;
+}
+/** Convert a chosen window to the inclusive epoch-ms floor the gateway is asked for. */
+export declare function windowStartFor(now: number, windowDays: number): number;
+/**
+ * One matched source item, with its payload still attached.
+ *
+ * The payload is the difference between this and `BackfillItem`, and it is why
+ * this shape never crosses the wire: an aggregator event body is arbitrary
+ * upstream JSON that can hold an OAuth token. It stays inside the daemon, where
+ * grouping reads the group-key path against it and the prompt composer renders
+ * it into a brief.
+ */
+export interface MatchedSourceItem {
+    item: SourceItem;
+    reasons: BackfillItem["reasons"];
+}
+/** What one window read found, before anything is projected for a client. */
+export interface MatchedSourceScan {
+    matched: MatchedSourceItem[];
+    windowStart: number;
+    windowEnd: number;
+    scanned: number;
+    truncated: boolean;
+    cursor: string | null;
+}
+/**
+ * Read the window and run the filter's match over it, keeping payloads.
+ *
+ * Extracted from `runBackfill` (Story 5.3) so the APPLY path can resolve item
+ * keys to payload-bearing items through the SAME read and the SAME matcher the
+ * preview used. The alternative was a second scan-and-match in the apply
+ * handler, which is how the preview and the thing it previews start disagreeing
+ * about what matched.
+ *
+ * Throws on a gateway failure and never degrades to an empty result. An empty
+ * `matched` array means the window really held no match; a caller that cannot
+ * tell those apart will eventually tell a user "nothing matched" about a read
+ * that never happened.
+ *
+ * `stopWhen` lets a caller with a KNOWN finite target end the scan early. The
+ * preview passes nothing (it wants the whole window); the apply path passes
+ * "every key I was asked to resolve is now in hand", which is what keeps
+ * resolving a 6-item selection from paging a 90-day history to the cap.
+ */
+export declare function scanMatchedSourceItems(input: RunBackfillInput & {
+    stopWhen?: (matched: MatchedSourceItem[]) => boolean;
+}): Promise<MatchedSourceScan>;
+/**
+ * The backfill PREVIEW: scan, then project each survivor for a client.
+ *
+ * The projection is the security boundary. `MatchedSourceItem` carries the raw
+ * payload; `BackfillItem` carries a title, a subtitle and the reasons, which is
+ * the CONCLUSION rather than the evidence, because the evidence can hold a
+ * credential (ARCHITECTURE.md section 6 invariant 3).
+ */
+export declare function runBackfill(input: RunBackfillInput): Promise<BackfillResult>;
+//# sourceMappingURL=backfill.d.ts.map

package/dist/server/server/ingestion/backfill.js

@@ -0,0 +1,160 @@
+import { BACKFILL_DEFAULT_WINDOW_DAYS, BACKFILL_WINDOW_OPTIONS_DAYS, } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+import { evaluateMatch } from "./match.js";
+/**
+ * The backfill preview: read a window of a source's history, run the filter's
+ * match over it, and report every survivor.
+ *
+ * This module WRITES NOTHING. It resolves no MCP target, calls no
+ * ScheduleService, and holds no ledger handle that could write one (see
+ * `BackfillLedgerReader`). That is the whole guarantee of the feature - the act
+ * of looking at a backlog must not act on it - and it is asserted byte-for-byte
+ * against the ledger in `backfill.no-side-effects.test.ts`.
+ */
+/**
+ * The window constants live in the protocol package, not here: the picker in
+ * the app and the wire schema need the same list, and a second copy would let
+ * the UI offer a window the daemon rejects. Re-exported so every backfill
+ * constant is reachable from this module.
+ */
+export { BACKFILL_DEFAULT_WINDOW_DAYS, BACKFILL_WINDOW_OPTIONS_DAYS };
+/**
+ * The hard ceiling on one preview.
+ *
+ * Reached rather than exceeded: hitting it sets `truncated` and surfaces the
+ * scanned count to the user. A cap that is not rendered is a silent cap, and a
+ * silent cap turns "nothing else matched" into a claim nobody checked.
+ */
+export const BACKFILL_MAX_ITEMS = 500;
+/** Items requested per `listItems` call. Five full pages reach the cap. */
+export const BACKFILL_PAGE_SIZE = 100;
+/** Convert a chosen window to the inclusive epoch-ms floor the gateway is asked for. */
+export function windowStartFor(now, windowDays) {
+    return now - windowDays * 86400000;
+}
+/**
+ * Read the window and run the filter's match over it, keeping payloads.
+ *
+ * Extracted from `runBackfill` (Story 5.3) so the APPLY path can resolve item
+ * keys to payload-bearing items through the SAME read and the SAME matcher the
+ * preview used. The alternative was a second scan-and-match in the apply
+ * handler, which is how the preview and the thing it previews start disagreeing
+ * about what matched.
+ *
+ * Throws on a gateway failure and never degrades to an empty result. An empty
+ * `matched` array means the window really held no match; a caller that cannot
+ * tell those apart will eventually tell a user "nothing matched" about a read
+ * that never happened.
+ *
+ * `stopWhen` lets a caller with a KNOWN finite target end the scan early. The
+ * preview passes nothing (it wants the whole window); the apply path passes
+ * "every key I was asked to resolve is now in hand", which is what keeps
+ * resolving a 6-item selection from paging a 90-day history to the cap.
+ */
+export async function scanMatchedSourceItems(input) {
+    const windowEnd = input.now();
+    const windowStart = input.sinceMs;
+    const ceiling = Math.min(input.limit ?? BACKFILL_MAX_ITEMS, BACKFILL_MAX_ITEMS);
+    const matched = [];
+    const seenKeys = new Set();
+    let scanned = 0;
+    let cursor = null;
+    /** Set once an item older than the window is seen: history before this point is covered. */
+    let windowCovered = false;
+    while (scanned < ceiling) {
+        const page = await input.gateway.listItems({
+            externalUserId: input.externalUserId,
+            appSlug: input.filter.match.appSlug,
+            accountId: input.accountId,
+            sinceMs: windowStart,
+            cursor,
+            limit: Math.min(BACKFILL_PAGE_SIZE, ceiling - scanned),
+        });
+        for (const item of page.items) {
+            scanned += 1;
+            if (item.timestampMs < windowStart) {
+                // The boundary, not a reason to abandon the rest of the page: a
+                // gateway is not required to hand items back in any order, and
+                // breaking here would drop in-window items sitting behind this one.
+                windowCovered = true;
+                continue;
+            }
+            // Closed at the start, open at the end: an item stamped after the read
+            // began belongs to the next tick, not to this window.
+            if (item.timestampMs > windowEnd) {
+                continue;
+            }
+            // Paging can overlap. The union, exactly once each, keyed by the
+            // adapter's fingerprint - which is the ledger's key, so a duplicate here
+            // would also be a double ledger read.
+            if (seenKeys.has(item.key)) {
+                continue;
+            }
+            seenKeys.add(item.key);
+            const verdict = evaluateMatch(item, input.filter.match);
+            if (verdict.matched) {
+                matched.push({ item, reasons: verdict.reasons });
+            }
+        }
+        cursor = page.cursor;
+        // Stop the moment the window is covered or the source is exhausted. One
+        // more page after either would be a call whose result cannot change the
+        // answer.
+        if (windowCovered || cursor === null) {
+            break;
+        }
+        // A caller resolving a KNOWN key set has no use for the rest of the
+        // history. Evaluated against the matches found so far, so it can only ever
+        // end the scan early - it can never admit an item the match rejected.
+        if (input.stopWhen?.(matched) === true) {
+            break;
+        }
+    }
+    // Truncated means "stopped at the cap with more still available", NOT
+    // "scanned == cap". A window that holds exactly BACKFILL_MAX_ITEMS was read
+    // in full, and reporting it as shortened would send the user hunting for
+    // items that are all already on screen.
+    const truncated = !windowCovered && cursor !== null;
+    matched.sort((left, right) => right.item.timestampMs - left.item.timestampMs);
+    return { matched, windowStart, windowEnd, scanned, truncated, cursor };
+}
+/**
+ * The backfill PREVIEW: scan, then project each survivor for a client.
+ *
+ * The projection is the security boundary. `MatchedSourceItem` carries the raw
+ * payload; `BackfillItem` carries a title, a subtitle and the reasons, which is
+ * the CONCLUSION rather than the evidence, because the evidence can hold a
+ * credential (ARCHITECTURE.md section 6 invariant 3).
+ */
+export async function runBackfill(input) {
+    const { matched, windowStart, windowEnd, scanned, truncated, cursor } = await scanMatchedSourceItems(input);
+    const items = [];
+    for (const { item, reasons } of matched) {
+        const row = await input.ledger.get(item.key);
+        // Handled means a successful mark FOR THIS CONTENT. A row whose hash
+        // differs is an item that changed since it was handled, and the ledger's
+        // own rule re-fires that item - so the preview must show it as unhandled or
+        // it disagrees with what an armed filter would do.
+        const handled = row !== null && row.contentHash === item.contentHash && row.markedAt !== null;
+        items.push({
+            itemKey: item.key,
+            contentHash: item.contentHash,
+            title: item.title,
+            subtitle: item.subtitle,
+            timestampMs: item.timestampMs,
+            reasons,
+            handled,
+            handledAt: handled ? row.markedAt : null,
+        });
+    }
+    return {
+        items,
+        // Returned verbatim, never recomputed downstream: the screen describes the
+        // window the daemon actually read, not one it derives from its own clock.
+        windowStart,
+        windowEnd,
+        scanned,
+        truncated,
+        cursor: truncated ? cursor : null,
+    };
+}
+//# sourceMappingURL=backfill.js.map

package/dist/server/server/ingestion/cap-limiter.d.ts

@@ -0,0 +1,148 @@
+import type { CapReport } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+import type { CapBudgetStore } from "./ledger.js";
+export interface CapLimiterOptions {
+    /** The `IngestionLedger` instance, never a second connection to the same file. */
+    store: CapBudgetStore;
+    /** Mirrors `ScheduleService.now` and `IngestionLedger.now` so tests can pin the clock. */
+    now?: () => Date;
+}
+/** What one `admit()` call granted, and what it refused. */
+export interface CapAdmission<T> {
+    admitted: T[];
+    report: CapReport;
+}
+/**
+ * The narrow view `schedule/service.ts` imports with `import type`.
+ *
+ * Structural on purpose, mirroring `IngestionLedgerMaintenance`: `service.ts`
+ * carries the limiter through to Epic 2's dispatch loop and Epic 5's RPC path
+ * without ever pulling in the concrete class, so nothing sqlite-adjacent enters
+ * that module's resolution graph.
+ */
+export interface CapLimiterAdmit {
+    admit<T>(filterId: string, items: readonly T[], capPerDay: number): CapAdmission<T>;
+    release(filterId: string, count: number): void;
+    spent(filterId: string): {
+        used: number;
+        day: string;
+    };
+    admitDryRun<T>(filterId: string, items: readonly T[], capPerDay: number): CapAdmission<T>;
+    spentDryRun(filterId: string): {
+        used: number;
+        day: string;
+    };
+    preview<T>(filterId: string, items: readonly T[], capPerDay: number): CapAdmission<T>;
+}
+/**
+ * ONE instance per daemon, holding THREE budget verbs over the same filter id.
+ *
+ *   `admit`      the APPLY budget. Ledger-backed, on disk, survives a restart,
+ *                CONSUMES on every call. Bounds outward artifacts - merge
+ *                requests, Gmail drafts, reviewer pings.
+ *   `admitDryRun` the DRY-RUN tuning budget (Story 5.2). In memory on this
+ *                object, reset by a restart, CONSUMES on every call. Bounds
+ *                token spend and nothing else.
+ *   `preview`    read-only (Story 5.3). Reads the SAME on-disk counter `admit`
+ *                spends and returns the identical `CapReport`, CONSUMING
+ *                NOTHING. It exists because the confirmation screen has to show
+ *                a number before the user has agreed to it, and computing that
+ *                number with `admit` would spend the day's real budget on a
+ *                plan the user is still free to cancel.
+ *
+ * Three verbs, two counters, one object. Adding a fourth means saying which of
+ * the two substrates it spends and why.
+ */
+export declare class CapLimiter {
+    private readonly store;
+    private readonly now;
+    private readonly dryRunBudget;
+    constructor(options: CapLimiterOptions);
+    /**
+     * Reserve budget for `items` and hand back the ones that fit.
+     *
+     * Generic so the caller's item type survives the trip: the limiter never looks
+     * inside a `T`, and persists a COUNT and nothing derived from item content -
+     * which is what makes it structurally impossible for a payload carrying a
+     * credential to reach the ledger through this surface.
+     *
+     * `admitted` is `items.slice(0, granted)`, preserving input order, so the
+     * caller's own prioritisation decides which items make the cut.
+     */
+    admit<T>(filterId: string, items: readonly T[], capPerDay: number): CapAdmission<T>;
+    /**
+     * What `admit` WOULD grant right now, without granting it.
+     *
+     * Same validation, same arithmetic, same five-field `CapReport`, same order
+     * preservation, same single clock read - deliberately, so a reader comparing
+     * this with `admit` is comparing whether it WRITES and nothing else. The only
+     * difference a caller can observe is that calling this twice leaves the day's
+     * remaining budget exactly where one call left it, and where zero calls left
+     * it.
+     *
+     * It reads `readDispatched` rather than `reserveDispatches`, which is the
+     * whole implementation: `reserveDispatches` is the clamp that makes admission
+     * race-safe, and a preview has nothing to make safe because it decides
+     * nothing.
+     *
+     * A CAVEAT the caller owns, not this method: a preview is a statement about
+     * the instant it was taken. Between a preview and the admit that follows it,
+     * a tick can spend the budget, so the two reports can differ. Story 5.3's
+     * confirmation gate handles exactly that by recomputing on the confirming
+     * round trip and refusing on a mismatch (`stale-plan`) rather than by
+     * pretending the preview is a reservation.
+     */
+    preview<T>(filterId: string, items: readonly T[], capPerDay: number): CapAdmission<T>;
+    /**
+     * Give `count` of today's reservations back, for dispatches the caller
+     * decided not to make.
+     *
+     * Explicit and caller-stated on purpose. Story 3.3's hard-failure abort stops
+     * a tick after item 3 of 10 admitted; without this the other 7 would stay
+     * burned for the rest of the UTC day and the operator would see "cap reached"
+     * with three artifacts to show for it. `admit` still never refunds on its own.
+     */
+    release(filterId: string, count: number): void;
+    /**
+     * Today's spend, read-only.
+     *
+     * The `day` comes back with the number because a caller rendering "7 of 10
+     * used today" needs to know WHICH day it counted - a UI that read the number
+     * at 23:59:59 and rendered it at 00:00:01 would otherwise show yesterday's
+     * spend against today's cap with nothing to reveal the mismatch.
+     *
+     * This is what keeps `CapReport` at five fields: the report carries this
+     * batch's numbers, and the day's prior spend is a separate, indexed lookup.
+     */
+    spent(filterId: string): {
+        used: number;
+        day: string;
+    };
+    /**
+     * The dry-run counterpart of `admit`, spending the in-memory budget instead of
+     * the on-disk one.
+     *
+     * Same validation, same arithmetic, same `CapReport` shape, same order
+     * preservation - deliberately, so a reader comparing the two verbs is comparing
+     * WHERE the number is kept and nothing else. The only difference visible to a
+     * caller is that this one writes no row: a filter that has only ever been dry
+     * run has no `ingestion_dispatch_budget` row at all.
+     *
+     * The limiter admits GROUPS, not items, on both verbs, because `cap.runsPerDay`
+     * counts sessions (decision D-5.2-E). The generic never looks inside a `T`.
+     */
+    admitDryRun<T>(filterId: string, items: readonly T[], capPerDay: number): CapAdmission<T>;
+    /** Today's dry-run spend, read-only. Writes nothing, not even a day rollover. */
+    spentDryRun(filterId: string): {
+        used: number;
+        day: string;
+    };
+    /**
+     * Today's dry-run count for `filterId`, treating a stale day as zero.
+     *
+     * The rollover happens on READ rather than on a timer: nothing wakes up at UTC
+     * midnight to sweep this map, so a filter last dry-run yesterday reads as `0`
+     * today and only overwrites its entry when something is actually granted.
+     */
+    private readDryRunBudget;
+}
+//# sourceMappingURL=cap-limiter.d.ts.map