@hyperdrive.bot/fleet-server 0.3.99

package/dist/server/server/ingestion/cohort-store.js

@@ -0,0 +1,462 @@
+// THE COHORT STORE - durable memory of one batch apply, so a restart finishes
+// what a crash interrupted instead of orphaning it.
+//
+// Story 5.3 dispatches a cohort through the narrow `CohortRecorder` port and, in
+// the interim, records it in a Map. That map is gone after a restart while the
+// sessions it minted, and the merge requests and drafts they produced, are not:
+// the outward artifacts outlive the record of having made them. This module
+// closes that gap by writing the cohort to the SAME `paseo.sqlite` the ledger
+// uses, through the SAME handle.
+//
+// Three rules this file exists to keep, each of which has a test:
+//
+//   1. `dispatched` means dispatched. It moves on `pending -> running` and on
+//      nothing else, so a cohort that handed six sessions to a provider and
+//      finished none can never read as six done.
+//   2. Counts on the wire are DERIVED from the group rows (`GROUP BY status`),
+//      never read off the cohort's denormalized columns. The columns are
+//      maintained transactionally beside the rows and a test asserts the two
+//      agree after fifty randomized transitions, which turns the redundancy into
+//      a live check on the transaction boundary.
+//   3. Cancel touches `pending` and nothing else. A `running` group is a live
+//      session that may be mid-write to Gmail or GitLab, and the half-drafted
+//      email is the exact failure this whole path exists to avoid.
+import { COHORT_GROUP_STATUSES, } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+/**
+ * How long a FINISHED cohort is kept. This story's decision; the epic left it
+ * unspecified.
+ *
+ * `30`, matching `LEDGER_PRUNE_DAYS`, on purpose: a cohort and the ledger rows
+ * it produced are two halves of one account of an apply, and letting them expire
+ * on different clocks would leave a month where the items say "handled" and
+ * nothing on disk says by what.
+ */
+export const COHORT_RETENTION_DAYS = 30;
+/**
+ * Which statuses keep a cohort IN FLIGHT, named one by one.
+ *
+ * This is "enumerate what you silence, not what you want" made structural. A
+ * query that asked for `status IN ('pending','running')` would make a sixth
+ * status vanish from the progress line silently - the cohort would collapse
+ * early and read as finished while groups in the unnamed status were still
+ * moving. Here every member is named, so:
+ *
+ *   - adding a sixth `CohortGroupStatus` without adding it below is a TypeScript
+ *     compile error, not a runtime surprise, and
+ *   - a status added as `counted` makes a cohort stay non-terminal (noisy, and
+ *     recoverable) rather than disappear from the count (invisible, and not).
+ *
+ * `counted` = still in flight, holds the cohort open.
+ * `excluded` = terminal, deliberately silenced from the in-flight tally. It is
+ * still returned in `counts`; what it is excluded from is the "is there anything
+ * left to wait for" question.
+ *
+ * IF YOU ADD A STATUS: add it here too, or this file will not compile.
+ */
+export const COHORT_STATUS_DISPOSITION = {
+    pending: "counted",
+    running: "counted",
+    succeeded: "excluded",
+    failed: "excluded",
+    cancelled: "excluded",
+};
+/** The in-flight statuses, derived from the disposition so the two cannot drift. */
+const IN_FLIGHT_STATUSES = COHORT_GROUP_STATUSES.filter((status) => COHORT_STATUS_DISPOSITION[status] === "counted");
+/** `'pending','running'`, built from the disposition rather than typed out. */
+const IN_FLIGHT_SQL_LIST = IN_FLIGHT_STATUSES.map((status) => `'${status}'`).join(", ");
+/**
+ * The only transitions a group may make.
+ *
+ * `running -> cancelled` is absent DELIBERATELY. Cancel means "do not start what
+ * has not started"; a running group reaches `succeeded` or `failed` on its own.
+ * The same rule is enforced a second time by `cancelRemaining`'s `WHERE` clause,
+ * so neither path can quietly become the exception to the other.
+ *
+ * Every terminal status maps to an empty list, which is what makes
+ * `succeeded -> failed` and `cancelled -> running` refusals rather than
+ * last-writer-wins.
+ */
+const LEGAL_TRANSITIONS = {
+    pending: ["running", "cancelled"],
+    running: ["succeeded", "failed"],
+    succeeded: [],
+    failed: [],
+    cancelled: [],
+};
+/**
+ * Which denormalized cohort counter a legal transition increments.
+ *
+ * `dispatched` on `pending -> running` and nowhere else: that is the one column
+ * whose name is a claim about what the daemon did rather than what happened, and
+ * it must never be reachable from a completion path.
+ */
+const COUNTER_FOR_TRANSITION = {
+    pending: { running: "dispatched", cancelled: "cancelled" },
+    running: { succeeded: "succeeded", failed: "failed" },
+};
+/**
+ * A group was asked to make a transition the state machine does not allow.
+ *
+ * Thrown rather than swallowed, and carrying `from`/`to` structurally so a
+ * caller never has to regex the message. Follows `ScheduleTargetGoneError` in
+ * `schedule/service.ts`.
+ */
+export class CohortTransitionError extends Error {
+    constructor(params) {
+        super(`Group ${params.groupKey} of cohort ${params.cohortId} cannot go ${params.from} -> ${params.to}.`);
+        this.name = "CohortTransitionError";
+        this.cohortId = params.cohortId;
+        this.groupKey = params.groupKey;
+        this.from = params.from;
+        this.to = params.to;
+    }
+}
+/**
+ * A patch arrived for a group nobody recorded.
+ *
+ * A wiring bug, not a data condition, and named separately from
+ * `CohortTransitionError` so the two are never confused: one means the caller
+ * asked for something illegal, the other means the caller is talking about a row
+ * that does not exist. Silently ignoring it would leave a cohort reporting
+ * `pending` forever with no way to tell that from a run that never started.
+ */
+export class CohortGroupMissingError extends Error {
+    constructor(cohortId, groupKey) {
+        super(`No group ${groupKey} on cohort ${cohortId}.`);
+        this.name = "CohortGroupMissingError";
+        this.cohortId = cohortId;
+        this.groupKey = groupKey;
+    }
+}
+/**
+ * A cohort RPC refusal, carrying a closed-set code.
+ *
+ * Its own class rather than an `IngestionError` code, for the reason `ApplyError`
+ * states one file over: `IngestionError`'s set is consumed by `health.ts` to
+ * decide whether a SOURCE is `revoked` or `degraded`, and neither is true of "no
+ * such cohort" or "this daemon has no ledger". Adding these codes there would
+ * force a health verdict about an aggregator that is working perfectly.
+ *
+ * `code` is what reaches the client; the message may quote a local id, so it does
+ * not.
+ */
+export class CohortRpcError extends Error {
+    constructor(code, message) {
+        super(message);
+        this.name = "CohortRpcError";
+        this.code = code;
+    }
+}
+/** Every status at `0` before the `GROUP BY` rows are folded in. */
+function zeroCounts() {
+    // Built by iterating the union rather than by typing five zeroes, so a sixth
+    // status is seeded automatically and reads `0` rather than `undefined`.
+    const counts = {};
+    for (const status of COHORT_GROUP_STATUSES) {
+        counts[status] = 0;
+    }
+    return counts;
+}
+function isCohortGroupStatus(value) {
+    return typeof value === "string" && COHORT_GROUP_STATUSES.includes(value);
+}
+export class CohortStore {
+    constructor(options) {
+        this.db = options.db;
+        this.now = options.now;
+        this.insertCohortStmt = this.db.prepare(`INSERT INTO ingestion_cohorts
+         (cohort_id, filter_id, mode, created_at, updated_at, requested, groups_total,
+          dispatched, succeeded, failed, cancelled, truncated, cap_per_day, cancelled_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, ?, ?, NULL)`);
+        this.insertGroupStmt = this.db.prepare(`INSERT INTO ingestion_cohort_groups (cohort_id, group_key, item_keys, status, run_id, agent_id)
+       VALUES (?, ?, ?, 'pending', NULL, NULL)`);
+        this.selectCohortStmt = this.db.prepare("SELECT * FROM ingestion_cohorts WHERE cohort_id = ?");
+        this.selectGroupStmt = this.db.prepare("SELECT status FROM ingestion_cohort_groups WHERE cohort_id = ? AND group_key = ?");
+        // Rides `idx_cohort_groups_status`. THE source of every count on the wire.
+        this.countsStmt = this.db.prepare("SELECT status, COUNT(*) AS n FROM ingestion_cohort_groups WHERE cohort_id = ? GROUP BY status");
+        // The `AND status = ?` is the optimistic lock: if another writer moved the
+        // row between the read and this update, zero rows change and the transaction
+        // rolls back rather than overwriting a transition it never saw.
+        this.updateGroupStmt = this.db.prepare(`UPDATE ingestion_cohort_groups
+         SET status = ?,
+             run_id = COALESCE(?, run_id),
+             agent_id = COALESCE(?, agent_id)
+       WHERE cohort_id = ? AND group_key = ? AND status = ?`);
+        this.touchCohortStmt = this.db.prepare("UPDATE ingestion_cohorts SET updated_at = ? WHERE cohort_id = ?");
+        this.countRunningStmt = this.db.prepare("SELECT COUNT(*) AS n FROM ingestion_cohort_groups WHERE cohort_id = ? AND status = 'running'");
+        this.finishCancelStmt = this.db.prepare("UPDATE ingestion_cohorts SET cancelled = cancelled + ?, cancelled_at = ?, updated_at = ? WHERE cohort_id = ?");
+        // Only 'pending'. A 'running' group is a live session that may be mid-write to Gmail or GitLab; interrupting it is exactly the half-drafted-email failure this whole path exists to avoid.
+        this.cancelPendingStmt = this.db.prepare("UPDATE ingestion_cohort_groups SET status = 'cancelled' WHERE cohort_id = ? AND status = 'pending'");
+        // `cancelled_at IS NULL` is the join's whole point: a cancelled cohort's
+        // leftover pending groups must never be resumed by a restart, or the cancel
+        // would silently expire at the next boot.
+        this.resumePendingStmt = this.db.prepare(`SELECT g.cohort_id AS cohort_id, c.filter_id AS filter_id, g.group_key AS group_key, g.item_keys AS item_keys
+         FROM ingestion_cohort_groups g
+         JOIN ingestion_cohorts c ON c.cohort_id = g.cohort_id
+        WHERE g.status = 'pending' AND c.cancelled_at IS NULL
+        ORDER BY c.created_at ASC, g.group_key ASC`);
+        // Terminal is expressed as "no group in a counted status", derived from the
+        // disposition above, so a sixth in-flight status protects its cohorts from
+        // the pruner for free.
+        this.selectPrunableStmt = this.db.prepare(`SELECT cohort_id FROM ingestion_cohorts c
+        WHERE c.updated_at < ?
+          AND NOT EXISTS (
+            SELECT 1 FROM ingestion_cohort_groups g
+             WHERE g.cohort_id = c.cohort_id AND g.status IN (${IN_FLIGHT_SQL_LIST})
+          )`);
+        this.selectInterruptedStmt = this.db.prepare("SELECT cohort_id, COUNT(*) AS n FROM ingestion_cohort_groups WHERE status = 'running' GROUP BY cohort_id");
+        this.failInterruptedStmt = this.db.prepare("UPDATE ingestion_cohort_groups SET status = 'failed' WHERE status = 'running'");
+        this.addFailedStmt = this.db.prepare("UPDATE ingestion_cohorts SET failed = failed + ?, updated_at = ? WHERE cohort_id = ?");
+        this.deleteGroupsStmt = this.db.prepare("DELETE FROM ingestion_cohort_groups WHERE cohort_id = ?");
+        this.deleteCohortStmt = this.db.prepare("DELETE FROM ingestion_cohorts WHERE cohort_id = ?");
+    }
+    /**
+     * Record a cohort and one `pending` group per group, in ONE transaction.
+     *
+     * Half a cohort on disk - the header row without its groups - would report
+     * `groupsTotal: 6` above zero group rows, which `inspect` would collapse to
+     * `terminal: true` and a client would render as a finished batch that never
+     * ran. The transaction is what makes that state unreachable.
+     */
+    create(params) {
+        const at = this.now();
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            this.insertCohortStmt.run(params.cohortId, params.filterId, params.mode, at, at, params.requested, params.groupsTotal, params.truncated, params.capPerDay);
+            for (const group of params.groups) {
+                // Item KEYS, JSON-encoded. `fingerprint()` hashes, never source payload:
+                // there is nowhere in this row for a mail body or a header to land.
+                this.insertGroupStmt.run(params.cohortId, group.key, JSON.stringify([...group.itemKeys]));
+            }
+            this.db.exec("COMMIT");
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    /**
+     * Move one group, and the parent's counters, together or not at all.
+     *
+     * The group row and the denormalized counter are updated inside ONE
+     * transaction precisely so the two can be compared later: `inspect` derives
+     * its counts from the rows, and a divergence from the columns is a real
+     * transaction bug rather than a test artifact.
+     */
+    markGroup(params) {
+        const { cohortId, groupKey, status } = params;
+        const at = this.now();
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            const row = this.selectGroupStmt.get(cohortId, groupKey);
+            if (!row) {
+                throw new CohortGroupMissingError(cohortId, groupKey);
+            }
+            const from = row.status;
+            if (!isCohortGroupStatus(from)) {
+                // A status this build does not know about. Refusing is the only honest
+                // answer: guessing a transition out of it would silently overwrite a row
+                // written by a version that understood more than this one does.
+                throw new CohortGroupMissingError(cohortId, groupKey);
+            }
+            if (!LEGAL_TRANSITIONS[from].includes(status)) {
+                throw new CohortTransitionError({ cohortId, groupKey, from, to: status });
+            }
+            const changed = this.updateGroupStmt.run(status, params.runId ?? null, params.agentId ?? null, cohortId, groupKey, from).changes;
+            if (changed !== 1) {
+                // Lost the optimistic lock: someone moved the row between the read and
+                // the write. Reported as the transition it actually is rather than
+                // written anyway.
+                throw new CohortTransitionError({ cohortId, groupKey, from, to: status });
+            }
+            const counter = COUNTER_FOR_TRANSITION[from]?.[status];
+            if (counter) {
+                // Column name comes from the closed map above, never from a caller.
+                this.db
+                    .prepare(`UPDATE ingestion_cohorts SET ${counter} = ${counter} + 1, updated_at = ? WHERE cohort_id = ?`)
+                    .run(at, cohortId);
+            }
+            else {
+                this.touchCohortStmt.run(at, cohortId);
+            }
+            this.db.exec("COMMIT");
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    /**
+     * One cohort, or `null` when no such cohort exists.
+     *
+     * `null` here means THIS COHORT DOES NOT EXIST and nothing else. A read that
+     * fails throws, so the caller can tell the two apart - an empty answer for
+     * both would render a vanished sqlite file as a batch that was never created.
+     */
+    inspect(cohortId) {
+        const row = this.selectCohortStmt.get(cohortId);
+        if (!row) {
+            return null;
+        }
+        const counts = zeroCounts();
+        for (const entry of this.countsStmt.all(cohortId)) {
+            if (isCohortGroupStatus(entry.status) && typeof entry.n === "number") {
+                counts[entry.status] = entry.n;
+            }
+        }
+        // Derived from the disposition, not from a hand-written `pending + running`,
+        // so an in-flight status added later keeps its cohort open automatically.
+        const inFlight = IN_FLIGHT_STATUSES.reduce((total, status) => total + counts[status], 0);
+        return {
+            cohortId: row.cohort_id,
+            filterId: row.filter_id,
+            mode: row.mode === "dry-run" ? "dry-run" : "apply",
+            createdAt: row.created_at,
+            updatedAt: row.updated_at,
+            requested: row.requested,
+            groupsTotal: row.groups_total,
+            dispatched: row.dispatched,
+            truncated: row.truncated,
+            capPerDay: row.cap_per_day,
+            cancelledAt: row.cancelled_at,
+            counts,
+            terminal: inFlight === 0,
+        };
+    }
+    /**
+     * Cancel what has not started, and say how many were deliberately left alone.
+     *
+     * `stillRunning` is not decoration. Without it the UI cannot tell the user that
+     * two sessions are still writing, and a cancel that stopped 3 of 5 groups reads
+     * identically to a cancel that stopped everything.
+     */
+    cancelRemaining(cohortId) {
+        const at = this.now();
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            const cancelled = this.cancelPendingStmt.run(cohortId).changes;
+            const runningRow = this.countRunningStmt.get(cohortId);
+            const stillRunning = typeof runningRow?.n === "number" ? runningRow.n : 0;
+            // `cancelled_at` is stamped even when nothing was pending: the user asked
+            // for this cohort to stop taking on new work, and a restart must honour
+            // that whether or not there happened to be a pending group at the time.
+            this.finishCancelStmt.run(cancelled, at, at, cohortId);
+            this.db.exec("COMMIT");
+            return { cancelled, stillRunning: Number(stillRunning) };
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    /**
+     * Every group a restart still owes work for.
+     *
+     * `pending` only, and only on a cohort nobody cancelled. A `running` group is
+     * NOT here: its session died with the process, and Epic 3's
+     * `releaseStaleClaims()` is what turns its claimed-but-unmarked items back
+     * into eligible ones. Resuming it here as well would dispatch the same group
+     * twice for one gesture.
+     */
+    resumePending() {
+        const rows = this.resumePendingStmt.all();
+        return rows.map((row) => ({
+            cohortId: row.cohort_id,
+            filterId: row.filter_id,
+            groupKey: row.group_key,
+            itemKeys: parseItemKeys(row.item_keys),
+        }));
+    }
+    /**
+     * At BOOT ONLY: a group still marked `running` had its session die with the
+     * previous process. Record that as `failed`.
+     *
+     * This is the cohort-level counterpart to Epic 3's `releaseStaleClaims()`,
+     * which returns the ITEMS of such a group to eligible but knows nothing about
+     * cohorts. Without this, a crash leaves the group `running` forever: the cohort
+     * never reaches `terminal`, so the progress row polls until the tab is closed
+     * and the pruner is forbidden from ever collecting it.
+     *
+     * The safety of "every running group is dead" rests entirely on WHEN this
+     * runs. At `ScheduleService.start()` nothing in this process has dispatched
+     * anything yet, so a `running` row can only have been written by a process
+     * that is gone. Calling this at any other moment would mark a live session
+     * failed while it was mid-write, which is the exact thing `cancelRemaining`
+     * refuses to do. It is called from `resumeCohorts` and from nowhere else.
+     *
+     * `failed`, not `cancelled`: the session started, so a group that was mid-write
+     * may well have produced a partial artifact. Recording it as cancelled would
+     * claim nothing happened, and something may have.
+     */
+    failInterrupted() {
+        const rows = this.selectInterruptedStmt.all();
+        if (rows.length === 0) {
+            return 0;
+        }
+        const at = this.now();
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            this.failInterruptedStmt.run();
+            for (const row of rows) {
+                this.addFailedStmt.run(row.n, at, row.cohort_id);
+            }
+            this.db.exec("COMMIT");
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+        return rows.reduce((total, row) => total + row.n, 0);
+    }
+    /**
+     * Drop finished cohorts older than `COHORT_RETENTION_DAYS`.
+     *
+     * NEVER a non-terminal cohort, regardless of age: a cohort with a pending group
+     * is work this daemon still owes, and age is not evidence it is finished. A
+     * long-stalled cohort is a bug to see, not a row to delete.
+     *
+     * Groups are deleted explicitly. There is no foreign key between the two
+     * tables, so there is no cascade to rely on, and an orphaned group row would
+     * keep answering `resumePending` forever with no cohort to attach to.
+     */
+    prune(nowMs = this.now()) {
+        const cutoff = nowMs - COHORT_RETENTION_DAYS * 86400000;
+        const rows = this.selectPrunableStmt.all(cutoff);
+        if (rows.length === 0) {
+            return 0;
+        }
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            for (const row of rows) {
+                this.deleteGroupsStmt.run(row.cohort_id);
+                this.deleteCohortStmt.run(row.cohort_id);
+            }
+            this.db.exec("COMMIT");
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+        return rows.length;
+    }
+}
+/**
+ * `item_keys` back to an array of strings.
+ *
+ * A row whose JSON is unreadable yields `[]` rather than throwing: one corrupt
+ * group must not take down the resume pass for every other cohort on the daemon.
+ * An empty list is honest here - it dispatches nothing - where a throw would
+ * strand every group behind it.
+ */
+function parseItemKeys(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        return Array.isArray(parsed)
+            ? parsed.filter((key) => typeof key === "string")
+            : [];
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=cohort-store.js.map

package/dist/server/server/ingestion/composio/gateway.d.ts

@@ -0,0 +1,8 @@
+import type { ComposioConfig, SourceGateway } from "../types.js";
+/**
+ * The gateway's config block. Re-exported from the shared contract so the
+ * adapter's whole configuration surface is readable from this file alone.
+ */
+export type { ComposioConfig };
+export declare function createComposioGateway(config: ComposioConfig): SourceGateway;
+//# sourceMappingURL=gateway.d.ts.map