@hyperdrive.bot/fleet-server 0.3.99

package/dist/server/server/ingestion/run-projection.js

@@ -0,0 +1,76 @@
+// THE READ-TIME JOIN, and nothing else.
+//
+// `ScheduleRun` lives in a schedule JSON document; the verification columns live
+// in `paseo.sqlite`. The Inbox needs them side by side. This file is where they
+// meet, and it meets them at READ time on purpose: writing the verification
+// stamp back into the schedule document would rewrite the whole document on
+// every Inbox paint, because `ScheduleService.finishRun` persists a schedule by
+// mapping all of its runs and writing the entire `StoredSchedule` atomically
+// (epic D1/D2).
+//
+// Pure. No I/O, no clock, no logger - which is what lets it be tested without a
+// daemon at all. Precedent: `buildLoopRows` (`app/src/screens/loops-screen.tsx`).
+/**
+ * Join runs to ledger rows on `lastRunId`, newest first.
+ *
+ * ORDERING DEPENDENCY, carried here because it is the reason the join works at
+ * all: Story 3.2's `claim()` writes the run id BEFORE dispatch, so by the time a
+ * `ScheduleRun` exists its ledger row already names it. The arrow only ever
+ * points row -> run, never the other way, which is why the row is the index and
+ * the run is the lookup.
+ *
+ * `lastRunId` (ledger schema v4) and NOT `claimRunId`, which is the defect this
+ * projection shipped with. `claim_run_id` names the run holding the claim right
+ * now, so `mark()` nulls it when the agent finishes and `releaseStaleClaims()`
+ * nulls it on the next boot. Joining on it meant the join matched ONLY items
+ * whose dispatch was still in flight in this process - so a handled item, which
+ * is precisely what an Inbox row represents, never matched, and a restart
+ * emptied the list. `last_run_id` carries the same value with no clearing path.
+ *
+ * A run with NO matching ledger row is EXCLUDED rather than emitted with null
+ * verification fields. Such a run is not an ingestion item - it is a clock
+ * schedule's ordinary run that happens to live in the same document - and
+ * emitting it with `verdict: null` would put a row on the Inbox that has no
+ * item behind it and can never be verified. A ledger row whose `lastRunId` is
+ * null is likewise excluded: it was seen but never dispatched, so there is no
+ * run to show.
+ *
+ * Sorted by `startedAt` descending, tie-broken by `id` ascending. The tiebreak
+ * is not decoration: two runs minted in the same millisecond would otherwise
+ * order by whatever `Array.prototype.sort` happened to do, and the test would
+ * be a coin flip.
+ */
+export function buildIngestionRunViews(runs, rows) {
+    const byRunId = new Map();
+    for (const row of rows) {
+        if (row.lastRunId === null)
+            continue;
+        byRunId.set(row.lastRunId, row);
+    }
+    const views = [];
+    for (const run of runs) {
+        const row = byRunId.get(run.id);
+        if (!row)
+            continue;
+        views.push({
+            ...run,
+            itemKey: row.itemKey,
+            verifiedAt: row.verifiedAt,
+            verdict: row.verdict,
+            verifiedSignature: row.verifiedSignature,
+            reconcileFailCount: row.reconcileFailCount,
+            backoffUntil: row.backoffUntil,
+        });
+    }
+    views.sort((a, b) => {
+        if (a.startedAt !== b.startedAt)
+            return a.startedAt < b.startedAt ? 1 : -1;
+        if (a.id < b.id)
+            return -1;
+        if (a.id > b.id)
+            return 1;
+        return 0;
+    });
+    return views;
+}
+//# sourceMappingURL=run-projection.js.map

package/dist/server/server/ingestion/service.d.ts

@@ -0,0 +1,147 @@
+import type { Logger } from "pino";
+import type { SourceCatalogueEntry as WireSourceCatalogueEntry, SourceSummary } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+import type { StoredFilter } from "@hyperdrive.bot/fleet-protocol/ingestion/filter-types";
+import { type IngestionConfig, type SourceGateway, type SourceKind } from "./types.js";
+export interface IngestionServiceOptions {
+    paseoHome: string;
+    serverId: string;
+    logger: Logger;
+    config: IngestionConfig;
+    /** Injected so the connect-poll bound is assertable under fake timers. */
+    now?: () => Date;
+    /** Injected so AC 18 can prove no timer is created when nothing is configured. */
+    setIntervalFn?: typeof setInterval;
+    /**
+     * Injected so tests can drive a fake gateway without network or credentials.
+     * Defaults to the real `resolveSourceGateway`. This is an INPUT seam, not an
+     * exposure: nothing here ever hands a gateway back out to a caller.
+     */
+    resolveGateway?: (kind: SourceKind) => SourceGateway;
+}
+export interface SourceCataloguePage {
+    items: WireSourceCatalogueEntry[];
+    nextCursor: string | null;
+}
+export interface SourceConnectResult {
+    source: SourceSummary;
+    redirectUrl: string;
+}
+/**
+ * The daemon's ingestion surface.
+ *
+ * Everything credential-bearing lives behind this class and stops here. It owns
+ * the SourceStore, resolves gateways lazily per call, and runs the health
+ * sweep. It deliberately exposes NO gateway, NO McpTarget and NO IngestionConfig
+ * to a caller: the only shapes that leave are SourceSummary and the public
+ * catalogue metadata, neither of which has anywhere to put a secret.
+ */
+export declare class IngestionService {
+    private readonly store;
+    private readonly filterStore;
+    private readonly logger;
+    private readonly config;
+    private readonly externalUserId;
+    private readonly now;
+    private readonly setIntervalFn;
+    private readonly gatewayResolver;
+    private readonly connectPolls;
+    private sweepTimer;
+    constructor(options: IngestionServiceOptions);
+    /**
+     * Start the background health sweep, if and only if some aggregator is
+     * actually configured. With no credentials anywhere the feature is inert: no
+     * timer, no filesystem, no network.
+     */
+    start(): Promise<void>;
+    /** Idempotent. Cancels every in-flight connect poll so the daemon can exit. */
+    stop(): Promise<void>;
+    catalogue(kind: SourceKind, query: string, cursor?: string): Promise<SourceCataloguePage>;
+    /**
+     * Hand the user off to the aggregator's own authorization page.
+     *
+     * Returns as soon as the aggregator produces a redirect URL. It does NOT wait
+     * for the user to finish the popup: a 60 second RPC is a broken RPC. The
+     * confirmation is a detached poll owned by this service, and the client
+     * learns the outcome from its next source/list.
+     */
+    connect(kind: SourceKind, appSlug: string): Promise<SourceConnectResult>;
+    list(): Promise<SourceSummary[]>;
+    /** Check one source and persist the verdict. Null when the source is gone. */
+    check(sourceId: string): Promise<SourceSummary | null>;
+    private hasConfiguredGateway;
+    private resolveGateway;
+    private healthDeps;
+    /**
+     * Sweep every source once.
+     *
+     * Promise.allSettled, never Promise.all: one source whose check rejects must
+     * not abort the checks queued behind it (NFR7). A rejected entry and an
+     * unclassifiable failure are both logged at warn with the source id, then
+     * skipped.
+     */
+    private sweep;
+    /**
+     * Register the poll BEFORE the first await, so a stop() landing between here
+     * and the first gateway call cancels it deterministically rather than racing.
+     */
+    private startConnectPoll;
+    /**
+     * Bounded by construction: the loop condition is elapsed time against a
+     * literal window, so the loop cannot become unbounded without someone
+     * deleting that bound outright. AC 6 audits this file by grep, so do not
+     * write the unbounded-loop keyword pair even inside a comment.
+     */
+    private runConnectPoll;
+    /**
+     * A sleep that a cancel can cut short. ONE resolve path, reached either by the
+     * timer firing or by `state.wake()`, so a cancel during a sleep cannot settle
+     * the promise twice.
+     */
+    private sleep;
+    /**
+     * The ONE place a health verdict is written.
+     *
+     * Re-reads the record first so a source deleted while a poll or a sweep was
+     * in flight is not resurrected by a stale write.
+     */
+    private persistHealth;
+    /**
+     * FILTER AUTHORING (Epic 7).
+     *
+     * Thin pass-throughs to `FilterStore`, which is what keeps the layering the
+     * class header already claims: no gateway and no credential reaches a caller
+     * through these. `FilterStore.put()` re-parses before writing, so an untyped
+     * caller still cannot land a rail-shaped field on disk even though these
+     * methods sit behind a typed RPC.
+     */
+    listFilters(sourceId?: string): Promise<StoredFilter[]>;
+    /**
+     * `createdAt`/`updatedAt` are minted HERE, not accepted from the caller, for
+     * the same reason `id` is: a client that could choose them could forge the
+     * ordering `FilterStore.list()` sorts by.
+     */
+    createFilter(draft: Omit<StoredFilter, "id" | "createdAt" | "updatedAt">): Promise<StoredFilter>;
+    /**
+     * Returns null when the id resolves to nothing, which the RPC reports as
+     * `filter: null` with `error: null` - "no such filter" is an answer, not a
+     * failure.
+     *
+     * `status` is absent from the patch type on purpose: arming is a schedule
+     * verb (`schedule/create` with a source cadence), never a filter field write.
+     */
+    updateFilter(filterId: string, patch: Partial<Omit<StoredFilter, "id" | "createdAt" | "updatedAt" | "status">>): Promise<StoredFilter | null>;
+    /**
+     * `false` means the id resolved to nothing. Deleting twice is idempotent and
+     * is not an error - `FilterStore.delete` already uses `force: true` - so the
+     * existence check is what makes the answer informative rather than a
+     * uniformly cheerful `true`.
+     */
+    deleteFilter(filterId: string): Promise<boolean>;
+    /**
+     * Field by field on purpose. A `...stored` spread would mean a field added to
+     * StoredSource later reaches the wire the day it is added, with no diff on
+     * this file to review.
+     */
+    private toSummary;
+}
+//# sourceMappingURL=service.d.ts.map

package/dist/server/server/ingestion/service.js

@@ -0,0 +1,364 @@
+import { join } from "node:path";
+import { resolveSourceGateway } from "./adapters.js";
+import { FilterStore, filtersDir } from "./filters/store.js";
+import { sourceExternalUserId } from "./external-user-id.js";
+import { checkSource } from "./health.js";
+import { SourceStore } from "./sources/store.js";
+import { SOURCE_CAPABLE_KINDS, } from "./types.js";
+/**
+ * Five minutes between background sweeps, mirroring the literal-constant style
+ * of SCHEDULE_TICK_INTERVAL_MS. A source's health changes when a user revokes a
+ * grant at the aggregator, which is rare and never urgent, so a tighter
+ * interval would only spend the platform rate limit.
+ */
+const HEALTH_SWEEP_INTERVAL_MS = 300000;
+/** How often the post-connect confirmation poll asks the aggregator. */
+const CONNECT_POLL_INTERVAL_MS = 2000;
+/**
+ * How long the confirmation poll waits before giving up.
+ *
+ * A hard ceiling, not a hint. The poll is a detached background task, so an
+ * unbounded version would keep a daemon alive and keep calling an aggregator
+ * forever for a popup the user closed an hour ago.
+ */
+const CONNECT_POLL_TIMEOUT_MS = 60000;
+/** Recorded when the confirmation window closes without the grant landing. */
+const NOT_CONFIRMED = "authorization_not_confirmed";
+/** Recorded when a health check threw something we could not classify. */
+const CHECK_FAILED = "check_failed";
+/**
+ * The daemon's ingestion surface.
+ *
+ * Everything credential-bearing lives behind this class and stops here. It owns
+ * the SourceStore, resolves gateways lazily per call, and runs the health
+ * sweep. It deliberately exposes NO gateway, NO McpTarget and NO IngestionConfig
+ * to a caller: the only shapes that leave are SourceSummary and the public
+ * catalogue metadata, neither of which has anywhere to put a secret.
+ */
+export class IngestionService {
+    constructor(options) {
+        this.connectPolls = new Map();
+        this.sweepTimer = null;
+        this.store = new SourceStore(join(options.paseoHome, "sources"));
+        // `filtersDir()` rather than join(paseoHome, "filters"): the directory name
+        // has exactly one producer, and bootstrap's FilterStore uses the same one.
+        // Two stores over one directory is fine - FilterStore holds no state, reads
+        // per call and writes atomically.
+        this.filterStore = new FilterStore(filtersDir(options.paseoHome));
+        this.logger = options.logger.child({ module: "ingestion-service" });
+        this.config = options.config;
+        // external-user-id.ts is the ONE producer of this string. Call it once here
+        // and pass the result down; never assemble the prefixed id inline. A guard
+        // test enforces that rule by scanning this whole tree for the prefix.
+        this.externalUserId = sourceExternalUserId(options.serverId);
+        this.now = options.now ?? (() => new Date());
+        this.setIntervalFn = options.setIntervalFn ?? setInterval;
+        this.gatewayResolver =
+            options.resolveGateway ?? ((kind) => resolveSourceGateway(kind, { config: this.config }));
+    }
+    /**
+     * Start the background health sweep, if and only if some aggregator is
+     * actually configured. With no credentials anywhere the feature is inert: no
+     * timer, no filesystem, no network.
+     */
+    async start() {
+        if (this.sweepTimer) {
+            return;
+        }
+        if (!this.hasConfiguredGateway()) {
+            this.logger.info("ingestion not configured, health sweep disabled");
+            return;
+        }
+        const timer = this.setIntervalFn(() => {
+            void this.sweep().catch((error) => {
+                this.logger.error({ err: error }, "Failed to run source health sweep");
+            });
+        }, HEALTH_SWEEP_INTERVAL_MS);
+        timer.unref?.();
+        this.sweepTimer = timer;
+    }
+    /** Idempotent. Cancels every in-flight connect poll so the daemon can exit. */
+    async stop() {
+        if (this.sweepTimer) {
+            clearInterval(this.sweepTimer);
+            this.sweepTimer = null;
+        }
+        for (const poll of this.connectPolls.values()) {
+            poll.cancel();
+        }
+        this.connectPolls.clear();
+    }
+    async catalogue(kind, query, cursor) {
+        const page = await this.resolveGateway(kind).listSources(query, cursor);
+        return { items: page.items, nextCursor: page.nextCursor };
+    }
+    /**
+     * Hand the user off to the aggregator's own authorization page.
+     *
+     * Returns as soon as the aggregator produces a redirect URL. It does NOT wait
+     * for the user to finish the popup: a 60 second RPC is a broken RPC. The
+     * confirmation is a detached poll owned by this service, and the client
+     * learns the outcome from its next source/list.
+     */
+    async connect(kind, appSlug) {
+        const gateway = this.resolveGateway(kind);
+        const handoff = await gateway.connect(this.externalUserId, appSlug);
+        const timestamp = this.now().toISOString();
+        const source = await this.store.create({
+            kind,
+            appSlug,
+            displayName: appSlug,
+            imgSrc: "",
+            externalUserId: this.externalUserId,
+            externalAccountId: null,
+            status: "connecting",
+            createdAt: timestamp,
+            updatedAt: timestamp,
+            lastCheckedAt: null,
+            lastError: null,
+        });
+        this.startConnectPoll(source);
+        return { source: this.toSummary(source), redirectUrl: handoff.redirectUrl };
+    }
+    async list() {
+        const sources = await this.store.list();
+        return sources.map((source) => this.toSummary(source));
+    }
+    /** Check one source and persist the verdict. Null when the source is gone. */
+    async check(sourceId) {
+        const source = await this.store.get(sourceId);
+        if (!source) {
+            return null;
+        }
+        const health = await checkSource(source, this.healthDeps());
+        const updated = await this.persistHealth(sourceId, health);
+        return updated ? this.toSummary(updated) : null;
+    }
+    hasConfiguredGateway() {
+        return SOURCE_CAPABLE_KINDS.some((kind) => this.resolveGateway(kind).configured);
+    }
+    resolveGateway(kind) {
+        return this.gatewayResolver(kind);
+    }
+    healthDeps() {
+        return {
+            resolveGateway: (kind) => this.resolveGateway(kind),
+            externalUserId: this.externalUserId,
+            now: this.now,
+        };
+    }
+    /**
+     * Sweep every source once.
+     *
+     * Promise.allSettled, never Promise.all: one source whose check rejects must
+     * not abort the checks queued behind it (NFR7). A rejected entry and an
+     * unclassifiable failure are both logged at warn with the source id, then
+     * skipped.
+     */
+    async sweep() {
+        const sources = await this.store.list();
+        const results = await Promise.allSettled(sources.map((source) => this.check(source.id)));
+        results.forEach((result, index) => {
+            const sourceId = sources[index]?.id;
+            if (result.status === "rejected") {
+                this.logger.warn({ err: result.reason, sourceId }, "Source health check failed");
+                return;
+            }
+            if (result.value?.lastError === CHECK_FAILED) {
+                this.logger.warn({ sourceId }, "Source health check failed");
+            }
+        });
+    }
+    /**
+     * Register the poll BEFORE the first await, so a stop() landing between here
+     * and the first gateway call cancels it deterministically rather than racing.
+     */
+    startConnectPoll(source) {
+        const state = { cancelled: false, wake: null };
+        this.connectPolls.set(source.id, {
+            cancel: () => {
+                state.cancelled = true;
+                state.wake?.();
+            },
+        });
+        void this.runConnectPoll(source, state)
+            .catch((error) => {
+            this.logger.warn({ err: error, sourceId: source.id }, "Source connect poll failed");
+        })
+            .finally(() => {
+            this.connectPolls.delete(source.id);
+        });
+    }
+    /**
+     * Bounded by construction: the loop condition is elapsed time against a
+     * literal window, so the loop cannot become unbounded without someone
+     * deleting that bound outright. AC 6 audits this file by grep, so do not
+     * write the unbounded-loop keyword pair even inside a comment.
+     */
+    async runConnectPoll(source, state) {
+        const gateway = this.resolveGateway(source.kind);
+        const startedAt = this.now().getTime();
+        while (!state.cancelled && this.now().getTime() - startedAt <= CONNECT_POLL_TIMEOUT_MS) {
+            const accounts = await gateway.listAccounts(this.externalUserId, source.appSlug);
+            if (state.cancelled) {
+                return;
+            }
+            const active = accounts.find((account) => account.status === "active");
+            if (active) {
+                await this.persistHealth(source.id, { status: "active", lastError: null }, active.id);
+                return;
+            }
+            await this.sleep(CONNECT_POLL_INTERVAL_MS, state);
+        }
+        if (state.cancelled) {
+            return;
+        }
+        await this.persistHealth(source.id, { status: "degraded", lastError: NOT_CONFIRMED });
+    }
+    /**
+     * A sleep that a cancel can cut short. ONE resolve path, reached either by the
+     * timer firing or by `state.wake()`, so a cancel during a sleep cannot settle
+     * the promise twice.
+     */
+    sleep(ms, state) {
+        // The resolver is captured out of the executor so `finish` is the single
+        // settle path, reached by whichever of the timer or the cancel fires first.
+        // `settled` makes that explicit rather than leaning on resolve() being a
+        // no-op the second time.
+        let settle;
+        const sleeping = new Promise((resolve) => {
+            settle = resolve;
+        });
+        let settled = false;
+        let timer = null;
+        const finish = () => {
+            if (settled) {
+                return;
+            }
+            settled = true;
+            if (timer !== null) {
+                clearTimeout(timer);
+                timer = null;
+            }
+            state.wake = null;
+            settle();
+        };
+        timer = setTimeout(finish, ms);
+        timer.unref?.();
+        state.wake = finish;
+        return sleeping;
+    }
+    /**
+     * The ONE place a health verdict is written.
+     *
+     * Re-reads the record first so a source deleted while a poll or a sweep was
+     * in flight is not resurrected by a stale write.
+     */
+    async persistHealth(sourceId, health, externalAccountId) {
+        const current = await this.store.get(sourceId);
+        if (!current) {
+            return null;
+        }
+        const timestamp = this.now().toISOString();
+        const next = {
+            ...current,
+            status: health.status,
+            lastError: health.lastError,
+            lastCheckedAt: timestamp,
+            updatedAt: timestamp,
+            ...(externalAccountId !== undefined ? { externalAccountId } : {}),
+        };
+        await this.store.put(next);
+        return next;
+    }
+    /**
+     * FILTER AUTHORING (Epic 7).
+     *
+     * Thin pass-throughs to `FilterStore`, which is what keeps the layering the
+     * class header already claims: no gateway and no credential reaches a caller
+     * through these. `FilterStore.put()` re-parses before writing, so an untyped
+     * caller still cannot land a rail-shaped field on disk even though these
+     * methods sit behind a typed RPC.
+     */
+    async listFilters(sourceId) {
+        const filters = await this.filterStore.list();
+        if (sourceId === undefined) {
+            return filters;
+        }
+        return filters.filter((filter) => filter.sourceId === sourceId);
+    }
+    /**
+     * `createdAt`/`updatedAt` are minted HERE, not accepted from the caller, for
+     * the same reason `id` is: a client that could choose them could forge the
+     * ordering `FilterStore.list()` sorts by.
+     */
+    async createFilter(draft) {
+        const timestamp = this.now().toISOString();
+        return this.filterStore.create({ ...draft, createdAt: timestamp, updatedAt: timestamp });
+    }
+    /**
+     * Returns null when the id resolves to nothing, which the RPC reports as
+     * `filter: null` with `error: null` - "no such filter" is an answer, not a
+     * failure.
+     *
+     * `status` is absent from the patch type on purpose: arming is a schedule
+     * verb (`schedule/create` with a source cadence), never a filter field write.
+     */
+    async updateFilter(filterId, patch) {
+        const existing = await this.filterStore.get(filterId);
+        if (existing === null) {
+            return null;
+        }
+        const updated = {
+            ...existing,
+            ...patch,
+            id: existing.id,
+            createdAt: existing.createdAt,
+            // Pinned AFTER the spread, defence in depth. The wire schema already
+            // strips `status` from a patch, but this method is a plain TypeScript
+            // entry point: a future MCP tool, an internal caller or a test that skips
+            // the parse would otherwise mint a second arming path through a spread.
+            // Same reasoning as FilterStore.put re-parsing before it writes - an
+            // untyped caller must not be able to land the field at all.
+            status: existing.status,
+            updatedAt: this.now().toISOString(),
+        };
+        await this.filterStore.put(updated);
+        return updated;
+    }
+    /**
+     * `false` means the id resolved to nothing. Deleting twice is idempotent and
+     * is not an error - `FilterStore.delete` already uses `force: true` - so the
+     * existence check is what makes the answer informative rather than a
+     * uniformly cheerful `true`.
+     */
+    async deleteFilter(filterId) {
+        const existing = await this.filterStore.get(filterId);
+        if (existing === null) {
+            return false;
+        }
+        await this.filterStore.delete(filterId);
+        return true;
+    }
+    /**
+     * Field by field on purpose. A `...stored` spread would mean a field added to
+     * StoredSource later reaches the wire the day it is added, with no diff on
+     * this file to review.
+     */
+    toSummary(source) {
+        return {
+            id: source.id,
+            kind: source.kind,
+            appSlug: source.appSlug,
+            displayName: source.displayName,
+            imgSrc: source.imgSrc,
+            externalAccountId: source.externalAccountId,
+            status: source.status,
+            createdAt: source.createdAt,
+            updatedAt: source.updatedAt,
+            lastCheckedAt: source.lastCheckedAt,
+            lastError: source.lastError,
+        };
+    }
+}
+//# sourceMappingURL=service.js.map

package/dist/server/server/ingestion/sources/store.d.ts

@@ -0,0 +1,27 @@
+import { type StoredSource } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+/**
+ * File-backed persistence for connected ingestion sources, one JSON file per
+ * source under `dir` (which the caller supplies as `join(paseoHome, "sources")`).
+ *
+ * Structurally identical to `ScheduleStore` on purpose. Two call sites do not
+ * justify a shared generic, and diverging from that shape here, including the
+ * plain recursive `mkdir` rather than a 0700 directory, is a review finding:
+ * `$PASEO_HOME` itself is already created 0700 by `resolvePaseoHome`, so its
+ * children are unreachable to other users by path.
+ *
+ * Every write goes through `writeJsonFileAtomic`, and every read goes through
+ * `StoredSourceSchema.parse`, which is strict. A file that grew an unexpected
+ * key throws here rather than being silently dropped.
+ */
+export declare class SourceStore {
+    private readonly dir;
+    constructor(dir: string);
+    private filePath;
+    private ensureDir;
+    list(): Promise<StoredSource[]>;
+    get(id: string): Promise<StoredSource | null>;
+    create(source: Omit<StoredSource, "id">): Promise<StoredSource>;
+    put(source: StoredSource): Promise<void>;
+    delete(id: string): Promise<void>;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/server/server/ingestion/sources/store.js

@@ -0,0 +1,71 @@
+import { randomBytes } from "node:crypto";
+import { mkdir, readFile, readdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { StoredSourceSchema, } from "@hyperdrive.bot/fleet-protocol/ingestion/types";
+import { writeJsonFileAtomic } from "../../atomic-file.js";
+function generateSourceId() {
+    return randomBytes(4).toString("hex");
+}
+/**
+ * File-backed persistence for connected ingestion sources, one JSON file per
+ * source under `dir` (which the caller supplies as `join(paseoHome, "sources")`).
+ *
+ * Structurally identical to `ScheduleStore` on purpose. Two call sites do not
+ * justify a shared generic, and diverging from that shape here, including the
+ * plain recursive `mkdir` rather than a 0700 directory, is a review finding:
+ * `$PASEO_HOME` itself is already created 0700 by `resolvePaseoHome`, so its
+ * children are unreachable to other users by path.
+ *
+ * Every write goes through `writeJsonFileAtomic`, and every read goes through
+ * `StoredSourceSchema.parse`, which is strict. A file that grew an unexpected
+ * key throws here rather than being silently dropped.
+ */
+export class SourceStore {
+    constructor(dir) {
+        this.dir = dir;
+    }
+    filePath(id) {
+        return join(this.dir, `${id}.json`);
+    }
+    async ensureDir() {
+        await mkdir(this.dir, { recursive: true });
+    }
+    async list() {
+        await this.ensureDir();
+        const entries = await readdir(this.dir, { withFileTypes: true });
+        const sources = await Promise.all(entries
+            .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+            .map(async (entry) => {
+            const content = await readFile(join(this.dir, entry.name), "utf-8");
+            return StoredSourceSchema.parse(JSON.parse(content));
+        }));
+        return sources.sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+    }
+    async get(id) {
+        await this.ensureDir();
+        try {
+            const content = await readFile(this.filePath(id), "utf-8");
+            return StoredSourceSchema.parse(JSON.parse(content));
+        }
+        catch (error) {
+            if (error.code === "ENOENT") {
+                return null;
+            }
+            throw error;
+        }
+    }
+    async create(source) {
+        const created = { ...source, id: generateSourceId() };
+        await this.put(created);
+        return created;
+    }
+    async put(source) {
+        await this.ensureDir();
+        await writeJsonFileAtomic(this.filePath(source.id), source);
+    }
+    async delete(id) {
+        await this.ensureDir();
+        await rm(this.filePath(id), { force: true });
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/server/server/ingestion/testing/claim-and-hang.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=claim-and-hang.d.ts.map