@hyperdrive.bot/fleet-server 0.3.148 → 0.3.150

package/dist/server/server/ingestion/subscriptions/poller.js

@@ -0,0 +1,168 @@
+import { scanMatchedSourceItems, windowStartFor } from "../backfill.js";
+/** How often the poller reads its sources. */
+export const DEFAULT_POLL_INTERVAL_MS = 300000;
+/**
+ * How far back each poll reads.
+ *
+ * Wider than the interval on purpose: a daemon that was asleep, or a source
+ * whose aggregator buffer filled late, would otherwise leave a hole no later
+ * poll ever covers. Re-reading an item is free, because `PendingItemStore.append`
+ * ignores one whose `contentHash` is unchanged.
+ */
+export const POLL_WINDOW_DAYS = 1;
+/**
+ * Reads each subscribed source ONCE per tick and fans the result out to every
+ * filter, and through them to every subscriber.
+ *
+ * The shape exists because of a measured cost: today two filters armed on one
+ * source cause two independent `listItems` walks, with no cache and no shared
+ * scan. With N subscribers that is N reads of the same aggregator buffer per
+ * window. Matching is pure string comparison over a dotted path, so fanning out
+ * in memory costs CPU while reading costs network, rate limit and latency.
+ * One read, many matches.
+ *
+ * It owns its own timer and touches the schedule engine not at all. A
+ * subscription is not a schedule: it must survive its target being archived,
+ * and `sweepOrphanedSchedules` would complete it terminally at the next boot.
+ */
+export class SubscriptionPoller {
+    constructor(options) {
+        this.timer = null;
+        /** Single-flight. A slow aggregator must not let two ticks overlap. */
+        this.running = false;
+        this.options = options;
+        this.intervalMs = options.intervalMs ?? DEFAULT_POLL_INTERVAL_MS;
+        this.now = options.now ?? (() => Date.now());
+    }
+    start() {
+        if (this.timer)
+            return;
+        this.timer = setInterval(() => {
+            void this.tick().catch((error) => {
+                this.options.logger.warn({ err: error }, "Subscription poll tick failed");
+            });
+        }, this.intervalMs);
+        // Never keep the process alive for a poll.
+        this.timer.unref?.();
+        this.options.logger.info({ intervalMs: this.intervalMs }, "Subscription poller started");
+    }
+    stop() {
+        if (!this.timer)
+            return;
+        clearInterval(this.timer);
+        this.timer = null;
+    }
+    /**
+     * One pass: read every subscribed source once, match, notify, then drain.
+     *
+     * The drain at the end is what makes a notice held back by `busy` or
+     * `debounced` eventually go out even when the source has gone quiet.
+     */
+    async tick() {
+        if (this.running) {
+            return { sourcesRead: 0, filtersMatched: 0, notified: 0 };
+        }
+        this.running = true;
+        try {
+            return await this.runTick();
+        }
+        finally {
+            this.running = false;
+        }
+    }
+    async runTick() {
+        const { subscriptions, filterResolver, sourceStore, gatewayResolver, logger } = this.options;
+        const armed = (await subscriptions.list()).filter((s) => s.status === "armed");
+        const summary = { sourcesRead: 0, filtersMatched: 0, notified: 0 };
+        if (armed.length === 0) {
+            return summary;
+        }
+        // Resolve the distinct filters once. Several subscribers commonly share one.
+        const filters = new Map();
+        for (const subscription of armed) {
+            if (filters.has(subscription.filterId))
+                continue;
+            const filter = await filterResolver(subscription.filterId);
+            if (!filter) {
+                logger.warn({ subscriptionId: subscription.id, filterId: subscription.filterId }, "Subscription points at a filter that no longer exists");
+                continue;
+            }
+            if (filter.status !== "armed")
+                continue;
+            filters.set(filter.id, filter);
+        }
+        // Group by source: this is the line that turns N reads into one.
+        const bySource = new Map();
+        for (const filter of filters.values()) {
+            const list = bySource.get(filter.sourceId);
+            if (list)
+                list.push(filter);
+            else
+                bySource.set(filter.sourceId, [filter]);
+        }
+        const now = this.now();
+        for (const [sourceId, sourceFilters] of bySource) {
+            const source = await sourceStore.get(sourceId);
+            if (!source) {
+                logger.warn({ sourceId }, "Subscribed filter points at a source that no longer exists");
+                continue;
+            }
+            let gateway;
+            try {
+                gateway = gatewayResolver(source.kind);
+            }
+            catch (error) {
+                logger.warn({ err: error, sourceId, kind: source.kind }, "No gateway for source kind");
+                continue;
+            }
+            for (const filter of sourceFilters) {
+                try {
+                    const { matched } = await scanMatchedSourceItems({
+                        filter,
+                        gateway,
+                        externalUserId: source.externalUserId,
+                        accountId: source.externalAccountId ?? "",
+                        // Freshness is the pending store's job, keyed per subscriber, so
+                        // this read deliberately answers "already handled?" with null for
+                        // everything. Asking the shared ledger here would be a second,
+                        // filter-wide answer to a question that is per subscription.
+                        ledger: { get: () => Promise.resolve(null) },
+                        sinceMs: windowStartFor(now, POLL_WINDOW_DAYS),
+                        now: () => now,
+                    });
+                    summary.filtersMatched += 1;
+                    if (matched.length === 0)
+                        continue;
+                    const results = await this.options.notifier.notifyForFilter(filter, matched);
+                    summary.notified += results.filter((result) => result.outcome === "sent").length;
+                }
+                catch (error) {
+                    // A failing filter must not abandon its siblings on the same read.
+                    logger.warn({ err: error, filterId: filter.id, sourceId }, "Subscription scan failed");
+                }
+            }
+            summary.sourcesRead += 1;
+        }
+        const drained = await this.options.notifier.deliverAllPending();
+        summary.notified += drained.filter((result) => result.outcome === "sent").length;
+        // Retention runs on the TICK, not only at boot. Sweeping only at startup
+        // makes real retention `max(uptime, ttl)`, so a daemon up for forty days
+        // holds forty days of payloads while the docstring promises seven. That
+        // TTL is one of the three things containing a credential at rest, so it
+        // cannot depend on how often the process restarts. `IngestionService` does
+        // the same on its own health timer.
+        await this.sweep().catch((error) => {
+            this.options.logger.warn({ err: error }, "Subscription payload sweep failed");
+        });
+        return summary;
+    }
+    /** Delete pending payloads past their TTL. Called at boot, beside the other sweeps. */
+    async sweep() {
+        const removed = await this.options.pending.sweepExpired(this.now());
+        if (removed > 0) {
+            this.options.logger.info({ removed }, "Swept expired subscription payloads");
+        }
+        return removed;
+    }
+}
+//# sourceMappingURL=poller.js.map

package/dist/server/server/ingestion/subscriptions/reader.d.ts

@@ -0,0 +1,39 @@
+import type { StoredSubscription } from "@hyperdrive.bot/fleet-protocol/ingestion/subscription-types";
+import type { PendingItem, PendingItemStore } from "./pending-store.js";
+import type { SubscriptionStore } from "./store.js";
+/**
+ * The READ half of subscriptions, and the only half a model ever holds.
+ *
+ * Deliberately not the notifier and not the stores themselves. A tool catalog
+ * that could arm a subscription, or trigger a delivery, would put the delivery
+ * policy inside a model's reach; this interface cannot do either. It exists as
+ * an interface rather than a concrete class so the tool layer depends on three
+ * methods instead of on the ingestion module.
+ */
+export interface SubscriptionReader {
+    /**
+     * Subscriptions the CALLER owns, never the daemon's whole set.
+     *
+     * `callerAgentId` is not decoration. Without it this is an enumeration step:
+     * one session lists every subscription on the box, reads another session's
+     * agent id, and then reads that session's stored payloads, credentials
+     * included. "A local MCP tool" means every agent on this daemon, which is
+     * not the same thing as the owner.
+     */
+    list(callerAgentId: string | undefined): Promise<StoredSubscription[]>;
+    get(subscriptionId: string, callerAgentId: string | undefined): Promise<StoredSubscription | null>;
+    /**
+     * Stored items for the given keys, payload included.
+     *
+     * This is the ONLY path by which a persisted payload leaves the daemon, and
+     * it is a local MCP call: no client RPC exposes it. Keys that are not held
+     * are simply absent from the result, the same contract `resolveMatchedItems`
+     * already has, so a caller sees a shorter list rather than an exception.
+     */
+    resolveItems(subscriptionId: string, itemKeys: readonly string[], callerAgentId: string | undefined): Promise<PendingItem[]>;
+}
+export declare function createSubscriptionReader(deps: {
+    subscriptions: SubscriptionStore;
+    pending: PendingItemStore;
+}): SubscriptionReader;
+//# sourceMappingURL=reader.d.ts.map

package/dist/server/server/ingestion/subscriptions/reader.js

@@ -0,0 +1,30 @@
+export function createSubscriptionReader(deps) {
+    /**
+     * Fails CLOSED when the caller is unknown.
+     *
+     * An absent `callerAgentId` means the tool catalog was built without a
+     * caller identity, and the safe reading of "I do not know who is asking" is
+     * "you own nothing", not "you own everything".
+     */
+    const owns = (subscription, callerAgentId) => callerAgentId !== undefined && subscription.agentId === callerAgentId;
+    return {
+        list: async (callerAgentId) => (await deps.subscriptions.list()).filter((subscription) => owns(subscription, callerAgentId)),
+        get: async (subscriptionId, callerAgentId) => {
+            const subscription = await deps.subscriptions.get(subscriptionId);
+            if (!subscription || !owns(subscription, callerAgentId))
+                return null;
+            return subscription;
+        },
+        resolveItems: async (subscriptionId, itemKeys, callerAgentId) => {
+            // A subscription that no longer exists, or that belongs to someone else,
+            // resolves to nothing. Both answer the same way on purpose: a caller
+            // must not be able to tell "no such subscription" from "not yours" and
+            // use the difference to enumerate.
+            const subscription = await deps.subscriptions.get(subscriptionId);
+            if (!subscription || !owns(subscription, callerAgentId))
+                return [];
+            return deps.pending.resolve(subscriptionId, itemKeys);
+        },
+    };
+}
+//# sourceMappingURL=reader.js.map

package/dist/server/server/ingestion/subscriptions/store.d.ts

@@ -0,0 +1,35 @@
+import { type StoredSubscription } from "@hyperdrive.bot/fleet-protocol/ingestion/subscription-types";
+/**
+ * File-backed persistence for session subscriptions, one JSON file per
+ * subscription under `dir` (the caller supplies `join(paseoHome, "subscriptions")`).
+ *
+ * Structurally identical to `SourceStore` and `ScheduleStore`, deliberately: a
+ * third shape here would be a third set of atomicity and parse bugs to find.
+ *
+ * `StoredSubscriptionSchema` is strict and `list()` is readdir + Promise.all, so
+ * ONE unparseable file rejects the whole array. That is the same trade the
+ * source store makes, and it carries the same obligation: every field added to
+ * the schema later must carry `.default(...)`, or records written before it
+ * existed stop parsing and every subscription silently disappears. There is no
+ * migration step. `accountLabel` on `StoredSourceSchema` is the precedent.
+ */
+export declare class SubscriptionStore {
+    private readonly dir;
+    constructor(dir: string);
+    private filePath;
+    private ensureDir;
+    list(): Promise<StoredSubscription[]>;
+    get(id: string): Promise<StoredSubscription | null>;
+    /**
+     * Every armed subscription pointing at `filterId`.
+     *
+     * The fan-out read. A filter with three armed subscribers returns three
+     * records here, and each one is notified on its own budget: this is the
+     * function that makes one filter serve many sessions.
+     */
+    listArmedForFilter(filterId: string): Promise<StoredSubscription[]>;
+    create(subscription: Omit<StoredSubscription, "id">): Promise<StoredSubscription>;
+    put(subscription: StoredSubscription): Promise<void>;
+    delete(id: string): Promise<void>;
+}
+//# sourceMappingURL=store.d.ts.map

package/dist/server/server/ingestion/subscriptions/store.js

@@ -0,0 +1,82 @@
+import { randomBytes } from "node:crypto";
+import { mkdir, readFile, readdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { StoredSubscriptionSchema, } from "@hyperdrive.bot/fleet-protocol/ingestion/subscription-types";
+import { writeJsonFileAtomic } from "../../atomic-file.js";
+function generateSubscriptionId() {
+    return randomBytes(4).toString("hex");
+}
+/**
+ * File-backed persistence for session subscriptions, one JSON file per
+ * subscription under `dir` (the caller supplies `join(paseoHome, "subscriptions")`).
+ *
+ * Structurally identical to `SourceStore` and `ScheduleStore`, deliberately: a
+ * third shape here would be a third set of atomicity and parse bugs to find.
+ *
+ * `StoredSubscriptionSchema` is strict and `list()` is readdir + Promise.all, so
+ * ONE unparseable file rejects the whole array. That is the same trade the
+ * source store makes, and it carries the same obligation: every field added to
+ * the schema later must carry `.default(...)`, or records written before it
+ * existed stop parsing and every subscription silently disappears. There is no
+ * migration step. `accountLabel` on `StoredSourceSchema` is the precedent.
+ */
+export class SubscriptionStore {
+    constructor(dir) {
+        this.dir = dir;
+    }
+    filePath(id) {
+        return join(this.dir, `${id}.json`);
+    }
+    async ensureDir() {
+        await mkdir(this.dir, { recursive: true });
+    }
+    async list() {
+        await this.ensureDir();
+        const entries = await readdir(this.dir, { withFileTypes: true });
+        const subscriptions = await Promise.all(entries
+            .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+            .map(async (entry) => {
+            const content = await readFile(join(this.dir, entry.name), "utf-8");
+            return StoredSubscriptionSchema.parse(JSON.parse(content));
+        }));
+        return subscriptions.sort((left, right) => left.createdAt.localeCompare(right.createdAt));
+    }
+    async get(id) {
+        await this.ensureDir();
+        try {
+            const content = await readFile(this.filePath(id), "utf-8");
+            return StoredSubscriptionSchema.parse(JSON.parse(content));
+        }
+        catch (error) {
+            if (error.code === "ENOENT") {
+                return null;
+            }
+            throw error;
+        }
+    }
+    /**
+     * Every armed subscription pointing at `filterId`.
+     *
+     * The fan-out read. A filter with three armed subscribers returns three
+     * records here, and each one is notified on its own budget: this is the
+     * function that makes one filter serve many sessions.
+     */
+    async listArmedForFilter(filterId) {
+        const all = await this.list();
+        return all.filter((subscription) => subscription.filterId === filterId && subscription.status === "armed");
+    }
+    async create(subscription) {
+        const created = { ...subscription, id: generateSubscriptionId() };
+        await this.put(created);
+        return created;
+    }
+    async put(subscription) {
+        await this.ensureDir();
+        await writeJsonFileAtomic(this.filePath(subscription.id), subscription);
+    }
+    async delete(id) {
+        await this.ensureDir();
+        await rm(this.filePath(id), { force: true });
+    }
+}
+//# sourceMappingURL=store.js.map

package/dist/server/server/ingestion/types.d.ts

@@ -14,7 +14,7 @@
  * `Record<SourceKind, ...>` registry in `adapters.ts` turns that silence into
  * a compile error.
  */
-export declare const SOURCE_CAPABLE_KINDS: readonly ["pipedream", "composio"];
+export declare const SOURCE_CAPABLE_KINDS: readonly ["pipedream", "composio", "self-hosted"];
 export type SourceKind = (typeof SOURCE_CAPABLE_KINDS)[number];
 /** Where to reach an aggregator's remote MCP server for a single agent turn. */
 export interface McpTarget {
@@ -101,9 +101,22 @@ export interface PipedreamConfig {
 export interface ComposioConfig {
     apiKey?: string | undefined;
 }
+/**
+ * Where this daemon's OWN source bridge lives, and the token that opens it.
+ *
+ * Unlike the aggregator configs above, both fields are user-supplied and
+ * user-hosted: paseo holds no platform credential for a self-hosted bridge, it
+ * holds a bearer for one endpoint the user runs. `configured` is false when
+ * either is missing, which is how the sources screen knows not to offer it.
+ */
+export interface SelfHostedConfig {
+    url?: string | undefined;
+    token?: string | undefined;
+}
 export interface IngestionConfig {
     pipedream: PipedreamConfig;
     composio: ComposioConfig;
+    selfHosted?: SelfHostedConfig | undefined;
 }
 /**
  * A single-field object on purpose: later stories add a logger here without

package/dist/server/server/ingestion/types.js

@@ -14,5 +14,5 @@
  * `Record<SourceKind, ...>` registry in `adapters.ts` turns that silence into
  * a compile error.
  */
-export const SOURCE_CAPABLE_KINDS = ["pipedream", "composio"];
+export const SOURCE_CAPABLE_KINDS = ["pipedream", "composio", "self-hosted"];
 //# sourceMappingURL=types.js.map