@hyperdrive.bot/fleet-server 0.3.147 → 0.3.149
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/server/agent/mcp-server.js +64 -2
- package/dist/server/server/agent/session-digest-generator.js +33 -2
- package/dist/server/server/agent/session-digest.d.ts +71 -0
- package/dist/server/server/agent/session-digest.js +117 -1
- package/dist/server/server/agent/tools/paseo-tools.d.ts +9 -0
- package/dist/server/server/agent/tools/paseo-tools.js +76 -1
- package/dist/server/server/agent/tools/read-only-surface.d.ts +7 -0
- package/dist/server/server/agent/tools/read-only-surface.js +8 -0
- package/dist/server/server/bootstrap.js +71 -1
- package/dist/server/server/exports.d.ts +2 -0
- package/dist/server/server/exports.js +6 -0
- package/dist/server/server/ingestion/subscriptions/notification-prompt.d.ts +83 -0
- package/dist/server/server/ingestion/subscriptions/notification-prompt.js +96 -0
- package/dist/server/server/ingestion/subscriptions/notifier.d.ts +96 -0
- package/dist/server/server/ingestion/subscriptions/notifier.js +209 -0
- package/dist/server/server/ingestion/subscriptions/pending-store.d.ts +111 -0
- package/dist/server/server/ingestion/subscriptions/pending-store.js +254 -0
- package/dist/server/server/ingestion/subscriptions/poller.d.ts +73 -0
- package/dist/server/server/ingestion/subscriptions/poller.js +168 -0
- package/dist/server/server/ingestion/subscriptions/reader.d.ts +39 -0
- package/dist/server/server/ingestion/subscriptions/reader.js +30 -0
- package/dist/server/server/ingestion/subscriptions/store.d.ts +35 -0
- package/dist/server/server/ingestion/subscriptions/store.js +82 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-837630304edbf229f37aaa1d262c40d5.js → index-8747c529e5cb02149fe51570f7cb697b.js} +13 -13
- package/dist/server/web-ui/_expo/static/js/web/index-8747c529e5cb02149fe51570f7cb697b.js.br +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-837630304edbf229f37aaa1d262c40d5.js.gz → index-8747c529e5cb02149fe51570f7cb697b.js.gz} +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-837630304edbf229f37aaa1d262c40d5.js.map.br → index-8747c529e5cb02149fe51570f7cb697b.js.map.br} +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-837630304edbf229f37aaa1d262c40d5.js.map.gz → index-8747c529e5cb02149fe51570f7cb697b.js.map.gz} +0 -0
- package/dist/server/web-ui/index.html +1 -1
- package/dist/server/web-ui/index.html.br +0 -0
- package/dist/server/web-ui/index.html.gz +0 -0
- package/package.json +6 -6
- package/dist/server/web-ui/_expo/static/js/web/index-837630304edbf229f37aaa1d262c40d5.js.br +0 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
import { readFile, readdir, rm, stat } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { writeJsonFileAtomic } from "../../atomic-file.js";
|
|
5
|
+
import { ensurePrivateDirectory, ensurePrivateFile } from "../../private-files.js";
|
|
6
|
+
/**
|
|
7
|
+
* One detected item, held until the subscriber has been told about it.
|
|
8
|
+
*
|
|
9
|
+
* This record does two jobs that look separate and are not:
|
|
10
|
+
*
|
|
11
|
+
* 1. **The coalescing accumulator.** A subscriber that is mid-turn, or inside
|
|
12
|
+
* its debounce window, must not lose the event. The schedule engine's own
|
|
13
|
+
* failure path is the cautionary tale: `finishRun` has no branch reading
|
|
14
|
+
* `params.status`, so a run that failed because the agent was busy still
|
|
15
|
+
* advances `nextRunAt` a whole cadence period, which turns "deliver later"
|
|
16
|
+
* into "never". Holding the item here is the memory that path lacks.
|
|
17
|
+
* 2. **The payload for the fetch.** `ingestion_items` stores no payload and
|
|
18
|
+
* cohort rows store fingerprints, so without this the `get_subscription_items`
|
|
19
|
+
* tool would be a live network read that can come back empty for an item
|
|
20
|
+
* that left the window between the notice and the question.
|
|
21
|
+
*/
|
|
22
|
+
export const PendingItemSchema = z.strictObject({
|
|
23
|
+
itemKey: z.string(),
|
|
24
|
+
contentHash: z.string(),
|
|
25
|
+
title: z.string(),
|
|
26
|
+
subtitle: z.string(),
|
|
27
|
+
timestampMs: z.number(),
|
|
28
|
+
/** Epoch ms the daemon detected it. Drives the TTL sweep, never freshness. */
|
|
29
|
+
detectedAt: z.number(),
|
|
30
|
+
/** ISO 8601, or null while the item is still waiting to be announced. */
|
|
31
|
+
notifiedAt: z.string().nullable().default(null),
|
|
32
|
+
/**
|
|
33
|
+
* The raw aggregator event body.
|
|
34
|
+
*
|
|
35
|
+
* Stored because the user chose reliable fetch over a live re-read. The cost
|
|
36
|
+
* is real and is not hidden: an event body can contain a credential, so this
|
|
37
|
+
* file is the one place in ingestion that holds one at rest. Three things
|
|
38
|
+
* contain it, and none of them is optional: `$PASEO_HOME` is 0700, the value
|
|
39
|
+
* leaves the daemon ONLY through a local MCP tool and never through a client
|
|
40
|
+
* RPC, and `sweepExpired` deletes it on a TTL.
|
|
41
|
+
*/
|
|
42
|
+
payload: z.unknown(),
|
|
43
|
+
});
|
|
44
|
+
const PendingFileSchema = z.strictObject({
|
|
45
|
+
subscriptionId: z.string(),
|
|
46
|
+
items: z.array(PendingItemSchema).default([]),
|
|
47
|
+
});
|
|
48
|
+
/** How long a detected item is kept before the sweep deletes it. 7 days. */
|
|
49
|
+
export const DEFAULT_PENDING_TTL_MS = 7 * 86400000;
|
|
50
|
+
/**
|
|
51
|
+
* Per-subscription pending items, one JSON file each under `dir`.
|
|
52
|
+
*
|
|
53
|
+
* Deliberately NOT a sqlite table. `ledger-schema.ts` creates its tables with
|
|
54
|
+
* bare `CREATE TABLE IF NOT EXISTS` and carries no migration step, so adding a
|
|
55
|
+
* column there is a change every existing `paseo.sqlite` silently does not get.
|
|
56
|
+
* A JSON file per subscription has the store convention the rest of ingestion
|
|
57
|
+
* already uses, and its failure mode is one unreadable subscription rather than
|
|
58
|
+
* a schema the database disagrees with.
|
|
59
|
+
*/
|
|
60
|
+
export class PendingItemStore {
|
|
61
|
+
constructor(dir) {
|
|
62
|
+
this.dir = dir;
|
|
63
|
+
/**
|
|
64
|
+
* One serialization chain per subscription.
|
|
65
|
+
*
|
|
66
|
+
* Every mutation here is read-modify-write over a whole JSON file, so two
|
|
67
|
+
* overlapping calls for the same subscription would each read the pre-state
|
|
68
|
+
* and the second write would erase the first. Today the only caller is the
|
|
69
|
+
* poller, whose tick is single-flight and sequential, so the race cannot
|
|
70
|
+
* happen by accident. That is exactly the kind of safety that stops being
|
|
71
|
+
* true the first time someone calls `deliverPending` from an RPC handler,
|
|
72
|
+
* and the failure it produces is a silently dropped notification rather than
|
|
73
|
+
* an error. Keyed by id, so two different subscriptions still run in
|
|
74
|
+
* parallel.
|
|
75
|
+
*/
|
|
76
|
+
this.chains = new Map();
|
|
77
|
+
}
|
|
78
|
+
withLock(subscriptionId, work) {
|
|
79
|
+
const previous = this.chains.get(subscriptionId) ?? Promise.resolve();
|
|
80
|
+
// `catch` before chaining: one failed operation must not poison every
|
|
81
|
+
// later one on the same subscription.
|
|
82
|
+
const next = previous.then(work, work);
|
|
83
|
+
this.chains.set(subscriptionId, next.catch(() => undefined));
|
|
84
|
+
return next;
|
|
85
|
+
}
|
|
86
|
+
filePath(subscriptionId) {
|
|
87
|
+
return join(this.dir, `${subscriptionId}.json`);
|
|
88
|
+
}
|
|
89
|
+
async ensureDir() {
|
|
90
|
+
// 0700, not the default 0755.
|
|
91
|
+
//
|
|
92
|
+
// This is the one directory in ingestion that holds a raw aggregator event
|
|
93
|
+
// body at rest, and such a body can carry an OAuth token. `$PASEO_HOME` is
|
|
94
|
+
// already 0700, so a 0755 child is unreachable to another user by path
|
|
95
|
+
// anyway - which is exactly the argument `SourceStore` makes for using a
|
|
96
|
+
// plain mkdir. That argument is correct and still leaves this directory
|
|
97
|
+
// one misplaced `$PASEO_HOME` away from being world readable, and unlike a
|
|
98
|
+
// source record the contents here are secret. Defence in depth, measured:
|
|
99
|
+
// a bare recursive mkdir under umask 0022 produces 0755.
|
|
100
|
+
ensurePrivateDirectory(this.dir);
|
|
101
|
+
}
|
|
102
|
+
async read(subscriptionId) {
|
|
103
|
+
await this.ensureDir();
|
|
104
|
+
try {
|
|
105
|
+
const content = await readFile(this.filePath(subscriptionId), "utf-8");
|
|
106
|
+
return PendingFileSchema.parse(JSON.parse(content)).items;
|
|
107
|
+
}
|
|
108
|
+
catch (error) {
|
|
109
|
+
if (error.code === "ENOENT") {
|
|
110
|
+
return [];
|
|
111
|
+
}
|
|
112
|
+
throw error;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
async write(subscriptionId, items) {
|
|
116
|
+
await this.ensureDir();
|
|
117
|
+
const target = this.filePath(subscriptionId);
|
|
118
|
+
await writeJsonFileAtomic(target, { subscriptionId, items });
|
|
119
|
+
// 0600 on the file too, for the same reason and after the write, because
|
|
120
|
+
// the atomic write renames a fresh temp file into place each time.
|
|
121
|
+
ensurePrivateFile(target);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Record newly detected items, keyed by `itemKey`.
|
|
125
|
+
*
|
|
126
|
+
* An item whose `contentHash` CHANGED replaces the stored one and is
|
|
127
|
+
* announced again, which mirrors the ledger's own rule (`contentChanged` at
|
|
128
|
+
* `ledger.ts:368-373`): a changed item is a new event, not a duplicate. An
|
|
129
|
+
* item whose hash is identical is left exactly as it is, including its
|
|
130
|
+
* `notifiedAt`, so re-detecting it does not re-announce it.
|
|
131
|
+
*
|
|
132
|
+
* Returns the items that are genuinely new or changed, which is what the
|
|
133
|
+
* caller announces.
|
|
134
|
+
*/
|
|
135
|
+
async append(subscriptionId, incoming) {
|
|
136
|
+
return this.withLock(subscriptionId, () => this.appendLocked(subscriptionId, incoming));
|
|
137
|
+
}
|
|
138
|
+
async appendLocked(subscriptionId, incoming) {
|
|
139
|
+
const existing = await this.read(subscriptionId);
|
|
140
|
+
const byKey = new Map(existing.map((item) => [item.itemKey, item]));
|
|
141
|
+
const fresh = [];
|
|
142
|
+
for (const item of incoming) {
|
|
143
|
+
const previous = byKey.get(item.itemKey);
|
|
144
|
+
if (previous && previous.contentHash === item.contentHash) {
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
byKey.set(item.itemKey, item);
|
|
148
|
+
fresh.push(item);
|
|
149
|
+
}
|
|
150
|
+
if (fresh.length > 0) {
|
|
151
|
+
await this.write(subscriptionId, [...byKey.values()]);
|
|
152
|
+
}
|
|
153
|
+
return fresh;
|
|
154
|
+
}
|
|
155
|
+
/** Everything detected but not yet announced. The coalescing read. */
|
|
156
|
+
async listUnnotified(subscriptionId) {
|
|
157
|
+
const items = await this.read(subscriptionId);
|
|
158
|
+
return items.filter((item) => item.notifiedAt === null);
|
|
159
|
+
}
|
|
160
|
+
/** Stamp `notifiedAt` on the keys just announced, so they are not repeated. */
|
|
161
|
+
/**
|
|
162
|
+
* Stamp the rows just announced, matched on `(itemKey, contentHash)`.
|
|
163
|
+
*
|
|
164
|
+
* The hash is load-bearing, not belt and braces. The caller reads the
|
|
165
|
+
* waiting rows, awaits a network send, and only then marks: if an append
|
|
166
|
+
* lands in that gap carrying the SAME key with CHANGED content, the row on
|
|
167
|
+
* disk is a different event from the one the notice described. Matching on
|
|
168
|
+
* the key alone would stamp the new version as announced and it would never
|
|
169
|
+
* go out. This is the same rule the ledger states as "a changed item is a
|
|
170
|
+
* new event".
|
|
171
|
+
*/
|
|
172
|
+
async markNotified(subscriptionId, announced, notifiedAt) {
|
|
173
|
+
return this.withLock(subscriptionId, () => this.markNotifiedLocked(subscriptionId, announced, notifiedAt));
|
|
174
|
+
}
|
|
175
|
+
async markNotifiedLocked(subscriptionId, announced, notifiedAt) {
|
|
176
|
+
const wanted = new Set(announced.map((row) => `${row.itemKey}\u0000${row.contentHash}`));
|
|
177
|
+
const items = await this.read(subscriptionId);
|
|
178
|
+
// A loop rather than `map` with a spread: `no-map-spread` is on, and the
|
|
179
|
+
// rewritten row is built once per match instead of once per element.
|
|
180
|
+
const next = [];
|
|
181
|
+
for (const item of items) {
|
|
182
|
+
if (wanted.has(`${item.itemKey}\u0000${item.contentHash}`) && item.notifiedAt === null) {
|
|
183
|
+
next.push(Object.assign({}, item, { notifiedAt }));
|
|
184
|
+
continue;
|
|
185
|
+
}
|
|
186
|
+
next.push(item);
|
|
187
|
+
}
|
|
188
|
+
await this.write(subscriptionId, next);
|
|
189
|
+
}
|
|
190
|
+
/** Resolve keys to their stored items. Absent keys are simply not returned. */
|
|
191
|
+
async resolve(subscriptionId, itemKeys) {
|
|
192
|
+
const wanted = new Set(itemKeys);
|
|
193
|
+
const items = await this.read(subscriptionId);
|
|
194
|
+
return items.filter((item) => wanted.has(item.itemKey));
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* Delete items older than `ttlMs`, and the file itself once it is empty.
|
|
198
|
+
*
|
|
199
|
+
* This is the retention half of the credential-at-rest trade. It runs at
|
|
200
|
+
* boot beside the other sweeps rather than on a timer of its own: a payload
|
|
201
|
+
* that outlives its usefulness is the part of this design that ages badly.
|
|
202
|
+
*/
|
|
203
|
+
async sweepExpired(now, ttlMs = DEFAULT_PENDING_TTL_MS) {
|
|
204
|
+
await this.ensureDir();
|
|
205
|
+
const entries = await readdir(this.dir, { withFileTypes: true });
|
|
206
|
+
let removed = 0;
|
|
207
|
+
for (const entry of entries) {
|
|
208
|
+
if (!entry.isFile() || !entry.name.endsWith(".json"))
|
|
209
|
+
continue;
|
|
210
|
+
const subscriptionId = entry.name.slice(0, -".json".length);
|
|
211
|
+
const items = await this.read(subscriptionId);
|
|
212
|
+
const kept = items.filter((item) => now - item.detectedAt < ttlMs);
|
|
213
|
+
removed += items.length - kept.length;
|
|
214
|
+
if (kept.length === items.length)
|
|
215
|
+
continue;
|
|
216
|
+
// Inside the lock: this is a read-modify-write like every other mutator,
|
|
217
|
+
// and an append landing between the read above and the write below would
|
|
218
|
+
// be erased with no error and no log. Safe today only because the caller
|
|
219
|
+
// awaits the sweep before starting the timer, which is exactly the kind
|
|
220
|
+
// of caller-sequencing guarantee this class refuses to rely on.
|
|
221
|
+
await this.withLock(subscriptionId, async () => {
|
|
222
|
+
const current = await this.read(subscriptionId);
|
|
223
|
+
const survivors = current.filter((item) => now - item.detectedAt < ttlMs);
|
|
224
|
+
if (survivors.length === 0) {
|
|
225
|
+
await rm(this.filePath(subscriptionId), { force: true });
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
await this.write(subscriptionId, survivors);
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
return removed;
|
|
232
|
+
}
|
|
233
|
+
/** Drop everything for a subscription that no longer exists. */
|
|
234
|
+
async drop(subscriptionId) {
|
|
235
|
+
await this.withLock(subscriptionId, async () => {
|
|
236
|
+
await this.ensureDir();
|
|
237
|
+
await rm(this.filePath(subscriptionId), { force: true });
|
|
238
|
+
// The chain entry itself is dropped, so a deleted subscription does not
|
|
239
|
+
// leave a settled promise in the map for the life of the daemon.
|
|
240
|
+
this.chains.delete(subscriptionId);
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
/** Whether the backing directory exists yet. Used only by diagnostics. */
|
|
244
|
+
async exists() {
|
|
245
|
+
try {
|
|
246
|
+
await stat(this.dir);
|
|
247
|
+
return true;
|
|
248
|
+
}
|
|
249
|
+
catch {
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
//# sourceMappingURL=pending-store.js.map
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import type { StoredFilter } from "@hyperdrive.bot/fleet-protocol/ingestion/filter-types";
|
|
2
|
+
import type { Logger } from "pino";
|
|
3
|
+
import type { SourceStore } from "../sources/store.js";
|
|
4
|
+
import type { SourceGateway, SourceKind } from "../types.js";
|
|
5
|
+
import type { SubscriptionNotifier } from "./notifier.js";
|
|
6
|
+
import type { PendingItemStore } from "./pending-store.js";
|
|
7
|
+
import type { SubscriptionStore } from "./store.js";
|
|
8
|
+
/** How often the poller reads its sources. */
|
|
9
|
+
export declare const DEFAULT_POLL_INTERVAL_MS = 300000;
|
|
10
|
+
/**
|
|
11
|
+
* How far back each poll reads.
|
|
12
|
+
*
|
|
13
|
+
* Wider than the interval on purpose: a daemon that was asleep, or a source
|
|
14
|
+
* whose aggregator buffer filled late, would otherwise leave a hole no later
|
|
15
|
+
* poll ever covers. Re-reading an item is free, because `PendingItemStore.append`
|
|
16
|
+
* ignores one whose `contentHash` is unchanged.
|
|
17
|
+
*/
|
|
18
|
+
export declare const POLL_WINDOW_DAYS = 1;
|
|
19
|
+
export interface SubscriptionPollerOptions {
|
|
20
|
+
subscriptions: SubscriptionStore;
|
|
21
|
+
pending: PendingItemStore;
|
|
22
|
+
notifier: SubscriptionNotifier;
|
|
23
|
+
sourceStore: SourceStore;
|
|
24
|
+
filterResolver: (filterId: string) => Promise<StoredFilter | null>;
|
|
25
|
+
gatewayResolver: (kind: SourceKind) => SourceGateway;
|
|
26
|
+
logger: Logger;
|
|
27
|
+
intervalMs?: number;
|
|
28
|
+
now?: () => number;
|
|
29
|
+
}
|
|
30
|
+
export interface PollSummary {
|
|
31
|
+
/** Sources actually read. The number of NETWORK reads this tick cost. */
|
|
32
|
+
sourcesRead: number;
|
|
33
|
+
/** Filters evaluated against those reads, which costs CPU and not network. */
|
|
34
|
+
filtersMatched: number;
|
|
35
|
+
notified: number;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Reads each subscribed source ONCE per tick and fans the result out to every
|
|
39
|
+
* filter, and through them to every subscriber.
|
|
40
|
+
*
|
|
41
|
+
* The shape exists because of a measured cost: today two filters armed on one
|
|
42
|
+
* source cause two independent `listItems` walks, with no cache and no shared
|
|
43
|
+
* scan. With N subscribers that is N reads of the same aggregator buffer per
|
|
44
|
+
* window. Matching is pure string comparison over a dotted path, so fanning out
|
|
45
|
+
* in memory costs CPU while reading costs network, rate limit and latency.
|
|
46
|
+
* One read, many matches.
|
|
47
|
+
*
|
|
48
|
+
* It owns its own timer and touches the schedule engine not at all. A
|
|
49
|
+
* subscription is not a schedule: it must survive its target being archived,
|
|
50
|
+
* and `sweepOrphanedSchedules` would complete it terminally at the next boot.
|
|
51
|
+
*/
|
|
52
|
+
export declare class SubscriptionPoller {
|
|
53
|
+
private readonly options;
|
|
54
|
+
private readonly intervalMs;
|
|
55
|
+
private readonly now;
|
|
56
|
+
private timer;
|
|
57
|
+
/** Single-flight. A slow aggregator must not let two ticks overlap. */
|
|
58
|
+
private running;
|
|
59
|
+
constructor(options: SubscriptionPollerOptions);
|
|
60
|
+
start(): void;
|
|
61
|
+
stop(): void;
|
|
62
|
+
/**
|
|
63
|
+
* One pass: read every subscribed source once, match, notify, then drain.
|
|
64
|
+
*
|
|
65
|
+
* The drain at the end is what makes a notice held back by `busy` or
|
|
66
|
+
* `debounced` eventually go out even when the source has gone quiet.
|
|
67
|
+
*/
|
|
68
|
+
tick(): Promise<PollSummary>;
|
|
69
|
+
private runTick;
|
|
70
|
+
/** Delete pending payloads past their TTL. Called at boot, beside the other sweeps. */
|
|
71
|
+
sweep(): Promise<number>;
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=poller.d.ts.map
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import { scanMatchedSourceItems, windowStartFor } from "../backfill.js";
|
|
2
|
+
/** How often the poller reads its sources. */
|
|
3
|
+
export const DEFAULT_POLL_INTERVAL_MS = 300000;
|
|
4
|
+
/**
|
|
5
|
+
* How far back each poll reads.
|
|
6
|
+
*
|
|
7
|
+
* Wider than the interval on purpose: a daemon that was asleep, or a source
|
|
8
|
+
* whose aggregator buffer filled late, would otherwise leave a hole no later
|
|
9
|
+
* poll ever covers. Re-reading an item is free, because `PendingItemStore.append`
|
|
10
|
+
* ignores one whose `contentHash` is unchanged.
|
|
11
|
+
*/
|
|
12
|
+
export const POLL_WINDOW_DAYS = 1;
|
|
13
|
+
/**
|
|
14
|
+
* Reads each subscribed source ONCE per tick and fans the result out to every
|
|
15
|
+
* filter, and through them to every subscriber.
|
|
16
|
+
*
|
|
17
|
+
* The shape exists because of a measured cost: today two filters armed on one
|
|
18
|
+
* source cause two independent `listItems` walks, with no cache and no shared
|
|
19
|
+
* scan. With N subscribers that is N reads of the same aggregator buffer per
|
|
20
|
+
* window. Matching is pure string comparison over a dotted path, so fanning out
|
|
21
|
+
* in memory costs CPU while reading costs network, rate limit and latency.
|
|
22
|
+
* One read, many matches.
|
|
23
|
+
*
|
|
24
|
+
* It owns its own timer and touches the schedule engine not at all. A
|
|
25
|
+
* subscription is not a schedule: it must survive its target being archived,
|
|
26
|
+
* and `sweepOrphanedSchedules` would complete it terminally at the next boot.
|
|
27
|
+
*/
|
|
28
|
+
export class SubscriptionPoller {
|
|
29
|
+
constructor(options) {
|
|
30
|
+
this.timer = null;
|
|
31
|
+
/** Single-flight. A slow aggregator must not let two ticks overlap. */
|
|
32
|
+
this.running = false;
|
|
33
|
+
this.options = options;
|
|
34
|
+
this.intervalMs = options.intervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
35
|
+
this.now = options.now ?? (() => Date.now());
|
|
36
|
+
}
|
|
37
|
+
start() {
|
|
38
|
+
if (this.timer)
|
|
39
|
+
return;
|
|
40
|
+
this.timer = setInterval(() => {
|
|
41
|
+
void this.tick().catch((error) => {
|
|
42
|
+
this.options.logger.warn({ err: error }, "Subscription poll tick failed");
|
|
43
|
+
});
|
|
44
|
+
}, this.intervalMs);
|
|
45
|
+
// Never keep the process alive for a poll.
|
|
46
|
+
this.timer.unref?.();
|
|
47
|
+
this.options.logger.info({ intervalMs: this.intervalMs }, "Subscription poller started");
|
|
48
|
+
}
|
|
49
|
+
stop() {
|
|
50
|
+
if (!this.timer)
|
|
51
|
+
return;
|
|
52
|
+
clearInterval(this.timer);
|
|
53
|
+
this.timer = null;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* One pass: read every subscribed source once, match, notify, then drain.
|
|
57
|
+
*
|
|
58
|
+
* The drain at the end is what makes a notice held back by `busy` or
|
|
59
|
+
* `debounced` eventually go out even when the source has gone quiet.
|
|
60
|
+
*/
|
|
61
|
+
async tick() {
|
|
62
|
+
if (this.running) {
|
|
63
|
+
return { sourcesRead: 0, filtersMatched: 0, notified: 0 };
|
|
64
|
+
}
|
|
65
|
+
this.running = true;
|
|
66
|
+
try {
|
|
67
|
+
return await this.runTick();
|
|
68
|
+
}
|
|
69
|
+
finally {
|
|
70
|
+
this.running = false;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async runTick() {
|
|
74
|
+
const { subscriptions, filterResolver, sourceStore, gatewayResolver, logger } = this.options;
|
|
75
|
+
const armed = (await subscriptions.list()).filter((s) => s.status === "armed");
|
|
76
|
+
const summary = { sourcesRead: 0, filtersMatched: 0, notified: 0 };
|
|
77
|
+
if (armed.length === 0) {
|
|
78
|
+
return summary;
|
|
79
|
+
}
|
|
80
|
+
// Resolve the distinct filters once. Several subscribers commonly share one.
|
|
81
|
+
const filters = new Map();
|
|
82
|
+
for (const subscription of armed) {
|
|
83
|
+
if (filters.has(subscription.filterId))
|
|
84
|
+
continue;
|
|
85
|
+
const filter = await filterResolver(subscription.filterId);
|
|
86
|
+
if (!filter) {
|
|
87
|
+
logger.warn({ subscriptionId: subscription.id, filterId: subscription.filterId }, "Subscription points at a filter that no longer exists");
|
|
88
|
+
continue;
|
|
89
|
+
}
|
|
90
|
+
if (filter.status !== "armed")
|
|
91
|
+
continue;
|
|
92
|
+
filters.set(filter.id, filter);
|
|
93
|
+
}
|
|
94
|
+
// Group by source: this is the line that turns N reads into one.
|
|
95
|
+
const bySource = new Map();
|
|
96
|
+
for (const filter of filters.values()) {
|
|
97
|
+
const list = bySource.get(filter.sourceId);
|
|
98
|
+
if (list)
|
|
99
|
+
list.push(filter);
|
|
100
|
+
else
|
|
101
|
+
bySource.set(filter.sourceId, [filter]);
|
|
102
|
+
}
|
|
103
|
+
const now = this.now();
|
|
104
|
+
for (const [sourceId, sourceFilters] of bySource) {
|
|
105
|
+
const source = await sourceStore.get(sourceId);
|
|
106
|
+
if (!source) {
|
|
107
|
+
logger.warn({ sourceId }, "Subscribed filter points at a source that no longer exists");
|
|
108
|
+
continue;
|
|
109
|
+
}
|
|
110
|
+
let gateway;
|
|
111
|
+
try {
|
|
112
|
+
gateway = gatewayResolver(source.kind);
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
logger.warn({ err: error, sourceId, kind: source.kind }, "No gateway for source kind");
|
|
116
|
+
continue;
|
|
117
|
+
}
|
|
118
|
+
for (const filter of sourceFilters) {
|
|
119
|
+
try {
|
|
120
|
+
const { matched } = await scanMatchedSourceItems({
|
|
121
|
+
filter,
|
|
122
|
+
gateway,
|
|
123
|
+
externalUserId: source.externalUserId,
|
|
124
|
+
accountId: source.externalAccountId ?? "",
|
|
125
|
+
// Freshness is the pending store's job, keyed per subscriber, so
|
|
126
|
+
// this read deliberately answers "already handled?" with null for
|
|
127
|
+
// everything. Asking the shared ledger here would be a second,
|
|
128
|
+
// filter-wide answer to a question that is per subscription.
|
|
129
|
+
ledger: { get: () => Promise.resolve(null) },
|
|
130
|
+
sinceMs: windowStartFor(now, POLL_WINDOW_DAYS),
|
|
131
|
+
now: () => now,
|
|
132
|
+
});
|
|
133
|
+
summary.filtersMatched += 1;
|
|
134
|
+
if (matched.length === 0)
|
|
135
|
+
continue;
|
|
136
|
+
const results = await this.options.notifier.notifyForFilter(filter, matched);
|
|
137
|
+
summary.notified += results.filter((result) => result.outcome === "sent").length;
|
|
138
|
+
}
|
|
139
|
+
catch (error) {
|
|
140
|
+
// A failing filter must not abandon its siblings on the same read.
|
|
141
|
+
logger.warn({ err: error, filterId: filter.id, sourceId }, "Subscription scan failed");
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
summary.sourcesRead += 1;
|
|
145
|
+
}
|
|
146
|
+
const drained = await this.options.notifier.deliverAllPending();
|
|
147
|
+
summary.notified += drained.filter((result) => result.outcome === "sent").length;
|
|
148
|
+
// Retention runs on the TICK, not only at boot. Sweeping only at startup
|
|
149
|
+
// makes real retention `max(uptime, ttl)`, so a daemon up for forty days
|
|
150
|
+
// holds forty days of payloads while the docstring promises seven. That
|
|
151
|
+
// TTL is one of the three things containing a credential at rest, so it
|
|
152
|
+
// cannot depend on how often the process restarts. `IngestionService` does
|
|
153
|
+
// the same on its own health timer.
|
|
154
|
+
await this.sweep().catch((error) => {
|
|
155
|
+
this.options.logger.warn({ err: error }, "Subscription payload sweep failed");
|
|
156
|
+
});
|
|
157
|
+
return summary;
|
|
158
|
+
}
|
|
159
|
+
/** Delete pending payloads past their TTL. Called at boot, beside the other sweeps. */
|
|
160
|
+
async sweep() {
|
|
161
|
+
const removed = await this.options.pending.sweepExpired(this.now());
|
|
162
|
+
if (removed > 0) {
|
|
163
|
+
this.options.logger.info({ removed }, "Swept expired subscription payloads");
|
|
164
|
+
}
|
|
165
|
+
return removed;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
//# sourceMappingURL=poller.js.map
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { StoredSubscription } from "@hyperdrive.bot/fleet-protocol/ingestion/subscription-types";
|
|
2
|
+
import type { PendingItem, PendingItemStore } from "./pending-store.js";
|
|
3
|
+
import type { SubscriptionStore } from "./store.js";
|
|
4
|
+
/**
|
|
5
|
+
* The READ half of subscriptions, and the only half a model ever holds.
|
|
6
|
+
*
|
|
7
|
+
* Deliberately not the notifier and not the stores themselves. A tool catalog
|
|
8
|
+
* that could arm a subscription, or trigger a delivery, would put the delivery
|
|
9
|
+
* policy inside a model's reach; this interface cannot do either. It exists as
|
|
10
|
+
* an interface rather than a concrete class so the tool layer depends on three
|
|
11
|
+
* methods instead of on the ingestion module.
|
|
12
|
+
*/
|
|
13
|
+
export interface SubscriptionReader {
|
|
14
|
+
/**
|
|
15
|
+
* Subscriptions the CALLER owns, never the daemon's whole set.
|
|
16
|
+
*
|
|
17
|
+
* `callerAgentId` is not decoration. Without it this is an enumeration step:
|
|
18
|
+
* one session lists every subscription on the box, reads another session's
|
|
19
|
+
* agent id, and then reads that session's stored payloads, credentials
|
|
20
|
+
* included. "A local MCP tool" means every agent on this daemon, which is
|
|
21
|
+
* not the same thing as the owner.
|
|
22
|
+
*/
|
|
23
|
+
list(callerAgentId: string | undefined): Promise<StoredSubscription[]>;
|
|
24
|
+
get(subscriptionId: string, callerAgentId: string | undefined): Promise<StoredSubscription | null>;
|
|
25
|
+
/**
|
|
26
|
+
* Stored items for the given keys, payload included.
|
|
27
|
+
*
|
|
28
|
+
* This is the ONLY path by which a persisted payload leaves the daemon, and
|
|
29
|
+
* it is a local MCP call: no client RPC exposes it. Keys that are not held
|
|
30
|
+
* are simply absent from the result, the same contract `resolveMatchedItems`
|
|
31
|
+
* already has, so a caller sees a shorter list rather than an exception.
|
|
32
|
+
*/
|
|
33
|
+
resolveItems(subscriptionId: string, itemKeys: readonly string[], callerAgentId: string | undefined): Promise<PendingItem[]>;
|
|
34
|
+
}
|
|
35
|
+
export declare function createSubscriptionReader(deps: {
|
|
36
|
+
subscriptions: SubscriptionStore;
|
|
37
|
+
pending: PendingItemStore;
|
|
38
|
+
}): SubscriptionReader;
|
|
39
|
+
//# sourceMappingURL=reader.d.ts.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export function createSubscriptionReader(deps) {
|
|
2
|
+
/**
|
|
3
|
+
* Fails CLOSED when the caller is unknown.
|
|
4
|
+
*
|
|
5
|
+
* An absent `callerAgentId` means the tool catalog was built without a
|
|
6
|
+
* caller identity, and the safe reading of "I do not know who is asking" is
|
|
7
|
+
* "you own nothing", not "you own everything".
|
|
8
|
+
*/
|
|
9
|
+
const owns = (subscription, callerAgentId) => callerAgentId !== undefined && subscription.agentId === callerAgentId;
|
|
10
|
+
return {
|
|
11
|
+
list: async (callerAgentId) => (await deps.subscriptions.list()).filter((subscription) => owns(subscription, callerAgentId)),
|
|
12
|
+
get: async (subscriptionId, callerAgentId) => {
|
|
13
|
+
const subscription = await deps.subscriptions.get(subscriptionId);
|
|
14
|
+
if (!subscription || !owns(subscription, callerAgentId))
|
|
15
|
+
return null;
|
|
16
|
+
return subscription;
|
|
17
|
+
},
|
|
18
|
+
resolveItems: async (subscriptionId, itemKeys, callerAgentId) => {
|
|
19
|
+
// A subscription that no longer exists, or that belongs to someone else,
|
|
20
|
+
// resolves to nothing. Both answer the same way on purpose: a caller
|
|
21
|
+
// must not be able to tell "no such subscription" from "not yours" and
|
|
22
|
+
// use the difference to enumerate.
|
|
23
|
+
const subscription = await deps.subscriptions.get(subscriptionId);
|
|
24
|
+
if (!subscription || !owns(subscription, callerAgentId))
|
|
25
|
+
return [];
|
|
26
|
+
return deps.pending.resolve(subscriptionId, itemKeys);
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=reader.js.map
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { type StoredSubscription } from "@hyperdrive.bot/fleet-protocol/ingestion/subscription-types";
|
|
2
|
+
/**
|
|
3
|
+
* File-backed persistence for session subscriptions, one JSON file per
|
|
4
|
+
* subscription under `dir` (the caller supplies `join(paseoHome, "subscriptions")`).
|
|
5
|
+
*
|
|
6
|
+
* Structurally identical to `SourceStore` and `ScheduleStore`, deliberately: a
|
|
7
|
+
* third shape here would be a third set of atomicity and parse bugs to find.
|
|
8
|
+
*
|
|
9
|
+
* `StoredSubscriptionSchema` is strict and `list()` is readdir + Promise.all, so
|
|
10
|
+
* ONE unparseable file rejects the whole array. That is the same trade the
|
|
11
|
+
* source store makes, and it carries the same obligation: every field added to
|
|
12
|
+
* the schema later must carry `.default(...)`, or records written before it
|
|
13
|
+
* existed stop parsing and every subscription silently disappears. There is no
|
|
14
|
+
* migration step. `accountLabel` on `StoredSourceSchema` is the precedent.
|
|
15
|
+
*/
|
|
16
|
+
export declare class SubscriptionStore {
|
|
17
|
+
private readonly dir;
|
|
18
|
+
constructor(dir: string);
|
|
19
|
+
private filePath;
|
|
20
|
+
private ensureDir;
|
|
21
|
+
list(): Promise<StoredSubscription[]>;
|
|
22
|
+
get(id: string): Promise<StoredSubscription | null>;
|
|
23
|
+
/**
|
|
24
|
+
* Every armed subscription pointing at `filterId`.
|
|
25
|
+
*
|
|
26
|
+
* The fan-out read. A filter with three armed subscribers returns three
|
|
27
|
+
* records here, and each one is notified on its own budget: this is the
|
|
28
|
+
* function that makes one filter serve many sessions.
|
|
29
|
+
*/
|
|
30
|
+
listArmedForFilter(filterId: string): Promise<StoredSubscription[]>;
|
|
31
|
+
create(subscription: Omit<StoredSubscription, "id">): Promise<StoredSubscription>;
|
|
32
|
+
put(subscription: StoredSubscription): Promise<void>;
|
|
33
|
+
delete(id: string): Promise<void>;
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=store.d.ts.map
|