@hyperdrive.bot/fleet-server 0.3.164 → 0.3.165
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/extensions/daemon-backend.js +25 -0
- package/dist/server/server/agent/agent-storage.d.ts +67 -12
- package/dist/server/server/agent/agent-storage.js +208 -132
- package/dist/server/server/agent/providers/claude/transport/pty-query.js +10 -0
- package/dist/server/server/bootstrap.d.ts +5 -0
- package/dist/server/server/bootstrap.js +26 -3
- package/dist/server/server/chat/chat-service.d.ts +22 -0
- package/dist/server/server/chat/chat-service.js +46 -3
- package/dist/server/server/daemon-worker.js +1 -0
- package/dist/server/server/fleet/decision-service.d.ts +5 -0
- package/dist/server/server/fleet/decision-service.js +30 -0
- package/dist/server/server/loop-service.d.ts +95 -0
- package/dist/server/server/loop-service.js +64 -3
- package/dist/server/server/migrations/backfill-workspace-id.migration.js +3 -2
- package/dist/server/server/push/token-store.d.ts +5 -1
- package/dist/server/server/push/token-store.js +26 -5
- package/dist/server/server/schedule/service.js +4 -1
- package/dist/server/server/schedule/store.d.ts +21 -1
- package/dist/server/server/schedule/store.js +283 -2
- package/dist/server/server/search/indexer.d.ts +10 -0
- package/dist/server/server/search/indexer.js +51 -2
- package/dist/server/server/session.js +12 -13
- package/dist/server/server/state/agent-state.d.ts +39 -0
- package/dist/server/server/state/agent-state.js +94 -0
- package/dist/server/server/state/daemon-state.d.ts +30 -0
- package/dist/server/server/state/daemon-state.js +99 -0
- package/dist/server/server/state/legacy-import.d.ts +124 -0
- package/dist/server/server/state/legacy-import.js +682 -0
- package/dist/server/server/state/legacy-manifest.d.ts +73 -0
- package/dist/server/server/state/legacy-manifest.js +112 -0
- package/dist/server/server/state/legacy-mirror.d.ts +57 -0
- package/dist/server/server/state/legacy-mirror.js +114 -0
- package/dist/server/server/state/legacy-sources.d.ts +9 -0
- package/dist/server/server/state/legacy-sources.js +163 -0
- package/dist/server/server/state/state-db.d.ts +263 -0
- package/dist/server/server/state/state-db.js +772 -0
- package/dist/server/server/state/state-schema.d.ts +39 -0
- package/dist/server/server/state/state-schema.js +125 -0
- package/dist/server/server/state/state-worker.d.ts +48 -0
- package/dist/server/server/state/state-worker.js +165 -0
- package/dist/server/server/websocket-server.js +2 -1
- package/dist/server/server/workspace/session-openable.js +7 -0
- package/dist/server/server/workspace-reconciliation-service.js +14 -3
- package/dist/server/server/workspace-registry.d.ts +45 -2
- package/dist/server/server/workspace-registry.js +75 -4
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js → index-38580ed8926a5ddc569d744b8299d339.js} +4 -4
- package/dist/server/web-ui/_expo/static/js/web/index-38580ed8926a5ddc569d744b8299d339.js.br +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js.gz → index-38580ed8926a5ddc569d744b8299d339.js.gz} +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js.map.br → index-38580ed8926a5ddc569d744b8299d339.js.map.br} +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js.map.gz → index-38580ed8926a5ddc569d744b8299d339.js.map.gz} +0 -0
- package/dist/server/web-ui/index.html +1 -1
- package/dist/server/web-ui/index.html.br +0 -0
- package/dist/server/web-ui/index.html.gz +0 -0
- package/package.json +6 -6
- package/dist/server/web-ui/_expo/static/js/web/index-2ac1a7249c20c322a1f9a54563cfee62.js.br +0 -0
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
2
|
import { mkdir, readFile, readdir, rm, stat } from "node:fs/promises";
|
|
3
|
-
import { join } from "node:path";
|
|
3
|
+
import { basename, join } from "node:path";
|
|
4
4
|
import { StoredScheduleSchema, } from "@hyperdrive.bot/fleet-protocol/schedule/types";
|
|
5
5
|
import { writeJsonFileAtomic } from "../atomic-file.js";
|
|
6
|
+
import { ScopeSync } from "../state/state-db.js";
|
|
6
7
|
/**
|
|
7
8
|
* How many runs a schedule file keeps. Every run start and finish rewrites the
|
|
8
9
|
* whole file, and the history grew unbounded (5,510 runs, 2.3 MB in one real
|
|
@@ -73,12 +74,15 @@ function generateScheduleId() {
|
|
|
73
74
|
* changes back through `put`.
|
|
74
75
|
*/
|
|
75
76
|
export class ScheduleStore {
|
|
76
|
-
constructor(dir) {
|
|
77
|
+
constructor(dir, options = {}) {
|
|
77
78
|
this.dir = dir;
|
|
78
79
|
this.cache = new Map();
|
|
79
80
|
this.stamps = new Map();
|
|
80
81
|
this.validating = null;
|
|
81
82
|
this.revision = 0;
|
|
83
|
+
this.sqlite = options.stateDb
|
|
84
|
+
? new SqliteScheduleStore(options.stateDb, dir, options.logger)
|
|
85
|
+
: null;
|
|
82
86
|
}
|
|
83
87
|
filePath(id) {
|
|
84
88
|
return join(this.dir, `${id}.json`);
|
|
@@ -143,10 +147,14 @@ export class ScheduleStore {
|
|
|
143
147
|
}
|
|
144
148
|
}
|
|
145
149
|
async list() {
|
|
150
|
+
if (this.sqlite)
|
|
151
|
+
return this.sqlite.list();
|
|
146
152
|
await this.validate();
|
|
147
153
|
return [...this.cache.values()].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
|
|
148
154
|
}
|
|
149
155
|
async get(id) {
|
|
156
|
+
if (this.sqlite)
|
|
157
|
+
return this.sqlite.get(id);
|
|
150
158
|
await this.validate();
|
|
151
159
|
return this.cache.get(id) ?? null;
|
|
152
160
|
}
|
|
@@ -186,6 +194,8 @@ export class ScheduleStore {
|
|
|
186
194
|
return created;
|
|
187
195
|
}
|
|
188
196
|
async put(schedule) {
|
|
197
|
+
if (this.sqlite)
|
|
198
|
+
return this.sqlite.put(schedule);
|
|
189
199
|
await this.ensureDir();
|
|
190
200
|
const name = `${schedule.id}.json`;
|
|
191
201
|
const path = this.filePath(schedule.id);
|
|
@@ -207,6 +217,8 @@ export class ScheduleStore {
|
|
|
207
217
|
this.revision += 1;
|
|
208
218
|
}
|
|
209
219
|
async delete(id) {
|
|
220
|
+
if (this.sqlite)
|
|
221
|
+
return this.sqlite.delete(id);
|
|
210
222
|
await this.ensureDir();
|
|
211
223
|
await rm(this.filePath(id), { force: true });
|
|
212
224
|
this.cache.delete(id);
|
|
@@ -218,8 +230,277 @@ export class ScheduleStore {
|
|
|
218
230
|
* put, a delete), so a caller can reuse anything it derived from `list()`
|
|
219
231
|
* until the store actually changes.
|
|
220
232
|
*/
|
|
233
|
+
get version() {
|
|
234
|
+
return this.sqlite ? this.sqlite.version : this.revision;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
const MANIFEST_STORE = "schedules";
|
|
238
|
+
/** Upper bound on how stale an in-place edit of a schedule file can go unseen. */
|
|
239
|
+
export const EXTERNAL_EDIT_RECHECK_MS = 1000;
|
|
240
|
+
function runProgress(run) {
|
|
241
|
+
// A finished run beats a running one; between two of the same kind, the later.
|
|
242
|
+
return (run.status === "running" ? 0 : 1e15) + Date.parse(run.endedAt ?? run.startedAt);
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Merge a schedule file written by someone else into the daemon's copy.
|
|
246
|
+
*
|
|
247
|
+
* Top-level fields come from whichever side has the newer `updatedAt`, as one
|
|
248
|
+
* unit, so a field the newer side cleared is not resurrected by the older one.
|
|
249
|
+
* Runs merge by id: a run on one side only is kept (the cap drops the oldest
|
|
250
|
+
* afterwards, so runs the daemon trimmed stay trimmed), and for the same id
|
|
251
|
+
* the more advanced run wins, so a completion written on either side is never
|
|
252
|
+
* replaced by the other side's "running" copy.
|
|
253
|
+
*/
|
|
254
|
+
export function mergeExternalSchedule(daemon, file) {
|
|
255
|
+
const newer = Date.parse(file.updatedAt) >= Date.parse(daemon.updatedAt) ? file : daemon;
|
|
256
|
+
const byId = new Map();
|
|
257
|
+
for (const run of [...daemon.runs, ...file.runs]) {
|
|
258
|
+
const seen = byId.get(run.id);
|
|
259
|
+
if (!seen || runProgress(run) > runProgress(seen)) {
|
|
260
|
+
byId.set(run.id, run);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
const runs = [...byId.values()].sort((left, right) => left.startedAt.localeCompare(right.startedAt));
|
|
264
|
+
return { ...newer, runs };
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Schedules in `state.sqlite`, with the same public surface as the file store
|
|
268
|
+
* (which delegates here when the daemon has a state database). A run start or
|
|
269
|
+
* finish used to rewrite the whole
|
|
270
|
+
* file (2.3 MB for the busiest real schedule); here it writes the one run row
|
|
271
|
+
* that changed. The legacy file is still written, debounced, by the mirror.
|
|
272
|
+
*
|
|
273
|
+
* The directory stays an input as well: a file whose content no longer matches
|
|
274
|
+
* what this daemon last wrote (the legacy manifest) is read back in and merged
|
|
275
|
+
* (`takeExternalEdits`), and a file someone deleted deletes the schedule. That
|
|
276
|
+
* keeps the contract that a schedule hand-written into schedules/ is picked up
|
|
277
|
+
* by a running daemon. The mirror re-checks the file right before overwriting
|
|
278
|
+
* it, so a hand edit made while a write was pending is merged, not lost.
|
|
279
|
+
*/
|
|
280
|
+
class SqliteScheduleStore {
|
|
281
|
+
constructor(state, dir, logger) {
|
|
282
|
+
this.state = state;
|
|
283
|
+
this.dir = dir;
|
|
284
|
+
this.logger = logger;
|
|
285
|
+
this.cache = null;
|
|
286
|
+
this.loading = null;
|
|
287
|
+
this.revision = 0;
|
|
288
|
+
this.dirMtime = null;
|
|
289
|
+
this.lastFullCheck = 0;
|
|
290
|
+
this.schedules = state.doc("schedules");
|
|
291
|
+
this.runs = new ScopeSync(state.doc("schedule_runs"));
|
|
292
|
+
}
|
|
221
293
|
get version() {
|
|
222
294
|
return this.revision;
|
|
223
295
|
}
|
|
296
|
+
filePath(id) {
|
|
297
|
+
return join(this.dir, `${id}.json`);
|
|
298
|
+
}
|
|
299
|
+
async load() {
|
|
300
|
+
if (this.cache) {
|
|
301
|
+
const cache = this.cache;
|
|
302
|
+
const edited = await this.takeExternalEdits((id) => cache.get(id) ?? null);
|
|
303
|
+
for (const schedule of edited) {
|
|
304
|
+
cache.set(schedule.id, schedule);
|
|
305
|
+
}
|
|
306
|
+
if (edited.length > 0) {
|
|
307
|
+
this.revision += 1;
|
|
308
|
+
}
|
|
309
|
+
return cache;
|
|
310
|
+
}
|
|
311
|
+
this.loading ?? (this.loading = this.loadAll().finally(() => {
|
|
312
|
+
this.loading = null;
|
|
313
|
+
}));
|
|
314
|
+
return this.loading;
|
|
315
|
+
}
|
|
316
|
+
async list() {
|
|
317
|
+
const cache = await this.load();
|
|
318
|
+
return [...cache.values()].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
|
|
319
|
+
}
|
|
320
|
+
async get(id) {
|
|
321
|
+
return (await this.load()).get(id) ?? null;
|
|
322
|
+
}
|
|
323
|
+
async loadAll() {
|
|
324
|
+
await mkdir(this.dir, { recursive: true });
|
|
325
|
+
const runRows = this.state.doc("schedule_runs").loadAll();
|
|
326
|
+
this.runs.seed(runRows);
|
|
327
|
+
const runsBySchedule = new Map();
|
|
328
|
+
for (const row of runRows) {
|
|
329
|
+
const list = runsBySchedule.get(row.scope) ?? [];
|
|
330
|
+
list.push(JSON.parse(row.body));
|
|
331
|
+
runsBySchedule.set(row.scope, list);
|
|
332
|
+
}
|
|
333
|
+
const maxRunRev = new Map();
|
|
334
|
+
for (const row of runRows) {
|
|
335
|
+
maxRunRev.set(row.scope, Math.max(maxRunRev.get(row.scope) ?? 0, row.rev));
|
|
336
|
+
}
|
|
337
|
+
const loaded = new Map();
|
|
338
|
+
const behind = [];
|
|
339
|
+
const entries = this.state.manifest.entries(MANIFEST_STORE);
|
|
340
|
+
for (const row of this.schedules.loadAll("")) {
|
|
341
|
+
const schedule = JSON.parse(row.body);
|
|
342
|
+
schedule.runs = runsBySchedule.get(row.id) ?? [];
|
|
343
|
+
loaded.set(schedule.id, schedule);
|
|
344
|
+
// The process died before the mirror caught this schedule's file up.
|
|
345
|
+
const mirrored = entries.get(this.filePath(row.id))?.rev ?? -1;
|
|
346
|
+
if (Math.max(row.rev, maxRunRev.get(row.id) ?? 0) > mirrored)
|
|
347
|
+
behind.push(schedule);
|
|
348
|
+
}
|
|
349
|
+
this.cache = loaded;
|
|
350
|
+
for (const schedule of await this.takeExternalEdits((id) => loaded.get(id) ?? null)) {
|
|
351
|
+
loaded.set(schedule.id, schedule);
|
|
352
|
+
}
|
|
353
|
+
for (const schedule of behind) {
|
|
354
|
+
if (loaded.get(schedule.id) === schedule)
|
|
355
|
+
this.mirrorWrite(schedule);
|
|
356
|
+
}
|
|
357
|
+
this.revision += 1;
|
|
358
|
+
return loaded;
|
|
359
|
+
}
|
|
360
|
+
async put(input) {
|
|
361
|
+
const cache = await this.load();
|
|
362
|
+
const schedule = capScheduleRuns(input);
|
|
363
|
+
const { runs, ...rest } = schedule;
|
|
364
|
+
await Promise.all([
|
|
365
|
+
this.schedules.put("", schedule.id, rest),
|
|
366
|
+
this.runs.sync(schedule.id, runs.map((run) => ({ id: run.id, value: run })), { prune: true }),
|
|
367
|
+
]);
|
|
368
|
+
cache.set(schedule.id, schedule);
|
|
369
|
+
this.revision += 1;
|
|
370
|
+
this.mirrorWrite(schedule);
|
|
371
|
+
}
|
|
372
|
+
async delete(id) {
|
|
373
|
+
const cache = await this.load();
|
|
374
|
+
await Promise.all([this.schedules.remove("", id), this.runs.dropScope(id)]);
|
|
375
|
+
cache.delete(id);
|
|
376
|
+
this.revision += 1;
|
|
377
|
+
this.state.mirror.write(`schedule:${id}`, async () => {
|
|
378
|
+
await rm(this.filePath(id), { force: true });
|
|
379
|
+
await this.state.manifest.forget(this.filePath(id));
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
async takeExternalEdits(current) {
|
|
383
|
+
let dirMtime;
|
|
384
|
+
try {
|
|
385
|
+
dirMtime = (await stat(this.dir)).mtimeMs;
|
|
386
|
+
}
|
|
387
|
+
catch {
|
|
388
|
+
return [];
|
|
389
|
+
}
|
|
390
|
+
// A new or renamed file bumps the directory mtime and is read at once. An
|
|
391
|
+
// in-place edit does not, so the per-file stats also run at most once per
|
|
392
|
+
// EXTERNAL_EDIT_RECHECK_MS; between those a read costs one stat.
|
|
393
|
+
const now = Date.now();
|
|
394
|
+
if (dirMtime === this.dirMtime && now - this.lastFullCheck < EXTERNAL_EDIT_RECHECK_MS) {
|
|
395
|
+
return [];
|
|
396
|
+
}
|
|
397
|
+
this.dirMtime = dirMtime;
|
|
398
|
+
this.lastFullCheck = now;
|
|
399
|
+
const edited = [];
|
|
400
|
+
const entries = this.state.manifest.entries(MANIFEST_STORE);
|
|
401
|
+
const ids = new Set();
|
|
402
|
+
for (const entry of await readdir(this.dir, { withFileTypes: true })) {
|
|
403
|
+
if (!entry.isFile() || !entry.name.endsWith(".json") || entry.name.startsWith(".")) {
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
ids.add(entry.name.slice(0, -".json".length));
|
|
407
|
+
}
|
|
408
|
+
for (const entryPath of entries.keys()) {
|
|
409
|
+
ids.add(basename(entryPath, ".json"));
|
|
410
|
+
}
|
|
411
|
+
for (const id of ids) {
|
|
412
|
+
// Our own write in flight: the manifest catches up when it lands.
|
|
413
|
+
if (this.state.mirror.isRunning(`schedule:${id}`))
|
|
414
|
+
continue;
|
|
415
|
+
const path = this.filePath(id);
|
|
416
|
+
try {
|
|
417
|
+
const file = await this.state.manifest.classify(path, entries.get(path) ?? null);
|
|
418
|
+
if (file.status === "deleted") {
|
|
419
|
+
// Deleted by someone else: a deletion wins, so an active schedule is
|
|
420
|
+
// never brought back.
|
|
421
|
+
if (current(id)) {
|
|
422
|
+
await Promise.all([this.schedules.remove("", id), this.runs.dropScope(id)]);
|
|
423
|
+
this.cache?.delete(id);
|
|
424
|
+
this.revision += 1;
|
|
425
|
+
}
|
|
426
|
+
await this.state.manifest.forget(path);
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
if (file.status !== "changed" && file.status !== "created")
|
|
430
|
+
continue;
|
|
431
|
+
const schedule = await this.adoptForeign(id, file.content, current(id));
|
|
432
|
+
edited.push(schedule);
|
|
433
|
+
}
|
|
434
|
+
catch (error) {
|
|
435
|
+
// A half-written or invalid file is retried when it changes next; a
|
|
436
|
+
// failed commit is worth knowing about.
|
|
437
|
+
this.logger?.warn({ err: error, path }, "Failed to read back an externally edited schedule");
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
return edited;
|
|
441
|
+
}
|
|
442
|
+
/**
|
|
443
|
+
* Merge a file someone else wrote into the daemon's copy, commit it, record
|
|
444
|
+
* the file as seen, and queue the merged result back out when it differs.
|
|
445
|
+
*/
|
|
446
|
+
async adoptForeign(id, content, existing) {
|
|
447
|
+
const fromFile = StoredScheduleSchema.parse(JSON.parse(content));
|
|
448
|
+
const schedule = capScheduleRuns(existing ? mergeExternalSchedule(existing, fromFile) : fromFile);
|
|
449
|
+
const { runs, ...rest } = schedule;
|
|
450
|
+
await Promise.all([
|
|
451
|
+
this.schedules.put("", schedule.id, rest),
|
|
452
|
+
this.runs.sync(schedule.id, runs.map((run) => ({ id: run.id, value: run })), { prune: true }),
|
|
453
|
+
]);
|
|
454
|
+
this.cache?.set(schedule.id, schedule);
|
|
455
|
+
this.revision += 1;
|
|
456
|
+
await this.state.manifest.record(MANIFEST_STORE, this.filePath(id), this.state.mirrorRev());
|
|
457
|
+
if (existing) {
|
|
458
|
+
// The merge may differ from the file: write the result back out.
|
|
459
|
+
this.mirrorWrite(schedule);
|
|
460
|
+
}
|
|
461
|
+
return schedule;
|
|
462
|
+
}
|
|
463
|
+
mirrorWrite(schedule) {
|
|
464
|
+
const id = schedule.id;
|
|
465
|
+
this.state.mirror.write(`schedule:${id}`, async () => {
|
|
466
|
+
const path = this.filePath(id);
|
|
467
|
+
// Re-check right before overwriting: a hand edit (or delete) made while
|
|
468
|
+
// this write was pending is merged in first, never replaced.
|
|
469
|
+
const file = await this.state.manifest.classify(path);
|
|
470
|
+
let latest = this.cache?.get(id) ?? schedule;
|
|
471
|
+
if (file.status === "deleted") {
|
|
472
|
+
if (this.cache?.has(id)) {
|
|
473
|
+
await Promise.all([this.schedules.remove("", id), this.runs.dropScope(id)]);
|
|
474
|
+
this.cache.delete(id);
|
|
475
|
+
this.revision += 1;
|
|
476
|
+
}
|
|
477
|
+
await this.state.manifest.forget(path);
|
|
478
|
+
return;
|
|
479
|
+
}
|
|
480
|
+
if (file.status === "changed" || file.status === "created") {
|
|
481
|
+
try {
|
|
482
|
+
const fromFile = StoredScheduleSchema.parse(JSON.parse(file.content));
|
|
483
|
+
latest = capScheduleRuns(mergeExternalSchedule(latest, fromFile));
|
|
484
|
+
const { runs, ...rest } = latest;
|
|
485
|
+
await Promise.all([
|
|
486
|
+
this.schedules.put("", id, rest),
|
|
487
|
+
this.runs.sync(id, runs.map((run) => ({ id: run.id, value: run })), { prune: true }),
|
|
488
|
+
]);
|
|
489
|
+
this.cache?.set(id, latest);
|
|
490
|
+
this.revision += 1;
|
|
491
|
+
}
|
|
492
|
+
catch (error) {
|
|
493
|
+
this.logger?.warn({ err: error, path }, "Schedule file edited by hand is unreadable; overwriting it");
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
else if (!this.cache?.has(id)) {
|
|
497
|
+
return;
|
|
498
|
+
}
|
|
499
|
+
const rev = this.state.mirrorRev();
|
|
500
|
+
await writeJsonFileAtomic(path, latest);
|
|
501
|
+
await this.state.manifest.record(MANIFEST_STORE, path, rev);
|
|
502
|
+
this.dirMtime = null;
|
|
503
|
+
});
|
|
504
|
+
}
|
|
224
505
|
}
|
|
225
506
|
//# sourceMappingURL=store.js.map
|
|
@@ -26,6 +26,16 @@ export interface SourceAdapter {
|
|
|
26
26
|
discover(): Promise<DiscoveredFile[]>;
|
|
27
27
|
/** Parse one raw record into extractable blocks. */
|
|
28
28
|
extract(record: unknown): ExtractedRecord;
|
|
29
|
+
/**
|
|
30
|
+
* Whole-document sources not kept in files (paseo agents in state.sqlite):
|
|
31
|
+
* the document for a discovered entry, with a size and a version that
|
|
32
|
+
* changes whenever the document does. Null when it is gone.
|
|
33
|
+
*/
|
|
34
|
+
readDocument?(file: DiscoveredFile): Promise<{
|
|
35
|
+
size: number;
|
|
36
|
+
mtimeMs: number;
|
|
37
|
+
content: string;
|
|
38
|
+
} | null>;
|
|
29
39
|
}
|
|
30
40
|
export declare function createClaudeAdapter(root?: string): SourceAdapter;
|
|
31
41
|
export declare function createCodexAdapter(root?: string): SourceAdapter;
|
|
@@ -26,6 +26,8 @@ import fs from "node:fs/promises";
|
|
|
26
26
|
import os from "node:os";
|
|
27
27
|
import path from "node:path";
|
|
28
28
|
import { resolvePaseoHome } from "../paseo-home.js";
|
|
29
|
+
import { agentRecordPath } from "../agent/agent-storage.js";
|
|
30
|
+
import { stateDbFor } from "../state/state-db.js";
|
|
29
31
|
import { extractClaudeRecord, extractCodexRecord, extractPaseoAgent, } from "./extract.js";
|
|
30
32
|
// ---------------------------------------------------------------------------
|
|
31
33
|
// Discovery helpers
|
|
@@ -133,12 +135,49 @@ export function createPaseoAdapter(root) {
|
|
|
133
135
|
// adapter indexes title + labels so agents are findable by name, and the
|
|
134
136
|
// record's runtimeInfo.sessionId is what links a hit to real transcript
|
|
135
137
|
// text held by the provider adapters.
|
|
138
|
+
async readDocument(file) {
|
|
139
|
+
const stateDb = root === undefined ? stateDbFor(resolvePaseoHome()) : null;
|
|
140
|
+
if (!stateDb) {
|
|
141
|
+
const stat = await fs.stat(file.path);
|
|
142
|
+
return {
|
|
143
|
+
size: stat.size,
|
|
144
|
+
mtimeMs: stat.mtimeMs,
|
|
145
|
+
content: await fs.readFile(file.path, "utf8"),
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
const row = stateDb
|
|
149
|
+
.prepare("SELECT body, rev FROM agents WHERE scope = '' AND id = ? AND deleted_at IS NULL")
|
|
150
|
+
.get(file.sid);
|
|
151
|
+
if (!row || typeof row.body !== "string")
|
|
152
|
+
return null;
|
|
153
|
+
// rev changes on every write (a title set keeps updatedAt), so it is the version.
|
|
154
|
+
return { size: row.body.length, mtimeMs: Number(row.rev), content: row.body };
|
|
155
|
+
},
|
|
136
156
|
extract(record) {
|
|
137
157
|
const { meta, blocks } = extractPaseoAgent(record);
|
|
138
158
|
// The agent record's own cwd is authoritative for a paseo hit.
|
|
139
159
|
return { blocks, timestamp: meta?.updatedAt ?? null, sid: null, cwd: meta?.cwd ?? null };
|
|
140
160
|
},
|
|
141
161
|
async discover() {
|
|
162
|
+
// With state.sqlite the agent files are only a lagging mirror (and
|
|
163
|
+
// absent under PASEO_STATE_FILE_MIRROR=0): index the rows. Paths stay the
|
|
164
|
+
// legacy file paths, so index keys do not change with the backend.
|
|
165
|
+
const stateDb = root === undefined ? stateDbFor(resolvePaseoHome()) : null;
|
|
166
|
+
if (stateDb) {
|
|
167
|
+
stateDb.flushSync();
|
|
168
|
+
return stateDb
|
|
169
|
+
.prepare("SELECT id, json_extract(body, '$.cwd') AS cwd FROM agents WHERE scope = '' AND deleted_at IS NULL")
|
|
170
|
+
.all()
|
|
171
|
+
.map((row) => ({
|
|
172
|
+
path: agentRecordPath(agentsRoot, {
|
|
173
|
+
id: String(row.id),
|
|
174
|
+
cwd: String(row.cwd ?? ""),
|
|
175
|
+
}),
|
|
176
|
+
source: "paseo",
|
|
177
|
+
sid: String(row.id),
|
|
178
|
+
cwd: "",
|
|
179
|
+
}));
|
|
180
|
+
}
|
|
142
181
|
const files = await walkFiles(agentsRoot, ".json");
|
|
143
182
|
return files.map((p) => ({
|
|
144
183
|
path: p,
|
|
@@ -288,8 +327,18 @@ function buildChunks(args, consumable, isWholeDoc) {
|
|
|
288
327
|
async function indexOneFile(args) {
|
|
289
328
|
const { store, adapter, file, result, maxBytes, log } = args;
|
|
290
329
|
let stat;
|
|
330
|
+
let document = null;
|
|
291
331
|
try {
|
|
292
|
-
|
|
332
|
+
if (adapter.readDocument) {
|
|
333
|
+
const read = await adapter.readDocument(file);
|
|
334
|
+
if (!read)
|
|
335
|
+
return;
|
|
336
|
+
document = read.content;
|
|
337
|
+
stat = read;
|
|
338
|
+
}
|
|
339
|
+
else {
|
|
340
|
+
stat = await fs.stat(file.path);
|
|
341
|
+
}
|
|
293
342
|
}
|
|
294
343
|
catch {
|
|
295
344
|
return;
|
|
@@ -301,7 +350,7 @@ async function indexOneFile(args) {
|
|
|
301
350
|
const end = Math.min(stat.size, start + maxBytes);
|
|
302
351
|
let raw;
|
|
303
352
|
try {
|
|
304
|
-
raw = await readRange(file.path, start, end);
|
|
353
|
+
raw = document ?? (await readRange(file.path, start, end));
|
|
305
354
|
}
|
|
306
355
|
catch (err) {
|
|
307
356
|
result.errors += 1;
|
|
@@ -6622,21 +6622,20 @@ export class Session {
|
|
|
6622
6622
|
clearedAgentIds.push(agentId);
|
|
6623
6623
|
continue;
|
|
6624
6624
|
}
|
|
6625
|
-
|
|
6626
|
-
|
|
6627
|
-
|
|
6628
|
-
|
|
6629
|
-
|
|
6625
|
+
// Inside the agent's write queue, so a concurrent title or digest
|
|
6626
|
+
// write is never dropped by this read-then-write.
|
|
6627
|
+
const nextRecord = await this.agentStorage.update(agentId, (record) => record.internal || record.archivedAt || record.requiresAttention !== true
|
|
6628
|
+
? null
|
|
6629
|
+
: {
|
|
6630
|
+
...record,
|
|
6631
|
+
updatedAt: new Date().toISOString(),
|
|
6632
|
+
requiresAttention: false,
|
|
6633
|
+
attentionReason: null,
|
|
6634
|
+
attentionTimestamp: null,
|
|
6635
|
+
});
|
|
6636
|
+
if (!nextRecord) {
|
|
6630
6637
|
continue;
|
|
6631
6638
|
}
|
|
6632
|
-
const nextRecord = {
|
|
6633
|
-
...record,
|
|
6634
|
-
updatedAt: new Date().toISOString(),
|
|
6635
|
-
requiresAttention: false,
|
|
6636
|
-
attentionReason: null,
|
|
6637
|
-
attentionTimestamp: null,
|
|
6638
|
-
};
|
|
6639
|
-
await this.agentStorage.upsert(nextRecord);
|
|
6640
6639
|
const agent = this.buildStoredAgentPayload(nextRecord);
|
|
6641
6640
|
const project = await this.buildProjectPlacementForWorkspace(workspace);
|
|
6642
6641
|
this.emit({
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { Logger } from "pino";
|
|
2
|
+
import { type AgentRecordBackend, type StoredAgentRecord } from "../agent/agent-storage.js";
|
|
3
|
+
import type { AgentMirrorCatchUp } from "./legacy-import.js";
|
|
4
|
+
import type { ChangedRow, StateDb } from "./state-db.js";
|
|
5
|
+
/**
|
|
6
|
+
* Agent records in `state.sqlite`, one row per agent, with a daemon-wide `rev`
|
|
7
|
+
* and a tombstone on delete. Replaces the boot scan of every agent JSON (14,388
|
|
8
|
+
* files on the reference host) with one indexed read, and the per-update
|
|
9
|
+
* rewrite of a whole file with one row in a batched transaction.
|
|
10
|
+
*
|
|
11
|
+
* Every write is mirrored to the legacy file, debounced and off the request
|
|
12
|
+
* path, see `LegacyMirror`, and each file written or removed is recorded in the
|
|
13
|
+
* legacy manifest with the rev it reflects. Files the boot import found lagging
|
|
14
|
+
* the database (the process died inside the debounce window) are caught up on
|
|
15
|
+
* the first load.
|
|
16
|
+
*/
|
|
17
|
+
export declare class SqliteAgentRecordBackend implements AgentRecordBackend {
|
|
18
|
+
private readonly state;
|
|
19
|
+
private readonly catchUp;
|
|
20
|
+
private readonly table;
|
|
21
|
+
private readonly files;
|
|
22
|
+
/**
|
|
23
|
+
* The record the legacy file currently holds, per agent the mirror has
|
|
24
|
+
* touched. Debounced writes collapse, so the file to replace is the one the
|
|
25
|
+
* mirror last wrote (or the record loaded at boot), not the previous record in
|
|
26
|
+
* memory, which may never have reached disk.
|
|
27
|
+
*/
|
|
28
|
+
private readonly onDisk;
|
|
29
|
+
/** Rev each running mirror write reflects, by agent id. */
|
|
30
|
+
private readonly mirrorRevs;
|
|
31
|
+
constructor(state: StateDb, baseDir: string, logger: Logger, catchUp?: AgentMirrorCatchUp);
|
|
32
|
+
loadAll(): Promise<StoredAgentRecord[]>;
|
|
33
|
+
write(record: StoredAgentRecord, previous: StoredAgentRecord | null): Promise<void>;
|
|
34
|
+
/** `rev` is the row's own rev for exactly this content (re-review R1). */
|
|
35
|
+
private mirrorWrite;
|
|
36
|
+
remove(agentId: string, previous: StoredAgentRecord | null): Promise<void>;
|
|
37
|
+
listChangedSince(rev: number, limit?: number): ChangedRow[];
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=agent-state.d.ts.map
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { FileAgentRecordBackend, } from "../agent/agent-storage.js";
|
|
4
|
+
/**
|
|
5
|
+
* Agent records in `state.sqlite`, one row per agent, with a daemon-wide `rev`
|
|
6
|
+
* and a tombstone on delete. Replaces the boot scan of every agent JSON (14,388
|
|
7
|
+
* files on the reference host) with one indexed read, and the per-update
|
|
8
|
+
* rewrite of a whole file with one row in a batched transaction.
|
|
9
|
+
*
|
|
10
|
+
* Every write is mirrored to the legacy file, debounced and off the request
|
|
11
|
+
* path, see `LegacyMirror`, and each file written or removed is recorded in the
|
|
12
|
+
* legacy manifest with the rev it reflects. Files the boot import found lagging
|
|
13
|
+
* the database (the process died inside the debounce window) are caught up on
|
|
14
|
+
* the first load.
|
|
15
|
+
*/
|
|
16
|
+
export class SqliteAgentRecordBackend {
|
|
17
|
+
constructor(state, baseDir, logger, catchUp = { rewrite: [], unlink: [] }) {
|
|
18
|
+
this.state = state;
|
|
19
|
+
this.catchUp = catchUp;
|
|
20
|
+
/**
|
|
21
|
+
* The record the legacy file currently holds, per agent the mirror has
|
|
22
|
+
* touched. Debounced writes collapse, so the file to replace is the one the
|
|
23
|
+
* mirror last wrote (or the record loaded at boot), not the previous record in
|
|
24
|
+
* memory, which may never have reached disk.
|
|
25
|
+
*/
|
|
26
|
+
this.onDisk = new Map();
|
|
27
|
+
/** Rev each running mirror write reflects, by agent id. */
|
|
28
|
+
this.mirrorRevs = new Map();
|
|
29
|
+
this.table = state.doc("agents", { tombstones: true });
|
|
30
|
+
this.files = new FileAgentRecordBackend(baseDir, logger, {
|
|
31
|
+
written: (filePath) => state.manifest.record("agents", filePath, this.mirrorRevs.get(path.basename(filePath, ".json")) ?? 0),
|
|
32
|
+
removed: (filePath) => state.manifest.forget(filePath),
|
|
33
|
+
});
|
|
34
|
+
// Every file the manifest knows, so a cwd move or a delete also cleans up
|
|
35
|
+
// copies left at older paths.
|
|
36
|
+
for (const entry of state.manifest.entries("agents").values()) {
|
|
37
|
+
this.files.indexPath(path.basename(entry.path, ".json"), entry.path);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
async loadAll() {
|
|
41
|
+
// Bodies were validated when they were written or imported; re-running zod
|
|
42
|
+
// over 14k records here would cost more than the read it replaces.
|
|
43
|
+
const rows = this.table.loadAll("");
|
|
44
|
+
const records = rows.map((row) => JSON.parse(row.body));
|
|
45
|
+
if (this.state.mirror.enabled) {
|
|
46
|
+
const byId = new Map(records.map((record) => [record.id, record]));
|
|
47
|
+
const revs = new Map(rows.map((row) => [row.id, row.rev]));
|
|
48
|
+
for (const id of this.catchUp.rewrite) {
|
|
49
|
+
const record = byId.get(id);
|
|
50
|
+
if (record)
|
|
51
|
+
this.mirrorWrite(record, revs.get(id) ?? 0);
|
|
52
|
+
}
|
|
53
|
+
for (const filePath of this.catchUp.unlink) {
|
|
54
|
+
this.state.mirror.write(`agent-file:${filePath}`, async () => {
|
|
55
|
+
await fs.rm(filePath, { force: true });
|
|
56
|
+
await this.state.manifest.forget(filePath);
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return records;
|
|
61
|
+
}
|
|
62
|
+
async write(record, previous) {
|
|
63
|
+
const rev = await this.table.putWithRev("", record.id, record);
|
|
64
|
+
if (!this.onDisk.has(record.id))
|
|
65
|
+
this.onDisk.set(record.id, previous);
|
|
66
|
+
this.mirrorWrite(record, rev);
|
|
67
|
+
}
|
|
68
|
+
/** `rev` is the row's own rev for exactly this content (re-review R1). */
|
|
69
|
+
mirrorWrite(record, rev) {
|
|
70
|
+
this.state.mirror.write(`agent:${record.id}`, async () => {
|
|
71
|
+
this.mirrorRevs.set(record.id, rev);
|
|
72
|
+
try {
|
|
73
|
+
await this.files.write(record, this.onDisk.get(record.id) ?? null);
|
|
74
|
+
}
|
|
75
|
+
finally {
|
|
76
|
+
this.mirrorRevs.delete(record.id);
|
|
77
|
+
}
|
|
78
|
+
this.onDisk.set(record.id, record);
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
async remove(agentId, previous) {
|
|
82
|
+
await this.table.remove("", agentId);
|
|
83
|
+
if (!this.onDisk.has(agentId))
|
|
84
|
+
this.onDisk.set(agentId, previous);
|
|
85
|
+
this.state.mirror.write(`agent:${agentId}`, async () => {
|
|
86
|
+
await this.files.remove(agentId, this.onDisk.get(agentId) ?? null);
|
|
87
|
+
this.onDisk.delete(agentId);
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
listChangedSince(rev, limit) {
|
|
91
|
+
return this.table.listChangedSince(rev, limit);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=agent-state.js.map
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { Logger } from "pino";
|
|
2
|
+
import type { AgentRecordBackend } from "../agent/agent-storage.js";
|
|
3
|
+
import { type StateDb } from "./state-db.js";
|
|
4
|
+
/**
|
|
5
|
+
* Free space the first import needs, as a multiple of the legacy state it
|
|
6
|
+
* copies: the database itself (about 1x on a real install) plus the WAL, which
|
|
7
|
+
* grows to about 1.4x while checkpoints are paused during the import.
|
|
8
|
+
*/
|
|
9
|
+
export declare const IMPORT_SPACE_FACTOR = 2.5;
|
|
10
|
+
export interface DaemonState {
|
|
11
|
+
/** Null when node:sqlite is unavailable, the probe failed, or PASEO_STATE_SQLITE=0. */
|
|
12
|
+
stateDb: StateDb | null;
|
|
13
|
+
/** Undefined means AgentStorage keeps its legacy file backend. */
|
|
14
|
+
agentBackend: AgentRecordBackend | undefined;
|
|
15
|
+
close(): Promise<void>;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Open `state.sqlite` for a daemon, copy the legacy stores in (first boot) or
|
|
19
|
+
* re-sync files an older daemon changed (after a rollback), and attach it for
|
|
20
|
+
* the stores constructed deeper down. See data-layer-rework.md section 5.
|
|
21
|
+
*/
|
|
22
|
+
export declare function openDaemonState(options: {
|
|
23
|
+
paseoHome: string;
|
|
24
|
+
agentStoragePath: string;
|
|
25
|
+
logger: Logger;
|
|
26
|
+
elapsed: () => unknown;
|
|
27
|
+
/** Test seam: free bytes on the PASEO_HOME filesystem. */
|
|
28
|
+
freeBytes?: () => Promise<number>;
|
|
29
|
+
}): Promise<DaemonState>;
|
|
30
|
+
//# sourceMappingURL=daemon-state.d.ts.map
|