@hyperdrive.bot/fleet-server 0.3.164 → 0.3.166

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/server/extensions/daemon-backend.js +25 -0
  2. package/dist/server/server/agent/agent-storage.d.ts +67 -12
  3. package/dist/server/server/agent/agent-storage.js +208 -132
  4. package/dist/server/server/agent/providers/claude/transport/pty-query.js +10 -0
  5. package/dist/server/server/bootstrap.d.ts +5 -0
  6. package/dist/server/server/bootstrap.js +26 -3
  7. package/dist/server/server/chat/chat-service.d.ts +22 -0
  8. package/dist/server/server/chat/chat-service.js +46 -3
  9. package/dist/server/server/daemon-worker.js +1 -0
  10. package/dist/server/server/fleet/decision-service.d.ts +5 -0
  11. package/dist/server/server/fleet/decision-service.js +30 -0
  12. package/dist/server/server/loop-service.d.ts +95 -0
  13. package/dist/server/server/loop-service.js +64 -3
  14. package/dist/server/server/migrations/backfill-workspace-id.migration.js +3 -2
  15. package/dist/server/server/push/token-store.d.ts +5 -1
  16. package/dist/server/server/push/token-store.js +26 -5
  17. package/dist/server/server/schedule/service.js +4 -1
  18. package/dist/server/server/schedule/store.d.ts +21 -1
  19. package/dist/server/server/schedule/store.js +283 -2
  20. package/dist/server/server/search/indexer.d.ts +10 -0
  21. package/dist/server/server/search/indexer.js +51 -2
  22. package/dist/server/server/session.js +12 -13
  23. package/dist/server/server/state/agent-state.d.ts +39 -0
  24. package/dist/server/server/state/agent-state.js +94 -0
  25. package/dist/server/server/state/daemon-state.d.ts +30 -0
  26. package/dist/server/server/state/daemon-state.js +99 -0
  27. package/dist/server/server/state/legacy-import.d.ts +124 -0
  28. package/dist/server/server/state/legacy-import.js +682 -0
  29. package/dist/server/server/state/legacy-manifest.d.ts +73 -0
  30. package/dist/server/server/state/legacy-manifest.js +112 -0
  31. package/dist/server/server/state/legacy-mirror.d.ts +57 -0
  32. package/dist/server/server/state/legacy-mirror.js +114 -0
  33. package/dist/server/server/state/legacy-sources.d.ts +9 -0
  34. package/dist/server/server/state/legacy-sources.js +163 -0
  35. package/dist/server/server/state/state-db.d.ts +263 -0
  36. package/dist/server/server/state/state-db.js +772 -0
  37. package/dist/server/server/state/state-schema.d.ts +39 -0
  38. package/dist/server/server/state/state-schema.js +125 -0
  39. package/dist/server/server/state/state-worker.d.ts +48 -0
  40. package/dist/server/server/state/state-worker.js +165 -0
  41. package/dist/server/server/websocket-server.js +2 -1
  42. package/dist/server/server/workspace/session-openable.js +7 -0
  43. package/dist/server/server/workspace-reconciliation-service.js +14 -3
  44. package/dist/server/server/workspace-registry.d.ts +45 -2
  45. package/dist/server/server/workspace-registry.js +75 -4
  46. package/dist/server/web-ui/_expo/static/js/web/{desktop-attachment-bridge-491bc812b0659658291d67bc77c9e697.js → desktop-attachment-bridge-eafa6217277b83ec33d4967049da7697.js} +5 -5
  47. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-eafa6217277b83ec33d4967049da7697.js.br +0 -0
  48. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-eafa6217277b83ec33d4967049da7697.js.gz +0 -0
  49. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-eafa6217277b83ec33d4967049da7697.js.map.br +0 -0
  50. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-eafa6217277b83ec33d4967049da7697.js.map.gz +0 -0
  51. package/dist/server/web-ui/_expo/static/js/web/{desktop-attachment-store-6d63da0fd45a8d1001f62f227484f116.js → desktop-attachment-store-bdcb16b0ac512f03771e9926602de68f.js} +3 -3
  52. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-bdcb16b0ac512f03771e9926602de68f.js.br +0 -0
  53. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-bdcb16b0ac512f03771e9926602de68f.js.gz +0 -0
  54. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-bdcb16b0ac512f03771e9926602de68f.js.map.br +0 -0
  55. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-bdcb16b0ac512f03771e9926602de68f.js.map.gz +0 -0
  56. package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js → index-1ff4c43fd657ba6473b02d56dae75a25.js} +1530 -1527
  57. package/dist/server/web-ui/_expo/static/js/web/index-1ff4c43fd657ba6473b02d56dae75a25.js.br +0 -0
  58. package/dist/server/web-ui/_expo/static/js/web/index-1ff4c43fd657ba6473b02d56dae75a25.js.gz +0 -0
  59. package/dist/server/web-ui/_expo/static/js/web/index-1ff4c43fd657ba6473b02d56dae75a25.js.map.br +0 -0
  60. package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js.map.gz → index-1ff4c43fd657ba6473b02d56dae75a25.js.map.gz} +0 -0
  61. package/dist/server/web-ui/_expo/static/js/web/{indexeddb-attachment-store-040c97c7cac8a3ed86c75baa2ce39d61.js → indexeddb-attachment-store-c83a6261ece654d4dfc3566e3b929e73.js} +3 -3
  62. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-c83a6261ece654d4dfc3566e3b929e73.js.br +0 -0
  63. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-c83a6261ece654d4dfc3566e3b929e73.js.gz +0 -0
  64. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-c83a6261ece654d4dfc3566e3b929e73.js.map.br +0 -0
  65. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-c83a6261ece654d4dfc3566e3b929e73.js.map.gz +0 -0
  66. package/dist/server/web-ui/_expo/static/js/web/{native-file-attachment-store-ac3bbd454eba677c4fd092227d5b5b89.js → native-file-attachment-store-ff6e2b6baeb643378628c82f177b35c9.js} +5 -5
  67. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ff6e2b6baeb643378628c82f177b35c9.js.br +0 -0
  68. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ff6e2b6baeb643378628c82f177b35c9.js.gz +0 -0
  69. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ff6e2b6baeb643378628c82f177b35c9.js.map.br +0 -0
  70. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ff6e2b6baeb643378628c82f177b35c9.js.map.gz +0 -0
  71. package/dist/server/web-ui/_expo/static/js/web/{profiler-b509d7fffd6cf457a9c12588b80560cb.js → profiler-a28d140b5874532b9eef737c77fde610.js} +11 -11
  72. package/dist/server/web-ui/_expo/static/js/web/profiler-a28d140b5874532b9eef737c77fde610.js.br +0 -0
  73. package/dist/server/web-ui/_expo/static/js/web/profiler-a28d140b5874532b9eef737c77fde610.js.gz +0 -0
  74. package/dist/server/web-ui/_expo/static/js/web/profiler-a28d140b5874532b9eef737c77fde610.js.map.br +0 -0
  75. package/dist/server/web-ui/_expo/static/js/web/profiler-a28d140b5874532b9eef737c77fde610.js.map.gz +0 -0
  76. package/dist/server/web-ui/_expo/static/js/web/{startRecording-9c90e555a5ecdc643e277f9a98913aaf.js → startRecording-f05b37fde4fc248dba4cb646dda2fc3d.js} +60 -60
  77. package/dist/server/web-ui/_expo/static/js/web/startRecording-f05b37fde4fc248dba4cb646dda2fc3d.js.br +0 -0
  78. package/dist/server/web-ui/_expo/static/js/web/startRecording-f05b37fde4fc248dba4cb646dda2fc3d.js.gz +0 -0
  79. package/dist/server/web-ui/_expo/static/js/web/startRecording-f05b37fde4fc248dba4cb646dda2fc3d.js.map.br +0 -0
  80. package/dist/server/web-ui/_expo/static/js/web/{startRecording-9c90e555a5ecdc643e277f9a98913aaf.js.map.gz → startRecording-f05b37fde4fc248dba4cb646dda2fc3d.js.map.gz} +0 -0
  81. package/dist/server/web-ui/index.html +1 -1
  82. package/dist/server/web-ui/index.html.br +0 -0
  83. package/dist/server/web-ui/index.html.gz +0 -0
  84. package/package.json +6 -6
  85. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-491bc812b0659658291d67bc77c9e697.js.br +0 -0
  86. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-491bc812b0659658291d67bc77c9e697.js.gz +0 -0
  87. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-491bc812b0659658291d67bc77c9e697.js.map.br +0 -0
  88. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-bridge-491bc812b0659658291d67bc77c9e697.js.map.gz +0 -0
  89. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-6d63da0fd45a8d1001f62f227484f116.js.br +0 -0
  90. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-6d63da0fd45a8d1001f62f227484f116.js.gz +0 -0
  91. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-6d63da0fd45a8d1001f62f227484f116.js.map.br +0 -0
  92. package/dist/server/web-ui/_expo/static/js/web/desktop-attachment-store-6d63da0fd45a8d1001f62f227484f116.js.map.gz +0 -0
  93. package/dist/server/web-ui/_expo/static/js/web/index-2ac1a7249c20c322a1f9a54563cfee62.js.br +0 -0
  94. package/dist/server/web-ui/_expo/static/js/web/index-2ac1a7249c20c322a1f9a54563cfee62.js.gz +0 -0
  95. package/dist/server/web-ui/_expo/static/js/web/index-2ac1a7249c20c322a1f9a54563cfee62.js.map.br +0 -0
  96. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-040c97c7cac8a3ed86c75baa2ce39d61.js.br +0 -0
  97. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-040c97c7cac8a3ed86c75baa2ce39d61.js.gz +0 -0
  98. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-040c97c7cac8a3ed86c75baa2ce39d61.js.map.br +0 -0
  99. package/dist/server/web-ui/_expo/static/js/web/indexeddb-attachment-store-040c97c7cac8a3ed86c75baa2ce39d61.js.map.gz +0 -0
  100. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ac3bbd454eba677c4fd092227d5b5b89.js.br +0 -0
  101. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ac3bbd454eba677c4fd092227d5b5b89.js.gz +0 -0
  102. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ac3bbd454eba677c4fd092227d5b5b89.js.map.br +0 -0
  103. package/dist/server/web-ui/_expo/static/js/web/native-file-attachment-store-ac3bbd454eba677c4fd092227d5b5b89.js.map.gz +0 -0
  104. package/dist/server/web-ui/_expo/static/js/web/profiler-b509d7fffd6cf457a9c12588b80560cb.js.br +0 -0
  105. package/dist/server/web-ui/_expo/static/js/web/profiler-b509d7fffd6cf457a9c12588b80560cb.js.gz +0 -0
  106. package/dist/server/web-ui/_expo/static/js/web/profiler-b509d7fffd6cf457a9c12588b80560cb.js.map.br +0 -0
  107. package/dist/server/web-ui/_expo/static/js/web/profiler-b509d7fffd6cf457a9c12588b80560cb.js.map.gz +0 -0
  108. package/dist/server/web-ui/_expo/static/js/web/startRecording-9c90e555a5ecdc643e277f9a98913aaf.js.br +0 -0
  109. package/dist/server/web-ui/_expo/static/js/web/startRecording-9c90e555a5ecdc643e277f9a98913aaf.js.gz +0 -0
  110. package/dist/server/web-ui/_expo/static/js/web/startRecording-9c90e555a5ecdc643e277f9a98913aaf.js.map.br +0 -0
@@ -1,8 +1,9 @@
1
1
  import { randomBytes } from "node:crypto";
2
2
  import { mkdir, readFile, readdir, rm, stat } from "node:fs/promises";
3
- import { join } from "node:path";
3
+ import { basename, join } from "node:path";
4
4
  import { StoredScheduleSchema, } from "@hyperdrive.bot/fleet-protocol/schedule/types";
5
5
  import { writeJsonFileAtomic } from "../atomic-file.js";
6
+ import { ScopeSync } from "../state/state-db.js";
6
7
  /**
7
8
  * How many runs a schedule file keeps. Every run start and finish rewrites the
8
9
  * whole file, and the history grew unbounded (5,510 runs, 2.3 MB in one real
@@ -73,12 +74,15 @@ function generateScheduleId() {
73
74
  * changes back through `put`.
74
75
  */
75
76
  export class ScheduleStore {
76
- constructor(dir) {
77
+ constructor(dir, options = {}) {
77
78
  this.dir = dir;
78
79
  this.cache = new Map();
79
80
  this.stamps = new Map();
80
81
  this.validating = null;
81
82
  this.revision = 0;
83
+ this.sqlite = options.stateDb
84
+ ? new SqliteScheduleStore(options.stateDb, dir, options.logger)
85
+ : null;
82
86
  }
83
87
  filePath(id) {
84
88
  return join(this.dir, `${id}.json`);
@@ -143,10 +147,14 @@ export class ScheduleStore {
143
147
  }
144
148
  }
145
149
  async list() {
150
+ if (this.sqlite)
151
+ return this.sqlite.list();
146
152
  await this.validate();
147
153
  return [...this.cache.values()].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
148
154
  }
149
155
  async get(id) {
156
+ if (this.sqlite)
157
+ return this.sqlite.get(id);
150
158
  await this.validate();
151
159
  return this.cache.get(id) ?? null;
152
160
  }
@@ -186,6 +194,8 @@ export class ScheduleStore {
186
194
  return created;
187
195
  }
188
196
  async put(schedule) {
197
+ if (this.sqlite)
198
+ return this.sqlite.put(schedule);
189
199
  await this.ensureDir();
190
200
  const name = `${schedule.id}.json`;
191
201
  const path = this.filePath(schedule.id);
@@ -207,6 +217,8 @@ export class ScheduleStore {
207
217
  this.revision += 1;
208
218
  }
209
219
  async delete(id) {
220
+ if (this.sqlite)
221
+ return this.sqlite.delete(id);
210
222
  await this.ensureDir();
211
223
  await rm(this.filePath(id), { force: true });
212
224
  this.cache.delete(id);
@@ -218,8 +230,277 @@ export class ScheduleStore {
218
230
  * put, a delete), so a caller can reuse anything it derived from `list()`
219
231
  * until the store actually changes.
220
232
  */
233
+ get version() {
234
+ return this.sqlite ? this.sqlite.version : this.revision;
235
+ }
236
+ }
237
+ const MANIFEST_STORE = "schedules";
238
+ /** Upper bound on how stale an in-place edit of a schedule file can go unseen. */
239
+ export const EXTERNAL_EDIT_RECHECK_MS = 1000;
240
+ function runProgress(run) {
241
+ // A finished run beats a running one; between two of the same kind, the later.
242
+ return (run.status === "running" ? 0 : 1e15) + Date.parse(run.endedAt ?? run.startedAt);
243
+ }
244
+ /**
245
+ * Merge a schedule file written by someone else into the daemon's copy.
246
+ *
247
+ * Top-level fields come from whichever side has the newer `updatedAt`, as one
248
+ * unit, so a field the newer side cleared is not resurrected by the older one.
249
+ * Runs merge by id: a run on one side only is kept (the cap drops the oldest
250
+ * afterwards, so runs the daemon trimmed stay trimmed), and for the same id
251
+ * the more advanced run wins, so a completion written on either side is never
252
+ * replaced by the other side's "running" copy.
253
+ */
254
+ export function mergeExternalSchedule(daemon, file) {
255
+ const newer = Date.parse(file.updatedAt) >= Date.parse(daemon.updatedAt) ? file : daemon;
256
+ const byId = new Map();
257
+ for (const run of [...daemon.runs, ...file.runs]) {
258
+ const seen = byId.get(run.id);
259
+ if (!seen || runProgress(run) > runProgress(seen)) {
260
+ byId.set(run.id, run);
261
+ }
262
+ }
263
+ const runs = [...byId.values()].sort((left, right) => left.startedAt.localeCompare(right.startedAt));
264
+ return { ...newer, runs };
265
+ }
266
+ /**
267
+ * Schedules in `state.sqlite`, with the same public surface as the file store
268
+ * (which delegates here when the daemon has a state database). A run start or
269
+ * finish used to rewrite the whole
270
+ * file (2.3 MB for the busiest real schedule); here it writes the one run row
271
+ * that changed. The legacy file is still written, debounced, by the mirror.
272
+ *
273
+ * The directory stays an input as well: a file whose content no longer matches
274
+ * what this daemon last wrote (the legacy manifest) is read back in and merged
275
+ * (`takeExternalEdits`), and a file someone deleted deletes the schedule. That
276
+ * keeps the contract that a schedule hand-written into schedules/ is picked up
277
+ * by a running daemon. The mirror re-checks the file right before overwriting
278
+ * it, so a hand edit made while a write was pending is merged, not lost.
279
+ */
280
+ class SqliteScheduleStore {
281
+ constructor(state, dir, logger) {
282
+ this.state = state;
283
+ this.dir = dir;
284
+ this.logger = logger;
285
+ this.cache = null;
286
+ this.loading = null;
287
+ this.revision = 0;
288
+ this.dirMtime = null;
289
+ this.lastFullCheck = 0;
290
+ this.schedules = state.doc("schedules");
291
+ this.runs = new ScopeSync(state.doc("schedule_runs"));
292
+ }
221
293
  get version() {
222
294
  return this.revision;
223
295
  }
296
+ filePath(id) {
297
+ return join(this.dir, `${id}.json`);
298
+ }
299
+ async load() {
300
+ if (this.cache) {
301
+ const cache = this.cache;
302
+ const edited = await this.takeExternalEdits((id) => cache.get(id) ?? null);
303
+ for (const schedule of edited) {
304
+ cache.set(schedule.id, schedule);
305
+ }
306
+ if (edited.length > 0) {
307
+ this.revision += 1;
308
+ }
309
+ return cache;
310
+ }
311
+ this.loading ?? (this.loading = this.loadAll().finally(() => {
312
+ this.loading = null;
313
+ }));
314
+ return this.loading;
315
+ }
316
+ async list() {
317
+ const cache = await this.load();
318
+ return [...cache.values()].sort((left, right) => left.createdAt.localeCompare(right.createdAt));
319
+ }
320
+ async get(id) {
321
+ return (await this.load()).get(id) ?? null;
322
+ }
323
+ async loadAll() {
324
+ await mkdir(this.dir, { recursive: true });
325
+ const runRows = this.state.doc("schedule_runs").loadAll();
326
+ this.runs.seed(runRows);
327
+ const runsBySchedule = new Map();
328
+ for (const row of runRows) {
329
+ const list = runsBySchedule.get(row.scope) ?? [];
330
+ list.push(JSON.parse(row.body));
331
+ runsBySchedule.set(row.scope, list);
332
+ }
333
+ const maxRunRev = new Map();
334
+ for (const row of runRows) {
335
+ maxRunRev.set(row.scope, Math.max(maxRunRev.get(row.scope) ?? 0, row.rev));
336
+ }
337
+ const loaded = new Map();
338
+ const behind = [];
339
+ const entries = this.state.manifest.entries(MANIFEST_STORE);
340
+ for (const row of this.schedules.loadAll("")) {
341
+ const schedule = JSON.parse(row.body);
342
+ schedule.runs = runsBySchedule.get(row.id) ?? [];
343
+ loaded.set(schedule.id, schedule);
344
+ // The process died before the mirror caught this schedule's file up.
345
+ const mirrored = entries.get(this.filePath(row.id))?.rev ?? -1;
346
+ if (Math.max(row.rev, maxRunRev.get(row.id) ?? 0) > mirrored)
347
+ behind.push(schedule);
348
+ }
349
+ this.cache = loaded;
350
+ for (const schedule of await this.takeExternalEdits((id) => loaded.get(id) ?? null)) {
351
+ loaded.set(schedule.id, schedule);
352
+ }
353
+ for (const schedule of behind) {
354
+ if (loaded.get(schedule.id) === schedule)
355
+ this.mirrorWrite(schedule);
356
+ }
357
+ this.revision += 1;
358
+ return loaded;
359
+ }
360
+ async put(input) {
361
+ const cache = await this.load();
362
+ const schedule = capScheduleRuns(input);
363
+ const { runs, ...rest } = schedule;
364
+ await Promise.all([
365
+ this.schedules.put("", schedule.id, rest),
366
+ this.runs.sync(schedule.id, runs.map((run) => ({ id: run.id, value: run })), { prune: true }),
367
+ ]);
368
+ cache.set(schedule.id, schedule);
369
+ this.revision += 1;
370
+ this.mirrorWrite(schedule);
371
+ }
372
+ async delete(id) {
373
+ const cache = await this.load();
374
+ await Promise.all([this.schedules.remove("", id), this.runs.dropScope(id)]);
375
+ cache.delete(id);
376
+ this.revision += 1;
377
+ this.state.mirror.write(`schedule:${id}`, async () => {
378
+ await rm(this.filePath(id), { force: true });
379
+ await this.state.manifest.forget(this.filePath(id));
380
+ });
381
+ }
382
+ async takeExternalEdits(current) {
383
+ let dirMtime;
384
+ try {
385
+ dirMtime = (await stat(this.dir)).mtimeMs;
386
+ }
387
+ catch {
388
+ return [];
389
+ }
390
+ // A new or renamed file bumps the directory mtime and is read at once. An
391
+ // in-place edit does not, so the per-file stats also run at most once per
392
+ // EXTERNAL_EDIT_RECHECK_MS; between those a read costs one stat.
393
+ const now = Date.now();
394
+ if (dirMtime === this.dirMtime && now - this.lastFullCheck < EXTERNAL_EDIT_RECHECK_MS) {
395
+ return [];
396
+ }
397
+ this.dirMtime = dirMtime;
398
+ this.lastFullCheck = now;
399
+ const edited = [];
400
+ const entries = this.state.manifest.entries(MANIFEST_STORE);
401
+ const ids = new Set();
402
+ for (const entry of await readdir(this.dir, { withFileTypes: true })) {
403
+ if (!entry.isFile() || !entry.name.endsWith(".json") || entry.name.startsWith(".")) {
404
+ continue;
405
+ }
406
+ ids.add(entry.name.slice(0, -".json".length));
407
+ }
408
+ for (const entryPath of entries.keys()) {
409
+ ids.add(basename(entryPath, ".json"));
410
+ }
411
+ for (const id of ids) {
412
+ // Our own write in flight: the manifest catches up when it lands.
413
+ if (this.state.mirror.isRunning(`schedule:${id}`))
414
+ continue;
415
+ const path = this.filePath(id);
416
+ try {
417
+ const file = await this.state.manifest.classify(path, entries.get(path) ?? null);
418
+ if (file.status === "deleted") {
419
+ // Deleted by someone else: a deletion wins, so an active schedule is
420
+ // never brought back.
421
+ if (current(id)) {
422
+ await Promise.all([this.schedules.remove("", id), this.runs.dropScope(id)]);
423
+ this.cache?.delete(id);
424
+ this.revision += 1;
425
+ }
426
+ await this.state.manifest.forget(path);
427
+ continue;
428
+ }
429
+ if (file.status !== "changed" && file.status !== "created")
430
+ continue;
431
+ const schedule = await this.adoptForeign(id, file.content, current(id));
432
+ edited.push(schedule);
433
+ }
434
+ catch (error) {
435
+ // A half-written or invalid file is retried when it changes next; a
436
+ // failed commit is worth knowing about.
437
+ this.logger?.warn({ err: error, path }, "Failed to read back an externally edited schedule");
438
+ }
439
+ }
440
+ return edited;
441
+ }
442
+ /**
443
+ * Merge a file someone else wrote into the daemon's copy, commit it, record
444
+ * the file as seen, and queue the merged result back out when it differs.
445
+ */
446
+ async adoptForeign(id, content, existing) {
447
+ const fromFile = StoredScheduleSchema.parse(JSON.parse(content));
448
+ const schedule = capScheduleRuns(existing ? mergeExternalSchedule(existing, fromFile) : fromFile);
449
+ const { runs, ...rest } = schedule;
450
+ await Promise.all([
451
+ this.schedules.put("", schedule.id, rest),
452
+ this.runs.sync(schedule.id, runs.map((run) => ({ id: run.id, value: run })), { prune: true }),
453
+ ]);
454
+ this.cache?.set(schedule.id, schedule);
455
+ this.revision += 1;
456
+ await this.state.manifest.record(MANIFEST_STORE, this.filePath(id), this.state.mirrorRev());
457
+ if (existing) {
458
+ // The merge may differ from the file: write the result back out.
459
+ this.mirrorWrite(schedule);
460
+ }
461
+ return schedule;
462
+ }
463
+ mirrorWrite(schedule) {
464
+ const id = schedule.id;
465
+ this.state.mirror.write(`schedule:${id}`, async () => {
466
+ const path = this.filePath(id);
467
+ // Re-check right before overwriting: a hand edit (or delete) made while
468
+ // this write was pending is merged in first, never replaced.
469
+ const file = await this.state.manifest.classify(path);
470
+ let latest = this.cache?.get(id) ?? schedule;
471
+ if (file.status === "deleted") {
472
+ if (this.cache?.has(id)) {
473
+ await Promise.all([this.schedules.remove("", id), this.runs.dropScope(id)]);
474
+ this.cache.delete(id);
475
+ this.revision += 1;
476
+ }
477
+ await this.state.manifest.forget(path);
478
+ return;
479
+ }
480
+ if (file.status === "changed" || file.status === "created") {
481
+ try {
482
+ const fromFile = StoredScheduleSchema.parse(JSON.parse(file.content));
483
+ latest = capScheduleRuns(mergeExternalSchedule(latest, fromFile));
484
+ const { runs, ...rest } = latest;
485
+ await Promise.all([
486
+ this.schedules.put("", id, rest),
487
+ this.runs.sync(id, runs.map((run) => ({ id: run.id, value: run })), { prune: true }),
488
+ ]);
489
+ this.cache?.set(id, latest);
490
+ this.revision += 1;
491
+ }
492
+ catch (error) {
493
+ this.logger?.warn({ err: error, path }, "Schedule file edited by hand is unreadable; overwriting it");
494
+ }
495
+ }
496
+ else if (!this.cache?.has(id)) {
497
+ return;
498
+ }
499
+ const rev = this.state.mirrorRev();
500
+ await writeJsonFileAtomic(path, latest);
501
+ await this.state.manifest.record(MANIFEST_STORE, path, rev);
502
+ this.dirMtime = null;
503
+ });
504
+ }
224
505
  }
225
506
  //# sourceMappingURL=store.js.map
@@ -26,6 +26,16 @@ export interface SourceAdapter {
26
26
  discover(): Promise<DiscoveredFile[]>;
27
27
  /** Parse one raw record into extractable blocks. */
28
28
  extract(record: unknown): ExtractedRecord;
29
+ /**
30
+ * Whole-document sources not kept in files (paseo agents in state.sqlite):
31
+ * the document for a discovered entry, with a size and a version that
32
+ * changes whenever the document does. Null when it is gone.
33
+ */
34
+ readDocument?(file: DiscoveredFile): Promise<{
35
+ size: number;
36
+ mtimeMs: number;
37
+ content: string;
38
+ } | null>;
29
39
  }
30
40
  export declare function createClaudeAdapter(root?: string): SourceAdapter;
31
41
  export declare function createCodexAdapter(root?: string): SourceAdapter;
@@ -26,6 +26,8 @@ import fs from "node:fs/promises";
26
26
  import os from "node:os";
27
27
  import path from "node:path";
28
28
  import { resolvePaseoHome } from "../paseo-home.js";
29
+ import { agentRecordPath } from "../agent/agent-storage.js";
30
+ import { stateDbFor } from "../state/state-db.js";
29
31
  import { extractClaudeRecord, extractCodexRecord, extractPaseoAgent, } from "./extract.js";
30
32
  // ---------------------------------------------------------------------------
31
33
  // Discovery helpers
@@ -133,12 +135,49 @@ export function createPaseoAdapter(root) {
133
135
  // adapter indexes title + labels so agents are findable by name, and the
134
136
  // record's runtimeInfo.sessionId is what links a hit to real transcript
135
137
  // text held by the provider adapters.
138
+ async readDocument(file) {
139
+ const stateDb = root === undefined ? stateDbFor(resolvePaseoHome()) : null;
140
+ if (!stateDb) {
141
+ const stat = await fs.stat(file.path);
142
+ return {
143
+ size: stat.size,
144
+ mtimeMs: stat.mtimeMs,
145
+ content: await fs.readFile(file.path, "utf8"),
146
+ };
147
+ }
148
+ const row = stateDb
149
+ .prepare("SELECT body, rev FROM agents WHERE scope = '' AND id = ? AND deleted_at IS NULL")
150
+ .get(file.sid);
151
+ if (!row || typeof row.body !== "string")
152
+ return null;
153
+ // rev changes on every write (a title set keeps updatedAt), so it is the version.
154
+ return { size: row.body.length, mtimeMs: Number(row.rev), content: row.body };
155
+ },
136
156
  extract(record) {
137
157
  const { meta, blocks } = extractPaseoAgent(record);
138
158
  // The agent record's own cwd is authoritative for a paseo hit.
139
159
  return { blocks, timestamp: meta?.updatedAt ?? null, sid: null, cwd: meta?.cwd ?? null };
140
160
  },
141
161
  async discover() {
162
+ // With state.sqlite the agent files are only a lagging mirror (and
163
+ // absent under PASEO_STATE_FILE_MIRROR=0): index the rows. Paths stay the
164
+ // legacy file paths, so index keys do not change with the backend.
165
+ const stateDb = root === undefined ? stateDbFor(resolvePaseoHome()) : null;
166
+ if (stateDb) {
167
+ stateDb.flushSync();
168
+ return stateDb
169
+ .prepare("SELECT id, json_extract(body, '$.cwd') AS cwd FROM agents WHERE scope = '' AND deleted_at IS NULL")
170
+ .all()
171
+ .map((row) => ({
172
+ path: agentRecordPath(agentsRoot, {
173
+ id: String(row.id),
174
+ cwd: String(row.cwd ?? ""),
175
+ }),
176
+ source: "paseo",
177
+ sid: String(row.id),
178
+ cwd: "",
179
+ }));
180
+ }
142
181
  const files = await walkFiles(agentsRoot, ".json");
143
182
  return files.map((p) => ({
144
183
  path: p,
@@ -288,8 +327,18 @@ function buildChunks(args, consumable, isWholeDoc) {
288
327
  async function indexOneFile(args) {
289
328
  const { store, adapter, file, result, maxBytes, log } = args;
290
329
  let stat;
330
+ let document = null;
291
331
  try {
292
- stat = await fs.stat(file.path);
332
+ if (adapter.readDocument) {
333
+ const read = await adapter.readDocument(file);
334
+ if (!read)
335
+ return;
336
+ document = read.content;
337
+ stat = read;
338
+ }
339
+ else {
340
+ stat = await fs.stat(file.path);
341
+ }
293
342
  }
294
343
  catch {
295
344
  return;
@@ -301,7 +350,7 @@ async function indexOneFile(args) {
301
350
  const end = Math.min(stat.size, start + maxBytes);
302
351
  let raw;
303
352
  try {
304
- raw = await readRange(file.path, start, end);
353
+ raw = document ?? (await readRange(file.path, start, end));
305
354
  }
306
355
  catch (err) {
307
356
  result.errors += 1;
@@ -6622,21 +6622,20 @@ export class Session {
6622
6622
  clearedAgentIds.push(agentId);
6623
6623
  continue;
6624
6624
  }
6625
- const record = await this.agentStorage.get(agentId);
6626
- if (!record ||
6627
- record.internal ||
6628
- record.archivedAt ||
6629
- record.requiresAttention !== true) {
6625
+ // Inside the agent's write queue, so a concurrent title or digest
6626
+ // write is never dropped by this read-then-write.
6627
+ const nextRecord = await this.agentStorage.update(agentId, (record) => record.internal || record.archivedAt || record.requiresAttention !== true
6628
+ ? null
6629
+ : {
6630
+ ...record,
6631
+ updatedAt: new Date().toISOString(),
6632
+ requiresAttention: false,
6633
+ attentionReason: null,
6634
+ attentionTimestamp: null,
6635
+ });
6636
+ if (!nextRecord) {
6630
6637
  continue;
6631
6638
  }
6632
- const nextRecord = {
6633
- ...record,
6634
- updatedAt: new Date().toISOString(),
6635
- requiresAttention: false,
6636
- attentionReason: null,
6637
- attentionTimestamp: null,
6638
- };
6639
- await this.agentStorage.upsert(nextRecord);
6640
6639
  const agent = this.buildStoredAgentPayload(nextRecord);
6641
6640
  const project = await this.buildProjectPlacementForWorkspace(workspace);
6642
6641
  this.emit({
@@ -0,0 +1,39 @@
1
+ import type { Logger } from "pino";
2
+ import { type AgentRecordBackend, type StoredAgentRecord } from "../agent/agent-storage.js";
3
+ import type { AgentMirrorCatchUp } from "./legacy-import.js";
4
+ import type { ChangedRow, StateDb } from "./state-db.js";
5
+ /**
6
+ * Agent records in `state.sqlite`, one row per agent, with a daemon-wide `rev`
7
+ * and a tombstone on delete. Replaces the boot scan of every agent JSON (14,388
8
+ * files on the reference host) with one indexed read, and the per-update
9
+ * rewrite of a whole file with one row in a batched transaction.
10
+ *
11
+ * Every write is mirrored to the legacy file, debounced and off the request
12
+ * path, see `LegacyMirror`, and each file written or removed is recorded in the
13
+ * legacy manifest with the rev it reflects. Files the boot import found lagging
14
+ * the database (the process died inside the debounce window) are caught up on
15
+ * the first load.
16
+ */
17
+ export declare class SqliteAgentRecordBackend implements AgentRecordBackend {
18
+ private readonly state;
19
+ private readonly catchUp;
20
+ private readonly table;
21
+ private readonly files;
22
+ /**
23
+ * The record the legacy file currently holds, per agent the mirror has
24
+ * touched. Debounced writes collapse, so the file to replace is the one the
25
+ * mirror last wrote (or the record loaded at boot), not the previous record in
26
+ * memory, which may never have reached disk.
27
+ */
28
+ private readonly onDisk;
29
+ /** Rev each running mirror write reflects, by agent id. */
30
+ private readonly mirrorRevs;
31
+ constructor(state: StateDb, baseDir: string, logger: Logger, catchUp?: AgentMirrorCatchUp);
32
+ loadAll(): Promise<StoredAgentRecord[]>;
33
+ write(record: StoredAgentRecord, previous: StoredAgentRecord | null): Promise<void>;
34
+ /** `rev` is the row's own rev for exactly this content (re-review R1). */
35
+ private mirrorWrite;
36
+ remove(agentId: string, previous: StoredAgentRecord | null): Promise<void>;
37
+ listChangedSince(rev: number, limit?: number): ChangedRow[];
38
+ }
39
+ //# sourceMappingURL=agent-state.d.ts.map
@@ -0,0 +1,94 @@
1
+ import { promises as fs } from "node:fs";
2
+ import path from "node:path";
3
+ import { FileAgentRecordBackend, } from "../agent/agent-storage.js";
4
+ /**
5
+ * Agent records in `state.sqlite`, one row per agent, with a daemon-wide `rev`
6
+ * and a tombstone on delete. Replaces the boot scan of every agent JSON (14,388
7
+ * files on the reference host) with one indexed read, and the per-update
8
+ * rewrite of a whole file with one row in a batched transaction.
9
+ *
10
+ * Every write is mirrored to the legacy file, debounced and off the request
11
+ * path, see `LegacyMirror`, and each file written or removed is recorded in the
12
+ * legacy manifest with the rev it reflects. Files the boot import found lagging
13
+ * the database (the process died inside the debounce window) are caught up on
14
+ * the first load.
15
+ */
16
+ export class SqliteAgentRecordBackend {
17
+ constructor(state, baseDir, logger, catchUp = { rewrite: [], unlink: [] }) {
18
+ this.state = state;
19
+ this.catchUp = catchUp;
20
+ /**
21
+ * The record the legacy file currently holds, per agent the mirror has
22
+ * touched. Debounced writes collapse, so the file to replace is the one the
23
+ * mirror last wrote (or the record loaded at boot), not the previous record in
24
+ * memory, which may never have reached disk.
25
+ */
26
+ this.onDisk = new Map();
27
+ /** Rev each running mirror write reflects, by agent id. */
28
+ this.mirrorRevs = new Map();
29
+ this.table = state.doc("agents", { tombstones: true });
30
+ this.files = new FileAgentRecordBackend(baseDir, logger, {
31
+ written: (filePath) => state.manifest.record("agents", filePath, this.mirrorRevs.get(path.basename(filePath, ".json")) ?? 0),
32
+ removed: (filePath) => state.manifest.forget(filePath),
33
+ });
34
+ // Every file the manifest knows, so a cwd move or a delete also cleans up
35
+ // copies left at older paths.
36
+ for (const entry of state.manifest.entries("agents").values()) {
37
+ this.files.indexPath(path.basename(entry.path, ".json"), entry.path);
38
+ }
39
+ }
40
+ async loadAll() {
41
+ // Bodies were validated when they were written or imported; re-running zod
42
+ // over 14k records here would cost more than the read it replaces.
43
+ const rows = this.table.loadAll("");
44
+ const records = rows.map((row) => JSON.parse(row.body));
45
+ if (this.state.mirror.enabled) {
46
+ const byId = new Map(records.map((record) => [record.id, record]));
47
+ const revs = new Map(rows.map((row) => [row.id, row.rev]));
48
+ for (const id of this.catchUp.rewrite) {
49
+ const record = byId.get(id);
50
+ if (record)
51
+ this.mirrorWrite(record, revs.get(id) ?? 0);
52
+ }
53
+ for (const filePath of this.catchUp.unlink) {
54
+ this.state.mirror.write(`agent-file:${filePath}`, async () => {
55
+ await fs.rm(filePath, { force: true });
56
+ await this.state.manifest.forget(filePath);
57
+ });
58
+ }
59
+ }
60
+ return records;
61
+ }
62
+ async write(record, previous) {
63
+ const rev = await this.table.putWithRev("", record.id, record);
64
+ if (!this.onDisk.has(record.id))
65
+ this.onDisk.set(record.id, previous);
66
+ this.mirrorWrite(record, rev);
67
+ }
68
+ /** `rev` is the row's own rev for exactly this content (re-review R1). */
69
+ mirrorWrite(record, rev) {
70
+ this.state.mirror.write(`agent:${record.id}`, async () => {
71
+ this.mirrorRevs.set(record.id, rev);
72
+ try {
73
+ await this.files.write(record, this.onDisk.get(record.id) ?? null);
74
+ }
75
+ finally {
76
+ this.mirrorRevs.delete(record.id);
77
+ }
78
+ this.onDisk.set(record.id, record);
79
+ });
80
+ }
81
+ async remove(agentId, previous) {
82
+ await this.table.remove("", agentId);
83
+ if (!this.onDisk.has(agentId))
84
+ this.onDisk.set(agentId, previous);
85
+ this.state.mirror.write(`agent:${agentId}`, async () => {
86
+ await this.files.remove(agentId, this.onDisk.get(agentId) ?? null);
87
+ this.onDisk.delete(agentId);
88
+ });
89
+ }
90
+ listChangedSince(rev, limit) {
91
+ return this.table.listChangedSince(rev, limit);
92
+ }
93
+ }
94
+ //# sourceMappingURL=agent-state.js.map
@@ -0,0 +1,30 @@
1
+ import type { Logger } from "pino";
2
+ import type { AgentRecordBackend } from "../agent/agent-storage.js";
3
+ import { type StateDb } from "./state-db.js";
4
+ /**
5
+ * Free space the first import needs, as a multiple of the legacy state it
6
+ * copies: the database itself (about 1x on a real install) plus the WAL, which
7
+ * grows to about 1.4x while checkpoints are paused during the import.
8
+ */
9
+ export declare const IMPORT_SPACE_FACTOR = 2.5;
10
+ export interface DaemonState {
11
+ /** Null when node:sqlite is unavailable, the probe failed, or PASEO_STATE_SQLITE=0. */
12
+ stateDb: StateDb | null;
13
+ /** Undefined means AgentStorage keeps its legacy file backend. */
14
+ agentBackend: AgentRecordBackend | undefined;
15
+ close(): Promise<void>;
16
+ }
17
+ /**
18
+ * Open `state.sqlite` for a daemon, copy the legacy stores in (first boot) or
19
+ * re-sync files an older daemon changed (after a rollback), and attach it for
20
+ * the stores constructed deeper down. See data-layer-rework.md section 5.
21
+ */
22
+ export declare function openDaemonState(options: {
23
+ paseoHome: string;
24
+ agentStoragePath: string;
25
+ logger: Logger;
26
+ elapsed: () => unknown;
27
+ /** Test seam: free bytes on the PASEO_HOME filesystem. */
28
+ freeBytes?: () => Promise<number>;
29
+ }): Promise<DaemonState>;
30
+ //# sourceMappingURL=daemon-state.d.ts.map