@hyperdrive.bot/fleet-server 0.3.164 → 0.3.165
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/extensions/daemon-backend.js +25 -0
- package/dist/server/server/agent/agent-storage.d.ts +67 -12
- package/dist/server/server/agent/agent-storage.js +208 -132
- package/dist/server/server/agent/providers/claude/transport/pty-query.js +10 -0
- package/dist/server/server/bootstrap.d.ts +5 -0
- package/dist/server/server/bootstrap.js +26 -3
- package/dist/server/server/chat/chat-service.d.ts +22 -0
- package/dist/server/server/chat/chat-service.js +46 -3
- package/dist/server/server/daemon-worker.js +1 -0
- package/dist/server/server/fleet/decision-service.d.ts +5 -0
- package/dist/server/server/fleet/decision-service.js +30 -0
- package/dist/server/server/loop-service.d.ts +95 -0
- package/dist/server/server/loop-service.js +64 -3
- package/dist/server/server/migrations/backfill-workspace-id.migration.js +3 -2
- package/dist/server/server/push/token-store.d.ts +5 -1
- package/dist/server/server/push/token-store.js +26 -5
- package/dist/server/server/schedule/service.js +4 -1
- package/dist/server/server/schedule/store.d.ts +21 -1
- package/dist/server/server/schedule/store.js +283 -2
- package/dist/server/server/search/indexer.d.ts +10 -0
- package/dist/server/server/search/indexer.js +51 -2
- package/dist/server/server/session.js +12 -13
- package/dist/server/server/state/agent-state.d.ts +39 -0
- package/dist/server/server/state/agent-state.js +94 -0
- package/dist/server/server/state/daemon-state.d.ts +30 -0
- package/dist/server/server/state/daemon-state.js +99 -0
- package/dist/server/server/state/legacy-import.d.ts +124 -0
- package/dist/server/server/state/legacy-import.js +682 -0
- package/dist/server/server/state/legacy-manifest.d.ts +73 -0
- package/dist/server/server/state/legacy-manifest.js +112 -0
- package/dist/server/server/state/legacy-mirror.d.ts +57 -0
- package/dist/server/server/state/legacy-mirror.js +114 -0
- package/dist/server/server/state/legacy-sources.d.ts +9 -0
- package/dist/server/server/state/legacy-sources.js +163 -0
- package/dist/server/server/state/state-db.d.ts +263 -0
- package/dist/server/server/state/state-db.js +772 -0
- package/dist/server/server/state/state-schema.d.ts +39 -0
- package/dist/server/server/state/state-schema.js +125 -0
- package/dist/server/server/state/state-worker.d.ts +48 -0
- package/dist/server/server/state/state-worker.js +165 -0
- package/dist/server/server/websocket-server.js +2 -1
- package/dist/server/server/workspace/session-openable.js +7 -0
- package/dist/server/server/workspace-reconciliation-service.js +14 -3
- package/dist/server/server/workspace-registry.d.ts +45 -2
- package/dist/server/server/workspace-registry.js +75 -4
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js → index-38580ed8926a5ddc569d744b8299d339.js} +4 -4
- package/dist/server/web-ui/_expo/static/js/web/index-38580ed8926a5ddc569d744b8299d339.js.br +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js.gz → index-38580ed8926a5ddc569d744b8299d339.js.gz} +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js.map.br → index-38580ed8926a5ddc569d744b8299d339.js.map.br} +0 -0
- package/dist/server/web-ui/_expo/static/js/web/{index-2ac1a7249c20c322a1f9a54563cfee62.js.map.gz → index-38580ed8926a5ddc569d744b8299d339.js.map.gz} +0 -0
- package/dist/server/web-ui/index.html +1 -1
- package/dist/server/web-ui/index.html.br +0 -0
- package/dist/server/web-ui/index.html.gz +0 -0
- package/package.json +6 -6
- package/dist/server/web-ui/_expo/static/js/web/index-2ac1a7249c20c322a1f9a54563cfee62.js.br +0 -0
|
@@ -94,6 +94,7 @@ import { scheduleSessionDigestRefresh } from "./agent/session-digest-generator.j
|
|
|
94
94
|
import { CardMoveLog } from "./agent/card-move-log.js";
|
|
95
95
|
import { BlockerLog } from "./agent/blocker-log.js";
|
|
96
96
|
import { AUTHORSHIP_ENV_FLAG, createAuthorshipLedger } from "./agent/authorship-ledger.js";
|
|
97
|
+
import { openDaemonState } from "./state/daemon-state.js";
|
|
97
98
|
import { AUTHORSHIP_MARKUP_ENV_FLAG, AUTHORSHIP_SPEAKER_ENV, createAuthorshipMarkup, } from "./agent/authorship-markup.js";
|
|
98
99
|
import { JudgeRelayGate } from "./agent/judge-relay-gate.js";
|
|
99
100
|
import { JarvisJudge } from "./agent/jarvis-judge.js";
|
|
@@ -691,9 +692,21 @@ export async function createPaseoDaemon(config, rootLogger) {
|
|
|
691
692
|
if (config.serviceProxy?.standaloneListen) {
|
|
692
693
|
serviceProxyListenTarget = parseListenString(config.serviceProxy.standaloneListen);
|
|
693
694
|
}
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
695
|
+
// Non-blob daemon state in $PASEO_HOME/state.sqlite (data-layer-rework.md
|
|
696
|
+
// section 5). Null when node:sqlite is unavailable, the probe fails or
|
|
697
|
+
// PASEO_STATE_SQLITE=0: every store then stays on its legacy files. The
|
|
698
|
+
// first boot copies the files in (yielding batches, resumable); files are
|
|
699
|
+
// never deleted and stay mirrored for one release.
|
|
700
|
+
const daemonState = await openDaemonState({
|
|
701
|
+
paseoHome: config.paseoHome,
|
|
702
|
+
agentStoragePath: config.agentStoragePath,
|
|
703
|
+
logger,
|
|
704
|
+
elapsed,
|
|
705
|
+
});
|
|
706
|
+
const stateDb = daemonState.stateDb;
|
|
707
|
+
const agentStorage = new AgentStorage(config.agentStoragePath, logger, daemonState.agentBackend);
|
|
708
|
+
const projectRegistry = new FileBackedProjectRegistry(path.join(config.paseoHome, "projects", "projects.json"), logger, { stateDb });
|
|
709
|
+
workspaceRegistry = new FileBackedWorkspaceRegistry(path.join(config.paseoHome, "projects", "workspaces.json"), logger, { stateDb });
|
|
697
710
|
// Story PASEO-CHAT-ACTIONS-V1-story-3.003 — GET /workspace/:id/sessions.
|
|
698
711
|
registerWorkspaceSessionsRoute(app, workspaceRegistry, logger);
|
|
699
712
|
// MARRA CUSTOMIZATION — GET /sessions (global; walks the agent transcript
|
|
@@ -707,6 +720,7 @@ export async function createPaseoDaemon(config, rootLogger) {
|
|
|
707
720
|
const chatService = new FileBackedChatService({
|
|
708
721
|
paseoHome: config.paseoHome,
|
|
709
722
|
logger,
|
|
723
|
+
stateDb,
|
|
710
724
|
});
|
|
711
725
|
const github = createGitHubService();
|
|
712
726
|
const workspaceGitService = new WorkspaceGitServiceImpl({
|
|
@@ -918,6 +932,7 @@ export async function createPaseoDaemon(config, rootLogger) {
|
|
|
918
932
|
logger,
|
|
919
933
|
agentManager,
|
|
920
934
|
providerSnapshotManager,
|
|
935
|
+
stateDb,
|
|
921
936
|
});
|
|
922
937
|
await loopService.initialize();
|
|
923
938
|
logger.info({ elapsed: elapsed() }, "Loop service initialized");
|
|
@@ -1669,6 +1684,9 @@ export async function createPaseoDaemon(config, rootLogger) {
|
|
|
1669
1684
|
if (listenTarget.type === "socket" && existsSync(listenTarget.path)) {
|
|
1670
1685
|
unlinkSync(listenTarget.path);
|
|
1671
1686
|
}
|
|
1687
|
+
// Last: every store above may still write on its way down. Close flushes
|
|
1688
|
+
// the batch queue and the legacy file mirror.
|
|
1689
|
+
await daemonState.close();
|
|
1672
1690
|
};
|
|
1673
1691
|
return {
|
|
1674
1692
|
config,
|
|
@@ -1681,6 +1699,11 @@ export async function createPaseoDaemon(config, rootLogger) {
|
|
|
1681
1699
|
browserToolsBroker,
|
|
1682
1700
|
start,
|
|
1683
1701
|
stop,
|
|
1702
|
+
beginStateShutdown: () => {
|
|
1703
|
+
void daemonState.stateDb?.beginShutdown().catch((error) => {
|
|
1704
|
+
logger.warn({ err: error }, "Early state flush on shutdown failed");
|
|
1705
|
+
});
|
|
1706
|
+
},
|
|
1684
1707
|
getListenTarget: () => boundListenTarget,
|
|
1685
1708
|
};
|
|
1686
1709
|
}
|
|
@@ -1,5 +1,25 @@
|
|
|
1
1
|
import type pino from "pino";
|
|
2
|
+
import { z } from "zod";
|
|
3
|
+
import { type StateDb } from "../state/state-db.js";
|
|
2
4
|
import { type ChatMessage, type ChatRoomDetail } from "@hyperdrive.bot/fleet-protocol/chat/types";
|
|
5
|
+
export declare const ChatStorePayloadSchema: z.ZodObject<{
|
|
6
|
+
rooms: z.ZodArray<z.ZodObject<{
|
|
7
|
+
id: z.ZodString;
|
|
8
|
+
name: z.ZodString;
|
|
9
|
+
purpose: z.ZodNullable<z.ZodString>;
|
|
10
|
+
createdAt: z.ZodString;
|
|
11
|
+
updatedAt: z.ZodString;
|
|
12
|
+
}, z.core.$strip>>;
|
|
13
|
+
messages: z.ZodArray<z.ZodObject<{
|
|
14
|
+
id: z.ZodString;
|
|
15
|
+
roomId: z.ZodString;
|
|
16
|
+
authorAgentId: z.ZodString;
|
|
17
|
+
body: z.ZodString;
|
|
18
|
+
replyToMessageId: z.ZodNullable<z.ZodString>;
|
|
19
|
+
mentionAgentIds: z.ZodArray<z.ZodString>;
|
|
20
|
+
createdAt: z.ZodString;
|
|
21
|
+
}, z.core.$strip>>;
|
|
22
|
+
}, z.core.$strip>;
|
|
3
23
|
export declare function parseMentionAgentIds(body: string): string[];
|
|
4
24
|
export declare class ChatServiceError extends Error {
|
|
5
25
|
readonly code: string;
|
|
@@ -49,9 +69,11 @@ export declare class FileBackedChatService {
|
|
|
49
69
|
private readonly messagesByRoomId;
|
|
50
70
|
private persistQueue;
|
|
51
71
|
private readonly waitersByRoomId;
|
|
72
|
+
private readonly sqlite;
|
|
52
73
|
constructor(options: {
|
|
53
74
|
paseoHome: string;
|
|
54
75
|
logger: pino.Logger;
|
|
76
|
+
stateDb?: StateDb | null;
|
|
55
77
|
});
|
|
56
78
|
initialize(): Promise<void>;
|
|
57
79
|
createRoom(input: CreateChatRoomInput): Promise<ChatRoomDetail>;
|
|
@@ -3,8 +3,9 @@ import { promises as fs } from "node:fs";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { writeJsonFileAtomic } from "../atomic-file.js";
|
|
6
|
+
import { ScopeSync } from "../state/state-db.js";
|
|
6
7
|
import { ChatMessageSchema, ChatRoomDetailSchema, ChatRoomSchema, } from "@hyperdrive.bot/fleet-protocol/chat/types";
|
|
7
|
-
const ChatStorePayloadSchema = z.object({
|
|
8
|
+
export const ChatStorePayloadSchema = z.object({
|
|
8
9
|
rooms: z.array(ChatRoomSchema),
|
|
9
10
|
messages: z.array(ChatMessageSchema),
|
|
10
11
|
});
|
|
@@ -36,6 +37,41 @@ export class ChatServiceError extends Error {
|
|
|
36
37
|
this.code = code;
|
|
37
38
|
}
|
|
38
39
|
}
|
|
40
|
+
/**
|
|
41
|
+
* Chat in `state.sqlite`: rooms in `chat_rooms`, each message a row in
|
|
42
|
+
* `chat_messages` scoped by room (indexed by room and order, unbounded). A
|
|
43
|
+
* persist writes only what changed; chat/rooms.json is mirrored.
|
|
44
|
+
*/
|
|
45
|
+
class SqliteChatStore {
|
|
46
|
+
constructor(state, filePath) {
|
|
47
|
+
this.state = state;
|
|
48
|
+
this.filePath = filePath;
|
|
49
|
+
this.rooms = new ScopeSync(state.doc("chat_rooms"));
|
|
50
|
+
this.messages = new ScopeSync(state.doc("chat_messages"));
|
|
51
|
+
}
|
|
52
|
+
load() {
|
|
53
|
+
const roomRows = this.state.doc("chat_rooms").loadAll("");
|
|
54
|
+
const messageRows = this.state.doc("chat_messages").loadAll();
|
|
55
|
+
this.rooms.seed(roomRows);
|
|
56
|
+
this.messages.seed(messageRows);
|
|
57
|
+
return {
|
|
58
|
+
rooms: roomRows.map((row) => JSON.parse(row.body)),
|
|
59
|
+
messages: messageRows.map((row) => JSON.parse(row.body)),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
async save(payload, messagesByRoomId) {
|
|
63
|
+
await Promise.all([
|
|
64
|
+
this.rooms.sync("", payload.rooms.map((room) => ({ id: room.id, value: room })), { prune: true }),
|
|
65
|
+
...[...messagesByRoomId].map(([roomId, messages]) => this.messages.sync(roomId, messages.map((message) => ({ id: message.id, value: message })), { prune: true })),
|
|
66
|
+
this.messages.dropScopesExcept(new Set(messagesByRoomId.keys())),
|
|
67
|
+
]);
|
|
68
|
+
const rev = this.state.mirrorRev();
|
|
69
|
+
this.state.mirror.write(`chat:${this.filePath}`, async () => {
|
|
70
|
+
await writeJsonFileAtomic(this.filePath, payload);
|
|
71
|
+
await this.state.manifest.record("chat", this.filePath, rev);
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
39
75
|
export class FileBackedChatService {
|
|
40
76
|
constructor(options) {
|
|
41
77
|
this.loaded = false;
|
|
@@ -44,6 +80,7 @@ export class FileBackedChatService {
|
|
|
44
80
|
this.persistQueue = Promise.resolve();
|
|
45
81
|
this.waitersByRoomId = new Map();
|
|
46
82
|
this.filePath = path.join(options.paseoHome, "chat", "rooms.json");
|
|
83
|
+
this.sqlite = options.stateDb ? new SqliteChatStore(options.stateDb, this.filePath) : null;
|
|
47
84
|
this.logger = options.logger.child({ component: "chat-service" });
|
|
48
85
|
}
|
|
49
86
|
async initialize() {
|
|
@@ -216,8 +253,10 @@ export class FileBackedChatService {
|
|
|
216
253
|
this.rooms.clear();
|
|
217
254
|
this.messagesByRoomId.clear();
|
|
218
255
|
try {
|
|
219
|
-
const raw =
|
|
220
|
-
|
|
256
|
+
const raw = this.sqlite
|
|
257
|
+
? this.sqlite.load()
|
|
258
|
+
: JSON.parse(await fs.readFile(this.filePath, "utf8"));
|
|
259
|
+
const parsed = ChatStorePayloadSchema.parse(raw);
|
|
221
260
|
for (const room of parsed.rooms) {
|
|
222
261
|
this.rooms.set(room.id, room);
|
|
223
262
|
}
|
|
@@ -247,6 +286,10 @@ export class FileBackedChatService {
|
|
|
247
286
|
.flat()
|
|
248
287
|
.sort((left, right) => left.createdAt.localeCompare(right.createdAt)),
|
|
249
288
|
};
|
|
289
|
+
if (this.sqlite) {
|
|
290
|
+
await this.sqlite.save(payload, this.messagesByRoomId);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
250
293
|
await writeJsonFileAtomic(this.filePath, payload);
|
|
251
294
|
}
|
|
252
295
|
findRoomByName(name) {
|
|
@@ -87,6 +87,7 @@ async function main() {
|
|
|
87
87
|
const reason = options?.reason ?? `worker_received_${signal}`;
|
|
88
88
|
if (!shutdownPromise) {
|
|
89
89
|
logger.info({ signal, reason, ...getProcessDiagnostics() }, `${signal} received, shutting down gracefully...`);
|
|
90
|
+
daemon?.beginStateShutdown();
|
|
90
91
|
shutdownPromise = (async () => {
|
|
91
92
|
const forceExit = setTimeout(() => {
|
|
92
93
|
logger.warn({ signal, reason, ...getProcessDiagnostics() }, "Forcing shutdown - HTTP server didn't close in time");
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type CreateLoopDecisionInput, type LoopDecision, type LoopDecisionResolver } from "@hyperdrive.bot/fleet-protocol/fleet/decisions";
|
|
2
|
+
import { type StateDb } from "../state/state-db.js";
|
|
2
3
|
/**
|
|
3
4
|
* Store + lifecycle for loop decisions.
|
|
4
5
|
*
|
|
@@ -13,6 +14,8 @@ import { type CreateLoopDecisionInput, type LoopDecision, type LoopDecisionResol
|
|
|
13
14
|
export interface DecisionServiceOptions {
|
|
14
15
|
paseoHome: string;
|
|
15
16
|
now?: () => Date;
|
|
17
|
+
/** Defaults to the daemon's state.sqlite for this home, when one is attached. */
|
|
18
|
+
stateDb?: StateDb | null;
|
|
16
19
|
}
|
|
17
20
|
export declare class LoopDecisionService {
|
|
18
21
|
private readonly dir;
|
|
@@ -24,6 +27,8 @@ export declare class LoopDecisionService {
|
|
|
24
27
|
* every file, synchronously, on each RPC.
|
|
25
28
|
*/
|
|
26
29
|
private cached;
|
|
30
|
+
private readonly stateDb;
|
|
31
|
+
private readonly table;
|
|
27
32
|
constructor(options: DecisionServiceOptions);
|
|
28
33
|
private pathFor;
|
|
29
34
|
private writeAtomic;
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { mkdirSync, readFileSync, readdirSync, statSync, writeFileSync, renameSync } from "node:fs";
|
|
3
|
+
import { writeFile, rename, mkdir } from "node:fs/promises";
|
|
3
4
|
import { join } from "node:path";
|
|
4
5
|
import { assertDecisionIsSound, isDecisionPending, LoopDecisionSchema, sortDecisionsByUrgency, } from "@hyperdrive.bot/fleet-protocol/fleet/decisions";
|
|
6
|
+
import { stateDbFor } from "../state/state-db.js";
|
|
5
7
|
export class LoopDecisionService {
|
|
6
8
|
constructor(options) {
|
|
7
9
|
/**
|
|
@@ -13,11 +15,29 @@ export class LoopDecisionService {
|
|
|
13
15
|
this.cached = null;
|
|
14
16
|
this.dir = join(options.paseoHome, "fleet", "decisions");
|
|
15
17
|
this.now = options.now ?? (() => new Date());
|
|
18
|
+
this.stateDb = options.stateDb === undefined ? stateDbFor(options.paseoHome) : options.stateDb;
|
|
19
|
+
this.table = this.stateDb?.doc("fleet_decisions") ?? null;
|
|
16
20
|
}
|
|
17
21
|
pathFor(id) {
|
|
18
22
|
return join(this.dir, `${id}.json`);
|
|
19
23
|
}
|
|
20
24
|
writeAtomic(decision) {
|
|
25
|
+
if (this.table && this.stateDb) {
|
|
26
|
+
// Committed with the next batch; a read flushes the batch first. Several
|
|
27
|
+
// FleetService instances share one home, so reads go to the table rather
|
|
28
|
+
// than to a per-instance cache.
|
|
29
|
+
void this.table.put("", decision.id, decision).catch(() => undefined);
|
|
30
|
+
const target = this.pathFor(decision.id);
|
|
31
|
+
const stateDb = this.stateDb;
|
|
32
|
+
stateDb.mirror.write(`decision:${decision.id}`, async () => {
|
|
33
|
+
const rev = stateDb.mirrorRev();
|
|
34
|
+
await mkdir(this.dir, { recursive: true });
|
|
35
|
+
await writeFile(`${target}.tmp`, JSON.stringify(decision, null, 2), "utf8");
|
|
36
|
+
await rename(`${target}.tmp`, target);
|
|
37
|
+
await stateDb.manifest.record("fleet_decisions", target, rev);
|
|
38
|
+
});
|
|
39
|
+
return;
|
|
40
|
+
}
|
|
21
41
|
mkdirSync(this.dir, { recursive: true });
|
|
22
42
|
const target = this.pathFor(decision.id);
|
|
23
43
|
const temp = `${target}.tmp`;
|
|
@@ -58,6 +78,16 @@ export class LoopDecisionService {
|
|
|
58
78
|
return decision;
|
|
59
79
|
}
|
|
60
80
|
readAll() {
|
|
81
|
+
if (this.table) {
|
|
82
|
+
const decisions = [];
|
|
83
|
+
for (const row of this.table.loadAll("")) {
|
|
84
|
+
const parsed = LoopDecisionSchema.safeParse(JSON.parse(row.body));
|
|
85
|
+
if (parsed.success) {
|
|
86
|
+
decisions.push(parsed.data);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return decisions;
|
|
90
|
+
}
|
|
61
91
|
let mtimeMs;
|
|
62
92
|
try {
|
|
63
93
|
mtimeMs = statSync(this.dir).mtimeMs;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import type { Logger } from "pino";
|
|
3
|
+
import { type StateDb } from "./state/state-db.js";
|
|
3
4
|
import type { AgentManager } from "./agent/agent-manager.js";
|
|
4
5
|
import type { AgentProvider } from "./agent/agent-sdk-types.js";
|
|
5
6
|
import type { ProviderSnapshotManager } from "./agent/provider-snapshot-manager.js";
|
|
@@ -156,6 +157,98 @@ declare const LoopRecordSchema: z.ZodObject<{
|
|
|
156
157
|
activeWorkerAgentId: z.ZodNullable<z.ZodString>;
|
|
157
158
|
activeVerifierAgentId: z.ZodNullable<z.ZodString>;
|
|
158
159
|
}, z.core.$strip>;
|
|
160
|
+
export declare const StoredLoopsSchema: z.ZodArray<z.ZodObject<{
|
|
161
|
+
id: z.ZodString;
|
|
162
|
+
name: z.ZodNullable<z.ZodString>;
|
|
163
|
+
prompt: z.ZodString;
|
|
164
|
+
cwd: z.ZodString;
|
|
165
|
+
provider: z.ZodString;
|
|
166
|
+
model: z.ZodNullable<z.ZodString>;
|
|
167
|
+
modeId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
168
|
+
workerProvider: z.ZodNullable<z.ZodString>;
|
|
169
|
+
workerModel: z.ZodNullable<z.ZodString>;
|
|
170
|
+
verifierProvider: z.ZodNullable<z.ZodString>;
|
|
171
|
+
verifierModel: z.ZodNullable<z.ZodString>;
|
|
172
|
+
verifierModeId: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
173
|
+
verifyPrompt: z.ZodNullable<z.ZodString>;
|
|
174
|
+
verifyChecks: z.ZodArray<z.ZodString>;
|
|
175
|
+
archive: z.ZodBoolean;
|
|
176
|
+
sleepMs: z.ZodNumber;
|
|
177
|
+
maxIterations: z.ZodNullable<z.ZodNumber>;
|
|
178
|
+
maxTimeMs: z.ZodNullable<z.ZodNumber>;
|
|
179
|
+
status: z.ZodEnum<{
|
|
180
|
+
running: "running";
|
|
181
|
+
failed: "failed";
|
|
182
|
+
succeeded: "succeeded";
|
|
183
|
+
stopped: "stopped";
|
|
184
|
+
}>;
|
|
185
|
+
createdAt: z.ZodString;
|
|
186
|
+
updatedAt: z.ZodString;
|
|
187
|
+
startedAt: z.ZodString;
|
|
188
|
+
completedAt: z.ZodNullable<z.ZodString>;
|
|
189
|
+
stopRequestedAt: z.ZodNullable<z.ZodString>;
|
|
190
|
+
iterations: z.ZodArray<z.ZodObject<{
|
|
191
|
+
index: z.ZodNumber;
|
|
192
|
+
workerAgentId: z.ZodNullable<z.ZodString>;
|
|
193
|
+
workerStartedAt: z.ZodString;
|
|
194
|
+
workerCompletedAt: z.ZodNullable<z.ZodString>;
|
|
195
|
+
verifierAgentId: z.ZodNullable<z.ZodString>;
|
|
196
|
+
status: z.ZodEnum<{
|
|
197
|
+
running: "running";
|
|
198
|
+
failed: "failed";
|
|
199
|
+
succeeded: "succeeded";
|
|
200
|
+
stopped: "stopped";
|
|
201
|
+
}>;
|
|
202
|
+
workerOutcome: z.ZodNullable<z.ZodEnum<{
|
|
203
|
+
completed: "completed";
|
|
204
|
+
failed: "failed";
|
|
205
|
+
canceled: "canceled";
|
|
206
|
+
}>>;
|
|
207
|
+
failureReason: z.ZodNullable<z.ZodString>;
|
|
208
|
+
verifyChecks: z.ZodArray<z.ZodObject<{
|
|
209
|
+
command: z.ZodString;
|
|
210
|
+
exitCode: z.ZodNumber;
|
|
211
|
+
passed: z.ZodBoolean;
|
|
212
|
+
stdout: z.ZodString;
|
|
213
|
+
stderr: z.ZodString;
|
|
214
|
+
startedAt: z.ZodString;
|
|
215
|
+
completedAt: z.ZodString;
|
|
216
|
+
}, z.core.$strip>>;
|
|
217
|
+
verifyPrompt: z.ZodNullable<z.ZodObject<{
|
|
218
|
+
passed: z.ZodBoolean;
|
|
219
|
+
reason: z.ZodString;
|
|
220
|
+
verifierAgentId: z.ZodNullable<z.ZodString>;
|
|
221
|
+
startedAt: z.ZodString;
|
|
222
|
+
completedAt: z.ZodString;
|
|
223
|
+
}, z.core.$strip>>;
|
|
224
|
+
}, z.core.$strip>>;
|
|
225
|
+
logs: z.ZodArray<z.ZodObject<{
|
|
226
|
+
seq: z.ZodNumber;
|
|
227
|
+
timestamp: z.ZodString;
|
|
228
|
+
iteration: z.ZodNullable<z.ZodNumber>;
|
|
229
|
+
source: z.ZodEnum<{
|
|
230
|
+
loop: "loop";
|
|
231
|
+
worker: "worker";
|
|
232
|
+
verifier: "verifier";
|
|
233
|
+
"verify-check": "verify-check";
|
|
234
|
+
}>;
|
|
235
|
+
level: z.ZodEnum<{
|
|
236
|
+
error: "error";
|
|
237
|
+
info: "info";
|
|
238
|
+
}>;
|
|
239
|
+
text: z.ZodString;
|
|
240
|
+
}, z.core.$strip>>;
|
|
241
|
+
nextLogSeq: z.ZodNumber;
|
|
242
|
+
activeIteration: z.ZodNullable<z.ZodNumber>;
|
|
243
|
+
activeWorkerAgentId: z.ZodNullable<z.ZodString>;
|
|
244
|
+
activeVerifierAgentId: z.ZodNullable<z.ZodString>;
|
|
245
|
+
}, z.core.$strip>>;
|
|
246
|
+
/**
|
|
247
|
+
* Log entries a loop keeps. The oldest are dropped past this, in memory and on
|
|
248
|
+
* disk; a long-running loop used to grow its log without bound and rewrite all
|
|
249
|
+
* of it on every persist.
|
|
250
|
+
*/
|
|
251
|
+
export declare const LOOP_LOGS_RETAINED = 5000;
|
|
159
252
|
export type LoopStatus = z.infer<typeof LoopRecordSchema>["status"];
|
|
160
253
|
export type LoopLogEntry = z.infer<typeof LoopLogEntrySchema>;
|
|
161
254
|
export type LoopVerifyCheckResult = z.infer<typeof LoopVerifyCheckResultSchema>;
|
|
@@ -204,11 +297,13 @@ export declare class LoopService {
|
|
|
204
297
|
private readonly loops;
|
|
205
298
|
private persistQueue;
|
|
206
299
|
private readonly running;
|
|
300
|
+
private readonly sqlite;
|
|
207
301
|
constructor(options: {
|
|
208
302
|
paseoHome: string;
|
|
209
303
|
agentManager: AgentManager;
|
|
210
304
|
logger: Logger;
|
|
211
305
|
providerSnapshotManager: CreateConfigResolver;
|
|
306
|
+
stateDb?: StateDb | null;
|
|
212
307
|
});
|
|
213
308
|
initialize(): Promise<void>;
|
|
214
309
|
runLoop(input: LoopRunOptions): Promise<LoopRecord>;
|
|
@@ -3,6 +3,7 @@ import { promises as fs } from "node:fs";
|
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { z } from "zod";
|
|
5
5
|
import { writeJsonFileAtomic } from "./atomic-file.js";
|
|
6
|
+
import { ScopeSync } from "./state/state-db.js";
|
|
6
7
|
import { curateAgentActivity } from "./agent/activity-curator.js";
|
|
7
8
|
import { getStructuredAgentResponse } from "./agent/agent-response-loop.js";
|
|
8
9
|
import { execCommand, platformShell } from "../utils/spawn.js";
|
|
@@ -81,7 +82,57 @@ const LoopRecordSchema = z.object({
|
|
|
81
82
|
activeWorkerAgentId: z.string().nullable(),
|
|
82
83
|
activeVerifierAgentId: z.string().nullable(),
|
|
83
84
|
});
|
|
84
|
-
const StoredLoopsSchema = z.array(LoopRecordSchema);
|
|
85
|
+
export const StoredLoopsSchema = z.array(LoopRecordSchema);
|
|
86
|
+
/**
|
|
87
|
+
* Log entries a loop keeps. The oldest are dropped past this, in memory and on
|
|
88
|
+
* disk; a long-running loop used to grow its log without bound and rewrite all
|
|
89
|
+
* of it on every persist.
|
|
90
|
+
*/
|
|
91
|
+
export const LOOP_LOGS_RETAINED = 5000;
|
|
92
|
+
/**
|
|
93
|
+
* Loops in `state.sqlite`: the loop (iterations included, bounded by
|
|
94
|
+
* maxIterations) in `loops`, each log entry as a row in `loop_logs`. A persist
|
|
95
|
+
* writes only the loops and log entries that changed. loops/loops.json is
|
|
96
|
+
* mirrored.
|
|
97
|
+
*/
|
|
98
|
+
class SqliteLoopStore {
|
|
99
|
+
constructor(state, storePath) {
|
|
100
|
+
this.state = state;
|
|
101
|
+
this.storePath = storePath;
|
|
102
|
+
this.loopsTable = state.doc("loops");
|
|
103
|
+
this.loops = new ScopeSync(this.loopsTable);
|
|
104
|
+
this.logs = new ScopeSync(state.doc("loop_logs"));
|
|
105
|
+
}
|
|
106
|
+
load() {
|
|
107
|
+
const loopRows = this.loopsTable.loadAll("");
|
|
108
|
+
const logRows = this.state.doc("loop_logs").loadAll();
|
|
109
|
+
this.loops.seed(loopRows);
|
|
110
|
+
this.logs.seed(logRows);
|
|
111
|
+
const logsByLoop = new Map();
|
|
112
|
+
for (const row of logRows) {
|
|
113
|
+
const list = logsByLoop.get(row.scope) ?? [];
|
|
114
|
+
list.push(JSON.parse(row.body));
|
|
115
|
+
logsByLoop.set(row.scope, list);
|
|
116
|
+
}
|
|
117
|
+
return loopRows.map((row) => {
|
|
118
|
+
const loop = JSON.parse(row.body);
|
|
119
|
+
loop.logs = logsByLoop.get(row.id) ?? [];
|
|
120
|
+
return loop;
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
async save(records) {
|
|
124
|
+
await Promise.all([
|
|
125
|
+
this.loops.sync("", records.map(({ logs: _logs, ...rest }) => ({ id: rest.id, value: rest })), { prune: true }),
|
|
126
|
+
...records.map((record) => this.logs.sync(record.id, record.logs.map((entry) => ({ id: String(entry.seq).padStart(10, "0"), value: entry })), { prune: true })),
|
|
127
|
+
this.logs.dropScopesExcept(new Set(records.map((record) => record.id))),
|
|
128
|
+
]);
|
|
129
|
+
const rev = this.state.mirrorRev();
|
|
130
|
+
this.state.mirror.write(`loops:${this.storePath}`, async () => {
|
|
131
|
+
await writeJsonFileAtomic(this.storePath, records);
|
|
132
|
+
await this.state.manifest.record("loops", this.storePath, rev);
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
}
|
|
85
136
|
function nowIso() {
|
|
86
137
|
return new Date().toISOString();
|
|
87
138
|
}
|
|
@@ -214,6 +265,7 @@ export class LoopService {
|
|
|
214
265
|
this.persistQueue = Promise.resolve();
|
|
215
266
|
this.running = new Map();
|
|
216
267
|
this.storePath = path.join(options.paseoHome, "loops", "loops.json");
|
|
268
|
+
this.sqlite = options.stateDb ? new SqliteLoopStore(options.stateDb, this.storePath) : null;
|
|
217
269
|
this.logger = options.logger.child({ module: "loop-service" });
|
|
218
270
|
}
|
|
219
271
|
async initialize() {
|
|
@@ -222,8 +274,10 @@ export class LoopService {
|
|
|
222
274
|
}
|
|
223
275
|
this.loops.clear();
|
|
224
276
|
try {
|
|
225
|
-
const raw =
|
|
226
|
-
|
|
277
|
+
const raw = this.sqlite
|
|
278
|
+
? this.sqlite.load()
|
|
279
|
+
: JSON.parse(await fs.readFile(this.storePath, "utf8"));
|
|
280
|
+
const parsed = StoredLoopsSchema.parse(raw);
|
|
227
281
|
for (const record of parsed) {
|
|
228
282
|
if (record.status === "running") {
|
|
229
283
|
const recovered = cloneLoop(record);
|
|
@@ -737,6 +791,9 @@ export class LoopService {
|
|
|
737
791
|
});
|
|
738
792
|
loop.nextLogSeq += 1;
|
|
739
793
|
loop.updatedAt = nowIso();
|
|
794
|
+
if (loop.logs.length > LOOP_LOGS_RETAINED) {
|
|
795
|
+
loop.logs.splice(0, loop.logs.length - LOOP_LOGS_RETAINED);
|
|
796
|
+
}
|
|
740
797
|
}
|
|
741
798
|
requireLoop(idOrPrefix) {
|
|
742
799
|
const trimmed = idOrPrefix.trim();
|
|
@@ -759,6 +816,10 @@ export class LoopService {
|
|
|
759
816
|
async persist() {
|
|
760
817
|
const nextPersist = this.persistQueue.then(async () => {
|
|
761
818
|
const records = Array.from(this.loops.values()).sort((left, right) => left.createdAt.localeCompare(right.createdAt));
|
|
819
|
+
if (this.sqlite) {
|
|
820
|
+
await this.sqlite.save(records);
|
|
821
|
+
return;
|
|
822
|
+
}
|
|
762
823
|
await writeJsonFileAtomic(this.storePath, records);
|
|
763
824
|
return;
|
|
764
825
|
});
|
|
@@ -53,8 +53,9 @@ export async function backfillWorkspaceIdForLegacyAgents(options) {
|
|
|
53
53
|
if (!workspaceId) {
|
|
54
54
|
continue;
|
|
55
55
|
}
|
|
56
|
-
await options.agentStorage.
|
|
57
|
-
|
|
56
|
+
const written = await options.agentStorage.update(record.id, (latest) => latest.workspaceId ? null : { ...latest, workspaceId });
|
|
57
|
+
if (written)
|
|
58
|
+
migrated += 1;
|
|
58
59
|
}
|
|
59
60
|
if (migrated > 0) {
|
|
60
61
|
options.logger.info({ migrated }, "Backfilled workspaceId for legacy agent records");
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type pino from "pino";
|
|
2
|
+
import type { StateDb } from "../state/state-db.js";
|
|
2
3
|
/**
|
|
3
4
|
* Store for Expo push tokens.
|
|
4
5
|
*
|
|
@@ -13,7 +14,9 @@ export declare class PushTokenStore {
|
|
|
13
14
|
private tokens;
|
|
14
15
|
private readonly filePath;
|
|
15
16
|
private sink;
|
|
16
|
-
|
|
17
|
+
private readonly stateDb;
|
|
18
|
+
private readonly table;
|
|
19
|
+
constructor(logger: pino.Logger, filePath: string, stateDb?: StateDb | null);
|
|
17
20
|
/**
|
|
18
21
|
* Attach an optional sink that gets notified on each successful addToken.
|
|
19
22
|
* Used by bootstrap to bridge legacy WebSocket-registered Expo tokens into
|
|
@@ -25,5 +28,6 @@ export declare class PushTokenStore {
|
|
|
25
28
|
getAllTokens(): string[];
|
|
26
29
|
private loadFromDisk;
|
|
27
30
|
private persist;
|
|
31
|
+
private writeFile;
|
|
28
32
|
}
|
|
29
33
|
//# sourceMappingURL=token-store.d.ts.map
|
|
@@ -1,12 +1,19 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from "node:fs";
|
|
2
2
|
import { ensurePrivateFile, writePrivateFileAtomicSync } from "../private-files.js";
|
|
3
3
|
export class PushTokenStore {
|
|
4
|
-
constructor(logger, filePath) {
|
|
4
|
+
constructor(logger, filePath, stateDb = null) {
|
|
5
5
|
this.tokens = new Set();
|
|
6
6
|
this.sink = null;
|
|
7
7
|
this.logger = logger.child({ component: "token-store" });
|
|
8
8
|
this.filePath = filePath;
|
|
9
|
-
this.
|
|
9
|
+
this.stateDb = stateDb;
|
|
10
|
+
this.table = stateDb?.doc("push_tokens") ?? null;
|
|
11
|
+
if (this.table) {
|
|
12
|
+
this.tokens = new Set(this.table.loadAll("").map((row) => row.id));
|
|
13
|
+
}
|
|
14
|
+
else {
|
|
15
|
+
this.loadFromDisk();
|
|
16
|
+
}
|
|
10
17
|
}
|
|
11
18
|
/**
|
|
12
19
|
* Attach an optional sink that gets notified on each successful addToken.
|
|
@@ -23,7 +30,7 @@ export class PushTokenStore {
|
|
|
23
30
|
if (this.tokens.has(normalized))
|
|
24
31
|
return;
|
|
25
32
|
this.tokens.add(normalized);
|
|
26
|
-
this.persist();
|
|
33
|
+
this.persist(normalized, "add");
|
|
27
34
|
this.logger.debug({ total: this.tokens.size }, "Added token");
|
|
28
35
|
try {
|
|
29
36
|
this.sink?.onTokenAdded?.(normalized);
|
|
@@ -38,7 +45,7 @@ export class PushTokenStore {
|
|
|
38
45
|
return;
|
|
39
46
|
const deleted = this.tokens.delete(normalized);
|
|
40
47
|
if (deleted) {
|
|
41
|
-
this.persist();
|
|
48
|
+
this.persist(normalized, "remove");
|
|
42
49
|
this.logger.debug({ total: this.tokens.size }, "Removed token");
|
|
43
50
|
}
|
|
44
51
|
}
|
|
@@ -64,7 +71,21 @@ export class PushTokenStore {
|
|
|
64
71
|
this.logger.warn({ err }, "Failed to load push tokens");
|
|
65
72
|
}
|
|
66
73
|
}
|
|
67
|
-
persist() {
|
|
74
|
+
persist(token, change) {
|
|
75
|
+
if (this.table && this.stateDb) {
|
|
76
|
+
const write = change === "add" ? this.table.put("", token, { token }) : this.table.remove("", token);
|
|
77
|
+
write.catch((error) => this.logger.warn({ err: error }, "Failed to persist push tokens"));
|
|
78
|
+
const stateDb = this.stateDb;
|
|
79
|
+
stateDb.mirror.write(`push-tokens:${this.filePath}`, async () => {
|
|
80
|
+
const rev = stateDb.mirrorRev();
|
|
81
|
+
this.writeFile();
|
|
82
|
+
await stateDb.manifest.record("push_tokens", this.filePath, rev);
|
|
83
|
+
});
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
this.writeFile();
|
|
87
|
+
}
|
|
88
|
+
writeFile() {
|
|
68
89
|
try {
|
|
69
90
|
const payload = JSON.stringify({ tokens: Array.from(this.tokens) }, null, 2) + "\n";
|
|
70
91
|
writePrivateFileAtomicSync(this.filePath, payload);
|
|
@@ -5,6 +5,7 @@ import { curateAgentActivity } from "../agent/activity-curator.js";
|
|
|
5
5
|
import { ensureAgentLoaded } from "../agent/agent-loading.js";
|
|
6
6
|
import { formatSystemNotificationPrompt } from "../agent/agent-prompt.js";
|
|
7
7
|
import { resolveCreateAgentTitles } from "../agent/create-agent-title.js";
|
|
8
|
+
import { stateDbFor } from "../state/state-db.js";
|
|
8
9
|
import { ScheduleStore } from "./store.js";
|
|
9
10
|
import { computeNextRunAt, validateScheduleCadence } from "./cron.js";
|
|
10
11
|
import { buildSourceRunPrompt } from "./source-prompt.js";
|
|
@@ -212,7 +213,9 @@ export class ScheduleService {
|
|
|
212
213
|
this.agentScheduleFacts = null;
|
|
213
214
|
/** Store version the facts above were built from. */
|
|
214
215
|
this.agentScheduleFactsVersion = -1;
|
|
215
|
-
this.store = new ScheduleStore(join(options.paseoHome, "schedules")
|
|
216
|
+
this.store = new ScheduleStore(join(options.paseoHome, "schedules"), {
|
|
217
|
+
stateDb: stateDbFor(options.paseoHome),
|
|
218
|
+
});
|
|
216
219
|
this.logger = options.logger.child({ module: "schedule-service" });
|
|
217
220
|
this.agentManager = options.agentManager;
|
|
218
221
|
this.agentStorage = options.agentStorage;
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { type StoredSchedule } from "@hyperdrive.bot/fleet-protocol/schedule/types";
|
|
2
|
+
import type { Logger } from "pino";
|
|
3
|
+
import { type StateDb } from "../state/state-db.js";
|
|
2
4
|
/**
|
|
3
5
|
* How many runs a schedule file keeps. Every run start and finish rewrites the
|
|
4
6
|
* whole file, and the history grew unbounded (5,510 runs, 2.3 MB in one real
|
|
@@ -35,7 +37,12 @@ export declare class ScheduleStore {
|
|
|
35
37
|
private readonly stamps;
|
|
36
38
|
private validating;
|
|
37
39
|
private revision;
|
|
38
|
-
|
|
40
|
+
/** Set when the daemon runs on state.sqlite; every public method then delegates. */
|
|
41
|
+
private readonly sqlite;
|
|
42
|
+
constructor(dir: string, options?: {
|
|
43
|
+
stateDb?: StateDb | null;
|
|
44
|
+
logger?: Logger;
|
|
45
|
+
});
|
|
39
46
|
private filePath;
|
|
40
47
|
private ensureDir;
|
|
41
48
|
private readStamp;
|
|
@@ -61,4 +68,17 @@ export declare class ScheduleStore {
|
|
|
61
68
|
*/
|
|
62
69
|
get version(): number;
|
|
63
70
|
}
|
|
71
|
+
/** Upper bound on how stale an in-place edit of a schedule file can go unseen. */
|
|
72
|
+
export declare const EXTERNAL_EDIT_RECHECK_MS = 1000;
|
|
73
|
+
/**
|
|
74
|
+
* Merge a schedule file written by someone else into the daemon's copy.
|
|
75
|
+
*
|
|
76
|
+
* Top-level fields come from whichever side has the newer `updatedAt`, as one
|
|
77
|
+
* unit, so a field the newer side cleared is not resurrected by the older one.
|
|
78
|
+
* Runs merge by id: a run on one side only is kept (the cap drops the oldest
|
|
79
|
+
* afterwards, so runs the daemon trimmed stay trimmed), and for the same id
|
|
80
|
+
* the more advanced run wins, so a completion written on either side is never
|
|
81
|
+
* replaced by the other side's "running" copy.
|
|
82
|
+
*/
|
|
83
|
+
export declare function mergeExternalSchedule(daemon: StoredSchedule, file: StoredSchedule): StoredSchedule;
|
|
64
84
|
//# sourceMappingURL=store.d.ts.map
|