arisa 5.2.19 → 5.2.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LOW-MEMORY.md +49 -0
- package/README.md +2 -1
- package/package.json +7 -8
- package/src/core/agent/agent-turn-coordinator.js +59 -3
- package/src/core/agent/model-selection.js +4 -3
- package/src/core/agent/model-speed.js +11 -3
- package/src/core/artifacts/artifact-index.js +107 -0
- package/src/core/artifacts/artifact-store.js +15 -84
- package/src/core/artifacts/legacy-artifact-reader.js +46 -0
- package/src/core/config/config-defaults.js +4 -0
- package/src/core/tasks/task-database.js +128 -0
- package/src/core/tasks/task-store.js +42 -101
- package/src/core/tools/daemon-journal.js +115 -0
- package/src/core/tools/daemon-processes.js +10 -1
- package/src/core/tools/daemon-protocol.js +1 -1
- package/src/core/tools/daemon-worker.js +37 -5
- package/src/index.js +45 -5
- package/src/platform/paths.js +7 -1
- package/src/runtime/worker-recovery-report.js +7 -4
- package/src/transport/telegram/model-callback.js +3 -2
- package/src/transport/telegram/model-controls.js +3 -3
- package/src/transport/telegram/model-picker.js +1 -1
- package/src/transport/telegram/task-dispatcher.js +8 -8
- package/test/agent-turn-coordinator.test.js +48 -0
- package/test/artifact-index-memory.test.js +46 -0
- package/test/artifact-index-migration.test.js +88 -0
- package/test/artifact-store.test.js +3 -3
- package/test/cli-command.test.js +52 -0
- package/test/cli-memory.test.js +22 -0
- package/test/daemon-runtime.test.js +44 -4
- package/test/model-selection.test.js +11 -2
- package/test/paths.test.js +2 -0
- package/test/pi-compaction.test.js +1 -0
- package/test/pi-speed-integration.test.js +9 -8
- package/test/task-database.test.js +130 -0
- package/test/task-store.test.js +10 -5
- package/test/telegram-task-dispatcher.test.js +2 -5
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { chmodSync, mkdirSync, readFileSync, closeSync, openSync } from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { DatabaseSync } from "node:sqlite";
|
|
4
|
+
import { tasksDatabaseFile, tasksFile } from "../../platform/paths.js";
|
|
5
|
+
|
|
6
|
+
function createPrivateFile(file) {
|
|
7
|
+
mkdirSync(path.dirname(file), { recursive: true });
|
|
8
|
+
try {
|
|
9
|
+
closeSync(openSync(file, "wx", 0o600));
|
|
10
|
+
} catch (error) {
|
|
11
|
+
if (error.code !== "EEXIST") throw error;
|
|
12
|
+
}
|
|
13
|
+
chmodSync(file, 0o600);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function legacyTasks(file) {
|
|
17
|
+
let text;
|
|
18
|
+
try {
|
|
19
|
+
text = readFileSync(file, "utf8");
|
|
20
|
+
} catch (error) {
|
|
21
|
+
if (error.code === "ENOENT") return [];
|
|
22
|
+
throw error;
|
|
23
|
+
}
|
|
24
|
+
const tasks = JSON.parse(text);
|
|
25
|
+
if (!Array.isArray(tasks)) throw new Error("Legacy task storage must contain an array");
|
|
26
|
+
return tasks;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function taskValues(task) {
|
|
30
|
+
if (!task || typeof task.id !== "string" || !task.id || typeof task.status !== "string") {
|
|
31
|
+
throw new Error("Invalid task identity or status");
|
|
32
|
+
}
|
|
33
|
+
return [task.id, task.status, String(task.payload?.chatId ?? ""), task.kind ?? null,
|
|
34
|
+
Number.isFinite(Date.parse(task.runAt)) ? Date.parse(task.runAt) : null,
|
|
35
|
+
Number.isFinite(Date.parse(task.createdAt)) ? Date.parse(task.createdAt) : 0,
|
|
36
|
+
JSON.stringify(task)];
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export function insertTask(db, task) {
|
|
40
|
+
db.prepare("INSERT INTO tasks (id, status, chat_id, kind, run_at, created_at, data) VALUES (?, ?, ?, ?, ?, ?, ?)")
|
|
41
|
+
.run(...taskValues(task));
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function transaction(db, operation) {
|
|
45
|
+
db.exec("BEGIN IMMEDIATE");
|
|
46
|
+
try {
|
|
47
|
+
const result = operation();
|
|
48
|
+
db.exec("COMMIT");
|
|
49
|
+
return result;
|
|
50
|
+
} catch (error) {
|
|
51
|
+
db.exec("ROLLBACK");
|
|
52
|
+
throw error;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function initialize(db, legacyFile, migrate) {
|
|
57
|
+
const version = () => db.prepare("PRAGMA user_version").get().user_version;
|
|
58
|
+
if (version() === 1) return;
|
|
59
|
+
if (version() !== 0) throw new Error("Unsupported task database version");
|
|
60
|
+
transaction(db, () => {
|
|
61
|
+
if (version() !== 0) return;
|
|
62
|
+
db.exec(`CREATE TABLE tasks (
|
|
63
|
+
seq INTEGER PRIMARY KEY, id TEXT NOT NULL UNIQUE, status TEXT NOT NULL,
|
|
64
|
+
chat_id TEXT NOT NULL, kind TEXT, run_at INTEGER, created_at INTEGER NOT NULL, data TEXT NOT NULL
|
|
65
|
+
);
|
|
66
|
+
CREATE INDEX tasks_due ON tasks(run_at, created_at, id) WHERE status IN ('pending', 'blocked_auth');
|
|
67
|
+
CREATE INDEX tasks_chat ON tasks(chat_id, status);
|
|
68
|
+
CREATE INDEX tasks_status ON tasks(status);`);
|
|
69
|
+
for (const task of legacyTasks(legacyFile)) insertTask(db, migrate(task));
|
|
70
|
+
db.exec("PRAGMA user_version = 1");
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// All work is synchronous inside a connection/transaction. No lock is held across
|
|
75
|
+
// an await; separate processes use SQLite locking, not an in-process promise queue.
|
|
76
|
+
export function withTaskDatabase(migrate, operation, {
|
|
77
|
+
databaseFile = tasksDatabaseFile, legacyFile = tasksFile
|
|
78
|
+
} = {}) {
|
|
79
|
+
createPrivateFile(databaseFile);
|
|
80
|
+
const db = new DatabaseSync(databaseFile);
|
|
81
|
+
try {
|
|
82
|
+
db.exec("PRAGMA busy_timeout = 5000; PRAGMA cache_size = -1024; PRAGMA mmap_size = 0; PRAGMA synchronous = FULL");
|
|
83
|
+
initialize(db, legacyFile, migrate);
|
|
84
|
+
return operation(db);
|
|
85
|
+
} catch (error) {
|
|
86
|
+
throw new Error(`Task storage operation failed: ${databaseFile}`, { cause: error });
|
|
87
|
+
} finally {
|
|
88
|
+
db.close();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
export function selectTasks(db, filter = {}) {
|
|
93
|
+
if (filter.empty) return [];
|
|
94
|
+
const conditions = [];
|
|
95
|
+
const values = [];
|
|
96
|
+
for (const [key, column] of [["id", "id"], ["chatId", "chat_id"], ["status", "status"], ["kind", "kind"]]) {
|
|
97
|
+
if (filter[key]) {
|
|
98
|
+
conditions.push(`${column} = ?`);
|
|
99
|
+
values.push(String(filter[key]));
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
let order = "seq";
|
|
103
|
+
let limit = "";
|
|
104
|
+
if (filter.due) {
|
|
105
|
+
conditions.push("status IN ('pending', 'blocked_auth')", "run_at <= ?");
|
|
106
|
+
values.push(filter.due.now);
|
|
107
|
+
order = "run_at, created_at, id";
|
|
108
|
+
limit = " LIMIT ?";
|
|
109
|
+
values.push(filter.due.limit);
|
|
110
|
+
}
|
|
111
|
+
return db.prepare(`SELECT data FROM tasks ${filter.due ? "INDEXED BY tasks_due" : ""} ${conditions.length ? `WHERE ${conditions.join(" AND ")}` : ""} ORDER BY ${order}${limit}`)
|
|
112
|
+
.all(...values).map((row) => JSON.parse(row.data));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export function persistTaskChanges(db, before, tasks) {
|
|
116
|
+
const remaining = new Set(tasks.map((task) => task.id));
|
|
117
|
+
for (const id of before.keys()) {
|
|
118
|
+
if (!remaining.has(id)) db.prepare("DELETE FROM tasks WHERE id = ?").run(id);
|
|
119
|
+
}
|
|
120
|
+
for (const task of tasks) {
|
|
121
|
+
if (!before.has(task.id)) insertTask(db, task);
|
|
122
|
+
else if (before.get(task.id) !== JSON.stringify(task)) {
|
|
123
|
+
const [id, ...values] = taskValues(task);
|
|
124
|
+
db.prepare("UPDATE tasks SET status = ?, chat_id = ?, kind = ?, run_at = ?, created_at = ?, data = ? WHERE id = ?")
|
|
125
|
+
.run(...values, id);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
1
|
import crypto from "node:crypto";
|
|
4
|
-
import {
|
|
2
|
+
import { withTaskDatabase, transaction, selectTasks, persistTaskChanges } from "./task-database.js";
|
|
5
3
|
|
|
6
4
|
const DEFAULT_RETRY = Object.freeze({
|
|
7
5
|
maxAttempts: 3,
|
|
@@ -13,43 +11,6 @@ const DEFAULT_RETRY = Object.freeze({
|
|
|
13
11
|
const TERMINAL_STATUSES = new Set(["done", "failed", "outcome_uncertain"]);
|
|
14
12
|
const TERMINAL_PAYLOAD_KEYS = ["chatId", "toolName", "resourceId", "artifactId"];
|
|
15
13
|
|
|
16
|
-
const taskFileOperations = new Map();
|
|
17
|
-
|
|
18
|
-
async function serializeTaskFileOperation(operation) {
|
|
19
|
-
const previous = taskFileOperations.get(tasksFile) || Promise.resolve();
|
|
20
|
-
const current = previous.catch(() => {}).then(operation);
|
|
21
|
-
taskFileOperations.set(tasksFile, current);
|
|
22
|
-
try {
|
|
23
|
-
return await current;
|
|
24
|
-
} finally {
|
|
25
|
-
if (taskFileOperations.get(tasksFile) === current) taskFileOperations.delete(tasksFile);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
async function waitForTaskFileOperations() {
|
|
30
|
-
await (taskFileOperations.get(tasksFile) || Promise.resolve()).catch(() => {});
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
async function loadTasksFile() {
|
|
34
|
-
try {
|
|
35
|
-
const parsed = JSON.parse(await readFile(tasksFile, "utf8"));
|
|
36
|
-
return Array.isArray(parsed) ? parsed.map(migrateTask) : [];
|
|
37
|
-
} catch {
|
|
38
|
-
return [];
|
|
39
|
-
}
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
async function saveTasksFile(tasks) {
|
|
43
|
-
await mkdir(path.dirname(tasksFile), { recursive: true });
|
|
44
|
-
const temporaryFile = `${tasksFile}.${process.pid}.${crypto.randomUUID()}.tmp`;
|
|
45
|
-
try {
|
|
46
|
-
await writeFile(temporaryFile, `${JSON.stringify(tasks, null, 2)}\n`, { encoding: "utf8", mode: 0o600 });
|
|
47
|
-
await rename(temporaryFile, tasksFile);
|
|
48
|
-
} finally {
|
|
49
|
-
await rm(temporaryFile, { force: true }).catch(() => {});
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
|
|
53
14
|
function taskId() {
|
|
54
15
|
return crypto.randomUUID();
|
|
55
16
|
}
|
|
@@ -193,64 +154,51 @@ function failTask(task, error) {
|
|
|
193
154
|
}
|
|
194
155
|
|
|
195
156
|
export class TaskStore {
|
|
196
|
-
constructor() {
|
|
197
|
-
this.
|
|
157
|
+
constructor(storage = {}) {
|
|
158
|
+
this.storage = storage;
|
|
198
159
|
}
|
|
199
160
|
|
|
200
161
|
async init() {
|
|
201
|
-
|
|
162
|
+
this.read(() => undefined);
|
|
202
163
|
}
|
|
203
164
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
this.tasks = await loadTasksFile();
|
|
165
|
+
read(operation) {
|
|
166
|
+
return withTaskDatabase(migrateTask, operation, this.storage);
|
|
207
167
|
}
|
|
208
168
|
|
|
209
|
-
async mutate(operation) {
|
|
210
|
-
return
|
|
211
|
-
|
|
212
|
-
const
|
|
213
|
-
|
|
169
|
+
async mutate(operation, filter = {}) {
|
|
170
|
+
return this.read((db) => transaction(db, () => {
|
|
171
|
+
const tasks = selectTasks(db, filter);
|
|
172
|
+
const before = new Map(tasks.map((task) => [task.id, JSON.stringify(task)]));
|
|
173
|
+
const outcome = operation(tasks);
|
|
174
|
+
if (outcome?.then) throw new Error("Task mutations must be synchronous");
|
|
175
|
+
const { result, changed = true } = outcome;
|
|
176
|
+
if (changed) persistTaskChanges(db, before, tasks);
|
|
214
177
|
return result;
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
async save() {
|
|
219
|
-
const tasks = structuredClone(this.tasks || []);
|
|
220
|
-
return serializeTaskFileOperation(async () => {
|
|
221
|
-
this.tasks = tasks;
|
|
222
|
-
await saveTasksFile(tasks);
|
|
223
|
-
});
|
|
178
|
+
}));
|
|
224
179
|
}
|
|
225
180
|
|
|
226
181
|
async add(task, defaults = {}) {
|
|
227
|
-
return this.mutate(
|
|
182
|
+
return this.mutate((tasks) => {
|
|
228
183
|
const normalized = normalizeTask(task, defaults);
|
|
229
184
|
tasks.push(normalized);
|
|
230
185
|
return { result: structuredClone(normalized) };
|
|
231
|
-
});
|
|
186
|
+
}, { empty: true });
|
|
232
187
|
}
|
|
233
188
|
|
|
234
189
|
async addMany(tasksToAdd = [], defaults = {}) {
|
|
235
|
-
return this.mutate(
|
|
190
|
+
return this.mutate((tasks) => {
|
|
236
191
|
const created = tasksToAdd.map((task) => normalizeTask(task, defaults));
|
|
237
192
|
tasks.push(...created);
|
|
238
193
|
return { result: structuredClone(created), changed: created.length > 0 };
|
|
239
|
-
});
|
|
194
|
+
}, { empty: true });
|
|
240
195
|
}
|
|
241
196
|
|
|
242
197
|
async claimDue(limit = 10) {
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
&& task.runAt
|
|
248
|
-
&& !Number.isNaN(Date.parse(task.runAt))
|
|
249
|
-
&& Date.parse(task.runAt) <= now)
|
|
250
|
-
.sort((left, right) => Date.parse(left.runAt) - Date.parse(right.runAt)
|
|
251
|
-
|| Date.parse(left.createdAt || 0) - Date.parse(right.createdAt || 0)
|
|
252
|
-
|| String(left.id).localeCompare(String(right.id)))
|
|
253
|
-
.slice(0, limit);
|
|
198
|
+
const now = Date.now();
|
|
199
|
+
if (!Number.isSafeInteger(limit) || limit < 0) throw new Error("Invalid task claim limit");
|
|
200
|
+
return this.mutate((tasks) => {
|
|
201
|
+
const due = tasks;
|
|
254
202
|
|
|
255
203
|
const claimedAt = new Date(now).toISOString();
|
|
256
204
|
for (const task of due) {
|
|
@@ -263,21 +211,21 @@ export class TaskStore {
|
|
|
263
211
|
}
|
|
264
212
|
|
|
265
213
|
return { result: structuredClone(due), changed: due.length > 0 };
|
|
266
|
-
});
|
|
214
|
+
}, { due: { now, limit } });
|
|
267
215
|
}
|
|
268
216
|
|
|
269
217
|
async markExecutionStarted(taskId) {
|
|
270
|
-
return this.mutate(
|
|
218
|
+
return this.mutate((tasks) => {
|
|
271
219
|
const task = tasks.find((item) => item.id === taskId);
|
|
272
220
|
if (!task || task.status !== "running") return { result: null, changed: false };
|
|
273
221
|
task.executionStartedAt = new Date().toISOString();
|
|
274
222
|
task.updatedAt = task.executionStartedAt;
|
|
275
223
|
return { result: structuredClone(task) };
|
|
276
|
-
});
|
|
224
|
+
}, { id: taskId });
|
|
277
225
|
}
|
|
278
226
|
|
|
279
227
|
async recoverInterrupted() {
|
|
280
|
-
return this.mutate(
|
|
228
|
+
return this.mutate((tasks) => {
|
|
281
229
|
const recovered = [];
|
|
282
230
|
let compacted = false;
|
|
283
231
|
const now = Date.now();
|
|
@@ -336,7 +284,7 @@ export class TaskStore {
|
|
|
336
284
|
}
|
|
337
285
|
|
|
338
286
|
async complete(taskId) {
|
|
339
|
-
return this.mutate(
|
|
287
|
+
return this.mutate((tasks) => {
|
|
340
288
|
const task = tasks.find((item) => item.id === taskId);
|
|
341
289
|
if (!task) return { result: null, changed: false };
|
|
342
290
|
|
|
@@ -366,11 +314,11 @@ export class TaskStore {
|
|
|
366
314
|
task.updatedAt = completedAt;
|
|
367
315
|
compactTerminalTask(task);
|
|
368
316
|
return { result: structuredClone(task) };
|
|
369
|
-
});
|
|
317
|
+
}, { id: taskId });
|
|
370
318
|
}
|
|
371
319
|
|
|
372
320
|
async blockAuth(taskId, error, resolution = {}) {
|
|
373
|
-
return this.mutate(
|
|
321
|
+
return this.mutate((tasks) => {
|
|
374
322
|
const task = tasks.find((item) => item.id === taskId);
|
|
375
323
|
if (!task) return { result: null, changed: false };
|
|
376
324
|
const now = Date.now();
|
|
@@ -395,11 +343,11 @@ export class TaskStore {
|
|
|
395
343
|
delete task.executionStartedAt;
|
|
396
344
|
delete task.error;
|
|
397
345
|
return { result: { ...structuredClone(task), authBlockedNew: !wasBlocked } };
|
|
398
|
-
});
|
|
346
|
+
}, { id: taskId });
|
|
399
347
|
}
|
|
400
348
|
|
|
401
349
|
async retryOrFail(taskId, error, { retryable = true, outcomeUncertain = false } = {}) {
|
|
402
|
-
return this.mutate(
|
|
350
|
+
return this.mutate((tasks) => {
|
|
403
351
|
const task = tasks.find((item) => item.id === taskId);
|
|
404
352
|
if (!task) return { result: null, changed: false };
|
|
405
353
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -465,45 +413,38 @@ export class TaskStore {
|
|
|
465
413
|
delete task.claimedAt;
|
|
466
414
|
delete task.executionStartedAt;
|
|
467
415
|
return { result: structuredClone(task) };
|
|
468
|
-
});
|
|
416
|
+
}, { id: taskId });
|
|
469
417
|
}
|
|
470
418
|
|
|
471
419
|
async fail(taskId, error) {
|
|
472
|
-
return this.mutate(
|
|
420
|
+
return this.mutate((tasks) => {
|
|
473
421
|
const task = tasks.find((item) => item.id === taskId);
|
|
474
422
|
return task
|
|
475
423
|
? { result: failTask(task, error) }
|
|
476
424
|
: { result: null, changed: false };
|
|
477
|
-
});
|
|
425
|
+
}, { id: taskId });
|
|
478
426
|
}
|
|
479
427
|
|
|
480
428
|
async list(filter = {}) {
|
|
481
|
-
|
|
482
|
-
return this.tasks.filter((task) => {
|
|
483
|
-
if (filter.chatId && String(task.payload?.chatId) !== String(filter.chatId)) return false;
|
|
484
|
-
if (filter.status && task.status !== filter.status) return false;
|
|
485
|
-
if (filter.kind && task.kind !== filter.kind) return false;
|
|
486
|
-
return true;
|
|
487
|
-
}).map((task) => structuredClone(task));
|
|
429
|
+
return this.read((db) => selectTasks(db, filter));
|
|
488
430
|
}
|
|
489
431
|
|
|
490
432
|
async get(taskId) {
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
return task ? structuredClone(task) : null;
|
|
433
|
+
if (typeof taskId !== "string" || !taskId) return null;
|
|
434
|
+
return this.read((db) => selectTasks(db, { id: taskId })[0] || null);
|
|
494
435
|
}
|
|
495
436
|
|
|
496
437
|
async cancel(taskId) {
|
|
497
|
-
return this.mutate(
|
|
438
|
+
return this.mutate((tasks) => {
|
|
498
439
|
const index = tasks.findIndex((item) => item.id === taskId);
|
|
499
440
|
if (index === -1) return { result: null, changed: false };
|
|
500
441
|
const [task] = tasks.splice(index, 1);
|
|
501
442
|
return { result: structuredClone(task) };
|
|
502
|
-
});
|
|
443
|
+
}, { id: taskId });
|
|
503
444
|
}
|
|
504
445
|
|
|
505
446
|
async cancelAll(filter = {}) {
|
|
506
|
-
return this.mutate(
|
|
447
|
+
return this.mutate((tasks) => {
|
|
507
448
|
const removed = [];
|
|
508
449
|
const remaining = tasks.filter((task) => {
|
|
509
450
|
if (filter.chatId && String(task.payload?.chatId) !== String(filter.chatId)) return true;
|
|
@@ -514,6 +455,6 @@ export class TaskStore {
|
|
|
514
455
|
});
|
|
515
456
|
tasks.splice(0, tasks.length, ...remaining);
|
|
516
457
|
return { result: removed, changed: removed.length > 0 };
|
|
517
|
-
});
|
|
458
|
+
}, filter);
|
|
518
459
|
}
|
|
519
460
|
}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { mkdir, readdir, rename, rm, stat } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
|
|
4
|
+
const defaultRetentionMs = 24 * 60 * 60_000;
|
|
5
|
+
const defaultMaxCompleted = 2_048;
|
|
6
|
+
const operationBatchSize = 64;
|
|
7
|
+
|
|
8
|
+
function positiveInteger(value, fallback) {
|
|
9
|
+
return Number.isSafeInteger(value) && value > 0 ? value : fallback;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function resultId(file) {
|
|
13
|
+
return file.endsWith(".result.json") ? file.slice(0, -".result.json".length) : "";
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function activeId(file) {
|
|
17
|
+
return file.replace(/\.(?:request|processing)\.json$/, "");
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function entries(directory) {
|
|
21
|
+
try {
|
|
22
|
+
return await readdir(directory, { withFileTypes: true });
|
|
23
|
+
} catch (error) {
|
|
24
|
+
if (error?.code === "ENOENT") return [];
|
|
25
|
+
throw error;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
async function inBatches(items, operation) {
|
|
30
|
+
for (let index = 0; index < items.length; index += operationBatchSize) {
|
|
31
|
+
await Promise.all(items.slice(index, index + operationBatchSize).map(operation));
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async function completedRecords(directory, location, directoryEntries) {
|
|
36
|
+
const records = directoryEntries
|
|
37
|
+
.filter((entry) => entry.isFile() && entry.name.endsWith(".result.json"))
|
|
38
|
+
.map((entry) => ({
|
|
39
|
+
id: resultId(entry.name),
|
|
40
|
+
name: entry.name,
|
|
41
|
+
file: path.join(directory, entry.name),
|
|
42
|
+
location,
|
|
43
|
+
mtimeMs: 0
|
|
44
|
+
}));
|
|
45
|
+
await inBatches(records, async (record) => {
|
|
46
|
+
record.mtimeMs = (await stat(record.file)).mtimeMs;
|
|
47
|
+
});
|
|
48
|
+
return records;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
async function moveLegacyResult(record, resultsDir) {
|
|
52
|
+
const destination = path.join(resultsDir, record.name);
|
|
53
|
+
try {
|
|
54
|
+
await rename(record.file, destination);
|
|
55
|
+
} catch (error) {
|
|
56
|
+
if (error?.code !== "EEXIST") throw error;
|
|
57
|
+
await rm(record.file, { force: true });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export async function ensureDaemonJournal(paths) {
|
|
62
|
+
await Promise.all([
|
|
63
|
+
mkdir(paths.commandsDir, { recursive: true }),
|
|
64
|
+
mkdir(paths.resultsDir, { recursive: true })
|
|
65
|
+
]);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export async function maintainDaemonJournal(paths, policy = {}, { now = Date.now } = {}) {
|
|
69
|
+
const startedAt = Date.now();
|
|
70
|
+
await ensureDaemonJournal(paths);
|
|
71
|
+
const [commandEntries, resultEntries] = await Promise.all([
|
|
72
|
+
entries(paths.commandsDir),
|
|
73
|
+
entries(paths.resultsDir)
|
|
74
|
+
]);
|
|
75
|
+
const activeIds = new Set(commandEntries
|
|
76
|
+
.filter((entry) => entry.isFile() && /\.(?:request|processing)\.json$/.test(entry.name))
|
|
77
|
+
.map((entry) => activeId(entry.name)));
|
|
78
|
+
const completed = [
|
|
79
|
+
...await completedRecords(paths.resultsDir, "results", resultEntries),
|
|
80
|
+
...await completedRecords(paths.commandsDir, "legacy", commandEntries)
|
|
81
|
+
].sort((left, right) => right.mtimeMs - left.mtimeMs || left.name.localeCompare(right.name));
|
|
82
|
+
const retentionMs = positiveInteger(policy.journalRetentionMs, defaultRetentionMs);
|
|
83
|
+
const maxCompleted = positiveInteger(policy.journalMaxCompleted, defaultMaxCompleted);
|
|
84
|
+
const cutoff = now() - retentionMs;
|
|
85
|
+
const seen = new Set();
|
|
86
|
+
const retained = [];
|
|
87
|
+
const removed = [];
|
|
88
|
+
let retainedCompleted = 0;
|
|
89
|
+
|
|
90
|
+
for (const record of completed) {
|
|
91
|
+
const duplicate = seen.has(record.id);
|
|
92
|
+
const protectedByActiveJob = activeIds.has(record.id);
|
|
93
|
+
const withinRetention = record.mtimeMs >= cutoff;
|
|
94
|
+
const withinLimit = retainedCompleted < maxCompleted;
|
|
95
|
+
if (!duplicate && (protectedByActiveJob || (withinRetention && withinLimit))) {
|
|
96
|
+
seen.add(record.id);
|
|
97
|
+
retained.push(record);
|
|
98
|
+
if (!protectedByActiveJob) retainedCompleted += 1;
|
|
99
|
+
} else {
|
|
100
|
+
removed.push(record);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
await inBatches(removed, (record) => rm(record.file, { force: true }));
|
|
105
|
+
const legacy = retained.filter((record) => record.location === "legacy");
|
|
106
|
+
await inBatches(legacy, (record) => moveLegacyResult(record, paths.resultsDir));
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
active: activeIds.size,
|
|
110
|
+
completed: retained.length,
|
|
111
|
+
migrated: legacy.length,
|
|
112
|
+
pruned: removed.length,
|
|
113
|
+
scanMs: Math.max(0, Date.now() - startedAt)
|
|
114
|
+
};
|
|
115
|
+
}
|
|
@@ -56,6 +56,7 @@ export function daemonPaths(toolNameOrOptions, scope) {
|
|
|
56
56
|
instanceId: getDaemonInstanceId(identity.scope),
|
|
57
57
|
root,
|
|
58
58
|
commandsDir: path.join(root, "commands"),
|
|
59
|
+
resultsDir: path.join(root, "results"),
|
|
59
60
|
pidFile: path.join(root, "daemon.pid"),
|
|
60
61
|
metaFile: path.join(root, "daemon.meta.json"),
|
|
61
62
|
statusFile: path.join(root, "status.json"),
|
|
@@ -186,6 +187,13 @@ export async function readDaemonDiagnostic({ toolName, scope, autoStart = false
|
|
|
186
187
|
nextAt: status.nextRestartAt || null
|
|
187
188
|
},
|
|
188
189
|
disposition: daemonDisposition({ state, alive, autoStart: Boolean(autoStart), restartRequested }),
|
|
190
|
+
journal: status.journal && typeof status.journal === "object" ? {
|
|
191
|
+
active: Number(status.journal.active || 0),
|
|
192
|
+
completed: Number(status.journal.completed || 0),
|
|
193
|
+
migrated: Number(status.journal.migrated || 0),
|
|
194
|
+
pruned: Number(status.journal.pruned || 0),
|
|
195
|
+
scanMs: Number(status.journal.scanMs || 0)
|
|
196
|
+
} : null,
|
|
189
197
|
updatedAt: status.updatedAt || null,
|
|
190
198
|
logFile: paths.logFile
|
|
191
199
|
};
|
|
@@ -275,7 +283,8 @@ export async function unregisterManagedDaemon(toolNameOrOptions, { scope } = {})
|
|
|
275
283
|
rm(paths.startLockFile, { force: true }),
|
|
276
284
|
rm(paths.capabilityFile, { force: true }),
|
|
277
285
|
process.platform === "win32" ? Promise.resolve() : rm(paths.socketFile, { force: true }),
|
|
278
|
-
rm(paths.commandsDir, { recursive: true, force: true })
|
|
286
|
+
rm(paths.commandsDir, { recursive: true, force: true }),
|
|
287
|
+
rm(paths.resultsDir, { recursive: true, force: true })
|
|
279
288
|
]);
|
|
280
289
|
return { toolName: paths.toolName, scope: paths.scope, instanceId: paths.instanceId };
|
|
281
290
|
}
|
|
@@ -10,7 +10,7 @@ export function daemonJobPaths(paths, id) {
|
|
|
10
10
|
return {
|
|
11
11
|
request: path.join(paths.commandsDir, `${id}.request.json`),
|
|
12
12
|
processing: path.join(paths.commandsDir, `${id}.processing.json`),
|
|
13
|
-
result: path.join(paths.commandsDir, `${id}.result.json`)
|
|
13
|
+
result: path.join(paths.resultsDir || paths.commandsDir, `${id}.result.json`)
|
|
14
14
|
};
|
|
15
15
|
}
|
|
16
16
|
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import net from "node:net";
|
|
2
|
-
import { chmod,
|
|
2
|
+
import { chmod, readdir, rename, rm, unlink } from "node:fs/promises";
|
|
3
3
|
import {
|
|
4
4
|
ensureDaemonCapability,
|
|
5
5
|
readJson,
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
writeJson
|
|
8
8
|
} from "./daemon-processes.js";
|
|
9
9
|
import { loadDaemonPolicy } from "./daemon-policy.js";
|
|
10
|
+
import { ensureDaemonJournal, maintainDaemonJournal } from "./daemon-journal.js";
|
|
10
11
|
import {
|
|
11
12
|
DAEMON_CONTROL_FIELD,
|
|
12
13
|
DAEMON_PROTOCOL_VERSION,
|
|
@@ -37,7 +38,7 @@ export function createDaemonWorker({ toolName, paths }) {
|
|
|
37
38
|
let statusWrite = Promise.resolve();
|
|
38
39
|
|
|
39
40
|
async function ensure() {
|
|
40
|
-
await
|
|
41
|
+
await ensureDaemonJournal(paths);
|
|
41
42
|
}
|
|
42
43
|
|
|
43
44
|
async function getPid() {
|
|
@@ -96,6 +97,7 @@ export function createDaemonWorker({ toolName, paths }) {
|
|
|
96
97
|
streamBufferBytes: policy.streamBufferBytes || 1_048_576
|
|
97
98
|
};
|
|
98
99
|
const subscribers = new Map();
|
|
100
|
+
const deliveredSequences = new WeakMap();
|
|
99
101
|
const activeJobs = new Map();
|
|
100
102
|
const cancelledJobs = new Set();
|
|
101
103
|
let lastActivity = Date.now();
|
|
@@ -103,21 +105,42 @@ export function createDaemonWorker({ toolName, paths }) {
|
|
|
103
105
|
let exiting = false;
|
|
104
106
|
let acceptingWork = true;
|
|
105
107
|
let processRequested = false;
|
|
108
|
+
let journalMaintenance = Promise.resolve();
|
|
106
109
|
|
|
107
|
-
|
|
110
|
+
function maintainJournal() {
|
|
111
|
+
const operation = journalMaintenance
|
|
112
|
+
.catch(() => {})
|
|
113
|
+
.then(() => maintainDaemonJournal(paths, policy));
|
|
114
|
+
journalMaintenance = operation;
|
|
115
|
+
return operation;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const journal = await maintainJournal();
|
|
108
119
|
const capabilityToken = process.env.ARISA_DAEMON_CAPABILITY || await ensureDaemonCapability(paths);
|
|
109
120
|
await writeStatus({
|
|
110
121
|
state: "starting",
|
|
111
122
|
pid: process.pid,
|
|
112
123
|
heartbeatAt: new Date().toISOString(),
|
|
113
124
|
supportsRecovery: typeof recover === "function",
|
|
125
|
+
journal,
|
|
114
126
|
message: "Daemon work loop started; waiting for health check"
|
|
115
127
|
});
|
|
116
128
|
|
|
129
|
+
async function sendFrame(socket, frame) {
|
|
130
|
+
const delivered = deliveredSequences.get(socket) || new Map();
|
|
131
|
+
if ((delivered.get(frame.jobId) || 0) >= frame.sequence) return true;
|
|
132
|
+
const sent = await writeDaemonSocketFrame(socket, frame, ipcLimits);
|
|
133
|
+
if (sent) {
|
|
134
|
+
delivered.set(frame.jobId, frame.sequence);
|
|
135
|
+
deliveredSequences.set(socket, delivered);
|
|
136
|
+
}
|
|
137
|
+
return sent;
|
|
138
|
+
}
|
|
139
|
+
|
|
117
140
|
async function publish(frame) {
|
|
118
141
|
const sockets = [...(subscribers.get(frame.jobId) || [])];
|
|
119
142
|
for (const socket of sockets) {
|
|
120
|
-
if (!(await
|
|
143
|
+
if (!(await sendFrame(socket, frame))) subscribers.get(frame.jobId)?.delete(socket);
|
|
121
144
|
}
|
|
122
145
|
}
|
|
123
146
|
|
|
@@ -276,7 +299,7 @@ export function createDaemonWorker({ toolName, paths }) {
|
|
|
276
299
|
subscribers.get(jobId).add(socket);
|
|
277
300
|
subscribedJobs.add(jobId);
|
|
278
301
|
readJson(daemonJobPaths(paths, jobId).result, null).then((result) => {
|
|
279
|
-
if (result?.terminal) return
|
|
302
|
+
if (result?.terminal) return sendFrame(socket, result.terminal);
|
|
280
303
|
return processQueue();
|
|
281
304
|
}).catch(() => socket.destroy());
|
|
282
305
|
}
|
|
@@ -293,10 +316,19 @@ export function createDaemonWorker({ toolName, paths }) {
|
|
|
293
316
|
const heartbeatTimer = setInterval(() => {
|
|
294
317
|
writeStatus({ heartbeatAt: new Date().toISOString() }).catch(() => {});
|
|
295
318
|
}, policy.heartbeatIntervalMs);
|
|
319
|
+
const journalTimer = setInterval(() => {
|
|
320
|
+
maintainJournal()
|
|
321
|
+
.then((journal) => writeStatus({ journal }))
|
|
322
|
+
.catch((error) => writeStatus({
|
|
323
|
+
lastError: { at: new Date().toISOString(), phase: "journal", message: error?.message || String(error) }
|
|
324
|
+
}));
|
|
325
|
+
}, policy.journalSweepIntervalMs || 5 * 60_000);
|
|
326
|
+
journalTimer.unref?.();
|
|
296
327
|
const idleTimer = idleTimeoutMs > 0 ? setInterval(async () => {
|
|
297
328
|
if (processing || exiting || Date.now() - lastActivity <= idleTimeoutMs) return;
|
|
298
329
|
exiting = true;
|
|
299
330
|
clearInterval(heartbeatTimer);
|
|
331
|
+
clearInterval(journalTimer);
|
|
300
332
|
clearInterval(idleTimer);
|
|
301
333
|
await beforeExit?.();
|
|
302
334
|
await writeStatus({ state: "stopped", restartRequested: false, nextRestartAt: null, message: "Idle timeout reached" });
|