arisa 5.2.7 → 5.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -2
- package/package.json +1 -1
- package/src/core/agent/agent-manager.js +79 -4
- package/src/core/agent/agent-session-lifecycle.js +29 -6
- package/src/core/agent/agent-turn-coordinator.js +143 -0
- package/src/core/agent/pi-capability-tools.js +7 -4
- package/src/core/agent/session-history-reader.js +168 -0
- package/src/core/agent/session-preload-migration.js +162 -0
- package/src/core/agent/session-rotation.js +29 -0
- package/src/core/agent/worker-tool-fanout.js +117 -0
- package/src/core/capabilities/capability-service.js +3 -2
- package/src/core/config/config-defaults.js +29 -0
- package/src/core/tasks/task-runner.js +8 -4
- package/src/core/tasks/task-store.js +47 -4
- package/src/core/tools/daemon-processes.js +7 -0
- package/src/core/tools/memory-pressure.js +2 -2
- package/src/core/tools/tool-registry.js +35 -7
- package/src/core/tools/weighted-resource-governor.js +7 -6
- package/src/official-tools.lock.json +121 -47
- package/src/runtime/doctor.js +20 -73
- package/src/runtime/obsolete-daemon-reaper.js +43 -0
- package/src/runtime/process-inspection.js +78 -0
- package/src/runtime/tool-process-supervisor.js +35 -8
- package/src/runtime/tui.js +1 -1
- package/src/transport/telegram/bot.js +3 -1
- package/src/transport/telegram/chat-queue.js +6 -2
- package/src/transport/telegram/task-dispatcher.js +36 -11
- package/src/transport/telegram/telegram-prompt-controller.js +17 -4
- package/test/agent-turn-coordinator.test.js +45 -0
- package/test/context-and-task-bounds.test.js +2 -1
- package/test/daemon-runtime.test.js +2 -4
- package/test/doctor.test.js +19 -0
- package/test/memory-pressure.test.js +7 -2
- package/test/obsolete-daemon-reaper.test.js +61 -0
- package/test/official-tool-dependencies.test.js +7 -2
- package/test/pi-compaction.test.js +21 -0
- package/test/session-history-reader.test.js +84 -0
- package/test/session-preload-migration.test.js +120 -0
- package/test/session-rotation.test.js +110 -0
- package/test/task-store.test.js +32 -0
- package/test/telegram-prompt-controller.test.js +2 -1
- package/test/telegram-task-dispatcher.test.js +66 -5
- package/test/tool-registry-run.test.js +10 -1
- package/test/weighted-resource-governor.test.js +28 -0
- package/test/worker-tool-fanout.test.js +79 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import {
|
|
2
|
+
closeSync,
|
|
3
|
+
fsyncSync,
|
|
4
|
+
openSync,
|
|
5
|
+
renameSync,
|
|
6
|
+
statSync,
|
|
7
|
+
unlinkSync,
|
|
8
|
+
writeFileSync
|
|
9
|
+
} from "node:fs";
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { randomUUID } from "node:crypto";
|
|
12
|
+
import {
|
|
13
|
+
findMostRecentSessionFile,
|
|
14
|
+
inspectSessionForPreloadMigration
|
|
15
|
+
} from "./session-history-reader.js";
|
|
16
|
+
import { normalizeSessionRotationPolicy } from "./session-rotation.js";
|
|
17
|
+
|
|
18
|
+
const contextEntryTypes = new Set([
|
|
19
|
+
"message",
|
|
20
|
+
"custom_message",
|
|
21
|
+
"branch_summary",
|
|
22
|
+
"thinking_level_change",
|
|
23
|
+
"model_change"
|
|
24
|
+
]);
|
|
25
|
+
|
|
26
|
+
function createEntry(type, parentId, fields = {}) {
|
|
27
|
+
return {
|
|
28
|
+
type,
|
|
29
|
+
...fields,
|
|
30
|
+
id: randomUUID(),
|
|
31
|
+
parentId,
|
|
32
|
+
timestamp: fields.timestamp || new Date().toISOString()
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function appendDurably(filePath, entry) {
|
|
37
|
+
writeFileSync(filePath, `${JSON.stringify(entry)}\n`, { encoding: "utf8", flag: "a" });
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function syncFile(filePath) {
|
|
41
|
+
const descriptor = openSync(filePath, "r");
|
|
42
|
+
try {
|
|
43
|
+
fsyncSync(descriptor);
|
|
44
|
+
} finally {
|
|
45
|
+
closeSync(descriptor);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function syncDirectory(directory) {
|
|
50
|
+
const descriptor = openSync(directory, "r");
|
|
51
|
+
try {
|
|
52
|
+
fsyncSync(descriptor);
|
|
53
|
+
} finally {
|
|
54
|
+
closeSync(descriptor);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function migratedEntry(source, parentId) {
|
|
59
|
+
const { id: _id, parentId: _parentId, ...fields } = source;
|
|
60
|
+
return createEntry(source.type, parentId, fields);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function createPreloadMigrationChild({
|
|
64
|
+
sessionDir,
|
|
65
|
+
cwd,
|
|
66
|
+
migration,
|
|
67
|
+
operationalNotes = "",
|
|
68
|
+
now = new Date()
|
|
69
|
+
}) {
|
|
70
|
+
const timestamp = now.toISOString();
|
|
71
|
+
const sessionId = randomUUID();
|
|
72
|
+
const filenameTimestamp = timestamp.replace(/[:.]/g, "-");
|
|
73
|
+
const targetFile = path.join(sessionDir, `${filenameTimestamp}_${sessionId}.jsonl`);
|
|
74
|
+
const temporaryFile = path.join(sessionDir, `.session-migration-${sessionId}.tmp`);
|
|
75
|
+
const header = {
|
|
76
|
+
type: "session",
|
|
77
|
+
version: 3,
|
|
78
|
+
id: sessionId,
|
|
79
|
+
timestamp,
|
|
80
|
+
cwd,
|
|
81
|
+
parentSession: migration.sourceFile
|
|
82
|
+
};
|
|
83
|
+
let parentId = null;
|
|
84
|
+
let copiedEntries = 0;
|
|
85
|
+
|
|
86
|
+
try {
|
|
87
|
+
writeFileSync(temporaryFile, `${JSON.stringify(header)}\n`, { encoding: "utf8", flag: "wx" });
|
|
88
|
+
const notes = String(operationalNotes || "").trim();
|
|
89
|
+
if (notes) {
|
|
90
|
+
const entry = createEntry("custom_message", parentId, {
|
|
91
|
+
customType: "arisa-operational-notes",
|
|
92
|
+
content: notes,
|
|
93
|
+
display: false,
|
|
94
|
+
details: { source: "session-start" }
|
|
95
|
+
});
|
|
96
|
+
appendDurably(temporaryFile, entry);
|
|
97
|
+
parentId = entry.id;
|
|
98
|
+
}
|
|
99
|
+
const handoff = createEntry("custom_message", parentId, {
|
|
100
|
+
customType: "arisa-session-handoff",
|
|
101
|
+
content: [
|
|
102
|
+
"Automatic session migration before loading. Continue from this checkpoint:",
|
|
103
|
+
"",
|
|
104
|
+
migration.summary
|
|
105
|
+
].join("\n"),
|
|
106
|
+
display: false,
|
|
107
|
+
details: { source: "preload-migration" }
|
|
108
|
+
});
|
|
109
|
+
appendDurably(temporaryFile, handoff);
|
|
110
|
+
parentId = handoff.id;
|
|
111
|
+
|
|
112
|
+
for (const sourceEntry of migration.contextEntries) {
|
|
113
|
+
if (!contextEntryTypes.has(sourceEntry.type)) continue;
|
|
114
|
+
const entry = migratedEntry(sourceEntry, parentId);
|
|
115
|
+
appendDurably(temporaryFile, entry);
|
|
116
|
+
parentId = entry.id;
|
|
117
|
+
copiedEntries += 1;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
syncFile(temporaryFile);
|
|
121
|
+
renameSync(temporaryFile, targetFile);
|
|
122
|
+
syncDirectory(sessionDir);
|
|
123
|
+
return {
|
|
124
|
+
sourceFile: migration.sourceFile,
|
|
125
|
+
sourceBytes: migration.sourceBytes,
|
|
126
|
+
targetFile,
|
|
127
|
+
targetBytes: statSync(targetFile).size,
|
|
128
|
+
copiedEntries
|
|
129
|
+
};
|
|
130
|
+
} catch (error) {
|
|
131
|
+
try {
|
|
132
|
+
unlinkSync(temporaryFile);
|
|
133
|
+
} catch {}
|
|
134
|
+
throw error;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function migrateRecentSessionBeforeLoad({
|
|
139
|
+
sessionDir,
|
|
140
|
+
cwd,
|
|
141
|
+
policy,
|
|
142
|
+
operationalNotes = ""
|
|
143
|
+
}) {
|
|
144
|
+
const normalized = normalizeSessionRotationPolicy(policy);
|
|
145
|
+
if (!normalized.enabled) return null;
|
|
146
|
+
const sourceFile = findMostRecentSessionFile(sessionDir, cwd);
|
|
147
|
+
if (!sourceFile) return null;
|
|
148
|
+
const sourceBytes = statSync(sourceFile).size;
|
|
149
|
+
if (sourceBytes <= normalized.maxPersistedBytes) return null;
|
|
150
|
+
const migration = inspectSessionForPreloadMigration(sourceFile, normalized.maxPersistedBytes);
|
|
151
|
+
if (!migration) {
|
|
152
|
+
const error = new Error("Oversized Pi session could not be migrated safely before loading");
|
|
153
|
+
error.code = "PI_SESSION_PRELOAD_MIGRATION_UNAVAILABLE";
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
156
|
+
return createPreloadMigrationChild({
|
|
157
|
+
sessionDir,
|
|
158
|
+
cwd,
|
|
159
|
+
migration,
|
|
160
|
+
operationalNotes
|
|
161
|
+
});
|
|
162
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
const defaultCompactAtPersistedBytes = 24 * 1024 * 1024;
|
|
2
|
+
const defaultMaxPersistedBytes = 32 * 1024 * 1024;
|
|
3
|
+
|
|
4
|
+
export function normalizeSessionRotationPolicy(policy = {}) {
|
|
5
|
+
const maxPersistedBytes = Math.max(1, Number(policy?.maxPersistedBytes) || defaultMaxPersistedBytes);
|
|
6
|
+
return {
|
|
7
|
+
enabled: policy?.enabled !== false,
|
|
8
|
+
compactAtPersistedBytes: Math.min(
|
|
9
|
+
maxPersistedBytes,
|
|
10
|
+
Math.max(1, Number(policy?.compactAtPersistedBytes) || defaultCompactAtPersistedBytes)
|
|
11
|
+
),
|
|
12
|
+
maxPersistedBytes
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function compactionRotationRequest(event, persistedBytes, policy = {}) {
|
|
17
|
+
const normalized = normalizeSessionRotationPolicy(policy);
|
|
18
|
+
if (!normalized.enabled || event?.type !== "compaction_end" || event.aborted || event.errorMessage) return null;
|
|
19
|
+
const summary = String(event.result?.summary || "").trim();
|
|
20
|
+
if (!summary || Math.max(0, Number(persistedBytes) || 0) <= normalized.compactAtPersistedBytes) return null;
|
|
21
|
+
return {
|
|
22
|
+
handoff: [
|
|
23
|
+
"Automatic session rotation after compaction. Continue from this checkpoint:",
|
|
24
|
+
"",
|
|
25
|
+
summary
|
|
26
|
+
].join("\n"),
|
|
27
|
+
persistedBytes: Math.max(0, Number(persistedBytes) || 0)
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
const defaults = Object.freeze({
|
|
2
|
+
enabled: true,
|
|
3
|
+
maxConcurrent: 2,
|
|
4
|
+
pressureConcurrent: 1,
|
|
5
|
+
serializePercent: 60
|
|
6
|
+
});
|
|
7
|
+
|
|
8
|
+
function boundedInteger(value, fallback, minimum, maximum) {
|
|
9
|
+
const number = Number(value);
|
|
10
|
+
return Number.isFinite(number) ? Math.min(maximum, Math.max(minimum, Math.floor(number))) : fallback;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
function boundedPercent(value, fallback) {
|
|
14
|
+
const number = Number(value);
|
|
15
|
+
return Number.isFinite(number) ? Math.min(99, Math.max(1, number)) : fallback;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function normalizeToolFanoutConfig(config = {}) {
|
|
19
|
+
const maxConcurrent = boundedInteger(config.maxConcurrent, defaults.maxConcurrent, 1, 16);
|
|
20
|
+
return {
|
|
21
|
+
enabled: config.enabled !== false,
|
|
22
|
+
maxConcurrent,
|
|
23
|
+
pressureConcurrent: Math.min(
|
|
24
|
+
maxConcurrent,
|
|
25
|
+
boundedInteger(config.pressureConcurrent, defaults.pressureConcurrent, 1, 16)
|
|
26
|
+
),
|
|
27
|
+
serializePercent: boundedPercent(config.serializePercent, defaults.serializePercent)
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class WorkerToolFanoutController {
|
|
32
|
+
constructor({ heapCircuitBreaker, logger, config = {} }) {
|
|
33
|
+
this.heapCircuitBreaker = heapCircuitBreaker;
|
|
34
|
+
this.logger = logger;
|
|
35
|
+
this.active = 0;
|
|
36
|
+
this.queue = [];
|
|
37
|
+
this.draining = false;
|
|
38
|
+
this.metrics = {
|
|
39
|
+
peakActive: 0,
|
|
40
|
+
peakQueued: 0,
|
|
41
|
+
pressureSerializations: 0,
|
|
42
|
+
rejectedAdmissions: 0,
|
|
43
|
+
completed: 0
|
|
44
|
+
};
|
|
45
|
+
this.setConfig(config);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
setConfig(config = {}) {
|
|
49
|
+
this.config = normalizeToolFanoutConfig(config);
|
|
50
|
+
this.drain();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
capacity() {
|
|
54
|
+
if (!this.config.enabled) return Number.POSITIVE_INFINITY;
|
|
55
|
+
const pressure = this.heapCircuitBreaker.sample();
|
|
56
|
+
if (pressure.percent >= this.config.serializePercent) {
|
|
57
|
+
this.metrics.pressureSerializations += 1;
|
|
58
|
+
return this.config.pressureConcurrent;
|
|
59
|
+
}
|
|
60
|
+
return this.config.maxConcurrent;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async drain() {
|
|
64
|
+
if (this.draining) return;
|
|
65
|
+
this.draining = true;
|
|
66
|
+
try {
|
|
67
|
+
while (this.queue.length && this.active < this.capacity()) {
|
|
68
|
+
const job = this.queue.shift();
|
|
69
|
+
try {
|
|
70
|
+
await this.heapCircuitBreaker.admit();
|
|
71
|
+
} catch (error) {
|
|
72
|
+
this.metrics.rejectedAdmissions += 1;
|
|
73
|
+
job.reject(error);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
this.active += 1;
|
|
77
|
+
this.metrics.peakActive = Math.max(this.metrics.peakActive, this.active);
|
|
78
|
+
job.resolve();
|
|
79
|
+
}
|
|
80
|
+
} finally {
|
|
81
|
+
this.draining = false;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
acquire() {
|
|
86
|
+
if (!this.config.enabled) return Promise.resolve();
|
|
87
|
+
return new Promise((resolve, reject) => {
|
|
88
|
+
this.queue.push({ resolve, reject });
|
|
89
|
+
this.metrics.peakQueued = Math.max(this.metrics.peakQueued, this.queue.length);
|
|
90
|
+
this.drain();
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
release() {
|
|
95
|
+
if (this.config.enabled) this.active = Math.max(0, this.active - 1);
|
|
96
|
+
this.metrics.completed += 1;
|
|
97
|
+
this.drain();
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async run(work) {
|
|
101
|
+
await this.acquire();
|
|
102
|
+
try {
|
|
103
|
+
return await work();
|
|
104
|
+
} finally {
|
|
105
|
+
this.release();
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
getDiagnostic() {
|
|
110
|
+
return {
|
|
111
|
+
...this.config,
|
|
112
|
+
active: this.active,
|
|
113
|
+
queued: this.queue.length,
|
|
114
|
+
...this.metrics
|
|
115
|
+
};
|
|
116
|
+
}
|
|
117
|
+
}
|
|
@@ -71,11 +71,12 @@ export function selectScheduledTasks(tasks = [], { status, limit = defaultSchedu
|
|
|
71
71
|
maxScheduledTaskListLimit
|
|
72
72
|
);
|
|
73
73
|
const allTasks = Array.isArray(tasks) ? tasks : [];
|
|
74
|
+
const activeStatuses = new Set(["pending", "running", "blocked_auth"]);
|
|
74
75
|
const orderedTasks = status
|
|
75
76
|
? [...allTasks].reverse()
|
|
76
77
|
: [
|
|
77
|
-
...allTasks.filter((task) =>
|
|
78
|
-
...allTasks.filter((task) =>
|
|
78
|
+
...allTasks.filter((task) => activeStatuses.has(task.status)).reverse(),
|
|
79
|
+
...allTasks.filter((task) => !activeStatuses.has(task.status)).reverse()
|
|
79
80
|
];
|
|
80
81
|
const visibleTasks = orderedTasks.slice(0, resolvedLimit);
|
|
81
82
|
return {
|
|
@@ -72,6 +72,11 @@ export const piConfigDefaults = Object.freeze({
|
|
|
72
72
|
maxSessions: 3,
|
|
73
73
|
maxPersistedBytes: 48 * 1024 * 1024
|
|
74
74
|
}),
|
|
75
|
+
sessionRotation: Object.freeze({
|
|
76
|
+
enabled: true,
|
|
77
|
+
compactAtPersistedBytes: 24 * 1024 * 1024,
|
|
78
|
+
maxPersistedBytes: 32 * 1024 * 1024
|
|
79
|
+
}),
|
|
75
80
|
heapCircuitBreaker: Object.freeze({
|
|
76
81
|
enabled: true,
|
|
77
82
|
softPercent: 70,
|
|
@@ -79,6 +84,18 @@ export const piConfigDefaults = Object.freeze({
|
|
|
79
84
|
waitMs: 15_000,
|
|
80
85
|
pollMs: 500
|
|
81
86
|
}),
|
|
87
|
+
toolFanout: Object.freeze({
|
|
88
|
+
enabled: true,
|
|
89
|
+
maxConcurrent: 2,
|
|
90
|
+
pressureConcurrent: 1,
|
|
91
|
+
serializePercent: 60
|
|
92
|
+
}),
|
|
93
|
+
turnCoordinator: Object.freeze({
|
|
94
|
+
enabled: true,
|
|
95
|
+
backgroundQueueTtlMs: 10 * 60_000,
|
|
96
|
+
interactiveQueueTtlMs: 0,
|
|
97
|
+
maxQueued: 100
|
|
98
|
+
}),
|
|
82
99
|
compaction: Object.freeze({
|
|
83
100
|
enabled: true,
|
|
84
101
|
reserveTokens: 120_000,
|
|
@@ -133,10 +150,22 @@ export function applyConfigDefaults(config) {
|
|
|
133
150
|
...piConfigDefaults.sessionCache,
|
|
134
151
|
...(configuredPi.sessionCache || {})
|
|
135
152
|
},
|
|
153
|
+
sessionRotation: {
|
|
154
|
+
...piConfigDefaults.sessionRotation,
|
|
155
|
+
...(configuredPi.sessionRotation || {})
|
|
156
|
+
},
|
|
136
157
|
heapCircuitBreaker: {
|
|
137
158
|
...piConfigDefaults.heapCircuitBreaker,
|
|
138
159
|
...(configuredPi.heapCircuitBreaker || {})
|
|
139
160
|
},
|
|
161
|
+
toolFanout: {
|
|
162
|
+
...piConfigDefaults.toolFanout,
|
|
163
|
+
...(configuredPi.toolFanout || {})
|
|
164
|
+
},
|
|
165
|
+
turnCoordinator: {
|
|
166
|
+
...piConfigDefaults.turnCoordinator,
|
|
167
|
+
...(configuredPi.turnCoordinator || {})
|
|
168
|
+
},
|
|
140
169
|
compaction: {
|
|
141
170
|
...piConfigDefaults.compaction,
|
|
142
171
|
...(configuredPi.compaction || {})
|
|
@@ -18,7 +18,8 @@ export function createTaskRunner({ taskStore, dispatch, laneKey = (task) => task
|
|
|
18
18
|
async function reportTerminalFailure(task, updated, error) {
|
|
19
19
|
const isTerminal = updated?.status === "failed"
|
|
20
20
|
|| updated?.status === "outcome_uncertain"
|
|
21
|
-
|| updated?.terminalFailure === true
|
|
21
|
+
|| updated?.terminalFailure === true
|
|
22
|
+
|| updated?.authBlockedNew === true;
|
|
22
23
|
if (!isTerminal || typeof onTerminalFailure !== "function") return;
|
|
23
24
|
try {
|
|
24
25
|
await onTerminalFailure({ task, result: updated, error });
|
|
@@ -37,9 +38,12 @@ export function createTaskRunner({ taskStore, dispatch, laneKey = (task) => task
|
|
|
37
38
|
logger?.log("tasks", `task ${task.id} completed after confirmed execution`);
|
|
38
39
|
return { taskId: task.id, status: "completed" };
|
|
39
40
|
} catch (error) {
|
|
40
|
-
const
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
const updated = error?.authBlocked === true
|
|
42
|
+
? await taskStore.blockAuth(task.id, error, error.authResolution)
|
|
43
|
+
: await taskStore.retryOrFail(task.id, error, {
|
|
44
|
+
retryable: error?.retryable !== false,
|
|
45
|
+
...(error?.outcomeUncertain === true ? { outcomeUncertain: true } : {})
|
|
46
|
+
});
|
|
43
47
|
const status = updated?.status || "missing";
|
|
44
48
|
logger?.log("tasks", `task ${task.id} ${status}: ${errorMessage(error)}`);
|
|
45
49
|
await reportTerminalFailure(task, updated, error);
|
|
@@ -170,6 +170,16 @@ function retryDelayMs(task) {
|
|
|
170
170
|
return Math.max(1, Math.round(seconds * 1000));
|
|
171
171
|
}
|
|
172
172
|
|
|
173
|
+
function normalizeAuthResolution(resolution = {}) {
|
|
174
|
+
const value = resolution && typeof resolution === "object" && !Array.isArray(resolution) ? resolution : {};
|
|
175
|
+
const retryAfterSeconds = Math.max(300, boundedPositiveNumber(value.retryAfterSeconds, 3_600, 86_400));
|
|
176
|
+
const probeArgs = value.probeArgs && typeof value.probeArgs === "object" && !Array.isArray(value.probeArgs)
|
|
177
|
+
? structuredClone(value.probeArgs)
|
|
178
|
+
: {};
|
|
179
|
+
if (JSON.stringify(probeArgs).length > 4_096) throw new Error("Authentication probe arguments are too large");
|
|
180
|
+
return { retryAfterSeconds, probeArgs };
|
|
181
|
+
}
|
|
182
|
+
|
|
173
183
|
function failTask(task, error) {
|
|
174
184
|
const failedAt = new Date().toISOString();
|
|
175
185
|
task.status = "failed";
|
|
@@ -232,7 +242,7 @@ export class TaskStore {
|
|
|
232
242
|
return this.mutate(async (tasks) => {
|
|
233
243
|
const now = Date.now();
|
|
234
244
|
const due = tasks
|
|
235
|
-
.filter((task) => task.status
|
|
245
|
+
.filter((task) => ["pending", "blocked_auth"].includes(task.status)
|
|
236
246
|
&& task.runAt
|
|
237
247
|
&& !Number.isNaN(Date.parse(task.runAt))
|
|
238
248
|
&& Date.parse(task.runAt) <= now)
|
|
@@ -275,7 +285,7 @@ export class TaskStore {
|
|
|
275
285
|
if (task.status === "running") {
|
|
276
286
|
const interruptedAt = new Date(now).toISOString();
|
|
277
287
|
if (task.claimedAt && !task.executionStartedAt) {
|
|
278
|
-
task.status = "pending";
|
|
288
|
+
task.status = task.authBlock ? "blocked_auth" : "pending";
|
|
279
289
|
task.attempts = Math.max(0, Number(task.attempts || 0) - 1);
|
|
280
290
|
task.lastError = "execution interrupted before start";
|
|
281
291
|
task.lastFailedAt = interruptedAt;
|
|
@@ -294,8 +304,11 @@ export class TaskStore {
|
|
|
294
304
|
delete task.claimedAt;
|
|
295
305
|
delete task.executionStartedAt;
|
|
296
306
|
if (task.kind === "poll_tool") {
|
|
297
|
-
task.status = "pending";
|
|
298
|
-
|
|
307
|
+
task.status = task.authBlock ? "blocked_auth" : "pending";
|
|
308
|
+
const delayMs = task.authBlock
|
|
309
|
+
? Number(task.authBlock.retryAfterSeconds || 3_600) * 1_000
|
|
310
|
+
: retryDelayMs(task);
|
|
311
|
+
task.runAt = new Date(now + delayMs).toISOString();
|
|
299
312
|
delete task.startedAt;
|
|
300
313
|
} else {
|
|
301
314
|
const nextRunAt = computeNextRunAt(task, now);
|
|
@@ -337,6 +350,7 @@ export class TaskStore {
|
|
|
337
350
|
delete task.error;
|
|
338
351
|
delete task.lastOutcome;
|
|
339
352
|
delete task.consecutiveFailures;
|
|
353
|
+
delete task.authBlock;
|
|
340
354
|
delete task.claimedAt;
|
|
341
355
|
delete task.executionStartedAt;
|
|
342
356
|
if (nextRunAt) {
|
|
@@ -354,6 +368,35 @@ export class TaskStore {
|
|
|
354
368
|
});
|
|
355
369
|
}
|
|
356
370
|
|
|
371
|
+
async blockAuth(taskId, error, resolution = {}) {
|
|
372
|
+
return this.mutate(async (tasks) => {
|
|
373
|
+
const task = tasks.find((item) => item.id === taskId);
|
|
374
|
+
if (!task) return { result: null, changed: false };
|
|
375
|
+
const now = Date.now();
|
|
376
|
+
const checkedAt = new Date(now).toISOString();
|
|
377
|
+
const normalized = normalizeAuthResolution(resolution);
|
|
378
|
+
const wasBlocked = Boolean(task.authBlock);
|
|
379
|
+
task.status = "blocked_auth";
|
|
380
|
+
task.runAt = new Date(now + (normalized.retryAfterSeconds * 1_000)).toISOString();
|
|
381
|
+
task.attempts = 0;
|
|
382
|
+
task.authBlock = {
|
|
383
|
+
firstBlockedAt: task.authBlock?.firstBlockedAt || checkedAt,
|
|
384
|
+
lastCheckedAt: checkedAt,
|
|
385
|
+
...normalized
|
|
386
|
+
};
|
|
387
|
+
task.lastError = error instanceof Error ? error.message : String(error);
|
|
388
|
+
task.lastFailedAt = checkedAt;
|
|
389
|
+
task.updatedAt = checkedAt;
|
|
390
|
+
if (task.claimedAt) task.lastClaimedAt = task.claimedAt;
|
|
391
|
+
if (task.executionStartedAt) task.lastExecutionStartedAt = task.executionStartedAt;
|
|
392
|
+
delete task.startedAt;
|
|
393
|
+
delete task.claimedAt;
|
|
394
|
+
delete task.executionStartedAt;
|
|
395
|
+
delete task.error;
|
|
396
|
+
return { result: { ...structuredClone(task), authBlockedNew: !wasBlocked } };
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
|
|
357
400
|
async retryOrFail(taskId, error, { retryable = true, outcomeUncertain = false } = {}) {
|
|
358
401
|
return this.mutate(async (tasks) => {
|
|
359
402
|
const task = tasks.find((item) => item.id === taskId);
|
|
@@ -280,6 +280,13 @@ export async function unregisterManagedDaemon(toolNameOrOptions, { scope } = {})
|
|
|
280
280
|
return { toolName: paths.toolName, scope: paths.scope, instanceId: paths.instanceId };
|
|
281
281
|
}
|
|
282
282
|
|
|
283
|
+
export async function purgeManagedDaemon(toolNameOrOptions, { scope } = {}) {
|
|
284
|
+
const paths = daemonPaths(toolNameOrOptions, scope);
|
|
285
|
+
await unregisterManagedDaemon(toolNameOrOptions, { scope });
|
|
286
|
+
await rm(paths.root, { recursive: true, force: true });
|
|
287
|
+
return { toolName: paths.toolName, scope: paths.scope, instanceId: paths.instanceId };
|
|
288
|
+
}
|
|
289
|
+
|
|
283
290
|
export async function stopManagedDaemon(toolNameOrOptions, {
|
|
284
291
|
scope,
|
|
285
292
|
signal = "SIGTERM",
|
|
@@ -43,10 +43,10 @@ export async function readMemoryPressure({
|
|
|
43
43
|
};
|
|
44
44
|
}
|
|
45
45
|
|
|
46
|
-
export function memoryPressureReason(snapshot, policy) {
|
|
46
|
+
export function memoryPressureReason(snapshot, policy, { ignoreWorkerRss = false } = {}) {
|
|
47
47
|
const mebibyte = 1024 * 1024;
|
|
48
48
|
const workerRssMb = snapshot.workerRssBytes / mebibyte;
|
|
49
|
-
if (workerRssMb > policy.maxWorkerRssMb) {
|
|
49
|
+
if (!ignoreWorkerRss && workerRssMb > policy.maxWorkerRssMb) {
|
|
50
50
|
return `worker RSS ${Math.ceil(workerRssMb)} MiB exceeds the ${policy.maxWorkerRssMb} MiB limit`;
|
|
51
51
|
}
|
|
52
52
|
if (snapshot.swapTotalBytes > 0 && snapshot.swapUsedPercent > policy.maxSwapUsedPercent) {
|
|
@@ -42,6 +42,25 @@ function concurrentExecutionKey(name, chatId, request) {
|
|
|
42
42
|
return createHash("sha256").update(serialized).digest("hex");
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
export function readyDaemonAdmission(tool, diagnostic) {
|
|
46
|
+
const reusable = Boolean(tool?.daemon)
|
|
47
|
+
&& diagnostic?.alive === true
|
|
48
|
+
&& diagnostic?.state === "ready"
|
|
49
|
+
&& diagnostic?.restart?.requested !== true;
|
|
50
|
+
return reusable ? { ignoreWorkerRss: true } : {};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function declaredDaemonScope(tool, chatId) {
|
|
54
|
+
if (!tool?.daemon) return null;
|
|
55
|
+
return tool.daemon.scope === "chat"
|
|
56
|
+
? { type: "chat", chatId }
|
|
57
|
+
: { type: "global" };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function nativeDaemonScope(tool, chatId) {
|
|
61
|
+
return tool?.daemon?.protocol === "arisa-daemon-v1" ? declaredDaemonScope(tool, chatId) : null;
|
|
62
|
+
}
|
|
63
|
+
|
|
45
64
|
function executionForLease(execution, lease) {
|
|
46
65
|
if (!execution) return null;
|
|
47
66
|
return {
|
|
@@ -399,20 +418,29 @@ export class ToolRegistry {
|
|
|
399
418
|
let leaseOutcome = {};
|
|
400
419
|
let result;
|
|
401
420
|
try {
|
|
402
|
-
|
|
421
|
+
const admissionScope = declaredDaemonScope(tool, chatId);
|
|
422
|
+
let admission = {};
|
|
423
|
+
if (admissionScope) {
|
|
424
|
+
const diagnostic = await readDaemonDiagnostic({
|
|
425
|
+
toolName: name,
|
|
426
|
+
scope: admissionScope,
|
|
427
|
+
autoStart: Boolean(tool.daemon.autoStart)
|
|
428
|
+
}).catch(() => null);
|
|
429
|
+
admission = readyDaemonAdmission(tool, diagnostic);
|
|
430
|
+
if (admission.ignoreWorkerRss) this.logger?.log("tools", `reusing ready ${name} daemon without worker RSS spawn admission`);
|
|
431
|
+
}
|
|
432
|
+
lease = await this.executionGovernor.acquire(tool.execution, name, admission);
|
|
403
433
|
this.logger?.log("tools", `running ${name}`);
|
|
404
434
|
await mkdir(tmpDir, { recursive: true });
|
|
405
435
|
const skills = await this.resolveSkills(name);
|
|
406
436
|
const enrichedRequest = { ...request, chatId, skills };
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
? { type: "chat", chatId }
|
|
410
|
-
: { type: "global" };
|
|
437
|
+
const runtimeScope = nativeDaemonScope(tool, chatId);
|
|
438
|
+
if (runtimeScope) {
|
|
411
439
|
const runtime = createDaemonRuntime({
|
|
412
440
|
toolName: name,
|
|
413
441
|
entryPath: tool.entry,
|
|
414
|
-
scope,
|
|
415
|
-
startupContext:
|
|
442
|
+
scope: runtimeScope,
|
|
443
|
+
startupContext: runtimeScope.type === "chat" ? { chatId: String(chatId) } : {},
|
|
416
444
|
autoStart: Boolean(tool.daemon.autoStart)
|
|
417
445
|
});
|
|
418
446
|
result = await runtime.submit(enrichedRequest, { onEvent });
|
|
@@ -205,11 +205,11 @@ export class WeightedResourceGovernor {
|
|
|
205
205
|
request.resolve(this.createLease(request, waitedMs));
|
|
206
206
|
}
|
|
207
207
|
|
|
208
|
-
async pressureSnapshot(label, resourceClass) {
|
|
208
|
+
async pressureSnapshot(label, resourceClass, { ignoreWorkerRss = false } = {}) {
|
|
209
209
|
const pressure = await this.memoryPressure();
|
|
210
210
|
this.lastPressure = pressure;
|
|
211
211
|
this.lastMemoryBudgetMb = this.memoryBudgetMb(pressure);
|
|
212
|
-
const reason = memoryPressureReason(pressure, this.policy);
|
|
212
|
+
const reason = memoryPressureReason(pressure, this.policy, { ignoreWorkerRss });
|
|
213
213
|
if (!reason) return pressure;
|
|
214
214
|
const error = new Error(`Tool ${label} was not started because ${reason}`);
|
|
215
215
|
error.code = "TOOL_RESOURCE_PRESSURE";
|
|
@@ -234,7 +234,7 @@ export class WeightedResourceGovernor {
|
|
|
234
234
|
const next = state.queue[0];
|
|
235
235
|
if (!next) continue;
|
|
236
236
|
try {
|
|
237
|
-
const pressure = await this.pressureSnapshot(next.label, resourceClass);
|
|
237
|
+
const pressure = await this.pressureSnapshot(next.label, resourceClass, next.admission);
|
|
238
238
|
const budgetMb = this.memoryBudgetMb(pressure);
|
|
239
239
|
next.memoryMb = this.requestedMemoryMb(next.execution, next.label, budgetMb);
|
|
240
240
|
if (next.memoryMb < this.policy.minimumToolMemoryMb || !this.canGrant(next)) continue;
|
|
@@ -253,14 +253,14 @@ export class WeightedResourceGovernor {
|
|
|
253
253
|
}
|
|
254
254
|
}
|
|
255
255
|
|
|
256
|
-
async acquire(rawExecution, label = "tool") {
|
|
256
|
+
async acquire(rawExecution, label = "tool", admission = {}) {
|
|
257
257
|
const execution = normalizeToolExecution(rawExecution);
|
|
258
258
|
if (!execution) return noopLease();
|
|
259
259
|
const resourceClass = execution.resourceClass;
|
|
260
260
|
const capacity = this.capacityFor(resourceClass);
|
|
261
261
|
const weight = Math.min(execution.weight, capacity);
|
|
262
262
|
const state = this.stateFor(resourceClass);
|
|
263
|
-
const pressure = await this.pressureSnapshot(label, resourceClass);
|
|
263
|
+
const pressure = await this.pressureSnapshot(label, resourceClass, admission);
|
|
264
264
|
const budgetMb = this.memoryBudgetMb(pressure);
|
|
265
265
|
const memoryMb = this.requestedMemoryMb(execution, label, budgetMb);
|
|
266
266
|
if (budgetMb < this.policy.minimumToolMemoryMb || memoryMb < this.policy.minimumToolMemoryMb) {
|
|
@@ -274,7 +274,8 @@ export class WeightedResourceGovernor {
|
|
|
274
274
|
weight,
|
|
275
275
|
memoryMb,
|
|
276
276
|
label,
|
|
277
|
-
queuedAt: this.now()
|
|
277
|
+
queuedAt: this.now(),
|
|
278
|
+
admission: { ignoreWorkerRss: admission?.ignoreWorkerRss === true }
|
|
278
279
|
};
|
|
279
280
|
if (!state.queue.length && this.canGrant(request)) {
|
|
280
281
|
return new Promise((resolve) => this.grant({ ...request, resolve }));
|