@nanhara/hara 0.122.2 → 0.122.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +108 -0
- package/README.md +15 -2
- package/dist/agent/limits.js +51 -0
- package/dist/agent/loop.js +367 -112
- package/dist/agent/repeat-guard.js +49 -12
- package/dist/checkpoints.js +9 -0
- package/dist/cli.js +2 -1
- package/dist/config.js +10 -2
- package/dist/context/agents-md.js +21 -2
- package/dist/context/mentions.js +84 -1
- package/dist/context/workspace-scope.js +49 -0
- package/dist/cron/deliver.js +61 -38
- package/dist/cron/runner.js +423 -65
- package/dist/cron/schedule.js +23 -7
- package/dist/cron/store.js +709 -43
- package/dist/fs-walk.js +279 -7
- package/dist/fs-write.js +9 -0
- package/dist/gateway/feishu.js +351 -64
- package/dist/gateway/flows-pending.js +31 -17
- package/dist/gateway/flows.js +306 -31
- package/dist/gateway/outbound-files.js +20 -6
- package/dist/gateway/runtime-state.js +1485 -0
- package/dist/gateway/serve.js +550 -242
- package/dist/gateway/sessions.js +3 -3
- package/dist/gateway/telegram.js +279 -29
- package/dist/gateway/tts.js +182 -38
- package/dist/hooks.js +22 -22
- package/dist/index.js +466 -158
- package/dist/memory/store.js +8 -5
- package/dist/org/planner.js +11 -6
- package/dist/org/projects.js +3 -3
- package/dist/process-identity.js +52 -0
- package/dist/providers/bounded-turn.js +42 -0
- package/dist/recall.js +69 -1
- package/dist/runtime.js +24 -0
- package/dist/sandbox.js +54 -4
- package/dist/search/embed.js +13 -2
- package/dist/search/hybrid.js +6 -3
- package/dist/search/semindex.js +87 -5
- package/dist/security/guardian.js +3 -15
- package/dist/security/permissions.js +11 -0
- package/dist/serve/server.js +70 -42
- package/dist/serve/sessions.js +4 -3
- package/dist/sync-sleep.js +46 -0
- package/dist/tools/agent.js +1 -1
- package/dist/tools/ask_user.js +5 -1
- package/dist/tools/builtin.js +5 -2
- package/dist/tools/codebase.js +67 -18
- package/dist/tools/computer.js +149 -68
- package/dist/tools/cron.js +72 -18
- package/dist/tools/edit.js +3 -2
- package/dist/tools/external_agent.js +66 -12
- package/dist/tools/memory.js +25 -7
- package/dist/tools/patch.js +11 -2
- package/dist/tools/registry.js +16 -1
- package/dist/tools/search.js +68 -9
- package/dist/tools/send.js +1 -1
- package/dist/tools/skill.js +1 -1
- package/dist/tools/task.js +3 -3
- package/dist/tools/web.js +43 -13
- package/dist/tui/App.js +93 -25
- package/dist/vision.js +5 -6
- package/package.json +2 -2
- package/runtime-bootstrap.cjs +22 -0
|
@@ -0,0 +1,1485 @@
|
|
|
1
|
+
// Process/runtime state for long-lived chat gateways.
|
|
2
|
+
//
|
|
3
|
+
// Two invariants live here:
|
|
4
|
+
// 1. One Hara process owns a platform connection at a time. Multiple Feishu WebSockets for the same app
|
|
5
|
+
// receive the same event and produce duplicate replies, so startup fails before opening a second socket.
|
|
6
|
+
// 2. Message ids, bounded retry budgets, and immutable no-tool flow decisions survive a restart. Every file
|
|
7
|
+
// is credential-scoped private state; raw credentials are never written here.
|
|
8
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
9
|
+
import { closeSync, constants, fstatSync, fsyncSync, linkSync, lstatSync, openSync, readdirSync, readSync, unlinkSync, writeFileSync, } from "node:fs";
|
|
10
|
+
import { homedir } from "node:os";
|
|
11
|
+
import { join } from "node:path";
|
|
12
|
+
import { atomicWriteText, bindHaraPrivateStateWritePath, FileChangedError } from "../fs-write.js";
|
|
13
|
+
import { ensurePrivateStateSubdirectory, readPrivateStateFileSnapshot, } from "../security/private-state.js";
|
|
14
|
+
import { sleepSync } from "../sync-sleep.js";
|
|
15
|
+
import { compareProcessIdentity, defaultProcessIdentity } from "../process-identity.js";
|
|
16
|
+
const PLATFORM = /^[a-z0-9][a-z0-9_-]{0,63}$/;
|
|
17
|
+
const LOCK_BYTES = 4 * 1024;
|
|
18
|
+
const LOCK_WAIT_MS = 5_000;
|
|
19
|
+
const STORE_BYTES = 256 * 1024;
|
|
20
|
+
const DEFAULT_TTL_MS = 60 * 60_000;
|
|
21
|
+
const DEFAULT_CAPACITY = 2_048;
|
|
22
|
+
const DEFAULT_STARTUP_GRACE_MS = 30_000;
|
|
23
|
+
const MAX_INBOUND_ATTEMPTS = 3;
|
|
24
|
+
const EVENT_SPOOL_BYTES = 2 * 1024 * 1024;
|
|
25
|
+
const EVENT_SPOOL_ITEM_BYTES = 128 * 1024;
|
|
26
|
+
const EVENT_SPOOL_CAPACITY = 128;
|
|
27
|
+
const EVENT_SPOOL_MAX_ATTEMPTS = 5;
|
|
28
|
+
const FLOW_RUN_FILE_BYTES = 128 * 1024;
|
|
29
|
+
const FLOW_RUN_OUTPUT_BYTES = 64 * 1024;
|
|
30
|
+
const FLOW_RUN_TTL_MS = 24 * 60 * 60_000;
|
|
31
|
+
const FLOW_RUN_CAPACITY = 512;
|
|
32
|
+
const FLOW_RUN_MAX_ATTEMPTS = 3;
|
|
33
|
+
const FLOW_RUN_FILE = /^[a-f0-9]{64}\.json$/;
|
|
34
|
+
const RUN_OUTCOME_FILE_BYTES = 32 * 1024 * 1024;
|
|
35
|
+
const RUN_OUTCOME_MAX_REPLY_BYTES = 64 * 1024;
|
|
36
|
+
const RUN_OUTCOME_MAX_FILE_BYTES = 20 * 1024 * 1024;
|
|
37
|
+
const RUN_OUTCOME_MAX_FILES = 4;
|
|
38
|
+
const RUN_OUTCOME_TTL_MS = 24 * 60 * 60_000;
|
|
39
|
+
const RUN_OUTCOME_CAPACITY = 32;
|
|
40
|
+
const RUN_OUTCOME_TOMBSTONE_CAPACITY = 2_048;
|
|
41
|
+
const RUN_OUTCOME_FILE = /^[a-f0-9]{64}\.json$/;
|
|
42
|
+
function errorCode(error) {
|
|
43
|
+
return typeof error === "object" && error !== null && "code" in error
|
|
44
|
+
? String(error.code)
|
|
45
|
+
: undefined;
|
|
46
|
+
}
|
|
47
|
+
function checkedPlatform(value) {
|
|
48
|
+
const platform = value.trim().toLowerCase();
|
|
49
|
+
if (!PLATFORM.test(platform))
|
|
50
|
+
throw new Error(`invalid gateway platform '${value}'`);
|
|
51
|
+
return platform;
|
|
52
|
+
}
|
|
53
|
+
/** Private stable namespace: same bot credential collides, different bots do not, raw identity never hits disk/logs. */
|
|
54
|
+
export function gatewayRuntimeScope(platformValue, connectionIdentity) {
|
|
55
|
+
const platform = checkedPlatform(platformValue);
|
|
56
|
+
if (!connectionIdentity)
|
|
57
|
+
return platform;
|
|
58
|
+
const digest = createHash("sha256").update(connectionIdentity).digest("hex").slice(0, 16);
|
|
59
|
+
return checkedPlatform(`${platform}-${digest}`);
|
|
60
|
+
}
|
|
61
|
+
function stateDirectory(home) {
|
|
62
|
+
return ensurePrivateStateSubdirectory(home, [".hara", "gateway"]).path;
|
|
63
|
+
}
|
|
64
|
+
function defaultPidAlive(pid) {
|
|
65
|
+
try {
|
|
66
|
+
process.kill(pid, 0);
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
return errorCode(error) === "EPERM";
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
function leaseOwnerAlive(record, pidAlive, processIdentity) {
|
|
74
|
+
if (!pidAlive(record.pid))
|
|
75
|
+
return false;
|
|
76
|
+
// Old locks and temporarily unavailable OS probes are treated as live. This may require manual recovery,
|
|
77
|
+
// but can never steal a live process' lease. New locks carry an identity and safely detect PID reuse.
|
|
78
|
+
if (!record.birthIdentity)
|
|
79
|
+
return true;
|
|
80
|
+
const current = processIdentity(record.pid);
|
|
81
|
+
return compareProcessIdentity(record.birthIdentity, current) !== "different";
|
|
82
|
+
}
|
|
83
|
+
function validLease(value, platform) {
|
|
84
|
+
if (typeof value !== "object" || value === null || Array.isArray(value))
|
|
85
|
+
return null;
|
|
86
|
+
const record = value;
|
|
87
|
+
return Number.isSafeInteger(record.pid)
|
|
88
|
+
&& record.pid > 0
|
|
89
|
+
&& typeof record.token === "string"
|
|
90
|
+
&& /^[a-f0-9]{32}$/.test(record.token)
|
|
91
|
+
&& Number.isFinite(record.startedAt)
|
|
92
|
+
&& record.startedAt > 0
|
|
93
|
+
&& record.platform === platform
|
|
94
|
+
&& (record.birthIdentity === undefined
|
|
95
|
+
|| (typeof record.birthIdentity === "string" && /^[\x20-\x7e]{1,256}$/.test(record.birthIdentity)))
|
|
96
|
+
? record
|
|
97
|
+
: null;
|
|
98
|
+
}
|
|
99
|
+
function leaseStagingPath(path, record) {
|
|
100
|
+
return `${path}.${record.pid}.${record.token}.pending`;
|
|
101
|
+
}
|
|
102
|
+
function readLease(path, platform) {
|
|
103
|
+
const noFollow = typeof constants.O_NOFOLLOW === "number" ? constants.O_NOFOLLOW : 0;
|
|
104
|
+
const nonBlock = typeof constants.O_NONBLOCK === "number" ? constants.O_NONBLOCK : 0;
|
|
105
|
+
let fd;
|
|
106
|
+
try {
|
|
107
|
+
// O_NONBLOCK is inert for regular files and prevents a hostile FIFO/device at the lock path from hanging
|
|
108
|
+
// gateway startup before fstat can reject it.
|
|
109
|
+
fd = openSync(path, constants.O_RDONLY | noFollow | nonBlock);
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
if (errorCode(error) === "ENOENT")
|
|
113
|
+
return null;
|
|
114
|
+
throw error;
|
|
115
|
+
}
|
|
116
|
+
try {
|
|
117
|
+
const info = fstatSync(fd);
|
|
118
|
+
if (!info.isFile() || info.nlink < 1 || info.nlink > 2 || info.size <= 0 || info.size > LOCK_BYTES) {
|
|
119
|
+
throw new Error(`refusing malformed gateway instance lock: ${path}`);
|
|
120
|
+
}
|
|
121
|
+
const readBounded = () => {
|
|
122
|
+
const bytes = Buffer.alloc(info.size);
|
|
123
|
+
let offset = 0;
|
|
124
|
+
while (offset < bytes.length) {
|
|
125
|
+
const count = readSync(fd, bytes, offset, bytes.length - offset, offset);
|
|
126
|
+
if (count <= 0)
|
|
127
|
+
break;
|
|
128
|
+
offset += count;
|
|
129
|
+
}
|
|
130
|
+
if (offset !== bytes.length)
|
|
131
|
+
throw new Error(`refusing concurrently changed gateway instance lock: ${path}`);
|
|
132
|
+
return bytes;
|
|
133
|
+
};
|
|
134
|
+
let parsed;
|
|
135
|
+
try {
|
|
136
|
+
const first = readBounded();
|
|
137
|
+
const second = readBounded();
|
|
138
|
+
const after = fstatSync(fd);
|
|
139
|
+
if (!first.equals(second)
|
|
140
|
+
|| !after.isFile()
|
|
141
|
+
|| after.dev !== info.dev
|
|
142
|
+
|| after.ino !== info.ino
|
|
143
|
+
|| after.mode !== info.mode
|
|
144
|
+
|| after.nlink !== info.nlink
|
|
145
|
+
|| after.size !== info.size
|
|
146
|
+
|| after.mtimeMs !== info.mtimeMs
|
|
147
|
+
|| after.ctimeMs !== info.ctimeMs)
|
|
148
|
+
throw new Error("changed while reading");
|
|
149
|
+
parsed = JSON.parse(first.toString("utf8"));
|
|
150
|
+
}
|
|
151
|
+
catch {
|
|
152
|
+
throw new Error(`refusing malformed gateway instance lock: ${path}`);
|
|
153
|
+
}
|
|
154
|
+
const record = validLease(parsed, platform);
|
|
155
|
+
if (!record)
|
|
156
|
+
throw new Error(`refusing malformed gateway instance lock: ${path}`);
|
|
157
|
+
if (info.nlink === 1)
|
|
158
|
+
return { record, dev: info.dev, ino: info.ino };
|
|
159
|
+
// `writeLease` publishes with link(2): readers may observe two names for the same inode between the
|
|
160
|
+
// atomic no-replace link and staging cleanup. Accept only that exact private, token-bound sibling.
|
|
161
|
+
const stagingPath = leaseStagingPath(path, record);
|
|
162
|
+
let staging;
|
|
163
|
+
try {
|
|
164
|
+
staging = lstatSync(stagingPath);
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
const refreshed = fstatSync(fd);
|
|
168
|
+
if (refreshed.isFile()
|
|
169
|
+
&& refreshed.nlink === 1
|
|
170
|
+
&& refreshed.dev === info.dev
|
|
171
|
+
&& refreshed.ino === info.ino)
|
|
172
|
+
return { record, dev: info.dev, ino: info.ino };
|
|
173
|
+
throw new Error(`refusing malformed gateway instance lock: ${path}`);
|
|
174
|
+
}
|
|
175
|
+
if (!staging.isFile()
|
|
176
|
+
|| staging.isSymbolicLink()
|
|
177
|
+
|| staging.nlink !== 2
|
|
178
|
+
|| staging.dev !== info.dev
|
|
179
|
+
|| staging.ino !== info.ino)
|
|
180
|
+
throw new Error(`refusing malformed gateway instance lock: ${path}`);
|
|
181
|
+
return { record, dev: info.dev, ino: info.ino, stagingPath };
|
|
182
|
+
}
|
|
183
|
+
finally {
|
|
184
|
+
closeSync(fd);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
function sameLease(left, right) {
|
|
188
|
+
return Boolean(left
|
|
189
|
+
&& right
|
|
190
|
+
&& left.dev === right.dev
|
|
191
|
+
&& left.ino === right.ino
|
|
192
|
+
&& left.record.pid === right.record.pid
|
|
193
|
+
&& left.record.token === right.record.token);
|
|
194
|
+
}
|
|
195
|
+
function unlinkLease(path, expected) {
|
|
196
|
+
let current = lstatSync(path);
|
|
197
|
+
if (!current.isFile()
|
|
198
|
+
|| current.isSymbolicLink()
|
|
199
|
+
|| current.nlink < 1
|
|
200
|
+
|| current.nlink > 2
|
|
201
|
+
|| current.dev !== expected.dev
|
|
202
|
+
|| current.ino !== expected.ino)
|
|
203
|
+
throw new Error(`gateway instance lock changed before removal: ${path}`);
|
|
204
|
+
if (current.nlink === 2) {
|
|
205
|
+
if (!expected.stagingPath)
|
|
206
|
+
throw new Error(`gateway instance lock changed before removal: ${path}`);
|
|
207
|
+
const staging = lstatSync(expected.stagingPath);
|
|
208
|
+
if (!staging.isFile()
|
|
209
|
+
|| staging.isSymbolicLink()
|
|
210
|
+
|| staging.nlink !== 2
|
|
211
|
+
|| staging.dev !== expected.dev
|
|
212
|
+
|| staging.ino !== expected.ino)
|
|
213
|
+
throw new Error(`gateway instance lock changed before removal: ${path}`);
|
|
214
|
+
unlinkSync(expected.stagingPath);
|
|
215
|
+
current = lstatSync(path);
|
|
216
|
+
if (!current.isFile()
|
|
217
|
+
|| current.isSymbolicLink()
|
|
218
|
+
|| current.nlink !== 1
|
|
219
|
+
|| current.dev !== expected.dev
|
|
220
|
+
|| current.ino !== expected.ino)
|
|
221
|
+
throw new Error(`gateway instance lock changed before removal: ${path}`);
|
|
222
|
+
}
|
|
223
|
+
unlinkSync(path);
|
|
224
|
+
}
|
|
225
|
+
function writeLease(path, record) {
|
|
226
|
+
const noFollow = typeof constants.O_NOFOLLOW === "number" ? constants.O_NOFOLLOW : 0;
|
|
227
|
+
const nonBlock = typeof constants.O_NONBLOCK === "number" ? constants.O_NONBLOCK : 0;
|
|
228
|
+
const stagingPath = leaseStagingPath(path, record);
|
|
229
|
+
let fd;
|
|
230
|
+
let published = false;
|
|
231
|
+
try {
|
|
232
|
+
// Never expose an empty/partial owner record at the canonical path. The private staging inode is fully
|
|
233
|
+
// written + fsynced first; link(2) then publishes it atomically and refuses to replace an existing owner.
|
|
234
|
+
fd = openSync(stagingPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | noFollow | nonBlock, 0o600);
|
|
235
|
+
writeFileSync(fd, JSON.stringify(record), "utf8");
|
|
236
|
+
fsyncSync(fd);
|
|
237
|
+
closeSync(fd);
|
|
238
|
+
fd = undefined;
|
|
239
|
+
linkSync(stagingPath, path);
|
|
240
|
+
published = true;
|
|
241
|
+
try {
|
|
242
|
+
unlinkSync(stagingPath);
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
// A crash or cleanup failure here is safe: readLease recognizes only this token-bound second name.
|
|
246
|
+
}
|
|
247
|
+
const owned = readLease(path, record.platform);
|
|
248
|
+
if (!owned || owned.record.token !== record.token || owned.record.pid !== record.pid) {
|
|
249
|
+
throw new Error(`gateway instance lock changed during acquisition: ${path}`);
|
|
250
|
+
}
|
|
251
|
+
return owned;
|
|
252
|
+
}
|
|
253
|
+
finally {
|
|
254
|
+
if (fd !== undefined)
|
|
255
|
+
closeSync(fd);
|
|
256
|
+
if (!published) {
|
|
257
|
+
try {
|
|
258
|
+
unlinkSync(stagingPath);
|
|
259
|
+
}
|
|
260
|
+
catch {
|
|
261
|
+
// Preserve the acquisition error; an unlinked-to staging file cannot claim the platform.
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
/** Claim the one supported gateway process for a platform. Stale owners are reclaimed only after PID proof. */
|
|
267
|
+
export function acquireGatewayInstance(platformValue, options = {}) {
|
|
268
|
+
const platform = checkedPlatform(platformValue);
|
|
269
|
+
const displayPlatform = checkedPlatform(options.displayPlatform ?? platform);
|
|
270
|
+
const dir = stateDirectory(options.home ?? homedir());
|
|
271
|
+
const lock = join(dir, `instance-${platform}.lock`);
|
|
272
|
+
const reclaim = `${lock}.reclaim`;
|
|
273
|
+
const now = options.now ?? Date.now;
|
|
274
|
+
const pidAlive = options.pidAlive ?? defaultPidAlive;
|
|
275
|
+
const processIdentity = options.processIdentity ?? defaultProcessIdentity;
|
|
276
|
+
const birthIdentity = processIdentity(process.pid);
|
|
277
|
+
const claim = {
|
|
278
|
+
pid: process.pid,
|
|
279
|
+
token: randomBytes(16).toString("hex"),
|
|
280
|
+
startedAt: now(),
|
|
281
|
+
platform,
|
|
282
|
+
...(birthIdentity ? { birthIdentity } : {}),
|
|
283
|
+
};
|
|
284
|
+
const deadline = Date.now() + LOCK_WAIT_MS;
|
|
285
|
+
for (;;) {
|
|
286
|
+
const existingGuard = readLease(reclaim, platform);
|
|
287
|
+
if (existingGuard) {
|
|
288
|
+
if (!leaseOwnerAlive(existingGuard.record, pidAlive, processIdentity)) {
|
|
289
|
+
const current = readLease(reclaim, platform);
|
|
290
|
+
if (sameLease(existingGuard, current) && current && !leaseOwnerAlive(current.record, pidAlive, processIdentity)) {
|
|
291
|
+
unlinkLease(reclaim, current);
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
if (Date.now() >= deadline)
|
|
296
|
+
throw new Error(`gateway '${displayPlatform}' instance recovery is busy; retry shortly`);
|
|
297
|
+
sleepSync(10);
|
|
298
|
+
continue;
|
|
299
|
+
}
|
|
300
|
+
try {
|
|
301
|
+
const owned = writeLease(lock, claim);
|
|
302
|
+
let released = false;
|
|
303
|
+
return () => {
|
|
304
|
+
if (released)
|
|
305
|
+
return;
|
|
306
|
+
const current = readLease(lock, platform);
|
|
307
|
+
if (sameLease(owned, current) && current)
|
|
308
|
+
unlinkLease(lock, current);
|
|
309
|
+
released = true;
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
if (errorCode(error) !== "EEXIST")
|
|
314
|
+
throw error;
|
|
315
|
+
}
|
|
316
|
+
const held = readLease(lock, platform);
|
|
317
|
+
if (!held)
|
|
318
|
+
continue;
|
|
319
|
+
if (leaseOwnerAlive(held.record, pidAlive, processIdentity)) {
|
|
320
|
+
throw new Error(`gateway '${displayPlatform}' is already running (pid ${held.record.pid}); stop that process before starting another`);
|
|
321
|
+
}
|
|
322
|
+
const guardRecord = {
|
|
323
|
+
pid: process.pid,
|
|
324
|
+
token: randomBytes(16).toString("hex"),
|
|
325
|
+
startedAt: now(),
|
|
326
|
+
platform,
|
|
327
|
+
...(birthIdentity ? { birthIdentity } : {}),
|
|
328
|
+
};
|
|
329
|
+
let guard;
|
|
330
|
+
try {
|
|
331
|
+
guard = writeLease(reclaim, guardRecord);
|
|
332
|
+
const current = readLease(lock, platform);
|
|
333
|
+
if (sameLease(held, current) && current && !leaseOwnerAlive(current.record, pidAlive, processIdentity))
|
|
334
|
+
unlinkLease(lock, current);
|
|
335
|
+
}
|
|
336
|
+
catch (error) {
|
|
337
|
+
if (errorCode(error) !== "EEXIST")
|
|
338
|
+
throw error;
|
|
339
|
+
}
|
|
340
|
+
finally {
|
|
341
|
+
if (guard) {
|
|
342
|
+
const currentGuard = readLease(reclaim, platform);
|
|
343
|
+
if (sameLease(guard, currentGuard) && currentGuard)
|
|
344
|
+
unlinkLease(reclaim, currentGuard);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
if (Date.now() >= deadline)
|
|
348
|
+
throw new Error(`gateway '${displayPlatform}' stale instance could not be reclaimed`);
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
function parseProcessedStore(snapshot, path) {
|
|
352
|
+
if (!snapshot)
|
|
353
|
+
return [];
|
|
354
|
+
let parsed;
|
|
355
|
+
try {
|
|
356
|
+
parsed = JSON.parse(snapshot.text);
|
|
357
|
+
}
|
|
358
|
+
catch {
|
|
359
|
+
throw new Error(`invalid gateway message dedupe store: ${path}`);
|
|
360
|
+
}
|
|
361
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
|
362
|
+
throw new Error(`invalid gateway message dedupe store: ${path}`);
|
|
363
|
+
}
|
|
364
|
+
const store = parsed;
|
|
365
|
+
if (store.version !== 1 || !Array.isArray(store.messages) || store.messages.length > DEFAULT_CAPACITY * 4) {
|
|
366
|
+
throw new Error(`invalid gateway message dedupe store: ${path}`);
|
|
367
|
+
}
|
|
368
|
+
const messages = [];
|
|
369
|
+
for (const entry of store.messages) {
|
|
370
|
+
if (typeof entry !== "object"
|
|
371
|
+
|| entry === null
|
|
372
|
+
|| typeof entry.id !== "string"
|
|
373
|
+
|| !entry.id
|
|
374
|
+
|| entry.id.length > 512
|
|
375
|
+
|| !Number.isFinite(entry.seenAt)
|
|
376
|
+
|| entry.seenAt <= 0)
|
|
377
|
+
throw new Error(`invalid gateway message dedupe store: ${path}`);
|
|
378
|
+
const failedAttempts = entry.failedAttempts;
|
|
379
|
+
if (failedAttempts !== undefined
|
|
380
|
+
&& (!Number.isInteger(failedAttempts) || failedAttempts < 1 || failedAttempts >= MAX_INBOUND_ATTEMPTS))
|
|
381
|
+
throw new Error(`invalid gateway message dedupe store: ${path}`);
|
|
382
|
+
messages.push({
|
|
383
|
+
id: entry.id,
|
|
384
|
+
seenAt: entry.seenAt,
|
|
385
|
+
...(failedAttempts !== undefined ? { failedAttempts } : {}),
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
return messages;
|
|
389
|
+
}
|
|
390
|
+
/** A serialized, crash-safe, bounded id cache. Completions and real-failure budgets survive restarts. */
|
|
391
|
+
export class GatewayMessageDeduper {
|
|
392
|
+
path;
|
|
393
|
+
dir;
|
|
394
|
+
startedAt;
|
|
395
|
+
now;
|
|
396
|
+
ttlMs;
|
|
397
|
+
capacity;
|
|
398
|
+
startupGraceMs;
|
|
399
|
+
inFlight = new Map();
|
|
400
|
+
failures = new Map();
|
|
401
|
+
queue = Promise.resolve();
|
|
402
|
+
constructor(platform, options) {
|
|
403
|
+
this.dir = stateDirectory(options.home ?? homedir());
|
|
404
|
+
this.path = join(this.dir, `processed-${platform}.json`);
|
|
405
|
+
this.now = options.now ?? Date.now;
|
|
406
|
+
this.startedAt = this.now();
|
|
407
|
+
this.ttlMs = Math.max(1_000, Math.min(options.ttlMs ?? DEFAULT_TTL_MS, 24 * 60 * 60_000));
|
|
408
|
+
this.capacity = Math.max(1, Math.min(options.capacity ?? DEFAULT_CAPACITY, DEFAULT_CAPACITY));
|
|
409
|
+
this.startupGraceMs = Math.max(0, Math.min(options.startupGraceMs ?? DEFAULT_STARTUP_GRACE_MS, 5 * 60_000));
|
|
410
|
+
}
|
|
411
|
+
static async open(platformValue, options = {}) {
|
|
412
|
+
const instance = new GatewayMessageDeduper(checkedPlatform(platformValue), options);
|
|
413
|
+
const snapshot = await readPrivateStateFileSnapshot(instance.path, STORE_BYTES);
|
|
414
|
+
parseProcessedStore(snapshot, instance.path); // fail closed before opening the platform socket
|
|
415
|
+
return instance;
|
|
416
|
+
}
|
|
417
|
+
async claim(messageId, createdAtMs, options = {}) {
|
|
418
|
+
// A platform may redeliver while the first callback is still running. Wait for that exact claim to settle:
|
|
419
|
+
// success means duplicate, failure means one waiter takes over. Never await while holding `queue`, because
|
|
420
|
+
// the owner's complete/release operation must use the same serializer.
|
|
421
|
+
for (;;) {
|
|
422
|
+
const attempt = await this.serialize(() => this.tryClaimOne(messageId, createdAtMs, options.durable === true));
|
|
423
|
+
if (attempt.kind === "claim")
|
|
424
|
+
return attempt.claim;
|
|
425
|
+
if (attempt.kind === "duplicate")
|
|
426
|
+
return null;
|
|
427
|
+
if (await attempt.outcome)
|
|
428
|
+
return null;
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
serialize(task) {
|
|
432
|
+
const operation = this.queue.then(task);
|
|
433
|
+
this.queue = operation.then(() => undefined, () => undefined);
|
|
434
|
+
return operation;
|
|
435
|
+
}
|
|
436
|
+
async tryClaimOne(messageId, createdAtMs, durable = false) {
|
|
437
|
+
if (messageId === undefined)
|
|
438
|
+
return { kind: "claim", claim: noOpMessageClaim() }; // adapters without stable ids keep their existing behavior
|
|
439
|
+
const id = messageId.trim();
|
|
440
|
+
if (!id || id.length > 512 || id.includes("\0"))
|
|
441
|
+
return { kind: "duplicate" };
|
|
442
|
+
if (!durable && Number.isFinite(createdAtMs) && createdAtMs < this.startedAt - this.startupGraceMs) {
|
|
443
|
+
return { kind: "duplicate" };
|
|
444
|
+
}
|
|
445
|
+
const active = this.inFlight.get(id);
|
|
446
|
+
if (active)
|
|
447
|
+
return { kind: "wait", outcome: active.outcome };
|
|
448
|
+
const now = this.now();
|
|
449
|
+
const snapshot = await readPrivateStateFileSnapshot(this.path, STORE_BYTES);
|
|
450
|
+
const cutoff = now - this.ttlMs;
|
|
451
|
+
const recent = parseProcessedStore(snapshot, this.path).filter((entry) => entry.seenAt >= cutoff);
|
|
452
|
+
const persisted = recent.find((entry) => entry.id === id);
|
|
453
|
+
if (persisted && persisted.failedAttempts === undefined)
|
|
454
|
+
return { kind: "duplicate" };
|
|
455
|
+
const memoryFailure = this.failures.get(id);
|
|
456
|
+
if (memoryFailure && now - memoryFailure.lastAt >= this.ttlMs)
|
|
457
|
+
this.failures.delete(id);
|
|
458
|
+
if (persisted?.failedAttempts !== undefined) {
|
|
459
|
+
const current = this.failures.get(id);
|
|
460
|
+
if (!current || current.attempts < persisted.failedAttempts) {
|
|
461
|
+
this.failures.set(id, { attempts: persisted.failedAttempts, lastAt: persisted.seenAt, alarmed: false });
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
const failed = this.failures.get(id);
|
|
465
|
+
if (failed && failed.attempts >= MAX_INBOUND_ATTEMPTS) {
|
|
466
|
+
await this.deadLetter(id, failed);
|
|
467
|
+
return { kind: "duplicate" };
|
|
468
|
+
}
|
|
469
|
+
let resolveOutcome;
|
|
470
|
+
const activeClaim = {
|
|
471
|
+
outcome: new Promise((resolve) => { resolveOutcome = resolve; }),
|
|
472
|
+
resolve: (completed) => resolveOutcome(completed),
|
|
473
|
+
};
|
|
474
|
+
this.inFlight.set(id, activeClaim);
|
|
475
|
+
let settlement;
|
|
476
|
+
const settle = (mode) => {
|
|
477
|
+
if (settlement)
|
|
478
|
+
return settlement;
|
|
479
|
+
settlement = this.serialize(async () => {
|
|
480
|
+
try {
|
|
481
|
+
let completed = false;
|
|
482
|
+
let exhausted = false;
|
|
483
|
+
if (mode === "complete") {
|
|
484
|
+
try {
|
|
485
|
+
await this.persistCompleted(id);
|
|
486
|
+
this.failures.delete(id);
|
|
487
|
+
completed = true;
|
|
488
|
+
}
|
|
489
|
+
catch (error) {
|
|
490
|
+
const failure = this.recordFailure(id);
|
|
491
|
+
if (failure.attempts < MAX_INBOUND_ATTEMPTS)
|
|
492
|
+
throw error;
|
|
493
|
+
await this.deadLetter(id, failure);
|
|
494
|
+
completed = true; // process-local exhaustion still breaks a loop when the state file is broken
|
|
495
|
+
exhausted = true;
|
|
496
|
+
}
|
|
497
|
+
}
|
|
498
|
+
else if (mode === "fail") {
|
|
499
|
+
const failure = this.recordFailure(id);
|
|
500
|
+
if (failure.attempts >= MAX_INBOUND_ATTEMPTS) {
|
|
501
|
+
await this.deadLetter(id, failure);
|
|
502
|
+
completed = true;
|
|
503
|
+
exhausted = true;
|
|
504
|
+
}
|
|
505
|
+
else {
|
|
506
|
+
await this.persistFailure(id, failure);
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
if (this.inFlight.get(id) === activeClaim)
|
|
510
|
+
this.inFlight.delete(id);
|
|
511
|
+
activeClaim.resolve(completed);
|
|
512
|
+
return { exhausted };
|
|
513
|
+
}
|
|
514
|
+
catch (error) {
|
|
515
|
+
if (this.inFlight.get(id) === activeClaim)
|
|
516
|
+
this.inFlight.delete(id);
|
|
517
|
+
activeClaim.resolve(false);
|
|
518
|
+
throw error;
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
return settlement;
|
|
522
|
+
};
|
|
523
|
+
return {
|
|
524
|
+
kind: "claim",
|
|
525
|
+
claim: {
|
|
526
|
+
complete: async () => { await settle("complete"); },
|
|
527
|
+
release: async () => { await settle("release"); },
|
|
528
|
+
fail: async () => (await settle("fail")).exhausted,
|
|
529
|
+
},
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
recordFailure(id) {
|
|
533
|
+
const now = this.now();
|
|
534
|
+
const current = this.failures.get(id);
|
|
535
|
+
this.failures.delete(id);
|
|
536
|
+
const next = { attempts: (current?.attempts ?? 0) + 1, lastAt: now, alarmed: current?.alarmed ?? false };
|
|
537
|
+
this.failures.set(id, next);
|
|
538
|
+
// The same bounded capacity as the persistent store prevents rotating failed ids from growing memory.
|
|
539
|
+
while (this.failures.size > this.capacity) {
|
|
540
|
+
const oldest = this.failures.keys().next().value;
|
|
541
|
+
if (!oldest)
|
|
542
|
+
break;
|
|
543
|
+
this.failures.delete(oldest);
|
|
544
|
+
}
|
|
545
|
+
return next;
|
|
546
|
+
}
|
|
547
|
+
async deadLetter(id, failure) {
|
|
548
|
+
let persisted = false;
|
|
549
|
+
try {
|
|
550
|
+
await this.persistCompleted(id);
|
|
551
|
+
persisted = true;
|
|
552
|
+
}
|
|
553
|
+
catch {
|
|
554
|
+
// Continue with the process-local exhausted state. A broken disk must not turn a full-auto coding event
|
|
555
|
+
// back into an infinite execution loop; the explicit alarm below makes durability loss observable.
|
|
556
|
+
}
|
|
557
|
+
if (!failure.alarmed) {
|
|
558
|
+
failure.alarmed = true;
|
|
559
|
+
const digest = createHash("sha256").update(id).digest("hex").slice(0, 12);
|
|
560
|
+
console.error(`hara gateway: ALERT inbound event ${digest} stopped after ${MAX_INBOUND_ATTEMPTS} failed attempts — acknowledged to break the execution loop${persisted ? "" : " (dedupe persistence unavailable)"}`);
|
|
561
|
+
}
|
|
562
|
+
if (persisted)
|
|
563
|
+
this.failures.delete(id);
|
|
564
|
+
}
|
|
565
|
+
async persistCompleted(id) {
|
|
566
|
+
const now = this.now();
|
|
567
|
+
const snapshot = await readPrivateStateFileSnapshot(this.path, STORE_BYTES);
|
|
568
|
+
const cutoff = now - this.ttlMs;
|
|
569
|
+
const recent = parseProcessedStore(snapshot, this.path).filter((entry) => entry.seenAt >= cutoff && entry.id !== id);
|
|
570
|
+
recent.push({ id, seenAt: now });
|
|
571
|
+
recent.sort((left, right) => left.seenAt - right.seenAt || left.id.localeCompare(right.id));
|
|
572
|
+
let messages = recent.slice(-this.capacity);
|
|
573
|
+
let payload = JSON.stringify({ version: 1, messages }, null, 2) + "\n";
|
|
574
|
+
while (Buffer.byteLength(payload, "utf8") > STORE_BYTES && messages.length > 1) {
|
|
575
|
+
const averageBytes = Math.max(1, Math.ceil(Buffer.byteLength(payload, "utf8") / messages.length));
|
|
576
|
+
const excess = Buffer.byteLength(payload, "utf8") - STORE_BYTES;
|
|
577
|
+
messages = messages.slice(Math.max(1, Math.ceil(excess / averageBytes)));
|
|
578
|
+
payload = JSON.stringify({ version: 1, messages }, null, 2) + "\n";
|
|
579
|
+
}
|
|
580
|
+
if (Buffer.byteLength(payload, "utf8") > STORE_BYTES) {
|
|
581
|
+
throw new Error(`gateway message id is too large for the bounded dedupe store: ${this.path}`);
|
|
582
|
+
}
|
|
583
|
+
const boundary = bindHaraPrivateStateWritePath(this.path, this.dir, "write gateway message dedupe state");
|
|
584
|
+
await atomicWriteText(this.path, payload, {
|
|
585
|
+
expected: snapshot?.text ?? null,
|
|
586
|
+
expectedIdentity: snapshot
|
|
587
|
+
? { dev: snapshot.dev, ino: snapshot.ino, mode: snapshot.mode, nlink: snapshot.nlink }
|
|
588
|
+
: undefined,
|
|
589
|
+
mode: 0o600,
|
|
590
|
+
boundary,
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
async persistFailure(id, failure) {
|
|
594
|
+
const snapshot = await readPrivateStateFileSnapshot(this.path, STORE_BYTES);
|
|
595
|
+
const cutoff = failure.lastAt - this.ttlMs;
|
|
596
|
+
const recent = parseProcessedStore(snapshot, this.path).filter((entry) => entry.seenAt >= cutoff && entry.id !== id);
|
|
597
|
+
recent.push({ id, seenAt: failure.lastAt, failedAttempts: failure.attempts });
|
|
598
|
+
recent.sort((left, right) => left.seenAt - right.seenAt || left.id.localeCompare(right.id));
|
|
599
|
+
let messages = recent.slice(-this.capacity);
|
|
600
|
+
let payload = JSON.stringify({ version: 1, messages }, null, 2) + "\n";
|
|
601
|
+
while (Buffer.byteLength(payload, "utf8") > STORE_BYTES && messages.length > 1) {
|
|
602
|
+
const averageBytes = Math.max(1, Math.ceil(Buffer.byteLength(payload, "utf8") / messages.length));
|
|
603
|
+
const excess = Buffer.byteLength(payload, "utf8") - STORE_BYTES;
|
|
604
|
+
messages = messages.slice(Math.max(1, Math.ceil(excess / averageBytes)));
|
|
605
|
+
payload = JSON.stringify({ version: 1, messages }, null, 2) + "\n";
|
|
606
|
+
}
|
|
607
|
+
if (Buffer.byteLength(payload, "utf8") > STORE_BYTES) {
|
|
608
|
+
throw new Error(`gateway failure id is too large for the bounded dedupe store: ${this.path}`);
|
|
609
|
+
}
|
|
610
|
+
const boundary = bindHaraPrivateStateWritePath(this.path, this.dir, "write gateway message failure state");
|
|
611
|
+
await atomicWriteText(this.path, payload, {
|
|
612
|
+
expected: snapshot?.text ?? null,
|
|
613
|
+
expectedIdentity: snapshot
|
|
614
|
+
? { dev: snapshot.dev, ino: snapshot.ino, mode: snapshot.mode, nlink: snapshot.nlink }
|
|
615
|
+
: undefined,
|
|
616
|
+
mode: 0o600,
|
|
617
|
+
boundary,
|
|
618
|
+
});
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
function noOpMessageClaim() {
|
|
622
|
+
return { complete: async () => { }, release: async () => { }, fail: async () => false };
|
|
623
|
+
}
|
|
624
|
+
function parseEventSpool(snapshot, path) {
|
|
625
|
+
if (!snapshot)
|
|
626
|
+
return [];
|
|
627
|
+
let parsed;
|
|
628
|
+
try {
|
|
629
|
+
parsed = JSON.parse(snapshot.text);
|
|
630
|
+
}
|
|
631
|
+
catch {
|
|
632
|
+
throw new Error(`invalid gateway event spool: ${path}`);
|
|
633
|
+
}
|
|
634
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
635
|
+
throw new Error(`invalid gateway event spool: ${path}`);
|
|
636
|
+
}
|
|
637
|
+
const store = parsed;
|
|
638
|
+
if (store.version !== 1 || !Array.isArray(store.items) || store.items.length > EVENT_SPOOL_CAPACITY) {
|
|
639
|
+
throw new Error(`invalid gateway event spool: ${path}`);
|
|
640
|
+
}
|
|
641
|
+
const seen = new Set();
|
|
642
|
+
return store.items.map((value) => {
|
|
643
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
644
|
+
throw new Error(`invalid gateway event spool: ${path}`);
|
|
645
|
+
const item = value;
|
|
646
|
+
if (typeof item.id !== "string"
|
|
647
|
+
|| !item.id
|
|
648
|
+
|| item.id.length > 512
|
|
649
|
+
|| item.id.includes("\0")
|
|
650
|
+
|| seen.has(item.id)
|
|
651
|
+
|| !Number.isFinite(item.receivedAt)
|
|
652
|
+
|| item.receivedAt <= 0
|
|
653
|
+
|| !Number.isSafeInteger(item.attempts)
|
|
654
|
+
|| item.attempts < 0
|
|
655
|
+
|| item.attempts >= EVENT_SPOOL_MAX_ATTEMPTS
|
|
656
|
+
|| !Number.isFinite(item.availableAt)
|
|
657
|
+
|| item.availableAt <= 0
|
|
658
|
+
|| !item.payload
|
|
659
|
+
|| typeof item.payload !== "object"
|
|
660
|
+
|| Array.isArray(item.payload))
|
|
661
|
+
throw new Error(`invalid gateway event spool: ${path}`);
|
|
662
|
+
const payloadText = JSON.stringify(item.payload);
|
|
663
|
+
if (Buffer.byteLength(payloadText, "utf8") > EVENT_SPOOL_ITEM_BYTES) {
|
|
664
|
+
throw new Error(`invalid gateway event spool: ${path}`);
|
|
665
|
+
}
|
|
666
|
+
seen.add(item.id);
|
|
667
|
+
return item;
|
|
668
|
+
});
|
|
669
|
+
}
|
|
670
|
+
/** Private crash-safe queue used when a platform protocol requires an ACK before agent work can finish.
|
|
671
|
+
* Payloads are bounded and owner-only; ids are credential-scoped by the caller's spool namespace. */
|
|
672
|
+
export class GatewayEventSpool {
|
|
673
|
+
path;
|
|
674
|
+
dir;
|
|
675
|
+
now;
|
|
676
|
+
items;
|
|
677
|
+
expectedText;
|
|
678
|
+
expectedIdentity;
|
|
679
|
+
leased = new Set();
|
|
680
|
+
queue = Promise.resolve();
|
|
681
|
+
constructor(scope, options, snapshot) {
|
|
682
|
+
this.dir = stateDirectory(options.home ?? homedir());
|
|
683
|
+
this.path = join(this.dir, `inbound-${scope}.json`);
|
|
684
|
+
this.now = options.now ?? Date.now;
|
|
685
|
+
this.items = parseEventSpool(snapshot, this.path);
|
|
686
|
+
this.expectedText = snapshot?.text ?? null;
|
|
687
|
+
this.expectedIdentity = snapshot
|
|
688
|
+
? { dev: snapshot.dev, ino: snapshot.ino, mode: snapshot.mode, nlink: snapshot.nlink }
|
|
689
|
+
: undefined;
|
|
690
|
+
}
|
|
691
|
+
static async open(scopeValue, options = {}) {
|
|
692
|
+
const scope = checkedPlatform(scopeValue);
|
|
693
|
+
const dir = stateDirectory(options.home ?? homedir());
|
|
694
|
+
const path = join(dir, `inbound-${scope}.json`);
|
|
695
|
+
const snapshot = await readPrivateStateFileSnapshot(path, EVENT_SPOOL_BYTES);
|
|
696
|
+
return new GatewayEventSpool(scope, options, snapshot);
|
|
697
|
+
}
|
|
698
|
+
serialize(task) {
|
|
699
|
+
const operation = this.queue.then(task);
|
|
700
|
+
this.queue = operation.then(() => undefined, () => undefined);
|
|
701
|
+
return operation;
|
|
702
|
+
}
|
|
703
|
+
async commit(items) {
|
|
704
|
+
const text = JSON.stringify({ version: 1, items }, null, 2) + "\n";
|
|
705
|
+
if (Buffer.byteLength(text, "utf8") > EVENT_SPOOL_BYTES)
|
|
706
|
+
throw new Error("gateway event spool is full");
|
|
707
|
+
const boundary = bindHaraPrivateStateWritePath(this.path, this.dir, "write gateway event spool");
|
|
708
|
+
const written = await atomicWriteText(this.path, text, {
|
|
709
|
+
expected: this.expectedText,
|
|
710
|
+
expectedIdentity: this.expectedIdentity,
|
|
711
|
+
mode: 0o600,
|
|
712
|
+
boundary,
|
|
713
|
+
});
|
|
714
|
+
this.items = items;
|
|
715
|
+
this.expectedText = text;
|
|
716
|
+
this.expectedIdentity = { dev: written.dev, ino: written.ino, mode: written.mode, nlink: written.nlink };
|
|
717
|
+
}
|
|
718
|
+
/** Persist before ACK. false means the same platform event is already queued/in progress. */
|
|
719
|
+
enqueue(idValue, payload) {
|
|
720
|
+
return this.serialize(async () => {
|
|
721
|
+
const id = idValue.trim();
|
|
722
|
+
if (!id || id.length > 512 || id.includes("\0"))
|
|
723
|
+
throw new Error("invalid gateway event id");
|
|
724
|
+
if (this.items.some((item) => item.id === id))
|
|
725
|
+
return false;
|
|
726
|
+
if (this.items.length >= EVENT_SPOOL_CAPACITY)
|
|
727
|
+
throw new Error("gateway event spool is full");
|
|
728
|
+
let payloadText;
|
|
729
|
+
try {
|
|
730
|
+
payloadText = JSON.stringify(payload);
|
|
731
|
+
}
|
|
732
|
+
catch {
|
|
733
|
+
throw new Error("gateway event payload is not serializable");
|
|
734
|
+
}
|
|
735
|
+
if (!payloadText || Buffer.byteLength(payloadText, "utf8") > EVENT_SPOOL_ITEM_BYTES) {
|
|
736
|
+
throw new Error("gateway event payload exceeds the spool limit");
|
|
737
|
+
}
|
|
738
|
+
const safePayload = JSON.parse(payloadText);
|
|
739
|
+
if (!safePayload || typeof safePayload !== "object" || Array.isArray(safePayload)) {
|
|
740
|
+
throw new Error("gateway event payload must be an object");
|
|
741
|
+
}
|
|
742
|
+
const now = this.now();
|
|
743
|
+
await this.commit([...this.items, { id, payload: safePayload, receivedAt: now, attempts: 0, availableAt: now }]);
|
|
744
|
+
return true;
|
|
745
|
+
});
|
|
746
|
+
}
|
|
747
|
+
nextReady() {
|
|
748
|
+
return this.serialize(async () => {
|
|
749
|
+
const now = this.now();
|
|
750
|
+
const item = this.items.find((candidate) => candidate.availableAt <= now && !this.leased.has(candidate.id));
|
|
751
|
+
if (item)
|
|
752
|
+
this.leased.add(item.id);
|
|
753
|
+
return item ? { id: item.id, payload: item.payload, attempts: item.attempts } : null;
|
|
754
|
+
});
|
|
755
|
+
}
|
|
756
|
+
complete(id) {
|
|
757
|
+
return this.serialize(async () => {
|
|
758
|
+
this.leased.delete(id);
|
|
759
|
+
const items = this.items.filter((item) => item.id !== id);
|
|
760
|
+
if (items.length !== this.items.length)
|
|
761
|
+
await this.commit(items);
|
|
762
|
+
});
|
|
763
|
+
}
|
|
764
|
+
retry(id) {
|
|
765
|
+
return this.serialize(async () => {
|
|
766
|
+
this.leased.delete(id);
|
|
767
|
+
const index = this.items.findIndex((item) => item.id === id);
|
|
768
|
+
if (index < 0)
|
|
769
|
+
return { exhausted: false, attempts: 0, retryAfterMs: 0 };
|
|
770
|
+
const attempts = this.items[index].attempts + 1;
|
|
771
|
+
if (attempts >= EVENT_SPOOL_MAX_ATTEMPTS) {
|
|
772
|
+
await this.commit(this.items.filter((item) => item.id !== id));
|
|
773
|
+
return { exhausted: true, attempts, retryAfterMs: 0 };
|
|
774
|
+
}
|
|
775
|
+
const retryAfterMs = Math.min(30_000, 2_000 * (2 ** Math.max(0, attempts - 1)));
|
|
776
|
+
const next = this.items.slice();
|
|
777
|
+
next[index] = { ...next[index], attempts, availableAt: this.now() + retryAfterMs };
|
|
778
|
+
await this.commit(next);
|
|
779
|
+
return { exhausted: false, attempts, retryAfterMs };
|
|
780
|
+
});
|
|
781
|
+
}
|
|
782
|
+
/** Keep the durable item but relinquish this process-local worker lease (used only during shutdown). */
|
|
783
|
+
release(id) {
|
|
784
|
+
return this.serialize(async () => { this.leased.delete(id); });
|
|
785
|
+
}
|
|
786
|
+
}
|
|
787
|
+
function checkedFlowDigest(value, label) {
|
|
788
|
+
const digest = value.trim();
|
|
789
|
+
if (!/^[a-f0-9]{64}$/.test(digest))
|
|
790
|
+
throw new Error(`invalid gateway flow ${label}`);
|
|
791
|
+
return digest;
|
|
792
|
+
}
|
|
793
|
+
function parseStoredFlowRun(snapshot, path) {
|
|
794
|
+
let value;
|
|
795
|
+
try {
|
|
796
|
+
value = JSON.parse(snapshot.text);
|
|
797
|
+
}
|
|
798
|
+
catch {
|
|
799
|
+
throw new Error(`invalid gateway flow run: ${path}`);
|
|
800
|
+
}
|
|
801
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
802
|
+
throw new Error(`invalid gateway flow run: ${path}`);
|
|
803
|
+
const stored = value;
|
|
804
|
+
if (stored.version !== 1
|
|
805
|
+
|| typeof stored.source !== "string"
|
|
806
|
+
|| !/^[a-f0-9]{64}$/.test(stored.source)
|
|
807
|
+
|| !Number.isFinite(stored.createdAt)
|
|
808
|
+
|| stored.createdAt <= 0
|
|
809
|
+
|| !Number.isFinite(stored.updatedAt)
|
|
810
|
+
|| stored.updatedAt < stored.createdAt
|
|
811
|
+
|| !Number.isInteger(stored.attempts)
|
|
812
|
+
|| stored.attempts < 0
|
|
813
|
+
|| stored.attempts > FLOW_RUN_MAX_ATTEMPTS
|
|
814
|
+
|| !Number.isFinite(stored.nextAttemptAt)
|
|
815
|
+
|| stored.nextAttemptAt < 0
|
|
816
|
+
|| !["running", "ready", "failed", "complete", "exhausted"].includes(String(stored.status))
|
|
817
|
+
|| typeof stored.alarmed !== "boolean"
|
|
818
|
+
|| (stored.output !== undefined && (typeof stored.output !== "string"
|
|
819
|
+
|| Buffer.byteLength(stored.output, "utf8") > FLOW_RUN_OUTPUT_BYTES))
|
|
820
|
+
|| ((stored.status === "ready" || stored.status === "complete") && stored.output === undefined)
|
|
821
|
+
|| (stored.status === "exhausted" && stored.attempts < FLOW_RUN_MAX_ATTEMPTS))
|
|
822
|
+
throw new Error(`invalid gateway flow run: ${path}`);
|
|
823
|
+
return stored;
|
|
824
|
+
}
|
|
825
|
+
/**
|
|
826
|
+
* Private per-rule flow decisions and attempt budgets. A model decision is committed before the first delivery,
|
|
827
|
+
* then reused byte-for-byte by every retry. One file per source/rule avoids rewriting unrelated model output.
|
|
828
|
+
*/
|
|
829
|
+
export class GatewayFlowRunStore {
|
|
830
|
+
dir;
|
|
831
|
+
now;
|
|
832
|
+
queue = Promise.resolve();
|
|
833
|
+
constructor(scope, options) {
|
|
834
|
+
const home = options.home ?? homedir();
|
|
835
|
+
this.dir = ensurePrivateStateSubdirectory(home, [".hara", "gateway", `flow-runs-${scope}`]).path;
|
|
836
|
+
this.now = options.now ?? Date.now;
|
|
837
|
+
}
|
|
838
|
+
static async open(scopeValue, options = {}) {
|
|
839
|
+
const store = new GatewayFlowRunStore(checkedPlatform(scopeValue), options);
|
|
840
|
+
await store.prune();
|
|
841
|
+
return store;
|
|
842
|
+
}
|
|
843
|
+
serialize(task) {
|
|
844
|
+
const operation = this.queue.then(task);
|
|
845
|
+
this.queue = operation.then(() => undefined, () => undefined);
|
|
846
|
+
return operation;
|
|
847
|
+
}
|
|
848
|
+
runPath(runKeyValue) {
|
|
849
|
+
return join(this.dir, `${checkedFlowDigest(runKeyValue, "run key")}.json`);
|
|
850
|
+
}
|
|
851
|
+
removeSnapshot(path, snapshot) {
|
|
852
|
+
const current = lstatSync(path);
|
|
853
|
+
if (!current.isFile()
|
|
854
|
+
|| current.isSymbolicLink()
|
|
855
|
+
|| current.nlink !== 1
|
|
856
|
+
|| current.dev !== snapshot.dev
|
|
857
|
+
|| current.ino !== snapshot.ino)
|
|
858
|
+
throw new Error(`gateway flow run changed before removal: ${path}`);
|
|
859
|
+
unlinkSync(path);
|
|
860
|
+
}
|
|
861
|
+
async writeRecord(path, snapshot, record) {
|
|
862
|
+
const text = JSON.stringify(record) + "\n";
|
|
863
|
+
if (Buffer.byteLength(text, "utf8") > FLOW_RUN_FILE_BYTES)
|
|
864
|
+
throw new Error("gateway flow decision exceeds the private cache limit");
|
|
865
|
+
const boundary = bindHaraPrivateStateWritePath(path, this.dir, "write gateway flow run");
|
|
866
|
+
await atomicWriteText(path, text, {
|
|
867
|
+
expected: snapshot?.text ?? null,
|
|
868
|
+
expectedIdentity: snapshot
|
|
869
|
+
? { dev: snapshot.dev, ino: snapshot.ino, mode: snapshot.mode, nlink: snapshot.nlink }
|
|
870
|
+
: undefined,
|
|
871
|
+
mode: 0o600,
|
|
872
|
+
boundary,
|
|
873
|
+
});
|
|
874
|
+
}
|
|
875
|
+
async prune() {
|
|
876
|
+
const cutoff = this.now() - FLOW_RUN_TTL_MS;
|
|
877
|
+
for (const name of readdirSync(this.dir)) {
|
|
878
|
+
if (!FLOW_RUN_FILE.test(name))
|
|
879
|
+
continue;
|
|
880
|
+
const path = join(this.dir, name);
|
|
881
|
+
try {
|
|
882
|
+
const snapshot = await readPrivateStateFileSnapshot(path, FLOW_RUN_FILE_BYTES);
|
|
883
|
+
if (!snapshot)
|
|
884
|
+
continue;
|
|
885
|
+
const record = parseStoredFlowRun(snapshot, path);
|
|
886
|
+
// The absolute lifetime is deliberately not refreshed by retries. It matches the 24-hour effect-receipt
|
|
887
|
+
// horizon and prevents a poison event from retaining model text forever.
|
|
888
|
+
if (record.createdAt < cutoff)
|
|
889
|
+
this.removeSnapshot(path, snapshot);
|
|
890
|
+
}
|
|
891
|
+
catch (error) {
|
|
892
|
+
if (error?.code !== "ENOENT")
|
|
893
|
+
throw error;
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
async claim(runKeyValue, sourceValue) {
|
|
898
|
+
const runKey = checkedFlowDigest(runKeyValue, "run key");
|
|
899
|
+
const source = checkedFlowDigest(sourceValue, "source key");
|
|
900
|
+
return this.serialize(async () => {
|
|
901
|
+
const path = this.runPath(runKey);
|
|
902
|
+
let snapshot = await readPrivateStateFileSnapshot(path, FLOW_RUN_FILE_BYTES);
|
|
903
|
+
let existing = snapshot ? parseStoredFlowRun(snapshot, path) : undefined;
|
|
904
|
+
const now = this.now();
|
|
905
|
+
if (snapshot && existing && now - existing.createdAt >= FLOW_RUN_TTL_MS) {
|
|
906
|
+
this.removeSnapshot(path, snapshot);
|
|
907
|
+
snapshot = null;
|
|
908
|
+
existing = undefined;
|
|
909
|
+
}
|
|
910
|
+
if (existing && existing.source !== source)
|
|
911
|
+
throw new Error(`gateway flow source mismatch: ${path}`);
|
|
912
|
+
if (existing?.status === "complete")
|
|
913
|
+
return { kind: "complete" };
|
|
914
|
+
if (existing?.status === "exhausted")
|
|
915
|
+
return { kind: "exhausted", alarm: !existing.alarmed };
|
|
916
|
+
if (existing?.status === "failed" && now < existing.nextAttemptAt) {
|
|
917
|
+
return { kind: "backoff", retryAfterMs: existing.nextAttemptAt - now };
|
|
918
|
+
}
|
|
919
|
+
if (existing && existing.attempts >= FLOW_RUN_MAX_ATTEMPTS) {
|
|
920
|
+
const exhausted = { ...existing, status: "exhausted", nextAttemptAt: 0, updatedAt: now };
|
|
921
|
+
await this.writeRecord(path, snapshot, exhausted);
|
|
922
|
+
return { kind: "exhausted", alarm: !exhausted.alarmed };
|
|
923
|
+
}
|
|
924
|
+
if (!existing) {
|
|
925
|
+
let names = readdirSync(this.dir).filter((name) => FLOW_RUN_FILE.test(name));
|
|
926
|
+
if (names.length >= FLOW_RUN_CAPACITY) {
|
|
927
|
+
await this.prune();
|
|
928
|
+
names = readdirSync(this.dir).filter((name) => FLOW_RUN_FILE.test(name));
|
|
929
|
+
if (names.length >= FLOW_RUN_CAPACITY) {
|
|
930
|
+
throw new Error("gateway flow decision cache is full; acknowledged-event cleanup or operator recovery is required");
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
const original = existing ? { ...existing } : undefined;
|
|
935
|
+
const running = {
|
|
936
|
+
version: 1,
|
|
937
|
+
source,
|
|
938
|
+
createdAt: existing?.createdAt ?? now,
|
|
939
|
+
updatedAt: now,
|
|
940
|
+
attempts: (existing?.attempts ?? 0) + 1,
|
|
941
|
+
nextAttemptAt: 0,
|
|
942
|
+
status: "running",
|
|
943
|
+
alarmed: existing?.alarmed ?? false,
|
|
944
|
+
...(existing?.output !== undefined ? { output: existing.output } : {}),
|
|
945
|
+
};
|
|
946
|
+
await this.writeRecord(path, snapshot, running);
|
|
947
|
+
let settled = false;
|
|
948
|
+
let failedResult;
|
|
949
|
+
const mutateActive = async (mutate) => {
|
|
950
|
+
await this.serialize(async () => {
|
|
951
|
+
const currentSnapshot = await readPrivateStateFileSnapshot(path, FLOW_RUN_FILE_BYTES);
|
|
952
|
+
if (!currentSnapshot)
|
|
953
|
+
throw new Error("gateway flow run disappeared while active");
|
|
954
|
+
const current = parseStoredFlowRun(currentSnapshot, path);
|
|
955
|
+
if (current.status !== "running" || current.attempts !== running.attempts || current.source !== source) {
|
|
956
|
+
throw new Error("gateway flow run changed while active");
|
|
957
|
+
}
|
|
958
|
+
await mutate(current, currentSnapshot);
|
|
959
|
+
});
|
|
960
|
+
};
|
|
961
|
+
const claim = {
|
|
962
|
+
retry: existing !== undefined,
|
|
963
|
+
output: existing?.output,
|
|
964
|
+
saveOutput: async (output) => {
|
|
965
|
+
if (settled)
|
|
966
|
+
throw new Error("gateway flow run is already settled");
|
|
967
|
+
if (Buffer.byteLength(output, "utf8") > FLOW_RUN_OUTPUT_BYTES)
|
|
968
|
+
throw new Error("gateway flow decision exceeds the private cache limit");
|
|
969
|
+
await mutateActive(async (current, currentSnapshot) => {
|
|
970
|
+
if (current.output !== undefined && current.output !== output)
|
|
971
|
+
throw new Error("gateway flow decision is immutable across retries");
|
|
972
|
+
await this.writeRecord(path, currentSnapshot, { ...current, output, updatedAt: this.now() });
|
|
973
|
+
});
|
|
974
|
+
},
|
|
975
|
+
complete: async () => {
|
|
976
|
+
if (settled)
|
|
977
|
+
return;
|
|
978
|
+
await mutateActive(async (current, currentSnapshot) => {
|
|
979
|
+
if (current.output === undefined)
|
|
980
|
+
throw new Error("gateway flow decision was not persisted before completion");
|
|
981
|
+
await this.writeRecord(path, currentSnapshot, { ...current, status: "complete", nextAttemptAt: 0, updatedAt: this.now() });
|
|
982
|
+
});
|
|
983
|
+
settled = true;
|
|
984
|
+
},
|
|
985
|
+
fail: async () => {
|
|
986
|
+
if (failedResult)
|
|
987
|
+
return failedResult;
|
|
988
|
+
if (settled)
|
|
989
|
+
return { exhausted: false, alarm: false };
|
|
990
|
+
await mutateActive(async (current, currentSnapshot) => {
|
|
991
|
+
const exhausted = current.attempts >= FLOW_RUN_MAX_ATTEMPTS;
|
|
992
|
+
const retryAfterMs = Math.min(30_000, 2_000 * (2 ** Math.max(0, current.attempts - 1)));
|
|
993
|
+
const next = {
|
|
994
|
+
...current,
|
|
995
|
+
status: exhausted ? "exhausted" : "failed",
|
|
996
|
+
nextAttemptAt: exhausted ? 0 : this.now() + retryAfterMs,
|
|
997
|
+
updatedAt: this.now(),
|
|
998
|
+
};
|
|
999
|
+
await this.writeRecord(path, currentSnapshot, next);
|
|
1000
|
+
failedResult = { exhausted, alarm: exhausted && !next.alarmed };
|
|
1001
|
+
});
|
|
1002
|
+
settled = true;
|
|
1003
|
+
return failedResult ?? { exhausted: false, alarm: false };
|
|
1004
|
+
},
|
|
1005
|
+
release: async () => {
|
|
1006
|
+
if (settled)
|
|
1007
|
+
return;
|
|
1008
|
+
await mutateActive(async (current, currentSnapshot) => {
|
|
1009
|
+
const output = current.output ?? original?.output;
|
|
1010
|
+
if (!original && output === undefined) {
|
|
1011
|
+
this.removeSnapshot(path, currentSnapshot);
|
|
1012
|
+
return;
|
|
1013
|
+
}
|
|
1014
|
+
const restored = original
|
|
1015
|
+
? {
|
|
1016
|
+
...original,
|
|
1017
|
+
updatedAt: this.now(),
|
|
1018
|
+
...(output !== undefined ? { output } : {}),
|
|
1019
|
+
...(output !== undefined && original.output === undefined ? { status: "ready", nextAttemptAt: 0 } : {}),
|
|
1020
|
+
}
|
|
1021
|
+
: {
|
|
1022
|
+
version: 1,
|
|
1023
|
+
source,
|
|
1024
|
+
createdAt: running.createdAt,
|
|
1025
|
+
updatedAt: this.now(),
|
|
1026
|
+
attempts: 0,
|
|
1027
|
+
nextAttemptAt: 0,
|
|
1028
|
+
status: "ready",
|
|
1029
|
+
alarmed: false,
|
|
1030
|
+
output: output,
|
|
1031
|
+
};
|
|
1032
|
+
await this.writeRecord(path, currentSnapshot, restored);
|
|
1033
|
+
});
|
|
1034
|
+
settled = true;
|
|
1035
|
+
},
|
|
1036
|
+
markAlarmed: async () => {
|
|
1037
|
+
await this.markAlarmed(runKey);
|
|
1038
|
+
},
|
|
1039
|
+
};
|
|
1040
|
+
return { kind: "claim", claim };
|
|
1041
|
+
});
|
|
1042
|
+
}
|
|
1043
|
+
async markAlarmed(runKeyValue) {
|
|
1044
|
+
const path = this.runPath(runKeyValue);
|
|
1045
|
+
await this.serialize(async () => {
|
|
1046
|
+
const snapshot = await readPrivateStateFileSnapshot(path, FLOW_RUN_FILE_BYTES);
|
|
1047
|
+
if (!snapshot)
|
|
1048
|
+
return;
|
|
1049
|
+
const current = parseStoredFlowRun(snapshot, path);
|
|
1050
|
+
if (current.status !== "exhausted" || current.alarmed)
|
|
1051
|
+
return;
|
|
1052
|
+
await this.writeRecord(path, snapshot, { ...current, alarmed: true, updatedAt: this.now() });
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
/** Platform ACK cleanup. Both arguments and filenames are opaque hashes, so private message ids never land here. */
|
|
1056
|
+
async removeSource(sourceValue) {
|
|
1057
|
+
if (!sourceValue)
|
|
1058
|
+
return;
|
|
1059
|
+
const source = checkedFlowDigest(sourceValue, "source key");
|
|
1060
|
+
await this.serialize(async () => {
|
|
1061
|
+
for (const name of readdirSync(this.dir)) {
|
|
1062
|
+
if (!FLOW_RUN_FILE.test(name))
|
|
1063
|
+
continue;
|
|
1064
|
+
const path = join(this.dir, name);
|
|
1065
|
+
const snapshot = await readPrivateStateFileSnapshot(path, FLOW_RUN_FILE_BYTES);
|
|
1066
|
+
if (!snapshot)
|
|
1067
|
+
continue;
|
|
1068
|
+
const record = parseStoredFlowRun(snapshot, path);
|
|
1069
|
+
if (record.source === source)
|
|
1070
|
+
this.removeSnapshot(path, snapshot);
|
|
1071
|
+
}
|
|
1072
|
+
});
|
|
1073
|
+
}
|
|
1074
|
+
/** Focused diagnostics/tests; values are bounded private state and the key itself is already opaque. */
|
|
1075
|
+
async load(runKeyValue) {
|
|
1076
|
+
const path = this.runPath(runKeyValue);
|
|
1077
|
+
const snapshot = await readPrivateStateFileSnapshot(path, FLOW_RUN_FILE_BYTES);
|
|
1078
|
+
if (!snapshot)
|
|
1079
|
+
return null;
|
|
1080
|
+
const record = parseStoredFlowRun(snapshot, path);
|
|
1081
|
+
return {
|
|
1082
|
+
status: record.status,
|
|
1083
|
+
attempts: record.attempts,
|
|
1084
|
+
...(record.output !== undefined ? { output: record.output } : {}),
|
|
1085
|
+
alarmed: record.alarmed,
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
function parseRunOutcome(snapshot, path) {
|
|
1090
|
+
let value;
|
|
1091
|
+
try {
|
|
1092
|
+
value = JSON.parse(snapshot.text);
|
|
1093
|
+
}
|
|
1094
|
+
catch {
|
|
1095
|
+
throw new Error(`invalid gateway run outcome: ${path}`);
|
|
1096
|
+
}
|
|
1097
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1098
|
+
throw new Error(`invalid gateway run outcome: ${path}`);
|
|
1099
|
+
const stored = value;
|
|
1100
|
+
if (stored.version !== 1
|
|
1101
|
+
|| !["running", "complete", "terminal", "acknowledged"].includes(String(stored.status))
|
|
1102
|
+
|| !Number.isFinite(stored.createdAt)
|
|
1103
|
+
|| stored.createdAt <= 0
|
|
1104
|
+
|| typeof stored.reply !== "string"
|
|
1105
|
+
|| Buffer.byteLength(stored.reply, "utf8") > RUN_OUTCOME_MAX_REPLY_BYTES
|
|
1106
|
+
|| !Array.isArray(stored.files)
|
|
1107
|
+
|| stored.files.length > RUN_OUTCOME_MAX_FILES)
|
|
1108
|
+
throw new Error(`invalid gateway run outcome: ${path}`);
|
|
1109
|
+
if (stored.status !== "complete") {
|
|
1110
|
+
if (stored.reply || stored.files.length || stored.voice !== undefined)
|
|
1111
|
+
throw new Error(`invalid gateway run outcome: ${path}`);
|
|
1112
|
+
return { status: stored.status === "running" ? "running" : "terminal" };
|
|
1113
|
+
}
|
|
1114
|
+
let total = 0;
|
|
1115
|
+
const decodePayload = (file) => {
|
|
1116
|
+
if (!file
|
|
1117
|
+
|| typeof file !== "object"
|
|
1118
|
+
|| typeof file.safeName !== "string"
|
|
1119
|
+
|| !file.safeName
|
|
1120
|
+
|| file.safeName.length > 128
|
|
1121
|
+
|| /[\\/\0]/u.test(file.safeName)
|
|
1122
|
+
|| typeof file.bytes !== "string"
|
|
1123
|
+
|| !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(file.bytes))
|
|
1124
|
+
throw new Error(`invalid gateway run outcome: ${path}`);
|
|
1125
|
+
const payload = file;
|
|
1126
|
+
const bytes = Buffer.from(payload.bytes, "base64");
|
|
1127
|
+
total += bytes.length;
|
|
1128
|
+
if (bytes.length > RUN_OUTCOME_MAX_FILE_BYTES || total > RUN_OUTCOME_MAX_FILE_BYTES) {
|
|
1129
|
+
throw new Error(`invalid gateway run outcome: ${path}`);
|
|
1130
|
+
}
|
|
1131
|
+
return { safeName: payload.safeName, bytes };
|
|
1132
|
+
};
|
|
1133
|
+
const files = stored.files.map(decodePayload);
|
|
1134
|
+
const voice = stored.voice === undefined ? undefined : decodePayload(stored.voice);
|
|
1135
|
+
return { status: "complete", reply: stored.reply, files, ...(voice ? { voice } : {}) };
|
|
1136
|
+
}
|
|
1137
|
+
function runOutcomeStorageStatus(snapshot, path) {
|
|
1138
|
+
parseRunOutcome(snapshot, path); // strict full-shape validation first
|
|
1139
|
+
return JSON.parse(snapshot.text).status;
|
|
1140
|
+
}
|
|
1141
|
+
/** Stores a started tombstone before a default coding run and its completed result before chat delivery.
|
|
1142
|
+
* A crash/restart therefore never guesses that coding is safe to repeat: an unfinished tombstone becomes an
|
|
1143
|
+
* explicit recovery notice, while a completed immutable result is only resent. Each message gets a separate
|
|
1144
|
+
* bounded private file so a large attachment never rewrites unrelated outcomes. */
|
|
1145
|
+
export class GatewayRunOutcomeStore {
|
|
1146
|
+
dir;
|
|
1147
|
+
now;
|
|
1148
|
+
queue = Promise.resolve();
|
|
1149
|
+
constructor(scope, options) {
|
|
1150
|
+
const home = options.home ?? homedir();
|
|
1151
|
+
this.dir = ensurePrivateStateSubdirectory(home, [".hara", "gateway", `run-outcomes-${scope}`]).path;
|
|
1152
|
+
this.now = options.now ?? Date.now;
|
|
1153
|
+
}
|
|
1154
|
+
static async open(scopeValue, options = {}) {
|
|
1155
|
+
const store = new GatewayRunOutcomeStore(checkedPlatform(scopeValue), options);
|
|
1156
|
+
await store.prune();
|
|
1157
|
+
return store;
|
|
1158
|
+
}
|
|
1159
|
+
outcomePath(messageIdValue) {
|
|
1160
|
+
const messageId = messageIdValue.trim();
|
|
1161
|
+
if (!messageId || messageId.length > 512 || messageId.includes("\0"))
|
|
1162
|
+
throw new Error("invalid gateway outcome message id");
|
|
1163
|
+
return join(this.dir, `${createHash("sha256").update(messageId).digest("hex")}.json`);
|
|
1164
|
+
}
|
|
1165
|
+
serialize(task) {
|
|
1166
|
+
const operation = this.queue.then(task);
|
|
1167
|
+
this.queue = operation.then(() => undefined, () => undefined);
|
|
1168
|
+
return operation;
|
|
1169
|
+
}
|
|
1170
|
+
removeSnapshot(path, snapshot) {
|
|
1171
|
+
const current = lstatSync(path);
|
|
1172
|
+
if (!current.isFile()
|
|
1173
|
+
|| current.isSymbolicLink()
|
|
1174
|
+
|| current.nlink !== 1
|
|
1175
|
+
|| current.dev !== snapshot.dev
|
|
1176
|
+
|| current.ino !== snapshot.ino)
|
|
1177
|
+
throw new Error(`gateway run outcome changed before removal: ${path}`);
|
|
1178
|
+
unlinkSync(path);
|
|
1179
|
+
}
|
|
1180
|
+
async replaceWithMarker(path, snapshot, status) {
|
|
1181
|
+
const text = JSON.stringify({
|
|
1182
|
+
version: 1,
|
|
1183
|
+
status,
|
|
1184
|
+
createdAt: this.now(),
|
|
1185
|
+
reply: "",
|
|
1186
|
+
files: [],
|
|
1187
|
+
}) + "\n";
|
|
1188
|
+
const boundary = bindHaraPrivateStateWritePath(path, this.dir, `${status} gateway run outcome`);
|
|
1189
|
+
await atomicWriteText(path, text, {
|
|
1190
|
+
expected: snapshot.text,
|
|
1191
|
+
expectedIdentity: {
|
|
1192
|
+
dev: snapshot.dev,
|
|
1193
|
+
ino: snapshot.ino,
|
|
1194
|
+
mode: snapshot.mode,
|
|
1195
|
+
nlink: snapshot.nlink,
|
|
1196
|
+
},
|
|
1197
|
+
mode: 0o600,
|
|
1198
|
+
boundary,
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
async prune() {
|
|
1202
|
+
const now = this.now();
|
|
1203
|
+
for (const name of readdirSync(this.dir)) {
|
|
1204
|
+
if (!RUN_OUTCOME_FILE.test(name))
|
|
1205
|
+
continue;
|
|
1206
|
+
const path = join(this.dir, name);
|
|
1207
|
+
try {
|
|
1208
|
+
const snapshot = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1209
|
+
if (!snapshot)
|
|
1210
|
+
continue;
|
|
1211
|
+
const storageStatus = runOutcomeStorageStatus(snapshot, path);
|
|
1212
|
+
if (storageStatus === "acknowledged") {
|
|
1213
|
+
// The adapter invokes remove only after the platform ACK is durable. If unlink failed after that
|
|
1214
|
+
// acknowledgement marker was committed, startup can finish the deletion without guessing.
|
|
1215
|
+
this.removeSnapshot(path, snapshot);
|
|
1216
|
+
}
|
|
1217
|
+
else if (storageStatus === "complete" && now - snapshot.mtimeMs >= RUN_OUTCOME_TTL_MS) {
|
|
1218
|
+
// Completed output can contain up to 20 MiB of attachment bytes, so payload retention has a TTL.
|
|
1219
|
+
// Never delete the execution marker, though: an offline durable platform event could arrive later
|
|
1220
|
+
// and otherwise rerun full-auto coding. A distinct terminal status remains fail-closed on redelivery,
|
|
1221
|
+
// but no longer consumes one of the 32 payload/in-progress slots.
|
|
1222
|
+
await this.replaceWithMarker(path, snapshot, "terminal");
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
catch (error) {
|
|
1226
|
+
if (error?.code !== "ENOENT")
|
|
1227
|
+
throw error;
|
|
1228
|
+
}
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
/** Make one active slot without ever deleting an ambiguous execution marker. Completed payloads can be
|
|
1232
|
+
* compacted to terminal tombstones; 32 genuinely running records require explicit operator recovery. */
|
|
1233
|
+
async makeRoomForStart() {
|
|
1234
|
+
const records = [];
|
|
1235
|
+
for (const name of readdirSync(this.dir)) {
|
|
1236
|
+
if (!RUN_OUTCOME_FILE.test(name))
|
|
1237
|
+
continue;
|
|
1238
|
+
const path = join(this.dir, name);
|
|
1239
|
+
const snapshot = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1240
|
+
if (!snapshot)
|
|
1241
|
+
continue;
|
|
1242
|
+
const status = runOutcomeStorageStatus(snapshot, path);
|
|
1243
|
+
if (status === "acknowledged") {
|
|
1244
|
+
this.removeSnapshot(path, snapshot);
|
|
1245
|
+
continue;
|
|
1246
|
+
}
|
|
1247
|
+
records.push({ path, snapshot, status });
|
|
1248
|
+
}
|
|
1249
|
+
const active = records.filter((record) => record.status === "running" || record.status === "complete");
|
|
1250
|
+
if (records.length >= RUN_OUTCOME_CAPACITY + RUN_OUTCOME_TOMBSTONE_CAPACITY) {
|
|
1251
|
+
throw new Error("gateway run outcome tombstone cache is full; inspect the workspace and platform history, then recover "
|
|
1252
|
+
+ "one known original message id with `hara gateway --platform <name> --recover-outcome <id> "
|
|
1253
|
+
+ "--confirm-recovery delete-terminal:<id>`; Hara will not bulk-delete markers");
|
|
1254
|
+
}
|
|
1255
|
+
if (active.length < RUN_OUTCOME_CAPACITY) {
|
|
1256
|
+
return;
|
|
1257
|
+
}
|
|
1258
|
+
const slotsNeeded = active.length - RUN_OUTCOME_CAPACITY + 1;
|
|
1259
|
+
const compactable = active
|
|
1260
|
+
.filter((record) => record.status === "complete")
|
|
1261
|
+
.sort((left, right) => left.snapshot.mtimeMs - right.snapshot.mtimeMs);
|
|
1262
|
+
if (compactable.length < slotsNeeded) {
|
|
1263
|
+
const ambiguous = active.length - compactable.length;
|
|
1264
|
+
throw new Error(`gateway run outcome cache has ${ambiguous} ambiguous interrupted runs; Hara will not rerun or delete them automatically. `
|
|
1265
|
+
+ "After reviewing one original message id, free an active slot with `hara gateway --platform <name> "
|
|
1266
|
+
+ "--recover-outcome <id> --confirm-recovery terminalize:<id>`");
|
|
1267
|
+
}
|
|
1268
|
+
for (const record of compactable.slice(0, slotsNeeded)) {
|
|
1269
|
+
await this.replaceWithMarker(record.path, record.snapshot, "terminal");
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
async load(messageId) {
|
|
1273
|
+
if (messageId === undefined)
|
|
1274
|
+
return null;
|
|
1275
|
+
const path = this.outcomePath(messageId);
|
|
1276
|
+
const snapshot = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1277
|
+
return snapshot ? parseRunOutcome(snapshot, path) : null;
|
|
1278
|
+
}
|
|
1279
|
+
/** Atomically records that execution is about to begin. null means this caller owns the fresh tombstone. */
|
|
1280
|
+
async start(messageId) {
|
|
1281
|
+
if (messageId === undefined)
|
|
1282
|
+
return null;
|
|
1283
|
+
return this.serialize(async () => {
|
|
1284
|
+
const path = this.outcomePath(messageId);
|
|
1285
|
+
const existing = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1286
|
+
if (existing)
|
|
1287
|
+
return parseRunOutcome(existing, path);
|
|
1288
|
+
// The common path has fewer than 32 files and must not decode unrelated cached attachments merely to
|
|
1289
|
+
// start a run. A full/terminal-heavy directory takes the slower validated compaction path.
|
|
1290
|
+
if (readdirSync(this.dir).filter((name) => RUN_OUTCOME_FILE.test(name)).length >= RUN_OUTCOME_CAPACITY) {
|
|
1291
|
+
await this.makeRoomForStart();
|
|
1292
|
+
}
|
|
1293
|
+
const text = JSON.stringify({
|
|
1294
|
+
version: 1,
|
|
1295
|
+
status: "running",
|
|
1296
|
+
createdAt: this.now(),
|
|
1297
|
+
reply: "",
|
|
1298
|
+
files: [],
|
|
1299
|
+
}) + "\n";
|
|
1300
|
+
const boundary = bindHaraPrivateStateWritePath(path, this.dir, "start gateway run outcome");
|
|
1301
|
+
try {
|
|
1302
|
+
await atomicWriteText(path, text, { expected: null, mode: 0o600, boundary });
|
|
1303
|
+
return null;
|
|
1304
|
+
}
|
|
1305
|
+
catch (error) {
|
|
1306
|
+
// Another local callback/process can only win by creating the same credential/message tombstone. Load
|
|
1307
|
+
// that durable result instead of treating the race as permission to execute a second time.
|
|
1308
|
+
if (!(error instanceof FileChangedError))
|
|
1309
|
+
throw error;
|
|
1310
|
+
const raced = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1311
|
+
if (!raced)
|
|
1312
|
+
throw error;
|
|
1313
|
+
return parseRunOutcome(raced, path);
|
|
1314
|
+
}
|
|
1315
|
+
});
|
|
1316
|
+
}
|
|
1317
|
+
/** Replaces a started tombstone with immutable output. The first completed execution always wins. */
|
|
1318
|
+
async finish(messageId, outcome) {
|
|
1319
|
+
if (messageId === undefined)
|
|
1320
|
+
return;
|
|
1321
|
+
await this.serialize(async () => {
|
|
1322
|
+
const path = this.outcomePath(messageId);
|
|
1323
|
+
if (Buffer.byteLength(outcome.reply, "utf8") > RUN_OUTCOME_MAX_REPLY_BYTES || outcome.files.length > RUN_OUTCOME_MAX_FILES) {
|
|
1324
|
+
throw new Error("gateway run outcome exceeds the private cache limit");
|
|
1325
|
+
}
|
|
1326
|
+
let total = 0;
|
|
1327
|
+
const files = outcome.files.map((file) => {
|
|
1328
|
+
if (!file.safeName || file.safeName.length > 128 || /[\\/\0]/u.test(file.safeName)) {
|
|
1329
|
+
throw new Error("gateway run outcome has an invalid attachment name");
|
|
1330
|
+
}
|
|
1331
|
+
total += file.bytes.length;
|
|
1332
|
+
if (file.bytes.length > RUN_OUTCOME_MAX_FILE_BYTES || total > RUN_OUTCOME_MAX_FILE_BYTES) {
|
|
1333
|
+
throw new Error("gateway run outcome exceeds the private cache limit");
|
|
1334
|
+
}
|
|
1335
|
+
return { safeName: file.safeName, bytes: file.bytes.toString("base64") };
|
|
1336
|
+
});
|
|
1337
|
+
let voice;
|
|
1338
|
+
if (outcome.voice) {
|
|
1339
|
+
if (!outcome.voice.safeName || outcome.voice.safeName.length > 128 || /[\\/\0]/u.test(outcome.voice.safeName)) {
|
|
1340
|
+
throw new Error("gateway run outcome has an invalid voice attachment name");
|
|
1341
|
+
}
|
|
1342
|
+
total += outcome.voice.bytes.length;
|
|
1343
|
+
if (outcome.voice.bytes.length > RUN_OUTCOME_MAX_FILE_BYTES || total > RUN_OUTCOME_MAX_FILE_BYTES) {
|
|
1344
|
+
throw new Error("gateway run outcome exceeds the private cache limit");
|
|
1345
|
+
}
|
|
1346
|
+
voice = { safeName: outcome.voice.safeName, bytes: outcome.voice.bytes.toString("base64") };
|
|
1347
|
+
}
|
|
1348
|
+
const text = JSON.stringify({
|
|
1349
|
+
version: 1,
|
|
1350
|
+
status: "complete",
|
|
1351
|
+
createdAt: this.now(),
|
|
1352
|
+
reply: outcome.reply,
|
|
1353
|
+
files,
|
|
1354
|
+
...(voice ? { voice } : {}),
|
|
1355
|
+
}) + "\n";
|
|
1356
|
+
if (Buffer.byteLength(text, "utf8") > RUN_OUTCOME_FILE_BYTES)
|
|
1357
|
+
throw new Error("gateway run outcome exceeds the private cache limit");
|
|
1358
|
+
const existing = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1359
|
+
if (!existing)
|
|
1360
|
+
throw new Error("gateway run outcome was not started; refusing an unguarded completion");
|
|
1361
|
+
if (parseRunOutcome(existing, path).status === "complete")
|
|
1362
|
+
return;
|
|
1363
|
+
const boundary = bindHaraPrivateStateWritePath(path, this.dir, "write gateway run outcome");
|
|
1364
|
+
try {
|
|
1365
|
+
await atomicWriteText(path, text, {
|
|
1366
|
+
expected: existing.text,
|
|
1367
|
+
expectedIdentity: {
|
|
1368
|
+
dev: existing.dev,
|
|
1369
|
+
ino: existing.ino,
|
|
1370
|
+
mode: existing.mode,
|
|
1371
|
+
nlink: existing.nlink,
|
|
1372
|
+
},
|
|
1373
|
+
mode: 0o600,
|
|
1374
|
+
boundary,
|
|
1375
|
+
});
|
|
1376
|
+
}
|
|
1377
|
+
catch (error) {
|
|
1378
|
+
if (!(error instanceof FileChangedError))
|
|
1379
|
+
throw error;
|
|
1380
|
+
const raced = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1381
|
+
if (raced && parseRunOutcome(raced, path).status === "complete")
|
|
1382
|
+
return;
|
|
1383
|
+
throw error;
|
|
1384
|
+
}
|
|
1385
|
+
});
|
|
1386
|
+
}
|
|
1387
|
+
async remove(messageId) {
|
|
1388
|
+
if (messageId === undefined)
|
|
1389
|
+
return;
|
|
1390
|
+
await this.serialize(async () => {
|
|
1391
|
+
const path = this.outcomePath(messageId);
|
|
1392
|
+
const snapshot = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1393
|
+
if (!snapshot)
|
|
1394
|
+
return;
|
|
1395
|
+
// Persist proof that the adapter already observed a durable platform ACK before attempting unlink. If
|
|
1396
|
+
// deletion itself fails or the process crashes, prune() can safely finish it on restart.
|
|
1397
|
+
if (runOutcomeStorageStatus(snapshot, path) !== "acknowledged") {
|
|
1398
|
+
await this.replaceWithMarker(path, snapshot, "acknowledged");
|
|
1399
|
+
}
|
|
1400
|
+
const acknowledged = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1401
|
+
if (acknowledged)
|
|
1402
|
+
this.removeSnapshot(path, acknowledged);
|
|
1403
|
+
});
|
|
1404
|
+
}
|
|
1405
|
+
/** Explicit, single-record operator recovery by the original platform message id.
|
|
1406
|
+
*
|
|
1407
|
+
* `terminalize:<id>` can only preserve a running marker as a compact terminal marker; it never deletes it.
|
|
1408
|
+
* `delete-terminal:<id>` can only delete a marker that is already terminal. Keeping these as distinct
|
|
1409
|
+
* confirmations makes repeating the first command harmless and prevents an accidental running deletion.
|
|
1410
|
+
*/
|
|
1411
|
+
async recover(messageIdValue, confirmationValue) {
|
|
1412
|
+
const messageId = messageIdValue.trim();
|
|
1413
|
+
if (!messageId || messageId.length > 512 || messageId.includes("\0")) {
|
|
1414
|
+
throw new Error("invalid gateway outcome message id");
|
|
1415
|
+
}
|
|
1416
|
+
const terminalizeConfirmation = `terminalize:${messageId}`;
|
|
1417
|
+
const deleteConfirmation = `delete-terminal:${messageId}`;
|
|
1418
|
+
if (confirmationValue !== terminalizeConfirmation && confirmationValue !== deleteConfirmation) {
|
|
1419
|
+
throw new Error("confirmation must exactly match terminalize:<message-id> or delete-terminal:<message-id>");
|
|
1420
|
+
}
|
|
1421
|
+
return this.serialize(async () => {
|
|
1422
|
+
const path = this.outcomePath(messageId);
|
|
1423
|
+
const snapshot = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1424
|
+
if (!snapshot)
|
|
1425
|
+
return "missing";
|
|
1426
|
+
const status = runOutcomeStorageStatus(snapshot, path);
|
|
1427
|
+
if (status === "complete") {
|
|
1428
|
+
throw new Error("refusing to discard a completed gateway outcome before its platform acknowledgement");
|
|
1429
|
+
}
|
|
1430
|
+
if (confirmationValue === terminalizeConfirmation) {
|
|
1431
|
+
if (status === "terminal" || status === "acknowledged")
|
|
1432
|
+
return "already-terminal";
|
|
1433
|
+
await this.replaceWithMarker(path, snapshot, "terminal");
|
|
1434
|
+
return "terminalized";
|
|
1435
|
+
}
|
|
1436
|
+
if (status === "running") {
|
|
1437
|
+
throw new Error("refusing to delete an ambiguous running outcome; confirm terminalize:<message-id> first");
|
|
1438
|
+
}
|
|
1439
|
+
// An acknowledged marker already carries durable proof that normal post-ACK cleanup may delete it.
|
|
1440
|
+
// For terminal markers, this explicit message-id-bound confirmation is the operator's recovery proof.
|
|
1441
|
+
if (status !== "acknowledged")
|
|
1442
|
+
await this.replaceWithMarker(path, snapshot, "acknowledged");
|
|
1443
|
+
const acknowledged = await readPrivateStateFileSnapshot(path, RUN_OUTCOME_FILE_BYTES);
|
|
1444
|
+
if (acknowledged)
|
|
1445
|
+
this.removeSnapshot(path, acknowledged);
|
|
1446
|
+
return "removed";
|
|
1447
|
+
});
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
/** Tracks adapter callbacks that may outlive socket shutdown, so an instance lease is never released early. */
|
|
1451
|
+
export class GatewayInboundTracker {
|
|
1452
|
+
tasks = new Set();
|
|
1453
|
+
get size() {
|
|
1454
|
+
return this.tasks.size;
|
|
1455
|
+
}
|
|
1456
|
+
track(task) {
|
|
1457
|
+
this.tasks.add(task);
|
|
1458
|
+
void task.then(() => this.tasks.delete(task), () => this.tasks.delete(task));
|
|
1459
|
+
return task;
|
|
1460
|
+
}
|
|
1461
|
+
async waitForIdle() {
|
|
1462
|
+
while (this.tasks.size)
|
|
1463
|
+
await Promise.allSettled([...this.tasks]);
|
|
1464
|
+
}
|
|
1465
|
+
async drain(timeoutMs = 15_000) {
|
|
1466
|
+
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 0)
|
|
1467
|
+
throw new RangeError("timeout must be a non-negative integer");
|
|
1468
|
+
const deadline = Date.now() + timeoutMs;
|
|
1469
|
+
if (!this.tasks.size)
|
|
1470
|
+
return true;
|
|
1471
|
+
const remaining = deadline - Date.now();
|
|
1472
|
+
if (remaining <= 0)
|
|
1473
|
+
return false;
|
|
1474
|
+
let timer;
|
|
1475
|
+
const settled = await Promise.race([
|
|
1476
|
+
this.waitForIdle().then(() => true),
|
|
1477
|
+
new Promise((resolve) => {
|
|
1478
|
+
timer = setTimeout(() => resolve(false), remaining);
|
|
1479
|
+
}),
|
|
1480
|
+
]);
|
|
1481
|
+
if (timer)
|
|
1482
|
+
clearTimeout(timer);
|
|
1483
|
+
return settled;
|
|
1484
|
+
}
|
|
1485
|
+
}
|