@sema-agent/core 7.2.0 → 7.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/dist/agents/cross-session-envelope.d.ts +7 -0
- package/dist/agents/cross-session-envelope.js +4 -0
- package/dist/agents/list-agents-tool.d.ts +55 -0
- package/dist/agents/list-agents-tool.js +94 -0
- package/dist/agents/peer-admission.d.ts +17 -1
- package/dist/agents/peer-admission.js +19 -2
- package/dist/agents/peer-directory.d.ts +208 -0
- package/dist/agents/peer-directory.js +272 -0
- package/dist/agents/peer-session-drain.d.ts +159 -0
- package/dist/agents/peer-session-drain.js +245 -0
- package/dist/agents/send-message-tool.d.ts +31 -0
- package/dist/agents/send-message-tool.js +145 -4
- package/dist/agents/subagent-steps.d.ts +11 -0
- package/dist/agents/subagent-steps.js +27 -4
- package/dist/brain/status-sink.d.ts +10 -0
- package/dist/brain/status-sink.js +13 -4
- package/dist/brain/stream-engine.d.ts +11 -0
- package/dist/brain/stream-engine.js +39 -3
- package/dist/core/arg-summary.d.ts +13 -3
- package/dist/core/arg-summary.js +138 -7
- package/dist/core/auto-mode-arming.d.ts +11 -0
- package/dist/core/auto-mode-arming.js +7 -1
- package/dist/core/auto-mode-prompt.d.ts +5 -0
- package/dist/core/auto-mode-prompt.js +2 -1
- package/dist/core/auto-mode-rebuild.d.ts +2 -1
- package/dist/core/auto-mode-rebuild.js +2 -0
- package/dist/core/checkpoint-store.d.ts +14 -0
- package/dist/core/checkpoint-store.js +4 -3
- package/dist/core/governance-codes.d.ts +1 -1
- package/dist/core/governance-codes.js +6 -0
- package/dist/core/mailbox-store.d.ts +89 -2
- package/dist/core/mailbox-store.js +77 -2
- package/dist/core/permission-rule-model.d.ts +9 -0
- package/dist/core/permission-rule-model.js +4 -1
- package/dist/core/runner/prepare-task.d.ts +20 -0
- package/dist/core/runner/prepare-task.js +152 -37
- package/dist/core/runner/runtask.js +5 -2
- package/dist/core/runner/tool-output-projection.js +1 -0
- package/dist/core/store-contracts/mailbox-store-contract.d.ts +23 -0
- package/dist/core/store-contracts/mailbox-store-contract.js +157 -1
- package/dist/core/task-notification.d.ts +38 -9
- package/dist/core/task-notification.js +8 -2
- package/dist/core/tools.js +1 -0
- package/dist/core/types.d.ts +176 -26
- package/dist/core/wiring-manifest.d.ts +62 -5
- package/dist/core/wiring-manifest.js +9 -0
- package/dist/engine/harness/agent-harness.d.ts +1 -0
- package/dist/engine/harness/agent-harness.js +3 -0
- package/dist/engine/harness/types.d.ts +3 -0
- package/dist/engine/loop/agent-loop.d.ts +7 -0
- package/dist/engine/loop/agent-loop.js +79 -0
- package/dist/engine/loop/types.d.ts +42 -0
- package/dist/index.d.ts +12 -5
- package/dist/index.js +11 -4
- package/dist/internal/harness-types.d.ts +1 -1
- package/dist/stores/cc/mailbox-store.d.ts +1 -1
- package/dist/stores/cc/mailbox-store.js +13 -0
- package/dist/stores/file/adoption/marker.d.ts +1 -1
- package/dist/stores/file/mailbox-store.d.ts +57 -0
- package/dist/stores/file/mailbox-store.js +369 -18
- package/dist/tools/fs/fs-write.js +69 -3
- package/package.json +1 -1
- package/test/export-surface.snapshot.json +121 -1
|
@@ -1,7 +1,44 @@
|
|
|
1
1
|
import { type MailboxAppendMessage, type MailboxLease, type MailboxStore } from "../../core/mailbox-store.js";
|
|
2
|
+
/** The waiter's refusal when a box lock stays held by a LIVE process past `lockWaitMs`. Carried as
|
|
3
|
+
* `.code` on a plain Error (consumers branch on the string); the message names the lock and the holder. */
|
|
4
|
+
export declare const MAILBOX_BOX_LOCK_TIMEOUT_CODE = "mailbox.box_lock_timeout";
|
|
5
|
+
/** The holder's refusal when its lock was found to have changed hands at release — the critical
|
|
6
|
+
* section may have overlapped another process's; its effects are reported as NOT trustworthy. */
|
|
7
|
+
export declare const MAILBOX_BOX_LOCK_COMPROMISED_CODE = "mailbox.box_lock_compromised";
|
|
8
|
+
/**
|
|
9
|
+
* Acquire the cross-process BOX LOCK (design/385 §1.2⑥ ring 1) at `lockPath` — `<box>.jsonl.lock`.
|
|
10
|
+
* Resolves with a release function once held; rejects with `mailbox.box_lock_timeout` when the lock is
|
|
11
|
+
* not obtainable within `waitMs` — a LIVE holder keeps it, or it is unreadable/unreclaimable and needs
|
|
12
|
+
* an operator (the message names which). Exported at module level for the real-process acceptance
|
|
13
|
+
* run (a fixture that holds the lock and dies / stays alive); not part of the package surface.
|
|
14
|
+
*
|
|
15
|
+
* Reclaim protocol (a holder that DIED — the pid recorded in the lock no longer exists):
|
|
16
|
+
* - ONE reclaimer per stale lock instance: a reclaimer first publishes a GATE named after the stale
|
|
17
|
+
* lock's own nonce (`<lock>.reclaim.<nonce>`, `O_EXCL`); only the gate winner touches the lock, so
|
|
18
|
+
* no second reclaimer can remove the stale file underneath the winner and let a fresh live lock
|
|
19
|
+
* appear at the path in between — the interleaving that would otherwise let the winner move a
|
|
20
|
+
* live holder's lock aside. Losers back off like any waiter.
|
|
21
|
+
* - the winner renames the stale file aside, re-reads the moved file and confirms it is exactly the
|
|
22
|
+
* (pid, nonce) it judged (belt: a mismatch is moved back untouched), unlinks it, then removes the
|
|
23
|
+
* gate. A gate or an aside file older than {@link BOX_LOCK_RECLAIM_GRACE_MS} is the residue of a
|
|
24
|
+
* reclaimer that died mid-reclaim and is cleared by the next one.
|
|
25
|
+
* - a lock whose recorded pid is ALIVE is never reclaimed. Liveness is `kill(pid, 0)`: the processes
|
|
26
|
+
* that share a box must share the host's pid namespace (a container that mounts the same volume
|
|
27
|
+
* from another pid namespace cannot see the holder and would judge every lock dead). A dead
|
|
28
|
+
* holder's pid reused by an unrelated long-lived process reads as alive; the waiter then times out
|
|
29
|
+
* LOUDLY naming the pid, and the lock file is removed by hand — the accepted failure, chosen over
|
|
30
|
+
* an age rule that would reclaim a live-but-slow holder.
|
|
31
|
+
* - the holder verifies its nonce at release; a lock that changed hands underneath it surfaces as
|
|
32
|
+
* `mailbox.box_lock_compromised`, never as a silent success.
|
|
33
|
+
*/
|
|
34
|
+
export declare function acquireMailboxBoxLock(lockPath: string, waitMs: number): Promise<() => void>;
|
|
2
35
|
export interface FileMailboxStoreOptions {
|
|
3
36
|
/** When false, ledger appends are NOT fsync'd. Default true. */
|
|
4
37
|
fsync?: boolean;
|
|
38
|
+
/** design/385 — how long an operation waits for a box lock a LIVE process holds before failing with
|
|
39
|
+
* `mailbox.box_lock_timeout`. Default 10 000 ms. Must be a finite number > 0 (anything else refuses
|
|
40
|
+
* at construction — a bad knob never becomes a silent default). */
|
|
41
|
+
lockWaitMs?: number;
|
|
5
42
|
/** Compact a box's event log into a snapshot-form log once it exceeds this many events. Default 500. */
|
|
6
43
|
compactEvery?: number;
|
|
7
44
|
/** Disclosure seat (review-510 family; server pickup receipt named this store's gap): a corrupt
|
|
@@ -14,10 +51,14 @@ export interface FileMailboxStoreOptions {
|
|
|
14
51
|
}) => void;
|
|
15
52
|
}
|
|
16
53
|
export declare class FileMailboxStore implements MailboxStore {
|
|
54
|
+
/** design/385 — the whole cross-process set (box lock + fresh re-read + descriptor reopen +
|
|
55
|
+
* compaction under the lock) is in place and accepted by `mailboxCrossProcessContract`. */
|
|
56
|
+
readonly crossProcessSafe = true;
|
|
17
57
|
private readonly dir;
|
|
18
58
|
private readonly tmpDir;
|
|
19
59
|
private readonly fsyncEnabled;
|
|
20
60
|
private readonly compactEvery;
|
|
61
|
+
private readonly lockWaitMs;
|
|
21
62
|
/** Paths THIS instance has touched → the EXACT BoxState it took a ref on (server[1523] 对等复审
|
|
22
63
|
* 发现1: after a drop/reap deletes the shared box, another instance's stale close() must not
|
|
23
64
|
* decrement a REBUILT box's refs — identity is checked, not just the path). */
|
|
@@ -53,6 +94,22 @@ export declare class FileMailboxStore implements MailboxStore {
|
|
|
53
94
|
* file talked about the rule.
|
|
54
95
|
*/
|
|
55
96
|
private lockKey;
|
|
97
|
+
/**
|
|
98
|
+
* The two-ring critical section (module header): the in-process mutex on the canonical key, then the
|
|
99
|
+
* cross-process box lock on the verbatim path. `fn` runs with both held; the box lock is released on
|
|
100
|
+
* every exit. A compromised release (the lock changed hands) surfaces as a throw AFTER `fn` — an
|
|
101
|
+
* operation whose critical section may have overlapped another process's is never reported as a
|
|
102
|
+
* success; when `fn` itself threw, its own error wins.
|
|
103
|
+
*/
|
|
104
|
+
private withBox;
|
|
105
|
+
/**
|
|
106
|
+
* design/385 ring 2 — bring a CACHED box up to date with the file, inside the box lock. A mark that
|
|
107
|
+
* matches means nothing happened since this process last left the file at rest; a moved mark means
|
|
108
|
+
* another process wrote (append/lease/ack/release, or a compaction/drop that replaced the inode),
|
|
109
|
+
* and the cache is replayed from the file. A moved INODE also swap-closes the descriptor (ring 3):
|
|
110
|
+
* the next write reopens the current file instead of landing on the unlinked one.
|
|
111
|
+
*/
|
|
112
|
+
private reconcile;
|
|
56
113
|
private load;
|
|
57
114
|
private commit;
|
|
58
115
|
private compact;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import { join, resolve, sep } from "node:path";
|
|
2
|
+
import { randomBytes } from "node:crypto";
|
|
2
3
|
import { assertRetentionPolicy } from "../../core/retention-policy.js";
|
|
3
|
-
import { existsSync, readdirSync, realpathSync } from "node:fs";
|
|
4
|
-
import { newestSentAt, } from "../../core/mailbox-store.js";
|
|
4
|
+
import { existsSync, linkSync, readdirSync, readFileSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { detachMailboxMessage, newestSentAt, readMailboxPeerMeta, } from "../../core/mailbox-store.js";
|
|
5
6
|
import { AppendLog, atomicWriteFile, canonicalStoreKey, ensureDir, readJsonlRecords, sanitizeScope, sanitizePathComponent } from "./fs-atomic.js";
|
|
6
7
|
import { assertAdoptionBootGate } from "./adoption/marker.js";
|
|
7
8
|
function realpathSyncSafe(p) {
|
|
@@ -12,6 +13,31 @@ function realpathSyncSafe(p) {
|
|
|
12
13
|
return p;
|
|
13
14
|
}
|
|
14
15
|
}
|
|
16
|
+
function diskMarkOf(path) {
|
|
17
|
+
try {
|
|
18
|
+
const st = statSync(path);
|
|
19
|
+
return { ino: st.ino, size: st.size, mtimeMs: st.mtimeMs };
|
|
20
|
+
}
|
|
21
|
+
catch (err) {
|
|
22
|
+
if (err.code === "ENOENT")
|
|
23
|
+
return undefined;
|
|
24
|
+
throw err;
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
const UNKNOWN_MARK = { ino: -1, size: -1, mtimeMs: -1 };
|
|
28
|
+
function markAfterWrite(path) {
|
|
29
|
+
try {
|
|
30
|
+
return diskMarkOf(path);
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return UNKNOWN_MARK;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function sameDiskMark(a, b) {
|
|
37
|
+
if (a === undefined || b === undefined)
|
|
38
|
+
return a === b;
|
|
39
|
+
return a.ino === b.ino && a.size === b.size && a.mtimeMs === b.mtimeMs;
|
|
40
|
+
}
|
|
15
41
|
const sharedBoxes = new Map();
|
|
16
42
|
function listBoxFiles(scopeDir) {
|
|
17
43
|
try {
|
|
@@ -35,17 +61,291 @@ function withPathLock(key, fn) {
|
|
|
35
61
|
});
|
|
36
62
|
return run;
|
|
37
63
|
}
|
|
64
|
+
export const MAILBOX_BOX_LOCK_TIMEOUT_CODE = "mailbox.box_lock_timeout";
|
|
65
|
+
export const MAILBOX_BOX_LOCK_COMPROMISED_CODE = "mailbox.box_lock_compromised";
|
|
66
|
+
const BOX_LOCK_RECLAIM_GRACE_MS = 5_000;
|
|
67
|
+
const BOX_LOCK_BACKOFF_MIN_MS = 1;
|
|
68
|
+
const BOX_LOCK_BACKOFF_MAX_MS = 16;
|
|
69
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
70
|
+
const abandonedLockNonces = new Set();
|
|
71
|
+
function ownerAlive(owner) {
|
|
72
|
+
if (owner.pid === process.pid)
|
|
73
|
+
return !abandonedLockNonces.has(owner.nonce);
|
|
74
|
+
return isProcessAlive(owner.pid);
|
|
75
|
+
}
|
|
76
|
+
function isProcessAlive(pid) {
|
|
77
|
+
try {
|
|
78
|
+
process.kill(pid, 0);
|
|
79
|
+
return true;
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
return err.code !== "ESRCH";
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function readLockOwner(lockPath) {
|
|
86
|
+
let raw;
|
|
87
|
+
try {
|
|
88
|
+
raw = readFileSync(lockPath, "utf8");
|
|
89
|
+
}
|
|
90
|
+
catch (err) {
|
|
91
|
+
if (err.code === "ENOENT")
|
|
92
|
+
return undefined;
|
|
93
|
+
return "unobservable";
|
|
94
|
+
}
|
|
95
|
+
const [pidText, nonce, rest] = raw.split("\n");
|
|
96
|
+
if (pidText === undefined || !/^[1-9][0-9]*$/.test(pidText) || nonce === undefined || !/^[0-9a-f]{16}$/.test(nonce) || rest !== "")
|
|
97
|
+
return "unreadable";
|
|
98
|
+
return { pid: Number.parseInt(pidText, 10), nonce };
|
|
99
|
+
}
|
|
100
|
+
function publishLockFile(lockPath, owner) {
|
|
101
|
+
const tmp = `${lockPath}.pub.${owner.pid}.${owner.nonce}`;
|
|
102
|
+
writeFileSync(tmp, `${owner.pid}\n${owner.nonce}\n`, { mode: 0o600 });
|
|
103
|
+
try {
|
|
104
|
+
linkSync(tmp, lockPath);
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
catch (err) {
|
|
108
|
+
const back = readLockOwner(lockPath);
|
|
109
|
+
if (back !== undefined && back !== "unreadable" && back !== "unobservable" && back.pid === owner.pid && back.nonce === owner.nonce)
|
|
110
|
+
return true;
|
|
111
|
+
if (back === "unobservable")
|
|
112
|
+
abandonedLockNonces.add(owner.nonce);
|
|
113
|
+
if (err.code === "EEXIST")
|
|
114
|
+
return false;
|
|
115
|
+
throw err;
|
|
116
|
+
}
|
|
117
|
+
finally {
|
|
118
|
+
try {
|
|
119
|
+
unlinkSync(tmp);
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
export async function acquireMailboxBoxLock(lockPath, waitMs) {
|
|
126
|
+
const mine = { pid: process.pid, nonce: randomBytes(8).toString("hex") };
|
|
127
|
+
const deadline = Date.now() + waitMs;
|
|
128
|
+
let backoff = BOX_LOCK_BACKOFF_MIN_MS;
|
|
129
|
+
let holderNote = "unknown holder";
|
|
130
|
+
let swept = false;
|
|
131
|
+
for (;;) {
|
|
132
|
+
let published;
|
|
133
|
+
try {
|
|
134
|
+
published = publishLockFile(lockPath, mine);
|
|
135
|
+
}
|
|
136
|
+
catch (err) {
|
|
137
|
+
if (err.code !== "ENOENT")
|
|
138
|
+
throw err;
|
|
139
|
+
ensureDir(join(lockPath, ".."));
|
|
140
|
+
continue;
|
|
141
|
+
}
|
|
142
|
+
if (published) {
|
|
143
|
+
return () => {
|
|
144
|
+
let now = readLockOwner(lockPath);
|
|
145
|
+
if (now === "unobservable")
|
|
146
|
+
now = readLockOwner(lockPath);
|
|
147
|
+
if (now === "unobservable") {
|
|
148
|
+
abandonedLockNonces.add(mine.nonce);
|
|
149
|
+
console.warn(`[sema] file mailbox: the box lock ${lockPath} could not be read back at release (a read fault); it is kept and still names this live process (pid ${process.pid}). ` +
|
|
150
|
+
`This process reclaims it at its next use of the box; another process waits on it and times out naming this pid until then.`);
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
if (now === undefined || now === "unreadable" || now.nonce !== mine.nonce) {
|
|
154
|
+
const e = new Error(`FileMailboxStore: the box lock ${lockPath} changed hands while this process held it (a stale reclaim overlapped) — the operation just performed may have raced another process; do not trust it`);
|
|
155
|
+
e.code = MAILBOX_BOX_LOCK_COMPROMISED_CODE;
|
|
156
|
+
e.lockPath = lockPath;
|
|
157
|
+
throw e;
|
|
158
|
+
}
|
|
159
|
+
try {
|
|
160
|
+
unlinkSync(lockPath);
|
|
161
|
+
}
|
|
162
|
+
catch (first) {
|
|
163
|
+
const again = readLockOwner(lockPath);
|
|
164
|
+
if (again === undefined || again === "unreadable" || (again !== "unobservable" && again.nonce !== mine.nonce))
|
|
165
|
+
return;
|
|
166
|
+
try {
|
|
167
|
+
if (again === "unobservable")
|
|
168
|
+
throw first;
|
|
169
|
+
unlinkSync(lockPath);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
abandonedLockNonces.add(mine.nonce);
|
|
173
|
+
console.warn(`[sema] file mailbox: the box lock ${lockPath} could not be handed back (${first.message}); it still names this live process (pid ${process.pid}). ` +
|
|
174
|
+
`This process reclaims it at its next use of the box; another process waits on it and times out naming this pid until then.`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
const owner = readLockOwner(lockPath);
|
|
180
|
+
if (owner === undefined)
|
|
181
|
+
continue;
|
|
182
|
+
let reclaim = "blocked";
|
|
183
|
+
if (owner === "unreadable") {
|
|
184
|
+
holderNote = "unreadable lock file — remove it by hand if no process holds it";
|
|
185
|
+
}
|
|
186
|
+
else if (owner === "unobservable") {
|
|
187
|
+
holderNote = "the lock file could not be read (a read fault)";
|
|
188
|
+
}
|
|
189
|
+
else if (ownerAlive(owner)) {
|
|
190
|
+
holderNote = owner.pid === process.pid ? `pid ${owner.pid} (this process — another isolate or thread of it)` : `pid ${owner.pid}`;
|
|
191
|
+
}
|
|
192
|
+
else {
|
|
193
|
+
const r = reclaimStaleBoxLock(lockPath, owner, mine);
|
|
194
|
+
reclaim = r.outcome;
|
|
195
|
+
holderNote = r.note;
|
|
196
|
+
}
|
|
197
|
+
if (reclaim === "done")
|
|
198
|
+
continue;
|
|
199
|
+
if (!swept) {
|
|
200
|
+
swept = true;
|
|
201
|
+
sweepLockResidue(join(lockPath, ".."), lockPath.slice(join(lockPath, "..").length + 1));
|
|
202
|
+
}
|
|
203
|
+
if (Date.now() >= deadline) {
|
|
204
|
+
const e = new Error(`FileMailboxStore: the box lock ${lockPath} was not obtainable within ${waitMs}ms (${holderNote}) — refusing to touch the box without it`);
|
|
205
|
+
e.code = MAILBOX_BOX_LOCK_TIMEOUT_CODE;
|
|
206
|
+
e.lockPath = lockPath;
|
|
207
|
+
if (owner !== "unreadable" && owner !== "unobservable")
|
|
208
|
+
e.holderPid = owner.pid;
|
|
209
|
+
throw e;
|
|
210
|
+
}
|
|
211
|
+
await sleep(backoff / 2 + Math.random() * (backoff / 2));
|
|
212
|
+
backoff = Math.min(BOX_LOCK_BACKOFF_MAX_MS, backoff * 2);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
function ageOf(path) {
|
|
216
|
+
try {
|
|
217
|
+
return Date.now() - statSync(path).mtimeMs;
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
return undefined;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
function reclaimStaleBoxLock(lockPath, stale, mine) {
|
|
224
|
+
const gate = `${lockPath}.reclaim.${stale.nonce}`;
|
|
225
|
+
let gateOwned;
|
|
226
|
+
try {
|
|
227
|
+
gateOwned = publishLockFile(gate, mine);
|
|
228
|
+
}
|
|
229
|
+
catch (err) {
|
|
230
|
+
return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; the reclaim gate could not be published: ${err.message}` };
|
|
231
|
+
}
|
|
232
|
+
if (!gateOwned) {
|
|
233
|
+
const g = readLockOwner(gate);
|
|
234
|
+
if (g === "unobservable")
|
|
235
|
+
return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; its reclaim gate could not be read (a read fault)` };
|
|
236
|
+
const gateLive = g !== undefined && g !== "unreadable" && ownerAlive(g);
|
|
237
|
+
if (gateLive)
|
|
238
|
+
return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; pid ${g.pid} is reclaiming it` };
|
|
239
|
+
if (g === undefined || g !== "unreadable" || (ageOf(gate) ?? 0) > BOX_LOCK_RECLAIM_GRACE_MS) {
|
|
240
|
+
try {
|
|
241
|
+
unlinkSync(gate);
|
|
242
|
+
}
|
|
243
|
+
catch {
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid}; a previous reclaim's gate is being cleared` };
|
|
247
|
+
}
|
|
248
|
+
try {
|
|
249
|
+
const again = readLockOwner(lockPath);
|
|
250
|
+
if (again === undefined)
|
|
251
|
+
return { outcome: "done", note: "" };
|
|
252
|
+
if (again === "unobservable")
|
|
253
|
+
return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid} could not be re-read under the gate (a read fault)` };
|
|
254
|
+
if (again === "unreadable" || again.pid !== stale.pid || again.nonce !== stale.nonce) {
|
|
255
|
+
return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid} was reclaimed meanwhile; re-judging the current holder` };
|
|
256
|
+
}
|
|
257
|
+
const aside = `${lockPath}.stale.${mine.pid}.${mine.nonce}`;
|
|
258
|
+
try {
|
|
259
|
+
renameSync(lockPath, aside);
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
if (err.code === "ENOENT")
|
|
263
|
+
return { outcome: "done", note: "" };
|
|
264
|
+
return { outcome: "blocked", note: `stale lock of dead pid ${stale.pid} cannot be moved aside: ${err.message}` };
|
|
265
|
+
}
|
|
266
|
+
try {
|
|
267
|
+
unlinkSync(aside);
|
|
268
|
+
}
|
|
269
|
+
catch {
|
|
270
|
+
}
|
|
271
|
+
sweepLockResidue(join(lockPath, ".."), lockPath.slice(join(lockPath, "..").length + 1));
|
|
272
|
+
return { outcome: "done", note: "" };
|
|
273
|
+
}
|
|
274
|
+
finally {
|
|
275
|
+
try {
|
|
276
|
+
unlinkSync(gate);
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
const RESIDUE_NAME_RE = /\.jsonl\.lock\.(?:pub|stale)\.([1-9][0-9]*)\.([0-9a-f]{16})$/;
|
|
283
|
+
const GATE_NAME_RE = /\.jsonl\.lock\.reclaim\.[0-9a-f]{16}$/;
|
|
284
|
+
function sweepLockResidue(dir, lockName) {
|
|
285
|
+
let names;
|
|
286
|
+
try {
|
|
287
|
+
names = readdirSync(dir);
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
292
|
+
const now = Date.now();
|
|
293
|
+
for (const name of names) {
|
|
294
|
+
if (lockName !== undefined && !name.startsWith(`${lockName}.`))
|
|
295
|
+
continue;
|
|
296
|
+
const isGate = GATE_NAME_RE.test(name);
|
|
297
|
+
const named = RESIDUE_NAME_RE.exec(name);
|
|
298
|
+
if (!isGate && named === null)
|
|
299
|
+
continue;
|
|
300
|
+
const p = join(dir, name);
|
|
301
|
+
let owner;
|
|
302
|
+
if (isGate) {
|
|
303
|
+
const g = readLockOwner(p);
|
|
304
|
+
if (g === "unobservable")
|
|
305
|
+
continue;
|
|
306
|
+
owner = g !== undefined && g !== "unreadable" ? g : undefined;
|
|
307
|
+
}
|
|
308
|
+
else {
|
|
309
|
+
owner = { pid: Number.parseInt(named[1], 10), nonce: named[2] };
|
|
310
|
+
}
|
|
311
|
+
if (owner !== undefined && ownerAlive(owner))
|
|
312
|
+
continue;
|
|
313
|
+
if (owner === undefined) {
|
|
314
|
+
let age;
|
|
315
|
+
try {
|
|
316
|
+
age = now - statSync(p).mtimeMs;
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
continue;
|
|
320
|
+
}
|
|
321
|
+
if (age <= BOX_LOCK_RECLAIM_GRACE_MS)
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
try {
|
|
325
|
+
unlinkSync(p);
|
|
326
|
+
}
|
|
327
|
+
catch {
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|
|
38
331
|
export class FileMailboxStore {
|
|
332
|
+
crossProcessSafe = true;
|
|
39
333
|
dir;
|
|
40
334
|
tmpDir;
|
|
41
335
|
fsyncEnabled;
|
|
42
336
|
compactEvery;
|
|
337
|
+
lockWaitMs;
|
|
43
338
|
touched = new Map();
|
|
44
339
|
discloseCorrupt;
|
|
45
340
|
constructor(root, opts = {}) {
|
|
46
341
|
assertAdoptionBootGate(root, "FileMailboxStore");
|
|
47
342
|
this.fsyncEnabled = opts.fsync !== false;
|
|
48
343
|
this.compactEvery = opts.compactEvery ?? 500;
|
|
344
|
+
const lockWaitMs = opts.lockWaitMs ?? 10_000;
|
|
345
|
+
if (typeof lockWaitMs !== "number" || !Number.isFinite(lockWaitMs) || lockWaitMs <= 0) {
|
|
346
|
+
throw new Error(`FileMailboxStore: lockWaitMs must be a finite number > 0, got ${JSON.stringify(opts.lockWaitMs)}`);
|
|
347
|
+
}
|
|
348
|
+
this.lockWaitMs = lockWaitMs;
|
|
49
349
|
const sink = opts.onCorruptRead;
|
|
50
350
|
this.discloseCorrupt =
|
|
51
351
|
sink === undefined
|
|
@@ -70,6 +370,40 @@ export class FileMailboxStore {
|
|
|
70
370
|
lockKey(scope, handle) {
|
|
71
371
|
return canonicalStoreKey(this.boxPath(scope, handle));
|
|
72
372
|
}
|
|
373
|
+
withBox(key, path, fn) {
|
|
374
|
+
return withPathLock(key, async () => {
|
|
375
|
+
const release = await acquireMailboxBoxLock(`${path}.lock`, this.lockWaitMs);
|
|
376
|
+
let result;
|
|
377
|
+
try {
|
|
378
|
+
result = await fn();
|
|
379
|
+
}
|
|
380
|
+
catch (err) {
|
|
381
|
+
try {
|
|
382
|
+
release();
|
|
383
|
+
}
|
|
384
|
+
catch {
|
|
385
|
+
}
|
|
386
|
+
throw err;
|
|
387
|
+
}
|
|
388
|
+
release();
|
|
389
|
+
return result;
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
reconcile(b) {
|
|
393
|
+
const now = diskMarkOf(b.path);
|
|
394
|
+
if (sameDiskMark(now, b.mark))
|
|
395
|
+
return;
|
|
396
|
+
if (now === undefined || b.mark === undefined || now.ino !== b.mark.ino)
|
|
397
|
+
b.log.closeForSwap();
|
|
398
|
+
const events = readJsonlRecords(b.path, this.discloseCorrupt);
|
|
399
|
+
b.messages = [];
|
|
400
|
+
b.nextSeq = 1;
|
|
401
|
+
delete b.lease;
|
|
402
|
+
for (const ev of events)
|
|
403
|
+
applyEvent(b, ev);
|
|
404
|
+
b.events = Math.max(0, events.length - (b.messages.length + 1 + (b.lease !== undefined ? 1 : 0)));
|
|
405
|
+
b.mark = now;
|
|
406
|
+
}
|
|
73
407
|
load(scope, handle) {
|
|
74
408
|
const path = this.boxPath(scope, handle);
|
|
75
409
|
const key = canonicalStoreKey(path);
|
|
@@ -79,13 +413,15 @@ export class FileMailboxStore {
|
|
|
79
413
|
b.refs++;
|
|
80
414
|
this.touched.set(key, b);
|
|
81
415
|
}
|
|
416
|
+
this.reconcile(b);
|
|
82
417
|
return b;
|
|
83
418
|
}
|
|
84
419
|
ensureDir(join(this.dir, sanitizeScope(scope)));
|
|
85
420
|
const events = readJsonlRecords(path, this.discloseCorrupt);
|
|
86
|
-
const state = { messages: [], nextSeq: 1, log: new AppendLog(path), events: events.length, refs: 1, path };
|
|
421
|
+
const state = { messages: [], nextSeq: 1, log: new AppendLog(path), events: events.length, refs: 1, path, mark: undefined };
|
|
87
422
|
for (const ev of events)
|
|
88
423
|
applyEvent(state, ev);
|
|
424
|
+
state.mark = diskMarkOf(path);
|
|
89
425
|
sharedBoxes.set(key, state);
|
|
90
426
|
this.touched.set(key, state);
|
|
91
427
|
return state;
|
|
@@ -101,6 +437,7 @@ export class FileMailboxStore {
|
|
|
101
437
|
catch {
|
|
102
438
|
}
|
|
103
439
|
}
|
|
440
|
+
b.mark = markAfterWrite(path);
|
|
104
441
|
}
|
|
105
442
|
compact(b, path) {
|
|
106
443
|
const lines = [];
|
|
@@ -112,20 +449,29 @@ export class FileMailboxStore {
|
|
|
112
449
|
b.log.closeForSwap();
|
|
113
450
|
atomicWriteFile(this.tmpDir, path, lines.length ? `${lines.join("\n")}\n` : "");
|
|
114
451
|
b.events = 0;
|
|
452
|
+
b.mark = markAfterWrite(path);
|
|
115
453
|
}
|
|
116
454
|
async append(scope, handle, msg) {
|
|
117
455
|
if (scope === undefined || scope === "")
|
|
118
456
|
throw new Error("MailboxStore.append: refusing a message without a scope");
|
|
119
|
-
|
|
457
|
+
const peerMeta = readMailboxPeerMeta(msg.peerMeta);
|
|
458
|
+
return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
|
|
120
459
|
const b = this.load(scope, handle);
|
|
121
460
|
const seq = b.nextSeq;
|
|
122
|
-
const m = {
|
|
461
|
+
const m = {
|
|
462
|
+
seq,
|
|
463
|
+
...(msg.from !== undefined ? { from: msg.from } : {}),
|
|
464
|
+
content: msg.content,
|
|
465
|
+
sentAt: msg.sentAt,
|
|
466
|
+
...(msg.hopChain !== undefined ? { hopChain: [...msg.hopChain] } : {}),
|
|
467
|
+
...(peerMeta !== undefined ? { peerMeta } : {}),
|
|
468
|
+
};
|
|
123
469
|
this.commit(b, this.boxPath(scope, handle), { t: "append", m });
|
|
124
470
|
return seq;
|
|
125
471
|
});
|
|
126
472
|
}
|
|
127
473
|
async claimLease(scope, handle, owner, ttlMs, now = Date.now()) {
|
|
128
|
-
return
|
|
474
|
+
return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
|
|
129
475
|
const b = this.load(scope, handle);
|
|
130
476
|
if (b.messages.length === 0)
|
|
131
477
|
return null;
|
|
@@ -133,11 +479,11 @@ export class FileMailboxStore {
|
|
|
133
479
|
return null;
|
|
134
480
|
const maxSeq = b.messages[b.messages.length - 1].seq;
|
|
135
481
|
this.commit(b, this.boxPath(scope, handle), { t: "lease", owner, expiresAt: now + ttlMs, maxSeq });
|
|
136
|
-
return { messages: b.messages.map((m) => (
|
|
482
|
+
return { messages: b.messages.map((m) => detachMailboxMessage(m)), maxSeq };
|
|
137
483
|
});
|
|
138
484
|
}
|
|
139
485
|
async ack(scope, handle, owner, upToSeq) {
|
|
140
|
-
return
|
|
486
|
+
return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
|
|
141
487
|
const b = this.load(scope, handle);
|
|
142
488
|
if (b.lease === undefined || b.lease.owner !== owner)
|
|
143
489
|
return;
|
|
@@ -145,7 +491,7 @@ export class FileMailboxStore {
|
|
|
145
491
|
});
|
|
146
492
|
}
|
|
147
493
|
async releaseLease(scope, handle, owner) {
|
|
148
|
-
return
|
|
494
|
+
return this.withBox(this.lockKey(scope, handle), this.boxPath(scope, handle), () => {
|
|
149
495
|
const b = this.load(scope, handle);
|
|
150
496
|
if (b.lease?.owner === owner)
|
|
151
497
|
this.commit(b, this.boxPath(scope, handle), { t: "release", owner });
|
|
@@ -154,8 +500,9 @@ export class FileMailboxStore {
|
|
|
154
500
|
async peekCount(scope, handle) {
|
|
155
501
|
const path = this.boxPath(scope, handle);
|
|
156
502
|
const loaded = sharedBoxes.get(canonicalStoreKey(path));
|
|
157
|
-
if (loaded !== undefined)
|
|
503
|
+
if (loaded !== undefined && sameDiskMark(diskMarkOf(path), loaded.mark)) {
|
|
158
504
|
return withPathLock(this.lockKey(scope, handle), () => loaded.messages.length);
|
|
505
|
+
}
|
|
159
506
|
if (!existsSync(path))
|
|
160
507
|
return 0;
|
|
161
508
|
return withPathLock(this.lockKey(scope, handle), () => {
|
|
@@ -166,9 +513,9 @@ export class FileMailboxStore {
|
|
|
166
513
|
});
|
|
167
514
|
}
|
|
168
515
|
async drop(scope, handle) {
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
516
|
+
const path = this.boxPath(scope, handle);
|
|
517
|
+
const key = canonicalStoreKey(path);
|
|
518
|
+
return this.withBox(key, path, () => {
|
|
172
519
|
const b = sharedBoxes.get(key);
|
|
173
520
|
if (b) {
|
|
174
521
|
b.log.close();
|
|
@@ -180,6 +527,7 @@ export class FileMailboxStore {
|
|
|
180
527
|
}
|
|
181
528
|
async reap(scope, now, opts) {
|
|
182
529
|
assertRetentionPolicy("MailboxStore.reap", opts);
|
|
530
|
+
sweepLockResidue(join(this.dir, sanitizeScope(scope)));
|
|
183
531
|
if (opts?.maxAgeMs === undefined)
|
|
184
532
|
return 0;
|
|
185
533
|
let dropped = 0;
|
|
@@ -189,13 +537,16 @@ export class FileMailboxStore {
|
|
|
189
537
|
for (const [key, b] of [...sharedBoxes]) {
|
|
190
538
|
if (!b.path.startsWith(prefix))
|
|
191
539
|
continue;
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
540
|
+
if (sameDiskMark(diskMarkOf(b.path), b.mark)) {
|
|
541
|
+
const seen = newestSentAt(b.messages);
|
|
542
|
+
if (seen === undefined || seen >= now - maxAgeMs)
|
|
543
|
+
continue;
|
|
544
|
+
}
|
|
545
|
+
const swept = await this.withBox(key, b.path, () => {
|
|
196
546
|
const live = sharedBoxes.get(key);
|
|
197
547
|
if (live === undefined || live !== b)
|
|
198
548
|
return false;
|
|
549
|
+
this.reconcile(b);
|
|
199
550
|
const newest = newestSentAt(b.messages);
|
|
200
551
|
if (newest === undefined || newest >= now - maxAgeMs)
|
|
201
552
|
return false;
|
|
@@ -212,7 +563,7 @@ export class FileMailboxStore {
|
|
|
212
563
|
const key = canonicalStoreKey(path);
|
|
213
564
|
if (sharedBoxes.has(key))
|
|
214
565
|
continue;
|
|
215
|
-
const swept = await
|
|
566
|
+
const swept = await this.withBox(key, path, () => {
|
|
216
567
|
if (sharedBoxes.has(key))
|
|
217
568
|
return false;
|
|
218
569
|
const b = { messages: [], nextSeq: 1 };
|