@cello-protocol/daemon 0.0.49 → 0.0.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/cello-daemon.js +13 -0
- package/dist/bin/cello-daemon.js.map +1 -1
- package/dist/connect-or-start.d.ts +11 -11
- package/dist/connect-or-start.d.ts.map +1 -1
- package/dist/connect-or-start.js +97 -39
- package/dist/connect-or-start.js.map +1 -1
- package/dist/daemon.d.ts +11 -0
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +127 -26
- package/dist/daemon.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/ipc-server.d.ts.map +1 -1
- package/dist/ipc-server.js +79 -13
- package/dist/ipc-server.js.map +1 -1
- package/dist/lock-file.d.ts +12 -0
- package/dist/lock-file.d.ts.map +1 -1
- package/dist/lock-file.js +36 -1
- package/dist/lock-file.js.map +1 -1
- package/dist/park-envelope.d.ts +93 -0
- package/dist/park-envelope.d.ts.map +1 -0
- package/dist/park-envelope.js +172 -0
- package/dist/park-envelope.js.map +1 -0
- package/dist/retry-queue.d.ts.map +1 -1
- package/dist/retry-queue.js +13 -1
- package/dist/retry-queue.js.map +1 -1
- package/dist/session-node-manager.d.ts +31 -13
- package/dist/session-node-manager.d.ts.map +1 -1
- package/dist/session-node-manager.js +104 -26
- package/dist/session-node-manager.js.map +1 -1
- package/dist/singleton-lock.d.ts +85 -0
- package/dist/singleton-lock.d.ts.map +1 -0
- package/dist/singleton-lock.js +208 -0
- package/dist/singleton-lock.js.map +1 -0
- package/dist/sqlcipher-db.d.ts +28 -0
- package/dist/sqlcipher-db.d.ts.map +1 -1
- package/dist/sqlcipher-db.js +6 -1
- package/dist/sqlcipher-db.js.map +1 -1
- package/package.json +4 -4
package/dist/bin/cello-daemon.js
CHANGED
|
@@ -14,6 +14,10 @@ import { join } from "node:path";
|
|
|
14
14
|
import { startDaemon } from "../daemon.js";
|
|
15
15
|
import { createDirectoryEndpointResolver } from "../directory-bootstrap.js";
|
|
16
16
|
import { buildManifestDeps } from "../manifest-deps.js";
|
|
17
|
+
// EXIT_ALREADY_RUNNING is distinct from 1 (generic startup failure) so a caller can tell "lost the
|
|
18
|
+
// race" from "broken" — connectOrStart relies on exactly that distinction, so the constant is shared
|
|
19
|
+
// rather than written down twice.
|
|
20
|
+
import { DaemonAlreadyRunningError, EXIT_ALREADY_RUNNING } from "../singleton-lock.js";
|
|
17
21
|
const MAX_CONNECTIONS = 16;
|
|
18
22
|
// Composition root: stdout JSON logger
|
|
19
23
|
const logger = {
|
|
@@ -111,6 +115,15 @@ async function main() {
|
|
|
111
115
|
});
|
|
112
116
|
}
|
|
113
117
|
main().catch((err) => {
|
|
118
|
+
// DOD-SINGLE-DAEMON-1 (AC2): losing the singleton race is not a crash — it is the system working.
|
|
119
|
+
// Say so in one plain line an operator can act on, name the pid that holds the lock, and exit
|
|
120
|
+
// non-zero without a stack trace. Two daemons is the silent, wrong outcome; this is the loud, right
|
|
121
|
+
// one.
|
|
122
|
+
if (err instanceof DaemonAlreadyRunningError) {
|
|
123
|
+
logger.info("daemon.start.refused", { reason: "already_running", holderPid: err.holderPid });
|
|
124
|
+
process.stderr.write(`${err.message}\n`);
|
|
125
|
+
process.exit(EXIT_ALREADY_RUNNING);
|
|
126
|
+
}
|
|
114
127
|
logger.error("daemon.startup.failed", {
|
|
115
128
|
error: err instanceof Error ? err.message : String(err),
|
|
116
129
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cello-daemon.js","sourceRoot":"","sources":["../../src/bin/cello-daemon.ts"],"names":[],"mappings":";AACA;;;;;;;;;GASG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,+BAA+B,EAAE,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"cello-daemon.js","sourceRoot":"","sources":["../../src/bin/cello-daemon.ts"],"names":[],"mappings":";AACA;;;;;;;;;GASG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,+BAA+B,EAAE,MAAM,2BAA2B,CAAC;AAC5E,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,mGAAmG;AACnG,qGAAqG;AACrG,kCAAkC;AAClC,OAAO,EAAE,yBAAyB,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAGvF,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,uCAAuC;AACvC,MAAM,MAAM,GAAW;IACrB,KAAK,CAAC,KAAa,EAAE,OAAgC;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACjG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,KAAa,EAAE,OAAgC;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAChG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IACD,IAAI,CAAC,KAAa,EAAE,OAAgC;QAClD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QAChG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;IACD,KAAK,CAAC,KAAa,EAAE,OAAgC;QACnD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACjG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IACpC,CAAC;CACF,CAAC;AAEF,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;AACpE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;AACjD,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;AACnD,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,OAAO,CAAC;AAErD,KAAK,UAAU,IAAI;IACjB,gFAAgF;IAChF,iFAAiF;IACjF,8EAA8E;IAC9E,0BAA0B;IAC1B,iGAAiG;IACjG,kGAAkG;IAClG,mGAAmG;IACnG,mGAAmG;IACnG,MAAM,yBAAyB,GAAG,+BAA+B,CAAC,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpG,6FAA6F;IAC7F,+FAA+F;IAC/F,kGAAkG;IAClG,kGAAkG;IAClG,mGAAmG;IACnG,4FAA4F;IAC5F,mGAAmG;IACnG,2EAA2E;IAC3E,mGAAmG;IACnG,gGAAgG;IAChG,MAAM,QAAQ,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAE3C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,QAAQ;QACR,UAAU;QACV,YAAY;QACZ,cAAc,EAAE,eAAe;QAC/B,OAAO;QACP,MAAM;QACN,yBAAyB;QACzB,GAAG,QAAQ;KACZ,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,KAAK,EAAE,MAAc,EAAiB,EAAE;QACvD,MAAM,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IAEF,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;QACzB,IAAI,CAAC;YACH,QAAQ,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACzC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;oBACrC,MAAM,EAAE,SAAS;oBACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;gBACrC,MAAM,EAAE,SAAS;gBACjB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,IAAI,CAAC;YACH,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACxC,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;oBACrC,MAAM,EAAE,QAAQ;oBAChB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBACxD,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE;gBACrC,MAAM,EAAE,QAAQ;gBAChB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACxD,CAAC,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;IAC5B,kGAAkG;IAClG,8FAA8F;IAC9F,oGAAoG;IACpG,OAAO;IACP,IAAI,GAAG,YAAY,yBAAyB,EAAE,CAAC;QAC7C,MAAM,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,MAAM,EAAE,iBAAiB,EAAE,SAAS,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC;QAC7F,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;QACzC,OAAO,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACrC,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE;QACpC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;KACxD,CAAC,CAAC;IACH,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -3,17 +3,17 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Pseudocode:
|
|
5
5
|
* 1. connectOrStart(celloDir, logger):
|
|
6
|
-
* a.
|
|
7
|
-
* b.
|
|
8
|
-
* -
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* -
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
6
|
+
* a. Try to connect to the daemon socket. If it answers, we are done.
|
|
7
|
+
* b. Otherwise ask the KERNEL whether a daemon holds the singleton lock (DOD-SINGLE-DAEMON-1):
|
|
8
|
+
* - HELD → a daemon is alive but not answering (mid-startup, or its socket was destroyed).
|
|
9
|
+
* Retry the connection, then fail loudly. NEVER spawn a second daemon beside it,
|
|
10
|
+
* and never delete its lock.
|
|
11
|
+
* - FREE → no daemon is running, whatever daemon.lock claims. Clear the stale lock, spawn,
|
|
12
|
+
* and wait for the socket to accept connections.
|
|
13
|
+
*
|
|
14
|
+
* The lock FILE is not consulted to make this decision — only the OS lock is. A pid in a JSON file
|
|
15
|
+
* can be dead, or REUSED by an unrelated process (in which case `isProcessAlive` says "alive"
|
|
16
|
+
* forever, and a daemon that trusted it would never start again).
|
|
17
17
|
*/
|
|
18
18
|
import { type IpcClient } from "./ipc-client.js";
|
|
19
19
|
import type { Logger } from "./types.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connect-or-start.d.ts","sourceRoot":"","sources":["../src/connect-or-start.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;
|
|
1
|
+
{"version":3,"file":"connect-or-start.d.ts","sourceRoot":"","sources":["../src/connect-or-start.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAQH,OAAO,EAAmB,KAAK,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAEzC,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,SAAS,CAAC;IAClB,cAAc,EAAE,OAAO,CAAC;CACzB;AAED,wBAAsB,cAAc,CAClC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,aAAa,CAAC,CAqDxB"}
|
package/dist/connect-or-start.js
CHANGED
|
@@ -3,58 +3,105 @@
|
|
|
3
3
|
*
|
|
4
4
|
* Pseudocode:
|
|
5
5
|
* 1. connectOrStart(celloDir, logger):
|
|
6
|
-
* a.
|
|
7
|
-
* b.
|
|
8
|
-
* -
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* -
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
6
|
+
* a. Try to connect to the daemon socket. If it answers, we are done.
|
|
7
|
+
* b. Otherwise ask the KERNEL whether a daemon holds the singleton lock (DOD-SINGLE-DAEMON-1):
|
|
8
|
+
* - HELD → a daemon is alive but not answering (mid-startup, or its socket was destroyed).
|
|
9
|
+
* Retry the connection, then fail loudly. NEVER spawn a second daemon beside it,
|
|
10
|
+
* and never delete its lock.
|
|
11
|
+
* - FREE → no daemon is running, whatever daemon.lock claims. Clear the stale lock, spawn,
|
|
12
|
+
* and wait for the socket to accept connections.
|
|
13
|
+
*
|
|
14
|
+
* The lock FILE is not consulted to make this decision — only the OS lock is. A pid in a JSON file
|
|
15
|
+
* can be dead, or REUSED by an unrelated process (in which case `isProcessAlive` says "alive"
|
|
16
|
+
* forever, and a daemon that trusted it would never start again).
|
|
17
17
|
*/
|
|
18
18
|
import { spawn } from "node:child_process";
|
|
19
19
|
import { openSync } from "node:fs";
|
|
20
|
-
import { readFile } from "node:fs/promises";
|
|
20
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
21
21
|
import { join } from "node:path";
|
|
22
|
-
import { readLock, removeLock
|
|
22
|
+
import { readLock, removeLock } from "./lock-file.js";
|
|
23
|
+
import { probeSingletonLock, SINGLETON_LOCK_FILENAME, EXIT_ALREADY_RUNNING } from "./singleton-lock.js";
|
|
23
24
|
import { connectToDaemon } from "./ipc-client.js";
|
|
24
25
|
export async function connectOrStart(celloDir, logger, daemonBin) {
|
|
26
|
+
// A CELLO_DIR that does not exist yet (first ever run) would make the singleton probe below fail to
|
|
27
|
+
// open its lock file, and "I could not ask the kernel" would be reported as "I don't know whether a
|
|
28
|
+
// daemon is running" — a blind spot invented out of an empty directory. Create it first; the daemon
|
|
29
|
+
// does the same on startup, so this changes nothing except the quality of the answer.
|
|
30
|
+
await mkdir(celloDir, { recursive: true });
|
|
25
31
|
const lockFilePath = join(celloDir, "daemon.lock");
|
|
26
|
-
// Read existing lock
|
|
27
32
|
const lock = await readLock(lockFilePath);
|
|
33
|
+
// The socket path is deterministic; the lock's copy is metadata, and it may be stale.
|
|
34
|
+
const socketPath = lock?.socketPath ?? join(celloDir, "daemon.sock");
|
|
35
|
+
// 1. If a daemon is answering, we are done. This is the overwhelmingly common path.
|
|
36
|
+
try {
|
|
37
|
+
return { client: await connectToDaemon(socketPath), alreadyRunning: true };
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
// Not answering. WHY it is not answering is the whole question — see below.
|
|
41
|
+
}
|
|
42
|
+
// 2. Ask the KERNEL whether a daemon is alive, not `daemon.lock`.
|
|
43
|
+
//
|
|
44
|
+
// DOD-SINGLE-DAEMON-1 (AC3): the old code asked `isProcessAlive(lock.pid)` and, when the socket
|
|
45
|
+
// did not answer, called the lock "stale", deleted it, and spawned a second daemon onto the live
|
|
46
|
+
// one's database. Both of its beliefs were unreliable: a pid can be REUSED (so a dead daemon's lock
|
|
47
|
+
// looks alive forever), and a live daemon can legitimately not answer yet (it is mid-startup, or an
|
|
48
|
+
// exiting orphan deleted its socket). The lock file cannot tell these apart. The OS lock can.
|
|
49
|
+
const probe = probeSingletonLock(celloDir, logger);
|
|
50
|
+
if (probe === "held") {
|
|
51
|
+
// A daemon is holding the lock right now. Spawning beside it is never correct. It may simply be
|
|
52
|
+
// mid-startup, so give it a moment to come up.
|
|
53
|
+
const client = await connectWithRetry(celloDir, socketPath, logger, lock?.pid ?? null);
|
|
54
|
+
return { client, alreadyRunning: true };
|
|
55
|
+
}
|
|
56
|
+
if (probe === "unknown") {
|
|
57
|
+
// We could not ask the kernel. That is NOT permission to spawn: "I don't know whether a daemon is
|
|
58
|
+
// running" must never be resolved by starting a second one. Fail loud with the real reason.
|
|
59
|
+
throw new Error("Could not determine whether a daemon is already running (the singleton lock could not be " +
|
|
60
|
+
`checked at ${celloDir}). Refusing to start a second daemon on a guess. See the ` +
|
|
61
|
+
"daemon.singleton.probe.failed log line for the underlying error.");
|
|
62
|
+
}
|
|
63
|
+
// 3. The lock is genuinely free, so no daemon is running — whatever `daemon.lock` claims. Only now
|
|
64
|
+
// is spawning correct.
|
|
28
65
|
if (lock) {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
try {
|
|
32
|
-
const client = await connectToDaemon(lock.socketPath);
|
|
33
|
-
return { client, alreadyRunning: true };
|
|
34
|
-
}
|
|
35
|
-
catch (err) {
|
|
36
|
-
logger.info("daemon.lock.stale", {
|
|
37
|
-
pid: lock.pid,
|
|
38
|
-
reason: "socket_unreachable",
|
|
39
|
-
error: err instanceof Error ? err.message : String(err),
|
|
40
|
-
});
|
|
41
|
-
await removeLock(lockFilePath, logger);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
else {
|
|
45
|
-
// PID is dead — stale lock
|
|
46
|
-
logger.info("daemon.lock.stale", {
|
|
47
|
-
pid: lock.pid,
|
|
48
|
-
reason: "process_dead",
|
|
49
|
-
});
|
|
50
|
-
await removeLock(lockFilePath, logger);
|
|
51
|
-
}
|
|
66
|
+
logger.info("daemon.lock.stale", { pid: lock.pid, reason: "no_singleton_holder" });
|
|
67
|
+
await removeLock(lockFilePath, logger);
|
|
52
68
|
}
|
|
53
|
-
// Start a new daemon
|
|
54
69
|
const client = await spawnDaemon(celloDir, daemonBin, lockFilePath, logger);
|
|
55
70
|
return { client, alreadyRunning: false };
|
|
56
71
|
}
|
|
57
|
-
|
|
72
|
+
/**
|
|
73
|
+
* A daemon holds the singleton lock but is not answering. Wait for it, then fail LOUDLY.
|
|
74
|
+
*
|
|
75
|
+
* The loud failure is the point (SI). The alternative — the old behaviour — was to quietly start a
|
|
76
|
+
* second daemon, which is the very thing that gives one machine two writers on one hash chain. An
|
|
77
|
+
* operator who is told "a daemon is running but not answering" can act; an operator silently given a
|
|
78
|
+
* second daemon cannot even see the problem.
|
|
79
|
+
*/
|
|
80
|
+
async function connectWithRetry(celloDir, socketPath, logger, holderPid) {
|
|
81
|
+
const deadline = Date.now() + 10_000;
|
|
82
|
+
let lastError;
|
|
83
|
+
while (Date.now() < deadline) {
|
|
84
|
+
try {
|
|
85
|
+
return await connectToDaemon(socketPath);
|
|
86
|
+
}
|
|
87
|
+
catch (err) {
|
|
88
|
+
lastError = err;
|
|
89
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
logger.error("daemon.unreachable", { holderPid, socketPath });
|
|
93
|
+
// The pid comes from daemon.lock — the very file this unit exists because it cannot be trusted. It
|
|
94
|
+
// may be stale, or name a process that merely inherited the number. So it is offered as a hint, and
|
|
95
|
+
// the operator is pointed at the authoritative answer: whoever holds the lock file open IS the
|
|
96
|
+
// daemon. Telling someone to `kill` an unverified pid is how you get them to kill something else.
|
|
97
|
+
const hint = holderPid === null ? "" : ` (daemon.lock names pid ${holderPid}, which may be stale)`;
|
|
98
|
+
throw new Error(`A daemon is already running${hint} and holds the singleton lock, but is not answering on ` +
|
|
99
|
+
`${socketPath} (${lastError instanceof Error ? lastError.message : String(lastError)}). ` +
|
|
100
|
+
"Refusing to start a second daemon beside it — two daemons on one database is the failure this " +
|
|
101
|
+
`prevents. To find the real holder: \`lsof ${join(celloDir, SINGLETON_LOCK_FILENAME)}\`, then stop ` +
|
|
102
|
+
"that process and try again.");
|
|
103
|
+
}
|
|
104
|
+
async function spawnDaemon(celloDir, daemonBin, lockFilePath, logger) {
|
|
58
105
|
const socketPath = join(celloDir, "daemon.sock");
|
|
59
106
|
const logPath = join(celloDir, "daemon.log");
|
|
60
107
|
// The daemon's stdout/stderr go to a LOG FILE, never a pipe to this process.
|
|
@@ -91,6 +138,17 @@ async function spawnDaemon(celloDir, daemonBin, _lockFilePath, _logger) {
|
|
|
91
138
|
// Not accepting yet (ENOENT/ECONNREFUSED). If the child has already exited, it
|
|
92
139
|
// died during startup — surface the log tail rather than spinning to the deadline.
|
|
93
140
|
if (exitCode !== undefined) {
|
|
141
|
+
// ...unless it exited because it LOST THE SINGLETON RACE. Two callers can legitimately probe
|
|
142
|
+
// "free" at the same instant (the probe is not a reservation) and both spawn; the kernel then
|
|
143
|
+
// picks exactly one winner and the other exits EXIT_ALREADY_RUNNING. That is the design
|
|
144
|
+
// working, not a failure: a healthy daemon is coming up right now. Reporting "Daemon exited
|
|
145
|
+
// (code 3) during startup" to an operator whose daemon is fine would be this unit's own lie.
|
|
146
|
+
// The distinct exit code exists precisely so we can tell these apart — so use it.
|
|
147
|
+
if (exitCode === EXIT_ALREADY_RUNNING) {
|
|
148
|
+
const winner = await readLock(lockFilePath);
|
|
149
|
+
logger.info("daemon.spawn.lost_race", { holderPid: winner?.pid ?? null });
|
|
150
|
+
return await connectWithRetry(celloDir, socketPath, logger, winner?.pid ?? null);
|
|
151
|
+
}
|
|
94
152
|
throw new Error(`Daemon exited (code ${exitCode}) during startup.\n${await daemonLogTail(logPath)}`);
|
|
95
153
|
}
|
|
96
154
|
await new Promise((r) => setTimeout(r, 150));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"connect-or-start.js","sourceRoot":"","sources":["../src/connect-or-start.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"connect-or-start.js","sourceRoot":"","sources":["../src/connect-or-start.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AACxG,OAAO,EAAE,eAAe,EAAkB,MAAM,iBAAiB,CAAC;AAQlE,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,QAAgB,EAChB,MAAc,EACd,SAAiB;IAEjB,oGAAoG;IACpG,oGAAoG;IACpG,oGAAoG;IACpG,sFAAsF;IACtF,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE3C,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC1C,sFAAsF;IACtF,MAAM,UAAU,GAAG,IAAI,EAAE,UAAU,IAAI,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAErE,oFAAoF;IACpF,IAAI,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC,UAAU,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;IAC7E,CAAC;IAAC,MAAM,CAAC;QACP,4EAA4E;IAC9E,CAAC;IAED,kEAAkE;IAClE,EAAE;IACF,gGAAgG;IAChG,iGAAiG;IACjG,oGAAoG;IACpG,oGAAoG;IACpG,8FAA8F;IAC9F,MAAM,KAAK,GAAG,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEnD,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;QACrB,gGAAgG;QAChG,+CAA+C;QAC/C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC;QACvF,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;IAED,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,kGAAkG;QAClG,4FAA4F;QAC5F,MAAM,IAAI,KAAK,CACb,2FAA2F;YAC3F,cAAc,QAAQ,2DAA2D;YACjF,kEAAkE,CACnE,CAAC;IACJ,CAAC;IAED,mGAAmG;IACnG,0BAA0B;IAC1B,IAAI,IAAI,EAAE,CAAC;QACT,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC,CAAC;QACnF,MAAM,UAAU,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IAC5E,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC;AAC3C,CAAC;AAED;;;;;;;GAOG;AACH,KAAK,UAAU,gBAAgB,CAC7B,QAAgB,EAChB,UAAkB,EAClB,MAAc,EACd,SAAwB;IAExB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;IACrC,IAAI,SAAkB,CAAC;IACvB,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,OAAO,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,SAAS,GAAG,GAAG,CAAC;YAChB,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,oBAAoB,EAAE,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC,CAAC;IAC9D,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,kGAAkG;IAClG,MAAM,IAAI,GAAG,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,2BAA2B,SAAS,uBAAuB,CAAC;IACnG,MAAM,IAAI,KAAK,CACb,8BAA8B,IAAI,yDAAyD;QAC3F,GAAG,UAAU,KAAK,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK;QACzF,gGAAgG;QAChG,6CAA6C,IAAI,CAAC,QAAQ,EAAE,uBAAuB,CAAC,gBAAgB;QACpG,6BAA6B,CAC9B,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,QAAgB,EAChB,SAAiB,EACjB,YAAoB,EACpB,MAAc;IAEd,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAE7C,6EAA6E;IAC7E,iFAAiF;IACjF,yFAAyF;IACzF,qFAAqF;IACrF,uFAAuF;IACvF,2EAA2E;IAC3E,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,SAAS,CAAC,EAAE;QACjD,QAAQ,EAAE,IAAI;QACd,KAAK,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;QAC3B,GAAG,EAAE;YACH,GAAG,OAAO,CAAC,GAAG;YACd,SAAS,EAAE,QAAQ;SACpB;KACF,CAAC,CAAC;IACH,KAAK,CAAC,KAAK,EAAE,CAAC;IAEd,uEAAuE;IACvE,mFAAmF;IACnF,oFAAoF;IACpF,IAAI,UAAU,GAAiB,IAAI,CAAC;IACpC,IAAI,QAAmC,CAAC;IACxC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE,GAAG,UAAU,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACzD,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAmB,EAAE,EAAE,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAEhE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC;IACrC,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;QAC7B,IAAI,UAAU;YAAE,MAAM,UAAU,CAAC;QACjC,IAAI,CAAC;YACH,OAAO,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,+EAA+E;YAC/E,mFAAmF;YACnF,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;gBAC3B,6FAA6F;gBAC7F,8FAA8F;gBAC9F,wFAAwF;gBACxF,4FAA4F;gBAC5F,6FAA6F;gBAC7F,kFAAkF;gBAClF,IAAI,QAAQ,KAAK,oBAAoB,EAAE,CAAC;oBACtC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,YAAY,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,CAAC;oBAC1E,OAAO,MAAM,gBAAgB,CAAC,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC;gBACnF,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,QAAQ,sBAAsB,MAAM,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACvG,CAAC;YACD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,8CAA8C,MAAM,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;AAChG,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,OAAe;IAC1C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChG,OAAO,uCAAuC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,qBAAqB,OAAO,GAAG,CAAC;IACzC,CAAC;AACH,CAAC"}
|
package/dist/daemon.d.ts
CHANGED
|
@@ -62,5 +62,16 @@ export interface DaemonHandle {
|
|
|
62
62
|
getAutoNatService(): IAutoNatService;
|
|
63
63
|
}
|
|
64
64
|
export declare const INBOUND_SESSION_TTL_MS: number;
|
|
65
|
+
/**
|
|
66
|
+
* DOD-SINGLE-DAEMON-1: take the singleton lock, and make sure it is released if startup fails
|
|
67
|
+
* ANYWHERE after that point — not just on the two failure paths that happened to think of it.
|
|
68
|
+
*
|
|
69
|
+
* The real binary exits on a startup failure and the kernel reclaims the lock regardless, so this is
|
|
70
|
+
* not a production-safety hole. It is a DIAGNOSTIC one, and a nasty kind: an in-process caller (every
|
|
71
|
+
* vitest daemon, any embedder) whose startup throws — a corrupt database, a bad key, an EACCES on the
|
|
72
|
+
* lock file — would leak the lock and then be told, on the next attempt in that process, "another
|
|
73
|
+
* daemon is already running". The true cause is replaced by a lie, and the directory is wedged for the
|
|
74
|
+
* life of the process.
|
|
75
|
+
*/
|
|
65
76
|
export declare function startDaemon(config: DaemonConfig): Promise<DaemonHandle>;
|
|
66
77
|
//# sourceMappingURL=daemon.d.ts.map
|
package/dist/daemon.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EAQrB,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../src/daemon.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAKH,OAAO,KAAK,EACV,YAAY,EACZ,oBAAoB,EAQrB,MAAM,YAAY,CAAC;AAMpB,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAU/D,OAAO,EAAoD,KAAK,SAAS,EAAE,MAAM,2BAA2B,CAAC;AAuB7G,OAAO,KAAK,EAAE,kBAAkB,EAA+C,MAAM,yBAAyB,CAAC;AAQ/G,OAAO,EAAoB,KAAK,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA0InF,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,SAAS,IAAI,oBAAoB,CAAC;IAClC;;;;OAIG;IACH,qBAAqB,IAAI,kBAAkB,CAAC;IAC5C;;;;;;;;OAQG;IACH,gBAAgB,IAAI,SAAS,GAAG,IAAI,CAAC;IACrC;;;;OAIG;IACH,oBAAoB,IAAI,kBAAkB,CAAC;IAC3C;;;OAGG;IACH,iBAAiB,IAAI,eAAe,CAAC;CACtC;AA2CD,eAAO,MAAM,sBAAsB,QAAsB,CAAC;AAE1D;;;;;;;;;;GAUG;AACH,wBAAsB,WAAW,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAS7E"}
|
package/dist/daemon.js
CHANGED
|
@@ -29,7 +29,8 @@ import { mkdir } from "node:fs/promises";
|
|
|
29
29
|
import { randomUUID, createHash } from "node:crypto";
|
|
30
30
|
import { dirname, join } from "node:path";
|
|
31
31
|
import { loadAgents } from "./agent-loader.js";
|
|
32
|
-
import { acquireLock,
|
|
32
|
+
import { acquireLock, removeLockIfOwned } from "./lock-file.js";
|
|
33
|
+
import { acquireSingletonLock } from "./singleton-lock.js";
|
|
33
34
|
import { createIpcServer } from "./ipc-server.js";
|
|
34
35
|
import { renderForSurface } from "./vocabulary.js";
|
|
35
36
|
import { SessionNodeManager } from "./session-node-manager.js";
|
|
@@ -48,12 +49,13 @@ import { RegistrationManager } from "./registration-manager.js";
|
|
|
48
49
|
import { DaemonRegistrationContext } from "./registration-context.js";
|
|
49
50
|
import { DbRegistrationPersistence, DbIdentityStore } from "./db-identity-store.js";
|
|
50
51
|
import { DbManifestVersionStore } from "./manifest-version-store-db.js";
|
|
51
|
-
import { verify as ed25519Verify,
|
|
52
|
+
import { verify as ed25519Verify, generateKLocalSeed, InMemoryKeyProvider, hash as cryptoHash } from "@cello-protocol/crypto";
|
|
52
53
|
import { attemptSealUpgrade as attemptSealUpgradeImpl, verifyUpgradeConfirmedCert } from "./seal-upgrade.js";
|
|
53
54
|
import { upgradeAbsentToRecovered, hasAbsentParticipant } from "./seal-receipt-upgrade.js";
|
|
54
55
|
// CELLO-M7-MSG-001 (AC-013/AC-018): the single application content-size cap, enforced
|
|
55
56
|
// at the send point here (the receive point lives in the transport content decode).
|
|
56
57
|
import { MAX_CONTENT_BYTES, computeGenesisPrevRoot, buildAgentRevocationTbs, MONIKER_RE, validateMoniker } from "@cello-protocol/protocol-types";
|
|
58
|
+
import { sealParkEnvelope } from "./park-envelope.js";
|
|
57
59
|
import { resolveCelloEnv, createTransportSelector, isProductionVariant, } from "./transport-composition.js";
|
|
58
60
|
import { selectAdvertisedAddress } from "./transport-selector.js";
|
|
59
61
|
import { parseSessionAssignment, sessionRequestErrorReason, parseDiscoveryLookupResult, discoveryLookupErrorReason, extractOfferedMoniker, resolveOutboundMoniker } from "./session-assignment-parser.js";
|
|
@@ -183,7 +185,29 @@ class ProductionSessionNodeFactory {
|
|
|
183
185
|
// M8C-TTL-1: receiver-side session-request TTL. CORE ships the DoD's own 24h default;
|
|
184
186
|
// per-agent configurability is PARKED on M9-CFG-001 (D17 — same pattern as D14/D15/D16).
|
|
185
187
|
export const INBOUND_SESSION_TTL_MS = 24 * 60 * 60 * 1000;
|
|
188
|
+
/**
|
|
189
|
+
* DOD-SINGLE-DAEMON-1: take the singleton lock, and make sure it is released if startup fails
|
|
190
|
+
* ANYWHERE after that point — not just on the two failure paths that happened to think of it.
|
|
191
|
+
*
|
|
192
|
+
* The real binary exits on a startup failure and the kernel reclaims the lock regardless, so this is
|
|
193
|
+
* not a production-safety hole. It is a DIAGNOSTIC one, and a nasty kind: an in-process caller (every
|
|
194
|
+
* vitest daemon, any embedder) whose startup throws — a corrupt database, a bad key, an EACCES on the
|
|
195
|
+
* lock file — would leak the lock and then be told, on the next attempt in that process, "another
|
|
196
|
+
* daemon is already running". The true cause is replaced by a lie, and the directory is wedged for the
|
|
197
|
+
* life of the process.
|
|
198
|
+
*/
|
|
186
199
|
export async function startDaemon(config) {
|
|
200
|
+
await mkdir(config.celloDir, { recursive: true });
|
|
201
|
+
const singletonLock = acquireSingletonLock(config.celloDir, config.logger);
|
|
202
|
+
try {
|
|
203
|
+
return await startDaemonHoldingLock(config, singletonLock);
|
|
204
|
+
}
|
|
205
|
+
catch (err) {
|
|
206
|
+
singletonLock.release();
|
|
207
|
+
throw err;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
async function startDaemonHoldingLock(config, singletonLock) {
|
|
187
211
|
const { celloDir, socketPath, lockFilePath, maxConnections, version, logger, manifestProvider, manifestRootKeys, manifestThreshold, manifestVersionStore: injectedManifestVersionStore, manifestPollScheduler, directoryHttpUrl, signalingConnect, challengeVerifier, directoryEndpointResolver, sessionNodeFactory, sessionNegotiator, getRelayCircuitAddress, telegramBotClient: injectedTelegramBotClient, } = config;
|
|
188
212
|
// CELLO-M7-TRANSPORT-001: composition-root selection of the transport selector.
|
|
189
213
|
// Driven by CELLO_ENV; fails fast at startup (here, not at first session) when a
|
|
@@ -211,6 +235,14 @@ export async function startDaemon(config) {
|
|
|
211
235
|
// encrypted DB (AC-008), not a manifest-version.json file. ──
|
|
212
236
|
await mkdir(celloDir, { recursive: true });
|
|
213
237
|
await mkdir(dirname(socketPath), { recursive: true });
|
|
238
|
+
// DOD-SINGLE-DAEMON-1: the caller already took the kernel's exclusive lock (see startDaemon) —
|
|
239
|
+
// BEFORE this function touches anything that assumes a single writer. A daemon that loses that race
|
|
240
|
+
// never reaches here at all: it never opens the database, never registers an agent, never connects
|
|
241
|
+
// to the directory. Two daemons sharing an identity is how a hash chain gets two leaves at the same
|
|
242
|
+
// index, and the seal then attests to the damage.
|
|
243
|
+
//
|
|
244
|
+
// AC4: the advisory JSON keeps its metadata role (it is how the NEXT process learns our pid), but
|
|
245
|
+
// the OS lock is what decides whether a daemon may run. This file never gates startup.
|
|
214
246
|
await acquireLock(lockFilePath, { pid: process.pid, socketPath, version });
|
|
215
247
|
// M9-CORE-001: one security-gateway client, shared by both seams — the outbound screen in
|
|
216
248
|
// cello_send and the inbound screen inside SessionNodeManager. Absent config falls back to a
|
|
@@ -341,7 +373,9 @@ export async function startDaemon(config) {
|
|
|
341
373
|
sessionNodeManager.getDb().close();
|
|
342
374
|
}
|
|
343
375
|
catch { /* ignore */ }
|
|
344
|
-
await
|
|
376
|
+
await removeLockIfOwned(lockFilePath, process.pid, logger).catch(() => { });
|
|
377
|
+
// The singleton lock is released by startDaemon's catch — every throw out of this function goes
|
|
378
|
+
// through it, so no failure path can leak the lock by forgetting.
|
|
345
379
|
throw new Error("Manifest verification failed. The daemon cannot start with an unverified manifest when manifestProvider is configured. " +
|
|
346
380
|
"Check the logs for the specific failure reason (manifest_signature_invalid, manifest_expired, or manifest_version_rollback).");
|
|
347
381
|
}
|
|
@@ -1508,7 +1542,7 @@ export async function startDaemon(config) {
|
|
|
1508
1542
|
// the session's relay endpoint; we seal the content to the recipient (E2E — the relay never sees
|
|
1509
1543
|
// plaintext, INV-3) and deposit it to that relay's store-and-forward mailbox via the standing
|
|
1510
1544
|
// receiver node. The recipient pulls + recovers it at the witnessed sequence (R1) on next online.
|
|
1511
|
-
sessionNodeManager.setContentParkHook(async ({ sessionId, recipientPubkeyHex, relayPeerId, relayAddrs, contentHashHex, content, structure1Cbor, structure2Cbor }) => {
|
|
1545
|
+
sessionNodeManager.setContentParkHook(async ({ agentName, sessionId, recipientPubkeyHex, relayPeerId, relayAddrs, contentHashHex, content, structure1Cbor, structure2Cbor }) => {
|
|
1512
1546
|
const node = sessionNodeManager.getStandingReceiverNode();
|
|
1513
1547
|
if (!node) {
|
|
1514
1548
|
const reason = "standing_receiver_unavailable";
|
|
@@ -1519,10 +1553,32 @@ export async function startDaemon(config) {
|
|
|
1519
1553
|
// parked when nothing was ever deposited.
|
|
1520
1554
|
return { ok: false, reason };
|
|
1521
1555
|
}
|
|
1556
|
+
// SEC-1: sign the entry as the SENDING agent. Without a key we cannot produce an envelope the
|
|
1557
|
+
// recipient will accept, so fail LOUD rather than depositing something that will be refused on
|
|
1558
|
+
// recovery (a deposit the recipient must reject is worse than no deposit — it looks delivered).
|
|
1559
|
+
const senderKp = keyProviders.get(agentName);
|
|
1560
|
+
if (!senderKp) {
|
|
1561
|
+
const reason = "signing_key_unavailable";
|
|
1562
|
+
logger.warn("content.park.deposit.failed", { agentName, sessionId, contentHash: contentHashHex, reason });
|
|
1563
|
+
return { ok: false, reason };
|
|
1564
|
+
}
|
|
1522
1565
|
const recipientPubkey = Buffer.from(recipientPubkeyHex, "hex");
|
|
1566
|
+
const contentHashBytes = Buffer.from(contentHashHex, "hex");
|
|
1567
|
+
logger.info("content.park.signed", { agentName, sessionId, contentHash: contentHashHex });
|
|
1523
1568
|
// DOD-MSG-4 (2b): seal the ORDERING ENVELOPE (content + the relay's signed Structure2), not bare
|
|
1524
1569
|
// content, so the parked entry is self-ordering on recover. The relay still holds only ciphertext.
|
|
1525
|
-
|
|
1570
|
+
// SEC-1: sealParkEnvelope is the SOLE producer — it signs (sender's K_local, over the
|
|
1571
|
+
// session/recipient/content binding) and seals in one place, so the two park sites cannot drift
|
|
1572
|
+
// apart on what gets signed.
|
|
1573
|
+
const ciphertext = await sealParkEnvelope({
|
|
1574
|
+
signer: senderKp,
|
|
1575
|
+
sessionIdHex: sessionId,
|
|
1576
|
+
recipientPubkey,
|
|
1577
|
+
contentHash: contentHashBytes,
|
|
1578
|
+
content,
|
|
1579
|
+
structure1Cbor,
|
|
1580
|
+
structure2Cbor,
|
|
1581
|
+
});
|
|
1526
1582
|
const client = new ContentParkClient({ relayPeerId, relayAddrs: [...relayAddrs], logger });
|
|
1527
1583
|
const res = await client.deposit(node, {
|
|
1528
1584
|
recipientPubkey,
|
|
@@ -1573,11 +1629,29 @@ export async function startDaemon(config) {
|
|
|
1573
1629
|
const node = sessionNodeManager.getStandingReceiverNode(record.agent_name);
|
|
1574
1630
|
if (!node)
|
|
1575
1631
|
return { parked: false, error: "standing_receiver_unavailable" };
|
|
1632
|
+
// SEC-1: the crash backstop must sign too — an unsigned re-park would be REFUSED on recovery,
|
|
1633
|
+
// which would turn the message-loss backstop into a message-loss cause. This is why the SEC-1
|
|
1634
|
+
// signature binds to the sender's own K_local and NOT to the relay's ordering record: the
|
|
1635
|
+
// ordering record is not persisted in retry_queue (see below) and so cannot be reproduced here,
|
|
1636
|
+
// but the KEY can — the owning agent still holds it after a crash. Hence: NO schema migration.
|
|
1637
|
+
const senderKp = keyProviders.get(ownerName);
|
|
1638
|
+
if (!senderKp)
|
|
1639
|
+
return { parked: false, error: "signing_key_unavailable" };
|
|
1576
1640
|
const recipientPubkey = Buffer.from(record.counterparty_pubkey, "hex");
|
|
1641
|
+
const contentHashBytes = Buffer.from(entry.contentHashHex, "hex");
|
|
1642
|
+
logger.info("content.park.signed", { agentName: ownerName, sessionId: entry.sessionId, contentHash: entry.contentHashHex, source: "startup_flush" });
|
|
1577
1643
|
// DOD-MSG-4 (2b): seal the envelope shape too (content only — the durable awaiting queue does not
|
|
1578
1644
|
// persist the ordering record, so a crash-backstop re-park recovers in arrival order; the common
|
|
1579
1645
|
// live-park path above carries the full Structure2). Keeps ONE envelope format on the recover side.
|
|
1580
|
-
|
|
1646
|
+
// SEC-1: same sole producer as the live hook — the backstop signs from the persisted
|
|
1647
|
+
// (sessionId, recipient, contentHash), which is why it survives a crash with NO schema migration.
|
|
1648
|
+
const ciphertext = await sealParkEnvelope({
|
|
1649
|
+
signer: senderKp,
|
|
1650
|
+
sessionIdHex: entry.sessionId,
|
|
1651
|
+
recipientPubkey,
|
|
1652
|
+
contentHash: contentHashBytes,
|
|
1653
|
+
content: entry.contentBlob,
|
|
1654
|
+
});
|
|
1581
1655
|
const client = new ContentParkClient({ relayPeerId: ep.relayPeerId, relayAddrs: [...ep.relayAddrs], logger });
|
|
1582
1656
|
const res = await client.deposit(node, {
|
|
1583
1657
|
recipientPubkey,
|
|
@@ -3706,22 +3780,28 @@ export async function startDaemon(config) {
|
|
|
3706
3780
|
const client = new ContentParkClient({ relayPeerId, relayAddrs, logger });
|
|
3707
3781
|
const entries = await client.pull(node, Buffer.from(recipientPubkey, "hex"), kp);
|
|
3708
3782
|
let recovered = 0;
|
|
3783
|
+
// SEC-1 / review M2: a REFUSED entry must not vanish behind `ok:true`. Report it — an operator
|
|
3784
|
+
// seeing {ok:true, pulled:3, recovered:0} would otherwise have three messages evaporate with the
|
|
3785
|
+
// only explanation buried in the daemon log. This is both the injection signal and, given the
|
|
3786
|
+
// deliberate no-tolerant-window rollout, the lagging-peer signal.
|
|
3787
|
+
const refusals = [];
|
|
3788
|
+
// AC7: one correlationId per recover flow, threaded through every event it emits.
|
|
3789
|
+
const correlationId = randomUUID();
|
|
3709
3790
|
for (const e of entries) {
|
|
3710
3791
|
const unsealed = await kp.openContentSeal(e.ciphertext);
|
|
3711
3792
|
if (!unsealed) {
|
|
3712
3793
|
logger.warn("content.recover.unseal_failed", { sessionId: e.sessionIdHex, contentHash: e.contentHashHex });
|
|
3713
3794
|
continue;
|
|
3714
3795
|
}
|
|
3715
|
-
//
|
|
3716
|
-
//
|
|
3717
|
-
//
|
|
3718
|
-
//
|
|
3719
|
-
|
|
3796
|
+
// SEC-1: authenticate THEN ingest — fused in recoverParkedEntry, which is now the ONLY way
|
|
3797
|
+
// parked content can enter the transcript. It verifies the sender's signature over
|
|
3798
|
+
// (session_id, recipient_pubkey, content_hash) and that the signer is this session's
|
|
3799
|
+
// counterparty, and it still verifies + records the DOD-MSG-4 ordering record when present.
|
|
3800
|
+
// Relay deposit is unauthenticated and the seal is anonymous, so WITHOUT this gate anyone
|
|
3801
|
+
// holding the recipient's public key — the relay above all — could inject content that gets
|
|
3802
|
+
// attributed to the honest counterparty and then notarized by the bilateral seal.
|
|
3720
3803
|
const contentHashBytes = Buffer.from(e.contentHashHex, "hex");
|
|
3721
|
-
|
|
3722
|
-
sessionNodeManager.recordOrderingRecord(recipientAgent.name, e.sessionIdHex, env.structure1Cbor, env.structure2Cbor, contentHashBytes);
|
|
3723
|
-
}
|
|
3724
|
-
const ingest = await sessionNodeManager.ingestReceivedContent(recipientAgent.name, e.sessionIdHex, env.content, contentHashBytes);
|
|
3804
|
+
const ingest = await sessionNodeManager.recoverParkedEntry(recipientAgent.name, e.sessionIdHex, Buffer.from(recipientPubkey, "hex"), unsealed, contentHashBytes, correlationId);
|
|
3725
3805
|
if (ingest.ok && ingest.held) {
|
|
3726
3806
|
// DOD-MSG-4 (review finding #4): a held entry is NOT yet an appended leaf — its sequence is
|
|
3727
3807
|
// the FUTURE canonical index, not a completed recovery. Do not count it as recovered; log it
|
|
@@ -3758,10 +3838,14 @@ export async function startDaemon(config) {
|
|
|
3758
3838
|
}
|
|
3759
3839
|
}
|
|
3760
3840
|
else {
|
|
3761
|
-
|
|
3841
|
+
// SEC-1 (M2): carry the refusal out to the caller, not just to the log. Deliberately still
|
|
3842
|
+
// NOT confirm-deleted — a forgery must not be able to evict itself from the mailbox, and a
|
|
3843
|
+
// genuine v1-peer message must not be destroyed by our refusing it.
|
|
3844
|
+
refusals.push({ contentHash: e.contentHashHex, sessionId: e.sessionIdHex, reason: ingest.reason });
|
|
3845
|
+
logger.warn("content.recover.ingest_failed", { sessionId: e.sessionIdHex, contentHash: e.contentHashHex, reason: ingest.reason, correlationId });
|
|
3762
3846
|
}
|
|
3763
3847
|
}
|
|
3764
|
-
return { ok: true, recovered, pulled: entries.length };
|
|
3848
|
+
return { ok: true, recovered, pulled: entries.length, refused: refusals.length, refusals };
|
|
3765
3849
|
}
|
|
3766
3850
|
// DOD-MSG-4 (auto-recover-on-reconnect): when an agent comes online, drain its parked mailbox from
|
|
3767
3851
|
// every relay it has sessions on — symmetric to the SENDER's flushAwaitingContent. Without this,
|
|
@@ -5828,7 +5912,8 @@ export async function startDaemon(config) {
|
|
|
5828
5912
|
await ipcServer.start();
|
|
5829
5913
|
}
|
|
5830
5914
|
catch (err) {
|
|
5831
|
-
|
|
5915
|
+
// As above: startDaemon's catch releases the singleton lock for us.
|
|
5916
|
+
await removeLockIfOwned(lockFilePath, process.pid, logger);
|
|
5832
5917
|
throw err;
|
|
5833
5918
|
}
|
|
5834
5919
|
// MCP-002: Instantiate NotificationDispatcher (wired to IPC server)
|
|
@@ -5909,14 +5994,30 @@ export async function startDaemon(config) {
|
|
|
5909
5994
|
manifestPollScheduler.cancel();
|
|
5910
5995
|
}
|
|
5911
5996
|
logger.info("daemon.stopped", { pid: process.pid, reason });
|
|
5912
|
-
|
|
5913
|
-
|
|
5914
|
-
|
|
5915
|
-
|
|
5916
|
-
|
|
5917
|
-
|
|
5918
|
-
|
|
5919
|
-
|
|
5997
|
+
try {
|
|
5998
|
+
// CONN-001 (code-review LOW): stopAllSignaling() stops the shared manager AND every per-agent
|
|
5999
|
+
// manager (best-effort). The previously-separate per-agent stop loop here was redundant and
|
|
6000
|
+
// unguarded (would abort the rest of shutdown if stop() ever threw on a second call) — removed.
|
|
6001
|
+
await stopAllSignaling();
|
|
6002
|
+
// Gracefully mark active sessions interrupted (AC-009) before stopping IPC
|
|
6003
|
+
await sessionNodeManager.gracefulShutdown();
|
|
6004
|
+
await ipcServer.stop();
|
|
6005
|
+
}
|
|
6006
|
+
finally {
|
|
6007
|
+
// DOD-SINGLE-DAEMON-1: in a `finally`, because a throw anywhere above must not leave the lock
|
|
6008
|
+
// held. In the real binary the process exits and the kernel reclaims it — but an in-process
|
|
6009
|
+
// caller (vitest, an embedder) whose shutdown throws would otherwise find every subsequent
|
|
6010
|
+
// startDaemon in that process reporting "another daemon is already running", with the real
|
|
6011
|
+
// cause thrown away. That is the same leak F2 fixed on the startup path.
|
|
6012
|
+
//
|
|
6013
|
+
// DOD-DAEMON-CLEANUP-1 (AC1): the lock file goes only if it is still OURS. Another daemon may
|
|
6014
|
+
// have taken it over while we ran, and deleting a live daemon's lock is what makes `cello
|
|
6015
|
+
// logout` say "No daemon running" and the next `cello login` spawn a third one.
|
|
6016
|
+
await removeLockIfOwned(lockFilePath, process.pid, logger).catch(() => { });
|
|
6017
|
+
// Released LAST: while we hold it, no successor daemon can start, which is what lets
|
|
6018
|
+
// ipcServer.stop()'s socket re-check above be race-free.
|
|
6019
|
+
singletonLock.release();
|
|
6020
|
+
}
|
|
5920
6021
|
}
|
|
5921
6022
|
function getSessionNodeManager() {
|
|
5922
6023
|
return sessionNodeManager;
|