@north-light/crouter 0.3.253 → 0.3.255
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/attach/__tests__/group-activity.test.js +81 -0
- package/dist/clients/attach/render/group-activity.d.ts +9 -1
- package/dist/clients/attach/render/group-activity.js +75 -111
- package/dist/clients/attach/render/group-recap.js +24 -9
- package/dist/clients/attach/viewer.js +508 -509
- package/dist/commands/memory/delete.js +2 -0
- package/dist/core/__tests__/daemon-boot.test.js +5 -5
- package/dist/core/__tests__/preview-mirror-cap.test.js +16 -0
- package/dist/core/exclusive-lock.js +13 -2
- package/dist/core/io.d.ts +11 -2
- package/dist/core/io.js +10 -11
- package/dist/core/substrate/surface-match.js +7 -0
- package/dist/daemon/__tests__/autostart-storm.test.d.ts +1 -0
- package/dist/daemon/__tests__/autostart-storm.test.js +166 -0
- package/dist/daemon/__tests__/integration/migration-startup.test.js +4 -7
- package/dist/daemon/crtrd-cli.js +15 -1
- package/dist/daemon/crtrd.js +14 -1
- package/dist/daemon/manage.d.ts +24 -3
- package/dist/daemon/manage.js +98 -12
- package/dist/daemon/startup-policy.d.ts +5 -0
- package/dist/daemon/startup-policy.js +5 -0
- package/dist/migrations/activation.d.ts +9 -0
- package/dist/migrations/activation.js +14 -1
- package/dist/pi-extensions/canvas-bash-valve.js +4 -0
- package/dist/pi-extensions/canvas-preview-result.js +32 -7
- package/dist/shared/generated-context.d.ts +2 -2
- package/dist/shared/generated-context.js +4 -4
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
|
@@ -37,6 +37,7 @@ export const deleteLeaf = defineLeaf({
|
|
|
37
37
|
selectorParam('dir', {}, 'Deletes from that one store, including a non-winning duplicate no target view resolves to.'),
|
|
38
38
|
],
|
|
39
39
|
output: [
|
|
40
|
+
{ name: 'name', type: 'string', required: true, constraint: 'Full canonical name of the document that was deleted, as resolution settled it.' },
|
|
40
41
|
{ name: 'scope', type: 'string', required: true, constraint: 'Scope the document was deleted from: node, user, project, or profile.' },
|
|
41
42
|
{ name: 'path', type: 'string', required: true, constraint: 'Absolute path of the file that was removed.' },
|
|
42
43
|
{ name: 'log_path', type: 'string', required: true, constraint: 'Absolute path to the document’s revision log, which OUTLIVES the document — the delete is appended to it as a tombstone carrying the full final text, still readable with `crtr memory history`.' },
|
|
@@ -87,6 +88,7 @@ export const deleteLeaf = defineLeaf({
|
|
|
87
88
|
// this same log — one log per physical path over its whole life.
|
|
88
89
|
appendHistoryRecord(logPath, buildHistoryRecord({ op: 'delete', before, after: '' }));
|
|
89
90
|
return {
|
|
91
|
+
name: doc.name,
|
|
90
92
|
scope: doc.scope,
|
|
91
93
|
path: doc.path,
|
|
92
94
|
log_path: logPath,
|
|
@@ -193,17 +193,17 @@ test('verifyDaemonStartup waits for pidfile, live pid, and serving socket before
|
|
|
193
193
|
assert.equal(socketServing, true, 'the Unix API socket must serve before success');
|
|
194
194
|
assert.equal(slept >= 40, true, 'the guard must actually poll through socket readiness');
|
|
195
195
|
});
|
|
196
|
-
test('runDaemon
|
|
196
|
+
test('runDaemon claims ownership before activation and readiness', () => {
|
|
197
197
|
const source = readFileSync(fileURLToPath(new URL('../../daemon/crtrd.ts', import.meta.url)), 'utf8');
|
|
198
198
|
const runDaemonAt = source.indexOf('export async function runDaemon');
|
|
199
199
|
const activationAt = source.indexOf('await ensureOnDiskMigrations();', runDaemonAt);
|
|
200
200
|
const ownershipAt = source.indexOf('claimDaemonOwnership(', runDaemonAt);
|
|
201
201
|
const readinessAt = source.indexOf('writePidfile();', runDaemonAt);
|
|
202
202
|
assert.ok(runDaemonAt >= 0, 'runDaemon exists');
|
|
203
|
-
assert.ok(
|
|
204
|
-
assert.equal(source.slice(runDaemonAt,
|
|
205
|
-
assert.ok(
|
|
206
|
-
assert.ok(
|
|
203
|
+
assert.ok(ownershipAt > runDaemonAt, 'runDaemon claims ownership');
|
|
204
|
+
assert.equal(source.slice(runDaemonAt, ownershipAt).includes('await '), false, 'ownership is claimed before startup can await');
|
|
205
|
+
assert.ok(ownershipAt < activationAt, 'ownership precedes on-disk activation');
|
|
206
|
+
assert.ok(activationAt < readinessAt, 'activation completes before readiness is published');
|
|
207
207
|
});
|
|
208
208
|
// Regression for the cold-guest recreate blocker: the production startup window
|
|
209
209
|
// must tolerate a slower cold start than the old 500ms bound.
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
// this channel — bypassing pi's 50 KB stdout truncation entirely — and five
|
|
4
4
|
// such calls made a 16 MB transcript that Northlight could no longer open.
|
|
5
5
|
// The writer is the one choke point, so the writer is what this pins.
|
|
6
|
+
//
|
|
7
|
+
// The file accumulates one record per crtr invocation in a bash call, so the
|
|
8
|
+
// bound is on the file total: chaining commands must not be a way past it.
|
|
6
9
|
import { strict as assert } from 'node:assert';
|
|
7
10
|
import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
8
11
|
import { tmpdir } from 'node:os';
|
|
@@ -35,3 +38,16 @@ test('a bounded record still mirrors verbatim', () => {
|
|
|
35
38
|
assert.equal(record.path, 'integration run');
|
|
36
39
|
assert.equal(record.result.output, 'small');
|
|
37
40
|
});
|
|
41
|
+
test('chained invocations accumulate as JSONL, and stop at the file budget', () => {
|
|
42
|
+
const resultPath = join(dir, 'chained.json');
|
|
43
|
+
process.env[PREVIEW_RESULT_PATH_ENV] = resultPath;
|
|
44
|
+
const quarter = 'x'.repeat(Math.floor(PREVIEW_RESULT_MAX_BYTES / 4));
|
|
45
|
+
for (let i = 0; i < 6; i += 1) {
|
|
46
|
+
beginPreview('memory read');
|
|
47
|
+
publishPreviewResult({ i, output: quarter });
|
|
48
|
+
}
|
|
49
|
+
const lines = readFileSync(resultPath, 'utf8').split('\n').filter((line) => line !== '');
|
|
50
|
+
assert.equal(lines.length, 3, 'records land in order until the budget is spent, then are dropped whole');
|
|
51
|
+
assert.deepEqual(lines.map((line) => JSON.parse(line).result.i), [0, 1, 2]);
|
|
52
|
+
assert.ok(readFileSync(resultPath).byteLength <= PREVIEW_RESULT_MAX_BYTES, 'the accumulated file stays within the bound');
|
|
53
|
+
});
|
|
@@ -1,9 +1,16 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readdirSync, rmSync, rmdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { randomUUID } from 'node:crypto';
|
|
3
3
|
const MARKER_PREFIX = 'owner.';
|
|
4
|
+
// Contention backoff. The first polls stay tight so an uncontended hand-off is
|
|
5
|
+
// still effectively immediate; the interval then doubles to a ceiling so a lock
|
|
6
|
+
// held across a long operation (a corpus migration, say) cannot turn a crowd of
|
|
7
|
+
// waiters into a filesystem hot loop — N waiters at a fixed 10ms is N*100
|
|
8
|
+
// tryAcquire() syscall bursts per second on one directory.
|
|
4
9
|
const POLL_MS = 10;
|
|
10
|
+
const POLL_MAX_MS = 250;
|
|
5
11
|
const DEFAULT_STALE_MS = 30_000;
|
|
6
12
|
const DEFAULT_TIMEOUT_MS = 5_000;
|
|
13
|
+
function nextPollMs(current) { return Math.min(current * 2, POLL_MAX_MS); }
|
|
7
14
|
function pause(ms) { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); }
|
|
8
15
|
function pauseAsync(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); }
|
|
9
16
|
function markerPath(path, token) { return `${path}/${MARKER_PREFIX}${token}`; }
|
|
@@ -107,12 +114,14 @@ function tryAcquire(path, staleMs) {
|
|
|
107
114
|
export function withExclusiveDirectoryLock(path, operation, options = {}) {
|
|
108
115
|
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
|
109
116
|
const deadline = Date.now() + (options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
117
|
+
let pollMs = POLL_MS;
|
|
110
118
|
let lock = tryAcquire(path, staleMs);
|
|
111
119
|
while (lock === null) {
|
|
112
120
|
if (Date.now() >= deadline) {
|
|
113
121
|
throw options.timeoutError?.() ?? new Error(`timed out waiting for the exclusive lock: ${path}`);
|
|
114
122
|
}
|
|
115
|
-
pause(
|
|
123
|
+
pause(Math.min(pollMs, Math.max(0, deadline - Date.now())));
|
|
124
|
+
pollMs = nextPollMs(pollMs);
|
|
116
125
|
lock = tryAcquire(path, staleMs);
|
|
117
126
|
}
|
|
118
127
|
try {
|
|
@@ -127,12 +136,14 @@ export function withExclusiveDirectoryLock(path, operation, options = {}) {
|
|
|
127
136
|
export async function withExclusiveDirectoryLockAsync(path, operation, options = {}) {
|
|
128
137
|
const staleMs = options.staleMs ?? DEFAULT_STALE_MS;
|
|
129
138
|
const deadline = Date.now() + (options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
|
|
139
|
+
let pollMs = POLL_MS;
|
|
130
140
|
let lock = tryAcquire(path, staleMs);
|
|
131
141
|
while (lock === null) {
|
|
132
142
|
if (Date.now() >= deadline) {
|
|
133
143
|
throw options.timeoutError?.() ?? new Error(`timed out waiting for the exclusive lock: ${path}`);
|
|
134
144
|
}
|
|
135
|
-
await pauseAsync(Math.min(
|
|
145
|
+
await pauseAsync(Math.min(pollMs, Math.max(0, deadline - Date.now())));
|
|
146
|
+
pollMs = nextPollMs(pollMs);
|
|
136
147
|
lock = tryAcquire(path, staleMs);
|
|
137
148
|
}
|
|
138
149
|
try {
|
package/dist/core/io.d.ts
CHANGED
|
@@ -3,7 +3,12 @@ import { type ExitCodeValue } from '../types.js';
|
|
|
3
3
|
import { ApiError } from '../api/index.js';
|
|
4
4
|
/** The private result mirror for one crtr CLI invocation launched by the
|
|
5
5
|
* canvas bash-preview extension. This deliberately never enters stdout: bash
|
|
6
|
-
* text is model context, while Pi tool-result details are viewer-only.
|
|
6
|
+
* text is model context, while Pi tool-result details are viewer-only.
|
|
7
|
+
*
|
|
8
|
+
* One bash tool call may run several crtr invocations (`read a && read b`), so
|
|
9
|
+
* the mirror file is JSONL: one record per invocation, appended in order. Each
|
|
10
|
+
* invocation is its own process and `beginPreview` resets this module's state,
|
|
11
|
+
* so the appends compose across them with no shared state or locking. */
|
|
7
12
|
export interface CrtrPreviewRecord {
|
|
8
13
|
path: string;
|
|
9
14
|
result?: Record<string, unknown>;
|
|
@@ -17,7 +22,11 @@ export interface CrtrPreviewRecord {
|
|
|
17
22
|
* stdout truncation never sees this channel. The mirror write is the one
|
|
18
23
|
* choke point every record passes, so the bound lives here: an oversized
|
|
19
24
|
* record is dropped, and the viewer falls back to rendering the agent-facing
|
|
20
|
-
* stdout text, which pi bounds itself.
|
|
25
|
+
* stdout text, which pi bounds itself.
|
|
26
|
+
*
|
|
27
|
+
* Because records accumulate, this bounds the FILE TOTAL, not one line: a
|
|
28
|
+
* record that would overrun the remaining budget is dropped whole, so every
|
|
29
|
+
* line on disk stays valid JSON. */
|
|
21
30
|
export declare const PREVIEW_RESULT_MAX_BYTES: number;
|
|
22
31
|
/** Begin collecting the current leaf's structured response, if its caller
|
|
23
32
|
* supplied the private per-tool-call result path. */
|
package/dist/core/io.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// the model, not data it parses); structured errors; stderr is diagnostics only
|
|
4
4
|
// and never carries the result. The raw JSON object is available behind the
|
|
5
5
|
// `--json` global for tooling. See the cli-design reference.
|
|
6
|
-
import {
|
|
6
|
+
import { appendFileSync, statSync } from 'node:fs';
|
|
7
7
|
import { CrtrError } from './errors.js';
|
|
8
8
|
import { ExitCode } from '../types.js';
|
|
9
9
|
import { PREVIEW_RESULT_PATH_ENV } from './preview-result-path.js';
|
|
@@ -19,7 +19,11 @@ let preview;
|
|
|
19
19
|
* stdout truncation never sees this channel. The mirror write is the one
|
|
20
20
|
* choke point every record passes, so the bound lives here: an oversized
|
|
21
21
|
* record is dropped, and the viewer falls back to rendering the agent-facing
|
|
22
|
-
* stdout text, which pi bounds itself.
|
|
22
|
+
* stdout text, which pi bounds itself.
|
|
23
|
+
*
|
|
24
|
+
* Because records accumulate, this bounds the FILE TOTAL, not one line: a
|
|
25
|
+
* record that would overrun the remaining budget is dropped whole, so every
|
|
26
|
+
* line on disk stays valid JSON. */
|
|
23
27
|
export const PREVIEW_RESULT_MAX_BYTES = 256 * 1024;
|
|
24
28
|
/** Begin collecting the current leaf's structured response, if its caller
|
|
25
29
|
* supplied the private per-tool-call result path. */
|
|
@@ -32,20 +36,15 @@ function publishPreview(record) {
|
|
|
32
36
|
if (resultPath === undefined || resultPath === '')
|
|
33
37
|
return;
|
|
34
38
|
const line = `${JSON.stringify({ path: preview?.path ?? '', ...record })}\n`;
|
|
35
|
-
if (Buffer.byteLength(line, 'utf8') > PREVIEW_RESULT_MAX_BYTES)
|
|
36
|
-
return;
|
|
37
|
-
const tempPath = `${resultPath}.${process.pid}.tmp`;
|
|
38
39
|
try {
|
|
39
|
-
|
|
40
|
-
|
|
40
|
+
const spent = statSync(resultPath, { throwIfNoEntry: false })?.size ?? 0;
|
|
41
|
+
if (spent + Buffer.byteLength(line, 'utf8') > PREVIEW_RESULT_MAX_BYTES)
|
|
42
|
+
return;
|
|
43
|
+
appendFileSync(resultPath, line, 'utf8');
|
|
41
44
|
}
|
|
42
45
|
catch {
|
|
43
46
|
// Preview transport is optional UI metadata. Its failure must never alter
|
|
44
47
|
// the CLI's stdout/error contract.
|
|
45
|
-
try {
|
|
46
|
-
writeFileSync(tempPath, '', 'utf8');
|
|
47
|
-
}
|
|
48
|
-
catch { /* nothing to clean */ }
|
|
49
48
|
}
|
|
50
49
|
}
|
|
51
50
|
/** Publish the single-object or collected JSONL result after leaf dispatch. */
|
|
@@ -102,6 +102,10 @@ export function matchesMemoryReadEntry(entry, routingAnchor, subject, name) {
|
|
|
102
102
|
return false;
|
|
103
103
|
return entry.match.some((g) => memoryReadGlobMatches(g, routingAnchor, name));
|
|
104
104
|
}
|
|
105
|
+
/** Split a command into shell segments on unquoted `;`, newline, `&&`, `||`,
|
|
106
|
+
* and `|`. A single `&`, subshell parentheses, and backticks are left inside
|
|
107
|
+
* their segment — this is a best-effort guardrail, not a shell parser.
|
|
108
|
+
* Heredoc bodies are a separate pre-match concern (see `stripHeredocs`). */
|
|
105
109
|
function splitCommandSegments(command) {
|
|
106
110
|
const segments = [];
|
|
107
111
|
let current = '';
|
|
@@ -152,6 +156,7 @@ function splitCommandSegments(command) {
|
|
|
152
156
|
finish();
|
|
153
157
|
return segments;
|
|
154
158
|
}
|
|
159
|
+
/** Tokenize one segment into shell words, marking quoted spans opaque. */
|
|
155
160
|
function tokenizeCommandSegment(segment) {
|
|
156
161
|
const tokens = [];
|
|
157
162
|
let text = '';
|
|
@@ -207,6 +212,8 @@ function tokenizeCommandSegment(segment) {
|
|
|
207
212
|
finish();
|
|
208
213
|
return tokens;
|
|
209
214
|
}
|
|
215
|
+
/** Every non-empty command segment as its env-stripped token list: leading
|
|
216
|
+
* `VAR=value` assignments are dropped so `tokens[0]` is the executable. */
|
|
210
217
|
function commandSegments(command) {
|
|
211
218
|
return splitCommandSegments(command)
|
|
212
219
|
.map(tokenizeCommandSegment)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { test, beforeEach, after } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { spawn } from 'node:child_process';
|
|
7
|
+
import { DaemonStartupBlockedError, daemonAutostartSuppression, ensureDaemon, resetDaemonAutostart, verifyDaemonStartup, } from '../manage.js';
|
|
8
|
+
import { DAEMON_EXIT_STARTUP_BLOCKED } from '../startup-policy.js';
|
|
9
|
+
import { withExclusiveDirectoryLockAsync } from '../../core/exclusive-lock.js';
|
|
10
|
+
// REGRESSION (daemon respawn storm): a daemon whose startup hit a STANDING
|
|
11
|
+
// condition — a blocked on-disk migration — exited, its attach clients saw a
|
|
12
|
+
// cold socket, autostarted a replacement, and that replacement hit the same
|
|
13
|
+
// wall. Nothing tracked the repeat and nothing reaped the losers, so failed
|
|
14
|
+
// starts accumulated into a pile that pegged the machine. Three properties keep
|
|
15
|
+
// that from recurring: the terminal exit is TYPED, a typed failure SUPPRESSES
|
|
16
|
+
// further autostarts, and an ordinary failure at least BACKS OFF.
|
|
17
|
+
const settle = async () => { await new Promise((resolve) => setImmediate(resolve)); };
|
|
18
|
+
// The test lane exports CRTR_NO_DAEMON_AUTOSTART=1 so no suite ever spawns a
|
|
19
|
+
// real daemon. These tests drive ensureDaemon through injected deps — nothing
|
|
20
|
+
// is spawned — so the guard has to come off for the gate under test to be
|
|
21
|
+
// reachable at all.
|
|
22
|
+
const autostartEnv = process.env['CRTR_NO_DAEMON_AUTOSTART'];
|
|
23
|
+
beforeEach(() => {
|
|
24
|
+
delete process.env['CRTR_NO_DAEMON_AUTOSTART'];
|
|
25
|
+
resetDaemonAutostart();
|
|
26
|
+
});
|
|
27
|
+
after(() => {
|
|
28
|
+
if (autostartEnv === undefined)
|
|
29
|
+
delete process.env['CRTR_NO_DAEMON_AUTOSTART'];
|
|
30
|
+
else
|
|
31
|
+
process.env['CRTR_NO_DAEMON_AUTOSTART'] = autostartEnv;
|
|
32
|
+
resetDaemonAutostart();
|
|
33
|
+
});
|
|
34
|
+
test('the blocked-startup exit code surfaces as a typed terminal error, not a generic exit', async () => {
|
|
35
|
+
let exited = null;
|
|
36
|
+
const child = spawn(process.execPath, ['-e', `process.exit(${DAEMON_EXIT_STARTUP_BLOCKED})`], { stdio: 'ignore' });
|
|
37
|
+
child.once('exit', (code, signal) => {
|
|
38
|
+
exited = { code, signal };
|
|
39
|
+
});
|
|
40
|
+
await assert.rejects(async () => verifyDaemonStartup(77, 2_000, {
|
|
41
|
+
readPidfile: () => null,
|
|
42
|
+
isPidAlive: () => false,
|
|
43
|
+
isDaemonServing: async () => false,
|
|
44
|
+
childExited: () => exited,
|
|
45
|
+
}), (error) => {
|
|
46
|
+
assert.ok(error instanceof DaemonStartupBlockedError, 'must be typed, so the spawner can stop retrying');
|
|
47
|
+
assert.match(error.message, /crtr sys migrate/);
|
|
48
|
+
return true;
|
|
49
|
+
});
|
|
50
|
+
});
|
|
51
|
+
test('a blocked start suppresses every later autostart instead of respawning into the same wall', async () => {
|
|
52
|
+
let spawns = 0;
|
|
53
|
+
const deps = {
|
|
54
|
+
isDaemonRunning: () => false,
|
|
55
|
+
spawnDaemon: async () => {
|
|
56
|
+
spawns += 1;
|
|
57
|
+
throw new DaemonStartupBlockedError(4242);
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
ensureDaemon(deps);
|
|
61
|
+
await settle();
|
|
62
|
+
assert.equal(spawns, 1);
|
|
63
|
+
assert.match(daemonAutostartSuppression() ?? '', /crtr sys migrate/);
|
|
64
|
+
// The storm: every redial of every attach client lands here. None may spawn.
|
|
65
|
+
for (let i = 0; i < 50; i++)
|
|
66
|
+
ensureDaemon(deps);
|
|
67
|
+
await settle();
|
|
68
|
+
assert.equal(spawns, 1, 'a standing failure must never be retried');
|
|
69
|
+
});
|
|
70
|
+
test('an ordinary spawn failure backs off, then retries once the window passes', async () => {
|
|
71
|
+
let spawns = 0;
|
|
72
|
+
let now = 1_000_000;
|
|
73
|
+
const deps = {
|
|
74
|
+
isDaemonRunning: () => false,
|
|
75
|
+
now: () => now,
|
|
76
|
+
spawnDaemon: async () => {
|
|
77
|
+
spawns += 1;
|
|
78
|
+
throw new Error('dist/daemon/crtrd-cli.js missing');
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
ensureDaemon(deps);
|
|
82
|
+
await settle();
|
|
83
|
+
assert.equal(spawns, 1);
|
|
84
|
+
for (let i = 0; i < 20; i++)
|
|
85
|
+
ensureDaemon(deps);
|
|
86
|
+
await settle();
|
|
87
|
+
assert.equal(spawns, 1, 'retries inside the backoff window must be dropped');
|
|
88
|
+
now += 5_000;
|
|
89
|
+
ensureDaemon(deps);
|
|
90
|
+
await settle();
|
|
91
|
+
assert.equal(spawns, 2, 'the window expiring re-opens exactly one attempt');
|
|
92
|
+
// Backoff grows, so the second failure buys a longer quiet period than the first.
|
|
93
|
+
now += 5_000;
|
|
94
|
+
ensureDaemon(deps);
|
|
95
|
+
await settle();
|
|
96
|
+
assert.equal(spawns, 2, 'the second failure must back off further than the first');
|
|
97
|
+
});
|
|
98
|
+
test('a spawn already in flight blocks a concurrent autostart', async () => {
|
|
99
|
+
let spawns = 0;
|
|
100
|
+
let release = () => { };
|
|
101
|
+
const gate = new Promise((resolve) => { release = resolve; });
|
|
102
|
+
const deps = {
|
|
103
|
+
isDaemonRunning: () => false,
|
|
104
|
+
spawnDaemon: async () => {
|
|
105
|
+
spawns += 1;
|
|
106
|
+
await gate;
|
|
107
|
+
return { started: true, pid: 1 };
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
ensureDaemon(deps);
|
|
111
|
+
for (let i = 0; i < 10; i++)
|
|
112
|
+
ensureDaemon(deps);
|
|
113
|
+
await settle();
|
|
114
|
+
assert.equal(spawns, 1, 'concurrent callers must not each spawn a daemon');
|
|
115
|
+
release();
|
|
116
|
+
await settle();
|
|
117
|
+
});
|
|
118
|
+
test('a success clears the backoff so the next cold socket starts immediately', async () => {
|
|
119
|
+
let spawns = 0;
|
|
120
|
+
let fail = true;
|
|
121
|
+
let now = 1_000_000;
|
|
122
|
+
const deps = {
|
|
123
|
+
isDaemonRunning: () => false,
|
|
124
|
+
now: () => now,
|
|
125
|
+
spawnDaemon: async () => {
|
|
126
|
+
spawns += 1;
|
|
127
|
+
if (fail)
|
|
128
|
+
throw new Error('transient');
|
|
129
|
+
return { started: true, pid: 1 };
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
ensureDaemon(deps);
|
|
133
|
+
await settle();
|
|
134
|
+
fail = false;
|
|
135
|
+
now += 5_000;
|
|
136
|
+
ensureDaemon(deps);
|
|
137
|
+
await settle();
|
|
138
|
+
assert.equal(spawns, 2);
|
|
139
|
+
ensureDaemon(deps);
|
|
140
|
+
await settle();
|
|
141
|
+
assert.equal(spawns, 3, 'a healthy start must not leave a backoff behind');
|
|
142
|
+
});
|
|
143
|
+
// The pile pegged the machine because each waiter re-probed the lock every 10ms
|
|
144
|
+
// with no backoff. Backing off must not make an uncontended hand-off sluggish.
|
|
145
|
+
test('a contended lock waiter still acquires promptly once the holder releases', async () => {
|
|
146
|
+
const home = mkdtempSync(join(tmpdir(), 'crtr-lock-backoff-'));
|
|
147
|
+
const lockPath = join(home, 'contended.lock');
|
|
148
|
+
try {
|
|
149
|
+
let released = 0;
|
|
150
|
+
const holder = withExclusiveDirectoryLockAsync(lockPath, async () => {
|
|
151
|
+
await new Promise((resolve) => setTimeout(resolve, 300));
|
|
152
|
+
released = Date.now();
|
|
153
|
+
}, { timeoutMs: 5_000 });
|
|
154
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
155
|
+
let acquired = 0;
|
|
156
|
+
const waiter = withExclusiveDirectoryLockAsync(lockPath, async () => {
|
|
157
|
+
acquired = Date.now();
|
|
158
|
+
}, { timeoutMs: 5_000 });
|
|
159
|
+
await Promise.all([holder, waiter]);
|
|
160
|
+
assert.ok(acquired >= released, 'the waiter must not enter before the holder leaves');
|
|
161
|
+
assert.ok(acquired - released < 1_000, `hand-off took ${acquired - released}ms — backoff ceiling is too coarse`);
|
|
162
|
+
}
|
|
163
|
+
finally {
|
|
164
|
+
rmSync(home, { recursive: true, force: true });
|
|
165
|
+
}
|
|
166
|
+
});
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import assert from 'node:assert/strict';
|
|
2
|
-
import { existsSync, mkdirSync, readFileSync,
|
|
2
|
+
import { existsSync, mkdirSync, readFileSync, statSync, watch, writeFileSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import test from 'node:test';
|
|
5
5
|
import { onDiskMigrationIdentity } from '../../../migrations/registry.js';
|
|
@@ -52,7 +52,7 @@ test('source daemon migrates the pre-migration corpus and writes its marker befo
|
|
|
52
52
|
assert.deepEqual(JSON.parse(readFileSync(markerPath, 'utf8')), onDiskMigrationIdentity());
|
|
53
53
|
assert.equal(existsSync(fixture.pidfile), true, 'health readiness follows the pidfile readiness mirror');
|
|
54
54
|
assert.equal(existsSync(fixture.socket), true, 'the unique unix API listener is bound at readiness');
|
|
55
|
-
assert.equal(existsSync(dbPath), true, 'daemon ownership
|
|
55
|
+
assert.equal(existsSync(dbPath), true, 'daemon ownership has opened canvas.db by readiness');
|
|
56
56
|
assert.ok(statSync(markerPath).mtimeMs <= statSync(fixture.pidfile).mtimeMs, 'the completion marker is written no later than the readiness pidfile');
|
|
57
57
|
assert.ok(statSync(markerPath).mtimeMs <= statSync(fixture.socket).mtimeMs, 'the completion marker is written no later than the API listener');
|
|
58
58
|
}
|
|
@@ -62,7 +62,7 @@ test('source daemon migrates the pre-migration corpus and writes its marker befo
|
|
|
62
62
|
assert.deepEqual(exit, { code: 0, signal: null }, `daemon must stop cleanly; stderr:\n${daemon.stderr()}`);
|
|
63
63
|
}
|
|
64
64
|
});
|
|
65
|
-
test('a real migration blocker exits before
|
|
65
|
+
test('a real migration blocker exits before pidfile, listeners, or broker effects', { timeout: 30_000 }, async () => {
|
|
66
66
|
const fixture = await createSourceDaemonFixture();
|
|
67
67
|
const blockerPath = writeUserMemory(fixture.home, 'blocked.md', `---\nkind: knowledge\nbroken: [\n---\n# Blocked\n`);
|
|
68
68
|
const effectNames = new Set();
|
|
@@ -82,14 +82,11 @@ test('a real migration blocker exits before database, pidfile, listeners, or bro
|
|
|
82
82
|
assert.match(daemon.stderr(), new RegExp(blockerPath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')));
|
|
83
83
|
assert.match(daemon.stderr(), /Repair: crtr sys migrate/);
|
|
84
84
|
assert.equal(wasReady, false, 'the unique TCP endpoint never answered health');
|
|
85
|
-
const databaseFiles = readdirSync(fixture.canvasHome).filter((name) => name.startsWith('canvas.db'));
|
|
86
|
-
assert.deepEqual(databaseFiles, [], 'canvas.db and its journals were never created');
|
|
87
85
|
assert.equal(existsSync(fixture.pidfile), false, 'readiness pidfile was never created');
|
|
88
86
|
assert.equal(existsSync(fixture.socket), false, 'unix API listener was never created');
|
|
89
87
|
assert.equal(existsSync(join(fixture.canvasHome, 'on-disk-migrations.json')), false, 'blocker never gets a completion marker');
|
|
90
|
-
assert.equal(existsSync(join(fixture.canvasHome, 'nodes')), false, 'no node/broker state was created');
|
|
91
88
|
assert.deepEqual(brokerPidsForCanvas(fixture.canvasHome), [], 'no broker process declares the isolated canvas');
|
|
92
|
-
assert.deepEqual([...effectNames].filter((name) => name
|
|
89
|
+
assert.deepEqual([...effectNames].filter((name) => name === 'ready.pid' || name === 'crtrd.sock'), [], `filesystem watcher saw no transient readiness or broker effects (all events: ${[...effectNames].join(', ')})`);
|
|
93
90
|
}
|
|
94
91
|
finally {
|
|
95
92
|
fixture.dispose();
|
package/dist/daemon/crtrd-cli.js
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// raw fatal boundary; a winning daemon stays alive through its scheduled loop.
|
|
4
4
|
import '../suppress-experimental-warnings.js';
|
|
5
5
|
import { runDaemon } from './crtrd.js';
|
|
6
|
+
import { MigrationBlockedError } from '../migrations/activation.js';
|
|
7
|
+
import { DAEMON_EXIT_STARTUP_BLOCKED } from './startup-policy.js';
|
|
6
8
|
import { envCrtrdTcp } from '../shared/env.js';
|
|
7
9
|
/** Extract `--tcp <host:port>` (or `--tcp=<host:port>`) from argv; falls back to
|
|
8
10
|
* `CRTRD_TCP`. Absent → unix socket only (§4.1). */
|
|
@@ -16,4 +18,16 @@ function parseTcpArg(argv) {
|
|
|
16
18
|
}
|
|
17
19
|
return envCrtrdTcp();
|
|
18
20
|
}
|
|
19
|
-
|
|
21
|
+
// A blocked migration is the one startup failure that is guaranteed to recur.
|
|
22
|
+
// Exit on its own code — with the repair line intact on stderr — so the
|
|
23
|
+
// spawner suppresses further autostarts rather than respawning into the same
|
|
24
|
+
// wall. Every other failure keeps reaching Node's raw fatal boundary.
|
|
25
|
+
try {
|
|
26
|
+
await runDaemon({ tcp: parseTcpArg(process.argv.slice(2)) });
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (!(error instanceof MigrationBlockedError))
|
|
30
|
+
throw error;
|
|
31
|
+
process.stderr.write(`[crtrd] startup blocked\n${error.message}\n`);
|
|
32
|
+
process.exit(DAEMON_EXIT_STARTUP_BLOCKED);
|
|
33
|
+
}
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -316,7 +316,6 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
|
|
|
316
316
|
* quiesces admitted work, removes its readiness mirror, and releases its exact
|
|
317
317
|
* claim. */
|
|
318
318
|
export async function runDaemon(opts = {}) {
|
|
319
|
-
await ensureOnDiskMigrations();
|
|
320
319
|
const ownership = claimDaemonOwnership(randomBytes(16).toString('hex'));
|
|
321
320
|
if (!ownership.claimed) {
|
|
322
321
|
// A losing claim isn't always a singleton race: `own_identity_indeterminate`
|
|
@@ -338,6 +337,20 @@ export async function runDaemon(opts = {}) {
|
|
|
338
337
|
return async () => { };
|
|
339
338
|
}
|
|
340
339
|
const claim = ownership.claim;
|
|
340
|
+
try {
|
|
341
|
+
await ensureOnDiskMigrations();
|
|
342
|
+
}
|
|
343
|
+
catch (migrationError) {
|
|
344
|
+
try {
|
|
345
|
+
const { released } = releaseDaemonOwnership(claim);
|
|
346
|
+
if (!released)
|
|
347
|
+
throw new Error('daemon ownership claim was not released');
|
|
348
|
+
}
|
|
349
|
+
catch (cleanupError) {
|
|
350
|
+
throw new AggregateError([migrationError, cleanupError], 'daemon startup failed and cleanup also failed');
|
|
351
|
+
}
|
|
352
|
+
throw migrationError;
|
|
353
|
+
}
|
|
341
354
|
let admissionOpen = true;
|
|
342
355
|
let interval;
|
|
343
356
|
let laneTail = Promise.resolve();
|
package/dist/daemon/manage.d.ts
CHANGED
|
@@ -70,6 +70,13 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
|
|
|
70
70
|
} | null;
|
|
71
71
|
isDaemonServing?: () => Promise<boolean>;
|
|
72
72
|
}
|
|
73
|
+
/** Thrown when the spawned daemon exited on `DAEMON_EXIT_STARTUP_BLOCKED` — a
|
|
74
|
+
* standing condition (a blocked on-disk migration) that the very next spawn
|
|
75
|
+
* would hit identically. Callers must stop retrying and surface the repair;
|
|
76
|
+
* the daemon has already written the detail to `crtrd.err`. */
|
|
77
|
+
export declare class DaemonStartupBlockedError extends Error {
|
|
78
|
+
constructor(pid: number);
|
|
79
|
+
}
|
|
73
80
|
/** Tiny bounded post-spawn guard: wait for the daemon pidfile and live pid to
|
|
74
81
|
* appear before reporting success. This catches the "started:true but not yet
|
|
75
82
|
* plausibly alive" race without turning startup into a retry loop.
|
|
@@ -124,8 +131,22 @@ export declare function spawnDaemon(): Promise<SpawnDaemonResult>;
|
|
|
124
131
|
* explicit `crtr sys daemon start` calls `spawnDaemon` directly and is
|
|
125
132
|
* deliberately unaffected. */
|
|
126
133
|
export declare function autostartDisabled(): boolean;
|
|
127
|
-
/**
|
|
128
|
-
*
|
|
134
|
+
/** Why implicit autostart is currently suppressed, or null when it is free to
|
|
135
|
+
* run. Set only by a `DaemonStartupBlockedError`, which no retry can clear —
|
|
136
|
+
* an explicit `crtr sys daemon start` still calls `spawnDaemon` directly and
|
|
137
|
+
* reports the failure in full. */
|
|
138
|
+
export declare function daemonAutostartSuppression(): string | null;
|
|
139
|
+
/** Clear the suppression and the backoff — for a caller that has repaired the
|
|
140
|
+
* standing condition and wants autostart live again without a new process. */
|
|
141
|
+
export declare function resetDaemonAutostart(): void;
|
|
142
|
+
export interface EnsureDaemonDeps {
|
|
143
|
+
isDaemonRunning?: () => boolean;
|
|
144
|
+
spawnDaemon?: () => Promise<SpawnDaemonResult>;
|
|
145
|
+
now?: () => number;
|
|
146
|
+
}
|
|
147
|
+
/** Start the daemon if it is not already running. No-op if already up, if
|
|
148
|
+
* autostart is suppressed for this invocation, while an attempt is in flight,
|
|
149
|
+
* or while backing off from a recent failure.
|
|
129
150
|
* Silently swallows spawn errors (the canvas still works without the daemon;
|
|
130
151
|
* nodes just won't be auto-revived). */
|
|
131
|
-
export declare function ensureDaemon(): void;
|
|
152
|
+
export declare function ensureDaemon(deps?: EnsureDaemonDeps): void;
|