@north-light/crouter 0.3.300 → 0.3.301
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/attach/session/profile-files.d.ts +6 -3
- package/dist/clients/attach/session/profile-files.js +39 -10
- package/dist/clients/attach/viewer.js +663 -663
- package/dist/commands/node/create.js +1 -1
- package/dist/commands/node/lifecycle.js +29 -2
- package/dist/core/__tests__/human-action-delivery.test.js +13 -11
- package/dist/core/__tests__/node-outcome.test.js +1 -6
- package/dist/core/bash-jobs.d.ts +22 -0
- package/dist/core/bash-jobs.js +38 -1
- package/dist/core/canvas/delivery-contract.d.ts +19 -0
- package/dist/core/canvas/delivery-contract.js +8 -0
- package/dist/core/canvas/human-deliveries.d.ts +3 -17
- package/dist/core/canvas/node-outcome-deliveries.d.ts +3 -19
- package/dist/core/canvas/node-outcome-deliveries.js +0 -4
- package/dist/core/keybindings/catalog.js +1 -1
- package/dist/daemon/__tests__/node-outcome-delivery.test.js +3 -2
- package/dist/daemon/delivery/run-delivery.d.ts +25 -0
- package/dist/daemon/delivery/run-delivery.js +164 -0
- package/dist/daemon/human/deliver-action.d.ts +1 -14
- package/dist/daemon/human/deliver-action.js +5 -165
- package/dist/daemon/node-outcome/deliver-outcome.d.ts +1 -12
- package/dist/daemon/node-outcome/deliver-outcome.js +6 -152
- package/dist/pi-extensions/canvas-bash-valve.js +7 -10
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
|
@@ -83,7 +83,7 @@ function nodeNewParams() {
|
|
|
83
83
|
{ kind: 'flag', name: 'parent', type: 'string', required: false, constraint: 'Parent node id. Defaults to the calling node.' },
|
|
84
84
|
{ kind: 'flag', name: 'node-id', type: 'string', required: false, constraint: 'Exact id for the new node; lowercase letters, digits, and hyphens only, at most 128 characters. Rejected if already present.' },
|
|
85
85
|
{ kind: 'flag', name: 'root', type: 'bool', required: false, constraint: 'Spawn an independent resident root with no parent or subscription. Mutually exclusive with --worktree.' },
|
|
86
|
-
{ kind: 'flag', name: 'resident', type: 'bool', required: false, constraint: 'Birth the node as resident: it parks and stays wakeable without being forced to submit a final. Applies to managed children and roots; roots default to resident.' },
|
|
86
|
+
{ kind: 'flag', name: 'resident', type: 'bool', required: false, constraint: 'Birth the node as resident: it parks and stays wakeable without being forced to submit a final. Pass it ONLY when the user will open this node’s viewer pane and work with it back and forth; a child that only other nodes message stays terminal. Applies to managed children and roots; roots default to resident.' },
|
|
87
87
|
{ kind: 'flag', name: 'worktree', type: 'string', required: false, constraint: 'Local base branch for a crouter-managed worktree for this child. Mutually exclusive with --root.' },
|
|
88
88
|
{ kind: 'flag', name: 'fork-from', type: 'string', required: false, constraint: 'Fork the new node from an existing node id, absolute session path, or partial pi session uuid.' },
|
|
89
89
|
{
|
|
@@ -3,6 +3,8 @@ import { envNodeId } from '../../shared/env.js';
|
|
|
3
3
|
import { defineBranch, defineLeaf } from '../../core/command.js';
|
|
4
4
|
import { InputError } from '../../core/io.js';
|
|
5
5
|
import { cliClient, getNodeOrNull, rethrowAsCliError } from '../api-client.js';
|
|
6
|
+
import { contextDir } from '../../core/canvas/paths.js';
|
|
7
|
+
import { DEFAULT_BASH_NICENESS, MAX_BASH_NICENESS, writeBashNiceness } from '../../core/bash-jobs.js';
|
|
6
8
|
import { nodeReviveLeaf } from '../node-lifecycle-revive.js';
|
|
7
9
|
import { assertKind, kindsStateBlock } from './create.js';
|
|
8
10
|
import { MODEL_SPEC_FORMS } from '../../core/runtime/model-selection.js';
|
|
@@ -140,6 +142,7 @@ export const nodeConfig = defineLeaf({
|
|
|
140
142
|
{ kind: 'flag', name: 'kind', type: 'string', required: false, constraint: 'Persona kind. The <kinds> list below names every top-level installable kind and when to use each; a registered sub-kind is valid too by exact path but not listed here.' },
|
|
141
143
|
{ kind: 'flag', name: 'mode', type: 'enum', choices: ['base', 'orchestrator'], required: false, constraint: 'Set persona mode headlessly. base is hands-on; orchestrator holds a roadmap and delegates. orchestrator seeds a roadmap scaffold if absent.' },
|
|
142
144
|
{ kind: 'flag', name: 'name', type: 'string', required: false, constraint: 'Rename the node; if the node has a live window, also rename that viewer window.' },
|
|
145
|
+
{ kind: 'flag', name: 'nice', type: 'int', required: false, constraint: `Scheduler niceness every command this node's bash tool starts at, 0–${MAX_BASH_NICENESS} (default ${DEFAULT_BASH_NICENESS}). ${DEFAULT_BASH_NICENESS} keeps the node's builds and test suites below the daemon, the brokers, the viewers, and the person's own shell; 0 lets them compete on equal terms with everything else on the host; ${MAX_BASH_NICENESS} makes them yield to it. Applies to the node's very next command, foreground and background alike.` },
|
|
143
146
|
{ kind: 'flag', name: 'cwd', type: 'string', required: false, constraint: 'Repair-only replacement launch directory. Must be a non-empty absolute existing directory and used alone; the daemon accepts it only for a dormant, non-forked node with no open managed worktree whose recorded cwd is gone. Clears the saved Pi session, so the next revive starts fresh while durable node work remains.' },
|
|
144
147
|
],
|
|
145
148
|
output: [
|
|
@@ -149,6 +152,7 @@ export const nodeConfig = defineLeaf({
|
|
|
149
152
|
{ name: 'kind', type: 'string', required: false, constraint: 'The new kind, present only when --kind changed it.' },
|
|
150
153
|
{ name: 'mode', type: 'string', required: false, constraint: 'The new mode, present only when --mode changed it.' },
|
|
151
154
|
{ name: 'name', type: 'string', required: false, constraint: 'The new name, present only when --name changed it.' },
|
|
155
|
+
{ name: 'nice', type: 'number', required: false, constraint: 'The new bash niceness, present only when --nice changed it.' },
|
|
152
156
|
{ name: 'cwd', type: 'string', required: false, constraint: 'The lexically resolved replacement directory, present only after a cwd repair.' },
|
|
153
157
|
],
|
|
154
158
|
outputKind: 'object',
|
|
@@ -157,6 +161,7 @@ export const nodeConfig = defineLeaf({
|
|
|
157
161
|
'Kind, mode, and lifecycle changes rebuild the launch spec from the fresh node meta; setting mode to orchestrator seeds a roadmap scaffold if absent.',
|
|
158
162
|
'Name changes update the row and, when the node already has a live window, rename that viewer window to the new full name.',
|
|
159
163
|
'A cwd repair writes only the lexically resolved cwd and clears saved Pi session pointers; the old transcript and durable node work remain untouched.',
|
|
164
|
+
'A niceness change is stored with the node\'s own bash-job state, so it takes effect on the next command without a revive and survives one.',
|
|
160
165
|
],
|
|
161
166
|
dynamicState: () => kindsStateBlock(),
|
|
162
167
|
},
|
|
@@ -169,7 +174,7 @@ export const nodeConfig = defineLeaf({
|
|
|
169
174
|
// clean listing error, and re-validated server-side against the TARGET
|
|
170
175
|
// node's scope as a backstop.
|
|
171
176
|
const cwdSpec = input['cwd'];
|
|
172
|
-
if (cwdSpec !== undefined && ['model', 'kind', 'lifecycle', 'mode', 'name'].some((flag) => input[flag] !== undefined)) {
|
|
177
|
+
if (cwdSpec !== undefined && ['model', 'kind', 'lifecycle', 'mode', 'name', 'nice'].some((flag) => input[flag] !== undefined)) {
|
|
173
178
|
throw new InputError({ error: 'cwd_not_exclusive', message: 'cwd repair cannot be combined with another config flag', field: 'cwd', next: 'Pass --cwd alone.' });
|
|
174
179
|
}
|
|
175
180
|
const modelSpec = input['model']?.trim();
|
|
@@ -180,6 +185,15 @@ export const nodeConfig = defineLeaf({
|
|
|
180
185
|
const lifecycleSpec = input['lifecycle']?.trim();
|
|
181
186
|
const modeSpec = input['mode']?.trim();
|
|
182
187
|
const nameSpec = input['name'];
|
|
188
|
+
const niceSpec = input['nice'];
|
|
189
|
+
if (niceSpec !== undefined && (niceSpec < 0 || niceSpec > MAX_BASH_NICENESS)) {
|
|
190
|
+
throw new InputError({
|
|
191
|
+
error: 'bad_nice',
|
|
192
|
+
message: `niceness must be between 0 and ${MAX_BASH_NICENESS}: ${niceSpec}`,
|
|
193
|
+
field: 'nice',
|
|
194
|
+
next: `A negative niceness needs root, so pass 0–${MAX_BASH_NICENESS}.`,
|
|
195
|
+
});
|
|
196
|
+
}
|
|
183
197
|
if (kindSpec !== undefined)
|
|
184
198
|
assertKind(kindSpec);
|
|
185
199
|
if (lifecycleSpec !== undefined && lifecycleSpec.toLowerCase() !== 'terminal' && lifecycleSpec.toLowerCase() !== 'resident') {
|
|
@@ -214,8 +228,17 @@ export const nodeConfig = defineLeaf({
|
|
|
214
228
|
patch.cwd = cwdSpec;
|
|
215
229
|
applied.push('cwd');
|
|
216
230
|
}
|
|
231
|
+
// Niceness is node-local bash-lane state, not canvas state: it is stored
|
|
232
|
+
// beside this node's job dirs so the valve reads it when it starts the next
|
|
233
|
+
// command, rather than waiting for a launch recipe to be rebuilt.
|
|
234
|
+
if (niceSpec !== undefined) {
|
|
235
|
+
writeBashNiceness(contextDir(nodeId), niceSpec);
|
|
236
|
+
applied.push('nice');
|
|
237
|
+
if (applied.length === 1)
|
|
238
|
+
return { node_id: nodeId, nice: niceSpec };
|
|
239
|
+
}
|
|
217
240
|
if (applied.length === 0) {
|
|
218
|
-
throw new InputError({ error: 'no_change', message: 'no config changes requested', next: 'Pass at least one of --model, --lifecycle, --kind, --mode, --name, --cwd.' });
|
|
241
|
+
throw new InputError({ error: 'no_change', message: 'no config changes requested', next: 'Pass at least one of --model, --lifecycle, --kind, --mode, --name, --nice, --cwd.' });
|
|
219
242
|
}
|
|
220
243
|
const detail = await cliClient()
|
|
221
244
|
.patchConfig(nodeId, patch)
|
|
@@ -233,6 +256,8 @@ export const nodeConfig = defineLeaf({
|
|
|
233
256
|
result.mode = detail.mode;
|
|
234
257
|
if (applied.includes('name'))
|
|
235
258
|
result.name = detail.name;
|
|
259
|
+
if (applied.includes('nice'))
|
|
260
|
+
result.nice = niceSpec;
|
|
236
261
|
return result;
|
|
237
262
|
},
|
|
238
263
|
render: (r) => {
|
|
@@ -249,6 +274,8 @@ export const nodeConfig = defineLeaf({
|
|
|
249
274
|
bits.push(`mode=${r['mode']}`);
|
|
250
275
|
if (r['name'] !== undefined)
|
|
251
276
|
bits.push(`name=${r['name']}`);
|
|
277
|
+
if (r['nice'] !== undefined)
|
|
278
|
+
bits.push(`nice=${r['nice']}`);
|
|
252
279
|
return `Configured ${r['node_id']} — ${bits.join(', ')}`;
|
|
253
280
|
},
|
|
254
281
|
});
|
|
@@ -5,7 +5,9 @@ import { tmpdir } from 'node:os';
|
|
|
5
5
|
import { join } from 'node:path';
|
|
6
6
|
import { closeDb } from '../canvas/db.js';
|
|
7
7
|
import { claimActionDelivery, enqueueActionDelivery, readActionDelivery, } from '../canvas/human-deliveries.js';
|
|
8
|
-
import {
|
|
8
|
+
import { deliveryBackoffMs } from '../canvas/delivery-contract.js';
|
|
9
|
+
import { deliverySettlementForOutcome } from '../../daemon/delivery/run-delivery.js';
|
|
10
|
+
import { deliverAction } from '../../daemon/human/deliver-action.js';
|
|
9
11
|
let home;
|
|
10
12
|
const priorHome = process.env['CRTR_HOME'];
|
|
11
13
|
beforeEach(() => {
|
|
@@ -25,14 +27,14 @@ after(() => {
|
|
|
25
27
|
else
|
|
26
28
|
process.env['CRTR_HOME'] = priorHome;
|
|
27
29
|
});
|
|
28
|
-
test('
|
|
29
|
-
assert.equal(
|
|
30
|
-
assert.equal(
|
|
31
|
-
assert.equal(
|
|
32
|
-
assert.equal(
|
|
33
|
-
assert.equal(
|
|
30
|
+
test('delivery retries follow the durable backoff and exit boundary', () => {
|
|
31
|
+
assert.equal(deliveryBackoffMs(1), 5_000);
|
|
32
|
+
assert.equal(deliveryBackoffMs(2), 10_000);
|
|
33
|
+
assert.equal(deliveryBackoffMs(3), 20_000);
|
|
34
|
+
assert.equal(deliveryBackoffMs(11), 3_600_000);
|
|
35
|
+
assert.equal(deliveryBackoffMs(100), 3_600_000);
|
|
34
36
|
const now = 1_000_000;
|
|
35
|
-
const outcome = (exitCode, signal = null) =>
|
|
37
|
+
const outcome = (exitCode, signal = null) => deliverySettlementForOutcome({
|
|
36
38
|
exitCode,
|
|
37
39
|
signal,
|
|
38
40
|
timedOut: false,
|
|
@@ -50,14 +52,14 @@ test('action delivery retries follow the durable backoff and exit boundary', ()
|
|
|
50
52
|
});
|
|
51
53
|
assert.equal(outcome(79).state, 'pending');
|
|
52
54
|
assert.equal(outcome(null, 'SIGTERM').state, 'pending');
|
|
53
|
-
assert.equal(
|
|
55
|
+
assert.equal(deliverySettlementForOutcome({
|
|
54
56
|
exitCode: null,
|
|
55
57
|
signal: null,
|
|
56
58
|
timedOut: true,
|
|
57
59
|
stderr: '',
|
|
58
60
|
}, 1, now).state, 'pending');
|
|
59
61
|
// The kill escalation's signal is the only way to tell the graceful term from the SIGKILL.
|
|
60
|
-
assert.deepEqual(
|
|
62
|
+
assert.deepEqual(deliverySettlementForOutcome({
|
|
61
63
|
exitCode: null,
|
|
62
64
|
signal: 'SIGKILL',
|
|
63
65
|
timedOut: true,
|
|
@@ -67,7 +69,7 @@ test('action delivery retries follow the durable backoff and exit boundary', ()
|
|
|
67
69
|
nextAttemptAt: now + 5_000,
|
|
68
70
|
failure: { kind: 'timeout', signal: 'SIGKILL', stderr: 'stalled' },
|
|
69
71
|
});
|
|
70
|
-
assert.equal(
|
|
72
|
+
assert.equal(deliverySettlementForOutcome({
|
|
71
73
|
exitCode: null,
|
|
72
74
|
signal: null,
|
|
73
75
|
timedOut: false,
|
|
@@ -6,7 +6,7 @@ import { tmpdir } from 'node:os';
|
|
|
6
6
|
import { join } from 'node:path';
|
|
7
7
|
import { closeDb, MIGRATIONS } from '../canvas/db.js';
|
|
8
8
|
import { createNode, getRow } from '../canvas/canvas.js';
|
|
9
|
-
import { armOutcomeDelivery, claimOutcomeDelivery,
|
|
9
|
+
import { armOutcomeDelivery, claimOutcomeDelivery, readOutcomeDelivery, recoverRunningOutcomeDeliveries, settleOutcomeDelivery, } from '../canvas/node-outcome-deliveries.js';
|
|
10
10
|
import { transition } from '../runtime/lifecycle.js';
|
|
11
11
|
import { composeNodeOutcomeDocument } from '../runtime/outcome-document.js';
|
|
12
12
|
import { pushFinal } from '../feed/feed.js';
|
|
@@ -153,11 +153,6 @@ test('outcome delivery state is fenced, recoverable, and follows the durable bac
|
|
|
153
153
|
nextAttemptAt: readOutcomeDelivery('recover').nextAttemptAt,
|
|
154
154
|
claimOwner: readOutcomeDelivery('recover').claimOwner,
|
|
155
155
|
}, { state: 'pending', nextAttemptAt: recoveredAt, claimOwner: null });
|
|
156
|
-
assert.equal(outcomeDeliveryBackoffMs(1), 5_000);
|
|
157
|
-
assert.equal(outcomeDeliveryBackoffMs(2), 10_000);
|
|
158
|
-
assert.equal(outcomeDeliveryBackoffMs(3), 20_000);
|
|
159
|
-
assert.equal(outcomeDeliveryBackoffMs(11), 3_600_000);
|
|
160
|
-
assert.equal(outcomeDeliveryBackoffMs(100), 3_600_000);
|
|
161
156
|
});
|
|
162
157
|
test('v39 migrates a v38 database and backfills finalized and terminal rows', () => {
|
|
163
158
|
const dbPath = join(home, 'v38.db');
|
package/dist/core/bash-jobs.d.ts
CHANGED
|
@@ -36,6 +36,28 @@ export declare function readJobPgid(contextDir: string, jobId: string): number |
|
|
|
36
36
|
/** The optional human-readable job label. Jobs created before labels existed
|
|
37
37
|
* have no file and deliberately project null. */
|
|
38
38
|
export declare function readJobPurpose(contextDir: string, jobId: string): string | null;
|
|
39
|
+
/** Scheduler niceness every command a node runs is started with. Agent work —
|
|
40
|
+
* builds, test suites, typechecks — sits below the daemon, the brokers, the
|
|
41
|
+
* viewers, and the person's own shell, so a fan-out of release builds cannot
|
|
42
|
+
* starve them: the scheduler only honors this under contention and it costs
|
|
43
|
+
* nothing on an idle host. */
|
|
44
|
+
export declare const DEFAULT_BASH_NICENESS = 10;
|
|
45
|
+
/** The politest priority the scheduler accepts. A NEGATIVE niceness (more
|
|
46
|
+
* priority than the broker itself) needs root: `nice` prints a permission error
|
|
47
|
+
* into the command's own log and runs it at the inherited priority anyway, so
|
|
48
|
+
* the override range stops at 0. */
|
|
49
|
+
export declare const MAX_BASH_NICENESS = 19;
|
|
50
|
+
/** Per-node override of the bash niceness, written by `crtr node config --nice`
|
|
51
|
+
* and read by the valve when it starts each command. Node-local like the job
|
|
52
|
+
* dirs beside it, so a change applies to the node's very next command without
|
|
53
|
+
* waiting for a revive. */
|
|
54
|
+
export declare function bashNicenessPath(contextDir: string): string;
|
|
55
|
+
/** The niceness this node's commands run at. Any missing, malformed, or
|
|
56
|
+
* out-of-range file reads as the default rather than an odd priority. */
|
|
57
|
+
export declare function readBashNiceness(contextDir: string): number;
|
|
58
|
+
/** Set the node's bash niceness. The caller owns validation; the store keeps
|
|
59
|
+
* one integer so the valve's read stays a single file read per command. */
|
|
60
|
+
export declare function writeBashNiceness(contextDir: string, niceness: number): void;
|
|
39
61
|
/** The auto-background cancel deadline (epoch ms), or undefined for a job with
|
|
40
62
|
* no deadline (a deliberate handoff, or a legacy job dir). A non-finite or
|
|
41
63
|
* malformed file reads as no deadline rather than an instant expiry. */
|
package/dist/core/bash-jobs.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// job.bg, while foreground completion renames it to job.done. The detached
|
|
4
4
|
// supervisor can therefore outlive pi and still decide whether to report exit.
|
|
5
5
|
import { randomBytes } from 'node:crypto';
|
|
6
|
-
import { closeSync, existsSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
6
|
+
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
8
|
const MAX_BASH_JOB_PURPOSE_BYTES = 280;
|
|
9
9
|
/** A purpose is display text, never shell input. Keep the file-backed control
|
|
@@ -66,6 +66,43 @@ export function readJobPurpose(contextDir, jobId) {
|
|
|
66
66
|
return null;
|
|
67
67
|
}
|
|
68
68
|
}
|
|
69
|
+
/** Scheduler niceness every command a node runs is started with. Agent work —
|
|
70
|
+
* builds, test suites, typechecks — sits below the daemon, the brokers, the
|
|
71
|
+
* viewers, and the person's own shell, so a fan-out of release builds cannot
|
|
72
|
+
* starve them: the scheduler only honors this under contention and it costs
|
|
73
|
+
* nothing on an idle host. */
|
|
74
|
+
export const DEFAULT_BASH_NICENESS = 10;
|
|
75
|
+
/** The politest priority the scheduler accepts. A NEGATIVE niceness (more
|
|
76
|
+
* priority than the broker itself) needs root: `nice` prints a permission error
|
|
77
|
+
* into the command's own log and runs it at the inherited priority anyway, so
|
|
78
|
+
* the override range stops at 0. */
|
|
79
|
+
export const MAX_BASH_NICENESS = 19;
|
|
80
|
+
/** Per-node override of the bash niceness, written by `crtr node config --nice`
|
|
81
|
+
* and read by the valve when it starts each command. Node-local like the job
|
|
82
|
+
* dirs beside it, so a change applies to the node's very next command without
|
|
83
|
+
* waiting for a revive. */
|
|
84
|
+
export function bashNicenessPath(contextDir) {
|
|
85
|
+
return join(bashJobsDir(contextDir), 'niceness');
|
|
86
|
+
}
|
|
87
|
+
/** The niceness this node's commands run at. Any missing, malformed, or
|
|
88
|
+
* out-of-range file reads as the default rather than an odd priority. */
|
|
89
|
+
export function readBashNiceness(contextDir) {
|
|
90
|
+
try {
|
|
91
|
+
const n = Number(readFileSync(bashNicenessPath(contextDir), 'utf8').trim());
|
|
92
|
+
if (!Number.isInteger(n) || n < 0 || n > MAX_BASH_NICENESS)
|
|
93
|
+
return DEFAULT_BASH_NICENESS;
|
|
94
|
+
return n;
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
return DEFAULT_BASH_NICENESS;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** Set the node's bash niceness. The caller owns validation; the store keeps
|
|
101
|
+
* one integer so the valve's read stays a single file read per command. */
|
|
102
|
+
export function writeBashNiceness(contextDir, niceness) {
|
|
103
|
+
mkdirSync(bashJobsDir(contextDir), { recursive: true });
|
|
104
|
+
writeFileSync(bashNicenessPath(contextDir), `${Math.trunc(niceness)}\n`);
|
|
105
|
+
}
|
|
69
106
|
/** The auto-background cancel deadline (epoch ms), or undefined for a job with
|
|
70
107
|
* no deadline (a deliberate handoff, or a legacy job dir). A non-finite or
|
|
71
108
|
* malformed file reads as no deadline rather than an instant expiry. */
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export interface DeliveryFailure {
|
|
2
|
+
kind: 'exit' | 'signal' | 'timeout' | 'spawn_error';
|
|
3
|
+
exitCode?: number;
|
|
4
|
+
signal?: string;
|
|
5
|
+
message?: string;
|
|
6
|
+
stderr?: string;
|
|
7
|
+
}
|
|
8
|
+
export type DeliverySettlement = {
|
|
9
|
+
state: 'accepted';
|
|
10
|
+
} | {
|
|
11
|
+
state: 'permanent_failed';
|
|
12
|
+
failure: DeliveryFailure;
|
|
13
|
+
} | {
|
|
14
|
+
state: 'pending';
|
|
15
|
+
nextAttemptAt: number;
|
|
16
|
+
failure: DeliveryFailure;
|
|
17
|
+
};
|
|
18
|
+
/** Durable exponential retry delay for the attempt that just failed. */
|
|
19
|
+
export declare function deliveryBackoffMs(attempt: number): number;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// The durable contract shared by every retried delivery: what a failed attempt
|
|
2
|
+
// records, how an attempt settles, and when the next attempt becomes due. The
|
|
3
|
+
// canvas modules own the rows; `daemon/delivery/run-delivery.ts` owns the
|
|
4
|
+
// process that produces these settlements.
|
|
5
|
+
/** Durable exponential retry delay for the attempt that just failed. */
|
|
6
|
+
export function deliveryBackoffMs(attempt) {
|
|
7
|
+
return Math.min(5_000 * 2 ** (attempt - 1), 3_600_000);
|
|
8
|
+
}
|
|
@@ -1,11 +1,6 @@
|
|
|
1
|
+
import type { DeliveryFailure, DeliverySettlement } from './delivery-contract.js';
|
|
1
2
|
export type ActionDeliveryState = 'pending' | 'running' | 'accepted' | 'permanent_failed';
|
|
2
|
-
export
|
|
3
|
-
kind: 'exit' | 'signal' | 'timeout' | 'spawn_error';
|
|
4
|
-
exitCode?: number;
|
|
5
|
-
signal?: string;
|
|
6
|
-
message?: string;
|
|
7
|
-
stderr?: string;
|
|
8
|
-
}
|
|
3
|
+
export type ActionDeliveryFailure = DeliveryFailure;
|
|
9
4
|
export interface ActionDeliveryRecord {
|
|
10
5
|
requestId: string;
|
|
11
6
|
state: ActionDeliveryState;
|
|
@@ -37,16 +32,7 @@ export declare function readActionDelivery(requestId: string): ActionDeliveryRec
|
|
|
37
32
|
export declare function dueActionDeliveries(now: number): ActionDeliveryRecord[];
|
|
38
33
|
/** Atomically start one due delivery attempt. A null result lost the claim. */
|
|
39
34
|
export declare function claimActionDelivery(requestId: string, now: number, claimOwner: string): ActionDeliveryRecord | null;
|
|
40
|
-
export type ActionDeliverySettlement =
|
|
41
|
-
state: 'accepted';
|
|
42
|
-
} | {
|
|
43
|
-
state: 'permanent_failed';
|
|
44
|
-
failure: ActionDeliveryFailure;
|
|
45
|
-
} | {
|
|
46
|
-
state: 'pending';
|
|
47
|
-
nextAttemptAt: number;
|
|
48
|
-
failure: ActionDeliveryFailure;
|
|
49
|
-
};
|
|
35
|
+
export type ActionDeliverySettlement = DeliverySettlement;
|
|
50
36
|
/** Settle only the still-owned running attempt, so a stale process cannot overwrite a newer claim. */
|
|
51
37
|
export declare function settleActionDelivery(requestId: string, attempt: number, claimOwner: string, settlement: ActionDeliverySettlement, now: number): boolean;
|
|
52
38
|
/** A new daemon deliberately redelivers interrupted completions at least once. */
|
|
@@ -1,14 +1,7 @@
|
|
|
1
|
+
import type { DeliveryFailure, DeliverySettlement } from './delivery-contract.js';
|
|
1
2
|
import { type NodeOutcomeKind, type TerminalReason } from './types.js';
|
|
2
3
|
export type OutcomeDeliveryState = 'armed' | 'pending' | 'running' | 'accepted' | 'permanent_failed';
|
|
3
|
-
export
|
|
4
|
-
kind: 'exit' | 'signal' | 'timeout' | 'spawn_error';
|
|
5
|
-
exitCode?: number;
|
|
6
|
-
signal?: string;
|
|
7
|
-
message?: string;
|
|
8
|
-
stderr?: string;
|
|
9
|
-
}
|
|
10
|
-
/** Durable exponential retry delay for the attempt that just failed. */
|
|
11
|
-
export declare function outcomeDeliveryBackoffMs(attempt: number): number;
|
|
4
|
+
export type OutcomeDeliveryFailure = DeliveryFailure;
|
|
12
5
|
export interface OutcomeDeliveryRecord {
|
|
13
6
|
nodeId: string;
|
|
14
7
|
state: OutcomeDeliveryState;
|
|
@@ -61,16 +54,7 @@ export declare function armOutcomeDelivery(row: {
|
|
|
61
54
|
export declare function readOutcomeDelivery(nodeId: string): OutcomeDeliveryRecord | null;
|
|
62
55
|
export declare function dueOutcomeDeliveries(now: number): OutcomeDeliveryRecord[];
|
|
63
56
|
export declare function claimOutcomeDelivery(nodeId: string, now: number, claimOwner: string): OutcomeDeliveryRecord | null;
|
|
64
|
-
export type OutcomeDeliverySettlement =
|
|
65
|
-
state: 'accepted';
|
|
66
|
-
} | {
|
|
67
|
-
state: 'permanent_failed';
|
|
68
|
-
failure: OutcomeDeliveryFailure;
|
|
69
|
-
} | {
|
|
70
|
-
state: 'pending';
|
|
71
|
-
nextAttemptAt: number;
|
|
72
|
-
failure: OutcomeDeliveryFailure;
|
|
73
|
-
};
|
|
57
|
+
export type OutcomeDeliverySettlement = DeliverySettlement;
|
|
74
58
|
/** Settle only the still-owned running attempt, fencing stale delivery processes. */
|
|
75
59
|
export declare function settleOutcomeDelivery(nodeId: string, attempt: number, claimOwner: string, settlement: OutcomeDeliverySettlement, now: number): boolean;
|
|
76
60
|
export declare function recoverRunningOutcomeDeliveries(now: number): number;
|
|
@@ -3,10 +3,6 @@ import { openDb, withCanvasWrite } from './db.js';
|
|
|
3
3
|
import { getNode } from './canvas.js';
|
|
4
4
|
import { OUTCOME_PAYLOAD_MAX_BYTES } from './types.js';
|
|
5
5
|
import { composeNodeOutcomeDocument } from '../runtime/outcome-document.js';
|
|
6
|
-
/** Durable exponential retry delay for the attempt that just failed. */
|
|
7
|
-
export function outcomeDeliveryBackoffMs(attempt) {
|
|
8
|
-
return Math.min(5_000 * 2 ** (attempt - 1), 3_600_000);
|
|
9
|
-
}
|
|
10
6
|
function serializedPayload(payload) {
|
|
11
7
|
if (payload === undefined)
|
|
12
8
|
return null;
|
|
@@ -84,7 +84,7 @@ export const BINDING_CATALOG = Object.freeze([
|
|
|
84
84
|
...attachEntry('crtr.tmux.menu.attach.model-ladder.previous', 'crtr.attach.model-ladder.previous', 'Previous model', 'Move to the previous model ladder rung.', ['['], ['alt+shift+m']),
|
|
85
85
|
...attachEntry('crtr.tmux.menu.attach.command.inspect', 'crtr.attach.command.inspect', 'Inspect loaded command', 'Open the loaded slash command’s expanded prompt.', ['h'], ['alt+shift+h']),
|
|
86
86
|
...attachEntry('crtr.tmux.menu.attach.file-review', 'crtr.attach.file-review', 'Review a file', 'Open a line-anchored review of a file named in the transcript.', [], ['alt+shift+r']),
|
|
87
|
-
...attachEntry('crtr.tmux.menu.attach.profile-files', 'crtr.attach.profile-files', 'Search profile files', 'Fuzzy-match path segments across the node family’s working directories, profile projects and memory, and ancestor crouter memory stores; Ctrl+Y copies a selected path.', ['f'], ['ctrl+f']),
|
|
87
|
+
...attachEntry('crtr.tmux.menu.attach.profile-files', 'crtr.attach.profile-files', 'Search profile files', 'Fuzzy-match path segments across the node family’s working directories, their context and reports directories, profile projects and memory, and ancestor crouter memory stores; Ctrl+Y copies a selected path.', ['f'], ['ctrl+f']),
|
|
88
88
|
...attachEntry('crtr.tmux.menu.attach.search', 'crtr.attach.search', 'Search transcript', 'Search the transcript as it is currently displayed — folded tool output is not searched until it is unfolded.', ['/'], ['alt+/']),
|
|
89
89
|
...attachEntry('crtr.tmux.menu.attach.whip', 'crtr.attach.whip', 'Whip agent', 'Interrupt the agent and send a playful command.', ['w'], []),
|
|
90
90
|
b('crtr.attach.scroll.up', 'Attach', 'Scroll up', 'Scroll the transcript one line up.', ['shift+up'], 'terminal', ATTACH_CONTEXTS),
|
|
@@ -8,7 +8,8 @@ import { createNode } from '../../core/canvas/canvas.js';
|
|
|
8
8
|
import { isPidAlive, killProcessGroup } from '../../core/canvas/pid.js';
|
|
9
9
|
import { armOutcomeDelivery, claimOutcomeDelivery, readOutcomeDelivery } from '../../core/canvas/node-outcome-deliveries.js';
|
|
10
10
|
import { transition } from '../../core/runtime/lifecycle.js';
|
|
11
|
-
import {
|
|
11
|
+
import { DELIVERY_TIMEOUT_MS } from '../delivery/run-delivery.js';
|
|
12
|
+
import { deliverOutcome } from '../node-outcome/deliver-outcome.js';
|
|
12
13
|
let home;
|
|
13
14
|
let processGroupFiles;
|
|
14
15
|
const priorHome = process.env.CRTR_HOME;
|
|
@@ -70,7 +71,7 @@ test('timeout SIGKILL escalation reaps a SIGTERM-ignoring descendant after its l
|
|
|
70
71
|
processGroupFiles.push(processGroupPid);
|
|
71
72
|
const delivery = claimScript('timeout', `#!/bin/sh\necho $$ > ${processGroupPid}\n(\n trap '' TERM\n while :; do sleep 30; done\n) &\necho $! > ${childPid}\ntrap 'exit 0' TERM\nwhile :; do sleep 30; done\n`);
|
|
72
73
|
const originalSetTimeout = global.setTimeout;
|
|
73
|
-
global.setTimeout = ((callback, delay, ...args) => originalSetTimeout(callback, delay ===
|
|
74
|
+
global.setTimeout = ((callback, delay, ...args) => originalSetTimeout(callback, delay === DELIVERY_TIMEOUT_MS ? 1_000 : delay, ...args));
|
|
74
75
|
try {
|
|
75
76
|
await deliverOutcome(delivery, 'test-daemon');
|
|
76
77
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { type DeliverySettlement } from '../../core/canvas/delivery-contract.js';
|
|
2
|
+
export declare const DELIVERY_TIMEOUT_MS = 120000;
|
|
3
|
+
export declare const DELIVERY_STDERR_TAIL_BYTES: number;
|
|
4
|
+
export interface DeliveryProcessOutcome {
|
|
5
|
+
exitCode: number | null;
|
|
6
|
+
signal: string | null;
|
|
7
|
+
timedOut: boolean;
|
|
8
|
+
spawnError?: Error;
|
|
9
|
+
stderr: string;
|
|
10
|
+
}
|
|
11
|
+
/** Process contract mapping. stdout deliberately does not participate. */
|
|
12
|
+
export declare function deliverySettlementForOutcome(outcome: DeliveryProcessOutcome, attempt: number, now: number): DeliverySettlement;
|
|
13
|
+
/** One already-claimed attempt: where to deliver, what to deliver, and which
|
|
14
|
+
* attempt this is. The caller resolved argv and cwd when the target was
|
|
15
|
+
* registered, so a later config edit cannot redirect this delivery. */
|
|
16
|
+
export interface DeliveryJob {
|
|
17
|
+
argv: string[];
|
|
18
|
+
cwd: string;
|
|
19
|
+
/** The stored document; its `delivery.attempt` is stamped per attempt. */
|
|
20
|
+
document: unknown;
|
|
21
|
+
attempt: number;
|
|
22
|
+
}
|
|
23
|
+
/** Spawn and settle one already-claimed delivery. `settle` writes the outcome
|
|
24
|
+
* to whichever durable row owns this attempt. */
|
|
25
|
+
export declare function runDelivery(job: DeliveryJob, settle: (settlement: DeliverySettlement) => void): Promise<void>;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// One process contract for every durable delivery crouter retries: spawn the
|
|
2
|
+
// declared target, hand it the document on stdin, and settle the attempt from
|
|
3
|
+
// its exit code. Human-request completions and node outcomes differ only in the
|
|
4
|
+
// row they settle, so both call this and neither reimplements the protocol.
|
|
5
|
+
import { spawn } from 'node:child_process';
|
|
6
|
+
import { envHomeOverride } from '../../shared/env.js';
|
|
7
|
+
import { deliveryBackoffMs } from '../../core/canvas/delivery-contract.js';
|
|
8
|
+
import { killProcessGroup } from '../../core/canvas/pid.js';
|
|
9
|
+
import { buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
|
|
10
|
+
export const DELIVERY_TIMEOUT_MS = 120_000;
|
|
11
|
+
const DELIVERY_KILL_GRACE_MS = 2_000;
|
|
12
|
+
export const DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
|
|
13
|
+
/** Process contract mapping. stdout deliberately does not participate. */
|
|
14
|
+
export function deliverySettlementForOutcome(outcome, attempt, now) {
|
|
15
|
+
if (outcome.spawnError !== undefined) {
|
|
16
|
+
return {
|
|
17
|
+
state: 'pending',
|
|
18
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
19
|
+
failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
if (outcome.timedOut) {
|
|
23
|
+
return {
|
|
24
|
+
state: 'pending',
|
|
25
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
26
|
+
// The kill escalation's own signal distinguishes the graceful term from the SIGKILL.
|
|
27
|
+
failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
if (outcome.signal !== null) {
|
|
31
|
+
return {
|
|
32
|
+
state: 'pending',
|
|
33
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
34
|
+
failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
if (outcome.exitCode === 0)
|
|
38
|
+
return { state: 'accepted' };
|
|
39
|
+
if (outcome.exitCode === 78) {
|
|
40
|
+
return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
state: 'pending',
|
|
44
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
45
|
+
failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function failure(kind, fields) {
|
|
49
|
+
return { kind, ...fields };
|
|
50
|
+
}
|
|
51
|
+
function stderrTailCapture() {
|
|
52
|
+
let tail = Buffer.alloc(0);
|
|
53
|
+
return {
|
|
54
|
+
take(chunk) {
|
|
55
|
+
tail = chunk.length >= DELIVERY_STDERR_TAIL_BYTES
|
|
56
|
+
? chunk.subarray(chunk.length - DELIVERY_STDERR_TAIL_BYTES)
|
|
57
|
+
: Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - DELIVERY_STDERR_TAIL_BYTES));
|
|
58
|
+
},
|
|
59
|
+
decode() {
|
|
60
|
+
return tail.toString('utf8');
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/** The stored document is immutable; attempt is its only per-delivery mutation. */
|
|
65
|
+
function documentForAttempt(document, attempt) {
|
|
66
|
+
const stored = document;
|
|
67
|
+
const delivery = stored.delivery;
|
|
68
|
+
return JSON.stringify({ ...stored, delivery: { ...delivery, attempt } });
|
|
69
|
+
}
|
|
70
|
+
/** The delivery child crosses the same consent boundary as every other
|
|
71
|
+
* crouter spawn: nothing of crtrd's own environment reaches it except what
|
|
72
|
+
* `spawnEnv.allow` and the operational base admit for the target's own cwd.
|
|
73
|
+
* No profile participates — a delivery target is declared in scope config alone. */
|
|
74
|
+
function deliveryEnv(cwd) {
|
|
75
|
+
const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
|
|
76
|
+
// Runtime identity, not host state: a target that calls `crtr` must reach
|
|
77
|
+
// the daemon that delivered to it, exactly as a cron body does.
|
|
78
|
+
const canvasHome = envHomeOverride();
|
|
79
|
+
if (canvasHome !== undefined && canvasHome !== '')
|
|
80
|
+
env['CRTR_HOME'] = canvasHome;
|
|
81
|
+
return env;
|
|
82
|
+
}
|
|
83
|
+
/** Spawn and settle one already-claimed delivery. `settle` writes the outcome
|
|
84
|
+
* to whichever durable row owns this attempt. */
|
|
85
|
+
export async function runDelivery(job, settle) {
|
|
86
|
+
const stderr = stderrTailCapture();
|
|
87
|
+
let document;
|
|
88
|
+
try {
|
|
89
|
+
document = documentForAttempt(job.document, job.attempt);
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
settle(deliverySettlementForOutcome({
|
|
93
|
+
exitCode: null,
|
|
94
|
+
signal: null,
|
|
95
|
+
timedOut: false,
|
|
96
|
+
spawnError: error,
|
|
97
|
+
stderr: stderr.decode(),
|
|
98
|
+
}, job.attempt, Date.now()));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
let child;
|
|
102
|
+
try {
|
|
103
|
+
child = spawn(job.argv[0], job.argv.slice(1), {
|
|
104
|
+
cwd: job.cwd,
|
|
105
|
+
env: deliveryEnv(job.cwd),
|
|
106
|
+
detached: true,
|
|
107
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
settle(deliverySettlementForOutcome({
|
|
112
|
+
exitCode: null,
|
|
113
|
+
signal: null,
|
|
114
|
+
timedOut: false,
|
|
115
|
+
spawnError: error,
|
|
116
|
+
stderr: stderr.decode(),
|
|
117
|
+
}, job.attempt, Date.now()));
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
await new Promise((resolve) => {
|
|
121
|
+
let settled = false;
|
|
122
|
+
let timedOut = false;
|
|
123
|
+
let killTimer;
|
|
124
|
+
let timeout;
|
|
125
|
+
const finish = (outcome) => {
|
|
126
|
+
if (settled)
|
|
127
|
+
return;
|
|
128
|
+
settled = true;
|
|
129
|
+
if (timeout !== undefined)
|
|
130
|
+
clearTimeout(timeout);
|
|
131
|
+
// A timed-out leader can exit while a SIGTERM-ignoring group member remains.
|
|
132
|
+
if (!outcome.timedOut && killTimer !== undefined)
|
|
133
|
+
clearTimeout(killTimer);
|
|
134
|
+
child.stdin?.destroy();
|
|
135
|
+
child.stdout?.destroy();
|
|
136
|
+
child.stderr?.destroy();
|
|
137
|
+
settle(deliverySettlementForOutcome(outcome, job.attempt, Date.now()));
|
|
138
|
+
resolve();
|
|
139
|
+
};
|
|
140
|
+
child.stdout?.on('data', () => { }); // Drain output; stdout is never a control channel.
|
|
141
|
+
child.stderr?.on('data', (chunk) => stderr.take(chunk));
|
|
142
|
+
child.stdin?.on('error', () => { }); // The child may exit before consuming stdin; its exit code is authoritative.
|
|
143
|
+
child.on('error', (error) => {
|
|
144
|
+
finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
|
|
145
|
+
});
|
|
146
|
+
child.on('exit', (exitCode, signal) => {
|
|
147
|
+
finish({ exitCode, signal, timedOut, stderr: stderr.decode() });
|
|
148
|
+
});
|
|
149
|
+
const killWithEscalation = () => {
|
|
150
|
+
if (child.pid === undefined)
|
|
151
|
+
return;
|
|
152
|
+
killProcessGroup(child.pid);
|
|
153
|
+
killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), DELIVERY_KILL_GRACE_MS);
|
|
154
|
+
killTimer.unref?.();
|
|
155
|
+
};
|
|
156
|
+
// The deadline starts at spawn, not after stdin write or any output.
|
|
157
|
+
timeout = setTimeout(() => {
|
|
158
|
+
timedOut = true;
|
|
159
|
+
killWithEscalation();
|
|
160
|
+
}, DELIVERY_TIMEOUT_MS);
|
|
161
|
+
timeout.unref?.();
|
|
162
|
+
child.stdin?.end(document, 'utf8');
|
|
163
|
+
});
|
|
164
|
+
}
|
|
@@ -1,16 +1,3 @@
|
|
|
1
|
-
import { type ActionDeliveryRecord
|
|
2
|
-
export declare const ACTION_DELIVERY_TIMEOUT_MS = 120000;
|
|
3
|
-
export declare const ACTION_DELIVERY_STDERR_TAIL_BYTES: number;
|
|
4
|
-
/** Durable exponential retry delay for the attempt that just failed. */
|
|
5
|
-
export declare function actionDeliveryBackoffMs(attempt: number): number;
|
|
6
|
-
export interface ActionDeliveryProcessOutcome {
|
|
7
|
-
exitCode: number | null;
|
|
8
|
-
signal: string | null;
|
|
9
|
-
timedOut: boolean;
|
|
10
|
-
spawnError?: Error;
|
|
11
|
-
stderr: string;
|
|
12
|
-
}
|
|
13
|
-
/** Process contract mapping. stdout deliberately does not participate. */
|
|
14
|
-
export declare function actionDeliverySettlementForOutcome(outcome: ActionDeliveryProcessOutcome, attempt: number, now: number): ActionDeliverySettlement;
|
|
1
|
+
import { type ActionDeliveryRecord } from '../../core/canvas/human-deliveries.js';
|
|
15
2
|
/** Spawn and settle one already-claimed action delivery. */
|
|
16
3
|
export declare function deliverAction(delivery: ActionDeliveryRecord, claimOwner: string): Promise<void>;
|