@north-light/crouter 0.3.300 → 0.3.302
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/dto/reports.d.ts +8 -8
- package/dist/clients/attach/session/profile-files.d.ts +6 -3
- package/dist/clients/attach/session/profile-files.js +39 -10
- package/dist/clients/attach/viewer.js +663 -663
- package/dist/commands/memory/__tests__/command-selector-and-mutation-guards.test.js +30 -1
- package/dist/commands/memory/read.js +12 -14
- package/dist/commands/node/create.js +1 -1
- package/dist/commands/node/lifecycle.js +36 -8
- package/dist/commands/push.js +4 -4
- package/dist/core/__tests__/human-action-delivery.test.js +13 -11
- package/dist/core/__tests__/integration/revive.test.js +3 -3
- package/dist/core/__tests__/node-outcome.test.js +1 -6
- package/dist/core/__tests__/seam/dormancy-release.test.js +42 -0
- package/dist/core/__tests__/stop-guard.test.js +64 -0
- package/dist/core/bash-jobs.d.ts +22 -0
- package/dist/core/bash-jobs.js +38 -1
- package/dist/core/canvas/crons.d.ts +20 -4
- package/dist/core/canvas/crons.js +54 -7
- package/dist/core/canvas/delivery-contract.d.ts +19 -0
- package/dist/core/canvas/delivery-contract.js +8 -0
- package/dist/core/canvas/human-deliveries.d.ts +3 -17
- package/dist/core/canvas/node-outcome-deliveries.d.ts +3 -19
- package/dist/core/canvas/node-outcome-deliveries.js +0 -4
- package/dist/core/keybindings/catalog.js +1 -1
- package/dist/core/memory-resolver.d.ts +3 -1
- package/dist/core/memory-resolver.js +3 -3
- package/dist/core/nested-stores.d.ts +1 -1
- package/dist/core/nested-stores.js +3 -3
- package/dist/core/runtime/bearings-render.js +1 -1
- package/dist/core/runtime/close.js +8 -6
- package/dist/core/runtime/stop-guard.js +6 -6
- package/dist/core/worktree.js +3 -6
- package/dist/daemon/__tests__/node-outcome-delivery.test.js +3 -2
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +21 -11
- package/dist/daemon/api/handlers/reports.js +1 -1
- package/dist/daemon/api/map.d.ts +1 -1
- package/dist/daemon/api/map.js +2 -2
- package/dist/daemon/cron/sinks.d.ts +1 -1
- package/dist/daemon/cron/sinks.js +2 -2
- package/dist/daemon/delivery/run-delivery.d.ts +25 -0
- package/dist/daemon/delivery/run-delivery.js +164 -0
- package/dist/daemon/human/deliver-action.d.ts +1 -14
- package/dist/daemon/human/deliver-action.js +5 -165
- package/dist/daemon/node-outcome/deliver-outcome.d.ts +1 -12
- package/dist/daemon/node-outcome/deliver-outcome.js +6 -152
- package/dist/daemon/reconcilers/live-obligation.js +2 -2
- package/dist/pi-extensions/canvas-bash-valve.js +7 -10
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
- package/dist/daemon/cron-sink.d.ts +0 -26
- package/dist/daemon/cron-sink.js +0 -43
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// One process contract for every durable delivery crouter retries: spawn the
|
|
2
|
+
// declared target, hand it the document on stdin, and settle the attempt from
|
|
3
|
+
// its exit code. Human-request completions and node outcomes differ only in the
|
|
4
|
+
// row they settle, so both call this and neither reimplements the protocol.
|
|
5
|
+
import { spawn } from 'node:child_process';
|
|
6
|
+
import { envHomeOverride } from '../../shared/env.js';
|
|
7
|
+
import { deliveryBackoffMs } from '../../core/canvas/delivery-contract.js';
|
|
8
|
+
import { killProcessGroup } from '../../core/canvas/pid.js';
|
|
9
|
+
import { buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
|
|
10
|
+
export const DELIVERY_TIMEOUT_MS = 120_000;
|
|
11
|
+
const DELIVERY_KILL_GRACE_MS = 2_000;
|
|
12
|
+
export const DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
|
|
13
|
+
/** Process contract mapping. stdout deliberately does not participate. */
|
|
14
|
+
export function deliverySettlementForOutcome(outcome, attempt, now) {
|
|
15
|
+
if (outcome.spawnError !== undefined) {
|
|
16
|
+
return {
|
|
17
|
+
state: 'pending',
|
|
18
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
19
|
+
failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
if (outcome.timedOut) {
|
|
23
|
+
return {
|
|
24
|
+
state: 'pending',
|
|
25
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
26
|
+
// The kill escalation's own signal distinguishes the graceful term from the SIGKILL.
|
|
27
|
+
failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
if (outcome.signal !== null) {
|
|
31
|
+
return {
|
|
32
|
+
state: 'pending',
|
|
33
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
34
|
+
failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
if (outcome.exitCode === 0)
|
|
38
|
+
return { state: 'accepted' };
|
|
39
|
+
if (outcome.exitCode === 78) {
|
|
40
|
+
return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
|
|
41
|
+
}
|
|
42
|
+
return {
|
|
43
|
+
state: 'pending',
|
|
44
|
+
nextAttemptAt: now + deliveryBackoffMs(attempt),
|
|
45
|
+
failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
function failure(kind, fields) {
|
|
49
|
+
return { kind, ...fields };
|
|
50
|
+
}
|
|
51
|
+
function stderrTailCapture() {
|
|
52
|
+
let tail = Buffer.alloc(0);
|
|
53
|
+
return {
|
|
54
|
+
take(chunk) {
|
|
55
|
+
tail = chunk.length >= DELIVERY_STDERR_TAIL_BYTES
|
|
56
|
+
? chunk.subarray(chunk.length - DELIVERY_STDERR_TAIL_BYTES)
|
|
57
|
+
: Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - DELIVERY_STDERR_TAIL_BYTES));
|
|
58
|
+
},
|
|
59
|
+
decode() {
|
|
60
|
+
return tail.toString('utf8');
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
/** The stored document is immutable; attempt is its only per-delivery mutation. */
|
|
65
|
+
function documentForAttempt(document, attempt) {
|
|
66
|
+
const stored = document;
|
|
67
|
+
const delivery = stored.delivery;
|
|
68
|
+
return JSON.stringify({ ...stored, delivery: { ...delivery, attempt } });
|
|
69
|
+
}
|
|
70
|
+
/** The delivery child crosses the same consent boundary as every other
|
|
71
|
+
* crouter spawn: nothing of crtrd's own environment reaches it except what
|
|
72
|
+
* `spawnEnv.allow` and the operational base admit for the target's own cwd.
|
|
73
|
+
* No profile participates — a delivery target is declared in scope config alone. */
|
|
74
|
+
function deliveryEnv(cwd) {
|
|
75
|
+
const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
|
|
76
|
+
// Runtime identity, not host state: a target that calls `crtr` must reach
|
|
77
|
+
// the daemon that delivered to it, exactly as a cron body does.
|
|
78
|
+
const canvasHome = envHomeOverride();
|
|
79
|
+
if (canvasHome !== undefined && canvasHome !== '')
|
|
80
|
+
env['CRTR_HOME'] = canvasHome;
|
|
81
|
+
return env;
|
|
82
|
+
}
|
|
83
|
+
/** Spawn and settle one already-claimed delivery. `settle` writes the outcome
|
|
84
|
+
* to whichever durable row owns this attempt. */
|
|
85
|
+
export async function runDelivery(job, settle) {
|
|
86
|
+
const stderr = stderrTailCapture();
|
|
87
|
+
let document;
|
|
88
|
+
try {
|
|
89
|
+
document = documentForAttempt(job.document, job.attempt);
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
settle(deliverySettlementForOutcome({
|
|
93
|
+
exitCode: null,
|
|
94
|
+
signal: null,
|
|
95
|
+
timedOut: false,
|
|
96
|
+
spawnError: error,
|
|
97
|
+
stderr: stderr.decode(),
|
|
98
|
+
}, job.attempt, Date.now()));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
let child;
|
|
102
|
+
try {
|
|
103
|
+
child = spawn(job.argv[0], job.argv.slice(1), {
|
|
104
|
+
cwd: job.cwd,
|
|
105
|
+
env: deliveryEnv(job.cwd),
|
|
106
|
+
detached: true,
|
|
107
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
108
|
+
});
|
|
109
|
+
}
|
|
110
|
+
catch (error) {
|
|
111
|
+
settle(deliverySettlementForOutcome({
|
|
112
|
+
exitCode: null,
|
|
113
|
+
signal: null,
|
|
114
|
+
timedOut: false,
|
|
115
|
+
spawnError: error,
|
|
116
|
+
stderr: stderr.decode(),
|
|
117
|
+
}, job.attempt, Date.now()));
|
|
118
|
+
return;
|
|
119
|
+
}
|
|
120
|
+
await new Promise((resolve) => {
|
|
121
|
+
let settled = false;
|
|
122
|
+
let timedOut = false;
|
|
123
|
+
let killTimer;
|
|
124
|
+
let timeout;
|
|
125
|
+
const finish = (outcome) => {
|
|
126
|
+
if (settled)
|
|
127
|
+
return;
|
|
128
|
+
settled = true;
|
|
129
|
+
if (timeout !== undefined)
|
|
130
|
+
clearTimeout(timeout);
|
|
131
|
+
// A timed-out leader can exit while a SIGTERM-ignoring group member remains.
|
|
132
|
+
if (!outcome.timedOut && killTimer !== undefined)
|
|
133
|
+
clearTimeout(killTimer);
|
|
134
|
+
child.stdin?.destroy();
|
|
135
|
+
child.stdout?.destroy();
|
|
136
|
+
child.stderr?.destroy();
|
|
137
|
+
settle(deliverySettlementForOutcome(outcome, job.attempt, Date.now()));
|
|
138
|
+
resolve();
|
|
139
|
+
};
|
|
140
|
+
child.stdout?.on('data', () => { }); // Drain output; stdout is never a control channel.
|
|
141
|
+
child.stderr?.on('data', (chunk) => stderr.take(chunk));
|
|
142
|
+
child.stdin?.on('error', () => { }); // The child may exit before consuming stdin; its exit code is authoritative.
|
|
143
|
+
child.on('error', (error) => {
|
|
144
|
+
finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
|
|
145
|
+
});
|
|
146
|
+
child.on('exit', (exitCode, signal) => {
|
|
147
|
+
finish({ exitCode, signal, timedOut, stderr: stderr.decode() });
|
|
148
|
+
});
|
|
149
|
+
const killWithEscalation = () => {
|
|
150
|
+
if (child.pid === undefined)
|
|
151
|
+
return;
|
|
152
|
+
killProcessGroup(child.pid);
|
|
153
|
+
killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), DELIVERY_KILL_GRACE_MS);
|
|
154
|
+
killTimer.unref?.();
|
|
155
|
+
};
|
|
156
|
+
// The deadline starts at spawn, not after stdin write or any output.
|
|
157
|
+
timeout = setTimeout(() => {
|
|
158
|
+
timedOut = true;
|
|
159
|
+
killWithEscalation();
|
|
160
|
+
}, DELIVERY_TIMEOUT_MS);
|
|
161
|
+
timeout.unref?.();
|
|
162
|
+
child.stdin?.end(document, 'utf8');
|
|
163
|
+
});
|
|
164
|
+
}
|
|
@@ -1,16 +1,3 @@
|
|
|
1
|
-
import { type ActionDeliveryRecord
|
|
2
|
-
export declare const ACTION_DELIVERY_TIMEOUT_MS = 120000;
|
|
3
|
-
export declare const ACTION_DELIVERY_STDERR_TAIL_BYTES: number;
|
|
4
|
-
/** Durable exponential retry delay for the attempt that just failed. */
|
|
5
|
-
export declare function actionDeliveryBackoffMs(attempt: number): number;
|
|
6
|
-
export interface ActionDeliveryProcessOutcome {
|
|
7
|
-
exitCode: number | null;
|
|
8
|
-
signal: string | null;
|
|
9
|
-
timedOut: boolean;
|
|
10
|
-
spawnError?: Error;
|
|
11
|
-
stderr: string;
|
|
12
|
-
}
|
|
13
|
-
/** Process contract mapping. stdout deliberately does not participate. */
|
|
14
|
-
export declare function actionDeliverySettlementForOutcome(outcome: ActionDeliveryProcessOutcome, attempt: number, now: number): ActionDeliverySettlement;
|
|
1
|
+
import { type ActionDeliveryRecord } from '../../core/canvas/human-deliveries.js';
|
|
15
2
|
/** Spawn and settle one already-claimed action delivery. */
|
|
16
3
|
export declare function deliverAction(delivery: ActionDeliveryRecord, claimOwner: string): Promise<void>;
|
|
@@ -1,169 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
// Delivery of one settled human request to its declared completion action.
|
|
2
|
+
// The process contract lives in `daemon/delivery/run-delivery.ts`; this file
|
|
3
|
+
// binds it to the action-delivery row.
|
|
3
4
|
import { settleActionDelivery, } from '../../core/canvas/human-deliveries.js';
|
|
4
|
-
import {
|
|
5
|
-
import { buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
|
|
6
|
-
export const ACTION_DELIVERY_TIMEOUT_MS = 120_000;
|
|
7
|
-
const ACTION_DELIVERY_KILL_GRACE_MS = 2_000;
|
|
8
|
-
export const ACTION_DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
|
|
9
|
-
/** Durable exponential retry delay for the attempt that just failed. */
|
|
10
|
-
export function actionDeliveryBackoffMs(attempt) {
|
|
11
|
-
return Math.min(5_000 * 2 ** (attempt - 1), 3_600_000);
|
|
12
|
-
}
|
|
13
|
-
/** Process contract mapping. stdout deliberately does not participate. */
|
|
14
|
-
export function actionDeliverySettlementForOutcome(outcome, attempt, now) {
|
|
15
|
-
if (outcome.spawnError !== undefined) {
|
|
16
|
-
return {
|
|
17
|
-
state: 'pending',
|
|
18
|
-
nextAttemptAt: now + actionDeliveryBackoffMs(attempt),
|
|
19
|
-
failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
|
|
20
|
-
};
|
|
21
|
-
}
|
|
22
|
-
if (outcome.timedOut) {
|
|
23
|
-
return {
|
|
24
|
-
state: 'pending',
|
|
25
|
-
nextAttemptAt: now + actionDeliveryBackoffMs(attempt),
|
|
26
|
-
// The kill escalation's own signal distinguishes the graceful term from the SIGKILL.
|
|
27
|
-
failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
|
|
28
|
-
};
|
|
29
|
-
}
|
|
30
|
-
if (outcome.signal !== null) {
|
|
31
|
-
return {
|
|
32
|
-
state: 'pending',
|
|
33
|
-
nextAttemptAt: now + actionDeliveryBackoffMs(attempt),
|
|
34
|
-
failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
|
|
35
|
-
};
|
|
36
|
-
}
|
|
37
|
-
if (outcome.exitCode === 0)
|
|
38
|
-
return { state: 'accepted' };
|
|
39
|
-
if (outcome.exitCode === 78) {
|
|
40
|
-
return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
|
|
41
|
-
}
|
|
42
|
-
return {
|
|
43
|
-
state: 'pending',
|
|
44
|
-
nextAttemptAt: now + actionDeliveryBackoffMs(attempt),
|
|
45
|
-
failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
function failure(kind, fields) {
|
|
49
|
-
return { kind, ...fields };
|
|
50
|
-
}
|
|
51
|
-
function stderrTailCapture() {
|
|
52
|
-
let tail = Buffer.alloc(0);
|
|
53
|
-
return {
|
|
54
|
-
take(chunk) {
|
|
55
|
-
tail = chunk.length >= ACTION_DELIVERY_STDERR_TAIL_BYTES
|
|
56
|
-
? chunk.subarray(chunk.length - ACTION_DELIVERY_STDERR_TAIL_BYTES)
|
|
57
|
-
: Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - ACTION_DELIVERY_STDERR_TAIL_BYTES));
|
|
58
|
-
},
|
|
59
|
-
decode() {
|
|
60
|
-
return tail.toString('utf8');
|
|
61
|
-
},
|
|
62
|
-
};
|
|
63
|
-
}
|
|
64
|
-
function completionForAttempt(completion, attempt) {
|
|
65
|
-
const document = completion;
|
|
66
|
-
const delivery = document.delivery;
|
|
67
|
-
return JSON.stringify({ ...document, delivery: { ...delivery, attempt } });
|
|
68
|
-
}
|
|
5
|
+
import { runDelivery } from '../delivery/run-delivery.js';
|
|
69
6
|
/** Spawn and settle one already-claimed action delivery. */
|
|
70
7
|
export async function deliverAction(delivery, claimOwner) {
|
|
71
|
-
|
|
72
|
-
let completion;
|
|
73
|
-
try {
|
|
74
|
-
completion = completionForAttempt(delivery.completion, delivery.attempt);
|
|
75
|
-
}
|
|
76
|
-
catch (error) {
|
|
77
|
-
settle(delivery, claimOwner, actionDeliverySettlementForOutcome({
|
|
78
|
-
exitCode: null,
|
|
79
|
-
signal: null,
|
|
80
|
-
timedOut: false,
|
|
81
|
-
spawnError: error,
|
|
82
|
-
stderr: stderr.decode(),
|
|
83
|
-
}, delivery.attempt, Date.now()));
|
|
84
|
-
return;
|
|
85
|
-
}
|
|
86
|
-
let child;
|
|
87
|
-
try {
|
|
88
|
-
child = spawn(delivery.argv[0], delivery.argv.slice(1), {
|
|
89
|
-
cwd: delivery.cwd,
|
|
90
|
-
env: actionDeliveryEnv(delivery.cwd),
|
|
91
|
-
detached: true,
|
|
92
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
93
|
-
});
|
|
94
|
-
}
|
|
95
|
-
catch (error) {
|
|
96
|
-
settle(delivery, claimOwner, actionDeliverySettlementForOutcome({
|
|
97
|
-
exitCode: null,
|
|
98
|
-
signal: null,
|
|
99
|
-
timedOut: false,
|
|
100
|
-
spawnError: error,
|
|
101
|
-
stderr: stderr.decode(),
|
|
102
|
-
}, delivery.attempt, Date.now()));
|
|
103
|
-
return;
|
|
104
|
-
}
|
|
105
|
-
await new Promise((resolve) => {
|
|
106
|
-
let settled = false;
|
|
107
|
-
let timedOut = false;
|
|
108
|
-
let killTimer;
|
|
109
|
-
let timeout;
|
|
110
|
-
const finish = (outcome) => {
|
|
111
|
-
if (settled)
|
|
112
|
-
return;
|
|
113
|
-
settled = true;
|
|
114
|
-
if (timeout !== undefined)
|
|
115
|
-
clearTimeout(timeout);
|
|
116
|
-
if (killTimer !== undefined)
|
|
117
|
-
clearTimeout(killTimer);
|
|
118
|
-
child.stdin?.destroy();
|
|
119
|
-
child.stdout?.destroy();
|
|
120
|
-
child.stderr?.destroy();
|
|
121
|
-
settle(delivery, claimOwner, actionDeliverySettlementForOutcome(outcome, delivery.attempt, Date.now()));
|
|
122
|
-
resolve();
|
|
123
|
-
};
|
|
124
|
-
child.stdout?.on('data', () => { }); // Drain diagnostics; stdout is never a control channel.
|
|
125
|
-
child.stderr?.on('data', (chunk) => stderr.take(chunk));
|
|
126
|
-
child.stdin?.on('error', () => { }); // A child may exit before draining a large completion; its exit code remains authoritative.
|
|
127
|
-
child.on('error', (error) => {
|
|
128
|
-
finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
|
|
129
|
-
});
|
|
130
|
-
child.on('exit', (exitCode, signal) => {
|
|
131
|
-
finish({
|
|
132
|
-
exitCode,
|
|
133
|
-
signal,
|
|
134
|
-
timedOut,
|
|
135
|
-
stderr: stderr.decode(),
|
|
136
|
-
});
|
|
137
|
-
});
|
|
138
|
-
const killWithEscalation = () => {
|
|
139
|
-
if (child.pid === undefined)
|
|
140
|
-
return;
|
|
141
|
-
killProcessGroup(child.pid);
|
|
142
|
-
killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), ACTION_DELIVERY_KILL_GRACE_MS);
|
|
143
|
-
killTimer.unref?.();
|
|
144
|
-
};
|
|
145
|
-
// The deadline starts at spawn, not after stdin write or any output.
|
|
146
|
-
timeout = setTimeout(() => {
|
|
147
|
-
timedOut = true;
|
|
148
|
-
killWithEscalation();
|
|
149
|
-
}, ACTION_DELIVERY_TIMEOUT_MS);
|
|
150
|
-
timeout.unref?.();
|
|
151
|
-
child.stdin?.end(completion, 'utf8');
|
|
152
|
-
});
|
|
153
|
-
}
|
|
154
|
-
/** The delivery child crosses the same consent boundary as every other
|
|
155
|
-
* crouter spawn: nothing of crtrd's own environment reaches it except what
|
|
156
|
-
* `spawnEnv.allow` and the operational base admit for the action's own cwd.
|
|
157
|
-
* No profile participates — an action is declared in scope config alone. */
|
|
158
|
-
function actionDeliveryEnv(cwd) {
|
|
159
|
-
const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
|
|
160
|
-
// Runtime identity, not host state: an action that calls `crtr` must reach
|
|
161
|
-
// the daemon that delivered to it, exactly as a cron body does.
|
|
162
|
-
const canvasHome = envHomeOverride();
|
|
163
|
-
if (canvasHome !== undefined && canvasHome !== '')
|
|
164
|
-
env['CRTR_HOME'] = canvasHome;
|
|
165
|
-
return env;
|
|
166
|
-
}
|
|
167
|
-
function settle(delivery, claimOwner, settlement) {
|
|
168
|
-
settleActionDelivery(delivery.requestId, delivery.attempt, claimOwner, settlement, Date.now());
|
|
8
|
+
await runDelivery({ argv: delivery.argv, cwd: delivery.cwd, document: delivery.completion, attempt: delivery.attempt }, (settlement) => settleActionDelivery(delivery.requestId, delivery.attempt, claimOwner, settlement, Date.now()));
|
|
169
9
|
}
|
|
@@ -1,14 +1,3 @@
|
|
|
1
|
-
import { type OutcomeDeliveryRecord
|
|
2
|
-
export declare const OUTCOME_DELIVERY_TIMEOUT_MS = 120000;
|
|
3
|
-
export declare const OUTCOME_DELIVERY_STDERR_TAIL_BYTES: number;
|
|
4
|
-
export interface OutcomeDeliveryProcessOutcome {
|
|
5
|
-
exitCode: number | null;
|
|
6
|
-
signal: string | null;
|
|
7
|
-
timedOut: boolean;
|
|
8
|
-
spawnError?: Error;
|
|
9
|
-
stderr: string;
|
|
10
|
-
}
|
|
11
|
-
/** The delivery target's exit code is its complete acknowledgment protocol. */
|
|
12
|
-
export declare function outcomeDeliverySettlementForOutcome(outcome: OutcomeDeliveryProcessOutcome, attempt: number, now: number): OutcomeDeliverySettlement;
|
|
1
|
+
import { type OutcomeDeliveryRecord } from '../../core/canvas/node-outcome-deliveries.js';
|
|
13
2
|
/** Spawn and settle one already-claimed outcome delivery. */
|
|
14
3
|
export declare function deliverOutcome(delivery: OutcomeDeliveryRecord, claimOwner: string): Promise<void>;
|
|
@@ -1,155 +1,9 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
6
|
-
export const OUTCOME_DELIVERY_TIMEOUT_MS = 120_000;
|
|
7
|
-
const OUTCOME_DELIVERY_KILL_GRACE_MS = 2_000;
|
|
8
|
-
export const OUTCOME_DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
|
|
9
|
-
/** The delivery target's exit code is its complete acknowledgment protocol. */
|
|
10
|
-
export function outcomeDeliverySettlementForOutcome(outcome, attempt, now) {
|
|
11
|
-
if (outcome.spawnError !== undefined) {
|
|
12
|
-
return {
|
|
13
|
-
state: 'pending',
|
|
14
|
-
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
15
|
-
failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
|
|
16
|
-
};
|
|
17
|
-
}
|
|
18
|
-
if (outcome.timedOut) {
|
|
19
|
-
return {
|
|
20
|
-
state: 'pending',
|
|
21
|
-
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
22
|
-
failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
|
|
23
|
-
};
|
|
24
|
-
}
|
|
25
|
-
if (outcome.signal !== null) {
|
|
26
|
-
return {
|
|
27
|
-
state: 'pending',
|
|
28
|
-
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
29
|
-
failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
|
|
30
|
-
};
|
|
31
|
-
}
|
|
32
|
-
if (outcome.exitCode === 0)
|
|
33
|
-
return { state: 'accepted' };
|
|
34
|
-
if (outcome.exitCode === 78) {
|
|
35
|
-
return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
|
|
36
|
-
}
|
|
37
|
-
return {
|
|
38
|
-
state: 'pending',
|
|
39
|
-
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
40
|
-
failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
|
|
41
|
-
};
|
|
42
|
-
}
|
|
43
|
-
function failure(kind, fields) {
|
|
44
|
-
return { kind, ...fields };
|
|
45
|
-
}
|
|
46
|
-
function stderrTailCapture() {
|
|
47
|
-
let tail = Buffer.alloc(0);
|
|
48
|
-
return {
|
|
49
|
-
take(chunk) {
|
|
50
|
-
tail = chunk.length >= OUTCOME_DELIVERY_STDERR_TAIL_BYTES
|
|
51
|
-
? chunk.subarray(chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES)
|
|
52
|
-
: Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES));
|
|
53
|
-
},
|
|
54
|
-
decode() {
|
|
55
|
-
return tail.toString('utf8');
|
|
56
|
-
},
|
|
57
|
-
};
|
|
58
|
-
}
|
|
59
|
-
/** The stored document is immutable; attempt is its only per-delivery mutation. */
|
|
60
|
-
function documentForAttempt(document, attempt) {
|
|
61
|
-
const outcome = document;
|
|
62
|
-
const delivery = outcome.delivery;
|
|
63
|
-
return JSON.stringify({ ...outcome, delivery: { ...delivery, attempt } });
|
|
64
|
-
}
|
|
1
|
+
// Delivery of one node's terminal outcome to its registered action target.
|
|
2
|
+
// The process contract lives in `daemon/delivery/run-delivery.ts`; this file
|
|
3
|
+
// binds it to the outcome-delivery row.
|
|
4
|
+
import { settleOutcomeDelivery, } from '../../core/canvas/node-outcome-deliveries.js';
|
|
5
|
+
import { runDelivery } from '../delivery/run-delivery.js';
|
|
65
6
|
/** Spawn and settle one already-claimed outcome delivery. */
|
|
66
7
|
export async function deliverOutcome(delivery, claimOwner) {
|
|
67
|
-
|
|
68
|
-
let document;
|
|
69
|
-
try {
|
|
70
|
-
document = documentForAttempt(delivery.document, delivery.attempt);
|
|
71
|
-
}
|
|
72
|
-
catch (error) {
|
|
73
|
-
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
|
|
74
|
-
exitCode: null,
|
|
75
|
-
signal: null,
|
|
76
|
-
timedOut: false,
|
|
77
|
-
spawnError: error,
|
|
78
|
-
stderr: stderr.decode(),
|
|
79
|
-
}, delivery.attempt, Date.now()));
|
|
80
|
-
return;
|
|
81
|
-
}
|
|
82
|
-
let child;
|
|
83
|
-
try {
|
|
84
|
-
child = spawn(delivery.argv[0], delivery.argv.slice(1), {
|
|
85
|
-
cwd: delivery.cwd,
|
|
86
|
-
env: outcomeDeliveryEnv(delivery.cwd),
|
|
87
|
-
detached: true,
|
|
88
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
89
|
-
});
|
|
90
|
-
}
|
|
91
|
-
catch (error) {
|
|
92
|
-
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
|
|
93
|
-
exitCode: null,
|
|
94
|
-
signal: null,
|
|
95
|
-
timedOut: false,
|
|
96
|
-
spawnError: error,
|
|
97
|
-
stderr: stderr.decode(),
|
|
98
|
-
}, delivery.attempt, Date.now()));
|
|
99
|
-
return;
|
|
100
|
-
}
|
|
101
|
-
await new Promise((resolve) => {
|
|
102
|
-
let settled = false;
|
|
103
|
-
let timedOut = false;
|
|
104
|
-
let killTimer;
|
|
105
|
-
let timeout;
|
|
106
|
-
const finish = (outcome) => {
|
|
107
|
-
if (settled)
|
|
108
|
-
return;
|
|
109
|
-
settled = true;
|
|
110
|
-
if (timeout !== undefined)
|
|
111
|
-
clearTimeout(timeout);
|
|
112
|
-
// A timed-out leader can exit while a SIGTERM-ignoring group member remains.
|
|
113
|
-
if (!outcome.timedOut && killTimer !== undefined)
|
|
114
|
-
clearTimeout(killTimer);
|
|
115
|
-
child.stdin?.destroy();
|
|
116
|
-
child.stdout?.destroy();
|
|
117
|
-
child.stderr?.destroy();
|
|
118
|
-
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome(outcome, delivery.attempt, Date.now()));
|
|
119
|
-
resolve();
|
|
120
|
-
};
|
|
121
|
-
child.stdout?.on('data', () => { }); // Drain output; stdout is never a control channel.
|
|
122
|
-
child.stderr?.on('data', (chunk) => stderr.take(chunk));
|
|
123
|
-
child.stdin?.on('error', () => { }); // The child may exit before consuming stdin; its exit code is authoritative.
|
|
124
|
-
child.on('error', (error) => {
|
|
125
|
-
finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
|
|
126
|
-
});
|
|
127
|
-
child.on('exit', (exitCode, signal) => {
|
|
128
|
-
finish({ exitCode, signal, timedOut, stderr: stderr.decode() });
|
|
129
|
-
});
|
|
130
|
-
const killWithEscalation = () => {
|
|
131
|
-
if (child.pid === undefined)
|
|
132
|
-
return;
|
|
133
|
-
killProcessGroup(child.pid);
|
|
134
|
-
killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), OUTCOME_DELIVERY_KILL_GRACE_MS);
|
|
135
|
-
killTimer.unref?.();
|
|
136
|
-
};
|
|
137
|
-
timeout = setTimeout(() => {
|
|
138
|
-
timedOut = true;
|
|
139
|
-
killWithEscalation();
|
|
140
|
-
}, OUTCOME_DELIVERY_TIMEOUT_MS);
|
|
141
|
-
timeout.unref?.();
|
|
142
|
-
child.stdin?.end(document, 'utf8');
|
|
143
|
-
});
|
|
144
|
-
}
|
|
145
|
-
/** Outcome targets cross the same consent boundary as every other crouter spawn. */
|
|
146
|
-
function outcomeDeliveryEnv(cwd) {
|
|
147
|
-
const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
|
|
148
|
-
const canvasHome = envHomeOverride();
|
|
149
|
-
if (canvasHome !== undefined && canvasHome !== '')
|
|
150
|
-
env['CRTR_HOME'] = canvasHome;
|
|
151
|
-
return env;
|
|
152
|
-
}
|
|
153
|
-
function settle(delivery, claimOwner, settlement) {
|
|
154
|
-
settleOutcomeDelivery(delivery.nodeId, delivery.attempt, claimOwner, settlement, Date.now());
|
|
8
|
+
await runDelivery({ argv: delivery.argv, cwd: delivery.cwd, document: delivery.document, attempt: delivery.attempt }, (settlement) => settleOutcomeDelivery(delivery.nodeId, delivery.attempt, claimOwner, settlement, Date.now()));
|
|
155
9
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { newestReportRef, hasActiveLiveSubscription, hasLiveMessageWait, hasLiveSubscriber,
|
|
1
|
+
import { newestReportRef, hasActiveLiveSubscription, hasLiveMessageWait, hasLiveSubscriber, hasPendingCronWake, subscribersOf, } from '../../core/canvas/index.js';
|
|
2
2
|
import { fullName } from '../../core/canvas/labels.js';
|
|
3
3
|
import { countTickets } from '../../core/canvas/attention.js';
|
|
4
4
|
import { hasUnseenWakeMail, readCursor, readInboxSince } from '../../core/feed/inbox.js';
|
|
@@ -12,7 +12,7 @@ import { transition } from '../../core/runtime/lifecycle.js';
|
|
|
12
12
|
export function hasLiveObligation(nodeId) {
|
|
13
13
|
return hasActiveLiveSubscription(nodeId)
|
|
14
14
|
|| hasLiveMessageWait(nodeId)
|
|
15
|
-
||
|
|
15
|
+
|| hasPendingCronWake(nodeId)
|
|
16
16
|
|| countTickets(nodeId) > 0
|
|
17
17
|
|| readOutputRequest(nodeId).state !== 'absent';
|
|
18
18
|
}
|
|
@@ -30,7 +30,7 @@ import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, rmS
|
|
|
30
30
|
import { homedir } from 'node:os';
|
|
31
31
|
import { dirname } from 'node:path';
|
|
32
32
|
import { performance } from 'node:perf_hooks';
|
|
33
|
-
import { backgroundBashJob, bashJobPaths, formatBashElapsed, newBashJobId, normalizeBashJobPurpose, writeJobDeadlineMs } from '../core/bash-jobs.js';
|
|
33
|
+
import { backgroundBashJob, bashJobPaths, formatBashElapsed, newBashJobId, normalizeBashJobPurpose, readBashNiceness, writeJobDeadlineMs } from '../core/bash-jobs.js';
|
|
34
34
|
import { readConfig } from '../core/config.js';
|
|
35
35
|
import { PREVIEW_RESULT_PATH_ENV, previewResultPath } from '../core/preview-result-path.js';
|
|
36
36
|
import { resolveAdmittedHostCommand } from '../core/runtime/spawn-env.js';
|
|
@@ -50,23 +50,20 @@ function resolveValveMs() {
|
|
|
50
50
|
// lives only in cmd.sh):
|
|
51
51
|
// $1 cmd.sh $2 job.log $3 job.exit $4 job.run
|
|
52
52
|
// $5 job.done $6 job.bg $7 jobId $8 nodeId $9 shell path
|
|
53
|
+
// ${10} niceness
|
|
53
54
|
//
|
|
54
55
|
// `--` before the paths is a dummy $0 (a standard `bash -c script -- args`
|
|
55
56
|
// idiom) so the real payload lands at $1 onward.
|
|
56
57
|
// The elapsed rendering mirrors formatBashElapsed() in core/bash-jobs.ts
|
|
57
58
|
// (`12s` / `3m 40s` / `1h 04m`) so the completion message reads the same as the
|
|
58
59
|
// handoff notice; it measures TOTAL runtime, foreground portion included.
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
* scheduler only honors this under contention and it costs nothing on an idle
|
|
63
|
-
* host. Only the command is niced; the supervisor that reports its finish
|
|
64
|
-
* runs at normal priority so completion notices stay prompt. */
|
|
65
|
-
const COMMAND_NICENESS = 10;
|
|
60
|
+
// Only the command is niced (at ${10}, the node's own niceness); the supervisor
|
|
61
|
+
// that reports its finish runs at normal priority so completion notices stay
|
|
62
|
+
// prompt.
|
|
66
63
|
const SUPERVISOR_SCRIPT = [
|
|
67
64
|
'SECONDS=0',
|
|
68
65
|
'n=$(command -v nice)',
|
|
69
|
-
|
|
66
|
+
'${n:+"$n" -n "${10}"} "${9}" "$1" > "$2" 2>&1',
|
|
70
67
|
'ec=$?',
|
|
71
68
|
'd=$SECONDS',
|
|
72
69
|
'echo "$ec" > "$3"',
|
|
@@ -337,7 +334,7 @@ export function createValveOperations(nodeId, contextDir, takePurpose = () => nu
|
|
|
337
334
|
childEnv = { ...childEnv, [PREVIEW_RESULT_PATH_ENV]: previewPath };
|
|
338
335
|
}
|
|
339
336
|
const shellPath = resolveAdmittedHostCommand('bash', childEnv, execution.cwd);
|
|
340
|
-
const child = spawn(shellPath, ['-c', SUPERVISOR_SCRIPT, '--', paths.cmdSh, paths.jobLog, paths.jobExit, paths.jobRun, paths.jobDone, paths.jobBg, paths.jobId, nodeId, shellPath], {
|
|
337
|
+
const child = spawn(shellPath, ['-c', SUPERVISOR_SCRIPT, '--', paths.cmdSh, paths.jobLog, paths.jobExit, paths.jobRun, paths.jobDone, paths.jobBg, paths.jobId, nodeId, shellPath, String(readBashNiceness(contextDir))], {
|
|
341
338
|
cwd: execution.cwd,
|
|
342
339
|
detached: true,
|
|
343
340
|
stdio: 'ignore',
|
package/package.json
CHANGED
package/runtime.lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/crouter",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.302",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@north-light/crouter",
|
|
9
|
-
"version": "0.3.
|
|
9
|
+
"version": "0.3.302",
|
|
10
10
|
"hasInstallScript": true,
|
|
11
11
|
"license": "MIT",
|
|
12
12
|
"workspaces": [
|
|
@@ -5206,17 +5206,17 @@
|
|
|
5206
5206
|
},
|
|
5207
5207
|
"packages/crouter-api": {
|
|
5208
5208
|
"name": "@north-light/crouter-api",
|
|
5209
|
-
"version": "0.3.
|
|
5209
|
+
"version": "0.3.302",
|
|
5210
5210
|
"license": "UNLICENSED"
|
|
5211
5211
|
},
|
|
5212
5212
|
"packages/crouter-env-docker": {
|
|
5213
5213
|
"name": "@north-light/crouter-env-docker",
|
|
5214
|
-
"version": "0.3.
|
|
5214
|
+
"version": "0.3.302",
|
|
5215
5215
|
"license": "UNLICENSED"
|
|
5216
5216
|
},
|
|
5217
5217
|
"packages/crouter-sdk": {
|
|
5218
5218
|
"name": "@north-light/crouter-sdk",
|
|
5219
|
-
"version": "0.3.
|
|
5219
|
+
"version": "0.3.302",
|
|
5220
5220
|
"license": "UNLICENSED",
|
|
5221
5221
|
"dependencies": {
|
|
5222
5222
|
"@north-light/crouter-api": "^0.3.295"
|