@north-light/cli 0.3.525 → 0.3.526
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/canvas/migrations.js +53 -53
- package/dist/core/canvas/recovery.d.ts +13 -0
- package/dist/core/canvas/recovery.js +2 -2
- package/dist/daemon/crtrd.js +8 -8
- package/dist/daemon/reconcilers/broker-supervision.d.ts +6 -2
- package/dist/daemon/reconcilers/broker-supervision.js +4 -4
- package/package.json +2 -2
- package/packages/crouter-identity/package.json +1 -1
- package/runtime.lock.json +12 -12
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
var Te=Object.defineProperty;var E=(e,t)=>Te(e,"name",{value:t,configurable:!0});import{randomBytes as P}from"node:crypto";import{parseScope as ae}from"@north-light/identity";import{importNodeSqlAuthority as de,recreateNodeRowsView as _e}from"./node-sql-migration.js";import{migrateRecoveryWitnesses as
|
|
1
|
+
var Te=Object.defineProperty;var E=(e,t)=>Te(e,"name",{value:t,configurable:!0});import{randomBytes as P}from"node:crypto";import{parseScope as ae}from"@north-light/identity";import{importNodeSqlAuthority as de,recreateNodeRowsView as _e}from"./node-sql-migration.js";import{addFaultNotifiedSince as Ne,migrateRecoveryWitnesses as ce}from"./recovery.js";import{migrateMessagingState as Le}from"./deliveries.js";import{migrateHumanRequestsState as le}from"../human/requests.js";import{existsSync as U,readFileSync as M,readdirSync as k,realpathSync as ue,statSync as pe}from"node:fs";import{basename as Re,join as j,resolve as Y}from"node:path";import{appSpace as Ae}from"../layout.js";import{emitEvent as Oe}from"../events/emit.js";import{isSafeNodeId as Ie,nodesRoot as h,nodeMetaPath as x}from"./paths.js";import{convertLegacyEdges as me}from"../../migrations/004-canvas-documents/legacy-edges.js";import{customExtensionPaths as fe}from"./extensions.js";import{readMetaObject as K,updateJsonFileDurably as q,withMetaLock as Xe,writeMetaObjectLocked as Ue}from"./meta-file.js";function Ce(e){e.exec(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS nodes (
|
|
3
3
|
node_id TEXT PRIMARY KEY,
|
|
4
4
|
name TEXT NOT NULL,
|
|
@@ -23,7 +23,7 @@ CREATE TABLE IF NOT EXISTS edges (
|
|
|
23
23
|
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(type, to_id);
|
|
24
24
|
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(type, from_id);
|
|
25
25
|
CREATE INDEX IF NOT EXISTS idx_nodes_status ON nodes(status);
|
|
26
|
-
`)}E(
|
|
26
|
+
`)}E(Ce,"baselineSchema");function b(e,t){return new Set(e.prepare(`PRAGMA table_info(${t})`).all().map(n=>n.name))}E(b,"tableColumns");function $(e,t,n,o){t.has(n)||(e.exec(o),t.add(n))}E($,"addColumnIfMissing");function O(e){return b(e,"nodes")}E(O,"nodeColumns");const c=$;function Se(e){const t=O(e);c(e,t,"intent","ALTER TABLE nodes ADD COLUMN intent TEXT;"),c(e,t,"pi_pid","ALTER TABLE nodes ADD COLUMN pi_pid INTEGER;"),c(e,t,"window","ALTER TABLE nodes ADD COLUMN window TEXT;"),c(e,t,"tmux_session","ALTER TABLE nodes ADD COLUMN tmux_session TEXT;")}E(Se,"addRuntimeColumns");function De(e){const t=h();if(!U(t))return;const n=e.prepare('UPDATE nodes SET intent = ?, pi_pid = ?, "window" = ?, tmux_session = ? WHERE node_id = ?');for(const o of k(t)){const T=x(o);if(!U(T))continue;let i;try{i=JSON.parse(M(T,"utf8"))}catch{continue}n.run(i.intent??null,i.pi_pid??null,i.window??null,i.tmux_session??null,o)}}E(De,"backfillRuntime");function ge(e){B(e,3,()=>{if(!(e.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'edges'").get()!==void 0)){e.exec(`
|
|
27
27
|
CREATE TABLE edges (
|
|
28
28
|
type TEXT NOT NULL,
|
|
29
29
|
from_id TEXT NOT NULL,
|
|
@@ -57,14 +57,14 @@ DROP TABLE edges;
|
|
|
57
57
|
ALTER TABLE edges_new RENAME TO edges;
|
|
58
58
|
CREATE INDEX IF NOT EXISTS idx_edges_to ON edges(type, to_id);
|
|
59
59
|
CREATE INDEX IF NOT EXISTS idx_edges_from ON edges(type, from_id);
|
|
60
|
-
`);const o=e.prepare("SELECT COUNT(*) AS n FROM edges").get().n;if(o!==n)throw new Error(`edges FK rebuild lost rows: expected ${n} non-orphan edge(s), got ${o}`);const T=e.prepare("PRAGMA foreign_key_check").all();if(T.length>0)throw new Error(`edges FK rebuild left ${T.length} foreign-key violation(s)`)})}E(
|
|
60
|
+
`);const o=e.prepare("SELECT COUNT(*) AS n FROM edges").get().n;if(o!==n)throw new Error(`edges FK rebuild lost rows: expected ${n} non-orphan edge(s), got ${o}`);const T=e.prepare("PRAGMA foreign_key_check").all();if(T.length>0)throw new Error(`edges FK rebuild left ${T.length} foreign-key violation(s)`)})}E(ge,"edgesForeignKeyCascade");function we(e){const t=O(e);c(e,t,"pane","ALTER TABLE nodes ADD COLUMN pane TEXT;")}E(we,"addPaneColumn");function ye(e){e.exec(`
|
|
61
61
|
CREATE TABLE IF NOT EXISTS focuses (
|
|
62
62
|
focus_id TEXT PRIMARY KEY, -- stable internal id for the viewport
|
|
63
63
|
pane TEXT, -- the durable %pane_id realizing the focus
|
|
64
64
|
session TEXT, -- derived cache of the user session (reconciled from pane)
|
|
65
65
|
node_id TEXT NOT NULL UNIQUE -- the node shown; UNIQUE \u2192 a node occupies <=1 focus
|
|
66
66
|
);
|
|
67
|
-
`)}E(
|
|
67
|
+
`)}E(ye,"addFocusesTable");function ve(e){e.exec(`
|
|
68
68
|
CREATE TABLE IF NOT EXISTS wakeups (
|
|
69
69
|
wakeup_id TEXT PRIMARY KEY,
|
|
70
70
|
node_id TEXT, -- TARGET + node-anchored cancel anchor; NULL = canvas-detached
|
|
@@ -80,7 +80,7 @@ CREATE INDEX IF NOT EXISTS idx_wakeups_fire ON wakeups(fire_at);
|
|
|
80
80
|
CREATE INDEX IF NOT EXISTS idx_wakeups_node ON wakeups(node_id); -- by-node cancel / list
|
|
81
81
|
CREATE INDEX IF NOT EXISTS idx_wakeups_owner ON wakeups(owner_id); -- by-owner reap (ruling A)
|
|
82
82
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_wakeups_deadline ON wakeups(node_id) WHERE kind = 'deadline';
|
|
83
|
-
`)}E(
|
|
83
|
+
`)}E(ve,"appendWakeupsTable");function he(e){e.exec(`
|
|
84
84
|
CREATE TABLE IF NOT EXISTS triggers (
|
|
85
85
|
trigger_id TEXT PRIMARY KEY,
|
|
86
86
|
owner_id TEXT NOT NULL,
|
|
@@ -131,12 +131,12 @@ SELECT
|
|
|
131
131
|
FROM wakeups;
|
|
132
132
|
|
|
133
133
|
DROP TABLE wakeups;
|
|
134
|
-
`)}E(
|
|
134
|
+
`)}E(he,"migrateWakeupsToTriggers");function xe(e){const t=O(e);c(e,t,"host_kind","ALTER TABLE nodes ADD COLUMN host_kind TEXT;")}E(xe,"addHostKindColumn");function Fe(e){e.exec(`
|
|
135
135
|
CREATE TABLE IF NOT EXISTS canvas_meta (
|
|
136
136
|
key TEXT PRIMARY KEY,
|
|
137
137
|
value TEXT NOT NULL
|
|
138
138
|
);
|
|
139
|
-
`)}E(
|
|
139
|
+
`)}E(Fe,"addCanvasMetaTable");function Me(e){const t=O(e);c(e,t,"profile_id","ALTER TABLE nodes ADD COLUMN profile_id TEXT;")}E(Me,"addProfileIdColumn");function ke(e){const t=O(e);c(e,t,"pi_pid_identity","ALTER TABLE nodes ADD COLUMN pi_pid_identity TEXT;")}E(ke,"addPidIdentityColumn");function je(e){const t=O(e);c(e,t,"waiting_for","ALTER TABLE nodes ADD COLUMN waiting_for TEXT;")}E(je,"addMessageWaitColumn");function J(e){const t=O(e);if(t.size===0)return;c(e,t,"final_report","ALTER TABLE nodes ADD COLUMN final_report TEXT;"),c(e,t,"finalized_at","ALTER TABLE nodes ADD COLUMN finalized_at TEXT;");const n=e.prepare("SELECT node_id FROM nodes WHERE status = 'done' AND final_report IS NULL").all(),o=e.prepare("UPDATE nodes SET final_report = ?, finalized_at = ? WHERE node_id = ? AND final_report IS NULL");for(const{node_id:T}of n){const i=j(h(),T,"reports");if(!U(i))continue;let a=null;for(const N of k(i))if(N.endsWith(".md"))try{const L=M(j(i,N),"utf8"),l=/^---\nnode: [^\n]+\nkind: final\nts: ([^\n]+)\n---\n/.exec(L);if(l===null||Number.isNaN(Date.parse(l[1])))continue;const r={name:N,ts:l[1]};(a===null||r.ts>a.ts||r.ts===a.ts&&r.name>a.name)&&(a=r)}catch{}a!==null&&o.run(a.name,a.ts,T)}}E(J,"addCanonicalFinalColumns");function He(e){const t=h();if(U(t))for(const n of k(t)){const o=x(n);U(o)&&q(o,T=>{const i=T.launch;if(typeof i!="object"||i===null||Array.isArray(i))return!1;const a=i.extensions;if(!Array.isArray(a)||!a.every(L=>typeof L=="string"))return!1;const N=fe(a);return N.length===a.length?!1:(i.extensions=N,!0)})}}E(He,"normalizePersistedLaunchExtensions");function Ge(e){const t=O(e);c(e,t,"waiting_for_generation","ALTER TABLE nodes ADD COLUMN waiting_for_generation INTEGER NOT NULL DEFAULT 0;")}E(Ge,"addMessageWaitGenerationColumn");function be(e){e.exec(`
|
|
140
140
|
CREATE TABLE IF NOT EXISTS consult_outbox (
|
|
141
141
|
node_id TEXT PRIMARY KEY, -- legacy work node id
|
|
142
142
|
dir TEXT NOT NULL, -- legacy ticket directory
|
|
@@ -150,15 +150,15 @@ CREATE INDEX IF NOT EXISTS idx_consult_outbox_status ON consult_outbox(status);
|
|
|
150
150
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_dir ON consult_outbox(dir);
|
|
151
151
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_dir_request_status ON consult_outbox(dir, request_id, status);
|
|
152
152
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_dir_status ON consult_outbox(dir, status);
|
|
153
|
-
`)}E(
|
|
153
|
+
`)}E(be,"addHumanWorkOutboxTable");function Be(e){const t=b(e,"consult_outbox"),n=E((o,T)=>$(e,t,o,T),"add");n("work_kind","ALTER TABLE consult_outbox ADD COLUMN work_kind TEXT NOT NULL DEFAULT 'follow_up';"),n("interaction_id","ALTER TABLE consult_outbox ADD COLUMN interaction_id TEXT;"),n("generation_id","ALTER TABLE consult_outbox ADD COLUMN generation_id TEXT;"),n("interaction_snapshot","ALTER TABLE consult_outbox ADD COLUMN interaction_snapshot TEXT;"),n("handler_command","ALTER TABLE consult_outbox ADD COLUMN handler_command TEXT;"),n("handler_args_json","ALTER TABLE consult_outbox ADD COLUMN handler_args_json TEXT;"),n("claim_token","ALTER TABLE consult_outbox ADD COLUMN claim_token TEXT;"),e.exec(`
|
|
154
154
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_human_work_status ON consult_outbox(work_kind, status);
|
|
155
155
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_human_work_dir_status ON consult_outbox(dir, work_kind, status);
|
|
156
156
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_human_work_request_status ON consult_outbox(dir, request_id, work_kind, status);
|
|
157
157
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_consult_outbox_visual_request ON consult_outbox(dir, request_id, work_kind) WHERE work_kind = 'visual';
|
|
158
|
-
`)}E(
|
|
158
|
+
`)}E(Be,"addHumanWorkColumns");function We(e){e.exec(`
|
|
159
159
|
CREATE INDEX IF NOT EXISTS idx_nodes_parent ON nodes(parent);
|
|
160
160
|
CREATE INDEX IF NOT EXISTS idx_nodes_created ON nodes(created);
|
|
161
|
-
`)}E(
|
|
161
|
+
`)}E(We,"addNodeParentCreatedIndexes");function Pe(e){e.exec(`
|
|
162
162
|
CREATE TABLE IF NOT EXISTS crons (
|
|
163
163
|
cron_id TEXT PRIMARY KEY,
|
|
164
164
|
name TEXT NOT NULL, -- agent-legible label; how it shows in list
|
|
@@ -213,22 +213,22 @@ CREATE TABLE IF NOT EXISTS cron_runs ( -- bounded ring, last 20 p
|
|
|
213
213
|
FOREIGN KEY (cron_id) REFERENCES crons(cron_id) ON DELETE CASCADE
|
|
214
214
|
);
|
|
215
215
|
CREATE INDEX IF NOT EXISTS idx_cron_runs_cron ON cron_runs(cron_id, started);
|
|
216
|
-
`)}E(
|
|
217
|
-
`).includes(o))return o;n++}}E(
|
|
216
|
+
`)}E(Pe,"addCronsTables");function g(e){return`'${e.replace(/'/g,`'"'"'`)}'`}E(g,"shellSingleQuote");function Ye(e,t){let n=0;for(;;){const o=`${t}_${n}`;if(!e.split(`
|
|
217
|
+
`).includes(o))return o;n++}}E(Ye,"uniqueHeredocDelimiter");function Q(e,t,n){const o=Ye(t,n);return`${e} <<'${o}'
|
|
218
218
|
${t}
|
|
219
|
-
${o}`}E(Q,"heredocCommand");function
|
|
219
|
+
${o}`}E(Q,"heredocCommand");function Ke(e){if(typeof e!="string")return{};try{const t=JSON.parse(e);return typeof t!="object"||t===null||Array.isArray(t)?{}:t}catch{return{}}}E(Ke,"migrationPayload");function X(e,t,n){const o=e[t];if(typeof o!="string"||o.trim()==="")throw new Error(`trigger ${n} payload requires non-empty ${t}`);return o}E(X,"requiredString");function H(e,t){const n=e[t];return typeof n=="string"&&n.trim()!==""?n:void 0}E(H,"optionalString");function w(e,t){const n=e[t];return typeof n=="string"&&n.trim()!==""?n:void 0}E(w,"optionalColumn");function z(e,t,n,o){return Q(`crtr node message send --to ${g(e)} --tier ${g(t)}`,n,`CRTR_MIGRATED_${o.replace(/[^A-Za-z0-9]/g,"_")}`)}E(z,"migratedMessageCommand");function qe(e,t){return`printf '%s\\n' "$CRTR_GATE_STDOUT" | crtr node message send --to ${g(e)} --tier ${g(t)}`}E(qe,"migratedGateStdoutMessageCommand");function $e(e,t){const n=[`crtr node new --kind ${g(X(e,"kind",t))}`],o=[["mode","mode"],["cwd","cwd"],["name","name"],["parent","parent"],["worktreeBase","worktree"],["forkFrom","fork-from"],["model","model"],["profile","profile"],["situationalContext","situational-context"]];for(const[i,a]of o){const N=H(e,i);N!==void 0&&n.push(`--${a} ${g(N)}`)}e.root===!0&&n.push("--root");const T=X(e,"prompt",t);return Q(n.join(" "),T,`CRTR_MIGRATED_${t.replace(/[^A-Za-z0-9]/g,"_")}`)}E($e,"migratedSpawnCommand");const V="migration.v22.skipped";function Je(e){const t=e.prepare("SELECT cwd FROM nodes WHERE parent IS NULL ORDER BY created LIMIT 1").get();return(t!==void 0?w(t,"cwd"):void 0)??process.cwd()}E(Je,"ambientMigrationCwd");function Qe(e){let t;try{t=e.prepare("SELECT eval_pid FROM triggers WHERE eval_state = 'running' AND eval_pid IS NOT NULL").all()}catch{return}for(const n of t){const o=n.eval_pid;if(!(typeof o!="number"||!Number.isInteger(o)||o<=1))try{process.kill(-o,"SIGTERM")}catch{}}}E(Qe,"reapRunningPredicateEvaluations");function ze(e){const t=[],n=e.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'triggers'").get()!==void 0;n&&Qe(e);let o=[];if(n)try{o=e.prepare("SELECT * FROM triggers ORDER BY created, trigger_id").all()}catch(a){t.push({trigger_id:"<unreadable table>",reason:`triggers table could not be read: ${a instanceof Error?a.message:String(a)}`})}const T=Je(e),i=e.prepare(`
|
|
220
220
|
INSERT OR REPLACE INTO crons (
|
|
221
221
|
cron_id, name, created_by, command, fire_at, recur, tz, expires_at,
|
|
222
222
|
anchor_node, cancel_on_wake, cwd, env_json, profile, scope,
|
|
223
223
|
run_timeout_s, overlap, on_output, sink, tier, created, updated
|
|
224
|
-
) VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, NULL, ?, 'profile', ?, 'skip', 'on-failure', '{}', ?, ?, ?)`);for(const a of o){const N=w(a,"trigger_id");if(N===void 0){t.push({trigger_id:"<unidentified row>",reason:"row carries no trigger_id"});continue}try{
|
|
224
|
+
) VALUES (?, ?, ?, ?, ?, ?, NULL, ?, ?, ?, ?, NULL, ?, 'profile', ?, 'skip', 'on-failure', '{}', ?, ?, ?)`);for(const a of o){const N=w(a,"trigger_id");if(N===void 0){t.push({trigger_id:"<unidentified row>",reason:"row carries no trigger_id"});continue}try{Ve(e,i,a,N,T)}catch(L){t.push({trigger_id:N,reason:L instanceof Error?L.message:String(L)})}}t.length>0&&(e.prepare("INSERT INTO canvas_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(V,JSON.stringify(t)),console.warn(`[canvas migration v22] ${t.length} retired schedule(s) could not be converted to crons and were dropped; see canvas_meta['${V}']: `+t.map(a=>`${a.trigger_id} (${a.reason})`).join("; "))),e.exec(`
|
|
225
225
|
DROP INDEX IF EXISTS idx_triggers_fire;
|
|
226
226
|
DROP INDEX IF EXISTS idx_triggers_node;
|
|
227
227
|
DROP INDEX IF EXISTS idx_triggers_owner;
|
|
228
228
|
DROP INDEX IF EXISTS idx_triggers_eval;
|
|
229
229
|
DROP INDEX IF EXISTS idx_triggers_deadline;
|
|
230
230
|
DROP TABLE IF EXISTS triggers;
|
|
231
|
-
`)}E(
|
|
231
|
+
`)}E(ze,"migrateTriggersToCrons");function Ve(e,t,n,o,T){const i=X(n,"owner_id",o),a=e.prepare("SELECT cwd, profile_id FROM nodes WHERE node_id = ?").get(i),N=(a===void 0?void 0:w(a,"cwd"))??T,L=a===void 0?null:w(a,"profile_id")??null,l=w(n,"node_id")??null,r=X(n,"action",o),_=X(n,"action_mode",o),d=X(n,"trigger_type",o),u=Ke(n.payload),p=w(n,"created")??new Date().toISOString(),I=_==="deadline_message"?"urgent":H(u,"tier")??"normal",R=H(u,"label");let s,A,m=!1;if(r==="msg"&&(_==="inbox_message"||_==="deadline_message")){if(l===null)throw new Error("has no message target");const f=H(u,"body");f!==void 0?s=z(l,I,f,o):d==="when"?(m=!0,s=qe(l,I)):s=z(l,I,R??"Scheduled message (no body was recorded when this schedule was armed).",o),A=R??(_==="deadline_message"?"deadline":"scheduled message")}else if(r==="msg"&&_==="fresh_revive"){if(l===null)throw new Error("has no revive target");s=`crtr node lifecycle revive ${g(l)} --fresh`,A=R??"scheduled fresh revive"}else if(r==="new"&&_==="node_birth")s=$e(u,o),A=H(u,"name")??`scheduled ${X(u,"kind",o)} node`;else throw new Error(`has unsupported action ${r}/${_}`);let D,y,G=null;if(d==="when"){D=X(n,"next_eval_at",o),y=X(n,"eval_every",o),G=w(n,"timeout_at")??null;const f=X(n,"when_script",o);s=`${m?`if ! CRTR_GATE_STDOUT="$(
|
|
232
232
|
${f}
|
|
233
233
|
)"; then`:`if ! (
|
|
234
234
|
${f}
|
|
@@ -237,14 +237,14 @@ ${f}
|
|
|
237
237
|
fi
|
|
238
238
|
{
|
|
239
239
|
${s}
|
|
240
|
-
} && crtr cron cancel "$CRTR_CRON_ID"`}else if(d==="at"||d==="every"||d==="until")D=X(n,"fire_at",o),y=d==="every"?X(n,"recur",o):null;else throw new Error(`has unknown timing type ${d}`);if(_==="deadline_message"&&l!==null){const f=e.prepare("SELECT cron_id FROM crons WHERE anchor_node = ? AND cancel_on_wake = 1 AND cron_id <> ?").get(l,o);if(f!==void 0)throw new Error(`node ${l} already has a migrated deadline (${String(f.cron_id)}); only one is allowed`)}t.run(o,A,i,s,D,y,G,l,_==="deadline_message"?1:0,N,L,300,I,p,p)}E(
|
|
240
|
+
} && crtr cron cancel "$CRTR_CRON_ID"`}else if(d==="at"||d==="every"||d==="until")D=X(n,"fire_at",o),y=d==="every"?X(n,"recur",o):null;else throw new Error(`has unknown timing type ${d}`);if(_==="deadline_message"&&l!==null){const f=e.prepare("SELECT cron_id FROM crons WHERE anchor_node = ? AND cancel_on_wake = 1 AND cron_id <> ?").get(l,o);if(f!==void 0)throw new Error(`node ${l} already has a migrated deadline (${String(f.cron_id)}); only one is allowed`)}t.run(o,A,i,s,D,y,G,l,_==="deadline_message"?1:0,N,L,300,I,p,p)}E(Ve,"convertTriggerRow");function Ze(e){const t=b(e,"crons");t.size===0||t.has("run_pid_identity")||e.exec("ALTER TABLE crons ADD COLUMN run_pid_identity TEXT;")}E(Ze,"addCronRunPidIdentityColumn");function et(e){const t=b(e,"crons");t.size!==0&&(t.has("run_out_of_band")||e.exec("ALTER TABLE crons ADD COLUMN run_out_of_band INTEGER NOT NULL DEFAULT 0;"),t.has("queued")&&e.exec("ALTER TABLE crons DROP COLUMN queued;"))}E(et,"addCronOutOfBandColumn");function tt(e){e.exec(`
|
|
241
241
|
CREATE TABLE IF NOT EXISTS warm_pool (
|
|
242
242
|
node_id TEXT PRIMARY KEY REFERENCES nodes(node_id) ON DELETE CASCADE,
|
|
243
243
|
recipe_key TEXT NOT NULL, -- frozen launch tuple this spare was minted for
|
|
244
244
|
created TEXT NOT NULL -- ISO 8601, mint time (oldest spare claimed first)
|
|
245
245
|
);
|
|
246
246
|
CREATE INDEX IF NOT EXISTS idx_warm_pool_recipe ON warm_pool(recipe_key, created);
|
|
247
|
-
`)}E(
|
|
247
|
+
`)}E(tt,"addWarmPoolTable");function nt(e){const t=e.prepare("SELECT node_id, dir, root, request_id, work_kind, status, final_report, updated_at FROM consult_outbox").all();e.exec(`
|
|
248
248
|
CREATE TABLE consult_outbox_new (
|
|
249
249
|
node_id TEXT PRIMARY KEY,
|
|
250
250
|
ticket_node_id TEXT NOT NULL,
|
|
@@ -256,7 +256,7 @@ CREATE TABLE consult_outbox_new (
|
|
|
256
256
|
);
|
|
257
257
|
`);const n=e.prepare(`
|
|
258
258
|
INSERT INTO consult_outbox_new (node_id, ticket_node_id, request_id, work_kind, status, final_report, updated_at)
|
|
259
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`);let o=0;for(const T of t){const i=
|
|
259
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`);let o=0;for(const T of t){const i=Re(T.dir);if(!(T.root===Y(T.root)&&T.dir===j(T.root,i))||!Ie(i)){o+=1;continue}n.run(T.node_id,i,T.request_id,T.work_kind,T.status,T.final_report,T.updated_at)}o>0&&console.warn(`[canvas migration v26] dropped ${o} consult_outbox row(s) whose legacy ticket path cannot safely map to a ticket node id`),e.exec(`
|
|
260
260
|
DROP TABLE consult_outbox;
|
|
261
261
|
ALTER TABLE consult_outbox_new RENAME TO consult_outbox;
|
|
262
262
|
|
|
@@ -265,7 +265,7 @@ CREATE INDEX IF NOT EXISTS idx_consult_outbox_human_work_status ON consult_outbo
|
|
|
265
265
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_ticket_status ON consult_outbox(ticket_node_id, work_kind, status);
|
|
266
266
|
CREATE INDEX IF NOT EXISTS idx_consult_outbox_ticket_request_status ON consult_outbox(ticket_node_id, request_id, work_kind, status);
|
|
267
267
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_consult_outbox_visual_request ON consult_outbox(ticket_node_id, request_id, work_kind) WHERE work_kind = 'visual';
|
|
268
|
-
`)}E(
|
|
268
|
+
`)}E(nt,"narrowHumanWorkOutbox");function Et(e){e.exec(`
|
|
269
269
|
CREATE TABLE IF NOT EXISTS reviews (
|
|
270
270
|
review_id TEXT PRIMARY KEY,
|
|
271
271
|
idempotency_key TEXT NOT NULL UNIQUE,
|
|
@@ -309,7 +309,7 @@ CREATE INDEX IF NOT EXISTS idx_reviews_state ON reviews(state);
|
|
|
309
309
|
CREATE INDEX IF NOT EXISTS idx_reviews_origin ON reviews(origin_node_id);
|
|
310
310
|
CREATE INDEX IF NOT EXISTS idx_reviews_file ON reviews(file);
|
|
311
311
|
CREATE INDEX IF NOT EXISTS idx_reviews_bridge ON reviews(bridge_node_id);
|
|
312
|
-
`)}E(
|
|
312
|
+
`)}E(Et,"addReviewsTable");function ot(e){e.exec(`
|
|
313
313
|
CREATE TABLE IF NOT EXISTS review_comments (
|
|
314
314
|
comment_id TEXT PRIMARY KEY,
|
|
315
315
|
review_id TEXT NOT NULL,
|
|
@@ -354,13 +354,13 @@ CREATE INDEX IF NOT EXISTS idx_review_comment_events_review_seq
|
|
|
354
354
|
ON review_comment_events(review_id, seq);
|
|
355
355
|
CREATE INDEX IF NOT EXISTS idx_review_comment_events_comment_seq
|
|
356
356
|
ON review_comment_events(comment_id, seq);
|
|
357
|
-
`)}E(
|
|
357
|
+
`)}E(ot,"addReviewCommentTables");function it(e){e.exec(`
|
|
358
358
|
ALTER TABLE review_comments
|
|
359
359
|
ADD COLUMN anchor_state TEXT NOT NULL DEFAULT 'live' CHECK (anchor_state IN ('live', 'detached'));
|
|
360
|
-
`)}E(
|
|
360
|
+
`)}E(it,"addReviewCommentAnchorState");function st(e){e.exec("DROP TABLE IF EXISTS consult_outbox")}E(st,"dropConsultOutbox");function rt(e){e.exec("ALTER TABLE reviews ADD COLUMN submit_requested_at TEXT")}E(rt,"addReviewSubmitRequested");function Tt(e){e.exec(`
|
|
361
361
|
ALTER TABLE crons ADD COLUMN held INTEGER NOT NULL DEFAULT 0;
|
|
362
362
|
ALTER TABLE crons ADD COLUMN last_poke_at TEXT;
|
|
363
|
-
`)}E(
|
|
363
|
+
`)}E(Tt,"addCronHeldColumns");function at(e){e.exec(`
|
|
364
364
|
CREATE TABLE human_action_deliveries (
|
|
365
365
|
request_id TEXT PRIMARY KEY,
|
|
366
366
|
state TEXT NOT NULL,
|
|
@@ -378,7 +378,7 @@ CREATE TABLE human_action_deliveries (
|
|
|
378
378
|
updated_at INTEGER NOT NULL
|
|
379
379
|
);
|
|
380
380
|
CREATE INDEX idx_human_action_deliveries_due ON human_action_deliveries(state, next_attempt_at);
|
|
381
|
-
`)}E(
|
|
381
|
+
`)}E(at,"addHumanActionDeliveries");function dt(e){const t=O(e);c(e,t,"pi_session_file","ALTER TABLE nodes ADD COLUMN pi_session_file TEXT;");for(const n of k(h(),{withFileTypes:!0}))if(n.isDirectory())try{const o=JSON.parse(M(x(n.name),"utf8"));typeof o.pi_session_file=="string"&&e.prepare("UPDATE nodes SET pi_session_file = ? WHERE node_id = ?").run(o.pi_session_file,n.name)}catch{}e.exec("CREATE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);")}E(dt,"addNodeSessionFileIndex");function Z(e,t){const n=E(N=>{const L=Y(N);return U(L)?ue(L):L},"canonical"),o=e.prepare("UPDATE nodes SET pi_session_file = ? WHERE node_id = ?");e.exec("DROP INDEX IF EXISTS idx_nodes_pi_session_file;");const T=e.prepare("SELECT node_id FROM nodes ORDER BY created ASC, node_id ASC").all();for(const{node_id:N}of T){const L=x(N);if(K(L)===null){o.run(null,N);continue}const l=Xe(L,()=>{const r=K(L);if(r===null)throw new Error(`node ${N} session metadata changed during ownership migration`);if(typeof r.pi_session_file!="string")return null;const _=r.pi_session_file,d=n(_);return d!==_&&(r.pi_session_file=d,Ue(L,r)),d});o.run(l,N)}const i=e.prepare(`
|
|
382
382
|
SELECT pi_session_file
|
|
383
383
|
FROM nodes
|
|
384
384
|
WHERE pi_session_file IS NOT NULL
|
|
@@ -389,7 +389,7 @@ SELECT node_id
|
|
|
389
389
|
FROM nodes
|
|
390
390
|
WHERE pi_session_file = ?
|
|
391
391
|
ORDER BY created ASC, node_id ASC
|
|
392
|
-
`).all(N);for(const{node_id:l}of L.slice(1)){if(!q(x(l),_=>{const d=_.pi_session_file;if(typeof d!="string"||n(d)!==N)throw new Error(`node ${l} session metadata changed during ownership migration`);return _.pi_session_id=null,_.pi_session_file=null,!0}))throw new Error(`node ${l} session metadata changed during ownership migration`);a.run(l)}}e.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);")}E(Z,"reconcileNodeSessionFileOwners");function
|
|
392
|
+
`).all(N);for(const{node_id:l}of L.slice(1)){if(!q(x(l),_=>{const d=_.pi_session_file;if(typeof d!="string"||n(d)!==N)throw new Error(`node ${l} session metadata changed during ownership migration`);return _.pi_session_id=null,_.pi_session_file=null,!0}))throw new Error(`node ${l} session metadata changed during ownership migration`);a.run(l)}}e.exec("CREATE UNIQUE INDEX IF NOT EXISTS idx_nodes_pi_session_file ON nodes(pi_session_file);")}E(Z,"reconcileNodeSessionFileOwners");function _t(e){Z(e,!0)}E(_t,"makeNodeSessionFileUnique");function Nt(e){Z(e,!1)}E(Nt,"repairNodeSessionFileUniqueIndex");function ct(e){const t=O(e);c(e,t,"frozen_at","ALTER TABLE nodes ADD COLUMN frozen_at TEXT;"),c(e,t,"respawn_failures","ALTER TABLE nodes ADD COLUMN respawn_failures INTEGER NOT NULL DEFAULT 0;"),c(e,t,"respawn_not_before","ALTER TABLE nodes ADD COLUMN respawn_not_before TEXT;"),c(e,t,"launched_at","ALTER TABLE nodes ADD COLUMN launched_at TEXT;"),c(e,t,"last_settled_at","ALTER TABLE nodes ADD COLUMN last_settled_at TEXT;"),c(e,t,"terminal_reason","ALTER TABLE nodes ADD COLUMN terminal_reason TEXT;"),e.exec(`
|
|
393
393
|
UPDATE nodes SET terminal_reason = CASE
|
|
394
394
|
WHEN status = 'done' AND final_report IS NOT NULL THEN 'finalized'
|
|
395
395
|
WHEN status = 'done' AND intent = 'parked' THEN 'parked'
|
|
@@ -398,7 +398,7 @@ UPDATE nodes SET terminal_reason = CASE
|
|
|
398
398
|
WHEN status = 'dead' THEN 'crashed'
|
|
399
399
|
END
|
|
400
400
|
WHERE status IN ('done','canceled','dead') AND terminal_reason IS NULL;
|
|
401
|
-
`)}E(
|
|
401
|
+
`)}E(ct,"addNodeLifecycleColumns");function Lt(e){const t=O(e);c(e,t,"description","ALTER TABLE nodes ADD COLUMN description TEXT;"),c(e,t,"cycles","ALTER TABLE nodes ADD COLUMN cycles INTEGER;");const n=h();if(!U(n))return;const o=e.prepare("UPDATE nodes SET description = ?, cycles = ? WHERE node_id = ?");for(const T of k(n)){const i=x(T);if(!U(i))continue;let a;try{a=JSON.parse(M(i,"utf8"))}catch{continue}const N=typeof a.description=="string"?a.description:null,L=typeof a.cycles=="number"?a.cycles:null;o.run(N,L,T)}}E(Lt,"addNodeLabelColumns");function lt(e){const t=O(e);c(e,t,"outcome_kind","ALTER TABLE nodes ADD COLUMN outcome_kind TEXT;"),c(e,t,"outcome_reason","ALTER TABLE nodes ADD COLUMN outcome_reason TEXT;"),c(e,t,"outcome_at","ALTER TABLE nodes ADD COLUMN outcome_at TEXT;"),c(e,t,"outcome_revision","ALTER TABLE nodes ADD COLUMN outcome_revision INTEGER NOT NULL DEFAULT 0;"),c(e,t,"outcome_detail_json","ALTER TABLE nodes ADD COLUMN outcome_detail_json TEXT;"),c(e,t,"deadline_at","ALTER TABLE nodes ADD COLUMN deadline_at TEXT;"),e.exec(`
|
|
402
402
|
CREATE TABLE IF NOT EXISTS node_outcome_deliveries (
|
|
403
403
|
node_id TEXT PRIMARY KEY,
|
|
404
404
|
state TEXT NOT NULL,
|
|
@@ -431,17 +431,17 @@ UPDATE nodes SET outcome_kind='failure',
|
|
|
431
431
|
outcome_reason=COALESCE(terminal_reason,'crashed'),
|
|
432
432
|
outcome_at=COALESCE(finalized_at, created), outcome_revision=1
|
|
433
433
|
WHERE final_report IS NULL AND status IN ('done','dead','canceled') AND outcome_at IS NULL;
|
|
434
|
-
`)}E(
|
|
434
|
+
`)}E(lt,"addNodeOutcomeAndDeadline");function ut(e){const t=O(e);c(e,t,"telemetry_tokens_in","ALTER TABLE nodes ADD COLUMN telemetry_tokens_in INTEGER;"),c(e,t,"telemetry_context_tokens","ALTER TABLE nodes ADD COLUMN telemetry_context_tokens INTEGER;"),c(e,t,"telemetry_last_activity","ALTER TABLE nodes ADD COLUMN telemetry_last_activity TEXT;"),c(e,t,"telemetry_updated_at","ALTER TABLE nodes ADD COLUMN telemetry_updated_at TEXT;");const n=e.prepare(`UPDATE nodes
|
|
435
435
|
SET telemetry_tokens_in = ?, telemetry_context_tokens = ?,
|
|
436
436
|
telemetry_last_activity = ?, telemetry_updated_at = ?
|
|
437
|
-
WHERE node_id = ?`);for(const{node_id:o}of e.prepare("SELECT node_id FROM nodes").all()){const T=j(h(),o,"job","telemetry.json");if(U(T))try{const i=JSON.parse(M(T,"utf8")),a=typeof i.tokens_in=="number"&&Number.isFinite(i.tokens_in)?i.tokens_in:null,N=typeof i.context_tokens=="number"&&Number.isFinite(i.context_tokens)?i.context_tokens:null,L=typeof i.last_activity=="string"?i.last_activity:null,l=
|
|
437
|
+
WHERE node_id = ?`);for(const{node_id:o}of e.prepare("SELECT node_id FROM nodes").all()){const T=j(h(),o,"job","telemetry.json");if(U(T))try{const i=JSON.parse(M(T,"utf8")),a=typeof i.tokens_in=="number"&&Number.isFinite(i.tokens_in)?i.tokens_in:null,N=typeof i.context_tokens=="number"&&Number.isFinite(i.context_tokens)?i.context_tokens:null,L=typeof i.last_activity=="string"?i.last_activity:null,l=pe(T).mtime.toISOString();n.run(a,N,L,l,o)}catch{}}}E(ut,"addNodeTelemetryColumns");function pt(e){e.exec(`
|
|
438
438
|
CREATE INDEX IF NOT EXISTS deliveries_external_due_order
|
|
439
439
|
ON deliveries(state, next_attempt_at, created_at, delivery_id)
|
|
440
440
|
WHERE destination_kind = 'external_action' AND canceled_at IS NULL;
|
|
441
441
|
DROP INDEX IF EXISTS deliveries_external_due;
|
|
442
442
|
CREATE INDEX IF NOT EXISTS idx_nodes_parent_created ON nodes(parent, created);
|
|
443
443
|
CREATE INDEX IF NOT EXISTS idx_edges_type_created ON edges(type, created);
|
|
444
|
-
`)}E(
|
|
444
|
+
`)}E(pt,"addRecurringLaneIndexes");function Rt(e){_e(e)}E(Rt,"projectLiveNodeState");function At(e){c(e,O(e),"scopes_json","ALTER TABLE nodes ADD COLUMN scopes_json TEXT;")}E(At,"addNodeScopes");function Ot(e){const t=O(e);c(e,t,"grantee","ALTER TABLE nodes ADD COLUMN grantee TEXT NOT NULL DEFAULT 'user:local';"),c(e,t,"sub","ALTER TABLE nodes ADD COLUMN sub TEXT;"),c(e,t,"grant_id","ALTER TABLE nodes ADD COLUMN grant_id TEXT;"),c(e,t,"client_id","ALTER TABLE nodes ADD COLUMN client_id TEXT;"),c(e,t,"metadata_json","ALTER TABLE nodes ADD COLUMN metadata_json TEXT;"),c(e,t,"run_id","ALTER TABLE nodes ADD COLUMN run_id TEXT;"),c(e,t,"event_cursor","ALTER TABLE nodes ADD COLUMN event_cursor INTEGER;"),c(e,t,"usage_json","ALTER TABLE nodes ADD COLUMN usage_json TEXT;"),c(e,t,"deleting_at","ALTER TABLE nodes ADD COLUMN deleting_at TEXT;"),e.exec(`
|
|
445
445
|
ALTER TABLE sessions ADD COLUMN event_offset INTEGER;
|
|
446
446
|
CREATE INDEX idx_nodes_run_id ON nodes(run_id);
|
|
447
447
|
-- Child nodes may be pruned while their root run and event history remain.
|
|
@@ -466,10 +466,10 @@ CREATE INDEX IF NOT EXISTS idx_edges_type_created ON edges(type, created);
|
|
|
466
466
|
PRIMARY KEY (grantee, key)
|
|
467
467
|
);
|
|
468
468
|
CREATE INDEX idx_idempotency_keys_created_at ON idempotency_keys(created_at);
|
|
469
|
-
`)}E(
|
|
469
|
+
`)}E(Ot,"addRunFoundation");function It(e){e.exec("ALTER TABLE nodes ADD COLUMN last_activity_at TEXT;"),e.exec(`UPDATE nodes SET last_activity_at = (
|
|
470
470
|
SELECT e.ts FROM run_events e WHERE e.run_id = nodes.node_id
|
|
471
471
|
ORDER BY e.sequence_number DESC LIMIT 1
|
|
472
|
-
) WHERE run_id = node_id`)}E(
|
|
472
|
+
) WHERE run_id = node_id`)}E(It,"addRunLastActivity");function mt(e){e.exec(`
|
|
473
473
|
CREATE TABLE structured_requests (
|
|
474
474
|
request_id TEXT PRIMARY KEY,
|
|
475
475
|
node_id TEXT REFERENCES nodes(node_id) ON DELETE CASCADE,
|
|
@@ -494,7 +494,7 @@ CREATE TABLE structured_requests (
|
|
|
494
494
|
CREATE UNIQUE INDEX idx_structured_requests_one_pending_node ON structured_requests(node_id) WHERE state = 'pending';
|
|
495
495
|
CREATE INDEX idx_structured_requests_expiry ON structured_requests(state, deadline_at);
|
|
496
496
|
CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done_at) WHERE effects_done_at IS NULL;
|
|
497
|
-
`)}E(
|
|
497
|
+
`)}E(mt,"addStructuredRequests");function ft(e){e.exec(`
|
|
498
498
|
CREATE TABLE grants (
|
|
499
499
|
id TEXT PRIMARY KEY, runtime TEXT NOT NULL, sub TEXT NOT NULL, grantee TEXT NOT NULL,
|
|
500
500
|
rev INTEGER NOT NULL, scopes_json TEXT NOT NULL, resolved_scopes_json TEXT NOT NULL,
|
|
@@ -503,13 +503,13 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
503
503
|
);
|
|
504
504
|
CREATE UNIQUE INDEX idx_grants_grantee_sub ON grants(grantee, sub);
|
|
505
505
|
CREATE TABLE runtime_state (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
|
506
|
-
`)}E(
|
|
506
|
+
`)}E(ft,"addGrantMirror");function Xt(e){e.exec("UPDATE nodes SET grantee = 'app:terminal' WHERE grantee = 'user:local';")}E(Xt,"migrateTerminalGrantees");function Ut(e){const t={llm:"crtr:llm",act:"crtr:act",schedule:"crtr:schedule","memory:read":"crtr:memory:read:user","memory:write":"crtr:memory:rw:user"},n=e.prepare("SELECT node_id, scopes_json FROM nodes WHERE scopes_json IS NOT NULL").all(),o=e.prepare("UPDATE nodes SET scopes_json = ? WHERE node_id = ?");for(const T of n){const i=JSON.parse(T.scopes_json);if(!Array.isArray(i))throw new Error(`invalid launched scopes for node ${T.node_id}`);const a=[];for(const N of i){if(typeof N!="string")continue;const L=t[N]??N;try{ae(L),a.push(L)}catch(l){if(!(l instanceof TypeError))throw l}}o.run(JSON.stringify([...new Set(a)]),T.node_id)}}E(Ut,"migrateLaunchedScopes");function Ct(e){e.exec(`CREATE TABLE node_tokens (
|
|
507
507
|
broker_process_id TEXT PRIMARY KEY REFERENCES executions(execution_id) ON DELETE CASCADE,
|
|
508
508
|
node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
|
|
509
509
|
token_hash TEXT NOT NULL UNIQUE, grantee TEXT NOT NULL, uid INTEGER NOT NULL,
|
|
510
510
|
sub TEXT NOT NULL, grant TEXT, client_id TEXT, scopes_json TEXT,
|
|
511
511
|
issued_at TEXT NOT NULL, revoked_at TEXT
|
|
512
|
-
);`)}E(
|
|
512
|
+
);`)}E(Ct,"addNodeTokens");function C(e,t){return e.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(t)!==void 0}E(C,"hasTable");function S(e,t,n){return e.prepare(`PRAGMA table_info(${t})`).all().some(o=>o.name===n)}E(S,"hasColumn");function ee(e){C(e,"memory_reads")||e.exec(`CREATE TABLE memory_reads (
|
|
513
513
|
node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
|
|
514
514
|
name TEXT NOT NULL,
|
|
515
515
|
scope TEXT NOT NULL,
|
|
@@ -523,7 +523,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
523
523
|
created TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ('active', 'retired'))
|
|
524
524
|
);`)}E(ne,"addAppUsers");function Ee(e){S(e,"crons","grantee")||(e.exec("ALTER TABLE crons ADD COLUMN grantee TEXT NOT NULL DEFAULT 'app:terminal'"),e.exec(`UPDATE crons SET grantee = COALESCE(
|
|
525
525
|
(SELECT COALESCE(nodes.grantee, 'app:terminal') FROM nodes WHERE nodes.node_id = crons.anchor_node),
|
|
526
|
-
'app:terminal') WHERE anchor_node IS NOT NULL`))}E(Ee,"addCronGrantee");function
|
|
526
|
+
'app:terminal') WHERE anchor_node IS NOT NULL`))}E(Ee,"addCronGrantee");function St(e){ee(e),te(e),ne(e),Ee(e)}E(St,"convergeSharedVersions");function Dt(e){(e.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'app_users'").get()?.sql??"").includes("'removed'")||e.exec(`CREATE TABLE app_users_new (
|
|
527
527
|
uid INTEGER PRIMARY KEY, app TEXT NOT NULL,
|
|
528
528
|
created TEXT NOT NULL, status TEXT NOT NULL CHECK (status IN ('active', 'removed')),
|
|
529
529
|
removed_at TEXT
|
|
@@ -533,7 +533,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
533
533
|
CASE status WHEN 'active' THEN NULL ELSE created END FROM app_users;
|
|
534
534
|
DROP TABLE app_users;
|
|
535
535
|
ALTER TABLE app_users_new RENAME TO app_users;
|
|
536
|
-
CREATE UNIQUE INDEX idx_app_users_active_app ON app_users(app) WHERE status = 'active';`)}E(
|
|
536
|
+
CREATE UNIQUE INDEX idx_app_users_active_app ON app_users(app) WHERE status = 'active';`)}E(Dt,"appUsersRemoved");function oe(e){C(e,"objects")||(e.exec(`
|
|
537
537
|
CREATE TABLE objects (
|
|
538
538
|
rowid INTEGER PRIMARY KEY,
|
|
539
539
|
id TEXT NOT NULL UNIQUE,
|
|
@@ -659,7 +659,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
659
659
|
WHERE NOT EXISTS (SELECT 1 FROM identities i WHERE i.external_id = n.grantee AND i.scheme IN ('app', 'local-user'))`).all();for(const{grantee:i}of T){let a;if(i==="user:local")a={type:"person",scheme:"local-user",handle:"user"};else if(/^app:[a-z0-9][a-z0-9._-]*$/.test(i))a={type:"app",scheme:"app",handle:i.slice(4)};else throw new Error(`canvas objects migration: node grantee ${JSON.stringify(i)} is neither app:<name> nor user:local`);const{rowid:N}=n.get(`${Date.now().toString(36)}-${P(4).toString("hex")}`,a.type,a.handle,t);o.run(N,a.scheme,i)}e.exec(`INSERT INTO objects (id, type, kind, owner, privacy, name, preview, created_at)
|
|
660
660
|
SELECT n.node_id, 'node', n.kind, i.object, 'public', n.name, '', n.created
|
|
661
661
|
FROM nodes n JOIN identities i ON i.external_id = n.grantee AND i.scheme IN ('app', 'local-user')
|
|
662
|
-
WHERE NOT EXISTS (SELECT 1 FROM objects o WHERE o.id = n.node_id)`)}E(ie,"copyNodesIntoObjects");function
|
|
662
|
+
WHERE NOT EXISTS (SELECT 1 FROM objects o WHERE o.id = n.node_id)`)}E(ie,"copyNodesIntoObjects");function gt(e){if(oe(e),!S(e,"nodes","name"))return;ie(e);const t=e.prepare("SELECT COUNT(*) AS n FROM nodes WHERE node_id NOT IN (SELECT id FROM objects)").get();if(t.n>0)throw new Error(`canvasObjectsCut: ${t.n} node(s) have no objects row`);e.exec(`
|
|
663
663
|
ALTER TABLE edges RENAME TO legacy_edges;
|
|
664
664
|
CREATE TABLE edges (
|
|
665
665
|
type TEXT NOT NULL CHECK (type IN ('acted-on','links-to')),
|
|
@@ -707,7 +707,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
707
707
|
LEFT JOIN sessions s ON s.node_id = n.node_id AND s.unbound_at IS NULL
|
|
708
708
|
LEFT JOIN node_faults f ON f.node_id = n.node_id AND f.active = 1;
|
|
709
709
|
ANALYZE;
|
|
710
|
-
`)}E(
|
|
710
|
+
`)}E(gt,"canvasObjectsCut");function wt(e){e.exec(`CREATE TABLE IF NOT EXISTS uploads (
|
|
711
711
|
upload_id TEXT PRIMARY KEY, grantee TEXT NOT NULL, name TEXT NOT NULL, media_type TEXT NOT NULL,
|
|
712
712
|
size INTEGER NOT NULL, created_at TEXT NOT NULL, delete_pending_since TEXT
|
|
713
713
|
);
|
|
@@ -723,7 +723,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
723
723
|
url TEXT NOT NULL, created_by TEXT NOT NULL, created_at TEXT NOT NULL, expires_at TEXT,
|
|
724
724
|
status TEXT NOT NULL CHECK (status IN ('on', 'off')), object_deleted_at TEXT
|
|
725
725
|
);
|
|
726
|
-
CREATE INDEX IF NOT EXISTS idx_shares_created_by ON shares(created_by);`)}E(
|
|
726
|
+
CREATE INDEX IF NOT EXISTS idx_shares_created_by ON shares(created_by);`)}E(wt,"addFileTransfer");function yt(e){e.exec(`CREATE TABLE IF NOT EXISTS daemon_actions (
|
|
727
727
|
action_id TEXT PRIMARY KEY,
|
|
728
728
|
adapter_json TEXT NOT NULL,
|
|
729
729
|
body_json TEXT,
|
|
@@ -762,7 +762,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
762
762
|
DELETE FROM messages
|
|
763
763
|
WHERE message_id IN (SELECT message_id FROM moved_actions WHERE message_id IS NOT NULL)
|
|
764
764
|
AND NOT EXISTS (SELECT 1 FROM deliveries d WHERE d.message_id = messages.message_id);
|
|
765
|
-
DROP TABLE moved_actions;`)}E(
|
|
765
|
+
DROP TABLE moved_actions;`)}E(yt,"moveDaemonActions");function vt(e){B(e,61,()=>{if(!C(e,"objects"))return;const t=e.prepare("SELECT COUNT(*) AS n FROM objects").get().n,n=e.prepare("SELECT name, sql FROM sqlite_master WHERE type = 'view' AND sql LIKE '%objects%'").all();for(const i of n)e.exec(`DROP VIEW "${i.name}"`);e.exec(`
|
|
766
766
|
CREATE TABLE objects_new (
|
|
767
767
|
rowid INTEGER PRIMARY KEY,
|
|
768
768
|
id TEXT NOT NULL UNIQUE,
|
|
@@ -783,10 +783,10 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
783
783
|
CREATE UNIQUE INDEX objects_handle ON objects(type, name) WHERE type IN ('person','app','repo','profile');
|
|
784
784
|
CREATE INDEX objects_owner_type ON objects(owner, type, kind);
|
|
785
785
|
CREATE INDEX objects_type_kind ON objects(type, kind);
|
|
786
|
-
`);for(const i of n)e.exec(i.sql);const o=e.prepare("SELECT COUNT(*) AS n FROM objects").get().n;if(o!==t)throw new Error(`objects rebuild lost rows: expected ${t}, got ${o}`);const T=e.prepare("PRAGMA foreign_key_check").all();if(T.length>0)throw new Error(`objects rebuild left ${T.length} foreign-key violation(s)`)})}E(
|
|
786
|
+
`);for(const i of n)e.exec(i.sql);const o=e.prepare("SELECT COUNT(*) AS n FROM objects").get().n;if(o!==t)throw new Error(`objects rebuild lost rows: expected ${t}, got ${o}`);const T=e.prepare("PRAGMA foreign_key_check").all();if(T.length>0)throw new Error(`objects rebuild left ${T.length} foreign-key violation(s)`)})}E(vt,"objectsWithoutTypeCheck");function ht(e){C(e,"deliveries")&&e.exec(`
|
|
787
787
|
CREATE INDEX IF NOT EXISTS deliveries_destination ON deliveries(destination_id, wake, state, created_at);
|
|
788
788
|
DROP INDEX IF EXISTS deliveries_node_inbox;
|
|
789
|
-
`)}E(
|
|
789
|
+
`)}E(ht,"addDeliveriesDestinationIndex");function xt(e){e.exec(`
|
|
790
790
|
CREATE TABLE object_kinds (
|
|
791
791
|
name TEXT PRIMARY KEY, owner INTEGER NOT NULL REFERENCES objects(rowid),
|
|
792
792
|
description TEXT NOT NULL DEFAULT '', events_json TEXT NOT NULL,
|
|
@@ -809,14 +809,14 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
809
809
|
DROP INDEX objects_doc_name;
|
|
810
810
|
CREATE UNIQUE INDEX objects_named ON objects(owner, name)
|
|
811
811
|
WHERE deleted_at IS NULL AND (type = 'document' OR instr(type, '.') > 0);
|
|
812
|
-
`)}E(
|
|
812
|
+
`)}E(xt,"customObjectKinds");function Ft(e){e.exec("CREATE INDEX IF NOT EXISTS revision_parents_parent ON revision_parents(parent);")}E(Ft,"addRevisionParentsParentIndex");function Mt(e){C(e,"run_events")&&e.exec(`
|
|
813
813
|
DELETE FROM run_events
|
|
814
814
|
WHERE type IN ('node.thinking.delta', 'node.output_text.delta')
|
|
815
815
|
AND EXISTS (SELECT 1 FROM run_events done
|
|
816
816
|
WHERE done.run_id = run_events.run_id AND done.node_id = run_events.node_id
|
|
817
817
|
AND done.type IN ('node.thinking.done', 'node.output_text.done', 'usage')
|
|
818
818
|
AND done.sequence_number > run_events.sequence_number);
|
|
819
|
-
`)}E(
|
|
819
|
+
`)}E(Mt,"dropSpentRunEventDeltas");function kt(e){e.exec("CREATE INDEX IF NOT EXISTS idx_run_events_node_type ON run_events(run_id, node_id, type);")}E(kt,"addRunEventsNodeTypeIndex");function jt(e){B(e,63,()=>{if(!C(e,"objects")||!C(e,"deliveries"))return;if(S(e,"objects","reception")||e.exec("ALTER TABLE objects ADD COLUMN reception INTEGER NOT NULL DEFAULT 0"),S(e,"objects","listen_json")||e.exec("ALTER TABLE objects ADD COLUMN listen_json TEXT"),e.exec(`UPDATE objects SET reception = CASE WHEN n.status IN ('active','idle') THEN 2 ELSE 1 END
|
|
820
820
|
FROM nodes n WHERE n.node_id = objects.id`),S(e,"nodes","listen_default_json")){e.exec(`UPDATE objects SET listen_json = n.listen_default_json FROM nodes n
|
|
821
821
|
WHERE n.node_id = objects.id AND n.listen_default_json IS NOT NULL`);const n=e.prepare("SELECT name, sql FROM sqlite_master WHERE type = 'view' AND sql LIKE '%nodes%'").all();for(const o of n)e.exec(`DROP VIEW "${o.name}"`);e.exec("ALTER TABLE nodes DROP COLUMN listen_default_json");for(const o of n)e.exec(o.sql)}if(e.exec(`
|
|
822
822
|
DROP TRIGGER IF EXISTS nodes_reception_insert;
|
|
@@ -862,7 +862,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
862
862
|
CREATE INDEX deliveries_handed_off ON deliveries(claim_owner) WHERE state = 'handed_off';
|
|
863
863
|
CREATE INDEX deliveries_message ON deliveries(message_id);
|
|
864
864
|
CREATE UNIQUE INDEX deliveries_idempotency ON deliveries(idempotency_key) WHERE idempotency_key IS NOT NULL;
|
|
865
|
-
`);const o=e.prepare("SELECT COUNT(*) AS n FROM deliveries").get().n;if(o!==n)throw new Error(`deliveries rebuild lost rows: expected ${n}, got ${o}`)}const t=e.prepare("PRAGMA foreign_key_check").all();if(t.length>0)throw new Error(`v64 left ${t.length} foreign-key violation(s)`)})}E(
|
|
865
|
+
`);const o=e.prepare("SELECT COUNT(*) AS n FROM deliveries").get().n;if(o!==n)throw new Error(`deliveries rebuild lost rows: expected ${n}, got ${o}`)}const t=e.prepare("PRAGMA foreign_key_check").all();if(t.length>0)throw new Error(`v64 left ${t.length} foreign-key violation(s)`)})}E(jt,"nodesOnObjectInterface");function Ht(e){B(e,66,()=>{if(e.exec(`CREATE TABLE IF NOT EXISTS custom_objects (
|
|
866
866
|
object INTEGER PRIMARY KEY REFERENCES objects(rowid) ON DELETE CASCADE,
|
|
867
867
|
receiver_json TEXT,
|
|
868
868
|
payload_json TEXT
|
|
@@ -916,7 +916,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
916
916
|
CREATE INDEX deliveries_handed_off ON deliveries(claim_owner) WHERE state = 'handed_off';
|
|
917
917
|
CREATE INDEX deliveries_message ON deliveries(message_id);
|
|
918
918
|
CREATE UNIQUE INDEX deliveries_idempotency ON deliveries(idempotency_key) WHERE idempotency_key IS NOT NULL;
|
|
919
|
-
`);for(const s of I)e.exec(s.sql);const R=e.prepare("SELECT COUNT(*) AS n FROM deliveries").get().n;if(R!==p)throw new Error(`v67 deliveries rebuild lost rows: expected ${p}, got ${R}`)}const t=e.prepare("PRAGMA foreign_key_check").all();if(t.length)throw new Error(`v67 left ${t.length} foreign-key violation(s)`)})}E(
|
|
919
|
+
`);for(const s of I)e.exec(s.sql);const R=e.prepare("SELECT COUNT(*) AS n FROM deliveries").get().n;if(R!==p)throw new Error(`v67 deliveries rebuild lost rows: expected ${p}, got ${R}`)}const t=e.prepare("PRAGMA foreign_key_check").all();if(t.length)throw new Error(`v67 left ${t.length} foreign-key violation(s)`)})}E(Ht,"outcomeActionsOnObjects");function Gt(e){e.exec(`
|
|
920
920
|
CREATE TABLE IF NOT EXISTS create_requests (
|
|
921
921
|
request_key TEXT PRIMARY KEY,
|
|
922
922
|
node_id TEXT,
|
|
@@ -924,7 +924,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
924
924
|
state TEXT NOT NULL DEFAULT 'reserved' CHECK (state IN ('reserved', 'warm_claimed', 'complete'))
|
|
925
925
|
);
|
|
926
926
|
CREATE INDEX IF NOT EXISTS idx_create_requests_expires ON create_requests(expires_at);
|
|
927
|
-
`)}E(
|
|
927
|
+
`)}E(Gt,"addCreateRequests");function bt(e){for(const[r,_]of[["sub","TEXT"],["grant_id","TEXT"],["scopes_json","TEXT"],["origin_run_id","TEXT"],["run_execution_id","TEXT"]])S(e,"crons",r)||e.exec(`ALTER TABLE crons ADD COLUMN ${r} ${_}`);const t=[],n=E(r=>e.prepare("SELECT id, sub, resolved_scopes_json FROM grants WHERE grantee = ? ORDER BY CASE status WHEN 'active' THEN 0 ELSE 1 END, rev DESC LIMIT 1").get(r),"grantRow"),o=E(r=>e.prepare("SELECT grantee, sub, resolved_scopes_json FROM grants WHERE id = ?").get(r),"grantById"),T=E(r=>{try{const _=JSON.parse(r);return Array.isArray(_)?_.flatMap(d=>d&&d.mode==="always"&&typeof d.scope=="string"?[d.scope]:[]):[]}catch{return[]}},"snapshot"),i=E((r,_)=>{if(_===null)return null;const d=e.prepare("SELECT grantee, sub, grant_id, scopes_json, run_id FROM node_rows WHERE node_id = ?").get(_);if(d===void 0||d.grantee!==r)return null;const u=r==="app:terminal";let p=null;if(d.scopes_json!==null){let A;try{A=JSON.parse(d.scopes_json)}catch{return{pause:"bad_scopes"}}if(!Array.isArray(A)||!A.every(m=>typeof m=="string"))return{pause:"bad_scopes"};p=A}const I=u?n(r):void 0;let R=d.grant_id??I?.id??null;R===null&&!u&&d.sub!==null&&(R=e.prepare("SELECT id FROM grants WHERE grantee = ? AND sub = ? ORDER BY CASE status WHEN 'active' THEN 0 ELSE 1 END, rev DESC LIMIT 1").get(r,d.sub)?.id??null);const s=R===null?void 0:o(R);return R!==null&&(s===void 0||s.grantee!==r||d.sub!==null&&s.sub!==d.sub)?{pause:"no_grant"}:R===null&&!u?{pause:"no_grant"}:(p===null&&!u&&(p=T(s.resolved_scopes_json)),{principal:{sub:d.sub??s?.sub??I?.sub??"user:local",grant_id:R,scopes:p,origin_run_id:d.run_id}})},"fromCreator"),a=E(r=>{const _=n(r);return r==="app:terminal"?{sub:_?.sub??"user:local",grant_id:_?.id??null,scopes:null,origin_run_id:null}:_===void 0?null:{sub:_.sub,grant_id:_.id,scopes:T(_.resolved_scopes_json),origin_run_id:null}},"fromGrantee"),N=e.prepare("SELECT cron_id, grantee, created_by, isolated, cwd FROM crons WHERE sub IS NULL").all(),L=e.prepare("UPDATE crons SET sub = ?, grant_id = ?, scopes_json = ?, origin_run_id = ? WHERE cron_id = ?"),l=new Map;for(const r of N){const _=i(r.grantee,r.created_by),d=_===null?a(r.grantee):"principal"in _?_.principal:null;if(d===null){const u=_!==null&&"pause"in _?_.pause:"no_grant";L.run("user:local",null,null,null,r.cron_id),e.prepare("UPDATE crons SET state = 'paused' WHERE cron_id = ?").run(r.cron_id),t.push({event:`cron.migrated.${u}`,fields:{cron_id:r.cron_id,grantee:r.grantee}})}else l.set(r.cron_id,d),L.run(d.sub,d.grant_id,d.scopes===null?null:JSON.stringify(d.scopes),d.origin_run_id,r.cron_id);if(r.isolated!==0){const u=j(Ae(r.grantee).root,"crons",r.cron_id);e.prepare("UPDATE crons SET cwd = ? WHERE cron_id = ?").run(u,r.cron_id),t.push({event:"cron.migrated.isolated_cwd",fields:{cron_id:r.cron_id,old_cwd:r.cwd,cwd:u}})}}if(C(e,"daemon_actions")){const r=e.prepare("SELECT action_id, adapter_json FROM daemon_actions WHERE state IN ('pending', 'handed_off') AND json_extract(adapter_json, '$.adapter') = 'cron_spawn' AND json_extract(adapter_json, '$.principal') IS NULL").all();for(const _ of r){const d=JSON.parse(_.adapter_json),u=typeof d.cronId=="string"?d.cronId:null,p=u===null?void 0:e.prepare("SELECT grantee, sub, grant_id, scopes_json, origin_run_id, isolated, created_by FROM crons WHERE cron_id = ?").get(u);let I=null;if(p!==void 0&&!(p.grant_id===null&&p.grantee!=="app:terminal"))I={cron_id:u,grantee:p.grantee,sub:p.sub,grant_id:p.grant_id,scopes:p.scopes_json===null?null:JSON.parse(p.scopes_json),origin_run_id:p.origin_run_id,isolated:p.isolated!==0,creator:p.created_by};else if(p===void 0){const R=typeof d.parent=="string"?d.parent:null,s=R===null?void 0:e.prepare("SELECT grantee FROM node_rows WHERE node_id = ?").get(R),A=s===void 0?null:i(s.grantee,R),m=A===null||!("principal"in A)?null:A.principal;s!==void 0&&m!==null&&(I={cron_id:u,grantee:s.grantee,sub:m.sub,grant_id:m.grant_id,scopes:m.scopes,origin_run_id:m.origin_run_id,isolated:d.isolated===!0,creator:R})}if(I!==null)e.prepare("UPDATE daemon_actions SET adapter_json = ? WHERE action_id = ?").run(JSON.stringify({...d,principal:I}),_.action_id);else{const R=Date.now();e.prepare("UPDATE daemon_actions SET state = 'permanent_failed', failed_at = ?, updated_at = ?, claimed_at = NULL, claim_owner = NULL, next_attempt_at = NULL, last_failure_json = ? WHERE action_id = ?").run(R,R,JSON.stringify({kind:"grant_refused",message:"the cron has no derivable principal to authorize this birth"}),_.action_id),t.push({event:"cron.birth.refused",fields:{cron_id:u,action_id:_.action_id,reason:"grant_missing"}})}}}for(const{event:r,fields:_}of t)Oe({level:"warn",event:r,fields:_})}E(bt,"addCronPrincipal");const F=[Ce,Se,De,ge,we,ye,ve,xe,he,Fe,Me,ke,je,J,He,J,Ge,be,Be,We,Pe,ze,Ze,et,tt,nt,Et,ot,it,st,rt,Tt,at,dt,ct,_t,Nt,Lt,lt,e=>{de(e),ce(e),Le(e),le(e)},pt,Rt,ut,At,Ot,It,ft,Ct,Xt,Ut,ee,te,ne,Ee,St,Dt,oe,gt,me,wt,yt,vt,ht,jt,kt,Mt,Ht,Ft,e=>e.exec(`
|
|
928
928
|
CREATE TABLE audit_rows (
|
|
929
929
|
id INTEGER PRIMARY KEY, request_id TEXT NOT NULL, time TEXT NOT NULL, grantee TEXT, sub TEXT,
|
|
930
930
|
client_id TEXT, class TEXT, operation TEXT NOT NULL, scope_checked TEXT,
|
|
@@ -955,7 +955,7 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
955
955
|
ALTER TABLE audit_rows ADD COLUMN question_id TEXT;
|
|
956
956
|
ALTER TABLE provider_approvals ADD COLUMN call_shape TEXT;
|
|
957
957
|
ALTER TABLE provider_approvals ADD COLUMN prepared_files TEXT;
|
|
958
|
-
`),
|
|
958
|
+
`),xt,e=>e.exec(`
|
|
959
959
|
CREATE INDEX idx_human_requests_page_source ON human_requests(
|
|
960
960
|
CASE WHEN json_valid(specification_json) THEN json_extract(specification_json, '$.source.nodeId') END
|
|
961
961
|
) WHERE kind = 'page';
|
|
@@ -986,14 +986,14 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
986
986
|
-- that registered a custom object fixes the run that object reads from.
|
|
987
987
|
CREATE TABLE deleted_isolated_nodes (node_id TEXT PRIMARY KEY, run_id TEXT NOT NULL);
|
|
988
988
|
ALTER TABLE custom_objects ADD COLUMN registered_by TEXT;
|
|
989
|
-
`),
|
|
989
|
+
`),mt,Gt,e=>e.exec("ALTER TABLE grants ADD COLUMN dependencies_json TEXT NOT NULL DEFAULT '{}'"),e=>e.exec("ALTER TABLE grants ADD COLUMN run_notice_url TEXT"),e=>e.exec(`
|
|
990
990
|
CREATE TABLE usage (
|
|
991
991
|
grantee TEXT NOT NULL, month TEXT NOT NULL, payer TEXT NOT NULL, source TEXT NOT NULL, served_model TEXT NOT NULL,
|
|
992
992
|
input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0, cached_tokens INTEGER NOT NULL DEFAULT 0,
|
|
993
993
|
usd_equivalent REAL,
|
|
994
994
|
PRIMARY KEY (grantee, month, payer, source, served_model)
|
|
995
995
|
);
|
|
996
|
-
`),
|
|
996
|
+
`),bt,e=>e.exec("ALTER TABLE audit_rows ADD COLUMN via TEXT"),e=>e.exec(`
|
|
997
997
|
ALTER TABLE objects ADD COLUMN home_machine TEXT;
|
|
998
998
|
CREATE TABLE peer_stubs (
|
|
999
999
|
object INTEGER PRIMARY KEY REFERENCES objects(rowid) ON DELETE CASCADE,
|
|
@@ -1046,13 +1046,13 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
1046
1046
|
CREATE UNIQUE INDEX objects_named ON objects(owner, name)
|
|
1047
1047
|
WHERE deleted_at IS NULL AND home_machine IS NULL AND (type = 'document' OR instr(type, '.') > 0);
|
|
1048
1048
|
CREATE UNIQUE INDEX objects_handle ON objects(type, name) WHERE home_machine IS NULL AND type IN ('person','app','repo','profile');
|
|
1049
|
-
`),e=>e.exec("ALTER TABLE grants ADD COLUMN inference_endpoint_json TEXT"),e=>e.exec("ALTER TABLE nodes ADD COLUMN reply_at TEXT; ALTER TABLE nodes ADD COLUMN read_at TEXT;"),Bt,
|
|
1049
|
+
`),e=>e.exec("ALTER TABLE grants ADD COLUMN inference_endpoint_json TEXT"),e=>e.exec("ALTER TABLE nodes ADD COLUMN reply_at TEXT; ALTER TABLE nodes ADD COLUMN read_at TEXT;"),Wt,Bt,e=>e.exec(`
|
|
1050
1050
|
CREATE TABLE IF NOT EXISTS unresolved_links (
|
|
1051
1051
|
src INTEGER NOT NULL REFERENCES objects(rowid) ON DELETE CASCADE,
|
|
1052
1052
|
name TEXT NOT NULL,
|
|
1053
1053
|
PRIMARY KEY (src, name)
|
|
1054
1054
|
) WITHOUT ROWID;
|
|
1055
|
-
`)];function
|
|
1055
|
+
`),Ne];function Bt(e){e.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'waits'").get()===void 0&&e.exec("CREATE TABLE waits (wait_id TEXT PRIMARY KEY, node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE, kind TEXT, target_id TEXT, generation INTEGER, created_at INTEGER, satisfied_at INTEGER)");const t=e.prepare("SELECT COUNT(*) AS n FROM waits").get().n;e.exec(`
|
|
1056
1056
|
CREATE TABLE waits_new (
|
|
1057
1057
|
wait_id TEXT PRIMARY KEY,
|
|
1058
1058
|
node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
|
|
@@ -1070,4 +1070,4 @@ CREATE INDEX idx_structured_requests_effects ON structured_requests(effects_done
|
|
|
1070
1070
|
CREATE UNIQUE INDEX waits_live_object ON waits(node_id, target_id) WHERE satisfied_at IS NULL AND kind = 'object';
|
|
1071
1071
|
CREATE INDEX waits_target ON waits(target_id) WHERE satisfied_at IS NULL;
|
|
1072
1072
|
CREATE INDEX waits_node ON waits(node_id);
|
|
1073
|
-
`);const n=e.prepare("SELECT COUNT(*) AS n FROM waits").get().n;if(n!==t)throw new Error(`v96 waits rebuild lost rows: expected ${t}, got ${n}`)}E(
|
|
1073
|
+
`);const n=e.prepare("SELECT COUNT(*) AS n FROM waits").get().n;if(n!==t)throw new Error(`v96 waits rebuild lost rows: expected ${t}, got ${n}`)}E(Bt,"addObjectWaits");function Wt(e){const t=O(e);for(const n of["waiting_for","waiting_for_generation"])t.has(n)&&e.exec(`ALTER TABLE nodes DROP COLUMN ${n}`)}E(Wt,"dropNodeWaitColumns");function B(e,t,n){const o=e.prepare("PRAGMA foreign_keys").get().foreign_keys;e.exec("PRAGMA foreign_keys = OFF;");try{e.exec("BEGIN IMMEDIATE");try{W(e)===t&&(n(),e.exec(`PRAGMA user_version = ${t+1}`)),e.exec("COMMIT")}catch(T){try{e.exec("ROLLBACK")}catch{}throw T}}finally{e.exec(`PRAGMA foreign_keys = ${o};`)}}E(B,"selfTransactionalMigration");const Pt=new Set([3,61,63,66]);function W(e){return e.prepare("PRAGMA user_version").get().user_version}E(W,"currentUserVersion");function se(e,t){F[t](e),e.exec(`PRAGMA user_version = ${t+1}`)}E(se,"applyMigration");function Tn(e){const t=W(e);if(t>F.length)throw new Error(`unsupported canvas schema version ${t} (latest supported ${F.length})`);if(t===F.length)return;const n=e.prepare("PRAGMA busy_timeout").get().timeout;e.exec("PRAGMA busy_timeout = 60000;");try{Yt(e)}finally{e.exec(`PRAGMA busy_timeout = ${n};`)}}E(Tn,"migrate");function Yt(e){for(;;){const t=W(e);if(t>=F.length)return;if(Pt.has(t)){if(e.isTransaction)throw new Error(`canvas migration v${t+1} requires autocommit`);F[t](e);continue}if(e.isTransaction){se(e,t);continue}e.exec("BEGIN IMMEDIATE");try{W(e)===t&&se(e,t),e.exec("COMMIT")}catch(n){try{e.exec("ROLLBACK")}catch{}throw n}}}E(Yt,"runPendingMigrations");export{F as MIGRATIONS,Tn as migrate};
|
|
@@ -15,6 +15,8 @@ export interface FaultRecordInput {
|
|
|
15
15
|
}
|
|
16
16
|
export interface StoredFault extends Fault {
|
|
17
17
|
faultId: string;
|
|
18
|
+
/** The episode start (`since`) a fatal-fault notice was last published for; unset until one is sent. */
|
|
19
|
+
notifiedSince?: string;
|
|
18
20
|
}
|
|
19
21
|
export interface ProviderRetryEpisode {
|
|
20
22
|
state: 'pending' | 'admitted' | 'invalidated';
|
|
@@ -58,6 +60,12 @@ export declare function readInterruptedTurn(nodeId: string): {
|
|
|
58
60
|
* exists before the kickoff is admitted. */
|
|
59
61
|
export declare function hasAdmittedTurn(nodeId: string): boolean;
|
|
60
62
|
export declare function recordFault(nodeId: string, input: FaultRecordInput): StoredFault | null;
|
|
63
|
+
/** Record that the fatal-fault notice for this fault row's current episode has been published. Callers run it in the write that publishes. */
|
|
64
|
+
export declare function markFaultNotified(faultId: string): boolean;
|
|
65
|
+
/** Nodes whose active fatal provider fault has not been announced yet, read from the fault table so a node with no live broker is still found. */
|
|
66
|
+
export declare function listUnnotifiedFatalFaultNodes(): string[];
|
|
67
|
+
/** The daemon registers the one function that publishes a fatal provider fault's notice. Every clear of an unannounced fatal fault runs it first, so no clear can drop a notice. */
|
|
68
|
+
export declare function setFatalFaultAnnouncer(announce: ((nodeId: string) => void) | null): void;
|
|
61
69
|
export declare function clearFault(nodeId: string, opts?: {
|
|
62
70
|
link?: FaultLink;
|
|
63
71
|
preserveEpisode?: boolean;
|
|
@@ -97,4 +105,9 @@ export interface PendingProviderRetry {
|
|
|
97
105
|
export declare function listPendingProviderRetries(): PendingProviderRetry[];
|
|
98
106
|
export declare function beginBootFaultAttempt(nodeId: string): BootFaultAttempt;
|
|
99
107
|
/** Import legacy witness and recovery files inside the parent's one shared consolidation transaction. */
|
|
108
|
+
/** v98 — `notified_since` on a fault row: the episode start its fatal notice was published for.
|
|
109
|
+
* No backfill: a fatal fault already active at upgrade reads as unannounced and is announced once more,
|
|
110
|
+
* because the old in-memory record cannot be rebuilt and a backfill could silence a fault never announced.
|
|
111
|
+
* Guarded like v95: a fixture that starts past v39 never created the table. */
|
|
112
|
+
export declare function addFaultNotifiedSince(db: DatabaseSync): void;
|
|
100
113
|
export declare function migrateRecoveryWitnesses(db: DatabaseSync): void;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
var
|
|
1
|
+
var ie=Object.defineProperty;var o=(e,n)=>ie(e,"name",{value:n,configurable:!0});import{prepared as s}from"../sqlite-statements.js";import{randomUUID as F}from"node:crypto";import{existsSync as f,readdirSync as oe,readFileSync as se,realpathSync as de,statSync as ae}from"node:fs";import{join as N,resolve as ue}from"node:path";import{openDb as E,prepared as ce,withCanvasWrite as l}from"./db.js";import{jobDir as _e,nodesRoot as C}from"./paths.js";import{errorClassFromFault as $}from"../fault-classifier.js";import{parseProviderRateLimit as le}from"../provider-failure.js";import{emitEvent as j}from"../events/emit.js";import{operationIdContext as B,validateOperationId as J}from"../events/operation-id.js";const Me=5*6e4,pe=500,K=["pi\u2192provider","viewer\u2194broker","relay\u2194broker","viewer\u2194crtrd","daemon\u2192node","crtr\u2192pi"],Ee=[...K,"broker\u2194crtrd"],fe=["rate-limit","overloaded","connection","auth","protocol","context-overflow","other","wedged","model-not-found"],ye=["auto","manual","fatal"],Te=["sdk","daemon","client"],Ne=["pending","admitted","invalidated"];function c(){return new Date().toISOString()}o(c,"now");function b(e,n){return new Set(e.prepare(`PRAGMA table_info(${n})`).all().map(({name:t})=>t))}o(b,"tableColumns");function ge(e){const n=b(e,"executions"),t=[["turn_open_at","TEXT"],["turn_progress_at","TEXT"],["turn_closed_at","TEXT"],["turn_end_reason","TEXT CHECK (turn_end_reason IS NULL OR turn_end_reason IN ('settled', 'teardown-aborted'))"],["exit_diagnostic_json","TEXT"]];for(const[r,i]of t)n.has(r)||e.exec(`ALTER TABLE executions ADD COLUMN ${r} ${i}`)}o(ge,"addExecutionColumns");function Le(e){e.exec(`
|
|
2
2
|
CREATE TABLE IF NOT EXISTS node_faults (
|
|
3
3
|
fault_id TEXT PRIMARY KEY,
|
|
4
4
|
node_id TEXT NOT NULL REFERENCES nodes(node_id) ON DELETE CASCADE,
|
|
@@ -23,4 +23,4 @@ CREATE TABLE IF NOT EXISTS node_faults (
|
|
|
23
23
|
);
|
|
24
24
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_active ON node_faults(node_id) WHERE active = 1;
|
|
25
25
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_node_faults_episode ON node_faults(node_id) WHERE episode_state IS NOT NULL;
|
|
26
|
-
`)}o(ge,"createFaultTable");function g(e,n){return typeof n=="string"&&e.includes(n)}o(g,"contains");function F(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}o(F,"isRecord");function K(e){if(!F(e)||typeof e.provider!="string"||e.provider===""||typeof e.model!="string"||e.model==="")return;const n=typeof e.status=="number"&&Number.isFinite(e.status)?e.status:void 0,t=typeof e.code=="string"||typeof e.code=="number"?e.code:void 0,r=typeof e.errno=="string"?e.errno:void 0,i=typeof e.body=="string"?e.body:void 0,d=_e(e.ratelimit),a=Array.isArray(e.causes)?e.causes.flatMap(u=>{if(!F(u))return[];const f={...typeof u.message=="string"?{message:u.message}:{},...typeof u.status=="number"&&Number.isFinite(u.status)?{status:u.status}:{},...typeof u.code=="string"||typeof u.code=="number"?{code:u.code}:{},...typeof u.errno=="string"?{errno:u.errno}:{},...typeof u.body=="string"?{body:u.body}:{}};return Object.keys(f).length===0?[]:[f]}):void 0;return{provider:e.provider,model:e.model,...n===void 0?{}:{status:n},...t===void 0?{}:{code:t},...r===void 0?{}:{errno:r},...i===void 0?{}:{body:i},...a===void 0||a.length===0?{}:{causes:a},...d===void 0?{}:{ratelimit:d}}}o(K,"parseProviderError");function Y(e){try{const n=JSON.parse(e);if(!g(pe,n.link)||typeof n.op!="string"||n.op===""||!g(Ee,n.kind)||typeof n.message!="string"||typeof n.since!="string")return null;const t=j(n.operation_id);if(!F(n.retry)||!g(fe,n.retry.disposition)||n.retry.by!==void 0&&!g(ye,n.retry.by)||n.retry.attempt!==void 0&&typeof n.retry.attempt!="number"||n.retry.max!==void 0&&typeof n.retry.max!="number"||n.retry.nextAt!==void 0&&typeof n.retry.nextAt!="string"||n.anchorEntryId!==void 0&&(typeof n.anchorEntryId!="string"||n.anchorEntryId===""))return null;const r=n.providerError===void 0?void 0:K(n.providerError);return n.providerError!==void 0&&r===void 0?null:{link:n.link,op:n.op,kind:n.kind,retry:{disposition:n.retry.disposition,...n.retry.by===void 0?{}:{by:n.retry.by},...n.retry.attempt===void 0?{}:{attempt:n.retry.attempt},...n.retry.max===void 0?{}:{max:n.retry.max},...n.retry.nextAt===void 0?{}:{nextAt:n.retry.nextAt}},message:n.message,since:n.since,operation_id:t,...n.anchorEntryId===void 0?{}:{anchorEntryId:n.anchorEntryId},...r===void 0?{}:{providerError:r}}}catch{return null}}o(Y,"parseLegacyFault");function Le(e){const n=ae(e);try{return se(n)}catch{return n}}o(Le,"canonicalPath");function me(e){try{const n=JSON.parse(e);if(!g(Te,n.state)||typeof n.sessionFile!="string"||n.sessionFile==="")return null;const t=JSON.stringify(n.fault),r=Y(t);if(r===null||r.link!=="pi\u2192provider"||r.retry.disposition!=="auto"||r.retry.by!=="daemon")return null;const i=n.anchorEntryId;return i!==null&&typeof i!="string"||(r.anchorEntryId??null)!==i?null:{state:n.state,sessionFile:Le(n.sessionFile),fault:r}}catch{return null}}o(me,"parseLegacyEpisode");function G(e){return e!==void 0?j(e):$.current()??$.fresh(t=>t)}o(G,"resolveOperationId");function Se(e){return e.slice(0,le)}o(Se,"clampMessage");function R(e){return e.link==="pi\u2192provider"&&e.retry.disposition==="auto"&&e.retry.by==="daemon"?3:e.retry.disposition==="fatal"?2:1}o(R,"faultPriority");function V(e){return{link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operationId:e.operation_id,anchorEntryId:e.anchorEntryId??null,retryDisposition:e.retry.disposition,retryBy:e.retry.by??null,retryAttempt:e.retry.attempt??null,retryMax:e.retry.max??null,retryNextAt:e.retry.nextAt??null,providerError:e.providerError===void 0?null:JSON.stringify(e.providerError)}}o(V,"encodeFault");function H(e){const n=e.provider_error_json,t=typeof n=="string"?K(JSON.parse(n)):void 0;if(typeof n=="string"&&t===void 0)throw new Error(`invalid provider error on stored fault ${e.fault_id}`);const r={disposition:e.retry_disposition,...e.retry_by===null?{}:{by:e.retry_by},...e.retry_attempt===null?{}:{attempt:e.retry_attempt},...e.retry_max===null?{}:{max:e.retry_max},...e.retry_next_at===null?{}:{nextAt:e.retry_next_at}};return{faultId:e.fault_id,link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operation_id:e.operation_id,retry:r,...e.anchor_entry_id===null?{}:{anchorEntryId:e.anchor_entry_id},...t===void 0?{}:{providerError:t}}}o(H,"hydrateFault");function L(e,n){const t=ue(e,"SELECT * FROM node_faults WHERE node_id = ? AND active = 1").get(n);return t===void 0?null:H(t)}o(L,"activeFault");function Q(e){if(!g(B,e.link))throw new Error(`fault link is not durable: ${e.link}`);return{link:e.link,op:e.op,kind:e.kind,retry:e.retry,message:Se(e.message),since:e.since??c(),operation_id:G(e.operation_id),...typeof e.anchorEntryId=="string"&&e.anchorEntryId!==""?{anchorEntryId:e.anchorEntryId}:{},...e.providerError===void 0?{}:{providerError:e.providerError}}}o(Q,"storedFault");function q(e,n){X({level:n.retry.disposition==="fatal"?"error":"warn",event:"fault.recorded",operation_id:n.operation_id,node_id:e,error_class:w(n),fields:{link:n.link,op:n.op,kind:n.kind,retry:n.retry,message:n.message,...n.anchorEntryId===void 0?{}:{anchor_entry_id:n.anchorEntryId},...n.providerError===void 0?{}:{provider_error:n.providerError}}})}o(q,"emitRecordedFault");function W(e,n,t,r,i){const d=C(),a=V(t),u=c();return s(e,"INSERT INTO node_faults (fault_id, node_id, link, op, kind, message, since, operation_id, anchor_entry_id, retry_disposition, retry_by, retry_attempt, retry_max, retry_next_at, provider_error_json, active, episode_state, episode_binding_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(d,n,a.link,a.op,a.kind,a.message,a.since,a.operationId,a.anchorEntryId,a.retryDisposition,a.retryBy,a.retryAttempt,a.retryMax,a.retryNextAt,a.providerError,r?1:0,i?.state??null,i?.bindingId??null,u,u),{...t,faultId:d}}o(W,"insertFault");function z(e,n,t,r){const i=V(t);return s(e,"UPDATE node_faults SET link = ?, op = ?, kind = ?, message = ?, since = ?, operation_id = ?, anchor_entry_id = ?, retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, provider_error_json = ?, episode_state = COALESCE(?, episode_state), episode_binding_id = COALESCE(?, episode_binding_id), updated_at = ? WHERE fault_id = ?").run(i.link,i.op,i.kind,i.message,i.since,i.operationId,i.anchorEntryId,i.retryDisposition,i.retryBy,i.retryAttempt,i.retryMax,i.retryNextAt,i.providerError,r?.state??null,r?.bindingId??null,c(),n),{...t,faultId:n}}o(z,"updateFault");function D(e,n){s(e,"DELETE FROM node_faults WHERE fault_id = ? AND active = 0 AND episode_state IS NULL").run(n)}o(D,"deleteIfUnretained");function Z(e,n,t,r){const i=L(e,n);if(i===null){const d=s(e,"SELECT fault_id, link, kind, since, operation_id FROM node_faults WHERE node_id = ? AND active = 0 AND episode_state IS NOT NULL").get(n);return d!==void 0&&d.link===t.link&&d.kind===t.kind&&d.since===t.since&&d.operation_id===t.operation_id?(s(e,"UPDATE node_faults SET active = 1, updated_at = ? WHERE fault_id = ?").run(c(),d.fault_id),z(e,d.fault_id,t)):W(e,n,t,!0,r)}if(i.link===t.link&&i.kind===t.kind){const d=t.retry.disposition==="auto"?{...i.retry,...t.retry}:t.retry,u=i.retry.disposition===t.retry.disposition&&(t.retry.disposition!=="fatal"||i.message===t.message)?i.since:t.since;return z(e,i.faultId,{...t,since:u,anchorEntryId:i.anchorEntryId??t.anchorEntryId,retry:d},r)}return i.link!==t.link&&R(i)>R(t)?null:(s(e,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(e,i.faultId),W(e,n,t,!0,r))}o(Z,"replaceActiveFault");function m(e,n,t){return s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND session_id = ? AND unbound_at IS NULL").get(n,t)?.binding_id??null}o(m,"liveBinding");function Pe(e,n){return l(t=>{const r=c();return s(t,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,e).changes>0?!0:s(t,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ? AND state != 'exited'").run(r,r,e).changes>0})}o(Pe,"openTurn");function Me(e){return l(n=>s(n,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(c(),e).changes>0)}o(Me,"recordTurnProgress");function we(e,n){return l(t=>{const r=c(),i=s(t,"UPDATE executions SET turn_closed_at = ?, turn_end_reason = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,n.endReason,e).changes>0;if(i&&n.endReason==="settled"){const d="(SELECT node_id FROM executions WHERE execution_id = ?)";n.viewerPresent===!0?s(t,`UPDATE nodes SET reply_at = ?, read_at = ? WHERE node_id = ${d}`).run(r,r,e):s(t,`UPDATE nodes SET reply_at = ? WHERE node_id = ${d}`).run(r,e)}return i})}o(we,"closeTurn");function Xe(e){return l(n=>s(n,"UPDATE nodes SET read_at = ? WHERE node_id = ?").run(c(),e).changes>0)}o(Xe,"markNodeRead");function $e(e,n){if(!Number.isInteger(n.consecutivePasses)||n.consecutivePasses<1||n.routes.length===0||!n.routes.every(t=>t!==""))throw new Error("invalid transport-poison verdict");return l(t=>s(t,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ? AND state != 'exited'").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:n.consecutivePasses,routes:n.routes}}),e).changes>0)}o($e,"recordTransportPoisonVerdict");function je(e){const n=s(E(),"SELECT exit_diagnostic_json FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);if(n===void 0||n.exit_diagnostic_json===null)return null;try{const t=JSON.parse(n.exit_diagnostic_json),r=t.transport_poison;return t.schema!=="crtr.execution-exit-diagnostic/v1"||r===void 0||!Number.isInteger(r.consecutive_passes)||r.consecutive_passes<1||!Array.isArray(r.routes)||r.routes.length===0||!r.routes.every(i=>typeof i=="string"&&i!=="")?null:{consecutivePasses:r.consecutive_passes,routes:r.routes}}catch{return null}}o(je,"readTransportPoisonVerdict");function Be(e){const n=s(E(),"SELECT turn_open_at, turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND state != 'exited'").get(e);return n===void 0?null:{busy:n.turn_open_at!==null&&n.turn_closed_at===null,progressAt:n.turn_progress_at}}o(Be,"readLiveTurn");function Je(e){const n=E(),t=s(n,"SELECT turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL ORDER BY rowid DESC LIMIT 1").get(e);if(t===void 0)return null;const r=s(n,"SELECT turn_end_reason FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);return{interrupted:t.turn_closed_at===null||r?.turn_end_reason==="teardown-aborted",progressAt:t.turn_progress_at}}o(Je,"readInterruptedTurn");function Ke(e){return s(E(),"SELECT 1 FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL LIMIT 1").get(e)!==void 0}o(Ke,"hasAdmittedTurn");function Ye(e,n){const t=Q(n);return q(e,t),l(r=>Z(r,e,t))}o(Ye,"recordFault");function Ge(e,n){const t=L(E(),e);return t===null||n?.link!==void 0&&t.link!==n.link?!1:l(r=>{const i=L(r,e);if(i===null||n?.link!==void 0&&i.link!==n.link)return!1;const d=s(r,"SELECT episode_state FROM node_faults WHERE fault_id = ?").get(i.faultId);s(r,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),d.episode_state==="pending"&&n?.preserveEpisode!==!0&&s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(r,i.faultId);const a=G();return X({level:"info",event:"fault.cleared",operation_id:a,node_id:e,error_class:w(i),fields:{cleared_fault_operation_id:i.operation_id,link:i.link,op:i.op,kind:i.kind,retry:i.retry,message:i.message,since:i.since,...i.anchorEntryId===void 0?{}:{anchor_entry_id:i.anchorEntryId}}}),!0})}o(Ge,"clearFault");function ee(e){return L(E(),e)}o(ee,"readFault");function Ve(e,n){const t=Q(n.input);if(t.link!=="pi\u2192provider"||t.retry.disposition!=="auto"||t.retry.by!=="daemon")throw new Error("provider retry episode requires an automatic daemon provider fault");return q(e,t),l(r=>{const i=m(r,e,n.sessionId);if(i===null)return null;const d=L(r,e);if(d!==null&&(d.link!==t.link||d.kind!==t.kind)&&R(d)>R(t))return null;const a=s(r,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_state IS NOT NULL").all(e);s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_state IS NOT NULL").run(c(),e);for(const f of a)D(r,f.fault_id);const u=Z(r,e,t,{state:"pending",bindingId:i});return u===null?null:(s(r,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run("pending",i,c(),u.faultId),u)})}o(Ve,"recordPendingEpisode");function Qe(e,n){return l(t=>{if(s(t,"SELECT 1 FROM nodes WHERE node_id = ? AND intent IS 'refresh'").get(e)!==void 0)return!1;const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'admitted', retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, updated_at = ? WHERE node_id = ? AND fault_id = ? AND episode_state = 'pending' AND episode_binding_id = ?").run(n.retry.disposition,n.retry.by??null,n.retry.attempt??null,n.retry.max??null,n.retry.nextAt??null,c(),e,n.faultId,r).changes>0})}o(Qe,"admitEpisode");function qe(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'pending', updated_at = ? WHERE node_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(qe,"reopenEpisode");function ze(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'invalidated', updated_at = ? WHERE node_id = ? AND episode_state != 'invalidated' AND episode_state IS NOT NULL AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(ze,"invalidateEpisode");function Ze(e,n){return l(t=>{const r=m(t,e,n.sessionId);if(r===null)return!1;const i=s(t,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").all(e,r);if(i.length===0)return!1;s(t,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").run(c(),e,r);for(const d of i)D(t,d.fault_id);return!0})}o(Ze,"clearEpisode");function en(e){const n=s(E(),"SELECT f.*, s.session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.node_id = ? AND f.episode_state IS NOT NULL").get(e);return n===void 0?null:{state:n.episode_state,sessionId:n.session_id,fault:H(n)}}o(en,"readEpisode");function nn(){return s(E(),"SELECT f.*, s.session_id AS bound_session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.active = 1 AND f.episode_state = 'pending' AND s.unbound_at IS NULL AND s.session_id IS NOT NULL AND f.link = 'pi\u2192provider' AND f.retry_disposition = 'auto' AND f.retry_by = 'daemon' AND f.kind IN ('connection', 'rate-limit', 'overloaded')").all().map(n=>({nodeId:n.node_id,sessionId:n.bound_session_id,fault:H(n),updatedAt:n.updated_at}))}o(nn,"listPendingProviderRetries");function tn(e){const n=ee(e)===null;return{explanation(){if(!n)return null;const t=ee(e);if(t===null||t.link!=="crtr\u2192pi"||t.retry.disposition!=="fatal")return null;const r=t.op.trim(),i=t.message.trim();return r===""||i===""?null:`${r}: ${i}`}}}o(tn,"beginBootFaultAttempt");function ne(e,n){try{return de(e).mtime.toISOString()}catch{return n}}o(ne,"markerMtime");function P(e,n){if(!p(e))return null;try{return oe(e,"utf8")}catch(t){if(!n)return null;throw new Error(`cannot import recovery input ${e}: ${t.message}`)}}o(P,"legacyFile");function Ae(e,n){return s(e,"SELECT execution_id, started_at FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(n)??null}o(Ae,"latestExecution");function xe(e,n){const r=J(e,"nodes").has("launched_at")?s(e,"SELECT COALESCE(launched_at, created) AS started_at FROM nodes WHERE node_id = ?").get(n):s(e,"SELECT created AS started_at FROM nodes WHERE node_id = ?").get(n);if(r===void 0)throw new Error(`cannot import recovery witness for unknown node ${n}`);const i=C();return s(e,"INSERT INTO executions (execution_id, node_id, daemon_epoch, pid, process_identity, state, started_at, ended_at, exit_reason) VALUES (?, ?, NULL, NULL, NULL, 'exited', ?, ?, 'pre-consolidation')").run(i,n,r.started_at,r.started_at),{execution_id:i,started_at:r.started_at}}o(xe,"syntheticExecution");function te(e,n,t,r,i){return W(e,n,t,r,i)}o(te,"insertMigratedFault");function Ie(e,n,t,r){const i=s(e,"SELECT binding_id, node_id FROM sessions WHERE transcript_path = ? AND unbound_at IS NULL").get(t);if(i!==void 0){if(i.node_id!==n)throw new Error(`cannot import provider retry ${r}: transcript is live-owned by ${i.node_id}`);return i.binding_id}const d=s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND transcript_path = ? ORDER BY rowid DESC LIMIT 1").get(n,t);if(d!==void 0)return d.binding_id;const a=s(e,"SELECT created FROM nodes WHERE node_id = ?").get(n);if(a===void 0)throw new Error(`cannot import provider retry ${r}: unknown node ${n}`);const u=C();return s(e,"INSERT INTO sessions (binding_id, session_id, node_id, transcript_path, source_session_id, fork_entry_id, created_at, bound_at, unbound_at) VALUES (?, NULL, ?, ?, NULL, NULL, ?, ?, ?)").run(u,n,t,a.created,a.created,c()),u}o(Ie,"migrationBinding");function rn(e){if(Ne(e),ge(e),!p(b()))return;const n=ie(b(),{withFileTypes:!0}).filter(r=>r.isDirectory()).sort((r,i)=>r.name.localeCompare(i.name)),t=s(e,"SELECT 1 FROM nodes WHERE node_id = ?");for(const r of n){const i=r.name,d=ce(i),a=N(d,"busy"),u=N(d,"clean-abort"),f=N(d,"transport-poison.json"),h=N(d,"fault"),S=N(d,"provider-retry"),O=p(a),U=p(u);if(!O&&!U&&!p(f)&&!p(h)&&!p(S)||!p(N(b(),i,"meta.json"))&&t.get(i)===void 0)continue;const k=P(f,!1),A=P(h,!0),x=P(S,!0);if(!O&&!U&&k===null&&A===null&&x===null)continue;let y=Ae(e,i);if(y===null&&(y=xe(e,i)),O){const _=ne(a,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ?").run(_,_,y.execution_id)}else if(U){const _=ne(u,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = ?, turn_end_reason = 'teardown-aborted' WHERE execution_id = ?").run(_,_,_,y.execution_id)}if(k!==null)try{const _=JSON.parse(k);Number.isInteger(_.consecutivePasses)&&_.consecutivePasses>=1&&Array.isArray(_.routes)&&_.routes.length>0&&_.routes.every(M=>typeof M=="string"&&M!=="")&&s(e,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ?").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:_.consecutivePasses,routes:_.routes}}),y.execution_id)}catch{}const I=A===null?null:Y(A);if(A!==null&&I===null)throw new Error(`cannot import recovery fault ${h}`);const T=x===null?null:me(x);if(x!==null&&T===null)throw new Error(`cannot import provider retry ${S}`);let v=I===null||I.link==="broker\u2194crtrd"?null:te(e,i,I,!0);if(T!==null){const _=Ie(e,i,T.sessionFile,S);v!==null&&v.since===T.fault.since?s(e,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run(T.state,_,c(),v.faultId):te(e,i,T.fault,!1,{state:T.state,bindingId:_})}}}o(rn,"migrateRecoveryWitnesses");export{We as FAULT_QUIET_MS,Qe as admitEpisode,tn as beginBootFaultAttempt,Ze as clearEpisode,Ge as clearFault,we as closeTurn,Ke as hasAdmittedTurn,H as hydrateFault,ze as invalidateEpisode,nn as listPendingProviderRetries,Xe as markNodeRead,rn as migrateRecoveryWitnesses,Pe as openTurn,K as parseProviderError,en as readEpisode,ee as readFault,Je as readInterruptedTurn,Be as readLiveTurn,je as readTransportPoisonVerdict,Ye as recordFault,Ve as recordPendingEpisode,$e as recordTransportPoisonVerdict,Me as recordTurnProgress,qe as reopenEpisode};
|
|
26
|
+
`)}o(Le,"createFaultTable");function g(e,n){return typeof n=="string"&&e.includes(n)}o(g,"contains");function H(e){return typeof e=="object"&&e!==null&&!Array.isArray(e)}o(H,"isRecord");function Y(e){if(!H(e)||typeof e.provider!="string"||e.provider===""||typeof e.model!="string"||e.model==="")return;const n=typeof e.status=="number"&&Number.isFinite(e.status)?e.status:void 0,t=typeof e.code=="string"||typeof e.code=="number"?e.code:void 0,r=typeof e.errno=="string"?e.errno:void 0,i=typeof e.body=="string"?e.body:void 0,d=le(e.ratelimit),a=Array.isArray(e.causes)?e.causes.flatMap(u=>{if(!H(u))return[];const p={...typeof u.message=="string"?{message:u.message}:{},...typeof u.status=="number"&&Number.isFinite(u.status)?{status:u.status}:{},...typeof u.code=="string"||typeof u.code=="number"?{code:u.code}:{},...typeof u.errno=="string"?{errno:u.errno}:{},...typeof u.body=="string"?{body:u.body}:{}};return Object.keys(p).length===0?[]:[p]}):void 0;return{provider:e.provider,model:e.model,...n===void 0?{}:{status:n},...t===void 0?{}:{code:t},...r===void 0?{}:{errno:r},...i===void 0?{}:{body:i},...a===void 0||a.length===0?{}:{causes:a},...d===void 0?{}:{ratelimit:d}}}o(Y,"parseProviderError");function G(e){try{const n=JSON.parse(e);if(!g(Ee,n.link)||typeof n.op!="string"||n.op===""||!g(fe,n.kind)||typeof n.message!="string"||typeof n.since!="string")return null;const t=J(n.operation_id);if(!H(n.retry)||!g(ye,n.retry.disposition)||n.retry.by!==void 0&&!g(Te,n.retry.by)||n.retry.attempt!==void 0&&typeof n.retry.attempt!="number"||n.retry.max!==void 0&&typeof n.retry.max!="number"||n.retry.nextAt!==void 0&&typeof n.retry.nextAt!="string"||n.anchorEntryId!==void 0&&(typeof n.anchorEntryId!="string"||n.anchorEntryId===""))return null;const r=n.providerError===void 0?void 0:Y(n.providerError);return n.providerError!==void 0&&r===void 0?null:{link:n.link,op:n.op,kind:n.kind,retry:{disposition:n.retry.disposition,...n.retry.by===void 0?{}:{by:n.retry.by},...n.retry.attempt===void 0?{}:{attempt:n.retry.attempt},...n.retry.max===void 0?{}:{max:n.retry.max},...n.retry.nextAt===void 0?{}:{nextAt:n.retry.nextAt}},message:n.message,since:n.since,operation_id:t,...n.anchorEntryId===void 0?{}:{anchorEntryId:n.anchorEntryId},...r===void 0?{}:{providerError:r}}}catch{return null}}o(G,"parseLegacyFault");function me(e){const n=ue(e);try{return de(n)}catch{return n}}o(me,"canonicalPath");function Ae(e){try{const n=JSON.parse(e);if(!g(Ne,n.state)||typeof n.sessionFile!="string"||n.sessionFile==="")return null;const t=JSON.stringify(n.fault),r=G(t);if(r===null||r.link!=="pi\u2192provider"||r.retry.disposition!=="auto"||r.retry.by!=="daemon")return null;const i=n.anchorEntryId;return i!==null&&typeof i!="string"||(r.anchorEntryId??null)!==i?null:{state:n.state,sessionFile:me(n.sessionFile),fault:r}}catch{return null}}o(Ae,"parseLegacyEpisode");function V(e){return e!==void 0?J(e):B.current()??B.fresh(t=>t)}o(V,"resolveOperationId");function Se(e){return e.slice(0,pe)}o(Se,"clampMessage");function I(e){return e.link==="pi\u2192provider"&&e.retry.disposition==="auto"&&e.retry.by==="daemon"?3:e.retry.disposition==="fatal"?2:1}o(I,"faultPriority");function Q(e){return{link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operationId:e.operation_id,anchorEntryId:e.anchorEntryId??null,retryDisposition:e.retry.disposition,retryBy:e.retry.by??null,retryAttempt:e.retry.attempt??null,retryMax:e.retry.max??null,retryNextAt:e.retry.nextAt??null,providerError:e.providerError===void 0?null:JSON.stringify(e.providerError)}}o(Q,"encodeFault");function W(e){const n=e.provider_error_json,t=typeof n=="string"?Y(JSON.parse(n)):void 0;if(typeof n=="string"&&t===void 0)throw new Error(`invalid provider error on stored fault ${e.fault_id}`);const r={disposition:e.retry_disposition,...e.retry_by===null?{}:{by:e.retry_by},...e.retry_attempt===null?{}:{attempt:e.retry_attempt},...e.retry_max===null?{}:{max:e.retry_max},...e.retry_next_at===null?{}:{nextAt:e.retry_next_at}};return{faultId:e.fault_id,link:e.link,op:e.op,kind:e.kind,message:e.message,since:e.since,operation_id:e.operation_id,retry:r,...e.anchor_entry_id===null?{}:{anchorEntryId:e.anchor_entry_id},...t===void 0?{}:{providerError:t},...typeof e.notified_since=="string"?{notifiedSince:e.notified_since}:{}}}o(W,"hydrateFault");function L(e,n){const t=ce(e,"SELECT * FROM node_faults WHERE node_id = ? AND active = 1").get(n);return t===void 0?null:W(t)}o(L,"activeFault");function q(e){if(!g(K,e.link))throw new Error(`fault link is not durable: ${e.link}`);return{link:e.link,op:e.op,kind:e.kind,retry:e.retry,message:Se(e.message),since:e.since??c(),operation_id:V(e.operation_id),...typeof e.anchorEntryId=="string"&&e.anchorEntryId!==""?{anchorEntryId:e.anchorEntryId}:{},...e.providerError===void 0?{}:{providerError:e.providerError}}}o(q,"storedFault");function z(e,n){j({level:n.retry.disposition==="fatal"?"error":"warn",event:"fault.recorded",operation_id:n.operation_id,node_id:e,error_class:$(n),fields:{link:n.link,op:n.op,kind:n.kind,retry:n.retry,message:n.message,...n.anchorEntryId===void 0?{}:{anchor_entry_id:n.anchorEntryId},...n.providerError===void 0?{}:{provider_error:n.providerError}}})}o(z,"emitRecordedFault");function P(e,n,t,r,i){const d=F(),a=Q(t),u=c();return s(e,"INSERT INTO node_faults (fault_id, node_id, link, op, kind, message, since, operation_id, anchor_entry_id, retry_disposition, retry_by, retry_attempt, retry_max, retry_next_at, provider_error_json, active, episode_state, episode_binding_id, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").run(d,n,a.link,a.op,a.kind,a.message,a.since,a.operationId,a.anchorEntryId,a.retryDisposition,a.retryBy,a.retryAttempt,a.retryMax,a.retryNextAt,a.providerError,r?1:0,i?.state??null,i?.bindingId??null,u,u),{...t,faultId:d}}o(P,"insertFault");function Z(e,n,t,r){const i=Q(t);return s(e,"UPDATE node_faults SET link = ?, op = ?, kind = ?, message = ?, since = ?, operation_id = ?, anchor_entry_id = ?, retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, provider_error_json = ?, episode_state = COALESCE(?, episode_state), episode_binding_id = COALESCE(?, episode_binding_id), updated_at = ? WHERE fault_id = ?").run(i.link,i.op,i.kind,i.message,i.since,i.operationId,i.anchorEntryId,i.retryDisposition,i.retryBy,i.retryAttempt,i.retryMax,i.retryNextAt,i.providerError,r?.state??null,r?.bindingId??null,c(),n),{...t,faultId:n}}o(Z,"updateFault");function D(e,n){s(e,"DELETE FROM node_faults WHERE fault_id = ? AND active = 0 AND episode_state IS NULL").run(n)}o(D,"deleteIfUnretained");function ee(e,n,t,r){const i=L(e,n);if(i===null){const d=s(e,"SELECT fault_id, link, kind, since, operation_id FROM node_faults WHERE node_id = ? AND active = 0 AND episode_state IS NOT NULL").get(n);return d!==void 0&&d.link===t.link&&d.kind===t.kind&&d.since===t.since&&d.operation_id===t.operation_id?(s(e,"UPDATE node_faults SET active = 1, updated_at = ? WHERE fault_id = ?").run(c(),d.fault_id),Z(e,d.fault_id,t)):P(e,n,t,!0,r)}if(i.link===t.link&&i.kind===t.kind){const d=t.retry.disposition==="auto"?{...i.retry,...t.retry}:t.retry,a=i.retry.disposition===t.retry.disposition&&(t.retry.disposition!=="fatal"||i.message===t.message),u=a?i.since:t.since,p=Z(e,i.faultId,{...t,since:u,anchorEntryId:i.anchorEntryId??t.anchorEntryId,retry:d},r);return a||s(e,"UPDATE node_faults SET notified_since = NULL WHERE fault_id = ?").run(i.faultId),p}return i.link!==t.link&&I(i)>I(t)?null:(s(e,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(e,i.faultId),P(e,n,t,!0,r))}o(ee,"replaceActiveFault");function m(e,n,t){return s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND session_id = ? AND unbound_at IS NULL").get(n,t)?.binding_id??null}o(m,"liveBinding");function we(e,n){return l(t=>{const r=c();return s(t,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,e).changes>0?!0:s(t,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ? AND state != 'exited'").run(r,r,e).changes>0})}o(we,"openTurn");function Xe(e){return l(n=>s(n,"UPDATE executions SET turn_progress_at = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(c(),e).changes>0)}o(Xe,"recordTurnProgress");function $e(e,n){return l(t=>{const r=c(),i=s(t,"UPDATE executions SET turn_closed_at = ?, turn_end_reason = ? WHERE execution_id = ? AND state != 'exited' AND turn_open_at IS NOT NULL AND turn_closed_at IS NULL").run(r,n.endReason,e).changes>0;if(i&&n.endReason==="settled"){const d="(SELECT node_id FROM executions WHERE execution_id = ?)";n.viewerPresent===!0?s(t,`UPDATE nodes SET reply_at = ?, read_at = ? WHERE node_id = ${d}`).run(r,r,e):s(t,`UPDATE nodes SET reply_at = ? WHERE node_id = ${d}`).run(r,e)}return i})}o($e,"closeTurn");function je(e){return l(n=>s(n,"UPDATE nodes SET read_at = ? WHERE node_id = ?").run(c(),e).changes>0)}o(je,"markNodeRead");function Be(e,n){if(!Number.isInteger(n.consecutivePasses)||n.consecutivePasses<1||n.routes.length===0||!n.routes.every(t=>t!==""))throw new Error("invalid transport-poison verdict");return l(t=>s(t,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ? AND state != 'exited'").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:n.consecutivePasses,routes:n.routes}}),e).changes>0)}o(Be,"recordTransportPoisonVerdict");function Je(e){const n=s(E(),"SELECT exit_diagnostic_json FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);if(n===void 0||n.exit_diagnostic_json===null)return null;try{const t=JSON.parse(n.exit_diagnostic_json),r=t.transport_poison;return t.schema!=="crtr.execution-exit-diagnostic/v1"||r===void 0||!Number.isInteger(r.consecutive_passes)||r.consecutive_passes<1||!Array.isArray(r.routes)||r.routes.length===0||!r.routes.every(i=>typeof i=="string"&&i!=="")?null:{consecutivePasses:r.consecutive_passes,routes:r.routes}}catch{return null}}o(Je,"readTransportPoisonVerdict");function Ke(e){const n=s(E(),"SELECT turn_open_at, turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND state != 'exited'").get(e);return n===void 0?null:{busy:n.turn_open_at!==null&&n.turn_closed_at===null,progressAt:n.turn_progress_at}}o(Ke,"readLiveTurn");function Ye(e){const n=E(),t=s(n,"SELECT turn_closed_at, turn_progress_at FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL ORDER BY rowid DESC LIMIT 1").get(e);if(t===void 0)return null;const r=s(n,"SELECT turn_end_reason FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(e);return{interrupted:t.turn_closed_at===null||r?.turn_end_reason==="teardown-aborted",progressAt:t.turn_progress_at}}o(Ye,"readInterruptedTurn");function Ge(e){return s(E(),"SELECT 1 FROM executions WHERE node_id = ? AND turn_open_at IS NOT NULL LIMIT 1").get(e)!==void 0}o(Ge,"hasAdmittedTurn");function Ve(e,n){const t=q(n);return z(e,t),l(r=>ee(r,e,t))}o(Ve,"recordFault");function Qe(e){return l(n=>s(n,"UPDATE node_faults SET notified_since = since, updated_at = ? WHERE fault_id = ?").run(c(),e).changes>0)}o(Qe,"markFaultNotified");function xe(e){return e.link==="pi\u2192provider"&&e.retry.disposition==="fatal"&&e.notifiedSince!==e.since}o(xe,"isUnnotifiedFatalProviderFault");function qe(){return s(E(),"SELECT f.node_id FROM node_faults f JOIN nodes n ON n.node_id = f.node_id WHERE f.active = 1 AND f.link = 'pi\u2192provider' AND f.retry_disposition = 'fatal' AND (f.notified_since IS NULL OR f.notified_since != f.since) AND n.status IN ('active', 'idle')").all().map(e=>e.node_id)}o(qe,"listUnnotifiedFatalFaultNodes");let M=null;function ze(e){M=e}o(ze,"setFatalFaultAnnouncer");function Ze(e,n){const t=L(E(),e);return t===null||n?.link!==void 0&&t.link!==n.link?!1:l(r=>{const i=L(r,e);if(i===null||n?.link!==void 0&&i.link!==n.link)return!1;M!==null&&xe(i)&&M(e);const d=s(r,"SELECT episode_state FROM node_faults WHERE fault_id = ?").get(i.faultId);s(r,"UPDATE node_faults SET active = 0, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),d.episode_state==="pending"&&n?.preserveEpisode!==!0&&s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE fault_id = ?").run(c(),i.faultId),D(r,i.faultId);const a=V();return j({level:"info",event:"fault.cleared",operation_id:a,node_id:e,error_class:$(i),fields:{cleared_fault_operation_id:i.operation_id,link:i.link,op:i.op,kind:i.kind,retry:i.retry,message:i.message,since:i.since,...i.anchorEntryId===void 0?{}:{anchor_entry_id:i.anchorEntryId}}}),!0})}o(Ze,"clearFault");function ne(e){return L(E(),e)}o(ne,"readFault");function en(e,n){const t=q(n.input);if(t.link!=="pi\u2192provider"||t.retry.disposition!=="auto"||t.retry.by!=="daemon")throw new Error("provider retry episode requires an automatic daemon provider fault");return z(e,t),l(r=>{const i=m(r,e,n.sessionId);if(i===null)return null;const d=L(r,e);if(d!==null&&(d.link!==t.link||d.kind!==t.kind)&&I(d)>I(t))return null;const a=s(r,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_state IS NOT NULL").all(e);s(r,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_state IS NOT NULL").run(c(),e);for(const p of a)D(r,p.fault_id);const u=ee(r,e,t,{state:"pending",bindingId:i});return u===null?null:(s(r,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run("pending",i,c(),u.faultId),u)})}o(en,"recordPendingEpisode");function nn(e,n){return l(t=>{if(s(t,"SELECT 1 FROM nodes WHERE node_id = ? AND intent IS 'refresh'").get(e)!==void 0)return!1;const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'admitted', retry_disposition = ?, retry_by = ?, retry_attempt = ?, retry_max = ?, retry_next_at = ?, updated_at = ? WHERE node_id = ? AND fault_id = ? AND episode_state = 'pending' AND episode_binding_id = ?").run(n.retry.disposition,n.retry.by??null,n.retry.attempt??null,n.retry.max??null,n.retry.nextAt??null,c(),e,n.faultId,r).changes>0})}o(nn,"admitEpisode");function tn(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'pending', updated_at = ? WHERE node_id = ? AND episode_state = 'admitted' AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(tn,"reopenEpisode");function rn(e,n){return l(t=>{const r=m(t,e,n.sessionId);return r===null?!1:s(t,"UPDATE node_faults SET episode_state = 'invalidated', updated_at = ? WHERE node_id = ? AND episode_state != 'invalidated' AND episode_state IS NOT NULL AND episode_binding_id = ?").run(c(),e,r).changes>0})}o(rn,"invalidateEpisode");function on(e,n){return l(t=>{const r=m(t,e,n.sessionId);if(r===null)return!1;const i=s(t,"SELECT fault_id FROM node_faults WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").all(e,r);if(i.length===0)return!1;s(t,"UPDATE node_faults SET episode_state = NULL, episode_binding_id = NULL, updated_at = ? WHERE node_id = ? AND episode_binding_id = ? AND episode_state IS NOT NULL").run(c(),e,r);for(const d of i)D(t,d.fault_id);return!0})}o(on,"clearEpisode");function sn(e){const n=s(E(),"SELECT f.*, s.session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.node_id = ? AND f.episode_state IS NOT NULL").get(e);return n===void 0?null:{state:n.episode_state,sessionId:n.session_id,fault:W(n)}}o(sn,"readEpisode");function dn(){return s(E(),"SELECT f.*, s.session_id AS bound_session_id FROM node_faults f JOIN sessions s ON s.binding_id = f.episode_binding_id WHERE f.active = 1 AND f.episode_state = 'pending' AND s.unbound_at IS NULL AND s.session_id IS NOT NULL AND f.link = 'pi\u2192provider' AND f.retry_disposition = 'auto' AND f.retry_by = 'daemon' AND f.kind IN ('connection', 'rate-limit', 'overloaded')").all().map(n=>({nodeId:n.node_id,sessionId:n.bound_session_id,fault:W(n),updatedAt:n.updated_at}))}o(dn,"listPendingProviderRetries");function an(e){const n=ne(e)===null;return{explanation(){if(!n)return null;const t=ne(e);if(t===null||t.link!=="crtr\u2192pi"||t.retry.disposition!=="fatal")return null;const r=t.op.trim(),i=t.message.trim();return r===""||i===""?null:`${r}: ${i}`}}}o(an,"beginBootFaultAttempt");function te(e,n){try{return ae(e).mtime.toISOString()}catch{return n}}o(te,"markerMtime");function w(e,n){if(!f(e))return null;try{return se(e,"utf8")}catch(t){if(!n)return null;throw new Error(`cannot import recovery input ${e}: ${t.message}`)}}o(w,"legacyFile");function Re(e,n){return s(e,"SELECT execution_id, started_at FROM executions WHERE node_id = ? ORDER BY rowid DESC LIMIT 1").get(n)??null}o(Re,"latestExecution");function Ie(e,n){const r=b(e,"nodes").has("launched_at")?s(e,"SELECT COALESCE(launched_at, created) AS started_at FROM nodes WHERE node_id = ?").get(n):s(e,"SELECT created AS started_at FROM nodes WHERE node_id = ?").get(n);if(r===void 0)throw new Error(`cannot import recovery witness for unknown node ${n}`);const i=F();return s(e,"INSERT INTO executions (execution_id, node_id, daemon_epoch, pid, process_identity, state, started_at, ended_at, exit_reason) VALUES (?, ?, NULL, NULL, NULL, 'exited', ?, ?, 'pre-consolidation')").run(i,n,r.started_at,r.started_at),{execution_id:i,started_at:r.started_at}}o(Ie,"syntheticExecution");function re(e,n,t,r,i){return P(e,n,t,r,i)}o(re,"insertMigratedFault");function De(e,n,t,r){const i=s(e,"SELECT binding_id, node_id FROM sessions WHERE transcript_path = ? AND unbound_at IS NULL").get(t);if(i!==void 0){if(i.node_id!==n)throw new Error(`cannot import provider retry ${r}: transcript is live-owned by ${i.node_id}`);return i.binding_id}const d=s(e,"SELECT binding_id FROM sessions WHERE node_id = ? AND transcript_path = ? ORDER BY rowid DESC LIMIT 1").get(n,t);if(d!==void 0)return d.binding_id;const a=s(e,"SELECT created FROM nodes WHERE node_id = ?").get(n);if(a===void 0)throw new Error(`cannot import provider retry ${r}: unknown node ${n}`);const u=F();return s(e,"INSERT INTO sessions (binding_id, session_id, node_id, transcript_path, source_session_id, fork_entry_id, created_at, bound_at, unbound_at) VALUES (?, NULL, ?, ?, NULL, NULL, ?, ?, ?)").run(u,n,t,a.created,a.created,c()),u}o(De,"migrationBinding");function un(e){e.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'node_faults'").get()!==void 0&&(b(e,"node_faults").has("notified_since")||e.exec("ALTER TABLE node_faults ADD COLUMN notified_since TEXT"))}o(un,"addFaultNotifiedSince");function cn(e){if(ge(e),Le(e),!f(C()))return;const n=oe(C(),{withFileTypes:!0}).filter(r=>r.isDirectory()).sort((r,i)=>r.name.localeCompare(i.name)),t=s(e,"SELECT 1 FROM nodes WHERE node_id = ?");for(const r of n){const i=r.name,d=_e(i),a=N(d,"busy"),u=N(d,"clean-abort"),p=N(d,"transport-poison.json"),U=N(d,"fault"),A=N(d,"provider-retry"),O=f(a),h=f(u);if(!O&&!h&&!f(p)&&!f(U)&&!f(A)||!f(N(C(),i,"meta.json"))&&t.get(i)===void 0)continue;const k=w(p,!1),S=w(U,!0),x=w(A,!0);if(!O&&!h&&k===null&&S===null&&x===null)continue;let y=Re(e,i);if(y===null&&(y=Ie(e,i)),O){const _=te(a,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = NULL, turn_end_reason = NULL WHERE execution_id = ?").run(_,_,y.execution_id)}else if(h){const _=te(u,y.started_at);s(e,"UPDATE executions SET turn_open_at = ?, turn_progress_at = ?, turn_closed_at = ?, turn_end_reason = 'teardown-aborted' WHERE execution_id = ?").run(_,_,_,y.execution_id)}if(k!==null)try{const _=JSON.parse(k);Number.isInteger(_.consecutivePasses)&&_.consecutivePasses>=1&&Array.isArray(_.routes)&&_.routes.length>0&&_.routes.every(X=>typeof X=="string"&&X!=="")&&s(e,"UPDATE executions SET exit_diagnostic_json = ? WHERE execution_id = ?").run(JSON.stringify({schema:"crtr.execution-exit-diagnostic/v1",transport_poison:{consecutive_passes:_.consecutivePasses,routes:_.routes}}),y.execution_id)}catch{}const R=S===null?null:G(S);if(S!==null&&R===null)throw new Error(`cannot import recovery fault ${U}`);const T=x===null?null:Ae(x);if(x!==null&&T===null)throw new Error(`cannot import provider retry ${A}`);let v=R===null||R.link==="broker\u2194crtrd"?null:re(e,i,R,!0);if(T!==null){const _=De(e,i,T.sessionFile,A);v!==null&&v.since===T.fault.since?s(e,"UPDATE node_faults SET episode_state = ?, episode_binding_id = ?, updated_at = ? WHERE fault_id = ?").run(T.state,_,c(),v.faultId):re(e,i,T.fault,!1,{state:T.state,bindingId:_})}}}o(cn,"migrateRecoveryWitnesses");export{Me as FAULT_QUIET_MS,un as addFaultNotifiedSince,nn as admitEpisode,an as beginBootFaultAttempt,on as clearEpisode,Ze as clearFault,$e as closeTurn,Ge as hasAdmittedTurn,W as hydrateFault,rn as invalidateEpisode,dn as listPendingProviderRetries,qe as listUnnotifiedFatalFaultNodes,Qe as markFaultNotified,je as markNodeRead,cn as migrateRecoveryWitnesses,we as openTurn,Y as parseProviderError,sn as readEpisode,ne as readFault,Ye as readInterruptedTurn,Ke as readLiveTurn,Je as readTransportPoisonVerdict,Ve as recordFault,en as recordPendingEpisode,Be as recordTransportPoisonVerdict,Xe as recordTurnProgress,tn as reopenEpisode,ze as setFatalFaultAnnouncer};
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
var Pe=Object.defineProperty;var a=(t,n)=>Pe(t,"name",{value:n,configurable:!0});import{startPeerOutbox as xe}from"./peer/outbox.js";import{reportMachineStatus as Ie}from"../core/peer/directory.js";import{createPeerRelay as De}from"./peer/relay-service.js";import{writeFileSync as Te,readFileSync as Ce,rmSync as $e,mkdirSync as ee,existsSync as E,chmodSync as Le}from"node:fs";import{createServer as re}from"node:http";import{dirname as Ae}from"node:path";import{envCrtrdTcp as Ne,envDocumentPort as Me}from"../shared/env.js";import{serveCanvasConversationStore as Oe}from"../core/canvas/sessions.js";import{pollRunObservers as Ge}from"./runs/observer.js";import{randomBytes as Fe}from"node:crypto";import{apiSocketPath as He,crtrHome as je}from"../core/canvas/paths.js";import{activeAppUsers as Be}from"../core/spaces/users.js";import{ensureAppSpace as qe,ensureUserFiles as ze}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as D}from"../launcher/client.js";import{installLauncherEventSink as We}from"./launcher-events.js";import{initializeEmptyLayout as Ue,layoutNeedsMigration as Ke,socketDir as te}from"../core/layout.js";import{migrateLooseKeys as Je}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as Ve}from"../core/profiles/relocate.js";import{getNode as Qe,listNodesForSupervision as ne,setFrozen as oe}from"../core/canvas/index.js";import{createApiServer as Ye}from"./api/server.js";import{createDocumentListener as Xe}from"./api/document-listener.js";import{documentSecret as Ze}from"./api/document-secret.js";import{createAppListener as er}from"./api/app-listener.js";import{installLocalTerminalGrant as rr}from"../core/grants/local.js";import{onRunEventAppended as tr}from"../core/runs/events.js";import{runNotices as nr}from"./run-notices.js";import{createGrantSync as or,GRANT_SYNC_INTERVAL_S as ir}from"../core/grants/sync.js";import{runRemovalCascades as sr}from"../core/grants/remove.js";import{sweepSandboxes as ar}from"../core/grants/on-sync.js";import{rebuildCronExecutions as cr}from"./cron/run-socket.js";import{hasSyncedGrants as dr}from"../core/grants/mirror.js";import{readConfig as ie}from"../core/config.js";import{reconcileUndeliveredTickets as lr}from"./human/sweep.js";import{resetHandedOffDeliveries as pr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as mr}from"../core/canvas/daemon-actions.js";import{sweepReviews as fr}from"./review/sweep.js";import{pidfilePath as j}from"./pidfile.js";import{rendererWarning as ur}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as hr}from"./supervise-cadence.js";import{createResumeDetector as vr}from"./resume-detector.js";import{BrokerSupervisionReconciler as wr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as gr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as yr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as Sr}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as _r}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as br}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as kr}from"./reconcilers/action-lane.js";import{endLiveExecution as Er,migrateLegacyExecutionIdentities as Rr}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as Pr}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as xr}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as Ir}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as Dr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as Tr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as Cr}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as $r}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Lr}from"./structured-requests/boot.js";import{captureLivenessSnapshot as se,capturePidCommand as Ar,captureTeardownSnapshot as ae,isPidAlive as Nr,isAnyPidAlive as ce,killProcessTreePids as de}from"../core/canvas/pid.js";import{bindFleet as Mr,boundFleet as Or}from"../core/runtime/fleet.js";import{bindDaemonControl as Gr,clearDaemonControl as Fr}from"./control.js";import{execDaemon as Hr,spawnDaemon as jr}from"./manage.js";import{reconcileAndMigrateBoot as Br}from"../core/canvas/boot.js";import{resolveInstallId as qr}from"../core/canvas/install-id.js";import{setInstallId as zr}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Wr,releaseDaemonOwnership as le}from"../core/canvas/daemon-owner.js";import{migrate as Ur,openDb as pe,withCanvasWrite as Kr,bindDaemonCanvasOpenHook as Jr,clearDaemonCanvasOpenHook as me}from"../core/canvas/db.js";import{brokerSignalBus as B}from"./broker-signals/bus.js";import{brokerSignalStreams as Vr}from"./api/handlers/broker-signals.js";import{reconcilePackages as Qr}from"../core/graph/package-reconcile.js";import{startRepoSync as Yr}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Xr,reapStaleSpares as Zr}from"../core/runtime/warm-pool.js";import{DaemonFleet as et}from"./fleet.js";import{BOOT_MIGRATIONS as rt}from"./boot-migrations.js";import{applyJoinedOwner as tt}from"./joined-owner.js";import{emitEvent as f}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as nt}from"../core/events/source.js";import{activateCanvasImport as ot,MigrationBlockedError as it,runForcedCanvasImport as st}from"../migrations/activation.js";import{convertLegacyProviderStore as at,convertLegacyScopedStore as ct}from"../core/scoped-state/migrate.js";import{userScopeRoot as dt}from"../core/scope.js";import{secretsDir as lt}from"../core/layout.js";function q(t,n){return t.includes(` --canvas-home ${n} `)}a(q,"brokerCommandDeclaresCanvasHome");function z(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}a(z,"brokerCommandEpoch");const pt=5e3,fe=150;async function mt(t){const n=se();if(n===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const c=je(),u=[];for(const[s,m]of n.processes){const y=m.command;if(m.zombie||!y.includes("broker-cli.js")||!q(y,c)||z(y)===t)continue;const b=y.trim().split(/\s+/).at(-1)??"";b===""||b.endsWith("broker-cli.js")||u.push({pid:s,nodeId:b})}let h=0;const o=[];for(const s of u){const m=Ar(s.pid);if(m===null||!m.includes("broker-cli.js")||!q(m,c)||z(m)===t||m.trim().split(/\s+/).at(-1)!==s.nodeId)continue;const y=ae(s.pid,null);if(!y.reused){try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):process.kill(s.pid,"SIGTERM")}catch{continue}h++,o.push({pid:s.pid,snap:y})}}const g=Date.now()+pt;for(;Date.now()<g&&o.some(s=>ce(s.snap.tree.length>0?s.snap.tree:[s.pid]));)await new Promise(s=>setTimeout(s,fe));let w=0;for(const s of o){const m=s.snap.tree.length>0?s.snap.tree:[s.pid];ce(m)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):de(m,"SIGKILL",s.snap.identities),w++)}w>0&&await new Promise(s=>setTimeout(s,fe));const d=se();if(d===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const p=[];for(const[s,m]of d.processes){const y=m.command;m.zombie||!y.includes("broker-cli.js")||q(y,c)&&z(y)!==t&&p.push(s)}if(p.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${p.join(", ")}`);f({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:u.length,sigtermed:h,sigkilled:w}})}a(mt,"reapPriorEpochBrokers");const ft=5e3;async function ut(t){const n=Array.from(t.entries());if(n.length===0)return;for(const[o]of n){const g=Qe(o)?.status;(g==="active"||g==="idle")&&oe(o,new Date().toISOString())}const c=n.map(([,o])=>({pid:o.pid,snap:ae(o.pid,null),exited:o.handle.exited}));for(const o of c)if(!o.snap.reused)try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):process.kill(o.pid,"SIGTERM")}catch{}const u=new Promise(o=>{setTimeout(o,ft).unref()});await Promise.allSettled(c.map(o=>Promise.race([o.exited.then(()=>{}),u])));const h=[];for(const o of c)Nr(o.pid)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):de(o.snap.tree.length>0?o.snap.tree:[o.pid],"SIGKILL",o.snap.identities),h.push(o));await Promise.all(h.map(o=>o.exited));for(const[o]of n)Er(o,{reason:"daemon-teardown"});f({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:c.length,sigkilled:h.length}})}a(ut,"teardownFleet");function ue(){ee(Ae(j()),{recursive:!0}),Te(j(),String(process.pid),"utf8")}a(ue,"writePidfile");function he(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}a(he,"isMissingPathError");function ve(t){const n=j();let c;try{c=Ce(n,"utf8")}catch(u){if(he(u))return;throw u}if(c.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{$e(n)}catch(u){if(he(u))return;throw u}}a(ve,"removeOwnedPidfileMirror");const ht={acceptsDetachedWork:a(()=>!0,"acceptsDetachedWork"),registerDetached:a(t=>{t.catch(()=>{})},"registerDetached")},W=new wr,vt=new gr,wt=new yr,gt=new Sr,yt=new _r,St=new br,_t=new kr,bt=new xr,kt=new Ir,Et=new Dr,Rt=new Tr,Pt=new Cr,xt=new $r;async function It(t=Date.now(),n=ht){const c=Or(),u=await W.runFleetPreamble(c);let h,o;try{const w=ne();h=w.filter(d=>d.status==="active"||d.status==="idle"),o=w.filter(d=>d.status==="done"||d.status==="canceled"||d.status==="dead")}catch(w){f({level:"error",event:"canvas.node_list.failed",error:w});return}await W.run(t,{rows:h,fleet:c,lifecycle:n}),vt.run(t,{rows:h});const g=await wt.run(t,{rows:[...h,...o],fleet:c});gt.run(t),xt.run(t),yt.run(t,{lifecycle:n}),St.run(t,{lifecycle:n}),_t.run(t,{lifecycle:n}),bt.run(t,{lifecycle:n}),kt.run(t,{lifecycle:n,fleet:c}),Et.run(t,{lifecycle:n}),Ge(),Pt.run(t,{fleet:c,lifecycle:n}),await Rt.run(t,{rows:h}),W.warnIfSaturated({fleet:c,lifecycle:n,thresholds:u,frozen:g})}a(It,"superviseTick");async function Dt(t){ee(te(),{recursive:!0,mode:493}),process.platform==="linux"&&Le(te(),493);const n=He();if(E(n))throw new Error(`layout refusal cannot bind occupied socket: ${n}`);const c=re((d,p)=>{if(d.method==="GET"&&d.url==="/healthz"){p.writeHead(200,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}p.writeHead(503,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((d,p)=>{c.once("error",p),c.listen(n,()=>{c.off("error",p),d()})});let u;const h=t.tcp??Ne();if(h){const d=/^(.*):(\d+)$/.exec(h);if(!d||Number(d[2])>65535)throw new Error(`invalid refusal TCP address: ${h}`);const p=re(c.listeners("request")[0]);await new Promise((s,m)=>{p.once("error",m),p.listen(Number(d[2]),d[1],()=>{p.off("error",m),s()})}),u=p}ue();let o;const g=a(()=>(o??=(async()=>{c.closeAllConnections(),u?.closeAllConnections(),await Promise.all([c,u].filter(d=>d!==void 0).map(d=>new Promise((p,s)=>d.close(m=>m?s(m):p())))),ve(process.pid),process.off("SIGINT",w),process.off("SIGTERM",w)})(),o),"cleanup"),w=a(()=>{g().then(()=>process.exit(0),d=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(d)}
|
|
2
|
-
`),process.exit(1)})},"onSignal");return process.on("SIGINT",w),process.on("SIGTERM",w),g}a(
|
|
1
|
+
var Pe=Object.defineProperty;var a=(t,n)=>Pe(t,"name",{value:n,configurable:!0});import{startPeerOutbox as xe}from"./peer/outbox.js";import{reportMachineStatus as Ie}from"../core/peer/directory.js";import{createPeerRelay as De}from"./peer/relay-service.js";import{writeFileSync as Te,readFileSync as Ce,rmSync as $e,mkdirSync as ee,existsSync as E,chmodSync as Le}from"node:fs";import{createServer as re}from"node:http";import{dirname as Ae}from"node:path";import{envCrtrdTcp as Ne,envDocumentPort as Me}from"../shared/env.js";import{serveCanvasConversationStore as Oe}from"../core/canvas/sessions.js";import{pollRunObservers as Fe}from"./runs/observer.js";import{randomBytes as Ge}from"node:crypto";import{apiSocketPath as He,crtrHome as je}from"../core/canvas/paths.js";import{activeAppUsers as Be}from"../core/spaces/users.js";import{ensureAppSpace as qe,ensureUserFiles as ze}from"../core/spaces/space.js";import{LAUNCHER_SOCKET as x}from"../launcher/spec.js";import{launcherRequest as D}from"../launcher/client.js";import{installLauncherEventSink as We}from"./launcher-events.js";import{initializeEmptyLayout as Ue,layoutNeedsMigration as Ke,socketDir as te}from"../core/layout.js";import{migrateLooseKeys as Je}from"../core/layout-migrate/loose-keys.js";import{relocateProfileFolders as Ve}from"../core/profiles/relocate.js";import{getNode as Qe,listNodesForSupervision as ne,setFrozen as oe}from"../core/canvas/index.js";import{createApiServer as Ye}from"./api/server.js";import{createDocumentListener as Xe}from"./api/document-listener.js";import{documentSecret as Ze}from"./api/document-secret.js";import{createAppListener as er}from"./api/app-listener.js";import{installLocalTerminalGrant as rr}from"../core/grants/local.js";import{onRunEventAppended as tr}from"../core/runs/events.js";import{runNotices as nr}from"./run-notices.js";import{createGrantSync as or,GRANT_SYNC_INTERVAL_S as ir}from"../core/grants/sync.js";import{runRemovalCascades as sr}from"../core/grants/remove.js";import{sweepSandboxes as ar}from"../core/grants/on-sync.js";import{rebuildCronExecutions as cr}from"./cron/run-socket.js";import{hasSyncedGrants as dr}from"../core/grants/mirror.js";import{readConfig as ie}from"../core/config.js";import{reconcileUndeliveredTickets as lr}from"./human/sweep.js";import{resetHandedOffDeliveries as pr}from"../core/canvas/deliveries.js";import{resetHandedOffDaemonActions as mr}from"../core/canvas/daemon-actions.js";import{sweepReviews as fr}from"./review/sweep.js";import{pidfilePath as B}from"./pidfile.js";import{rendererWarning as ur}from"../core/runtime/package-health.js";import{DEFAULT_INTERVAL_MS as hr}from"./supervise-cadence.js";import{createResumeDetector as vr}from"./resume-detector.js";import{setFatalFaultAnnouncer as wr}from"../core/canvas/recovery.js";import{BrokerSupervisionReconciler as gr}from"./reconcilers/broker-supervision.js";import{ControllerDeathReconciler as yr}from"./reconcilers/controller-death.js";import{NodeLifecycleReconciler as Sr}from"./reconcilers/node-lifecycle/tick.js";import{NodeDeadlineReconciler as _r}from"./reconcilers/node-deadline.js";import{CronLaneReconciler as br}from"./reconcilers/cron-lane.js";import{DeliveryLaneReconciler as kr}from"./reconcilers/delivery-lane.js";import{ActionLaneReconciler as Er}from"./reconcilers/action-lane.js";import{endLiveExecution as Rr,migrateLegacyExecutionIdentities as Pr}from"../core/canvas/executions.js";import{revokePriorEpochNodeTokens as xr}from"../core/canvas/node-tokens.js";import{PendingReviewSubmitReconciler as Ir}from"./reconcilers/pending-review-submit.js";import{StorageMaintenanceReconciler as Dr}from"./reconcilers/storage-maintenance.js";import{StorageDueWorkReconciler as Tr}from"./reconcilers/storage-due-work.js";import{BashDeadlineReconciler as Cr}from"./reconcilers/bash-deadline.js";import{FaultRetryBackstopReconciler as $r}from"./reconcilers/fault-retry-backstop.js";import{StructuredRequestReconciler as Lr}from"./reconcilers/structured-request-lane.js";import{reconcileStructuredRequestSidecars as Ar}from"./structured-requests/boot.js";import{captureLivenessSnapshot as se,capturePidCommand as Nr,captureTeardownSnapshot as ae,isPidAlive as Mr,isAnyPidAlive as ce,killProcessTreePids as de}from"../core/canvas/pid.js";import{bindFleet as Or,boundFleet as Fr}from"../core/runtime/fleet.js";import{bindDaemonControl as Gr,clearDaemonControl as Hr}from"./control.js";import{execDaemon as jr,spawnDaemon as Br}from"./manage.js";import{reconcileAndMigrateBoot as qr}from"../core/canvas/boot.js";import{resolveInstallId as zr}from"../core/canvas/install-id.js";import{setInstallId as Wr}from"../core/runtime/nodes.js";import{claimDaemonOwnership as Ur,releaseDaemonOwnership as le}from"../core/canvas/daemon-owner.js";import{migrate as Kr,openDb as pe,withCanvasWrite as Jr,bindDaemonCanvasOpenHook as Vr,clearDaemonCanvasOpenHook as me}from"../core/canvas/db.js";import{brokerSignalBus as q}from"./broker-signals/bus.js";import{brokerSignalStreams as Qr}from"./api/handlers/broker-signals.js";import{reconcilePackages as Yr}from"../core/graph/package-reconcile.js";import{startRepoSync as Xr}from"../core/graph/repo-sync/index.js";import{prewarmRecentRecipes as Zr,reapStaleSpares as et}from"../core/runtime/warm-pool.js";import{DaemonFleet as rt}from"./fleet.js";import{BOOT_MIGRATIONS as tt}from"./boot-migrations.js";import{applyJoinedOwner as nt}from"./joined-owner.js";import{emitEvent as f}from"../core/events/emit.js";import{operationIdContext as _}from"../core/events/operation-id.js";import{bindDaemonEventSource as ot}from"../core/events/source.js";import{activateCanvasImport as it,MigrationBlockedError as st,runForcedCanvasImport as at}from"../migrations/activation.js";import{convertLegacyProviderStore as ct,convertLegacyScopedStore as dt}from"../core/scoped-state/migrate.js";import{userScopeRoot as lt}from"../core/scope.js";import{secretsDir as pt}from"../core/layout.js";function z(t,n){return t.includes(` --canvas-home ${n} `)}a(z,"brokerCommandDeclaresCanvasHome");function W(t){return/ --epoch (\S+) /.exec(t)?.[1]??null}a(W,"brokerCommandEpoch");const mt=5e3,fe=150;async function ft(t){const n=se();if(n===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness"}}),new Error("prior-epoch broker census could not capture the process table");const c=je(),u=[];for(const[s,m]of n.processes){const y=m.command;if(m.zombie||!y.includes("broker-cli.js")||!z(y,c)||W(y)===t)continue;const b=y.trim().split(/\s+/).at(-1)??"";b===""||b.endsWith("broker-cli.js")||u.push({pid:s,nodeId:b})}let h=0;const o=[];for(const s of u){const m=Nr(s.pid);if(m===null||!m.includes("broker-cli.js")||!z(m,c)||W(m)===t||m.trim().split(/\s+/).at(-1)!==s.nodeId)continue;const y=ae(s.pid,null);if(!y.reused){try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):process.kill(s.pid,"SIGTERM")}catch{continue}h++,o.push({pid:s.pid,snap:y})}}const g=Date.now()+mt;for(;Date.now()<g&&o.some(s=>ce(s.snap.tree.length>0?s.snap.tree:[s.pid]));)await new Promise(s=>setTimeout(s,fe));let w=0;for(const s of o){const m=s.snap.tree.length>0?s.snap.tree:[s.pid];ce(m)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:s.pid}):de(m,"SIGKILL",s.snap.identities),w++)}w>0&&await new Promise(s=>setTimeout(s,fe));const d=se();if(d===null)throw f({level:"error",event:"daemon.census.snapshot_failed",fields:{invariant:"prior-epoch brokers are reaped before readiness",stage:"final"}}),new Error("prior-epoch broker census could not capture the final process table");const p=[];for(const[s,m]of d.processes){const y=m.command;m.zombie||!y.includes("broker-cli.js")||z(y,c)&&W(y)!==t&&p.push(s)}if(p.length>0)throw new Error(`prior-epoch broker census could not prove termination of pid(s): ${p.join(", ")}`);f({level:"info",event:"daemon.census.completed",fields:{prior_epoch_brokers:u.length,sigtermed:h,sigkilled:w}})}a(ft,"reapPriorEpochBrokers");const ut=5e3;async function ht(t){const n=Array.from(t.entries());if(n.length===0)return;for(const[o]of n){const g=Qe(o)?.status;(g==="active"||g==="idle")&&oe(o,new Date().toISOString())}const c=n.map(([,o])=>({pid:o.pid,snap:ae(o.pid,null),exited:o.handle.exited}));for(const o of c)if(!o.snap.reused)try{process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):process.kill(o.pid,"SIGTERM")}catch{}const u=new Promise(o=>{setTimeout(o,ut).unref()});await Promise.allSettled(c.map(o=>Promise.race([o.exited.then(()=>{}),u])));const h=[];for(const o of c)Mr(o.pid)&&(process.platform==="linux"&&E(x)?await D({operation:"stop",pid:o.pid}):de(o.snap.tree.length>0?o.snap.tree:[o.pid],"SIGKILL",o.snap.identities),h.push(o));await Promise.all(h.map(o=>o.exited));for(const[o]of n)Rr(o,{reason:"daemon-teardown"});f({level:"info",event:"daemon.fleet_teardown.completed",fields:{brokers:c.length,sigkilled:h.length}})}a(ht,"teardownFleet");function ue(){ee(Ae(B()),{recursive:!0}),Te(B(),String(process.pid),"utf8")}a(ue,"writePidfile");function he(t){return typeof t=="object"&&t!==null&&t.code==="ENOENT"}a(he,"isMissingPathError");function ve(t){const n=B();let c;try{c=Ce(n,"utf8")}catch(u){if(he(u))return;throw u}if(c.trim()!==String(t))throw new Error(`daemon readiness mirror does not name this invocation (expected pid ${t})`);try{$e(n)}catch(u){if(he(u))return;throw u}}a(ve,"removeOwnedPidfileMirror");const vt={acceptsDetachedWork:a(()=>!0,"acceptsDetachedWork"),registerDetached:a(t=>{t.catch(()=>{})},"registerDetached")},O=new gr;wr(t=>O.handleFatalFault(t,!1));const wt=new yr,gt=new Sr,yt=new _r,St=new br,_t=new kr,bt=new Er,kt=new Ir,Et=new Dr,Rt=new Tr,Pt=new Cr,xt=new $r,It=new Lr;async function Dt(t=Date.now(),n=vt){const c=Fr(),u=await O.runFleetPreamble(c);let h,o;try{const w=ne();h=w.filter(d=>d.status==="active"||d.status==="idle"),o=w.filter(d=>d.status==="done"||d.status==="canceled"||d.status==="dead")}catch(w){f({level:"error",event:"canvas.node_list.failed",error:w});return}await O.run(t,{rows:h,fleet:c,lifecycle:n}),wt.run(t,{rows:h});const g=await gt.run(t,{rows:[...h,...o],fleet:c});yt.run(t),It.run(t),St.run(t,{lifecycle:n}),_t.run(t,{lifecycle:n}),bt.run(t,{lifecycle:n}),kt.run(t,{lifecycle:n}),Et.run(t,{lifecycle:n,fleet:c}),Rt.run(t,{lifecycle:n}),Fe(),xt.run(t,{fleet:c,lifecycle:n}),await Pt.run(t,{rows:h}),O.warnIfSaturated({fleet:c,lifecycle:n,thresholds:u,frozen:g})}a(Dt,"superviseTick");async function Tt(t){ee(te(),{recursive:!0,mode:493}),process.platform==="linux"&&Le(te(),493);const n=He();if(E(n))throw new Error(`layout refusal cannot bind occupied socket: ${n}`);const c=re((d,p)=>{if(d.method==="GET"&&d.url==="/healthz"){p.writeHead(200,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({ok:!1,daemon_up:!0,store_accessible:!1,brokers_reconciled:!1,startup_phase:"blocked",startup_blocked:"layout_migration_required"}));return}p.writeHead(503,{"content-type":"application/json; charset=utf-8"}),p.end(JSON.stringify({error:{code:"layout_migration_required",message:"run crtr sys migrate while the runtime is offline",type:"server_error",origin:"runtime",retryable:!1}}))});await new Promise((d,p)=>{c.once("error",p),c.listen(n,()=>{c.off("error",p),d()})});let u;const h=t.tcp??Ne();if(h){const d=/^(.*):(\d+)$/.exec(h);if(!d||Number(d[2])>65535)throw new Error(`invalid refusal TCP address: ${h}`);const p=re(c.listeners("request")[0]);await new Promise((s,m)=>{p.once("error",m),p.listen(Number(d[2]),d[1],()=>{p.off("error",m),s()})}),u=p}ue();let o;const g=a(()=>(o??=(async()=>{c.closeAllConnections(),u?.closeAllConnections(),await Promise.all([c,u].filter(d=>d!==void 0).map(d=>new Promise((p,s)=>d.close(m=>m?s(m):p())))),ve(process.pid),process.off("SIGINT",w),process.off("SIGTERM",w)})(),o),"cleanup"),w=a(()=>{g().then(()=>process.exit(0),d=>{process.stderr.write(`[crtrd] layout refusal cleanup failed: ${String(d)}
|
|
2
|
+
`),process.exit(1)})},"onSignal");return process.on("SIGINT",w),process.on("SIGTERM",w),g}a(Tt,"serveLayoutRefusal");async function Ct(t={}){if(Ke())return Tt(t);Ue();const n=Ur(Ge(16).toString("hex"));if(!n.claimed){if(n.reason==="incumbent_alive")process.stderr.write(`[crtrd] already running (pid ${n.incumbentPid??"?"})
|
|
3
3
|
`);else{const r=[n.incumbentPid!==void 0?`pid ${n.incumbentPid}`:void 0,n.verdict!==void 0?`verdict ${n.verdict}`:void 0].filter(i=>i!==void 0),e=r.length>0?` (${r.join(", ")})`:"";process.stderr.write(`[crtrd] claim lost: ${n.reason}${e}
|
|
4
|
-
`),n.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(n.reason==="own_identity_indeterminate"||n.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const c=n.claim;let u=null;try{await
|
|
5
|
-
`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:i}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:l}=await import("node:os");i("user",S=>{S.paths??={},S.paths.user_files??=l()})}}await
|
|
6
|
-
`);
|
|
4
|
+
`),n.reason==="own_identity_indeterminate"&&process.stderr.write("[crtrd] cannot read its own process identity. crouter needs /proc on Linux, or the `ps` command elsewhere (Debian/Ubuntu: apt-get install procps).\n"),(n.reason==="own_identity_indeterminate"||n.reason==="incumbent_malformed")&&(process.exitCode=1)}return async()=>{}}const c=n.claim;let u=null;try{await ft(c.token),dt({kind:"user",root:lt()});for(const e of Ve())process.stderr.write(`[crtrd] ${e}
|
|
5
|
+
`);if(process.platform==="darwin"){const{readConfig:e,updateConfig:i}=await import("../core/config.js");if(e("user").paths?.user_files===void 0){const{homedir:l}=await import("node:os");i("user",S=>{S.paths??={},S.paths.user_files??=l()})}}await ct(pt());for(const e of Je())process.stderr.write(`[crtrd] ${e}
|
|
6
|
+
`);Kr(pe()),Vr(e=>q.install(e));const r=Jr(e=>Yr(e));for(const e of r.conflicts)process.stderr.write(`[crtrd] plugin documents not reconciled: ${e.message}
|
|
7
7
|
`);for(const e of r.invalid)process.stderr.write(`[crtrd] package file is not a document: ${e.path}: ${e.error}
|
|
8
8
|
`);for(const e of r.quarantined)process.stderr.write(`[crtrd] plugin "${e.name}" has a damaged ${e.file} (${e.reason}); ${e.movedTo!==void 0?`moved to ${e.movedTo}`:"left in place and skipped"}
|
|
9
9
|
`);for(const e of r.kinds.refused)process.stderr.write(`[crtrd] plugin "${e.plugin}" kind ${e.kind} not registered: ${e.message}
|
|
10
10
|
`);for(const e of r.kinds.invalid)process.stderr.write(`[crtrd] plugin "${e.plugin}" declares invalid object kinds or scopes; no kinds registered and no app token: ${e.error}
|
|
11
|
-
`);u=
|
|
11
|
+
`);u=nt(),process.platform==="linux"&&E(x)&&(We(),await D({operation:"users.sync",users:Be()})),xr(c.token)}catch(r){me(),q.close();try{const{released:e}=le(c);if(!e)throw new Error("daemon ownership claim was not released")}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}let h=!0,o,g=Promise.resolve(),w=!1,d,p;const s=new Set;let m=!1,y=!1,b,A,F,U=!1,N,T,C,$,G;const we=vr();let R,v={phase:"initializing",message:"daemon startup is running the canvas import and normal services are unavailable",blockerPaths:[]},I="initial startup activation",ge=null;function ye(){return v.phase==="ready"?null:{phase:v.phase,message:v.message,blockerPaths:v.blockerPaths}}a(ye,"startupBlock");function k(r){s.add(r),r.finally(()=>{s.delete(r)}).catch(()=>{})}a(k,"registerDetached");const K={acceptsDetachedWork:a(()=>h,"acceptsDetachedWork"),registerDetached:k};Xr(K,pe);async function Se(){try{h=!1,Qr.close(),me(),q.close(),d!==void 0&&(clearInterval(d),d=void 0),C!==void 0&&(clearInterval(C),C=void 0),G?.(),G=void 0;let r;do r=g,await r;while(r!==g);for(;s.size>0;)await Promise.allSettled([...s]);if(p!==void 0&&(await ht(p),p.dispose()),F!==void 0&&await F.close(),A!==void 0&&!U&&(U=!0,await A.close()),$!==void 0){const e=$;$=void 0,await e.stop()}if(T!==void 0){const e=T;T=void 0,await e.close()}if(N!==void 0){const e=N;N=void 0,await e.close()}if(m||(ve(process.pid),m=!0),m&&!y){const{released:e}=le(c);if(!e)throw new Error("daemon ownership claim was not released");y=!0}}finally{Hr(),process.off("SIGINT",Q),process.off("SIGTERM",Y),process.off("exit",X)}}a(Se,"cleanupAttempt");const M=1e3;let J=!1;function _e(){return J?{graceMs:M}:(J=!0,_.fresh(()=>{f({level:"info",event:"daemon.handover.armed",fields:{pid:process.pid,grace_ms:M}})}),setTimeout(()=>{be()},M),{graceMs:M})}a(_e,"requestHandover");async function be(){try{await P()}catch(r){const e=r instanceof Error?r.stack??r.message:String(r);process.stderr.write(`[crtrd] handover cleanup failed:
|
|
12
12
|
${e}
|
|
13
|
-
`),process.exit(1)}try{process.platform==="linux"&&E(x)&&(await D({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await
|
|
13
|
+
`),process.exit(1)}try{process.platform==="linux"&&E(x)&&(await D({operation:"daemon.handover"}),process.exit(0)),process.pid===1&&await jr();const r=await Br({waitForJoinLock:!0});_.fresh(()=>{f({level:"info",event:"daemon.handover.completed",fields:{predecessor_pid:process.pid,successor_pid:r.pid??r.existing_pid??null,spawned:r.started}})})}catch(r){_.fresh(()=>{f({level:"error",event:"daemon.handover.successor_failed",error:r})});const e=r instanceof Error?r.message:String(r);process.stderr.write(`[crtrd] handover successor failed to start: ${e}
|
|
14
14
|
`),process.exit(1)}process.exit(0)}a(be,"performHandover");function P(){if(b!==void 0)return b;const r=Se();return b=r,r.then(()=>{b===r&&(b=void 0)},()=>{b===r&&(b=void 0)}),r}a(P,"requestCleanup");async function V(r){try{await P(),process.exit(0)}catch(e){process.exitCode=1;const i=e instanceof Error?e.stack??`${e.name}: ${e.message}`:String(e);process.stderr.write(`[crtrd] ${r} cleanup failed:
|
|
15
15
|
${i}
|
|
16
|
-
`),process.exit(1)}}a(V,"handleSignal");function Q(){V("SIGINT")}a(Q,"onSigint");function Y(){V("SIGTERM")}a(Y,"onSigterm");function X(){P().catch(()=>{})}a(X,"onExit");function
|
|
16
|
+
`),process.exit(1)}}a(V,"handleSignal");function Q(){V("SIGINT")}a(Q,"onSigint");function Y(){V("SIGTERM")}a(Y,"onSigterm");function X(){P().catch(()=>{})}a(X,"onExit");function H(r){h&&(g=g.then(r).then(()=>{},e=>{_.fresh(()=>{f({level:"error",event:"daemon.lane_job.failed",error:e})})}))}a(H,"enqueue");function Z(){!h||w||(we.check()&&R&&(f({level:"info",event:"daemon.resume.detected"}),k(R().catch(()=>{}))),w=!0,H(async()=>{if(w=!1,!!h)try{await Dt(Date.now(),K)}catch(r){_.fresh(()=>{f({level:"error",event:"daemon.tick.failed",error:r})})}}))}a(Z,"enqueueTick");let L=null;async function ke(){if(L!==null)throw{status:500,code:"admission_failed",message:`daemon admission previously failed: ${L instanceof Error?L.message:String(L)}`};if(v.phase==="ready")return{admitted:!1,already_admitted:!0,startup_phase:"ready"};if(v.phase!=="prepared"&&v.phase!=="recovering")throw{status:409,code:"not_prepared",message:v.message};const r=v.phase==="prepared";try{return await j(),v={phase:"ready",message:"",blockerPaths:[]},r?{admitted:!0,already_admitted:!1,startup_phase:"ready"}:{admitted:!1,already_admitted:!0,startup_phase:"ready"}}catch(e){throw L=e,{status:500,code:"admission_failed",message:`daemon admission failed: ${e instanceof Error?e.message:String(e)}`}}}a(ke,"admitStartup");async function Ee(r){if(I!==null)throw{status:409,code:"migration_in_progress",message:`the canvas import is already running (${I})`};I="repair";try{const e=await at({dryRun:r.dryRun});if(r.dryRun||e.pending||v.phase!=="blocked")return{result:e,resumed:!1};if(t.prepare)return v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]},{result:e,resumed:!1};try{return await j(),v={phase:"ready",message:"",blockerPaths:[]},{result:e,resumed:!0}}catch(i){const l=i instanceof Error?i.message:String(i);return setTimeout(()=>{P().catch(()=>{})},1e3),{result:e,resumed:!1,resumeError:l}}}finally{I=null}}a(Ee,"repairMigrations");async function Re(){o=t.intervalMs??hr;const r=c.token;p=new rt({epoch:r,enqueue:H}),Or(p),v={phase:"recovering",message:"daemon startup recovery is running and normal services are unavailable",blockerPaths:[]},_.fresh(()=>{f({level:"info",event:"daemon.started",fields:{pid:process.pid,interval_ms:o}})});const e=zr();if(Wr(e),_.fresh(()=>{f({level:"info",event:"daemon.install_id.resolved",fields:{install_id:e}})}),_.fresh(()=>{let i;try{i=ur()}catch{return}i!==null&&f({level:"warn",event:"daemon.startup.warning",fields:{stage:"termrender",warning:i.slice(0,2048)}})}),await _.fresh(async()=>{await fr(),await lr(),pr(Date.now()),mr(Date.now())}),_.fresh(()=>{const{reconcile:i,migrated:l}=qr(Pr);i.changed&&f({level:"info",event:"daemon.boot_change.reconciled",fields:{cleared_count:i.cleared.length,same_boot_proven:i.sameBootProven}}),l.length>0&&f({level:"info",event:"daemon.pid_identity.migrated",fields:{migrated_count:l.length}})}),_.fresh(()=>{for(const i of ne())i.status!=="active"||i.pi_pid!=null||i.frozen_at!=null||i.respawn_not_before!=null||(oe(i.node_id,new Date().toISOString()),f({level:"warn",event:"daemon.startup.launch_debt_recorded",node_id:i.node_id}))}),_.fresh(()=>{Ar();for(const i of tt){const l=i.apply();l.changed&&f({level:"info",event:"daemon.boot_migration.applied",fields:{migration:i.id,...l.summary}})}}),k(cr()),R){const i=a(()=>{k(R().catch(()=>{})),k(Ie().then(l=>$?.observe(l)).then(()=>{},l=>{f({level:"warn",event:"daemon.peer.status_failed",error:l})}))},"refresh");k(sr().catch(()=>{})),k(ar().then(({completion:l})=>l).catch(l=>{f({level:"error",event:"daemon.grants.sweep_failed",error:l})})),i(),C=setInterval(i,ir*1e3),C.unref(),G=xe({joined:!0,register:k})}H(()=>{et(),Zr()}),d=setInterval(Z,o),Z()}a(Re,"completeStartup");function j(){return ge??=Re().catch(r=>{throw v={phase:"initializing",message:"daemon startup recovery failed and normal services are unavailable",blockerPaths:[]},r})}a(j,"resumeStartup");try{process.on("SIGINT",Q),process.on("SIGTERM",Y),process.on("exit",X),ot(),u!==null&&f({level:"info",event:"daemon.joined_owner.applied",fields:{owner:u.owner,rows:u.rows}}),N=await Oe();const r=c.token;Gr({epoch:r,requestHandover:_e,registerDetached:k,startupBlock:ye,admitStartup:ke,repairMigrations:Ee}),ue(),A=Ye({tcp:t.tcp}),await A.ready;const e=Me()??ie("user").document_port,i=Xe(e,Ze());try{await i.ready,F=i}catch(S){f({level:"warn",event:"daemon.document_listener.bind_failed",fields:{port:e,error:S instanceof Error?S.message:String(S)}})}const l=ie("user").runtime;l?.issuer?(R=or(l.issuer,k),l.unsandboxed===!0&&($=De({syncGrants:R})),tr(S=>nr().appended(S))):rr(),l?.unsandboxed===!0&&l.listener!==void 0&&f({level:"warn",event:"daemon.app_listener.refused_unsandboxed",fields:{reason:"runtime.unsandboxed is set; runtime.listener is ignored"}}),l!==void 0&&l.listener!==void 0&&l.unsandboxed!==!0&&(T=er(l,()=>dr(l.issuer.replace(/\/$/,"")),R),await T.ready);try{await it()}catch(S){if(I=null,S instanceof st)return v={phase:"blocked",message:S.message,blockerPaths:S.blockerPaths},P;throw S}process.platform==="linux"&&E(x)&&(ze(),await qe("app:terminal"));try{t.prepare?v={phase:"prepared",message:"daemon storage is prepared and normal services are unavailable until admission",blockerPaths:[]}:(await j(),v={phase:"ready",message:"",blockerPaths:[]})}finally{I=null}return P}catch(r){try{await P()}catch(e){throw new AggregateError([r,e],"daemon startup failed and cleanup also failed")}throw r}}a(Ct,"runDaemon");var co=Ct;export{z as brokerCommandDeclaresCanvasHome,W as brokerCommandEpoch,co as default,Ct as runDaemon,Dt as superviseTick,ht as teardownFleet};
|
|
@@ -61,7 +61,6 @@ export declare function treeHasDescendants(psOutput: string, rootPid: number): b
|
|
|
61
61
|
export declare class BrokerSupervisionReconciler {
|
|
62
62
|
private readonly yieldStalls;
|
|
63
63
|
private readonly wedgeNotifiedAt;
|
|
64
|
-
private readonly fatalFaultNotifiedAt;
|
|
65
64
|
private readonly unattendedSince;
|
|
66
65
|
private readonly sampleTree;
|
|
67
66
|
private readonly signal;
|
|
@@ -93,6 +92,11 @@ export declare class BrokerSupervisionReconciler {
|
|
|
93
92
|
private signalYieldStall;
|
|
94
93
|
private readTreeSample;
|
|
95
94
|
private handleWedgeDetection;
|
|
96
|
-
|
|
95
|
+
/** Publish a fatal provider fault's notice once per episode and, for a node that settles on it, end the node.
|
|
96
|
+
* The notices, the `notified_since` mark and the terminal transition commit in one write, so a daemon exit
|
|
97
|
+
* leaves the fault either fully announced or fully unannounced. The daemon registers this as the announcer
|
|
98
|
+
* every clear of an unannounced fatal fault runs first with `settle` false: a clear ends the fault on purpose
|
|
99
|
+
* (a revive, a completed turn), so it announces but does not end the node. */
|
|
100
|
+
handleFatalFault(id: string, settle?: boolean): void;
|
|
97
101
|
}
|
|
98
102
|
export {};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
var Y=Object.defineProperty;var
|
|
2
|
-
`)){const i=
|
|
3
|
-
`).map(async h=>{if(h!=="")try{await
|
|
4
|
-
`)[0]??"").slice(0,200),r
|
|
1
|
+
var Y=Object.defineProperty;var p=(s,e)=>Y(s,"name",{value:e,configurable:!0});import{existsSync as H}from"node:fs";import{LAUNCHER_SOCKET as K}from"../../launcher/spec.js";import{launcherRequest as V}from"../../launcher/client.js";import{execFile as z}from"node:child_process";import{promisify as j}from"node:util";import{getNode as T,isWarmSpare as J,subscribersOf as Q}from"../../core/canvas/index.js";import{isSafeNodeId as c}from"../../core/canvas/paths.js";import{contextDir as X}from"../../core/canvas/node-agent-paths.js";import{fullName as C}from"../../core/canvas/labels.js";import{recordedPidLiveness as D}from"../../core/canvas/pid.js";import{transition as Z}from"../../core/runtime/lifecycle.js";import{listUnnotifiedFatalFaultNodes as ee,markFaultNotified as te,readFault as O,readLiveTurn as E,recordFault as ne}from"../../core/canvas/recovery.js";import{errorClassFromFault as re}from"../../core/fault-classifier.js";import{nextFaultRetry as ie}from"../../core/runtime/fault-recovery.js";import{publishNotice as R}from"../../core/runtime/close.js";import{openDb as L,withCanvasWrite as ae}from"../../core/canvas/db.js";import{getById as se}from"../../core/graph/objects.js";import{watcherIdsOf as oe}from"../../core/graph/watches.js";import{headlessBrokerHost as S}from"../../core/runtime/host.js";import{emitEvent as u}from"../../core/events/emit.js";import{operationIdContext as F}from"../../core/events/operation-id.js";import{hasUnseenWakeMail as le,resetStaleExecutionMail as de}from"../../core/canvas/deliveries.js";import{activeBackgroundBashJobs as ce}from"../../core/bash-jobs.js";import{focusOf as ue}from"../../core/runtime/placement.js";import{hasAttachedBrokerViewers as fe}from"../api/viewer-presence.js";import{deliverLive as pe}from"../../core/runtime/interactive-deliver.js";import{BrokerUnreachableError as me}from"../../core/runtime/broker-request.js";import{formatCard as he,PARK_SUMMARY_PROMPT as _e}from"../../shared/generated-context.js";import{brokerThresholdsForDaemon as ge}from"../../core/runtime/fleet.js";import{readConfig as ke}from"../../core/config.js";import{beginPendingPark as ve,consumePendingPark as N,pendingParkIds as ye,pendingParkStartedAt as be}from"../park-pending.js";import{shouldParkAfterSummary as we}from"../park-activity.js";import{applyParkEvent as P,parkEvent as Se}from"./live-obligation.js";import{memoryDeniedRecently as Ne}from"../../core/runtime/memory-admission.js";import{envNoDaemonAutostart as Pe,envTestUnattendedParkMs as $e,parkSummaryGraceMs as Te,envUnattendedParkMs as Me}from"../../shared/env.js";const M=j(z);function Ae(s){return E(s)?.busy===!0}p(Ae,"liveBusy");function yt(s){const e=E(s)?.progressAt;if(e==null)return null;const t=Date.parse(e);return Number.isNaN(t)?null:t}p(yt,"liveTurnProgressMs");function G(s,e){return e===null?null:`${s}:${e}`}p(G,"incarnationOf");function I(s){const e=E(s.node_id),t=e?.busy===!0&&e.progressAt!==null?Date.parse(e.progressAt):Number.NaN,r=s.launched_at===null||s.launched_at===void 0?Number.NaN:Date.parse(s.launched_at);return!Number.isFinite(t)||!Number.isFinite(r)||t<r?null:t}p(I,"busySinceForIncarnation");const W=3*6e4,U=2e4,x=20*6e4,Ee=2,Fe=5*6e4;function Ie(s,e){return s===null||s.parent!=null?!1:s.lifecycle==="terminal"?!0:e==="cap_exhausted"&&s.run_id===s.node_id}p(Ie,"settlesOnFatalProviderFault");function xe(){const s=Me();if(s!==void 0)return s;const e=ke("user").lifecycle.unattendedParkMs;return Pe()?$e()??e:e}p(xe,"unattendedParkMsForDaemon");function B(){return Te()}p(B,"parkSummaryGraceMsForDaemon");function Ce(s,e,t,r){return s!==!0||e!=="refresh"||t?"leave":r===null||r<W?"pending":"kill"}p(Ce,"yieldStallVerdict");function De(s){return!s.busy||s.quietForMs===null||s.quietForMs<x?"leave":s.cpuPercent===null||s.hasDescendants!==!1&&s.cpuPercent>Ee?"pending":"wedged"}p(De,"wedgeVerdict");function q(s){const e=new Map,t=new Map;for(const r of s.split(`
|
|
2
|
+
`)){const i=r.trim();if(i==="")continue;const n=i.split(/\s+/);if(n.length<3)continue;const l=Number(n[0]),o=Number(n[1]),a=Number(n[2]);if(!Number.isFinite(l)||!Number.isFinite(o)||!Number.isFinite(a))continue;e.set(l,a);const m=t.get(o);m===void 0?t.set(o,[l]):m.push(l)}return{byPid:e,childrenOf:t}}p(q,"parsePsTree");function Oe(s,e){const{byPid:t,childrenOf:r}=q(s);if(!t.has(e))return null;let i=0;const n=[e],l=new Set;for(;n.length>0;){const o=n.pop();if(l.has(o))continue;l.add(o),i+=t.get(o)??0;const a=r.get(o);a!==void 0&&n.push(...a)}return i}p(Oe,"sumTreeCpu");function Re(s,e){const{byPid:t,childrenOf:r}=q(s);return t.has(e)?(r.get(e)??[]).length>0:null}p(Re,"treeHasDescendants");class bt{static{p(this,"BrokerSupervisionReconciler")}yieldStalls=new Map;wedgeNotifiedAt=new Map;unattendedSince=new Map;sampleTree;signal;lastOverCapWarnAt=Number.NEGATIVE_INFINITY;constructor(e={}){this.sampleTree=e.readTreeSample??(t=>this.readTreeSample(t)),this.signal=e.signal??((t,r)=>{if(process.platform==="linux"&&H(K)){if(r!=="SIGTERM"&&r!=="SIGKILL")throw new Error(`unsupported launcher signal ${r}`);return V({operation:"stop",pid:t,signal:r}).then(()=>{})}process.kill(t,r)})}async runFleetPreamble(e){return ge()}warnIfSaturated(e){if(e.fleet.size()<e.thresholds.warning||!e.lifecycle.acceptsDetachedWork())return;const t=F.fresh(()=>this.warnOverCapInTmux(e.fleet.size(),e.thresholds,e.frozen));e.lifecycle.registerDetached(t)}async run(e,t){const r=xe();for(const i of ee())await F.fresh(async()=>{try{this.handleFatalFault(i)}catch(n){u({level:"error",event:"broker.supervision.failed",...c(i)?{node_id:i}:{fields:{affected_node_id:i}},error:n})}});for(const i of t.rows){const n=i.node_id,l=t.fleet.get(n);if(l===void 0||i.status!=="active"&&i.status!=="idle"){this.unattendedSince.delete(n);continue}await F.fresh(async()=>{try{await this.handleYieldStall(i,l,t.fleet,e),await this.handleWedgeDetection(i,l,t.fleet,e),this.handleUnattendedParking(n,T(n),l.pid,e,t,r)}catch(o){this.unattendedSince.delete(n),u({level:"error",event:"broker.supervision.failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:o})}})}for(const i of this.unattendedSince.keys())t.fleet.has(i)||this.unattendedSince.delete(i);for(const i of ye())t.fleet.has(i)||N(i);de(e)}parkResetCondition(e,t,r,i){if(t.pi_session_id==null||t.pi_pid!==r||t.status==="done"||t.status==="canceled"||t.status==="dead"||t.intent!==null||ce(X(e)).length>0||ue(e)!==null||fe(e))return!0;const n=O(e);return n!==null&&n.retry.disposition==="auto"&&n.retry.by==="daemon"&&(n.kind==="connection"||n.kind==="rate-limit"||n.kind==="overloaded")&&(n.retry.nextAt!==void 0||ie(n,i)!==null)?!0:le(e)}handleUnattendedParking(e,t,r,i,n,l){if(J(e)){this.unattendedSince.delete(e),N(e);return}if(t===null){this.unattendedSince.delete(e),N(e);return}const o=this.parkResetCondition(e,t,r,i),a=be(e);if(a!==void 0){if(i-a<B())return;const _=N(e);if(_===void 0||(this.unattendedSince.delete(e),!we(e,t,_)))return;u({level:"warn",event:"broker.park.summary_expired",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},grace_ms:B()}}),P(e,"park"),S.teardown(e);return}if(o||Ae(e)){this.unattendedSince.delete(e);return}const m=this.unattendedSince.get(e);if(m===void 0){this.unattendedSince.set(e,i);return}const d=Se(e,t.lifecycle),h=d!=="cancel"&&Ne(t.grantee??"app:terminal",i)?0:l;if(!(i-m<h)){if(this.unattendedSince.delete(e),d==="release"){P(e,d),S.teardown(e);return}if(d==="cancel"){P(e,d),S.teardown(e);return}if(t.grantee!==void 0&&t.grantee!=="app:terminal"){P(e,"park"),S.teardown(e);return}n.lifecycle.acceptsDetachedWork()&&(ve(e,i),n.lifecycle.registerDetached(this.requestParkSummary(e)))}}async requestParkSummary(e){try{const t=await pe(e,he("park",{},_e),{origin:{type:"runtime",kind:"park"}});u({level:"info",event:"broker.park.summary_requested",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},route:t}})}catch(t){if(N(e)===void 0)return;if(t instanceof me){P(e,"park"),S.teardown(e);return}u({level:"error",event:"broker.park.summary_failed",...c(e)?{node_id:e}:{fields:{affected_node_id:e}},error:t})}}async warnOverCapInTmux(e,t,r){const i=Date.now();if(i-this.lastOverCapWarnAt<Fe)return;this.lastOverCapWarnAt=i;const{warning:n,automaticReviveCap:l}=t,o=r.stalled+r.waitingOnMail;u({level:"warn",event:"broker.capacity.warning",fields:{live_count:e,warning_threshold:n,hard_cap:l,frozen_stalled:r.stalled,frozen_waiting_on_mail:r.waitingOnMail}});const a=o===0?"":` \u2014 ${o} frozen (${r.stalled} stalled, ${r.waitingOnMail} waiting on mail)`,m=`\u26A0 crtrd: ${e} live brokers (warning ${n}; hard cap ${l})${a} \u2014 possible broker leak. ps -ax | grep broker-cli.js`;try{const d=await M("tmux",["list-clients","-F","#{client_name}"],{encoding:"utf8",timeout:2e3});if(typeof d.stdout!="string")return;await Promise.allSettled(d.stdout.split(`
|
|
3
|
+
`).map(async h=>{if(h!=="")try{await M("tmux",["display-message","-C","-d","10000","-c",h,m],{encoding:"utf8",timeout:2e3})}catch{try{await M("tmux",["display-message","-c",h,m],{encoding:"utf8",timeout:2e3})}catch(_){u({level:"error",event:"broker.capacity.tmux_display_failed",error:_,fields:{client:h}})}}}))}catch(d){u({level:"error",event:"broker.capacity.tmux_list_clients_failed",error:d})}}async handleYieldStall(e,t,r,i){const n=e.node_id,l=G(t.pid,e.pi_pid_identity);if(e.intent!=="refresh"||I(e)!==null||l===null){l!==null&&this.yieldStalls.delete(l);return}let o=this.yieldStalls.get(l);if(o===void 0){o={since:i},this.yieldStalls.set(l,o);return}if(Ce(!0,e.intent,!1,i-o.since)!=="kill")return;const a=T(n);if(!(r.get(n)!==t||a?.pi_pid!==t.pid||a.pi_pid_identity!==e.pi_pid_identity||D(t.pid,e.pi_pid_identity)!=="alive"))if(o.termedAt===void 0){if(!await this.signalYieldStall(n,t.pid,"SIGTERM"))return;o.termedAt=i,u({level:"warn",event:"broker.yield_stall.sigterm",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.since,grace_ms:W}})}else i-o.termedAt>=U&&await this.signalYieldStall(n,t.pid,"SIGKILL")&&u({level:"warn",event:"broker.yield_stall.sigkill",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,elapsed_ms:i-o.termedAt,grace_ms:U,stall_elapsed_ms:i-o.since}})}async signalYieldStall(e,t,r){try{return await this.signal(t,r),!0}catch(i){return u({level:"error",event:"broker.yield_stall.signal_failed",...c(e)?{node_id:e}:{},error:i,fields:{...c(e)?{}:{affected_node_id:e},pid:t,signal:r}}),!1}}async readTreeSample(e){try{const{stdout:t}=await M("ps",["-ax","-o","pid=,ppid=,pcpu="],{encoding:"utf8",timeout:2e3});if(typeof t!="string")return null;const r=Oe(t,e),i=Re(t,e);return r===null||i===null?null:{cpuPercent:r,hasDescendants:i}}catch{return null}}async handleWedgeDetection(e,t,r,i){const n=e.node_id,l=G(t.pid,e.pi_pid_identity),o=I(e);if(o===null||l===null){l!==null&&this.wedgeNotifiedAt.delete(l);return}const a=i-o,m=a>=x?await this.sampleTree(t.pid):null,d=T(n),h=d===null?null:I(d);if(r.get(n)!==t||d===null||d.pi_pid!==t.pid||d.pi_pid_identity!==e.pi_pid_identity||h!==o)return;const _=m?.cpuPercent??null,$=m?.hasDescendants??null;if(De({busy:!0,quietForMs:a,cpuPercent:_,hasDescendants:$})!=="wedged"||this.wedgeNotifiedAt.get(l)===o)return;this.wedgeNotifiedAt.set(l,o);const v=C(d),g=Math.round((a??x)/6e4),y=$??!0,k=!y&&D(t.pid,e.pi_pid_identity)!=="alive"?"pid_identity_unproven":null;let f=!1,b=!1,w;if(y)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and its process is near-idle. It will NOT recover on its own \u2014 read \`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this node) so its turn can resume with no lost context.`;else if(k!==null)w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did not SIGTERM the broker because its PID identity cannot be confirmed alive.`;else{f=!0;try{await this.signal(t.pid,"SIGTERM"),b=!0,w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`}catch(A){u({level:"error",event:"broker.wedge.signal_failed",...c(n)?{node_id:n}:{},error:A,fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,signal:"SIGTERM"}}),w=`Child wedged \u2014 ${v} (${n}) is alive and mid-turn but has shown NO engine progress for ${g}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon attempted SIGTERM to the broker, but the signal call failed.`}}u({level:"error",event:"broker.wedge.detected",...c(n)?{node_id:n}:{},fields:{...c(n)?{}:{affected_node_id:n},pid:t.pid,quiet_ms:a,cpu_percent:_,has_descendants:y,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}}});try{ne(n,{link:"daemon\u2192node",op:"engine wedge",kind:"wedged",retry:{disposition:"manual"},message:w})}catch(A){u({level:"error",event:"broker.wedge.fault_record_failed",...c(n)?{node_id:n}:{fields:{affected_node_id:n}},error:A})}R(n,"notice.crash",w,{reason:"child-wedged",child:n,quiet_ms:a,cpu_percent:_,kick_attempted:f,kicked:b,...k===null?{}:{kick_suppression_reason:k}})}handleFatalFault(e,t=!0){const r=O(e);if(r===null||r.retry.disposition!=="fatal"||r.link!=="pi\u2192provider"||r.notifiedSince===r.since)return;const i=T(e),n=i!==null?C(i):e,l=r.kind==="auth",o=(r.message.split(`
|
|
4
|
+
`)[0]??"").slice(0,200),a=r.providerError,m=a===void 0?"":` Provider: ${a.provider}/${a.model}.`+(a.status===void 0?"":` HTTP status: ${a.status}.`)+(a.errno===void 0?"":` Errno: ${a.errno}.`)+(a.code===void 0?"":` Code: ${a.code}.`)+(a.body===void 0?"":` Body: ${a.body.slice(0,200)}.`)+(a.ratelimit===void 0?"":` Rate limit: ${a.ratelimit.scope??"account"}${a.ratelimit.claim===void 0?"":` (${a.ratelimit.claim})`}${a.ratelimit.model===void 0?"":` on ${a.ratelimit.model}`}.`),d=r.kind==="context-overflow",h=Ie(i,typeof a?.code=="string"?a.code:void 0),_=d||h,$=d?`exceeded its model context window and is now dead. Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${o}`:`got a non-retryable ${r.kind} error and will NOT recover on its own; `+(h?"its terminal run has failed. ":"it stays alive, and waking it clears the fault as soon as one turn completes \u2014 a wake that hits the same cause fails again. "+(l?"Its provider credential is dead \u2014 re-authenticate (`/login`, or re-auth crtr); it recovers on its next wake once the auth reloads. ":""))+`Error: ${o}.${m}`,v=d?"failed":"hit a fatal provider fault",g=new Set(Q(e).map(f=>f.node_id)),y=oe(L(),se(L(),e)),k=[{relation:"Child",recipients:y.filter(f=>g.has(f))},{relation:"Watched node",recipients:y.filter(f=>!g.has(f))}];u({level:"error",event:"fault.fatal.fanout",...c(e)?{node_id:e}:{},fields:{...c(e)?{}:{affected_node_id:e},kind:r.kind,link:r.link,terminal_overflow:d,terminal_provider_fault:h}}),ae(()=>{for(const{relation:f,recipients:b}of k)b.length!==0&&R(e,"notice.crash",`${f} ${v} \u2014 ${n} (${e}) ${$}`,{reason:d?"child-context-overflow":"child-fault-fatal",child:e,kind:r.kind,message:o,...a===void 0?{}:{provider_error:a}},"normal",null,b);te(r.faultId),t&&_&&Z(e,"crash",{reason:d?"context_overflow":a?.code==="cap_exhausted"?"cap_exhausted":"provider_fatal",outcome:{detail:{schema:"crtr.node-outcome-detail/v1",message:r.message,fault_kind:r.kind,error_class:re(r).class}}})}),t&&_&&S.teardown(e).then(f=>{f.confirmed||u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f.error??new Error("Broker teardown not confirmed")})},f=>u({level:"error",event:"fault.fatal.teardown_failed",node_id:e,error:f}))}}export{bt as BrokerSupervisionReconciler,Ee as WEDGE_CPU_MAX_PERCENT,x as WEDGE_QUIET_MS,W as YIELD_STALL_GRACE_MS,B as parkSummaryGraceMsForDaemon,Ie as settlesOnFatalProviderFault,Oe as sumTreeCpu,Re as treeHasDescendants,xe as unattendedParkMsForDaemon,De as wedgeVerdict,Ce as yieldStallVerdict};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.526",
|
|
4
4
|
"description": "crtr — agent runtime with memory, plugins, and marketplaces",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -139,7 +139,7 @@
|
|
|
139
139
|
},
|
|
140
140
|
"license": "UNLICENSED",
|
|
141
141
|
"dependencies": {
|
|
142
|
-
"@north-light/identity": "0.3.
|
|
142
|
+
"@north-light/identity": "0.3.526",
|
|
143
143
|
"@earendil-works/pi-agent-core": "1.1.0",
|
|
144
144
|
"@earendil-works/pi-ai": "1.1.0",
|
|
145
145
|
"@earendil-works/pi-coding-agent": "1.1.0",
|
package/runtime.lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@north-light/cli",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.526",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@north-light/cli",
|
|
9
|
-
"version": "0.3.
|
|
9
|
+
"version": "0.3.526",
|
|
10
10
|
"hasInstallScript": true,
|
|
11
11
|
"license": "UNLICENSED",
|
|
12
12
|
"workspaces": [
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
"@milkdown/preset-gfm": "^7.22.3",
|
|
25
25
|
"@milkdown/prose": "^7.22.3",
|
|
26
26
|
"@milkdown/utils": "^7.22.3",
|
|
27
|
-
"@north-light/identity": "0.3.
|
|
27
|
+
"@north-light/identity": "0.3.526",
|
|
28
28
|
"cron-parser": "^5.6.0",
|
|
29
29
|
"esbuild": "^0.27.7",
|
|
30
30
|
"jose": "^6.2.12",
|
|
@@ -12468,17 +12468,17 @@
|
|
|
12468
12468
|
},
|
|
12469
12469
|
"packages/crouter-api": {
|
|
12470
12470
|
"name": "@north-light/api",
|
|
12471
|
-
"version": "0.3.
|
|
12471
|
+
"version": "0.3.526",
|
|
12472
12472
|
"license": "UNLICENSED"
|
|
12473
12473
|
},
|
|
12474
12474
|
"packages/crouter-env-docker": {
|
|
12475
12475
|
"name": "@north-light/env-docker",
|
|
12476
|
-
"version": "0.3.
|
|
12476
|
+
"version": "0.3.526",
|
|
12477
12477
|
"license": "UNLICENSED"
|
|
12478
12478
|
},
|
|
12479
12479
|
"packages/crouter-identity": {
|
|
12480
12480
|
"name": "@north-light/identity",
|
|
12481
|
-
"version": "0.3.
|
|
12481
|
+
"version": "0.3.526",
|
|
12482
12482
|
"license": "UNLICENSED",
|
|
12483
12483
|
"dependencies": {
|
|
12484
12484
|
"jose": "^6.2.1"
|
|
@@ -12486,20 +12486,20 @@
|
|
|
12486
12486
|
},
|
|
12487
12487
|
"packages/crouter-plugin": {
|
|
12488
12488
|
"name": "@north-light/plugin",
|
|
12489
|
-
"version": "0.3.
|
|
12489
|
+
"version": "0.3.526",
|
|
12490
12490
|
"license": "UNLICENSED",
|
|
12491
12491
|
"dependencies": {
|
|
12492
|
-
"@north-light/api": "0.3.
|
|
12493
|
-
"@north-light/identity": "0.3.
|
|
12492
|
+
"@north-light/api": "0.3.526",
|
|
12493
|
+
"@north-light/identity": "0.3.526"
|
|
12494
12494
|
}
|
|
12495
12495
|
},
|
|
12496
12496
|
"packages/crouter-sdk": {
|
|
12497
12497
|
"name": "@north-light/sdk",
|
|
12498
|
-
"version": "0.3.
|
|
12498
|
+
"version": "0.3.526",
|
|
12499
12499
|
"license": "UNLICENSED",
|
|
12500
12500
|
"dependencies": {
|
|
12501
|
-
"@north-light/api": "0.3.
|
|
12502
|
-
"@north-light/identity": "0.3.
|
|
12501
|
+
"@north-light/api": "0.3.526",
|
|
12502
|
+
"@north-light/identity": "0.3.526",
|
|
12503
12503
|
"jose": "^6.2.1"
|
|
12504
12504
|
},
|
|
12505
12505
|
"bin": {
|