@addai/node 0.24.1 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/capabilities.d.ts +23 -0
- package/dist/capabilities.js +48 -0
- package/dist/codex-spawn.js +19 -31
- package/dist/command-runner.js +35 -2
- package/dist/desktop/creds.d.ts +15 -0
- package/dist/desktop/creds.js +28 -0
- package/dist/desktop/manager.d.ts +19 -0
- package/dist/desktop/manager.js +87 -0
- package/dist/desktop/relay-client.js +29 -5
- package/dist/flows/engine-loader.d.ts +8 -0
- package/dist/flows/engine-loader.js +40 -0
- package/dist/flows/host.d.ts +40 -0
- package/dist/flows/host.js +125 -0
- package/dist/flows/pump.d.ts +31 -0
- package/dist/flows/pump.js +264 -0
- package/dist/flows/run.d.ts +7 -0
- package/dist/flows/run.js +219 -0
- package/dist/gemini-spawn.js +54 -2
- package/dist/grok-spawn.js +10 -39
- package/dist/heartbeat.js +24 -0
- package/dist/index.js +22 -4
- package/dist/mcp-headers.d.ts +12 -6
- package/dist/mcp-headers.js +12 -6
- package/dist/session-runner.js +32 -1
- package/dist/subagent-usage.d.ts +19 -0
- package/dist/subagent-usage.js +173 -0
- package/dist/types.d.ts +8 -0
- package/package.json +2 -1
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// What the shared engine needs from this machine, and what it must ask the
|
|
3
|
+
// server for.
|
|
4
|
+
//
|
|
5
|
+
// The engine itself is host-agnostic: it walks the graph and calls handlers.
|
|
6
|
+
// This file is the half of the contract that only makes sense here — the
|
|
7
|
+
// twenty-five local handlers run in this process, and the fifteen privileged
|
|
8
|
+
// ones become a round trip to flow-node-host, because they need a
|
|
9
|
+
// service-role Supabase client or the credential encryption key and this
|
|
10
|
+
// machine has neither.
|
|
11
|
+
//
|
|
12
|
+
// It is worth being precise about why that split is not a limitation we are
|
|
13
|
+
// working around. `httpRequest` and `code` running HERE is the entire feature:
|
|
14
|
+
// they reach this machine's LAN, its VPN, its filesystem and its IP address,
|
|
15
|
+
// which is the thing no amount of cloud capacity substitutes for. Everything
|
|
16
|
+
// else is bookkeeping the server was always going to be better at.
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.buildRegistry = buildRegistry;
|
|
19
|
+
exports.reportNodeLog = reportNodeLog;
|
|
20
|
+
const engine_loader_1 = require("./engine-loader");
|
|
21
|
+
const supabase_client_1 = require("../supabase-client");
|
|
22
|
+
const store_1 = require("../store");
|
|
23
|
+
const config_1 = require("../config");
|
|
24
|
+
const PROXY_TIMEOUT_MS = 5 * 60_000;
|
|
25
|
+
/**
|
|
26
|
+
* Ask the server to run one privileged node.
|
|
27
|
+
*
|
|
28
|
+
* Deliberately NOT retried in here. A privileged node is very often a write —
|
|
29
|
+
* enqueueing an agent request, creating a review task, sending a notification —
|
|
30
|
+
* and a transparent retry on a timeout would do it twice. The engine's own
|
|
31
|
+
* error handling decides what happens next, in the open, where the flow author
|
|
32
|
+
* can see it on the node.
|
|
33
|
+
*/
|
|
34
|
+
async function proxyOne(runId, node, inputs, parentOutputs, inputData, viaReturnEdge) {
|
|
35
|
+
const pairing = (0, store_1.readPairing)();
|
|
36
|
+
if (!pairing)
|
|
37
|
+
throw new Error('not paired — cannot reach flow-node-host');
|
|
38
|
+
const controller = new AbortController();
|
|
39
|
+
const timer = setTimeout(() => controller.abort(), PROXY_TIMEOUT_MS);
|
|
40
|
+
try {
|
|
41
|
+
const res = await fetch(`${config_1.SUPABASE_URL}/functions/v1/flow-node-host`, {
|
|
42
|
+
method: 'POST',
|
|
43
|
+
headers: {
|
|
44
|
+
'Content-Type': 'application/json',
|
|
45
|
+
'Authorization': `Bearer ${config_1.SUPABASE_ANON_KEY}`,
|
|
46
|
+
'apikey': config_1.SUPABASE_ANON_KEY,
|
|
47
|
+
'x-daemon-token': pairing.daemonToken,
|
|
48
|
+
},
|
|
49
|
+
body: JSON.stringify({
|
|
50
|
+
run_id: runId,
|
|
51
|
+
node,
|
|
52
|
+
inputs,
|
|
53
|
+
parent_outputs: parentOutputs,
|
|
54
|
+
input_data: inputData,
|
|
55
|
+
via_return_edge: viaReturnEdge,
|
|
56
|
+
}),
|
|
57
|
+
signal: controller.signal,
|
|
58
|
+
});
|
|
59
|
+
const text = await res.text();
|
|
60
|
+
let body = null;
|
|
61
|
+
try {
|
|
62
|
+
body = text ? JSON.parse(text) : null;
|
|
63
|
+
}
|
|
64
|
+
catch { /* non-JSON below */ }
|
|
65
|
+
if (!res.ok || body?.success === false) {
|
|
66
|
+
// Carry the server's own message through. It is what lands on the failed
|
|
67
|
+
// node in the run view, and "proxy failed (500)" is the difference
|
|
68
|
+
// between a five-minute fix and an afternoon.
|
|
69
|
+
throw new Error(body?.error || `flow-node-host returned ${res.status}: ${text.slice(0, 300)}`);
|
|
70
|
+
}
|
|
71
|
+
return body?.output ?? {};
|
|
72
|
+
}
|
|
73
|
+
catch (err) {
|
|
74
|
+
if (err.name === 'AbortError') {
|
|
75
|
+
throw new Error(`"${node.type}" did not finish on the server within ${PROXY_TIMEOUT_MS / 60000} minutes.`);
|
|
76
|
+
}
|
|
77
|
+
throw err;
|
|
78
|
+
}
|
|
79
|
+
finally {
|
|
80
|
+
clearTimeout(timer);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* The handler registry this machine runs a flow with.
|
|
85
|
+
*
|
|
86
|
+
* Built once per run so the proxy closes over that run's id — the server
|
|
87
|
+
* authorises each proxied node by checking this daemon still holds the claim
|
|
88
|
+
* on it.
|
|
89
|
+
*/
|
|
90
|
+
async function buildRegistry(run) {
|
|
91
|
+
const { createHandlerRegistry, findNodeInputs } = await (0, engine_loader_1.loadEngine)();
|
|
92
|
+
return createHandlerRegistry({
|
|
93
|
+
proxyFactory: (type) => async (args) => {
|
|
94
|
+
const node = args?.node ?? {};
|
|
95
|
+
const parentOutputs = args?.parentOutputs ?? {};
|
|
96
|
+
// The server's handlers take the `from_<sourceId>` input map that
|
|
97
|
+
// Engine A builds, not the whole outputs bag. Computing it here with the
|
|
98
|
+
// engine's own function is what keeps a proxied node seeing exactly what
|
|
99
|
+
// it would see running inline.
|
|
100
|
+
const inputs = findNodeInputs(node.id, run.edges || [], parentOutputs);
|
|
101
|
+
return proxyOne(run.id, { ...node, type }, inputs, parentOutputs, args?.inputData ?? {}, args?.context?.viaReturnEdge === true);
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Write one node event to the same place a cloud run writes it.
|
|
107
|
+
*
|
|
108
|
+
* Not optional polish. If a run on this machine is not as legible in the run
|
|
109
|
+
* viewer as a cloud run, self-hosting becomes the option nobody can debug, and
|
|
110
|
+
* people go back to the cloud for reasons that have nothing to do with where
|
|
111
|
+
* the work belongs.
|
|
112
|
+
*
|
|
113
|
+
* Fire-and-forget on purpose: a lost log line must never fail a flow.
|
|
114
|
+
*/
|
|
115
|
+
function reportNodeLog(run, node, status, output) {
|
|
116
|
+
void (0, supabase_client_1.rpc)('insert_node_log', {
|
|
117
|
+
p_run_id: run.id,
|
|
118
|
+
p_node_id: node?.id ?? null,
|
|
119
|
+
p_node_type: node?.type ?? null,
|
|
120
|
+
p_node_label: node?.data?.label ?? null,
|
|
121
|
+
p_status: status,
|
|
122
|
+
p_output: output ?? null,
|
|
123
|
+
p_workspace_id: run.workspace_id,
|
|
124
|
+
}).catch(() => { });
|
|
125
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export declare const DEFAULT_MAX_CONCURRENT_FLOWS = 4;
|
|
2
|
+
/** Clamp whatever the server sent into something the pump can act on. */
|
|
3
|
+
export declare function clampMaxConcurrentFlows(value: unknown): number;
|
|
4
|
+
/** Called by the heartbeat. Lowering it never interrupts a run in flight —
|
|
5
|
+
* those finish; the pump just stops claiming until it is back under the line. */
|
|
6
|
+
export declare function setMaxConcurrentFlows(value: unknown): void;
|
|
7
|
+
/** Called by the heartbeat. Turning flows on starts claiming immediately;
|
|
8
|
+
* turning it off stops claiming but lets what is running finish. */
|
|
9
|
+
export declare function setFlowsEnabled(value: unknown): void;
|
|
10
|
+
export declare function flowsEnabled(): boolean;
|
|
11
|
+
export declare function maxConcurrentFlows(): number;
|
|
12
|
+
export declare function inflightFlowCount(): number;
|
|
13
|
+
/** Runs this daemon is holding right now. Reported every 30s so the server's
|
|
14
|
+
* reclaim never requeues a healthy run that simply has nothing to say — a
|
|
15
|
+
* flow parked in a slow HTTP call is quiet, not dead. */
|
|
16
|
+
export declare function activeFlowRunIds(): string[];
|
|
17
|
+
export declare function start(): void;
|
|
18
|
+
export declare function stop(): void;
|
|
19
|
+
/**
|
|
20
|
+
* Claim right now, because something said there is work.
|
|
21
|
+
*
|
|
22
|
+
* Deliberately does not clear the backoff window: if the pump is backing off
|
|
23
|
+
* because Supabase is unhealthy, a nudge is no reason to start hammering it.
|
|
24
|
+
*/
|
|
25
|
+
export declare function pumpNow(): void;
|
|
26
|
+
/** Wait for in-flight runs, with a ceiling so a wedged run cannot block
|
|
27
|
+
* shutdown forever. */
|
|
28
|
+
export declare function drain(timeoutMs: number): Promise<{
|
|
29
|
+
drained: number;
|
|
30
|
+
timedOut: number;
|
|
31
|
+
}>;
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Claims +Ai Flows runs targeted at this machine and executes them.
|
|
3
|
+
//
|
|
4
|
+
// A deliberate sibling of request-pump, not a variation on it. The two solve
|
|
5
|
+
// the same problem — claim work from Supabase, hold a bounded number of it,
|
|
6
|
+
// heartbeat what you are holding, drain on shutdown — and every difference
|
|
7
|
+
// between them would be a difference in how the box behaves under load that
|
|
8
|
+
// nobody chose. Where the shapes match, they match on purpose.
|
|
9
|
+
//
|
|
10
|
+
// The one thing that is genuinely different is the ceiling. Flow runs and
|
|
11
|
+
// agent runs count separately, because they compete for the same machine but
|
|
12
|
+
// answer to different people: a burst of flows must never mean a chat goes
|
|
13
|
+
// unanswered, and a long agent session must never stall the flows queue.
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.DEFAULT_MAX_CONCURRENT_FLOWS = void 0;
|
|
16
|
+
exports.clampMaxConcurrentFlows = clampMaxConcurrentFlows;
|
|
17
|
+
exports.setMaxConcurrentFlows = setMaxConcurrentFlows;
|
|
18
|
+
exports.setFlowsEnabled = setFlowsEnabled;
|
|
19
|
+
exports.flowsEnabled = flowsEnabled;
|
|
20
|
+
exports.maxConcurrentFlows = maxConcurrentFlows;
|
|
21
|
+
exports.inflightFlowCount = inflightFlowCount;
|
|
22
|
+
exports.activeFlowRunIds = activeFlowRunIds;
|
|
23
|
+
exports.start = start;
|
|
24
|
+
exports.stop = stop;
|
|
25
|
+
exports.pumpNow = pumpNow;
|
|
26
|
+
exports.drain = drain;
|
|
27
|
+
const supabase_client_1 = require("../supabase-client");
|
|
28
|
+
const store_1 = require("../store");
|
|
29
|
+
const node_health_1 = require("../node-health");
|
|
30
|
+
const diskguard_1 = require("../diskguard");
|
|
31
|
+
const run_1 = require("./run");
|
|
32
|
+
// Same 2s cadence as request-pump, for the same reason: 250ms polling from a
|
|
33
|
+
// handful of daemons was enough to saturate the project's connection pool.
|
|
34
|
+
// Successful claims re-tick immediately so a queued burst still drains fast.
|
|
35
|
+
const POLL_INTERVAL_MS = 2_000;
|
|
36
|
+
// How many flow runs this machine holds at once. Mirrors max_concurrent_runs:
|
|
37
|
+
// the server's number, pushed in by the heartbeat, defaulting to 4 and
|
|
38
|
+
// specifically to 4 when the server says NOTHING. A server without the
|
|
39
|
+
// migration must read as "the behaviour you had yesterday", never as 0 (a node
|
|
40
|
+
// that accepts nothing) or unbounded.
|
|
41
|
+
exports.DEFAULT_MAX_CONCURRENT_FLOWS = 4;
|
|
42
|
+
const MAX_MAX_CONCURRENT_FLOWS = 32;
|
|
43
|
+
let maxConcurrent = exports.DEFAULT_MAX_CONCURRENT_FLOWS;
|
|
44
|
+
// Off until the server says otherwise. Unlike desktops — where "can this
|
|
45
|
+
// machine?" is answered by whether Docker is installed — the engine ships
|
|
46
|
+
// inside the daemon, so capability is never the question and consent is.
|
|
47
|
+
let enabled = false;
|
|
48
|
+
/** Clamp whatever the server sent into something the pump can act on. */
|
|
49
|
+
function clampMaxConcurrentFlows(value) {
|
|
50
|
+
// Absent is NOT zero. Number(null) and Number('') are both 0, which would
|
|
51
|
+
// quietly throttle the node to a single run — or with a naive clamp, shut it.
|
|
52
|
+
if (typeof value !== 'number' && typeof value !== 'string')
|
|
53
|
+
return exports.DEFAULT_MAX_CONCURRENT_FLOWS;
|
|
54
|
+
if (typeof value === 'string' && value.trim() === '')
|
|
55
|
+
return exports.DEFAULT_MAX_CONCURRENT_FLOWS;
|
|
56
|
+
const n = typeof value === 'number' ? value : Number(value);
|
|
57
|
+
if (!Number.isFinite(n))
|
|
58
|
+
return exports.DEFAULT_MAX_CONCURRENT_FLOWS;
|
|
59
|
+
return Math.min(MAX_MAX_CONCURRENT_FLOWS, Math.max(1, Math.floor(n)));
|
|
60
|
+
}
|
|
61
|
+
/** Called by the heartbeat. Lowering it never interrupts a run in flight —
|
|
62
|
+
* those finish; the pump just stops claiming until it is back under the line. */
|
|
63
|
+
function setMaxConcurrentFlows(value) {
|
|
64
|
+
const next = clampMaxConcurrentFlows(value);
|
|
65
|
+
if (next === maxConcurrent)
|
|
66
|
+
return;
|
|
67
|
+
console.log(`[flows-pump] flows at once: ${maxConcurrent} -> ${next}`);
|
|
68
|
+
maxConcurrent = next;
|
|
69
|
+
// Raising it should take effect now, not in two seconds — the usual reason
|
|
70
|
+
// to raise it is that work is queued up behind it.
|
|
71
|
+
if (enabled && !stopped && inflight < maxConcurrent)
|
|
72
|
+
queueMicrotask(() => { void tick(); });
|
|
73
|
+
}
|
|
74
|
+
/** Called by the heartbeat. Turning flows on starts claiming immediately;
|
|
75
|
+
* turning it off stops claiming but lets what is running finish. */
|
|
76
|
+
function setFlowsEnabled(value) {
|
|
77
|
+
const next = value === true;
|
|
78
|
+
if (next === enabled)
|
|
79
|
+
return;
|
|
80
|
+
enabled = next;
|
|
81
|
+
console.log(`[flows-pump] flows ${next ? 'enabled' : 'disabled'} for this machine`);
|
|
82
|
+
if (next && !stopped)
|
|
83
|
+
queueMicrotask(() => { void tick(); });
|
|
84
|
+
}
|
|
85
|
+
function flowsEnabled() { return enabled; }
|
|
86
|
+
function maxConcurrentFlows() { return maxConcurrent; }
|
|
87
|
+
function inflightFlowCount() { return inflight; }
|
|
88
|
+
/** Runs this daemon is holding right now. Reported every 30s so the server's
|
|
89
|
+
* reclaim never requeues a healthy run that simply has nothing to say — a
|
|
90
|
+
* flow parked in a slow HTTP call is quiet, not dead. */
|
|
91
|
+
function activeFlowRunIds() { return [...activeRunIds]; }
|
|
92
|
+
// Anti-leak backstop ONLY: how long the pump holds a slot for a run whose
|
|
93
|
+
// promise never settles. It does not stop the run and is not a time limit.
|
|
94
|
+
const SLOT_LEAK_BACKSTOP_MS = 12 * 60 * 60 * 1000;
|
|
95
|
+
let timer = null;
|
|
96
|
+
let stopped = false;
|
|
97
|
+
let inflight = 0;
|
|
98
|
+
const activeRunIds = new Set();
|
|
99
|
+
const inflightPromises = new Set();
|
|
100
|
+
// Failure backoff, deduped logging — same shape as request-pump so an outage
|
|
101
|
+
// produces ten log lines rather than a quarter of a million.
|
|
102
|
+
let consecutiveFailures = 0;
|
|
103
|
+
let lastLoggedError = '';
|
|
104
|
+
let lastLoggedAt = 0;
|
|
105
|
+
let nextClaimAllowedAt = 0;
|
|
106
|
+
let lastHealthBlock = '';
|
|
107
|
+
let lastHealthBlockAt = 0;
|
|
108
|
+
async function tick() {
|
|
109
|
+
if (stopped || !enabled)
|
|
110
|
+
return;
|
|
111
|
+
if (inflight >= maxConcurrent)
|
|
112
|
+
return;
|
|
113
|
+
if (Date.now() < nextClaimAllowedAt)
|
|
114
|
+
return;
|
|
115
|
+
// The daemon-wide circuit breaker: when Supabase is in distress every poller
|
|
116
|
+
// pauses, not just this one.
|
|
117
|
+
if ((0, supabase_client_1.rpcShouldSkip)())
|
|
118
|
+
return;
|
|
119
|
+
const pairing = (0, store_1.readPairing)();
|
|
120
|
+
if (!pairing)
|
|
121
|
+
return;
|
|
122
|
+
// Don't accept work this machine cannot finish. A full scratch disk fails
|
|
123
|
+
// every code node and every attachment write; refusing the claim leaves the
|
|
124
|
+
// run queued for a healthy machine instead of burning one of its three
|
|
125
|
+
// attempts here.
|
|
126
|
+
const health = (0, node_health_1.evaluateNodeHealth)({ diskUsedPct: (0, node_health_1.readDiskUsedPct)((0, diskguard_1.getDiskGuard)().config.tempRoot) });
|
|
127
|
+
if (!health.healthy) {
|
|
128
|
+
const msg = health.blockers.map(b => b.message).join('; ');
|
|
129
|
+
const now = Date.now();
|
|
130
|
+
if (msg !== lastHealthBlock || (now - lastHealthBlockAt) > 60_000) {
|
|
131
|
+
console.warn(`[flows-pump] refusing claim — node unhealthy: ${msg}`);
|
|
132
|
+
lastHealthBlock = msg;
|
|
133
|
+
lastHealthBlockAt = now;
|
|
134
|
+
}
|
|
135
|
+
void (0, diskguard_1.getDiskGuard)().sweep('flows-health-gate').catch(() => { });
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
let row = null;
|
|
139
|
+
try {
|
|
140
|
+
const result = await (0, supabase_client_1.rpc)('flow_run_claim_next', {
|
|
141
|
+
p_token: pairing.daemonToken,
|
|
142
|
+
});
|
|
143
|
+
row = (result && result.id) ? result : null;
|
|
144
|
+
if (consecutiveFailures > 0) {
|
|
145
|
+
console.log(`[flows-pump] recovered after ${consecutiveFailures} consecutive failures`);
|
|
146
|
+
consecutiveFailures = 0;
|
|
147
|
+
lastLoggedError = '';
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
catch (err) {
|
|
151
|
+
if (err instanceof supabase_client_1.RpcError && err.status === 401) {
|
|
152
|
+
console.error('[flows-pump] unauthorized; will retry on next heartbeat.');
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
155
|
+
consecutiveFailures++;
|
|
156
|
+
const backoffMs = Math.min(60_000, POLL_INTERVAL_MS * Math.pow(2, Math.min(consecutiveFailures, 8)));
|
|
157
|
+
nextClaimAllowedAt = Date.now() + backoffMs;
|
|
158
|
+
const msg = err.message;
|
|
159
|
+
const now = Date.now();
|
|
160
|
+
if (msg !== lastLoggedError || (now - lastLoggedAt) > 60_000) {
|
|
161
|
+
console.error(`[flows-pump] claim failed (#${consecutiveFailures}, backing off ${backoffMs}ms):`, msg);
|
|
162
|
+
lastLoggedError = msg;
|
|
163
|
+
lastLoggedAt = now;
|
|
164
|
+
}
|
|
165
|
+
return;
|
|
166
|
+
}
|
|
167
|
+
if (!row)
|
|
168
|
+
return;
|
|
169
|
+
// Never run the same run twice at once. The claim RPC should make this
|
|
170
|
+
// impossible, but a duplicate execution of a flow means duplicate writes to
|
|
171
|
+
// whatever it touches — the one failure here that is not recoverable by
|
|
172
|
+
// trying again.
|
|
173
|
+
if (activeRunIds.has(row.id)) {
|
|
174
|
+
console.warn(`[flows-pump] skipping re-claim of in-flight run [run=${row.id}]`);
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
inflight++;
|
|
178
|
+
const runId = row.id;
|
|
179
|
+
activeRunIds.add(runId);
|
|
180
|
+
// Per-run liveness, on its own timer. The 30s fleet heartbeat covers the
|
|
181
|
+
// normal case; this one exists so a single run that outlives a heartbeat
|
|
182
|
+
// hiccup still refreshes its own claim.
|
|
183
|
+
const touch = setInterval(() => {
|
|
184
|
+
void (0, supabase_client_1.rpc)('flow_run_touch', {
|
|
185
|
+
p_token: pairing.daemonToken,
|
|
186
|
+
p_run_id: runId,
|
|
187
|
+
}).catch(() => { });
|
|
188
|
+
}, 60_000);
|
|
189
|
+
touch.unref?.();
|
|
190
|
+
let resolveInflight;
|
|
191
|
+
const inflightPromise = new Promise((r) => { resolveInflight = r; });
|
|
192
|
+
inflightPromises.add(inflightPromise);
|
|
193
|
+
let backstopFired = false;
|
|
194
|
+
const backstop = new Promise((resolve) => {
|
|
195
|
+
const t = setTimeout(() => {
|
|
196
|
+
backstopFired = true;
|
|
197
|
+
console.error(`[flows-pump] run exceeded the ${SLOT_LEAK_BACKSTOP_MS / 3600000}h slot backstop `
|
|
198
|
+
+ `[run=${runId}] — freeing the slot. The run continues; the server's reclaim `
|
|
199
|
+
+ 'handles the row only if it really is dead.');
|
|
200
|
+
resolve();
|
|
201
|
+
}, SLOT_LEAK_BACKSTOP_MS);
|
|
202
|
+
t.unref?.();
|
|
203
|
+
});
|
|
204
|
+
const runPromise = (0, run_1.runFlow)(row).catch(err => console.error(`[flows-pump] runFlow threw [run=${runId}]: ${err.message}`));
|
|
205
|
+
// Drain tracking follows the REAL run, not the slot: a run that outlives its
|
|
206
|
+
// backstop must still hold up a graceful shutdown, or a restart would hand
|
|
207
|
+
// over mid-flow.
|
|
208
|
+
void runPromise.finally(() => {
|
|
209
|
+
clearInterval(touch);
|
|
210
|
+
activeRunIds.delete(runId);
|
|
211
|
+
inflightPromises.delete(inflightPromise);
|
|
212
|
+
resolveInflight();
|
|
213
|
+
});
|
|
214
|
+
Promise.race([runPromise, backstop]).finally(() => {
|
|
215
|
+
inflight = backstopFired ? Math.max(0, inflight - 1) : inflight - 1;
|
|
216
|
+
if (!stopped && enabled && inflight < maxConcurrent) {
|
|
217
|
+
queueMicrotask(() => { void tick(); });
|
|
218
|
+
}
|
|
219
|
+
});
|
|
220
|
+
// Back-to-back claims should not wait for the next interval.
|
|
221
|
+
if (!stopped && enabled && inflight < maxConcurrent) {
|
|
222
|
+
queueMicrotask(() => { void tick(); });
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
function start() {
|
|
226
|
+
if (timer || stopped)
|
|
227
|
+
return;
|
|
228
|
+
void tick();
|
|
229
|
+
timer = setInterval(() => { void tick(); }, POLL_INTERVAL_MS);
|
|
230
|
+
}
|
|
231
|
+
function stop() {
|
|
232
|
+
stopped = true;
|
|
233
|
+
if (timer) {
|
|
234
|
+
clearInterval(timer);
|
|
235
|
+
timer = null;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* Claim right now, because something said there is work.
|
|
240
|
+
*
|
|
241
|
+
* Deliberately does not clear the backoff window: if the pump is backing off
|
|
242
|
+
* because Supabase is unhealthy, a nudge is no reason to start hammering it.
|
|
243
|
+
*/
|
|
244
|
+
function pumpNow() {
|
|
245
|
+
if (stopped || !enabled)
|
|
246
|
+
return;
|
|
247
|
+
void tick();
|
|
248
|
+
}
|
|
249
|
+
/** Wait for in-flight runs, with a ceiling so a wedged run cannot block
|
|
250
|
+
* shutdown forever. */
|
|
251
|
+
async function drain(timeoutMs) {
|
|
252
|
+
if (inflightPromises.size === 0)
|
|
253
|
+
return { drained: 0, timedOut: 0 };
|
|
254
|
+
const started = inflightPromises.size;
|
|
255
|
+
const deadline = new Promise((resolve) => {
|
|
256
|
+
const t = setTimeout(resolve, timeoutMs);
|
|
257
|
+
t.unref?.();
|
|
258
|
+
});
|
|
259
|
+
await Promise.race([
|
|
260
|
+
Promise.allSettled([...inflightPromises]).then(() => undefined),
|
|
261
|
+
deadline,
|
|
262
|
+
]);
|
|
263
|
+
return { drained: started - inflightPromises.size, timedOut: inflightPromises.size };
|
|
264
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { type FlowRunRow } from './host';
|
|
2
|
+
/**
|
|
3
|
+
* Run it. Resolves when the run has reached a terminal or paused state and
|
|
4
|
+
* that state has been reported; never throws, because the pump's slot
|
|
5
|
+
* accounting is built on this settling exactly once.
|
|
6
|
+
*/
|
|
7
|
+
export declare function runFlow(run: FlowRunRow): Promise<void>;
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// Execute one claimed flow run on this machine.
|
|
3
|
+
//
|
|
4
|
+
// The engine does the walking. What this file owns is the part the engine
|
|
5
|
+
// deliberately knows nothing about: turning a database row into a graph call,
|
|
6
|
+
// keeping the server informed while it runs, and writing exactly one terminal
|
|
7
|
+
// answer at the end.
|
|
8
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
+
if (k2 === undefined) k2 = k;
|
|
10
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
+
}
|
|
14
|
+
Object.defineProperty(o, k2, desc);
|
|
15
|
+
}) : (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
o[k2] = m[k];
|
|
18
|
+
}));
|
|
19
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
20
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
21
|
+
}) : function(o, v) {
|
|
22
|
+
o["default"] = v;
|
|
23
|
+
});
|
|
24
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
25
|
+
var ownKeys = function(o) {
|
|
26
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
+
var ar = [];
|
|
28
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
+
return ar;
|
|
30
|
+
};
|
|
31
|
+
return ownKeys(o);
|
|
32
|
+
};
|
|
33
|
+
return function (mod) {
|
|
34
|
+
if (mod && mod.__esModule) return mod;
|
|
35
|
+
var result = {};
|
|
36
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
+
__setModuleDefault(result, mod);
|
|
38
|
+
return result;
|
|
39
|
+
};
|
|
40
|
+
})();
|
|
41
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
+
exports.runFlow = runFlow;
|
|
43
|
+
const os = __importStar(require("os"));
|
|
44
|
+
const engine_loader_1 = require("./engine-loader");
|
|
45
|
+
const supabase_client_1 = require("../supabase-client");
|
|
46
|
+
const store_1 = require("../store");
|
|
47
|
+
const host_1 = require("./host");
|
|
48
|
+
/**
|
|
49
|
+
* What the engine's result means for the run row.
|
|
50
|
+
*
|
|
51
|
+
* An end condition outranks the walk's own verdict: a flow that hit a goal
|
|
52
|
+
* "completed" as far as the graph is concerned, but goal and exit are the
|
|
53
|
+
* answers the runs list and the stats are counted on, and a node run that
|
|
54
|
+
* reported plain 'completed' for a goal would quietly skew both.
|
|
55
|
+
*/
|
|
56
|
+
function resolveStatus(result) {
|
|
57
|
+
const endType = result?.endConditionResult?.type;
|
|
58
|
+
if (endType === 'goal')
|
|
59
|
+
return 'goal';
|
|
60
|
+
if (endType === 'exit')
|
|
61
|
+
return 'exit';
|
|
62
|
+
switch (result?.status) {
|
|
63
|
+
case 'completed': return 'completed';
|
|
64
|
+
case 'cancelled': return 'cancelled';
|
|
65
|
+
case 'paused': return 'paused';
|
|
66
|
+
case 'failed': return 'failed';
|
|
67
|
+
default: return 'completed';
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
async function complete(run, status, fields = {}) {
|
|
71
|
+
const pairing = (0, store_1.readPairing)();
|
|
72
|
+
if (!pairing)
|
|
73
|
+
return;
|
|
74
|
+
try {
|
|
75
|
+
await (0, supabase_client_1.rpc)('flow_run_complete', {
|
|
76
|
+
p_token: pairing.daemonToken,
|
|
77
|
+
p_run_id: run.id,
|
|
78
|
+
p_status: status,
|
|
79
|
+
p_node_outputs: fields.nodeOutputs ?? null,
|
|
80
|
+
p_error: fields.error ?? null,
|
|
81
|
+
p_current_node_id: fields.currentNodeId ?? null,
|
|
82
|
+
p_metadata: fields.metadata ?? null,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
catch (err) {
|
|
86
|
+
// The run is finished on this machine either way. Losing this write means
|
|
87
|
+
// the server's reclaim sweep eventually requeues it, which is the correct
|
|
88
|
+
// fallback — better a second attempt than a run stuck 'running' forever.
|
|
89
|
+
console.error(`[flows] could not report ${status} [run=${run.id}]:`, err.message);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Run it. Resolves when the run has reached a terminal or paused state and
|
|
94
|
+
* that state has been reported; never throws, because the pump's slot
|
|
95
|
+
* accounting is built on this settling exactly once.
|
|
96
|
+
*/
|
|
97
|
+
async function runFlow(run) {
|
|
98
|
+
const label = `${run.flow_id?.slice(0, 8) ?? '?'} run=${run.id.slice(0, 8)}`;
|
|
99
|
+
const started = Date.now();
|
|
100
|
+
console.log(`[flows] starting ${label}`);
|
|
101
|
+
// Resuming: the row carries what already ran, so a run that paused on a form
|
|
102
|
+
// and came back does not re-execute the nodes before it. This is also what
|
|
103
|
+
// makes a run survive being handed to a different machine mid-flight.
|
|
104
|
+
const previousOutputs = run.node_outputs || {};
|
|
105
|
+
const previouslyExecuted = new Set(Object.keys(previousOutputs));
|
|
106
|
+
let lastNodeId = run.current_node_id;
|
|
107
|
+
try {
|
|
108
|
+
// Inside the try, deliberately. If the engine cannot load — a broken
|
|
109
|
+
// install, a native sandbox that never built — this run has to be marked
|
|
110
|
+
// failed with that message on it. Loading above the try would throw past
|
|
111
|
+
// every handler below and leave the run sitting 'running' until the
|
|
112
|
+
// server's reclaim sweep noticed, three minutes later, with no reason
|
|
113
|
+
// attached.
|
|
114
|
+
//
|
|
115
|
+
// The error classes must come from the loaded module too: `instanceof`
|
|
116
|
+
// against a separately imported copy would never match, and every paused
|
|
117
|
+
// flow would be recorded as a failure.
|
|
118
|
+
const { executeWorkflowGraph, FlowPausedError, FlowCancelledError } = await (0, engine_loader_1.loadEngine)();
|
|
119
|
+
const result = await executeWorkflowGraph({
|
|
120
|
+
nodes: run.nodes || [],
|
|
121
|
+
edges: run.edges || [],
|
|
122
|
+
inputData: run.trigger_data || {},
|
|
123
|
+
// Where to pick up. A fresh run has none and starts at its trigger; a
|
|
124
|
+
// run the cloud handed back after a form or a wait carries the node it
|
|
125
|
+
// stopped at. Without this a resumed flow would walk from the top —
|
|
126
|
+
// previouslyExecuted stops it re-running anything, but a flow whose
|
|
127
|
+
// trigger is a webhook has nothing to walk from.
|
|
128
|
+
startFromNode: run.resume_from_node_id || null,
|
|
129
|
+
previousOutputs,
|
|
130
|
+
previouslyExecuted,
|
|
131
|
+
endConditions: null,
|
|
132
|
+
hooks: {
|
|
133
|
+
onNodeStarted: (node) => {
|
|
134
|
+
lastNodeId = node?.id ?? lastNodeId;
|
|
135
|
+
},
|
|
136
|
+
onNodeCompleted: (node, output) => {
|
|
137
|
+
(0, host_1.reportNodeLog)(run, node, 'success', output);
|
|
138
|
+
},
|
|
139
|
+
onNodeError: (node, error) => {
|
|
140
|
+
(0, host_1.reportNodeLog)(run, node, 'error', {
|
|
141
|
+
error: error?.message || String(error),
|
|
142
|
+
success: false,
|
|
143
|
+
});
|
|
144
|
+
},
|
|
145
|
+
},
|
|
146
|
+
context: {
|
|
147
|
+
handlers: await (0, host_1.buildRegistry)(run),
|
|
148
|
+
workspaceId: run.workspace_id,
|
|
149
|
+
flowRunId: run.id,
|
|
150
|
+
flowId: run.flow_id,
|
|
151
|
+
versionId: run.aiflow_version_id,
|
|
152
|
+
isTest: run.is_test === true,
|
|
153
|
+
variables: run.resolved_variables || [],
|
|
154
|
+
// Cancellation. The engine asks between nodes, so a cancel from the
|
|
155
|
+
// runs list stops this machine at the next boundary rather than after
|
|
156
|
+
// the whole flow. Failures here answer "unknown" rather than throwing:
|
|
157
|
+
// a blip talking to Supabase must not kill a healthy run.
|
|
158
|
+
checkRunStatus: async (rid) => {
|
|
159
|
+
try {
|
|
160
|
+
const pairing = (0, store_1.readPairing)();
|
|
161
|
+
if (!pairing)
|
|
162
|
+
return undefined;
|
|
163
|
+
return await (0, supabase_client_1.rpc)('flow_run_status', {
|
|
164
|
+
p_token: pairing.daemonToken,
|
|
165
|
+
p_run_id: rid,
|
|
166
|
+
}) ?? undefined;
|
|
167
|
+
}
|
|
168
|
+
catch {
|
|
169
|
+
return undefined;
|
|
170
|
+
}
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
});
|
|
174
|
+
const status = resolveStatus(result);
|
|
175
|
+
await complete(run, status, {
|
|
176
|
+
nodeOutputs: result?.nodeOutputs ?? null,
|
|
177
|
+
// A flow can fail without throwing: the engine catches a handled node
|
|
178
|
+
// error and RETURNS status 'failed' with the reason on `error`. Missing
|
|
179
|
+
// it meant a node run showed "failed" in the runs list with nothing
|
|
180
|
+
// beside it, which is the least useful thing a failed run can say.
|
|
181
|
+
error: result?.error ?? null,
|
|
182
|
+
currentNodeId: lastNodeId,
|
|
183
|
+
metadata: {
|
|
184
|
+
// So the run view can say which machine ran it, and why it is not in
|
|
185
|
+
// the cloud's logs.
|
|
186
|
+
ran_on: 'node',
|
|
187
|
+
runtime_hostname: os.hostname(),
|
|
188
|
+
duration_ms: Date.now() - started,
|
|
189
|
+
},
|
|
190
|
+
});
|
|
191
|
+
console.log(`[flows] ${status} ${label} in ${((Date.now() - started) / 1000).toFixed(1)}s`);
|
|
192
|
+
}
|
|
193
|
+
catch (err) {
|
|
194
|
+
// A pause is a normal outcome, not a failure: the flow is sitting on a
|
|
195
|
+
// form, a webhook or a timer. It releases the slot and the claim; whichever
|
|
196
|
+
// machine is free when it resumes picks it up.
|
|
197
|
+
//
|
|
198
|
+
// Named rather than `instanceof` because the classes live inside the try's
|
|
199
|
+
// scope, and because a run that failed to load the engine at all has no
|
|
200
|
+
// classes to compare against.
|
|
201
|
+
const kind = err?.constructor?.name;
|
|
202
|
+
if (kind === 'FlowPausedError') {
|
|
203
|
+
await complete(run, 'waiting', {
|
|
204
|
+
nodeOutputs: err?.nodeOutputs ?? null,
|
|
205
|
+
currentNodeId: lastNodeId,
|
|
206
|
+
});
|
|
207
|
+
console.log(`[flows] waiting ${label}`);
|
|
208
|
+
return;
|
|
209
|
+
}
|
|
210
|
+
if (kind === 'FlowCancelledError') {
|
|
211
|
+
await complete(run, 'cancelled', { currentNodeId: lastNodeId });
|
|
212
|
+
console.log(`[flows] cancelled ${label}`);
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
const message = err?.message || String(err);
|
|
216
|
+
await complete(run, 'failed', { error: message, currentNodeId: lastNodeId });
|
|
217
|
+
console.error(`[flows] failed ${label}: ${message}`);
|
|
218
|
+
}
|
|
219
|
+
}
|