c8ctl-plugin-nano 1.70.2 → 1.70.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -2
- package/agent-checkpoint.mjs +243 -6
- package/agent-instance.mjs +180 -22
- package/c8ctl-plugin.js +2313 -121
- package/package.json +8 -8
- package/supervisor.dist.js +11 -11
package/agent-instance.mjs
CHANGED
|
@@ -200,23 +200,48 @@ function normalizeSdkText(value, max) {
|
|
|
200
200
|
* error — an "SDK status 400" message alone is useless. Tolerant of the various
|
|
201
201
|
* error shapes the `@camunda8/orchestration-cluster-api` client and the underlying
|
|
202
202
|
* transport surface (statusCode / status / nested response / body / data).
|
|
203
|
+
*
|
|
204
|
+
* Walks a BOUNDED `cause` chain (issue #289 review): the SDK/transport frequently
|
|
205
|
+
* wraps the real rejection in an outer error whose own `status`/`response` is unset,
|
|
206
|
+
* leaving the HTTP status only on `err.cause` (this repository already treats
|
|
207
|
+
* `err.cause` as a supported SDK-status shape — `isLeaseLostSettleError`,
|
|
208
|
+
* c8ctl-plugin.js). Without the walk a cause-wrapped 409/404 surfaces
|
|
209
|
+
* `status === null`, so it is neither ADOPTED (`parseAssociatedInstanceKey`) nor
|
|
210
|
+
* classified PERMANENT (`isPermanentCreateStatus`) and continues the retry loop this
|
|
211
|
+
* change exists to stop. The first non-null status / non-empty body found from the
|
|
212
|
+
* outermost error inward wins (the outermost shape is the most specific surface), and
|
|
213
|
+
* the walk is depth-capped so a cyclic cause can't loop.
|
|
203
214
|
*/
|
|
204
215
|
export function describeSdkError(err) {
|
|
205
216
|
if (err == null) return { status: null, body: null, message: String(err) };
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
(
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
217
|
+
let status = null;
|
|
218
|
+
let body = null;
|
|
219
|
+
let message = null;
|
|
220
|
+
let node = err;
|
|
221
|
+
for (let depth = 0; node != null && typeof node === 'object' && depth <= 4; depth += 1) {
|
|
222
|
+
if (status == null) {
|
|
223
|
+
status =
|
|
224
|
+
node.statusCode ??
|
|
225
|
+
node.status ??
|
|
226
|
+
node.response?.status ??
|
|
227
|
+
node.response?.statusCode ??
|
|
228
|
+
(typeof node.code === 'number' ? node.code : null) ??
|
|
229
|
+
null;
|
|
230
|
+
}
|
|
231
|
+
if (body == null) {
|
|
232
|
+
body =
|
|
233
|
+
node.body ??
|
|
234
|
+
node.responseBody ??
|
|
235
|
+
node.response?.body ??
|
|
236
|
+
node.response?.data ??
|
|
237
|
+
node.data ??
|
|
238
|
+
null;
|
|
239
|
+
}
|
|
240
|
+
if (message == null && isNonBlank(node.message)) {
|
|
241
|
+
message = String(node.message);
|
|
242
|
+
}
|
|
243
|
+
node = node.cause;
|
|
244
|
+
}
|
|
220
245
|
if (body != null && typeof body !== 'string') {
|
|
221
246
|
try {
|
|
222
247
|
body = JSON.stringify(body);
|
|
@@ -230,9 +255,9 @@ export function describeSdkError(err) {
|
|
|
230
255
|
// body must not sneak past the body cap) — issue #230. Matches the raw engine
|
|
231
256
|
// adapter's 500-char cap (supervisor-engine.mjs readErrorBody).
|
|
232
257
|
const normalizedBody = body != null ? normalizeSdkText(body, SDK_ERROR_BODY_MAX) : null;
|
|
233
|
-
const rawMessage =
|
|
234
|
-
const
|
|
235
|
-
return { status: status ?? null, body: normalizedBody, message };
|
|
258
|
+
const rawMessage = message != null ? message : String(err);
|
|
259
|
+
const normalizedMessage = normalizeSdkText(rawMessage, SDK_ERROR_BODY_MAX);
|
|
260
|
+
return { status: status ?? null, body: normalizedBody, message: normalizedMessage };
|
|
236
261
|
}
|
|
237
262
|
|
|
238
263
|
/**
|
|
@@ -261,6 +286,51 @@ export function isPermanentTerminalStatus(status) {
|
|
|
261
286
|
return Number.isFinite(n) && n >= 400 && n < 500;
|
|
262
287
|
}
|
|
263
288
|
|
|
289
|
+
/**
|
|
290
|
+
* True when a createAgentInstance HTTP status is a PERMANENT rejection that a blind
|
|
291
|
+
* retry can never turn into a mint (issue #289): the retry loop was observed burning
|
|
292
|
+
* up to 79 attempts against a permanent 4xx (a 409 "already associated", a 404
|
|
293
|
+
* "element instance is not active", a 400 "invalid agent history"). A 4xx will not
|
|
294
|
+
* succeed on reissue, so the producer stops retrying immediately and stays inert for
|
|
295
|
+
* the rest of the run. 5xx / network / timeout / unknown statuses are potentially
|
|
296
|
+
* transient and still retry — fail-safe toward recovery.
|
|
297
|
+
*
|
|
298
|
+
* Mirrors `isPermanentTerminalStatus`'s two carve-outs (429 rate-limit, 408 request
|
|
299
|
+
* timeout) since those CAN succeed on a later attempt. A 409 is permanent here but is
|
|
300
|
+
* handled specially by the caller BEFORE this check: a 409 that names an existing
|
|
301
|
+
* agent instance is ADOPTED (`parseAssociatedInstanceKey`), not abandoned.
|
|
302
|
+
*/
|
|
303
|
+
export function isPermanentCreateStatus(status) {
|
|
304
|
+
const n = Number(status);
|
|
305
|
+
if (n === 429 || n === 408) return false;
|
|
306
|
+
return Number.isFinite(n) && n >= 400 && n < 500;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Extract the already-associated AgentInstance key from a createAgentInstance 409
|
|
311
|
+
* rejection (issue #289). When an activation's run is interrupted the producer leaves
|
|
312
|
+
* the instance open so a retry can continue it; the retry then calls
|
|
313
|
+
* createAgentInstance again and the engine rejects with HTTP 409 and a body/message
|
|
314
|
+
* like "element instance 247794 is already associated with agent instance 247990".
|
|
315
|
+
* That conflict is PERMANENT — the element instance already owns instance 247990 — so
|
|
316
|
+
* the correct response is to ADOPT 247990 (resume appending to it), not to retry the
|
|
317
|
+
* create 79 times. Returns the associated instance key as a string, or null when the
|
|
318
|
+
* error is not a parseable "already associated" conflict.
|
|
319
|
+
*
|
|
320
|
+
* Reads BOTH the normalized message and body (describeSdkError surfaces either),
|
|
321
|
+
* tolerant of surrounding text and casing.
|
|
322
|
+
*/
|
|
323
|
+
export function parseAssociatedInstanceKey(desc) {
|
|
324
|
+
if (!isPlainObject(desc)) return null;
|
|
325
|
+
const re = /already associated with agent instance\s+["']?(\d+)["']?/i;
|
|
326
|
+
for (const field of [desc.message, desc.body]) {
|
|
327
|
+
if (!isNonBlank(field)) continue;
|
|
328
|
+
const m = re.exec(String(field));
|
|
329
|
+
if (m && isNonBlank(m[1])) return String(m[1]).trim();
|
|
330
|
+
}
|
|
331
|
+
return null;
|
|
332
|
+
}
|
|
333
|
+
|
|
264
334
|
// Redact a lease token down to a presence + short tail so it can be logged for
|
|
265
335
|
// correlation without leaking the opaque fence value. For a short token a last-4
|
|
266
336
|
// tail would reveal most (or all) of the value, so emit a fixed redacted marker
|
|
@@ -505,10 +575,20 @@ export function activationNamespace(job = {}) {
|
|
|
505
575
|
return token ? shortHash(token) : '';
|
|
506
576
|
}
|
|
507
577
|
|
|
508
|
-
// Extract per-call metrics from an ACP update when the agent carries them
|
|
509
|
-
//
|
|
510
|
-
//
|
|
511
|
-
//
|
|
578
|
+
// Extract per-call metrics from an ACP update when the agent carries them.
|
|
579
|
+
//
|
|
580
|
+
// Issue #289 — WHY instance token metrics are near-always 0: the ACP `session/update`
|
|
581
|
+
// stream this producer ingests does NOT carry per-turn input/output token counts. ACP's
|
|
582
|
+
// only usage signal is `usage_update`, which reports context-window OCCUPANCY
|
|
583
|
+
// (`{ used, size, cost }`), not `inputTokens`/`outputTokens` — a documented ACP fidelity
|
|
584
|
+
// gap (see `@nanobpm/agentic` ACP_FIDELITY_GAPS, "usage / token accounting"). The
|
|
585
|
+
// classifier deliberately DROPS it rather than mis-attribute occupancy as token counts,
|
|
586
|
+
// so an ACP-mode run records `modelCalls`/`toolCalls` (derived from the turns) but
|
|
587
|
+
// `metrics.inputTokens/outputTokens = 0`. That is a PROTOCOL limitation of ACP, not a
|
|
588
|
+
// producer bug: a native (non-ACP) transcript carries the real accounting. This reader
|
|
589
|
+
// therefore only records real token counts when an update ACTUALLY carries them (some
|
|
590
|
+
// agents attach a `usage`/`_meta.usage` block to a message) and never fabricates them
|
|
591
|
+
// from context-window data. Returns undefined when nothing usable is present.
|
|
512
592
|
function extractMetrics(update) {
|
|
513
593
|
const src =
|
|
514
594
|
(isPlainObject(update?.usage) && update.usage) ||
|
|
@@ -661,6 +741,13 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
661
741
|
// transient create failure — the latter is RETRIED, never permanently disabled
|
|
662
742
|
// (issue #230).
|
|
663
743
|
let disabled = !usable;
|
|
744
|
+
// Set when a create attempt was made and PERMANENTLY rejected (a 4xx that will never
|
|
745
|
+
// mint on a blind retry — see isPermanentCreateStatus). Distinct from a construction-
|
|
746
|
+
// time unmet precondition (`!usable`): the preconditions were valid, the ENGINE
|
|
747
|
+
// rejected the create. Holds the HTTP status so the caller can report "activation
|
|
748
|
+
// attempted but permanently rejected (HTTP N)" rather than "unknown precondition"
|
|
749
|
+
// (issue #289 review).
|
|
750
|
+
let permanentCreateStatus = null;
|
|
664
751
|
let agentInstanceKey = null;
|
|
665
752
|
// Create-retry bookkeeping. `creating` dedups a concurrent in-flight attempt;
|
|
666
753
|
// `createAttempts` / `lastCreateAttemptAt` pace the exponential backoff so a
|
|
@@ -1168,6 +1255,44 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
1168
1255
|
return false;
|
|
1169
1256
|
}
|
|
1170
1257
|
const d = describeSdkError(err);
|
|
1258
|
+
// Issue #289 — ADOPT, don't re-create, on a 409 "already associated" conflict.
|
|
1259
|
+
// When a prior activation's run was interrupted the producer left the instance
|
|
1260
|
+
// open for a retry to continue; the retry (same elementInstanceKey) then gets a
|
|
1261
|
+
// 409 naming the existing agent instance. The conflict is PERMANENT, so blindly
|
|
1262
|
+
// retrying the create burns dozens of attempts (observed up to 79) and the run
|
|
1263
|
+
// records no transcript. Instead, adopt the named instance and resume appending
|
|
1264
|
+
// to it — exactly the continuation the engine's open instance was left for.
|
|
1265
|
+
const adoptKey = (Number(d.status) === 409) ? parseAssociatedInstanceKey(d) : null;
|
|
1266
|
+
if (adoptKey) {
|
|
1267
|
+
agentInstanceKey = adoptKey;
|
|
1268
|
+
if (!activatedAt) activatedAt = now();
|
|
1269
|
+
if (loopIteration < 1) loopIteration = 1;
|
|
1270
|
+
logger?.info?.(
|
|
1271
|
+
`AgentInstance ${agentInstanceKey} ADOPTED for element instance ${elementInstanceKey} ` +
|
|
1272
|
+
`(job ${jobKey}, attempt ${attempt}) ${correlation()} — createAgentInstance returned ` +
|
|
1273
|
+
`409 (element instance already associated); resuming appends on the existing instance ` +
|
|
1274
|
+
`instead of retrying the create.`,
|
|
1275
|
+
);
|
|
1276
|
+
replayPreMintBuffer();
|
|
1277
|
+
return true;
|
|
1278
|
+
}
|
|
1279
|
+
// Issue #289 — a PERMANENT 4xx create rejection will never mint on a blind retry
|
|
1280
|
+
// (a 404 "element instance is not active" — the job was cancelled mid-create — or a
|
|
1281
|
+
// 400 "invalid agent history"). STOP retrying immediately and leave the producer
|
|
1282
|
+
// inert for the rest of the run rather than burning attempts against a dying lease
|
|
1283
|
+
// (the retry loop was observed making up to 79 futile attempts). A 409 is permanent
|
|
1284
|
+
// too, but is ADOPTED above when it names an instance; a 409 without a parseable key
|
|
1285
|
+
// falls through here and stops. The canonical shaped diagnostic below still carries
|
|
1286
|
+
// the status/body/lease/model/correlation — only its retry clause changes.
|
|
1287
|
+
const permanent = isPermanentCreateStatus(d.status);
|
|
1288
|
+
if (permanent) {
|
|
1289
|
+
disabled = true;
|
|
1290
|
+
permanentCreateStatus = Number(d.status);
|
|
1291
|
+
}
|
|
1292
|
+
const effectiveClause = permanent
|
|
1293
|
+
? `not retried — the rejection is permanent (HTTP ${d.status}); no durable ` +
|
|
1294
|
+
`transcript will be recorded for this run (job completion unaffected).`
|
|
1295
|
+
: retryClause;
|
|
1171
1296
|
// ONE canonical create-failure diagnostic (issue #230 / #229): the shaped
|
|
1172
1297
|
// key=value line carrying the HTTP status + engine body (a lease fence vs a
|
|
1173
1298
|
// schema error), the masked lease, model/provider, the correlation keys, and
|
|
@@ -1177,7 +1302,7 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
1177
1302
|
`AgentInstance producer: createAgentInstance failed (attempt ${attempt}) — ` +
|
|
1178
1303
|
`status=${d.status ?? 'n/a'} message=${d.message} body=${d.body ?? 'n/a'} ` +
|
|
1179
1304
|
`jobLease=${leaseTokenLabel(leaseToken)} model=${oneLine(def.model)} provider=${oneLine(def.provider)} ` +
|
|
1180
|
-
`${correlation()}; ${
|
|
1305
|
+
`${correlation()}; ${effectiveClause}`,
|
|
1181
1306
|
);
|
|
1182
1307
|
return false;
|
|
1183
1308
|
} finally {
|
|
@@ -1550,6 +1675,39 @@ export function createAgentInstanceProducer(opts = {}) {
|
|
|
1550
1675
|
get active() {
|
|
1551
1676
|
return !!agentInstanceKey && !disabled;
|
|
1552
1677
|
},
|
|
1678
|
+
/**
|
|
1679
|
+
* When the producer is PERMANENTLY unavailable (`disabled` at construction — not a
|
|
1680
|
+
* transient create failure), reports WHICH precondition(s) failed so the worker can
|
|
1681
|
+
* log a specific cause instead of merging "SDK lacks create/update" with "classifier
|
|
1682
|
+
* unavailable" (issue #289). Returns a short, comma-joined reason string, or null
|
|
1683
|
+
* when the producer is usable (or was disabled LATER by a permanent create rejection
|
|
1684
|
+
* rather than an unmet precondition). Deterministic — derived only from the
|
|
1685
|
+
* construction-time preconditions.
|
|
1686
|
+
*/
|
|
1687
|
+
get unavailableReason() {
|
|
1688
|
+
if (usable) return null;
|
|
1689
|
+
const reasons = [];
|
|
1690
|
+
if (!camunda) reasons.push('no host SDK client');
|
|
1691
|
+
else {
|
|
1692
|
+
if (typeof camunda[SDK_CREATE] !== 'function') reasons.push(`SDK lacks ${SDK_CREATE}`);
|
|
1693
|
+
if (typeof camunda[SDK_UPDATE] !== 'function') reasons.push(`SDK lacks ${SDK_UPDATE}`);
|
|
1694
|
+
}
|
|
1695
|
+
if (!classify) reasons.push('ACP classifier unavailable');
|
|
1696
|
+
if (!isExternalAgentJob(job)) reasons.push('not an external agent job (no lease token / elementInstanceKey)');
|
|
1697
|
+
return reasons.length ? reasons.join('; ') : null;
|
|
1698
|
+
},
|
|
1699
|
+
/**
|
|
1700
|
+
* The HTTP status of a create that was ATTEMPTED and PERMANENTLY rejected by the
|
|
1701
|
+
* engine (a 4xx that will never mint on retry — see isPermanentCreateStatus), or
|
|
1702
|
+
* null when no such rejection occurred. Distinct from `unavailableReason`, which
|
|
1703
|
+
* reports a construction-time unmet precondition: here the preconditions were
|
|
1704
|
+
* valid, the producer activated, and the create was rejected. Lets the worker log
|
|
1705
|
+
* "activation attempted but permanently rejected (HTTP N)" instead of the
|
|
1706
|
+
* misleading "activation not attempted: unknown precondition" (issue #289 review).
|
|
1707
|
+
*/
|
|
1708
|
+
get permanentlyRejected() {
|
|
1709
|
+
return permanentCreateStatus;
|
|
1710
|
+
},
|
|
1553
1711
|
/**
|
|
1554
1712
|
* True when the producer is NOT yet active but its create is still armed to retry
|
|
1555
1713
|
* on the ACP hot path (issue #230): a create was attempted (createAttempts > 0)
|