c8ctl-plugin-nano 1.70.2 → 1.70.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -200,23 +200,48 @@ function normalizeSdkText(value, max) {
200
200
  * error — an "SDK status 400" message alone is useless. Tolerant of the various
201
201
  * error shapes the `@camunda8/orchestration-cluster-api` client and the underlying
202
202
  * transport surface (statusCode / status / nested response / body / data).
203
+ *
204
+ * Walks a BOUNDED `cause` chain (issue #289 review): the SDK/transport frequently
205
+ * wraps the real rejection in an outer error whose own `status`/`response` is unset,
206
+ * leaving the HTTP status only on `err.cause` (this repository already treats
207
+ * `err.cause` as a supported SDK-status shape — `isLeaseLostSettleError`,
208
+ * c8ctl-plugin.js). Without the walk a cause-wrapped 409/404 surfaces
209
+ * `status === null`, so it is neither ADOPTED (`parseAssociatedInstanceKey`) nor
210
+ * classified PERMANENT (`isPermanentCreateStatus`) and continues the retry loop this
211
+ * change exists to stop. The first non-null status / non-empty body found from the
212
+ * outermost error inward wins (the outermost shape is the most specific surface), and
213
+ * the walk is depth-capped so a cyclic cause can't loop.
203
214
  */
204
215
  export function describeSdkError(err) {
205
216
  if (err == null) return { status: null, body: null, message: String(err) };
206
- const status =
207
- err.statusCode ??
208
- err.status ??
209
- err.response?.status ??
210
- err.response?.statusCode ??
211
- (typeof err.code === 'number' ? err.code : null) ??
212
- null;
213
- let body =
214
- err.body ??
215
- err.responseBody ??
216
- err.response?.body ??
217
- err.response?.data ??
218
- err.data ??
219
- null;
217
+ let status = null;
218
+ let body = null;
219
+ let message = null;
220
+ let node = err;
221
+ for (let depth = 0; node != null && typeof node === 'object' && depth <= 4; depth += 1) {
222
+ if (status == null) {
223
+ status =
224
+ node.statusCode ??
225
+ node.status ??
226
+ node.response?.status ??
227
+ node.response?.statusCode ??
228
+ (typeof node.code === 'number' ? node.code : null) ??
229
+ null;
230
+ }
231
+ if (body == null) {
232
+ body =
233
+ node.body ??
234
+ node.responseBody ??
235
+ node.response?.body ??
236
+ node.response?.data ??
237
+ node.data ??
238
+ null;
239
+ }
240
+ if (message == null && isNonBlank(node.message)) {
241
+ message = String(node.message);
242
+ }
243
+ node = node.cause;
244
+ }
220
245
  if (body != null && typeof body !== 'string') {
221
246
  try {
222
247
  body = JSON.stringify(body);
@@ -230,9 +255,9 @@ export function describeSdkError(err) {
230
255
  // body must not sneak past the body cap) — issue #230. Matches the raw engine
231
256
  // adapter's 500-char cap (supervisor-engine.mjs readErrorBody).
232
257
  const normalizedBody = body != null ? normalizeSdkText(body, SDK_ERROR_BODY_MAX) : null;
233
- const rawMessage = isNonBlank(err.message) ? String(err.message) : String(err);
234
- const message = normalizeSdkText(rawMessage, SDK_ERROR_BODY_MAX);
235
- return { status: status ?? null, body: normalizedBody, message };
258
+ const rawMessage = message != null ? message : String(err);
259
+ const normalizedMessage = normalizeSdkText(rawMessage, SDK_ERROR_BODY_MAX);
260
+ return { status: status ?? null, body: normalizedBody, message: normalizedMessage };
236
261
  }
237
262
 
238
263
  /**
@@ -261,6 +286,51 @@ export function isPermanentTerminalStatus(status) {
261
286
  return Number.isFinite(n) && n >= 400 && n < 500;
262
287
  }
263
288
 
289
+ /**
290
+ * True when a createAgentInstance HTTP status is a PERMANENT rejection that a blind
291
+ * retry can never turn into a mint (issue #289): the retry loop was observed burning
292
+ * up to 79 attempts against a permanent 4xx (a 409 "already associated", a 404
293
+ * "element instance is not active", a 400 "invalid agent history"). A 4xx will not
294
+ * succeed on reissue, so the producer stops retrying immediately and stays inert for
295
+ * the rest of the run. 5xx / network / timeout / unknown statuses are potentially
296
+ * transient and still retry — fail-safe toward recovery.
297
+ *
298
+ * Mirrors `isPermanentTerminalStatus`'s two carve-outs (429 rate-limit, 408 request
299
+ * timeout) since those CAN succeed on a later attempt. A 409 is permanent here but is
300
+ * handled specially by the caller BEFORE this check: a 409 that names an existing
301
+ * agent instance is ADOPTED (`parseAssociatedInstanceKey`), not abandoned.
302
+ */
303
+ export function isPermanentCreateStatus(status) {
304
+ const n = Number(status);
305
+ if (n === 429 || n === 408) return false;
306
+ return Number.isFinite(n) && n >= 400 && n < 500;
307
+ }
308
+
309
+ /**
310
+ * Extract the already-associated AgentInstance key from a createAgentInstance 409
311
+ * rejection (issue #289). When an activation's run is interrupted the producer leaves
312
+ * the instance open so a retry can continue it; the retry then calls
313
+ * createAgentInstance again and the engine rejects with HTTP 409 and a body/message
314
+ * like "element instance 247794 is already associated with agent instance 247990".
315
+ * That conflict is PERMANENT — the element instance already owns instance 247990 — so
316
+ * the correct response is to ADOPT 247990 (resume appending to it), not to retry the
317
+ * create 79 times. Returns the associated instance key as a string, or null when the
318
+ * error is not a parseable "already associated" conflict.
319
+ *
320
+ * Reads BOTH the normalized message and body (describeSdkError surfaces either),
321
+ * tolerant of surrounding text and casing.
322
+ */
323
+ export function parseAssociatedInstanceKey(desc) {
324
+ if (!isPlainObject(desc)) return null;
325
+ const re = /already associated with agent instance\s+["']?(\d+)["']?/i;
326
+ for (const field of [desc.message, desc.body]) {
327
+ if (!isNonBlank(field)) continue;
328
+ const m = re.exec(String(field));
329
+ if (m && isNonBlank(m[1])) return String(m[1]).trim();
330
+ }
331
+ return null;
332
+ }
333
+
264
334
  // Redact a lease token down to a presence + short tail so it can be logged for
265
335
  // correlation without leaking the opaque fence value. For a short token a last-4
266
336
  // tail would reveal most (or all) of the value, so emit a fixed redacted marker
@@ -505,10 +575,20 @@ export function activationNamespace(job = {}) {
505
575
  return token ? shortHash(token) : '';
506
576
  }
507
577
 
508
- // Extract per-call metrics from an ACP update when the agent carries them (many
509
- // ACP agents do not — a documented ACP fidelity gap — so this is usually absent).
510
- // Reads the common usage locations and maps to the AgentHistory item metric field
511
- // names. Returns undefined when nothing usable is present.
578
+ // Extract per-call metrics from an ACP update when the agent carries them.
579
+ //
580
+ // Issue #289 — WHY instance token metrics are near-always 0: the ACP `session/update`
581
+ // stream this producer ingests does NOT carry per-turn input/output token counts. ACP's
582
+ // only usage signal is `usage_update`, which reports context-window OCCUPANCY
583
+ // (`{ used, size, cost }`), not `inputTokens`/`outputTokens` — a documented ACP fidelity
584
+ // gap (see `@nanobpm/agentic` ACP_FIDELITY_GAPS, "usage / token accounting"). The
585
+ // classifier deliberately DROPS it rather than mis-attribute occupancy as token counts,
586
+ // so an ACP-mode run records `modelCalls`/`toolCalls` (derived from the turns) but
587
+ // `metrics.inputTokens/outputTokens = 0`. That is a PROTOCOL limitation of ACP, not a
588
+ // producer bug: a native (non-ACP) transcript carries the real accounting. This reader
589
+ // therefore only records real token counts when an update ACTUALLY carries them (some
590
+ // agents attach a `usage`/`_meta.usage` block to a message) and never fabricates them
591
+ // from context-window data. Returns undefined when nothing usable is present.
512
592
  function extractMetrics(update) {
513
593
  const src =
514
594
  (isPlainObject(update?.usage) && update.usage) ||
@@ -661,6 +741,13 @@ export function createAgentInstanceProducer(opts = {}) {
661
741
  // transient create failure — the latter is RETRIED, never permanently disabled
662
742
  // (issue #230).
663
743
  let disabled = !usable;
744
+ // Set when a create attempt was made and PERMANENTLY rejected (a 4xx that will never
745
+ // mint on a blind retry — see isPermanentCreateStatus). Distinct from a construction-
746
+ // time unmet precondition (`!usable`): the preconditions were valid, the ENGINE
747
+ // rejected the create. Holds the HTTP status so the caller can report "activation
748
+ // attempted but permanently rejected (HTTP N)" rather than "unknown precondition"
749
+ // (issue #289 review).
750
+ let permanentCreateStatus = null;
664
751
  let agentInstanceKey = null;
665
752
  // Create-retry bookkeeping. `creating` dedups a concurrent in-flight attempt;
666
753
  // `createAttempts` / `lastCreateAttemptAt` pace the exponential backoff so a
@@ -1168,6 +1255,44 @@ export function createAgentInstanceProducer(opts = {}) {
1168
1255
  return false;
1169
1256
  }
1170
1257
  const d = describeSdkError(err);
1258
+ // Issue #289 — ADOPT, don't re-create, on a 409 "already associated" conflict.
1259
+ // When a prior activation's run was interrupted the producer left the instance
1260
+ // open for a retry to continue; the retry (same elementInstanceKey) then gets a
1261
+ // 409 naming the existing agent instance. The conflict is PERMANENT, so blindly
1262
+ // retrying the create burns dozens of attempts (observed up to 79) and the run
1263
+ // records no transcript. Instead, adopt the named instance and resume appending
1264
+ // to it — exactly the continuation the engine's open instance was left for.
1265
+ const adoptKey = (Number(d.status) === 409) ? parseAssociatedInstanceKey(d) : null;
1266
+ if (adoptKey) {
1267
+ agentInstanceKey = adoptKey;
1268
+ if (!activatedAt) activatedAt = now();
1269
+ if (loopIteration < 1) loopIteration = 1;
1270
+ logger?.info?.(
1271
+ `AgentInstance ${agentInstanceKey} ADOPTED for element instance ${elementInstanceKey} ` +
1272
+ `(job ${jobKey}, attempt ${attempt}) ${correlation()} — createAgentInstance returned ` +
1273
+ `409 (element instance already associated); resuming appends on the existing instance ` +
1274
+ `instead of retrying the create.`,
1275
+ );
1276
+ replayPreMintBuffer();
1277
+ return true;
1278
+ }
1279
+ // Issue #289 — a PERMANENT 4xx create rejection will never mint on a blind retry
1280
+ // (a 404 "element instance is not active" — the job was cancelled mid-create — or a
1281
+ // 400 "invalid agent history"). STOP retrying immediately and leave the producer
1282
+ // inert for the rest of the run rather than burning attempts against a dying lease
1283
+ // (the retry loop was observed making up to 79 futile attempts). A 409 is permanent
1284
+ // too, but is ADOPTED above when it names an instance; a 409 without a parseable key
1285
+ // falls through here and stops. The canonical shaped diagnostic below still carries
1286
+ // the status/body/lease/model/correlation — only its retry clause changes.
1287
+ const permanent = isPermanentCreateStatus(d.status);
1288
+ if (permanent) {
1289
+ disabled = true;
1290
+ permanentCreateStatus = Number(d.status);
1291
+ }
1292
+ const effectiveClause = permanent
1293
+ ? `not retried — the rejection is permanent (HTTP ${d.status}); no durable ` +
1294
+ `transcript will be recorded for this run (job completion unaffected).`
1295
+ : retryClause;
1171
1296
  // ONE canonical create-failure diagnostic (issue #230 / #229): the shaped
1172
1297
  // key=value line carrying the HTTP status + engine body (a lease fence vs a
1173
1298
  // schema error), the masked lease, model/provider, the correlation keys, and
@@ -1177,7 +1302,7 @@ export function createAgentInstanceProducer(opts = {}) {
1177
1302
  `AgentInstance producer: createAgentInstance failed (attempt ${attempt}) — ` +
1178
1303
  `status=${d.status ?? 'n/a'} message=${d.message} body=${d.body ?? 'n/a'} ` +
1179
1304
  `jobLease=${leaseTokenLabel(leaseToken)} model=${oneLine(def.model)} provider=${oneLine(def.provider)} ` +
1180
- `${correlation()}; ${retryClause}`,
1305
+ `${correlation()}; ${effectiveClause}`,
1181
1306
  );
1182
1307
  return false;
1183
1308
  } finally {
@@ -1550,6 +1675,39 @@ export function createAgentInstanceProducer(opts = {}) {
1550
1675
  get active() {
1551
1676
  return !!agentInstanceKey && !disabled;
1552
1677
  },
1678
+ /**
1679
+ * When the producer is PERMANENTLY unavailable (`disabled` at construction — not a
1680
+ * transient create failure), reports WHICH precondition(s) failed so the worker can
1681
+ * log a specific cause instead of merging "SDK lacks create/update" with "classifier
1682
+ * unavailable" (issue #289). Returns a short, comma-joined reason string, or null
1683
+ * when the producer is usable (or was disabled LATER by a permanent create rejection
1684
+ * rather than an unmet precondition). Deterministic — derived only from the
1685
+ * construction-time preconditions.
1686
+ */
1687
+ get unavailableReason() {
1688
+ if (usable) return null;
1689
+ const reasons = [];
1690
+ if (!camunda) reasons.push('no host SDK client');
1691
+ else {
1692
+ if (typeof camunda[SDK_CREATE] !== 'function') reasons.push(`SDK lacks ${SDK_CREATE}`);
1693
+ if (typeof camunda[SDK_UPDATE] !== 'function') reasons.push(`SDK lacks ${SDK_UPDATE}`);
1694
+ }
1695
+ if (!classify) reasons.push('ACP classifier unavailable');
1696
+ if (!isExternalAgentJob(job)) reasons.push('not an external agent job (no lease token / elementInstanceKey)');
1697
+ return reasons.length ? reasons.join('; ') : null;
1698
+ },
1699
+ /**
1700
+ * The HTTP status of a create that was ATTEMPTED and PERMANENTLY rejected by the
1701
+ * engine (a 4xx that will never mint on retry — see isPermanentCreateStatus), or
1702
+ * null when no such rejection occurred. Distinct from `unavailableReason`, which
1703
+ * reports a construction-time unmet precondition: here the preconditions were
1704
+ * valid, the producer activated, and the create was rejected. Lets the worker log
1705
+ * "activation attempted but permanently rejected (HTTP N)" instead of the
1706
+ * misleading "activation not attempted: unknown precondition" (issue #289 review).
1707
+ */
1708
+ get permanentlyRejected() {
1709
+ return permanentCreateStatus;
1710
+ },
1553
1711
  /**
1554
1712
  * True when the producer is NOT yet active but its create is still armed to retry
1555
1713
  * on the ACP hot path (issue #230): a create was attempted (createAttempts > 0)