@elisym/cli 0.13.0 → 0.15.1

package/dist/index.js

@@ -8,7 +8,7 @@ import { generateSecretKey, getPublicKey, nip19, verifyEvent } from 'nostr-tools
 import YAML from 'yaml';
 import { Command } from 'commander';
 import { createHash } from 'node:crypto';
-import { LlmHealthMonitor, startLlmRecovery, createFreeLlmLimiterSet, ScriptBillingExhaustedError, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey } from '@elisym/sdk/llm-health';
+import { LlmHealthMonitor, startLlmRecovery, createFreeLlmLimiterSet, ScriptBillingExhaustedError, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey, LlmHealthError } from '@elisym/sdk/llm-health';
 import { lookup } from 'node:dns/promises';
 import { Socket } from 'node:net';
 import pino from 'pino';
@@ -1959,6 +1959,7 @@ var payment = new SolanaPaymentStrategy();
 var LEDGER_GC_INTERVAL_MS = 60 * 60 * 1e3;
 var LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1e3;
 var TOTAL_JOB_TIMEOUT_MS = 5 * 60 * 1e3;
+var MAX_PAID_AGE_MS = 24 * 60 * 60 * 1e3;
 var SIG_PATH_TIMEOUT_MS = 60 * 1e3;
 function resolveJobPrice(tags, skills) {
   const skill = skills.route(tags);
@@ -1969,6 +1970,22 @@ function resolveJobAsset(tags, skills) {
   return skill?.asset ?? NATIVE_SOL;
 }
 var BILLING_BODY_MARKERS3 = ["credit balance", "billing", "insufficient", "insufficient_quota"];
+var SCRIPT_BILLING_INVALID_MARKERS = [
+  "credit balance",
+  "billing",
+  "insufficient",
+  "insufficient_quota",
+  "x-api-key",
+  "invalid api key",
+  "invalid_api_key",
+  "authentication_error",
+  "unauthorized",
+  "unauthenticated"
+];
+function scriptMessageLooksLikeBillingOrInvalid(message) {
+  const lower = message.toLowerCase();
+  return SCRIPT_BILLING_INVALID_MARKERS.some((marker) => lower.includes(marker));
+}
 var AGENT_UNAVAILABLE_MESSAGE = "Agent temporarily unavailable";
 var AgentUnavailableError = class extends Error {
   constructor() {
@@ -2069,6 +2086,32 @@ var AgentRuntime = class {
    * Anything else is a transient/skill error and does NOT touch health
    * state - the recovery loop should not be poisoned by skill bugs.
    */
+  /**
+   * Build a "and N other model(s) for the same provider" suffix for
+   * cascade-narrating log lines. The SDK monitor cascades `invalid` /
+   * `billing` flips across every sibling pair sharing the same provider
+   * (shared API key); this helper just narrates that to the operator log
+   * so they can see why unrelated skills are now refusing jobs.
+   */
+  cascadeSuffix(provider, triggeringModel) {
+    if (!this.healthMonitor) {
+      return "";
+    }
+    let siblings = 0;
+    for (const entry of this.healthMonitor.snapshot()) {
+      if (entry.provider !== provider) {
+        continue;
+      }
+      if (entry.model === triggeringModel) {
+        continue;
+      }
+      siblings += 1;
+    }
+    if (siblings === 0) {
+      return "";
+    }
+    return ` (cascading to ${siblings} other model(s) for ${provider} sharing the same API key)`;
+  }
   markHealthFromExecuteError(skill, err, log, jobId) {
     if (!this.healthMonitor) {
       return false;
@@ -2084,7 +2127,7 @@ var AgentRuntime = class {
         return false;
       }
       log(
-        `${tag} Script signaled billing-exhausted (exit ${err.exitCode}). Marking ${provider}/${model} unhealthy; future jobs against this pair will be refused until recovery probe succeeds.`
+        `${tag} Script signaled billing-exhausted (exit ${err.exitCode}). Marking ${provider}/${model} unhealthy${this.cascadeSuffix(provider, model)}; future jobs against this pair will be refused until recovery probe succeeds.`
       );
       this.healthMonitor.markUnhealthyFromJob(provider, model, "billing", err.message);
       return true;
@@ -2108,11 +2151,32 @@ var AgentRuntime = class {
       const provider = skill.resolvedTriple.provider;
       const model = skill.resolvedTriple.model;
       log(
-        `${tag} LLM provider returned HTTP ${status} (${reason}). Marking ${provider}/${model} unhealthy; future jobs against this pair will be refused until recovery probe succeeds.`
+        `${tag} LLM provider returned HTTP ${status} (${reason}). Marking ${provider}/${model} unhealthy${this.cascadeSuffix(provider, model)}; future jobs against this pair will be refused until recovery probe succeeds.`
       );
       this.healthMonitor.markUnhealthyFromJob(provider, model, reason, body);
       return true;
     }
+    if (skill.mode !== "llm") {
+      const message = err instanceof Error ? err.message : String(err);
+      if (!scriptMessageLooksLikeBillingOrInvalid(message)) {
+        return false;
+      }
+      const provider = skill.llmOverride?.provider;
+      const model = skill.llmOverride?.model;
+      if (!provider || !model) {
+        log(
+          `${tag} Script failure looks like billing/invalid ("${message.slice(0, 120)}") but skill "${skill.name}" did not declare provider/model in SKILL.md - cannot gate future jobs.`
+        );
+        return false;
+      }
+      const lower = message.toLowerCase();
+      const reason = lower.includes("credit balance") || lower.includes("billing") || lower.includes("insufficient") ? "billing" : "invalid";
+      log(
+        `${tag} Script failure carries ${reason} signal in stderr. Marking ${provider}/${model} unhealthy${this.cascadeSuffix(provider, model)}; future jobs against this pair will be refused until recovery probe succeeds.`
+      );
+      this.healthMonitor.markUnhealthyFromJob(provider, model, reason, message.slice(0, 200));
+      return true;
+    }
     return false;
   }
   /** Fetch on-chain protocol config (fee, treasury). Always fetches fresh to avoid stale treasury. */
@@ -2276,9 +2340,15 @@ var AgentRuntime = class {
       const log = this.callbacks.onLog ?? console.log;
       log(`[${job.jobId.slice(0, 8)}] Error: ${e.message}`);
       const currentStatus = this.ledger.getStatus(job.jobId);
-      if (currentStatus !== "executed") {
+      const keepPaidForRecovery = e instanceof AgentUnavailableError && currentStatus === "paid";
+      if (currentStatus !== "executed" && !keepPaidForRecovery) {
         this.ledger.markFailed(job.jobId);
       }
+      if (keepPaidForRecovery) {
+        log(
+          `[${job.jobId.slice(0, 8)}] Keeping status=paid; recovery will re-execute when LLM pair recovers (24h cutoff).`
+        );
+      }
       this.callbacks.onJobError?.(job.jobId, e.message);
       let safeMessage;
       if (e instanceof AgentUnavailableError) {
@@ -2543,9 +2613,35 @@ var AgentRuntime = class {
     }
     const log = this.callbacks.onLog ?? console.log;
     log(`Recovering ${pending.length} pending jobs...`);
+    if (this.healthMonitor) {
+      const snap = this.healthMonitor.snapshot();
+      const unhealthyKeys = new Set(
+        snap.filter(
+          (entry) => entry.status === "invalid" || entry.status === "billing" || entry.status === "unavailable"
+        ).map((entry) => `${entry.provider}::${entry.model}`)
+      );
+      if (unhealthyKeys.size > 0) {
+        const hasPaidOnUnhealthy = pending.some((entry) => {
+          if (entry.status !== "paid") {
+            return false;
+          }
+          const skill = this.skills.route(entry.tags);
+          const pair = resolveHealthPair(skill);
+          return pair !== null && unhealthyKeys.has(`${pair.provider}::${pair.model}`);
+        });
+        if (hasPaidOnUnhealthy) {
+          void this.healthMonitor.refreshUnhealthy().catch(() => {
+          });
+        }
+      }
+    }
     for (const entry of pending) {
-      if (entry.retry_count >= this.config.recoveryMaxRetries) {
+      const ageMs = (Math.floor(Date.now() / 1e3) - entry.created_at) * 1e3;
+      const expired = ageMs > MAX_PAID_AGE_MS;
+      const exhaustedRetries = entry.retry_count >= this.config.recoveryMaxRetries;
+      if (expired || exhaustedRetries) {
         this.ledger.markFailed(entry.job_id);
+        const reason = expired ? "Job permanently failed: agent did not recover within 24 hours" : "Job permanently failed after maximum retries";
         if (entry.raw_event_json) {
           try {
             const rawEvent = JSON.parse(entry.raw_event_json);
@@ -2559,7 +2655,7 @@ var AgentRuntime = class {
                 encrypted: false,
                 rawEvent
               },
-              { type: "error", message: "Job permanently failed after maximum retries" }
+              { type: "error", message: reason }
             ).catch(() => {
             });
           } catch {
@@ -2592,7 +2688,6 @@ var AgentRuntime = class {
     this.jobAbortControllers.add(recoveryAbort);
     const timeout = setTimeout(() => recoveryAbort.abort(), TOTAL_JOB_TIMEOUT_MS);
     try {
-      this.ledger.incrementRetry(entry.job_id);
       const rawEvent = JSON.parse(entry.raw_event_json);
       const fakeJob = {
         jobId: entry.job_id,
@@ -2604,6 +2699,7 @@ var AgentRuntime = class {
         rawEvent
       };
       if (entry.status === "executed" && entry.result !== void 0) {
+        this.ledger.incrementRetry(entry.job_id);
         await this.transport.deliverResult(fakeJob, entry.result, entry.net_amount);
         this.ledger.markDelivered(entry.job_id);
         log(`[${entry.job_id.slice(0, 8)}] Recovery: re-delivered`);
@@ -2614,6 +2710,21 @@ var AgentRuntime = class {
           this.ledger.markFailed(entry.job_id);
           return;
         }
+        const healthPair = resolveHealthPair(skill);
+        if (this.healthMonitor && healthPair) {
+          try {
+            await this.healthMonitor.assertReady(healthPair.provider, healthPair.model);
+          } catch (err) {
+            if (err instanceof LlmHealthError) {
+              log(
+                `[${entry.job_id.slice(0, 8)}] Recovery: pair ${healthPair.provider}/${healthPair.model} still unhealthy (${err.reason}); waiting for recovery probe.`
+              );
+              return;
+            }
+            throw err;
+          }
+        }
+        this.ledger.incrementRetry(entry.job_id);
         if (skill.priceSubunits > 0 && !entry.net_amount) {
           if (entry.payment_request) {
             const verified = await this.reVerifyPayment(