@elisym/cli 0.13.0 → 0.14.0

package/dist/index.js

@@ -8,7 +8,7 @@ import { generateSecretKey, getPublicKey, nip19, verifyEvent } from 'nostr-tools
 import YAML from 'yaml';
 import { Command } from 'commander';
 import { createHash } from 'node:crypto';
-import { LlmHealthMonitor, startLlmRecovery, createFreeLlmLimiterSet, ScriptBillingExhaustedError, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey } from '@elisym/sdk/llm-health';
+import { LlmHealthMonitor, startLlmRecovery, createFreeLlmLimiterSet, ScriptBillingExhaustedError, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey, LlmHealthError } from '@elisym/sdk/llm-health';
 import { lookup } from 'node:dns/promises';
 import { Socket } from 'node:net';
 import pino from 'pino';
@@ -1959,6 +1959,7 @@ var payment = new SolanaPaymentStrategy();
 var LEDGER_GC_INTERVAL_MS = 60 * 60 * 1e3;
 var LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1e3;
 var TOTAL_JOB_TIMEOUT_MS = 5 * 60 * 1e3;
+var MAX_PAID_AGE_MS = 24 * 60 * 60 * 1e3;
 var SIG_PATH_TIMEOUT_MS = 60 * 1e3;
 function resolveJobPrice(tags, skills) {
   const skill = skills.route(tags);
@@ -2276,9 +2277,15 @@ var AgentRuntime = class {
       const log = this.callbacks.onLog ?? console.log;
       log(`[${job.jobId.slice(0, 8)}] Error: ${e.message}`);
       const currentStatus = this.ledger.getStatus(job.jobId);
-      if (currentStatus !== "executed") {
+      const keepPaidForRecovery = e instanceof AgentUnavailableError && currentStatus === "paid";
+      if (currentStatus !== "executed" && !keepPaidForRecovery) {
         this.ledger.markFailed(job.jobId);
       }
+      if (keepPaidForRecovery) {
+        log(
+          `[${job.jobId.slice(0, 8)}] Keeping status=paid; recovery will re-execute when LLM pair recovers (24h cutoff).`
+        );
+      }
       this.callbacks.onJobError?.(job.jobId, e.message);
       let safeMessage;
       if (e instanceof AgentUnavailableError) {
@@ -2543,9 +2550,35 @@ var AgentRuntime = class {
     }
     const log = this.callbacks.onLog ?? console.log;
     log(`Recovering ${pending.length} pending jobs...`);
+    if (this.healthMonitor) {
+      const snap = this.healthMonitor.snapshot();
+      const unhealthyKeys = new Set(
+        snap.filter(
+          (entry) => entry.status === "invalid" || entry.status === "billing" || entry.status === "unavailable"
+        ).map((entry) => `${entry.provider}::${entry.model}`)
+      );
+      if (unhealthyKeys.size > 0) {
+        const hasPaidOnUnhealthy = pending.some((entry) => {
+          if (entry.status !== "paid") {
+            return false;
+          }
+          const skill = this.skills.route(entry.tags);
+          const pair = resolveHealthPair(skill);
+          return pair !== null && unhealthyKeys.has(`${pair.provider}::${pair.model}`);
+        });
+        if (hasPaidOnUnhealthy) {
+          void this.healthMonitor.refreshUnhealthy().catch(() => {
+          });
+        }
+      }
+    }
     for (const entry of pending) {
-      if (entry.retry_count >= this.config.recoveryMaxRetries) {
+      const ageMs = (Math.floor(Date.now() / 1e3) - entry.created_at) * 1e3;
+      const expired = ageMs > MAX_PAID_AGE_MS;
+      const exhaustedRetries = entry.retry_count >= this.config.recoveryMaxRetries;
+      if (expired || exhaustedRetries) {
         this.ledger.markFailed(entry.job_id);
+        const reason = expired ? "Job permanently failed: agent did not recover within 24 hours" : "Job permanently failed after maximum retries";
         if (entry.raw_event_json) {
           try {
             const rawEvent = JSON.parse(entry.raw_event_json);
@@ -2559,7 +2592,7 @@ var AgentRuntime = class {
                 encrypted: false,
                 rawEvent
               },
-              { type: "error", message: "Job permanently failed after maximum retries" }
+              { type: "error", message: reason }
             ).catch(() => {
             });
           } catch {
@@ -2592,7 +2625,6 @@ var AgentRuntime = class {
     this.jobAbortControllers.add(recoveryAbort);
     const timeout = setTimeout(() => recoveryAbort.abort(), TOTAL_JOB_TIMEOUT_MS);
     try {
-      this.ledger.incrementRetry(entry.job_id);
       const rawEvent = JSON.parse(entry.raw_event_json);
       const fakeJob = {
         jobId: entry.job_id,
@@ -2604,6 +2636,7 @@ var AgentRuntime = class {
         rawEvent
       };
       if (entry.status === "executed" && entry.result !== void 0) {
+        this.ledger.incrementRetry(entry.job_id);
         await this.transport.deliverResult(fakeJob, entry.result, entry.net_amount);
         this.ledger.markDelivered(entry.job_id);
         log(`[${entry.job_id.slice(0, 8)}] Recovery: re-delivered`);
@@ -2614,6 +2647,21 @@ var AgentRuntime = class {
           this.ledger.markFailed(entry.job_id);
           return;
         }
+        const healthPair = resolveHealthPair(skill);
+        if (this.healthMonitor && healthPair) {
+          try {
+            await this.healthMonitor.assertReady(healthPair.provider, healthPair.model);
+          } catch (err) {
+            if (err instanceof LlmHealthError) {
+              log(
+                `[${entry.job_id.slice(0, 8)}] Recovery: pair ${healthPair.provider}/${healthPair.model} still unhealthy (${err.reason}); waiting for recovery probe.`
+              );
+              return;
+            }
+            throw err;
+          }
+        }
+        this.ledger.incrementRetry(entry.job_id);
         if (skill.priceSubunits > 0 && !entry.net_amount) {
           if (entry.payment_request) {
             const verified = await this.reVerifyPayment(