@elisym/cli 0.12.0 → 0.14.0

package/README.md

@@ -127,6 +127,8 @@ my-project/
 
 Skills are defined in `SKILL.md` files inside `./skills/<skill-name>/`. Each file has YAML frontmatter (between `---` delimiters) that describes the skill, followed by a markdown body that becomes the LLM system prompt.
 
+> Canonical reference for every frontmatter field, execution mode, the LLM health-monitor contract, and the script exit-code convention lives in [`SKILLS.md`](./SKILLS.md). The summary below covers the common cases; consult `SKILLS.md` for the full schema.
+
 ```markdown
 ---
 name: youtube-summary

package/dist/index.js

@@ -8,7 +8,7 @@ import { generateSecretKey, getPublicKey, nip19, verifyEvent } from 'nostr-tools
 import YAML from 'yaml';
 import { Command } from 'commander';
 import { createHash } from 'node:crypto';
-import { LlmHealthMonitor, startLlmRecovery, createFreeLlmLimiterSet, ScriptBillingExhaustedError, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey } from '@elisym/sdk/llm-health';
+import { LlmHealthMonitor, startLlmRecovery, createFreeLlmLimiterSet, ScriptBillingExhaustedError, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey, LlmHealthError } from '@elisym/sdk/llm-health';
 import { lookup } from 'node:dns/promises';
 import { Socket } from 'node:net';
 import pino from 'pino';
@@ -1959,6 +1959,7 @@ var payment = new SolanaPaymentStrategy();
 var LEDGER_GC_INTERVAL_MS = 60 * 60 * 1e3;
 var LEDGER_RETENTION_MS = 30 * 24 * 60 * 60 * 1e3;
 var TOTAL_JOB_TIMEOUT_MS = 5 * 60 * 1e3;
+var MAX_PAID_AGE_MS = 24 * 60 * 60 * 1e3;
 var SIG_PATH_TIMEOUT_MS = 60 * 1e3;
 function resolveJobPrice(tags, skills) {
   const skill = skills.route(tags);
@@ -1969,6 +1970,13 @@ function resolveJobAsset(tags, skills) {
   return skill?.asset ?? NATIVE_SOL;
 }
 var BILLING_BODY_MARKERS3 = ["credit balance", "billing", "insufficient", "insufficient_quota"];
+var AGENT_UNAVAILABLE_MESSAGE = "Agent temporarily unavailable";
+var AgentUnavailableError = class extends Error {
+  constructor() {
+    super(AGENT_UNAVAILABLE_MESSAGE);
+    this.name = "AgentUnavailableError";
+  }
+};
 function bodyLooksLikeBilling3(body) {
   const lower = body.toLowerCase();
   return BILLING_BODY_MARKERS3.some((marker) => lower.includes(marker));
@@ -2064,7 +2072,7 @@ var AgentRuntime = class {
    */
   markHealthFromExecuteError(skill, err, log, jobId) {
     if (!this.healthMonitor) {
-      return;
+      return false;
     }
     const tag = `[${jobId.slice(0, 8)}]`;
     if (err instanceof ScriptBillingExhaustedError) {
@@ -2074,19 +2082,19 @@ var AgentRuntime = class {
         log(
           `${tag} Script returned exit ${err.exitCode} (billing-exhausted) but skill "${skill.name}" did not declare provider/model in SKILL.md - cannot gate future jobs.`
         );
-        return;
+        return false;
       }
       log(
         `${tag} Script signaled billing-exhausted (exit ${err.exitCode}). Marking ${provider}/${model} unhealthy; future jobs against this pair will be refused until recovery probe succeeds.`
       );
       this.healthMonitor.markUnhealthyFromJob(provider, model, "billing", err.message);
-      return;
+      return true;
     }
     if (skill.mode === "llm" && skill.resolvedTriple) {
       const message = err instanceof Error ? err.message : String(err);
       const match = /API error:\s*(\d{3})\b\s*(.*)/i.exec(message);
       if (!match) {
-        return;
+        return false;
       }
       const status = Number(match[1]);
       const body = (match[2] ?? "").slice(0, 200);
@@ -2095,7 +2103,7 @@ var AgentRuntime = class {
       const isBilling400 = status === 400 && bodyLooksLikeBilling3(body);
       const isBilling429 = status === 429 && bodyLooksLikeBilling3(body);
       if (!isBillingStatus && !isAuthStatus && !isBilling400 && !isBilling429) {
-        return;
+        return false;
       }
       const reason = isAuthStatus ? "invalid" : "billing";
       const provider = skill.resolvedTriple.provider;
@@ -2104,7 +2112,9 @@ var AgentRuntime = class {
         `${tag} LLM provider returned HTTP ${status} (${reason}). Marking ${provider}/${model} unhealthy; future jobs against this pair will be refused until recovery probe succeeds.`
       );
       this.healthMonitor.markUnhealthyFromJob(provider, model, reason, body);
+      return true;
     }
+    return false;
   }
   /** Fetch on-chain protocol config (fee, treasury). Always fetches fresh to avoid stale treasury. */
   async fetchProtocolConfig() {
@@ -2267,11 +2277,24 @@ var AgentRuntime = class {
       const log = this.callbacks.onLog ?? console.log;
       log(`[${job.jobId.slice(0, 8)}] Error: ${e.message}`);
       const currentStatus = this.ledger.getStatus(job.jobId);
-      if (currentStatus !== "executed") {
+      const keepPaidForRecovery = e instanceof AgentUnavailableError && currentStatus === "paid";
+      if (currentStatus !== "executed" && !keepPaidForRecovery) {
         this.ledger.markFailed(job.jobId);
       }
+      if (keepPaidForRecovery) {
+        log(
+          `[${job.jobId.slice(0, 8)}] Keeping status=paid; recovery will re-execute when LLM pair recovers (24h cutoff).`
+        );
+      }
       this.callbacks.onJobError?.(job.jobId, e.message);
-      const safeMessage = e.message?.includes("API") ? "Internal processing error" : e.message ?? "Unknown error";
+      let safeMessage;
+      if (e instanceof AgentUnavailableError) {
+        safeMessage = e.message;
+      } else if (e.message?.includes("API")) {
+        safeMessage = "Internal processing error";
+      } else {
+        safeMessage = e.message ?? "Unknown error";
+      }
       await this.transport.sendFeedback(job, { type: "error", message: safeMessage }).catch(() => {
       });
     } finally {
@@ -2295,7 +2318,7 @@ var AgentRuntime = class {
         log(`[${job.jobId.slice(0, 8)}] LLM health gate refused job: ${detail}`);
         await this.transport.sendFeedback(job, {
           type: "error",
-          message: "Service temporarily unavailable, try again later"
+          message: AGENT_UNAVAILABLE_MESSAGE
         }).catch(() => {
         });
         return;
@@ -2348,7 +2371,10 @@ var AgentRuntime = class {
         { ...this.skillCtx, signal }
       );
     } catch (err) {
-      this.markHealthFromExecuteError(skill, err, log, job.jobId);
+      const flippedToUnhealthy = this.markHealthFromExecuteError(skill, err, log, job.jobId);
+      if (flippedToUnhealthy) {
+        throw new AgentUnavailableError();
+      }
       throw err;
     }
     this.ledger.markExecuted(job.jobId, output.data);
@@ -2524,9 +2550,35 @@ var AgentRuntime = class {
     }
     const log = this.callbacks.onLog ?? console.log;
     log(`Recovering ${pending.length} pending jobs...`);
+    if (this.healthMonitor) {
+      const snap = this.healthMonitor.snapshot();
+      const unhealthyKeys = new Set(
+        snap.filter(
+          (entry) => entry.status === "invalid" || entry.status === "billing" || entry.status === "unavailable"
+        ).map((entry) => `${entry.provider}::${entry.model}`)
+      );
+      if (unhealthyKeys.size > 0) {
+        const hasPaidOnUnhealthy = pending.some((entry) => {
+          if (entry.status !== "paid") {
+            return false;
+          }
+          const skill = this.skills.route(entry.tags);
+          const pair = resolveHealthPair(skill);
+          return pair !== null && unhealthyKeys.has(`${pair.provider}::${pair.model}`);
+        });
+        if (hasPaidOnUnhealthy) {
+          void this.healthMonitor.refreshUnhealthy().catch(() => {
+          });
+        }
+      }
+    }
     for (const entry of pending) {
-      if (entry.retry_count >= this.config.recoveryMaxRetries) {
+      const ageMs = (Math.floor(Date.now() / 1e3) - entry.created_at) * 1e3;
+      const expired = ageMs > MAX_PAID_AGE_MS;
+      const exhaustedRetries = entry.retry_count >= this.config.recoveryMaxRetries;
+      if (expired || exhaustedRetries) {
         this.ledger.markFailed(entry.job_id);
+        const reason = expired ? "Job permanently failed: agent did not recover within 24 hours" : "Job permanently failed after maximum retries";
         if (entry.raw_event_json) {
           try {
             const rawEvent = JSON.parse(entry.raw_event_json);
@@ -2540,7 +2592,7 @@ var AgentRuntime = class {
                 encrypted: false,
                 rawEvent
               },
-              { type: "error", message: "Job permanently failed after maximum retries" }
+              { type: "error", message: reason }
             ).catch(() => {
             });
           } catch {
@@ -2573,7 +2625,6 @@ var AgentRuntime = class {
     this.jobAbortControllers.add(recoveryAbort);
     const timeout = setTimeout(() => recoveryAbort.abort(), TOTAL_JOB_TIMEOUT_MS);
     try {
-      this.ledger.incrementRetry(entry.job_id);
       const rawEvent = JSON.parse(entry.raw_event_json);
       const fakeJob = {
         jobId: entry.job_id,
@@ -2585,6 +2636,7 @@ var AgentRuntime = class {
         rawEvent
       };
       if (entry.status === "executed" && entry.result !== void 0) {
+        this.ledger.incrementRetry(entry.job_id);
         await this.transport.deliverResult(fakeJob, entry.result, entry.net_amount);
         this.ledger.markDelivered(entry.job_id);
         log(`[${entry.job_id.slice(0, 8)}] Recovery: re-delivered`);
@@ -2595,6 +2647,21 @@ var AgentRuntime = class {
           this.ledger.markFailed(entry.job_id);
           return;
         }
+        const healthPair = resolveHealthPair(skill);
+        if (this.healthMonitor && healthPair) {
+          try {
+            await this.healthMonitor.assertReady(healthPair.provider, healthPair.model);
+          } catch (err) {
+            if (err instanceof LlmHealthError) {
+              log(
+                `[${entry.job_id.slice(0, 8)}] Recovery: pair ${healthPair.provider}/${healthPair.model} still unhealthy (${err.reason}); waiting for recovery probe.`
+              );
+              return;
+            }
+            throw err;
+          }
+        }
+        this.ledger.incrementRetry(entry.job_id);
         if (skill.priceSubunits > 0 && !entry.net_amount) {
           if (entry.payment_request) {
             const verified = await this.reVerifyPayment(