@elisym/cli 0.11.5 → 0.13.0

package/README.md

@@ -127,6 +127,8 @@ my-project/
 
 Skills are defined in `SKILL.md` files inside `./skills/<skill-name>/`. Each file has YAML frontmatter (between `---` delimiters) that describes the skill, followed by a markdown body that becomes the LLM system prompt.
 
+> Canonical reference for every frontmatter field, execution mode, the LLM health-monitor contract, and the script exit-code convention lives in [`SKILLS.md`](./SKILLS.md). The summary below covers the common cases; consult `SKILLS.md` for the full schema.
+
 ```markdown
 ---
 name: youtube-summary

package/dist/index.js

@@ -2,13 +2,13 @@
 import { readFileSync, readdirSync, statSync, renameSync, mkdirSync, writeFileSync } from 'node:fs';
 import { dirname, join, resolve, basename, relative, sep } from 'node:path';
 import { SolanaPaymentStrategy, validateAgentName, RELAYS, ElisymIdentity, formatSol, formatAssetAmount, USDC_SOLANA_DEVNET, ElisymClient, MediaService, jobRequestKind, DEFAULT_KIND_OFFSET, toDTag, DEFAULTS, makeCensor, DEFAULT_REDACT_PATHS, createSlidingWindowLimiter, getProtocolProgramId, getProtocolConfig, LIMITS, calculateProtocolFee, BoundedSet, KIND_JOB_FEEDBACK, NATIVE_SOL } from '@elisym/sdk';
-import { ElisymYamlSchema, resolveInHome, resolveInProject, createAgentDir, writeYamlInitial, writeSecrets, listAgents, loadAgent, writeYaml, agentPaths, readMediaCache, lookupCachedUrl, newCacheEntry, writeMediaCache } from '@elisym/sdk/agent-store';
+import { ElisymYamlSchema, resolveInHome, resolveInProject, createAgentDir, writeYamlInitial, writeExampleSkillTemplate, writeSecrets, listAgents, loadAgent, writeYaml, agentPaths, readMediaCache, lookupCachedUrl, newCacheEntry, writeMediaCache } from '@elisym/sdk/agent-store';
 import { isAddress, createSolanaRpc, address } from '@solana/kit';
 import { generateSecretKey, getPublicKey, nip19, verifyEvent } from 'nostr-tools';
 import YAML from 'yaml';
 import { Command } from 'commander';
 import { createHash } from 'node:crypto';
-import { LlmHealthMonitor, startLlmHeartbeat, createFreeLlmLimiterSet, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey } from '@elisym/sdk/llm-health';
+import { LlmHealthMonitor, startLlmRecovery, createFreeLlmLimiterSet, ScriptBillingExhaustedError, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey } from '@elisym/sdk/llm-health';
 import { lookup } from 'node:dns/promises';
 import { Socket } from 'node:net';
 import pino from 'pino';
@@ -341,7 +341,7 @@ var init_anthropic = __esm({
 // src/llm/providers/openai-compatible.ts
 function createOpenAICompatibleProvider(config) {
   const billingMarkers = [...DEFAULT_BILLING_MARKERS, ...config.extraBillingMarkers ?? []];
-  function bodyLooksLikeBilling3(body) {
+  function bodyLooksLikeBilling4(body) {
     const lower = body.toLowerCase();
     return billingMarkers.some((marker) => lower.includes(marker));
   }
@@ -423,7 +423,7 @@ function createOpenAICompatibleProvider(config) {
       if (response.status === 402) {
         return { ok: false, reason: "billing", status: response.status, body };
       }
-      if ((response.status === 400 || response.status === 429) && bodyLooksLikeBilling3(body)) {
+      if ((response.status === 400 || response.status === 429) && bodyLooksLikeBilling4(body)) {
         return { ok: false, reason: "billing", status: response.status, body };
       }
       return {
@@ -1163,6 +1163,7 @@ async function cmdInit(nameArg, options = {}) {
   }
   const llmApiKeys = Object.fromEntries(collectedKeys);
   await writeYamlInitial(created.dir, yaml);
+  await writeExampleSkillTemplate(created.dir);
   await writeSecrets(
     created.dir,
     {
@@ -1967,6 +1968,36 @@ function resolveJobAsset(tags, skills) {
   const skill = skills.route(tags);
   return skill?.asset ?? NATIVE_SOL;
 }
+var BILLING_BODY_MARKERS3 = ["credit balance", "billing", "insufficient", "insufficient_quota"];
+var AGENT_UNAVAILABLE_MESSAGE = "Agent temporarily unavailable";
+var AgentUnavailableError = class extends Error {
+  constructor() {
+    super(AGENT_UNAVAILABLE_MESSAGE);
+    this.name = "AgentUnavailableError";
+  }
+};
+function bodyLooksLikeBilling3(body) {
+  const lower = body.toLowerCase();
+  return BILLING_BODY_MARKERS3.some((marker) => lower.includes(marker));
+}
+function resolveHealthPair(skill) {
+  if (!skill) {
+    return null;
+  }
+  if (skill.mode === "llm" && skill.resolvedTriple) {
+    return {
+      provider: skill.resolvedTriple.provider,
+      model: skill.resolvedTriple.model
+    };
+  }
+  if (skill.mode !== "llm" && skill.llmOverride?.provider && skill.llmOverride?.model) {
+    return {
+      provider: skill.llmOverride.provider,
+      model: skill.llmOverride.model
+    };
+  }
+  return null;
+}
 var RATE_LIMIT_WINDOW_MS = 10 * 60 * 1e3;
 var MAX_JOBS_PER_CUSTOMER = 20;
 var GLOBAL_MAX_JOBS_PER_WINDOW = 200;
@@ -2012,6 +2043,78 @@ var AgentRuntime = class {
    * the operator's API key.
    */
   freeLlmLimiters = createFreeLlmLimiterSet();
+  /**
+   * Inspect an error thrown by `skill.execute()` and, when it carries a
+   * billing/invalid signal from the LLM provider, flip the matching
+   * (provider, model) pair to unhealthy via the health monitor. The next
+   * job hitting the same pair is then refused at the preflight gate
+   * before payment, so customers don't keep paying for jobs that will
+   * fail. Recovery happens through the lazy recovery loop.
+   *
+   * Two error shapes are recognized:
+   *
+   *   - `ScriptBillingExhaustedError` - thrown by SDK script skills when
+   *     the spawned process exits with `SCRIPT_EXIT_BILLING_EXHAUSTED`.
+   *     Pair comes from `skill.llmOverride` (operator-declared).
+   *
+   *   - LLM provider HTTP error from `mode: 'llm'` - bare `Error` whose
+   *     message starts with "<Provider> API error: <status> <body>" (the
+   *     format every CLI provider currently uses). We classify on status:
+   *     402 / 401 / 403 -> mark unhealthy. Body markers (`credit
+   *     balance`, `billing`, `insufficient`) catch the 400-on-billing case
+   *     Anthropic returns and the 429+`insufficient_quota` case OpenAI
+   *     and the openai-compatible providers (xAI/Google/DeepSeek) return.
+   *     Pair comes from `skill.resolvedTriple`.
+   *
+   * Anything else is a transient/skill error and does NOT touch health
+   * state - the recovery loop should not be poisoned by skill bugs.
+   */
+  markHealthFromExecuteError(skill, err, log, jobId) {
+    if (!this.healthMonitor) {
+      return false;
+    }
+    const tag = `[${jobId.slice(0, 8)}]`;
+    if (err instanceof ScriptBillingExhaustedError) {
+      const provider = skill.llmOverride?.provider;
+      const model = skill.llmOverride?.model;
+      if (!provider || !model) {
+        log(
+          `${tag} Script returned exit ${err.exitCode} (billing-exhausted) but skill "${skill.name}" did not declare provider/model in SKILL.md - cannot gate future jobs.`
+        );
+        return false;
+      }
+      log(
+        `${tag} Script signaled billing-exhausted (exit ${err.exitCode}). Marking ${provider}/${model} unhealthy; future jobs against this pair will be refused until recovery probe succeeds.`
+      );
+      this.healthMonitor.markUnhealthyFromJob(provider, model, "billing", err.message);
+      return true;
+    }
+    if (skill.mode === "llm" && skill.resolvedTriple) {
+      const message = err instanceof Error ? err.message : String(err);
+      const match = /API error:\s*(\d{3})\b\s*(.*)/i.exec(message);
+      if (!match) {
+        return false;
+      }
+      const status = Number(match[1]);
+      const body = (match[2] ?? "").slice(0, 200);
+      const isBillingStatus = status === 402;
+      const isAuthStatus = status === 401 || status === 403;
+      const isBilling400 = status === 400 && bodyLooksLikeBilling3(body);
+      const isBilling429 = status === 429 && bodyLooksLikeBilling3(body);
+      if (!isBillingStatus && !isAuthStatus && !isBilling400 && !isBilling429) {
+        return false;
+      }
+      const reason = isAuthStatus ? "invalid" : "billing";
+      const provider = skill.resolvedTriple.provider;
+      const model = skill.resolvedTriple.model;
+      log(
+        `${tag} LLM provider returned HTTP ${status} (${reason}). Marking ${provider}/${model} unhealthy; future jobs against this pair will be refused until recovery probe succeeds.`
+      );
+      this.healthMonitor.markUnhealthyFromJob(provider, model, reason, body);
+      return true;
+    }
+    return false;
+  }
   /** Fetch on-chain protocol config (fee, treasury). Always fetches fresh to avoid stale treasury. */
   async fetchProtocolConfig() {
     if (this.config.network !== "devnet") {
@@ -2177,7 +2280,14 @@ var AgentRuntime = class {
         this.ledger.markFailed(job.jobId);
       }
       this.callbacks.onJobError?.(job.jobId, e.message);
-      const safeMessage = e.message?.includes("API") ? "Internal processing error" : e.message ?? "Unknown error";
+      let safeMessage;
+      if (e instanceof AgentUnavailableError) {
+        safeMessage = e.message;
+      } else if (e.message?.includes("API")) {
+        safeMessage = "Internal processing error";
+      } else {
+        safeMessage = e.message ?? "Unknown error";
+      }
       await this.transport.sendFeedback(job, { type: "error", message: safeMessage }).catch(() => {
       });
     } finally {
@@ -2192,18 +2302,16 @@ var AgentRuntime = class {
       throw new Error(`Input too long: ${job.input.length} chars (max ${LIMITS.MAX_INPUT_LENGTH})`);
     }
     const matched = this.skills.route(job.tags);
-    if (this.healthMonitor && matched && matched.mode === "llm" && matched.resolvedTriple) {
+    const healthPair = resolveHealthPair(matched);
+    if (this.healthMonitor && healthPair) {
       try {
-        await this.healthMonitor.assertReady(
-          matched.resolvedTriple.provider,
-          matched.resolvedTriple.model
-        );
+        await this.healthMonitor.assertReady(healthPair.provider, healthPair.model);
       } catch (err) {
         const detail = err instanceof Error ? err.message : String(err);
         log(`[${job.jobId.slice(0, 8)}] LLM health gate refused job: ${detail}`);
         await this.transport.sendFeedback(job, {
           type: "error",
-          message: "Service temporarily unavailable, try again later"
+          message: AGENT_UNAVAILABLE_MESSAGE
         }).catch(() => {
         });
         return;
@@ -2244,15 +2352,24 @@ var AgentRuntime = class {
       throw new Error("No skill matched for tags: " + job.tags.join(", "));
     }
     log(`[${job.jobId.slice(0, 8)}] Executing skill: ${skill.name}`);
-    const output = await skill.execute(
-      {
-        data: job.input,
-        inputType: job.inputType,
-        tags: job.tags,
-        jobId: job.jobId
-      },
-      { ...this.skillCtx, signal }
-    );
+    let output;
+    try {
+      output = await skill.execute(
+        {
+          data: job.input,
+          inputType: job.inputType,
+          tags: job.tags,
+          jobId: job.jobId
+        },
+        { ...this.skillCtx, signal }
+      );
+    } catch (err) {
+      const flippedToUnhealthy = this.markHealthFromExecuteError(skill, err, log, job.jobId);
+      if (flippedToUnhealthy) {
+        throw new AgentUnavailableError();
+      }
+      throw err;
+    }
     this.ledger.markExecuted(job.jobId, output.data);
     log(`[${job.jobId.slice(0, 8)}] Skill completed, delivering result`);
     const eventId = await this.transport.deliverResult(job, output.data, netAmount);
@@ -2638,6 +2755,7 @@ var StaticFileSkill = class {
   image;
   imageFile;
   dir;
+  llmOverride;
   inner;
   constructor(params) {
     this.name = params.name;
@@ -2648,6 +2766,7 @@ var StaticFileSkill = class {
     this.image = params.image;
     this.imageFile = params.imageFile;
     this.dir = params.dir;
+    this.llmOverride = params.llmOverride;
     this.inner = new StaticFileSkill$1({
       name: params.name,
       description: params.description,
@@ -2673,6 +2792,7 @@ var StaticScriptSkill = class {
   image;
   imageFile;
   dir;
+  llmOverride;
   inner;
   constructor(params) {
     this.name = params.name;
@@ -2683,6 +2803,7 @@ var StaticScriptSkill = class {
     this.image = params.image;
     this.imageFile = params.imageFile;
     this.dir = params.dir;
+    this.llmOverride = params.llmOverride;
     this.inner = new StaticScriptSkill$1({
       name: params.name,
       description: params.description,
@@ -2711,6 +2832,7 @@ var DynamicScriptSkill = class {
   image;
   imageFile;
   dir;
+  llmOverride;
   inner;
   constructor(params) {
     this.name = params.name;
@@ -2721,6 +2843,7 @@ var DynamicScriptSkill = class {
     this.image = params.image;
     this.imageFile = params.imageFile;
     this.dir = params.dir;
+    this.llmOverride = params.llmOverride;
     this.inner = new DynamicScriptSkill$1({
       name: params.name,
       description: params.description,
@@ -2846,7 +2969,8 @@ function buildCliSkill(parsed, entryPath, scriptEnv) {
         outputFilePath,
         image: parsed.image,
         imageFile: parsed.imageFile,
-        dir: entryPath
+        dir: entryPath,
+        llmOverride: parsed.llmOverride
       });
       break;
     }
@@ -2874,7 +2998,8 @@ function buildCliSkill(parsed, entryPath, scriptEnv) {
         scriptEnv,
         image: parsed.image,
         imageFile: parsed.imageFile,
-        dir: entryPath
+        dir: entryPath,
+        llmOverride: parsed.llmOverride
       });
       break;
     }
@@ -3292,12 +3417,16 @@ async function cmdStart(nameArg, options = {}) {
     process.exit(1);
   }
   const llmSkills = allSkills.filter((skill) => skill.mode === "llm");
+  const scriptDepSkills = allSkills.filter(
+    (skill) => skill.mode !== "llm" && skill.llmOverride?.provider && skill.llmOverride?.model
+  );
+  const monitorPairs = /* @__PURE__ */ new Map();
   const triplesByKey = /* @__PURE__ */ new Map();
   const dependentSkillsByProvider = /* @__PURE__ */ new Map();
   let agentDefaultCacheKey;
   const llmClientCache = /* @__PURE__ */ new Map();
   const healthMonitor = new LlmHealthMonitor();
-  if (llmSkills.length > 0) {
+  if (llmSkills.length > 0 || scriptDepSkills.length > 0) {
     const resolutionErrors = [];
     for (const skill of llmSkills) {
       const skillMdPath = join(skillsDir, skill.name, "SKILL.md");
@@ -3311,6 +3440,10 @@ async function cmdStart(nameArg, options = {}) {
       }
       const cacheKey = cacheKeyFor(result);
       triplesByKey.set(cacheKey, result);
+      monitorPairs.set(`${result.provider}::${result.model}`, {
+        provider: result.provider,
+        model: result.model
+      });
       skill.resolvedTriple = {
         provider: result.provider,
         model: result.model,
@@ -3320,6 +3453,16 @@ async function cmdStart(nameArg, options = {}) {
       list.push(skill.name);
       dependentSkillsByProvider.set(result.provider, list);
     }
+    for (const skill of scriptDepSkills) {
+      const provider = skill.llmOverride?.provider;
+      const model = skill.llmOverride?.model;
+      monitorPairs.set(`${provider}::${model}`, { provider, model });
+      const list = dependentSkillsByProvider.get(provider) ?? [];
+      if (!list.includes(skill.name)) {
+        list.push(skill.name);
+      }
+      dependentSkillsByProvider.set(provider, list);
+    }
     if (resolutionErrors.length > 0) {
       for (const message of resolutionErrors) {
         console.error(`  ! ${message}`);
@@ -3359,27 +3502,27 @@ async function cmdStart(nameArg, options = {}) {
       console.error("");
       process.exit(1);
     }
-    for (const [, triple] of triplesByKey) {
-      const apiKey = keyByProvider.get(triple.provider);
+    for (const [, pair] of monitorPairs) {
+      const apiKey = keyByProvider.get(pair.provider);
       if (!apiKey) {
         continue;
       }
-      const envVar = getLlmProvider(triple.provider)?.envVar ?? `${triple.provider.toUpperCase()}_API_KEY`;
-      process.stdout.write(`  Verifying ${triple.provider} ${triple.model}... `);
-      const verification = await verifyLlmApiKeyDeep(triple.provider, apiKey, triple.model);
-      const descriptor = getLlmProvider(triple.provider);
-      const verifyFn = async (signal) => descriptor ? descriptor.verifyKeyDeep(apiKey, triple.model, signal) : { ok: false, reason: "unavailable", error: "no descriptor" };
+      const envVar = getLlmProvider(pair.provider)?.envVar ?? `${pair.provider.toUpperCase()}_API_KEY`;
+      process.stdout.write(`  Verifying ${pair.provider} ${pair.model}... `);
+      const verification = await verifyLlmApiKeyDeep(pair.provider, apiKey, pair.model);
+      const descriptor = getLlmProvider(pair.provider);
+      const verifyFn = async (signal) => descriptor ? descriptor.verifyKeyDeep(apiKey, pair.model, signal) : { ok: false, reason: "unavailable", error: "no descriptor" };
       healthMonitor.register({
-        provider: triple.provider,
-        model: triple.model,
+        provider: pair.provider,
+        model: pair.model,
         verifyFn
       });
-      healthMonitor.seed(triple.provider, triple.model, verification);
+      healthMonitor.seed(pair.provider, pair.model, verification);
       if (verification.ok) {
         console.log("ok");
       } else if (verification.reason === "invalid") {
         console.log("INVALID");
-        console.error(`  ! ${triple.provider} rejected the API key (HTTP ${verification.status}).`);
+        console.error(`  ! ${pair.provider} rejected the API key (HTTP ${verification.status}).`);
         console.error(
           `    Update it via \`npx @elisym/cli profile ${agentName}\` or set ${envVar} to a valid key.
 `
@@ -3389,7 +3532,7 @@ async function cmdStart(nameArg, options = {}) {
         console.log("BILLING");
         const detail = verification.body ? ` ${verification.body.slice(0, 200)}` : "";
         console.error(
-          `  ! ${triple.provider} reports a billing/quota issue for ${triple.model}.${detail}`
+          `  ! ${pair.provider} reports a billing/quota issue for ${pair.model}.${detail}`
         );
         console.error(
           `    Top up the account at the provider console, or set ${envVar} to a key on a funded org.
@@ -3399,7 +3542,7 @@ async function cmdStart(nameArg, options = {}) {
       } else {
         console.log("unavailable");
         console.warn(
-          `  ! Could not verify ${triple.provider} ${triple.model} (${verification.error}). Continuing - jobs will fail if the key is invalid.
+          `  ! Could not verify ${pair.provider} ${pair.model} (${verification.error}). Continuing - jobs will fail if the key is invalid.
 `
         );
       }
@@ -3421,7 +3564,7 @@ async function cmdStart(nameArg, options = {}) {
       llmClientCache.set(cacheKey, createLlmClient(config));
     }
   } else {
-    console.log("  No LLM skills loaded; skipping LLM key check.\n");
+    console.log("  No LLM-dependent skills loaded; skipping LLM key check.\n");
   }
   const agentDefaultClient = agentDefaultCacheKey !== void 0 ? llmClientCache.get(agentDefaultCacheKey) : void 0;
   const getLlm = (override) => {
@@ -3618,12 +3761,12 @@ async function cmdStart(nameArg, options = {}) {
     logger
   });
   let llmHeartbeat;
-  if (llmSkills.length > 0) {
-    llmHeartbeat = startLlmHeartbeat({
+  if (monitorPairs.size > 0) {
+    llmHeartbeat = startLlmRecovery({
       monitor: healthMonitor,
       log: diagLog
     });
-    diagLog("LLM health monitor armed (10min TTL, 10min heartbeat).");
+    diagLog("LLM health monitor armed (lazy recovery, 5min interval).");
   }
   const runtime = new AgentRuntime(
     transport,