@elisym/cli 0.10.0 → 0.11.0

package/dist/index.js

@@ -8,6 +8,7 @@ import { generateSecretKey, getPublicKey, nip19, verifyEvent } from 'nostr-tools
 import YAML from 'yaml';
 import { Command } from 'commander';
 import { createHash } from 'node:crypto';
+import { LlmHealthMonitor, startLlmHeartbeat, createFreeLlmLimiterSet, FREE_LLM_GLOBAL_KEY, freeLlmCustomerKey } from '@elisym/sdk/llm-health';
 import { lookup } from 'node:dns/promises';
 import { Socket } from 'node:net';
 import pino from 'pino';
@@ -149,7 +150,54 @@ async function verifyKey(apiKey, signal) {
     return { ok: false, reason: "unavailable", error: message };
   }
 }
-var DEFAULT_MODEL, DEFAULT_MAX_TOKENS, FALLBACK_MODELS, AnthropicClient, ANTHROPIC_PROVIDER;
+function bodyLooksLikeBilling(body) {
+  const lower = body.toLowerCase();
+  return BILLING_BODY_MARKERS.some((marker) => lower.includes(marker));
+}
+async function verifyKeyDeep(apiKey, model, signal) {
+  try {
+    const response = await fetchWithTimeout(
+      "https://api.anthropic.com/v1/messages",
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "x-api-key": apiKey,
+          "anthropic-version": "2023-06-01"
+        },
+        body: JSON.stringify({
+          model,
+          max_tokens: 1,
+          messages: [{ role: "user", content: "." }]
+        })
+      },
+      signal
+    );
+    if (response.ok) {
+      await response.body?.cancel().catch(() => void 0);
+      return { ok: true };
+    }
+    const body = (await response.text().catch(() => "")).slice(0, 500);
+    if (response.status === 401 || response.status === 403) {
+      return { ok: false, reason: "invalid", status: response.status, body };
+    }
+    if (response.status === 402) {
+      return { ok: false, reason: "billing", status: response.status, body };
+    }
+    if (response.status === 400 && bodyLooksLikeBilling(body)) {
+      return { ok: false, reason: "billing", status: response.status, body };
+    }
+    return {
+      ok: false,
+      reason: "unavailable",
+      error: `HTTP ${response.status}: ${body.slice(0, 200)}`
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { ok: false, reason: "unavailable", error: message };
+  }
+}
+var DEFAULT_MODEL, DEFAULT_MAX_TOKENS, FALLBACK_MODELS, AnthropicClient, BILLING_BODY_MARKERS, ANTHROPIC_PROVIDER;
 var init_anthropic = __esm({
   "src/llm/providers/anthropic.ts"() {
     init_http();
@@ -270,6 +318,7 @@ var init_anthropic = __esm({
         ];
       }
     };
+    BILLING_BODY_MARKERS = ["credit balance", "billing", "insufficient"];
     ANTHROPIC_PROVIDER = {
       id: "anthropic",
       displayName: "Anthropic (Claude)",
@@ -278,6 +327,7 @@ var init_anthropic = __esm({
       fallbackModels: FALLBACK_MODELS,
       fetchModels,
       verifyKey,
+      verifyKeyDeep,
       createClient: (config) => new AnthropicClient({
         apiKey: config.apiKey,
         model: config.model ?? DEFAULT_MODEL,
@@ -342,7 +392,57 @@ async function verifyKey2(apiKey, signal) {
     return { ok: false, reason: "unavailable", error: message };
   }
 }
-var DEFAULT_MODEL2, DEFAULT_MAX_TOKENS2, FALLBACK_MODELS2, OpenAIClient, OPENAI_PROVIDER;
+function bodyLooksLikeBilling2(body) {
+  const lower = body.toLowerCase();
+  return BILLING_BODY_MARKERS2.some((marker) => lower.includes(marker));
+}
+async function verifyKeyDeep2(apiKey, model, signal) {
+  try {
+    const reasoning = isOpenAIReasoningModel(model);
+    const response = await fetchWithTimeout(
+      "https://api.openai.com/v1/chat/completions",
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Authorization: `Bearer ${apiKey}`
+        },
+        body: JSON.stringify({
+          model,
+          ...reasoning ? { max_completion_tokens: 1 } : { max_tokens: 1 },
+          messages: [{ role: "user", content: "." }]
+        })
+      },
+      signal
+    );
+    if (response.ok) {
+      await response.body?.cancel().catch(() => void 0);
+      return { ok: true };
+    }
+    const body = (await response.text().catch(() => "")).slice(0, 500);
+    if (response.status === 401 || response.status === 403) {
+      return { ok: false, reason: "invalid", status: response.status, body };
+    }
+    if (response.status === 402) {
+      return { ok: false, reason: "billing", status: response.status, body };
+    }
+    if (response.status === 429 && bodyLooksLikeBilling2(body)) {
+      return { ok: false, reason: "billing", status: response.status, body };
+    }
+    if (response.status === 400 && bodyLooksLikeBilling2(body)) {
+      return { ok: false, reason: "billing", status: response.status, body };
+    }
+    return {
+      ok: false,
+      reason: "unavailable",
+      error: `HTTP ${response.status}: ${body.slice(0, 200)}`
+    };
+  } catch (error) {
+    const message = error instanceof Error ? error.message : String(error);
+    return { ok: false, reason: "unavailable", error: message };
+  }
+}
+var DEFAULT_MODEL2, DEFAULT_MAX_TOKENS2, FALLBACK_MODELS2, OpenAIClient, BILLING_BODY_MARKERS2, OPENAI_PROVIDER;
 var init_openai = __esm({
   "src/llm/providers/openai.ts"() {
     init_http();
@@ -466,6 +566,7 @@ var init_openai = __esm({
         }));
       }
     };
+    BILLING_BODY_MARKERS2 = ["credit balance", "billing", "insufficient_quota", "insufficient"];
     OPENAI_PROVIDER = {
       id: "openai",
       displayName: "OpenAI (GPT)",
@@ -474,6 +575,7 @@ var init_openai = __esm({
       fallbackModels: FALLBACK_MODELS2,
       fetchModels: fetchModels2,
       verifyKey: verifyKey2,
+      verifyKeyDeep: verifyKeyDeep2,
       createClient: (config) => new OpenAIClient({
         apiKey: config.apiKey,
         model: config.model ?? DEFAULT_MODEL2,
@@ -526,7 +628,7 @@ function createLlmClient(config) {
     logUsage: config.logUsage
   });
 }
-async function verifyLlmApiKey(provider, apiKey, signal) {
+async function verifyLlmApiKeyDeep(provider, apiKey, model, signal) {
   const descriptor = getLlmProvider(provider);
   if (!descriptor) {
     return {
@@ -535,7 +637,7 @@ async function verifyLlmApiKey(provider, apiKey, signal) {
       error: `Unknown LLM provider "${provider}"`
     };
   }
-  return descriptor.verifyKey(apiKey, signal);
+  return descriptor.verifyKeyDeep(apiKey, model, signal);
 }
 var init_llm = __esm({
   "src/llm/index.ts"() {
@@ -1546,13 +1648,14 @@ var GLOBAL_MAX_JOBS_PER_WINDOW = 200;
 var MAX_TRACKED_CUSTOMERS = 1e3;
 var GLOBAL_LIMITER_KEY = "__global__";
 var AgentRuntime = class {
-  constructor(transport, skills, skillCtx, config, ledger, callbacks = {}) {
+  constructor(transport, skills, skillCtx, config, ledger, callbacks = {}, healthMonitor) {
     this.transport = transport;
     this.skills = skills;
     this.skillCtx = skillCtx;
     this.config = config;
     this.ledger = ledger;
     this.callbacks = callbacks;
+    this.healthMonitor = healthMonitor;
     this.limit = pLimit(config.maxConcurrentJobs);
     this.maxQueueSize = config.maxQueueSize ?? config.maxConcurrentJobs * 10;
   }
@@ -1577,6 +1680,13 @@ var AgentRuntime = class {
     maxPerWindow: GLOBAL_MAX_JOBS_PER_WINDOW,
     maxKeys: 1
   });
+  /**
+   * Two-tier limiter applied only to free LLM skills (mode='llm', price=0).
+   * Existing per-customer + global limiters above remain in effect for all
+   * jobs; this set adds tighter caps to prevent unpaid spam from draining
+   * the operator's API key.
+   */
+  freeLlmLimiters = createFreeLlmLimiterSet();
   /** Fetch on-chain protocol config (fee, treasury). Always fetches fresh to avoid stale treasury. */
   async fetchProtocolConfig() {
     if (this.config.network !== "devnet") {
@@ -1633,8 +1743,33 @@ var AgentRuntime = class {
         });
         return;
       }
+      const matched = this.skills.route(job.tags);
+      const isFreeLlm = matched?.mode === "llm" && matched.priceSubunits === 0;
+      let perCustomerLimiter;
+      let perSkillKey;
+      if (isFreeLlm && matched) {
+        if (!this.freeLlmLimiters.globalLimiter.peek(FREE_LLM_GLOBAL_KEY).allowed) {
+          this.transport.sendFeedback(job, { type: "error", message: "Rate limited, try again later" }).catch(() => {
+          });
+          return;
+        }
+        perCustomerLimiter = this.freeLlmLimiters.getPerCustomerLimiter(
+          matched.name,
+          matched.rateLimit
+        );
+        perSkillKey = freeLlmCustomerKey(job.customerId, matched.name);
+        if (!perCustomerLimiter.peek(perSkillKey).allowed) {
+          this.transport.sendFeedback(job, { type: "error", message: "Rate limited, try again later" }).catch(() => {
+          });
+          return;
+        }
+      }
       this.customerLimiter.check(job.customerId);
       this.globalLimiter.check(GLOBAL_LIMITER_KEY);
+      if (isFreeLlm && perCustomerLimiter && perSkillKey) {
+        this.freeLlmLimiters.globalLimiter.check(FREE_LLM_GLOBAL_KEY);
+        perCustomerLimiter.check(perSkillKey);
+      }
       this.callbacks.onJobReceived?.(job);
       this.inFlight.add(job.jobId);
       this.pending++;
@@ -1662,10 +1797,12 @@ var AgentRuntime = class {
       process.on("SIGTERM", onSignal);
     });
   }
-  /** Drop expired hits from both sliding-window limiters. */
+  /** Drop expired hits from every sliding-window limiter. */
   cleanupRateLimits() {
     this.customerLimiter.prune();
     this.globalLimiter.prune();
+    this.freeLlmLimiters.globalLimiter.prune();
+    this.freeLlmLimiters.prunePerCustomer();
   }
   stop() {
     if (this.stopped) {
@@ -1729,6 +1866,24 @@ var AgentRuntime = class {
     if (job.input.length > LIMITS.MAX_INPUT_LENGTH) {
       throw new Error(`Input too long: ${job.input.length} chars (max ${LIMITS.MAX_INPUT_LENGTH})`);
     }
+    const matched = this.skills.route(job.tags);
+    if (this.healthMonitor && matched && matched.mode === "llm" && matched.resolvedTriple) {
+      try {
+        await this.healthMonitor.assertReady(
+          matched.resolvedTriple.provider,
+          matched.resolvedTriple.model
+        );
+      } catch (err) {
+        const detail = err instanceof Error ? err.message : String(err);
+        log(`[${job.jobId.slice(0, 8)}] LLM health gate refused job: ${detail}`);
+        await this.transport.sendFeedback(job, {
+          type: "error",
+          message: "Service temporarily unavailable, try again later"
+        }).catch(() => {
+        });
+        return;
+      }
+    }
     const jobPrice = resolveJobPrice(job.tags, this.skills);
     const jobAsset = resolveJobAsset(job.tags, this.skills);
     let netAmount;
@@ -2325,9 +2480,10 @@ var ScriptSkill = class {
 
 // src/skill/loader.ts
 function buildCliSkill(parsed, entryPath) {
+  let skill;
   switch (parsed.mode) {
     case "llm":
-      return new ScriptSkill(
+      skill = new ScriptSkill(
         parsed.name,
         parsed.description,
         parsed.capabilities,
@@ -2341,6 +2497,7 @@ function buildCliSkill(parsed, entryPath) {
         parsed.maxToolRounds,
         parsed.llmOverride
       );
+      break;
     case "static-file": {
       if (parsed.outputFile === void 0) {
         throw new Error(
@@ -2353,7 +2510,7 @@ function buildCliSkill(parsed, entryPath) {
           `SKILL.md "${parsed.name}": "output_file" must stay inside the skill directory`
         );
       }
-      return new StaticFileSkill({
+      skill = new StaticFileSkill({
         name: parsed.name,
         description: parsed.description,
         capabilities: parsed.capabilities,
@@ -2364,6 +2521,7 @@ function buildCliSkill(parsed, entryPath) {
         imageFile: parsed.imageFile,
         dir: entryPath
       });
+      break;
     }
     case "static-script":
     case "dynamic-script": {
@@ -2377,7 +2535,7 @@ function buildCliSkill(parsed, entryPath) {
         throw new Error(`SKILL.md "${parsed.name}": "script" must stay inside the skill directory`);
       }
       const Ctor = parsed.mode === "static-script" ? StaticScriptSkill : DynamicScriptSkill;
-      return new Ctor({
+      skill = new Ctor({
         name: parsed.name,
         description: parsed.description,
         capabilities: parsed.capabilities,
@@ -2390,8 +2548,13 @@ function buildCliSkill(parsed, entryPath) {
         imageFile: parsed.imageFile,
         dir: entryPath
       });
+      break;
     }
   }
+  if (parsed.rateLimit) {
+    skill.rateLimit = parsed.rateLimit;
+  }
+  return skill;
 }
 function loadSkillsFromDir(skillsDir) {
   const skills = [];
@@ -2797,6 +2960,7 @@ async function cmdStart(nameArg, options = {}) {
   const dependentSkillsByProvider = /* @__PURE__ */ new Map();
   let agentDefaultCacheKey;
   const llmClientCache = /* @__PURE__ */ new Map();
+  const healthMonitor = new LlmHealthMonitor();
   if (llmSkills.length > 0) {
     const resolutionErrors = [];
     for (const skill of llmSkills) {
@@ -2811,6 +2975,11 @@ async function cmdStart(nameArg, options = {}) {
       }
       const cacheKey = cacheKeyFor(result);
       triplesByKey.set(cacheKey, result);
+      skill.resolvedTriple = {
+        provider: result.provider,
+        model: result.model,
+        maxTokens: result.maxTokens
+      };
       const list = dependentSkillsByProvider.get(result.provider) ?? [];
       list.push(skill.name);
       dependentSkillsByProvider.set(result.provider, list);
@@ -2854,28 +3023,47 @@ async function cmdStart(nameArg, options = {}) {
       console.error("");
       process.exit(1);
     }
-    for (const provider of keyByProvider.keys()) {
-      const apiKey = keyByProvider.get(provider);
+    for (const [, triple] of triplesByKey) {
+      const apiKey = keyByProvider.get(triple.provider);
       if (!apiKey) {
         continue;
       }
-      const envVar = getLlmProvider(provider)?.envVar ?? `${provider.toUpperCase()}_API_KEY`;
-      process.stdout.write(`  Verifying ${provider} API key... `);
-      const verification = await verifyLlmApiKey(provider, apiKey);
+      const envVar = getLlmProvider(triple.provider)?.envVar ?? `${triple.provider.toUpperCase()}_API_KEY`;
+      process.stdout.write(`  Verifying ${triple.provider} ${triple.model}... `);
+      const verification = await verifyLlmApiKeyDeep(triple.provider, apiKey, triple.model);
+      const descriptor = getLlmProvider(triple.provider);
+      const verifyFn = async (signal) => descriptor ? descriptor.verifyKeyDeep(apiKey, triple.model, signal) : { ok: false, reason: "unavailable", error: "no descriptor" };
+      healthMonitor.register({
+        provider: triple.provider,
+        model: triple.model,
+        verifyFn
+      });
+      healthMonitor.seed(triple.provider, triple.model, verification);
       if (verification.ok) {
         console.log("ok");
       } else if (verification.reason === "invalid") {
         console.log("INVALID");
-        console.error(`  ! ${provider} rejected the API key (HTTP ${verification.status}).`);
+        console.error(`  ! ${triple.provider} rejected the API key (HTTP ${verification.status}).`);
         console.error(
           `    Update it via \`npx @elisym/cli profile ${agentName}\` or set ${envVar} to a valid key.
+`
+        );
+        process.exit(1);
+      } else if (verification.reason === "billing") {
+        console.log("BILLING");
+        const detail = verification.body ? ` ${verification.body.slice(0, 200)}` : "";
+        console.error(
+          `  ! ${triple.provider} reports a billing/quota issue for ${triple.model}.${detail}`
+        );
+        console.error(
+          `    Top up the account at the provider console, or set ${envVar} to a key on a funded org.
 `
         );
         process.exit(1);
       } else {
         console.log("unavailable");
         console.warn(
-          `  ! Could not verify ${provider} API key (${verification.error}). Continuing - jobs will fail if the key is invalid.
+          `  ! Could not verify ${triple.provider} ${triple.model} (${verification.error}). Continuing - jobs will fail if the key is invalid.
 `
         );
       }
@@ -3093,28 +3281,45 @@ async function cmdStart(nameArg, options = {}) {
     log: diagLog,
     logger
   });
-  const runtime = new AgentRuntime(transport, registry, skillCtx, runtimeConfig, ledger, {
-    onJobReceived: (job) => {
-      const cap = job.tags.find((t) => t !== "elisym") ?? "unknown";
-      process.stdout.write(`  [job] ${job.jobId.slice(0, 16)} | cap=${cap}
+  let llmHeartbeat;
+  if (llmSkills.length > 0) {
+    llmHeartbeat = startLlmHeartbeat({
+      monitor: healthMonitor,
+      log: diagLog
+    });
+    diagLog("LLM health monitor armed (10min TTL, 10min heartbeat).");
+  }
+  const runtime = new AgentRuntime(
+    transport,
+    registry,
+    skillCtx,
+    runtimeConfig,
+    ledger,
+    {
+      onJobReceived: (job) => {
+        const cap = job.tags.find((t) => t !== "elisym") ?? "unknown";
+        process.stdout.write(`  [job] ${job.jobId.slice(0, 16)} | cap=${cap}
 `);
-      logger.info({ event: "job_received", jobId: job.jobId, capability: cap });
-    },
-    onJobCompleted: (jobId) => {
-      process.stdout.write(`  [job] ${jobId.slice(0, 16)} | delivered
+        logger.info({ event: "job_received", jobId: job.jobId, capability: cap });
+      },
+      onJobCompleted: (jobId) => {
+        process.stdout.write(`  [job] ${jobId.slice(0, 16)} | delivered
 `);
-      logger.info({ event: "job_delivered", jobId });
-    },
-    onJobError: (jobId, error) => {
-      process.stderr.write(`  [job] ${jobId.slice(0, 16)} | error: ${error}
+        logger.info({ event: "job_delivered", jobId });
+      },
+      onJobError: (jobId, error) => {
+        process.stderr.write(`  [job] ${jobId.slice(0, 16)} | error: ${error}
 `);
-      logger.error({ event: "job_error", jobId, error });
+        logger.error({ event: "job_error", jobId, error });
+      },
+      onLog: diagLog,
+      onStop: () => {
+        watchdog.stop();
+        llmHeartbeat?.stop();
+      }
     },
-    onLog: diagLog,
-    onStop: () => {
-      watchdog.stop();
-    }
-  });
+    healthMonitor
+  );
   console.log("  * Running. Press Ctrl+C to stop.\n");
   await runtime.run();
 }