ag-ai 0.4.3 → 0.4.5

package/README.md

@@ -62,6 +62,20 @@ pool is cooling down waits for the soonest model to recover and retries, at
 most `MAX_COOLDOWN_WAITS` times and never past the request timeout; with a
 fallback it falls back immediately instead.
 
+## Runner Quota Fallback
+
+The cluster runner resolves each `prefer` tier through `runtime/model-tiers.json`. When the
+tier's model fails with a quota error (HTTP 429, or a ChatGPT "usage limit"), the job walks
+the text models of the tier's `fallback` provider in the OpenCode catalog: the tier's family
+first (pro for `quality`, flash for `fast`), then by output price as a quality proxy. Image,
+audio, embedding, alias (`-latest`) and models without tool calling are skipped.
+
+A model that fails with 429 is skipped for the provider's requested wait, otherwise for 60 s
+doubling to 15 min; a 404 skips it for an hour. Cooldowns live in the runner's memory. When
+every model is cooling down, the one that recovers soonest is tried. Requests naming an
+explicit model never fall back. Completed results and failure reports carry the model that
+actually ran.
+
 ## Tooling
 
 Formatting and linting use `e7-oxfmt` / `e7-oxlint` from

package/dist/k8s/assets/network-init.sh

@@ -13,7 +13,7 @@ iptables-restore <<'RULES'
 :OUTPUT DROP [0:0]
 -A INPUT -i lo -j ACCEPT
 -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
--A INPUT -s __GENERATION_CIDR__ -p tcp --dport __GENERATION_PORT__ -j ACCEPT
+__GENERATION_INGRESS__
 -A OUTPUT -o lo -j ACCEPT
 -A OUTPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT
 __DNS_RULES__

package/dist/k8s/runner.js

@@ -5,6 +5,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.runnerStack = void 0;
 const node_fs_1 = require("node:fs");
+const node_net_1 = require("node:net");
 const node_path_1 = __importDefault(require("node:path"));
 const ag_rpi_mega_1 = require("ag-rpi-mega");
 const common_1 = require("./common");
@@ -12,12 +13,14 @@ const networkInitTemplate = (0, node_fs_1.readFileSync)(node_path_1.default.join
 /** The published runtime is installed into a writable emptyDir at pod start. */
 const AG_AI_RUNTIME_DIR = '/opt/ag-ai';
 const AG_AI_BRIDGE = `${AG_AI_RUNTIME_DIR}/node_modules/ag-ai/runtime/bridge.mjs`;
-const networkInitScript = ({ generationCidr, generationPort, opencodePort, }) => networkInitTemplate
-    .replaceAll('__GENERATION_CIDR__', generationCidr)
+const networkInitScript = ({ generationCidrs, generationPort, opencodePort, }) => networkInitTemplate
+    .replaceAll('__GENERATION_INGRESS__', generationCidrs
+    .map((cidr) => `-A INPUT -s ${cidr} -p tcp --dport ${generationPort} -j ACCEPT`)
+    .join('\n'))
     .replaceAll('__GENERATION_PORT__', String(generationPort))
     .replaceAll('__OPENCODE_PORT__', String(opencodePort))
     .replaceAll('__DNS_RULES__', '-A OUTPUT -p udp --dport 53 -j ACCEPT\n-A OUTPUT -p tcp --dport 53 -j ACCEPT')
-    .replaceAll('__GENERATION_REJECT__', `-A OUTPUT -d ${generationCidr} -j REJECT`);
+    .replaceAll('__GENERATION_REJECT__', generationCidrs.map((cidr) => `-A OUTPUT -d ${cidr} -j REJECT`).join('\n'));
 const healthProbe = (readiness) => ({
     exec: {
         command: [
@@ -38,6 +41,25 @@ const runnerStack = (context) => {
     const generationCidr = context.env.AG_AI_GENERATION_CIDR;
     if (!generationCidr)
         throw new Error('AG_AI_GENERATION_CIDR is required');
+    const clusterCidr = context.env.AG_AI_GENERATION_CLUSTER_CIDR;
+    const generationCidrs = [generationCidr, ...(clusterCidr ? [clusterCidr] : [])];
+    for (const cidr of generationCidrs) {
+        const [address, prefix, ...extra] = cidr.split('/');
+        const minimum = address.startsWith('10.')
+            ? 8
+            : address.startsWith('192.168.')
+                ? 16
+                : /^172\.(1[6-9]|2\d|3[01])\./.test(address)
+                    ? 12
+                    : 33;
+        if ((0, node_net_1.isIP)(address) !== 4 ||
+            extra.length ||
+            !/^\d{1,2}$/.test(prefix) ||
+            Number(prefix) < minimum ||
+            Number(prefix) > 32) {
+            throw new Error('Generation ingress requires a valid private IPv4 CIDR');
+        }
+    }
     const imageProvider = context.env.AG_AI_IMAGE_PROVIDER ?? 'openai';
     const opencodeProvider = context.env.AG_AI_OPENCODE_PROVIDER;
     const agAiPackage = context.env.AG_AI_PACKAGE;
@@ -61,7 +83,7 @@ const runnerStack = (context) => {
         preserveData: true,
         data: {
             'network-init.sh': networkInitScript({
-                generationCidr,
+                generationCidrs,
                 generationPort: common_1.GENERATION_PORT,
                 opencodePort: common_1.OPENCODE_PORT,
             }),
@@ -138,9 +160,10 @@ const runnerStack = (context) => {
         changeNFSFilePerms: false,
     });
     const spec = deployment.spec;
-    // oxlint-disable-next-line typescript/no-unnecessary-condition -- the SDK class type hides runtime optionality.
-    const pod = spec?.template?.spec;
-    if (!spec || !pod)
+    if (!spec)
+        throw new Error('runner deployment is missing its specification');
+    const pod = spec.template.spec;
+    if (!pod)
         throw new Error('runner deployment is missing its pod specification');
     // A shared store must have only one writer during rollouts.
     spec.strategy = { type: 'Recreate' };
@@ -253,7 +276,7 @@ const runnerStack = (context) => {
         spec: {
             type: 'LoadBalancer',
             loadBalancerIP: metalLbAddress,
-            loadBalancerSourceRanges: [generationCidr],
+            loadBalancerSourceRanges: generationCidrs,
             externalTrafficPolicy: 'Local',
             allocateLoadBalancerNodePorts: false,
             selector: { app: common_1.RUNNER_APP },
@@ -279,7 +302,7 @@ const runnerStack = (context) => {
             policyTypes: ['Ingress'],
             ingress: [
                 {
-                    from: [{ ipBlock: { cidr: generationCidr } }],
+                    from: generationCidrs.map((cidr) => ({ ipBlock: { cidr } })),
                     ports: [{ protocol: 'TCP', port: common_1.GENERATION_PORT }],
                 },
             ],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ag-ai",
-  "version": "0.4.3",
+  "version": "0.4.5",
   "description": "Provider-neutral AI client and cluster runtime for the rpi-mega cluster.",
   "license": "ISC",
   "author": "admin@gec.dev",

package/runtime/bridge.mjs

@@ -9,7 +9,7 @@ import { fileURLToPath } from 'node:url';
 
 import { createOpenCodeExecutor, DEFAULT_OPENCODE_URL } from './executor/opencode.mjs';
 import { createImageAdapter } from './image/adapter.mjs';
-import { normalizeGenerationRequest } from './protocol.mjs';
+import { fallbackModels, normalizeGenerationRequest, requestOnModel } from './protocol.mjs';
 import { createStatusReporter, createWorkSignal, startPushChannel } from './push.mjs';
 import { createLocalRunner, startGenerationServer } from './server.mjs';
 import { GenerationStore } from './store.mjs';
@@ -31,6 +31,9 @@ const HEALTH_INTERVAL_MS = 30_000;
 const CLAIM_INTERVAL_MS = 5_000;
 const REQUEST_TIMEOUT_MS = 15_000;
 const TURN_TIMEOUT_MS = 30 * 60 * 1000;
+const QUOTA_COOLDOWN_BASE_MS = 60_000;
+const QUOTA_COOLDOWN_MAX_MS = 15 * 60_000;
+const UNAVAILABLE_COOLDOWN_MS = 60 * 60_000;
 const SHUTDOWN_TIMEOUT_MS = 10_000;
 const MAX_HTTP_JSON_BYTES = 2 * 1024 * 1024;
 const MAX_HTTP_INPUT_BYTES = 64 * 1024 * 1024;
@@ -394,12 +397,13 @@ class RunnerClient {
     });
   }
 
-  fail(jobId, leaseToken, error, outcome) {
+  fail(jobId, leaseToken, error, outcome, model) {
     return this.request('POST', `jobs/${encodeURIComponent(jobId)}/fail`, {
       leaseToken,
       body: {
         error: error.slice(0, 10_000),
         outcome,
+        ...(model === undefined ? {} : { model }),
       },
     });
   }
@@ -575,6 +579,32 @@ const parseStructuredOutput = (text, outputSchema) => {
   }
 };
 
+/** Models to try in order: those not cooling down, or else the one that recovers soonest. */
+const modelsToTry = (candidates, cooldowns, now) => {
+  const ready = candidates.filter((id) => (cooldowns.get(id)?.until ?? 0) <= now);
+  if (ready.length > 0) return ready;
+  return [
+    candidates.reduce((soonest, id) =>
+      cooldowns.get(id).until < cooldowns.get(soonest).until ? id : soonest,
+    ),
+  ];
+};
+
+/**
+ * Skip a model for the provider's requested wait, or back off from a minute to 15 after
+ * repeated quota failures; an unavailable model is skipped for an hour. In memory only:
+ * a restarted runner retries every model once.
+ */
+const coolDown = (cooldowns, id, error, now) => {
+  const hits = (cooldowns.get(id)?.hits ?? 0) + 1;
+  const delay =
+    error.status === 404
+      ? UNAVAILABLE_COOLDOWN_MS
+      : (error.retryAfterMs ??
+        Math.min(QUOTA_COOLDOWN_BASE_MS * 2 ** (hits - 1), QUOTA_COOLDOWN_MAX_MS));
+  cooldowns.set(id, { hits, until: now + delay });
+};
+
 const safeErrorOutcome = (error) =>
   error instanceof JobError
     ? error.outcome
@@ -641,6 +671,7 @@ const processJob = async (runner, job, config, state, source, analytics) => {
   }, LEASE_HEARTBEAT_INTERVAL_MS);
 
   let turnTimer;
+  let model;
   try {
     if (job.leaseExpiresAt <= Date.now()) {
       throw new JobError('claimed job lease has already expired', 'unknown');
@@ -648,7 +679,8 @@ const processJob = async (runner, job, config, state, source, analytics) => {
     const inputEnvelope = await runner.input(job.id, job.leaseToken);
     void analytics.emit('job', source, 'input-loaded');
     const models = state.health.models ?? [];
-    const normalizedInput = normalizeGenerationRequest(inputEnvelope, models);
+    let normalizedInput = normalizeGenerationRequest(inputEnvelope, models);
+    model = normalizedInput.model;
     void analytics.emit('job', source, 'input-validated');
 
     const jobDirectoryName = `runner-${job.id}`;
@@ -670,10 +702,38 @@ const processJob = async (runner, job, config, state, source, analytics) => {
 
     turnTimer = setTimeout(() => cancellation.cancel('turn timeout'), TURN_TIMEOUT_MS);
     void analytics.emit('job', source, 'turn-started');
-    const result = await state.executor.execute(normalizedInput, {
-      signal: cancellation.signal,
-      directory: mainJobDirectory,
-    });
+    const execute = (request) =>
+      state.executor.execute(request, { signal: cancellation.signal, directory: mainJobDirectory });
+    const cooldowns = (state.cooldowns ??= new Map());
+    const candidates = [normalizedInput.model, ...fallbackModels(normalizedInput, models)];
+    let result;
+    let lastError;
+    for (const candidate of modelsToTry(candidates, cooldowns, Date.now())) {
+      let request;
+      try {
+        request =
+          candidate === normalizedInput.model
+            ? normalizedInput
+            : requestOnModel(normalizedInput, candidate, models);
+      } catch {
+        continue; // This model cannot accept the request's inputs or outputs.
+      }
+      model = candidate;
+      try {
+        result = await execute(request);
+        cooldowns.delete(candidate);
+        normalizedInput = request;
+        break;
+      } catch (error) {
+        // oxlint-disable-next-line typescript/no-unnecessary-condition -- cancelled is set by the turn timer and lease checks.
+        if (cancellation.cancelled || (error?.status !== 429 && error?.status !== 404)) throw error;
+        coolDown(cooldowns, candidate, error, Date.now());
+        console.error(`[runner] job ${job.id}: ${candidate} is out of quota or unavailable`);
+        void analytics.emit('job', source, 'turn-fallback');
+        lastError = error;
+      }
+    }
+    if (result === undefined) throw lastError ?? new JobError('no model can run this request');
     void analytics.emit('job', source, 'turn-completed');
     // oxlint-disable-next-line typescript/no-unnecessary-condition -- leaseLost is set by the lease timer.
     if (leaseLost) {
@@ -726,7 +786,7 @@ const processJob = async (runner, job, config, state, source, analytics) => {
     const responseBytes = Buffer.from(
       JSON.stringify({
         output,
-        model: normalizedInput.model,
+        model,
         generatedAt: Number.isInteger(result.generatedAt) ? result.generatedAt : Date.now(),
         ...(result.usage === undefined ? {} : { usage: result.usage }),
       }),
@@ -764,7 +824,7 @@ const processJob = async (runner, job, config, state, source, analytics) => {
     void analytics.emit('job', source, 'failed', outcome);
     const diagnostic = redact(error, [job.leaseToken]);
     try {
-      await runner.fail(job.id, job.leaseToken, diagnostic, outcome);
+      await runner.fail(job.id, job.leaseToken, diagnostic, outcome, model);
       void analytics.emit('job', source, 'failure-recorded', outcome);
     } catch (failError) {
       if (!(failError instanceof RunnerHttpError && failError.status === 409)) {

package/runtime/executor/opencode.mjs

@@ -17,13 +17,22 @@ const MAX_TEXT_BYTES = 2 * 1024 * 1024;
  */
 
 class ExecutorError extends Error {
-  constructor(message, { status } = {}) {
+  constructor(message, { status, retryAfterMs } = {}) {
     super(message);
     this.name = 'ExecutorError';
     if (status !== undefined) this.status = status;
+    if (retryAfterMs !== undefined) this.retryAfterMs = retryAfterMs;
   }
 }
 
+/** The wait a provider quota message asks for, such as "Please retry in 2h5m12.03s". */
+const retryAfterMs = (message) => {
+  const match = /retry in (?:(\d+)h)?(?:(\d+)m(?!s))?(?:(\d+(?:\.\d+)?)s)?/i.exec(message);
+  const seconds =
+    (Number(match?.[1] ?? 0) * 60 + Number(match?.[2] ?? 0)) * 60 + Number(match?.[3] ?? 0);
+  return seconds > 0 ? Math.ceil(seconds * 1000) : undefined;
+};
+
 const asObject = (value) => (value && typeof value === 'object' ? value : null);
 
 const boundedString = (value, maximum, name) => {
@@ -385,10 +394,16 @@ const createOpenCodeExecutor = async ({
         );
         const info = asObject(response?.info);
         if (info?.error) {
-          const message = asObject(info.error)?.data;
-          throw new ExecutorError(
-            `OpenCode turn failed: ${asObject(message)?.message ?? info.error.name ?? 'unknown error'}`,
-          );
+          const data = asObject(asObject(info.error)?.data);
+          const message = data?.message ?? info.error.name ?? 'unknown error';
+          // A ChatGPT plan reports exhaustion as a usage limit, not always with its 429 status.
+          const quota = /usage limit|exceeded your current quota/i.test(String(message));
+          const status = quota ? 429 : data?.statusCode;
+          const retryAfter = retryAfterMs(String(message));
+          throw new ExecutorError(`OpenCode turn failed: ${message}`, {
+            ...(Number.isInteger(status) ? { status } : {}),
+            ...(retryAfter === undefined ? {} : { retryAfterMs: retryAfter }),
+          });
         }
         const structured = structuredFromResponse(info, response?.parts);
         text =
@@ -437,6 +452,7 @@ const createOpenCodeExecutor = async ({
         const modelId = typeof model?.id === 'string' && model.id.length > 0 ? model.id : key;
         const inputModalities = ['text'];
         if (model?.capabilities?.input?.image === true) inputModalities.push('image');
+        const output = asObject(model?.capabilities?.output);
         const declaredEfforts = reasoningEffortsFromModel(model);
         const supportedReasoningEfforts =
           declaredEfforts.length > 0 ? declaredEfforts : [DEFAULT_REASONING_EFFORT];
@@ -450,6 +466,15 @@ const createOpenCodeExecutor = async ({
           prefer: ['quality'],
           isDefault: defaults[providerId] === modelId,
           imageGeneration: imageAdapter !== undefined,
+          // Catalog facts used only to rank quota fallbacks.
+          textOnly:
+            output?.text === true &&
+            output.image !== true &&
+            output.audio !== true &&
+            output.video !== true,
+          toolCall: model?.capabilities?.toolcall === true,
+          ...(typeof model?.cost?.output === 'number' ? { outputCost: model.cost.output } : {}),
+          ...(typeof model?.release_date === 'string' ? { releaseDate: model.release_date } : {}),
         });
       }
     }

package/runtime/model-tiers.json

@@ -1,4 +1,12 @@
 {
-  "fast": { "model": "gpt-6-luna", "reasoningEffort": "medium" },
-  "quality": { "model": "gpt-6.1-sol", "reasoningEffort": "medium" }
+  "fast": {
+    "model": "gpt-6-luna",
+    "reasoningEffort": "medium",
+    "fallback": "google"
+  },
+  "quality": {
+    "model": "gpt-6.1-sol",
+    "reasoningEffort": "medium",
+    "fallback": "google"
+  }
 }

package/runtime/protocol.mjs

@@ -7,7 +7,9 @@ import { readFileSync } from 'node:fs';
 
 /**
  * The only place that names a model: each `prefer` tier maps to one model and
- * effort. Callers send a tier; the model is resolved when the call is made.
+ * effort, plus the provider whose free models a tier falls back to when its own is out
+ * of quota.
+ * Callers send a tier; the model is resolved when the call is made.
  */
 const MODEL_TIERS = JSON.parse(
   readFileSync(new URL('./model-tiers.json', import.meta.url), 'utf8'),
@@ -325,6 +327,43 @@ const normalizeGenerationRequest = (value, models) => {
   };
 };
 
+// Catalog variants that are not general text models, or alias one that is listed already.
+const FALLBACK_EXCLUDED = ['embedding', 'computer-use', 'customtools', 'latest', 'deep-research'];
+
+/**
+ * Text models of the tier's fallback provider, from highest to lowest quality, for a
+ * request running its tier's own model; an explicit model never falls back. Output price
+ * ranks quality, after the tier's family (pro for quality, flash for fast) as before.
+ */
+const fallbackModels = (request, models) => {
+  const tier = MODEL_TIERS[request.prefer ?? 'quality'];
+  if (!tier.fallback) return [];
+  if (request.model !== tier.model && !request.model.endsWith(`/${tier.model}`)) return [];
+  const family = request.prefer === 'fast' ? 'flash' : 'pro';
+  return (Array.isArray(models) ? models : [])
+    .filter(
+      (model) =>
+        typeof model?.id === 'string' &&
+        model.id.startsWith(`${tier.fallback}/`) &&
+        model.textOnly === true &&
+        model.toolCall === true &&
+        !FALLBACK_EXCLUDED.some((word) => model.id.includes(word)),
+    )
+    .sort(
+      (a, b) =>
+        Number(b.id.includes(family)) - Number(a.id.includes(family)) ||
+        (b.outputCost ?? 0) - (a.outputCost ?? 0) ||
+        String(b.releaseDate ?? '').localeCompare(String(a.releaseDate ?? '')),
+    )
+    .map((model) => model.id);
+};
+
+/** The same normalized request on another model, with that model's own default effort. */
+const requestOnModel = (request, model, models) => {
+  const { reasoningEffort: _reasoningEffort, ...rest } = request;
+  return normalizeGenerationRequest({ ...rest, model }, models);
+};
+
 const requestDigest = (request) =>
   createHash('sha256').update(JSON.stringify(request)).digest('hex');
 const idempotencyDigest = (key) => createHash('sha256').update(key).digest('hex');
@@ -361,10 +400,12 @@ export {
   MAX_INPUT_ATTACHMENT_BYTES,
   MAX_INPUT_TOTAL_BYTES,
   GenerationProtocolError,
+  fallbackModels,
   idempotencyDigest,
   normalizeGenerationRequest,
   normalizeIdempotencyKey,
   normalizeModelCatalog,
   publicModelCatalog,
   requestDigest,
+  requestOnModel,
 };

package/runtime/server.mjs

@@ -229,7 +229,9 @@ const genericResult = (record, responseBytes, imageBytes) => {
     !Number.isInteger(response.generatedAt) ||
     response.generatedAt < 0 ||
     typeof response.model !== 'string' ||
-    response.model !== record.model ||
+    // A tier's quota fallback may run a different model than the one recorded.
+    response.model.length === 0 ||
+    response.model.length > 160 ||
     !Array.isArray(response.output) ||
     response.output.length > 64
   ) {