@loreai/gateway 0.13.3 → 0.14.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loreai/gateway",
-  "version": "0.13.3",
+  "version": "0.14.0",
   "type": "module",
   "license": "FSL-1.1-Apache-2.0",
   "description": "Lore as a transparent LLM proxy — context management for any AI coding client",
@@ -13,23 +13,26 @@
     }
   },
   "bin": {
-    "lore-gateway": "./dist/index.js"
+    "lore": "./dist/bin.cjs",
+    "lore-gateway": "./dist/bin.cjs"
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
     "build": "bun run script/build.ts",
+    "bundle": "bun run script/bundle.ts",
+    "build:binary": "bun run script/build.ts --binary",
     "start": "bun run src/index.ts"
   },
-  "dependencies": {
-    "@loreai/core": "0.13.3"
-  },
+  "dependencies": {},
   "files": [
     "src/",
     "dist/",
+    "!dist/**/*.map",
     "README.md",
     "LICENSE"
   ],
   "engines": {
+    "node": ">=22.15",
     "bun": ">=1.2.0"
   },
   "repository": {
@@ -49,5 +52,10 @@
     "anthropic",
     "openai"
   ],
-  "author": "BYK"
+  "author": "BYK",
+  "devDependencies": {
+    "@loreai/core": "0.14.0",
+    "@sentry/bun": "^10.52.0",
+    "binpunch": "^1.0.0"
+  }
 }

package/src/batch-queue.ts

@@ -135,6 +135,9 @@ export function createBatchLLMClient(
   let flushTimer: ReturnType<typeof setInterval> | null = null;
   let shuttingDown = false;
 
+  /** Credentials whose batch API access has been permanently disabled (401/403). */
+  const disabledBatchAuth = new Set<string>();
+
   // Stats
   let totalQueued = 0;
   let totalBatched = 0;
@@ -169,7 +172,19 @@ export function createBatchLLMClient(
 
       if (!response.ok) {
         const text = await response.text().catch(() => "(no body)");
-        log.error(`batch create failed: ${response.status} ${response.statusText} — ${text}`);
+        // Permanent auth errors — disable batch API for this credential
+        if (response.status === 401 || response.status === 403) {
+          const key = authKey(auth);
+          if (!disabledBatchAuth.has(key)) {
+            disabledBatchAuth.add(key);
+            log.warn(
+              `batch API disabled for this credential (${response.status}): ${text}. ` +
+                `Future worker calls will use individual requests.`,
+            );
+          }
+        } else {
+          log.error(`batch create failed: ${response.status} ${response.statusText} — ${text}`);
+        }
         // Fall back to synchronous for all items
         await fallbackAll(items);
         return;
@@ -231,6 +246,11 @@ export function createBatchLLMClient(
     }
 
     for (const { auth, items } of byAuth.values()) {
+      // Skip batch API for credentials with permanent auth failures
+      if (disabledBatchAuth.has(authKey(auth))) {
+        await fallbackAll(items);
+        continue;
+      }
       await submitBatch(auth, items);
     }
   }

package/src/cache-analytics.ts

@@ -0,0 +1,344 @@
+/**
+ * Cache analytics — deterministic cache-bust detection using API request
+ * body prefix comparison and Anthropic response cache fields.
+ *
+ * Instead of fingerprinting internal message representations, this module
+ * compares the actual serialized JSON request body byte-for-byte across
+ * turns. When the prefix diverges, it maps the byte offset back to a
+ * semantic location in the JSON structure (e.g. "messages[3].content[1]").
+ *
+ * The API response's `cache_read_input_tokens` and
+ * `cache_creation_input_tokens` provide ground-truth confirmation.
+ *
+ * Request bodies are stored zstd-compressed (~99.9% reduction on
+ * repetitive JSON) to keep per-session memory overhead low.
+ */
+
+import type {
+  CacheAnalytics,
+  CacheTurnAnalysis,
+  GatewayUsage,
+} from "./translate/types.ts";
+import { log } from "@loreai/core";
+
+// ---------------------------------------------------------------------------
+// Compression helpers (Bun built-in zstd, zero dependencies)
+// ---------------------------------------------------------------------------
+
+export function compressBody(body: string): Uint8Array {
+  return Bun.zstdCompressSync(Buffer.from(body));
+}
+
+export function decompressBody(compressed: Uint8Array): string {
+  return Buffer.from(
+    Bun.zstdDecompressSync(compressed as Uint8Array<ArrayBuffer>),
+  ).toString();
+}
+
+// ---------------------------------------------------------------------------
+// Byte-level prefix comparison
+// ---------------------------------------------------------------------------
+
+/**
+ * Find the byte offset where two strings first differ.
+ * Returns the length of the shorter string if one is a prefix of the other.
+ */
+export function findDivergenceOffset(a: string, b: string): number {
+  const len = Math.min(a.length, b.length);
+  for (let i = 0; i < len; i++) {
+    if (a[i] !== b[i]) return i;
+  }
+  return len;
+}
+
+// ---------------------------------------------------------------------------
+// Semantic location mapping
+// ---------------------------------------------------------------------------
+
+/**
+ * Map a byte offset in a serialized JSON string to a semantic JSON path.
+ *
+ * Walks the JSON structure character-by-character, tracking the current
+ * path (keys and array indices). Stops when we reach the target offset.
+ *
+ * Returns a human-readable path like "messages[3].content[1].text" or
+ * "system" or "tools[2].name".
+ */
+export function mapOffsetToJsonPath(json: string, offset: number): string {
+  if (offset >= json.length) return "<end>";
+  if (offset === 0) return "<start>";
+
+  // Stack-based JSON path tracker. Each frame represents a nesting level.
+  // - object frames: { kind: "object", key: current key or "" }
+  // - array frames:  { kind: "array", index: current element index }
+  type Frame =
+    | { kind: "object"; key: string }
+    | { kind: "array"; index: number };
+
+  const stack: Frame[] = [];
+  let inString = false;
+  let escaped = false;
+  let currentKey = "";
+  let collectingKey = false;
+
+  for (let i = 0; i < json.length && i < offset; i++) {
+    const ch = json[i];
+
+    if (inString) {
+      if (escaped) {
+        escaped = false;
+        if (collectingKey) currentKey += ch;
+        continue;
+      }
+      if (ch === "\\") {
+        escaped = true;
+        if (collectingKey) currentKey += ch;
+        continue;
+      }
+      if (ch === '"') {
+        inString = false;
+        collectingKey = false;
+        continue;
+      }
+      if (collectingKey) currentKey += ch;
+      continue;
+    }
+
+    switch (ch) {
+      case '"':
+        inString = true;
+        // Determine if this quote starts a key (object context, key position)
+        collectingKey = isObjectKeyPosition(json, i);
+        if (collectingKey) currentKey = "";
+        break;
+
+      case "{":
+        stack.push({ kind: "object", key: "" });
+        break;
+
+      case "[":
+        stack.push({ kind: "array", index: 0 });
+        break;
+
+      case "}":
+      case "]":
+        stack.pop();
+        break;
+
+      case ":":
+        // Assign the collected key to the current object frame
+        if (currentKey && stack.length > 0) {
+          const top = stack[stack.length - 1];
+          if (top.kind === "object") {
+            top.key = currentKey;
+          }
+          currentKey = "";
+        }
+        break;
+
+      case ",":
+        // In array context, advance to the next element
+        if (stack.length > 0) {
+          const top = stack[stack.length - 1];
+          if (top.kind === "array") {
+            top.index++;
+          }
+        }
+        break;
+    }
+  }
+
+  // Build path from the stack
+  const parts: string[] = [];
+  for (const frame of stack) {
+    if (frame.kind === "object" && frame.key) {
+      parts.push(parts.length === 0 ? frame.key : `.${frame.key}`);
+    } else if (frame.kind === "array") {
+      parts.push(`[${frame.index}]`);
+    }
+  }
+
+  return parts.length === 0 ? "<root>" : parts.join("");
+}
+
+/**
+ * Determine if a '"' at `offset` starts an object key.
+ * Scans backwards to find the last structural character — if it's '{' or ','
+ * we're in key position (not value position).
+ */
+function isObjectKeyPosition(json: string, offset: number): boolean {
+  for (let i = offset - 1; i >= 0; i--) {
+    const ch = json[i];
+    if (ch === " " || ch === "\n" || ch === "\r" || ch === "\t") continue;
+    // After '{' or ',' in object = key position
+    return ch === "{" || ch === ",";
+  }
+  return false;
+}
+
+// ---------------------------------------------------------------------------
+// Divergence reason inference
+// ---------------------------------------------------------------------------
+
+/**
+ * Infer a human-readable reason from the semantic path.
+ */
+export function inferDivergenceReason(
+  path: string,
+  prevLength: number,
+  currLength: number,
+): string {
+  if (path === "<end>") {
+    return currLength > prevLength
+      ? "new content appended"
+      : "content truncated";
+  }
+  if (path === "<start>") return "request structure changed from start";
+  if (path === "<root>") return "top-level structure changed";
+
+  if (path === "system" || path.startsWith("system"))
+    return "system prompt changed";
+  if (path === "model") return "model changed";
+  if (path === "max_tokens") return "max_tokens changed";
+  if (path === "tools" || path.startsWith("tools"))
+    return "tool definitions changed";
+
+  // messages[N] patterns
+  const msgMatch = path.match(/^messages\[(\d+)\]/);
+  if (msgMatch) {
+    const idx = parseInt(msgMatch[1], 10);
+    const rest = path.slice(msgMatch[0].length);
+
+    if (!rest) return `message ${idx} structure changed`;
+    if (rest === ".role") return `message ${idx} role changed`;
+    if (rest.startsWith(".content"))
+      return `message ${idx} content changed`;
+  }
+
+  return `changed at ${path}`;
+}
+
+// ---------------------------------------------------------------------------
+// Main analytics function
+// ---------------------------------------------------------------------------
+
+/**
+ * Analyze cache performance for a turn. Compares the current request body
+ * with the previous turn's (stored compressed) and incorporates the API
+ * response's cache usage fields.
+ *
+ * Updates `analytics` state in-place and returns the per-turn analysis.
+ */
+export function analyzeCacheTurn(
+  analytics: CacheAnalytics,
+  currentBody: string,
+  usage: GatewayUsage,
+  sessionID?: string,
+): CacheTurnAnalysis {
+  analytics.turnCount++;
+
+  const cacheRead = usage.cacheReadInputTokens ?? 0;
+  const cacheCreation = usage.cacheCreationInputTokens ?? 0;
+  const inputTokens = usage.inputTokens ?? 0;
+  const totalInput = cacheRead + cacheCreation + inputTokens;
+  const cacheHitRate = totalInput > 0 ? cacheRead / totalInput : 0;
+
+  // Track confirmed busts (API says no cache hit + new cache written)
+  if (cacheRead === 0 && cacheCreation > 0 && analytics.turnCount > 1) {
+    analytics.bustCount++;
+  }
+
+  // Default values for first turn (no previous body to compare)
+  let prefixMatchBytes = 0;
+  let prefixMatchPercent = 0;
+  let divergencePoint = "<first-turn>";
+  let divergenceReason = "first turn — no previous request to compare";
+
+  // Compare with previous body if available
+  if (analytics.lastRequestBody !== null) {
+    const prevBody = decompressBody(analytics.lastRequestBody);
+    const prevLength = prevBody.length;
+    const currLength = currentBody.length;
+
+    prefixMatchBytes = findDivergenceOffset(prevBody, currentBody);
+    const minLength = Math.min(prevLength, currLength);
+    prefixMatchPercent = minLength > 0 ? prefixMatchBytes / minLength : 0;
+
+    if (prefixMatchBytes < minLength) {
+      // Actual divergence within the shared prefix
+      divergencePoint = mapOffsetToJsonPath(currentBody, prefixMatchBytes);
+      divergenceReason = inferDivergenceReason(
+        divergencePoint,
+        prevLength,
+        currLength,
+      );
+    } else if (prevLength !== currLength) {
+      // One is a prefix of the other — new content appended/removed
+      divergencePoint = "<end>";
+      divergenceReason = currLength > prevLength
+        ? "new content appended (likely new messages)"
+        : "content truncated (context window compressed)";
+    } else {
+      // Identical bodies
+      divergencePoint = "<identical>";
+      divergenceReason = "request bodies are identical";
+    }
+  }
+
+  // Store compressed body for next turn
+  analytics.lastRequestBody = compressBody(currentBody);
+  analytics.lastRequestBodyLength = currentBody.length;
+  analytics.lastCacheRead = cacheRead;
+  analytics.lastCacheCreation = cacheCreation;
+
+  const result: CacheTurnAnalysis = {
+    turn: analytics.turnCount,
+    cacheRead,
+    cacheCreation,
+    inputTokens,
+    cacheHitRate,
+    prefixMatchBytes,
+    prefixMatchPercent,
+    divergencePoint,
+    divergenceReason,
+  };
+
+  // Log structured analysis
+  if (analytics.turnCount > 1) {
+    const bustStr = cacheRead === 0 && cacheCreation > 0 ? " [BUST]" : "";
+    const sidStr = sessionID ? ` session=${sessionID.slice(0, 16)}` : "";
+    log.info(
+      `cache-analytics:${sidStr} turn=${result.turn}` +
+        ` hit=${(result.cacheHitRate * 100).toFixed(0)}%` +
+        ` read=${cacheRead} create=${cacheCreation} input=${inputTokens}` +
+        ` prefixMatch=${(result.prefixMatchPercent * 100).toFixed(1)}%` +
+        ` (${prefixMatchBytes}/${analytics.lastRequestBodyLength}B)` +
+        ` divergence="${divergencePoint}" reason="${divergenceReason}"` +
+        bustStr,
+    );
+  }
+
+  return result;
+}
+
+/**
+ * Log a cache analytics summary for the session.
+ * Suitable for calling on session cleanup or periodically.
+ */
+export function logCacheAnalyticsSummary(
+  sessionID: string,
+  analytics: CacheAnalytics,
+): void {
+  if (analytics.turnCount === 0) return;
+
+  const bustRate = analytics.turnCount > 1
+    ? analytics.bustCount / (analytics.turnCount - 1)
+    : 0;
+
+  log.info(
+    `cache-analytics summary: session=${sessionID.slice(0, 16)}` +
+      ` turns=${analytics.turnCount}` +
+      ` busts=${analytics.bustCount}` +
+      ` bustRate=${(bustRate * 100).toFixed(0)}%`,
+  );
+}

package/src/cli/agents.ts

@@ -0,0 +1,107 @@
+/**
+ * Agent registry — known AI coding agents that can be launched through
+ * the gateway.
+ *
+ * Each agent defines:
+ *  - How to detect it (binary name on PATH)
+ *  - What env vars to set so it talks through the gateway
+ */
+
+// ---------------------------------------------------------------------------
+// which() — cross-runtime binary lookup
+// ---------------------------------------------------------------------------
+
+/**
+ * Find a binary on PATH. Uses Bun.which() when available (Bun runtime),
+ * falls back to `which`/`where` via child_process (Node.js runtime).
+ */
+function which(binary: string): string | null {
+  // Bun runtime
+  if (typeof Bun !== "undefined" && typeof Bun.which === "function") {
+    return Bun.which(binary);
+  }
+
+  // Node.js runtime
+  try {
+    const { execFileSync } = require("node:child_process") as typeof import("node:child_process");
+    const cmd = process.platform === "win32" ? "where" : "which";
+    const result = execFileSync(cmd, [binary], {
+      encoding: "utf8",
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+    const path = result.trim().split("\n")[0];
+    return path || null;
+  } catch {
+    return null;
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Agent definitions
+// ---------------------------------------------------------------------------
+
+export interface AgentDef {
+  /** Internal identifier, e.g. "claude-code" */
+  name: string;
+  /** Human-readable name, e.g. "Claude Code" */
+  displayName: string;
+  /** Binary to search for on PATH */
+  binary: string;
+  /** Returns the binary path if found, or null */
+  detect: () => string | null;
+  /** Env vars to inject given the gateway URL (e.g. "http://127.0.0.1:6969") */
+  envVars: (gatewayUrl: string) => Record<string, string>;
+}
+
+export const AGENTS: AgentDef[] = [
+  {
+    name: "claude-code",
+    displayName: "Claude Code",
+    binary: "claude",
+    detect: () => which("claude"),
+    envVars: (url) => ({ ANTHROPIC_BASE_URL: url }),
+  },
+  {
+    name: "codex",
+    displayName: "Codex",
+    binary: "codex",
+    detect: () => which("codex"),
+    envVars: (url) => ({ OPENAI_BASE_URL: `${url}/v1` }),
+  },
+  {
+    name: "pi",
+    displayName: "Pi",
+    binary: "pi",
+    detect: () => which("pi"),
+    envVars: (url) => ({ ANTHROPIC_BASE_URL: url }),
+  },
+  {
+    name: "opencode",
+    displayName: "OpenCode",
+    binary: "opencode",
+    detect: () => which("opencode"),
+    envVars: (url) => ({ OPENAI_BASE_URL: `${url}/v1` }),
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Detection
+// ---------------------------------------------------------------------------
+
+export interface DetectedAgent {
+  def: AgentDef;
+  path: string;
+}
+
+/**
+ * Scan PATH for all known agents. Returns the ones found with their
+ * binary paths.
+ */
+export function detectAgents(): DetectedAgent[] {
+  const found: DetectedAgent[] = [];
+  for (const def of AGENTS) {
+    const path = def.detect();
+    if (path) found.push({ def, path });
+  }
+  return found;
+}

package/src/cli/bin.ts

@@ -0,0 +1,11 @@
+/**
+ * Binary entry point — called when running the standalone Bun binary
+ * or directly via `bun run src/cli/bin.ts`.
+ */
+import "../../instrument";
+import { _cli } from "./main";
+
+_cli().catch((e) => {
+  if (e) console.error(e);
+  process.exit(1);
+});

package/src/cli/help.ts

@@ -0,0 +1,55 @@
+/**
+ * CLI help text — printed by `lore help` and `lore --help`.
+ */
+import { VERSION } from "./version";
+
+const USAGE = `
+lore v${VERSION} — context management proxy for AI coding agents
+
+Usage:
+  lore [command] [options]
+
+Commands:
+  run [command...]    Start gateway and launch an AI agent (default)
+  start               Start the gateway server only
+  upgrade [version]   Update lore to the latest (or specified) version
+                      Flags: --check, --force, --offline, --channel <ch>
+  help                Show this help text
+
+Options:
+  -p, --port <port>   Gateway port (default: 6969, env: LORE_LISTEN_PORT)
+  -H, --host <host>   Gateway host (default: 127.0.0.1, env: LORE_LISTEN_HOST)
+  -d, --debug         Enable debug logging (env: LORE_DEBUG=1)
+  -v, --version       Print version and exit
+  -h, --help          Show this help text
+
+Examples:
+  lore                          # Auto-detect agent and launch with gateway
+  lore run claude               # Launch Claude Code through the gateway
+  lore run opencode             # Launch OpenCode through the gateway
+  lore start                    # Start gateway only (set ANTHROPIC_BASE_URL yourself)
+  lore start -p 8080            # Start gateway on a custom port
+  lore upgrade                  # Upgrade to latest version
+  lore upgrade --check          # Check for updates without installing
+  lore upgrade --force          # Force re-download even if up to date
+  lore upgrade nightly          # Switch to nightly channel and update
+  lore upgrade stable           # Switch back to stable channel
+  lore upgrade 0.14.0           # Install a specific version
+  lore upgrade --offline        # Upgrade from cached patches (no network)
+
+Environment variables:
+  LORE_LISTEN_PORT              Gateway port (overridden by --port)
+  LORE_LISTEN_HOST              Gateway host (overridden by --host)
+  LORE_UPSTREAM_ANTHROPIC       Upstream Anthropic API URL
+  LORE_UPSTREAM_OPENAI          Upstream OpenAI API URL
+  LORE_DEBUG                    Enable debug logging (1 or true)
+  LORE_NO_UPDATE_CHECK          Disable background update checks (set to 1)
+`.trimStart();
+
+export function printHelp(): void {
+  console.log(USAGE);
+}
+
+export function printVersion(): void {
+  console.log(VERSION);
+}