nexus-agents 2.166.3 → 2.167.0

package/dist/{chunk-WOY6TQCH.js → chunk-ZSUN7BHE.js}

@@ -1,37 +1,50 @@
 import {
   resolveToken
-} from "./chunk-UWRDWX6L.js";
+} from "./chunk-UNIDAVVR.js";
 import {
   REGISTRY_PATH,
   getProjectRoot,
   loadPapersRegistry,
   loadTechniquesRegistry,
   savePapersRegistry
-} from "./chunk-6LXTT6AK.js";
+} from "./chunk-PC4Y4VPX.js";
 import {
   resolveInsideRoot
 } from "./chunk-NUBSJGQZ.js";
 import {
   getToolMemory
-} from "./chunk-I7OZP6FS.js";
+} from "./chunk-KM3YEIEN.js";
 import {
   capitalizeWords
 } from "./chunk-6T3EPABN.js";
+import {
+  extractSystemPrompt,
+  generateCacheKey,
+  messagesToAnthropicFormat
+} from "./chunk-QZUG667L.js";
 import {
   API_TIMEOUTS,
   CLI_NAMES,
   CLI_SUBPROCESS_TIMEOUTS,
+  ErrorCode,
+  JsonlStore,
+  NexusError,
   StrategyDistiller,
   createLogger,
+  err,
   getErrorMessage,
   getOutcomeStore,
   getTimeProvider,
+  ok,
   registerPersistentDistillerFactory
-} from "./chunk-Z66EEQ4A.js";
+} from "./chunk-CURHM4TH.js";
 import {
   ensureLearningDir,
   getRulesFile
 } from "./chunk-AXTD4TFK.js";
+import {
+  nexusDataPath
+} from "./chunk-DHVMSIT5.js";
 
 // src/learning/strategy-distiller-persistence.ts
 import { writeFileSync, readFileSync, renameSync, unlinkSync, existsSync } from "fs";
@@ -704,8 +717,8 @@ function parseArxivXml(arxivId, xml) {
 function createArxivError(code, arxivId, message, cause) {
   return { code, message, arxivId, cause };
 }
-function getErrorCodeFromException(err) {
-  if (err instanceof Error && err.name === "TimeoutError") {
+function getErrorCodeFromException(err2) {
+  if (err2 instanceof Error && err2.name === "TimeoutError") {
     return "TIMEOUT";
   }
   return "NETWORK";
@@ -735,10 +748,10 @@ async function fetchArxivMetadataResult(arxivId) {
       };
     }
     return { ok: true, value: metadata };
-  } catch (err) {
-    const code = getErrorCodeFromException(err);
+  } catch (err2) {
+    const code = getErrorCodeFromException(err2);
     const message = code === "TIMEOUT" ? `arXiv API request timed out after ${String(ARXIV_API_TIMEOUT_MS / 1e3)} seconds` : "Network error while fetching from arXiv API";
-    return { ok: false, error: createArxivError(code, arxivId, message, err) };
+    return { ok: false, error: createArxivError(code, arxivId, message, err2) };
   }
 }
 async function paperExists(arxivId) {
@@ -2421,6 +2434,403 @@ function clamp01(n) {
   return Math.min(1, Math.max(0, n));
 }
 
+// src/context/token-counter.ts
+import Anthropic from "@anthropic-ai/sdk";
+import { GoogleGenAI } from "@google/genai";
+import { encoding_for_model } from "tiktoken";
+
+// src/context/token-counter-types.ts
+var TokenCounterProvider = {
+  ANTHROPIC: "anthropic",
+  GEMINI: "gemini",
+  OPENAI: "openai"
+};
+var TokenCountError = class extends NexusError {
+  constructor(message, options) {
+    super(message, { code: ErrorCode.MODEL_ERROR, ...options });
+    this.name = "TokenCountError";
+  }
+};
+var CHARS_PER_TOKEN2 = {
+  anthropic: 3.5,
+  // Claude models
+  gemini: 4,
+  // Gemini models
+  openai: 4,
+  // GPT models
+  default: 4
+  // Generic fallback
+};
+var DEFAULT_MAX_CACHE_SIZE = 1e3;
+var DEFAULT_CACHE_TTL_MS = 5 * 60 * 1e3;
+var DEFAULT_TIKTOKEN_MODEL = "gpt-4o";
+var TIKTOKEN_MODEL_MAP = {
+  "gpt-4o": "gpt-4o",
+  "gpt-4o-mini": "gpt-4o",
+  "gpt-4-turbo": "gpt-4-turbo",
+  "gpt-4": "gpt-4",
+  "gpt-3.5-turbo": "gpt-3.5-turbo",
+  o1: "o1",
+  "o1-mini": "o1",
+  "o1-preview": "o1"
+};
+
+// src/context/token-counter.ts
+var TokenCounter = class {
+  anthropicClient;
+  geminiClient;
+  cache;
+  maxCacheSize;
+  cacheTtlMs;
+  tiktokenEncoder;
+  currentTiktokenModel;
+  /**
+   * Creates a new TokenCounter instance.
+   *
+   * @param config - Token counter configuration
+   */
+  constructor(config = {}) {
+    this.maxCacheSize = config.maxCacheSize ?? DEFAULT_MAX_CACHE_SIZE;
+    this.cacheTtlMs = config.cacheTtlMs ?? DEFAULT_CACHE_TTL_MS;
+    this.cache = /* @__PURE__ */ new Map();
+    if (config.anthropicApiKey !== void 0 && config.anthropicApiKey !== "") {
+      this.anthropicClient = new Anthropic({ apiKey: config.anthropicApiKey });
+    }
+    if (config.googleApiKey !== void 0 && config.googleApiKey !== "") {
+      this.geminiClient = new GoogleGenAI({ apiKey: config.googleApiKey });
+    }
+  }
+  /**
+   * Count tokens for Anthropic/Claude models via API.
+   */
+  async countAnthropic(messages, model) {
+    const cacheKey = generateCacheKey(messages, TokenCounterProvider.ANTHROPIC, model);
+    const cached = this.getCached(cacheKey);
+    if (cached !== void 0) {
+      return ok({ count: cached.count, cached: true, provider: cached.provider, model });
+    }
+    if (this.anthropicClient === void 0) {
+      return err(
+        new TokenCountError("Anthropic API key not configured", {
+          context: { provider: "anthropic", model }
+        })
+      );
+    }
+    try {
+      const anthropicMessages = messagesToAnthropicFormat(messages);
+      const systemPrompt = extractSystemPrompt(messages);
+      const params = {
+        model,
+        messages: anthropicMessages
+      };
+      if (systemPrompt !== void 0) {
+        params.system = systemPrompt;
+      }
+      const response = await this.anthropicClient.messages.countTokens(params);
+      const count = response.input_tokens;
+      this.setCache(cacheKey, {
+        count,
+        provider: TokenCounterProvider.ANTHROPIC,
+        model,
+        timestamp: getTimeProvider().now()
+      });
+      return ok({ count, cached: false, provider: TokenCounterProvider.ANTHROPIC, model });
+    } catch (error) {
+      const cause = error instanceof Error ? error : new Error(String(error));
+      return err(
+        new TokenCountError(`Anthropic token counting failed: ${cause.message}`, {
+          cause,
+          context: { provider: "anthropic", model }
+        })
+      );
+    }
+  }
+  /**
+   * Count tokens for Gemini models via API.
+   */
+  async countGemini(content, model) {
+    const cacheKey = generateCacheKey(content, TokenCounterProvider.GEMINI, model);
+    const cached = this.getCached(cacheKey);
+    if (cached !== void 0) {
+      return ok({ count: cached.count, cached: true, provider: cached.provider, model });
+    }
+    if (this.geminiClient === void 0) {
+      return err(
+        new TokenCountError("Google API key not configured", {
+          context: { provider: "gemini", model }
+        })
+      );
+    }
+    try {
+      const response = await this.geminiClient.models.countTokens({
+        model,
+        contents: content
+      });
+      const count = response.totalTokens ?? 0;
+      this.setCache(cacheKey, {
+        count,
+        provider: TokenCounterProvider.GEMINI,
+        model,
+        timestamp: getTimeProvider().now()
+      });
+      return ok({ count, cached: false, provider: TokenCounterProvider.GEMINI, model });
+    } catch (error) {
+      const cause = error instanceof Error ? error : new Error(String(error));
+      return err(
+        new TokenCountError(`Gemini token counting failed: ${cause.message}`, {
+          cause,
+          context: { provider: "gemini", model }
+        })
+      );
+    }
+  }
+  /**
+   * Count tokens for OpenAI models using local tiktoken.
+   */
+  countOpenAI(text, model = DEFAULT_TIKTOKEN_MODEL) {
+    const cacheKey = generateCacheKey(text, TokenCounterProvider.OPENAI, model);
+    const cached = this.getCached(cacheKey);
+    if (cached !== void 0) {
+      return ok({ count: cached.count, cached: true, provider: cached.provider, model });
+    }
+    try {
+      const encoder = this.getOrCreateTiktokenEncoder(model);
+      const tokens = encoder.encode(text);
+      const count = tokens.length;
+      this.setCache(cacheKey, {
+        count,
+        provider: TokenCounterProvider.OPENAI,
+        model,
+        timestamp: getTimeProvider().now()
+      });
+      return ok({ count, cached: false, provider: TokenCounterProvider.OPENAI, model });
+    } catch (error) {
+      const cause = error instanceof Error ? error : new Error(String(error));
+      return err(
+        new TokenCountError(`OpenAI token counting failed: ${cause.message}`, {
+          cause,
+          context: { provider: "openai", model }
+        })
+      );
+    }
+  }
+  /**
+   * Estimate tokens offline using character-based heuristic.
+   * Uses ~4 characters per token as a general approximation.
+   */
+  estimate(text) {
+    if (text.length === 0) {
+      return 0;
+    }
+    return Math.ceil(text.length / CHARS_PER_TOKEN2.default);
+  }
+  /**
+   * Estimate tokens for a specific provider.
+   */
+  estimateForProvider(text, provider) {
+    if (text.length === 0) {
+      return 0;
+    }
+    const charsPerToken = CHARS_PER_TOKEN2[provider];
+    return Math.ceil(text.length / charsPerToken);
+  }
+  /**
+   * Clear the token count cache.
+   */
+  clearCache() {
+    this.cache.clear();
+  }
+  /**
+   * Get current cache statistics.
+   */
+  getCacheStats() {
+    return {
+      size: this.cache.size,
+      maxSize: this.maxCacheSize,
+      ttlMs: this.cacheTtlMs
+    };
+  }
+  /**
+   * Gets or creates a tiktoken encoder for the specified model.
+   */
+  getOrCreateTiktokenEncoder(model) {
+    const tiktokenModel = TIKTOKEN_MODEL_MAP[model] ?? DEFAULT_TIKTOKEN_MODEL;
+    if (this.tiktokenEncoder !== void 0 && this.currentTiktokenModel === tiktokenModel) {
+      return this.tiktokenEncoder;
+    }
+    if (this.tiktokenEncoder !== void 0) {
+      this.tiktokenEncoder.free();
+    }
+    this.tiktokenEncoder = encoding_for_model(
+      tiktokenModel
+    );
+    this.currentTiktokenModel = tiktokenModel;
+    return this.tiktokenEncoder;
+  }
+  /**
+   * Gets a cached entry if valid.
+   */
+  getCached(key) {
+    const entry = this.cache.get(key);
+    if (entry === void 0) {
+      return void 0;
+    }
+    if (getTimeProvider().now() - entry.timestamp > this.cacheTtlMs) {
+      this.cache.delete(key);
+      return void 0;
+    }
+    return entry;
+  }
+  /**
+   * Sets a cache entry, evicting oldest if at capacity.
+   */
+  setCache(key, entry) {
+    if (this.cache.size >= this.maxCacheSize) {
+      const firstKey = this.cache.keys().next();
+      if (firstKey.done !== true) {
+        this.cache.delete(firstKey.value);
+      }
+    }
+    this.cache.set(key, entry);
+  }
+  /**
+   * Frees resources (tiktoken encoder).
+   * Call this when done with the counter.
+   */
+  dispose() {
+    if (this.tiktokenEncoder !== void 0) {
+      this.tiktokenEncoder.free();
+      this.tiktokenEncoder = void 0;
+      this.currentTiktokenModel = void 0;
+    }
+    this.cache.clear();
+  }
+};
+function createTokenCounter(config = {}) {
+  return new TokenCounter(config);
+}
+
+// src/context/token-ledger.ts
+import { z as z5 } from "zod";
+var TokenLedgerEventSchema = z5.object({
+  /** Schema version. Forward-compat for on-disk evolution. */
+  v: z5.literal(1),
+  /** ISO-8601 timestamp of the call. */
+  ts: z5.iso.datetime(),
+  /** Tool/call-site identifier (e.g. an MCP tool name or module.function). */
+  tool: z5.string().min(1).max(200),
+  /** Context-source classification. See {@link CONTEXT_SOURCE_TAGS}. */
+  contextSource: z5.string().min(1).max(64),
+  /** Input tokens attributed to this call/source. */
+  inputTokens: z5.number().nonnegative(),
+  /** Output tokens attributed to this call. `0` when the call is context-only. */
+  outputTokens: z5.number().nonnegative(),
+  /** Model identifier, when the calling surface exposes one. */
+  model: z5.string().min(1).max(200).optional(),
+  /** Task/operation identifier, mirroring {@link TokenUsageRecord.taskId}. */
+  taskId: z5.string().min(1).max(200).optional(),
+  /**
+   * Free-form call-site variant (e.g. `'ranked' | 'legacy'` for the
+   * `context-retriever.ts` rendering path, #4252 wiring). Optional dimension
+   * so producers that have no variant to report don't need a placeholder.
+   */
+  variant: z5.string().min(1).max(100).optional()
+});
+var DEFAULT_MAX_EVENTS = 5e4;
+var LEDGER_SUBDIR = "token-ledger";
+var LEDGER_FILE = "ledger.jsonl";
+var TokenLedger = class {
+  store;
+  constructor(config) {
+    this.store = new JsonlStore({
+      filePath: config?.filePath ?? nexusDataPath(LEDGER_SUBDIR, LEDGER_FILE),
+      schema: TokenLedgerEventSchema,
+      maxRecords: config?.maxEvents ?? DEFAULT_MAX_EVENTS,
+      component: "TokenLedger"
+    });
+  }
+  /**
+   * Record one call's token usage. `v`/`ts` are stamped here so callers pass
+   * only the observed signal. Best-effort durability (inherited from
+   * JsonlStore): never throws into the observed operation.
+   */
+  record(event) {
+    const record = {
+      v: 1,
+      ts: event.ts ?? (/* @__PURE__ */ new Date()).toISOString(),
+      tool: event.tool,
+      contextSource: event.contextSource,
+      inputTokens: event.inputTokens,
+      outputTokens: event.outputTokens ?? 0,
+      ...event.model !== void 0 ? { model: event.model } : {},
+      ...event.taskId !== void 0 ? { taskId: event.taskId } : {},
+      ...event.variant !== void 0 ? { variant: event.variant } : {}
+    };
+    this.store.append(record);
+  }
+  /** Total retained events. */
+  size() {
+    return this.store.count();
+  }
+  /** All retained events, oldest first. Mainly for tests/inspection. */
+  all() {
+    return this.store.all();
+  }
+  /**
+   * Aggregate totals by `contextSource` tag and by `tool`, optionally
+   * restricted to a time window (#4252 query surface). Pure over the current
+   * retained event set — no I/O.
+   *
+   * Manual before/after comparison (the #4251 A/B measurement need) is
+   * possible today by calling this twice with windows that bracket a change,
+   * e.g. `summarize({ untilMs: deployedAt })` vs.
+   * `summarize({ sinceMs: deployedAt })`. A dedicated harness that fixes a
+   * task sample and runs it under two configurations is deliberately out of
+   * scope here — tracked as a follow-up under epic #4251.
+   */
+  summarize(window = {}) {
+    const events = this.store.all().filter((e) => withinWindow(e.ts, window));
+    return aggregateEvents(events);
+  }
+};
+function withinWindow(ts, window) {
+  const t = Date.parse(ts);
+  if (window.sinceMs !== void 0 && t < window.sinceMs) return false;
+  if (window.untilMs !== void 0 && t >= window.untilMs) return false;
+  return true;
+}
+function emptyAggregate() {
+  return { entries: 0, inputTokens: 0, outputTokens: 0, totalTokens: 0 };
+}
+function addEvent(agg, e) {
+  return {
+    entries: agg.entries + 1,
+    inputTokens: agg.inputTokens + e.inputTokens,
+    outputTokens: agg.outputTokens + e.outputTokens,
+    totalTokens: agg.totalTokens + e.inputTokens + e.outputTokens
+  };
+}
+function aggregateEvents(events) {
+  let overall = emptyAggregate();
+  const bySource = /* @__PURE__ */ new Map();
+  const byTool = /* @__PURE__ */ new Map();
+  for (const e of events) {
+    overall = addEvent(overall, e);
+    bySource.set(e.contextSource, addEvent(bySource.get(e.contextSource) ?? emptyAggregate(), e));
+    byTool.set(e.tool, addEvent(byTool.get(e.tool) ?? emptyAggregate(), e));
+  }
+  return {
+    overall,
+    bySource: Object.fromEntries(bySource),
+    byTool: Object.fromEntries(byTool)
+  };
+}
+var singleton;
+function getTokenLedger() {
+  singleton ??= new TokenLedger();
+  return singleton;
+}
+
 // src/context/context-retriever.ts
 var DEFAULT_LIMIT = 5;
 async function getContextForTask(options) {
@@ -2580,8 +2990,21 @@ function oneLine2(value) {
   return value.replace(/\s+/g, " ").trim().slice(0, MAX_SUMMARY_FIELD);
 }
 function summarizeContextForPrompt(ctx, budgetTokens = DEFAULT_CONTEXT_BUDGET_TOKENS) {
-  const rendered = process.env[CONTEXT_RANKED_FLAG] === "1" ? summarizeRankedContext(ctx) : summarizeLegacyContext(ctx);
-  return clampRenderedContext(rendered, budgetTokens);
+  const ranked = process.env[CONTEXT_RANKED_FLAG] === "1";
+  const rendered = ranked ? summarizeRankedContext(ctx) : summarizeLegacyContext(ctx);
+  const clamped = clampRenderedContext(rendered, budgetTokens);
+  recordAssembledContextTokens(clamped, ranked);
+  return clamped;
+}
+var contextTokenCounter = createTokenCounter();
+function recordAssembledContextTokens(rendered, ranked) {
+  if (rendered === "") return;
+  getTokenLedger().record({
+    tool: "context-retriever.summarizeContextForPrompt",
+    contextSource: "memory-backend",
+    inputTokens: contextTokenCounter.estimate(rendered),
+    variant: ranked ? "ranked" : "legacy"
+  });
 }
 var DEFAULT_CONTEXT_BUDGET_TOKENS = 2500;
 var CLIP_NOTICE_RESERVE_TOKENS = 30;
@@ -2702,6 +3125,10 @@ export {
   handleCheckCommand,
   formatResearchIssueBody,
   createResearchIssue,
+  TokenCounterProvider,
+  TokenCountError,
+  TokenCounter,
+  createTokenCounter,
   getContextForTask,
   selectRelevantResearch,
   getResearchInsightsForTask,
@@ -2710,4 +3137,4 @@ export {
   DEFAULT_CONTEXT_BUDGET_TOKENS,
   getContextPromptPrefix
 };
-//# sourceMappingURL=chunk-WOY6TQCH.js.map
+//# sourceMappingURL=chunk-ZSUN7BHE.js.map