@autorag/librarian 2.13.2 → 2.13.4

package/dist/core.js

@@ -15098,6 +15098,51 @@ var init_permission_gate = __esm(() => {
   };
 });
 
+// src/cloud/context-overflow.ts
+import {
+  createAssistantMessageEventStream,
+  isContextOverflow
+} from "@earendil-works/pi-ai";
+import { getApiProvider } from "@earendil-works/pi-ai/compat";
+function recordRequestEstimate(model, estimatedTokens) {
+  lastEstimates.set(modelKey(model), estimatedTokens);
+}
+function noteContextOverflow(model) {
+  const key = modelKey(model);
+  const failed = lastEstimates.get(key);
+  if (failed === undefined)
+    return;
+  ceilings.set(key, Math.min(Math.floor(failed / 2), ceilings.get(key) ?? Number.POSITIVE_INFINITY));
+}
+function contextOverflowCeiling(model) {
+  return ceilings.get(modelKey(model));
+}
+function streamDazziSimple(model, context, options) {
+  const api = getApiProvider(model.api);
+  if (api === undefined)
+    throw new Error(`No API provider registered for api: ${model.api}`);
+  const inner = api.streamSimple(model, context, options);
+  const stream = createAssistantMessageEventStream();
+  (async () => {
+    try {
+      for await (const event of inner) {
+        if (event.type === "error" && event.error.stopReason === "error" && isContextOverflow(event.error)) {
+          noteContextOverflow(model);
+        }
+        stream.push(event);
+      }
+    } finally {
+      stream.end();
+    }
+  })();
+  return stream;
+}
+var lastEstimates, ceilings, modelKey = (model) => `${model.provider}/${model.id}`;
+var init_context_overflow = __esm(() => {
+  lastEstimates = new Map;
+  ceilings = new Map;
+});
+
 // src/cloud/authorize.ts
 function buildAuthorizeUrl(params) {
   const url = new URL(`${normalizeBaseUrl(params.baseUrl)}${AUTHORIZE_PATH}`);
@@ -15453,6 +15498,7 @@ function createDazziProviderConfig(env = process.env) {
     name: PROVIDER_NAME,
     baseUrl: endpoints.apiBaseUrl,
     api: "openai-responses",
+    streamSimple: streamDazziSimple,
     oauth: {
       name: PROVIDER_NAME,
       login: (callbacks) => loginDazzi(callbacks, { baseUrl }),
@@ -15468,6 +15514,7 @@ async function registerDazziProvider(runtime, env = process.env) {
 }
 var init_provider2 = __esm(() => {
   init_config();
+  init_context_overflow();
   init_models();
   init_oauth_login();
 });
@@ -24287,413 +24334,6 @@ var init_slack = __esm(() => {
   };
 });
 
-// src/datasource/skills/spotlight/connector.ts
-import { spawn as spawn19 } from "node:child_process";
-import { createHash as createHash19 } from "node:crypto";
-import { readFile as fsReadFile, stat as stat5 } from "node:fs/promises";
-import { basename as basename11, extname as extname7 } from "node:path";
-
-class SpotlightConnector {
-  options;
-  run;
-  readFile;
-  platform;
-  constructor(options = {}) {
-    this.options = options;
-    this.run = options.run ?? spawnRunner;
-    this.readFile = options.readFile ?? ((path) => fsReadFile(path, "utf8"));
-    this.platform = options.platform ?? process.platform;
-  }
-  async probe() {
-    const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
-    const failure = await this.readinessFailure(queries);
-    return failure ?? { ok: true, documents: [] };
-  }
-  async search(query, limit = DEFAULT_MAX_DOCUMENTS4, acceptDocument) {
-    const trimmed = query.trim();
-    if (trimmed.length === 0 || limit <= 0)
-      return { ok: true, documents: [] };
-    const queries = (this.options.queries ?? []).map((query) => query.trim()).filter((query) => query.length > 0);
-    const failure = await this.readinessFailure(queries);
-    if (failure !== undefined)
-      return failure;
-    const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS11;
-    const maxPerQuery = this.options.maxResultsPerQuery ?? DEFAULT_MAX_RESULTS_PER_QUERY;
-    const maxDocuments = Math.min(this.options.maxDocuments ?? DEFAULT_MAX_DOCUMENTS4, limit);
-    const seen = new Set;
-    const paths = [];
-    const textQuery = trimmed.split(/\s+/u).map((term) => `"${term.replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"")}"`).join(" AND ");
-    let mdfindMissing = false;
-    let failedQueries = 0;
-    const failures = [];
-    for (const filter of queries) {
-      const query = `(${filter}) AND (${textQuery})`;
-      const args = this.options.onlyIn !== undefined ? ["-onlyin", this.options.onlyIn, query] : [query];
-      const result = await this.run("mdfind", args, timeoutMs);
-      if (!result.ok) {
-        if (result.error === "ENOENT") {
-          mdfindMissing = true;
-          break;
-        }
-        failedQueries += 1;
-        failures.push(result);
-        continue;
-      }
-      let kept = 0;
-      for (const line of result.stdout.split(`
-`)) {
-        const path = line.trim();
-        if (path.length === 0 || seen.has(path))
-          continue;
-        if (kept >= maxPerQuery)
-          break;
-        seen.add(path);
-        paths.push(path);
-        kept += 1;
-      }
-    }
-    if (mdfindMissing) {
-      return { ok: false, reason: "unavailable", message: "mdfind binary not found; Spotlight requires macOS" };
-    }
-    if (failedQueries === queries.length) {
-      return {
-        ok: false,
-        reason: "unavailable",
-        message: "every configured mdfind query failed",
-        stderr: failures[0]?.stderr,
-        stdout: failures[0]?.stdout,
-        code: failures[0]?.code
-      };
-    }
-    const documents = [];
-    let permissionDenied = 0;
-    let readFailures = 0;
-    const maxBytes = this.options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE2;
-    const maxContentChars = this.options.maxContentChars ?? MAX_CONTENT_CHARS4;
-    for (const path of paths) {
-      if (documents.length >= maxDocuments)
-        break;
-      const docId = createHash19("sha256").update(path).digest("hex").slice(0, 16);
-      if (acceptDocument !== undefined && !acceptDocument(docId))
-        continue;
-      try {
-        const info = await stat5(path);
-        if (!info.isFile() || info.size > maxBytes)
-          continue;
-        const text = await this.readFile(path);
-        const content = text.trim().slice(0, maxContentChars);
-        if (content.length === 0)
-          continue;
-        documents.push({
-          docId,
-          hierarchy: ["files"],
-          title: basename11(path),
-          content,
-          publishedAt: info.mtimeMs,
-          metadata: { path, extension: extname7(path).toLowerCase() }
-        });
-      } catch (error) {
-        const code = error.code;
-        if (code === "EPERM" || code === "EACCES")
-          permissionDenied += 1;
-        else
-          readFailures += 1;
-      }
-    }
-    if (documents.length === 0 && permissionDenied > 0) {
-      return {
-        ok: false,
-        reason: "permission",
-        message: `macOS denied access to all Spotlight hits; ${FDA_GUIDANCE}`
-      };
-    }
-    if (documents.length === 0 && failedQueries > 0) {
-      return {
-        ok: false,
-        reason: "unavailable",
-        message: "some mdfind queries failed and no readable hits remained",
-        stderr: failures[0]?.stderr,
-        stdout: failures[0]?.stdout,
-        code: failures[0]?.code
-      };
-    }
-    const warnings = [];
-    if (failedQueries > 0)
-      warnings.push(`${failedQueries} mdfind quer(y/ies) failed`);
-    if (permissionDenied > 0) {
-      warnings.push(`${permissionDenied} Spotlight hit(s) were denied by macOS privacy protections; ${FDA_GUIDANCE}`);
-    }
-    if (readFailures > 0)
-      warnings.push(`${readFailures} Spotlight hit(s) failed to read`);
-    return { ok: true, documents, ...warnings.length > 0 ? { warnings } : {} };
-  }
-  async readinessFailure(queries) {
-    if (this.platform !== "darwin") {
-      return { ok: false, reason: "unavailable", message: "spotlight datasource is supported on macOS only" };
-    }
-    if (queries.length === 0) {
-      return { ok: false, reason: "not-configured", message: "no spotlight queries configured" };
-    }
-    const timeoutMs = this.options.timeoutMs ?? DEFAULT_TIMEOUT_MS11;
-    const status = await this.run("mdutil", ["-s", "/"], timeoutMs);
-    if (status.ok && /indexing disabled/iu.test(status.stdout)) {
-      return {
-        ok: false,
-        reason: "unavailable",
-        message: "Spotlight indexing is disabled; enable it with `sudo mdutil -i on /`"
-      };
-    }
-    return;
-  }
-}
-var DEFAULT_MAX_RESULTS_PER_QUERY = 100, DEFAULT_MAX_DOCUMENTS4 = 300, DEFAULT_MAX_BYTES_PER_FILE2, DEFAULT_TIMEOUT_MS11 = 15000, MAX_CONTENT_CHARS4 = 1e5, MAX_BUFFER_BYTES, FDA_GUIDANCE = "grant Full Disk Access to the terminal or app running AutoRAG (System Settings -> Privacy & Security -> Full Disk Access), then restart it", spawnRunner = (command, args, timeoutMs) => new Promise((resolve) => {
-  const child = spawn19(command, [...args], { stdio: ["ignore", "pipe", "pipe"] });
-  let stdout = "";
-  let stderr = "";
-  let settled = false;
-  const finish = (result) => {
-    if (settled)
-      return;
-    settled = true;
-    clearTimeout(timer);
-    resolve(result);
-  };
-  const timer = setTimeout(() => {
-    child.kill("SIGKILL");
-    finish({ ok: false, stdout, stderr: "process timed out", code: null });
-  }, timeoutMs);
-  child.stdout.setEncoding("utf8");
-  child.stderr.setEncoding("utf8");
-  child.stdout.on("data", (chunk) => {
-    if (stdout.length < MAX_BUFFER_BYTES)
-      stdout += chunk;
-  });
-  child.stderr.on("data", (chunk) => {
-    if (stderr.length < MAX_BUFFER_BYTES)
-      stderr += chunk;
-  });
-  child.on("error", (error) => {
-    finish({ ok: false, stdout, stderr: error.message, code: null, error: error.code });
-  });
-  child.on("close", (code) => {
-    finish({ ok: code === 0, stdout, stderr, code });
-  });
-});
-var init_connector5 = __esm(() => {
-  DEFAULT_MAX_BYTES_PER_FILE2 = 256 * 1024;
-  MAX_BUFFER_BYTES = 4 * 1024 * 1024;
-});
-
-// src/datasource/skills/spotlight/skill.ts
-class SpotlightSkill {
-  connector;
-  skillName;
-  instanceId;
-  instances;
-  tags;
-  lastIndexedAt;
-  lastError;
-  constructor(options = {}) {
-    this.skillName = options.skillName ?? "spotlight";
-    this.instanceId = options.instanceId ?? "default";
-    this.instances = options.instances !== undefined && options.instances.length > 0 ? options.instances : [this.instanceId];
-    this.tags = options.tags ?? DEFAULT_TAGS4;
-    this.connector = options.connector ?? new SpotlightConnector(options.connectorOptions ?? {});
-  }
-  describe() {
-    return {
-      name: this.skillName,
-      id: this.skillName,
-      type: "spotlight-search",
-      description: "Live macOS Spotlight search via the built-in mdfind CLI",
-      capabilities: ["files", "macos-only", "local-search", "live-search", "scoped", "external-cli"],
-      tags: this.tags,
-      status: "active",
-      requiresExternalCli: true,
-      datasourceId: this.skillName,
-      instanceId: this.instanceId,
-      instances: this.instances
-    };
-  }
-  polling() {
-    return {
-      mode: "none",
-      ...this.lastIndexedAt !== undefined ? { lastIndexedAt: this.lastIndexedAt } : {},
-      ...this.lastError !== undefined ? { lastError: this.lastError } : {}
-    };
-  }
-  async index() {
-    try {
-      const result = await this.connector.probe();
-      if (!result.ok) {
-        const code = connectorFailureToDiagnosticCode(result.reason);
-        const message = boundDiagnosticText(result.message ?? result.reason);
-        this.lastError = message;
-        return this.fail(code, message);
-      }
-      this.lastError = undefined;
-      this.lastIndexedAt = Date.now();
-      return {
-        ok: true,
-        instanceId: this.instanceId,
-        skill: this.skillName,
-        chunkCount: 0,
-        indexedAt: this.lastIndexedAt,
-        diagnostics: [this.note(LIVE_NOTE, "datasource-empty", "info")]
-      };
-    } catch (error) {
-      const message = boundDiagnosticText(error instanceof Error ? error.message : "spotlight probe failed");
-      this.lastError = message;
-      return this.fail("datasource-unavailable", message);
-    }
-  }
-  retrievalMethods() {
-    return [new SpotlightLiveMethod(this)];
-  }
-  describeSources() {
-    return this.instances.map((instanceId) => ({
-      source: datasourceSourcePath(this.skillName, instanceId),
-      datasourceId: this.skillName,
-      skill: this.skillName,
-      instanceId,
-      contentType: "document",
-      metadata: { datasourceId: this.skillName, instanceId, tags: this.tags }
-    }));
-  }
-  skillManifest() {
-    const scopes = this.instances.map((instanceId) => `- \`${datasourceSourcePath(this.skillName, instanceId)}\``).join(`
-`);
-    return {
-      name: `datasource-${this.skillName}`,
-      description: "Search the current macOS Spotlight index directly. AutoRAG keeps no file snapshot.",
-      content: [
-        "# macOS Spotlight (live mdfind search)",
-        "",
-        "Search files indexed by macOS Spotlight via the built-in `mdfind` CLI. macOS owns and refreshes the index; AutoRAG does not copy files into a local snapshot.",
-        "",
-        "## Platform requirements",
-        "- macOS only: this datasource is unavailable on any other platform.",
-        "- Spotlight indexing must be enabled (`sudo mdutil -i on /`; check with `mdutil -s /`).",
-        "- Full Disk Access may be required to read protected hits. Grant it under System Settings -> Privacy & Security -> Full Disk Access, then restart the host app.",
-        "",
-        "## How to search",
-        `Call the dedicated \`${datasourceSearchToolName(this.skillName)}\` tool with a query and optional narrowing scope. Available scopes:`,
-        scopes,
-        "Each configured Spotlight filter is AND-ed with the user query. Only the requested hits are read; refresh performs a readiness probe without running `mdfind`.",
-        "Result metadata carries the real absolute file path (`path`) so hits are traceable back to disk. If paths must not leave this machine, run AutoRAG with a local LLM.",
-        "",
-        "## Native CLI",
-        "- `mdfind <query>` searches Spotlight; `mdfind -onlyin <dir> <query>` scopes to a directory; `mdfind -name <text>` matches file names.",
-        "- `mdls <file>` shows Spotlight metadata for one file.",
-        "- `mdutil -s /` checks index status; `sudo mdutil -i on /` enables indexing."
-      ].join(`
-`)
-    };
-  }
-  async search(query, options) {
-    const trimmed = query.trim();
-    if (trimmed.length === 0)
-      return [];
-    const scope = options.scope?.trim() ? normalizeVirtualPath2(options.scope) : undefined;
-    if (scope !== undefined) {
-      const chunksRoot = normalizeVirtualPath2(`/${this.skillName}/${this.instanceId}/chunks`);
-      const scopeProbe = datasourceSourcePath(this.skillName, this.instanceId, "__scope_probe__");
-      if (!matchesDatasourceScope(chunksRoot, scope) && !matchesDatasourceScope(scopeProbe, scope) && !scope.startsWith(`${chunksRoot}/`)) {
-        return [];
-      }
-    }
-    const topK = options.topK ?? DEFAULT_TOP_K9;
-    const result = await this.connector.search(trimmed, topK, (docId) => matchesDatasourceScope(datasourceSourcePath(this.skillName, this.instanceId, docId), scope));
-    if (!result.ok) {
-      throw datasourceCliError(this.skillName, "search", {
-        reason: result.reason,
-        stderr: result.stderr ?? result.message,
-        stdout: result.stdout,
-        code: result.code
-      });
-    }
-    const hits = [];
-    for (const [index, document] of result.documents.entries()) {
-      const source = datasourceSourcePath(this.skillName, this.instanceId, document.docId);
-      if (!matchesDatasourceScope(source, options.scope))
-        continue;
-      hits.push({
-        id: `${this.skillName}:${this.instanceId}:${document.docId}`,
-        content: document.content,
-        source,
-        score: 1 / (index + 1),
-        metadata: {
-          ...document.metadata,
-          method: `${this.skillName}-live`,
-          datasourceId: this.skillName,
-          instanceId: this.instanceId,
-          chunkId: document.docId,
-          ...document.title !== undefined ? { title: document.title } : {},
-          ...document.hierarchy !== undefined && document.hierarchy.length > 0 ? { hierarchy: document.hierarchy.join("/") } : {},
-          ...document.publishedAt !== undefined ? { publishedAt: document.publishedAt } : {},
-          ...result.warnings !== undefined && result.warnings.length > 0 ? { warnings: result.warnings } : {}
-        }
-      });
-      if (hits.length >= topK)
-        break;
-    }
-    return hits;
-  }
-  fail(code, message) {
-    return {
-      ok: false,
-      instanceId: this.instanceId,
-      skill: this.skillName,
-      indexedAt: Date.now(),
-      diagnostics: [
-        this.note(message, code, code === "datasource-unavailable" || code === "datasource-empty" ? "warning" : "error")
-      ],
-      error: message,
-      code,
-      message
-    };
-  }
-  note(message, code, severity) {
-    return { code, severity, message, instanceId: this.instanceId, source: this.skillName };
-  }
-}
-
-class SpotlightLiveMethod {
-  skill;
-  constructor(skill) {
-    this.skill = skill;
-  }
-  describe() {
-    const descriptor = this.skill.describe();
-    return {
-      name: `${descriptor.datasourceId ?? "spotlight"}-live`,
-      type: "remote",
-      description: "Live macOS Spotlight search via mdfind",
-      status: "active",
-      capabilities: ["live-search", "scoped", "external-cli", "path-opaque-sources"],
-      datasourceId: descriptor.datasourceId,
-      tags: descriptor.tags
-    };
-  }
-  retrieve(query, options) {
-    return this.skill.search(query, options);
-  }
-}
-var DEFAULT_TAGS4, DEFAULT_TOP_K9 = 20, LIVE_NOTE = "live-search datasource: macOS owns the Spotlight index; AutoRAG stores no snapshot";
-var init_skill12 = __esm(() => {
-  init_connector();
-  init_errors4();
-  init_scope2();
-  init_connector5();
-  DEFAULT_TAGS4 = ["spotlight", "files"];
-});
-
-// src/datasource/skills/spotlight/index.ts
-var init_spotlight = __esm(() => {
-  init_connector5();
-  init_skill12();
-});
-
 // src/datasource/skills/telecrawl/index.ts
 function globalArgs3(options) {
   return [
@@ -25098,7 +24738,6 @@ var init_factory = __esm(() => {
   init_obsidian();
   init_rss();
   init_slack();
-  init_spotlight();
   init_telecrawl();
   init_wacrawl();
   BUILDERS = {
@@ -25235,12 +24874,6 @@ var init_factory = __esm(() => {
       ...common(config, workspaceRoot),
       skillName: registrationName,
       connectorOptions: config.connector
-    }),
-    spotlight: (config, _workspaceRoot, registrationName) => new SpotlightSkill({
-      ...config.instanceId !== undefined ? { instanceId: config.instanceId } : {},
-      ...config.tags !== undefined ? { tags: config.tags } : {},
-      skillName: registrationName,
-      connectorOptions: config.connector
     })
   };
   PLATFORM_ONLY_SKILLS = { kakao: "darwin" };
@@ -25250,7 +24883,7 @@ var init_factory = __esm(() => {
 // src/cli/config.ts
 import { randomUUID as randomUUID6 } from "node:crypto";
 import { existsSync as existsSync27, mkdirSync as mkdirSync22, readFileSync as readFileSync24, unlinkSync as unlinkSync3, writeFileSync as writeFileSync15 } from "node:fs";
-import { basename as basename12, dirname as dirname23, isAbsolute as isAbsolute11, join as join42, resolve as resolve15 } from "node:path";
+import { basename as basename11, dirname as dirname23, isAbsolute as isAbsolute11, join as join42, resolve as resolve15 } from "node:path";
 import { findEnvKeys, getEnvApiKey } from "@earendil-works/pi-ai/compat";
 import { getAgentDir as getAgentDir2, ModelRuntime, SettingsManager } from "@earendil-works/pi-coding-agent";
 function resolveConfigPath(input) {
@@ -25324,7 +24957,7 @@ function acquireConfigWriteLock(configPath) {
   });
 }
 function replaceFileAtomically(path, contents, assertCommitAllowed) {
-  const temporaryPath = join42(dirname23(path), `.${basename12(path)}.${process.pid}.${randomUUID6()}.tmp`);
+  const temporaryPath = join42(dirname23(path), `.${basename11(path)}.${process.pid}.${randomUUID6()}.tmp`);
   try {
     writeFileSync15(temporaryPath, contents, { encoding: "utf8", flag: "wx", flush: true, mode: 384 });
     assertCommitAllowed?.();
@@ -26835,6 +26468,42 @@ function loopGuardPrompt(rule) {
 `);
 }
 
+// src/agent/token-count.ts
+import { createHash as createHash19 } from "node:crypto";
+import { countTokens } from "gpt-tokenizer/encoding/o200k_base";
+function estimateTextTokens(text) {
+  return Math.max(Math.ceil(text.length / 4), countText(text));
+}
+function tokensBeyondCharGuess(parts) {
+  let excess = 0;
+  for (const part of parts)
+    excess += Math.max(0, countText(part) - Math.ceil(part.length / 4));
+  return excess;
+}
+function countText(text) {
+  if (text.length === 0)
+    return 0;
+  if (text.length < CACHE_MIN_CHARS)
+    return countTokens(text, PLAIN_TEXT);
+  const key = `${text.length}:${createHash19("sha1").update(text).digest("base64")}`;
+  const hit = cache.get(key);
+  if (hit !== undefined) {
+    cache.delete(key);
+    cache.set(key, hit);
+    return hit;
+  }
+  const count = countTokens(text, PLAIN_TEXT);
+  cache.set(key, count);
+  if (cache.size > CACHE_ENTRIES)
+    cache.delete(cache.keys().next().value);
+  return count;
+}
+var PLAIN_TEXT, CACHE_MIN_CHARS = 2000, CACHE_ENTRIES = 64, cache;
+var init_token_count = __esm(() => {
+  PLAIN_TEXT = { disallowedSpecial: new Set };
+  cache = new Map;
+});
+
 // src/agent/context-token-guard.ts
 import { calculateContextTokens, estimateTokens } from "@earendil-works/pi-coding-agent";
 function createContextTokenGuardedTransform(model, base, fixedOverheadTokens = () => 0) {
@@ -26850,12 +26519,20 @@ function withBoundedOutput(model) {
   return { ...model, maxTokens: MAX_OUTPUT_TOKENS };
 }
 function guardAgentContextMessages(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
+  const guarded = guardWithinBudget(model, messages, useReportedUsage, fixedOverheadTokens);
+  if (model !== undefined && guarded.length > 0) {
+    recordRequestEstimate(model, estimateContextMessageTokens(guarded, false) + fixedOverheadTokens);
+  }
+  return guarded;
+}
+function guardWithinBudget(model, messages, useReportedUsage = true, fixedOverheadTokens = 0) {
   if (model === undefined || !Number.isFinite(model.contextWindow) || model.contextWindow <= 0 || messages.length === 0) {
     return messages;
   }
   const usableContext = Math.floor(model.contextWindow / CONTEXT_HEADROOM);
   const outputReserve = Math.min(Math.floor(usableContext / 4), MAX_OUTPUT_TOKENS, Number.isFinite(model.maxTokens) ? Math.max(0, model.maxTokens) : 0);
-  const inputBudget = Math.max(0, usableContext - outputReserve);
+  const overflowCeiling = contextOverflowCeiling(model);
+  const inputBudget = Math.min(Math.max(0, usableContext - outputReserve), overflowCeiling ?? Number.POSITIVE_INFINITY);
   const budget = Math.max(0, inputBudget - fixedOverheadTokens);
   const reported = useReportedUsage && findLastAssistantUsage(messages) !== undefined;
   if (estimateContextMessageTokens(messages, useReportedUsage) <= (reported ? inputBudget : budget)) {
@@ -26877,50 +26554,40 @@ function guardAgentContextMessages(model, messages, useReportedUsage = true, fix
   }
   let guarded = messages;
   if (candidates.length > 0) {
-    const fixedTokens = estimateContextMessageTokens(base, false);
-    const maximumOmitted = candidates.reduce((total, section) => total + section.blocks.length, 0);
+    const entries = candidates.flatMap((section, sectionIndex) => section.blocks.map((block, blockIndex) => ({ block, sectionIndex, blockIndex })));
     const omissionMarker = (count) => `
 
 ${GUARD_MARKER} Retrieval candidates omitted: ${count}.
 `;
-    let remaining = Math.max(0, budget - fixedTokens - Math.ceil(omissionMarker(maximumOmitted).length / 4));
-    const kept = candidates.map(() => []);
-    let omitted = 0;
-    let firstOmitted = -1;
-    for (const [sectionIndex, section] of candidates.entries()) {
-      for (const block of section.blocks) {
-        const tokens = Math.max(1, Math.ceil(block.length / 4));
-        if (tokens <= remaining) {
-          kept[sectionIndex]?.push(block);
-          remaining -= tokens;
-          continue;
-        }
-        omitted += section.blocks.length - (kept[sectionIndex]?.length ?? 0);
-        for (const later of candidates.slice(sectionIndex + 1))
-          omitted += later.blocks.length;
-        firstOmitted = sectionIndex;
-        break;
-      }
-      if (firstOmitted !== -1)
+    const markerTokens = candidates.length * Math.ceil(omissionMarker(entries.length).length / 4);
+    let remaining = Math.max(0, budget - estimateContextMessageTokens(base, false) - markerTokens);
+    entries.sort((a, b) => a.block.rank - b.block.rank || a.sectionIndex - b.sectionIndex || a.blockIndex - b.blockIndex);
+    const kept = new Set;
+    for (const { block } of entries) {
+      const tokens = Math.max(1, estimateTextTokens(block.text));
+      if (tokens > remaining)
         break;
+      remaining -= tokens;
+      kept.add(block);
     }
-    if (omitted > 0) {
-      const candidateByMessage = new Map;
-      for (const [sectionIndex, messageIndex] of candidateIndexes.entries())
-        candidateByMessage.set(messageIndex, sectionIndex);
+    if (kept.size < entries.length) {
+      const sectionByMessage = new Map(candidateIndexes.map((messageIndex, i) => [messageIndex, candidates[i]]));
       guarded = base.map((message, index) => {
-        const sectionIndex = candidateByMessage.get(index);
-        if (sectionIndex === undefined)
+        const section = sectionByMessage.get(index);
+        if (section === undefined)
           return message;
-        const section = candidates[sectionIndex];
-        const marker = sectionIndex === firstOmitted ? omissionMarker(omitted) : "";
-        return replaceTextContent(message, `${section.prefix}${kept[sectionIndex]?.join("") ?? ""}${marker}${section.suffix}`);
+        const keptBlocks = section.blocks.filter((block) => kept.has(block));
+        const omitted = section.blocks.length - keptBlocks.length;
+        if (omitted === 0)
+          return messages[index] ?? message;
+        const marker = omissionMarker(omitted);
+        return replaceTextContent(message, `${section.prefix}${keptBlocks.map((block) => block.text).join("")}${marker}${section.suffix}`);
       });
     }
   }
-  if (estimateContextMessageTokens(guarded, false) <= budget && guarded !== messages)
+  if (guarded !== messages && estimateContextMessageTokens(guarded, false) <= budget)
     return guarded;
-  return trimFixedContext(guarded, budget);
+  return fitNewestToolResult(guarded, budget);
 }
 function estimateContextMessageTokens(messages, useReportedUsage = false) {
   if (useReportedUsage) {
@@ -26931,82 +26598,37 @@ function estimateContextMessageTokens(messages, useReportedUsage = false) {
   }
   return messages.reduce((total, message) => total + estimateMessageTokens(message), 0);
 }
-function trimFixedContext(messages, inputBudget) {
-  const trimmed = messages.slice();
-  const lastIndex = trimmed.length - 1;
-  const queryIndex = trimmed.reduce((found, message, index) => message.role === "user" ? index : found, -1);
-  const fits = () => estimateContextMessageTokens(trimmed, false) <= inputBudget;
-  const fitInto = (index, floorTokens) => {
-    const message = trimmed[index];
-    if (message === undefined)
-      return;
-    const others = trimmed.reduce((total, other, i) => i === index ? total : total + estimateMessageTokens(other), 0);
-    trimmed[index] = fitMessageToTokens(message, Math.max(floorTokens, inputBudget - others));
-  };
-  for (const [index, message] of trimmed.entries()) {
-    if (index === lastIndex || index === queryIndex || message.role === "system")
-      continue;
-    const replacement = replaceMessageWithMarker(message);
-    if (replacement === message || estimateMessageTokens(replacement) >= estimateMessageTokens(message))
-      continue;
-    trimmed[index] = replacement;
-    if (fits())
-      return trimmed;
-  }
-  if (lastIndex !== queryIndex && trimmed[lastIndex]?.role !== "system") {
-    fitInto(lastIndex, 0);
-    if (fits())
-      return trimmed;
-  }
-  for (const [index, message] of trimmed.entries()) {
-    if (message.role !== "system")
-      continue;
-    fitInto(index, 0);
-    if (fits())
-      return trimmed;
-  }
-  if (queryIndex >= 0)
-    fitInto(queryIndex, MIN_QUERY_TOKENS);
-  return trimmed;
+function fitNewestToolResult(messages, inputBudget) {
+  const lastIndex = messages.length - 1;
+  const last = messages[lastIndex];
+  if (last?.role !== "toolResult")
+    return messages;
+  const others = messages.reduce((total, message, index) => index === lastIndex ? total : total + estimateMessageTokens(message), 0);
+  if (others >= inputBudget)
+    return messages;
+  const fitted = fitMessageToTokens(last, inputBudget - others);
+  return fitted === last ? messages : [...messages.slice(0, lastIndex), fitted];
 }
 function fitMessageToTokens(message, maxTokens) {
   const original = textContent(message);
   if (original === undefined)
-    return stripNonTextContent(message);
+    return message;
   const marker = `
 
 ${GUARD_MARKER}
 `;
-  let low = 0;
-  let high = original.length;
-  let best = replaceTextContent(message, marker);
-  while (low <= high) {
-    const middle = Math.floor((low + high) / 2);
-    const candidate = replaceTextContent(message, excerpt(original, middle, marker));
-    if (estimateMessageTokens(candidate) <= maxTokens) {
-      best = candidate;
-      low = middle + 1;
-    } else {
-      high = middle - 1;
-    }
+  const overhead = estimateMessageTokens(replaceTextContent(message, ""));
+  const density = Math.max(0, estimateMessageTokens(message) - overhead) / Math.max(1, original.length);
+  let chars = density > 0 ? Math.floor((maxTokens - overhead) / density) : original.length;
+  for (let attempt = 0;attempt < MAX_FIT_ATTEMPTS && chars > 0; attempt++) {
+    const candidate = replaceTextContent(message, excerpt(original, chars, marker));
+    const tokens = estimateMessageTokens(candidate);
+    if (tokens <= maxTokens)
+      return candidate;
+    const scaled = Math.floor(chars * Math.max(0, maxTokens - overhead) / Math.max(1, tokens - overhead) * 0.98);
+    chars = Math.min(chars - 1, scaled);
   }
-  return best;
-}
-function stripNonTextContent(message) {
-  if (message.role !== "user" || typeof message.content === "string")
-    return replaceMessageWithMarker(message);
-  const marker = `
-
-${GUARD_MARKER}
-`;
-  let replacedImage = false;
-  const content = message.content.map((block) => {
-    if (block.type === "text")
-      return block;
-    replacedImage = true;
-    return { type: "text", text: marker };
-  });
-  return replacedImage ? { ...message, content } : message;
+  return replaceTextContent(message, marker);
 }
 function excerpt(text, maxChars, marker) {
   if (text.length <= maxChars)
@@ -27016,32 +26638,54 @@ function excerpt(text, maxChars, marker) {
   const tail = Math.max(0, available - head);
   return `${text.slice(0, head)}${marker}${tail > 0 ? text.slice(-tail) : ""}`;
 }
-function replaceMessageWithMarker(message) {
-  return replaceTextContent(message, `
-
-${GUARD_MARKER}
-`);
-}
 function estimateMessageTokens(message) {
   if (message.role === "system")
     return estimateSystemMessageTokens(message);
-  const estimate = estimateTokens(message);
+  const estimate = estimateTokens(message) + tokensBeyondCharGuess(messageTextParts(message));
   return message.role === "toolResult" ? estimate + Math.ceil(message.toolName.length / 4) : estimate;
 }
 function estimateSystemMessageTokens(message) {
-  let chars = typeof message.content === "string" ? message.content.length : 0;
-  if (Array.isArray(message.content)) {
+  const parts = [];
+  let chars = 0;
+  if (typeof message.content === "string")
+    parts.push(message.content);
+  else {
     for (const block of message.content) {
       if (block.type === "text")
-        chars += block.text.length;
+        parts.push(block.text);
       else if (block.type === "image")
         chars += 4800;
     }
   }
-  chars += JSON.stringify(message.sections ?? {}).length;
-  chars += JSON.stringify(message.toolsAdded ?? []).length;
-  chars += JSON.stringify(message.toolsRemoved ?? []).length;
-  return Math.ceil(chars / 4);
+  parts.push(JSON.stringify(message.sections ?? {}), JSON.stringify(message.toolsAdded ?? []), JSON.stringify(message.toolsRemoved ?? []));
+  for (const part of parts)
+    chars += part.length;
+  return Math.ceil(chars / 4) + tokensBeyondCharGuess(parts);
+}
+function messageTextParts(message) {
+  switch (message.role) {
+    case "user":
+    case "custom":
+    case "toolResult":
+      return typeof message.content === "string" ? [message.content] : message.content.flatMap((block) => block.type === "text" ? [block.text] : []);
+    case "assistant":
+      return message.content.flatMap((block) => {
+        if (block.type === "text")
+          return [block.text];
+        if (block.type === "thinking")
+          return [block.thinking];
+        if (block.type === "toolCall")
+          return [`${block.name}${JSON.stringify(block.arguments)}`];
+        return [];
+      });
+    case "bashExecution":
+      return [message.command, message.output];
+    case "branchSummary":
+    case "compactionSummary":
+      return [message.summary];
+    default:
+      return [];
+  }
 }
 function findLastAssistantUsage(messages) {
   for (let index = messages.length - 1;index >= 0; index--) {
@@ -27115,7 +26759,7 @@ function parseCandidateSection(message, text) {
     const sectionEnd = findBaselineSectionEnd(text, sectionStart);
     if (!/^\[e?\d+\] /mu.test(text.slice(sectionStart, sectionEnd)))
       return;
-    return splitCandidates(text, sectionStart, sectionEnd);
+    return splitCandidates(text, sectionStart, sectionEnd, { lists: true });
   }
   if (message.role !== "toolResult")
     return;
@@ -27126,7 +26770,9 @@ function parseCandidateSection(message, text) {
     const directiveStart = text.indexOf(`
 
 directive:`);
-    return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, /^- [^\n]*(?:\n|$)/gmu);
+    return splitCandidates(text, 0, directiveStart >= 0 ? directiveStart : text.length, {
+      pattern: /^- [^\n]*(?:\n|$)/gmu
+    });
   }
   const diagnosticsStart = text.indexOf(`
 
@@ -27148,17 +26794,23 @@ Formatting and content rules`
   const positions = endMarkers.map((marker) => text.indexOf(marker, startIndex)).filter((index) => index >= 0);
   return positions.length > 0 ? Math.min(...positions) : text.length;
 }
-function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [^\n]*(?:\n|$)/gmu) {
+function splitCandidates(text, sectionStart, sectionEnd, options = {}) {
   const region = text.slice(sectionStart, sectionEnd);
-  const matches = [...region.matchAll(pattern)];
+  const matches = [...region.matchAll(options.pattern ?? /^\[e?\d+\] [^\n]*(?:\n|$)/gmu)];
   const firstStart = matches[0]?.index;
   if (firstStart === undefined)
     return;
   const blocks = [];
+  let rank = 0;
+  let heading = "";
   for (const [index, match] of matches.entries()) {
     const start = match.index ?? 0;
     const end = index + 1 < matches.length ? matches[index + 1]?.index ?? region.length : region.length;
-    blocks.push(region.slice(start, end));
+    const raw = region.slice(start, end);
+    const tail = options.lists === true ? LIST_HEADING_TAIL.exec(raw) : null;
+    blocks.push({ text: heading + (tail === null ? raw : raw.slice(0, tail.index)), rank });
+    rank = tail === null ? rank + 1 : 0;
+    heading = tail === null ? "" : tail[0];
   }
   return {
     prefix: text.slice(0, sectionStart) + region.slice(0, firstStart),
@@ -27166,8 +26818,10 @@ function splitCandidates(text, sectionStart, sectionEnd, pattern = /^\[e?\d+\] [
     blocks
   };
 }
-var CONTEXT_HEADROOM = 1.2, MAX_OUTPUT_TOKENS = 32000, MIN_QUERY_TOKENS = 2000, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES;
+var CONTEXT_HEADROOM = 1.2, MAX_OUTPUT_TOKENS = 32000, GUARD_MARKER = "[AutoRAG context guard: omitted context to stay within the model context budget.]", RETRIEVAL_TOOL_PREFIX = "search_datasource_", RETRIEVAL_TOOL_NAMES, MAX_FIT_ATTEMPTS = 6, LIST_HEADING_TAIL;
 var init_context_token_guard = __esm(() => {
+  init_context_overflow();
+  init_token_count();
   RETRIEVAL_TOOL_NAMES = {
     jikji_find: true,
     search_all_documents: true,
@@ -27176,6 +26830,7 @@ var init_context_token_guard = __esm(() => {
     everything_search: true,
     fsearch_search: true
   };
+  LIST_HEADING_TAIL = /\n[^\n]*initial candidates[^\n]*:\n$/u;
 });
 
 // src/agent/current-date.ts
@@ -27621,7 +27276,7 @@ function guardContext(pi, contextTransform) {
   pi.on("context", async (event, ctx) => {
     const activeTools = new Set(pi.getActiveTools());
     const toolDeclarations = pi.getAllTools().filter((tool) => activeTools.has(tool.name)).map(({ name, description, parameters }) => ({ name, description, parameters }));
-    const fixedOverheadTokens = Math.ceil((ctx.getSystemPrompt().length + JSON.stringify(toolDeclarations).length) / 4);
+    const fixedOverheadTokens = estimateTextTokens(ctx.getSystemPrompt()) + estimateTextTokens(JSON.stringify(toolDeclarations));
     return {
       messages: await createContextTokenGuardedTransform(ctx.model, contextTransform, () => fixedOverheadTokens)(event.messages)
     };
@@ -27855,6 +27510,7 @@ var init_pi_session = __esm(() => {
   init_intent_tracing();
   init_mcp_client();
   init_request_limits();
+  init_token_count();
   init_welcome_header();
   PI_BUILTIN_TOOL_NAMES = ["read", "bash", "edit", "write", "grep", "find", "ls"];
 });
@@ -29043,7 +28699,7 @@ var init_automation_tool = __esm(() => {
 });
 
 // src/agent/citation-links.ts
-import { basename as basename13, isAbsolute as isAbsolute12 } from "node:path";
+import { basename as basename12, isAbsolute as isAbsolute12 } from "node:path";
 import { pathToFileURL as pathToFileURL3 } from "node:url";
 function openableTarget(source) {
   if (/^https?:\/\//iu.test(source)) {
@@ -29061,7 +28717,7 @@ function linkCitations(markdown, sourceOf, isFile) {
   for (const marker of answerMarkers(markdown, isFile)) {
     if (marker.kind === "number")
       continue;
-    const labels = marker.kind === "file" ? [{ label: basename13(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
+    const labels = marker.kind === "file" ? [{ label: basename12(marker.path), source: marker.path }] : marker.ids.map((id) => ({ label: id, source: sourceOf(id) }));
     let linked = false;
     const parts = labels.map(({ label, source }) => {
       const target = source === undefined ? undefined : openableTarget(source);
@@ -30300,7 +29956,7 @@ class DazziJevBackend {
     this.apiKey = apiKey;
     this.baseUrl = options.baseUrl ?? resolveBaseUrl(env);
     this.defaultModel = options.defaultModel ?? DAZZI_DEFAULT_JEV_MODEL;
-    this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS12;
+    this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS11;
     this.maxRetries = options.maxRetries ?? DEFAULT_MAX_RETRIES;
     this.fetchImpl = options.fetchImpl;
   }
@@ -30487,7 +30143,7 @@ async function extractErrorMessage(response) {
 function sleep6(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
-var JEV_PATH, DAZZI_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS12 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
+var JEV_PATH, DAZZI_DEFAULT_JEV_MODEL = "jev-1.13", DEFAULT_TIMEOUT_MS11 = 1e4, DEFAULT_MAX_RETRIES = 2, RETRY_BASE_MS = 500, RETRY_MAX_MS = 5000;
 var init_jev_backend = __esm(() => {
   init_config();
   init_credential();
@@ -31971,7 +31627,7 @@ var init_page_loader = __esm(() => {
 });
 
 // src/web/fetch/url-target.ts
-import { extname as extname8 } from "node:path";
+import { extname as extname7 } from "node:path";
 function repairCollapsedScheme(value) {
   const m = value.match(/^(https?):\/(?!\/)/i);
   return m ? `${m[1]}://${value.slice(m[0].length)}` : value;
@@ -31990,7 +31646,7 @@ function getFilenameExtensionHint(filename) {
   const lower = filename.toLowerCase();
   if (lower.endsWith(".tar.gz"))
     return ".tar.gz";
-  return extname8(filename).toLowerCase();
+  return extname7(filename).toLowerCase();
 }
 function getExtensionHint(url, contentDisposition) {
   if (contentDisposition) {
@@ -32167,7 +31823,7 @@ var init_convert2 = __esm(() => {
 });
 
 // src/web/fetch/feeds.ts
-import { extname as extname9 } from "node:path";
+import { extname as extname8 } from "node:path";
 import { XMLParser as XMLParser3 } from "fast-xml-parser";
 function getHtmlAttribute(tag, attribute) {
   const pattern = new RegExp(`\\b${attribute}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'=<>]+))`, "i");
@@ -32230,7 +31886,7 @@ function extractDocumentLinks(html, baseUrl) {
       const href = getHtmlAttribute(tag, "href");
       if (!href)
         continue;
-      const ext = extname9(href).toLowerCase();
+      const ext = extname8(href).toLowerCase();
       if (!CONVERTIBLE_EXTENSIONS2.has(ext))
         continue;
       const resolved = href.startsWith("http") ? href : new URL(href, baseUrl).href;
@@ -33898,7 +33554,7 @@ var init_web_search_tool = __esm(() => {
 // src/agent/agent.ts
 import { randomUUID as randomUUID7 } from "node:crypto";
 import { existsSync as existsSync34, watch as fsWatch, mkdirSync as mkdirSync25, realpathSync as realpathSync7, statSync as statSync9, writeFileSync as writeFileSync18 } from "node:fs";
-import { basename as basename14, dirname as dirname26, join as join52, resolve as resolve18 } from "node:path";
+import { basename as basename13, dirname as dirname26, join as join52, resolve as resolve18 } from "node:path";
 import { clampThinkingLevel } from "@earendil-works/pi-ai/compat";
 import { getAgentDir as getAgentDir5, keyText } from "@earendil-works/pi-coding-agent";
 function positiveLimit(value, fallback, path) {
@@ -36821,7 +36477,7 @@ function toMinSyncReasonDiagnostic(reason) {
 function pinExcludedPath(path) {
   const resolvedPath = resolve18(path);
   try {
-    return join52(realpathSync7(dirname26(resolvedPath)), basename14(resolvedPath));
+    return join52(realpathSync7(dirname26(resolvedPath)), basename13(resolvedPath));
   } catch {
     return resolvedPath;
   }