@firstpick/pi-utils 0.1.5 → 0.1.7

package/README.md

@@ -19,7 +19,12 @@ Shared helper utilities used by `@firstpick/pi-extension-*` packages.
 - `slugify(input, options?)`
 - `formatTokens(count)`
 - `estimateTokensFromCharCount(charCount)`
+- `estimateTokensFromText(text)`
 - `estimatePromptInjectionTokens(systemPrompt)`
+- `estimateInitialPromptInput(options)`
+- `collectInitialPromptCalibration(sessionDir, maxSamples?)`
+- `buildInitialPromptCalibrationRecord(args)`
+- `appendInitialPromptCalibrationRecord(appendEntry, record)`
 - `delay(ms)`
 - `createExtensionWorkingIndicator(ctx, initialMessage, options?)`
 - `withExtensionWorkingIndicator(ctx, initialMessage, run, options?)`

package/index.ts

@@ -2,6 +2,7 @@ export * from "./src/paths";
 export * from "./src/env";
 export * from "./src/text";
 export * from "./src/tokens";
+export * from "./src/prompt-calibration";
 export * from "./src/async";
 export * from "./src/ui/working-indicator";
 export * from "./src/local-wiki";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firstpick/pi-utils",
-  "version": "0.1.5",
+  "version": "0.1.7",
   "description": "Shared utilities for Firstpick Pi extension packages.",
   "main": "index.ts",
   "exports": {
@@ -9,6 +9,7 @@
     "./env": "./src/env.ts",
     "./text": "./src/text.ts",
     "./tokens": "./src/tokens.ts",
+    "./prompt-calibration": "./src/prompt-calibration.ts",
     "./async": "./src/async.ts",
     "./ui": "./src/ui/working-indicator.ts",
     "./local-wiki": "./src/local-wiki.ts"

package/src/local-wiki.ts

@@ -59,6 +59,8 @@ export interface LocalWikiEngineConfig {
   fileExtensions: RegExp;
   format: LocalWikiFormat;
   queryExpansions?: Record<string, string[]>;
+  searchStopwords?: Iterable<string>;
+  termWeights?: Record<string, number>;
   missingDocsMessage?: string;
   ignoredDirs?: string[];
   sourceName?: (filePath: string, docsPath: string) => string | undefined;
@@ -74,6 +76,7 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
   const metadataCache = path.join(config.cacheDir, "metadata.json");
   const ignoredDirs = new Set([".git", "node_modules", "result", ...(config.ignoredDirs ?? [])]);
   const missingDocsMessage = config.missingDocsMessage ?? `Local ${config.displayName} docs are not available at ${config.docsPath}.`;
+  const searchStopwords = new Set([...(config.searchStopwords ?? [])].map((word) => normalizeQuery(word)).filter(Boolean));
 
   async function localExists(filePath: string): Promise<boolean> {
     try { await fsp.access(filePath); return true; } catch { return false; }
@@ -119,7 +122,36 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
   }
 
   function stripMarkdownDecorators(input: string): string {
-    return input.replace(/^#+\s*/, "").replace(/[*_`~]/g, "").replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1").trim();
+    return input
+      .replace(/^#+\s*/, "")
+      .replace(/\s*\{#[^}]+\}\s*$/g, "")
+      .replace(/[*_`~]/g, "")
+      .replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1")
+      .trim();
+  }
+
+  function stripYamlFrontmatter(markdown: string): string {
+    return markdown.replace(/^---\s*\n[\s\S]*?\n---\s*\n?/, "");
+  }
+
+  function yamlFrontmatterTitle(markdown: string): string | undefined {
+    const frontmatter = markdown.match(/^---\s*\n([\s\S]*?)\n---\s*\n?/);
+    const raw = frontmatter?.[1]?.match(/^title:\s*["']?(.+?)["']?\s*$/m)?.[1];
+    return raw ? stripMarkdownDecorators(raw) : undefined;
+  }
+
+  function firstMarkdownHeading(markdown: string): string | undefined {
+    let inFence = false;
+    for (const line of stripYamlFrontmatter(markdown).split(/\n/)) {
+      if (/^\s*(```|~~~)/.test(line)) {
+        inFence = !inFence;
+        continue;
+      }
+      if (inFence) continue;
+      const match = line.match(/^#\s+(.+)$/);
+      if (match) return match[1].trim();
+    }
+    return undefined;
   }
 
   function decodeEntities(input: string): string {
@@ -137,10 +169,17 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
   }
 
   function markdownSections(markdown: string, fallbackTitle: string): LocalWikiSection[] {
+    const body = stripYamlFrontmatter(markdown);
     const sections: LocalWikiSection[] = [];
     let current: LocalWikiSection | undefined;
-    for (const line of markdown.split(/\n/)) {
-      const match = line.match(/^(#{1,6})\s+(.+)$/);
+    let inFence = false;
+    for (const line of body.split(/\n/)) {
+      if (/^\s*(```|~~~)/.test(line)) {
+        inFence = !inFence;
+        if (current) current.text += `${line}\n`;
+        continue;
+      }
+      const match = !inFence ? line.match(/^(#{1,6})\s+(.+)$/) : undefined;
       if (match) {
         const title = stripMarkdownDecorators(match[2]);
         if (title.toLowerCase() === "contents") continue;
@@ -151,7 +190,7 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
       }
       if (current) current.text += `${line}\n`;
     }
-    if (!current) sections.push({ title: fallbackTitle, level: 1, anchor: anchorFromHeading(fallbackTitle), text: markdown.trim() });
+    if (!current) sections.push({ title: fallbackTitle, level: 1, anchor: anchorFromHeading(fallbackTitle), text: body.trim() });
     else current.text = current.text.trim();
     return sections;
   }
@@ -168,7 +207,7 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
   }
 
   function markdownTitle(markdown: string, filePath: string): string {
-    return stripMarkdownDecorators(markdown.match(/^#\s+(.+)$/m)?.[1]?.trim() || titleFromPath(filePath));
+    return stripMarkdownDecorators(yamlFrontmatterTitle(markdown) || firstMarkdownHeading(markdown) || titleFromPath(filePath));
   }
 
   function htmlTitle(html: string, filePath: string): string {
@@ -211,10 +250,11 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
 
   function parsePage(raw: string, filePath: string, mtimeMs: number): LocalWikiPage {
     const title = config.format === "html" ? htmlTitle(raw, filePath) : markdownTitle(raw, filePath);
-    const baseText = config.format === "html" ? htmlToText(raw) : normalizeWhitespace(raw);
+    const markdownBody = config.format === "html" ? raw : stripYamlFrontmatter(raw);
+    const baseText = config.format === "html" ? htmlToText(raw) : normalizeWhitespace(markdownBody);
     const text = config.transformText?.(baseText, title, filePath) ?? baseText;
     const sections = markdownSections(text, title);
-    return { title, slug: path.relative(config.docsPath, filePath).replace(config.fileExtensions, ""), path: filePath, source: config.sourceName?.(filePath, config.docsPath), headings: sections.map((s) => s.title), sections, links: config.format === "html" ? htmlLinks(raw, filePath) : markdownLinks(text, filePath), text, mtimeMs };
+    return { title, slug: path.relative(config.docsPath, filePath).replace(config.fileExtensions, ""), path: filePath, source: config.sourceName?.(filePath, config.docsPath), headings: sections.map((s) => s.title), sections, links: config.format === "html" ? htmlLinks(raw, filePath) : markdownLinks(markdownBody, filePath), text, mtimeMs };
   }
 
   function limitText(text: string, maxChars = 12000): { text: string; truncated: boolean } {
@@ -257,12 +297,21 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
   }
 
   function expandQuery(query: string): string[] {
-    const tokens = normalizeQuery(query).split(/\s+/).filter(Boolean);
+    const tokens = normalizeQuery(query).split(/\s+/).filter((token) => token && !searchStopwords.has(token));
     const expanded = new Set(tokens);
-    for (const token of tokens) for (const extra of config.queryExpansions?.[token] ?? []) expanded.add(normalizeQuery(extra));
+    for (const token of tokens) {
+      for (const extra of config.queryExpansions?.[token] ?? []) {
+        const normalized = normalizeQuery(extra);
+        if (normalized && !searchStopwords.has(normalized)) expanded.add(normalized);
+      }
+    }
     return [...expanded].filter(Boolean);
   }
 
+  function tokenWeight(token: string): number {
+    return config.termWeights?.[token] ?? 1;
+  }
+
   function makeSnippet(text: string, tokens: string[], max = 280): string | undefined {
     const lower = text.toLowerCase();
     const index = tokens.map((t) => lower.indexOf(t.toLowerCase())).filter((i) => i >= 0).sort((a, b) => a - b)[0];
@@ -282,14 +331,15 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
     const matchedFields = new Set<string>();
     const scoreExplanation: string[] = [];
     for (const token of tokens) {
-      if (title.includes(token)) { score += 25; matchedFields.add("title"); scoreExplanation.push(`title matched '${token}'`); }
-      if (slug.includes(token)) { score += 12; matchedFields.add("slug"); }
-      if (source.includes(token)) { score += 8; matchedFields.add("source"); }
-      if (headings.includes(token)) { score += 10; matchedFields.add("headings"); }
+      const weight = tokenWeight(token);
+      if (title.includes(token)) { score += 25 * weight; matchedFields.add("title"); scoreExplanation.push(`title matched '${token}'`); }
+      if (slug.includes(token)) { score += 12 * weight; matchedFields.add("slug"); }
+      if (source.includes(token)) { score += 8 * weight; matchedFields.add("source"); }
+      if (headings.includes(token)) { score += 10 * weight; matchedFields.add("headings"); }
       const textMatches = text.split(token).length - 1;
-      if (textMatches > 0) { score += Math.min(15, textMatches); matchedFields.add("text"); }
+      if (textMatches > 0) { score += Math.min(15, textMatches) * weight; matchedFields.add("text"); }
     }
-    return score > 0 ? { title: page.title, path: page.path, source: page.source, score, matchedFields: [...matchedFields], scoreExplanation, snippet: makeSnippet(page.text, tokens) } : undefined;
+    return score > 0 ? { title: page.title, path: page.path, source: page.source, score: Number(score.toFixed(2)), matchedFields: [...matchedFields], scoreExplanation, snippet: makeSnippet(page.text, tokens) } : undefined;
   }
 
   function findPage(pages: LocalWikiPage[], pageRef: string): LocalWikiPage | undefined {
@@ -308,9 +358,9 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
   async function search(params: { query: string; limit?: number; includeSnippets?: boolean }) {
     const { pages } = await loadCache();
     const tokens = expandQuery(params.query);
-    const limit = Math.max(1, Math.min(params.limit ?? 10, 50));
+    const limit = Math.max(1, Math.min(params.limit ?? 8, 50));
     const results = pages.map((p) => scorePage(p, tokens)).filter((x): x is LocalWikiSearchResult => Boolean(x)).sort((a, b) => b.score - a.score).slice(0, limit);
-    return { query: params.query, expandedTokens: tokens, results: params.includeSnippets === false ? results.map(({ snippet, ...rest }) => rest) : results };
+    return { query: params.query, expandedTokens: tokens, results: params.includeSnippets === true ? results : results.map(({ snippet, ...rest }) => rest) };
   }
 
   async function loadPage(pageRef: string): Promise<LocalWikiPage> {
@@ -326,23 +376,27 @@ export function createLocalWikiEngine(config: LocalWikiEngineConfig) {
     return { title: page.title, source: page.source, path: page.path, citation: `${page.path} — ${page.title}`, truncated: limited.truncated, text: limited.text };
   }
 
-  async function sections(params: { page: string }) {
+  async function sections(params: { page: string; maxSections?: number }) {
     const page = await loadPage(params.page);
-    return { title: page.title, source: page.source, path: page.path, sections: page.sections.map((s) => ({ title: s.title, level: s.level, anchor: s.anchor })) };
+    const maxSections = Math.max(1, Math.min(params.maxSections ?? 80, 300));
+    const selected = page.sections.slice(0, maxSections);
+    return { title: page.title, source: page.source, path: page.path, sectionCount: page.sections.length, omittedSectionCount: Math.max(0, page.sections.length - selected.length), sections: selected.map((s) => ({ title: s.title, level: s.level, anchor: s.anchor })) };
   }
 
-  async function extract(params: { page: string; section?: string; query?: string; maxChars?: number }) {
+  async function extract(params: { page: string; section?: string; query?: string; maxChars?: number; maxSections?: number }) {
     const page = await loadPage(params.page);
     let matchedSections = page.sections;
     if (params.section) { const needle = normalizeQuery(params.section); matchedSections = matchedSections.filter((s) => normalizeQuery(s.title).includes(needle)); }
     if (params.query) {
       const tokens = expandQuery(params.query);
-      matchedSections = matchedSections.map((section) => ({ section, score: tokens.reduce((sum, token) => sum + (normalizeQuery(`${section.title} ${section.text}`).includes(token) ? 1 : 0), 0) })).filter((i) => i.score > 0).sort((a, b) => b.score - a.score).map((i) => i.section);
+      matchedSections = matchedSections.map((section) => ({ section, score: tokens.reduce((sum, token) => sum + (normalizeQuery(`${section.title} ${section.text}`).includes(token) ? tokenWeight(token) : 0), 0) })).filter((i) => i.score > 0).sort((a, b) => b.score - a.score).map((i) => i.section);
     }
-    if (!params.section && !params.query) matchedSections = matchedSections.slice(0, 5);
+    const maxSections = Math.max(1, Math.min(params.maxSections ?? (params.section || params.query ? 6 : 5), 50));
+    const totalMatchedSections = matchedSections.length;
+    matchedSections = matchedSections.slice(0, maxSections);
     const joined = matchedSections.map((s) => `${"#".repeat(Math.min(s.level, 6))} ${s.title}\n\n${s.text}`).join("\n\n");
-    const limited = limitText(joined || page.text, params.maxChars ?? 12000);
-    return { title: page.title, source: page.source, path: page.path, citation: `${page.path} — ${matchedSections.map((s) => s.title).join(", ") || page.title}`, matchedSections: matchedSections.map((s) => ({ title: s.title, level: s.level, anchor: s.anchor })), truncated: limited.truncated, text: limited.text };
+    const limited = limitText(joined || page.text, params.maxChars ?? 10000);
+    return { title: page.title, source: page.source, path: page.path, citation: `${page.path} — ${matchedSections.map((s) => s.title).join(", ") || page.title}`, matchedSections: matchedSections.map((s) => ({ title: s.title, level: s.level, anchor: s.anchor })), totalMatchedSections, omittedSectionCount: Math.max(0, totalMatchedSections - matchedSections.length), truncated: limited.truncated, text: limited.text };
   }
 
   async function related(params: { page: string; limit?: number }) {

package/src/prompt-calibration.ts

@@ -0,0 +1,125 @@
+import * as fs from "node:fs";
+import { resolve } from "node:path";
+import type { InitialPromptCalibration, InitialPromptInputEstimate } from "./tokens";
+
+export const INITIAL_PROMPT_CALIBRATION_CUSTOM_TYPE = "stats_initial_prompt_estimate";
+
+export type InitialPromptCalibrationSample = {
+  ratio: number;
+  timestamp: string;
+};
+
+export type InitialPromptCalibrationRecord = {
+  version: 1;
+  ratio: number;
+  estimatedUncalibratedTokens: number;
+  estimatedFinalTokens: number;
+  actualInitialInputTokens: number;
+  actualInjectedTokens: number;
+  firstUserTokens: number;
+  provider: string;
+  model: string;
+  createdAt: string;
+};
+
+type AppendEntryLike = <T = unknown>(customType: string, data?: T) => void;
+
+function listSessionFiles(sessionDir: string): string[] {
+  try {
+    return fs
+      .readdirSync(sessionDir, { withFileTypes: true })
+      .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl"))
+      .map((entry) => resolve(sessionDir, entry.name));
+  } catch {
+    return [];
+  }
+}
+
+function quantile(values: number[], q: number): number {
+  if (values.length === 0) return 1;
+  const sorted = [...values].sort((a, b) => a - b);
+  const pos = (sorted.length - 1) * q;
+  const lower = Math.floor(pos);
+  const upper = Math.ceil(pos);
+  if (lower === upper) return sorted[lower] ?? 1;
+  const weight = pos - lower;
+  return (sorted[lower] ?? 1) * (1 - weight) + (sorted[upper] ?? 1) * weight;
+}
+
+export function collectInitialPromptCalibrationSamples(sessionDir: string, maxSamples = 100): InitialPromptCalibrationSample[] {
+  const samples: InitialPromptCalibrationSample[] = [];
+
+  for (const file of listSessionFiles(sessionDir)) {
+    let content: string;
+    try {
+      content = fs.readFileSync(file, "utf8");
+    } catch {
+      continue;
+    }
+
+    for (const line of content.split(/\r?\n/)) {
+      if (!line.trim()) continue;
+      try {
+        const entry = JSON.parse(line);
+        if (entry?.type !== "custom" || entry?.customType !== INITIAL_PROMPT_CALIBRATION_CUSTOM_TYPE) continue;
+        const ratio = Number(entry?.data?.ratio);
+        if (!Number.isFinite(ratio) || ratio <= 0.25 || ratio >= 4) continue;
+        samples.push({ ratio, timestamp: String(entry?.timestamp ?? entry?.data?.createdAt ?? "") });
+      } catch {
+        continue;
+      }
+    }
+  }
+
+  return samples.sort((a, b) => a.timestamp.localeCompare(b.timestamp)).slice(-maxSamples);
+}
+
+export function collectInitialPromptCalibration(sessionDir: string, maxSamples = 100): InitialPromptCalibration | null {
+  const ratios = collectInitialPromptCalibrationSamples(sessionDir, maxSamples).map((sample) => sample.ratio);
+  if (ratios.length === 0) return null;
+
+  const median = quantile(ratios, 0.5);
+  const q25 = quantile(ratios, 0.25);
+  const q75 = quantile(ratios, 0.75);
+  return {
+    multiplier: median,
+    lowMultiplier: Math.min(q25, median * 0.95),
+    highMultiplier: Math.max(q75, median * 1.05),
+    samples: ratios.length,
+  };
+}
+
+export function buildInitialPromptCalibrationRecord(args: {
+  estimate: InitialPromptInputEstimate;
+  actualInitialInputTokens: number;
+  firstUserTokens: number;
+  provider: string;
+  model: string;
+  createdAt?: string;
+}): InitialPromptCalibrationRecord | null {
+  const actualInjectedTokens = Math.max(0, args.actualInitialInputTokens - args.firstUserTokens);
+  const ratio = args.estimate.uncalibratedTotal > 0 ? actualInjectedTokens / args.estimate.uncalibratedTotal : 0;
+  if (!Number.isFinite(ratio) || ratio <= 0.25 || ratio >= 4) return null;
+
+  return {
+    version: 1,
+    ratio,
+    estimatedUncalibratedTokens: args.estimate.uncalibratedTotal,
+    estimatedFinalTokens: args.estimate.total,
+    actualInitialInputTokens: args.actualInitialInputTokens,
+    actualInjectedTokens,
+    firstUserTokens: args.firstUserTokens,
+    provider: args.provider,
+    model: args.model,
+    createdAt: args.createdAt ?? new Date().toISOString(),
+  };
+}
+
+export function appendInitialPromptCalibrationRecord(appendEntry: AppendEntryLike, record: InitialPromptCalibrationRecord): boolean {
+  try {
+    appendEntry(INITIAL_PROMPT_CALIBRATION_CUSTOM_TYPE, record);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/tokens.ts

@@ -1,3 +1,61 @@
+export type TokenEstimateConfidence = "estimated" | "calibrated" | "measured-after-call";
+
+export type InitialPromptToolInfo = {
+  name: string;
+  description?: string;
+  parameters?: unknown;
+};
+
+export type InitialPromptCalibration = {
+  multiplier?: number;
+  lowMultiplier?: number;
+  highMultiplier?: number;
+  samples?: number;
+};
+
+export type InitialPromptInputEstimate = {
+  /** Final best estimate after optional calibration. */
+  total: number;
+  /** Lower bound for dashboard/budget display. */
+  low: number;
+  /** Upper bound for dashboard/budget display. */
+  high: number;
+  /** Uncalibrated total: prompt text + tool schemas + framing. */
+  uncalibratedTotal: number;
+  /** Estimated tokens in Pi's assembled system prompt text. */
+  promptText: number;
+  /** Estimated tokens in provider-level active tool schemas. */
+  toolSchemas: number;
+  /** Provider/message/request framing allowance. */
+  framing: number;
+  /** Number of active tool schemas included in the estimate. */
+  toolCount: number;
+  /** Multiplier applied to uncalibratedTotal. */
+  calibrationMultiplier: number;
+  /** Calibration samples used for multiplier/range. */
+  calibrationSamples: number;
+  confidence: TokenEstimateConfidence;
+};
+
+export type EstimateInitialPromptInputOptions = {
+  systemPrompt: string;
+  activeTools?: string[];
+  allTools?: InitialPromptToolInfo[];
+  calibration?: InitialPromptCalibration | number | null;
+  /** Override request framing tokens. Defaults to a conservative provider-agnostic allowance. */
+  framingTokens?: number;
+};
+
+const ASCII_TOKENS_PER_CHAR = 0.25;
+const LATIN_EXTENDED_TOKENS_PER_CHAR = 0.5;
+const CJK_TOKENS_PER_CHAR = 1.2;
+const OTHER_UNICODE_TOKENS_PER_CHAR = 0.75;
+const EMOJI_TOKENS_PER_CODE_POINT = 2;
+
+const DEFAULT_REQUEST_FRAMING_TOKENS = 64;
+const SYSTEM_MESSAGE_FRAMING_TOKENS = 12;
+const TOOL_SCHEMA_FRAMING_TOKENS = 8;
+
 export function formatTokens(count: number): string {
   if (count < 1000) return count.toString();
   if (count < 10000) return `${(count / 1000).toFixed(1)}k`;
@@ -6,10 +64,173 @@ export function formatTokens(count: number): string {
   return `${Math.round(count / 1000000)}M`;
 }
 
+/**
+ * Legacy fast heuristic for callers that only have a character count.
+ */
 export function estimateTokensFromCharCount(charCount: number): number {
   return Math.max(0, Math.round(charCount / 4));
 }
 
+function isCjkLike(codePoint: number): boolean {
+  return (
+    (codePoint >= 0x3040 && codePoint <= 0x30ff) || // Hiragana/Katakana
+    (codePoint >= 0x3400 && codePoint <= 0x4dbf) || // CJK Extension A
+    (codePoint >= 0x4e00 && codePoint <= 0x9fff) || // CJK Unified Ideographs
+    (codePoint >= 0xac00 && codePoint <= 0xd7af) || // Hangul syllables
+    (codePoint >= 0xf900 && codePoint <= 0xfaff) || // CJK compatibility
+    (codePoint >= 0x20000 && codePoint <= 0x2fa1f)
+  );
+}
+
+function isEmojiLike(codePoint: number): boolean {
+  return (
+    (codePoint >= 0x1f000 && codePoint <= 0x1faff) ||
+    (codePoint >= 0x2600 && codePoint <= 0x27bf)
+  );
+}
+
+function isCombiningMark(codePoint: number): boolean {
+  return (
+    (codePoint >= 0x0300 && codePoint <= 0x036f) ||
+    (codePoint >= 0x1ab0 && codePoint <= 0x1aff) ||
+    (codePoint >= 0x1dc0 && codePoint <= 0x1dff) ||
+    (codePoint >= 0x20d0 && codePoint <= 0x20ff) ||
+    (codePoint >= 0xfe20 && codePoint <= 0xfe2f)
+  );
+}
+
+/**
+ * Provider-agnostic text token estimate.
+ *
+ * English/code remains close to the common chars/4 rule, while non-ASCII text is
+ * weighted higher to avoid underestimating CJK or emoji-heavy prompts.
+ */
+export function estimateTokensFromText(text: string): number {
+  if (!text) return 0;
+
+  let tokens = 0;
+  for (let i = 0; i < text.length; i++) {
+    const codePoint = text.codePointAt(i) ?? 0;
+    if (codePoint > 0xffff) i++;
+
+    if (codePoint <= 0x7f) {
+      tokens += ASCII_TOKENS_PER_CHAR;
+    } else if (isCombiningMark(codePoint)) {
+      // Combining marks usually merge into the previous token/grapheme.
+      continue;
+    } else if (isEmojiLike(codePoint)) {
+      tokens += EMOJI_TOKENS_PER_CODE_POINT;
+    } else if (isCjkLike(codePoint)) {
+      tokens += CJK_TOKENS_PER_CHAR;
+    } else if (codePoint <= 0x024f) {
+      tokens += LATIN_EXTENDED_TOKENS_PER_CHAR;
+    } else {
+      tokens += OTHER_UNICODE_TOKENS_PER_CHAR;
+    }
+  }
+
+  return Math.max(0, Math.ceil(tokens));
+}
+
 export function estimatePromptInjectionTokens(systemPrompt: string): number {
-  return estimateTokensFromCharCount(systemPrompt.length);
+  return estimateTokensFromText(systemPrompt);
+}
+
+function replacer(_key: string, value: unknown): unknown {
+  if (typeof value === "bigint") return value.toString();
+  if (typeof value === "function") return "[Function]";
+  if (typeof value === "symbol") return value.toString();
+  return value;
+}
+
+function stringifyForTokenEstimate(value: unknown): string {
+  try {
+    return JSON.stringify(value, replacer) ?? "";
+  } catch {
+    return String(value ?? "");
+  }
+}
+
+function buildActiveToolSchemaPayload(activeTools: string[] | undefined, allTools: InitialPromptToolInfo[] | undefined) {
+  if (!allTools || allTools.length === 0) return [];
+
+  const toolsByName = new Map<string, InitialPromptToolInfo>();
+  for (const tool of allTools) {
+    if (tool?.name && !toolsByName.has(tool.name)) {
+      toolsByName.set(tool.name, tool);
+    }
+  }
+
+  const orderedNames = activeTools && activeTools.length > 0 ? activeTools : Array.from(toolsByName.keys()).sort();
+  return orderedNames
+    .map((name) => toolsByName.get(name))
+    .filter((tool): tool is InitialPromptToolInfo => !!tool)
+    .map((tool) => ({
+      name: tool.name,
+      description: tool.description ?? "",
+      parameters: tool.parameters ?? {},
+    }));
+}
+
+function normalizeMultiplier(value: unknown, fallback: number): number {
+  const n = typeof value === "number" ? value : Number(value);
+  if (!Number.isFinite(n) || n <= 0) return fallback;
+  return Math.min(4, Math.max(0.25, n));
+}
+
+function resolveCalibration(calibration: InitialPromptCalibration | number | null | undefined): Required<InitialPromptCalibration> {
+  if (typeof calibration === "number") {
+    const multiplier = normalizeMultiplier(calibration, 1);
+    return {
+      multiplier,
+      lowMultiplier: multiplier * 0.95,
+      highMultiplier: multiplier * 1.05,
+      samples: 1,
+    };
+  }
+
+  const samples = Math.max(0, Math.floor(Number(calibration?.samples ?? 0) || 0));
+  const multiplier = normalizeMultiplier(calibration?.multiplier, 1);
+  const lowMultiplier = normalizeMultiplier(calibration?.lowMultiplier, samples > 0 ? multiplier * 0.95 : 0.85);
+  const highMultiplier = normalizeMultiplier(calibration?.highMultiplier, samples > 0 ? multiplier * 1.05 : 1.25);
+
+  return {
+    multiplier: samples > 0 ? multiplier : 1,
+    lowMultiplier: Math.min(lowMultiplier, highMultiplier),
+    highMultiplier: Math.max(lowMultiplier, highMultiplier),
+    samples,
+  };
+}
+
+export function estimateInitialPromptInput(options: EstimateInitialPromptInputOptions): InitialPromptInputEstimate {
+  const systemPrompt = options.systemPrompt ?? "";
+  const promptText = estimatePromptInjectionTokens(systemPrompt);
+  const toolPayload = buildActiveToolSchemaPayload(options.activeTools, options.allTools);
+  const toolSchemas = toolPayload.length > 0 ? estimateTokensFromText(stringifyForTokenEstimate(toolPayload)) : 0;
+  const framing = Math.max(
+    0,
+    Math.round(
+      options.framingTokens ??
+        DEFAULT_REQUEST_FRAMING_TOKENS + SYSTEM_MESSAGE_FRAMING_TOKENS + toolPayload.length * TOOL_SCHEMA_FRAMING_TOKENS,
+    ),
+  );
+  const uncalibratedTotal = Math.max(0, promptText + toolSchemas + framing);
+  const calibration = resolveCalibration(options.calibration);
+  const total = Math.max(0, Math.round(uncalibratedTotal * calibration.multiplier));
+  const low = Math.max(0, Math.round(uncalibratedTotal * calibration.lowMultiplier));
+  const high = Math.max(low, Math.round(uncalibratedTotal * calibration.highMultiplier));
+
+  return {
+    total,
+    low,
+    high,
+    uncalibratedTotal,
+    promptText,
+    toolSchemas,
+    framing,
+    toolCount: toolPayload.length,
+    calibrationMultiplier: calibration.multiplier,
+    calibrationSamples: calibration.samples,
+    confidence: calibration.samples > 0 ? "calibrated" : "estimated",
+  };
 }