@ashx-j/lunr-ai 0.2.27 → 0.2.28

package/dist/utils/estimate.js.map

@@ -1 +1 @@
-{"version":3,"file":"estimate.js","sourceRoot":"","sources":["../../src/utils/estimate.ts"],"names":[],"mappings":"AAaA,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAEnC,MAAM,UAAU,sBAAsB,CAAC,KAAY,EAAU;IAC5D,OAAO,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC;AAAA,CAC5F;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAU;IAClD,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,kBAAkB,CAAC;IAC3B,CAAC;AAAA,CACD;AAED,SAAS,gCAAgC,CAAC,OAAmD,EAAU;IACtG,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC,MAAM,CAAC;IAEvD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,OAAO;QAAE,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,qBAAqB,CAAC;IACxG,OAAO,KAAK,CAAC;AAAA,CACb;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAU;IACxD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,eAAe,CAAC,CAAC;AAAA,CAChD;AAED,MAAM,UAAU,iCAAiC,CAAC,OAAmD,EAAU;IAC9G,OAAO,IAAI,CAAC,IAAI,CAAC,gCAAgC,CAAC,OAAO,CAAC,GAAG,eAAe,CAAC,CAAC;AAAA,CAC9E;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAgB,EAAU;IAC/D,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM;QAAE,OAAO,iCAAiC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACvF,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY;QAAE,OAAO,iCAAiC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAE7F,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3B,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;QAC5B,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YACtC,KAAK,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;QAChC,CAAC;aAAM,CAAC;YACP,KAAK,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;QACxE,CAAC;IACF,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,eAAe,CAAC,CAAC;AAAA,CAC1C;AAED,SAAS,yBAAyB,CAAC,QAA4B,EAA+C;IAC7G,IAAI,qBAAqB,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACrD,IAAI,SAAsD,CAAC;IAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,OAA2B,CAAC;YAC9C,0EAA0E;YAC1E,wEAAwE;YACxE,MAAM,oBAAoB,GAAG,SAAS,CAAC,SAAS,IAAI,qBAAqB,CAAC;YAC1E,IACC,oBAAoB;gBACpB,SAAS,CAAC,UAAU,KAAK,SAAS;gBAClC,SAAS,CAAC,UAAU,KAAK,OAAO;gBAChC,sBAAsB,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,EAC1C,CAAC;gBACF,SAAS,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAClD,CAAC;QACF,CAAC;QACD,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO,SAAS,CAAC;AAAA,CACjB;AAED,SAAS,gBAAgB,CAAC,QAA4B,EAAwB;IAC7E,MAAM,SAAS,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;IACtD,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC5D,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5D,cAAc,IAAI,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,WAAW,GAAG,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/G,CAAC;IAED,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,OAAO,IAAI,QAAQ;QAAE,MAAM,IAAI,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;AAAA,CAChF;AAED,SAAS,mBAAmB,CAAC,KAAkC,EAAU;IACxE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,OAAO,kBAAkB,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,SAAS,cAAc,CAAC,KAAmC,EAA+B;IACzF,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAAA,CAC5B;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAqC,EAAwB;IAClG,IAAI,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,GAAG,CACzB,OAAO,CAAC,QAAQ;aACd,KAAK,CAAC,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC;aAClC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,YAAY,CAAC;aAClD,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,eAAe,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxG,OAAO;YACN,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,eAAe;YACzC,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,cAAc,EAAE,QAAQ,CAAC,cAAc,GAAG,eAAe;YACzD,cAAc,EAAE,QAAQ,CAAC,cAAc;SACvC,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GACjB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAE5G,OAAO;QACN,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY;QACtC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,cAAc,EAAE,QAAQ,CAAC,cAAc,GAAG,YAAY;QACtD,cAAc,EAAE,QAAQ,CAAC,cAAc;KACvC,CAAC;AAAA,CACF","sourcesContent":["import type { AssistantMessage, Context, ImageContent, Message, TextContent, Tool, Usage } from \"../types.ts\";\n\nexport interface ContextUsageEstimate {\n\t/** Estimated total context tokens. */\n\ttokens: number;\n\t/** Tokens reported by the most recent applicable assistant usage block. */\n\tusageTokens: number;\n\t/** Estimated tokens after the most recent applicable assistant usage block. */\n\ttrailingTokens: number;\n\t/** Index of the applicable message that provided usage, or null when none exists. */\n\tlastUsageIndex: number | null;\n}\n\nconst CHARS_PER_TOKEN = 4;\nconst ESTIMATED_IMAGE_CHARS = 4800;\n\nexport function calculateContextTokens(usage: Usage): number {\n\treturn usage.totalTokens || usage.input + usage.output + usage.cacheRead + usage.cacheWrite;\n}\n\nfunction safeJsonStringify(value: unknown): string {\n\ttry {\n\t\treturn JSON.stringify(value) ?? \"undefined\";\n\t} catch {\n\t\treturn \"[unserializable]\";\n\t}\n}\n\nfunction estimateTextAndImageContentChars(content: string | Array<TextContent | ImageContent>): number {\n\tif (typeof content === \"string\") return content.length;\n\n\tlet chars = 0;\n\tfor (const block of content) chars += block.type === \"text\" ? block.text.length : ESTIMATED_IMAGE_CHARS;\n\treturn chars;\n}\n\nexport function estimateTextTokens(text: string): number {\n\treturn Math.ceil(text.length / CHARS_PER_TOKEN);\n}\n\nexport function estimateTextAndImageContentTokens(content: string | Array<TextContent | ImageContent>): number {\n\treturn Math.ceil(estimateTextAndImageContentChars(content) / CHARS_PER_TOKEN);\n}\n\nexport function estimateMessageTokens(message: Message): number {\n\tlet chars = 0;\n\n\tif (message.role === \"user\") return estimateTextAndImageContentTokens(message.content);\n\tif (message.role === \"toolResult\") return estimateTextAndImageContentTokens(message.content);\n\n\tfor (const block of message.content) {\n\t\tif (block.type === \"text\") {\n\t\t\tchars += block.text.length;\n\t\t} else if (block.type === \"thinking\") {\n\t\t\tchars += block.thinking.length;\n\t\t} else {\n\t\t\tchars += block.name.length + safeJsonStringify(block.arguments).length;\n\t\t}\n\t}\n\treturn Math.ceil(chars / CHARS_PER_TOKEN);\n}\n\nfunction getLastAssistantUsageInfo(messages: readonly Message[]): { usage: Usage; index: number } | undefined {\n\tlet latestPrefixTimestamp = Number.NEGATIVE_INFINITY;\n\tlet usageInfo: { usage: Usage; index: number } | undefined;\n\n\tfor (let i = 0; i < messages.length; i++) {\n\t\tconst message = messages[i];\n\t\tif (message.role === \"assistant\") {\n\t\t\tconst assistant = message as AssistantMessage;\n\t\t\t// A newer prefix message was inserted after this response (for example, a\n\t\t\t// compaction summary), so its usage cannot describe the current prefix.\n\t\t\tconst usageAppliesToPrefix = assistant.timestamp >= latestPrefixTimestamp;\n\t\t\tif (\n\t\t\t\tusageAppliesToPrefix &&\n\t\t\t\tassistant.stopReason !== \"aborted\" &&\n\t\t\t\tassistant.stopReason !== \"error\" &&\n\t\t\t\tcalculateContextTokens(assistant.usage) > 0\n\t\t\t) {\n\t\t\t\tusageInfo = { usage: assistant.usage, index: i };\n\t\t\t}\n\t\t}\n\t\tlatestPrefixTimestamp = Math.max(latestPrefixTimestamp, message.timestamp);\n\t}\n\n\treturn usageInfo;\n}\n\nfunction estimateMessages(messages: readonly Message[]): ContextUsageEstimate {\n\tconst usageInfo = getLastAssistantUsageInfo(messages);\n\tif (usageInfo) {\n\t\tconst usageTokens = calculateContextTokens(usageInfo.usage);\n\t\tlet trailingTokens = 0;\n\t\tfor (let i = usageInfo.index + 1; i < messages.length; i++) {\n\t\t\ttrailingTokens += estimateMessageTokens(messages[i]);\n\t\t}\n\t\treturn { tokens: usageTokens + trailingTokens, usageTokens, trailingTokens, lastUsageIndex: usageInfo.index };\n\t}\n\n\tlet tokens = 0;\n\tfor (const message of messages) tokens += estimateMessageTokens(message);\n\treturn { tokens, usageTokens: 0, trailingTokens: tokens, lastUsageIndex: null };\n}\n\nfunction estimateToolsTokens(tools: readonly Tool[] | undefined): number {\n\tif (!tools || tools.length === 0) return 0;\n\treturn estimateTextTokens(safeJsonStringify(tools));\n}\n\nfunction isMessageArray(value: Context | readonly Message[]): value is readonly Message[] {\n\treturn Array.isArray(value);\n}\n\nexport function estimateContextTokens(context: Context | readonly Message[]): ContextUsageEstimate {\n\tif (isMessageArray(context)) return estimateMessages(context);\n\n\tconst estimate = estimateMessages(context.messages);\n\tif (estimate.lastUsageIndex !== null) {\n\t\tconst addedNames = new Set(\n\t\t\tcontext.messages\n\t\t\t\t.slice(estimate.lastUsageIndex + 1)\n\t\t\t\t.filter((message) => message.role === \"toolResult\")\n\t\t\t\t.flatMap((message) => message.addedToolNames ?? []),\n\t\t);\n\t\tconst addedToolTokens = estimateToolsTokens(context.tools?.filter((tool) => addedNames.has(tool.name)));\n\t\treturn {\n\t\t\ttokens: estimate.tokens + addedToolTokens,\n\t\t\tusageTokens: estimate.usageTokens,\n\t\t\ttrailingTokens: estimate.trailingTokens + addedToolTokens,\n\t\t\tlastUsageIndex: estimate.lastUsageIndex,\n\t\t};\n\t}\n\n\tconst prefixTokens =\n\t\t(context.systemPrompt ? estimateTextTokens(context.systemPrompt) : 0) + estimateToolsTokens(context.tools);\n\n\treturn {\n\t\ttokens: estimate.tokens + prefixTokens,\n\t\tusageTokens: estimate.usageTokens,\n\t\ttrailingTokens: estimate.trailingTokens + prefixTokens,\n\t\tlastUsageIndex: estimate.lastUsageIndex,\n\t};\n}\n"]}
+{"version":3,"file":"estimate.js","sourceRoot":"","sources":["../../src/utils/estimate.ts"],"names":[],"mappings":"AAaA,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAEnC,MAAM,UAAU,sBAAsB,CAAC,KAAY,EAAU;IAC5D,OAAO,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC;AAAA,CAC5F;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAU;IAClD,IAAI,CAAC;QACJ,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,WAAW,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,kBAAkB,CAAC;IAC3B,CAAC;AAAA,CACD;AAED,yFAAyF;AACzF,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAU;IACxD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,IAAI;YAAE,KAAK,EAAE,CAAC;;YAC1C,QAAQ,EAAE,CAAC;IACjB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,eAAe,GAAG,QAAQ,CAAC,CAAC;AAAA,CACrD;AAED,iFAAiF;AACjF,MAAM,UAAU,qBAAqB,CAAC,OAAgD,EAAU;IAC/F,MAAM,UAAU,GAAG,iBAAiB,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IACxF,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;IACxG,OAAO,GAAG,UAAU,CAAC,MAAM,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;AAAA,CAC5C;AAED,MAAM,UAAU,iCAAiC,CAAC,OAAmD,EAAU;IAC9G,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;IACpE,OAAO,OAAO,CAAC,MAAM,CACpB,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CACd,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,qBAAqB,GAAG,eAAe,CAAC,EACzG,CAAC,CACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAgB,EAAU;IAC/D,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY;QAC3D,OAAO,iCAAiC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3D,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,CAC5B,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CACjB,MAAM;QACN,kBAAkB,CACjB,KAAK,CAAC,IAAI,KAAK,MAAM;YACpB,CAAC,CAAC,KAAK,CAAC,IAAI;YACZ,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU;gBAC1B,CAAC,CAAC,KAAK,CAAC,QAAQ;gBAChB,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CACnD,EACF,CAAC,CACD,CAAC;AAAA,CACF;AAED,SAAS,yBAAyB,CACjC,QAA4B,EAC5B,WAAoB,EAC0B;IAC9C,IAAI,qBAAqB,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACrD,IAAI,SAAsD,CAAC;IAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,OAAO,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,OAA2B,CAAC;YAC9C,0EAA0E;YAC1E,wEAAwE;YACxE,MAAM,oBAAoB,GAAG,SAAS,CAAC,SAAS,IAAI,qBAAqB,CAAC;YAC1E,IACC,oBAAoB;gBACpB,CAAC,WAAW,KAAK,SAAS;oBACzB,SAAS,CAAC,KAAK,CAAC,kBAAkB,KAAK,SAAS;oBAChD,SAAS,CAAC,KAAK,CAAC,kBAAkB,KAAK,WAAW,CAAC;gBACpD,SAAS,CAAC,UAAU,KAAK,SAAS;gBAClC,SAAS,CAAC,UAAU,KAAK,OAAO;gBAChC,sBAAsB,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,EAC1C,CAAC;gBACF,SAAS,GAAG,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;YAClD,CAAC;QACF,CAAC;QACD,qBAAqB,GAAG,IAAI,CAAC,GAAG,CAAC,qBAAqB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC5E,CAAC;IAED,OAAO,SAAS,CAAC;AAAA,CACjB;AAED,SAAS,gBAAgB,CAAC,QAA4B,EAAE,WAAoB,EAAwB;IACnG,MAAM,SAAS,GAAG,yBAAyB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IACnE,IAAI,SAAS,EAAE,CAAC;QACf,MAAM,WAAW,GAAG,sBAAsB,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC5D,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5D,cAAc,IAAI,qBAAqB,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,WAAW,GAAG,cAAc,EAAE,WAAW,EAAE,cAAc,EAAE,cAAc,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC;IAC/G,CAAC;IAED,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,OAAO,IAAI,QAAQ;QAAE,MAAM,IAAI,qBAAqB,CAAC,OAAO,CAAC,CAAC;IACzE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,EAAE,cAAc,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;AAAA,CAChF;AAED,SAAS,mBAAmB,CAAC,KAAkC,EAAU;IACxE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,OAAO,kBAAkB,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;AAAA,CACpD;AAED,SAAS,cAAc,CAAC,KAAmC,EAA+B;IACzF,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAAA,CAC5B;AAED,MAAM,UAAU,qBAAqB,CAAC,OAAqC,EAAwB;IAClG,IAAI,cAAc,CAAC,OAAO,CAAC;QAAE,OAAO,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE9D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,QAAQ,EAAE,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC;IACpF,IAAI,QAAQ,CAAC,cAAc,KAAK,IAAI,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,IAAI,GAAG,CACzB,OAAO,CAAC,QAAQ;aACd,KAAK,CAAC,QAAQ,CAAC,cAAc,GAAG,CAAC,CAAC;aAClC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,YAAY,CAAC;aAClD,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,IAAI,EAAE,CAAC,CACpD,CAAC;QACF,MAAM,eAAe,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACxG,OAAO;YACN,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,eAAe;YACzC,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,cAAc,EAAE,QAAQ,CAAC,cAAc,GAAG,eAAe;YACzD,cAAc,EAAE,QAAQ,CAAC,cAAc;SACvC,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GACjB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,mBAAmB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAE5G,OAAO;QACN,MAAM,EAAE,QAAQ,CAAC,MAAM,GAAG,YAAY;QACtC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,cAAc,EAAE,QAAQ,CAAC,cAAc,GAAG,YAAY;QACtD,cAAc,EAAE,QAAQ,CAAC,cAAc;KACvC,CAAC;AAAA,CACF","sourcesContent":["import type { AssistantMessage, Context, ImageContent, Message, TextContent, Tool, Usage } from \"../types.ts\";\n\nexport interface ContextUsageEstimate {\n\t/** Estimated total context tokens. */\n\ttokens: number;\n\t/** Tokens reported by the most recent applicable assistant usage block. */\n\tusageTokens: number;\n\t/** Estimated tokens after the most recent applicable assistant usage block. */\n\ttrailingTokens: number;\n\t/** Index of the applicable message that provided usage, or null when none exists. */\n\tlastUsageIndex: number | null;\n}\n\nconst CHARS_PER_TOKEN = 4;\nconst ESTIMATED_IMAGE_CHARS = 4800;\n\nexport function calculateContextTokens(usage: Usage): number {\n\treturn usage.totalTokens || usage.input + usage.output + usage.cacheRead + usage.cacheWrite;\n}\n\nfunction safeJsonStringify(value: unknown): string {\n\ttry {\n\t\treturn JSON.stringify(value) ?? \"undefined\";\n\t} catch {\n\t\treturn \"[unserializable]\";\n\t}\n}\n\n/** Local heuristic, not a provider tokenizer. Non-ASCII text often needs more tokens. */\nexport function estimateTextTokens(text: string): number {\n\tlet ascii = 0;\n\tlet nonAscii = 0;\n\tfor (const character of text) {\n\t\tif (character.codePointAt(0)! <= 0x7f) ascii++;\n\t\telse nonAscii++;\n\t}\n\treturn Math.ceil(ascii / CHARS_PER_TOKEN + nonAscii);\n}\n\n/** Only the stable request prefix; conversation growth is handled separately. */\nexport function getContextFingerprint(context: Pick<Context, \"systemPrompt\" | \"tools\">): string {\n\tconst serialized = safeJsonStringify([context.systemPrompt ?? \"\", context.tools ?? []]);\n\tlet hash = 2166136261;\n\tfor (let i = 0; i < serialized.length; i++) hash = Math.imul(hash ^ serialized.charCodeAt(i), 16777619);\n\treturn `${serialized.length}:${hash >>> 0}`;\n}\n\nexport function estimateTextAndImageContentTokens(content: string | Array<TextContent | ImageContent>): number {\n\tif (typeof content === \"string\") return estimateTextTokens(content);\n\treturn content.reduce(\n\t\t(sum, block) =>\n\t\t\tsum + (block.type === \"text\" ? estimateTextTokens(block.text) : ESTIMATED_IMAGE_CHARS / CHARS_PER_TOKEN),\n\t\t0,\n\t);\n}\n\nexport function estimateMessageTokens(message: Message): number {\n\tif (message.role === \"user\" || message.role === \"toolResult\")\n\t\treturn estimateTextAndImageContentTokens(message.content);\n\treturn message.content.reduce(\n\t\t(tokens, block) =>\n\t\t\ttokens +\n\t\t\testimateTextTokens(\n\t\t\t\tblock.type === \"text\"\n\t\t\t\t\t? block.text\n\t\t\t\t\t: block.type === \"thinking\"\n\t\t\t\t\t\t? block.thinking\n\t\t\t\t\t\t: block.name + safeJsonStringify(block.arguments),\n\t\t\t),\n\t\t0,\n\t);\n}\n\nfunction getLastAssistantUsageInfo(\n\tmessages: readonly Message[],\n\tfingerprint?: string,\n): { usage: Usage; index: number } | undefined {\n\tlet latestPrefixTimestamp = Number.NEGATIVE_INFINITY;\n\tlet usageInfo: { usage: Usage; index: number } | undefined;\n\n\tfor (let i = 0; i < messages.length; i++) {\n\t\tconst message = messages[i];\n\t\tif (message.role === \"assistant\") {\n\t\t\tconst assistant = message as AssistantMessage;\n\t\t\t// A newer prefix message was inserted after this response (for example, a\n\t\t\t// compaction summary), so its usage cannot describe the current prefix.\n\t\t\tconst usageAppliesToPrefix = assistant.timestamp >= latestPrefixTimestamp;\n\t\t\tif (\n\t\t\t\tusageAppliesToPrefix &&\n\t\t\t\t(fingerprint === undefined ||\n\t\t\t\t\tassistant.usage.contextFingerprint === undefined ||\n\t\t\t\t\tassistant.usage.contextFingerprint === fingerprint) &&\n\t\t\t\tassistant.stopReason !== \"aborted\" &&\n\t\t\t\tassistant.stopReason !== \"error\" &&\n\t\t\t\tcalculateContextTokens(assistant.usage) > 0\n\t\t\t) {\n\t\t\t\tusageInfo = { usage: assistant.usage, index: i };\n\t\t\t}\n\t\t}\n\t\tlatestPrefixTimestamp = Math.max(latestPrefixTimestamp, message.timestamp);\n\t}\n\n\treturn usageInfo;\n}\n\nfunction estimateMessages(messages: readonly Message[], fingerprint?: string): ContextUsageEstimate {\n\tconst usageInfo = getLastAssistantUsageInfo(messages, fingerprint);\n\tif (usageInfo) {\n\t\tconst usageTokens = calculateContextTokens(usageInfo.usage);\n\t\tlet trailingTokens = 0;\n\t\tfor (let i = usageInfo.index + 1; i < messages.length; i++) {\n\t\t\ttrailingTokens += estimateMessageTokens(messages[i]);\n\t\t}\n\t\treturn { tokens: usageTokens + trailingTokens, usageTokens, trailingTokens, lastUsageIndex: usageInfo.index };\n\t}\n\n\tlet tokens = 0;\n\tfor (const message of messages) tokens += estimateMessageTokens(message);\n\treturn { tokens, usageTokens: 0, trailingTokens: tokens, lastUsageIndex: null };\n}\n\nfunction estimateToolsTokens(tools: readonly Tool[] | undefined): number {\n\tif (!tools || tools.length === 0) return 0;\n\treturn estimateTextTokens(safeJsonStringify(tools));\n}\n\nfunction isMessageArray(value: Context | readonly Message[]): value is readonly Message[] {\n\treturn Array.isArray(value);\n}\n\nexport function estimateContextTokens(context: Context | readonly Message[]): ContextUsageEstimate {\n\tif (isMessageArray(context)) return estimateMessages(context);\n\n\tconst estimate = estimateMessages(context.messages, getContextFingerprint(context));\n\tif (estimate.lastUsageIndex !== null) {\n\t\tconst addedNames = new Set(\n\t\t\tcontext.messages\n\t\t\t\t.slice(estimate.lastUsageIndex + 1)\n\t\t\t\t.filter((message) => message.role === \"toolResult\")\n\t\t\t\t.flatMap((message) => message.addedToolNames ?? []),\n\t\t);\n\t\tconst addedToolTokens = estimateToolsTokens(context.tools?.filter((tool) => addedNames.has(tool.name)));\n\t\treturn {\n\t\t\ttokens: estimate.tokens + addedToolTokens,\n\t\t\tusageTokens: estimate.usageTokens,\n\t\t\ttrailingTokens: estimate.trailingTokens + addedToolTokens,\n\t\t\tlastUsageIndex: estimate.lastUsageIndex,\n\t\t};\n\t}\n\n\tconst prefixTokens =\n\t\t(context.systemPrompt ? estimateTextTokens(context.systemPrompt) : 0) + estimateToolsTokens(context.tools);\n\n\treturn {\n\t\ttokens: estimate.tokens + prefixTokens,\n\t\tusageTokens: estimate.usageTokens,\n\t\ttrailingTokens: estimate.trailingTokens + prefixTokens,\n\t\tlastUsageIndex: estimate.lastUsageIndex,\n\t};\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@ashx-j/lunr-ai",
-	"version": "0.2.27",
+	"version": "0.2.28",
 	"description": "Unified LLM API with automatic model discovery and provider configuration",
 	"type": "module",
 	"main": "./dist/index.js",