@fastpaca/cria 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +25 -22
- package/dist/dsl/builder.d.ts +243 -0
- package/dist/dsl/builder.d.ts.map +1 -0
- package/dist/dsl/builder.js +458 -0
- package/dist/dsl/builder.js.map +1 -0
- package/dist/dsl/index.d.ts +71 -0
- package/dist/dsl/index.d.ts.map +1 -0
- package/dist/dsl/index.js +82 -0
- package/dist/dsl/index.js.map +1 -0
- package/dist/dsl/strategies.d.ts +29 -0
- package/dist/dsl/strategies.d.ts.map +1 -0
- package/dist/dsl/strategies.js +62 -0
- package/dist/dsl/strategies.js.map +1 -0
- package/dist/dsl/summary.d.ts +50 -0
- package/dist/dsl/summary.d.ts.map +1 -0
- package/dist/dsl/summary.js +56 -0
- package/dist/dsl/summary.js.map +1 -0
- package/dist/dsl/templating.d.ts +31 -0
- package/dist/dsl/templating.d.ts.map +1 -0
- package/dist/dsl/templating.js +83 -0
- package/dist/dsl/templating.js.map +1 -0
- package/dist/{components → dsl}/vector-search.d.ts +11 -34
- package/dist/dsl/vector-search.d.ts.map +1 -0
- package/dist/{components → dsl}/vector-search.js +12 -34
- package/dist/dsl/vector-search.js.map +1 -0
- package/dist/eval/index.d.ts +13 -2
- package/dist/eval/index.d.ts.map +1 -1
- package/dist/eval/judge.d.ts +6 -5
- package/dist/eval/judge.d.ts.map +1 -1
- package/dist/eval/judge.js +18 -5
- package/dist/eval/judge.js.map +1 -1
- package/dist/index.d.ts +5 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -6
- package/dist/index.js.map +1 -1
- package/dist/instrumentation/otel.js +5 -14
- package/dist/instrumentation/otel.js.map +1 -1
- package/dist/providers/ai-sdk.d.ts +29 -0
- package/dist/providers/ai-sdk.d.ts.map +1 -0
- package/dist/providers/ai-sdk.js +104 -0
- package/dist/providers/ai-sdk.js.map +1 -0
- package/dist/providers/anthropic.d.ts +30 -0
- package/dist/providers/anthropic.d.ts.map +1 -0
- package/dist/providers/anthropic.js +160 -0
- package/dist/providers/anthropic.js.map +1 -0
- package/dist/providers/openai.d.ts +37 -0
- package/dist/providers/openai.d.ts.map +1 -0
- package/dist/providers/openai.js +164 -0
- package/dist/providers/openai.js.map +1 -0
- package/dist/render.d.ts +24 -22
- package/dist/render.d.ts.map +1 -1
- package/dist/render.js +191 -63
- package/dist/render.js.map +1 -1
- package/dist/testing/plaintext.d.ts +28 -0
- package/dist/testing/plaintext.d.ts.map +1 -0
- package/dist/testing/plaintext.js +71 -0
- package/dist/testing/plaintext.js.map +1 -0
- package/dist/types.d.ts +204 -163
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js +37 -100
- package/dist/types.js.map +1 -1
- package/package.json +7 -33
- package/dist/components/index.d.ts +0 -147
- package/dist/components/index.d.ts.map +0 -1
- package/dist/components/index.js +0 -181
- package/dist/components/index.js.map +0 -1
- package/dist/components/summary.d.ts +0 -89
- package/dist/components/summary.d.ts.map +0 -1
- package/dist/components/summary.js +0 -118
- package/dist/components/summary.js.map +0 -1
- package/dist/components/vector-search.d.ts.map +0 -1
- package/dist/components/vector-search.js.map +0 -1
- package/dist/dsl.d.ts +0 -233
- package/dist/dsl.d.ts.map +0 -1
- package/dist/dsl.js +0 -428
- package/dist/dsl.js.map +0 -1
- package/dist/integrations/ai-sdk.d.ts +0 -22
- package/dist/integrations/ai-sdk.d.ts.map +0 -1
- package/dist/integrations/ai-sdk.js +0 -172
- package/dist/integrations/ai-sdk.js.map +0 -1
- package/dist/integrations/anthropic.d.ts +0 -50
- package/dist/integrations/anthropic.d.ts.map +0 -1
- package/dist/integrations/anthropic.js +0 -220
- package/dist/integrations/anthropic.js.map +0 -1
- package/dist/integrations/openai.d.ts +0 -48
- package/dist/integrations/openai.d.ts.map +0 -1
- package/dist/integrations/openai.js +0 -265
- package/dist/integrations/openai.js.map +0 -1
- package/dist/jsx/index.d.ts +0 -6
- package/dist/jsx/index.d.ts.map +0 -1
- package/dist/jsx/index.js +0 -7
- package/dist/jsx/index.js.map +0 -1
- package/dist/jsx/jsx-dev-runtime.d.ts +0 -2
- package/dist/jsx/jsx-dev-runtime.d.ts.map +0 -1
- package/dist/jsx/jsx-dev-runtime.js +0 -3
- package/dist/jsx/jsx-dev-runtime.js.map +0 -1
- package/dist/jsx/jsx-runtime.d.ts +0 -40
- package/dist/jsx/jsx-runtime.d.ts.map +0 -1
- package/dist/jsx/jsx-runtime.js +0 -40
- package/dist/jsx/jsx-runtime.js.map +0 -1
- package/dist/renderers/markdown.d.ts +0 -3
- package/dist/renderers/markdown.d.ts.map +0 -1
- package/dist/renderers/markdown.js +0 -43
- package/dist/renderers/markdown.js.map +0 -1
- package/dist/renderers/shared.d.ts +0 -82
- package/dist/renderers/shared.d.ts.map +0 -1
- package/dist/renderers/shared.js +0 -156
- package/dist/renderers/shared.js.map +0 -1
- package/dist/snapshot.d.ts +0 -47
- package/dist/snapshot.d.ts.map +0 -1
- package/dist/snapshot.js +0 -144
- package/dist/snapshot.js.map +0 -1
- package/dist/tokenizers.d.ts +0 -14
- package/dist/tokenizers.d.ts.map +0 -1
- package/dist/tokenizers.js +0 -45
- package/dist/tokenizers.js.map +0 -1
package/dist/renderers/shared.js
DELETED
|
@@ -1,156 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Collect all message nodes from the prompt tree.
|
|
3
|
-
*/
|
|
4
|
-
export function collectMessageNodes(element, acc = []) {
|
|
5
|
-
if (element.kind === "message") {
|
|
6
|
-
acc.push(element);
|
|
7
|
-
return acc;
|
|
8
|
-
}
|
|
9
|
-
for (const child of element.children) {
|
|
10
|
-
if (typeof child !== "string") {
|
|
11
|
-
collectMessageNodes(child, acc);
|
|
12
|
-
}
|
|
13
|
-
}
|
|
14
|
-
return acc;
|
|
15
|
-
}
|
|
16
|
-
/**
|
|
17
|
-
* Extract semantic parts from prompt children.
|
|
18
|
-
*/
|
|
19
|
-
export function collectSemanticParts(children) {
|
|
20
|
-
const parts = [];
|
|
21
|
-
for (const child of children) {
|
|
22
|
-
if (typeof child === "string") {
|
|
23
|
-
if (child.length > 0) {
|
|
24
|
-
parts.push({ type: "text", text: child });
|
|
25
|
-
}
|
|
26
|
-
continue;
|
|
27
|
-
}
|
|
28
|
-
parts.push(...semanticPartsFromElement(child));
|
|
29
|
-
}
|
|
30
|
-
return parts;
|
|
31
|
-
}
|
|
32
|
-
/**
|
|
33
|
-
* Extract semantic parts from a single prompt element.
|
|
34
|
-
*/
|
|
35
|
-
export function semanticPartsFromElement(element) {
|
|
36
|
-
switch (element.kind) {
|
|
37
|
-
case "tool-call":
|
|
38
|
-
return [
|
|
39
|
-
{
|
|
40
|
-
type: "tool-call",
|
|
41
|
-
toolCallId: element.toolCallId,
|
|
42
|
-
toolName: element.toolName,
|
|
43
|
-
input: element.input,
|
|
44
|
-
},
|
|
45
|
-
];
|
|
46
|
-
case "tool-result":
|
|
47
|
-
return [
|
|
48
|
-
{
|
|
49
|
-
type: "tool-result",
|
|
50
|
-
toolCallId: element.toolCallId,
|
|
51
|
-
toolName: element.toolName,
|
|
52
|
-
output: element.output,
|
|
53
|
-
},
|
|
54
|
-
];
|
|
55
|
-
case "reasoning":
|
|
56
|
-
return element.text.length === 0
|
|
57
|
-
? []
|
|
58
|
-
: [{ type: "reasoning", text: element.text }];
|
|
59
|
-
case "message":
|
|
60
|
-
// Nested messages are ambiguous - skip
|
|
61
|
-
return [];
|
|
62
|
-
default:
|
|
63
|
-
return collectSemanticParts(element.children);
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Coalesce adjacent text parts into single parts.
|
|
68
|
-
*/
|
|
69
|
-
export function coalesceTextParts(parts) {
|
|
70
|
-
const result = [];
|
|
71
|
-
let buffer = "";
|
|
72
|
-
for (const part of parts) {
|
|
73
|
-
if (part.type === "text") {
|
|
74
|
-
buffer += part.text;
|
|
75
|
-
continue;
|
|
76
|
-
}
|
|
77
|
-
if (buffer.length > 0) {
|
|
78
|
-
result.push({ type: "text", text: buffer });
|
|
79
|
-
buffer = "";
|
|
80
|
-
}
|
|
81
|
-
result.push(part);
|
|
82
|
-
}
|
|
83
|
-
if (buffer.length > 0) {
|
|
84
|
-
result.push({ type: "text", text: buffer });
|
|
85
|
-
}
|
|
86
|
-
return result;
|
|
87
|
-
}
|
|
88
|
-
/**
|
|
89
|
-
* Safely stringify a value to JSON string.
|
|
90
|
-
* Returns string values directly, handles undefined, and catches JSON errors.
|
|
91
|
-
*
|
|
92
|
-
* @param value - The value to stringify
|
|
93
|
-
* @param pretty - If true, format with 2-space indentation (default: false)
|
|
94
|
-
*/
|
|
95
|
-
export function safeStringify(value, pretty = false) {
|
|
96
|
-
if (typeof value === "string") {
|
|
97
|
-
return value;
|
|
98
|
-
}
|
|
99
|
-
if (value === undefined) {
|
|
100
|
-
return "null";
|
|
101
|
-
}
|
|
102
|
-
try {
|
|
103
|
-
return ((pretty ? JSON.stringify(value, null, 2) : JSON.stringify(value)) ??
|
|
104
|
-
"null");
|
|
105
|
-
}
|
|
106
|
-
catch {
|
|
107
|
-
return String(value);
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
/**
|
|
111
|
-
* Categorize semantic parts by type for easier processing.
|
|
112
|
-
*/
|
|
113
|
-
export function categorizeParts(parts) {
|
|
114
|
-
const textParts = [];
|
|
115
|
-
const toolCallParts = [];
|
|
116
|
-
const toolResultParts = [];
|
|
117
|
-
const reasoningParts = [];
|
|
118
|
-
for (const part of parts) {
|
|
119
|
-
switch (part.type) {
|
|
120
|
-
case "text":
|
|
121
|
-
textParts.push(part);
|
|
122
|
-
break;
|
|
123
|
-
case "tool-call":
|
|
124
|
-
toolCallParts.push(part);
|
|
125
|
-
break;
|
|
126
|
-
case "tool-result":
|
|
127
|
-
toolResultParts.push(part);
|
|
128
|
-
break;
|
|
129
|
-
case "reasoning":
|
|
130
|
-
reasoningParts.push(part);
|
|
131
|
-
break;
|
|
132
|
-
default:
|
|
133
|
-
// Exhaustive check - TypeScript will error if a case is missing
|
|
134
|
-
break;
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
return { textParts, toolCallParts, toolResultParts, reasoningParts };
|
|
138
|
-
}
|
|
139
|
-
/**
|
|
140
|
-
* Extract text content from semantic parts.
|
|
141
|
-
* Optionally wraps reasoning parts in thinking tags.
|
|
142
|
-
*/
|
|
143
|
-
export function partsToText(parts, options = {}) {
|
|
144
|
-
const { wrapReasoning = false } = options;
|
|
145
|
-
let result = "";
|
|
146
|
-
for (const part of parts) {
|
|
147
|
-
if (part.type === "text") {
|
|
148
|
-
result += part.text;
|
|
149
|
-
}
|
|
150
|
-
else if (part.type === "reasoning" && wrapReasoning) {
|
|
151
|
-
result += `<thinking>\n${part.text}\n</thinking>\n`;
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
return result;
|
|
155
|
-
}
|
|
156
|
-
//# sourceMappingURL=shared.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../src/renderers/shared.ts"],"names":[],"mappings":"AAwBA;;GAEG;AACH,MAAM,UAAU,mBAAmB,CACjC,OAAsB,EACtB,MAAwB,EAAE;IAE1B,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,mBAAmB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,QAAwB;IAC3D,MAAM,KAAK,GAAmB,EAAE,CAAC;IAEjC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,SAAS;QACX,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACtC,OAAsB;IAEtB,QAAQ,OAAO,CAAC,IAAI,EAAE,CAAC;QACrB,KAAK,WAAW;YACd,OAAO;gBACL;oBACE,IAAI,EAAE,WAAW;oBACjB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;iBACrB;aACF,CAAC;QACJ,KAAK,aAAa;YAChB,OAAO;gBACL;oBACE,IAAI,EAAE,aAAa;oBACnB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,MAAM,EAAE,OAAO,CAAC,MAAM;iBACvB;aACF,CAAC;QACJ,KAAK,WAAW;YACd,OAAO,OAAO,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;gBAC9B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,KAAK,SAAS;YACZ,uCAAuC;YACvC,OAAO,EAAE,CAAC;QACZ;YACE,OAAO,oBAAoB,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAClD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAC/B,KAA8B;IAE9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,IAAI,MAAM,GAAG,EAAE,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC;YACpB,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;YAC5C,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,KAAc,EAAE,MAAM,GAAG,KAAK;IAC1D,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,OAAO,CACL,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACjE,MAAM,CACP,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAYD;;GAEG;AACH,MAAM,UAAU,eAAe,CAC7B,KAA8B;IAE9B,MAAM,SAAS,GAAe,EAAE,CAAC;IACjC,MAAM,aAAa,GAAmB,EAAE,CAAC;IACzC,MAAM,eAAe,GAAqB,EAAE,CAAC;IAC7C,MAAM,cAAc,GAAoB,EAAE,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;YAClB,KAAK,MAAM;gBACT,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,WAAW;gBACd,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACzB,MAAM;YACR,KAAK,aAAa;gBAChB,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,WAAW;gBACd,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,MAAM;YACR;gBACE,gEAAgE;gBAChE,MAAM;QACV,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC;AACvE,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAA8B,EAC9B,UAAuC,EAAE;IAEzC,MAAM,EAAE,aAAa,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAC1C,IAAI,MAAM,GAAG,EAAE,CAAC;IAEhB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC;QACtB,CAAC;aAAM,IAAI,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,aAAa,EAAE,CAAC;YACtD,MAAM,IAAI,eAAe,IAAI,CAAC,IAAI,iBAAiB,CAAC;QACtD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/snapshot.d.ts
DELETED
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import type { RenderHooks } from "./render";
|
|
2
|
-
import type { MaybePromise, PromptElement, PromptRenderer, Tokenizer } from "./types";
|
|
3
|
-
export type SnapshotChild = string | SnapshotElement;
|
|
4
|
-
export interface SnapshotElement extends Omit<PromptElement, "children" | "strategy" | "context"> {
|
|
5
|
-
children: SnapshotChild[];
|
|
6
|
-
tokens: number;
|
|
7
|
-
hash: string;
|
|
8
|
-
}
|
|
9
|
-
export interface Snapshot {
|
|
10
|
-
root: SnapshotElement;
|
|
11
|
-
totalTokens: number;
|
|
12
|
-
hash: string;
|
|
13
|
-
}
|
|
14
|
-
export interface SnapshotOptions {
|
|
15
|
-
tokenizer: Tokenizer;
|
|
16
|
-
renderer?: PromptRenderer<unknown>;
|
|
17
|
-
}
|
|
18
|
-
export interface SnapshotHooksOptions {
|
|
19
|
-
tokenizer: Tokenizer;
|
|
20
|
-
renderer?: PromptRenderer<unknown>;
|
|
21
|
-
onSnapshot: (snapshot: Snapshot) => MaybePromise<void>;
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Create a deterministic snapshot of a fitted prompt tree.
|
|
25
|
-
*
|
|
26
|
-
* - Keeps the tree structure (no flattening) and mirrors PromptElement shape.
|
|
27
|
-
* - Adds per-node token counts (using renderer.tokenString) and stable hashes.
|
|
28
|
-
* - Excludes strategy/context from the snapshot; this is for observability only.
|
|
29
|
-
*/
|
|
30
|
-
export declare function createSnapshot(element: PromptElement, { tokenizer, renderer }: SnapshotOptions): Snapshot;
|
|
31
|
-
export declare function createSnapshotHooks({ tokenizer, renderer, onSnapshot, }: SnapshotHooksOptions): RenderHooks;
|
|
32
|
-
export interface DiffEntry {
|
|
33
|
-
path: readonly number[];
|
|
34
|
-
node: SnapshotChild;
|
|
35
|
-
}
|
|
36
|
-
export interface DiffChange {
|
|
37
|
-
path: readonly number[];
|
|
38
|
-
before: SnapshotChild;
|
|
39
|
-
after: SnapshotChild;
|
|
40
|
-
}
|
|
41
|
-
export interface SnapshotDiff {
|
|
42
|
-
added: DiffEntry[];
|
|
43
|
-
removed: DiffEntry[];
|
|
44
|
-
changed: DiffChange[];
|
|
45
|
-
}
|
|
46
|
-
export declare function diffSnapshots(before: Snapshot, after: Snapshot): SnapshotDiff;
|
|
47
|
-
//# sourceMappingURL=snapshot.d.ts.map
|
package/dist/snapshot.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../src/snapshot.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5C,OAAO,KAAK,EACV,YAAY,EAEZ,aAAa,EACb,cAAc,EAEd,SAAS,EACV,MAAM,SAAS,CAAC;AAEjB,MAAM,MAAM,aAAa,GAAG,MAAM,GAAG,eAAe,CAAC;AAErD,MAAM,WAAW,eACf,SAAQ,IAAI,CAAC,aAAa,EAAE,UAAU,GAAG,UAAU,GAAG,SAAS,CAAC;IAChE,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,eAAe,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC;IACnC,UAAU,EAAE,CAAC,QAAQ,EAAE,QAAQ,KAAK,YAAY,CAAC,IAAI,CAAC,CAAC;CACxD;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAC5B,OAAO,EAAE,aAAa,EACtB,EAAE,SAAS,EAAE,QAA2B,EAAE,EAAE,eAAe,GAC1D,QAAQ,CAOV;AAED,wBAAgB,mBAAmB,CAAC,EAClC,SAAS,EACT,QAAQ,EACR,UAAU,GACX,EAAE,oBAAoB,GAAG,WAAW,CAapC;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACxB,IAAI,EAAE,aAAa,CAAC;CACrB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACxB,MAAM,EAAE,aAAa,CAAC;IACtB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,SAAS,EAAE,CAAC;IACnB,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,OAAO,EAAE,UAAU,EAAE,CAAC;CACvB;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,GAAG,YAAY,CAQ7E"}
|
package/dist/snapshot.js
DELETED
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
import { sha256 } from "@noble/hashes/sha256";
|
|
2
|
-
import { bytesToHex, utf8ToBytes } from "@noble/hashes/utils";
|
|
3
|
-
import { markdownRenderer } from "./renderers/markdown";
|
|
4
|
-
/**
|
|
5
|
-
* Create a deterministic snapshot of a fitted prompt tree.
|
|
6
|
-
*
|
|
7
|
-
* - Keeps the tree structure (no flattening) and mirrors PromptElement shape.
|
|
8
|
-
* - Adds per-node token counts (using renderer.tokenString) and stable hashes.
|
|
9
|
-
* - Excludes strategy/context from the snapshot; this is for observability only.
|
|
10
|
-
*/
|
|
11
|
-
export function createSnapshot(element, { tokenizer, renderer = markdownRenderer }) {
|
|
12
|
-
const root = snapshotElement(element, tokenizer, renderer);
|
|
13
|
-
return {
|
|
14
|
-
root,
|
|
15
|
-
totalTokens: root.tokens,
|
|
16
|
-
hash: root.hash,
|
|
17
|
-
};
|
|
18
|
-
}
|
|
19
|
-
export function createSnapshotHooks({ tokenizer, renderer, onSnapshot, }) {
|
|
20
|
-
return {
|
|
21
|
-
onFitComplete: async (event) => {
|
|
22
|
-
if (!event.result) {
|
|
23
|
-
return;
|
|
24
|
-
}
|
|
25
|
-
const snapshot = createSnapshot(event.result, {
|
|
26
|
-
tokenizer,
|
|
27
|
-
...(renderer && { renderer }),
|
|
28
|
-
});
|
|
29
|
-
await onSnapshot(snapshot);
|
|
30
|
-
},
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
export function diffSnapshots(before, after) {
|
|
34
|
-
const added = [];
|
|
35
|
-
const removed = [];
|
|
36
|
-
const changed = [];
|
|
37
|
-
compareNodes(before.root, after.root, [], added, removed, changed);
|
|
38
|
-
return { added, removed, changed };
|
|
39
|
-
}
|
|
40
|
-
function compareNodes(before, after, path, added, removed, changed) {
|
|
41
|
-
if (handlePrimitives(before, after, path, added, removed, changed)) {
|
|
42
|
-
return;
|
|
43
|
-
}
|
|
44
|
-
const beforeElement = before;
|
|
45
|
-
const afterElement = after;
|
|
46
|
-
recordElementChange(beforeElement, afterElement, path, changed);
|
|
47
|
-
compareChildren(beforeElement, afterElement, path, added, removed, changed);
|
|
48
|
-
}
|
|
49
|
-
function handlePrimitives(before, after, path, added, removed, changed) {
|
|
50
|
-
if (typeof before === "string" && typeof after === "string") {
|
|
51
|
-
if (before !== after) {
|
|
52
|
-
changed.push({ path, before, after });
|
|
53
|
-
}
|
|
54
|
-
return true;
|
|
55
|
-
}
|
|
56
|
-
if (typeof before === "string") {
|
|
57
|
-
removed.push({ path, node: before });
|
|
58
|
-
added.push({ path, node: after });
|
|
59
|
-
return true;
|
|
60
|
-
}
|
|
61
|
-
if (typeof after === "string") {
|
|
62
|
-
removed.push({ path, node: before });
|
|
63
|
-
added.push({ path, node: after });
|
|
64
|
-
return true;
|
|
65
|
-
}
|
|
66
|
-
return false;
|
|
67
|
-
}
|
|
68
|
-
function recordElementChange(before, after, path, changed) {
|
|
69
|
-
const identityChanged = elementIdentity(before) !== elementIdentity(after);
|
|
70
|
-
const contentChanged = before.hash !== after.hash || before.tokens !== after.tokens;
|
|
71
|
-
if (identityChanged || contentChanged) {
|
|
72
|
-
changed.push({ path, before, after });
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
function compareChildren(before, after, path, added, removed, changed) {
|
|
76
|
-
const maxChildren = Math.max(before.children.length, after.children.length);
|
|
77
|
-
for (let i = 0; i < maxChildren; i++) {
|
|
78
|
-
const beforeChild = before.children[i];
|
|
79
|
-
const afterChild = after.children[i];
|
|
80
|
-
const childPath = [...path, i];
|
|
81
|
-
if (beforeChild === undefined && afterChild !== undefined) {
|
|
82
|
-
added.push({ path: childPath, node: afterChild });
|
|
83
|
-
continue;
|
|
84
|
-
}
|
|
85
|
-
if (beforeChild !== undefined && afterChild === undefined) {
|
|
86
|
-
removed.push({ path: childPath, node: beforeChild });
|
|
87
|
-
continue;
|
|
88
|
-
}
|
|
89
|
-
if (beforeChild !== undefined && afterChild !== undefined) {
|
|
90
|
-
compareNodes(beforeChild, afterChild, childPath, added, removed, changed);
|
|
91
|
-
}
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
function elementIdentity(element) {
|
|
95
|
-
return element.id ?? `${element.kind ?? "region"}:${element.priority}`;
|
|
96
|
-
}
|
|
97
|
-
function snapshotElement(element, tokenizer, renderer) {
|
|
98
|
-
const childSnapshots = element.children.map((child) => typeof child === "string"
|
|
99
|
-
? child
|
|
100
|
-
: snapshotElement(child, tokenizer, renderer));
|
|
101
|
-
const contentProjection = renderer.tokenString(element);
|
|
102
|
-
const tokens = tokenizer(contentProjection);
|
|
103
|
-
const childHashes = childSnapshots.map((child) => typeof child === "string" ? hashString(child) : child.hash);
|
|
104
|
-
const hasToolId = element.kind === "tool-call" || element.kind === "tool-result";
|
|
105
|
-
const role = element.kind === "message" ? element.role : undefined;
|
|
106
|
-
const text = element.kind === "reasoning" ? element.text : undefined;
|
|
107
|
-
const toolCallId = hasToolId ? element.toolCallId : undefined;
|
|
108
|
-
const toolName = hasToolId ? element.toolName : undefined;
|
|
109
|
-
const hash = hashElement({
|
|
110
|
-
kind: element.kind,
|
|
111
|
-
priority: element.priority,
|
|
112
|
-
...(role !== undefined ? { role } : {}),
|
|
113
|
-
...(text !== undefined ? { text } : {}),
|
|
114
|
-
...(toolCallId !== undefined ? { toolCallId } : {}),
|
|
115
|
-
...(toolName !== undefined ? { toolName } : {}),
|
|
116
|
-
...(element.id !== undefined ? { id: element.id } : {}),
|
|
117
|
-
tokens,
|
|
118
|
-
childHashes,
|
|
119
|
-
});
|
|
120
|
-
return {
|
|
121
|
-
...element,
|
|
122
|
-
children: childSnapshots,
|
|
123
|
-
tokens,
|
|
124
|
-
hash,
|
|
125
|
-
};
|
|
126
|
-
}
|
|
127
|
-
function hashElement(input) {
|
|
128
|
-
const payload = {
|
|
129
|
-
kind: input.kind ?? "region",
|
|
130
|
-
priority: input.priority,
|
|
131
|
-
role: input.role,
|
|
132
|
-
text: input.text,
|
|
133
|
-
toolCallId: input.toolCallId,
|
|
134
|
-
toolName: input.toolName,
|
|
135
|
-
id: input.id,
|
|
136
|
-
tokens: input.tokens,
|
|
137
|
-
children: input.childHashes,
|
|
138
|
-
};
|
|
139
|
-
return hashString(JSON.stringify(payload));
|
|
140
|
-
}
|
|
141
|
-
function hashString(value) {
|
|
142
|
-
return bytesToHex(sha256(utf8ToBytes(value)));
|
|
143
|
-
}
|
|
144
|
-
//# sourceMappingURL=snapshot.js.map
|
package/dist/snapshot.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"snapshot.js","sourceRoot":"","sources":["../src/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,sBAAsB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAE9D,OAAO,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAoCxD;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAC5B,OAAsB,EACtB,EAAE,SAAS,EAAE,QAAQ,GAAG,gBAAgB,EAAmB;IAE3D,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC3D,OAAO;QACL,IAAI;QACJ,WAAW,EAAE,IAAI,CAAC,MAAM;QACxB,IAAI,EAAE,IAAI,CAAC,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,EAClC,SAAS,EACT,QAAQ,EACR,UAAU,GACW;IACrB,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;YAC7B,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,EAAE;gBAC5C,SAAS;gBACT,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;aAC9B,CAAC,CAAC;YACH,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC7B,CAAC;KACF,CAAC;AACJ,CAAC;AAmBD,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,KAAe;IAC7D,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,MAAM,OAAO,GAAiB,EAAE,CAAC;IAEjC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAEnE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,YAAY,CACnB,MAAqB,EACrB,KAAoB,EACpB,IAAc,EACd,KAAkB,EAClB,OAAoB,EACpB,OAAqB;IAErB,IAAI,gBAAgB,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,CAAC;QACnE,OAAO;IACT,CAAC;IAED,MAAM,aAAa,GAAG,MAAyB,CAAC;IAChD,MAAM,YAAY,GAAG,KAAwB,CAAC;IAE9C,mBAAmB,CAAC,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;IAChE,eAAe,CAAC,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CACvB,MAAqB,EACrB,KAAoB,EACpB,IAAuB,EACvB,KAAkB,EAClB,OAAoB,EACpB,OAAqB;IAErB,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC5D,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACxC,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAuB,EACvB,KAAsB,EACtB,IAAuB,EACvB,OAAqB;IAErB,MAAM,eAAe,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,eAAe,CAAC,KAAK,CAAC,CAAC;IAC3E,MAAM,cAAc,GAClB,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC;IAC/D,IAAI,eAAe,IAAI,cAAc,EAAE,CAAC;QACtC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACxC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CACtB,MAAuB,EACvB,KAAsB,EACtB,IAAc,EACd,KAAkB,EAClB,OAAoB,EACpB,OAAqB;IAErB,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC5E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvC,MAAM,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACrC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC;QAE/B,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC1D,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;YAClD,SAAS;QACX,CAAC;QACD,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC1D,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;YACrD,SAAS;QACX,CAAC;QACD,IAAI,WAAW,KAAK,SAAS,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC1D,YAAY,CAAC,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAAwB;IAC/C,OAAO,OAAO,CAAC,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,QAAQ,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,eAAe,CACtB,OAAsB,EACtB,SAAoB,EACpB,QAAiC;IAEjC,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CACzC,CAAC,KAAkB,EAAiB,EAAE,CACpC,OAAO,KAAK,KAAK,QAAQ;QACvB,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,CAClD,CAAC;IAEF,MAAM,iBAAiB,GAAG,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACxD,MAAM,MAAM,GAAG,SAAS,CAAC,iBAAiB,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,KAAoB,EAAU,EAAE,CACtE,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAC3D,CAAC;IAEF,MAAM,SAAS,GACb,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,CAAC;IAEjE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACnE,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;IACrE,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9D,MAAM,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IAE1D,MAAM,IAAI,GAAG,WAAW,CAAC;QACvB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,GAAG,CAAC,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,MAAM;QACN,WAAW;KACZ,CAAC,CAAC;IAEH,OAAO;QACL,GAAG,OAAO;QACV,QAAQ,EAAE,cAAc;QACxB,MAAM;QACN,IAAI;KACL,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,KAUpB;IACC,MAAM,OAAO,GAAG;QACd,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,QAAQ;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,QAAQ,EAAE,KAAK,CAAC,WAAW;KAC5B,CAAC;IACF,OAAO,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAChD,CAAC"}
|
package/dist/tokenizers.d.ts
DELETED
|
@@ -1,14 +0,0 @@
|
|
|
1
|
-
import type { Tokenizer } from "./types";
|
|
2
|
-
/**
|
|
3
|
-
* Rough heuristic tokenizer: ~4 characters per token.
|
|
4
|
-
*
|
|
5
|
-
* Suitable as a default estimate when no model-specific tokenizer is available.
|
|
6
|
-
* For accurate budgeting, pass a model-aware tokenizer (e.g. tiktoken).
|
|
7
|
-
*/
|
|
8
|
-
export declare const approximateTokenizer: Tokenizer;
|
|
9
|
-
/**
|
|
10
|
-
* Best-effort tokenizer using tiktoken. Falls back to approximate if the encoding
|
|
11
|
-
* is unavailable.
|
|
12
|
-
*/
|
|
13
|
-
export declare function tiktokenTokenizer(modelHint?: string): Tokenizer;
|
|
14
|
-
//# sourceMappingURL=tokenizers.d.ts.map
|
package/dist/tokenizers.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizers.d.ts","sourceRoot":"","sources":["../src/tokenizers.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,EAAE,SACP,CAAC;AA6B7B;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAO/D"}
|
package/dist/tokenizers.js
DELETED
|
@@ -1,45 +0,0 @@
|
|
|
1
|
-
import { encodingForModel, getEncoding, } from "js-tiktoken";
|
|
2
|
-
/**
|
|
3
|
-
* Rough heuristic tokenizer: ~4 characters per token.
|
|
4
|
-
*
|
|
5
|
-
* Suitable as a default estimate when no model-specific tokenizer is available.
|
|
6
|
-
* For accurate budgeting, pass a model-aware tokenizer (e.g. tiktoken).
|
|
7
|
-
*/
|
|
8
|
-
export const approximateTokenizer = (text) => Math.ceil(text.length / 4);
|
|
9
|
-
const tiktokenCache = new Map();
|
|
10
|
-
function getTiktokenEncoder(modelHint) {
|
|
11
|
-
const cacheKey = modelHint ?? "cl100k_base";
|
|
12
|
-
const cached = tiktokenCache.get(cacheKey);
|
|
13
|
-
if (cached) {
|
|
14
|
-
return cached;
|
|
15
|
-
}
|
|
16
|
-
try {
|
|
17
|
-
const encoder = modelHint
|
|
18
|
-
? encodingForModel(modelHint)
|
|
19
|
-
: getEncoding("cl100k_base");
|
|
20
|
-
tiktokenCache.set(cacheKey, encoder);
|
|
21
|
-
return encoder;
|
|
22
|
-
}
|
|
23
|
-
catch (error) {
|
|
24
|
-
// Fall through to allow approximate fallback
|
|
25
|
-
const debugEnv = process.env
|
|
26
|
-
.DEBUG;
|
|
27
|
-
if (debugEnv?.includes("cria:tokenizer")) {
|
|
28
|
-
// eslint-disable-next-line no-console
|
|
29
|
-
console.warn("Falling back to approximate tokenizer:", error);
|
|
30
|
-
}
|
|
31
|
-
return null;
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
/**
|
|
35
|
-
* Best-effort tokenizer using tiktoken. Falls back to approximate if the encoding
|
|
36
|
-
* is unavailable.
|
|
37
|
-
*/
|
|
38
|
-
export function tiktokenTokenizer(modelHint) {
|
|
39
|
-
const encoder = getTiktokenEncoder(modelHint);
|
|
40
|
-
if (!encoder) {
|
|
41
|
-
return approximateTokenizer;
|
|
42
|
-
}
|
|
43
|
-
return (text) => encoder.encode(text).length;
|
|
44
|
-
}
|
|
45
|
-
//# sourceMappingURL=tokenizers.js.map
|
package/dist/tokenizers.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"tokenizers.js","sourceRoot":"","sources":["../src/tokenizers.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,WAAW,GAGZ,MAAM,aAAa,CAAC;AAGrB;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAc,CAAC,IAAI,EAAE,EAAE,CACtD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAE7B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAC;AAElD,SAAS,kBAAkB,CAAC,SAAkB;IAC5C,MAAM,QAAQ,GAAG,SAAS,IAAI,aAAa,CAAC;IAC5C,MAAM,MAAM,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC3C,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,SAAS;YACvB,CAAC,CAAC,gBAAgB,CAAC,SAA0B,CAAC;YAC9C,CAAC,CAAC,WAAW,CAAC,aAAa,CAAC,CAAC;QAC/B,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACrC,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,6CAA6C;QAC7C,MAAM,QAAQ,GAAI,OAAO,CAAC,GAA8C;aACrE,KAAK,CAAC;QACT,IAAI,QAAQ,EAAE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YACzC,sCAAsC;YACtC,OAAO,CAAC,IAAI,CAAC,wCAAwC,EAAE,KAAK,CAAC,CAAC;QAChE,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAkB;IAClD,MAAM,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,CAAC;IAC9C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;AACvD,CAAC"}
|