@sohqureshi/tokenwise 1.0.7 → 1.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -5
- package/dist/{chunk-IOMOVRC7.js → chunk-2TDVL6BJ.js} +2 -2
- package/dist/chunk-BN3L7TUV.js +241 -0
- package/dist/{chunk-P6KXISNT.js → chunk-FBJ3DHS6.js} +2 -2
- package/dist/chunk-L3HY6AWL.js +69 -0
- package/dist/cli.cjs +169 -19
- package/dist/cli.js +5 -5
- package/dist/core/analyze.cjs +32 -18
- package/dist/core/analyze.js +2 -2
- package/dist/core/natural.cjs +136 -0
- package/dist/core/natural.js +1 -1
- package/dist/core/token.cjs +31 -17
- package/dist/core/token.d.cts +2 -8
- package/dist/core/token.d.ts +2 -8
- package/dist/core/token.js +1 -1
- package/dist/index.cjs +168 -18
- package/dist/index.js +4 -4
- package/package.json +1 -1
- package/dist/chunk-D4CFTFM3.js +0 -105
- package/dist/chunk-U433WUUT.js +0 -55
package/dist/core/analyze.cjs
CHANGED
|
@@ -150,8 +150,18 @@ function serializeForTokenEstimate(value) {
|
|
|
150
150
|
function estimateTokensHeuristic(value) {
|
|
151
151
|
return Math.ceil(serializeForTokenEstimate(value).length / 4);
|
|
152
152
|
}
|
|
153
|
+
function expectedEncodingForModel(model) {
|
|
154
|
+
const normalizedModel = model.toLowerCase();
|
|
155
|
+
if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
|
|
156
|
+
return "o200k_base";
|
|
157
|
+
}
|
|
158
|
+
if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
|
|
159
|
+
return "r50k_base";
|
|
160
|
+
}
|
|
161
|
+
return "cl100k_base";
|
|
162
|
+
}
|
|
153
163
|
function estimateTokensWithMeta(value, options = {}) {
|
|
154
|
-
const { exact =
|
|
164
|
+
const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
|
|
155
165
|
const heuristic = estimateTokensHeuristic(value);
|
|
156
166
|
const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
|
|
157
167
|
if (!exact) {
|
|
@@ -160,36 +170,40 @@ function estimateTokensWithMeta(value, options = {}) {
|
|
|
160
170
|
try {
|
|
161
171
|
const tiktoken = require2("tiktoken");
|
|
162
172
|
const encoder = tiktoken.encoding_for_model(model);
|
|
163
|
-
const
|
|
164
|
-
|
|
165
|
-
let encodingName = null;
|
|
166
|
-
if (encoder.name) encodingName = encoder.name;
|
|
173
|
+
const count = encoder.encode(serializeForTokenEstimate(value)).length;
|
|
174
|
+
let encodingName = encoder.name;
|
|
167
175
|
if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
|
|
168
176
|
try {
|
|
169
177
|
encodingName = tiktoken.model_to_encoding(model);
|
|
170
|
-
} catch
|
|
171
|
-
encodingName =
|
|
178
|
+
} catch {
|
|
179
|
+
encodingName = void 0;
|
|
172
180
|
}
|
|
173
181
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
+
encodingName ?? (encodingName = expectedEncodingForModel(model));
|
|
183
|
+
encoder.free?.();
|
|
184
|
+
return {
|
|
185
|
+
count,
|
|
186
|
+
estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
|
|
187
|
+
};
|
|
188
|
+
} catch {
|
|
189
|
+
const expectedEncoding = expectedEncodingForModel(model);
|
|
182
190
|
if (!fallbackToHeuristic) {
|
|
183
|
-
return {
|
|
191
|
+
return {
|
|
192
|
+
count: heuristic,
|
|
193
|
+
estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
|
|
194
|
+
};
|
|
184
195
|
}
|
|
185
|
-
return {
|
|
196
|
+
return {
|
|
197
|
+
count: heuristic,
|
|
198
|
+
estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
|
|
199
|
+
};
|
|
186
200
|
}
|
|
187
201
|
}
|
|
188
202
|
|
|
189
203
|
// src/core/analyze.ts
|
|
190
204
|
function analyze(input, options = {}) {
|
|
191
205
|
const {
|
|
192
|
-
exact =
|
|
206
|
+
exact = true,
|
|
193
207
|
model = "gpt-4o-mini",
|
|
194
208
|
fallbackToHeuristic = true
|
|
195
209
|
} = options;
|
package/dist/core/analyze.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
analyze
|
|
3
|
-
} from "../chunk-
|
|
3
|
+
} from "../chunk-2TDVL6BJ.js";
|
|
4
4
|
import "../chunk-6VRLDRJK.js";
|
|
5
5
|
import "../chunk-XE36GLJP.js";
|
|
6
6
|
import "../chunk-L7BC62MT.js";
|
|
7
7
|
import "../chunk-ZD536GZF.js";
|
|
8
|
-
import "../chunk-
|
|
8
|
+
import "../chunk-L3HY6AWL.js";
|
|
9
9
|
export {
|
|
10
10
|
analyze
|
|
11
11
|
};
|
package/dist/core/natural.cjs
CHANGED
|
@@ -42,6 +42,15 @@ function toNatural(data, depth = 0) {
|
|
|
42
42
|
return String(data);
|
|
43
43
|
}
|
|
44
44
|
function buildContextualStory(obj, depth = 0) {
|
|
45
|
+
const semanticStory = buildSemanticStory(obj);
|
|
46
|
+
if (semanticStory) return semanticStory;
|
|
47
|
+
const nestedEntity = Object.values(obj).find((value) => {
|
|
48
|
+
return isPlainObject(value) && buildSemanticStory(value) !== null;
|
|
49
|
+
});
|
|
50
|
+
if (nestedEntity) {
|
|
51
|
+
const nestedStory = buildSemanticStory(nestedEntity);
|
|
52
|
+
if (nestedStory) return nestedStory;
|
|
53
|
+
}
|
|
45
54
|
let name = obj.name || obj.userName || obj.user;
|
|
46
55
|
if (typeof name === "object" && name !== null && name.name) {
|
|
47
56
|
name = name.name;
|
|
@@ -66,6 +75,133 @@ function buildContextualStory(obj, depth = 0) {
|
|
|
66
75
|
}
|
|
67
76
|
return story + ".";
|
|
68
77
|
}
|
|
78
|
+
function buildSemanticStory(obj) {
|
|
79
|
+
if (typeof obj.holderName === "string" && typeof obj.policyNumber === "string") {
|
|
80
|
+
const subject2 = `${obj.holderName} has policy ${obj.policyNumber}`;
|
|
81
|
+
const details = [
|
|
82
|
+
typeof obj.planType === "string" ? `a ${obj.planType} plan` : null,
|
|
83
|
+
typeof obj.premiumAmount === "number" ? `with a premium of ${formatNumber(obj.premiumAmount)}` : null
|
|
84
|
+
].filter((value) => value !== null);
|
|
85
|
+
const sentences = [`${subject2}${details.length ? `, ${details.join(", ")}` : ""}.`];
|
|
86
|
+
if (isPlainObject(obj.claim)) {
|
|
87
|
+
const claim = describeClaim(obj.claim);
|
|
88
|
+
if (claim) sentences.push(claim);
|
|
89
|
+
}
|
|
90
|
+
if (Array.isArray(obj.dependents) && obj.dependents.length > 0) {
|
|
91
|
+
sentences.push(`${obj.holderName}'s dependents are ${joinNaturalList(obj.dependents.map(String))}.`);
|
|
92
|
+
}
|
|
93
|
+
return sentences.join(" ");
|
|
94
|
+
}
|
|
95
|
+
const subject = findSubject(obj);
|
|
96
|
+
const stateEntry = findSemanticEntry(obj, [
|
|
97
|
+
"status",
|
|
98
|
+
"state",
|
|
99
|
+
"condition",
|
|
100
|
+
"stage",
|
|
101
|
+
"role",
|
|
102
|
+
"type",
|
|
103
|
+
"category",
|
|
104
|
+
"classification",
|
|
105
|
+
"priority",
|
|
106
|
+
"phase",
|
|
107
|
+
"mode",
|
|
108
|
+
"availability",
|
|
109
|
+
"outcome",
|
|
110
|
+
"result",
|
|
111
|
+
"health",
|
|
112
|
+
"progress",
|
|
113
|
+
"visibility",
|
|
114
|
+
"access",
|
|
115
|
+
"membership",
|
|
116
|
+
"sentiment",
|
|
117
|
+
"severity"
|
|
118
|
+
]);
|
|
119
|
+
if (subject && stateEntry && typeof stateEntry[1] === "string") {
|
|
120
|
+
const details = Object.entries(obj).filter(([key, value]) => key !== stateEntry[0] && key !== subject.key && shouldDescribeSemantically(key, value)).map(([key, value]) => `${camelToWords(key)} ${formatSemanticValue(value)}`);
|
|
121
|
+
return `${subject.value} is ${stateEntry[1]}${details.length ? ` and has ${details.join(", ")}` : ""}.`;
|
|
122
|
+
}
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
function findSubject(obj) {
|
|
126
|
+
const subjectKeys = [
|
|
127
|
+
"name",
|
|
128
|
+
"title",
|
|
129
|
+
"label",
|
|
130
|
+
"displayName",
|
|
131
|
+
"entityName",
|
|
132
|
+
"fullName",
|
|
133
|
+
"userName",
|
|
134
|
+
"username",
|
|
135
|
+
"personName",
|
|
136
|
+
"customerName",
|
|
137
|
+
"clientName",
|
|
138
|
+
"ownerName",
|
|
139
|
+
"accountName",
|
|
140
|
+
"companyName",
|
|
141
|
+
"organizationName",
|
|
142
|
+
"teamName",
|
|
143
|
+
"departmentName",
|
|
144
|
+
"projectName",
|
|
145
|
+
"productName",
|
|
146
|
+
"serviceName",
|
|
147
|
+
"resourceName",
|
|
148
|
+
"fileName",
|
|
149
|
+
"deviceName",
|
|
150
|
+
"hostName",
|
|
151
|
+
"applicationName",
|
|
152
|
+
"appName",
|
|
153
|
+
"taskName",
|
|
154
|
+
"eventName",
|
|
155
|
+
"itemName",
|
|
156
|
+
"orderName",
|
|
157
|
+
"patientName",
|
|
158
|
+
"employeeName"
|
|
159
|
+
];
|
|
160
|
+
const entries = Object.entries(obj);
|
|
161
|
+
for (const key of subjectKeys) {
|
|
162
|
+
if (typeof obj[key] === "string" && obj[key].trim()) {
|
|
163
|
+
return { key, value: obj[key].trim() };
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
const nameEntry = entries.find(([key, value]) => {
|
|
167
|
+
return key.toLowerCase().includes("name") && typeof value === "string" && value.trim().length > 0;
|
|
168
|
+
});
|
|
169
|
+
if (nameEntry) {
|
|
170
|
+
return { key: nameEntry[0], value: nameEntry[1].trim() };
|
|
171
|
+
}
|
|
172
|
+
return null;
|
|
173
|
+
}
|
|
174
|
+
function findSemanticEntry(obj, keys) {
|
|
175
|
+
const entries = Object.entries(obj);
|
|
176
|
+
const exactEntry = entries.find(([key]) => keys.includes(key));
|
|
177
|
+
if (exactEntry) return exactEntry;
|
|
178
|
+
return entries.find(([key]) => {
|
|
179
|
+
const normalizedKey = key.toLowerCase();
|
|
180
|
+
return keys.some((semanticKey) => normalizedKey.includes(semanticKey.toLowerCase()));
|
|
181
|
+
}) ?? null;
|
|
182
|
+
}
|
|
183
|
+
function shouldDescribeSemantically(key, value) {
|
|
184
|
+
return (typeof value === "string" || typeof value === "number" || typeof value === "boolean") && !["id", "timestamp", "createdAt", "updatedAt", "apiKey", "debug"].includes(key);
|
|
185
|
+
}
|
|
186
|
+
function formatSemanticValue(value) {
|
|
187
|
+
if (typeof value === "number") return formatNumber(value);
|
|
188
|
+
if (typeof value === "boolean") return value ? "enabled" : "disabled";
|
|
189
|
+
return String(value);
|
|
190
|
+
}
|
|
191
|
+
function describeClaim(claim) {
|
|
192
|
+
const parts = [];
|
|
193
|
+
if (typeof claim.status === "string") parts.push(`The claim is ${claim.status}`);
|
|
194
|
+
if (typeof claim.requestedAmount === "number") {
|
|
195
|
+
parts.push(`for ${formatNumber(claim.requestedAmount)}`);
|
|
196
|
+
}
|
|
197
|
+
if (typeof claim.claimNumber === "string") {
|
|
198
|
+
parts.push(`(reference ${claim.claimNumber})`);
|
|
199
|
+
}
|
|
200
|
+
return parts.length ? `${parts.join(" ")}.` : null;
|
|
201
|
+
}
|
|
202
|
+
function formatNumber(value) {
|
|
203
|
+
return new Intl.NumberFormat("en-US").format(value);
|
|
204
|
+
}
|
|
69
205
|
function formatPropertyClause(key, value, depth) {
|
|
70
206
|
const naturalKey = camelToWords(key);
|
|
71
207
|
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
package/dist/core/natural.js
CHANGED
package/dist/core/token.cjs
CHANGED
|
@@ -36,11 +36,21 @@ function serializeForTokenEstimate(value) {
|
|
|
36
36
|
function estimateTokensHeuristic(value) {
|
|
37
37
|
return Math.ceil(serializeForTokenEstimate(value).length / 4);
|
|
38
38
|
}
|
|
39
|
+
function expectedEncodingForModel(model) {
|
|
40
|
+
const normalizedModel = model.toLowerCase();
|
|
41
|
+
if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
|
|
42
|
+
return "o200k_base";
|
|
43
|
+
}
|
|
44
|
+
if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
|
|
45
|
+
return "r50k_base";
|
|
46
|
+
}
|
|
47
|
+
return "cl100k_base";
|
|
48
|
+
}
|
|
39
49
|
function estimateTokens(value, options = {}) {
|
|
40
50
|
return estimateTokensWithMeta(value, options).count;
|
|
41
51
|
}
|
|
42
52
|
function estimateTokensWithMeta(value, options = {}) {
|
|
43
|
-
const { exact =
|
|
53
|
+
const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
|
|
44
54
|
const heuristic = estimateTokensHeuristic(value);
|
|
45
55
|
const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
|
|
46
56
|
if (!exact) {
|
|
@@ -49,29 +59,33 @@ function estimateTokensWithMeta(value, options = {}) {
|
|
|
49
59
|
try {
|
|
50
60
|
const tiktoken = require2("tiktoken");
|
|
51
61
|
const encoder = tiktoken.encoding_for_model(model);
|
|
52
|
-
const
|
|
53
|
-
|
|
54
|
-
let encodingName = null;
|
|
55
|
-
if (encoder.name) encodingName = encoder.name;
|
|
62
|
+
const count = encoder.encode(serializeForTokenEstimate(value)).length;
|
|
63
|
+
let encodingName = encoder.name;
|
|
56
64
|
if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
|
|
57
65
|
try {
|
|
58
66
|
encodingName = tiktoken.model_to_encoding(model);
|
|
59
|
-
} catch
|
|
60
|
-
encodingName =
|
|
67
|
+
} catch {
|
|
68
|
+
encodingName = void 0;
|
|
61
69
|
}
|
|
62
70
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
+
encodingName ?? (encodingName = expectedEncodingForModel(model));
|
|
72
|
+
encoder.free?.();
|
|
73
|
+
return {
|
|
74
|
+
count,
|
|
75
|
+
estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
|
|
76
|
+
};
|
|
77
|
+
} catch {
|
|
78
|
+
const expectedEncoding = expectedEncodingForModel(model);
|
|
71
79
|
if (!fallbackToHeuristic) {
|
|
72
|
-
return {
|
|
80
|
+
return {
|
|
81
|
+
count: heuristic,
|
|
82
|
+
estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
|
|
83
|
+
};
|
|
73
84
|
}
|
|
74
|
-
return {
|
|
85
|
+
return {
|
|
86
|
+
count: heuristic,
|
|
87
|
+
estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
|
|
88
|
+
};
|
|
75
89
|
}
|
|
76
90
|
}
|
|
77
91
|
// Annotate the CommonJS export names for ESM import in node:
|
package/dist/core/token.d.cts
CHANGED
|
@@ -4,20 +4,14 @@ interface TokenEstimateOptions {
|
|
|
4
4
|
fallbackToHeuristic?: boolean;
|
|
5
5
|
}
|
|
6
6
|
declare function serializeForTokenEstimate(value: unknown): string;
|
|
7
|
-
/**
|
|
8
|
-
* Estimates tokens using a transparent four-characters-per-token heuristic.
|
|
9
|
-
* JSON values are serialized first, matching the compact form normally sent
|
|
10
|
-
* to an API. Set exact=true to attempt a model-specific tokenizer count when
|
|
11
|
-
* the runtime has the tokenizer installed.
|
|
12
|
-
*/
|
|
13
7
|
type TokenEstimateResult = {
|
|
14
8
|
count: number;
|
|
15
9
|
estimator: string;
|
|
16
10
|
};
|
|
17
11
|
declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
|
|
18
12
|
/**
|
|
19
|
-
* Returns
|
|
20
|
-
*
|
|
13
|
+
* Returns an exact model tokenizer count by default. Set exact=false to use the
|
|
14
|
+
* lightweight four-characters-per-token fallback explicitly.
|
|
21
15
|
*/
|
|
22
16
|
declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
|
|
23
17
|
|
package/dist/core/token.d.ts
CHANGED
|
@@ -4,20 +4,14 @@ interface TokenEstimateOptions {
|
|
|
4
4
|
fallbackToHeuristic?: boolean;
|
|
5
5
|
}
|
|
6
6
|
declare function serializeForTokenEstimate(value: unknown): string;
|
|
7
|
-
/**
|
|
8
|
-
* Estimates tokens using a transparent four-characters-per-token heuristic.
|
|
9
|
-
* JSON values are serialized first, matching the compact form normally sent
|
|
10
|
-
* to an API. Set exact=true to attempt a model-specific tokenizer count when
|
|
11
|
-
* the runtime has the tokenizer installed.
|
|
12
|
-
*/
|
|
13
7
|
type TokenEstimateResult = {
|
|
14
8
|
count: number;
|
|
15
9
|
estimator: string;
|
|
16
10
|
};
|
|
17
11
|
declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
|
|
18
12
|
/**
|
|
19
|
-
* Returns
|
|
20
|
-
*
|
|
13
|
+
* Returns an exact model tokenizer count by default. Set exact=false to use the
|
|
14
|
+
* lightweight four-characters-per-token fallback explicitly.
|
|
21
15
|
*/
|
|
22
16
|
declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
|
|
23
17
|
|
package/dist/core/token.js
CHANGED
package/dist/index.cjs
CHANGED
|
@@ -159,11 +159,21 @@ function serializeForTokenEstimate(value) {
|
|
|
159
159
|
function estimateTokensHeuristic(value) {
|
|
160
160
|
return Math.ceil(serializeForTokenEstimate(value).length / 4);
|
|
161
161
|
}
|
|
162
|
+
function expectedEncodingForModel(model) {
|
|
163
|
+
const normalizedModel = model.toLowerCase();
|
|
164
|
+
if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
|
|
165
|
+
return "o200k_base";
|
|
166
|
+
}
|
|
167
|
+
if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
|
|
168
|
+
return "r50k_base";
|
|
169
|
+
}
|
|
170
|
+
return "cl100k_base";
|
|
171
|
+
}
|
|
162
172
|
function estimateTokens(value, options = {}) {
|
|
163
173
|
return estimateTokensWithMeta(value, options).count;
|
|
164
174
|
}
|
|
165
175
|
function estimateTokensWithMeta(value, options = {}) {
|
|
166
|
-
const { exact =
|
|
176
|
+
const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
|
|
167
177
|
const heuristic = estimateTokensHeuristic(value);
|
|
168
178
|
const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
|
|
169
179
|
if (!exact) {
|
|
@@ -172,36 +182,40 @@ function estimateTokensWithMeta(value, options = {}) {
|
|
|
172
182
|
try {
|
|
173
183
|
const tiktoken = require2("tiktoken");
|
|
174
184
|
const encoder = tiktoken.encoding_for_model(model);
|
|
175
|
-
const
|
|
176
|
-
|
|
177
|
-
let encodingName = null;
|
|
178
|
-
if (encoder.name) encodingName = encoder.name;
|
|
185
|
+
const count = encoder.encode(serializeForTokenEstimate(value)).length;
|
|
186
|
+
let encodingName = encoder.name;
|
|
179
187
|
if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
|
|
180
188
|
try {
|
|
181
189
|
encodingName = tiktoken.model_to_encoding(model);
|
|
182
|
-
} catch
|
|
183
|
-
encodingName =
|
|
190
|
+
} catch {
|
|
191
|
+
encodingName = void 0;
|
|
184
192
|
}
|
|
185
193
|
}
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
+
encodingName ?? (encodingName = expectedEncodingForModel(model));
|
|
195
|
+
encoder.free?.();
|
|
196
|
+
return {
|
|
197
|
+
count,
|
|
198
|
+
estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
|
|
199
|
+
};
|
|
200
|
+
} catch {
|
|
201
|
+
const expectedEncoding = expectedEncodingForModel(model);
|
|
194
202
|
if (!fallbackToHeuristic) {
|
|
195
|
-
return {
|
|
203
|
+
return {
|
|
204
|
+
count: heuristic,
|
|
205
|
+
estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
|
|
206
|
+
};
|
|
196
207
|
}
|
|
197
|
-
return {
|
|
208
|
+
return {
|
|
209
|
+
count: heuristic,
|
|
210
|
+
estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
|
|
211
|
+
};
|
|
198
212
|
}
|
|
199
213
|
}
|
|
200
214
|
|
|
201
215
|
// src/core/analyze.ts
|
|
202
216
|
function analyze(input, options = {}) {
|
|
203
217
|
const {
|
|
204
|
-
exact =
|
|
218
|
+
exact = true,
|
|
205
219
|
model = "gpt-4o-mini",
|
|
206
220
|
fallbackToHeuristic = true
|
|
207
221
|
} = options;
|
|
@@ -280,6 +294,15 @@ function toNatural(data, depth = 0) {
|
|
|
280
294
|
return String(data);
|
|
281
295
|
}
|
|
282
296
|
function buildContextualStory(obj, depth = 0) {
|
|
297
|
+
const semanticStory = buildSemanticStory(obj);
|
|
298
|
+
if (semanticStory) return semanticStory;
|
|
299
|
+
const nestedEntity = Object.values(obj).find((value) => {
|
|
300
|
+
return isPlainObject2(value) && buildSemanticStory(value) !== null;
|
|
301
|
+
});
|
|
302
|
+
if (nestedEntity) {
|
|
303
|
+
const nestedStory = buildSemanticStory(nestedEntity);
|
|
304
|
+
if (nestedStory) return nestedStory;
|
|
305
|
+
}
|
|
283
306
|
let name = obj.name || obj.userName || obj.user;
|
|
284
307
|
if (typeof name === "object" && name !== null && name.name) {
|
|
285
308
|
name = name.name;
|
|
@@ -304,6 +327,133 @@ function buildContextualStory(obj, depth = 0) {
|
|
|
304
327
|
}
|
|
305
328
|
return story + ".";
|
|
306
329
|
}
|
|
330
|
+
function buildSemanticStory(obj) {
|
|
331
|
+
if (typeof obj.holderName === "string" && typeof obj.policyNumber === "string") {
|
|
332
|
+
const subject2 = `${obj.holderName} has policy ${obj.policyNumber}`;
|
|
333
|
+
const details = [
|
|
334
|
+
typeof obj.planType === "string" ? `a ${obj.planType} plan` : null,
|
|
335
|
+
typeof obj.premiumAmount === "number" ? `with a premium of ${formatNumber(obj.premiumAmount)}` : null
|
|
336
|
+
].filter((value) => value !== null);
|
|
337
|
+
const sentences = [`${subject2}${details.length ? `, ${details.join(", ")}` : ""}.`];
|
|
338
|
+
if (isPlainObject2(obj.claim)) {
|
|
339
|
+
const claim = describeClaim(obj.claim);
|
|
340
|
+
if (claim) sentences.push(claim);
|
|
341
|
+
}
|
|
342
|
+
if (Array.isArray(obj.dependents) && obj.dependents.length > 0) {
|
|
343
|
+
sentences.push(`${obj.holderName}'s dependents are ${joinNaturalList(obj.dependents.map(String))}.`);
|
|
344
|
+
}
|
|
345
|
+
return sentences.join(" ");
|
|
346
|
+
}
|
|
347
|
+
const subject = findSubject(obj);
|
|
348
|
+
const stateEntry = findSemanticEntry(obj, [
|
|
349
|
+
"status",
|
|
350
|
+
"state",
|
|
351
|
+
"condition",
|
|
352
|
+
"stage",
|
|
353
|
+
"role",
|
|
354
|
+
"type",
|
|
355
|
+
"category",
|
|
356
|
+
"classification",
|
|
357
|
+
"priority",
|
|
358
|
+
"phase",
|
|
359
|
+
"mode",
|
|
360
|
+
"availability",
|
|
361
|
+
"outcome",
|
|
362
|
+
"result",
|
|
363
|
+
"health",
|
|
364
|
+
"progress",
|
|
365
|
+
"visibility",
|
|
366
|
+
"access",
|
|
367
|
+
"membership",
|
|
368
|
+
"sentiment",
|
|
369
|
+
"severity"
|
|
370
|
+
]);
|
|
371
|
+
if (subject && stateEntry && typeof stateEntry[1] === "string") {
|
|
372
|
+
const details = Object.entries(obj).filter(([key, value]) => key !== stateEntry[0] && key !== subject.key && shouldDescribeSemantically(key, value)).map(([key, value]) => `${camelToWords(key)} ${formatSemanticValue(value)}`);
|
|
373
|
+
return `${subject.value} is ${stateEntry[1]}${details.length ? ` and has ${details.join(", ")}` : ""}.`;
|
|
374
|
+
}
|
|
375
|
+
return null;
|
|
376
|
+
}
|
|
377
|
+
function findSubject(obj) {
|
|
378
|
+
const subjectKeys = [
|
|
379
|
+
"name",
|
|
380
|
+
"title",
|
|
381
|
+
"label",
|
|
382
|
+
"displayName",
|
|
383
|
+
"entityName",
|
|
384
|
+
"fullName",
|
|
385
|
+
"userName",
|
|
386
|
+
"username",
|
|
387
|
+
"personName",
|
|
388
|
+
"customerName",
|
|
389
|
+
"clientName",
|
|
390
|
+
"ownerName",
|
|
391
|
+
"accountName",
|
|
392
|
+
"companyName",
|
|
393
|
+
"organizationName",
|
|
394
|
+
"teamName",
|
|
395
|
+
"departmentName",
|
|
396
|
+
"projectName",
|
|
397
|
+
"productName",
|
|
398
|
+
"serviceName",
|
|
399
|
+
"resourceName",
|
|
400
|
+
"fileName",
|
|
401
|
+
"deviceName",
|
|
402
|
+
"hostName",
|
|
403
|
+
"applicationName",
|
|
404
|
+
"appName",
|
|
405
|
+
"taskName",
|
|
406
|
+
"eventName",
|
|
407
|
+
"itemName",
|
|
408
|
+
"orderName",
|
|
409
|
+
"patientName",
|
|
410
|
+
"employeeName"
|
|
411
|
+
];
|
|
412
|
+
const entries = Object.entries(obj);
|
|
413
|
+
for (const key of subjectKeys) {
|
|
414
|
+
if (typeof obj[key] === "string" && obj[key].trim()) {
|
|
415
|
+
return { key, value: obj[key].trim() };
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
const nameEntry = entries.find(([key, value]) => {
|
|
419
|
+
return key.toLowerCase().includes("name") && typeof value === "string" && value.trim().length > 0;
|
|
420
|
+
});
|
|
421
|
+
if (nameEntry) {
|
|
422
|
+
return { key: nameEntry[0], value: nameEntry[1].trim() };
|
|
423
|
+
}
|
|
424
|
+
return null;
|
|
425
|
+
}
|
|
426
|
+
function findSemanticEntry(obj, keys) {
|
|
427
|
+
const entries = Object.entries(obj);
|
|
428
|
+
const exactEntry = entries.find(([key]) => keys.includes(key));
|
|
429
|
+
if (exactEntry) return exactEntry;
|
|
430
|
+
return entries.find(([key]) => {
|
|
431
|
+
const normalizedKey = key.toLowerCase();
|
|
432
|
+
return keys.some((semanticKey) => normalizedKey.includes(semanticKey.toLowerCase()));
|
|
433
|
+
}) ?? null;
|
|
434
|
+
}
|
|
435
|
+
function shouldDescribeSemantically(key, value) {
|
|
436
|
+
return (typeof value === "string" || typeof value === "number" || typeof value === "boolean") && !["id", "timestamp", "createdAt", "updatedAt", "apiKey", "debug"].includes(key);
|
|
437
|
+
}
|
|
438
|
+
function formatSemanticValue(value) {
|
|
439
|
+
if (typeof value === "number") return formatNumber(value);
|
|
440
|
+
if (typeof value === "boolean") return value ? "enabled" : "disabled";
|
|
441
|
+
return String(value);
|
|
442
|
+
}
|
|
443
|
+
function describeClaim(claim) {
|
|
444
|
+
const parts = [];
|
|
445
|
+
if (typeof claim.status === "string") parts.push(`The claim is ${claim.status}`);
|
|
446
|
+
if (typeof claim.requestedAmount === "number") {
|
|
447
|
+
parts.push(`for ${formatNumber(claim.requestedAmount)}`);
|
|
448
|
+
}
|
|
449
|
+
if (typeof claim.claimNumber === "string") {
|
|
450
|
+
parts.push(`(reference ${claim.claimNumber})`);
|
|
451
|
+
}
|
|
452
|
+
return parts.length ? `${parts.join(" ")}.` : null;
|
|
453
|
+
}
|
|
454
|
+
function formatNumber(value) {
|
|
455
|
+
return new Intl.NumberFormat("en-US").format(value);
|
|
456
|
+
}
|
|
307
457
|
function formatPropertyClause(key, value, depth) {
|
|
308
458
|
const naturalKey = camelToWords(key);
|
|
309
459
|
if (typeof value === "object" && value !== null && !Array.isArray(value)) {
|
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
AIChain
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-FBJ3DHS6.js";
|
|
4
4
|
import {
|
|
5
5
|
analyze
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-2TDVL6BJ.js";
|
|
7
7
|
import {
|
|
8
8
|
toTOON
|
|
9
9
|
} from "./chunk-6VRLDRJK.js";
|
|
@@ -15,14 +15,14 @@ import {
|
|
|
15
15
|
} from "./chunk-L7BC62MT.js";
|
|
16
16
|
import {
|
|
17
17
|
toNatural
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-BN3L7TUV.js";
|
|
19
19
|
import {
|
|
20
20
|
prune
|
|
21
21
|
} from "./chunk-ZD536GZF.js";
|
|
22
22
|
import {
|
|
23
23
|
estimateTokens,
|
|
24
24
|
serializeForTokenEstimate
|
|
25
|
-
} from "./chunk-
|
|
25
|
+
} from "./chunk-L3HY6AWL.js";
|
|
26
26
|
|
|
27
27
|
// src/index.ts
|
|
28
28
|
function ai(data) {
|