@opencode/codemode 0.0.0-dev-1153 → 0.0.0-dev-1160
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/tool-runtime.d.ts +7 -3
- package/dist/tool-runtime.js +212 -38
- package/package.json +1 -1
package/dist/tool-runtime.d.ts
CHANGED
|
@@ -71,10 +71,14 @@ export type Prepared<R = never> = {
|
|
|
71
71
|
};
|
|
72
72
|
export type SearchEntry = {
|
|
73
73
|
readonly description: ToolDescription;
|
|
74
|
-
/**
|
|
75
|
-
readonly
|
|
76
|
-
|
|
74
|
+
/** Position in the full search index, for scoring through the word index. */
|
|
75
|
+
readonly position: number;
|
|
76
|
+
/** The tool's searchable text by field, read when the word index is built. */
|
|
77
|
+
readonly text: () => Readonly<Record<SearchField, ReadonlyArray<string>>>;
|
|
78
|
+
/** Words in the tool's own name (the last path segment). */
|
|
79
|
+
readonly nameLength: number;
|
|
77
80
|
};
|
|
81
|
+
type SearchField = "namespace" | "name" | "description" | "parameters" | "namespaceDescription";
|
|
78
82
|
/** Exact callable signature of the built-in `search` function, for host-owned instructions. */
|
|
79
83
|
export declare const searchSignature: string;
|
|
80
84
|
export declare const prepare: <R>(tools: Tools<R>) => Prepared<R>;
|
package/dist/tool-runtime.js
CHANGED
|
@@ -99,34 +99,203 @@ const describeTool = (visible) => {
|
|
|
99
99
|
},
|
|
100
100
|
};
|
|
101
101
|
};
|
|
102
|
-
|
|
102
|
+
// Standard field-weighted BM25 (k1 1.2, b 0.75) plus a preference for short names. Compared against other rankers on
|
|
103
|
+
// model-written and real search queries over an ~11k-tool catalog (anomalyco/evals, dumber/evals/tool-search).
|
|
104
|
+
const fieldWeights = {
|
|
105
|
+
name: 3,
|
|
106
|
+
namespace: 2,
|
|
107
|
+
description: 1,
|
|
108
|
+
namespaceDescription: 0.4,
|
|
109
|
+
parameters: 0.3,
|
|
110
|
+
};
|
|
111
|
+
const searchFields = Object.keys(fieldWeights);
|
|
112
|
+
const k1 = 1.2;
|
|
113
|
+
const lengthNormalization = 0.75;
|
|
114
|
+
/** score × (1 + brevityBoost / words in the tool name): "users" prefers `list_users` over `list_user_groups`. */
|
|
115
|
+
const brevityBoost = 0.5;
|
|
116
|
+
/** A query word of 4+ letters also matches longer words it starts with, at this weight: `term` → `terminal`. */
|
|
117
|
+
const prefixWeight = 0.3;
|
|
118
|
+
/** Words split at case changes and non-alphanumerics: `listUserGroups` → list, user, groups. */
|
|
119
|
+
const splitWords = (text) => text
|
|
103
120
|
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
104
121
|
.toLowerCase()
|
|
105
122
|
.split(/[^a-z0-9]+/)
|
|
106
|
-
.filter((
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
if (
|
|
110
|
-
|
|
111
|
-
if (
|
|
112
|
-
|
|
113
|
-
|
|
123
|
+
.filter((word) => word.length > 0);
|
|
124
|
+
/** Plural endings only, applied the same everywhere: `settings` → `setting`, `statuses` → `status`, `vms` → `vm`. */
|
|
125
|
+
const stem = (word) => {
|
|
126
|
+
if (word.length > 4 && word.endsWith("ies"))
|
|
127
|
+
return word.slice(0, -3) + "y";
|
|
128
|
+
if (word.length > 4 && /(?:ss|sh|ch|x|z|us)es$/.test(word))
|
|
129
|
+
return word.slice(0, -2);
|
|
130
|
+
if (word.length > 2 && word.endsWith("s") && !/(?:ss|us|is)$/.test(word))
|
|
131
|
+
return word.slice(0, -1);
|
|
132
|
+
return word;
|
|
133
|
+
};
|
|
134
|
+
/** Searchable words of a text: its split words, plus each camelCase word whole (`CronJob` → cron, job, cronjob). */
|
|
135
|
+
const searchWords = (text) => splitWords(text)
|
|
136
|
+
.concat((text.match(/[A-Za-z0-9]*[a-z0-9][A-Z][A-Za-z0-9]*/g) ?? []).map((word) => word.toLowerCase()))
|
|
137
|
+
.map(stem);
|
|
138
|
+
const fieldWords = (text, cache) => {
|
|
139
|
+
const cached = cache.get(text);
|
|
140
|
+
if (cached)
|
|
141
|
+
return cached;
|
|
142
|
+
const words = searchWords(text);
|
|
143
|
+
cache.set(text, words);
|
|
144
|
+
return words;
|
|
145
|
+
};
|
|
146
|
+
// Built once per search index on the first query.
|
|
147
|
+
const wordIndexes = new WeakMap();
|
|
148
|
+
// Locations with the same tools (global MCP servers) share one word index, keyed by a hash of the indexed text. An
|
|
149
|
+
// index lives while some search index uses it; once none does it can be collected and its key is removed. A hash
|
|
150
|
+
// collision (about 1 in 37 trillion with 1,000 distinct catalogs alive) would only misrank results: they still come
|
|
151
|
+
// from the caller's own tools, and the tool count in the key keeps positions in range.
|
|
152
|
+
const shared = typeof WeakRef === "undefined" || typeof FinalizationRegistry === "undefined"
|
|
153
|
+
? undefined
|
|
154
|
+
: {
|
|
155
|
+
indexes: new Map(),
|
|
156
|
+
cleanup: new FinalizationRegistry(({ key, ref }) => {
|
|
157
|
+
// A newer index may have been stored under the same key since this one was dropped.
|
|
158
|
+
if (shared?.indexes.get(key) === ref)
|
|
159
|
+
shared.indexes.delete(key);
|
|
160
|
+
}),
|
|
161
|
+
};
|
|
162
|
+
const wordIndexFor = (index) => {
|
|
163
|
+
const cached = wordIndexes.get(index);
|
|
164
|
+
if (cached)
|
|
165
|
+
return cached;
|
|
166
|
+
const texts = index.map((entry) => entry.text());
|
|
167
|
+
const key = shared ? textKey(texts) : undefined;
|
|
168
|
+
const reused = key === undefined ? undefined : shared?.indexes.get(key)?.deref();
|
|
169
|
+
if (reused) {
|
|
170
|
+
wordIndexes.set(index, reused);
|
|
171
|
+
return reused;
|
|
172
|
+
}
|
|
173
|
+
const built = buildWordIndex(texts);
|
|
174
|
+
wordIndexes.set(index, built);
|
|
175
|
+
if (shared && key !== undefined) {
|
|
176
|
+
const ref = new WeakRef(built);
|
|
177
|
+
shared.indexes.set(key, ref);
|
|
178
|
+
shared.cleanup.register(built, { key, ref });
|
|
179
|
+
}
|
|
180
|
+
return built;
|
|
181
|
+
};
|
|
182
|
+
/** Tool count plus two independent 32-bit FNV-1a hashes of every indexed field, in index order. */
|
|
183
|
+
const textKey = (texts) => {
|
|
184
|
+
let first = 0x811c9dc5;
|
|
185
|
+
let second = 0x9e3779b9;
|
|
186
|
+
const add = (text) => {
|
|
187
|
+
for (let i = 0; i < text.length; i++) {
|
|
188
|
+
const code = text.charCodeAt(i);
|
|
189
|
+
first = Math.imul(first ^ code, 0x01000193);
|
|
190
|
+
second = Math.imul(second ^ code, 0x5bd1e995);
|
|
191
|
+
second ^= second >>> 15;
|
|
192
|
+
}
|
|
193
|
+
// Separators keep field boundaries distinct: ["ab", "c"] and ["a", "bc"] hash differently.
|
|
194
|
+
first = Math.imul(first ^ 0x1f, 0x01000193);
|
|
195
|
+
second = Math.imul(second ^ 0x1f, 0x5bd1e995);
|
|
196
|
+
};
|
|
197
|
+
for (const text of texts)
|
|
198
|
+
for (const field of searchFields) {
|
|
199
|
+
for (const part of text[field])
|
|
200
|
+
add(part);
|
|
201
|
+
add("\u0000");
|
|
202
|
+
}
|
|
203
|
+
return `${texts.length}:${(first >>> 0).toString(36)}:${(second >>> 0).toString(36)}`;
|
|
204
|
+
};
|
|
205
|
+
const buildWordIndex = (texts) => {
|
|
206
|
+
// Field texts are split once here and dropped afterwards; descriptions repeat across tools, so each text is split
|
|
207
|
+
// once and shared while building.
|
|
208
|
+
const cache = new Map();
|
|
209
|
+
const words = (texts) => texts.flatMap((text) => fieldWords(text, cache));
|
|
210
|
+
const fields = texts.map((text) => {
|
|
211
|
+
return {
|
|
212
|
+
namespace: words(text.namespace),
|
|
213
|
+
name: words(text.name),
|
|
214
|
+
description: words(text.description),
|
|
215
|
+
parameters: words(text.parameters),
|
|
216
|
+
namespaceDescription: words(text.namespaceDescription),
|
|
217
|
+
};
|
|
218
|
+
});
|
|
219
|
+
const averageLength = Object.fromEntries(searchFields.map((field) => [
|
|
220
|
+
field,
|
|
221
|
+
fields.reduce((sum, entry) => sum + entry[field].length, 0) / texts.length || 1,
|
|
222
|
+
]));
|
|
223
|
+
const building = new Map();
|
|
224
|
+
const weights = new Map();
|
|
225
|
+
fields.forEach((entry, position) => {
|
|
226
|
+
weights.clear();
|
|
227
|
+
for (const field of searchFields) {
|
|
228
|
+
const weight = fieldWeights[field] /
|
|
229
|
+
(1 - lengthNormalization + (lengthNormalization * entry[field].length) / averageLength[field]);
|
|
230
|
+
for (const word of entry[field])
|
|
231
|
+
weights.set(word, (weights.get(word) ?? 0) + weight);
|
|
232
|
+
}
|
|
233
|
+
for (const [word, weight] of weights) {
|
|
234
|
+
let posting = building.get(word);
|
|
235
|
+
if (!posting)
|
|
236
|
+
building.set(word, (posting = { positions: [], weights: [] }));
|
|
237
|
+
posting.positions.push(position);
|
|
238
|
+
posting.weights.push(weight);
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
const postings = new Map([...building].map(([word, posting]) => [
|
|
242
|
+
word,
|
|
243
|
+
{ positions: Int32Array.from(posting.positions), weights: Float32Array.from(posting.weights) },
|
|
244
|
+
]));
|
|
245
|
+
return { postings, documents: texts.length, words: [...postings.keys()].sort() };
|
|
114
246
|
};
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
247
|
+
/** Indexed words longer than a 4+ letter query word that start with it. */
|
|
248
|
+
const prefixMatches = (words, term) => {
|
|
249
|
+
if (term.length < 4)
|
|
250
|
+
return [];
|
|
251
|
+
let low = 0;
|
|
252
|
+
let high = words.words.length;
|
|
253
|
+
while (low < high) {
|
|
254
|
+
const middle = (low + high) >> 1;
|
|
255
|
+
if (words.words[middle] < term)
|
|
256
|
+
low = middle + 1;
|
|
257
|
+
else
|
|
258
|
+
high = middle;
|
|
259
|
+
}
|
|
260
|
+
const matches = [];
|
|
261
|
+
for (let i = low; i < words.words.length && words.words[i].startsWith(term); i++)
|
|
262
|
+
if (words.words[i] !== term)
|
|
263
|
+
matches.push([words.words[i], prefixWeight]);
|
|
264
|
+
return matches;
|
|
265
|
+
};
|
|
266
|
+
const rank = (entries, query, index) => {
|
|
267
|
+
const terms = searchWords(query);
|
|
268
|
+
if (terms.length === 0)
|
|
269
|
+
return [...entries].sort((left, right) => compareText(left.description.path, right.description.path));
|
|
270
|
+
const words = wordIndexFor(index);
|
|
271
|
+
const allowed = entries === index ? undefined : new Set(entries.map((entry) => entry.position));
|
|
272
|
+
const scores = new Map();
|
|
273
|
+
for (const term of terms) {
|
|
274
|
+
// Each tool counts a query word once, through its best-scoring match.
|
|
275
|
+
const best = new Map();
|
|
276
|
+
for (const [word, factor] of [[term, 1], ...prefixMatches(words, term)]) {
|
|
277
|
+
const posting = words.postings.get(word);
|
|
278
|
+
if (!posting)
|
|
279
|
+
continue;
|
|
280
|
+
const idf = Math.log(1 + (words.documents - posting.positions.length + 0.5) / (posting.positions.length + 0.5));
|
|
281
|
+
for (let i = 0; i < posting.positions.length; i++) {
|
|
282
|
+
const position = posting.positions[i];
|
|
283
|
+
if (allowed && !allowed.has(position))
|
|
284
|
+
continue;
|
|
285
|
+
const weight = posting.weights[i];
|
|
286
|
+
const score = (factor * idf * weight * (k1 + 1)) / (weight + k1);
|
|
287
|
+
if (score > (best.get(position) ?? 0))
|
|
288
|
+
best.set(position, score);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
for (const [position, score] of best)
|
|
292
|
+
scores.set(position, (scores.get(position) ?? 0) + score);
|
|
293
|
+
}
|
|
294
|
+
return [...scores]
|
|
295
|
+
.map(([position, score]) => ({
|
|
296
|
+
entry: index[position],
|
|
297
|
+
score: score * (1 + brevityBoost / Math.max(1, index[position].nameLength)),
|
|
298
|
+
}))
|
|
130
299
|
.sort((left, right) => right.score - left.score || compareText(left.entry.description.path, right.entry.description.path))
|
|
131
300
|
.map(({ entry }) => entry);
|
|
132
301
|
};
|
|
@@ -154,7 +323,7 @@ const makeSearchTool = (searchIndex) => ({
|
|
|
154
323
|
const exact = pathQuery === ""
|
|
155
324
|
? undefined
|
|
156
325
|
: scoped.find((entry) => entry.description.path === pathQuery || toolExpression(entry.description.path) === trimmed);
|
|
157
|
-
const ranked = exact !== undefined ? [exact] : rank(scoped,
|
|
326
|
+
const ranked = exact !== undefined ? [exact] : rank(scoped, pathQuery, searchIndex);
|
|
158
327
|
const items = ranked.slice(offset, offset + (request.limit ?? defaultSearchLimit)).map(({ description }) => ({
|
|
159
328
|
...description,
|
|
160
329
|
path: toolExpression(description.path),
|
|
@@ -172,18 +341,23 @@ export const searchSignature = (() => {
|
|
|
172
341
|
const tool = makeSearchTool([]);
|
|
173
342
|
return `search(${inputTypeScript(tool, true)}): ${outputTypeScript(tool, true)}`;
|
|
174
343
|
})();
|
|
175
|
-
const toSearchEntry = (visible) =>
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
visible
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
344
|
+
const toSearchEntry = (visible, position) => {
|
|
345
|
+
const segments = visible.path.split(".");
|
|
346
|
+
const name = segments.at(-1) ?? "";
|
|
347
|
+
return {
|
|
348
|
+
description: describeTool(visible),
|
|
349
|
+
position,
|
|
350
|
+
text: () => ({
|
|
351
|
+
namespace: segments.slice(0, -1),
|
|
352
|
+
name: [name],
|
|
353
|
+
description: [visible.tool.description],
|
|
354
|
+
// Parameter names only: their descriptions are half of all words and didn't improve ranking.
|
|
355
|
+
parameters: inputProperties(visible.tool).map((property) => property.name),
|
|
356
|
+
namespaceDescription: visible.namespaces.map((namespace) => namespace.description ?? ""),
|
|
357
|
+
}),
|
|
358
|
+
nameLength: splitWords(name).length,
|
|
359
|
+
};
|
|
360
|
+
};
|
|
187
361
|
export const prepare = (tools) => {
|
|
188
362
|
const root = toolTrie(tools);
|
|
189
363
|
let searchIndex;
|
|
@@ -217,7 +391,7 @@ const resolve = (root, path, index) => {
|
|
|
217
391
|
if (node === undefined) {
|
|
218
392
|
const name = segments.join(".");
|
|
219
393
|
const ns = segments.length > 1 && root.children.has(segments[0]) ? segments[0] : undefined;
|
|
220
|
-
const closest = rank(ns ? index.filter((entry) => entry.description.path.startsWith(`${ns}.`)) : index, ns ? segments.slice(1).join(" ") : name)[0];
|
|
394
|
+
const closest = rank(ns ? index.filter((entry) => entry.description.path.startsWith(`${ns}.`)) : index, ns ? segments.slice(1).join(" ") : name, index)[0];
|
|
221
395
|
throw new ToolRuntimeError("UnknownTool", closest
|
|
222
396
|
? `Unknown tool '${name}'. Did you mean ${toolExpression(closest.description.path)}?`
|
|
223
397
|
: `Unknown tool '${name}'.`, ["Use search to find available tools."]);
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "https://json.schemastore.org/package.json",
|
|
3
3
|
"name": "@opencode/codemode",
|
|
4
|
-
"version": "0.0.0-dev-
|
|
4
|
+
"version": "0.0.0-dev-1160",
|
|
5
5
|
"description": "Effect-native confined code execution over schema-described tools",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"license": "MIT",
|