@opencode/codemode 0.0.0-dev-1155 → 0.0.0-dev-1160

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -71,10 +71,14 @@ export type Prepared<R = never> = {
71
71
  };
72
72
  export type SearchEntry = {
73
73
  readonly description: ToolDescription;
74
- /** The path split into words, so `zones` matches `get_zones` as a word rather than as a substring of `timezones`. */
75
- readonly pathWords: ReadonlyArray<string>;
76
- readonly searchText: string;
74
+ /** Position in the full search index, for scoring through the word index. */
75
+ readonly position: number;
76
+ /** The tool's searchable text by field, read when the word index is built. */
77
+ readonly text: () => Readonly<Record<SearchField, ReadonlyArray<string>>>;
78
+ /** Words in the tool's own name (the last path segment). */
79
+ readonly nameLength: number;
77
80
  };
81
+ type SearchField = "namespace" | "name" | "description" | "parameters" | "namespaceDescription";
78
82
  /** Exact callable signature of the built-in `search` function, for host-owned instructions. */
79
83
  export declare const searchSignature: string;
80
84
  export declare const prepare: <R>(tools: Tools<R>) => Prepared<R>;
@@ -99,34 +99,203 @@ const describeTool = (visible) => {
99
99
  },
100
100
  };
101
101
  };
102
- const tokenize = (query) => query
102
+ // Standard field-weighted BM25 (k1 1.2, b 0.75) plus a preference for short names. Compared against other rankers on
103
+ // model-written and real search queries over an ~11k-tool catalog (anomalyco/evals, dumber/evals/tool-search).
104
+ const fieldWeights = {
105
+ name: 3,
106
+ namespace: 2,
107
+ description: 1,
108
+ namespaceDescription: 0.4,
109
+ parameters: 0.3,
110
+ };
111
+ const searchFields = Object.keys(fieldWeights);
112
+ const k1 = 1.2;
113
+ const lengthNormalization = 0.75;
114
+ /** score × (1 + brevityBoost / words in the tool name): "users" prefers `list_users` over `list_user_groups`. */
115
+ const brevityBoost = 0.5;
116
+ /** A query word of 4+ letters also matches longer words it starts with, at this weight: `term` → `terminal`. */
117
+ const prefixWeight = 0.3;
118
+ /** Words split at case changes and non-alphanumerics: `listUserGroups` → list, user, groups. */
119
+ const splitWords = (text) => text
103
120
  .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
104
121
  .toLowerCase()
105
122
  .split(/[^a-z0-9]+/)
106
- .filter((term) => term.length > 0 && term !== "*");
107
- const termForms = (term) => {
108
- const forms = [term];
109
- if (term.endsWith("es") && term.length > 3)
110
- forms.push(term.slice(0, -2));
111
- if (term.endsWith("s") && term.length > 2)
112
- forms.push(term.slice(0, -1));
113
- return forms;
123
+ .filter((word) => word.length > 0);
124
+ /** Plural endings only, applied the same everywhere: `settings` → `setting`, `statuses` → `status`, `vms` → `vm`. */
125
+ const stem = (word) => {
126
+ if (word.length > 4 && word.endsWith("ies"))
127
+ return word.slice(0, -3) + "y";
128
+ if (word.length > 4 && /(?:ss|sh|ch|x|z|us)es$/.test(word))
129
+ return word.slice(0, -2);
130
+ if (word.length > 2 && word.endsWith("s") && !/(?:ss|us|is)$/.test(word))
131
+ return word.slice(0, -1);
132
+ return word;
133
+ };
134
+ /** Searchable words of a text: its split words, plus each camelCase word whole (`CronJob` → cron, job, cronjob). */
135
+ const searchWords = (text) => splitWords(text)
136
+ .concat((text.match(/[A-Za-z0-9]*[a-z0-9][A-Z][A-Za-z0-9]*/g) ?? []).map((word) => word.toLowerCase()))
137
+ .map(stem);
138
+ const fieldWords = (text, cache) => {
139
+ const cached = cache.get(text);
140
+ if (cached)
141
+ return cached;
142
+ const words = searchWords(text);
143
+ cache.set(text, words);
144
+ return words;
145
+ };
146
+ // Built once per search index on the first query.
147
+ const wordIndexes = new WeakMap();
148
+ // Locations with the same tools (global MCP servers) share one word index, keyed by a hash of the indexed text. An
149
+ // index lives while some search index uses it; once none does it can be collected and its key is removed. A hash
150
+ // collision (about 1 in 37 trillion with 1,000 distinct catalogs alive) would only misrank results: they still come
151
+ // from the caller's own tools, and the tool count in the key keeps positions in range.
152
+ const shared = typeof WeakRef === "undefined" || typeof FinalizationRegistry === "undefined"
153
+ ? undefined
154
+ : {
155
+ indexes: new Map(),
156
+ cleanup: new FinalizationRegistry(({ key, ref }) => {
157
+ // A newer index may have been stored under the same key since this one was dropped.
158
+ if (shared?.indexes.get(key) === ref)
159
+ shared.indexes.delete(key);
160
+ }),
161
+ };
162
+ const wordIndexFor = (index) => {
163
+ const cached = wordIndexes.get(index);
164
+ if (cached)
165
+ return cached;
166
+ const texts = index.map((entry) => entry.text());
167
+ const key = shared ? textKey(texts) : undefined;
168
+ const reused = key === undefined ? undefined : shared?.indexes.get(key)?.deref();
169
+ if (reused) {
170
+ wordIndexes.set(index, reused);
171
+ return reused;
172
+ }
173
+ const built = buildWordIndex(texts);
174
+ wordIndexes.set(index, built);
175
+ if (shared && key !== undefined) {
176
+ const ref = new WeakRef(built);
177
+ shared.indexes.set(key, ref);
178
+ shared.cleanup.register(built, { key, ref });
179
+ }
180
+ return built;
181
+ };
182
+ /** Tool count plus two independent 32-bit FNV-1a hashes of every indexed field, in index order. */
183
+ const textKey = (texts) => {
184
+ let first = 0x811c9dc5;
185
+ let second = 0x9e3779b9;
186
+ const add = (text) => {
187
+ for (let i = 0; i < text.length; i++) {
188
+ const code = text.charCodeAt(i);
189
+ first = Math.imul(first ^ code, 0x01000193);
190
+ second = Math.imul(second ^ code, 0x5bd1e995);
191
+ second ^= second >>> 15;
192
+ }
193
+ // Separators keep field boundaries distinct: ["ab", "c"] and ["a", "bc"] hash differently.
194
+ first = Math.imul(first ^ 0x1f, 0x01000193);
195
+ second = Math.imul(second ^ 0x1f, 0x5bd1e995);
196
+ };
197
+ for (const text of texts)
198
+ for (const field of searchFields) {
199
+ for (const part of text[field])
200
+ add(part);
201
+ add("\u0000");
202
+ }
203
+ return `${texts.length}:${(first >>> 0).toString(36)}:${(second >>> 0).toString(36)}`;
204
+ };
205
+ const buildWordIndex = (texts) => {
206
+ // Field texts are split once here and dropped afterwards; descriptions repeat across tools, so each text is split
207
+ // once and shared while building.
208
+ const cache = new Map();
209
+ const words = (texts) => texts.flatMap((text) => fieldWords(text, cache));
210
+ const fields = texts.map((text) => {
211
+ return {
212
+ namespace: words(text.namespace),
213
+ name: words(text.name),
214
+ description: words(text.description),
215
+ parameters: words(text.parameters),
216
+ namespaceDescription: words(text.namespaceDescription),
217
+ };
218
+ });
219
+ const averageLength = Object.fromEntries(searchFields.map((field) => [
220
+ field,
221
+ fields.reduce((sum, entry) => sum + entry[field].length, 0) / texts.length || 1,
222
+ ]));
223
+ const building = new Map();
224
+ const weights = new Map();
225
+ fields.forEach((entry, position) => {
226
+ weights.clear();
227
+ for (const field of searchFields) {
228
+ const weight = fieldWeights[field] /
229
+ (1 - lengthNormalization + (lengthNormalization * entry[field].length) / averageLength[field]);
230
+ for (const word of entry[field])
231
+ weights.set(word, (weights.get(word) ?? 0) + weight);
232
+ }
233
+ for (const [word, weight] of weights) {
234
+ let posting = building.get(word);
235
+ if (!posting)
236
+ building.set(word, (posting = { positions: [], weights: [] }));
237
+ posting.positions.push(position);
238
+ posting.weights.push(weight);
239
+ }
240
+ });
241
+ const postings = new Map([...building].map(([word, posting]) => [
242
+ word,
243
+ { positions: Int32Array.from(posting.positions), weights: Float32Array.from(posting.weights) },
244
+ ]));
245
+ return { postings, documents: texts.length, words: [...postings.keys()].sort() };
114
246
  };
115
- const rank = (entries, query) => {
116
- const terms = tokenize(query).map(termForms);
117
- return entries
118
- .map((entry) => {
119
- const path = entry.description.path.toLowerCase();
120
- const description = entry.description.description.toLowerCase();
121
- const score = terms.reduce((total, forms) => total +
122
- (forms.some((form) => path === form || path.endsWith(`.${form}`)) ? 20 : 0) +
123
- (forms.some((form) => entry.pathWords.includes(form)) ? 12 : 0) +
124
- (forms.some((form) => path.includes(form)) ? 8 : 0) +
125
- (forms.some((form) => description.includes(form)) ? 4 : 0) +
126
- (forms.some((form) => entry.searchText.includes(form)) ? 2 : 0), 0);
127
- return { entry, score };
128
- })
129
- .filter(({ score }) => terms.length === 0 || score > 0)
247
+ /** Indexed words longer than a 4+ letter query word that start with it. */
248
+ const prefixMatches = (words, term) => {
249
+ if (term.length < 4)
250
+ return [];
251
+ let low = 0;
252
+ let high = words.words.length;
253
+ while (low < high) {
254
+ const middle = (low + high) >> 1;
255
+ if (words.words[middle] < term)
256
+ low = middle + 1;
257
+ else
258
+ high = middle;
259
+ }
260
+ const matches = [];
261
+ for (let i = low; i < words.words.length && words.words[i].startsWith(term); i++)
262
+ if (words.words[i] !== term)
263
+ matches.push([words.words[i], prefixWeight]);
264
+ return matches;
265
+ };
266
+ const rank = (entries, query, index) => {
267
+ const terms = searchWords(query);
268
+ if (terms.length === 0)
269
+ return [...entries].sort((left, right) => compareText(left.description.path, right.description.path));
270
+ const words = wordIndexFor(index);
271
+ const allowed = entries === index ? undefined : new Set(entries.map((entry) => entry.position));
272
+ const scores = new Map();
273
+ for (const term of terms) {
274
+ // Each tool counts a query word once, through its best-scoring match.
275
+ const best = new Map();
276
+ for (const [word, factor] of [[term, 1], ...prefixMatches(words, term)]) {
277
+ const posting = words.postings.get(word);
278
+ if (!posting)
279
+ continue;
280
+ const idf = Math.log(1 + (words.documents - posting.positions.length + 0.5) / (posting.positions.length + 0.5));
281
+ for (let i = 0; i < posting.positions.length; i++) {
282
+ const position = posting.positions[i];
283
+ if (allowed && !allowed.has(position))
284
+ continue;
285
+ const weight = posting.weights[i];
286
+ const score = (factor * idf * weight * (k1 + 1)) / (weight + k1);
287
+ if (score > (best.get(position) ?? 0))
288
+ best.set(position, score);
289
+ }
290
+ }
291
+ for (const [position, score] of best)
292
+ scores.set(position, (scores.get(position) ?? 0) + score);
293
+ }
294
+ return [...scores]
295
+ .map(([position, score]) => ({
296
+ entry: index[position],
297
+ score: score * (1 + brevityBoost / Math.max(1, index[position].nameLength)),
298
+ }))
130
299
  .sort((left, right) => right.score - left.score || compareText(left.entry.description.path, right.entry.description.path))
131
300
  .map(({ entry }) => entry);
132
301
  };
@@ -154,7 +323,7 @@ const makeSearchTool = (searchIndex) => ({
154
323
  const exact = pathQuery === ""
155
324
  ? undefined
156
325
  : scoped.find((entry) => entry.description.path === pathQuery || toolExpression(entry.description.path) === trimmed);
157
- const ranked = exact !== undefined ? [exact] : rank(scoped, query);
326
+ const ranked = exact !== undefined ? [exact] : rank(scoped, pathQuery, searchIndex);
158
327
  const items = ranked.slice(offset, offset + (request.limit ?? defaultSearchLimit)).map(({ description }) => ({
159
328
  ...description,
160
329
  path: toolExpression(description.path),
@@ -172,18 +341,23 @@ export const searchSignature = (() => {
172
341
  const tool = makeSearchTool([]);
173
342
  return `search(${inputTypeScript(tool, true)}): ${outputTypeScript(tool, true)}`;
174
343
  })();
175
- const toSearchEntry = (visible) => ({
176
- description: describeTool(visible),
177
- pathWords: tokenize(visible.path),
178
- searchText: [
179
- visible.path,
180
- visible.tool.description,
181
- ...visible.namespaces.flatMap((namespace) => (namespace.description === undefined ? [] : [namespace.description])),
182
- ...inputProperties(visible.tool).flatMap(({ name, description: property }) => property === undefined ? [name] : [name, property]),
183
- ]
184
- .join("\n")
185
- .toLowerCase(),
186
- });
344
+ const toSearchEntry = (visible, position) => {
345
+ const segments = visible.path.split(".");
346
+ const name = segments.at(-1) ?? "";
347
+ return {
348
+ description: describeTool(visible),
349
+ position,
350
+ text: () => ({
351
+ namespace: segments.slice(0, -1),
352
+ name: [name],
353
+ description: [visible.tool.description],
354
+ // Parameter names only: their descriptions are half of all words and didn't improve ranking.
355
+ parameters: inputProperties(visible.tool).map((property) => property.name),
356
+ namespaceDescription: visible.namespaces.map((namespace) => namespace.description ?? ""),
357
+ }),
358
+ nameLength: splitWords(name).length,
359
+ };
360
+ };
187
361
  export const prepare = (tools) => {
188
362
  const root = toolTrie(tools);
189
363
  let searchIndex;
@@ -217,7 +391,7 @@ const resolve = (root, path, index) => {
217
391
  if (node === undefined) {
218
392
  const name = segments.join(".");
219
393
  const ns = segments.length > 1 && root.children.has(segments[0]) ? segments[0] : undefined;
220
- const closest = rank(ns ? index.filter((entry) => entry.description.path.startsWith(`${ns}.`)) : index, ns ? segments.slice(1).join(" ") : name)[0];
394
+ const closest = rank(ns ? index.filter((entry) => entry.description.path.startsWith(`${ns}.`)) : index, ns ? segments.slice(1).join(" ") : name, index)[0];
221
395
  throw new ToolRuntimeError("UnknownTool", closest
222
396
  ? `Unknown tool '${name}'. Did you mean ${toolExpression(closest.description.path)}?`
223
397
  : `Unknown tool '${name}'.`, ["Use search to find available tools."]);
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "$schema": "https://json.schemastore.org/package.json",
3
3
  "name": "@opencode/codemode",
4
- "version": "0.0.0-dev-1155",
4
+ "version": "0.0.0-dev-1160",
5
5
  "description": "Effect-native confined code execution over schema-described tools",
6
6
  "type": "module",
7
7
  "license": "MIT",