@agentpro/scroll 0.0.0-stage → 2.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/AGENTS.md +55 -0
  2. package/LICENSE +21 -0
  3. package/README.md +293 -2
  4. package/SPEC.md +1079 -0
  5. package/bin/scroll.js +8 -0
  6. package/docs/prompt-caching.md +128 -0
  7. package/index.js +33 -0
  8. package/lib/audit.js +358 -0
  9. package/lib/auth.js +849 -0
  10. package/lib/chat-sdk/admin-controller.js +289 -0
  11. package/lib/chat-sdk/debouncer.js +94 -0
  12. package/lib/chat-sdk/group-grants.js +150 -0
  13. package/lib/chat-sdk/index.js +222 -0
  14. package/lib/chat-sdk/long-task-runner.js +413 -0
  15. package/lib/chat-session.js +1788 -0
  16. package/lib/chat-telemetry.js +557 -0
  17. package/lib/codex-app-server.js +851 -0
  18. package/lib/digest.js +112 -0
  19. package/lib/effects.js +157 -0
  20. package/lib/eval.js +408 -0
  21. package/lib/grounding.js +356 -0
  22. package/lib/installers.js +176 -0
  23. package/lib/knowledge.js +335 -0
  24. package/lib/ledger.js +38 -0
  25. package/lib/loop.js +197 -0
  26. package/lib/mcp-client.js +926 -0
  27. package/lib/memory-cli.js +66 -0
  28. package/lib/memory-tracker.js +1472 -0
  29. package/lib/memory.js +1279 -0
  30. package/lib/people-cli.js +81 -0
  31. package/lib/people.js +534 -0
  32. package/lib/permissions.js +272 -0
  33. package/lib/providers.js +1572 -0
  34. package/lib/runtime.js +928 -0
  35. package/lib/sandbox.js +744 -0
  36. package/lib/scheduler.js +402 -0
  37. package/lib/scroll.js +1225 -0
  38. package/lib/skill-cli.js +120 -0
  39. package/lib/skill-eval.js +156 -0
  40. package/lib/skills.js +860 -0
  41. package/lib/style.js +126 -0
  42. package/lib/tool-retriever.js +489 -0
  43. package/lib/tools.js +1833 -0
  44. package/lib/user.js +711 -0
  45. package/lib/vault.js +151 -0
  46. package/package.json +88 -4
  47. package/templates/agent/IDENTITY.md +56 -0
  48. package/templates/agent/SOUL.md +23 -0
  49. package/templates/agent/TOOLS.md +25 -0
  50. package/templates/agent/evals/case-02-citation-discipline.md +29 -0
  51. package/templates/agent/evals/case-03-boundary.md +27 -0
  52. package/templates/agent/evals/case-example.md +43 -0
  53. package/templates/agent/hard-rules.md +11 -0
  54. package/templates/agent/memory/lessons.md +5 -0
  55. package/templates/agent/memory/log/.gitkeep +0 -0
  56. package/templates/external-safety-contract.md +5 -0
  57. package/templates/long-task-planner.md +17 -0
  58. package/templates/work/LOOP.md +37 -0
  59. package/templates/work/WORK.md +50 -0
  60. package/types/chat.d.ts +61 -0
  61. package/types/index.d.ts +112 -0
package/lib/style.js ADDED
@@ -0,0 +1,126 @@
1
+ // SCROLL — machine-enforced `style:` block & voice-hook safety checks (v1.9 §43).
2
+ //
3
+ // Only keys with deterministic code enforcement are allowed inside `IDENTITY.style`:
4
+ // bubbles (integer >= 1): max outgoing message bubbles
5
+ // max_chars (integer >= 20): hard character ceiling (1 regenerate, else hold)
6
+ // emoji (integer >= 0): max emoji characters allowed (0 strips all emojis)
7
+ // follow_up (integer >= 0): max trailing follow-up questions allowed (0 strips filler follow-ups)
8
+ //
9
+ // Any other key (e.g. warmth, humor, tone) fails `scroll check` — prose tone belongs in SOUL.md
10
+ // or a voice plugin that translates settings into concrete wording without bare numeric sliders.
11
+ //
12
+ // Deterministic, zero dependencies, pure ASCII source. Node >= 20.
13
+ import { extractClaims, verifyClaimsInvariant } from './grounding.js';
14
+
15
+ export const ENFORCED_STYLE_KEYS = new Set(['bubbles', 'max_chars', 'emoji', 'follow_up']);
16
+ export const SPEAKS_AS_VALUES = new Set(['self', 'owner', 'org']);
17
+
18
+ // Matches Unicode emoji pictographs/symbols (ASCII-safe Unicode property escape)
19
+ const EMOJI_RE = /\p{Extended_Pictographic}/gu;
20
+
21
+ // Detects bare numeric sliders leaked into prompt text (e.g. "warmth: 4/5", "humor = 3")
22
+ const BARE_SLIDER_RE = /\b(?:warmth|humor|playfulness|directness|agency|communion|initiative|verbosity|formality|empathy|tone)\s*[:=]\s*\d+(?:\s*\/\s*\d+)?\b/i;
23
+
24
+ export function hasBareNumericSlider(text) {
25
+ return BARE_SLIDER_RE.test(String(text ?? ''));
26
+ }
27
+
28
+ export function validateStyle(style) {
29
+ const errors = [];
30
+ if (style === undefined || style === null) return { errors };
31
+ if (typeof style !== 'object' || Array.isArray(style)) {
32
+ return { errors: ['IDENTITY: style must be a map { bubbles, max_chars, emoji, follow_up }'] };
33
+ }
34
+ for (const [k, v] of Object.entries(style)) {
35
+ if (!ENFORCED_STYLE_KEYS.has(k)) {
36
+ errors.push(`IDENTITY: style.${k} has no machine enforcement in code — only [${[...ENFORCED_STYLE_KEYS].join(', ')}] are allowed; move qualitative tone to SOUL.md or a voice plugin`);
37
+ continue;
38
+ }
39
+ if (typeof v !== 'number' || !Number.isInteger(v)) {
40
+ errors.push(`IDENTITY: style.${k} must be an integer`);
41
+ continue;
42
+ }
43
+ if (k === 'bubbles' && v < 1) errors.push('IDENTITY: style.bubbles must be >= 1');
44
+ if (k === 'max_chars' && v < 20) errors.push('IDENTITY: style.max_chars must be >= 20');
45
+ if ((k === 'emoji' || k === 'follow_up') && v < 0) errors.push(`IDENTITY: style.${k} must be >= 0`);
46
+ }
47
+ return { errors };
48
+ }
49
+
50
+ function stripRedundantMarkdown(text) {
51
+ let s = String(text ?? '').replace(/\r\n?/g, '\n');
52
+ // Unwrap bold/italic markers without touching numbers, codes, or links
53
+ s = s.replace(/\*\*([^*\n]+)\*\*/g, '$1');
54
+ s = s.replace(/__([^_\n]+)__/g, '$1');
55
+ // Strip leading markdown heading hashes while preserving line text
56
+ s = s.replace(/^#{1,6}\s+/gm, '');
57
+ return s.trim();
58
+ }
59
+
60
+ function capEmojis(text, maxEmoji) {
61
+ if (maxEmoji == null) return text;
62
+ let seen = 0;
63
+ return text.replace(EMOJI_RE, (m) => {
64
+ seen++;
65
+ return seen <= maxEmoji ? m : '';
66
+ }).replace(/[ \t]{2,}/g, ' ').replace(/ +\n/g, '\n').trim();
67
+ }
68
+
69
+ // Strip trailing filler follow-up questions when follow_up === 0, ONLY if removing them
70
+ // drops ZERO structured claims (amounts, codes, dates, links).
71
+ function capFollowUps(text, maxFollowUp) {
72
+ if (maxFollowUp == null) return text;
73
+ const parts = text.split(/(?<=[.!?])\s+|\n+/);
74
+ const out = [];
75
+ let questionsKept = 0;
76
+ for (let i = 0; i < parts.length; i++) {
77
+ const p = parts[i].trim();
78
+ if (!p) continue;
79
+ const isQ = p.endsWith('?');
80
+ if (isQ) {
81
+ const hasClaim = extractClaims(p).length > 0;
82
+ if (!hasClaim && out.length > 0 && questionsKept >= maxFollowUp) continue;
83
+ questionsKept++;
84
+ }
85
+ out.push(p);
86
+ }
87
+ return out.join('\n').trim();
88
+ }
89
+
90
+ function splitAndCapBubbles(text, maxBubbles) {
91
+ const rawChunks = String(text ?? '')
92
+ .split(/\n{2,}|\n/)
93
+ .map((x) => x.trim())
94
+ .filter(Boolean);
95
+ if (!rawChunks.length) return [''];
96
+ if (!maxBubbles || rawChunks.length <= maxBubbles) return rawChunks;
97
+ // Merge overflow chunks into the last allowed bubble (never drop text)
98
+ const kept = rawChunks.slice(0, maxBubbles - 1);
99
+ const tail = rawChunks.slice(maxBubbles - 1).join(' ');
100
+ kept.push(tail);
101
+ return kept;
102
+ }
103
+
104
+ // Machine-enforce `IDENTITY.style` on a reply without ever altering numbers, codes, dates, or links.
105
+ export function enforceStyle(rawText, style = {}) {
106
+ const original = String(rawText ?? '').trim();
107
+ const s = style && typeof style === 'object' ? style : {};
108
+ let cleaned = stripRedundantMarkdown(original);
109
+ if (typeof s.emoji === 'number') cleaned = capEmojis(cleaned, s.emoji);
110
+ if (typeof s.follow_up === 'number') cleaned = capFollowUps(cleaned, s.follow_up);
111
+ const bubbles = splitAndCapBubbles(cleaned, typeof s.bubbles === 'number' ? s.bubbles : null);
112
+ const text = bubbles.join('\n');
113
+ const inv = verifyClaimsInvariant(original, text);
114
+ // Fail-safe: if formatting ever perturbed a claim, fall back to original text split cleanly
115
+ const safeBubbles = inv.ok ? bubbles : splitAndCapBubbles(original, typeof s.bubbles === 'number' ? s.bubbles : null);
116
+ const safeText = safeBubbles.join('\n');
117
+ const overMaxChars = typeof s.max_chars === 'number' && safeText.length > s.max_chars;
118
+ return {
119
+ text: safeText,
120
+ bubbles: safeBubbles,
121
+ chars: safeText.length,
122
+ overMaxChars,
123
+ needsRegenerate: overMaxChars,
124
+ invariantOk: verifyClaimsInvariant(original, safeText).ok,
125
+ };
126
+ }
@@ -0,0 +1,489 @@
1
+ // SCROLL 2.5 -- §47.2 Zero-Dependency Deterministic Tool Retriever, Host JSON Schema Validator,
2
+ // and Frozen `<available_tools>` Region A Catalog Builder.
3
+ //
4
+ // Designed for strict §13 Prompt Cache invariance:
5
+ // - Catalog C (deployment-frozen, speaker-independent) renders `<available_tools>` once.
6
+ // - Visible V (per-turn volatile) filters `tool_search` results inside Region C.
7
+ // - Executable E (per-call) unwraps `tool_invoke`, validates args via `validateToolArgs`,
8
+ // checks `schemaHash` against pinned lockfile, and routes through `executeTool()`.
9
+ //
10
+ // Zero runtime dependencies. Node >= 20 built-ins only.
11
+ import crypto from 'node:crypto';
12
+ import { foldAscii } from './knowledge.js';
13
+ import { canonicalJson } from './chat-telemetry.js';
14
+
15
+ const TOOL_STOPWORDS = new Set([
16
+ 'the', 'and', 'for', 'with', 'from', 'that', 'this', 'into', 'onto', 'over', 'under',
17
+ 'your', 'their', 'about', 'using', 'used', 'tool', 'tools', 'function', 'call', 'get',
18
+ 'set', 'run', 'use', 'return', 'returns', 'data', 'info', 'information', 'object',
19
+ 'cua', 'cho', 'voi', 'trong', 'ngoai', 'nay', 'kia', 'mot', 'cac', 'nhung', 'duoc',
20
+ ]);
21
+
22
+ // Compact bilingual English/Vietnamese operational synonym groups (pure ASCII after foldAscii)
23
+ const SYNONYM_GROUPS = [
24
+ ['search', 'find', 'lookup', 'query', 'tim', 'tra', 'kiem'],
25
+ ['read', 'view', 'show', 'inspect', 'fetch', 'doc', 'xem', 'lay'],
26
+ ['write', 'create', 'add', 'insert', 'new', 'tao', 'them', 'ghi'],
27
+ ['update', 'edit', 'modify', 'patch', 'change', 'sua', 'cap', 'doi'],
28
+ ['delete', 'remove', 'erase', 'forget', 'drop', 'xoa', 'huy', 'bo'],
29
+ ['send', 'post', 'notify', 'message', 'reply', 'gui', 'nhan', 'bao'],
30
+ ['mail', 'email', 'inbox', 'gmail', 'outlook', 'thu'],
31
+ ['calendar', 'schedule', 'cron', 'reminder', 'meeting', 'event', 'lich', 'hen', 'nhac'],
32
+ ['browser', 'web', 'page', 'tab', 'click', 'screenshot', 'dom', 'trinh', 'duyet', 'trang'],
33
+ ['deploy', 'release', 'build', 'service', 'server', 'logs', 'trien', 'khai'],
34
+ ['file', 'folder', 'dir', 'directory', 'path', 'document', 'tep', 'thu', 'muc'],
35
+ ['memory', 'remember', 'fact', 'lesson', 'note', 'nho', 'ky', 'uc', 'bai', 'hoc'],
36
+ ['refund', 'invoice', 'payment', 'bill', 'order', 'price', 'hoan', 'tien', 'hoa', 'don', 'gia'],
37
+ ['issue', 'ticket', 'task', 'bug', 'pull', 'request', 'repo', 'commit', 'branch', 'viec'],
38
+ ['database', 'db', 'table', 'sql', 'record', 'row', 'notion', 'page', 'bang'],
39
+ ['photo', 'image', 'picture', 'camera', 'screen', 'anh', 'hinh', 'chup'],
40
+ ['shell', 'exec', 'command', 'terminal', 'bash', 'script', 'lenh', 'chay'],
41
+ ];
42
+
43
+ const SYNONYM_MAP = new Map();
44
+ for (const group of SYNONYM_GROUPS) {
45
+ for (const term of group) {
46
+ const existing = SYNONYM_MAP.get(term) || new Set();
47
+ for (const peer of group) {
48
+ if (peer !== term) existing.add(peer);
49
+ }
50
+ SYNONYM_MAP.set(term, existing);
51
+ }
52
+ }
53
+
54
+ export function tokenizeIdentifier(text) {
55
+ const split = String(text ?? '')
56
+ .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
57
+ .replace(/[._:/-]+/g, ' ');
58
+ return (foldAscii(split).toLowerCase().match(/[a-z0-9]+/g) || [])
59
+ .filter((t) => t.length >= 2 && !TOOL_STOPWORDS.has(t));
60
+ }
61
+
62
+ function makeBigrams(tokens) {
63
+ const out = [];
64
+ for (let i = 0; i < tokens.length - 1; i++) {
65
+ out.push(`${tokens[i]} ${tokens[i + 1]}`);
66
+ }
67
+ return out;
68
+ }
69
+
70
+ function sanitizeInline(text, maxLen = 80) {
71
+ const cleaned = String(text ?? '')
72
+ .replace(/[<>]/g, '')
73
+ .replace(/[\r\n\t]+/g, ' ')
74
+ .replace(/\s+/g, ' ')
75
+ .trim();
76
+ if (cleaned.length <= maxLen) return cleaned;
77
+ return cleaned.slice(0, Math.max(8, maxLen - 3)).trim() + '...';
78
+ }
79
+
80
+ function firstSentence(text, maxLen = 80) {
81
+ const s = sanitizeInline(text, maxLen * 2);
82
+ const dot = s.search(/[.!?](?:\s|$)/);
83
+ const base = dot > 8 && dot < maxLen ? s.slice(0, dot + 1) : s;
84
+ return sanitizeInline(base, maxLen);
85
+ }
86
+
87
+ export function computeSchemaHash({ name, description = '', parameters = {} } = {}) {
88
+ return crypto.createHash('sha256')
89
+ .update(canonicalJson({
90
+ name: String(name || ''),
91
+ description: String(description || ''),
92
+ parameters: parameters || { type: 'object', properties: {} },
93
+ }))
94
+ .digest('hex')
95
+ .slice(0, 24);
96
+ }
97
+
98
+ export function computeDescHash(description = '') {
99
+ return crypto.createHash('sha256').update(String(description || '')).digest('hex').slice(0, 16);
100
+ }
101
+
102
+ // ── Zero-Dep Host JSON Schema Subset Validator (for `tool_invoke` safety) ──────
103
+ function matchesType(val, expected) {
104
+ if (Array.isArray(expected)) return expected.some((t) => matchesType(val, t));
105
+ if (expected === 'null') return val === null;
106
+ if (expected === 'array') return Array.isArray(val);
107
+ if (expected === 'object') return val !== null && typeof val === 'object' && !Array.isArray(val);
108
+ if (expected === 'integer') return typeof val === 'number' && Number.isInteger(val);
109
+ if (expected === 'number') return typeof val === 'number' && Number.isFinite(val);
110
+ if (expected === 'string') return typeof val === 'string';
111
+ if (expected === 'boolean') return typeof val === 'boolean';
112
+ return true;
113
+ }
114
+
115
+ export function validateToolArgs(args, schema = {}, pathPrefix = '$') {
116
+ const errors = [];
117
+ if (!schema || typeof schema !== 'object') return { ok: true, errors };
118
+
119
+ const val = args;
120
+ if (schema.type && !matchesType(val, schema.type)) {
121
+ errors.push(`${pathPrefix}: expected type ${JSON.stringify(schema.type)}, got ${val === null ? 'null' : (Array.isArray(val) ? 'array' : typeof val)}`);
122
+ return { ok: false, errors };
123
+ }
124
+ if (Array.isArray(schema.enum) && !schema.enum.includes(val)) {
125
+ errors.push(`${pathPrefix}: value must be one of ${JSON.stringify(schema.enum)}`);
126
+ }
127
+ if (typeof val === 'string') {
128
+ if (typeof schema.minLength === 'number' && val.length < schema.minLength) {
129
+ errors.push(`${pathPrefix}: length ${val.length} < minLength ${schema.minLength}`);
130
+ }
131
+ if (typeof schema.maxLength === 'number' && val.length > schema.maxLength) {
132
+ errors.push(`${pathPrefix}: length ${val.length} > maxLength ${schema.maxLength}`);
133
+ }
134
+ }
135
+ if (typeof val === 'number') {
136
+ if (typeof schema.minimum === 'number' && val < schema.minimum) {
137
+ errors.push(`${pathPrefix}: ${val} < minimum ${schema.minimum}`);
138
+ }
139
+ if (typeof schema.maximum === 'number' && val > schema.maximum) {
140
+ errors.push(`${pathPrefix}: ${val} > maximum ${schema.maximum}`);
141
+ }
142
+ }
143
+ if (Array.isArray(val) && schema.items && typeof schema.items === 'object') {
144
+ for (let i = 0; i < val.length; i++) {
145
+ const sub = validateToolArgs(val[i], schema.items, `${pathPrefix}[${i}]`);
146
+ if (!sub.ok) errors.push(...sub.errors);
147
+ }
148
+ }
149
+ if (val && typeof val === 'object' && !Array.isArray(val)) {
150
+ const props = schema.properties && typeof schema.properties === 'object' ? schema.properties : {};
151
+ if (Array.isArray(schema.required)) {
152
+ for (const req of schema.required) {
153
+ if (val[req] === undefined) {
154
+ errors.push(`${pathPrefix}.${req}: required property missing`);
155
+ }
156
+ }
157
+ }
158
+ for (const [k, v] of Object.entries(val)) {
159
+ if (Object.prototype.hasOwnProperty.call(props, k)) {
160
+ const sub = validateToolArgs(v, props[k], `${pathPrefix}.${k}`);
161
+ if (!sub.ok) errors.push(...sub.errors);
162
+ } else if (schema.additionalProperties === false) {
163
+ errors.push(`${pathPrefix}.${k}: unknown property not allowed`);
164
+ }
165
+ }
166
+ }
167
+ return { ok: errors.length === 0, errors };
168
+ }
169
+
170
+ // ── Extract Parameter & Enum Tokens for Multi-Field BM25F ──────────────────────
171
+ function extractParamTokens(schema) {
172
+ const out = [];
173
+ if (!schema || typeof schema !== 'object') return out;
174
+ const props = schema.properties && typeof schema.properties === 'object' ? schema.properties : {};
175
+ for (const [key, def] of Object.entries(props)) {
176
+ out.push(...tokenizeIdentifier(key));
177
+ if (def && typeof def === 'object') {
178
+ if (def.description) out.push(...tokenizeIdentifier(def.description));
179
+ if (Array.isArray(def.enum)) {
180
+ for (const e of def.enum) out.push(...tokenizeIdentifier(String(e)));
181
+ }
182
+ if (def.properties && typeof def.properties === 'object') {
183
+ for (const subKey of Object.keys(def.properties)) out.push(...tokenizeIdentifier(subKey));
184
+ }
185
+ }
186
+ }
187
+ return out;
188
+ }
189
+
190
+ function inferNamespace(tool) {
191
+ if (tool.namespace) return String(tool.namespace);
192
+ if (tool.server) return `mcp.${tool.server}`;
193
+ const m = /^mcp\.([^.]+)\./.exec(String(tool.name || ''));
194
+ if (m) return `mcp.${m[1]}`;
195
+ return 'builtin';
196
+ }
197
+
198
+ function inferShortName(tool) {
199
+ const n = String(tool.name || '');
200
+ const parts = n.split('.');
201
+ return parts.length >= 3 && parts[0] === 'mcp' ? parts.slice(2).join('.') : n;
202
+ }
203
+
204
+ function cosineSim(a, b) {
205
+ if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length || !a.length) return 0;
206
+ let dot = 0, na = 0, nb = 0;
207
+ for (let i = 0; i < a.length; i++) {
208
+ dot += a[i] * b[i];
209
+ na += a[i] * a[i];
210
+ nb += b[i] * b[i];
211
+ }
212
+ if (!na || !nb) return 0;
213
+ return dot / (Math.sqrt(na) * Math.sqrt(nb));
214
+ }
215
+
216
+ // ── Multi-Field Weighted BM25F Tool Retriever ──────────────────────────────────
217
+ export function buildToolRetriever(toolsList = [], { embedFn = null } = {}) {
218
+ const docs = (toolsList || []).map((t) => {
219
+ const ns = inferNamespace(t);
220
+ const shortName = inferShortName(t);
221
+ const nameTokens = tokenizeIdentifier(`${t.name} ${shortName}`);
222
+ const nsTagTokens = tokenizeIdentifier(`${ns} ${(Array.isArray(t.tags) ? t.tags.join(' ') : '')}`);
223
+ const paramTokens = extractParamTokens(t.parameters);
224
+ const descTokens = tokenizeIdentifier(`${t.shortDescription || ''} ${t.description || ''}`);
225
+
226
+ const fieldTf = (arr) => {
227
+ const map = new Map();
228
+ for (const tok of arr) map.set(tok, (map.get(tok) || 0) + 1);
229
+ return map;
230
+ };
231
+ const allTerms = new Set([...nameTokens, ...nsTagTokens, ...paramTokens, ...descTokens]);
232
+ return {
233
+ tool: t,
234
+ nameLower: String(t.name || '').toLowerCase(),
235
+ shortLower: shortName.toLowerCase(),
236
+ namespace: ns,
237
+ nameTf: fieldTf(nameTokens),
238
+ nameLen: nameTokens.length || 1,
239
+ nsTf: fieldTf(nsTagTokens),
240
+ nsLen: nsTagTokens.length || 1,
241
+ paramTf: fieldTf(paramTokens),
242
+ paramLen: paramTokens.length || 1,
243
+ descTf: fieldTf(descTokens),
244
+ descLen: descTokens.length || 1,
245
+ nameBigrams: new Set(makeBigrams(nameTokens)),
246
+ descBigrams: new Set(makeBigrams([...nameTokens, ...descTokens])),
247
+ allTerms,
248
+ };
249
+ });
250
+
251
+ const N = Math.max(1, docs.length);
252
+ const df = new Map();
253
+ let sumName = 0, sumNs = 0, sumParam = 0, sumDesc = 0;
254
+ for (const d of docs) {
255
+ sumName += d.nameLen;
256
+ sumNs += d.nsLen;
257
+ sumParam += d.paramLen;
258
+ sumDesc += d.descLen;
259
+ for (const term of d.allTerms) df.set(term, (df.get(term) || 0) + 1);
260
+ }
261
+ const avgName = sumName / N;
262
+ const avgNs = sumNs / N;
263
+ const avgParam = sumParam / N;
264
+ const avgDesc = sumDesc / N;
265
+
266
+ const idf = (term) => Math.log(1 + (N - (df.get(term) || 0) + 0.5) / ((df.get(term) || 0) + 0.5));
267
+ const bm25Field = (tf, len, avgLen, k1 = 1.5, b = 0.75) => {
268
+ if (!tf) return 0;
269
+ return (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (len / Math.max(1, avgLen))));
270
+ };
271
+
272
+ let cachedDocVectors = null;
273
+
274
+ function scoreOneQuery(doc, rawQuery) {
275
+ const qLower = String(rawQuery || '').trim().toLowerCase();
276
+ if (!qLower) return 0;
277
+ let score = 0;
278
+ if (doc.nameLower === qLower || doc.shortLower === qLower) {
279
+ score += 14.0;
280
+ } else if (doc.nameLower.includes(qLower) && qLower.length >= 3) {
281
+ score += 6.0;
282
+ }
283
+
284
+ const qTokens = tokenizeIdentifier(rawQuery);
285
+ const expanded = new Set();
286
+ for (const qt of qTokens) {
287
+ const w = idf(qt);
288
+ score += 3.0 * w * bm25Field(doc.nameTf.get(qt) || 0, doc.nameLen, avgName);
289
+ score += 2.2 * w * bm25Field(doc.nsTf.get(qt) || 0, doc.nsLen, avgNs);
290
+ score += 1.4 * w * bm25Field(doc.paramTf.get(qt) || 0, doc.paramLen, avgParam);
291
+ score += 1.0 * w * bm25Field(doc.descTf.get(qt) || 0, doc.descLen, avgDesc);
292
+ const syns = SYNONYM_MAP.get(qt);
293
+ if (syns) {
294
+ for (const s of syns) {
295
+ if (!qTokens.includes(s)) expanded.add(s);
296
+ }
297
+ }
298
+ }
299
+ for (const syn of expanded) {
300
+ const w = idf(syn) * 0.55;
301
+ score += 2.0 * w * bm25Field(doc.nameTf.get(syn) || 0, doc.nameLen, avgName);
302
+ score += 1.0 * w * bm25Field(doc.descTf.get(syn) || 0, doc.descLen, avgDesc);
303
+ }
304
+
305
+ const qBigrams = makeBigrams(qTokens);
306
+ for (const bg of qBigrams) {
307
+ if (doc.nameBigrams.has(bg)) score += 3.0;
308
+ else if (doc.descBigrams.has(bg)) score += 1.6;
309
+ }
310
+ return score;
311
+ }
312
+
313
+ function search({
314
+ query = '',
315
+ queries = null,
316
+ namespace = null,
317
+ risk = 'all',
318
+ limit = 5,
319
+ filterFn = null,
320
+ } = {}) {
321
+ const queryList = Array.isArray(queries) && queries.length
322
+ ? queries.map(String).filter((s) => s.trim())
323
+ : (String(query || '').trim() ? [String(query).trim()] : []);
324
+ const cap = Math.min(10, Math.max(1, Number(limit || 5)));
325
+ const nsFilter = namespace ? String(namespace).toLowerCase() : null;
326
+
327
+ const scored = [];
328
+ for (const d of docs) {
329
+ if (nsFilter && d.namespace.toLowerCase() !== nsFilter && !d.namespace.toLowerCase().endsWith(`.${nsFilter}`)) {
330
+ continue;
331
+ }
332
+ if (risk === 'read_only' && d.tool.tier !== 'read_only') continue;
333
+ if (risk === 'write' && d.tool.tier === 'read_only') continue;
334
+ if (typeof filterFn === 'function' && !filterFn(d.tool)) continue;
335
+
336
+ let best = 0;
337
+ if (!queryList.length) {
338
+ best = 0.01;
339
+ } else {
340
+ for (const q of queryList) {
341
+ const s = scoreOneQuery(d, q);
342
+ if (s > best) best = s;
343
+ }
344
+ }
345
+ if (best > 0) {
346
+ scored.push({ doc: d, score: best });
347
+ }
348
+ }
349
+
350
+ scored.sort((a, b) => (b.score - a.score) || a.doc.nameLower.localeCompare(b.doc.nameLower));
351
+ const namespaces = [...new Set(docs.filter((d) => !filterFn || filterFn(d.tool)).map((d) => d.namespace))].sort();
352
+ return {
353
+ queries: queryList,
354
+ totalMatches: scored.length,
355
+ availableNamespaces: namespaces,
356
+ hits: scored.slice(0, cap).map(({ doc, score }) => ({
357
+ score: Number(score.toFixed(3)),
358
+ namespace: doc.namespace,
359
+ ...doc.tool,
360
+ })),
361
+ };
362
+ }
363
+
364
+ async function searchAsync(opts = {}) {
365
+ const lex = search(opts);
366
+ if (typeof embedFn !== 'function' || !docs.length) return lex;
367
+ const qText = Array.isArray(opts.queries) && opts.queries.length
368
+ ? opts.queries.join(' | ')
369
+ : String(opts.query || '').trim();
370
+ if (!qText) return lex;
371
+
372
+ try {
373
+ if (!cachedDocVectors) {
374
+ const texts = docs.map((d) => `${d.tool.name} ${d.namespace} ${d.tool.shortDescription || d.tool.description || ''}`);
375
+ cachedDocVectors = await embedFn(texts);
376
+ }
377
+ const [qVec] = await embedFn([qText]);
378
+ if (!Array.isArray(qVec) || !Array.isArray(cachedDocVectors)) return lex;
379
+
380
+ const lexByName = new Map(lex.hits.map((h) => [h.name, h.score]));
381
+ const maxLex = Math.max(1, ...lex.hits.map((h) => h.score));
382
+ const fused = [];
383
+ for (let i = 0; i < docs.length; i++) {
384
+ const d = docs[i];
385
+ if (typeof opts.filterFn === 'function' && !opts.filterFn(d.tool)) continue;
386
+ const lScore = (lexByName.get(d.tool.name) || 0) / maxLex;
387
+ const sScore = Math.max(0, cosineSim(qVec, cachedDocVectors[i]));
388
+ const score = 0.65 * lScore + 0.35 * sScore;
389
+ if (score > 0.08) fused.push({ doc: d, score });
390
+ }
391
+ fused.sort((a, b) => (b.score - a.score) || a.doc.nameLower.localeCompare(b.doc.nameLower));
392
+ const cap = Math.min(10, Math.max(1, Number(opts.limit || 5)));
393
+ return {
394
+ ...lex,
395
+ totalMatches: fused.length,
396
+ hits: fused.slice(0, cap).map(({ doc, score }) => ({
397
+ score: Number(score.toFixed(3)),
398
+ namespace: doc.namespace,
399
+ ...doc.tool,
400
+ })),
401
+ };
402
+ } catch {
403
+ return lex;
404
+ }
405
+ }
406
+
407
+ return {
408
+ count: docs.length,
409
+ search,
410
+ searchAsync,
411
+ };
412
+ }
413
+
414
+ // ── Frozen `<available_tools>` Region A Renderer (§13 Cache-Safe, ≤ 1,500 tok) ─
415
+ export function buildAvailableToolsIndex(deferredTools = [], {
416
+ maxIndexTokens = 1500,
417
+ namespaceDescriptions = {},
418
+ } = {}) {
419
+ const sorted = [...(deferredTools || [])].sort((a, b) => String(a.name).localeCompare(String(b.name)));
420
+ if (!sorted.length) {
421
+ return { text: '', hash: 'empty', deferredCount: 0, estimatedTokens: 0, tier: 'none' };
422
+ }
423
+
424
+ const byNs = new Map();
425
+ for (const t of sorted) {
426
+ const ns = inferNamespace(t);
427
+ if (!byNs.has(ns)) byNs.set(ns, []);
428
+ byNs.get(ns).push(t);
429
+ }
430
+ const nsKeys = [...byNs.keys()].sort();
431
+ const maxChars = Math.max(800, Number(maxIndexTokens || 1500) * 4);
432
+
433
+ const headerLine = 'Additional tools are loaded on demand. Call tool_search({ queries: [...] }) to inspect exact parameter schemas, then call tool_invoke({ name, arguments }).';
434
+
435
+ const renderTier = (mode) => {
436
+ const lines = [headerLine];
437
+ for (const ns of nsKeys) {
438
+ const list = byNs.get(ns);
439
+ const nsDesc = namespaceDescriptions[ns] ? ` -- ${sanitizeInline(namespaceDescriptions[ns], 80)}` : '';
440
+ if (mode === 'detailed') {
441
+ lines.push(`## ${sanitizeInline(ns, 48)} (${list.length})${nsDesc}`);
442
+ for (const t of list) {
443
+ const desc = t.shortDescription ? sanitizeInline(t.shortDescription, 80) : firstSentence(t.description || '', 80);
444
+ lines.push(`- ${sanitizeInline(t.name, 64)}: ${desc}`);
445
+ }
446
+ } else if (mode === 'compact') {
447
+ lines.push(`## ${sanitizeInline(ns, 48)} (${list.length} tools)${nsDesc}`);
448
+ const reads = list.filter((x) => x.tier === 'read_only').map((x) => inferShortName(x));
449
+ const writes = list.filter((x) => x.tier !== 'read_only').map((x) => inferShortName(x));
450
+ if (reads.length) {
451
+ const head = reads.slice(0, 4).join(', ');
452
+ lines.push(` read: ${head}${reads.length > 4 ? ` (+${reads.length - 4})` : ''}`);
453
+ }
454
+ if (writes.length) {
455
+ const head = writes.slice(0, 4).join(', ');
456
+ lines.push(` write: ${head}${writes.length > 4 ? ` (+${writes.length - 4})` : ''}`);
457
+ }
458
+ } else {
459
+ const keywords = [...new Set(list.flatMap((t) => tokenizeIdentifier(inferShortName(t))).slice(0, 8))].join(', ');
460
+ lines.push(`- ${sanitizeInline(ns, 48)} (${list.length} tools)${nsDesc || (keywords ? `: ${keywords}` : '')}`);
461
+ }
462
+ }
463
+ return lines.join('\n');
464
+ };
465
+
466
+ let chosenTier = sorted.length <= 40 ? 'detailed' : (sorted.length <= 150 ? 'compact' : 'namespace');
467
+ let body = renderTier(chosenTier);
468
+ if (body.length > maxChars && chosenTier === 'detailed') {
469
+ chosenTier = 'compact';
470
+ body = renderTier(chosenTier);
471
+ }
472
+ if (body.length > maxChars && chosenTier === 'compact') {
473
+ chosenTier = 'namespace';
474
+ body = renderTier(chosenTier);
475
+ }
476
+ if (body.length > maxChars) {
477
+ body = body.slice(0, maxChars - 36).trim() + '\n... (use tool_search to explore)';
478
+ }
479
+
480
+ const hash = crypto.createHash('sha256').update(body).digest('hex').slice(0, 12);
481
+ const text = `<available_tools snapshot="${hash}" deferred="${sorted.length}">\n${body}\n</available_tools>`;
482
+ return {
483
+ text,
484
+ hash,
485
+ deferredCount: sorted.length,
486
+ estimatedTokens: Math.ceil(text.length / 4),
487
+ tier: chosenTier,
488
+ };
489
+ }