@juspay/neurolink 9.89.0 → 9.91.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/dist/adapters/tts/googleTTSHandler.d.ts +10 -0
  3. package/dist/adapters/tts/googleTTSHandler.js +27 -18
  4. package/dist/agent/directTools.js +1 -1
  5. package/dist/browser/neurolink.min.js +387 -365
  6. package/dist/core/baseProvider.d.ts +37 -3
  7. package/dist/core/baseProvider.js +167 -44
  8. package/dist/core/modules/GenerationHandler.js +7 -1
  9. package/dist/core/modules/ToolsManager.js +5 -0
  10. package/dist/core/toolDedup.js +4 -1
  11. package/dist/index.d.ts +1 -1
  12. package/dist/index.js +1 -1
  13. package/dist/lib/adapters/tts/googleTTSHandler.d.ts +10 -0
  14. package/dist/lib/adapters/tts/googleTTSHandler.js +27 -18
  15. package/dist/lib/agent/directTools.js +1 -1
  16. package/dist/lib/core/baseProvider.d.ts +37 -3
  17. package/dist/lib/core/baseProvider.js +167 -44
  18. package/dist/lib/core/modules/GenerationHandler.js +7 -1
  19. package/dist/lib/core/modules/ToolsManager.js +5 -0
  20. package/dist/lib/core/toolDedup.js +4 -1
  21. package/dist/lib/index.d.ts +1 -1
  22. package/dist/lib/index.js +1 -1
  23. package/dist/lib/mcp/toolRegistry.js +7 -2
  24. package/dist/lib/neurolink.d.ts +31 -1
  25. package/dist/lib/neurolink.js +151 -26
  26. package/dist/lib/rag/index.d.ts +1 -0
  27. package/dist/lib/rag/index.js +2 -0
  28. package/dist/lib/rag/stores/chroma.d.ts +90 -0
  29. package/dist/lib/rag/stores/chroma.js +281 -0
  30. package/dist/lib/rag/stores/index.d.ts +21 -0
  31. package/dist/lib/rag/stores/index.js +22 -0
  32. package/dist/lib/rag/stores/pgvector.d.ts +95 -0
  33. package/dist/lib/rag/stores/pgvector.js +400 -0
  34. package/dist/lib/rag/stores/pinecone.d.ts +85 -0
  35. package/dist/lib/rag/stores/pinecone.js +159 -0
  36. package/dist/lib/server/routes/agentRoutes.js +25 -2
  37. package/dist/lib/tools/toolDiscovery.d.ts +48 -0
  38. package/dist/lib/tools/toolDiscovery.js +231 -0
  39. package/dist/lib/tools/toolGate.d.ts +16 -0
  40. package/dist/lib/tools/toolGate.js +51 -0
  41. package/dist/lib/tools/toolPolicy.d.ts +40 -0
  42. package/dist/lib/tools/toolPolicy.js +194 -0
  43. package/dist/lib/types/config.d.ts +43 -3
  44. package/dist/lib/types/index.d.ts +3 -0
  45. package/dist/lib/types/index.js +3 -0
  46. package/dist/lib/types/providers.d.ts +8 -0
  47. package/dist/lib/types/rag.d.ts +30 -0
  48. package/dist/lib/types/toolResolution.d.ts +73 -0
  49. package/dist/lib/types/toolResolution.js +11 -0
  50. package/dist/lib/types/vectorStoreChroma.d.ts +67 -0
  51. package/dist/lib/types/vectorStoreChroma.js +12 -0
  52. package/dist/lib/types/vectorStorePinecone.d.ts +48 -0
  53. package/dist/lib/types/vectorStorePinecone.js +12 -0
  54. package/dist/mcp/toolRegistry.js +7 -2
  55. package/dist/neurolink.d.ts +31 -1
  56. package/dist/neurolink.js +151 -26
  57. package/dist/rag/index.d.ts +1 -0
  58. package/dist/rag/index.js +2 -0
  59. package/dist/rag/stores/chroma.d.ts +90 -0
  60. package/dist/rag/stores/chroma.js +280 -0
  61. package/dist/rag/stores/index.d.ts +21 -0
  62. package/dist/rag/stores/index.js +21 -0
  63. package/dist/rag/stores/pgvector.d.ts +95 -0
  64. package/dist/rag/stores/pgvector.js +399 -0
  65. package/dist/rag/stores/pinecone.d.ts +85 -0
  66. package/dist/rag/stores/pinecone.js +158 -0
  67. package/dist/server/routes/agentRoutes.js +25 -2
  68. package/dist/tools/toolDiscovery.d.ts +48 -0
  69. package/dist/tools/toolDiscovery.js +230 -0
  70. package/dist/tools/toolGate.d.ts +16 -0
  71. package/dist/tools/toolGate.js +50 -0
  72. package/dist/tools/toolPolicy.d.ts +40 -0
  73. package/dist/tools/toolPolicy.js +193 -0
  74. package/dist/types/config.d.ts +43 -3
  75. package/dist/types/index.d.ts +3 -0
  76. package/dist/types/index.js +3 -0
  77. package/dist/types/providers.d.ts +8 -0
  78. package/dist/types/rag.d.ts +30 -0
  79. package/dist/types/toolResolution.d.ts +73 -0
  80. package/dist/types/toolResolution.js +10 -0
  81. package/dist/types/vectorStoreChroma.d.ts +67 -0
  82. package/dist/types/vectorStoreChroma.js +11 -0
  83. package/dist/types/vectorStorePinecone.d.ts +48 -0
  84. package/dist/types/vectorStorePinecone.js +11 -0
  85. package/package.json +12 -7
@@ -0,0 +1,158 @@
1
+ /**
2
+ * Pinecone Vector Store Adapter
3
+ *
4
+ * Implements the `VectorStore` contract (see `src/lib/types/rag.ts`) against
5
+ * a Pinecone index, using client injection: callers construct their own
6
+ * `@pinecone-database/pinecone` client/index and pass it in. No Pinecone SDK
7
+ * is a runtime dependency of this package — `PineconeIndexLike` below is a
8
+ * minimal structural interface satisfied by the real `Index` object (and by
9
+ * hand-written mocks in tests).
10
+ *
11
+ * ## indexName -> namespace mapping
12
+ *
13
+ * The `VectorStore` contract's `indexName` parameter is mapped onto a
14
+ * Pinecone **namespace**, not a Pinecone index. Pinecone ties one client
15
+ * `Index` object to a single physical index (created ahead of time via
16
+ * Pinecone's control-plane API); namespaces are Pinecone's mechanism for
17
+ * partitioning data *within* that one index, and are the closest match to
18
+ * this SDK's per-call `indexName` concept.
19
+ *
20
+ * - If the injected client exposes `.namespace(name)` (as the real
21
+ * `@pinecone-database/pinecone` `Index.namespace()` does), every
22
+ * query/upsert/delete call routes through `client.namespace(indexName)`.
23
+ * - If the injected client does not expose `.namespace` (e.g. a caller
24
+ * already pre-scoped their client to a fixed namespace, or a minimal
25
+ * mock), calls go straight to the injected client and `indexName` has
26
+ * no further effect. Document this for your callers if you construct a
27
+ * `PineconeVectorStore` from a namespace-fixed client.
28
+ *
29
+ * ## Metadata filter translation
30
+ *
31
+ * `MetadataFilter` (Mongo/Sift-style) is translated to Pinecone's native
32
+ * filter DSL, which already shares the same `$op` shape for comparison
33
+ * operators. Supported operators: `$eq`, `$ne`, `$gt`, `$gte`, `$lt`,
34
+ * `$lte`, `$in`, `$nin` (field-level) and `$and` / `$or` (logical,
35
+ * recursively translated). Any other operator — `$not`, `$nor`, `$exists`,
36
+ * `$contains`, `$regex`, `$size` — has no Pinecone equivalent and throws a
37
+ * clear `Error` naming the offending operator/field rather than silently
38
+ * mis-filtering (Pinecone's filter DSL has no negation, existence, string
39
+ * containment, regex, or array-length operators).
40
+ */
41
+ /** Comparison operators Pinecone's filter DSL can express, verbatim. */
42
+ const SUPPORTED_COMPARISON_OPS = new Set([
43
+ "$eq",
44
+ "$ne",
45
+ "$gt",
46
+ "$gte",
47
+ "$lt",
48
+ "$lte",
49
+ "$in",
50
+ "$nin",
51
+ ]);
52
+ /** Logical operators Pinecone's filter DSL can express, verbatim. */
53
+ const SUPPORTED_LOGICAL_OPS = new Set(["$and", "$or"]);
54
+ /**
55
+ * Translate a `MetadataFilter` into Pinecone's native filter object.
56
+ *
57
+ * Throws a descriptive `Error` for any operator Pinecone cannot express
58
+ * ($not, $nor, $exists, $contains, $regex, $size) instead of silently
59
+ * dropping or mis-applying it.
60
+ */
61
+ export function translatePineconeFilter(filter) {
62
+ const result = {};
63
+ for (const [key, value] of Object.entries(filter)) {
64
+ if (key.startsWith("$")) {
65
+ if (!SUPPORTED_LOGICAL_OPS.has(key)) {
66
+ throw new Error(`PineconeVectorStore: MetadataFilter operator "${key}" cannot be translated to Pinecone's filter DSL ` +
67
+ `(supported logical operators: ${[...SUPPORTED_LOGICAL_OPS].join(", ")}). ` +
68
+ `Restructure the filter to avoid "${key}".`);
69
+ }
70
+ const subFilters = value;
71
+ result[key] = subFilters.map((sub) => translatePineconeFilter(sub));
72
+ continue;
73
+ }
74
+ if (typeof value === "object" && value !== null && !Array.isArray(value)) {
75
+ const ops = value;
76
+ const translatedOps = {};
77
+ for (const opKey of Object.keys(ops)) {
78
+ if (!SUPPORTED_COMPARISON_OPS.has(opKey)) {
79
+ throw new Error(`PineconeVectorStore: MetadataFilter operator "${opKey}" on field "${key}" cannot be translated to ` +
80
+ `Pinecone's filter DSL (supported: ${[...SUPPORTED_COMPARISON_OPS].join(", ")}). ` +
81
+ `Restructure the filter to avoid "${opKey}" on "${key}".`);
82
+ }
83
+ translatedOps[opKey] = ops[opKey];
84
+ }
85
+ result[key] = translatedOps;
86
+ }
87
+ else {
88
+ // Primitive/array/null value: Pinecone treats a bare value as
89
+ // implicit equality, matching InMemoryVectorStore's direct-equality
90
+ // fallback for non-object filter values.
91
+ result[key] = value;
92
+ }
93
+ }
94
+ return result;
95
+ }
96
+ /**
97
+ * Pinecone-backed implementation of the `VectorStore` contract.
98
+ *
99
+ * @example
100
+ * ```typescript
101
+ * import { Pinecone } from '@pinecone-database/pinecone';
102
+ * import { PineconeVectorStore } from '@juspay/neurolink/rag';
103
+ *
104
+ * const client = new Pinecone({ apiKey: process.env.PINECONE_API_KEY! });
105
+ * const store = new PineconeVectorStore(client.index('my-index'));
106
+ *
107
+ * await store.upsert('tenant-a', [{ id: '1', vector: [...], metadata: { text: 'hello' } }]);
108
+ * const results = await store.query({ indexName: 'tenant-a', queryVector: [...], topK: 5 });
109
+ * ```
110
+ */
111
+ export class PineconeVectorStore {
112
+ client;
113
+ constructor(client) {
114
+ this.client = client;
115
+ }
116
+ /** Resolve the Pinecone client scoped to `indexName` (see class docs re: namespace mapping). */
117
+ resolveClient(indexName) {
118
+ return this.client.namespace
119
+ ? this.client.namespace(indexName)
120
+ : this.client;
121
+ }
122
+ async query(params) {
123
+ const { indexName, queryVector, topK = 10, filter, includeVectors = false, } = params;
124
+ const target = this.resolveClient(indexName);
125
+ const response = await target.query({
126
+ vector: queryVector,
127
+ topK,
128
+ ...(filter ? { filter: translatePineconeFilter(filter) } : {}),
129
+ includeMetadata: true,
130
+ includeValues: includeVectors,
131
+ });
132
+ const matches = response.matches ?? [];
133
+ return matches.map((match) => {
134
+ const metadata = match.metadata ?? {};
135
+ return {
136
+ id: match.id,
137
+ score: match.score,
138
+ text: metadata.text,
139
+ metadata,
140
+ ...(includeVectors && match.values ? { vector: match.values } : {}),
141
+ };
142
+ });
143
+ }
144
+ /** Add or update vectors in the namespace mapped from `indexName`. */
145
+ async upsert(indexName, items) {
146
+ const target = this.resolveClient(indexName);
147
+ await target.upsert(items.map((item) => ({
148
+ id: item.id,
149
+ values: item.vector,
150
+ metadata: item.metadata ?? {},
151
+ })));
152
+ }
153
+ /** Delete vectors by id from the namespace mapped from `indexName`. */
154
+ async delete(indexName, ids) {
155
+ const target = this.resolveClient(indexName);
156
+ await target.deleteMany(ids);
157
+ }
158
+ }
@@ -6,6 +6,27 @@ import { SpanStatusCode } from "@opentelemetry/api";
6
6
  import { ProviderFactory } from "../../factories/providerFactory.js";
7
7
  import { withSpan } from "../../telemetry/withSpan.js";
8
8
  import { tracers } from "../../telemetry/tracers.js";
9
+ import { logger } from "../../utils/logger.js";
10
+ /**
11
+ * Resolve `request.tools` (an array of tool NAMES shared by the execute and
12
+ * stream endpoints) into a `toolFilter` whitelist. The field was previously
13
+ * accepted and silently ignored, so entries are validated against registered
14
+ * names and the whole field fails open (undefined — keep all tools) when
15
+ * nothing matches: existing clients sending unknown names must not lose
16
+ * their tool set.
17
+ */
18
+ async function resolveRequestToolFilter(ctx, requestedTools) {
19
+ if (!Array.isArray(requestedTools) || requestedTools.length === 0) {
20
+ return undefined;
21
+ }
22
+ const registered = new Set((await ctx.neurolink.getAllAvailableTools()).map((t) => t.name));
23
+ const known = requestedTools.filter((name) => typeof name === "string" && registered.has(name));
24
+ if (known.length > 0) {
25
+ return known;
26
+ }
27
+ logger.warn("[agentRoutes] Ignoring request.tools — no entries match registered tool names (legacy fail-open)", { requested: requestedTools });
28
+ return undefined;
29
+ }
9
30
  import { createStreamRedactor } from "../utils/redaction.js";
10
31
  import { AgentExecuteRequestSchema, createErrorResponse as createError, EmbedManyRequestSchema, EmbedRequestSchema, SkillCreateRequestSchema, SkillUpdateRequestSchema, validateRequest, } from "../utils/validation.js";
11
32
  /**
@@ -65,6 +86,7 @@ export function createAgentRoutes(basePath = "/api") {
65
86
  const input = typeof request.input === "string"
66
87
  ? { text: request.input }
67
88
  : request.input;
89
+ const requestToolFilter = await resolveRequestToolFilter(ctx, request.tools);
68
90
  const result = await ctx.neurolink.generate({
69
91
  input,
70
92
  provider: request.provider,
@@ -72,8 +94,7 @@ export function createAgentRoutes(basePath = "/api") {
72
94
  systemPrompt: request.systemPrompt,
73
95
  temperature: request.temperature,
74
96
  maxTokens: request.maxTokens,
75
- // Note: tools should be passed as Record<string, Tool> in generate options
76
- // If request.tools is an array of tool names, we skip them
97
+ ...(requestToolFilter ? { toolFilter: requestToolFilter } : {}),
77
98
  context: {
78
99
  // When an authenticated user context exists (set by auth middleware),
79
100
  // always use its IDs to prevent caller-supplied impersonation.
@@ -118,6 +139,7 @@ export function createAgentRoutes(basePath = "/api") {
118
139
  const input = typeof request.input === "string"
119
140
  ? { text: request.input }
120
141
  : request.input;
142
+ const streamToolFilter = await resolveRequestToolFilter(ctx, request.tools);
121
143
  const result = await ctx.neurolink.stream({
122
144
  input,
123
145
  provider: request.provider,
@@ -125,6 +147,7 @@ export function createAgentRoutes(basePath = "/api") {
125
147
  systemPrompt: request.systemPrompt,
126
148
  temperature: request.temperature,
127
149
  maxTokens: request.maxTokens,
150
+ ...(streamToolFilter ? { toolFilter: streamToolFilter } : {}),
128
151
  context: {
129
152
  // When an authenticated user context exists (set by auth middleware),
130
153
  // always use its IDs to prevent caller-supplied impersonation.
@@ -0,0 +1,48 @@
1
+ /**
2
+ * On-demand tool discovery (`tools.discovery: true`).
3
+ *
4
+ * Instead of sending every external MCP tool's full schema on every request,
5
+ * the model receives the hot set (built-in tools, per-call tools, explicitly
6
+ * requested tools, and previously discovered tools) plus ONE `search_tools`
7
+ * meta-tool whose description embeds a compact name+summary catalog of the
8
+ * deferred tools. Calling `search_tools` loads matching tools:
9
+ * - immediately into the live tool record (providers that re-read the tools
10
+ * record between agent-loop steps — the AI SDK path — can call them on the
11
+ * very next step), and
12
+ * - into the session pin set (all providers include them in full on every
13
+ * subsequent call of the session).
14
+ *
15
+ * Evidence for this design: catalogs past ~30-50 tools measurably degrade
16
+ * tool-selection accuracy, and deferral+search both cuts definition tokens by
17
+ * ~85% and RAISES selection accuracy (Anthropic MCP-eval 49%→74% on Opus 4).
18
+ * The catalog construction (byte-stable, name-sorted, char-budgeted with a
19
+ * degradation ladder) mirrors the proven skills-index pattern in
20
+ * src/lib/skills/skillTools.ts.
21
+ */
22
+ import type { Tool } from "../types/index.js";
23
+ /**
24
+ * Above this many tools, a WARN suggests enabling `tools.discovery` when it
25
+ * is off. Chosen from the measured degradation range (30-50 tools).
26
+ */
27
+ export declare const LARGE_CATALOG_WARN_THRESHOLD = 40;
28
+ /** True when the value is a search_tools meta-tool generated by this module. */
29
+ export declare function isDiscoveryMetaTool(tool: unknown): boolean;
30
+ /**
31
+ * Partition a (post-gate) tools record for discovery: deferrable tools that
32
+ * are not pinned are removed from the record and replaced by one
33
+ * `search_tools` meta-tool. Returns the hot record; when nothing is
34
+ * deferrable the input record is returned unchanged.
35
+ *
36
+ * The returned record is mutated by search_tools on hydration (call-scoped —
37
+ * BaseProvider builds a fresh merged record per call), which is what makes
38
+ * discovered tools callable on subsequent steps of the same agent loop on
39
+ * providers that re-read the record between steps.
40
+ */
41
+ export declare function partitionToolsForDiscovery(tools: Record<string, Tool>, args: {
42
+ /** External-tool names present in `tools` that are allowed to defer. */
43
+ deferrableNames: string[];
44
+ /** Session-pinned names (previously discovered) — never deferred. */
45
+ pinnedNames: ReadonlySet<string>;
46
+ /** Called with newly discovered names — persists session pins. */
47
+ onHydrate: (names: string[]) => void;
48
+ }): Record<string, Tool>;
@@ -0,0 +1,230 @@
1
+ /**
2
+ * On-demand tool discovery (`tools.discovery: true`).
3
+ *
4
+ * Instead of sending every external MCP tool's full schema on every request,
5
+ * the model receives the hot set (built-in tools, per-call tools, explicitly
6
+ * requested tools, and previously discovered tools) plus ONE `search_tools`
7
+ * meta-tool whose description embeds a compact name+summary catalog of the
8
+ * deferred tools. Calling `search_tools` loads matching tools:
9
+ * - immediately into the live tool record (providers that re-read the tools
10
+ * record between agent-loop steps — the AI SDK path — can call them on the
11
+ * very next step), and
12
+ * - into the session pin set (all providers include them in full on every
13
+ * subsequent call of the session).
14
+ *
15
+ * Evidence for this design: catalogs past ~30-50 tools measurably degrade
16
+ * tool-selection accuracy, and deferral+search both cuts definition tokens by
17
+ * ~85% and RAISES selection accuracy (Anthropic MCP-eval 49%→74% on Opus 4).
18
+ * The catalog construction (byte-stable, name-sorted, char-budgeted with a
19
+ * degradation ladder) mirrors the proven skills-index pattern in
20
+ * src/lib/skills/skillTools.ts.
21
+ */
22
+ import { z } from "zod";
23
+ import { tool as createAISDKTool } from "../utils/tool.js";
24
+ import { convertZodToJsonSchema } from "../utils/schemaConversion.js";
25
+ import { logger } from "../utils/logger.js";
26
+ /**
27
+ * Above this many tools, a WARN suggests enabling `tools.discovery` when it
28
+ * is off. Chosen from the measured degradation range (30-50 tools).
29
+ */
30
+ export const LARGE_CATALOG_WARN_THRESHOLD = 40;
31
+ /** Char budget for the catalog embedded in the search_tools description. */
32
+ const CATALOG_CHAR_BUDGET = 15_000;
33
+ /** Max tools loaded per search_tools call. */
34
+ const MAX_SEARCH_RESULTS = 5;
35
+ const SUMMARY_MAX_CHARS = 100;
36
+ /**
37
+ * Marker property stamped on the generated search_tools meta-tool so
38
+ * re-entrant resolution passes (e.g. the stream → generate fallback, whose
39
+ * options.tools already contain a partitioned record) can distinguish OUR
40
+ * stale meta-tool from a real user tool that happens to share the name.
41
+ */
42
+ const DISCOVERY_META_TOOL_MARKER = "__nlDiscoveryMetaTool";
43
+ /** True when the value is a search_tools meta-tool generated by this module. */
44
+ export function isDiscoveryMetaTool(tool) {
45
+ return (!!tool &&
46
+ typeof tool === "object" &&
47
+ tool[DISCOVERY_META_TOOL_MARKER] === true);
48
+ }
49
+ let warnedSearchToolsCollision = false;
50
+ function warnSearchToolsCollisionOnce() {
51
+ if (warnedSearchToolsCollision) {
52
+ return;
53
+ }
54
+ warnedSearchToolsCollision = true;
55
+ logger.warn("[ToolDiscovery] A registered tool is already named 'search_tools' — on-demand discovery is skipped to avoid shadowing it. Rename that tool to use tools.discovery.");
56
+ }
57
+ /** First sentence of a description, capped for the catalog line. */
58
+ function summarize(description) {
59
+ const firstSentence = description.split(/(?<=[.!?])\s/)[0] ?? description;
60
+ const trimmed = firstSentence.trim().replace(/\s+/g, " ");
61
+ return trimmed.length > SUMMARY_MAX_CHARS
62
+ ? trimmed.slice(0, SUMMARY_MAX_CHARS - 1) + "…"
63
+ : trimmed;
64
+ }
65
+ /**
66
+ * Render the deferred-tool catalog, byte-stable (name-sorted input) and
67
+ * char-budgeted with a degradation ladder: full one-liners → name-only lines
68
+ * → truncated with a count note. Entries are never silently dropped without
69
+ * the note, so the model always knows the catalog's true size.
70
+ */
71
+ function renderCatalog(entries) {
72
+ const full = entries.map((e) => `- ${e.name}: ${e.summary}`);
73
+ let text = full.join("\n");
74
+ if (text.length <= CATALOG_CHAR_BUDGET) {
75
+ return text;
76
+ }
77
+ const nameOnly = entries.map((e) => `- ${e.name}`);
78
+ text = nameOnly.join("\n");
79
+ if (text.length <= CATALOG_CHAR_BUDGET) {
80
+ return text;
81
+ }
82
+ const kept = [];
83
+ let used = 0;
84
+ for (const line of nameOnly) {
85
+ if (used + line.length + 1 > CATALOG_CHAR_BUDGET - 80) {
86
+ break;
87
+ }
88
+ kept.push(line);
89
+ used += line.length + 1;
90
+ }
91
+ const remaining = entries.length - kept.length;
92
+ return `${kept.join("\n")}\n… and ${remaining} more (search by task description to find them)`;
93
+ }
94
+ /** Lexical relevance score of one catalog entry for a query. Zero = no match. */
95
+ function scoreEntry(queryLower, queryTokens, entry, fullDescription) {
96
+ const nameLower = entry.name.toLowerCase();
97
+ if (nameLower === queryLower) {
98
+ return 100;
99
+ }
100
+ let score = 0;
101
+ if (queryLower.includes(nameLower) || nameLower.includes(queryLower)) {
102
+ score += 20;
103
+ }
104
+ const haystack = `${nameLower} ${fullDescription.toLowerCase()}`;
105
+ for (const token of queryTokens) {
106
+ if (token.length < 3) {
107
+ continue;
108
+ }
109
+ if (nameLower.includes(token)) {
110
+ score += 8;
111
+ }
112
+ else if (haystack.includes(token)) {
113
+ score += 2;
114
+ }
115
+ }
116
+ if (entry.serverId && queryLower.includes(entry.serverId.toLowerCase())) {
117
+ score += 3;
118
+ }
119
+ return score;
120
+ }
121
+ /**
122
+ * Partition a (post-gate) tools record for discovery: deferrable tools that
123
+ * are not pinned are removed from the record and replaced by one
124
+ * `search_tools` meta-tool. Returns the hot record; when nothing is
125
+ * deferrable the input record is returned unchanged.
126
+ *
127
+ * The returned record is mutated by search_tools on hydration (call-scoped —
128
+ * BaseProvider builds a fresh merged record per call), which is what makes
129
+ * discovered tools callable on subsequent steps of the same agent loop on
130
+ * providers that re-read the record between steps.
131
+ */
132
+ export function partitionToolsForDiscovery(tools, args) {
133
+ const deferredNames = args.deferrableNames
134
+ .filter((n) => n in tools && !args.pinnedNames.has(n))
135
+ .sort();
136
+ if (deferredNames.length === 0) {
137
+ return tools;
138
+ }
139
+ // Never shadow a real tool that happens to be named "search_tools" (a
140
+ // custom tool or an external MCP tool could claim the name). Discovery is
141
+ // skipped for the request rather than silently replacing the caller's tool.
142
+ if ("search_tools" in tools) {
143
+ warnSearchToolsCollisionOnce();
144
+ return tools;
145
+ }
146
+ const deferredSet = new Set(deferredNames);
147
+ // Null prototype: a tool named "__proto__" must become an own entry, not a
148
+ // prototype mutation (mirrors BaseProvider.sortToolRecord) — this record is
149
+ // also mutated later by search_tools hydration.
150
+ const hot = Object.create(null);
151
+ for (const [name, toolDef] of Object.entries(tools)) {
152
+ if (!deferredSet.has(name)) {
153
+ hot[name] = toolDef;
154
+ }
155
+ }
156
+ const entries = deferredNames.map((name) => {
157
+ const t = tools[name];
158
+ return {
159
+ name,
160
+ summary: summarize(t.description ?? ""),
161
+ serverId: typeof t.serverId === "string" ? t.serverId : undefined,
162
+ };
163
+ });
164
+ hot["search_tools"] = buildSearchTool(entries, tools, hot, args.onHydrate);
165
+ logger.debug("[ToolDiscovery] Deferred external tools behind search_tools", {
166
+ hotCount: Object.keys(hot).length - 1,
167
+ deferredCount: deferredNames.length,
168
+ pinnedCount: args.pinnedNames.size,
169
+ });
170
+ return hot;
171
+ }
172
+ function buildSearchTool(entries, allTools, hotRecord, onHydrate) {
173
+ const catalog = renderCatalog(entries);
174
+ const searchTool = createAISDKTool({
175
+ description: `Search and load additional tools on demand. ${entries.length} more tools are available but not yet loaded:\n` +
176
+ `${catalog}\n` +
177
+ `Call this with a task description or an exact tool name; matching tools are loaded and become directly callable.`,
178
+ inputSchema: z.object({
179
+ query: z
180
+ .string()
181
+ .describe("What you need to do, or an exact tool name to load"),
182
+ }),
183
+ execute: async ({ query }) => {
184
+ const queryLower = query.toLowerCase().trim();
185
+ const queryTokens = queryLower.split(/[^a-z0-9_.-]+/).filter(Boolean);
186
+ const scored = entries
187
+ .map((entry) => {
188
+ const full = allTools[entry.name];
189
+ return {
190
+ entry,
191
+ score: scoreEntry(queryLower, queryTokens, entry, full?.description ?? ""),
192
+ };
193
+ })
194
+ .filter((s) => s.score > 0)
195
+ .sort((a, b) => b.score - a.score || (a.entry.name < b.entry.name ? -1 : 1))
196
+ .slice(0, MAX_SEARCH_RESULTS);
197
+ if (scored.length === 0) {
198
+ return {
199
+ found: 0,
200
+ message: "No tools matched. Try different words, or use one of the exact names from the catalog in this tool's description.",
201
+ };
202
+ }
203
+ const loaded = scored.map(({ entry }) => {
204
+ const toolDef = allTools[entry.name];
205
+ // Live hydration: callable on subsequent loop steps where the
206
+ // provider re-reads the record.
207
+ hotRecord[entry.name] = toolDef;
208
+ const def = toolDef;
209
+ return {
210
+ name: entry.name,
211
+ description: def.description ?? "",
212
+ inputSchema: convertZodToJsonSchema(def.inputSchema),
213
+ };
214
+ });
215
+ // Session pinning: included in full on every subsequent call.
216
+ onHydrate(loaded.map((t) => t.name));
217
+ logger.debug("[ToolDiscovery] search_tools hydrated tools", {
218
+ query,
219
+ loaded: loaded.map((t) => t.name),
220
+ });
221
+ return {
222
+ found: loaded.length,
223
+ tools: loaded,
224
+ message: "These tools are now loaded for this session. Call them directly by name with arguments matching their inputSchema. If a call reports the tool as unavailable, it will be available from the next message onward.",
225
+ };
226
+ },
227
+ });
228
+ searchTool[DISCOVERY_META_TOOL_MARKER] = true;
229
+ return searchTool;
230
+ }
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Tool gate — applies a ResolvedToolPolicy to a tools record. This is the
3
+ * ONE place where include/exclude filtering of the native tool set happens;
4
+ * every generate/stream path reaches it via BaseProvider.
5
+ *
6
+ * Generic over the tool value type so it works for AI-SDK Tool records,
7
+ * ToolInfo listings, and provider wire formats alike.
8
+ */
9
+ import type { ResolvedToolPolicy } from "../types/index.js";
10
+ /**
11
+ * Filter a tools record according to the resolved policy.
12
+ * Order: enabled kill-switch → include allowlist → exclude denylist.
13
+ * Preserves the input record's key order for surviving tools (callers rely
14
+ * on deterministic ordering for provider prompt-cache stability).
15
+ */
16
+ export declare function applyToolGate<T>(tools: Record<string, T>, policy: ResolvedToolPolicy): Record<string, T>;
@@ -0,0 +1,50 @@
1
+ /**
2
+ * Tool gate — applies a ResolvedToolPolicy to a tools record. This is the
3
+ * ONE place where include/exclude filtering of the native tool set happens;
4
+ * every generate/stream path reaches it via BaseProvider.
5
+ *
6
+ * Generic over the tool value type so it works for AI-SDK Tool records,
7
+ * ToolInfo listings, and provider wire formats alike.
8
+ */
9
+ import { toolNameMatcher } from "./toolPolicy.js";
10
+ /**
11
+ * Filter a tools record according to the resolved policy.
12
+ * Order: enabled kill-switch → include allowlist → exclude denylist.
13
+ * Preserves the input record's key order for surviving tools (callers rely
14
+ * on deterministic ordering for provider prompt-cache stability).
15
+ */
16
+ export function applyToolGate(tools, policy) {
17
+ if (!policy.enabled) {
18
+ return {};
19
+ }
20
+ const include = policy.include;
21
+ const includeBound = policy.includeBound;
22
+ const hasExclude = policy.exclude.length > 0;
23
+ if (include === undefined && includeBound === undefined && !hasExclude) {
24
+ return tools;
25
+ }
26
+ const includeMatcher = include !== undefined ? toolNameMatcher(include) : undefined;
27
+ // Secondary allowlist clause ANDed with `include` (see ResolvedToolPolicy):
28
+ // a name must match BOTH when a per-call filter is bounded by the
29
+ // instance-level tools.include.
30
+ const boundMatcher = includeBound !== undefined ? toolNameMatcher(includeBound) : undefined;
31
+ const excludeMatcher = hasExclude
32
+ ? toolNameMatcher(policy.exclude)
33
+ : undefined;
34
+ // Null prototype: a tool named "__proto__" must become an own entry, not
35
+ // a prototype mutation (mirrors BaseProvider.sortToolRecord).
36
+ const result = Object.create(null);
37
+ for (const [name, tool] of Object.entries(tools)) {
38
+ if (includeMatcher && !includeMatcher(name)) {
39
+ continue;
40
+ }
41
+ if (boundMatcher && !boundMatcher(name)) {
42
+ continue;
43
+ }
44
+ if (excludeMatcher && excludeMatcher(name)) {
45
+ continue;
46
+ }
47
+ result[name] = tool;
48
+ }
49
+ return result;
50
+ }
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Tool policy resolution — the single, pure mapping from every tool-selection
3
+ * surface (per-call legacy options + instance-level `tools` config) to one
4
+ * ResolvedToolPolicy that the gate applies.
5
+ *
6
+ * Legacy semantics are preserved deliberately:
7
+ * - `toolFilter: []` and `enabledToolNames: []` are no-ops (fail-open), as
8
+ * they always were.
9
+ * - Malformed shapes (non-string-arrays) are ignored with a once-per-process
10
+ * WARN instead of throwing — some internal callers historically passed
11
+ * wrong shapes and relied on the silent no-op.
12
+ * - `enabledToolNames` now filters the native tool set (its docs always
13
+ * promised this; it previously only filtered the system-prompt listing) —
14
+ * but ONLY when `toolFilter` is absent. When both are set, `toolFilter`
15
+ * alone bounds the native set, exactly as before this refactor: the native
16
+ * set must never become WIDER than a caller's `toolFilter`.
17
+ *
18
+ * New semantics:
19
+ * - `tools.include: []` (the instance config surface) means NO tools
20
+ * (fail-closed); legacy empty arrays stay fail-open as above.
21
+ * - A per-call `toolFilter` is bounded by `tools.include` — it can narrow
22
+ * the instance allowlist but never widen past it.
23
+ * - Pattern entries containing `*` match as globs (e.g. `"github*"`) on ALL
24
+ * surfaces. Real tool names cannot contain `*` (MCP names are sanitized),
25
+ * so a legacy list entry with `*` previously matched nothing (dead entry);
26
+ * it now matches as a glob. Entries without `*` match exactly, unchanged.
27
+ */
28
+ import type { ResolvedToolPolicy, ToolPolicyResolutionInput } from "../types/index.js";
29
+ /**
30
+ * Compile a list of tool-name patterns into a matcher. Patterns without `*`
31
+ * match exactly (case-sensitive, preserving legacy toolFilter semantics);
32
+ * patterns containing `*` match as globs.
33
+ */
34
+ export declare function toolNameMatcher(patterns: string[]): (name: string) => boolean;
35
+ /**
36
+ * Resolve the effective tool policy for one request. Pure function — all
37
+ * inputs are explicit, so the legacy→policy mapping is unit-testable as a
38
+ * table.
39
+ */
40
+ export declare function resolveToolPolicy(input: ToolPolicyResolutionInput): ResolvedToolPolicy;