@zackbart/connecta 0.15.1 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (335) hide show
  1. package/AGENTS.md +40 -20
  2. package/CHANGELOG.md +653 -0
  3. package/README.md +25 -5
  4. package/bin/connecta.mjs +44 -3
  5. package/dist/access-tokens.d.ts +0 -1
  6. package/dist/access-tokens.js +0 -1
  7. package/dist/activity.d.ts +50 -2
  8. package/dist/activity.js +34 -1
  9. package/dist/apps-shell.d.ts +0 -1
  10. package/dist/apps-shell.js +0 -1
  11. package/dist/auth/bearer.d.ts +0 -1
  12. package/dist/auth/bearer.js +0 -1
  13. package/dist/auth/clerk.d.ts +0 -1
  14. package/dist/auth/clerk.js +45 -9
  15. package/dist/auth/downstream-oauth.d.ts +0 -1
  16. package/dist/auth/downstream-oauth.js +0 -1
  17. package/dist/call-admission.d.ts +0 -1
  18. package/dist/call-admission.js +0 -1
  19. package/dist/catalog-drift.d.ts +99 -0
  20. package/dist/catalog-drift.js +251 -0
  21. package/dist/catalog-fingerprint.d.ts +0 -1
  22. package/dist/catalog-fingerprint.js +0 -1
  23. package/dist/catalog-limits.d.ts +0 -1
  24. package/dist/catalog-limits.js +0 -1
  25. package/dist/catalog-service.d.ts +0 -1
  26. package/dist/catalog-service.js +41 -5
  27. package/dist/catalog.d.ts +11 -1
  28. package/dist/catalog.js +20 -1
  29. package/dist/concurrency.d.ts +0 -1
  30. package/dist/concurrency.js +0 -1
  31. package/dist/connector-scope.d.ts +0 -1
  32. package/dist/connector-scope.js +0 -1
  33. package/dist/connectors/api.d.ts +29 -21
  34. package/dist/connectors/api.js +45 -19
  35. package/dist/connectors/guarded-fetch.d.ts +106 -0
  36. package/dist/connectors/guarded-fetch.js +289 -0
  37. package/dist/connectors/remote-mcp.d.ts +0 -1
  38. package/dist/connectors/remote-mcp.js +0 -1
  39. package/dist/credentials.d.ts +0 -1
  40. package/dist/credentials.js +0 -1
  41. package/dist/errors.d.ts +30 -3
  42. package/dist/errors.js +1 -1
  43. package/dist/execute.d.ts +0 -1
  44. package/dist/execute.js +0 -1
  45. package/dist/executor-admission.d.ts +11 -1
  46. package/dist/executor-admission.js +28 -1
  47. package/dist/executor-result.d.ts +0 -1
  48. package/dist/executor-result.js +0 -1
  49. package/dist/executors/quickjs-child.d.ts +0 -1
  50. package/dist/executors/quickjs-child.js +0 -1
  51. package/dist/executors/quickjs-protocol.d.ts +0 -1
  52. package/dist/executors/quickjs-protocol.js +0 -1
  53. package/dist/executors/quickjs-runtime.d.ts +0 -1
  54. package/dist/executors/quickjs-runtime.js +0 -1
  55. package/dist/executors/quickjs.d.ts +0 -1
  56. package/dist/executors/quickjs.js +2 -1
  57. package/dist/favicon.d.ts +0 -1
  58. package/dist/favicon.js +0 -1
  59. package/dist/index.d.ts +2 -3
  60. package/dist/index.js +24 -7
  61. package/dist/invocation.d.ts +0 -1
  62. package/dist/invocation.js +0 -1
  63. package/dist/json-schema.d.ts +0 -1
  64. package/dist/json-schema.js +0 -1
  65. package/dist/mcp-result.d.ts +0 -1
  66. package/dist/mcp-result.js +0 -1
  67. package/dist/meta-tools.d.ts +0 -1
  68. package/dist/meta-tools.js +0 -1
  69. package/dist/node.d.ts +0 -1
  70. package/dist/node.js +0 -1
  71. package/dist/operator-ui/generated.d.ts +2 -3
  72. package/dist/operator-ui/generated.js +3 -4
  73. package/dist/operator-ui/model.d.ts +10 -1
  74. package/dist/operator-ui/model.js +0 -1
  75. package/dist/operator-ui/view.d.ts +167 -0
  76. package/dist/operator-ui/view.js +270 -0
  77. package/dist/providers/cloudflare.d.ts +0 -1
  78. package/dist/providers/cloudflare.js +261 -299
  79. package/dist/providers/linear.d.ts +26 -5
  80. package/dist/providers/linear.js +32 -61
  81. package/dist/providers/mixpanel.d.ts +35 -4
  82. package/dist/providers/mixpanel.js +53 -79
  83. package/dist/providers/notion.d.ts +0 -1
  84. package/dist/providers/notion.js +87 -42
  85. package/dist/providers/stripe.d.ts +13 -1
  86. package/dist/providers/stripe.js +31 -60
  87. package/dist/registry.d.ts +34 -2
  88. package/dist/registry.js +85 -5
  89. package/dist/retired-toolkits.d.ts +0 -1
  90. package/dist/retired-toolkits.js +0 -1
  91. package/dist/routes/access-tokens.d.ts +0 -1
  92. package/dist/routes/access-tokens.js +0 -1
  93. package/dist/routes/activity.d.ts +0 -1
  94. package/dist/routes/activity.js +0 -1
  95. package/dist/routes/credentials.d.ts +0 -1
  96. package/dist/routes/credentials.js +0 -1
  97. package/dist/routes/mcp.d.ts +0 -1
  98. package/dist/routes/mcp.js +0 -1
  99. package/dist/routes/oauth.d.ts +0 -1
  100. package/dist/routes/oauth.js +0 -1
  101. package/dist/routes/shared.d.ts +2 -1
  102. package/dist/routes/shared.js +0 -1
  103. package/dist/routes/ui.d.ts +0 -1
  104. package/dist/routes/ui.js +0 -1
  105. package/dist/server.d.ts +0 -1
  106. package/dist/server.js +11 -1
  107. package/dist/skills.d.ts +0 -1
  108. package/dist/skills.js +0 -1
  109. package/dist/storage/file.d.ts +0 -1
  110. package/dist/storage/file.js +0 -1
  111. package/dist/storage/memory.d.ts +0 -1
  112. package/dist/storage/memory.js +0 -1
  113. package/dist/timeout.d.ts +0 -1
  114. package/dist/timeout.js +0 -1
  115. package/dist/tool-safety.d.ts +0 -1
  116. package/dist/tool-safety.js +0 -1
  117. package/dist/types.d.ts +45 -1
  118. package/dist/types.js +0 -1
  119. package/dist/ui.d.ts +0 -1
  120. package/dist/ui.js +13 -137
  121. package/dist/validate.d.ts +13 -15
  122. package/dist/validate.js +24 -11
  123. package/dist/version.d.ts +1 -2
  124. package/dist/version.js +1 -2
  125. package/documentation/architecture.md +186 -5
  126. package/documentation/auth.md +11 -0
  127. package/documentation/call-admission.md +142 -6
  128. package/documentation/cloudflare.md +92 -10
  129. package/documentation/code-first-exploration.md +2 -2
  130. package/documentation/code-mode.md +14 -14
  131. package/documentation/connector-guides.md +172 -5
  132. package/documentation/connectors.md +89 -0
  133. package/documentation/linear.md +20 -2
  134. package/documentation/mcp-ui-design.md +2 -2
  135. package/documentation/meta-tools.md +7 -1
  136. package/documentation/mixpanel.md +42 -9
  137. package/documentation/notion.md +23 -2
  138. package/documentation/operations.md +325 -5
  139. package/documentation/operator-ui.md +120 -5
  140. package/documentation/provider-audit.md +160 -0
  141. package/documentation/provider-conventions.md +667 -0
  142. package/documentation/request-admission.md +167 -5
  143. package/documentation/rich-output-design.md +1 -2
  144. package/documentation/stripe.md +15 -0
  145. package/documentation/upgrading.md +454 -0
  146. package/ethos.md +14 -9
  147. package/examples/worker/README.md +102 -12
  148. package/examples/worker/src/index.ts +46 -1
  149. package/examples/worker/wrangler.jsonc +14 -1
  150. package/package.json +10 -6
  151. package/templates/node/.dockerignore +10 -0
  152. package/templates/node/.env.example +40 -3
  153. package/templates/node/AGENTS.md +16 -1
  154. package/templates/node/Dockerfile +49 -0
  155. package/templates/node/README.md +117 -2
  156. package/templates/node/docker-compose.yml +39 -0
  157. package/templates/node/package.json +1 -1
  158. package/templates/node/src/file-activity.ts +182 -0
  159. package/templates/node/src/index.ts +80 -4
  160. package/assets/connecta-clay-hero.png +0 -0
  161. package/dist/access-tokens.d.ts.map +0 -1
  162. package/dist/access-tokens.js.map +0 -1
  163. package/dist/activity.d.ts.map +0 -1
  164. package/dist/activity.js.map +0 -1
  165. package/dist/apps-shell.d.ts.map +0 -1
  166. package/dist/apps-shell.js.map +0 -1
  167. package/dist/auth/bearer.d.ts.map +0 -1
  168. package/dist/auth/bearer.js.map +0 -1
  169. package/dist/auth/clerk.d.ts.map +0 -1
  170. package/dist/auth/clerk.js.map +0 -1
  171. package/dist/auth/downstream-oauth.d.ts.map +0 -1
  172. package/dist/auth/downstream-oauth.js.map +0 -1
  173. package/dist/call-admission.d.ts.map +0 -1
  174. package/dist/call-admission.js.map +0 -1
  175. package/dist/catalog-fingerprint.d.ts.map +0 -1
  176. package/dist/catalog-fingerprint.js.map +0 -1
  177. package/dist/catalog-limits.d.ts.map +0 -1
  178. package/dist/catalog-limits.js.map +0 -1
  179. package/dist/catalog-service.d.ts.map +0 -1
  180. package/dist/catalog-service.js.map +0 -1
  181. package/dist/catalog.d.ts.map +0 -1
  182. package/dist/catalog.js.map +0 -1
  183. package/dist/concurrency.d.ts.map +0 -1
  184. package/dist/concurrency.js.map +0 -1
  185. package/dist/connector-scope.d.ts.map +0 -1
  186. package/dist/connector-scope.js.map +0 -1
  187. package/dist/connectors/api.d.ts.map +0 -1
  188. package/dist/connectors/api.js.map +0 -1
  189. package/dist/connectors/remote-mcp.d.ts.map +0 -1
  190. package/dist/connectors/remote-mcp.js.map +0 -1
  191. package/dist/credentials.d.ts.map +0 -1
  192. package/dist/credentials.js.map +0 -1
  193. package/dist/errors.d.ts.map +0 -1
  194. package/dist/errors.js.map +0 -1
  195. package/dist/execute.d.ts.map +0 -1
  196. package/dist/execute.js.map +0 -1
  197. package/dist/executor-admission.d.ts.map +0 -1
  198. package/dist/executor-admission.js.map +0 -1
  199. package/dist/executor-result.d.ts.map +0 -1
  200. package/dist/executor-result.js.map +0 -1
  201. package/dist/executors/quickjs-child.d.ts.map +0 -1
  202. package/dist/executors/quickjs-child.js.map +0 -1
  203. package/dist/executors/quickjs-protocol.d.ts.map +0 -1
  204. package/dist/executors/quickjs-protocol.js.map +0 -1
  205. package/dist/executors/quickjs-runtime.d.ts.map +0 -1
  206. package/dist/executors/quickjs-runtime.js.map +0 -1
  207. package/dist/executors/quickjs.d.ts.map +0 -1
  208. package/dist/executors/quickjs.js.map +0 -1
  209. package/dist/favicon.d.ts.map +0 -1
  210. package/dist/favicon.js.map +0 -1
  211. package/dist/index.d.ts.map +0 -1
  212. package/dist/index.js.map +0 -1
  213. package/dist/invocation.d.ts.map +0 -1
  214. package/dist/invocation.js.map +0 -1
  215. package/dist/json-schema.d.ts.map +0 -1
  216. package/dist/json-schema.js.map +0 -1
  217. package/dist/mcp-result.d.ts.map +0 -1
  218. package/dist/mcp-result.js.map +0 -1
  219. package/dist/meta-tools.d.ts.map +0 -1
  220. package/dist/meta-tools.js.map +0 -1
  221. package/dist/node.d.ts.map +0 -1
  222. package/dist/node.js.map +0 -1
  223. package/dist/operator-ui/generated.d.ts.map +0 -1
  224. package/dist/operator-ui/generated.js.map +0 -1
  225. package/dist/operator-ui/model.d.ts.map +0 -1
  226. package/dist/operator-ui/model.js.map +0 -1
  227. package/dist/providers/cloudflare.d.ts.map +0 -1
  228. package/dist/providers/cloudflare.js.map +0 -1
  229. package/dist/providers/linear.d.ts.map +0 -1
  230. package/dist/providers/linear.js.map +0 -1
  231. package/dist/providers/mixpanel.d.ts.map +0 -1
  232. package/dist/providers/mixpanel.js.map +0 -1
  233. package/dist/providers/notion.d.ts.map +0 -1
  234. package/dist/providers/notion.js.map +0 -1
  235. package/dist/providers/stripe.d.ts.map +0 -1
  236. package/dist/providers/stripe.js.map +0 -1
  237. package/dist/registry.d.ts.map +0 -1
  238. package/dist/registry.js.map +0 -1
  239. package/dist/retired-toolkits.d.ts.map +0 -1
  240. package/dist/retired-toolkits.js.map +0 -1
  241. package/dist/routes/access-tokens.d.ts.map +0 -1
  242. package/dist/routes/access-tokens.js.map +0 -1
  243. package/dist/routes/activity.d.ts.map +0 -1
  244. package/dist/routes/activity.js.map +0 -1
  245. package/dist/routes/credentials.d.ts.map +0 -1
  246. package/dist/routes/credentials.js.map +0 -1
  247. package/dist/routes/mcp.d.ts.map +0 -1
  248. package/dist/routes/mcp.js.map +0 -1
  249. package/dist/routes/oauth.d.ts.map +0 -1
  250. package/dist/routes/oauth.js.map +0 -1
  251. package/dist/routes/shared.d.ts.map +0 -1
  252. package/dist/routes/shared.js.map +0 -1
  253. package/dist/routes/ui.d.ts.map +0 -1
  254. package/dist/routes/ui.js.map +0 -1
  255. package/dist/server.d.ts.map +0 -1
  256. package/dist/server.js.map +0 -1
  257. package/dist/skills.d.ts.map +0 -1
  258. package/dist/skills.js.map +0 -1
  259. package/dist/storage/file.d.ts.map +0 -1
  260. package/dist/storage/file.js.map +0 -1
  261. package/dist/storage/memory.d.ts.map +0 -1
  262. package/dist/storage/memory.js.map +0 -1
  263. package/dist/timeout.d.ts.map +0 -1
  264. package/dist/timeout.js.map +0 -1
  265. package/dist/tool-safety.d.ts.map +0 -1
  266. package/dist/tool-safety.js.map +0 -1
  267. package/dist/types.d.ts.map +0 -1
  268. package/dist/types.js.map +0 -1
  269. package/dist/ui.d.ts.map +0 -1
  270. package/dist/ui.js.map +0 -1
  271. package/dist/validate.d.ts.map +0 -1
  272. package/dist/validate.js.map +0 -1
  273. package/dist/version.d.ts.map +0 -1
  274. package/dist/version.js.map +0 -1
  275. package/examples/node/README.md +0 -52
  276. package/examples/node/src/index.ts +0 -71
  277. package/src/access-tokens.ts +0 -289
  278. package/src/activity.ts +0 -234
  279. package/src/apps-shell.ts +0 -370
  280. package/src/auth/bearer.ts +0 -61
  281. package/src/auth/clerk.ts +0 -553
  282. package/src/auth/downstream-oauth.ts +0 -575
  283. package/src/call-admission.ts +0 -521
  284. package/src/catalog-fingerprint.ts +0 -52
  285. package/src/catalog-limits.ts +0 -11
  286. package/src/catalog-service.ts +0 -1153
  287. package/src/catalog.ts +0 -790
  288. package/src/concurrency.ts +0 -39
  289. package/src/connector-scope.ts +0 -66
  290. package/src/connectors/api.ts +0 -159
  291. package/src/connectors/remote-mcp.ts +0 -1090
  292. package/src/credentials.ts +0 -428
  293. package/src/errors.ts +0 -364
  294. package/src/execute.ts +0 -1406
  295. package/src/executor-admission.ts +0 -328
  296. package/src/executor-result.ts +0 -107
  297. package/src/executors/quickjs-child.ts +0 -172
  298. package/src/executors/quickjs-protocol.ts +0 -99
  299. package/src/executors/quickjs-runtime.ts +0 -470
  300. package/src/executors/quickjs.ts +0 -776
  301. package/src/favicon.ts +0 -53
  302. package/src/index.ts +0 -666
  303. package/src/invocation.ts +0 -655
  304. package/src/json-schema.ts +0 -11
  305. package/src/mcp-result.ts +0 -43
  306. package/src/meta-tools.ts +0 -1607
  307. package/src/node.ts +0 -206
  308. package/src/operator-ui/browser.css +0 -423
  309. package/src/operator-ui/browser.ts +0 -1281
  310. package/src/operator-ui/generated.ts +0 -4
  311. package/src/operator-ui/model.ts +0 -111
  312. package/src/providers/cloudflare.ts +0 -3999
  313. package/src/providers/linear.ts +0 -302
  314. package/src/providers/mixpanel.ts +0 -229
  315. package/src/providers/notion.ts +0 -1879
  316. package/src/providers/stripe.ts +0 -307
  317. package/src/registry.ts +0 -1085
  318. package/src/retired-toolkits.ts +0 -14
  319. package/src/routes/access-tokens.ts +0 -115
  320. package/src/routes/activity.ts +0 -215
  321. package/src/routes/credentials.ts +0 -254
  322. package/src/routes/mcp.ts +0 -429
  323. package/src/routes/oauth.ts +0 -322
  324. package/src/routes/shared.ts +0 -247
  325. package/src/routes/ui.ts +0 -152
  326. package/src/server.ts +0 -195
  327. package/src/skills.ts +0 -271
  328. package/src/storage/file.ts +0 -123
  329. package/src/storage/memory.ts +0 -44
  330. package/src/timeout.ts +0 -84
  331. package/src/tool-safety.ts +0 -15
  332. package/src/types.ts +0 -523
  333. package/src/ui.ts +0 -744
  334. package/src/validate.ts +0 -387
  335. package/src/version.ts +0 -7
package/src/catalog.ts DELETED
@@ -1,790 +0,0 @@
1
- import type { JsonSchema, ToolDef } from "./types.js";
2
-
3
- const DEFAULT_DESCRIPTION_LENGTH = 240;
4
- const DISCOVERY_DESCRIPTION_LENGTH = 160;
5
- export const MAX_COMPACT_DISCOVERY_SCHEMA_BYTES = 1_024;
6
- const MAX_COMPACT_DISCOVERY_ENUM_BYTES =
7
- MAX_COMPACT_DISCOVERY_SCHEMA_BYTES / 4;
8
- const schemaEncoder = new TextEncoder();
9
- const COMPACT_DISCOVERY_TRUNCATION = " /* truncated */";
10
-
11
- export function summarizeDescription(
12
- text: string | undefined,
13
- full: boolean,
14
- ): string | undefined {
15
- return summarizeToLength(text, full, DEFAULT_DESCRIPTION_LENGTH);
16
- }
17
-
18
- export function summarizeDiscoveryDescription(
19
- text: string | undefined,
20
- full: boolean,
21
- ): string | undefined {
22
- return summarizeToLength(text, full, DISCOVERY_DESCRIPTION_LENGTH);
23
- }
24
-
25
- function summarizeToLength(
26
- text: string | undefined,
27
- full: boolean,
28
- maxLength: number,
29
- ): string | undefined {
30
- if (!text) return undefined;
31
- if (full) return text;
32
- const compact = text.replace(/\s+/g, " ").trim();
33
- if (compact.length <= maxLength) return compact;
34
- return `${compact.slice(0, maxLength - 1).trimEnd()}…`;
35
- }
36
-
37
- function lexicalTokens(text: string): string[] {
38
- return text
39
- .replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
40
- .replace(/([a-z0-9])([A-Z])/g, "$1 $2")
41
- .toLowerCase()
42
- .replace(/[^a-z0-9]+/g, " ")
43
- .trim()
44
- .split(/\s+/)
45
- .filter(Boolean);
46
- }
47
-
48
- function normalized(text: string): string {
49
- return lexicalTokens(text).join(" ");
50
- }
51
-
52
- /** Distinct normalized terms in query order, shared by ranking and feedback. */
53
- export function lexicalQueryTerms(query: string): string[] {
54
- return [...new Set(lexicalTokens(query))];
55
- }
56
-
57
- /**
58
- * Conversational framing selected by the #188 research run before the #189
59
- * holdout existed. Action-bearing terms such as get/list/search/find/create
60
- * deliberately remain: arbitrary connector catalogs need those distinctions.
61
- */
62
- const CONVERSATIONAL_QUERY_WORDS = new Set([
63
- "a",
64
- "all",
65
- "an",
66
- "and",
67
- "are",
68
- "can",
69
- "could",
70
- "current",
71
- "for",
72
- "from",
73
- "i",
74
- "in",
75
- "into",
76
- "it",
77
- "latest",
78
- "let",
79
- "me",
80
- "most",
81
- "of",
82
- "on",
83
- "our",
84
- "please",
85
- "right",
86
- "show",
87
- "that",
88
- "the",
89
- "this",
90
- "to",
91
- "up",
92
- "want",
93
- "when",
94
- "which",
95
- "with",
96
- "you",
97
- ]);
98
-
99
- /**
100
- * Remove conversational framing before the all-term/partial decision. If
101
- * every term is framing, retain the original query rather than turning a
102
- * search into an unfiltered catalog browse.
103
- */
104
- export function lexicalSearchQuery(query: string): string {
105
- const terms = normalized(query).split(/\s+/).filter(Boolean);
106
- const contentTerms = terms.filter(
107
- (term) => !CONVERSATIONAL_QUERY_WORDS.has(term),
108
- );
109
- return contentTerms.length > 0 ? contentTerms.join(" ") : query;
110
- }
111
-
112
- interface SearchDocument {
113
- tool: ToolDef;
114
- name: string;
115
- nameTokens: string[];
116
- descriptionTokens: string[];
117
- }
118
-
119
- interface SearchIndex {
120
- documents: SearchDocument[];
121
- nameTokenDocuments: ReadonlyMap<string, readonly ToolDef[]>;
122
- descriptionTokenDocuments: ReadonlyMap<string, readonly ToolDef[]>;
123
- }
124
-
125
- export type LexicalMatchMode = "all" | "partial";
126
-
127
- export interface RankedTool {
128
- tool: ToolDef;
129
- score: number;
130
- order: number;
131
- exactName: boolean;
132
- matchedTermCount: number;
133
- }
134
-
135
- const searchIndexes = new WeakMap<ToolDef[], SearchIndex>();
136
-
137
- function indexFor(tools: ToolDef[]): SearchIndex {
138
- let index = searchIndexes.get(tools);
139
- if (!index) {
140
- const nameTokenDocuments = new Map<string, ToolDef[]>();
141
- const descriptionTokenDocuments = new Map<string, ToolDef[]>();
142
- const addTokens = (
143
- target: Map<string, ToolDef[]>,
144
- tokens: string[],
145
- tool: ToolDef,
146
- ) => {
147
- for (const token of new Set(tokens)) {
148
- const documents = target.get(token) ?? [];
149
- documents.push(tool);
150
- target.set(token, documents);
151
- }
152
- };
153
- const documents = tools.map((tool) => {
154
- const nameTokens = lexicalTokens(tool.name);
155
- const descriptionTokens = lexicalTokens(tool.description ?? "");
156
- addTokens(nameTokenDocuments, nameTokens, tool);
157
- addTokens(descriptionTokenDocuments, descriptionTokens, tool);
158
- return {
159
- tool,
160
- name: nameTokens.join(" "),
161
- nameTokens,
162
- descriptionTokens,
163
- };
164
- });
165
- index = {
166
- documents,
167
- nameTokenDocuments,
168
- descriptionTokenDocuments,
169
- };
170
- searchIndexes.set(tools, index);
171
- }
172
- return index;
173
- }
174
-
175
- function documentsFor(tools: ToolDef[]): SearchDocument[] {
176
- return indexFor(tools).documents;
177
- }
178
-
179
- /**
180
- * Whole-token equality is the ordinary lexical match. A deliberately narrow
181
- * inflection check retains useful singular/plural and past-tense recall
182
- * without bringing back arbitrary substring matches (`list` must not match
183
- * `enlist`, and `record` must not match the noun `recording`).
184
- */
185
- function inflectionVariants(base: string): string[] {
186
- return [
187
- `${base}s`,
188
- `${base}es`,
189
- `${base}ed`,
190
- ...(base.endsWith("e") ? [`${base}d`] : []),
191
- ...(base.endsWith("y")
192
- ? [
193
- `${base.slice(0, -1)}ies`,
194
- `${base.slice(0, -1)}ied`,
195
- ]
196
- : []),
197
- ];
198
- }
199
-
200
- function matchingTokenCandidates(term: string): Set<string> {
201
- const candidates = new Set([term, ...inflectionVariants(term)]);
202
- const possibleBases = [
203
- ...(term.endsWith("s") ? [term.slice(0, -1)] : []),
204
- ...(term.endsWith("es") ? [term.slice(0, -2)] : []),
205
- ...(term.endsWith("ed") ? [term.slice(0, -2)] : []),
206
- ...(term.endsWith("d") ? [term.slice(0, -1)] : []),
207
- ...(term.endsWith("ies")
208
- ? [`${term.slice(0, -3)}y`]
209
- : []),
210
- ...(term.endsWith("ied")
211
- ? [`${term.slice(0, -3)}y`]
212
- : []),
213
- ];
214
- for (const base of possibleBases) {
215
- if (base && inflectionVariants(base).includes(term)) {
216
- candidates.add(base);
217
- }
218
- }
219
- return candidates;
220
- }
221
-
222
- export interface LexicalCorpusStatistics {
223
- documentCount: number;
224
- documentFrequency: ReadonlyMap<string, number>;
225
- nameMatches: ReadonlyMap<string, ReadonlySet<ToolDef>>;
226
- descriptionMatches: ReadonlyMap<string, ReadonlySet<ToolDef>>;
227
- }
228
-
229
- /**
230
- * Compute query-specific document frequencies across every available catalog.
231
- * The caller does this once per search and shares the result with each
232
- * connector rank, so ubiquitous words contribute less than discriminative
233
- * ones without making any action word a stopword.
234
- */
235
- export function lexicalCorpusStatistics(
236
- toolSets: ToolDef[][],
237
- query: string,
238
- ): LexicalCorpusStatistics {
239
- const terms = lexicalQueryTerms(query);
240
- if (terms.length === 0) {
241
- return {
242
- documentCount: toolSets.reduce(
243
- (total, tools) => total + tools.length,
244
- 0,
245
- ),
246
- documentFrequency: new Map(),
247
- nameMatches: new Map(),
248
- descriptionMatches: new Map(),
249
- };
250
- }
251
- const nameMatches = new Map<string, Set<ToolDef>>();
252
- const descriptionMatches = new Map<string, Set<ToolDef>>();
253
- for (const term of terms) {
254
- const termNameMatches = new Set<ToolDef>();
255
- const termDescriptionMatches = new Set<ToolDef>();
256
- for (const tools of toolSets) {
257
- const index = indexFor(tools);
258
- for (const candidate of matchingTokenCandidates(term)) {
259
- for (const tool of index.nameTokenDocuments.get(candidate) ?? []) {
260
- termNameMatches.add(tool);
261
- }
262
- for (
263
- const tool of
264
- index.descriptionTokenDocuments.get(candidate) ?? []
265
- ) {
266
- termDescriptionMatches.add(tool);
267
- }
268
- }
269
- }
270
- nameMatches.set(term, termNameMatches);
271
- descriptionMatches.set(term, termDescriptionMatches);
272
- }
273
- const documentFrequency = new Map(
274
- terms.map((term) => [
275
- term,
276
- new Set([
277
- ...(nameMatches.get(term) ?? []),
278
- ...(descriptionMatches.get(term) ?? []),
279
- ]).size,
280
- ]),
281
- );
282
- return {
283
- documentCount: toolSets.reduce(
284
- (total, tools) => total + tools.length,
285
- 0,
286
- ),
287
- documentFrequency,
288
- nameMatches,
289
- descriptionMatches,
290
- };
291
- }
292
-
293
- function inverseDocumentFrequency(
294
- term: string,
295
- statistics: LexicalCorpusStatistics,
296
- ): number {
297
- const frequency = statistics.documentFrequency.get(term) ?? 0;
298
- return Math.log(
299
- 1 +
300
- (statistics.documentCount - frequency + 0.5) /
301
- (frequency + 0.5),
302
- );
303
- }
304
-
305
- function scoreDocument(
306
- doc: SearchDocument,
307
- phrase: string,
308
- terms: string[],
309
- mode: LexicalMatchMode,
310
- statistics: LexicalCorpusStatistics,
311
- ): { score: number; matchedTermCount: number } | null {
312
- if (!phrase) return { score: 0, matchedTermCount: 0 };
313
- const matchedTerms = terms.filter((term) =>
314
- statistics.nameMatches.get(term)?.has(doc.tool) ||
315
- statistics.descriptionMatches.get(term)?.has(doc.tool),
316
- );
317
- if (mode === "all" && matchedTerms.length !== terms.length) return null;
318
- if (matchedTerms.length === 0) return null;
319
-
320
- // Coverage remains meaningful in partial mode, but is IDF-weighted rather
321
- // than a raw term count: one rare domain term can beat several ubiquitous
322
- // catalog verbs.
323
- let score = matchedTerms.reduce(
324
- (total, term) =>
325
- total + 4 * inverseDocumentFrequency(term, statistics),
326
- 0,
327
- );
328
- const phraseWeight = terms.reduce(
329
- (total, term) =>
330
- total + inverseDocumentFrequency(term, statistics),
331
- 0,
332
- );
333
- if (doc.name === phrase) score += 40 * phraseWeight;
334
- else if (doc.name.startsWith(`${phrase} `)) score += 24 * phraseWeight;
335
- else if (` ${doc.name} `.includes(` ${phrase} `)) {
336
- score += 16 * phraseWeight;
337
- }
338
-
339
- for (const term of matchedTerms) {
340
- const weight = inverseDocumentFrequency(term, statistics);
341
- if (doc.nameTokens.includes(term)) score += 12 * weight;
342
- else if (statistics.nameMatches.get(term)?.has(doc.tool)) {
343
- score += 8 * weight;
344
- }
345
- if (doc.descriptionTokens.includes(term)) score += 3 * weight;
346
- else if (statistics.descriptionMatches.get(term)?.has(doc.tool)) {
347
- score += 1.5 * weight;
348
- }
349
- }
350
- return { score, matchedTermCount: matchedTerms.length };
351
- }
352
-
353
- function queryContainsExactName(doc: SearchDocument, phrase: string): boolean {
354
- if (!doc.name || !phrase) return false;
355
- return (` ${phrase} `).includes(` ${doc.name} `);
356
- }
357
-
358
- /**
359
- * Rank a connector's tools while caching its normalized plain-data index.
360
- * `exactNameQuery` may retain framing removed from the scoring query: those
361
- * words are weak term evidence, but remain part of a real tool-name phrase.
362
- */
363
- export function rankTools(
364
- tools: ToolDef[],
365
- query: string,
366
- mode: LexicalMatchMode = "all",
367
- statistics: LexicalCorpusStatistics = lexicalCorpusStatistics(
368
- [tools],
369
- query,
370
- ),
371
- exactNameQuery: string = query,
372
- ): RankedTool[] {
373
- const phrase = normalized(query);
374
- const exactNamePhrase = normalized(exactNameQuery);
375
- const terms = [...new Set(phrase.split(/\s+/).filter(Boolean))];
376
- const ranked: RankedTool[] = [];
377
- documentsFor(tools).forEach((doc, order) => {
378
- const scored = scoreDocument(doc, phrase, terms, mode, statistics);
379
- if (scored !== null) {
380
- ranked.push({
381
- tool: doc.tool,
382
- score: scored.score,
383
- order,
384
- exactName: queryContainsExactName(doc, exactNamePhrase),
385
- matchedTermCount: scored.matchedTermCount,
386
- });
387
- }
388
- });
389
- return ranked;
390
- }
391
-
392
- function refName(ref: string): string {
393
- return ref.split("/").pop() ?? ref;
394
- }
395
-
396
- /**
397
- * Whether a schema declares anything renderSchema knows how to render on its
398
- * own. Used to decide if the non-allOf half of a schema is worth rendering:
399
- * without this, a plain `{ allOf: [...] }` would render its (empty) local half
400
- * through the raw-JSON fallback and emit `{} & …`.
401
- */
402
- function declaresShape(s: Record<string, unknown>): boolean {
403
- return (
404
- typeof s.$ref === "string" ||
405
- Array.isArray(s.oneOf) ||
406
- Array.isArray(s.anyOf) ||
407
- Array.isArray(s.enum) ||
408
- s.const !== undefined ||
409
- s.items !== undefined ||
410
- s.properties !== undefined ||
411
- s.type !== undefined
412
- );
413
- }
414
-
415
- /**
416
- * Parenthesize a top-level union so it doesn't read as part of a surrounding
417
- * `&`. Only separators outside braces count, so a nested union or a property
418
- * description containing a pipe doesn't trigger stray parentheses.
419
- */
420
- function grouped(part: string): string {
421
- let nesting = 0;
422
- for (let i = 0; i < part.length; i += 1) {
423
- const char = part[i];
424
- if (char === "{" || char === "(" || char === "[") nesting += 1;
425
- else if (char === "}" || char === ")" || char === "]") nesting -= 1;
426
- else if (nesting === 0 && part.startsWith(" | ", i)) return `(${part})`;
427
- }
428
- return part;
429
- }
430
-
431
- function renderEnum(
432
- values: unknown[],
433
- byteLimit: number | undefined,
434
- onTruncated: (() => void) | undefined,
435
- ): string {
436
- if (values.length === 0) return "never";
437
- const renderedValues = values.map((value) => JSON.stringify(value));
438
- const full = renderedValues.join(" | ");
439
- if (
440
- byteLimit === undefined ||
441
- schemaEncoder.encode(full).length <= byteLimit
442
- ) {
443
- return full;
444
- }
445
-
446
- onTruncated?.();
447
- const marker = (omitted: number) =>
448
- `unknown /* ${omitted} enum ${omitted === 1 ? "value" : "values"} omitted */`;
449
- let rendered = `(${marker(values.length)})`;
450
- const prefix: string[] = [];
451
- for (let index = 0; index < renderedValues.length - 1; index += 1) {
452
- prefix.push(renderedValues[index] as string);
453
- const omitted = renderedValues.length - prefix.length;
454
- const candidate = `(${prefix.join(" | ")} | ${marker(omitted)})`;
455
- if (schemaEncoder.encode(candidate).length > byteLimit) break;
456
- rendered = candidate;
457
- }
458
- return rendered;
459
- }
460
-
461
- function renderSchema(
462
- schema: unknown,
463
- defs: Record<string, unknown>,
464
- seen: Set<string>,
465
- depth: number,
466
- options: {
467
- propertyDescriptions: boolean;
468
- requiredFirst: boolean;
469
- enumByteLimit?: number;
470
- onEnumTruncated?: () => void;
471
- },
472
- ): string {
473
- if (depth > 4) return "…";
474
- if (schema === null || typeof schema !== "object") {
475
- return JSON.stringify(schema);
476
- }
477
- const s = schema as Record<string, unknown>;
478
-
479
- // allOf composes rather than replaces: it is checked before every other
480
- // keyword, and renders the schema's own shape alongside its members instead
481
- // of returning early. A schema carrying both allOf and properties (the usual
482
- // OpenAPI-derived "extend this base" shape, and equally legal with $ref,
483
- // enum, const, or items) would otherwise silently drop whichever half lost
484
- // the branch race. The schema's own shape comes first, being the more
485
- // specific half, and is rendered at the current depth because its members
486
- // sit at this nesting level, not one below.
487
- if (Array.isArray(s.allOf)) {
488
- const { allOf: _members, ...own } = s;
489
- const parts = declaresShape(own)
490
- ? [renderSchema(own, defs, seen, depth, options)]
491
- : [];
492
- for (const member of s.allOf) {
493
- parts.push(renderSchema(member, defs, seen, depth + 1, options));
494
- }
495
- if (parts.length === 0) return "unknown";
496
- if (parts.length === 1) return parts[0] as string;
497
- return parts.map(grouped).join(" & ");
498
- }
499
-
500
- if (typeof s.$ref === "string") {
501
- const name = refName(s.$ref);
502
- if (seen.has(name)) return name;
503
- const target = defs[name];
504
- if (target === undefined) return name;
505
- seen.add(name);
506
- const rendered = renderSchema(target, defs, seen, depth, options);
507
- seen.delete(name);
508
- return rendered;
509
- }
510
-
511
- const union = (s.oneOf ?? s.anyOf) as unknown[] | undefined;
512
- if (Array.isArray(union)) {
513
- return (
514
- union
515
- .map((u) => renderSchema(u, defs, seen, depth + 1, options))
516
- .join(" | ") ||
517
- "unknown"
518
- );
519
- }
520
- if (Array.isArray(s.enum)) {
521
- return renderEnum(s.enum, options.enumByteLimit, options.onEnumTruncated);
522
- }
523
- // Checked before type/properties so a discriminator like
524
- // { type: "string", const: "emoji" } renders as "emoji" rather than string.
525
- // JSON.stringify(undefined) returns undefined (not a string), so an explicit
526
- // `const: undefined` must fall through to the regular type rendering.
527
- if (s.const !== undefined) return JSON.stringify(s.const);
528
-
529
- const type = s.type;
530
- if (type === "array" || s.items) {
531
- const items = s.items
532
- ? renderSchema(s.items, defs, seen, depth + 1, options)
533
- : "unknown";
534
- return `${items}[]`;
535
- }
536
- if (type === "object" || s.properties) {
537
- const props = (s.properties ?? {}) as Record<string, unknown>;
538
- const required = new Set(
539
- (Array.isArray(s.required) ? s.required : []) as string[],
540
- );
541
- const declaredKeys = Object.keys(props);
542
- const keys = options.requiredFirst
543
- ? [
544
- ...declaredKeys.filter((key) => required.has(key)),
545
- ...declaredKeys.filter((key) => !required.has(key)),
546
- ]
547
- : declaredKeys;
548
- if (keys.length === 0) return "{}";
549
- return `{ ${keys
550
- .map((key) => {
551
- const optional = required.has(key) ? "" : "?";
552
- const rendered = renderSchema(
553
- props[key],
554
- defs,
555
- seen,
556
- depth + 1,
557
- options,
558
- );
559
- const description = (
560
- props[key] as Record<string, unknown> | null
561
- )?.description;
562
- const comment =
563
- options.propertyDescriptions && typeof description === "string"
564
- ? ` // ${description}`
565
- : "";
566
- return `${key}${optional}: ${rendered}${comment}`;
567
- })
568
- .join(", ")} }`;
569
- }
570
- if (typeof type === "string") return type;
571
- if (Array.isArray(type)) return type.join(" | ");
572
- return JSON.stringify(schema);
573
- }
574
-
575
- const compactSchemas = new WeakMap<JsonSchema, string>();
576
-
577
- /** Render and cache a compact TypeScript-like representation of JSON Schema. */
578
- export function compactSchema(schema: JsonSchema): string {
579
- const cached = compactSchemas.get(schema);
580
- if (cached) return cached;
581
- const defs = {
582
- ...(schema.$defs as Record<string, unknown>),
583
- ...(schema.definitions as Record<string, unknown>),
584
- };
585
- let rendered: string;
586
- try {
587
- rendered = renderSchema(schema, defs, new Set(), 0, {
588
- propertyDescriptions: true,
589
- requiredFirst: false,
590
- });
591
- } catch {
592
- rendered = JSON.stringify(schema);
593
- }
594
- compactSchemas.set(schema, rendered);
595
- return rendered;
596
- }
597
-
598
- export interface CompactDiscoverySchema {
599
- text: string;
600
- truncated: boolean;
601
- }
602
-
603
- const compactDiscoverySchemas = new WeakMap<
604
- JsonSchema,
605
- CompactDiscoverySchema
606
- >();
607
-
608
- /**
609
- * A valid, bounded replacement for a discovery shape too large to carry.
610
- *
611
- * Required object keys come first and every retained key is JSON-quoted, so
612
- * arbitrary downstream names remain valid TypeScript property signatures.
613
- * Types become `unknown`: pretending a severed nested type is exact would be
614
- * worse than making the existing truncation flag's recovery route explicit.
615
- */
616
- function truncatedDiscoverySchema(schema: JsonSchema): string {
617
- const keys = schemaObjectKeys(schema);
618
- if (!keys) return `unknown${COMPACT_DISCOVERY_TRUNCATION}`;
619
- const required = new Set(keys.required);
620
- const ordered = [
621
- ...keys.properties.filter((key) => required.has(key)),
622
- ...keys.properties.filter((key) => !required.has(key)),
623
- ];
624
- const parts: string[] = [];
625
- for (const key of ordered) {
626
- const part = `${JSON.stringify(key)}${required.has(key) ? "" : "?"}: unknown`;
627
- const candidate = `{ ${[...parts, part].join(", ")} }${COMPACT_DISCOVERY_TRUNCATION}`;
628
- if (
629
- schemaEncoder.encode(candidate).length >
630
- MAX_COMPACT_DISCOVERY_SCHEMA_BYTES
631
- ) {
632
- break;
633
- }
634
- parts.push(part);
635
- }
636
- if (parts.length === 0 && ordered.length > 0) {
637
- return `unknown${COMPACT_DISCOVERY_TRUNCATION}`;
638
- }
639
- return `{ ${parts.join(", ")} }${COMPACT_DISCOVERY_TRUNCATION}`;
640
- }
641
-
642
- /**
643
- * Render the schema shape carried by search results.
644
- *
645
- * Search is a routing step, so repeated property prose does not earn its
646
- * context cost there. Required inputs render first, and the result has a hard
647
- * UTF-8 budget; exact JSON and the prose-rich compact rendering remain
648
- * available through the existing full retrieval paths.
649
- */
650
- export function compactDiscoverySchema(
651
- schema: JsonSchema,
652
- ): CompactDiscoverySchema {
653
- const cached = compactDiscoverySchemas.get(schema);
654
- if (cached) return cached;
655
- const defs = {
656
- ...(schema.$defs as Record<string, unknown>),
657
- ...(schema.definitions as Record<string, unknown>),
658
- };
659
- let rendered: string;
660
- let enumTruncated = false;
661
- try {
662
- rendered = renderSchema(schema, defs, new Set(), 0, {
663
- propertyDescriptions: false,
664
- requiredFirst: true,
665
- // Three near-cap enums spend about three quarters of the complete shape
666
- // budget, leaving the final quarter for surrounding syntax before the
667
- // unchanged global fallback applies. Whole values keep this UTF-8 safe.
668
- enumByteLimit: MAX_COMPACT_DISCOVERY_ENUM_BYTES,
669
- onEnumTruncated: () => {
670
- enumTruncated = true;
671
- },
672
- });
673
- } catch {
674
- rendered = JSON.stringify(schema);
675
- }
676
- const bytes = schemaEncoder.encode(rendered);
677
- let result: CompactDiscoverySchema;
678
- if (bytes.length <= MAX_COMPACT_DISCOVERY_SCHEMA_BYTES) {
679
- result = { text: rendered, truncated: enumTruncated };
680
- } else {
681
- result = {
682
- text: truncatedDiscoverySchema(schema),
683
- truncated: true,
684
- };
685
- }
686
- compactDiscoverySchemas.set(schema, result);
687
- return result;
688
- }
689
-
690
- /** The property and required names a schema resolves to, or undefined. */
691
- export interface SchemaObjectKeys {
692
- properties: string[];
693
- required: string[];
694
- }
695
-
696
- /**
697
- * Walk a schema the way renderSchema does — composing `allOf` and resolving
698
- * `$ref` against the root's `$defs`/`definitions` — and collect the top-level
699
- * property names it would render. A shallow `Object.keys(schema.properties)`
700
- * disagrees with the rendered compact schema for exactly the shapes real
701
- * connectors emit (a top-level `$ref` to a `$defs` entry, or the OpenAPI
702
- * "extend this base" `allOf`), which is worse than no metadata at all: it
703
- * reports an empty field list for a tool that plainly has fields.
704
- *
705
- * Returns undefined when the schema is not an object shape at all — a union,
706
- * array, enum, or unresolvable `$ref`. Absent metadata tells a caller to read
707
- * the rendered schema instead; an empty array would claim the tool takes no
708
- * fields.
709
- */
710
- export function schemaObjectKeys(
711
- schema: JsonSchema | undefined,
712
- ): SchemaObjectKeys | undefined {
713
- if (!schema) return undefined;
714
- const defs = {
715
- ...(schema.$defs as Record<string, unknown>),
716
- ...(schema.definitions as Record<string, unknown>),
717
- };
718
- try {
719
- return objectKeys(schema, defs, new Set(), 0);
720
- } catch {
721
- return undefined;
722
- }
723
- }
724
-
725
- /** Merge in declaration order, first occurrence winning, as renderSchema renders. */
726
- function mergedKeys(
727
- parts: readonly SchemaObjectKeys[],
728
- ): SchemaObjectKeys | undefined {
729
- if (parts.length === 0) return undefined;
730
- return {
731
- properties: [...new Set(parts.flatMap((part) => part.properties))],
732
- required: [...new Set(parts.flatMap((part) => part.required))],
733
- };
734
- }
735
-
736
- /** The key-collecting twin of renderSchema; the branch order must match it. */
737
- function objectKeys(
738
- schema: unknown,
739
- defs: Record<string, unknown>,
740
- seen: Set<string>,
741
- depth: number,
742
- ): SchemaObjectKeys | undefined {
743
- if (depth > 4) return undefined;
744
- if (schema === null || typeof schema !== "object") return undefined;
745
- const s = schema as Record<string, unknown>;
746
-
747
- if (Array.isArray(s.allOf)) {
748
- const { allOf: _members, ...own } = s;
749
- const parts = declaresShape(own)
750
- ? [objectKeys(own, defs, seen, depth)]
751
- : [];
752
- for (const member of s.allOf) {
753
- parts.push(objectKeys(member, defs, seen, depth + 1));
754
- }
755
- // An allOf whose members are not all object shapes renders as an
756
- // intersection with a non-object half; no single key list describes it.
757
- return parts.every((part) => part !== undefined)
758
- ? mergedKeys(parts as SchemaObjectKeys[])
759
- : undefined;
760
- }
761
-
762
- if (typeof s.$ref === "string") {
763
- const name = refName(s.$ref);
764
- if (seen.has(name)) return undefined;
765
- const target = defs[name];
766
- if (target === undefined) return undefined;
767
- seen.add(name);
768
- const resolved = objectKeys(target, defs, seen, depth);
769
- seen.delete(name);
770
- return resolved;
771
- }
772
-
773
- if (Array.isArray(s.oneOf ?? s.anyOf)) return undefined;
774
- if (Array.isArray(s.enum)) return undefined;
775
- if (s.const !== undefined) return undefined;
776
- if (s.type === "array" || s.items) return undefined;
777
- if (s.type === "object" || s.properties) {
778
- const props = s.properties;
779
- if (props === null || Array.isArray(props) || typeof props !== "object") {
780
- return { properties: [], required: [] };
781
- }
782
- return {
783
- properties: Object.keys(props as Record<string, unknown>),
784
- required: Array.isArray(s.required)
785
- ? s.required.filter((key): key is string => typeof key === "string")
786
- : [],
787
- };
788
- }
789
- return undefined;
790
- }