@zackbart/connecta 0.15.1 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +40 -20
- package/CHANGELOG.md +653 -0
- package/README.md +25 -5
- package/bin/connecta.mjs +44 -3
- package/dist/access-tokens.d.ts +0 -1
- package/dist/access-tokens.js +0 -1
- package/dist/activity.d.ts +50 -2
- package/dist/activity.js +34 -1
- package/dist/apps-shell.d.ts +0 -1
- package/dist/apps-shell.js +0 -1
- package/dist/auth/bearer.d.ts +0 -1
- package/dist/auth/bearer.js +0 -1
- package/dist/auth/clerk.d.ts +0 -1
- package/dist/auth/clerk.js +45 -9
- package/dist/auth/downstream-oauth.d.ts +0 -1
- package/dist/auth/downstream-oauth.js +0 -1
- package/dist/call-admission.d.ts +0 -1
- package/dist/call-admission.js +0 -1
- package/dist/catalog-drift.d.ts +99 -0
- package/dist/catalog-drift.js +251 -0
- package/dist/catalog-fingerprint.d.ts +0 -1
- package/dist/catalog-fingerprint.js +0 -1
- package/dist/catalog-limits.d.ts +0 -1
- package/dist/catalog-limits.js +0 -1
- package/dist/catalog-service.d.ts +0 -1
- package/dist/catalog-service.js +41 -5
- package/dist/catalog.d.ts +11 -1
- package/dist/catalog.js +20 -1
- package/dist/concurrency.d.ts +0 -1
- package/dist/concurrency.js +0 -1
- package/dist/connector-scope.d.ts +0 -1
- package/dist/connector-scope.js +0 -1
- package/dist/connectors/api.d.ts +29 -21
- package/dist/connectors/api.js +45 -19
- package/dist/connectors/guarded-fetch.d.ts +106 -0
- package/dist/connectors/guarded-fetch.js +289 -0
- package/dist/connectors/remote-mcp.d.ts +0 -1
- package/dist/connectors/remote-mcp.js +0 -1
- package/dist/credentials.d.ts +0 -1
- package/dist/credentials.js +0 -1
- package/dist/errors.d.ts +30 -3
- package/dist/errors.js +1 -1
- package/dist/execute.d.ts +0 -1
- package/dist/execute.js +0 -1
- package/dist/executor-admission.d.ts +11 -1
- package/dist/executor-admission.js +28 -1
- package/dist/executor-result.d.ts +0 -1
- package/dist/executor-result.js +0 -1
- package/dist/executors/quickjs-child.d.ts +0 -1
- package/dist/executors/quickjs-child.js +0 -1
- package/dist/executors/quickjs-protocol.d.ts +0 -1
- package/dist/executors/quickjs-protocol.js +0 -1
- package/dist/executors/quickjs-runtime.d.ts +0 -1
- package/dist/executors/quickjs-runtime.js +0 -1
- package/dist/executors/quickjs.d.ts +0 -1
- package/dist/executors/quickjs.js +2 -1
- package/dist/favicon.d.ts +0 -1
- package/dist/favicon.js +0 -1
- package/dist/index.d.ts +2 -3
- package/dist/index.js +24 -7
- package/dist/invocation.d.ts +0 -1
- package/dist/invocation.js +0 -1
- package/dist/json-schema.d.ts +0 -1
- package/dist/json-schema.js +0 -1
- package/dist/mcp-result.d.ts +0 -1
- package/dist/mcp-result.js +0 -1
- package/dist/meta-tools.d.ts +0 -1
- package/dist/meta-tools.js +0 -1
- package/dist/node.d.ts +0 -1
- package/dist/node.js +0 -1
- package/dist/operator-ui/generated.d.ts +2 -3
- package/dist/operator-ui/generated.js +3 -4
- package/dist/operator-ui/model.d.ts +10 -1
- package/dist/operator-ui/model.js +0 -1
- package/dist/operator-ui/view.d.ts +167 -0
- package/dist/operator-ui/view.js +270 -0
- package/dist/providers/cloudflare.d.ts +0 -1
- package/dist/providers/cloudflare.js +261 -299
- package/dist/providers/linear.d.ts +26 -5
- package/dist/providers/linear.js +32 -61
- package/dist/providers/mixpanel.d.ts +35 -4
- package/dist/providers/mixpanel.js +53 -79
- package/dist/providers/notion.d.ts +0 -1
- package/dist/providers/notion.js +87 -42
- package/dist/providers/stripe.d.ts +13 -1
- package/dist/providers/stripe.js +31 -60
- package/dist/registry.d.ts +34 -2
- package/dist/registry.js +85 -5
- package/dist/retired-toolkits.d.ts +0 -1
- package/dist/retired-toolkits.js +0 -1
- package/dist/routes/access-tokens.d.ts +0 -1
- package/dist/routes/access-tokens.js +0 -1
- package/dist/routes/activity.d.ts +0 -1
- package/dist/routes/activity.js +0 -1
- package/dist/routes/credentials.d.ts +0 -1
- package/dist/routes/credentials.js +0 -1
- package/dist/routes/mcp.d.ts +0 -1
- package/dist/routes/mcp.js +0 -1
- package/dist/routes/oauth.d.ts +0 -1
- package/dist/routes/oauth.js +0 -1
- package/dist/routes/shared.d.ts +2 -1
- package/dist/routes/shared.js +0 -1
- package/dist/routes/ui.d.ts +0 -1
- package/dist/routes/ui.js +0 -1
- package/dist/server.d.ts +0 -1
- package/dist/server.js +11 -1
- package/dist/skills.d.ts +0 -1
- package/dist/skills.js +0 -1
- package/dist/storage/file.d.ts +0 -1
- package/dist/storage/file.js +0 -1
- package/dist/storage/memory.d.ts +0 -1
- package/dist/storage/memory.js +0 -1
- package/dist/timeout.d.ts +0 -1
- package/dist/timeout.js +0 -1
- package/dist/tool-safety.d.ts +0 -1
- package/dist/tool-safety.js +0 -1
- package/dist/types.d.ts +45 -1
- package/dist/types.js +0 -1
- package/dist/ui.d.ts +0 -1
- package/dist/ui.js +13 -137
- package/dist/validate.d.ts +13 -15
- package/dist/validate.js +24 -11
- package/dist/version.d.ts +1 -2
- package/dist/version.js +1 -2
- package/documentation/architecture.md +186 -5
- package/documentation/auth.md +11 -0
- package/documentation/call-admission.md +142 -6
- package/documentation/cloudflare.md +92 -10
- package/documentation/code-first-exploration.md +2 -2
- package/documentation/code-mode.md +14 -14
- package/documentation/connector-guides.md +172 -5
- package/documentation/connectors.md +89 -0
- package/documentation/linear.md +20 -2
- package/documentation/mcp-ui-design.md +2 -2
- package/documentation/meta-tools.md +7 -1
- package/documentation/mixpanel.md +42 -9
- package/documentation/notion.md +23 -2
- package/documentation/operations.md +325 -5
- package/documentation/operator-ui.md +120 -5
- package/documentation/provider-audit.md +160 -0
- package/documentation/provider-conventions.md +667 -0
- package/documentation/request-admission.md +167 -5
- package/documentation/rich-output-design.md +1 -2
- package/documentation/stripe.md +15 -0
- package/documentation/upgrading.md +454 -0
- package/ethos.md +14 -9
- package/examples/worker/README.md +102 -12
- package/examples/worker/src/index.ts +46 -1
- package/examples/worker/wrangler.jsonc +14 -1
- package/package.json +10 -6
- package/templates/node/.dockerignore +10 -0
- package/templates/node/.env.example +40 -3
- package/templates/node/AGENTS.md +16 -1
- package/templates/node/Dockerfile +49 -0
- package/templates/node/README.md +117 -2
- package/templates/node/docker-compose.yml +39 -0
- package/templates/node/package.json +1 -1
- package/templates/node/src/file-activity.ts +182 -0
- package/templates/node/src/index.ts +80 -4
- package/assets/connecta-clay-hero.png +0 -0
- package/dist/access-tokens.d.ts.map +0 -1
- package/dist/access-tokens.js.map +0 -1
- package/dist/activity.d.ts.map +0 -1
- package/dist/activity.js.map +0 -1
- package/dist/apps-shell.d.ts.map +0 -1
- package/dist/apps-shell.js.map +0 -1
- package/dist/auth/bearer.d.ts.map +0 -1
- package/dist/auth/bearer.js.map +0 -1
- package/dist/auth/clerk.d.ts.map +0 -1
- package/dist/auth/clerk.js.map +0 -1
- package/dist/auth/downstream-oauth.d.ts.map +0 -1
- package/dist/auth/downstream-oauth.js.map +0 -1
- package/dist/call-admission.d.ts.map +0 -1
- package/dist/call-admission.js.map +0 -1
- package/dist/catalog-fingerprint.d.ts.map +0 -1
- package/dist/catalog-fingerprint.js.map +0 -1
- package/dist/catalog-limits.d.ts.map +0 -1
- package/dist/catalog-limits.js.map +0 -1
- package/dist/catalog-service.d.ts.map +0 -1
- package/dist/catalog-service.js.map +0 -1
- package/dist/catalog.d.ts.map +0 -1
- package/dist/catalog.js.map +0 -1
- package/dist/concurrency.d.ts.map +0 -1
- package/dist/concurrency.js.map +0 -1
- package/dist/connector-scope.d.ts.map +0 -1
- package/dist/connector-scope.js.map +0 -1
- package/dist/connectors/api.d.ts.map +0 -1
- package/dist/connectors/api.js.map +0 -1
- package/dist/connectors/remote-mcp.d.ts.map +0 -1
- package/dist/connectors/remote-mcp.js.map +0 -1
- package/dist/credentials.d.ts.map +0 -1
- package/dist/credentials.js.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js.map +0 -1
- package/dist/execute.d.ts.map +0 -1
- package/dist/execute.js.map +0 -1
- package/dist/executor-admission.d.ts.map +0 -1
- package/dist/executor-admission.js.map +0 -1
- package/dist/executor-result.d.ts.map +0 -1
- package/dist/executor-result.js.map +0 -1
- package/dist/executors/quickjs-child.d.ts.map +0 -1
- package/dist/executors/quickjs-child.js.map +0 -1
- package/dist/executors/quickjs-protocol.d.ts.map +0 -1
- package/dist/executors/quickjs-protocol.js.map +0 -1
- package/dist/executors/quickjs-runtime.d.ts.map +0 -1
- package/dist/executors/quickjs-runtime.js.map +0 -1
- package/dist/executors/quickjs.d.ts.map +0 -1
- package/dist/executors/quickjs.js.map +0 -1
- package/dist/favicon.d.ts.map +0 -1
- package/dist/favicon.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/invocation.d.ts.map +0 -1
- package/dist/invocation.js.map +0 -1
- package/dist/json-schema.d.ts.map +0 -1
- package/dist/json-schema.js.map +0 -1
- package/dist/mcp-result.d.ts.map +0 -1
- package/dist/mcp-result.js.map +0 -1
- package/dist/meta-tools.d.ts.map +0 -1
- package/dist/meta-tools.js.map +0 -1
- package/dist/node.d.ts.map +0 -1
- package/dist/node.js.map +0 -1
- package/dist/operator-ui/generated.d.ts.map +0 -1
- package/dist/operator-ui/generated.js.map +0 -1
- package/dist/operator-ui/model.d.ts.map +0 -1
- package/dist/operator-ui/model.js.map +0 -1
- package/dist/providers/cloudflare.d.ts.map +0 -1
- package/dist/providers/cloudflare.js.map +0 -1
- package/dist/providers/linear.d.ts.map +0 -1
- package/dist/providers/linear.js.map +0 -1
- package/dist/providers/mixpanel.d.ts.map +0 -1
- package/dist/providers/mixpanel.js.map +0 -1
- package/dist/providers/notion.d.ts.map +0 -1
- package/dist/providers/notion.js.map +0 -1
- package/dist/providers/stripe.d.ts.map +0 -1
- package/dist/providers/stripe.js.map +0 -1
- package/dist/registry.d.ts.map +0 -1
- package/dist/registry.js.map +0 -1
- package/dist/retired-toolkits.d.ts.map +0 -1
- package/dist/retired-toolkits.js.map +0 -1
- package/dist/routes/access-tokens.d.ts.map +0 -1
- package/dist/routes/access-tokens.js.map +0 -1
- package/dist/routes/activity.d.ts.map +0 -1
- package/dist/routes/activity.js.map +0 -1
- package/dist/routes/credentials.d.ts.map +0 -1
- package/dist/routes/credentials.js.map +0 -1
- package/dist/routes/mcp.d.ts.map +0 -1
- package/dist/routes/mcp.js.map +0 -1
- package/dist/routes/oauth.d.ts.map +0 -1
- package/dist/routes/oauth.js.map +0 -1
- package/dist/routes/shared.d.ts.map +0 -1
- package/dist/routes/shared.js.map +0 -1
- package/dist/routes/ui.d.ts.map +0 -1
- package/dist/routes/ui.js.map +0 -1
- package/dist/server.d.ts.map +0 -1
- package/dist/server.js.map +0 -1
- package/dist/skills.d.ts.map +0 -1
- package/dist/skills.js.map +0 -1
- package/dist/storage/file.d.ts.map +0 -1
- package/dist/storage/file.js.map +0 -1
- package/dist/storage/memory.d.ts.map +0 -1
- package/dist/storage/memory.js.map +0 -1
- package/dist/timeout.d.ts.map +0 -1
- package/dist/timeout.js.map +0 -1
- package/dist/tool-safety.d.ts.map +0 -1
- package/dist/tool-safety.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/ui.d.ts.map +0 -1
- package/dist/ui.js.map +0 -1
- package/dist/validate.d.ts.map +0 -1
- package/dist/validate.js.map +0 -1
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js.map +0 -1
- package/examples/node/README.md +0 -52
- package/examples/node/src/index.ts +0 -71
- package/src/access-tokens.ts +0 -289
- package/src/activity.ts +0 -234
- package/src/apps-shell.ts +0 -370
- package/src/auth/bearer.ts +0 -61
- package/src/auth/clerk.ts +0 -553
- package/src/auth/downstream-oauth.ts +0 -575
- package/src/call-admission.ts +0 -521
- package/src/catalog-fingerprint.ts +0 -52
- package/src/catalog-limits.ts +0 -11
- package/src/catalog-service.ts +0 -1153
- package/src/catalog.ts +0 -790
- package/src/concurrency.ts +0 -39
- package/src/connector-scope.ts +0 -66
- package/src/connectors/api.ts +0 -159
- package/src/connectors/remote-mcp.ts +0 -1090
- package/src/credentials.ts +0 -428
- package/src/errors.ts +0 -364
- package/src/execute.ts +0 -1406
- package/src/executor-admission.ts +0 -328
- package/src/executor-result.ts +0 -107
- package/src/executors/quickjs-child.ts +0 -172
- package/src/executors/quickjs-protocol.ts +0 -99
- package/src/executors/quickjs-runtime.ts +0 -470
- package/src/executors/quickjs.ts +0 -776
- package/src/favicon.ts +0 -53
- package/src/index.ts +0 -666
- package/src/invocation.ts +0 -655
- package/src/json-schema.ts +0 -11
- package/src/mcp-result.ts +0 -43
- package/src/meta-tools.ts +0 -1607
- package/src/node.ts +0 -206
- package/src/operator-ui/browser.css +0 -423
- package/src/operator-ui/browser.ts +0 -1281
- package/src/operator-ui/generated.ts +0 -4
- package/src/operator-ui/model.ts +0 -111
- package/src/providers/cloudflare.ts +0 -3999
- package/src/providers/linear.ts +0 -302
- package/src/providers/mixpanel.ts +0 -229
- package/src/providers/notion.ts +0 -1879
- package/src/providers/stripe.ts +0 -307
- package/src/registry.ts +0 -1085
- package/src/retired-toolkits.ts +0 -14
- package/src/routes/access-tokens.ts +0 -115
- package/src/routes/activity.ts +0 -215
- package/src/routes/credentials.ts +0 -254
- package/src/routes/mcp.ts +0 -429
- package/src/routes/oauth.ts +0 -322
- package/src/routes/shared.ts +0 -247
- package/src/routes/ui.ts +0 -152
- package/src/server.ts +0 -195
- package/src/skills.ts +0 -271
- package/src/storage/file.ts +0 -123
- package/src/storage/memory.ts +0 -44
- package/src/timeout.ts +0 -84
- package/src/tool-safety.ts +0 -15
- package/src/types.ts +0 -523
- package/src/ui.ts +0 -744
- package/src/validate.ts +0 -387
- package/src/version.ts +0 -7
package/src/catalog.ts
DELETED
|
@@ -1,790 +0,0 @@
|
|
|
1
|
-
import type { JsonSchema, ToolDef } from "./types.js";
|
|
2
|
-
|
|
3
|
-
const DEFAULT_DESCRIPTION_LENGTH = 240;
|
|
4
|
-
const DISCOVERY_DESCRIPTION_LENGTH = 160;
|
|
5
|
-
export const MAX_COMPACT_DISCOVERY_SCHEMA_BYTES = 1_024;
|
|
6
|
-
const MAX_COMPACT_DISCOVERY_ENUM_BYTES =
|
|
7
|
-
MAX_COMPACT_DISCOVERY_SCHEMA_BYTES / 4;
|
|
8
|
-
const schemaEncoder = new TextEncoder();
|
|
9
|
-
const COMPACT_DISCOVERY_TRUNCATION = " /* truncated */";
|
|
10
|
-
|
|
11
|
-
export function summarizeDescription(
|
|
12
|
-
text: string | undefined,
|
|
13
|
-
full: boolean,
|
|
14
|
-
): string | undefined {
|
|
15
|
-
return summarizeToLength(text, full, DEFAULT_DESCRIPTION_LENGTH);
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
export function summarizeDiscoveryDescription(
|
|
19
|
-
text: string | undefined,
|
|
20
|
-
full: boolean,
|
|
21
|
-
): string | undefined {
|
|
22
|
-
return summarizeToLength(text, full, DISCOVERY_DESCRIPTION_LENGTH);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
function summarizeToLength(
|
|
26
|
-
text: string | undefined,
|
|
27
|
-
full: boolean,
|
|
28
|
-
maxLength: number,
|
|
29
|
-
): string | undefined {
|
|
30
|
-
if (!text) return undefined;
|
|
31
|
-
if (full) return text;
|
|
32
|
-
const compact = text.replace(/\s+/g, " ").trim();
|
|
33
|
-
if (compact.length <= maxLength) return compact;
|
|
34
|
-
return `${compact.slice(0, maxLength - 1).trimEnd()}…`;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
function lexicalTokens(text: string): string[] {
|
|
38
|
-
return text
|
|
39
|
-
.replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2")
|
|
40
|
-
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
41
|
-
.toLowerCase()
|
|
42
|
-
.replace(/[^a-z0-9]+/g, " ")
|
|
43
|
-
.trim()
|
|
44
|
-
.split(/\s+/)
|
|
45
|
-
.filter(Boolean);
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function normalized(text: string): string {
|
|
49
|
-
return lexicalTokens(text).join(" ");
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
/** Distinct normalized terms in query order, shared by ranking and feedback. */
|
|
53
|
-
export function lexicalQueryTerms(query: string): string[] {
|
|
54
|
-
return [...new Set(lexicalTokens(query))];
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
/**
|
|
58
|
-
* Conversational framing selected by the #188 research run before the #189
|
|
59
|
-
* holdout existed. Action-bearing terms such as get/list/search/find/create
|
|
60
|
-
* deliberately remain: arbitrary connector catalogs need those distinctions.
|
|
61
|
-
*/
|
|
62
|
-
const CONVERSATIONAL_QUERY_WORDS = new Set([
|
|
63
|
-
"a",
|
|
64
|
-
"all",
|
|
65
|
-
"an",
|
|
66
|
-
"and",
|
|
67
|
-
"are",
|
|
68
|
-
"can",
|
|
69
|
-
"could",
|
|
70
|
-
"current",
|
|
71
|
-
"for",
|
|
72
|
-
"from",
|
|
73
|
-
"i",
|
|
74
|
-
"in",
|
|
75
|
-
"into",
|
|
76
|
-
"it",
|
|
77
|
-
"latest",
|
|
78
|
-
"let",
|
|
79
|
-
"me",
|
|
80
|
-
"most",
|
|
81
|
-
"of",
|
|
82
|
-
"on",
|
|
83
|
-
"our",
|
|
84
|
-
"please",
|
|
85
|
-
"right",
|
|
86
|
-
"show",
|
|
87
|
-
"that",
|
|
88
|
-
"the",
|
|
89
|
-
"this",
|
|
90
|
-
"to",
|
|
91
|
-
"up",
|
|
92
|
-
"want",
|
|
93
|
-
"when",
|
|
94
|
-
"which",
|
|
95
|
-
"with",
|
|
96
|
-
"you",
|
|
97
|
-
]);
|
|
98
|
-
|
|
99
|
-
/**
|
|
100
|
-
* Remove conversational framing before the all-term/partial decision. If
|
|
101
|
-
* every term is framing, retain the original query rather than turning a
|
|
102
|
-
* search into an unfiltered catalog browse.
|
|
103
|
-
*/
|
|
104
|
-
export function lexicalSearchQuery(query: string): string {
|
|
105
|
-
const terms = normalized(query).split(/\s+/).filter(Boolean);
|
|
106
|
-
const contentTerms = terms.filter(
|
|
107
|
-
(term) => !CONVERSATIONAL_QUERY_WORDS.has(term),
|
|
108
|
-
);
|
|
109
|
-
return contentTerms.length > 0 ? contentTerms.join(" ") : query;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
interface SearchDocument {
|
|
113
|
-
tool: ToolDef;
|
|
114
|
-
name: string;
|
|
115
|
-
nameTokens: string[];
|
|
116
|
-
descriptionTokens: string[];
|
|
117
|
-
}
|
|
118
|
-
|
|
119
|
-
interface SearchIndex {
|
|
120
|
-
documents: SearchDocument[];
|
|
121
|
-
nameTokenDocuments: ReadonlyMap<string, readonly ToolDef[]>;
|
|
122
|
-
descriptionTokenDocuments: ReadonlyMap<string, readonly ToolDef[]>;
|
|
123
|
-
}
|
|
124
|
-
|
|
125
|
-
export type LexicalMatchMode = "all" | "partial";
|
|
126
|
-
|
|
127
|
-
export interface RankedTool {
|
|
128
|
-
tool: ToolDef;
|
|
129
|
-
score: number;
|
|
130
|
-
order: number;
|
|
131
|
-
exactName: boolean;
|
|
132
|
-
matchedTermCount: number;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
const searchIndexes = new WeakMap<ToolDef[], SearchIndex>();
|
|
136
|
-
|
|
137
|
-
function indexFor(tools: ToolDef[]): SearchIndex {
|
|
138
|
-
let index = searchIndexes.get(tools);
|
|
139
|
-
if (!index) {
|
|
140
|
-
const nameTokenDocuments = new Map<string, ToolDef[]>();
|
|
141
|
-
const descriptionTokenDocuments = new Map<string, ToolDef[]>();
|
|
142
|
-
const addTokens = (
|
|
143
|
-
target: Map<string, ToolDef[]>,
|
|
144
|
-
tokens: string[],
|
|
145
|
-
tool: ToolDef,
|
|
146
|
-
) => {
|
|
147
|
-
for (const token of new Set(tokens)) {
|
|
148
|
-
const documents = target.get(token) ?? [];
|
|
149
|
-
documents.push(tool);
|
|
150
|
-
target.set(token, documents);
|
|
151
|
-
}
|
|
152
|
-
};
|
|
153
|
-
const documents = tools.map((tool) => {
|
|
154
|
-
const nameTokens = lexicalTokens(tool.name);
|
|
155
|
-
const descriptionTokens = lexicalTokens(tool.description ?? "");
|
|
156
|
-
addTokens(nameTokenDocuments, nameTokens, tool);
|
|
157
|
-
addTokens(descriptionTokenDocuments, descriptionTokens, tool);
|
|
158
|
-
return {
|
|
159
|
-
tool,
|
|
160
|
-
name: nameTokens.join(" "),
|
|
161
|
-
nameTokens,
|
|
162
|
-
descriptionTokens,
|
|
163
|
-
};
|
|
164
|
-
});
|
|
165
|
-
index = {
|
|
166
|
-
documents,
|
|
167
|
-
nameTokenDocuments,
|
|
168
|
-
descriptionTokenDocuments,
|
|
169
|
-
};
|
|
170
|
-
searchIndexes.set(tools, index);
|
|
171
|
-
}
|
|
172
|
-
return index;
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
function documentsFor(tools: ToolDef[]): SearchDocument[] {
|
|
176
|
-
return indexFor(tools).documents;
|
|
177
|
-
}
|
|
178
|
-
|
|
179
|
-
/**
|
|
180
|
-
* Whole-token equality is the ordinary lexical match. A deliberately narrow
|
|
181
|
-
* inflection check retains useful singular/plural and past-tense recall
|
|
182
|
-
* without bringing back arbitrary substring matches (`list` must not match
|
|
183
|
-
* `enlist`, and `record` must not match the noun `recording`).
|
|
184
|
-
*/
|
|
185
|
-
function inflectionVariants(base: string): string[] {
|
|
186
|
-
return [
|
|
187
|
-
`${base}s`,
|
|
188
|
-
`${base}es`,
|
|
189
|
-
`${base}ed`,
|
|
190
|
-
...(base.endsWith("e") ? [`${base}d`] : []),
|
|
191
|
-
...(base.endsWith("y")
|
|
192
|
-
? [
|
|
193
|
-
`${base.slice(0, -1)}ies`,
|
|
194
|
-
`${base.slice(0, -1)}ied`,
|
|
195
|
-
]
|
|
196
|
-
: []),
|
|
197
|
-
];
|
|
198
|
-
}
|
|
199
|
-
|
|
200
|
-
function matchingTokenCandidates(term: string): Set<string> {
|
|
201
|
-
const candidates = new Set([term, ...inflectionVariants(term)]);
|
|
202
|
-
const possibleBases = [
|
|
203
|
-
...(term.endsWith("s") ? [term.slice(0, -1)] : []),
|
|
204
|
-
...(term.endsWith("es") ? [term.slice(0, -2)] : []),
|
|
205
|
-
...(term.endsWith("ed") ? [term.slice(0, -2)] : []),
|
|
206
|
-
...(term.endsWith("d") ? [term.slice(0, -1)] : []),
|
|
207
|
-
...(term.endsWith("ies")
|
|
208
|
-
? [`${term.slice(0, -3)}y`]
|
|
209
|
-
: []),
|
|
210
|
-
...(term.endsWith("ied")
|
|
211
|
-
? [`${term.slice(0, -3)}y`]
|
|
212
|
-
: []),
|
|
213
|
-
];
|
|
214
|
-
for (const base of possibleBases) {
|
|
215
|
-
if (base && inflectionVariants(base).includes(term)) {
|
|
216
|
-
candidates.add(base);
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
return candidates;
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
export interface LexicalCorpusStatistics {
|
|
223
|
-
documentCount: number;
|
|
224
|
-
documentFrequency: ReadonlyMap<string, number>;
|
|
225
|
-
nameMatches: ReadonlyMap<string, ReadonlySet<ToolDef>>;
|
|
226
|
-
descriptionMatches: ReadonlyMap<string, ReadonlySet<ToolDef>>;
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
/**
|
|
230
|
-
* Compute query-specific document frequencies across every available catalog.
|
|
231
|
-
* The caller does this once per search and shares the result with each
|
|
232
|
-
* connector rank, so ubiquitous words contribute less than discriminative
|
|
233
|
-
* ones without making any action word a stopword.
|
|
234
|
-
*/
|
|
235
|
-
export function lexicalCorpusStatistics(
|
|
236
|
-
toolSets: ToolDef[][],
|
|
237
|
-
query: string,
|
|
238
|
-
): LexicalCorpusStatistics {
|
|
239
|
-
const terms = lexicalQueryTerms(query);
|
|
240
|
-
if (terms.length === 0) {
|
|
241
|
-
return {
|
|
242
|
-
documentCount: toolSets.reduce(
|
|
243
|
-
(total, tools) => total + tools.length,
|
|
244
|
-
0,
|
|
245
|
-
),
|
|
246
|
-
documentFrequency: new Map(),
|
|
247
|
-
nameMatches: new Map(),
|
|
248
|
-
descriptionMatches: new Map(),
|
|
249
|
-
};
|
|
250
|
-
}
|
|
251
|
-
const nameMatches = new Map<string, Set<ToolDef>>();
|
|
252
|
-
const descriptionMatches = new Map<string, Set<ToolDef>>();
|
|
253
|
-
for (const term of terms) {
|
|
254
|
-
const termNameMatches = new Set<ToolDef>();
|
|
255
|
-
const termDescriptionMatches = new Set<ToolDef>();
|
|
256
|
-
for (const tools of toolSets) {
|
|
257
|
-
const index = indexFor(tools);
|
|
258
|
-
for (const candidate of matchingTokenCandidates(term)) {
|
|
259
|
-
for (const tool of index.nameTokenDocuments.get(candidate) ?? []) {
|
|
260
|
-
termNameMatches.add(tool);
|
|
261
|
-
}
|
|
262
|
-
for (
|
|
263
|
-
const tool of
|
|
264
|
-
index.descriptionTokenDocuments.get(candidate) ?? []
|
|
265
|
-
) {
|
|
266
|
-
termDescriptionMatches.add(tool);
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
}
|
|
270
|
-
nameMatches.set(term, termNameMatches);
|
|
271
|
-
descriptionMatches.set(term, termDescriptionMatches);
|
|
272
|
-
}
|
|
273
|
-
const documentFrequency = new Map(
|
|
274
|
-
terms.map((term) => [
|
|
275
|
-
term,
|
|
276
|
-
new Set([
|
|
277
|
-
...(nameMatches.get(term) ?? []),
|
|
278
|
-
...(descriptionMatches.get(term) ?? []),
|
|
279
|
-
]).size,
|
|
280
|
-
]),
|
|
281
|
-
);
|
|
282
|
-
return {
|
|
283
|
-
documentCount: toolSets.reduce(
|
|
284
|
-
(total, tools) => total + tools.length,
|
|
285
|
-
0,
|
|
286
|
-
),
|
|
287
|
-
documentFrequency,
|
|
288
|
-
nameMatches,
|
|
289
|
-
descriptionMatches,
|
|
290
|
-
};
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
function inverseDocumentFrequency(
|
|
294
|
-
term: string,
|
|
295
|
-
statistics: LexicalCorpusStatistics,
|
|
296
|
-
): number {
|
|
297
|
-
const frequency = statistics.documentFrequency.get(term) ?? 0;
|
|
298
|
-
return Math.log(
|
|
299
|
-
1 +
|
|
300
|
-
(statistics.documentCount - frequency + 0.5) /
|
|
301
|
-
(frequency + 0.5),
|
|
302
|
-
);
|
|
303
|
-
}
|
|
304
|
-
|
|
305
|
-
function scoreDocument(
|
|
306
|
-
doc: SearchDocument,
|
|
307
|
-
phrase: string,
|
|
308
|
-
terms: string[],
|
|
309
|
-
mode: LexicalMatchMode,
|
|
310
|
-
statistics: LexicalCorpusStatistics,
|
|
311
|
-
): { score: number; matchedTermCount: number } | null {
|
|
312
|
-
if (!phrase) return { score: 0, matchedTermCount: 0 };
|
|
313
|
-
const matchedTerms = terms.filter((term) =>
|
|
314
|
-
statistics.nameMatches.get(term)?.has(doc.tool) ||
|
|
315
|
-
statistics.descriptionMatches.get(term)?.has(doc.tool),
|
|
316
|
-
);
|
|
317
|
-
if (mode === "all" && matchedTerms.length !== terms.length) return null;
|
|
318
|
-
if (matchedTerms.length === 0) return null;
|
|
319
|
-
|
|
320
|
-
// Coverage remains meaningful in partial mode, but is IDF-weighted rather
|
|
321
|
-
// than a raw term count: one rare domain term can beat several ubiquitous
|
|
322
|
-
// catalog verbs.
|
|
323
|
-
let score = matchedTerms.reduce(
|
|
324
|
-
(total, term) =>
|
|
325
|
-
total + 4 * inverseDocumentFrequency(term, statistics),
|
|
326
|
-
0,
|
|
327
|
-
);
|
|
328
|
-
const phraseWeight = terms.reduce(
|
|
329
|
-
(total, term) =>
|
|
330
|
-
total + inverseDocumentFrequency(term, statistics),
|
|
331
|
-
0,
|
|
332
|
-
);
|
|
333
|
-
if (doc.name === phrase) score += 40 * phraseWeight;
|
|
334
|
-
else if (doc.name.startsWith(`${phrase} `)) score += 24 * phraseWeight;
|
|
335
|
-
else if (` ${doc.name} `.includes(` ${phrase} `)) {
|
|
336
|
-
score += 16 * phraseWeight;
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
for (const term of matchedTerms) {
|
|
340
|
-
const weight = inverseDocumentFrequency(term, statistics);
|
|
341
|
-
if (doc.nameTokens.includes(term)) score += 12 * weight;
|
|
342
|
-
else if (statistics.nameMatches.get(term)?.has(doc.tool)) {
|
|
343
|
-
score += 8 * weight;
|
|
344
|
-
}
|
|
345
|
-
if (doc.descriptionTokens.includes(term)) score += 3 * weight;
|
|
346
|
-
else if (statistics.descriptionMatches.get(term)?.has(doc.tool)) {
|
|
347
|
-
score += 1.5 * weight;
|
|
348
|
-
}
|
|
349
|
-
}
|
|
350
|
-
return { score, matchedTermCount: matchedTerms.length };
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
function queryContainsExactName(doc: SearchDocument, phrase: string): boolean {
|
|
354
|
-
if (!doc.name || !phrase) return false;
|
|
355
|
-
return (` ${phrase} `).includes(` ${doc.name} `);
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
/**
|
|
359
|
-
* Rank a connector's tools while caching its normalized plain-data index.
|
|
360
|
-
* `exactNameQuery` may retain framing removed from the scoring query: those
|
|
361
|
-
* words are weak term evidence, but remain part of a real tool-name phrase.
|
|
362
|
-
*/
|
|
363
|
-
export function rankTools(
|
|
364
|
-
tools: ToolDef[],
|
|
365
|
-
query: string,
|
|
366
|
-
mode: LexicalMatchMode = "all",
|
|
367
|
-
statistics: LexicalCorpusStatistics = lexicalCorpusStatistics(
|
|
368
|
-
[tools],
|
|
369
|
-
query,
|
|
370
|
-
),
|
|
371
|
-
exactNameQuery: string = query,
|
|
372
|
-
): RankedTool[] {
|
|
373
|
-
const phrase = normalized(query);
|
|
374
|
-
const exactNamePhrase = normalized(exactNameQuery);
|
|
375
|
-
const terms = [...new Set(phrase.split(/\s+/).filter(Boolean))];
|
|
376
|
-
const ranked: RankedTool[] = [];
|
|
377
|
-
documentsFor(tools).forEach((doc, order) => {
|
|
378
|
-
const scored = scoreDocument(doc, phrase, terms, mode, statistics);
|
|
379
|
-
if (scored !== null) {
|
|
380
|
-
ranked.push({
|
|
381
|
-
tool: doc.tool,
|
|
382
|
-
score: scored.score,
|
|
383
|
-
order,
|
|
384
|
-
exactName: queryContainsExactName(doc, exactNamePhrase),
|
|
385
|
-
matchedTermCount: scored.matchedTermCount,
|
|
386
|
-
});
|
|
387
|
-
}
|
|
388
|
-
});
|
|
389
|
-
return ranked;
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
function refName(ref: string): string {
|
|
393
|
-
return ref.split("/").pop() ?? ref;
|
|
394
|
-
}
|
|
395
|
-
|
|
396
|
-
/**
|
|
397
|
-
* Whether a schema declares anything renderSchema knows how to render on its
|
|
398
|
-
* own. Used to decide if the non-allOf half of a schema is worth rendering:
|
|
399
|
-
* without this, a plain `{ allOf: [...] }` would render its (empty) local half
|
|
400
|
-
* through the raw-JSON fallback and emit `{} & …`.
|
|
401
|
-
*/
|
|
402
|
-
function declaresShape(s: Record<string, unknown>): boolean {
|
|
403
|
-
return (
|
|
404
|
-
typeof s.$ref === "string" ||
|
|
405
|
-
Array.isArray(s.oneOf) ||
|
|
406
|
-
Array.isArray(s.anyOf) ||
|
|
407
|
-
Array.isArray(s.enum) ||
|
|
408
|
-
s.const !== undefined ||
|
|
409
|
-
s.items !== undefined ||
|
|
410
|
-
s.properties !== undefined ||
|
|
411
|
-
s.type !== undefined
|
|
412
|
-
);
|
|
413
|
-
}
|
|
414
|
-
|
|
415
|
-
/**
|
|
416
|
-
* Parenthesize a top-level union so it doesn't read as part of a surrounding
|
|
417
|
-
* `&`. Only separators outside braces count, so a nested union or a property
|
|
418
|
-
* description containing a pipe doesn't trigger stray parentheses.
|
|
419
|
-
*/
|
|
420
|
-
function grouped(part: string): string {
|
|
421
|
-
let nesting = 0;
|
|
422
|
-
for (let i = 0; i < part.length; i += 1) {
|
|
423
|
-
const char = part[i];
|
|
424
|
-
if (char === "{" || char === "(" || char === "[") nesting += 1;
|
|
425
|
-
else if (char === "}" || char === ")" || char === "]") nesting -= 1;
|
|
426
|
-
else if (nesting === 0 && part.startsWith(" | ", i)) return `(${part})`;
|
|
427
|
-
}
|
|
428
|
-
return part;
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
function renderEnum(
|
|
432
|
-
values: unknown[],
|
|
433
|
-
byteLimit: number | undefined,
|
|
434
|
-
onTruncated: (() => void) | undefined,
|
|
435
|
-
): string {
|
|
436
|
-
if (values.length === 0) return "never";
|
|
437
|
-
const renderedValues = values.map((value) => JSON.stringify(value));
|
|
438
|
-
const full = renderedValues.join(" | ");
|
|
439
|
-
if (
|
|
440
|
-
byteLimit === undefined ||
|
|
441
|
-
schemaEncoder.encode(full).length <= byteLimit
|
|
442
|
-
) {
|
|
443
|
-
return full;
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
onTruncated?.();
|
|
447
|
-
const marker = (omitted: number) =>
|
|
448
|
-
`unknown /* ${omitted} enum ${omitted === 1 ? "value" : "values"} omitted */`;
|
|
449
|
-
let rendered = `(${marker(values.length)})`;
|
|
450
|
-
const prefix: string[] = [];
|
|
451
|
-
for (let index = 0; index < renderedValues.length - 1; index += 1) {
|
|
452
|
-
prefix.push(renderedValues[index] as string);
|
|
453
|
-
const omitted = renderedValues.length - prefix.length;
|
|
454
|
-
const candidate = `(${prefix.join(" | ")} | ${marker(omitted)})`;
|
|
455
|
-
if (schemaEncoder.encode(candidate).length > byteLimit) break;
|
|
456
|
-
rendered = candidate;
|
|
457
|
-
}
|
|
458
|
-
return rendered;
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
function renderSchema(
|
|
462
|
-
schema: unknown,
|
|
463
|
-
defs: Record<string, unknown>,
|
|
464
|
-
seen: Set<string>,
|
|
465
|
-
depth: number,
|
|
466
|
-
options: {
|
|
467
|
-
propertyDescriptions: boolean;
|
|
468
|
-
requiredFirst: boolean;
|
|
469
|
-
enumByteLimit?: number;
|
|
470
|
-
onEnumTruncated?: () => void;
|
|
471
|
-
},
|
|
472
|
-
): string {
|
|
473
|
-
if (depth > 4) return "…";
|
|
474
|
-
if (schema === null || typeof schema !== "object") {
|
|
475
|
-
return JSON.stringify(schema);
|
|
476
|
-
}
|
|
477
|
-
const s = schema as Record<string, unknown>;
|
|
478
|
-
|
|
479
|
-
// allOf composes rather than replaces: it is checked before every other
|
|
480
|
-
// keyword, and renders the schema's own shape alongside its members instead
|
|
481
|
-
// of returning early. A schema carrying both allOf and properties (the usual
|
|
482
|
-
// OpenAPI-derived "extend this base" shape, and equally legal with $ref,
|
|
483
|
-
// enum, const, or items) would otherwise silently drop whichever half lost
|
|
484
|
-
// the branch race. The schema's own shape comes first, being the more
|
|
485
|
-
// specific half, and is rendered at the current depth because its members
|
|
486
|
-
// sit at this nesting level, not one below.
|
|
487
|
-
if (Array.isArray(s.allOf)) {
|
|
488
|
-
const { allOf: _members, ...own } = s;
|
|
489
|
-
const parts = declaresShape(own)
|
|
490
|
-
? [renderSchema(own, defs, seen, depth, options)]
|
|
491
|
-
: [];
|
|
492
|
-
for (const member of s.allOf) {
|
|
493
|
-
parts.push(renderSchema(member, defs, seen, depth + 1, options));
|
|
494
|
-
}
|
|
495
|
-
if (parts.length === 0) return "unknown";
|
|
496
|
-
if (parts.length === 1) return parts[0] as string;
|
|
497
|
-
return parts.map(grouped).join(" & ");
|
|
498
|
-
}
|
|
499
|
-
|
|
500
|
-
if (typeof s.$ref === "string") {
|
|
501
|
-
const name = refName(s.$ref);
|
|
502
|
-
if (seen.has(name)) return name;
|
|
503
|
-
const target = defs[name];
|
|
504
|
-
if (target === undefined) return name;
|
|
505
|
-
seen.add(name);
|
|
506
|
-
const rendered = renderSchema(target, defs, seen, depth, options);
|
|
507
|
-
seen.delete(name);
|
|
508
|
-
return rendered;
|
|
509
|
-
}
|
|
510
|
-
|
|
511
|
-
const union = (s.oneOf ?? s.anyOf) as unknown[] | undefined;
|
|
512
|
-
if (Array.isArray(union)) {
|
|
513
|
-
return (
|
|
514
|
-
union
|
|
515
|
-
.map((u) => renderSchema(u, defs, seen, depth + 1, options))
|
|
516
|
-
.join(" | ") ||
|
|
517
|
-
"unknown"
|
|
518
|
-
);
|
|
519
|
-
}
|
|
520
|
-
if (Array.isArray(s.enum)) {
|
|
521
|
-
return renderEnum(s.enum, options.enumByteLimit, options.onEnumTruncated);
|
|
522
|
-
}
|
|
523
|
-
// Checked before type/properties so a discriminator like
|
|
524
|
-
// { type: "string", const: "emoji" } renders as "emoji" rather than string.
|
|
525
|
-
// JSON.stringify(undefined) returns undefined (not a string), so an explicit
|
|
526
|
-
// `const: undefined` must fall through to the regular type rendering.
|
|
527
|
-
if (s.const !== undefined) return JSON.stringify(s.const);
|
|
528
|
-
|
|
529
|
-
const type = s.type;
|
|
530
|
-
if (type === "array" || s.items) {
|
|
531
|
-
const items = s.items
|
|
532
|
-
? renderSchema(s.items, defs, seen, depth + 1, options)
|
|
533
|
-
: "unknown";
|
|
534
|
-
return `${items}[]`;
|
|
535
|
-
}
|
|
536
|
-
if (type === "object" || s.properties) {
|
|
537
|
-
const props = (s.properties ?? {}) as Record<string, unknown>;
|
|
538
|
-
const required = new Set(
|
|
539
|
-
(Array.isArray(s.required) ? s.required : []) as string[],
|
|
540
|
-
);
|
|
541
|
-
const declaredKeys = Object.keys(props);
|
|
542
|
-
const keys = options.requiredFirst
|
|
543
|
-
? [
|
|
544
|
-
...declaredKeys.filter((key) => required.has(key)),
|
|
545
|
-
...declaredKeys.filter((key) => !required.has(key)),
|
|
546
|
-
]
|
|
547
|
-
: declaredKeys;
|
|
548
|
-
if (keys.length === 0) return "{}";
|
|
549
|
-
return `{ ${keys
|
|
550
|
-
.map((key) => {
|
|
551
|
-
const optional = required.has(key) ? "" : "?";
|
|
552
|
-
const rendered = renderSchema(
|
|
553
|
-
props[key],
|
|
554
|
-
defs,
|
|
555
|
-
seen,
|
|
556
|
-
depth + 1,
|
|
557
|
-
options,
|
|
558
|
-
);
|
|
559
|
-
const description = (
|
|
560
|
-
props[key] as Record<string, unknown> | null
|
|
561
|
-
)?.description;
|
|
562
|
-
const comment =
|
|
563
|
-
options.propertyDescriptions && typeof description === "string"
|
|
564
|
-
? ` // ${description}`
|
|
565
|
-
: "";
|
|
566
|
-
return `${key}${optional}: ${rendered}${comment}`;
|
|
567
|
-
})
|
|
568
|
-
.join(", ")} }`;
|
|
569
|
-
}
|
|
570
|
-
if (typeof type === "string") return type;
|
|
571
|
-
if (Array.isArray(type)) return type.join(" | ");
|
|
572
|
-
return JSON.stringify(schema);
|
|
573
|
-
}
|
|
574
|
-
|
|
575
|
-
const compactSchemas = new WeakMap<JsonSchema, string>();
|
|
576
|
-
|
|
577
|
-
/** Render and cache a compact TypeScript-like representation of JSON Schema. */
|
|
578
|
-
export function compactSchema(schema: JsonSchema): string {
|
|
579
|
-
const cached = compactSchemas.get(schema);
|
|
580
|
-
if (cached) return cached;
|
|
581
|
-
const defs = {
|
|
582
|
-
...(schema.$defs as Record<string, unknown>),
|
|
583
|
-
...(schema.definitions as Record<string, unknown>),
|
|
584
|
-
};
|
|
585
|
-
let rendered: string;
|
|
586
|
-
try {
|
|
587
|
-
rendered = renderSchema(schema, defs, new Set(), 0, {
|
|
588
|
-
propertyDescriptions: true,
|
|
589
|
-
requiredFirst: false,
|
|
590
|
-
});
|
|
591
|
-
} catch {
|
|
592
|
-
rendered = JSON.stringify(schema);
|
|
593
|
-
}
|
|
594
|
-
compactSchemas.set(schema, rendered);
|
|
595
|
-
return rendered;
|
|
596
|
-
}
|
|
597
|
-
|
|
598
|
-
export interface CompactDiscoverySchema {
|
|
599
|
-
text: string;
|
|
600
|
-
truncated: boolean;
|
|
601
|
-
}
|
|
602
|
-
|
|
603
|
-
const compactDiscoverySchemas = new WeakMap<
|
|
604
|
-
JsonSchema,
|
|
605
|
-
CompactDiscoverySchema
|
|
606
|
-
>();
|
|
607
|
-
|
|
608
|
-
/**
|
|
609
|
-
* A valid, bounded replacement for a discovery shape too large to carry.
|
|
610
|
-
*
|
|
611
|
-
* Required object keys come first and every retained key is JSON-quoted, so
|
|
612
|
-
* arbitrary downstream names remain valid TypeScript property signatures.
|
|
613
|
-
* Types become `unknown`: pretending a severed nested type is exact would be
|
|
614
|
-
* worse than making the existing truncation flag's recovery route explicit.
|
|
615
|
-
*/
|
|
616
|
-
function truncatedDiscoverySchema(schema: JsonSchema): string {
|
|
617
|
-
const keys = schemaObjectKeys(schema);
|
|
618
|
-
if (!keys) return `unknown${COMPACT_DISCOVERY_TRUNCATION}`;
|
|
619
|
-
const required = new Set(keys.required);
|
|
620
|
-
const ordered = [
|
|
621
|
-
...keys.properties.filter((key) => required.has(key)),
|
|
622
|
-
...keys.properties.filter((key) => !required.has(key)),
|
|
623
|
-
];
|
|
624
|
-
const parts: string[] = [];
|
|
625
|
-
for (const key of ordered) {
|
|
626
|
-
const part = `${JSON.stringify(key)}${required.has(key) ? "" : "?"}: unknown`;
|
|
627
|
-
const candidate = `{ ${[...parts, part].join(", ")} }${COMPACT_DISCOVERY_TRUNCATION}`;
|
|
628
|
-
if (
|
|
629
|
-
schemaEncoder.encode(candidate).length >
|
|
630
|
-
MAX_COMPACT_DISCOVERY_SCHEMA_BYTES
|
|
631
|
-
) {
|
|
632
|
-
break;
|
|
633
|
-
}
|
|
634
|
-
parts.push(part);
|
|
635
|
-
}
|
|
636
|
-
if (parts.length === 0 && ordered.length > 0) {
|
|
637
|
-
return `unknown${COMPACT_DISCOVERY_TRUNCATION}`;
|
|
638
|
-
}
|
|
639
|
-
return `{ ${parts.join(", ")} }${COMPACT_DISCOVERY_TRUNCATION}`;
|
|
640
|
-
}
|
|
641
|
-
|
|
642
|
-
/**
|
|
643
|
-
* Render the schema shape carried by search results.
|
|
644
|
-
*
|
|
645
|
-
* Search is a routing step, so repeated property prose does not earn its
|
|
646
|
-
* context cost there. Required inputs render first, and the result has a hard
|
|
647
|
-
* UTF-8 budget; exact JSON and the prose-rich compact rendering remain
|
|
648
|
-
* available through the existing full retrieval paths.
|
|
649
|
-
*/
|
|
650
|
-
export function compactDiscoverySchema(
|
|
651
|
-
schema: JsonSchema,
|
|
652
|
-
): CompactDiscoverySchema {
|
|
653
|
-
const cached = compactDiscoverySchemas.get(schema);
|
|
654
|
-
if (cached) return cached;
|
|
655
|
-
const defs = {
|
|
656
|
-
...(schema.$defs as Record<string, unknown>),
|
|
657
|
-
...(schema.definitions as Record<string, unknown>),
|
|
658
|
-
};
|
|
659
|
-
let rendered: string;
|
|
660
|
-
let enumTruncated = false;
|
|
661
|
-
try {
|
|
662
|
-
rendered = renderSchema(schema, defs, new Set(), 0, {
|
|
663
|
-
propertyDescriptions: false,
|
|
664
|
-
requiredFirst: true,
|
|
665
|
-
// Three near-cap enums spend about three quarters of the complete shape
|
|
666
|
-
// budget, leaving the final quarter for surrounding syntax before the
|
|
667
|
-
// unchanged global fallback applies. Whole values keep this UTF-8 safe.
|
|
668
|
-
enumByteLimit: MAX_COMPACT_DISCOVERY_ENUM_BYTES,
|
|
669
|
-
onEnumTruncated: () => {
|
|
670
|
-
enumTruncated = true;
|
|
671
|
-
},
|
|
672
|
-
});
|
|
673
|
-
} catch {
|
|
674
|
-
rendered = JSON.stringify(schema);
|
|
675
|
-
}
|
|
676
|
-
const bytes = schemaEncoder.encode(rendered);
|
|
677
|
-
let result: CompactDiscoverySchema;
|
|
678
|
-
if (bytes.length <= MAX_COMPACT_DISCOVERY_SCHEMA_BYTES) {
|
|
679
|
-
result = { text: rendered, truncated: enumTruncated };
|
|
680
|
-
} else {
|
|
681
|
-
result = {
|
|
682
|
-
text: truncatedDiscoverySchema(schema),
|
|
683
|
-
truncated: true,
|
|
684
|
-
};
|
|
685
|
-
}
|
|
686
|
-
compactDiscoverySchemas.set(schema, result);
|
|
687
|
-
return result;
|
|
688
|
-
}
|
|
689
|
-
|
|
690
|
-
/** The property and required names a schema resolves to, or undefined. */
|
|
691
|
-
export interface SchemaObjectKeys {
|
|
692
|
-
properties: string[];
|
|
693
|
-
required: string[];
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
/**
|
|
697
|
-
* Walk a schema the way renderSchema does — composing `allOf` and resolving
|
|
698
|
-
* `$ref` against the root's `$defs`/`definitions` — and collect the top-level
|
|
699
|
-
* property names it would render. A shallow `Object.keys(schema.properties)`
|
|
700
|
-
* disagrees with the rendered compact schema for exactly the shapes real
|
|
701
|
-
* connectors emit (a top-level `$ref` to a `$defs` entry, or the OpenAPI
|
|
702
|
-
* "extend this base" `allOf`), which is worse than no metadata at all: it
|
|
703
|
-
* reports an empty field list for a tool that plainly has fields.
|
|
704
|
-
*
|
|
705
|
-
* Returns undefined when the schema is not an object shape at all — a union,
|
|
706
|
-
* array, enum, or unresolvable `$ref`. Absent metadata tells a caller to read
|
|
707
|
-
* the rendered schema instead; an empty array would claim the tool takes no
|
|
708
|
-
* fields.
|
|
709
|
-
*/
|
|
710
|
-
export function schemaObjectKeys(
|
|
711
|
-
schema: JsonSchema | undefined,
|
|
712
|
-
): SchemaObjectKeys | undefined {
|
|
713
|
-
if (!schema) return undefined;
|
|
714
|
-
const defs = {
|
|
715
|
-
...(schema.$defs as Record<string, unknown>),
|
|
716
|
-
...(schema.definitions as Record<string, unknown>),
|
|
717
|
-
};
|
|
718
|
-
try {
|
|
719
|
-
return objectKeys(schema, defs, new Set(), 0);
|
|
720
|
-
} catch {
|
|
721
|
-
return undefined;
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
|
|
725
|
-
/** Merge in declaration order, first occurrence winning, as renderSchema renders. */
|
|
726
|
-
function mergedKeys(
|
|
727
|
-
parts: readonly SchemaObjectKeys[],
|
|
728
|
-
): SchemaObjectKeys | undefined {
|
|
729
|
-
if (parts.length === 0) return undefined;
|
|
730
|
-
return {
|
|
731
|
-
properties: [...new Set(parts.flatMap((part) => part.properties))],
|
|
732
|
-
required: [...new Set(parts.flatMap((part) => part.required))],
|
|
733
|
-
};
|
|
734
|
-
}
|
|
735
|
-
|
|
736
|
-
/** The key-collecting twin of renderSchema; the branch order must match it. */
|
|
737
|
-
function objectKeys(
|
|
738
|
-
schema: unknown,
|
|
739
|
-
defs: Record<string, unknown>,
|
|
740
|
-
seen: Set<string>,
|
|
741
|
-
depth: number,
|
|
742
|
-
): SchemaObjectKeys | undefined {
|
|
743
|
-
if (depth > 4) return undefined;
|
|
744
|
-
if (schema === null || typeof schema !== "object") return undefined;
|
|
745
|
-
const s = schema as Record<string, unknown>;
|
|
746
|
-
|
|
747
|
-
if (Array.isArray(s.allOf)) {
|
|
748
|
-
const { allOf: _members, ...own } = s;
|
|
749
|
-
const parts = declaresShape(own)
|
|
750
|
-
? [objectKeys(own, defs, seen, depth)]
|
|
751
|
-
: [];
|
|
752
|
-
for (const member of s.allOf) {
|
|
753
|
-
parts.push(objectKeys(member, defs, seen, depth + 1));
|
|
754
|
-
}
|
|
755
|
-
// An allOf whose members are not all object shapes renders as an
|
|
756
|
-
// intersection with a non-object half; no single key list describes it.
|
|
757
|
-
return parts.every((part) => part !== undefined)
|
|
758
|
-
? mergedKeys(parts as SchemaObjectKeys[])
|
|
759
|
-
: undefined;
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
if (typeof s.$ref === "string") {
|
|
763
|
-
const name = refName(s.$ref);
|
|
764
|
-
if (seen.has(name)) return undefined;
|
|
765
|
-
const target = defs[name];
|
|
766
|
-
if (target === undefined) return undefined;
|
|
767
|
-
seen.add(name);
|
|
768
|
-
const resolved = objectKeys(target, defs, seen, depth);
|
|
769
|
-
seen.delete(name);
|
|
770
|
-
return resolved;
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
if (Array.isArray(s.oneOf ?? s.anyOf)) return undefined;
|
|
774
|
-
if (Array.isArray(s.enum)) return undefined;
|
|
775
|
-
if (s.const !== undefined) return undefined;
|
|
776
|
-
if (s.type === "array" || s.items) return undefined;
|
|
777
|
-
if (s.type === "object" || s.properties) {
|
|
778
|
-
const props = s.properties;
|
|
779
|
-
if (props === null || Array.isArray(props) || typeof props !== "object") {
|
|
780
|
-
return { properties: [], required: [] };
|
|
781
|
-
}
|
|
782
|
-
return {
|
|
783
|
-
properties: Object.keys(props as Record<string, unknown>),
|
|
784
|
-
required: Array.isArray(s.required)
|
|
785
|
-
? s.required.filter((key): key is string => typeof key === "string")
|
|
786
|
-
: [],
|
|
787
|
-
};
|
|
788
|
-
}
|
|
789
|
-
return undefined;
|
|
790
|
-
}
|