@zackbart/connecta 0.7.9 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/README.md +67 -144
- package/dist/auth/bearer.d.ts +1 -1
- package/dist/auth/bearer.js +4 -4
- package/dist/auth/bearer.js.map +1 -1
- package/dist/auth/clerk.d.ts +1 -1
- package/dist/auth/clerk.js +1 -1
- package/dist/call-admission.d.ts.map +1 -1
- package/dist/call-admission.js +9 -3
- package/dist/call-admission.js.map +1 -1
- package/dist/catalog-fingerprint.d.ts +20 -0
- package/dist/catalog-fingerprint.d.ts.map +1 -0
- package/dist/catalog-fingerprint.js +34 -0
- package/dist/catalog-fingerprint.js.map +1 -0
- package/dist/catalog-limits.d.ts +10 -0
- package/dist/catalog-limits.d.ts.map +1 -0
- package/dist/catalog-limits.js +10 -0
- package/dist/catalog-limits.js.map +1 -0
- package/dist/catalog-service.d.ts +138 -0
- package/dist/catalog-service.d.ts.map +1 -0
- package/dist/catalog-service.js +407 -0
- package/dist/catalog-service.js.map +1 -0
- package/dist/catalog.js +2 -2
- package/dist/catalog.js.map +1 -1
- package/dist/concurrency.d.ts +4 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/concurrency.js +31 -0
- package/dist/concurrency.js.map +1 -0
- package/dist/connectors/api.d.ts.map +1 -1
- package/dist/connectors/api.js +22 -12
- package/dist/connectors/api.js.map +1 -1
- package/dist/connectors/remote-mcp.d.ts.map +1 -1
- package/dist/connectors/remote-mcp.js +44 -19
- package/dist/connectors/remote-mcp.js.map +1 -1
- package/dist/credential-health.d.ts +0 -22
- package/dist/credential-health.d.ts.map +1 -1
- package/dist/credential-health.js +14 -10
- package/dist/credential-health.js.map +1 -1
- package/dist/credentials.d.ts +3 -2
- package/dist/credentials.d.ts.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/execute.d.ts +9 -8
- package/dist/execute.d.ts.map +1 -1
- package/dist/execute.js +144 -321
- package/dist/execute.js.map +1 -1
- package/dist/executor-result.d.ts +0 -3
- package/dist/executor-result.d.ts.map +1 -1
- package/dist/executor-result.js +2 -2
- package/dist/executor-result.js.map +1 -1
- package/dist/executors/quickjs-child.js +3 -3
- package/dist/executors/quickjs-child.js.map +1 -1
- package/dist/executors/quickjs-protocol.d.ts +4 -1
- package/dist/executors/quickjs-protocol.d.ts.map +1 -1
- package/dist/executors/quickjs-runtime.d.ts.map +1 -1
- package/dist/executors/quickjs-runtime.js +7 -2
- package/dist/executors/quickjs-runtime.js.map +1 -1
- package/dist/executors/quickjs.d.ts.map +1 -1
- package/dist/executors/quickjs.js +4 -1
- package/dist/executors/quickjs.js.map +1 -1
- package/dist/favicon.js +2 -2
- package/dist/favicon.js.map +1 -1
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +44 -17
- package/dist/index.js.map +1 -1
- package/dist/invocation.d.ts +96 -0
- package/dist/invocation.d.ts.map +1 -0
- package/dist/invocation.js +369 -0
- package/dist/invocation.js.map +1 -0
- package/dist/meta-tools.d.ts +8 -86
- package/dist/meta-tools.d.ts.map +1 -1
- package/dist/meta-tools.js +145 -583
- package/dist/meta-tools.js.map +1 -1
- package/dist/operator-ui/generated.d.ts +3 -0
- package/dist/operator-ui/generated.d.ts.map +1 -0
- package/dist/operator-ui/generated.js +5 -0
- package/dist/operator-ui/generated.js.map +1 -0
- package/dist/operator-ui/model.d.ts +94 -0
- package/dist/operator-ui/model.d.ts.map +1 -0
- package/dist/operator-ui/model.js +28 -0
- package/dist/operator-ui/model.js.map +1 -0
- package/dist/registry.d.ts +22 -6
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +328 -35
- package/dist/registry.js.map +1 -1
- package/dist/routes/activity.d.ts +3 -0
- package/dist/routes/activity.d.ts.map +1 -0
- package/dist/routes/activity.js +173 -0
- package/dist/routes/activity.js.map +1 -0
- package/dist/routes/credentials.d.ts +3 -0
- package/dist/routes/credentials.d.ts.map +1 -0
- package/dist/routes/credentials.js +181 -0
- package/dist/routes/credentials.js.map +1 -0
- package/dist/routes/mcp.d.ts +8 -0
- package/dist/routes/mcp.d.ts.map +1 -0
- package/dist/routes/mcp.js +383 -0
- package/dist/routes/mcp.js.map +1 -0
- package/dist/routes/oauth.d.ts +4 -0
- package/dist/routes/oauth.d.ts.map +1 -0
- package/dist/routes/oauth.js +268 -0
- package/dist/routes/oauth.js.map +1 -0
- package/dist/routes/shared.d.ts +97 -0
- package/dist/routes/shared.d.ts.map +1 -0
- package/dist/routes/shared.js +199 -0
- package/dist/routes/shared.js.map +1 -0
- package/dist/routes/ui.d.ts +3 -0
- package/dist/routes/ui.d.ts.map +1 -0
- package/dist/routes/ui.js +122 -0
- package/dist/routes/ui.js.map +1 -0
- package/dist/server.d.ts +9 -40
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +63 -1293
- package/dist/server.js.map +1 -1
- package/dist/skills.d.ts +0 -15
- package/dist/skills.d.ts.map +1 -1
- package/dist/skills.js +6 -5
- package/dist/skills.js.map +1 -1
- package/dist/storage/file.d.ts.map +1 -1
- package/dist/storage/file.js +3 -1
- package/dist/storage/file.js.map +1 -1
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +3 -1
- package/dist/storage/memory.js.map +1 -1
- package/dist/toolkits.d.ts +0 -22
- package/dist/toolkits.d.ts.map +1 -1
- package/dist/toolkits.js +2 -2
- package/dist/toolkits.js.map +1 -1
- package/dist/types.d.ts +9 -2
- package/dist/types.d.ts.map +1 -1
- package/dist/ui.d.ts +3 -93
- package/dist/ui.d.ts.map +1 -1
- package/dist/ui.js +26 -1393
- package/dist/ui.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +15 -8
- package/src/auth/bearer.ts +4 -4
- package/src/auth/clerk.ts +1 -1
- package/src/call-admission.ts +8 -5
- package/src/catalog-fingerprint.ts +52 -0
- package/src/catalog-limits.ts +11 -0
- package/src/catalog-service.ts +601 -0
- package/src/catalog.ts +2 -2
- package/src/concurrency.ts +39 -0
- package/src/connectors/api.ts +22 -12
- package/src/connectors/remote-mcp.ts +52 -22
- package/src/credential-health.ts +13 -10
- package/src/credentials.ts +2 -2
- package/src/errors.ts +1 -1
- package/src/execute.ts +194 -409
- package/src/executor-result.ts +2 -2
- package/src/executors/quickjs-child.ts +8 -4
- package/src/executors/quickjs-protocol.ts +1 -1
- package/src/executors/quickjs-runtime.ts +5 -1
- package/src/executors/quickjs.ts +11 -8
- package/src/favicon.ts +2 -2
- package/src/index.ts +49 -17
- package/src/invocation.ts +527 -0
- package/src/meta-tools.ts +366 -906
- package/src/operator-ui/browser.css +413 -0
- package/src/operator-ui/browser.ts +1121 -0
- package/src/operator-ui/generated.ts +4 -0
- package/src/operator-ui/model.ts +127 -0
- package/src/registry.ts +434 -44
- package/src/routes/activity.ts +220 -0
- package/src/routes/credentials.ts +266 -0
- package/src/routes/mcp.ts +485 -0
- package/src/routes/oauth.ts +328 -0
- package/src/routes/shared.ts +330 -0
- package/src/routes/ui.ts +153 -0
- package/src/server.ts +66 -1703
- package/src/skills.ts +6 -5
- package/src/storage/file.ts +3 -1
- package/src/storage/memory.ts +3 -1
- package/src/toolkits.ts +2 -2
- package/src/types.ts +9 -2
- package/src/ui.ts +52 -1493
- package/src/version.ts +1 -1
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filter by connector identity/description or tool name/description. A
|
|
3
|
+
* connector-level match stays visible even when it currently exposes no tools
|
|
4
|
+
* (for example while authorization is required).
|
|
5
|
+
*/
|
|
6
|
+
export function filterUiConnectors(connectors, query) {
|
|
7
|
+
const q = query.trim().toLowerCase();
|
|
8
|
+
const filtered = [];
|
|
9
|
+
for (const connector of connectors) {
|
|
10
|
+
const connectorText = [
|
|
11
|
+
connector.id,
|
|
12
|
+
connector.title,
|
|
13
|
+
connector.description,
|
|
14
|
+
connector.status,
|
|
15
|
+
]
|
|
16
|
+
.join(" ")
|
|
17
|
+
.toLowerCase();
|
|
18
|
+
const connectorMatches = Boolean(q && connectorText.includes(q));
|
|
19
|
+
const tools = connector.tools.filter((tool) => !q ||
|
|
20
|
+
connectorMatches ||
|
|
21
|
+
`${tool.name} ${tool.description ?? ""}`.toLowerCase().includes(q));
|
|
22
|
+
if (q && tools.length === 0 && !connectorMatches)
|
|
23
|
+
continue;
|
|
24
|
+
filtered.push({ connector, tools });
|
|
25
|
+
}
|
|
26
|
+
return filtered;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=model.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model.js","sourceRoot":"","sources":["../../src/operator-ui/model.ts"],"names":[],"mappings":"AA+FA;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAChC,UAAyB,EACzB,KAAa;IAEb,MAAM,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAA0B,EAAE,CAAC;IAC3C,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,MAAM,aAAa,GAAG;YACpB,SAAS,CAAC,EAAE;YACZ,SAAS,CAAC,KAAK;YACf,SAAS,CAAC,WAAW;YACrB,SAAS,CAAC,MAAM;SACjB;aACE,IAAI,CAAC,GAAG,CAAC;aACT,WAAW,EAAE,CAAC;QACjB,MAAM,gBAAgB,GAAG,OAAO,CAAC,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAClC,CAAC,IAAI,EAAE,EAAE,CACP,CAAC,CAAC;YACF,gBAAgB;YAChB,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,WAAW,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CACrE,CAAC;QACF,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,gBAAgB;YAAE,SAAS;QAC3D,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
package/dist/registry.d.ts
CHANGED
|
@@ -4,9 +4,6 @@ import { type CredentialCheckOptions, type CredentialCheckResult, type Credentia
|
|
|
4
4
|
import { type CallAdmissionPermit, type ConnectorCallAdmissionSnapshot } from "./call-admission.js";
|
|
5
5
|
import type { DeferredWork } from "./connector-scope.js";
|
|
6
6
|
import { type Toolkit } from "./toolkits.js";
|
|
7
|
-
export declare const DEFAULT_MAX_RESULT_BYTES = 50000;
|
|
8
|
-
/** Independent final-envelope boundary for `batch_call`. */
|
|
9
|
-
export declare const DEFAULT_MAX_BATCH_RESULT_BYTES = 100000;
|
|
10
7
|
/**
|
|
11
8
|
* Smallest accepted inline-result cap. One byte is pathological but harmless:
|
|
12
9
|
* `alignEndToCharBoundary` widens a window narrower than the codepoint at the
|
|
@@ -81,7 +78,7 @@ export interface RegistryOptions {
|
|
|
81
78
|
/** Tuning for the credential liveness checks (issue #24). */
|
|
82
79
|
credentialHealth?: CredentialHealthConfig;
|
|
83
80
|
}
|
|
84
|
-
type ConnectorOperationOptions = Pick<ConnectorContext, "signal" | "timeoutMs">;
|
|
81
|
+
export type ConnectorOperationOptions = Pick<ConnectorContext, "signal" | "timeoutMs">;
|
|
85
82
|
/**
|
|
86
83
|
* The registry surface a per-connection MCP server consumes: every meta-tool
|
|
87
84
|
* (`src/meta-tools.ts`) and the `execute_code` sandbox bridge (`src/execute.ts`)
|
|
@@ -147,6 +144,8 @@ export declare class Registry implements RegistryView {
|
|
|
147
144
|
private readonly catalogGenerations;
|
|
148
145
|
/** Serialize persisted catalog set/delete operations within this isolate. */
|
|
149
146
|
private readonly catalogMutations;
|
|
147
|
+
/** Same-request cold loads share one promise without retaining the request. */
|
|
148
|
+
private readonly requestCatalogLoads;
|
|
150
149
|
/** Deployment-wide observations — every call, whatever view made it. */
|
|
151
150
|
private readonly health;
|
|
152
151
|
private readonly ttlMs;
|
|
@@ -197,8 +196,15 @@ export declare class Registry implements RegistryView {
|
|
|
197
196
|
toolName: string;
|
|
198
197
|
} | null;
|
|
199
198
|
private catalogKey;
|
|
200
|
-
private
|
|
199
|
+
private catalogChunkKey;
|
|
200
|
+
private validLegacyCatalog;
|
|
201
|
+
private validCatalogTools;
|
|
202
|
+
private validCatalogManifest;
|
|
203
|
+
private parseCatalogManifest;
|
|
204
|
+
private splitCatalogChunks;
|
|
205
|
+
private readCatalog;
|
|
201
206
|
private storeCatalog;
|
|
207
|
+
private deleteCatalog;
|
|
202
208
|
private catalogGeneration;
|
|
203
209
|
private advanceCatalogGeneration;
|
|
204
210
|
/**
|
|
@@ -210,6 +216,17 @@ export declare class Registry implements RegistryView {
|
|
|
210
216
|
/** Force a live listTools refresh and replace both catalog cache layers. */
|
|
211
217
|
refreshTools(id: string, baseUrl: string, requestScope?: object, callOptions?: ConnectorOperationOptions): Promise<ToolDef[]>;
|
|
212
218
|
/** Cached listTools with in-memory + persisted serializable catalog layers. */
|
|
219
|
+
private loadTools;
|
|
220
|
+
/**
|
|
221
|
+
* Coalesce one connector's cold load inside one inbound request. The WeakMap
|
|
222
|
+
* neither roots the request scope nor lets its connector context escape into
|
|
223
|
+
* another request; settled entries are also removed eagerly.
|
|
224
|
+
*
|
|
225
|
+
* The first caller's `callOptions` govern the shared load: a later caller's
|
|
226
|
+
* signal or timeout neither cancels nor extends it, and an abort by the
|
|
227
|
+
* first caller rejects every coalesced caller. Within one request that is
|
|
228
|
+
* the deal being made — one fetch, one deadline.
|
|
229
|
+
*/
|
|
213
230
|
getTools(id: string, baseUrl: string, requestScope?: object, callOptions?: ConnectorOperationOptions): Promise<ToolDef[]>;
|
|
214
231
|
/** Return a cached catalog without performing storage or network I/O. */
|
|
215
232
|
peekTools(id: string): ToolDef[] | undefined;
|
|
@@ -364,5 +381,4 @@ export declare class ScopedRegistry implements RegistryView {
|
|
|
364
381
|
}): Promise<ConnectorStatus>;
|
|
365
382
|
invalidateStored(id: string): Promise<void>;
|
|
366
383
|
}
|
|
367
|
-
export {};
|
|
368
384
|
//# sourceMappingURL=registry.d.ts.map
|
package/dist/registry.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,eAAe,EACf,SAAS,EACT,MAAM,EACN,OAAO,EACR,MAAM,YAAY,CAAC;AACpB,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAEL,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACpC,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"registry.d.ts","sourceRoot":"","sources":["../src/registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,eAAe,EACf,SAAS,EACT,MAAM,EACN,OAAO,EACR,MAAM,YAAY,CAAC;AACpB,OAAO,EAEL,KAAK,eAAe,EACrB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAEL,KAAK,sBAAsB,EAC3B,KAAK,qBAAqB,EAC1B,KAAK,sBAAsB,EAC3B,KAAK,sBAAsB,EAC5B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAEL,KAAK,mBAAmB,EACxB,KAAK,8BAA8B,EACpC,MAAM,qBAAqB,CAAC;AAY7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAgB,KAAK,OAAO,EAAE,MAAM,eAAe,CAAC;AAU3D;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,IAAI,CAAC;AAEtC;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAE5D;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,SAAS,EAAE,MAAM,GAChB,MAAM,CAIR;AAuCD,MAAM,WAAW,iBAAiB;IAChC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAwC;IAErE,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAYlD,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAWlE,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS;CAI/C;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,SAAS,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,6DAA6D;IAC7D,gBAAgB,CAAC,EAAE,sBAAsB,CAAC;CAC3C;AA0CD,MAAM,MAAM,yBAAyB,GAAG,IAAI,CAC1C,gBAAgB,EAChB,QAAQ,GAAG,WAAW,CACvB,CAAC;AAEF;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,YAAY;IAC3B,kEAAkE;IAClE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,uEAAuE;IACvE,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,cAAc,IAAI,SAAS,EAAE,CAAC;IAC9B,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS,CAAC;IAChD,cAAc,CACZ,OAAO,EAAE,MAAM,GACd;QAAE,SAAS,EAAE,SAAS,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IACrD,QAAQ,CACN,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,yBAAyB,GACtC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACtB,YAAY,CACV,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,yBAAyB,GACtC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;IACtB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,EAAE,GAAG,SAAS,CAAC;IAC7C,UAAU,CACR,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,yBAAyB,GACtC,gBAAgB,CAAC;IACpB;;;OAGG;IACH,SAAS,CACP,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAC/D,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChC,cAAc,IAAI,SAAS,CAAC;IAC5B,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACnD,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;IACnE,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS,CAAC;IACrD,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC;IACxC,yEAAyE;IACzE,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IAClD,4EAA4E;IAC5E,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC,CAAC;IAC7E,kEAAkE;IAClE,sBAAsB,CACpB,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,SAAS,CACP,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,yBAAyB,GACtC,OAAO,CAAC,eAAe,CAAC,CAAC;IAC5B,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7C;AAED;;;;GAIG;AACH,qBAAa,QAAS,YAAW,YAAY;IA+BzC,OAAO,CAAC,QAAQ,CAAC,IAAI;IA9BvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgC;IAC3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAG1B;IACJ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAiC;IACvD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqB;IACjD,iFAAiF;IACjF,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA6B;IAChE,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAoC;IACrE,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAGhC;IACJ,wEAAwE;IACxE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAC7C,wDAAwD;IACxD,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,2DAA2D;IAC3D,QAAQ,CAAC,mBAAmB,EAAE,MAAM,CAAC;IACrC,gFAAgF;IAChF,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA0B;gBAGzD,UAAU,EAAE,SAAS,EAAE,EACN,IAAI,EAAE,eAAe;IAsDxC;;;;;;;OAOG;IACH,OAAO,CAAC,eAAe;IA4CvB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAuBxB,cAAc,IAAI,SAAS,EAAE;IAI7B,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAI/C,UAAU,CACR,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,GAAE,MAAW,EACzB,WAAW,GAAE,yBAA8B,GAC1C,gBAAgB;IA6BnB,SAAS,CACP,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAC/D,OAAO,CAAC,mBAAmB,CAAC;IAM/B,iEAAiE;IACjE,qBAAqB,IAAI,MAAM,CAAC,MAAM,EAAE,8BAA8B,CAAC;IASvE,gFAAgF;IAChF,kBAAkB,IAAI,IAAI;IAI1B;;;OAGG;IACH,cAAc,IAAI,SAAS;IAI3B,kEAAkE;IAClE,cAAc,CACZ,OAAO,EAAE,MAAM,GACd;QAAE,SAAS,EAAE,SAAS,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAQpD,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,kBAAkB;IAiB1B,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,oBAAoB;IA8B5B,OAAO,CAAC,oBAAoB;IAW5B,OAAO,CAAC,kBAAkB;YA4BZ,WAAW;YA6HX,YAAY;YA4CZ,aAAa;IA8B3B,OAAO,CAAC,iBAAiB;IAIzB,OAAO,CAAC,wBAAwB;IAIhC;;;;OAIG;IACH,OAAO,CAAC,sBAAsB;IAiB9B,4EAA4E;IACtE,YAAY,CAChB,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,GAAE,yBAA8B,GAC1C,OAAO,CAAC,OAAO,EAAE,CAAC;IA2DrB,+EAA+E;YACjE,SAAS;IA+DvB;;;;;;;;;OASG;IACG,QAAQ,CACZ,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,GAAE,yBAA8B,GAC1C,OAAO,CAAC,OAAO,EAAE,CAAC;IAyBrB,yEAAyE;IACzE,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,EAAE,GAAG,SAAS;IAO5C,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAIlD,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAIlE,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS;IAIpD;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIvC;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIjD;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAI5E,sBAAsB,CACpB,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,IAAI,CAAC;IAIhB;;;;;;OAMG;IACH,qBAAqB,CACnB,OAAO,EAAE,MAAM,EACf,IAAI,CAAC,EAAE,sBAAsB,EAC7B,KAAK,CAAC,EAAE,YAAY,GACnB,OAAO,CAAC,qBAAqB,EAAE,CAAC;IAInC;;;;;OAKG;IACH,0BAA0B,CACxB,OAAO,EAAE,MAAM,EACf,KAAK,CAAC,EAAE,YAAY,GACnB,OAAO,CAAC,qBAAqB,EAAE,CAAC,GAAG,SAAS;IAI/C;;;;;;OAMG;IACH,qBAAqB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhD,wDAAwD;IAClD,SAAS,CACb,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,GAAE,MAAW,EACzB,WAAW,GAAE,yBAA8B,GAC1C,OAAO,CAAC,eAAe,CAAC;IAmB3B,uEAAuE;IACvE,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAiB5B,uDAAuD;IACjD,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAgBlD;AA4BD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,qBAAa,cAAe,YAAW,YAAY;IAK/C,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAL1B,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAY;gBAGhB,IAAI,EAAE,QAAQ,EACd,OAAO,EAAE,OAAO;IAKnC,IAAI,cAAc,IAAI,MAAM,CAE3B;IAED,IAAI,mBAAmB,IAAI,MAAM,CAEhC;IAED,wCAAwC;IACxC,OAAO,CAAC,OAAO;IAMf,6EAA6E;IAC7E,OAAO,CAAC,gBAAgB;IAIxB,OAAO,CAAC,YAAY;IAIpB,cAAc,IAAI,SAAS,EAAE;IAM7B,YAAY,CAAC,EAAE,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAI/C,cAAc,CACZ,OAAO,EAAE,MAAM,GACd;QAAE,SAAS,EAAE,SAAS,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI;IAO9C,QAAQ,CACZ,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,GAAE,yBAA8B,GAC1C,OAAO,CAAC,OAAO,EAAE,CAAC;IAQf,YAAY,CAChB,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,GAAE,yBAA8B,GAC1C,OAAO,CAAC,OAAO,EAAE,CAAC;IAQrB,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,EAAE,GAAG,SAAS;IAM5C,UAAU,CACR,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,GAAE,MAAW,EACzB,WAAW,GAAE,yBAA8B,GAC1C,gBAAgB;IAQnB,SAAS,CACP,EAAE,EAAE,MAAM,EACV,KAAK,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAE,GAC/D,OAAO,CAAC,mBAAmB,CAAC;IAK/B;;;;;OAKG;IACH,cAAc,IAAI,SAAS;IAW3B,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI;IAMlD,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAMlE,SAAS,CAAC,EAAE,EAAE,MAAM,GAAG,iBAAiB,GAAG,SAAS;IAIpD;;;;;;OAMG;IACH,kBAAkB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIvC,yEAAyE;IACzE,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAIjD;;;;;;;;OAQG;IACH,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,sBAAsB,GAAG,SAAS,CAAC;IAMtE,sBAAsB,CAC1B,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,sBAAsB,GAC7B,OAAO,CAAC,IAAI,CAAC;IAKV,SAAS,CACb,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,GAAE,MAAW,EACzB,WAAW,GAAE;QAAE,MAAM,CAAC,EAAE,WAAW,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO,GAC7D,OAAO,CAAC,eAAe,CAAC;IAQrB,gBAAgB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAIlD"}
|
package/dist/registry.js
CHANGED
|
@@ -2,13 +2,17 @@ import { storedCredentialShape, } from "./credentials.js";
|
|
|
2
2
|
import { ConnectorCallError } from "./errors.js";
|
|
3
3
|
import { CredentialHealthChecker, } from "./credential-health.js";
|
|
4
4
|
import { ConnectorCallAdmissionController, } from "./call-admission.js";
|
|
5
|
+
import { fingerprintSerializedCatalog, snapshotCatalog, } from "./catalog-fingerprint.js";
|
|
6
|
+
import { MAX_CATALOG_CHUNK_BYTES, MAX_CATALOG_TOOLS, MAX_SERIALIZED_CATALOG_BYTES, } from "./catalog-limits.js";
|
|
7
|
+
import { mapSettledWithConcurrency } from "./concurrency.js";
|
|
5
8
|
import { splitAddress } from "./toolkits.js";
|
|
6
9
|
const ID_RE = /^[a-z0-9_-]+$/;
|
|
7
10
|
const DEFAULT_TTL_SECONDS = 300;
|
|
8
11
|
const DEFAULT_STALE_SECONDS = 3600;
|
|
9
|
-
|
|
12
|
+
const CATALOG_CHUNK_TTL_GRACE_SECONDS = 300;
|
|
13
|
+
const DEFAULT_MAX_RESULT_BYTES = 50_000;
|
|
10
14
|
/** Independent final-envelope boundary for `batch_call`. */
|
|
11
|
-
|
|
15
|
+
const DEFAULT_MAX_BATCH_RESULT_BYTES = 100_000;
|
|
12
16
|
/**
|
|
13
17
|
* Smallest accepted inline-result cap. One byte is pathological but harmless:
|
|
14
18
|
* `alignEndToCharBoundary` widens a window narrower than the codepoint at the
|
|
@@ -44,6 +48,7 @@ export function resolveMaxResultBytes(value, inherited) {
|
|
|
44
48
|
? value
|
|
45
49
|
: inherited;
|
|
46
50
|
}
|
|
51
|
+
const CATALOG_CHUNK_IO_CONCURRENCY = 4;
|
|
47
52
|
/**
|
|
48
53
|
* Recent real-call outcomes per connector, as observed by ONE view.
|
|
49
54
|
*
|
|
@@ -59,13 +64,14 @@ export class HealthLog {
|
|
|
59
64
|
observations = new Map();
|
|
60
65
|
recordSuccess(id, latencyMs) {
|
|
61
66
|
const previous = this.observations.get(id);
|
|
62
|
-
|
|
67
|
+
const observation = {
|
|
63
68
|
...previous,
|
|
64
69
|
lastSuccessAt: new Date().toISOString(),
|
|
65
70
|
lastLatencyMs: latencyMs,
|
|
66
71
|
consecutiveFailures: 0,
|
|
67
|
-
|
|
68
|
-
|
|
72
|
+
};
|
|
73
|
+
delete observation.lastError;
|
|
74
|
+
this.observations.set(id, observation);
|
|
69
75
|
}
|
|
70
76
|
recordFailure(id, latencyMs, error) {
|
|
71
77
|
const previous = this.observations.get(id);
|
|
@@ -92,6 +98,35 @@ function namespaced(storage, prefix) {
|
|
|
92
98
|
function msg(err) {
|
|
93
99
|
return err instanceof Error ? err.message : String(err);
|
|
94
100
|
}
|
|
101
|
+
/** WHATWG TextEncoder byte length without allocating another full buffer. */
|
|
102
|
+
function utf8ByteLength(value) {
|
|
103
|
+
let bytes = 0;
|
|
104
|
+
for (let index = 0; index < value.length; index++) {
|
|
105
|
+
const code = value.charCodeAt(index);
|
|
106
|
+
if (code <= 0x7f) {
|
|
107
|
+
bytes++;
|
|
108
|
+
}
|
|
109
|
+
else if (code <= 0x7ff) {
|
|
110
|
+
bytes += 2;
|
|
111
|
+
}
|
|
112
|
+
else if (code >= 0xd800 &&
|
|
113
|
+
code <= 0xdbff &&
|
|
114
|
+
index + 1 < value.length) {
|
|
115
|
+
const next = value.charCodeAt(index + 1);
|
|
116
|
+
if (next >= 0xdc00 && next <= 0xdfff) {
|
|
117
|
+
bytes += 4;
|
|
118
|
+
index++;
|
|
119
|
+
}
|
|
120
|
+
else {
|
|
121
|
+
bytes += 3;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
else {
|
|
125
|
+
bytes += 3;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return bytes;
|
|
129
|
+
}
|
|
95
130
|
/**
|
|
96
131
|
* Holds the connector set, resolves addresses, and caches per-connector tool
|
|
97
132
|
* lists in memory with a TTL. Connector failures are isolated: a broken
|
|
@@ -107,6 +142,8 @@ export class Registry {
|
|
|
107
142
|
catalogGenerations = new Map();
|
|
108
143
|
/** Serialize persisted catalog set/delete operations within this isolate. */
|
|
109
144
|
catalogMutations = new Map();
|
|
145
|
+
/** Same-request cold loads share one promise without retaining the request. */
|
|
146
|
+
requestCatalogLoads = new WeakMap();
|
|
110
147
|
/** Deployment-wide observations — every call, whatever view made it. */
|
|
111
148
|
health = new HealthLog();
|
|
112
149
|
ttlMs;
|
|
@@ -147,7 +184,9 @@ export class Registry {
|
|
|
147
184
|
contextFor: (id, baseUrl, requestScope) => this.contextFor(id, baseUrl, requestScope),
|
|
148
185
|
storage: opts.storage,
|
|
149
186
|
logger: opts.logger,
|
|
150
|
-
|
|
187
|
+
...(opts.credentialVault !== undefined
|
|
188
|
+
? { credentialVault: opts.credentialVault }
|
|
189
|
+
: {}),
|
|
151
190
|
}, opts.credentialHealth);
|
|
152
191
|
}
|
|
153
192
|
/**
|
|
@@ -279,40 +318,236 @@ export class Registry {
|
|
|
279
318
|
catalogKey(id) {
|
|
280
319
|
return `catalog:${id}`;
|
|
281
320
|
}
|
|
282
|
-
|
|
321
|
+
catalogChunkKey(id, revision, index) {
|
|
322
|
+
return `${this.catalogKey(id)}:chunk:${revision}:${index}`;
|
|
323
|
+
}
|
|
324
|
+
validLegacyCatalog(value) {
|
|
325
|
+
if (!value || typeof value !== "object")
|
|
326
|
+
return null;
|
|
327
|
+
const catalog = value;
|
|
328
|
+
if (!Array.isArray(catalog.tools) ||
|
|
329
|
+
typeof catalog.fetchedAt !== "number" ||
|
|
330
|
+
typeof catalog.expiresAt !== "number" ||
|
|
331
|
+
typeof catalog.staleUntil !== "number" ||
|
|
332
|
+
(catalog.fingerprint !== undefined &&
|
|
333
|
+
typeof catalog.fingerprint !== "string") ||
|
|
334
|
+
!this.validCatalogTools(catalog.tools)) {
|
|
335
|
+
return null;
|
|
336
|
+
}
|
|
337
|
+
return catalog;
|
|
338
|
+
}
|
|
339
|
+
validCatalogTools(value) {
|
|
340
|
+
return value.every((tool) => tool !== null &&
|
|
341
|
+
typeof tool === "object" &&
|
|
342
|
+
typeof tool.name === "string");
|
|
343
|
+
}
|
|
344
|
+
validCatalogManifest(value) {
|
|
345
|
+
if (!value || typeof value !== "object")
|
|
346
|
+
return null;
|
|
347
|
+
const manifest = value;
|
|
348
|
+
const maxChunks = Math.ceil(MAX_SERIALIZED_CATALOG_BYTES / MAX_CATALOG_CHUNK_BYTES) + 1;
|
|
349
|
+
if (manifest.version !== 2 ||
|
|
350
|
+
typeof manifest.revision !== "string" ||
|
|
351
|
+
!/^sha256:[0-9]{1,8}:[0-9a-f]{64}$/.test(manifest.revision) ||
|
|
352
|
+
!Number.isInteger(manifest.toolCount) ||
|
|
353
|
+
manifest.toolCount < 0 ||
|
|
354
|
+
manifest.toolCount > MAX_CATALOG_TOOLS ||
|
|
355
|
+
!Number.isInteger(manifest.byteCount) ||
|
|
356
|
+
manifest.byteCount < 2 ||
|
|
357
|
+
manifest.byteCount > MAX_SERIALIZED_CATALOG_BYTES ||
|
|
358
|
+
!manifest.revision.startsWith(`sha256:${manifest.byteCount}:`) ||
|
|
359
|
+
!Number.isInteger(manifest.chunkCount) ||
|
|
360
|
+
manifest.chunkCount < 1 ||
|
|
361
|
+
manifest.chunkCount > maxChunks ||
|
|
362
|
+
typeof manifest.fetchedAt !== "number" ||
|
|
363
|
+
typeof manifest.expiresAt !== "number" ||
|
|
364
|
+
typeof manifest.staleUntil !== "number") {
|
|
365
|
+
return null;
|
|
366
|
+
}
|
|
367
|
+
return manifest;
|
|
368
|
+
}
|
|
369
|
+
parseCatalogManifest(raw) {
|
|
370
|
+
if (!raw)
|
|
371
|
+
return null;
|
|
372
|
+
try {
|
|
373
|
+
return this.validCatalogManifest(JSON.parse(raw));
|
|
374
|
+
}
|
|
375
|
+
catch {
|
|
376
|
+
return null;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
splitCatalogChunks(snapshot) {
|
|
380
|
+
const chunks = [];
|
|
381
|
+
const decoder = new TextDecoder("utf-8", {
|
|
382
|
+
fatal: true,
|
|
383
|
+
ignoreBOM: true,
|
|
384
|
+
});
|
|
385
|
+
let offset = 0;
|
|
386
|
+
while (offset < snapshot.serializedBytes.byteLength) {
|
|
387
|
+
let end = Math.min(offset + MAX_CATALOG_CHUNK_BYTES, snapshot.serializedBytes.byteLength);
|
|
388
|
+
// Move a boundary that landed inside a multibyte UTF-8 sequence back to
|
|
389
|
+
// the next character start. Every stored string then remains valid UTF-8.
|
|
390
|
+
while (end < snapshot.serializedBytes.byteLength &&
|
|
391
|
+
((snapshot.serializedBytes[end] ?? 0) & 0xc0) === 0x80) {
|
|
392
|
+
end--;
|
|
393
|
+
}
|
|
394
|
+
chunks.push(decoder.decode(snapshot.serializedBytes.subarray(offset, end)));
|
|
395
|
+
offset = end;
|
|
396
|
+
}
|
|
397
|
+
return chunks;
|
|
398
|
+
}
|
|
399
|
+
async readCatalog(id, raw, now) {
|
|
283
400
|
if (!raw)
|
|
284
401
|
return null;
|
|
402
|
+
let parsed;
|
|
285
403
|
try {
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
404
|
+
parsed = JSON.parse(raw);
|
|
405
|
+
}
|
|
406
|
+
catch {
|
|
407
|
+
return null;
|
|
408
|
+
}
|
|
409
|
+
if (!parsed ||
|
|
410
|
+
typeof parsed !== "object" ||
|
|
411
|
+
parsed.version !== 2) {
|
|
412
|
+
const legacy = this.validLegacyCatalog(parsed);
|
|
413
|
+
if (!legacy || legacy.staleUntil <= now)
|
|
414
|
+
return null;
|
|
415
|
+
const snapshot = await snapshotCatalog(legacy.tools);
|
|
416
|
+
if (legacy.tools.length > MAX_CATALOG_TOOLS ||
|
|
417
|
+
snapshot.serializedBytes.byteLength > MAX_SERIALIZED_CATALOG_BYTES ||
|
|
418
|
+
(legacy.fingerprint !== undefined &&
|
|
419
|
+
legacy.fingerprint !== snapshot.fingerprint)) {
|
|
420
|
+
this.opts.logger.warn(`[connecta] connector "${id}" legacy catalog is oversized or has a fingerprint mismatch; ignoring persisted catalog.`);
|
|
421
|
+
return null;
|
|
422
|
+
}
|
|
423
|
+
return {
|
|
424
|
+
tools: legacy.tools,
|
|
425
|
+
fingerprint: snapshot.fingerprint,
|
|
426
|
+
fetchedAt: legacy.fetchedAt,
|
|
427
|
+
expiresAt: legacy.expiresAt,
|
|
428
|
+
staleUntil: legacy.staleUntil,
|
|
429
|
+
};
|
|
430
|
+
}
|
|
431
|
+
const manifest = this.validCatalogManifest(parsed);
|
|
432
|
+
if (!manifest) {
|
|
433
|
+
this.opts.logger.warn(`[connecta] connector "${id}" catalog manifest is invalid; ignoring persisted catalog.`);
|
|
434
|
+
return null;
|
|
435
|
+
}
|
|
436
|
+
if (manifest.staleUntil <= now)
|
|
437
|
+
return null;
|
|
438
|
+
const chunkReads = await mapSettledWithConcurrency(Array.from({ length: manifest.chunkCount }, (_, index) => index), CATALOG_CHUNK_IO_CONCURRENCY, (index) => this.opts.storage.get(this.catalogChunkKey(id, manifest.revision, index)));
|
|
439
|
+
const chunks = [];
|
|
440
|
+
let chunkBytes = 0;
|
|
441
|
+
for (const [index, result] of chunkReads.entries()) {
|
|
442
|
+
if (result.status === "rejected")
|
|
443
|
+
throw result.reason;
|
|
444
|
+
const chunk = result.value;
|
|
445
|
+
if (chunk === null) {
|
|
446
|
+
this.opts.logger.warn(`[connecta] connector "${id}" catalog chunk ${index + 1}/${manifest.chunkCount} is missing; ignoring persisted catalog.`);
|
|
294
447
|
return null;
|
|
295
448
|
}
|
|
296
|
-
|
|
449
|
+
const byteLength = utf8ByteLength(chunk);
|
|
450
|
+
chunkBytes += byteLength;
|
|
451
|
+
if (byteLength > MAX_CATALOG_CHUNK_BYTES ||
|
|
452
|
+
chunkBytes > manifest.byteCount) {
|
|
453
|
+
this.opts.logger.warn(`[connecta] connector "${id}" catalog chunk bounds do not match its manifest; ignoring persisted catalog.`);
|
|
454
|
+
return null;
|
|
455
|
+
}
|
|
456
|
+
chunks.push(chunk);
|
|
457
|
+
}
|
|
458
|
+
const serializedTools = chunks.join("");
|
|
459
|
+
const stored = await fingerprintSerializedCatalog(serializedTools);
|
|
460
|
+
if (stored.byteLength !== manifest.byteCount ||
|
|
461
|
+
stored.fingerprint !== manifest.revision) {
|
|
462
|
+
this.opts.logger.warn(`[connecta] connector "${id}" catalog fingerprint mismatch; ignoring persisted catalog.`);
|
|
463
|
+
return null;
|
|
464
|
+
}
|
|
465
|
+
let tools;
|
|
466
|
+
try {
|
|
467
|
+
tools = JSON.parse(serializedTools);
|
|
297
468
|
}
|
|
298
469
|
catch {
|
|
470
|
+
this.opts.logger.warn(`[connecta] connector "${id}" catalog chunks are torn; ignoring persisted catalog.`);
|
|
471
|
+
return null;
|
|
472
|
+
}
|
|
473
|
+
if (!Array.isArray(tools) || !this.validCatalogTools(tools)) {
|
|
474
|
+
this.opts.logger.warn(`[connecta] connector "${id}" catalog chunks contain invalid tools; ignoring persisted catalog.`);
|
|
475
|
+
return null;
|
|
476
|
+
}
|
|
477
|
+
if (tools.length !== manifest.toolCount) {
|
|
478
|
+
this.opts.logger.warn(`[connecta] connector "${id}" catalog tool count does not match its manifest; ignoring persisted catalog.`);
|
|
299
479
|
return null;
|
|
300
480
|
}
|
|
481
|
+
return {
|
|
482
|
+
tools,
|
|
483
|
+
fingerprint: stored.fingerprint,
|
|
484
|
+
fetchedAt: manifest.fetchedAt,
|
|
485
|
+
expiresAt: manifest.expiresAt,
|
|
486
|
+
staleUntil: manifest.staleUntil,
|
|
487
|
+
};
|
|
301
488
|
}
|
|
302
|
-
async storeCatalog(id,
|
|
489
|
+
async storeCatalog(id, snapshot) {
|
|
303
490
|
if (!this.persistToolCatalog)
|
|
304
491
|
return;
|
|
305
492
|
const fetchedAt = Date.now();
|
|
306
|
-
const
|
|
307
|
-
|
|
493
|
+
const expiresAt = fetchedAt + this.ttlMs;
|
|
494
|
+
const staleUntil = expiresAt + this.staleMs;
|
|
495
|
+
const ttlSeconds = Math.max(60, Math.ceil((this.ttlMs + this.staleMs) / 1000));
|
|
496
|
+
const chunks = this.splitCatalogChunks(snapshot);
|
|
497
|
+
const chunkWrites = await mapSettledWithConcurrency(chunks, CATALOG_CHUNK_IO_CONCURRENCY, (chunk, index) => this.opts.storage.set(this.catalogChunkKey(id, snapshot.fingerprint, index), chunk, { ttlSeconds: ttlSeconds + CATALOG_CHUNK_TTL_GRACE_SECONDS }));
|
|
498
|
+
for (const result of chunkWrites) {
|
|
499
|
+
if (result.status === "rejected")
|
|
500
|
+
throw result.reason;
|
|
501
|
+
}
|
|
502
|
+
// The manifest is the only publication point. A failed/partial chunk write
|
|
503
|
+
// therefore leaves the previous manifest authoritative (or no catalog);
|
|
504
|
+
// unreachable chunks carry a bounded TTL and require no prefix scan.
|
|
505
|
+
const manifest = {
|
|
506
|
+
version: 2,
|
|
507
|
+
revision: snapshot.fingerprint,
|
|
508
|
+
toolCount: snapshot.tools.length,
|
|
509
|
+
byteCount: snapshot.serializedBytes.byteLength,
|
|
510
|
+
chunkCount: chunks.length,
|
|
308
511
|
fetchedAt,
|
|
309
|
-
expiresAt
|
|
310
|
-
staleUntil
|
|
512
|
+
expiresAt,
|
|
513
|
+
staleUntil,
|
|
311
514
|
};
|
|
312
|
-
await this.opts.storage.set(this.catalogKey(id), JSON.stringify(
|
|
313
|
-
ttlSeconds
|
|
515
|
+
await this.opts.storage.set(this.catalogKey(id), JSON.stringify(manifest), {
|
|
516
|
+
ttlSeconds,
|
|
314
517
|
});
|
|
315
518
|
}
|
|
519
|
+
async deleteCatalog(id) {
|
|
520
|
+
let raw = null;
|
|
521
|
+
let readError;
|
|
522
|
+
try {
|
|
523
|
+
raw = await this.opts.storage.get(this.catalogKey(id));
|
|
524
|
+
}
|
|
525
|
+
catch (err) {
|
|
526
|
+
readError = err;
|
|
527
|
+
}
|
|
528
|
+
const manifest = this.parseCatalogManifest(raw);
|
|
529
|
+
// The root is authoritative, so attempt its deletion even when the
|
|
530
|
+
// best-effort read needed for physical chunk cleanup failed.
|
|
531
|
+
await this.opts.storage.delete(this.catalogKey(id));
|
|
532
|
+
if (!manifest) {
|
|
533
|
+
if (readError)
|
|
534
|
+
throw readError;
|
|
535
|
+
return;
|
|
536
|
+
}
|
|
537
|
+
let firstError;
|
|
538
|
+
for (let index = 0; index < manifest.chunkCount; index++) {
|
|
539
|
+
try {
|
|
540
|
+
await this.opts.storage.delete(this.catalogChunkKey(id, manifest.revision, index));
|
|
541
|
+
}
|
|
542
|
+
catch (err) {
|
|
543
|
+
firstError ??= err;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
if (readError)
|
|
547
|
+
throw readError;
|
|
548
|
+
if (firstError)
|
|
549
|
+
throw firstError;
|
|
550
|
+
}
|
|
316
551
|
catalogGeneration(id) {
|
|
317
552
|
return this.catalogGenerations.get(id) ?? 0;
|
|
318
553
|
}
|
|
@@ -342,22 +577,40 @@ export class Registry {
|
|
|
342
577
|
const connector = this.connectors.get(id);
|
|
343
578
|
if (!connector)
|
|
344
579
|
throw new Error(`Unknown connector "${id}"`);
|
|
580
|
+
if (connector.staticTools)
|
|
581
|
+
return connector.staticTools;
|
|
345
582
|
const generation = this.catalogGeneration(id);
|
|
346
|
-
const tools = connector.
|
|
347
|
-
? connector.staticTools
|
|
348
|
-
: await connector.listTools(this.contextFor(id, baseUrl, requestScope, callOptions));
|
|
583
|
+
const tools = await connector.listTools(this.contextFor(id, baseUrl, requestScope, callOptions));
|
|
349
584
|
// The caller that began this refresh may still use its result, but a
|
|
350
585
|
// credential/OAuth change that landed while listTools was in flight means
|
|
351
586
|
// the listing must not enter either shared cache layer.
|
|
352
587
|
if (generation !== this.catalogGeneration(id))
|
|
353
588
|
return tools;
|
|
354
|
-
|
|
589
|
+
if (tools.length > MAX_CATALOG_TOOLS) {
|
|
590
|
+
const message = `Connector "${id}" returned ${tools.length} tools, over the ` +
|
|
591
|
+
`${MAX_CATALOG_TOOLS}-tool catalog ceiling; refusing the complete catalog.`;
|
|
592
|
+
this.opts.logger.warn(`[connecta] ${message}`);
|
|
593
|
+
throw new Error(message);
|
|
594
|
+
}
|
|
355
595
|
const previous = this.cache.get(id);
|
|
356
|
-
const
|
|
357
|
-
|
|
358
|
-
|
|
596
|
+
const snapshot = await snapshotCatalog(tools);
|
|
597
|
+
if (generation !== this.catalogGeneration(id))
|
|
598
|
+
return tools;
|
|
599
|
+
if (snapshot.serializedBytes.byteLength > MAX_SERIALIZED_CATALOG_BYTES) {
|
|
600
|
+
const message = `Connector "${id}" returned a ${snapshot.serializedBytes.byteLength}-byte ` +
|
|
601
|
+
`serialized catalog, over the ${MAX_SERIALIZED_CATALOG_BYTES}-byte ceiling; ` +
|
|
602
|
+
"refusing the complete catalog.";
|
|
603
|
+
this.opts.logger.warn(`[connecta] ${message}`);
|
|
604
|
+
throw new Error(message);
|
|
605
|
+
}
|
|
606
|
+
const now = Date.now();
|
|
607
|
+
const catalogChanged = !previous || previous.fingerprint !== snapshot.fingerprint;
|
|
608
|
+
const shouldPersist = catalogChanged ||
|
|
609
|
+
(previous !== undefined && previous.exp <= now) ||
|
|
610
|
+
this.invalidated.has(id);
|
|
359
611
|
this.cache.set(id, {
|
|
360
612
|
tools,
|
|
613
|
+
fingerprint: snapshot.fingerprint,
|
|
361
614
|
exp: now + this.ttlMs,
|
|
362
615
|
staleUntil: now + this.ttlMs + this.staleMs,
|
|
363
616
|
});
|
|
@@ -367,7 +620,7 @@ export class Registry {
|
|
|
367
620
|
if (generation !== this.catalogGeneration(id))
|
|
368
621
|
return;
|
|
369
622
|
try {
|
|
370
|
-
await this.storeCatalog(id,
|
|
623
|
+
await this.storeCatalog(id, snapshot);
|
|
371
624
|
}
|
|
372
625
|
catch (err) {
|
|
373
626
|
this.opts.logger.warn(`[connecta] connector "${id}" catalog persistence failed: ${msg(err)}`);
|
|
@@ -377,7 +630,7 @@ export class Registry {
|
|
|
377
630
|
return tools;
|
|
378
631
|
}
|
|
379
632
|
/** Cached listTools with in-memory + persisted serializable catalog layers. */
|
|
380
|
-
async
|
|
633
|
+
async loadTools(id, baseUrl, requestScope, callOptions = {}) {
|
|
381
634
|
const connector = this.connectors.get(id);
|
|
382
635
|
if (!connector)
|
|
383
636
|
throw new Error(`Unknown connector "${id}"`);
|
|
@@ -393,7 +646,7 @@ export class Registry {
|
|
|
393
646
|
const generation = this.catalogGeneration(id);
|
|
394
647
|
let persisted = null;
|
|
395
648
|
try {
|
|
396
|
-
persisted = this.
|
|
649
|
+
persisted = await this.readCatalog(id, await this.opts.storage.get(this.catalogKey(id)), now);
|
|
397
650
|
}
|
|
398
651
|
catch (err) {
|
|
399
652
|
this.opts.logger.warn(`[connecta] connector "${id}" catalog read failed: ${msg(err)}`);
|
|
@@ -405,6 +658,7 @@ export class Registry {
|
|
|
405
658
|
if (persisted && persisted.staleUntil > now) {
|
|
406
659
|
this.cache.set(id, {
|
|
407
660
|
tools: persisted.tools,
|
|
661
|
+
fingerprint: persisted.fingerprint,
|
|
408
662
|
exp: persisted.expiresAt,
|
|
409
663
|
staleUntil: persisted.staleUntil,
|
|
410
664
|
});
|
|
@@ -426,6 +680,45 @@ export class Registry {
|
|
|
426
680
|
throw err;
|
|
427
681
|
}
|
|
428
682
|
}
|
|
683
|
+
/**
|
|
684
|
+
* Coalesce one connector's cold load inside one inbound request. The WeakMap
|
|
685
|
+
* neither roots the request scope nor lets its connector context escape into
|
|
686
|
+
* another request; settled entries are also removed eagerly.
|
|
687
|
+
*
|
|
688
|
+
* The first caller's `callOptions` govern the shared load: a later caller's
|
|
689
|
+
* signal or timeout neither cancels nor extends it, and an abort by the
|
|
690
|
+
* first caller rejects every coalesced caller. Within one request that is
|
|
691
|
+
* the deal being made — one fetch, one deadline.
|
|
692
|
+
*/
|
|
693
|
+
async getTools(id, baseUrl, requestScope, callOptions = {}) {
|
|
694
|
+
const connector = this.connectors.get(id);
|
|
695
|
+
if (!connector)
|
|
696
|
+
throw new Error(`Unknown connector "${id}"`);
|
|
697
|
+
if (connector.staticTools)
|
|
698
|
+
return connector.staticTools;
|
|
699
|
+
if (!requestScope) {
|
|
700
|
+
return this.loadTools(id, baseUrl, requestScope, callOptions);
|
|
701
|
+
}
|
|
702
|
+
let loads = this.requestCatalogLoads.get(requestScope);
|
|
703
|
+
if (!loads) {
|
|
704
|
+
loads = new Map();
|
|
705
|
+
this.requestCatalogLoads.set(requestScope, loads);
|
|
706
|
+
}
|
|
707
|
+
const existing = loads.get(id);
|
|
708
|
+
if (existing)
|
|
709
|
+
return existing;
|
|
710
|
+
const loading = this.loadTools(id, baseUrl, requestScope, callOptions);
|
|
711
|
+
loads.set(id, loading);
|
|
712
|
+
try {
|
|
713
|
+
return await loading;
|
|
714
|
+
}
|
|
715
|
+
finally {
|
|
716
|
+
if (loads.get(id) === loading)
|
|
717
|
+
loads.delete(id);
|
|
718
|
+
if (loads.size === 0)
|
|
719
|
+
this.requestCatalogLoads.delete(requestScope);
|
|
720
|
+
}
|
|
721
|
+
}
|
|
429
722
|
/** Return a cached catalog without performing storage or network I/O. */
|
|
430
723
|
peekTools(id) {
|
|
431
724
|
const connector = this.connectors.get(id);
|
|
@@ -535,7 +828,7 @@ export class Registry {
|
|
|
535
828
|
if (this.persistToolCatalog) {
|
|
536
829
|
void this.enqueueCatalogMutation(id, async () => {
|
|
537
830
|
try {
|
|
538
|
-
await this.
|
|
831
|
+
await this.deleteCatalog(id);
|
|
539
832
|
}
|
|
540
833
|
catch (err) {
|
|
541
834
|
this.opts.logger.warn(`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`);
|
|
@@ -551,7 +844,7 @@ export class Registry {
|
|
|
551
844
|
if (this.persistToolCatalog) {
|
|
552
845
|
await this.enqueueCatalogMutation(id, async () => {
|
|
553
846
|
try {
|
|
554
|
-
await this.
|
|
847
|
+
await this.deleteCatalog(id);
|
|
555
848
|
}
|
|
556
849
|
catch (err) {
|
|
557
850
|
this.opts.logger.warn(`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`);
|