@zackbart/connecta 0.7.9 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +135 -0
- package/README.md +47 -148
- package/dist/activity.d.ts +0 -8
- package/dist/activity.d.ts.map +1 -1
- package/dist/activity.js +0 -1
- package/dist/activity.js.map +1 -1
- package/dist/auth/bearer.d.ts +1 -7
- package/dist/auth/bearer.d.ts.map +1 -1
- package/dist/auth/bearer.js +3 -24
- package/dist/auth/bearer.js.map +1 -1
- package/dist/auth/clerk.d.ts +2 -11
- package/dist/auth/clerk.d.ts.map +1 -1
- package/dist/auth/clerk.js +31 -46
- package/dist/auth/clerk.js.map +1 -1
- package/dist/call-admission.d.ts.map +1 -1
- package/dist/call-admission.js +9 -4
- package/dist/call-admission.js.map +1 -1
- package/dist/catalog-fingerprint.d.ts +20 -0
- package/dist/catalog-fingerprint.d.ts.map +1 -0
- package/dist/catalog-fingerprint.js +34 -0
- package/dist/catalog-fingerprint.js.map +1 -0
- package/dist/catalog-limits.d.ts +10 -0
- package/dist/catalog-limits.d.ts.map +1 -0
- package/dist/catalog-limits.js +10 -0
- package/dist/catalog-limits.js.map +1 -0
- package/dist/catalog-service.d.ts +137 -0
- package/dist/catalog-service.d.ts.map +1 -0
- package/dist/catalog-service.js +401 -0
- package/dist/catalog-service.js.map +1 -0
- package/dist/catalog.d.ts +6 -0
- package/dist/catalog.d.ts.map +1 -1
- package/dist/catalog.js +53 -2
- package/dist/catalog.js.map +1 -1
- package/dist/concurrency.d.ts +4 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/concurrency.js +31 -0
- package/dist/concurrency.js.map +1 -0
- package/dist/connectors/api.d.ts.map +1 -1
- package/dist/connectors/api.js +22 -12
- package/dist/connectors/api.js.map +1 -1
- package/dist/connectors/remote-mcp.d.ts.map +1 -1
- package/dist/connectors/remote-mcp.js +53 -41
- package/dist/connectors/remote-mcp.js.map +1 -1
- package/dist/credentials.d.ts +3 -2
- package/dist/credentials.d.ts.map +1 -1
- package/dist/errors.d.ts +19 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/execute.d.ts +10 -16
- package/dist/execute.d.ts.map +1 -1
- package/dist/execute.js +143 -330
- package/dist/execute.js.map +1 -1
- package/dist/executor-result.d.ts +0 -3
- package/dist/executor-result.d.ts.map +1 -1
- package/dist/executor-result.js +3 -3
- package/dist/executor-result.js.map +1 -1
- package/dist/executors/quickjs-child.js +3 -3
- package/dist/executors/quickjs-child.js.map +1 -1
- package/dist/executors/quickjs-protocol.d.ts +4 -1
- package/dist/executors/quickjs-protocol.d.ts.map +1 -1
- package/dist/executors/quickjs-runtime.d.ts.map +1 -1
- package/dist/executors/quickjs-runtime.js +7 -2
- package/dist/executors/quickjs-runtime.js.map +1 -1
- package/dist/executors/quickjs.d.ts.map +1 -1
- package/dist/executors/quickjs.js +4 -1
- package/dist/executors/quickjs.js.map +1 -1
- package/dist/favicon.js +2 -2
- package/dist/favicon.js.map +1 -1
- package/dist/index.d.ts +8 -82
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +58 -104
- package/dist/index.js.map +1 -1
- package/dist/invocation.d.ts +101 -0
- package/dist/invocation.d.ts.map +1 -0
- package/dist/invocation.js +405 -0
- package/dist/invocation.js.map +1 -0
- package/dist/meta-tools.d.ts +9 -95
- package/dist/meta-tools.d.ts.map +1 -1
- package/dist/meta-tools.js +235 -725
- package/dist/meta-tools.js.map +1 -1
- package/dist/operator-ui/generated.d.ts +3 -0
- package/dist/operator-ui/generated.d.ts.map +1 -0
- package/dist/operator-ui/generated.js +5 -0
- package/dist/operator-ui/generated.js.map +1 -0
- package/dist/operator-ui/model.d.ts +73 -0
- package/dist/operator-ui/model.d.ts.map +1 -0
- package/dist/operator-ui/model.js +28 -0
- package/dist/operator-ui/model.js.map +1 -0
- package/dist/registry.d.ts +33 -184
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +345 -330
- package/dist/registry.js.map +1 -1
- package/dist/retired-toolkits.d.ts +3 -0
- package/dist/retired-toolkits.d.ts.map +1 -0
- package/dist/retired-toolkits.js +9 -0
- package/dist/retired-toolkits.js.map +1 -0
- package/dist/routes/activity.d.ts +3 -0
- package/dist/routes/activity.d.ts.map +1 -0
- package/dist/routes/activity.js +170 -0
- package/dist/routes/activity.js.map +1 -0
- package/dist/routes/credentials.d.ts +3 -0
- package/dist/routes/credentials.d.ts.map +1 -0
- package/dist/routes/credentials.js +170 -0
- package/dist/routes/credentials.js.map +1 -0
- package/dist/routes/mcp.d.ts +8 -0
- package/dist/routes/mcp.d.ts.map +1 -0
- package/dist/routes/mcp.js +270 -0
- package/dist/routes/mcp.js.map +1 -0
- package/dist/routes/oauth.d.ts +4 -0
- package/dist/routes/oauth.d.ts.map +1 -0
- package/dist/routes/oauth.js +263 -0
- package/dist/routes/oauth.js.map +1 -0
- package/dist/routes/shared.d.ts +76 -0
- package/dist/routes/shared.d.ts.map +1 -0
- package/dist/routes/shared.js +140 -0
- package/dist/routes/shared.js.map +1 -0
- package/dist/routes/ui.d.ts +3 -0
- package/dist/routes/ui.d.ts.map +1 -0
- package/dist/routes/ui.js +116 -0
- package/dist/routes/ui.js.map +1 -0
- package/dist/server.d.ts +9 -40
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +62 -1322
- package/dist/server.js.map +1 -1
- package/dist/skills.d.ts +3 -21
- package/dist/skills.d.ts.map +1 -1
- package/dist/skills.js +15 -43
- package/dist/skills.js.map +1 -1
- package/dist/storage/file.d.ts.map +1 -1
- package/dist/storage/file.js +3 -1
- package/dist/storage/file.js.map +1 -1
- package/dist/storage/memory.d.ts.map +1 -1
- package/dist/storage/memory.js +3 -1
- package/dist/storage/memory.js.map +1 -1
- package/dist/timeout.d.ts.map +1 -1
- package/dist/timeout.js +2 -4
- package/dist/timeout.js.map +1 -1
- package/dist/types.d.ts +9 -59
- package/dist/types.d.ts.map +1 -1
- package/dist/ui.d.ts +3 -94
- package/dist/ui.d.ts.map +1 -1
- package/dist/ui.js +30 -1427
- package/dist/ui.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +15 -8
- package/src/activity.ts +0 -9
- package/src/auth/bearer.ts +4 -33
- package/src/auth/clerk.ts +35 -48
- package/src/call-admission.ts +8 -6
- package/src/catalog-fingerprint.ts +52 -0
- package/src/catalog-limits.ts +11 -0
- package/src/catalog-service.ts +601 -0
- package/src/catalog.ts +57 -2
- package/src/concurrency.ts +39 -0
- package/src/connectors/api.ts +22 -12
- package/src/connectors/remote-mcp.ts +63 -45
- package/src/credentials.ts +2 -2
- package/src/errors.ts +21 -1
- package/src/execute.ts +193 -423
- package/src/executor-result.ts +3 -3
- package/src/executors/quickjs-child.ts +8 -4
- package/src/executors/quickjs-protocol.ts +1 -1
- package/src/executors/quickjs-runtime.ts +5 -1
- package/src/executors/quickjs.ts +11 -8
- package/src/favicon.ts +2 -2
- package/src/index.ts +70 -222
- package/src/invocation.ts +580 -0
- package/src/meta-tools.ts +393 -985
- package/src/operator-ui/browser.css +360 -0
- package/src/operator-ui/browser.ts +987 -0
- package/src/operator-ui/generated.ts +4 -0
- package/src/operator-ui/model.ts +105 -0
- package/src/registry.ts +459 -439
- package/src/retired-toolkits.ts +14 -0
- package/src/routes/activity.ts +215 -0
- package/src/routes/credentials.ts +254 -0
- package/src/routes/mcp.ts +339 -0
- package/src/routes/oauth.ts +322 -0
- package/src/routes/shared.ts +240 -0
- package/src/routes/ui.ts +143 -0
- package/src/server.ts +64 -1731
- package/src/skills.ts +15 -43
- package/src/storage/file.ts +3 -1
- package/src/storage/memory.ts +3 -1
- package/src/timeout.ts +2 -4
- package/src/types.ts +8 -60
- package/src/ui.ts +60 -1538
- package/src/version.ts +1 -1
- package/dist/credential-health.d.ts +0 -227
- package/dist/credential-health.d.ts.map +0 -1
- package/dist/credential-health.js +0 -606
- package/dist/credential-health.js.map +0 -1
- package/dist/toolkits.d.ts +0 -146
- package/dist/toolkits.d.ts.map +0 -1
- package/dist/toolkits.js +0 -322
- package/dist/toolkits.js.map +0 -1
- package/src/credential-health.ts +0 -832
- package/src/toolkits.ts +0 -461
package/src/registry.ts
CHANGED
|
@@ -11,27 +11,46 @@ import {
|
|
|
11
11
|
type CredentialVault,
|
|
12
12
|
} from "./credentials.js";
|
|
13
13
|
import { ConnectorCallError } from "./errors.js";
|
|
14
|
-
import {
|
|
15
|
-
CredentialHealthChecker,
|
|
16
|
-
type CredentialCheckOptions,
|
|
17
|
-
type CredentialCheckResult,
|
|
18
|
-
type CredentialHealthConfig,
|
|
19
|
-
type CredentialHealthRecord,
|
|
20
|
-
} from "./credential-health.js";
|
|
21
14
|
import {
|
|
22
15
|
ConnectorCallAdmissionController,
|
|
23
16
|
type CallAdmissionPermit,
|
|
24
17
|
type ConnectorCallAdmissionSnapshot,
|
|
25
18
|
} from "./call-admission.js";
|
|
26
|
-
import
|
|
27
|
-
|
|
19
|
+
import {
|
|
20
|
+
fingerprintSerializedCatalog,
|
|
21
|
+
snapshotCatalog,
|
|
22
|
+
type CatalogSnapshot,
|
|
23
|
+
} from "./catalog-fingerprint.js";
|
|
24
|
+
import {
|
|
25
|
+
MAX_CATALOG_CHUNK_BYTES,
|
|
26
|
+
MAX_CATALOG_TOOLS,
|
|
27
|
+
MAX_SERIALIZED_CATALOG_BYTES,
|
|
28
|
+
} from "./catalog-limits.js";
|
|
29
|
+
import { mapSettledWithConcurrency } from "./concurrency.js";
|
|
28
30
|
|
|
29
31
|
const ID_RE = /^[a-z0-9_-]+$/;
|
|
30
32
|
const DEFAULT_TTL_SECONDS = 300;
|
|
31
33
|
const DEFAULT_STALE_SECONDS = 3600;
|
|
32
|
-
|
|
34
|
+
const CATALOG_CHUNK_TTL_GRACE_SECONDS = 300;
|
|
35
|
+
const DEFAULT_MAX_RESULT_BYTES = 50_000;
|
|
36
|
+
const encoder = new TextEncoder();
|
|
33
37
|
/** Independent final-envelope boundary for `batch_call`. */
|
|
34
|
-
|
|
38
|
+
const DEFAULT_MAX_BATCH_RESULT_BYTES = 100_000;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Split `"<connectorId>.<toolName>"` on the first dot. Connector ids contain
|
|
42
|
+
* no dots, so a downstream tool name may.
|
|
43
|
+
*/
|
|
44
|
+
function splitAddress(
|
|
45
|
+
address: string,
|
|
46
|
+
): { connectorId: string; toolName: string } | null {
|
|
47
|
+
const dot = address.indexOf(".");
|
|
48
|
+
if (dot <= 0 || dot === address.length - 1) return null;
|
|
49
|
+
return {
|
|
50
|
+
connectorId: address.slice(0, dot),
|
|
51
|
+
toolName: address.slice(dot + 1),
|
|
52
|
+
};
|
|
53
|
+
}
|
|
35
54
|
|
|
36
55
|
/**
|
|
37
56
|
* Smallest accepted inline-result cap. One byte is pathological but harmless:
|
|
@@ -76,17 +95,41 @@ export function resolveMaxResultBytes(
|
|
|
76
95
|
|
|
77
96
|
interface CacheEntry {
|
|
78
97
|
tools: ToolDef[];
|
|
98
|
+
fingerprint: string;
|
|
79
99
|
exp: number; // epoch ms
|
|
80
100
|
staleUntil: number;
|
|
81
101
|
}
|
|
82
102
|
|
|
103
|
+
interface LegacyPersistedCatalog {
|
|
104
|
+
tools: ToolDef[];
|
|
105
|
+
/** Optional only for catalogs written before fingerprints were introduced. */
|
|
106
|
+
fingerprint?: string;
|
|
107
|
+
fetchedAt: number;
|
|
108
|
+
expiresAt: number;
|
|
109
|
+
staleUntil: number;
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
interface PersistedCatalogManifest {
|
|
113
|
+
version: 2;
|
|
114
|
+
revision: string;
|
|
115
|
+
toolCount: number;
|
|
116
|
+
byteCount: number;
|
|
117
|
+
chunkCount: number;
|
|
118
|
+
fetchedAt: number;
|
|
119
|
+
expiresAt: number;
|
|
120
|
+
staleUntil: number;
|
|
121
|
+
}
|
|
122
|
+
|
|
83
123
|
interface PersistedCatalog {
|
|
84
124
|
tools: ToolDef[];
|
|
125
|
+
fingerprint: string;
|
|
85
126
|
fetchedAt: number;
|
|
86
127
|
expiresAt: number;
|
|
87
128
|
staleUntil: number;
|
|
88
129
|
}
|
|
89
130
|
|
|
131
|
+
const CATALOG_CHUNK_IO_CONCURRENCY = 4;
|
|
132
|
+
|
|
90
133
|
export interface HealthObservation {
|
|
91
134
|
lastSuccessAt?: string;
|
|
92
135
|
lastFailureAt?: string;
|
|
@@ -96,28 +139,21 @@ export interface HealthObservation {
|
|
|
96
139
|
}
|
|
97
140
|
|
|
98
141
|
/**
|
|
99
|
-
* Recent real-call outcomes per connector
|
|
100
|
-
*
|
|
101
|
-
* The deployment keeps one log, and each toolkit keeps its own. That split is a
|
|
102
|
-
* scope boundary, not bookkeeping: `lastError` is a downstream error string
|
|
103
|
-
* that routinely names the tool that failed, and `list_connectors` returns the
|
|
104
|
-
* observation verbatim. Sharing one log would let a toolkit read back the tools
|
|
105
|
-
* and failures of a sibling toolkit through a connector they happen to share —
|
|
106
|
-
* and let a sibling's failures flip this view's reported status. A scoped view
|
|
107
|
-
* reads back only the calls it made itself.
|
|
142
|
+
* Recent real-call outcomes per connector.
|
|
108
143
|
*/
|
|
109
|
-
|
|
144
|
+
class HealthLog {
|
|
110
145
|
private readonly observations = new Map<string, HealthObservation>();
|
|
111
146
|
|
|
112
147
|
recordSuccess(id: string, latencyMs: number): void {
|
|
113
148
|
const previous = this.observations.get(id);
|
|
114
|
-
|
|
149
|
+
const observation: HealthObservation = {
|
|
115
150
|
...previous,
|
|
116
151
|
lastSuccessAt: new Date().toISOString(),
|
|
117
152
|
lastLatencyMs: latencyMs,
|
|
118
153
|
consecutiveFailures: 0,
|
|
119
|
-
|
|
120
|
-
|
|
154
|
+
};
|
|
155
|
+
delete observation.lastError;
|
|
156
|
+
this.observations.set(id, observation);
|
|
121
157
|
}
|
|
122
158
|
|
|
123
159
|
recordFailure(id: string, latencyMs: number, error: unknown): void {
|
|
@@ -155,8 +191,6 @@ export interface RegistryOptions {
|
|
|
155
191
|
* of bytes >= 1; anything else warns and falls back to 100_000.
|
|
156
192
|
*/
|
|
157
193
|
maxBatchResultBytes?: number;
|
|
158
|
-
/** Tuning for the credential liveness checks (issue #24). */
|
|
159
|
-
credentialHealth?: CredentialHealthConfig;
|
|
160
194
|
}
|
|
161
195
|
|
|
162
196
|
function namespaced(storage: KVStorage, prefix: string): KVStorage {
|
|
@@ -171,7 +205,7 @@ function msg(err: unknown): string {
|
|
|
171
205
|
return err instanceof Error ? err.message : String(err);
|
|
172
206
|
}
|
|
173
207
|
|
|
174
|
-
type ConnectorOperationOptions = Pick<
|
|
208
|
+
export type ConnectorOperationOptions = Pick<
|
|
175
209
|
ConnectorContext,
|
|
176
210
|
"signal" | "timeoutMs"
|
|
177
211
|
>;
|
|
@@ -181,12 +215,9 @@ type ConnectorOperationOptions = Pick<
|
|
|
181
215
|
* (`src/meta-tools.ts`) and the `execute_code` sandbox bridge (`src/execute.ts`)
|
|
182
216
|
* is typed against THIS, never against the concrete `Registry`.
|
|
183
217
|
*
|
|
184
|
-
*
|
|
185
|
-
*
|
|
186
|
-
*
|
|
187
|
-
* here is a compile error, and adding one here is a compile error until
|
|
188
|
-
* `ScopedRegistry` filters it — so a new meta-tool cannot quietly step around
|
|
189
|
-
* the boundary.
|
|
218
|
+
* The read-only seam remains useful even without scoped views: meta-tools can
|
|
219
|
+
* consume registry behavior without depending on the concrete implementation
|
|
220
|
+
* or its construction-only methods.
|
|
190
221
|
*/
|
|
191
222
|
export interface RegistryView {
|
|
192
223
|
/** Deployment-wide result-size cap threaded to the meta-tools. */
|
|
@@ -217,10 +248,7 @@ export interface RegistryView {
|
|
|
217
248
|
requestScope?: object,
|
|
218
249
|
callOptions?: ConnectorOperationOptions,
|
|
219
250
|
): ConnectorContext;
|
|
220
|
-
/**
|
|
221
|
-
* Acquire the connector's shared downstream-call permit. Scoped views
|
|
222
|
-
* delegate to the base registry so every toolkit contends on the same pool.
|
|
223
|
-
*/
|
|
251
|
+
/** Acquire the connector's shared downstream-call permit. */
|
|
224
252
|
admitCall(
|
|
225
253
|
id: string,
|
|
226
254
|
input: { toolName: string; args: unknown; signal?: AbortSignal },
|
|
@@ -230,15 +258,9 @@ export interface RegistryView {
|
|
|
230
258
|
recordFailure(id: string, latencyMs: number, error: unknown): void;
|
|
231
259
|
healthFor(id: string): HealthObservation | undefined;
|
|
232
260
|
hasObservedSuccess(id: string): boolean;
|
|
233
|
-
/** When ANY view last saw a successful call to `id`, deployment-wide. */
|
|
234
261
|
observedSuccessAt(id: string): string | undefined;
|
|
235
|
-
/**
|
|
236
|
-
|
|
237
|
-
/** Store a liveness verdict a live status check just produced. */
|
|
238
|
-
recordCredentialHealth(
|
|
239
|
-
id: string,
|
|
240
|
-
record: CredentialHealthRecord,
|
|
241
|
-
): Promise<void>;
|
|
262
|
+
/** Local declared-vs-stored credential mismatch, with no downstream I/O. */
|
|
263
|
+
credentialDriftFor(id: string): Promise<string | undefined>;
|
|
242
264
|
statusFor(
|
|
243
265
|
id: string,
|
|
244
266
|
baseUrl: string,
|
|
@@ -265,6 +287,11 @@ export class Registry implements RegistryView {
|
|
|
265
287
|
private readonly catalogGenerations = new Map<string, number>();
|
|
266
288
|
/** Serialize persisted catalog set/delete operations within this isolate. */
|
|
267
289
|
private readonly catalogMutations = new Map<string, Promise<void>>();
|
|
290
|
+
/** Same-request cold loads share one promise without retaining the request. */
|
|
291
|
+
private readonly requestCatalogLoads = new WeakMap<
|
|
292
|
+
object,
|
|
293
|
+
Map<string, Promise<ToolDef[]>>
|
|
294
|
+
>();
|
|
268
295
|
/** Deployment-wide observations — every call, whatever view made it. */
|
|
269
296
|
private readonly health = new HealthLog();
|
|
270
297
|
private readonly ttlMs: number;
|
|
@@ -274,8 +301,6 @@ export class Registry implements RegistryView {
|
|
|
274
301
|
readonly maxResultBytes: number;
|
|
275
302
|
/** Final batch envelope cap threaded to the meta-tools. */
|
|
276
303
|
readonly maxBatchResultBytes: number;
|
|
277
|
-
/** Proactive liveness checks over stored downstream credentials (issue #24). */
|
|
278
|
-
private readonly credentialHealth: CredentialHealthChecker;
|
|
279
304
|
|
|
280
305
|
constructor(
|
|
281
306
|
connectors: Connector[],
|
|
@@ -317,18 +342,6 @@ export class Registry implements RegistryView {
|
|
|
317
342
|
opts.maxResultBytes,
|
|
318
343
|
opts.maxBatchResultBytes,
|
|
319
344
|
);
|
|
320
|
-
this.credentialHealth = new CredentialHealthChecker(
|
|
321
|
-
{
|
|
322
|
-
listConnectors: () => this.listConnectors(),
|
|
323
|
-
getConnector: (id) => this.getConnector(id),
|
|
324
|
-
contextFor: (id, baseUrl, requestScope) =>
|
|
325
|
-
this.contextFor(id, baseUrl, requestScope),
|
|
326
|
-
storage: opts.storage,
|
|
327
|
-
logger: opts.logger,
|
|
328
|
-
credentialVault: opts.credentialVault,
|
|
329
|
-
},
|
|
330
|
-
opts.credentialHealth,
|
|
331
|
-
);
|
|
332
345
|
}
|
|
333
346
|
|
|
334
347
|
/**
|
|
@@ -500,47 +513,304 @@ export class Registry implements RegistryView {
|
|
|
500
513
|
return `catalog:${id}`;
|
|
501
514
|
}
|
|
502
515
|
|
|
503
|
-
private
|
|
516
|
+
private catalogChunkKey(id: string, revision: string, index: number): string {
|
|
517
|
+
return `${this.catalogKey(id)}:chunk:${revision}:${index}`;
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
private validLegacyCatalog(value: unknown): LegacyPersistedCatalog | null {
|
|
521
|
+
if (!value || typeof value !== "object") return null;
|
|
522
|
+
const catalog = value as Partial<LegacyPersistedCatalog>;
|
|
523
|
+
if (
|
|
524
|
+
!Array.isArray(catalog.tools) ||
|
|
525
|
+
typeof catalog.fetchedAt !== "number" ||
|
|
526
|
+
typeof catalog.expiresAt !== "number" ||
|
|
527
|
+
typeof catalog.staleUntil !== "number" ||
|
|
528
|
+
(catalog.fingerprint !== undefined &&
|
|
529
|
+
typeof catalog.fingerprint !== "string") ||
|
|
530
|
+
!this.validCatalogTools(catalog.tools)
|
|
531
|
+
) {
|
|
532
|
+
return null;
|
|
533
|
+
}
|
|
534
|
+
return catalog as LegacyPersistedCatalog;
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
private validCatalogTools(value: unknown[]): value is ToolDef[] {
|
|
538
|
+
return value.every(
|
|
539
|
+
(tool) =>
|
|
540
|
+
tool !== null &&
|
|
541
|
+
typeof tool === "object" &&
|
|
542
|
+
typeof (tool as ToolDef).name === "string",
|
|
543
|
+
);
|
|
544
|
+
}
|
|
545
|
+
|
|
546
|
+
private validCatalogManifest(
|
|
547
|
+
value: unknown,
|
|
548
|
+
): PersistedCatalogManifest | null {
|
|
549
|
+
if (!value || typeof value !== "object") return null;
|
|
550
|
+
const manifest = value as Partial<PersistedCatalogManifest>;
|
|
551
|
+
const maxChunks =
|
|
552
|
+
Math.ceil(MAX_SERIALIZED_CATALOG_BYTES / MAX_CATALOG_CHUNK_BYTES) + 1;
|
|
553
|
+
if (
|
|
554
|
+
manifest.version !== 2 ||
|
|
555
|
+
typeof manifest.revision !== "string" ||
|
|
556
|
+
!/^sha256:[0-9]{1,8}:[0-9a-f]{64}$/.test(manifest.revision) ||
|
|
557
|
+
!Number.isInteger(manifest.toolCount) ||
|
|
558
|
+
manifest.toolCount! < 0 ||
|
|
559
|
+
manifest.toolCount! > MAX_CATALOG_TOOLS ||
|
|
560
|
+
!Number.isInteger(manifest.byteCount) ||
|
|
561
|
+
manifest.byteCount! < 2 ||
|
|
562
|
+
manifest.byteCount! > MAX_SERIALIZED_CATALOG_BYTES ||
|
|
563
|
+
!manifest.revision.startsWith(`sha256:${manifest.byteCount}:`) ||
|
|
564
|
+
!Number.isInteger(manifest.chunkCount) ||
|
|
565
|
+
manifest.chunkCount! < 1 ||
|
|
566
|
+
manifest.chunkCount! > maxChunks ||
|
|
567
|
+
typeof manifest.fetchedAt !== "number" ||
|
|
568
|
+
typeof manifest.expiresAt !== "number" ||
|
|
569
|
+
typeof manifest.staleUntil !== "number"
|
|
570
|
+
) {
|
|
571
|
+
return null;
|
|
572
|
+
}
|
|
573
|
+
return manifest as PersistedCatalogManifest;
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
private parseCatalogManifest(
|
|
577
|
+
raw: string | null,
|
|
578
|
+
): PersistedCatalogManifest | null {
|
|
579
|
+
if (!raw) return null;
|
|
580
|
+
try {
|
|
581
|
+
return this.validCatalogManifest(JSON.parse(raw));
|
|
582
|
+
} catch {
|
|
583
|
+
return null;
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
private splitCatalogChunks(snapshot: CatalogSnapshot): string[] {
|
|
588
|
+
const chunks: string[] = [];
|
|
589
|
+
const decoder = new TextDecoder("utf-8", {
|
|
590
|
+
fatal: true,
|
|
591
|
+
ignoreBOM: true,
|
|
592
|
+
});
|
|
593
|
+
let offset = 0;
|
|
594
|
+
while (offset < snapshot.serializedBytes.byteLength) {
|
|
595
|
+
let end = Math.min(
|
|
596
|
+
offset + MAX_CATALOG_CHUNK_BYTES,
|
|
597
|
+
snapshot.serializedBytes.byteLength,
|
|
598
|
+
);
|
|
599
|
+
// Move a boundary that landed inside a multibyte UTF-8 sequence back to
|
|
600
|
+
// the next character start. Every stored string then remains valid UTF-8.
|
|
601
|
+
while (
|
|
602
|
+
end < snapshot.serializedBytes.byteLength &&
|
|
603
|
+
((snapshot.serializedBytes[end] ?? 0) & 0xc0) === 0x80
|
|
604
|
+
) {
|
|
605
|
+
end--;
|
|
606
|
+
}
|
|
607
|
+
chunks.push(
|
|
608
|
+
decoder.decode(snapshot.serializedBytes.subarray(offset, end)),
|
|
609
|
+
);
|
|
610
|
+
offset = end;
|
|
611
|
+
}
|
|
612
|
+
return chunks;
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
private async readCatalog(
|
|
616
|
+
id: string,
|
|
617
|
+
raw: string | null,
|
|
618
|
+
now: number,
|
|
619
|
+
): Promise<PersistedCatalog | null> {
|
|
504
620
|
if (!raw) return null;
|
|
621
|
+
let parsed: unknown;
|
|
505
622
|
try {
|
|
506
|
-
|
|
623
|
+
parsed = JSON.parse(raw);
|
|
624
|
+
} catch {
|
|
625
|
+
return null;
|
|
626
|
+
}
|
|
627
|
+
|
|
628
|
+
if (
|
|
629
|
+
!parsed ||
|
|
630
|
+
typeof parsed !== "object" ||
|
|
631
|
+
(parsed as { version?: unknown }).version !== 2
|
|
632
|
+
) {
|
|
633
|
+
const legacy = this.validLegacyCatalog(parsed);
|
|
634
|
+
if (!legacy || legacy.staleUntil <= now) return null;
|
|
635
|
+
const snapshot = await snapshotCatalog(legacy.tools);
|
|
636
|
+
if (
|
|
637
|
+
legacy.tools.length > MAX_CATALOG_TOOLS ||
|
|
638
|
+
snapshot.serializedBytes.byteLength > MAX_SERIALIZED_CATALOG_BYTES ||
|
|
639
|
+
(legacy.fingerprint !== undefined &&
|
|
640
|
+
legacy.fingerprint !== snapshot.fingerprint)
|
|
641
|
+
) {
|
|
642
|
+
this.opts.logger.warn(
|
|
643
|
+
`[connecta] connector "${id}" legacy catalog is oversized or has a fingerprint mismatch; ignoring persisted catalog.`,
|
|
644
|
+
);
|
|
645
|
+
return null;
|
|
646
|
+
}
|
|
647
|
+
return {
|
|
648
|
+
tools: legacy.tools,
|
|
649
|
+
fingerprint: snapshot.fingerprint,
|
|
650
|
+
fetchedAt: legacy.fetchedAt,
|
|
651
|
+
expiresAt: legacy.expiresAt,
|
|
652
|
+
staleUntil: legacy.staleUntil,
|
|
653
|
+
};
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
const manifest = this.validCatalogManifest(parsed);
|
|
657
|
+
if (!manifest) {
|
|
658
|
+
this.opts.logger.warn(
|
|
659
|
+
`[connecta] connector "${id}" catalog manifest is invalid; ignoring persisted catalog.`,
|
|
660
|
+
);
|
|
661
|
+
return null;
|
|
662
|
+
}
|
|
663
|
+
if (manifest.staleUntil <= now) return null;
|
|
664
|
+
|
|
665
|
+
const chunkReads = await mapSettledWithConcurrency(
|
|
666
|
+
Array.from({ length: manifest.chunkCount }, (_, index) => index),
|
|
667
|
+
CATALOG_CHUNK_IO_CONCURRENCY,
|
|
668
|
+
(index) =>
|
|
669
|
+
this.opts.storage.get(
|
|
670
|
+
this.catalogChunkKey(id, manifest.revision, index),
|
|
671
|
+
),
|
|
672
|
+
);
|
|
673
|
+
const chunks: string[] = [];
|
|
674
|
+
let chunkBytes = 0;
|
|
675
|
+
for (const [index, result] of chunkReads.entries()) {
|
|
676
|
+
if (result.status === "rejected") throw result.reason;
|
|
677
|
+
const chunk = result.value;
|
|
678
|
+
if (chunk === null) {
|
|
679
|
+
this.opts.logger.warn(
|
|
680
|
+
`[connecta] connector "${id}" catalog chunk ${index + 1}/${manifest.chunkCount} is missing; ignoring persisted catalog.`,
|
|
681
|
+
);
|
|
682
|
+
return null;
|
|
683
|
+
}
|
|
684
|
+
const byteLength = encoder.encode(chunk).byteLength;
|
|
685
|
+
chunkBytes += byteLength;
|
|
507
686
|
if (
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
typeof value.expiresAt !== "number" ||
|
|
511
|
-
typeof value.staleUntil !== "number" ||
|
|
512
|
-
!value.tools.every(
|
|
513
|
-
(tool) =>
|
|
514
|
-
tool !== null &&
|
|
515
|
-
typeof tool === "object" &&
|
|
516
|
-
typeof (tool as ToolDef).name === "string",
|
|
517
|
-
)
|
|
687
|
+
byteLength > MAX_CATALOG_CHUNK_BYTES ||
|
|
688
|
+
chunkBytes > manifest.byteCount
|
|
518
689
|
) {
|
|
690
|
+
this.opts.logger.warn(
|
|
691
|
+
`[connecta] connector "${id}" catalog chunk bounds do not match its manifest; ignoring persisted catalog.`,
|
|
692
|
+
);
|
|
519
693
|
return null;
|
|
520
694
|
}
|
|
521
|
-
|
|
695
|
+
chunks.push(chunk);
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
const serializedTools = chunks.join("");
|
|
699
|
+
const stored = await fingerprintSerializedCatalog(serializedTools);
|
|
700
|
+
if (
|
|
701
|
+
stored.byteLength !== manifest.byteCount ||
|
|
702
|
+
stored.fingerprint !== manifest.revision
|
|
703
|
+
) {
|
|
704
|
+
this.opts.logger.warn(
|
|
705
|
+
`[connecta] connector "${id}" catalog fingerprint mismatch; ignoring persisted catalog.`,
|
|
706
|
+
);
|
|
707
|
+
return null;
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
let tools: unknown;
|
|
711
|
+
try {
|
|
712
|
+
tools = JSON.parse(serializedTools);
|
|
522
713
|
} catch {
|
|
714
|
+
this.opts.logger.warn(
|
|
715
|
+
`[connecta] connector "${id}" catalog chunks are torn; ignoring persisted catalog.`,
|
|
716
|
+
);
|
|
523
717
|
return null;
|
|
524
718
|
}
|
|
719
|
+
if (!Array.isArray(tools) || !this.validCatalogTools(tools)) {
|
|
720
|
+
this.opts.logger.warn(
|
|
721
|
+
`[connecta] connector "${id}" catalog chunks contain invalid tools; ignoring persisted catalog.`,
|
|
722
|
+
);
|
|
723
|
+
return null;
|
|
724
|
+
}
|
|
725
|
+
if (tools.length !== manifest.toolCount) {
|
|
726
|
+
this.opts.logger.warn(
|
|
727
|
+
`[connecta] connector "${id}" catalog tool count does not match its manifest; ignoring persisted catalog.`,
|
|
728
|
+
);
|
|
729
|
+
return null;
|
|
730
|
+
}
|
|
731
|
+
return {
|
|
732
|
+
tools,
|
|
733
|
+
fingerprint: stored.fingerprint,
|
|
734
|
+
fetchedAt: manifest.fetchedAt,
|
|
735
|
+
expiresAt: manifest.expiresAt,
|
|
736
|
+
staleUntil: manifest.staleUntil,
|
|
737
|
+
};
|
|
525
738
|
}
|
|
526
739
|
|
|
527
|
-
private async storeCatalog(
|
|
740
|
+
private async storeCatalog(
|
|
741
|
+
id: string,
|
|
742
|
+
snapshot: CatalogSnapshot,
|
|
743
|
+
): Promise<void> {
|
|
528
744
|
if (!this.persistToolCatalog) return;
|
|
529
745
|
const fetchedAt = Date.now();
|
|
530
|
-
const
|
|
531
|
-
|
|
746
|
+
const expiresAt = fetchedAt + this.ttlMs;
|
|
747
|
+
const staleUntil = expiresAt + this.staleMs;
|
|
748
|
+
const ttlSeconds = Math.max(
|
|
749
|
+
60,
|
|
750
|
+
Math.ceil((this.ttlMs + this.staleMs) / 1000),
|
|
751
|
+
);
|
|
752
|
+
const chunks = this.splitCatalogChunks(snapshot);
|
|
753
|
+
const chunkWrites = await mapSettledWithConcurrency(
|
|
754
|
+
chunks,
|
|
755
|
+
CATALOG_CHUNK_IO_CONCURRENCY,
|
|
756
|
+
(chunk, index) =>
|
|
757
|
+
this.opts.storage.set(
|
|
758
|
+
this.catalogChunkKey(id, snapshot.fingerprint, index),
|
|
759
|
+
chunk,
|
|
760
|
+
{ ttlSeconds: ttlSeconds + CATALOG_CHUNK_TTL_GRACE_SECONDS },
|
|
761
|
+
),
|
|
762
|
+
);
|
|
763
|
+
for (const result of chunkWrites) {
|
|
764
|
+
if (result.status === "rejected") throw result.reason;
|
|
765
|
+
}
|
|
766
|
+
// The manifest is the only publication point. A failed/partial chunk write
|
|
767
|
+
// therefore leaves the previous manifest authoritative (or no catalog);
|
|
768
|
+
// unreachable chunks carry a bounded TTL and require no prefix scan.
|
|
769
|
+
const manifest: PersistedCatalogManifest = {
|
|
770
|
+
version: 2,
|
|
771
|
+
revision: snapshot.fingerprint,
|
|
772
|
+
toolCount: snapshot.tools.length,
|
|
773
|
+
byteCount: snapshot.serializedBytes.byteLength,
|
|
774
|
+
chunkCount: chunks.length,
|
|
532
775
|
fetchedAt,
|
|
533
|
-
expiresAt
|
|
534
|
-
staleUntil
|
|
776
|
+
expiresAt,
|
|
777
|
+
staleUntil,
|
|
535
778
|
};
|
|
536
|
-
await this.opts.storage.set(this.catalogKey(id), JSON.stringify(
|
|
537
|
-
ttlSeconds
|
|
538
|
-
60,
|
|
539
|
-
Math.ceil((this.ttlMs + this.staleMs) / 1000),
|
|
540
|
-
),
|
|
779
|
+
await this.opts.storage.set(this.catalogKey(id), JSON.stringify(manifest), {
|
|
780
|
+
ttlSeconds,
|
|
541
781
|
});
|
|
542
782
|
}
|
|
543
783
|
|
|
784
|
+
private async deleteCatalog(id: string): Promise<void> {
|
|
785
|
+
let raw: string | null = null;
|
|
786
|
+
let readError: unknown;
|
|
787
|
+
try {
|
|
788
|
+
raw = await this.opts.storage.get(this.catalogKey(id));
|
|
789
|
+
} catch (err) {
|
|
790
|
+
readError = err;
|
|
791
|
+
}
|
|
792
|
+
const manifest = this.parseCatalogManifest(raw);
|
|
793
|
+
// The root is authoritative, so attempt its deletion even when the
|
|
794
|
+
// best-effort read needed for physical chunk cleanup failed.
|
|
795
|
+
await this.opts.storage.delete(this.catalogKey(id));
|
|
796
|
+
if (!manifest) {
|
|
797
|
+
if (readError) throw readError;
|
|
798
|
+
return;
|
|
799
|
+
}
|
|
800
|
+
let firstError: unknown;
|
|
801
|
+
for (let index = 0; index < manifest.chunkCount; index++) {
|
|
802
|
+
try {
|
|
803
|
+
await this.opts.storage.delete(
|
|
804
|
+
this.catalogChunkKey(id, manifest.revision, index),
|
|
805
|
+
);
|
|
806
|
+
} catch (err) {
|
|
807
|
+
firstError ??= err;
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
if (readError) throw readError;
|
|
811
|
+
if (firstError) throw firstError;
|
|
812
|
+
}
|
|
813
|
+
|
|
544
814
|
private catalogGeneration(id: string): number {
|
|
545
815
|
return this.catalogGenerations.get(id) ?? 0;
|
|
546
816
|
}
|
|
@@ -580,25 +850,43 @@ export class Registry implements RegistryView {
|
|
|
580
850
|
): Promise<ToolDef[]> {
|
|
581
851
|
const connector = this.connectors.get(id);
|
|
582
852
|
if (!connector) throw new Error(`Unknown connector "${id}"`);
|
|
853
|
+
if (connector.staticTools) return connector.staticTools;
|
|
583
854
|
const generation = this.catalogGeneration(id);
|
|
584
|
-
const tools = connector.
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
this.contextFor(id, baseUrl, requestScope, callOptions),
|
|
588
|
-
);
|
|
855
|
+
const tools = await connector.listTools(
|
|
856
|
+
this.contextFor(id, baseUrl, requestScope, callOptions),
|
|
857
|
+
);
|
|
589
858
|
// The caller that began this refresh may still use its result, but a
|
|
590
859
|
// credential/OAuth change that landed while listTools was in flight means
|
|
591
860
|
// the listing must not enter either shared cache layer.
|
|
592
861
|
if (generation !== this.catalogGeneration(id)) return tools;
|
|
593
|
-
|
|
862
|
+
if (tools.length > MAX_CATALOG_TOOLS) {
|
|
863
|
+
const message =
|
|
864
|
+
`Connector "${id}" returned ${tools.length} tools, over the ` +
|
|
865
|
+
`${MAX_CATALOG_TOOLS}-tool catalog ceiling; refusing the complete catalog.`;
|
|
866
|
+
this.opts.logger.warn(`[connecta] ${message}`);
|
|
867
|
+
throw new Error(message);
|
|
868
|
+
}
|
|
594
869
|
const previous = this.cache.get(id);
|
|
870
|
+
const snapshot = await snapshotCatalog(tools);
|
|
871
|
+
if (generation !== this.catalogGeneration(id)) return tools;
|
|
872
|
+
if (snapshot.serializedBytes.byteLength > MAX_SERIALIZED_CATALOG_BYTES) {
|
|
873
|
+
const message =
|
|
874
|
+
`Connector "${id}" returned a ${snapshot.serializedBytes.byteLength}-byte ` +
|
|
875
|
+
`serialized catalog, over the ${MAX_SERIALIZED_CATALOG_BYTES}-byte ceiling; ` +
|
|
876
|
+
"refusing the complete catalog.";
|
|
877
|
+
this.opts.logger.warn(`[connecta] ${message}`);
|
|
878
|
+
throw new Error(message);
|
|
879
|
+
}
|
|
880
|
+
const now = Date.now();
|
|
595
881
|
const catalogChanged =
|
|
596
|
-
!previous ||
|
|
882
|
+
!previous || previous.fingerprint !== snapshot.fingerprint;
|
|
597
883
|
const shouldPersist =
|
|
598
|
-
|
|
599
|
-
(
|
|
884
|
+
catalogChanged ||
|
|
885
|
+
(previous !== undefined && previous.exp <= now) ||
|
|
886
|
+
this.invalidated.has(id);
|
|
600
887
|
this.cache.set(id, {
|
|
601
888
|
tools,
|
|
889
|
+
fingerprint: snapshot.fingerprint,
|
|
602
890
|
exp: now + this.ttlMs,
|
|
603
891
|
staleUntil: now + this.ttlMs + this.staleMs,
|
|
604
892
|
});
|
|
@@ -607,7 +895,7 @@ export class Registry implements RegistryView {
|
|
|
607
895
|
await this.enqueueCatalogMutation(id, async () => {
|
|
608
896
|
if (generation !== this.catalogGeneration(id)) return;
|
|
609
897
|
try {
|
|
610
|
-
await this.storeCatalog(id,
|
|
898
|
+
await this.storeCatalog(id, snapshot);
|
|
611
899
|
} catch (err) {
|
|
612
900
|
this.opts.logger.warn(
|
|
613
901
|
`[connecta] connector "${id}" catalog persistence failed: ${msg(err)}`,
|
|
@@ -619,7 +907,7 @@ export class Registry implements RegistryView {
|
|
|
619
907
|
}
|
|
620
908
|
|
|
621
909
|
/** Cached listTools with in-memory + persisted serializable catalog layers. */
|
|
622
|
-
async
|
|
910
|
+
private async loadTools(
|
|
623
911
|
id: string,
|
|
624
912
|
baseUrl: string,
|
|
625
913
|
requestScope?: object,
|
|
@@ -639,8 +927,10 @@ export class Registry implements RegistryView {
|
|
|
639
927
|
const generation = this.catalogGeneration(id);
|
|
640
928
|
let persisted: PersistedCatalog | null = null;
|
|
641
929
|
try {
|
|
642
|
-
persisted = this.
|
|
930
|
+
persisted = await this.readCatalog(
|
|
931
|
+
id,
|
|
643
932
|
await this.opts.storage.get(this.catalogKey(id)),
|
|
933
|
+
now,
|
|
644
934
|
);
|
|
645
935
|
} catch (err) {
|
|
646
936
|
this.opts.logger.warn(
|
|
@@ -654,6 +944,7 @@ export class Registry implements RegistryView {
|
|
|
654
944
|
if (persisted && persisted.staleUntil > now) {
|
|
655
945
|
this.cache.set(id, {
|
|
656
946
|
tools: persisted.tools,
|
|
947
|
+
fingerprint: persisted.fingerprint,
|
|
657
948
|
exp: persisted.expiresAt,
|
|
658
949
|
staleUntil: persisted.staleUntil,
|
|
659
950
|
});
|
|
@@ -679,6 +970,46 @@ export class Registry implements RegistryView {
|
|
|
679
970
|
}
|
|
680
971
|
}
|
|
681
972
|
|
|
973
|
+
/**
|
|
974
|
+
* Coalesce one connector's cold load inside one inbound request. The WeakMap
|
|
975
|
+
* neither roots the request scope nor lets its connector context escape into
|
|
976
|
+
* another request; settled entries are also removed eagerly.
|
|
977
|
+
*
|
|
978
|
+
* The first caller's `callOptions` govern the shared load: a later caller's
|
|
979
|
+
* signal or timeout neither cancels nor extends it, and an abort by the
|
|
980
|
+
* first caller rejects every coalesced caller. Within one request that is
|
|
981
|
+
* the deal being made — one fetch, one deadline.
|
|
982
|
+
*/
|
|
983
|
+
async getTools(
|
|
984
|
+
id: string,
|
|
985
|
+
baseUrl: string,
|
|
986
|
+
requestScope?: object,
|
|
987
|
+
callOptions: ConnectorOperationOptions = {},
|
|
988
|
+
): Promise<ToolDef[]> {
|
|
989
|
+
const connector = this.connectors.get(id);
|
|
990
|
+
if (!connector) throw new Error(`Unknown connector "${id}"`);
|
|
991
|
+
if (connector.staticTools) return connector.staticTools;
|
|
992
|
+
if (!requestScope) {
|
|
993
|
+
return this.loadTools(id, baseUrl, requestScope, callOptions);
|
|
994
|
+
}
|
|
995
|
+
|
|
996
|
+
let loads = this.requestCatalogLoads.get(requestScope);
|
|
997
|
+
if (!loads) {
|
|
998
|
+
loads = new Map();
|
|
999
|
+
this.requestCatalogLoads.set(requestScope, loads);
|
|
1000
|
+
}
|
|
1001
|
+
const existing = loads.get(id);
|
|
1002
|
+
if (existing) return existing;
|
|
1003
|
+
const loading = this.loadTools(id, baseUrl, requestScope, callOptions);
|
|
1004
|
+
loads.set(id, loading);
|
|
1005
|
+
try {
|
|
1006
|
+
return await loading;
|
|
1007
|
+
} finally {
|
|
1008
|
+
if (loads.get(id) === loading) loads.delete(id);
|
|
1009
|
+
if (loads.size === 0) this.requestCatalogLoads.delete(requestScope);
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
|
|
682
1013
|
/** Return a cached catalog without performing storage or network I/O. */
|
|
683
1014
|
peekTools(id: string): ToolDef[] | undefined {
|
|
684
1015
|
const connector = this.connectors.get(id);
|
|
@@ -699,83 +1030,30 @@ export class Registry implements RegistryView {
|
|
|
699
1030
|
return this.health.get(id);
|
|
700
1031
|
}
|
|
701
1032
|
|
|
702
|
-
/**
|
|
703
|
-
* Whether ANY view of this deployment has seen a successful call to `id` —
|
|
704
|
-
* a bare boolean, never the observation. Connector liveness (reachable,
|
|
705
|
-
* credentials still valid) is a deployment-level fact, not a per-view one, so
|
|
706
|
-
* `list_connectors` may classify a connector as ok/unknown from it. The
|
|
707
|
-
* observation itself — `lastError` above all, which names the tool that
|
|
708
|
-
* failed — stays strictly per view.
|
|
709
|
-
*/
|
|
1033
|
+
/** Whether this deployment has seen a successful call to `id`. */
|
|
710
1034
|
hasObservedSuccess(id: string): boolean {
|
|
711
1035
|
return this.observedSuccessAt(id) !== undefined;
|
|
712
1036
|
}
|
|
713
1037
|
|
|
714
|
-
/**
|
|
715
|
-
* The timestamp behind `hasObservedSuccess`, on the same deployment-wide
|
|
716
|
-
* terms and for the same reason: it says only *when* the connector last
|
|
717
|
-
* answered, never what was called or what failed. Credential health reads it
|
|
718
|
-
* to decide whether a failed verdict has been overtaken by real traffic.
|
|
719
|
-
*/
|
|
1038
|
+
/** The timestamp behind `hasObservedSuccess`. */
|
|
720
1039
|
observedSuccessAt(id: string): string | undefined {
|
|
721
1040
|
return this.health.get(id)?.lastSuccessAt;
|
|
722
1041
|
}
|
|
723
1042
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
): Promise<void> {
|
|
739
|
-
return this.credentialHealth.record(id, record);
|
|
740
|
-
}
|
|
741
|
-
|
|
742
|
-
/**
|
|
743
|
-
* Check stored downstream credentials now and return one outcome per
|
|
744
|
-
* connector considered. THE operator-facing entry point behind
|
|
745
|
-
* `Connecta.checkCredentials()`: wire it to a Worker cron trigger or a Node
|
|
746
|
-
* interval. Never rejects; connectors checked recently are reported as
|
|
747
|
-
* `fresh` unless `force` is set.
|
|
748
|
-
*/
|
|
749
|
-
checkCredentialHealth(
|
|
750
|
-
baseUrl: string,
|
|
751
|
-
opts?: CredentialCheckOptions,
|
|
752
|
-
defer?: DeferredWork,
|
|
753
|
-
): Promise<CredentialCheckResult[]> {
|
|
754
|
-
return this.credentialHealth.check(baseUrl, opts, defer);
|
|
755
|
-
}
|
|
756
|
-
|
|
757
|
-
/**
|
|
758
|
-
* The traffic-triggered sweep: a promise for the caller to defer (Workers:
|
|
759
|
-
* `ctx.waitUntil`), or `undefined` when nothing is due — which is the common
|
|
760
|
-
* case and costs no I/O. Called by the server after an authenticated request;
|
|
761
|
-
* the checker owns the rate limiting.
|
|
762
|
-
*/
|
|
763
|
-
sweepCredentialHealthIfDue(
|
|
764
|
-
baseUrl: string,
|
|
765
|
-
defer?: DeferredWork,
|
|
766
|
-
): Promise<CredentialCheckResult[]> | undefined {
|
|
767
|
-
return this.credentialHealth.sweepIfDue(baseUrl, defer);
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
/**
|
|
771
|
-
* Drop a connector's liveness verdict, because its credential just changed
|
|
772
|
-
* under us (OAuth callback completed, credential stored or removed on
|
|
773
|
-
* /credentials). A
|
|
774
|
-
* stale `auth_required` must not outlive the re-authorization that fixed it —
|
|
775
|
-
* that is the difference between recovery working and needing a restart.
|
|
776
|
-
*/
|
|
777
|
-
clearCredentialHealth(id: string): Promise<void> {
|
|
778
|
-
return this.credentialHealth.clear(id);
|
|
1043
|
+
async credentialDriftFor(id: string): Promise<string | undefined> {
|
|
1044
|
+
const credential = this.connectors.get(id)?.credential;
|
|
1045
|
+
const vault = this.opts.credentialVault;
|
|
1046
|
+
if (!credential || !vault) return undefined;
|
|
1047
|
+
try {
|
|
1048
|
+
const values = await vault.getAll(id);
|
|
1049
|
+
const shape = storedCredentialShape(credential, values);
|
|
1050
|
+
return shape.state === "mismatch" ? shape.message : undefined;
|
|
1051
|
+
} catch (error) {
|
|
1052
|
+
this.opts.logger.warn(
|
|
1053
|
+
`[connecta] connector "${id}" credential shape read failed: ${msg(error)}`,
|
|
1054
|
+
);
|
|
1055
|
+
return undefined;
|
|
1056
|
+
}
|
|
779
1057
|
}
|
|
780
1058
|
|
|
781
1059
|
/** Best-effort connector status for list_connectors. */
|
|
@@ -803,291 +1081,33 @@ export class Registry implements RegistryView {
|
|
|
803
1081
|
}
|
|
804
1082
|
}
|
|
805
1083
|
|
|
806
|
-
|
|
807
|
-
invalidate(id: string): void {
|
|
1084
|
+
private markCatalogInvalid(id: string): void {
|
|
808
1085
|
this.advanceCatalogGeneration(id);
|
|
809
1086
|
this.cache.delete(id);
|
|
810
1087
|
this.invalidated.add(id);
|
|
811
|
-
if (this.persistToolCatalog) {
|
|
812
|
-
void this.enqueueCatalogMutation(id, async () => {
|
|
813
|
-
try {
|
|
814
|
-
await this.opts.storage.delete(this.catalogKey(id));
|
|
815
|
-
} catch (err) {
|
|
816
|
-
this.opts.logger.warn(
|
|
817
|
-
`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`,
|
|
818
|
-
);
|
|
819
|
-
}
|
|
820
|
-
});
|
|
821
|
-
}
|
|
822
|
-
}
|
|
823
|
-
|
|
824
|
-
/** Drop both in-memory and persisted tool catalogs. */
|
|
825
|
-
async invalidateStored(id: string): Promise<void> {
|
|
826
|
-
this.advanceCatalogGeneration(id);
|
|
827
|
-
this.cache.delete(id);
|
|
828
|
-
this.invalidated.add(id);
|
|
829
|
-
if (this.persistToolCatalog) {
|
|
830
|
-
await this.enqueueCatalogMutation(id, async () => {
|
|
831
|
-
try {
|
|
832
|
-
await this.opts.storage.delete(this.catalogKey(id));
|
|
833
|
-
} catch (err) {
|
|
834
|
-
this.opts.logger.warn(
|
|
835
|
-
`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`,
|
|
836
|
-
);
|
|
837
|
-
}
|
|
838
|
-
});
|
|
839
|
-
}
|
|
840
1088
|
}
|
|
841
|
-
}
|
|
842
1089
|
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
/** The long-lived log one toolkit records into; created on first use. */
|
|
855
|
-
function toolkitHealthLog(base: Registry, toolkitName: string): HealthLog {
|
|
856
|
-
let logs = toolkitHealthLogs.get(base);
|
|
857
|
-
if (!logs) {
|
|
858
|
-
logs = new Map<string, HealthLog>();
|
|
859
|
-
toolkitHealthLogs.set(base, logs);
|
|
860
|
-
}
|
|
861
|
-
let log = logs.get(toolkitName);
|
|
862
|
-
if (!log) {
|
|
863
|
-
log = new HealthLog();
|
|
864
|
-
logs.set(toolkitName, log);
|
|
865
|
-
}
|
|
866
|
-
return log;
|
|
867
|
-
}
|
|
868
|
-
|
|
869
|
-
/**
|
|
870
|
-
* THE toolkit enforcement point (issue #22).
|
|
871
|
-
*
|
|
872
|
-
* A filtered VIEW of one long-lived `Registry`: same connectors, same tool
|
|
873
|
-
* caches, same health — narrowed to the connectors and tool addresses one
|
|
874
|
-
* toolkit selects. `serveMcp` builds it once per scoped connection and hands it
|
|
875
|
-
* to `registerMetaTools`/`registerExecuteTool`, so `list_connectors`,
|
|
876
|
-
* `search_tools`, `describe_tools`, `call_tool`, `call_destructive_tool`,
|
|
877
|
-
* `batch_call`, `authorize_connector`, `get_result`, `skills`, and the
|
|
878
|
-
* `execute_code` host bridge all inherit the boundary from here instead of
|
|
879
|
-
* re-implementing nine checks.
|
|
880
|
-
*
|
|
881
|
-
* Two invariants make it reviewable:
|
|
882
|
-
*
|
|
883
|
-
* 1. **Indistinguishability.** Out-of-scope input fails through the SAME code
|
|
884
|
-
* path, with the same error class and message, as input naming something
|
|
885
|
-
* that does not exist at all. An out-of-scope CONNECTOR disappears from
|
|
886
|
-
* `resolveAddress`/`getConnector` (→ `Unknown address` / `Unknown
|
|
887
|
-
* connector`); an out-of-scope TOOL disappears from the catalog this view
|
|
888
|
-
* returns (→ `Unknown tool "<t>" on connector "<c>"`), which is exactly the
|
|
889
|
-
* error a misspelled tool name already produced. Scoping deliberately does
|
|
890
|
-
* NOT reject at `resolveAddress` for tools: that would answer with the
|
|
891
|
-
* connector-level message and make "hidden here" distinguishable from
|
|
892
|
-
* "never existed".
|
|
893
|
-
* 2. **It filters views, never state.** Every read delegates to the shared
|
|
894
|
-
* registry and filters the returned array, so the tool cache, the persisted
|
|
895
|
-
* catalog, and the health map stay whole and shared across scopes.
|
|
896
|
-
*/
|
|
897
|
-
export class ScopedRegistry implements RegistryView {
|
|
898
|
-
/** This toolkit's own health observations — see HealthLog. */
|
|
899
|
-
private readonly health: HealthLog;
|
|
900
|
-
|
|
901
|
-
constructor(
|
|
902
|
-
private readonly base: Registry,
|
|
903
|
-
private readonly toolkit: Toolkit,
|
|
904
|
-
) {
|
|
905
|
-
this.health = toolkitHealthLog(base, toolkit.name);
|
|
906
|
-
}
|
|
907
|
-
|
|
908
|
-
get maxResultBytes(): number {
|
|
909
|
-
return this.base.maxResultBytes;
|
|
910
|
-
}
|
|
911
|
-
|
|
912
|
-
get maxBatchResultBytes(): number {
|
|
913
|
-
return this.base.maxBatchResultBytes;
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
/** In scope AND actually registered. */
|
|
917
|
-
private visible(id: string): boolean {
|
|
918
|
-
return (
|
|
919
|
-
this.toolkit.hasConnector(id) && this.base.getConnector(id) !== undefined
|
|
920
|
-
);
|
|
921
|
-
}
|
|
922
|
-
|
|
923
|
-
/** Byte-identical to what the unscoped registry throws for an unknown id. */
|
|
924
|
-
private unknownConnector(id: string): Error {
|
|
925
|
-
return new Error(`Unknown connector "${id}"`);
|
|
926
|
-
}
|
|
927
|
-
|
|
928
|
-
private inScopeTools(id: string, tools: ToolDef[]): ToolDef[] {
|
|
929
|
-
return tools.filter((tool) => this.toolkit.hasTool(id, tool.name));
|
|
930
|
-
}
|
|
931
|
-
|
|
932
|
-
listConnectors(): Connector[] {
|
|
933
|
-
return this.base
|
|
934
|
-
.listConnectors()
|
|
935
|
-
.filter((connector) => this.toolkit.hasConnector(connector.id));
|
|
936
|
-
}
|
|
937
|
-
|
|
938
|
-
getConnector(id: string): Connector | undefined {
|
|
939
|
-
return this.visible(id) ? this.base.getConnector(id) : undefined;
|
|
940
|
-
}
|
|
941
|
-
|
|
942
|
-
resolveAddress(
|
|
943
|
-
address: string,
|
|
944
|
-
): { connector: Connector; toolName: string } | null {
|
|
945
|
-
const resolved = this.base.resolveAddress(address);
|
|
946
|
-
if (!resolved) return null;
|
|
947
|
-
// Connector-level only — see invariant 1 above.
|
|
948
|
-
return this.toolkit.hasConnector(resolved.connector.id) ? resolved : null;
|
|
949
|
-
}
|
|
950
|
-
|
|
951
|
-
async getTools(
|
|
952
|
-
id: string,
|
|
953
|
-
baseUrl: string,
|
|
954
|
-
requestScope?: object,
|
|
955
|
-
callOptions: ConnectorOperationOptions = {},
|
|
956
|
-
): Promise<ToolDef[]> {
|
|
957
|
-
if (!this.visible(id)) throw this.unknownConnector(id);
|
|
958
|
-
return this.inScopeTools(
|
|
959
|
-
id,
|
|
960
|
-
await this.base.getTools(id, baseUrl, requestScope, callOptions),
|
|
961
|
-
);
|
|
962
|
-
}
|
|
963
|
-
|
|
964
|
-
async refreshTools(
|
|
965
|
-
id: string,
|
|
966
|
-
baseUrl: string,
|
|
967
|
-
requestScope?: object,
|
|
968
|
-
callOptions: ConnectorOperationOptions = {},
|
|
969
|
-
): Promise<ToolDef[]> {
|
|
970
|
-
if (!this.visible(id)) throw this.unknownConnector(id);
|
|
971
|
-
return this.inScopeTools(
|
|
972
|
-
id,
|
|
973
|
-
await this.base.refreshTools(id, baseUrl, requestScope, callOptions),
|
|
974
|
-
);
|
|
975
|
-
}
|
|
976
|
-
|
|
977
|
-
peekTools(id: string): ToolDef[] | undefined {
|
|
978
|
-
if (!this.visible(id)) return undefined;
|
|
979
|
-
const tools = this.base.peekTools(id);
|
|
980
|
-
return tools ? this.inScopeTools(id, tools) : undefined;
|
|
981
|
-
}
|
|
982
|
-
|
|
983
|
-
contextFor(
|
|
984
|
-
id: string,
|
|
985
|
-
baseUrl: string,
|
|
986
|
-
requestScope: object = {},
|
|
987
|
-
callOptions: ConnectorOperationOptions = {},
|
|
988
|
-
): ConnectorContext {
|
|
989
|
-
// Unreachable through the meta-tools (they resolve first), so a throw here
|
|
990
|
-
// is a loud backstop rather than a silent grant of connector storage and
|
|
991
|
-
// credentials to a scope that may not see the connector.
|
|
992
|
-
if (!this.visible(id)) throw this.unknownConnector(id);
|
|
993
|
-
return this.base.contextFor(id, baseUrl, requestScope, callOptions);
|
|
994
|
-
}
|
|
995
|
-
|
|
996
|
-
admitCall(
|
|
997
|
-
id: string,
|
|
998
|
-
input: { toolName: string; args: unknown; signal?: AbortSignal },
|
|
999
|
-
): Promise<CallAdmissionPermit> {
|
|
1000
|
-
if (!this.visible(id)) return Promise.reject(this.unknownConnector(id));
|
|
1001
|
-
return this.base.admitCall(id, input);
|
|
1002
|
-
}
|
|
1003
|
-
|
|
1004
|
-
/**
|
|
1005
|
-
* Stashed oversized results are bound to the scope that produced them: a
|
|
1006
|
-
* scoped session cannot page a result it could not have produced, and an id
|
|
1007
|
-
* from another scope reads back as the ordinary "Unknown or expired result
|
|
1008
|
-
* id". Unscoped sessions keep the historical `results:` prefix untouched.
|
|
1009
|
-
*/
|
|
1010
|
-
resultsStorage(): KVStorage {
|
|
1011
|
-
return namespaced(
|
|
1012
|
-
this.base.resultsStorage(),
|
|
1013
|
-
`toolkit:${this.toolkit.name}:`,
|
|
1014
|
-
);
|
|
1015
|
-
}
|
|
1016
|
-
|
|
1017
|
-
// Outcomes are recorded twice on purpose: the deployment-wide log keeps the
|
|
1018
|
-
// operator surfaces complete, while the toolkit's own log is the ONLY one
|
|
1019
|
-
// this view reads back — so `list_connectors` here never reports a sibling
|
|
1020
|
-
// toolkit's failures, or the tool names their error strings carry.
|
|
1021
|
-
recordSuccess(id: string, latencyMs: number): void {
|
|
1022
|
-
if (!this.visible(id)) return;
|
|
1023
|
-
this.base.recordSuccess(id, latencyMs);
|
|
1024
|
-
this.health.recordSuccess(id, latencyMs);
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
recordFailure(id: string, latencyMs: number, error: unknown): void {
|
|
1028
|
-
if (!this.visible(id)) return;
|
|
1029
|
-
this.base.recordFailure(id, latencyMs, error);
|
|
1030
|
-
this.health.recordFailure(id, latencyMs, error);
|
|
1031
|
-
}
|
|
1032
|
-
|
|
1033
|
-
healthFor(id: string): HealthObservation | undefined {
|
|
1034
|
-
return this.visible(id) ? this.health.get(id) : undefined;
|
|
1035
|
-
}
|
|
1036
|
-
|
|
1037
|
-
/**
|
|
1038
|
-
* Deliberately NOT per view: whether the connector has ever answered is a
|
|
1039
|
-
* fact about the connector, not about a team's traffic, and withholding it
|
|
1040
|
-
* would report every remote connector as "unknown" to a scoped session that
|
|
1041
|
-
* has not called it yet. It carries no tool name, error text, or count — the
|
|
1042
|
-
* per-view isolation of those, above, is unchanged.
|
|
1043
|
-
*/
|
|
1044
|
-
hasObservedSuccess(id: string): boolean {
|
|
1045
|
-
return this.visible(id) ? this.base.hasObservedSuccess(id) : false;
|
|
1046
|
-
}
|
|
1047
|
-
|
|
1048
|
-
/** Deployment-wide for the same reason as `hasObservedSuccess` above. */
|
|
1049
|
-
observedSuccessAt(id: string): string | undefined {
|
|
1050
|
-
return this.visible(id) ? this.base.observedSuccessAt(id) : undefined;
|
|
1051
|
-
}
|
|
1052
|
-
|
|
1053
|
-
/**
|
|
1054
|
-
* Also deliberately NOT per view, for the same reason as
|
|
1055
|
-
* `hasObservedSuccess`: whether the credential connecta stores for a connector
|
|
1056
|
-
* still works is a fact about the deployment's credential, not about a team's
|
|
1057
|
-
* traffic. Withholding it would leave a scoped session unable to see that the
|
|
1058
|
-
* connector it shares needs re-authorization. The verdict carries the
|
|
1059
|
-
* connector's own connector-level reason — never a tool name — so the per-view
|
|
1060
|
-
* isolation of `lastError` above is unchanged.
|
|
1061
|
-
*/
|
|
1062
|
-
credentialHealthFor(id: string): Promise<CredentialHealthRecord | undefined> {
|
|
1063
|
-
return this.visible(id)
|
|
1064
|
-
? this.base.credentialHealthFor(id)
|
|
1065
|
-
: Promise.resolve(undefined);
|
|
1066
|
-
}
|
|
1067
|
-
|
|
1068
|
-
async recordCredentialHealth(
|
|
1069
|
-
id: string,
|
|
1070
|
-
record: CredentialHealthRecord,
|
|
1071
|
-
): Promise<void> {
|
|
1072
|
-
if (!this.visible(id)) return;
|
|
1073
|
-
await this.base.recordCredentialHealth(id, record);
|
|
1090
|
+
private deleteStoredCatalog(id: string): Promise<void> {
|
|
1091
|
+
return this.enqueueCatalogMutation(id, async () => {
|
|
1092
|
+
try {
|
|
1093
|
+
await this.deleteCatalog(id);
|
|
1094
|
+
} catch (err) {
|
|
1095
|
+
this.opts.logger.warn(
|
|
1096
|
+
`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`,
|
|
1097
|
+
);
|
|
1098
|
+
}
|
|
1099
|
+
});
|
|
1074
1100
|
}
|
|
1075
1101
|
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
callOptions: { signal?: AbortSignal; timeoutMs?: number } = {},
|
|
1081
|
-
): Promise<ConnectorStatus> {
|
|
1082
|
-
// Same shape the unscoped registry returns for an unregistered id.
|
|
1083
|
-
if (!this.visible(id)) {
|
|
1084
|
-
return { state: "error", message: "Unknown connector" };
|
|
1085
|
-
}
|
|
1086
|
-
return this.base.statusFor(id, baseUrl, requestScope, callOptions);
|
|
1102
|
+
/** Drop a connector's cached tool list (e.g. after auth completes). */
|
|
1103
|
+
invalidate(id: string): void {
|
|
1104
|
+
this.markCatalogInvalid(id);
|
|
1105
|
+
if (this.persistToolCatalog) void this.deleteStoredCatalog(id);
|
|
1087
1106
|
}
|
|
1088
1107
|
|
|
1108
|
+
/** Drop both in-memory and persisted tool catalogs. */
|
|
1089
1109
|
async invalidateStored(id: string): Promise<void> {
|
|
1090
|
-
|
|
1091
|
-
await this.
|
|
1110
|
+
this.markCatalogInvalid(id);
|
|
1111
|
+
if (this.persistToolCatalog) await this.deleteStoredCatalog(id);
|
|
1092
1112
|
}
|
|
1093
1113
|
}
|