@zackbart/connecta 0.7.9 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/CHANGELOG.md +135 -0
  2. package/README.md +47 -148
  3. package/dist/activity.d.ts +0 -8
  4. package/dist/activity.d.ts.map +1 -1
  5. package/dist/activity.js +0 -1
  6. package/dist/activity.js.map +1 -1
  7. package/dist/auth/bearer.d.ts +1 -7
  8. package/dist/auth/bearer.d.ts.map +1 -1
  9. package/dist/auth/bearer.js +3 -24
  10. package/dist/auth/bearer.js.map +1 -1
  11. package/dist/auth/clerk.d.ts +2 -11
  12. package/dist/auth/clerk.d.ts.map +1 -1
  13. package/dist/auth/clerk.js +31 -46
  14. package/dist/auth/clerk.js.map +1 -1
  15. package/dist/call-admission.d.ts.map +1 -1
  16. package/dist/call-admission.js +9 -4
  17. package/dist/call-admission.js.map +1 -1
  18. package/dist/catalog-fingerprint.d.ts +20 -0
  19. package/dist/catalog-fingerprint.d.ts.map +1 -0
  20. package/dist/catalog-fingerprint.js +34 -0
  21. package/dist/catalog-fingerprint.js.map +1 -0
  22. package/dist/catalog-limits.d.ts +10 -0
  23. package/dist/catalog-limits.d.ts.map +1 -0
  24. package/dist/catalog-limits.js +10 -0
  25. package/dist/catalog-limits.js.map +1 -0
  26. package/dist/catalog-service.d.ts +137 -0
  27. package/dist/catalog-service.d.ts.map +1 -0
  28. package/dist/catalog-service.js +401 -0
  29. package/dist/catalog-service.js.map +1 -0
  30. package/dist/catalog.d.ts +6 -0
  31. package/dist/catalog.d.ts.map +1 -1
  32. package/dist/catalog.js +53 -2
  33. package/dist/catalog.js.map +1 -1
  34. package/dist/concurrency.d.ts +4 -0
  35. package/dist/concurrency.d.ts.map +1 -0
  36. package/dist/concurrency.js +31 -0
  37. package/dist/concurrency.js.map +1 -0
  38. package/dist/connectors/api.d.ts.map +1 -1
  39. package/dist/connectors/api.js +22 -12
  40. package/dist/connectors/api.js.map +1 -1
  41. package/dist/connectors/remote-mcp.d.ts.map +1 -1
  42. package/dist/connectors/remote-mcp.js +53 -41
  43. package/dist/connectors/remote-mcp.js.map +1 -1
  44. package/dist/credentials.d.ts +3 -2
  45. package/dist/credentials.d.ts.map +1 -1
  46. package/dist/errors.d.ts +19 -1
  47. package/dist/errors.d.ts.map +1 -1
  48. package/dist/errors.js.map +1 -1
  49. package/dist/execute.d.ts +10 -16
  50. package/dist/execute.d.ts.map +1 -1
  51. package/dist/execute.js +143 -330
  52. package/dist/execute.js.map +1 -1
  53. package/dist/executor-result.d.ts +0 -3
  54. package/dist/executor-result.d.ts.map +1 -1
  55. package/dist/executor-result.js +3 -3
  56. package/dist/executor-result.js.map +1 -1
  57. package/dist/executors/quickjs-child.js +3 -3
  58. package/dist/executors/quickjs-child.js.map +1 -1
  59. package/dist/executors/quickjs-protocol.d.ts +4 -1
  60. package/dist/executors/quickjs-protocol.d.ts.map +1 -1
  61. package/dist/executors/quickjs-runtime.d.ts.map +1 -1
  62. package/dist/executors/quickjs-runtime.js +7 -2
  63. package/dist/executors/quickjs-runtime.js.map +1 -1
  64. package/dist/executors/quickjs.d.ts.map +1 -1
  65. package/dist/executors/quickjs.js +4 -1
  66. package/dist/executors/quickjs.js.map +1 -1
  67. package/dist/favicon.js +2 -2
  68. package/dist/favicon.js.map +1 -1
  69. package/dist/index.d.ts +8 -82
  70. package/dist/index.d.ts.map +1 -1
  71. package/dist/index.js +58 -104
  72. package/dist/index.js.map +1 -1
  73. package/dist/invocation.d.ts +101 -0
  74. package/dist/invocation.d.ts.map +1 -0
  75. package/dist/invocation.js +405 -0
  76. package/dist/invocation.js.map +1 -0
  77. package/dist/meta-tools.d.ts +9 -95
  78. package/dist/meta-tools.d.ts.map +1 -1
  79. package/dist/meta-tools.js +235 -725
  80. package/dist/meta-tools.js.map +1 -1
  81. package/dist/operator-ui/generated.d.ts +3 -0
  82. package/dist/operator-ui/generated.d.ts.map +1 -0
  83. package/dist/operator-ui/generated.js +5 -0
  84. package/dist/operator-ui/generated.js.map +1 -0
  85. package/dist/operator-ui/model.d.ts +73 -0
  86. package/dist/operator-ui/model.d.ts.map +1 -0
  87. package/dist/operator-ui/model.js +28 -0
  88. package/dist/operator-ui/model.js.map +1 -0
  89. package/dist/registry.d.ts +33 -184
  90. package/dist/registry.d.ts.map +1 -1
  91. package/dist/registry.js +345 -330
  92. package/dist/registry.js.map +1 -1
  93. package/dist/retired-toolkits.d.ts +3 -0
  94. package/dist/retired-toolkits.d.ts.map +1 -0
  95. package/dist/retired-toolkits.js +9 -0
  96. package/dist/retired-toolkits.js.map +1 -0
  97. package/dist/routes/activity.d.ts +3 -0
  98. package/dist/routes/activity.d.ts.map +1 -0
  99. package/dist/routes/activity.js +170 -0
  100. package/dist/routes/activity.js.map +1 -0
  101. package/dist/routes/credentials.d.ts +3 -0
  102. package/dist/routes/credentials.d.ts.map +1 -0
  103. package/dist/routes/credentials.js +170 -0
  104. package/dist/routes/credentials.js.map +1 -0
  105. package/dist/routes/mcp.d.ts +8 -0
  106. package/dist/routes/mcp.d.ts.map +1 -0
  107. package/dist/routes/mcp.js +270 -0
  108. package/dist/routes/mcp.js.map +1 -0
  109. package/dist/routes/oauth.d.ts +4 -0
  110. package/dist/routes/oauth.d.ts.map +1 -0
  111. package/dist/routes/oauth.js +263 -0
  112. package/dist/routes/oauth.js.map +1 -0
  113. package/dist/routes/shared.d.ts +76 -0
  114. package/dist/routes/shared.d.ts.map +1 -0
  115. package/dist/routes/shared.js +140 -0
  116. package/dist/routes/shared.js.map +1 -0
  117. package/dist/routes/ui.d.ts +3 -0
  118. package/dist/routes/ui.d.ts.map +1 -0
  119. package/dist/routes/ui.js +116 -0
  120. package/dist/routes/ui.js.map +1 -0
  121. package/dist/server.d.ts +9 -40
  122. package/dist/server.d.ts.map +1 -1
  123. package/dist/server.js +62 -1322
  124. package/dist/server.js.map +1 -1
  125. package/dist/skills.d.ts +3 -21
  126. package/dist/skills.d.ts.map +1 -1
  127. package/dist/skills.js +15 -43
  128. package/dist/skills.js.map +1 -1
  129. package/dist/storage/file.d.ts.map +1 -1
  130. package/dist/storage/file.js +3 -1
  131. package/dist/storage/file.js.map +1 -1
  132. package/dist/storage/memory.d.ts.map +1 -1
  133. package/dist/storage/memory.js +3 -1
  134. package/dist/storage/memory.js.map +1 -1
  135. package/dist/timeout.d.ts.map +1 -1
  136. package/dist/timeout.js +2 -4
  137. package/dist/timeout.js.map +1 -1
  138. package/dist/types.d.ts +9 -59
  139. package/dist/types.d.ts.map +1 -1
  140. package/dist/ui.d.ts +3 -94
  141. package/dist/ui.d.ts.map +1 -1
  142. package/dist/ui.js +30 -1427
  143. package/dist/ui.js.map +1 -1
  144. package/dist/version.d.ts +1 -1
  145. package/dist/version.js +1 -1
  146. package/package.json +15 -8
  147. package/src/activity.ts +0 -9
  148. package/src/auth/bearer.ts +4 -33
  149. package/src/auth/clerk.ts +35 -48
  150. package/src/call-admission.ts +8 -6
  151. package/src/catalog-fingerprint.ts +52 -0
  152. package/src/catalog-limits.ts +11 -0
  153. package/src/catalog-service.ts +601 -0
  154. package/src/catalog.ts +57 -2
  155. package/src/concurrency.ts +39 -0
  156. package/src/connectors/api.ts +22 -12
  157. package/src/connectors/remote-mcp.ts +63 -45
  158. package/src/credentials.ts +2 -2
  159. package/src/errors.ts +21 -1
  160. package/src/execute.ts +193 -423
  161. package/src/executor-result.ts +3 -3
  162. package/src/executors/quickjs-child.ts +8 -4
  163. package/src/executors/quickjs-protocol.ts +1 -1
  164. package/src/executors/quickjs-runtime.ts +5 -1
  165. package/src/executors/quickjs.ts +11 -8
  166. package/src/favicon.ts +2 -2
  167. package/src/index.ts +70 -222
  168. package/src/invocation.ts +580 -0
  169. package/src/meta-tools.ts +393 -985
  170. package/src/operator-ui/browser.css +360 -0
  171. package/src/operator-ui/browser.ts +987 -0
  172. package/src/operator-ui/generated.ts +4 -0
  173. package/src/operator-ui/model.ts +105 -0
  174. package/src/registry.ts +459 -439
  175. package/src/retired-toolkits.ts +14 -0
  176. package/src/routes/activity.ts +215 -0
  177. package/src/routes/credentials.ts +254 -0
  178. package/src/routes/mcp.ts +339 -0
  179. package/src/routes/oauth.ts +322 -0
  180. package/src/routes/shared.ts +240 -0
  181. package/src/routes/ui.ts +143 -0
  182. package/src/server.ts +64 -1731
  183. package/src/skills.ts +15 -43
  184. package/src/storage/file.ts +3 -1
  185. package/src/storage/memory.ts +3 -1
  186. package/src/timeout.ts +2 -4
  187. package/src/types.ts +8 -60
  188. package/src/ui.ts +60 -1538
  189. package/src/version.ts +1 -1
  190. package/dist/credential-health.d.ts +0 -227
  191. package/dist/credential-health.d.ts.map +0 -1
  192. package/dist/credential-health.js +0 -606
  193. package/dist/credential-health.js.map +0 -1
  194. package/dist/toolkits.d.ts +0 -146
  195. package/dist/toolkits.d.ts.map +0 -1
  196. package/dist/toolkits.js +0 -322
  197. package/dist/toolkits.js.map +0 -1
  198. package/src/credential-health.ts +0 -832
  199. package/src/toolkits.ts +0 -461
package/dist/registry.js CHANGED
@@ -1,14 +1,30 @@
1
1
  import { storedCredentialShape, } from "./credentials.js";
2
2
  import { ConnectorCallError } from "./errors.js";
3
- import { CredentialHealthChecker, } from "./credential-health.js";
4
3
  import { ConnectorCallAdmissionController, } from "./call-admission.js";
5
- import { splitAddress } from "./toolkits.js";
4
+ import { fingerprintSerializedCatalog, snapshotCatalog, } from "./catalog-fingerprint.js";
5
+ import { MAX_CATALOG_CHUNK_BYTES, MAX_CATALOG_TOOLS, MAX_SERIALIZED_CATALOG_BYTES, } from "./catalog-limits.js";
6
+ import { mapSettledWithConcurrency } from "./concurrency.js";
6
7
  const ID_RE = /^[a-z0-9_-]+$/;
7
8
  const DEFAULT_TTL_SECONDS = 300;
8
9
  const DEFAULT_STALE_SECONDS = 3600;
9
- export const DEFAULT_MAX_RESULT_BYTES = 50_000;
10
+ const CATALOG_CHUNK_TTL_GRACE_SECONDS = 300;
11
+ const DEFAULT_MAX_RESULT_BYTES = 50_000;
12
+ const encoder = new TextEncoder();
10
13
  /** Independent final-envelope boundary for `batch_call`. */
11
- export const DEFAULT_MAX_BATCH_RESULT_BYTES = 100_000;
14
+ const DEFAULT_MAX_BATCH_RESULT_BYTES = 100_000;
15
+ /**
16
+ * Split `"<connectorId>.<toolName>"` on the first dot. Connector ids contain
17
+ * no dots, so a downstream tool name may.
18
+ */
19
+ function splitAddress(address) {
20
+ const dot = address.indexOf(".");
21
+ if (dot <= 0 || dot === address.length - 1)
22
+ return null;
23
+ return {
24
+ connectorId: address.slice(0, dot),
25
+ toolName: address.slice(dot + 1),
26
+ };
27
+ }
12
28
  /**
13
29
  * Smallest accepted inline-result cap. One byte is pathological but harmless:
14
30
  * `alignEndToCharBoundary` widens a window narrower than the codepoint at the
@@ -44,28 +60,22 @@ export function resolveMaxResultBytes(value, inherited) {
44
60
  ? value
45
61
  : inherited;
46
62
  }
63
+ const CATALOG_CHUNK_IO_CONCURRENCY = 4;
47
64
  /**
48
- * Recent real-call outcomes per connector, as observed by ONE view.
49
- *
50
- * The deployment keeps one log, and each toolkit keeps its own. That split is a
51
- * scope boundary, not bookkeeping: `lastError` is a downstream error string
52
- * that routinely names the tool that failed, and `list_connectors` returns the
53
- * observation verbatim. Sharing one log would let a toolkit read back the tools
54
- * and failures of a sibling toolkit through a connector they happen to share —
55
- * and let a sibling's failures flip this view's reported status. A scoped view
56
- * reads back only the calls it made itself.
65
+ * Recent real-call outcomes per connector.
57
66
  */
58
- export class HealthLog {
67
+ class HealthLog {
59
68
  observations = new Map();
60
69
  recordSuccess(id, latencyMs) {
61
70
  const previous = this.observations.get(id);
62
- this.observations.set(id, {
71
+ const observation = {
63
72
  ...previous,
64
73
  lastSuccessAt: new Date().toISOString(),
65
74
  lastLatencyMs: latencyMs,
66
75
  consecutiveFailures: 0,
67
- lastError: undefined,
68
- });
76
+ };
77
+ delete observation.lastError;
78
+ this.observations.set(id, observation);
69
79
  }
70
80
  recordFailure(id, latencyMs, error) {
71
81
  const previous = this.observations.get(id);
@@ -107,6 +117,8 @@ export class Registry {
107
117
  catalogGenerations = new Map();
108
118
  /** Serialize persisted catalog set/delete operations within this isolate. */
109
119
  catalogMutations = new Map();
120
+ /** Same-request cold loads share one promise without retaining the request. */
121
+ requestCatalogLoads = new WeakMap();
110
122
  /** Deployment-wide observations — every call, whatever view made it. */
111
123
  health = new HealthLog();
112
124
  ttlMs;
@@ -116,8 +128,6 @@ export class Registry {
116
128
  maxResultBytes;
117
129
  /** Final batch envelope cap threaded to the meta-tools. */
118
130
  maxBatchResultBytes;
119
- /** Proactive liveness checks over stored downstream credentials (issue #24). */
120
- credentialHealth;
121
131
  constructor(connectors, opts) {
122
132
  this.opts = opts;
123
133
  this.ttlMs =
@@ -141,14 +151,6 @@ export class Registry {
141
151
  }
142
152
  this.checkConventions(opts.logger);
143
153
  this.checkResultCaps(opts.logger, opts.maxResultBytes, opts.maxBatchResultBytes);
144
- this.credentialHealth = new CredentialHealthChecker({
145
- listConnectors: () => this.listConnectors(),
146
- getConnector: (id) => this.getConnector(id),
147
- contextFor: (id, baseUrl, requestScope) => this.contextFor(id, baseUrl, requestScope),
148
- storage: opts.storage,
149
- logger: opts.logger,
150
- credentialVault: opts.credentialVault,
151
- }, opts.credentialHealth);
152
154
  }
153
155
  /**
154
156
  * Warn once per unusable result cap, at construction time — the same
@@ -279,40 +281,236 @@ export class Registry {
279
281
  catalogKey(id) {
280
282
  return `catalog:${id}`;
281
283
  }
282
- validCatalog(raw) {
284
+ catalogChunkKey(id, revision, index) {
285
+ return `${this.catalogKey(id)}:chunk:${revision}:${index}`;
286
+ }
287
+ validLegacyCatalog(value) {
288
+ if (!value || typeof value !== "object")
289
+ return null;
290
+ const catalog = value;
291
+ if (!Array.isArray(catalog.tools) ||
292
+ typeof catalog.fetchedAt !== "number" ||
293
+ typeof catalog.expiresAt !== "number" ||
294
+ typeof catalog.staleUntil !== "number" ||
295
+ (catalog.fingerprint !== undefined &&
296
+ typeof catalog.fingerprint !== "string") ||
297
+ !this.validCatalogTools(catalog.tools)) {
298
+ return null;
299
+ }
300
+ return catalog;
301
+ }
302
+ validCatalogTools(value) {
303
+ return value.every((tool) => tool !== null &&
304
+ typeof tool === "object" &&
305
+ typeof tool.name === "string");
306
+ }
307
+ validCatalogManifest(value) {
308
+ if (!value || typeof value !== "object")
309
+ return null;
310
+ const manifest = value;
311
+ const maxChunks = Math.ceil(MAX_SERIALIZED_CATALOG_BYTES / MAX_CATALOG_CHUNK_BYTES) + 1;
312
+ if (manifest.version !== 2 ||
313
+ typeof manifest.revision !== "string" ||
314
+ !/^sha256:[0-9]{1,8}:[0-9a-f]{64}$/.test(manifest.revision) ||
315
+ !Number.isInteger(manifest.toolCount) ||
316
+ manifest.toolCount < 0 ||
317
+ manifest.toolCount > MAX_CATALOG_TOOLS ||
318
+ !Number.isInteger(manifest.byteCount) ||
319
+ manifest.byteCount < 2 ||
320
+ manifest.byteCount > MAX_SERIALIZED_CATALOG_BYTES ||
321
+ !manifest.revision.startsWith(`sha256:${manifest.byteCount}:`) ||
322
+ !Number.isInteger(manifest.chunkCount) ||
323
+ manifest.chunkCount < 1 ||
324
+ manifest.chunkCount > maxChunks ||
325
+ typeof manifest.fetchedAt !== "number" ||
326
+ typeof manifest.expiresAt !== "number" ||
327
+ typeof manifest.staleUntil !== "number") {
328
+ return null;
329
+ }
330
+ return manifest;
331
+ }
332
+ parseCatalogManifest(raw) {
333
+ if (!raw)
334
+ return null;
335
+ try {
336
+ return this.validCatalogManifest(JSON.parse(raw));
337
+ }
338
+ catch {
339
+ return null;
340
+ }
341
+ }
342
+ splitCatalogChunks(snapshot) {
343
+ const chunks = [];
344
+ const decoder = new TextDecoder("utf-8", {
345
+ fatal: true,
346
+ ignoreBOM: true,
347
+ });
348
+ let offset = 0;
349
+ while (offset < snapshot.serializedBytes.byteLength) {
350
+ let end = Math.min(offset + MAX_CATALOG_CHUNK_BYTES, snapshot.serializedBytes.byteLength);
351
+ // Move a boundary that landed inside a multibyte UTF-8 sequence back to
352
+ // the next character start. Every stored string then remains valid UTF-8.
353
+ while (end < snapshot.serializedBytes.byteLength &&
354
+ ((snapshot.serializedBytes[end] ?? 0) & 0xc0) === 0x80) {
355
+ end--;
356
+ }
357
+ chunks.push(decoder.decode(snapshot.serializedBytes.subarray(offset, end)));
358
+ offset = end;
359
+ }
360
+ return chunks;
361
+ }
362
+ async readCatalog(id, raw, now) {
283
363
  if (!raw)
284
364
  return null;
365
+ let parsed;
285
366
  try {
286
- const value = JSON.parse(raw);
287
- if (!Array.isArray(value.tools) ||
288
- typeof value.fetchedAt !== "number" ||
289
- typeof value.expiresAt !== "number" ||
290
- typeof value.staleUntil !== "number" ||
291
- !value.tools.every((tool) => tool !== null &&
292
- typeof tool === "object" &&
293
- typeof tool.name === "string")) {
367
+ parsed = JSON.parse(raw);
368
+ }
369
+ catch {
370
+ return null;
371
+ }
372
+ if (!parsed ||
373
+ typeof parsed !== "object" ||
374
+ parsed.version !== 2) {
375
+ const legacy = this.validLegacyCatalog(parsed);
376
+ if (!legacy || legacy.staleUntil <= now)
377
+ return null;
378
+ const snapshot = await snapshotCatalog(legacy.tools);
379
+ if (legacy.tools.length > MAX_CATALOG_TOOLS ||
380
+ snapshot.serializedBytes.byteLength > MAX_SERIALIZED_CATALOG_BYTES ||
381
+ (legacy.fingerprint !== undefined &&
382
+ legacy.fingerprint !== snapshot.fingerprint)) {
383
+ this.opts.logger.warn(`[connecta] connector "${id}" legacy catalog is oversized or has a fingerprint mismatch; ignoring persisted catalog.`);
384
+ return null;
385
+ }
386
+ return {
387
+ tools: legacy.tools,
388
+ fingerprint: snapshot.fingerprint,
389
+ fetchedAt: legacy.fetchedAt,
390
+ expiresAt: legacy.expiresAt,
391
+ staleUntil: legacy.staleUntil,
392
+ };
393
+ }
394
+ const manifest = this.validCatalogManifest(parsed);
395
+ if (!manifest) {
396
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog manifest is invalid; ignoring persisted catalog.`);
397
+ return null;
398
+ }
399
+ if (manifest.staleUntil <= now)
400
+ return null;
401
+ const chunkReads = await mapSettledWithConcurrency(Array.from({ length: manifest.chunkCount }, (_, index) => index), CATALOG_CHUNK_IO_CONCURRENCY, (index) => this.opts.storage.get(this.catalogChunkKey(id, manifest.revision, index)));
402
+ const chunks = [];
403
+ let chunkBytes = 0;
404
+ for (const [index, result] of chunkReads.entries()) {
405
+ if (result.status === "rejected")
406
+ throw result.reason;
407
+ const chunk = result.value;
408
+ if (chunk === null) {
409
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog chunk ${index + 1}/${manifest.chunkCount} is missing; ignoring persisted catalog.`);
294
410
  return null;
295
411
  }
296
- return value;
412
+ const byteLength = encoder.encode(chunk).byteLength;
413
+ chunkBytes += byteLength;
414
+ if (byteLength > MAX_CATALOG_CHUNK_BYTES ||
415
+ chunkBytes > manifest.byteCount) {
416
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog chunk bounds do not match its manifest; ignoring persisted catalog.`);
417
+ return null;
418
+ }
419
+ chunks.push(chunk);
420
+ }
421
+ const serializedTools = chunks.join("");
422
+ const stored = await fingerprintSerializedCatalog(serializedTools);
423
+ if (stored.byteLength !== manifest.byteCount ||
424
+ stored.fingerprint !== manifest.revision) {
425
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog fingerprint mismatch; ignoring persisted catalog.`);
426
+ return null;
427
+ }
428
+ let tools;
429
+ try {
430
+ tools = JSON.parse(serializedTools);
297
431
  }
298
432
  catch {
433
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog chunks are torn; ignoring persisted catalog.`);
299
434
  return null;
300
435
  }
436
+ if (!Array.isArray(tools) || !this.validCatalogTools(tools)) {
437
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog chunks contain invalid tools; ignoring persisted catalog.`);
438
+ return null;
439
+ }
440
+ if (tools.length !== manifest.toolCount) {
441
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog tool count does not match its manifest; ignoring persisted catalog.`);
442
+ return null;
443
+ }
444
+ return {
445
+ tools,
446
+ fingerprint: stored.fingerprint,
447
+ fetchedAt: manifest.fetchedAt,
448
+ expiresAt: manifest.expiresAt,
449
+ staleUntil: manifest.staleUntil,
450
+ };
301
451
  }
302
- async storeCatalog(id, tools) {
452
+ async storeCatalog(id, snapshot) {
303
453
  if (!this.persistToolCatalog)
304
454
  return;
305
455
  const fetchedAt = Date.now();
306
- const catalog = {
307
- tools,
456
+ const expiresAt = fetchedAt + this.ttlMs;
457
+ const staleUntil = expiresAt + this.staleMs;
458
+ const ttlSeconds = Math.max(60, Math.ceil((this.ttlMs + this.staleMs) / 1000));
459
+ const chunks = this.splitCatalogChunks(snapshot);
460
+ const chunkWrites = await mapSettledWithConcurrency(chunks, CATALOG_CHUNK_IO_CONCURRENCY, (chunk, index) => this.opts.storage.set(this.catalogChunkKey(id, snapshot.fingerprint, index), chunk, { ttlSeconds: ttlSeconds + CATALOG_CHUNK_TTL_GRACE_SECONDS }));
461
+ for (const result of chunkWrites) {
462
+ if (result.status === "rejected")
463
+ throw result.reason;
464
+ }
465
+ // The manifest is the only publication point. A failed/partial chunk write
466
+ // therefore leaves the previous manifest authoritative (or no catalog);
467
+ // unreachable chunks carry a bounded TTL and require no prefix scan.
468
+ const manifest = {
469
+ version: 2,
470
+ revision: snapshot.fingerprint,
471
+ toolCount: snapshot.tools.length,
472
+ byteCount: snapshot.serializedBytes.byteLength,
473
+ chunkCount: chunks.length,
308
474
  fetchedAt,
309
- expiresAt: fetchedAt + this.ttlMs,
310
- staleUntil: fetchedAt + this.ttlMs + this.staleMs,
475
+ expiresAt,
476
+ staleUntil,
311
477
  };
312
- await this.opts.storage.set(this.catalogKey(id), JSON.stringify(catalog), {
313
- ttlSeconds: Math.max(60, Math.ceil((this.ttlMs + this.staleMs) / 1000)),
478
+ await this.opts.storage.set(this.catalogKey(id), JSON.stringify(manifest), {
479
+ ttlSeconds,
314
480
  });
315
481
  }
482
+ async deleteCatalog(id) {
483
+ let raw = null;
484
+ let readError;
485
+ try {
486
+ raw = await this.opts.storage.get(this.catalogKey(id));
487
+ }
488
+ catch (err) {
489
+ readError = err;
490
+ }
491
+ const manifest = this.parseCatalogManifest(raw);
492
+ // The root is authoritative, so attempt its deletion even when the
493
+ // best-effort read needed for physical chunk cleanup failed.
494
+ await this.opts.storage.delete(this.catalogKey(id));
495
+ if (!manifest) {
496
+ if (readError)
497
+ throw readError;
498
+ return;
499
+ }
500
+ let firstError;
501
+ for (let index = 0; index < manifest.chunkCount; index++) {
502
+ try {
503
+ await this.opts.storage.delete(this.catalogChunkKey(id, manifest.revision, index));
504
+ }
505
+ catch (err) {
506
+ firstError ??= err;
507
+ }
508
+ }
509
+ if (readError)
510
+ throw readError;
511
+ if (firstError)
512
+ throw firstError;
513
+ }
316
514
  catalogGeneration(id) {
317
515
  return this.catalogGenerations.get(id) ?? 0;
318
516
  }
@@ -342,22 +540,40 @@ export class Registry {
342
540
  const connector = this.connectors.get(id);
343
541
  if (!connector)
344
542
  throw new Error(`Unknown connector "${id}"`);
543
+ if (connector.staticTools)
544
+ return connector.staticTools;
345
545
  const generation = this.catalogGeneration(id);
346
- const tools = connector.staticTools
347
- ? connector.staticTools
348
- : await connector.listTools(this.contextFor(id, baseUrl, requestScope, callOptions));
546
+ const tools = await connector.listTools(this.contextFor(id, baseUrl, requestScope, callOptions));
349
547
  // The caller that began this refresh may still use its result, but a
350
548
  // credential/OAuth change that landed while listTools was in flight means
351
549
  // the listing must not enter either shared cache layer.
352
550
  if (generation !== this.catalogGeneration(id))
353
551
  return tools;
354
- const now = Date.now();
552
+ if (tools.length > MAX_CATALOG_TOOLS) {
553
+ const message = `Connector "${id}" returned ${tools.length} tools, over the ` +
554
+ `${MAX_CATALOG_TOOLS}-tool catalog ceiling; refusing the complete catalog.`;
555
+ this.opts.logger.warn(`[connecta] ${message}`);
556
+ throw new Error(message);
557
+ }
355
558
  const previous = this.cache.get(id);
356
- const catalogChanged = !previous || JSON.stringify(previous.tools) !== JSON.stringify(tools);
357
- const shouldPersist = !connector.staticTools &&
358
- (catalogChanged || previous.exp <= now || this.invalidated.has(id));
559
+ const snapshot = await snapshotCatalog(tools);
560
+ if (generation !== this.catalogGeneration(id))
561
+ return tools;
562
+ if (snapshot.serializedBytes.byteLength > MAX_SERIALIZED_CATALOG_BYTES) {
563
+ const message = `Connector "${id}" returned a ${snapshot.serializedBytes.byteLength}-byte ` +
564
+ `serialized catalog, over the ${MAX_SERIALIZED_CATALOG_BYTES}-byte ceiling; ` +
565
+ "refusing the complete catalog.";
566
+ this.opts.logger.warn(`[connecta] ${message}`);
567
+ throw new Error(message);
568
+ }
569
+ const now = Date.now();
570
+ const catalogChanged = !previous || previous.fingerprint !== snapshot.fingerprint;
571
+ const shouldPersist = catalogChanged ||
572
+ (previous !== undefined && previous.exp <= now) ||
573
+ this.invalidated.has(id);
359
574
  this.cache.set(id, {
360
575
  tools,
576
+ fingerprint: snapshot.fingerprint,
361
577
  exp: now + this.ttlMs,
362
578
  staleUntil: now + this.ttlMs + this.staleMs,
363
579
  });
@@ -367,7 +583,7 @@ export class Registry {
367
583
  if (generation !== this.catalogGeneration(id))
368
584
  return;
369
585
  try {
370
- await this.storeCatalog(id, tools);
586
+ await this.storeCatalog(id, snapshot);
371
587
  }
372
588
  catch (err) {
373
589
  this.opts.logger.warn(`[connecta] connector "${id}" catalog persistence failed: ${msg(err)}`);
@@ -377,7 +593,7 @@ export class Registry {
377
593
  return tools;
378
594
  }
379
595
  /** Cached listTools with in-memory + persisted serializable catalog layers. */
380
- async getTools(id, baseUrl, requestScope, callOptions = {}) {
596
+ async loadTools(id, baseUrl, requestScope, callOptions = {}) {
381
597
  const connector = this.connectors.get(id);
382
598
  if (!connector)
383
599
  throw new Error(`Unknown connector "${id}"`);
@@ -393,7 +609,7 @@ export class Registry {
393
609
  const generation = this.catalogGeneration(id);
394
610
  let persisted = null;
395
611
  try {
396
- persisted = this.validCatalog(await this.opts.storage.get(this.catalogKey(id)));
612
+ persisted = await this.readCatalog(id, await this.opts.storage.get(this.catalogKey(id)), now);
397
613
  }
398
614
  catch (err) {
399
615
  this.opts.logger.warn(`[connecta] connector "${id}" catalog read failed: ${msg(err)}`);
@@ -405,6 +621,7 @@ export class Registry {
405
621
  if (persisted && persisted.staleUntil > now) {
406
622
  this.cache.set(id, {
407
623
  tools: persisted.tools,
624
+ fingerprint: persisted.fingerprint,
408
625
  exp: persisted.expiresAt,
409
626
  staleUntil: persisted.staleUntil,
410
627
  });
@@ -426,6 +643,45 @@ export class Registry {
426
643
  throw err;
427
644
  }
428
645
  }
646
+ /**
647
+ * Coalesce one connector's cold load inside one inbound request. The WeakMap
648
+ * neither roots the request scope nor lets its connector context escape into
649
+ * another request; settled entries are also removed eagerly.
650
+ *
651
+ * The first caller's `callOptions` govern the shared load: a later caller's
652
+ * signal or timeout neither cancels nor extends it, and an abort by the
653
+ * first caller rejects every coalesced caller. Within one request that is
654
+ * the deal being made — one fetch, one deadline.
655
+ */
656
+ async getTools(id, baseUrl, requestScope, callOptions = {}) {
657
+ const connector = this.connectors.get(id);
658
+ if (!connector)
659
+ throw new Error(`Unknown connector "${id}"`);
660
+ if (connector.staticTools)
661
+ return connector.staticTools;
662
+ if (!requestScope) {
663
+ return this.loadTools(id, baseUrl, requestScope, callOptions);
664
+ }
665
+ let loads = this.requestCatalogLoads.get(requestScope);
666
+ if (!loads) {
667
+ loads = new Map();
668
+ this.requestCatalogLoads.set(requestScope, loads);
669
+ }
670
+ const existing = loads.get(id);
671
+ if (existing)
672
+ return existing;
673
+ const loading = this.loadTools(id, baseUrl, requestScope, callOptions);
674
+ loads.set(id, loading);
675
+ try {
676
+ return await loading;
677
+ }
678
+ finally {
679
+ if (loads.get(id) === loading)
680
+ loads.delete(id);
681
+ if (loads.size === 0)
682
+ this.requestCatalogLoads.delete(requestScope);
683
+ }
684
+ }
429
685
  /** Return a cached catalog without performing storage or network I/O. */
430
686
  peekTools(id) {
431
687
  const connector = this.connectors.get(id);
@@ -443,67 +699,28 @@ export class Registry {
443
699
  healthFor(id) {
444
700
  return this.health.get(id);
445
701
  }
446
- /**
447
- * Whether ANY view of this deployment has seen a successful call to `id` —
448
- * a bare boolean, never the observation. Connector liveness (reachable,
449
- * credentials still valid) is a deployment-level fact, not a per-view one, so
450
- * `list_connectors` may classify a connector as ok/unknown from it. The
451
- * observation itself — `lastError` above all, which names the tool that
452
- * failed — stays strictly per view.
453
- */
702
+ /** Whether this deployment has seen a successful call to `id`. */
454
703
  hasObservedSuccess(id) {
455
704
  return this.observedSuccessAt(id) !== undefined;
456
705
  }
457
- /**
458
- * The timestamp behind `hasObservedSuccess`, on the same deployment-wide
459
- * terms and for the same reason: it says only *when* the connector last
460
- * answered, never what was called or what failed. Credential health reads it
461
- * to decide whether a failed verdict has been overtaken by real traffic.
462
- */
706
+ /** The timestamp behind `hasObservedSuccess`. */
463
707
  observedSuccessAt(id) {
464
708
  return this.health.get(id)?.lastSuccessAt;
465
709
  }
466
- /**
467
- * The last credential-liveness verdict for `id` — the layer that lets a cached
468
- * status read report `auth_required` before a real call discovers it. Read
469
- * from storage (mirrored in memory for a few seconds) rather than held in
470
- * memory alone, because on Workers the isolate that ran the check is usually
471
- * not the isolate answering this read.
472
- */
473
- credentialHealthFor(id) {
474
- return this.credentialHealth.healthFor(id);
475
- }
476
- recordCredentialHealth(id, record) {
477
- return this.credentialHealth.record(id, record);
478
- }
479
- /**
480
- * Check stored downstream credentials now and return one outcome per
481
- * connector considered. THE operator-facing entry point behind
482
- * `Connecta.checkCredentials()`: wire it to a Worker cron trigger or a Node
483
- * interval. Never rejects; connectors checked recently are reported as
484
- * `fresh` unless `force` is set.
485
- */
486
- checkCredentialHealth(baseUrl, opts, defer) {
487
- return this.credentialHealth.check(baseUrl, opts, defer);
488
- }
489
- /**
490
- * The traffic-triggered sweep: a promise for the caller to defer (Workers:
491
- * `ctx.waitUntil`), or `undefined` when nothing is due — which is the common
492
- * case and costs no I/O. Called by the server after an authenticated request;
493
- * the checker owns the rate limiting.
494
- */
495
- sweepCredentialHealthIfDue(baseUrl, defer) {
496
- return this.credentialHealth.sweepIfDue(baseUrl, defer);
497
- }
498
- /**
499
- * Drop a connector's liveness verdict, because its credential just changed
500
- * under us (OAuth callback completed, credential stored or removed on
501
- * /credentials). A
502
- * stale `auth_required` must not outlive the re-authorization that fixed it —
503
- * that is the difference between recovery working and needing a restart.
504
- */
505
- clearCredentialHealth(id) {
506
- return this.credentialHealth.clear(id);
710
+ async credentialDriftFor(id) {
711
+ const credential = this.connectors.get(id)?.credential;
712
+ const vault = this.opts.credentialVault;
713
+ if (!credential || !vault)
714
+ return undefined;
715
+ try {
716
+ const values = await vault.getAll(id);
717
+ const shape = storedCredentialShape(credential, values);
718
+ return shape.state === "mismatch" ? shape.message : undefined;
719
+ }
720
+ catch (error) {
721
+ this.opts.logger.warn(`[connecta] connector "${id}" credential shape read failed: ${msg(error)}`);
722
+ return undefined;
723
+ }
507
724
  }
508
725
  /** Best-effort connector status for list_connectors. */
509
726
  async statusFor(id, baseUrl, requestScope = {}, callOptions = {}) {
@@ -527,234 +744,32 @@ export class Registry {
527
744
  return { state: "error", message: msg(err) };
528
745
  }
529
746
  }
530
- /** Drop a connector's cached tool list (e.g. after auth completes). */
531
- invalidate(id) {
532
- this.advanceCatalogGeneration(id);
533
- this.cache.delete(id);
534
- this.invalidated.add(id);
535
- if (this.persistToolCatalog) {
536
- void this.enqueueCatalogMutation(id, async () => {
537
- try {
538
- await this.opts.storage.delete(this.catalogKey(id));
539
- }
540
- catch (err) {
541
- this.opts.logger.warn(`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`);
542
- }
543
- });
544
- }
545
- }
546
- /** Drop both in-memory and persisted tool catalogs. */
547
- async invalidateStored(id) {
747
+ markCatalogInvalid(id) {
548
748
  this.advanceCatalogGeneration(id);
549
749
  this.cache.delete(id);
550
750
  this.invalidated.add(id);
551
- if (this.persistToolCatalog) {
552
- await this.enqueueCatalogMutation(id, async () => {
553
- try {
554
- await this.opts.storage.delete(this.catalogKey(id));
555
- }
556
- catch (err) {
557
- this.opts.logger.warn(`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`);
558
- }
559
- });
560
- }
561
- }
562
- }
563
- /**
564
- * Per-toolkit HealthLogs, keyed by registry and then by toolkit name.
565
- *
566
- * Module-scoped rather than a `Registry` member on purpose: `Registry`'s type
567
- * is part of the public API surface (`Connecta.registry`), and this is internal
568
- * factoring only `ScopedRegistry` may touch. The WeakMap keeps the logs alive
569
- * exactly as long as their registry, and the inner map is bounded by the
570
- * number of configured toolkits.
571
- */
572
- const toolkitHealthLogs = new WeakMap();
573
- /** The long-lived log one toolkit records into; created on first use. */
574
- function toolkitHealthLog(base, toolkitName) {
575
- let logs = toolkitHealthLogs.get(base);
576
- if (!logs) {
577
- logs = new Map();
578
- toolkitHealthLogs.set(base, logs);
579
- }
580
- let log = logs.get(toolkitName);
581
- if (!log) {
582
- log = new HealthLog();
583
- logs.set(toolkitName, log);
584
- }
585
- return log;
586
- }
587
- /**
588
- * THE toolkit enforcement point (issue #22).
589
- *
590
- * A filtered VIEW of one long-lived `Registry`: same connectors, same tool
591
- * caches, same health — narrowed to the connectors and tool addresses one
592
- * toolkit selects. `serveMcp` builds it once per scoped connection and hands it
593
- * to `registerMetaTools`/`registerExecuteTool`, so `list_connectors`,
594
- * `search_tools`, `describe_tools`, `call_tool`, `call_destructive_tool`,
595
- * `batch_call`, `authorize_connector`, `get_result`, `skills`, and the
596
- * `execute_code` host bridge all inherit the boundary from here instead of
597
- * re-implementing nine checks.
598
- *
599
- * Two invariants make it reviewable:
600
- *
601
- * 1. **Indistinguishability.** Out-of-scope input fails through the SAME code
602
- * path, with the same error class and message, as input naming something
603
- * that does not exist at all. An out-of-scope CONNECTOR disappears from
604
- * `resolveAddress`/`getConnector` (→ `Unknown address` / `Unknown
605
- * connector`); an out-of-scope TOOL disappears from the catalog this view
606
- * returns (→ `Unknown tool "<t>" on connector "<c>"`), which is exactly the
607
- * error a misspelled tool name already produced. Scoping deliberately does
608
- * NOT reject at `resolveAddress` for tools: that would answer with the
609
- * connector-level message and make "hidden here" distinguishable from
610
- * "never existed".
611
- * 2. **It filters views, never state.** Every read delegates to the shared
612
- * registry and filters the returned array, so the tool cache, the persisted
613
- * catalog, and the health map stay whole and shared across scopes.
614
- */
615
- export class ScopedRegistry {
616
- base;
617
- toolkit;
618
- /** This toolkit's own health observations — see HealthLog. */
619
- health;
620
- constructor(base, toolkit) {
621
- this.base = base;
622
- this.toolkit = toolkit;
623
- this.health = toolkitHealthLog(base, toolkit.name);
624
- }
625
- get maxResultBytes() {
626
- return this.base.maxResultBytes;
627
- }
628
- get maxBatchResultBytes() {
629
- return this.base.maxBatchResultBytes;
630
- }
631
- /** In scope AND actually registered. */
632
- visible(id) {
633
- return (this.toolkit.hasConnector(id) && this.base.getConnector(id) !== undefined);
634
- }
635
- /** Byte-identical to what the unscoped registry throws for an unknown id. */
636
- unknownConnector(id) {
637
- return new Error(`Unknown connector "${id}"`);
638
- }
639
- inScopeTools(id, tools) {
640
- return tools.filter((tool) => this.toolkit.hasTool(id, tool.name));
641
- }
642
- listConnectors() {
643
- return this.base
644
- .listConnectors()
645
- .filter((connector) => this.toolkit.hasConnector(connector.id));
646
- }
647
- getConnector(id) {
648
- return this.visible(id) ? this.base.getConnector(id) : undefined;
649
- }
650
- resolveAddress(address) {
651
- const resolved = this.base.resolveAddress(address);
652
- if (!resolved)
653
- return null;
654
- // Connector-level only — see invariant 1 above.
655
- return this.toolkit.hasConnector(resolved.connector.id) ? resolved : null;
656
- }
657
- async getTools(id, baseUrl, requestScope, callOptions = {}) {
658
- if (!this.visible(id))
659
- throw this.unknownConnector(id);
660
- return this.inScopeTools(id, await this.base.getTools(id, baseUrl, requestScope, callOptions));
661
- }
662
- async refreshTools(id, baseUrl, requestScope, callOptions = {}) {
663
- if (!this.visible(id))
664
- throw this.unknownConnector(id);
665
- return this.inScopeTools(id, await this.base.refreshTools(id, baseUrl, requestScope, callOptions));
666
- }
667
- peekTools(id) {
668
- if (!this.visible(id))
669
- return undefined;
670
- const tools = this.base.peekTools(id);
671
- return tools ? this.inScopeTools(id, tools) : undefined;
672
- }
673
- contextFor(id, baseUrl, requestScope = {}, callOptions = {}) {
674
- // Unreachable through the meta-tools (they resolve first), so a throw here
675
- // is a loud backstop rather than a silent grant of connector storage and
676
- // credentials to a scope that may not see the connector.
677
- if (!this.visible(id))
678
- throw this.unknownConnector(id);
679
- return this.base.contextFor(id, baseUrl, requestScope, callOptions);
680
751
  }
681
- admitCall(id, input) {
682
- if (!this.visible(id))
683
- return Promise.reject(this.unknownConnector(id));
684
- return this.base.admitCall(id, input);
685
- }
686
- /**
687
- * Stashed oversized results are bound to the scope that produced them: a
688
- * scoped session cannot page a result it could not have produced, and an id
689
- * from another scope reads back as the ordinary "Unknown or expired result
690
- * id". Unscoped sessions keep the historical `results:` prefix untouched.
691
- */
692
- resultsStorage() {
693
- return namespaced(this.base.resultsStorage(), `toolkit:${this.toolkit.name}:`);
694
- }
695
- // Outcomes are recorded twice on purpose: the deployment-wide log keeps the
696
- // operator surfaces complete, while the toolkit's own log is the ONLY one
697
- // this view reads back — so `list_connectors` here never reports a sibling
698
- // toolkit's failures, or the tool names their error strings carry.
699
- recordSuccess(id, latencyMs) {
700
- if (!this.visible(id))
701
- return;
702
- this.base.recordSuccess(id, latencyMs);
703
- this.health.recordSuccess(id, latencyMs);
704
- }
705
- recordFailure(id, latencyMs, error) {
706
- if (!this.visible(id))
707
- return;
708
- this.base.recordFailure(id, latencyMs, error);
709
- this.health.recordFailure(id, latencyMs, error);
710
- }
711
- healthFor(id) {
712
- return this.visible(id) ? this.health.get(id) : undefined;
713
- }
714
- /**
715
- * Deliberately NOT per view: whether the connector has ever answered is a
716
- * fact about the connector, not about a team's traffic, and withholding it
717
- * would report every remote connector as "unknown" to a scoped session that
718
- * has not called it yet. It carries no tool name, error text, or count — the
719
- * per-view isolation of those, above, is unchanged.
720
- */
721
- hasObservedSuccess(id) {
722
- return this.visible(id) ? this.base.hasObservedSuccess(id) : false;
723
- }
724
- /** Deployment-wide for the same reason as `hasObservedSuccess` above. */
725
- observedSuccessAt(id) {
726
- return this.visible(id) ? this.base.observedSuccessAt(id) : undefined;
727
- }
728
- /**
729
- * Also deliberately NOT per view, for the same reason as
730
- * `hasObservedSuccess`: whether the credential connecta stores for a connector
731
- * still works is a fact about the deployment's credential, not about a team's
732
- * traffic. Withholding it would leave a scoped session unable to see that the
733
- * connector it shares needs re-authorization. The verdict carries the
734
- * connector's own connector-level reason — never a tool name — so the per-view
735
- * isolation of `lastError` above is unchanged.
736
- */
737
- credentialHealthFor(id) {
738
- return this.visible(id)
739
- ? this.base.credentialHealthFor(id)
740
- : Promise.resolve(undefined);
741
- }
742
- async recordCredentialHealth(id, record) {
743
- if (!this.visible(id))
744
- return;
745
- await this.base.recordCredentialHealth(id, record);
752
+ deleteStoredCatalog(id) {
753
+ return this.enqueueCatalogMutation(id, async () => {
754
+ try {
755
+ await this.deleteCatalog(id);
756
+ }
757
+ catch (err) {
758
+ this.opts.logger.warn(`[connecta] connector "${id}" catalog invalidation failed: ${msg(err)}`);
759
+ }
760
+ });
746
761
  }
747
- async statusFor(id, baseUrl, requestScope = {}, callOptions = {}) {
748
- // Same shape the unscoped registry returns for an unregistered id.
749
- if (!this.visible(id)) {
750
- return { state: "error", message: "Unknown connector" };
751
- }
752
- return this.base.statusFor(id, baseUrl, requestScope, callOptions);
762
+ /** Drop a connector's cached tool list (e.g. after auth completes). */
763
+ invalidate(id) {
764
+ this.markCatalogInvalid(id);
765
+ if (this.persistToolCatalog)
766
+ void this.deleteStoredCatalog(id);
753
767
  }
768
+ /** Drop both in-memory and persisted tool catalogs. */
754
769
  async invalidateStored(id) {
755
- if (!this.visible(id))
756
- return;
757
- await this.base.invalidateStored(id);
770
+ this.markCatalogInvalid(id);
771
+ if (this.persistToolCatalog)
772
+ await this.deleteStoredCatalog(id);
758
773
  }
759
774
  }
760
775
  //# sourceMappingURL=registry.js.map