@zackbart/connecta 0.41.0 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (148) hide show
  1. package/CHANGELOG.md +61 -0
  2. package/bin/connecta.mjs +26 -17
  3. package/bin/meta-tool-names.d.mts +3 -1
  4. package/bin/meta-tool-names.mjs +4 -1
  5. package/bin/version.mjs +1 -1
  6. package/dist/activity.d.ts +2 -1
  7. package/dist/activity.js +5 -5
  8. package/dist/advertised-schema.js +15 -0
  9. package/dist/agent-routes.js +31 -5
  10. package/dist/auth/downstream-oauth/refresh-coordinator.js +4 -1
  11. package/dist/auth/downstream-oauth.js +3 -0
  12. package/dist/auth/static-oauth.js +26 -18
  13. package/dist/auth-elicitation.js +118 -61
  14. package/dist/authorization-handoff.js +36 -3
  15. package/dist/catalog-cache.js +292 -24
  16. package/dist/catalog-freshness.js +12 -0
  17. package/dist/catalog-service.js +373 -70
  18. package/dist/catalog.js +147 -58
  19. package/dist/config-defaults.d.ts +2 -1
  20. package/dist/config-defaults.js +4 -1
  21. package/dist/config.d.ts +43 -17
  22. package/dist/config.js +26 -10
  23. package/dist/connectors/remote-mcp/catalog.js +52 -28
  24. package/dist/connectors/remote-mcp/connection.js +62 -5
  25. package/dist/connectors/remote-mcp.js +10 -1
  26. package/dist/describe-config.d.ts +3 -2
  27. package/dist/discovery-health.js +26 -11
  28. package/dist/errors.d.ts +52 -3
  29. package/dist/errors.js +86 -3
  30. package/dist/execute.js +476 -200
  31. package/dist/executor-admission.js +2 -0
  32. package/dist/executor-contract.js +6 -0
  33. package/dist/executor-result.js +2 -2
  34. package/dist/executors/quickjs-child.js +13 -4
  35. package/dist/executors/quickjs-runtime.d.ts +2 -0
  36. package/dist/executors/quickjs-runtime.js +36 -19
  37. package/dist/executors/quickjs.js +23 -6
  38. package/dist/guest-functions.js +5 -4
  39. package/dist/guest-runtime.js +37 -1
  40. package/dist/guest-types.d.ts +18 -0
  41. package/dist/http/client.js +7 -2
  42. package/dist/http/errors.js +9 -4
  43. package/dist/http/fields.js +17 -5
  44. package/dist/http/hatch.js +9 -3
  45. package/dist/http/paging.js +1 -1
  46. package/dist/http/provider.js +2 -6
  47. package/dist/http/vendor.js +6 -1
  48. package/dist/index.d.ts +1 -1
  49. package/dist/index.js +5 -0
  50. package/dist/invocation.js +98 -34
  51. package/dist/meta-output.js +24 -18
  52. package/dist/meta-tool-names.d.ts +1 -1
  53. package/dist/meta-tool-names.js +1 -1
  54. package/dist/meta-tools.js +259 -60
  55. package/dist/native-recovery.js +4 -1
  56. package/dist/native-tools.js +166 -51
  57. package/dist/node.js +3 -1
  58. package/dist/oauth-handoff.js +53 -43
  59. package/dist/operation-activity.js +2 -1
  60. package/dist/operation-vocabulary.js +6 -0
  61. package/dist/operator-record.d.ts +1 -1
  62. package/dist/operator-record.js +1 -0
  63. package/dist/operator-ui/generated.js +2 -2
  64. package/dist/program-result.js +2 -0
  65. package/dist/program-source.js +1 -1
  66. package/dist/program-writes.js +112 -19
  67. package/dist/provider.d.ts +0 -13
  68. package/dist/provider.js +0 -3
  69. package/dist/providers/_shared/google/kit.js +2 -2
  70. package/dist/providers/breeze/api.js +2 -2
  71. package/dist/providers/breeze/skill.generated.js +1 -1
  72. package/dist/providers/ccb/api.js +2 -2
  73. package/dist/providers/ccb/skill.generated.js +1 -1
  74. package/dist/providers/cloudflare/api.js +2 -2
  75. package/dist/providers/cloudflare/index.js +2 -0
  76. package/dist/providers/cloudflare/skill.generated.js +1 -1
  77. package/dist/providers/docs/skill.generated.js +1 -1
  78. package/dist/providers/drive/skill.generated.js +1 -1
  79. package/dist/providers/forms/index.js +1 -0
  80. package/dist/providers/forms/skill.generated.js +1 -1
  81. package/dist/providers/github/api.js +98 -6
  82. package/dist/providers/github/index.js +7 -2
  83. package/dist/providers/github/lib/auth.d.ts +3 -1
  84. package/dist/providers/github/lib/auth.js +43 -12
  85. package/dist/providers/github/lib/discovery.js +11 -1
  86. package/dist/providers/github/lib/operations.js +8 -0
  87. package/dist/providers/github/safety.js +39 -0
  88. package/dist/providers/github/skill.generated.js +1 -1
  89. package/dist/providers/github/tools.js +12 -5
  90. package/dist/providers/gmail/index.js +1 -0
  91. package/dist/providers/gmail/skill.generated.js +1 -1
  92. package/dist/providers/granola/skill.generated.js +1 -1
  93. package/dist/providers/overflow/index.js +2 -0
  94. package/dist/providers/overflow/skill.generated.js +1 -1
  95. package/dist/providers/planning-center/index.js +11 -5
  96. package/dist/providers/planning-center/lib/constants.d.ts +1 -1
  97. package/dist/providers/planning-center/skill.generated.js +1 -1
  98. package/dist/providers/quickbooks/index.js +2 -5
  99. package/dist/providers/quickbooks/skill.generated.js +1 -1
  100. package/dist/providers/sheets/index.js +1 -0
  101. package/dist/providers/sheets/skill.generated.js +1 -1
  102. package/dist/providers/slides/index.js +2 -2
  103. package/dist/providers/slides/skill.generated.js +1 -1
  104. package/dist/providers/stripe/skill.generated.js +1 -1
  105. package/dist/providers/tithely/index.js +1 -0
  106. package/dist/providers/tithely/skill.generated.js +1 -1
  107. package/dist/providers/vercel/api.js +2 -2
  108. package/dist/providers/vercel/skill.generated.js +1 -1
  109. package/dist/registry.d.ts +36 -4
  110. package/dist/registry.js +127 -27
  111. package/dist/result-preview.js +210 -0
  112. package/dist/result-seal.js +168 -0
  113. package/dist/route-names.js +13 -4
  114. package/dist/routes/connect.js +21 -13
  115. package/dist/routes/mcp.js +55 -26
  116. package/dist/routes/shared.d.ts +6 -0
  117. package/dist/runtime/call.js +15 -10
  118. package/dist/runtime/request-scope.js +36 -1
  119. package/dist/runtime/run.js +4 -1
  120. package/dist/sent-secrets.d.ts +20 -0
  121. package/dist/sent-secrets.js +49 -1
  122. package/dist/skills.js +100 -33
  123. package/dist/storage/keys.js +9 -6
  124. package/dist/storage/sql-native.js +1 -1
  125. package/dist/tool-safety.d.ts +33 -0
  126. package/dist/tool-safety.js +51 -15
  127. package/dist/types.d.ts +31 -0
  128. package/dist/usage-guide.js +14 -13
  129. package/dist/validate.js +29 -10
  130. package/dist/version.d.ts +1 -1
  131. package/dist/version.js +1 -1
  132. package/dist/worker.js +28 -8
  133. package/documentation/architecture.md +104 -42
  134. package/documentation/auth.md +46 -19
  135. package/documentation/code-mode.md +378 -116
  136. package/documentation/deploying.md +10 -9
  137. package/documentation/hosted-mcp-migration.md +9 -3
  138. package/documentation/integrating.md +27 -5
  139. package/documentation/meta-tools.md +655 -153
  140. package/documentation/operating.md +5 -3
  141. package/documentation/providers.md +18 -6
  142. package/examples/worker/README.md +3 -3
  143. package/guides/how-it-works.md +2 -1
  144. package/guides/safety-and-trust.md +1 -3
  145. package/package.json +1 -1
  146. package/templates/node/AGENTS.md +3 -3
  147. package/templates/node/README.md +1 -1
  148. package/templates/node/package.json +1 -1
package/dist/registry.js CHANGED
@@ -6,15 +6,15 @@ import { authCall, authInvoke } from "./auth/native.js";
6
6
  import { Clock, Duration, Effect, Random } from "effect";
7
7
  import { activityRequest } from "./activity-request.js";
8
8
  import { aggregateCallAdmissionSnapshots, ConnectorCallAdmissionController, } from "./call-admission.js";
9
- import { attachCatalogCache, catalogExpiry, catalogFetchedAt, customCatalogFallback, invalidateCatalogCache, observeUncachedCatalogRefresh, retainedCatalogIdentity, storeCustomCatalogFallback, } from "./catalog-cache.js";
10
- import { carryCatalogFreshness, markCatalogFreshness } from "./catalog-freshness.js";
9
+ import { attachCatalogCache, catalogExpiry, catalogFetchedAt, bindCustomCatalogListing, customCatalogFallback, customCatalogPeek, fenceCustomCatalog, invalidateCatalogCache, observeUncachedCatalogRefresh, retainedCatalogIdentity, storeCustomCatalogFallback, } from "./catalog-cache.js";
10
+ import { carryCatalogFreshness, markCatalogFreshness, markCatalogStale } from "./catalog-freshness.js";
11
11
  import { MAX_CATALOG_TOOLS, MAX_SERIALIZED_CATALOG_BYTES } from "./catalog-limits.js";
12
12
  import { CONFIG_DEFAULTS } from "./config-defaults.js";
13
13
  import { usesCallerIdentity } from "./connector-caller.js";
14
14
  import { forwardsPrincipal } from "./request-principal.js";
15
15
  import { canonicalJson } from "./canonical-json.js";
16
16
  import { storedCredentialShape } from "./credential-rules.js";
17
- import { ConnectorCallError } from "./errors.js";
17
+ import { ConnectorCallError, isAccessRefusal } from "./errors.js";
18
18
  import { attachOAuthPartition, oauthPartitionIdle } from "./oauth-partition.js";
19
19
  import { attachOAuthSealer, vaultOAuthSealer } from "./oauth-sealing.js";
20
20
  import { boundedStatus, failureRecord, failureStatus, logFailure, ownStatus } from "./operator-record.js";
@@ -34,7 +34,7 @@ import { redactCatalog, sentSecretsFor, sentSecretsForRequest } from "./sent-sec
34
34
  import { GUIDE_SUMMARY_LENGTH, normalizeGuideSummary, connectorGuide, connectorGuideSummary } from "./skills.js";
35
35
  import { jsonCodec, OAUTH_HANDOFF_TTL_SECONDS, oauthHandoffKeys, resultKeys, scopes, stashExpiryKeys, stashLedgerKeys, } from "./storage/keys.js";
36
36
  import { assertStaticToolNames, hasControlCharacters } from "./tool-name.js";
37
- import { carryToolClassification, classifyTool } from "./tool-safety.js";
37
+ import { carryToolClassification, classifyTool, sourceDeclaresWrite, } from "./tool-safety.js";
38
38
  import { isWorkspaceAddress } from "./workspace-address.js";
39
39
  import { workspaceSubjectOf } from "./workspace-subject.js";
40
40
  const ID_RE = /^[a-z0-9_-]+$/;
@@ -145,12 +145,18 @@ const STASH_WRITE_TIMEOUT_MS = 30_000;
145
145
  const STASH_COMPLETION_TIMEOUT_MS = 15_000;
146
146
  /** Reservation, chunk writes and completion share one finite outer budget. */
147
147
  const STASH_TIMEOUT_MS = STASH_WRITE_TIMEOUT_MS + STASH_COMPLETION_TIMEOUT_MS;
148
+ /** Chunks one release deletes at a time; a stash holds at most 32. */
149
+ const STASH_RELEASE_CONCURRENCY = 16;
148
150
  /** Stop waiting without starting background storage work after the deadline. */
149
151
  function completeStash(effect) {
150
152
  return effect.pipe(Effect.asVoid, Effect.timeoutOrElse({ duration: Duration.millis(STASH_COMPLETION_TIMEOUT_MS), orElse: () => Effect.void }),
151
153
  // onError finalizers are otherwise uninterruptible, including their I/O.
152
154
  Effect.interruptible);
153
155
  }
156
+ /** The reservation id of one result id, which is random and never reused. */
157
+ function stashCharge(id) {
158
+ return Effect.tryPromise({ try: () => sha256Hex(id), catch: (error) => error });
159
+ }
154
160
  /**
155
161
  * Live charges in a stored ledger. Anything malformed reads as empty: the
156
162
  * ledger bounds an advisory cache, and the next successful swap replaces it.
@@ -317,8 +323,8 @@ export class Registry {
317
323
  Object.freeze(this.classification);
318
324
  for (const [id, overrides] of Object.entries(this.classification)) {
319
325
  for (const [name, verdict] of Object.entries(overrides)) {
320
- if (!name || (verdict !== "read" && verdict !== "write" && verdict !== "hidden")) {
321
- throw new Error(`ConnectaConfig.classification.${id}.${name}: expected "read", "write", or "hidden"`);
326
+ if (!name || !["read", "read-repeatable", "write", "hidden"].includes(verdict)) {
327
+ throw new Error(`ConnectaConfig.classification.${id}.${name}: expected "read", "read-repeatable", "write", or "hidden"`);
322
328
  }
323
329
  }
324
330
  const connector = this.connectors.get(id);
@@ -617,8 +623,11 @@ export class Registry {
617
623
  * unconfirmed and retain their charge. Cleanup/release and settlement each
618
624
  * stop waiting after 15 seconds. Storage TTL reclaims the rows.
619
625
  */
620
- stashResult(id, chunks, ttlSeconds, partition = scopes.results) {
621
- return runOnPartition(registryEdgeProgram(Registry.prototype, this, "stashResult", id, chunks, ttlSeconds, partition), this.opts);
626
+ stashResult(id, chunks, ttlSeconds, options = {}) {
627
+ return runOnPartition(registryEdgeProgram(Registry.prototype, this, "stashResult", id, chunks, ttlSeconds, options), this.opts);
628
+ }
629
+ releaseResult(id, chunkCount, partition = scopes.results) {
630
+ return runOnPartition(registryEdgeProgram(Registry.prototype, this, "releaseResult", id, chunkCount, partition), this.opts);
622
631
  }
623
632
  /**
624
633
  * Storage namespaced to the root result partition, kept separate from any
@@ -681,7 +690,16 @@ export class Registry {
681
690
  if (connector.staticTools)
682
691
  return markCatalogFreshness(connector.staticTools, Infinity);
683
692
  const ctx = this.contextFor(id, baseUrl, requestScope, callOptions, scope);
684
- const listed = yield* authInvoke(connector, "listTools", [ctx]).pipe(Effect.catch((error) => Effect.gen(function* () {
693
+ // `remoteMcp()` binds and fences its own attempts, knowing how each was made.
694
+ const customListing = connector.peekCatalog
695
+ ? undefined
696
+ : yield* authCall(bindCustomCatalogListing, [ctx, id]).pipe(Effect.catch((failure) => Effect.sync(() => {
697
+ logFailure(this.opts.logger, "catalog read failed", failureRecord({ connector: id }, failure));
698
+ return undefined;
699
+ })));
700
+ const listed = yield* authInvoke(connector, "listTools", [ctx]).pipe(Effect.catch((error) => Effect.gen({ self: this }, function* () {
701
+ if (customListing && isAccessRefusal(error))
702
+ yield* authCall(fenceCustomCatalog, [ctx, id, customListing]).pipe(Effect.catch((failure) => Effect.sync(() => logFailure(this.opts.logger, "catalog fence failed", failureRecord({ connector: id }, failure)))));
685
703
  const fallback = error instanceof ConnectorCallError && error.code === "unavailable"
686
704
  ? yield* authCall(customCatalogFallback, [ctx, id]).pipe(Effect.catch(() => Effect.succeed(undefined)))
687
705
  : undefined;
@@ -697,12 +715,18 @@ export class Registry {
697
715
  const facts = frozenFacts(structuredClone(tools));
698
716
  const accepted = this.acceptToolNames(facts);
699
717
  if (catalogFetchedAt(ctx) === undefined) {
700
- for (const observe of [storeCustomCatalogFallback, observeUncachedCatalogRefresh]) {
701
- yield* authCall(observe, [ctx, id, facts]).pipe(Effect.catch((error) => Effect.sync(() => logFailure(this.opts.logger, "catalog refresh observation failed", failureRecord({ connector: id }, error)))));
718
+ for (const observe of [
719
+ authCall(storeCustomCatalogFallback, [ctx, facts, customListing]),
720
+ authCall(observeUncachedCatalogRefresh, [ctx, id, facts]),
721
+ ]) {
722
+ yield* observe.pipe(Effect.catch((error) => Effect.sync(() => logFailure(this.opts.logger, "catalog refresh observation failed", failureRecord({ connector: id }, error)))));
702
723
  }
703
724
  }
704
725
  const provenance = catalogExpiry(ctx);
705
726
  const freshUntil = provenance && !provenance.staleFallback ? provenance.expiresAt : 0;
727
+ // A transient-failure fallback answers the listing; discovery still reports its age.
728
+ if (provenance?.staleFallback)
729
+ markCatalogStale(accepted, { code: "unavailable", ageMs: Math.max(0, Date.now() - provenance.fetchedAt) });
706
730
  // Principal-forwarded listings belong to one caller, never to the shared
707
731
  // operator observation store. The actual catalog cache is identity-bound.
708
732
  if (!forwardsPrincipal(connector)) {
@@ -715,6 +739,30 @@ export class Registry {
715
739
  return markCatalogFreshness(accepted, freshUntil);
716
740
  });
717
741
  }
742
+ /** See {@link RegistryView.peekCatalog}. Same classification and filtering as `loadTools`. */
743
+ loadPeek(id, baseUrl, requestScope, callOptions = {}, scope) {
744
+ return Effect.gen({ self: this }, function* () {
745
+ const connector = this.connectors.get(id);
746
+ if (!connector || connector.staticTools)
747
+ return undefined;
748
+ const ctx = this.contextFor(id, baseUrl, requestScope, callOptions, scope);
749
+ const peek = connector.peekCatalog
750
+ ? yield* authInvoke(connector, "peekCatalog", [ctx])
751
+ : yield* authCall(customCatalogPeek, [ctx, id]);
752
+ if (!peek)
753
+ return undefined;
754
+ const facts = redactCatalog(ctx, peek.tools).map(({ classification: _ignored, ...fact }) => fact);
755
+ const overrides = this.classification[id];
756
+ return {
757
+ tools: this.acceptToolNames(frozenFacts(structuredClone(facts)))
758
+ .filter((tool) => overrides?.[tool.name] !== "hidden")
759
+ .map((tool) => this.publishTool(id, tool)),
760
+ fetchedAt: peek.fetchedAt,
761
+ // A custom catalog is always listed anew, so its retained copy is never current.
762
+ fresh: connector.peekCatalog !== undefined && peek.expiresAt > Date.now(),
763
+ };
764
+ });
765
+ }
718
766
  /** The served catalog excludes control-character names; raw caches stay complete. */
719
767
  acceptToolNames(tools) {
720
768
  const accepted = tools.filter((tool) => !hasControlCharacters(tool.name));
@@ -741,19 +789,20 @@ export class Registry {
741
789
  }
742
790
  publishTool(id, tool) {
743
791
  const override = this.classification[id]?.[tool.name];
792
+ const classification = classifyTool(tool, override === "hidden" ? undefined : override);
744
793
  return carryToolClassification(tool, {
745
794
  ...structuredClone(tool),
746
- classification: classifyTool(tool, override === "hidden" ? undefined : override),
795
+ classification,
747
796
  ...(override !== undefined
748
797
  ? {
749
798
  annotations: {
750
799
  ...structuredClone(tool.annotations),
751
- readOnlyHint: override === "read",
752
- destructiveHint: override === "write",
800
+ readOnlyHint: override !== "hidden" && classification === "read",
801
+ destructiveHint: override !== "hidden" && classification === "write",
753
802
  },
754
803
  }
755
804
  : {}),
756
- });
805
+ }, override === "read-repeatable" || undefined);
757
806
  }
758
807
  /** Construction-only view for the secret-free static config description. */
759
808
  describeStaticTools(id) {
@@ -771,19 +820,30 @@ export class Registry {
771
820
  return;
772
821
  this.overrideWarningCatalogs.set(partition, catalog);
773
822
  }
774
- const known = new Set(tools.map((tool) => tool.name));
775
- for (const name of Object.keys(overrides ?? {})) {
776
- if (known.has(name))
777
- continue;
778
- if (this.connectors.get(id)?.staticTools) {
823
+ const known = new Map(tools.map((tool) => [tool.name, tool]));
824
+ const staticCatalog = this.connectors.get(id)?.staticTools !== undefined;
825
+ for (const [name, override] of Object.entries(overrides ?? {})) {
826
+ const tool = known.get(name);
827
+ if (!tool && staticCatalog) {
779
828
  throw new Error(`ConnectaConfig.classification: connector "${id}" has no tool "${name}"`);
780
829
  }
781
830
  // Only deployment-authored names, never downstream text. JSON quoting
782
831
  // prevents control characters from forging operator log lines.
783
832
  const quoted = JSON.stringify(`${id}.${name}`).replace(/[\u2028\u2029]/g, (ch) => `\\u${ch.charCodeAt(0).toString(16)}`);
833
+ if (tool) {
834
+ // A static catalog cannot change, so the deployment's construction reports it once.
835
+ const reports = staticCatalog ? partition === undefined && this.opts.constructionChecks !== false : true;
836
+ if (override === "read" && reports && sourceDeclaresWrite(tool)) {
837
+ this.opts.logger.warn(`classification override ${quoted} to "read" is not applied because the connector declares this tool a write; it stays a write. "read-repeatable" serves it as a read and accepts that hosts may repeat it`);
838
+ }
839
+ continue;
840
+ }
784
841
  this.opts.logger.warn(`classification override ${quoted} is dormant because the runtime catalog lists no such tool`);
785
842
  }
786
843
  }
844
+ peekCatalog(id, baseUrl, requestScope, callOptions = {}, scope) {
845
+ return runEdge(registryEdgeProgram(Registry.prototype, this, "peekCatalog", id, baseUrl, requestScope, callOptions, scope));
846
+ }
787
847
  getTools(id, baseUrl, requestScope, callOptions = {}, scope) {
788
848
  return runEdge(registryEdgeProgram(Registry.prototype, this, "getTools", id, baseUrl, requestScope, callOptions, scope));
789
849
  }
@@ -818,7 +878,8 @@ export class Registry {
818
878
  }
819
879
  static {
820
880
  provideRegistryPrograms(Registry.prototype, {
821
- stashResult: (self, id, chunks, ttlSeconds, partition = scopes.results) => Effect.gen({ self }, function* () {
881
+ stashResult: (self, id, chunks, ttlSeconds, options = {}) => Effect.gen({ self }, function* () {
882
+ const partition = options.partition ?? scopes.results;
822
883
  const maxBytes = this.opts.results?.maxStashBytes ?? CONFIG_DEFAULTS.results.maxStashBytes;
823
884
  const maxEntries = this.opts.results?.maxStashEntries ?? CONFIG_DEFAULTS.results.maxStashEntries;
824
885
  // The paging envelope is ASCII, so its string length is its stored byte count.
@@ -826,8 +887,9 @@ export class Registry {
826
887
  if (bytes > maxBytes || maxEntries === 0)
827
888
  return false;
828
889
  const keys = chunks.map((_, index) => partition + resultKeys.chunk(id, index));
829
- // IDs belong to reservations, not reusable result IDs or subjects.
830
- const charge = crypto.randomUUID();
890
+ // A digest finds this reservation again on release without putting
891
+ // the result's ID in the deployment-wide ledger.
892
+ const charge = yield* stashCharge(id);
831
893
  const ttlMs = ttlSeconds * 1000;
832
894
  if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0)
833
895
  return false;
@@ -882,7 +944,8 @@ export class Registry {
882
944
  const remaining = Math.max(0, writeDeadline - (yield* Clock.currentTimeMillis));
883
945
  const written = yield* Effect.gen(function* () {
884
946
  const multi = getNativeStorage(self.opts.storage)?.setMany;
885
- for (let index = keys.length - 1; index >= 0;) {
947
+ const lowest = options.holdFirst ? 1 : 0;
948
+ for (let index = keys.length - 1; index >= lowest;) {
886
949
  const first = multi && index > 0 ? Math.max(1, index - 63) : index;
887
950
  const before = yield* Clock.currentTimeMillis;
888
951
  if (before >= writeDeadline || before >= chunkExpiry)
@@ -905,6 +968,15 @@ export class Registry {
905
968
  yield* completeStash(settle);
906
969
  return written;
907
970
  }).pipe(Effect.timeoutOrElse({ duration: Duration.millis(STASH_TIMEOUT_MS), orElse: () => Effect.succeed(false) }), Effect.provideService(Storage, self.opts.storage)),
971
+ releaseResult: (self, id, chunkCount, partition = scopes.results) => completeStash(Effect.gen(function* () {
972
+ const charge = yield* stashCharge(id);
973
+ // Bytes first: a failed delete keeps its charge, so the ledger never
974
+ // undercounts what storage holds.
975
+ yield* Effect.forEach(Array.from({ length: chunkCount }, (_, index) => partition + resultKeys.chunk(id, index)), (key) => storageDelete(key), { concurrency: STASH_RELEASE_CONCURRENCY, discard: true });
976
+ yield* swapStashLedger((live) => live.some((entry) => entry[0] === charge)
977
+ ? { entries: live.filter((entry) => entry[0] !== charge), result: undefined }
978
+ : { result: undefined });
979
+ })).pipe(Effect.ignore, Effect.provideService(Storage, self.opts.storage)),
908
980
  recheckResultAccess: () => Effect.succeed(true),
909
981
  storeOAuthHandoff: (self, connectorId, state, principalKey) => Effect.gen(function* () {
910
982
  const key = oauthHandoffKeys.handoff(connectorId, yield* Effect.tryPromise({ try: () => sha256Hex(state), catch: (error) => error }));
@@ -973,6 +1045,12 @@ export class Registry {
973
1045
  return yield* load.join(signal);
974
1046
  }));
975
1047
  }),
1048
+ peekCatalog: (self, id, baseUrl, requestScope, callOptions = {}, scope) => Effect.suspend(() => {
1049
+ if (!self.connectors.has(id))
1050
+ return Effect.succeed(undefined);
1051
+ const owner = self.contextFor(id, baseUrl, requestScope, callOptions, scope);
1052
+ return duringViewEffect(owner, (signal) => self.loadPeek(id, baseUrl, owner.requestScope, { ...callOptions, signal }, scope));
1053
+ }),
976
1054
  credentialDriftFor: (self, id, requestScope) => Effect.gen(function* () {
977
1055
  const credential = self.connectors.get(id)?.credential;
978
1056
  const vault = self.opts.credentialVault;
@@ -1097,6 +1175,9 @@ class ScopedRegistryView {
1097
1175
  getTools(...args) {
1098
1176
  return runEdge(registryEdgeProgram(ScopedRegistryView.prototype, this, "getTools", ...args));
1099
1177
  }
1178
+ peekCatalog(...args) {
1179
+ return runEdge(registryEdgeProgram(ScopedRegistryView.prototype, this, "peekCatalog", ...args));
1180
+ }
1100
1181
  contextFor(...args) {
1101
1182
  const registry = this.registryFor(args[0]);
1102
1183
  if (!registry)
@@ -1110,8 +1191,14 @@ class ScopedRegistryView {
1110
1191
  }
1111
1192
  return registry.admitCall(...args);
1112
1193
  }
1113
- stashResult(id, chunks, ttlSeconds) {
1114
- return this.root.stashResult(id, chunks, ttlSeconds, this.scope.subjectKey ? scopes.subject(this.scope.subjectKey) : scopes.results);
1194
+ stashResult(id, chunks, ttlSeconds, options = {}) {
1195
+ return this.root.stashResult(id, chunks, ttlSeconds, {
1196
+ ...options,
1197
+ partition: this.scope.subjectKey ? scopes.subject(this.scope.subjectKey) : scopes.results,
1198
+ });
1199
+ }
1200
+ releaseResult(id, chunkCount) {
1201
+ return this.root.releaseResult(id, chunkCount, this.scope.subjectKey ? scopes.subject(this.scope.subjectKey) : scopes.results);
1115
1202
  }
1116
1203
  resultIdentity() {
1117
1204
  return {
@@ -1158,7 +1245,11 @@ class ScopedRegistryView {
1158
1245
  }
1159
1246
  static {
1160
1247
  provideRegistryPrograms(ScopedRegistryView.prototype, {
1161
- stashResult: (self, id, chunks, ttlSeconds) => registryProgram(self.root, "stashResult", id, chunks, ttlSeconds, self.scope.subjectKey ? scopes.subject(self.scope.subjectKey) : scopes.results),
1248
+ releaseResult: (self, id, chunkCount) => registryProgram(self.root, "releaseResult", id, chunkCount, self.scope.subjectKey ? scopes.subject(self.scope.subjectKey) : scopes.results),
1249
+ stashResult: (self, id, chunks, ttlSeconds, options = {}) => registryProgram(self.root, "stashResult", id, chunks, ttlSeconds, {
1250
+ ...options,
1251
+ partition: self.scope.subjectKey ? scopes.subject(self.scope.subjectKey) : scopes.results,
1252
+ }),
1162
1253
  recheckResultAccess: (self, dependencies, classification, signal) => Effect.tryPromise({
1163
1254
  try: () => self.scope.currentResultAccess?.(dependencies, classification, signal) ?? Promise.resolve(true),
1164
1255
  catch: (error) => error,
@@ -1185,6 +1276,15 @@ class ScopedRegistryView {
1185
1276
  }
1186
1277
  return carryCatalogFreshness(tools, visible);
1187
1278
  }),
1279
+ peekCatalog: (self, ...args) => Effect.gen(function* () {
1280
+ const registry = self.registryFor(args[0]);
1281
+ if (!registry)
1282
+ return undefined;
1283
+ const peek = yield* registryProgram(registry, "peekCatalog", args[0], args[1], args[2], args[3], self.scope);
1284
+ const granted = self.scope.toolAccess?.get(args[0]);
1285
+ // A pool grant binds a stale listing exactly as it binds a live one.
1286
+ return peek && granted ? { ...peek, tools: peek.tools.filter((tool) => granted.has(tool.name)) } : peek;
1287
+ }),
1188
1288
  credentialDriftFor: (self, id, requestScope) => Effect.suspend(() => {
1189
1289
  const registry = self.registryFor(id);
1190
1290
  return registry
@@ -0,0 +1,210 @@
1
+ /**
2
+ * Schema-aware views of direct reads past their cap, as pure functions of redacted
3
+ * text. Presentation only: the stash and what a program sees stay byte-identical
4
+ * to the full result, and everything shown here also appears in it, unchanged.
5
+ * Rules: documentation/meta-tools.md#previews-of-reads-over-the-cap.
6
+ */
7
+ const enc = new TextEncoder();
8
+ const dec = new TextDecoder();
9
+ /** Bytes of whole rows a view shows. */
10
+ const ROW_BYTES = 1_500;
11
+ /** One row can pass ROW_BYTES up to this, so a wide list still shows its shape. */
12
+ const FIRST_ROW_BYTES = 3_000;
13
+ /** Everything outside the list stays whole, or the read is not a list. */
14
+ const REST_BYTES = 1_024;
15
+ const TEXT_HEAD_BYTES = 1_200;
16
+ const TEXT_TAIL_BYTES = 800;
17
+ const MIN_ROWS = 3;
18
+ const MAX_ROW_DEPTH = 3;
19
+ const MAX_KEYS = 40;
20
+ const MAX_KEY_CHARS = 64;
21
+ /** Rows sampled for key names. */
22
+ const KEY_SAMPLE_ROWS = 50;
23
+ function isRecord(value) {
24
+ return typeof value === "object" && value !== null && !Array.isArray(value);
25
+ }
26
+ const byteLength = (text) => enc.encode(text).length;
27
+ /** The largest list of records within MAX_ROW_DEPTH keys of the root. */
28
+ function dominantRows(root) {
29
+ let best;
30
+ const visit = (node, path) => {
31
+ if (Array.isArray(node)) {
32
+ if (node.length >= MIN_ROWS && node.every((row) => isRecord(row) || Array.isArray(row))) {
33
+ const bytes = byteLength(JSON.stringify(node));
34
+ if (!best || bytes > best.bytes)
35
+ best = { path, rows: node, bytes };
36
+ }
37
+ return;
38
+ }
39
+ if (!isRecord(node) || path.length >= MAX_ROW_DEPTH)
40
+ return;
41
+ for (const key of Object.keys(node))
42
+ visit(node[key], [...path, key]);
43
+ };
44
+ visit(root, []);
45
+ return best;
46
+ }
47
+ function withRows(root, path, rows) {
48
+ if (!path.length)
49
+ return rows;
50
+ const [head, ...tail] = path;
51
+ const record = root;
52
+ // Computed keys define own properties, so a key named __proto__ stays data.
53
+ return { ...record, [head]: withRows(record[head], tail, rows) };
54
+ }
55
+ function rowKeys(rows) {
56
+ const keys = new Set();
57
+ for (const row of rows.slice(0, KEY_SAMPLE_ROWS)) {
58
+ if (!isRecord(row))
59
+ continue;
60
+ for (const key of Object.keys(row))
61
+ if (key.length <= MAX_KEY_CHARS)
62
+ keys.add(key);
63
+ if (keys.size >= MAX_KEYS)
64
+ break;
65
+ }
66
+ return [...keys].slice(0, MAX_KEYS);
67
+ }
68
+ /** JavaScript that reads `path` from a parsed value; a program runs it, so it must be exact. */
69
+ function accessor(path) {
70
+ return path.map((key) => (/^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`)).join("");
71
+ }
72
+ function rowsPreview(parsed, total, cap) {
73
+ const found = dominantRows(parsed);
74
+ if (!found || found.bytes * 2 < total)
75
+ return undefined;
76
+ // The bytes outside the list; the ways it can be spelled do not change them.
77
+ const rest = byteLength(JSON.stringify(withRows(parsed, found.path, [])));
78
+ if (rest > REST_BYTES)
79
+ return undefined;
80
+ const firstRowBytes = Math.min(FIRST_ROW_BYTES, cap - rest);
81
+ const rowBytes = Math.min(ROW_BYTES, cap - rest);
82
+ const shown = [];
83
+ let used = 0;
84
+ for (const row of found.rows) {
85
+ const size = byteLength(JSON.stringify(row)) + 1;
86
+ if (used + size > (shown.length ? rowBytes : firstRowBytes))
87
+ break;
88
+ shown.push(row);
89
+ used += size;
90
+ }
91
+ const count = found.rows.length;
92
+ // An empty list would read as a list with no rows, and a whole one needs no sample.
93
+ if (!shown.length || shown.length === count)
94
+ return undefined;
95
+ const keys = rowKeys(found.rows);
96
+ const path = found.path.length ? accessor(found.path).replace(/^\./, "") : "(root)";
97
+ const body = JSON.stringify(withRows(parsed, found.path, shown));
98
+ if (byteLength(body) > cap)
99
+ return undefined;
100
+ return {
101
+ fields: { rows: { path, count, shown: shown.length, keys } },
102
+ body,
103
+ shown: `Sample only: ${shown.length === 1 ? "row 1" : `rows 1-${shown.length}`} of ${count} at ${path} follow, in the result's own shape, with every value outside the list; the other ${count - shown.length} are not shown.`,
104
+ noun: "rows",
105
+ reduce: `const rows = JSON.parse(text)${accessor(found.path)}; return { count: rows.length, rows: rows.slice(0, 5) };`,
106
+ };
107
+ }
108
+ function lineCount(text) {
109
+ let lines = 1;
110
+ for (let at = text.indexOf("\n"); at !== -1; at = text.indexOf("\n", at + 1))
111
+ lines++;
112
+ return lines;
113
+ }
114
+ /** Whether a byte continues a UTF-8 character rather than starting one. */
115
+ const continues = (byte) => byte !== undefined && (byte & 0xc0) === 0x80;
116
+ function textPreview(text, reading, cap) {
117
+ const bytes = enc.encode(text);
118
+ if (bytes.length <= TEXT_HEAD_BYTES + TEXT_TAIL_BYTES)
119
+ return undefined;
120
+ const marker = (omitted) => `\n…[${omitted} bytes omitted]…\n`;
121
+ let headEnd = TEXT_HEAD_BYTES;
122
+ while (continues(bytes[headEnd]))
123
+ headEnd--;
124
+ let tailStart = bytes.length - TEXT_TAIL_BYTES;
125
+ while (continues(bytes[tailStart]))
126
+ tailStart++;
127
+ const shownBytes = headEnd + bytes.length - tailStart;
128
+ const body = `${dec.decode(bytes.subarray(0, headEnd))}${marker(tailStart - headEnd)}${dec.decode(bytes.subarray(tailStart))}`;
129
+ if (byteLength(body) > cap)
130
+ return undefined;
131
+ const lines = lineCount(text);
132
+ return {
133
+ fields: { text: { lines, shownBytes } },
134
+ body,
135
+ shown: `Sample only: the first ${headEnd} and last ${bytes.length - tailStart} bytes of ${lines} lines follow; the ${tailStart - headEnd} bytes between them are not shown.`,
136
+ noun: "text",
137
+ reduce: `const lines = ${reading}.split("\\n"); return { lines: lines.length, matches: lines.filter((line) => /error|fail/i.test(line)).map((line) => line.slice(0, 500)).slice(0, 40) };`,
138
+ };
139
+ }
140
+ /** Parts of a JSON number token whose exact value is the same however it is spelled. */
141
+ function exactValue(token) {
142
+ const match = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(token);
143
+ if (!match)
144
+ return undefined;
145
+ const fraction = match[3] ?? "";
146
+ const digits = `${match[2]}${fraction}`;
147
+ let start = 0;
148
+ let end = digits.length;
149
+ while (digits[start] === "0")
150
+ start++;
151
+ while (end > start && digits[end - 1] === "0")
152
+ end--;
153
+ if (start === end)
154
+ return `${match[1]}0`;
155
+ return `${match[1]}${digits.slice(start, end)}e${Number(match[4] ?? 0) - fraction.length + digits.length - end}`;
156
+ }
157
+ /** Whether JSON.parse then JSON.stringify keeps this token's value, including its sign and range. */
158
+ function keepsNumber(token) {
159
+ const printed = String(Number(token));
160
+ return printed === token || exactValue(printed) === exactValue(token);
161
+ }
162
+ /**
163
+ * Whether every number in text JSON.parse has already accepted survives parsing.
164
+ * Both a view and the offered program parse it, so one lossy number anywhere,
165
+ * inside or outside the rows, declines the view. Strings are skipped, not read.
166
+ */
167
+ function keepsNumbers(json) {
168
+ let at = 0;
169
+ while (at < json.length) {
170
+ const code = json.charCodeAt(at);
171
+ if (code === 34) {
172
+ at++;
173
+ while (json.charCodeAt(at) !== 34)
174
+ at += json.charCodeAt(at) === 92 ? 2 : 1;
175
+ at++;
176
+ }
177
+ else if (code === 45 || (code >= 48 && code <= 57)) {
178
+ const start = at;
179
+ while (at < json.length && "+-.0123456789Ee".includes(json[at]))
180
+ at++;
181
+ if (!keepsNumber(json.slice(start, at)))
182
+ return false;
183
+ }
184
+ else
185
+ at++;
186
+ }
187
+ return true;
188
+ }
189
+ /**
190
+ * The view of a read past its cap, or undefined when its first bytes should stand:
191
+ * documents, scalars, shallow or deep lists a view would misstate, and numbers
192
+ * parsing would change.
193
+ *
194
+ * `format` is the result's own: an API connector's text arrives here as one
195
+ * JSON string, whose decoded text is what a person would call the result.
196
+ */
197
+ export function previewResult(text, format, cap) {
198
+ let parsed;
199
+ try {
200
+ parsed = JSON.parse(text);
201
+ }
202
+ catch {
203
+ return textPreview(text, "text", cap);
204
+ }
205
+ if (typeof parsed === "string" && format === "text")
206
+ return textPreview(parsed, "JSON.parse(text)", cap);
207
+ if (typeof parsed !== "object" || parsed === null || !keepsNumbers(text))
208
+ return undefined;
209
+ return rowsPreview(parsed, byteLength(text), cap);
210
+ }