@zackbart/connecta 0.41.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +61 -0
- package/bin/connecta.mjs +26 -17
- package/bin/meta-tool-names.d.mts +3 -1
- package/bin/meta-tool-names.mjs +4 -1
- package/bin/version.mjs +1 -1
- package/dist/activity.d.ts +2 -1
- package/dist/activity.js +5 -5
- package/dist/advertised-schema.js +15 -0
- package/dist/agent-routes.js +31 -5
- package/dist/auth/downstream-oauth/refresh-coordinator.js +4 -1
- package/dist/auth/downstream-oauth.js +3 -0
- package/dist/auth/static-oauth.js +26 -18
- package/dist/auth-elicitation.js +118 -61
- package/dist/authorization-handoff.js +36 -3
- package/dist/catalog-cache.js +292 -24
- package/dist/catalog-freshness.js +12 -0
- package/dist/catalog-service.js +373 -70
- package/dist/catalog.js +147 -58
- package/dist/config-defaults.d.ts +2 -1
- package/dist/config-defaults.js +4 -1
- package/dist/config.d.ts +43 -17
- package/dist/config.js +26 -10
- package/dist/connectors/remote-mcp/catalog.js +52 -28
- package/dist/connectors/remote-mcp/connection.js +62 -5
- package/dist/connectors/remote-mcp.js +10 -1
- package/dist/describe-config.d.ts +3 -2
- package/dist/discovery-health.js +26 -11
- package/dist/errors.d.ts +52 -3
- package/dist/errors.js +86 -3
- package/dist/execute.js +476 -200
- package/dist/executor-admission.js +2 -0
- package/dist/executor-contract.js +6 -0
- package/dist/executor-result.js +2 -2
- package/dist/executors/quickjs-child.js +13 -4
- package/dist/executors/quickjs-runtime.d.ts +2 -0
- package/dist/executors/quickjs-runtime.js +36 -19
- package/dist/executors/quickjs.js +23 -6
- package/dist/guest-functions.js +5 -4
- package/dist/guest-runtime.js +37 -1
- package/dist/guest-types.d.ts +18 -0
- package/dist/http/client.js +7 -2
- package/dist/http/errors.js +9 -4
- package/dist/http/fields.js +17 -5
- package/dist/http/hatch.js +9 -3
- package/dist/http/paging.js +1 -1
- package/dist/http/provider.js +2 -6
- package/dist/http/vendor.js +6 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +5 -0
- package/dist/invocation.js +98 -34
- package/dist/meta-output.js +24 -18
- package/dist/meta-tool-names.d.ts +1 -1
- package/dist/meta-tool-names.js +1 -1
- package/dist/meta-tools.js +259 -60
- package/dist/native-recovery.js +4 -1
- package/dist/native-tools.js +166 -51
- package/dist/node.js +3 -1
- package/dist/oauth-handoff.js +53 -43
- package/dist/operation-activity.js +2 -1
- package/dist/operation-vocabulary.js +6 -0
- package/dist/operator-record.d.ts +1 -1
- package/dist/operator-record.js +1 -0
- package/dist/operator-ui/generated.js +2 -2
- package/dist/program-result.js +2 -0
- package/dist/program-source.js +1 -1
- package/dist/program-writes.js +112 -19
- package/dist/provider.d.ts +0 -13
- package/dist/provider.js +0 -3
- package/dist/providers/_shared/google/kit.js +2 -2
- package/dist/providers/breeze/api.js +2 -2
- package/dist/providers/breeze/skill.generated.js +1 -1
- package/dist/providers/ccb/api.js +2 -2
- package/dist/providers/ccb/skill.generated.js +1 -1
- package/dist/providers/cloudflare/api.js +2 -2
- package/dist/providers/cloudflare/index.js +2 -0
- package/dist/providers/cloudflare/skill.generated.js +1 -1
- package/dist/providers/docs/skill.generated.js +1 -1
- package/dist/providers/drive/skill.generated.js +1 -1
- package/dist/providers/forms/index.js +1 -0
- package/dist/providers/forms/skill.generated.js +1 -1
- package/dist/providers/github/api.js +98 -6
- package/dist/providers/github/index.js +7 -2
- package/dist/providers/github/lib/auth.d.ts +3 -1
- package/dist/providers/github/lib/auth.js +43 -12
- package/dist/providers/github/lib/discovery.js +11 -1
- package/dist/providers/github/lib/operations.js +8 -0
- package/dist/providers/github/safety.js +39 -0
- package/dist/providers/github/skill.generated.js +1 -1
- package/dist/providers/github/tools.js +12 -5
- package/dist/providers/gmail/index.js +1 -0
- package/dist/providers/gmail/skill.generated.js +1 -1
- package/dist/providers/granola/skill.generated.js +1 -1
- package/dist/providers/overflow/index.js +2 -0
- package/dist/providers/overflow/skill.generated.js +1 -1
- package/dist/providers/planning-center/index.js +11 -5
- package/dist/providers/planning-center/lib/constants.d.ts +1 -1
- package/dist/providers/planning-center/skill.generated.js +1 -1
- package/dist/providers/quickbooks/index.js +2 -5
- package/dist/providers/quickbooks/skill.generated.js +1 -1
- package/dist/providers/sheets/index.js +1 -0
- package/dist/providers/sheets/skill.generated.js +1 -1
- package/dist/providers/slides/index.js +2 -2
- package/dist/providers/slides/skill.generated.js +1 -1
- package/dist/providers/stripe/skill.generated.js +1 -1
- package/dist/providers/tithely/index.js +1 -0
- package/dist/providers/tithely/skill.generated.js +1 -1
- package/dist/providers/vercel/api.js +2 -2
- package/dist/providers/vercel/skill.generated.js +1 -1
- package/dist/registry.d.ts +36 -4
- package/dist/registry.js +127 -27
- package/dist/result-preview.js +210 -0
- package/dist/result-seal.js +168 -0
- package/dist/route-names.js +13 -4
- package/dist/routes/connect.js +21 -13
- package/dist/routes/mcp.js +55 -26
- package/dist/routes/shared.d.ts +6 -0
- package/dist/runtime/call.js +15 -10
- package/dist/runtime/request-scope.js +36 -1
- package/dist/runtime/run.js +4 -1
- package/dist/sent-secrets.d.ts +20 -0
- package/dist/sent-secrets.js +49 -1
- package/dist/skills.js +100 -33
- package/dist/storage/keys.js +9 -6
- package/dist/storage/sql-native.js +1 -1
- package/dist/tool-safety.d.ts +33 -0
- package/dist/tool-safety.js +51 -15
- package/dist/types.d.ts +31 -0
- package/dist/usage-guide.js +14 -13
- package/dist/validate.js +29 -10
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worker.js +28 -8
- package/documentation/architecture.md +104 -42
- package/documentation/auth.md +46 -19
- package/documentation/code-mode.md +378 -116
- package/documentation/deploying.md +10 -9
- package/documentation/hosted-mcp-migration.md +9 -3
- package/documentation/integrating.md +27 -5
- package/documentation/meta-tools.md +655 -153
- package/documentation/operating.md +5 -3
- package/documentation/providers.md +18 -6
- package/examples/worker/README.md +3 -3
- package/guides/how-it-works.md +2 -1
- package/guides/safety-and-trust.md +1 -3
- package/package.json +1 -1
- package/templates/node/AGENTS.md +3 -3
- package/templates/node/README.md +1 -1
- package/templates/node/package.json +1 -1
package/dist/registry.js
CHANGED
|
@@ -6,15 +6,15 @@ import { authCall, authInvoke } from "./auth/native.js";
|
|
|
6
6
|
import { Clock, Duration, Effect, Random } from "effect";
|
|
7
7
|
import { activityRequest } from "./activity-request.js";
|
|
8
8
|
import { aggregateCallAdmissionSnapshots, ConnectorCallAdmissionController, } from "./call-admission.js";
|
|
9
|
-
import { attachCatalogCache, catalogExpiry, catalogFetchedAt, customCatalogFallback, invalidateCatalogCache, observeUncachedCatalogRefresh, retainedCatalogIdentity, storeCustomCatalogFallback, } from "./catalog-cache.js";
|
|
10
|
-
import { carryCatalogFreshness, markCatalogFreshness } from "./catalog-freshness.js";
|
|
9
|
+
import { attachCatalogCache, catalogExpiry, catalogFetchedAt, bindCustomCatalogListing, customCatalogFallback, customCatalogPeek, fenceCustomCatalog, invalidateCatalogCache, observeUncachedCatalogRefresh, retainedCatalogIdentity, storeCustomCatalogFallback, } from "./catalog-cache.js";
|
|
10
|
+
import { carryCatalogFreshness, markCatalogFreshness, markCatalogStale } from "./catalog-freshness.js";
|
|
11
11
|
import { MAX_CATALOG_TOOLS, MAX_SERIALIZED_CATALOG_BYTES } from "./catalog-limits.js";
|
|
12
12
|
import { CONFIG_DEFAULTS } from "./config-defaults.js";
|
|
13
13
|
import { usesCallerIdentity } from "./connector-caller.js";
|
|
14
14
|
import { forwardsPrincipal } from "./request-principal.js";
|
|
15
15
|
import { canonicalJson } from "./canonical-json.js";
|
|
16
16
|
import { storedCredentialShape } from "./credential-rules.js";
|
|
17
|
-
import { ConnectorCallError } from "./errors.js";
|
|
17
|
+
import { ConnectorCallError, isAccessRefusal } from "./errors.js";
|
|
18
18
|
import { attachOAuthPartition, oauthPartitionIdle } from "./oauth-partition.js";
|
|
19
19
|
import { attachOAuthSealer, vaultOAuthSealer } from "./oauth-sealing.js";
|
|
20
20
|
import { boundedStatus, failureRecord, failureStatus, logFailure, ownStatus } from "./operator-record.js";
|
|
@@ -34,7 +34,7 @@ import { redactCatalog, sentSecretsFor, sentSecretsForRequest } from "./sent-sec
|
|
|
34
34
|
import { GUIDE_SUMMARY_LENGTH, normalizeGuideSummary, connectorGuide, connectorGuideSummary } from "./skills.js";
|
|
35
35
|
import { jsonCodec, OAUTH_HANDOFF_TTL_SECONDS, oauthHandoffKeys, resultKeys, scopes, stashExpiryKeys, stashLedgerKeys, } from "./storage/keys.js";
|
|
36
36
|
import { assertStaticToolNames, hasControlCharacters } from "./tool-name.js";
|
|
37
|
-
import { carryToolClassification, classifyTool } from "./tool-safety.js";
|
|
37
|
+
import { carryToolClassification, classifyTool, sourceDeclaresWrite, } from "./tool-safety.js";
|
|
38
38
|
import { isWorkspaceAddress } from "./workspace-address.js";
|
|
39
39
|
import { workspaceSubjectOf } from "./workspace-subject.js";
|
|
40
40
|
const ID_RE = /^[a-z0-9_-]+$/;
|
|
@@ -145,12 +145,18 @@ const STASH_WRITE_TIMEOUT_MS = 30_000;
|
|
|
145
145
|
const STASH_COMPLETION_TIMEOUT_MS = 15_000;
|
|
146
146
|
/** Reservation, chunk writes and completion share one finite outer budget. */
|
|
147
147
|
const STASH_TIMEOUT_MS = STASH_WRITE_TIMEOUT_MS + STASH_COMPLETION_TIMEOUT_MS;
|
|
148
|
+
/** Chunks one release deletes at a time; a stash holds at most 32. */
|
|
149
|
+
const STASH_RELEASE_CONCURRENCY = 16;
|
|
148
150
|
/** Stop waiting without starting background storage work after the deadline. */
|
|
149
151
|
function completeStash(effect) {
|
|
150
152
|
return effect.pipe(Effect.asVoid, Effect.timeoutOrElse({ duration: Duration.millis(STASH_COMPLETION_TIMEOUT_MS), orElse: () => Effect.void }),
|
|
151
153
|
// onError finalizers are otherwise uninterruptible, including their I/O.
|
|
152
154
|
Effect.interruptible);
|
|
153
155
|
}
|
|
156
|
+
/** The reservation id of one result id, which is random and never reused. */
|
|
157
|
+
function stashCharge(id) {
|
|
158
|
+
return Effect.tryPromise({ try: () => sha256Hex(id), catch: (error) => error });
|
|
159
|
+
}
|
|
154
160
|
/**
|
|
155
161
|
* Live charges in a stored ledger. Anything malformed reads as empty: the
|
|
156
162
|
* ledger bounds an advisory cache, and the next successful swap replaces it.
|
|
@@ -317,8 +323,8 @@ export class Registry {
|
|
|
317
323
|
Object.freeze(this.classification);
|
|
318
324
|
for (const [id, overrides] of Object.entries(this.classification)) {
|
|
319
325
|
for (const [name, verdict] of Object.entries(overrides)) {
|
|
320
|
-
if (!name ||
|
|
321
|
-
throw new Error(`ConnectaConfig.classification.${id}.${name}: expected "read", "write", or "hidden"`);
|
|
326
|
+
if (!name || !["read", "read-repeatable", "write", "hidden"].includes(verdict)) {
|
|
327
|
+
throw new Error(`ConnectaConfig.classification.${id}.${name}: expected "read", "read-repeatable", "write", or "hidden"`);
|
|
322
328
|
}
|
|
323
329
|
}
|
|
324
330
|
const connector = this.connectors.get(id);
|
|
@@ -617,8 +623,11 @@ export class Registry {
|
|
|
617
623
|
* unconfirmed and retain their charge. Cleanup/release and settlement each
|
|
618
624
|
* stop waiting after 15 seconds. Storage TTL reclaims the rows.
|
|
619
625
|
*/
|
|
620
|
-
stashResult(id, chunks, ttlSeconds,
|
|
621
|
-
return runOnPartition(registryEdgeProgram(Registry.prototype, this, "stashResult", id, chunks, ttlSeconds,
|
|
626
|
+
stashResult(id, chunks, ttlSeconds, options = {}) {
|
|
627
|
+
return runOnPartition(registryEdgeProgram(Registry.prototype, this, "stashResult", id, chunks, ttlSeconds, options), this.opts);
|
|
628
|
+
}
|
|
629
|
+
releaseResult(id, chunkCount, partition = scopes.results) {
|
|
630
|
+
return runOnPartition(registryEdgeProgram(Registry.prototype, this, "releaseResult", id, chunkCount, partition), this.opts);
|
|
622
631
|
}
|
|
623
632
|
/**
|
|
624
633
|
* Storage namespaced to the root result partition, kept separate from any
|
|
@@ -681,7 +690,16 @@ export class Registry {
|
|
|
681
690
|
if (connector.staticTools)
|
|
682
691
|
return markCatalogFreshness(connector.staticTools, Infinity);
|
|
683
692
|
const ctx = this.contextFor(id, baseUrl, requestScope, callOptions, scope);
|
|
684
|
-
|
|
693
|
+
// `remoteMcp()` binds and fences its own attempts, knowing how each was made.
|
|
694
|
+
const customListing = connector.peekCatalog
|
|
695
|
+
? undefined
|
|
696
|
+
: yield* authCall(bindCustomCatalogListing, [ctx, id]).pipe(Effect.catch((failure) => Effect.sync(() => {
|
|
697
|
+
logFailure(this.opts.logger, "catalog read failed", failureRecord({ connector: id }, failure));
|
|
698
|
+
return undefined;
|
|
699
|
+
})));
|
|
700
|
+
const listed = yield* authInvoke(connector, "listTools", [ctx]).pipe(Effect.catch((error) => Effect.gen({ self: this }, function* () {
|
|
701
|
+
if (customListing && isAccessRefusal(error))
|
|
702
|
+
yield* authCall(fenceCustomCatalog, [ctx, id, customListing]).pipe(Effect.catch((failure) => Effect.sync(() => logFailure(this.opts.logger, "catalog fence failed", failureRecord({ connector: id }, failure)))));
|
|
685
703
|
const fallback = error instanceof ConnectorCallError && error.code === "unavailable"
|
|
686
704
|
? yield* authCall(customCatalogFallback, [ctx, id]).pipe(Effect.catch(() => Effect.succeed(undefined)))
|
|
687
705
|
: undefined;
|
|
@@ -697,12 +715,18 @@ export class Registry {
|
|
|
697
715
|
const facts = frozenFacts(structuredClone(tools));
|
|
698
716
|
const accepted = this.acceptToolNames(facts);
|
|
699
717
|
if (catalogFetchedAt(ctx) === undefined) {
|
|
700
|
-
for (const observe of [
|
|
701
|
-
|
|
718
|
+
for (const observe of [
|
|
719
|
+
authCall(storeCustomCatalogFallback, [ctx, facts, customListing]),
|
|
720
|
+
authCall(observeUncachedCatalogRefresh, [ctx, id, facts]),
|
|
721
|
+
]) {
|
|
722
|
+
yield* observe.pipe(Effect.catch((error) => Effect.sync(() => logFailure(this.opts.logger, "catalog refresh observation failed", failureRecord({ connector: id }, error)))));
|
|
702
723
|
}
|
|
703
724
|
}
|
|
704
725
|
const provenance = catalogExpiry(ctx);
|
|
705
726
|
const freshUntil = provenance && !provenance.staleFallback ? provenance.expiresAt : 0;
|
|
727
|
+
// A transient-failure fallback answers the listing; discovery still reports its age.
|
|
728
|
+
if (provenance?.staleFallback)
|
|
729
|
+
markCatalogStale(accepted, { code: "unavailable", ageMs: Math.max(0, Date.now() - provenance.fetchedAt) });
|
|
706
730
|
// Principal-forwarded listings belong to one caller, never to the shared
|
|
707
731
|
// operator observation store. The actual catalog cache is identity-bound.
|
|
708
732
|
if (!forwardsPrincipal(connector)) {
|
|
@@ -715,6 +739,30 @@ export class Registry {
|
|
|
715
739
|
return markCatalogFreshness(accepted, freshUntil);
|
|
716
740
|
});
|
|
717
741
|
}
|
|
742
|
+
/** See {@link RegistryView.peekCatalog}. Same classification and filtering as `loadTools`. */
|
|
743
|
+
loadPeek(id, baseUrl, requestScope, callOptions = {}, scope) {
|
|
744
|
+
return Effect.gen({ self: this }, function* () {
|
|
745
|
+
const connector = this.connectors.get(id);
|
|
746
|
+
if (!connector || connector.staticTools)
|
|
747
|
+
return undefined;
|
|
748
|
+
const ctx = this.contextFor(id, baseUrl, requestScope, callOptions, scope);
|
|
749
|
+
const peek = connector.peekCatalog
|
|
750
|
+
? yield* authInvoke(connector, "peekCatalog", [ctx])
|
|
751
|
+
: yield* authCall(customCatalogPeek, [ctx, id]);
|
|
752
|
+
if (!peek)
|
|
753
|
+
return undefined;
|
|
754
|
+
const facts = redactCatalog(ctx, peek.tools).map(({ classification: _ignored, ...fact }) => fact);
|
|
755
|
+
const overrides = this.classification[id];
|
|
756
|
+
return {
|
|
757
|
+
tools: this.acceptToolNames(frozenFacts(structuredClone(facts)))
|
|
758
|
+
.filter((tool) => overrides?.[tool.name] !== "hidden")
|
|
759
|
+
.map((tool) => this.publishTool(id, tool)),
|
|
760
|
+
fetchedAt: peek.fetchedAt,
|
|
761
|
+
// A custom catalog is always listed anew, so its retained copy is never current.
|
|
762
|
+
fresh: connector.peekCatalog !== undefined && peek.expiresAt > Date.now(),
|
|
763
|
+
};
|
|
764
|
+
});
|
|
765
|
+
}
|
|
718
766
|
/** The served catalog excludes control-character names; raw caches stay complete. */
|
|
719
767
|
acceptToolNames(tools) {
|
|
720
768
|
const accepted = tools.filter((tool) => !hasControlCharacters(tool.name));
|
|
@@ -741,19 +789,20 @@ export class Registry {
|
|
|
741
789
|
}
|
|
742
790
|
publishTool(id, tool) {
|
|
743
791
|
const override = this.classification[id]?.[tool.name];
|
|
792
|
+
const classification = classifyTool(tool, override === "hidden" ? undefined : override);
|
|
744
793
|
return carryToolClassification(tool, {
|
|
745
794
|
...structuredClone(tool),
|
|
746
|
-
classification
|
|
795
|
+
classification,
|
|
747
796
|
...(override !== undefined
|
|
748
797
|
? {
|
|
749
798
|
annotations: {
|
|
750
799
|
...structuredClone(tool.annotations),
|
|
751
|
-
readOnlyHint: override === "read",
|
|
752
|
-
destructiveHint: override === "write",
|
|
800
|
+
readOnlyHint: override !== "hidden" && classification === "read",
|
|
801
|
+
destructiveHint: override !== "hidden" && classification === "write",
|
|
753
802
|
},
|
|
754
803
|
}
|
|
755
804
|
: {}),
|
|
756
|
-
});
|
|
805
|
+
}, override === "read-repeatable" || undefined);
|
|
757
806
|
}
|
|
758
807
|
/** Construction-only view for the secret-free static config description. */
|
|
759
808
|
describeStaticTools(id) {
|
|
@@ -771,19 +820,30 @@ export class Registry {
|
|
|
771
820
|
return;
|
|
772
821
|
this.overrideWarningCatalogs.set(partition, catalog);
|
|
773
822
|
}
|
|
774
|
-
const known = new
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
if (
|
|
823
|
+
const known = new Map(tools.map((tool) => [tool.name, tool]));
|
|
824
|
+
const staticCatalog = this.connectors.get(id)?.staticTools !== undefined;
|
|
825
|
+
for (const [name, override] of Object.entries(overrides ?? {})) {
|
|
826
|
+
const tool = known.get(name);
|
|
827
|
+
if (!tool && staticCatalog) {
|
|
779
828
|
throw new Error(`ConnectaConfig.classification: connector "${id}" has no tool "${name}"`);
|
|
780
829
|
}
|
|
781
830
|
// Only deployment-authored names, never downstream text. JSON quoting
|
|
782
831
|
// prevents control characters from forging operator log lines.
|
|
783
832
|
const quoted = JSON.stringify(`${id}.${name}`).replace(/[\u2028\u2029]/g, (ch) => `\\u${ch.charCodeAt(0).toString(16)}`);
|
|
833
|
+
if (tool) {
|
|
834
|
+
// A static catalog cannot change, so the deployment's construction reports it once.
|
|
835
|
+
const reports = staticCatalog ? partition === undefined && this.opts.constructionChecks !== false : true;
|
|
836
|
+
if (override === "read" && reports && sourceDeclaresWrite(tool)) {
|
|
837
|
+
this.opts.logger.warn(`classification override ${quoted} to "read" is not applied because the connector declares this tool a write; it stays a write. "read-repeatable" serves it as a read and accepts that hosts may repeat it`);
|
|
838
|
+
}
|
|
839
|
+
continue;
|
|
840
|
+
}
|
|
784
841
|
this.opts.logger.warn(`classification override ${quoted} is dormant because the runtime catalog lists no such tool`);
|
|
785
842
|
}
|
|
786
843
|
}
|
|
844
|
+
peekCatalog(id, baseUrl, requestScope, callOptions = {}, scope) {
|
|
845
|
+
return runEdge(registryEdgeProgram(Registry.prototype, this, "peekCatalog", id, baseUrl, requestScope, callOptions, scope));
|
|
846
|
+
}
|
|
787
847
|
getTools(id, baseUrl, requestScope, callOptions = {}, scope) {
|
|
788
848
|
return runEdge(registryEdgeProgram(Registry.prototype, this, "getTools", id, baseUrl, requestScope, callOptions, scope));
|
|
789
849
|
}
|
|
@@ -818,7 +878,8 @@ export class Registry {
|
|
|
818
878
|
}
|
|
819
879
|
static {
|
|
820
880
|
provideRegistryPrograms(Registry.prototype, {
|
|
821
|
-
stashResult: (self, id, chunks, ttlSeconds,
|
|
881
|
+
stashResult: (self, id, chunks, ttlSeconds, options = {}) => Effect.gen({ self }, function* () {
|
|
882
|
+
const partition = options.partition ?? scopes.results;
|
|
822
883
|
const maxBytes = this.opts.results?.maxStashBytes ?? CONFIG_DEFAULTS.results.maxStashBytes;
|
|
823
884
|
const maxEntries = this.opts.results?.maxStashEntries ?? CONFIG_DEFAULTS.results.maxStashEntries;
|
|
824
885
|
// The paging envelope is ASCII, so its string length is its stored byte count.
|
|
@@ -826,8 +887,9 @@ export class Registry {
|
|
|
826
887
|
if (bytes > maxBytes || maxEntries === 0)
|
|
827
888
|
return false;
|
|
828
889
|
const keys = chunks.map((_, index) => partition + resultKeys.chunk(id, index));
|
|
829
|
-
//
|
|
830
|
-
|
|
890
|
+
// A digest finds this reservation again on release without putting
|
|
891
|
+
// the result's ID in the deployment-wide ledger.
|
|
892
|
+
const charge = yield* stashCharge(id);
|
|
831
893
|
const ttlMs = ttlSeconds * 1000;
|
|
832
894
|
if (!Number.isSafeInteger(ttlMs) || ttlMs <= 0)
|
|
833
895
|
return false;
|
|
@@ -882,7 +944,8 @@ export class Registry {
|
|
|
882
944
|
const remaining = Math.max(0, writeDeadline - (yield* Clock.currentTimeMillis));
|
|
883
945
|
const written = yield* Effect.gen(function* () {
|
|
884
946
|
const multi = getNativeStorage(self.opts.storage)?.setMany;
|
|
885
|
-
|
|
947
|
+
const lowest = options.holdFirst ? 1 : 0;
|
|
948
|
+
for (let index = keys.length - 1; index >= lowest;) {
|
|
886
949
|
const first = multi && index > 0 ? Math.max(1, index - 63) : index;
|
|
887
950
|
const before = yield* Clock.currentTimeMillis;
|
|
888
951
|
if (before >= writeDeadline || before >= chunkExpiry)
|
|
@@ -905,6 +968,15 @@ export class Registry {
|
|
|
905
968
|
yield* completeStash(settle);
|
|
906
969
|
return written;
|
|
907
970
|
}).pipe(Effect.timeoutOrElse({ duration: Duration.millis(STASH_TIMEOUT_MS), orElse: () => Effect.succeed(false) }), Effect.provideService(Storage, self.opts.storage)),
|
|
971
|
+
releaseResult: (self, id, chunkCount, partition = scopes.results) => completeStash(Effect.gen(function* () {
|
|
972
|
+
const charge = yield* stashCharge(id);
|
|
973
|
+
// Bytes first: a failed delete keeps its charge, so the ledger never
|
|
974
|
+
// undercounts what storage holds.
|
|
975
|
+
yield* Effect.forEach(Array.from({ length: chunkCount }, (_, index) => partition + resultKeys.chunk(id, index)), (key) => storageDelete(key), { concurrency: STASH_RELEASE_CONCURRENCY, discard: true });
|
|
976
|
+
yield* swapStashLedger((live) => live.some((entry) => entry[0] === charge)
|
|
977
|
+
? { entries: live.filter((entry) => entry[0] !== charge), result: undefined }
|
|
978
|
+
: { result: undefined });
|
|
979
|
+
})).pipe(Effect.ignore, Effect.provideService(Storage, self.opts.storage)),
|
|
908
980
|
recheckResultAccess: () => Effect.succeed(true),
|
|
909
981
|
storeOAuthHandoff: (self, connectorId, state, principalKey) => Effect.gen(function* () {
|
|
910
982
|
const key = oauthHandoffKeys.handoff(connectorId, yield* Effect.tryPromise({ try: () => sha256Hex(state), catch: (error) => error }));
|
|
@@ -973,6 +1045,12 @@ export class Registry {
|
|
|
973
1045
|
return yield* load.join(signal);
|
|
974
1046
|
}));
|
|
975
1047
|
}),
|
|
1048
|
+
peekCatalog: (self, id, baseUrl, requestScope, callOptions = {}, scope) => Effect.suspend(() => {
|
|
1049
|
+
if (!self.connectors.has(id))
|
|
1050
|
+
return Effect.succeed(undefined);
|
|
1051
|
+
const owner = self.contextFor(id, baseUrl, requestScope, callOptions, scope);
|
|
1052
|
+
return duringViewEffect(owner, (signal) => self.loadPeek(id, baseUrl, owner.requestScope, { ...callOptions, signal }, scope));
|
|
1053
|
+
}),
|
|
976
1054
|
credentialDriftFor: (self, id, requestScope) => Effect.gen(function* () {
|
|
977
1055
|
const credential = self.connectors.get(id)?.credential;
|
|
978
1056
|
const vault = self.opts.credentialVault;
|
|
@@ -1097,6 +1175,9 @@ class ScopedRegistryView {
|
|
|
1097
1175
|
getTools(...args) {
|
|
1098
1176
|
return runEdge(registryEdgeProgram(ScopedRegistryView.prototype, this, "getTools", ...args));
|
|
1099
1177
|
}
|
|
1178
|
+
peekCatalog(...args) {
|
|
1179
|
+
return runEdge(registryEdgeProgram(ScopedRegistryView.prototype, this, "peekCatalog", ...args));
|
|
1180
|
+
}
|
|
1100
1181
|
contextFor(...args) {
|
|
1101
1182
|
const registry = this.registryFor(args[0]);
|
|
1102
1183
|
if (!registry)
|
|
@@ -1110,8 +1191,14 @@ class ScopedRegistryView {
|
|
|
1110
1191
|
}
|
|
1111
1192
|
return registry.admitCall(...args);
|
|
1112
1193
|
}
|
|
1113
|
-
stashResult(id, chunks, ttlSeconds) {
|
|
1114
|
-
return this.root.stashResult(id, chunks, ttlSeconds,
|
|
1194
|
+
stashResult(id, chunks, ttlSeconds, options = {}) {
|
|
1195
|
+
return this.root.stashResult(id, chunks, ttlSeconds, {
|
|
1196
|
+
...options,
|
|
1197
|
+
partition: this.scope.subjectKey ? scopes.subject(this.scope.subjectKey) : scopes.results,
|
|
1198
|
+
});
|
|
1199
|
+
}
|
|
1200
|
+
releaseResult(id, chunkCount) {
|
|
1201
|
+
return this.root.releaseResult(id, chunkCount, this.scope.subjectKey ? scopes.subject(this.scope.subjectKey) : scopes.results);
|
|
1115
1202
|
}
|
|
1116
1203
|
resultIdentity() {
|
|
1117
1204
|
return {
|
|
@@ -1158,7 +1245,11 @@ class ScopedRegistryView {
|
|
|
1158
1245
|
}
|
|
1159
1246
|
static {
|
|
1160
1247
|
provideRegistryPrograms(ScopedRegistryView.prototype, {
|
|
1161
|
-
|
|
1248
|
+
releaseResult: (self, id, chunkCount) => registryProgram(self.root, "releaseResult", id, chunkCount, self.scope.subjectKey ? scopes.subject(self.scope.subjectKey) : scopes.results),
|
|
1249
|
+
stashResult: (self, id, chunks, ttlSeconds, options = {}) => registryProgram(self.root, "stashResult", id, chunks, ttlSeconds, {
|
|
1250
|
+
...options,
|
|
1251
|
+
partition: self.scope.subjectKey ? scopes.subject(self.scope.subjectKey) : scopes.results,
|
|
1252
|
+
}),
|
|
1162
1253
|
recheckResultAccess: (self, dependencies, classification, signal) => Effect.tryPromise({
|
|
1163
1254
|
try: () => self.scope.currentResultAccess?.(dependencies, classification, signal) ?? Promise.resolve(true),
|
|
1164
1255
|
catch: (error) => error,
|
|
@@ -1185,6 +1276,15 @@ class ScopedRegistryView {
|
|
|
1185
1276
|
}
|
|
1186
1277
|
return carryCatalogFreshness(tools, visible);
|
|
1187
1278
|
}),
|
|
1279
|
+
peekCatalog: (self, ...args) => Effect.gen(function* () {
|
|
1280
|
+
const registry = self.registryFor(args[0]);
|
|
1281
|
+
if (!registry)
|
|
1282
|
+
return undefined;
|
|
1283
|
+
const peek = yield* registryProgram(registry, "peekCatalog", args[0], args[1], args[2], args[3], self.scope);
|
|
1284
|
+
const granted = self.scope.toolAccess?.get(args[0]);
|
|
1285
|
+
// A pool grant binds a stale listing exactly as it binds a live one.
|
|
1286
|
+
return peek && granted ? { ...peek, tools: peek.tools.filter((tool) => granted.has(tool.name)) } : peek;
|
|
1287
|
+
}),
|
|
1188
1288
|
credentialDriftFor: (self, id, requestScope) => Effect.suspend(() => {
|
|
1189
1289
|
const registry = self.registryFor(id);
|
|
1190
1290
|
return registry
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema-aware views of direct reads past their cap, as pure functions of redacted
|
|
3
|
+
* text. Presentation only: the stash and what a program sees stay byte-identical
|
|
4
|
+
* to the full result, and everything shown here also appears in it, unchanged.
|
|
5
|
+
* Rules: documentation/meta-tools.md#previews-of-reads-over-the-cap.
|
|
6
|
+
*/
|
|
7
|
+
const enc = new TextEncoder();
|
|
8
|
+
const dec = new TextDecoder();
|
|
9
|
+
/** Bytes of whole rows a view shows. */
|
|
10
|
+
const ROW_BYTES = 1_500;
|
|
11
|
+
/** One row can pass ROW_BYTES up to this, so a wide list still shows its shape. */
|
|
12
|
+
const FIRST_ROW_BYTES = 3_000;
|
|
13
|
+
/** Everything outside the list stays whole, or the read is not a list. */
|
|
14
|
+
const REST_BYTES = 1_024;
|
|
15
|
+
const TEXT_HEAD_BYTES = 1_200;
|
|
16
|
+
const TEXT_TAIL_BYTES = 800;
|
|
17
|
+
const MIN_ROWS = 3;
|
|
18
|
+
const MAX_ROW_DEPTH = 3;
|
|
19
|
+
const MAX_KEYS = 40;
|
|
20
|
+
const MAX_KEY_CHARS = 64;
|
|
21
|
+
/** Rows sampled for key names. */
|
|
22
|
+
const KEY_SAMPLE_ROWS = 50;
|
|
23
|
+
function isRecord(value) {
|
|
24
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
25
|
+
}
|
|
26
|
+
const byteLength = (text) => enc.encode(text).length;
|
|
27
|
+
/** The largest list of records within MAX_ROW_DEPTH keys of the root. */
|
|
28
|
+
function dominantRows(root) {
|
|
29
|
+
let best;
|
|
30
|
+
const visit = (node, path) => {
|
|
31
|
+
if (Array.isArray(node)) {
|
|
32
|
+
if (node.length >= MIN_ROWS && node.every((row) => isRecord(row) || Array.isArray(row))) {
|
|
33
|
+
const bytes = byteLength(JSON.stringify(node));
|
|
34
|
+
if (!best || bytes > best.bytes)
|
|
35
|
+
best = { path, rows: node, bytes };
|
|
36
|
+
}
|
|
37
|
+
return;
|
|
38
|
+
}
|
|
39
|
+
if (!isRecord(node) || path.length >= MAX_ROW_DEPTH)
|
|
40
|
+
return;
|
|
41
|
+
for (const key of Object.keys(node))
|
|
42
|
+
visit(node[key], [...path, key]);
|
|
43
|
+
};
|
|
44
|
+
visit(root, []);
|
|
45
|
+
return best;
|
|
46
|
+
}
|
|
47
|
+
function withRows(root, path, rows) {
|
|
48
|
+
if (!path.length)
|
|
49
|
+
return rows;
|
|
50
|
+
const [head, ...tail] = path;
|
|
51
|
+
const record = root;
|
|
52
|
+
// Computed keys define own properties, so a key named __proto__ stays data.
|
|
53
|
+
return { ...record, [head]: withRows(record[head], tail, rows) };
|
|
54
|
+
}
|
|
55
|
+
function rowKeys(rows) {
|
|
56
|
+
const keys = new Set();
|
|
57
|
+
for (const row of rows.slice(0, KEY_SAMPLE_ROWS)) {
|
|
58
|
+
if (!isRecord(row))
|
|
59
|
+
continue;
|
|
60
|
+
for (const key of Object.keys(row))
|
|
61
|
+
if (key.length <= MAX_KEY_CHARS)
|
|
62
|
+
keys.add(key);
|
|
63
|
+
if (keys.size >= MAX_KEYS)
|
|
64
|
+
break;
|
|
65
|
+
}
|
|
66
|
+
return [...keys].slice(0, MAX_KEYS);
|
|
67
|
+
}
|
|
68
|
+
/** JavaScript that reads `path` from a parsed value; a program runs it, so it must be exact. */
|
|
69
|
+
function accessor(path) {
|
|
70
|
+
return path.map((key) => (/^[A-Za-z_$][\w$]*$/.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`)).join("");
|
|
71
|
+
}
|
|
72
|
+
function rowsPreview(parsed, total, cap) {
|
|
73
|
+
const found = dominantRows(parsed);
|
|
74
|
+
if (!found || found.bytes * 2 < total)
|
|
75
|
+
return undefined;
|
|
76
|
+
// The bytes outside the list; the ways it can be spelled do not change them.
|
|
77
|
+
const rest = byteLength(JSON.stringify(withRows(parsed, found.path, [])));
|
|
78
|
+
if (rest > REST_BYTES)
|
|
79
|
+
return undefined;
|
|
80
|
+
const firstRowBytes = Math.min(FIRST_ROW_BYTES, cap - rest);
|
|
81
|
+
const rowBytes = Math.min(ROW_BYTES, cap - rest);
|
|
82
|
+
const shown = [];
|
|
83
|
+
let used = 0;
|
|
84
|
+
for (const row of found.rows) {
|
|
85
|
+
const size = byteLength(JSON.stringify(row)) + 1;
|
|
86
|
+
if (used + size > (shown.length ? rowBytes : firstRowBytes))
|
|
87
|
+
break;
|
|
88
|
+
shown.push(row);
|
|
89
|
+
used += size;
|
|
90
|
+
}
|
|
91
|
+
const count = found.rows.length;
|
|
92
|
+
// An empty list would read as a list with no rows, and a whole one needs no sample.
|
|
93
|
+
if (!shown.length || shown.length === count)
|
|
94
|
+
return undefined;
|
|
95
|
+
const keys = rowKeys(found.rows);
|
|
96
|
+
const path = found.path.length ? accessor(found.path).replace(/^\./, "") : "(root)";
|
|
97
|
+
const body = JSON.stringify(withRows(parsed, found.path, shown));
|
|
98
|
+
if (byteLength(body) > cap)
|
|
99
|
+
return undefined;
|
|
100
|
+
return {
|
|
101
|
+
fields: { rows: { path, count, shown: shown.length, keys } },
|
|
102
|
+
body,
|
|
103
|
+
shown: `Sample only: ${shown.length === 1 ? "row 1" : `rows 1-${shown.length}`} of ${count} at ${path} follow, in the result's own shape, with every value outside the list; the other ${count - shown.length} are not shown.`,
|
|
104
|
+
noun: "rows",
|
|
105
|
+
reduce: `const rows = JSON.parse(text)${accessor(found.path)}; return { count: rows.length, rows: rows.slice(0, 5) };`,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
function lineCount(text) {
|
|
109
|
+
let lines = 1;
|
|
110
|
+
for (let at = text.indexOf("\n"); at !== -1; at = text.indexOf("\n", at + 1))
|
|
111
|
+
lines++;
|
|
112
|
+
return lines;
|
|
113
|
+
}
|
|
114
|
+
/** Whether a byte continues a UTF-8 character rather than starting one. */
|
|
115
|
+
const continues = (byte) => byte !== undefined && (byte & 0xc0) === 0x80;
|
|
116
|
+
function textPreview(text, reading, cap) {
|
|
117
|
+
const bytes = enc.encode(text);
|
|
118
|
+
if (bytes.length <= TEXT_HEAD_BYTES + TEXT_TAIL_BYTES)
|
|
119
|
+
return undefined;
|
|
120
|
+
const marker = (omitted) => `\n…[${omitted} bytes omitted]…\n`;
|
|
121
|
+
let headEnd = TEXT_HEAD_BYTES;
|
|
122
|
+
while (continues(bytes[headEnd]))
|
|
123
|
+
headEnd--;
|
|
124
|
+
let tailStart = bytes.length - TEXT_TAIL_BYTES;
|
|
125
|
+
while (continues(bytes[tailStart]))
|
|
126
|
+
tailStart++;
|
|
127
|
+
const shownBytes = headEnd + bytes.length - tailStart;
|
|
128
|
+
const body = `${dec.decode(bytes.subarray(0, headEnd))}${marker(tailStart - headEnd)}${dec.decode(bytes.subarray(tailStart))}`;
|
|
129
|
+
if (byteLength(body) > cap)
|
|
130
|
+
return undefined;
|
|
131
|
+
const lines = lineCount(text);
|
|
132
|
+
return {
|
|
133
|
+
fields: { text: { lines, shownBytes } },
|
|
134
|
+
body,
|
|
135
|
+
shown: `Sample only: the first ${headEnd} and last ${bytes.length - tailStart} bytes of ${lines} lines follow; the ${tailStart - headEnd} bytes between them are not shown.`,
|
|
136
|
+
noun: "text",
|
|
137
|
+
reduce: `const lines = ${reading}.split("\\n"); return { lines: lines.length, matches: lines.filter((line) => /error|fail/i.test(line)).map((line) => line.slice(0, 500)).slice(0, 40) };`,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
/** Parts of a JSON number token whose exact value is the same however it is spelled. */
|
|
141
|
+
function exactValue(token) {
|
|
142
|
+
const match = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(token);
|
|
143
|
+
if (!match)
|
|
144
|
+
return undefined;
|
|
145
|
+
const fraction = match[3] ?? "";
|
|
146
|
+
const digits = `${match[2]}${fraction}`;
|
|
147
|
+
let start = 0;
|
|
148
|
+
let end = digits.length;
|
|
149
|
+
while (digits[start] === "0")
|
|
150
|
+
start++;
|
|
151
|
+
while (end > start && digits[end - 1] === "0")
|
|
152
|
+
end--;
|
|
153
|
+
if (start === end)
|
|
154
|
+
return `${match[1]}0`;
|
|
155
|
+
return `${match[1]}${digits.slice(start, end)}e${Number(match[4] ?? 0) - fraction.length + digits.length - end}`;
|
|
156
|
+
}
|
|
157
|
+
/** Whether JSON.parse then JSON.stringify keeps this token's value, including its sign and range. */
|
|
158
|
+
function keepsNumber(token) {
|
|
159
|
+
const printed = String(Number(token));
|
|
160
|
+
return printed === token || exactValue(printed) === exactValue(token);
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Whether every number in text JSON.parse has already accepted survives parsing.
|
|
164
|
+
* Both a view and the offered program parse it, so one lossy number anywhere,
|
|
165
|
+
* inside or outside the rows, declines the view. Strings are skipped, not read.
|
|
166
|
+
*/
|
|
167
|
+
function keepsNumbers(json) {
|
|
168
|
+
let at = 0;
|
|
169
|
+
while (at < json.length) {
|
|
170
|
+
const code = json.charCodeAt(at);
|
|
171
|
+
if (code === 34) {
|
|
172
|
+
at++;
|
|
173
|
+
while (json.charCodeAt(at) !== 34)
|
|
174
|
+
at += json.charCodeAt(at) === 92 ? 2 : 1;
|
|
175
|
+
at++;
|
|
176
|
+
}
|
|
177
|
+
else if (code === 45 || (code >= 48 && code <= 57)) {
|
|
178
|
+
const start = at;
|
|
179
|
+
while (at < json.length && "+-.0123456789Ee".includes(json[at]))
|
|
180
|
+
at++;
|
|
181
|
+
if (!keepsNumber(json.slice(start, at)))
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
else
|
|
185
|
+
at++;
|
|
186
|
+
}
|
|
187
|
+
return true;
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* The view of a read past its cap, or undefined when its first bytes should stand:
|
|
191
|
+
* documents, scalars, shallow or deep lists a view would misstate, and numbers
|
|
192
|
+
* parsing would change.
|
|
193
|
+
*
|
|
194
|
+
* `format` is the result's own: an API connector's text arrives here as one
|
|
195
|
+
* JSON string, whose decoded text is what a person would call the result.
|
|
196
|
+
*/
|
|
197
|
+
export function previewResult(text, format, cap) {
|
|
198
|
+
let parsed;
|
|
199
|
+
try {
|
|
200
|
+
parsed = JSON.parse(text);
|
|
201
|
+
}
|
|
202
|
+
catch {
|
|
203
|
+
return textPreview(text, "text", cap);
|
|
204
|
+
}
|
|
205
|
+
if (typeof parsed === "string" && format === "text")
|
|
206
|
+
return textPreview(parsed, "JSON.parse(text)", cap);
|
|
207
|
+
if (typeof parsed !== "object" || parsed === null || !keepsNumbers(text))
|
|
208
|
+
return undefined;
|
|
209
|
+
return rowsPreview(parsed, byteLength(text), cap);
|
|
210
|
+
}
|