@zackbart/connecta 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +6 -5
- package/CHANGELOG.md +271 -0
- package/README.md +32 -15
- package/SECURITY.md +6 -5
- package/bin/connecta.mjs +5 -1
- package/dist/activity.d.ts +10 -3
- package/dist/activity.js +10 -1
- package/dist/artifacts/connector.d.ts +55 -0
- package/dist/artifacts/connector.js +724 -0
- package/dist/artifacts/document.js +76 -0
- package/dist/artifacts/guide.js +140 -0
- package/dist/artifacts/html-scan.js +329 -0
- package/dist/artifacts/json.js +68 -0
- package/dist/artifacts/kv-store.d.ts +20 -0
- package/dist/artifacts/kv-store.js +140 -0
- package/dist/artifacts/markdown.js +449 -0
- package/dist/artifacts/module.d.ts +37 -0
- package/dist/artifacts/module.js +60 -0
- package/dist/artifacts/operations.d.ts +195 -0
- package/dist/artifacts/operations.js +944 -0
- package/dist/artifacts/refresh.d.ts +53 -0
- package/dist/artifacts/refresh.js +235 -0
- package/dist/artifacts/routes.js +194 -0
- package/dist/artifacts/types.d.ts +214 -0
- package/dist/artifacts/types.js +22 -0
- package/dist/artifacts/validate.d.ts +62 -0
- package/dist/artifacts/validate.js +652 -0
- package/dist/artifacts.d.ts +6 -0
- package/dist/artifacts.js +9 -0
- package/dist/catalog-service.js +39 -29
- package/dist/catalog.js +42 -8
- package/dist/connector-access.d.ts +18 -4
- package/dist/connector-access.js +65 -20
- package/dist/connector-caller.d.ts +8 -0
- package/dist/connector-caller.js +19 -0
- package/dist/connectors/remote-mcp.js +16 -2
- package/dist/errors.d.ts +16 -1
- package/dist/errors.js +28 -0
- package/dist/execute.js +427 -70
- package/dist/executor-admission.d.ts +11 -1
- package/dist/executor-admission.js +79 -14
- package/dist/executors/quickjs.js +4 -2
- package/dist/index.d.ts +62 -6
- package/dist/index.js +230 -6
- package/dist/invocation.js +56 -10
- package/dist/meta-tools.js +7 -2
- package/dist/module-contracts.d.ts +21 -1
- package/dist/operator-ui/generated.js +2 -2
- package/dist/operator-ui/model.d.ts +62 -5
- package/dist/operator-ui/view.js +94 -5
- package/dist/program-source.js +116 -0
- package/dist/registry.d.ts +42 -6
- package/dist/registry.js +148 -47
- package/dist/resumable.js +1291 -0
- package/dist/routes/activity.js +6 -69
- package/dist/routes/actor-labels.js +114 -0
- package/dist/routes/credentials.js +27 -5
- package/dist/routes/mcp.js +116 -59
- package/dist/routes/oauth-management.js +22 -9
- package/dist/routes/operator.js +1 -0
- package/dist/routes/shared.d.ts +23 -1
- package/dist/routes/shared.js +20 -2
- package/dist/routes/ui-data.js +2 -1
- package/dist/routes/ui.js +13 -4
- package/dist/run-journal.js +347 -0
- package/dist/runtime/shared-read.js +106 -0
- package/dist/server.js +26 -0
- package/dist/skills.js +27 -8
- package/dist/timeout.js +26 -0
- package/dist/tool-safety.d.ts +35 -0
- package/dist/tool-safety.js +31 -0
- package/dist/types.d.ts +18 -0
- package/dist/ui.d.ts +12 -5
- package/dist/ui.js +32 -6
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/documentation/architecture.md +196 -19
- package/documentation/auth.md +83 -2
- package/documentation/code-mode.md +221 -48
- package/documentation/meta-tools.md +44 -25
- package/ethos.md +12 -10
- package/examples/worker/AGENTS.md +6 -0
- package/examples/worker/README.md +174 -10
- package/examples/worker/src/d1-activity-row.ts +14 -2
- package/examples/worker/src/d1-activity.ts +4 -2
- package/examples/worker/src/index.ts +36 -1
- package/examples/worker/src/r2-artifact-blobs.ts +29 -0
- package/examples/worker/wrangler.jsonc +5 -1
- package/package.json +6 -2
- package/templates/node/AGENTS.md +9 -2
- package/templates/node/README.md +31 -3
- package/templates/node/package.json +1 -1
- package/templates/node/src/index.ts +27 -0
package/dist/routes/shared.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Implementation } from "@modelcontextprotocol/server";
|
|
2
|
-
import type { ActivityModule, OperatorSurface } from "../module-contracts.js";
|
|
2
|
+
import type { ActivityModule, ArtifactsModule, OperatorSurface } from "../module-contracts.js";
|
|
3
3
|
import type { ActivityActor, ActivityReadGate, ActivityStore } from "../activity.js";
|
|
4
4
|
import type { CredentialVault } from "../credential-contract.js";
|
|
5
5
|
import type { DeferredWork } from "../connector-scope.js";
|
|
@@ -7,6 +7,7 @@ import type { AdmissionController } from "../executor-admission.js";
|
|
|
7
7
|
import type { Registry, ToolAccess } from "../registry.js";
|
|
8
8
|
import type { ResolvedPool } from "../connector-access.js";
|
|
9
9
|
import type { AuthenticatedIdentity, ConnectaBranding, Executor, InboundAuth, InboundAuthRuntimeContext, Logger } from "../types.js";
|
|
10
|
+
import type { ApprovalPolicy } from "../tool-safety.js";
|
|
10
11
|
import type { ConnectorPermission, ConnectaIdentityConfig } from "../index.js";
|
|
11
12
|
export { msg } from "../errors.js";
|
|
12
13
|
export interface ServerOptions {
|
|
@@ -16,6 +17,7 @@ export interface ServerOptions {
|
|
|
16
17
|
/** Validated named pools served at `/mcp/<name>`; empty when none declared. */
|
|
17
18
|
pools?: ReadonlyMap<string, ResolvedPool> | undefined;
|
|
18
19
|
publicUrl?: string | undefined;
|
|
20
|
+
artifactOrigin?: string | undefined;
|
|
19
21
|
allowedOrigins?: readonly string[] | "*" | undefined;
|
|
20
22
|
serverInfo: Implementation;
|
|
21
23
|
logger: Logger;
|
|
@@ -39,6 +41,15 @@ export interface ServerOptions {
|
|
|
39
41
|
hostCallTimeoutMs?: number | undefined;
|
|
40
42
|
/** Hard ceiling on one execute_code run, outside the sandbox. Default 120_000. */
|
|
41
43
|
watchdogMs?: number | undefined;
|
|
44
|
+
/** Resumable writes' settings; absent when they are off. */
|
|
45
|
+
resumable?: {
|
|
46
|
+
maxWrites: number;
|
|
47
|
+
ttlSeconds: number;
|
|
48
|
+
} | undefined;
|
|
49
|
+
/** Config approval exemptions (`execute.approval`), resolved at construction. */
|
|
50
|
+
approval?: ApprovalPolicy | undefined;
|
|
51
|
+
/** Writes one program may send (`execute.maxWrites`), exempt ones included. */
|
|
52
|
+
maxWrites?: number | undefined;
|
|
42
53
|
/** Required sandbox backing the execute_code meta-tool. */
|
|
43
54
|
executor: Executor;
|
|
44
55
|
/** Sanitized identity of the configured sandbox, when it has one. */
|
|
@@ -50,6 +61,8 @@ export interface ServerOptions {
|
|
|
50
61
|
/** Optional browser routes, with no implementation import in core. */
|
|
51
62
|
ui?: OperatorSurface | undefined;
|
|
52
63
|
activityModule?: ActivityModule | undefined;
|
|
64
|
+
/** Optional team pages; their routes mount only beside `ui`. */
|
|
65
|
+
artifactsModule?: ArtifactsModule | undefined;
|
|
53
66
|
/** Optional browser UI and OAuth result-page labels. */
|
|
54
67
|
branding?: ConnectaBranding | undefined;
|
|
55
68
|
}
|
|
@@ -86,6 +99,8 @@ export declare function authorize(request: Request, baseUrl: string, auth: Inbou
|
|
|
86
99
|
connectorIds: "all" | readonly string[];
|
|
87
100
|
/** Per-connector tool allowlist for connectors granted by address only. */
|
|
88
101
|
toolAccess?: ToolAccess;
|
|
102
|
+
/** Granted addresses that still require the catalog's read-only hint. */
|
|
103
|
+
guardedToolAccess?: ToolAccess;
|
|
89
104
|
operator: boolean;
|
|
90
105
|
credentialAdministration: ConnectorPermission;
|
|
91
106
|
personalConnection: ConnectorPermission;
|
|
@@ -108,6 +123,13 @@ type AuthorizedIdentity = Extract<Awaited<ReturnType<typeof authorize>>, {
|
|
|
108
123
|
}>;
|
|
109
124
|
/** Unknown configured ids refuse the complete view, including management rights. */
|
|
110
125
|
export declare function validateAuthPermissions(authz: AuthorizedIdentity, registry: Registry): void;
|
|
126
|
+
/**
|
|
127
|
+
* Whether this identity may open artifact pages: exactly whoever may read the
|
|
128
|
+
* `artifacts` connector — it is visible to them and `get_artifact` is among
|
|
129
|
+
* its tools they were granted. Viewing grants nothing a program could not
|
|
130
|
+
* already read (ethos, "Human routes manage auth, never capability").
|
|
131
|
+
*/
|
|
132
|
+
export declare function mayViewArtifacts(authz: AuthorizedIdentity, registry: Registry): boolean;
|
|
111
133
|
export declare function mayManageConnector(authz: AuthorizedIdentity, connector: {
|
|
112
134
|
id: string;
|
|
113
135
|
authScope?: "shared" | "personal";
|
package/dist/routes/shared.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { parseConnectorAccess } from "../connector-access.js";
|
|
2
2
|
import { identityStorageKey, validIdentityReference } from "../identity.js";
|
|
3
|
+
import { isExplicitlyReadOnly } from "../tool-safety.js";
|
|
3
4
|
export { msg } from "../errors.js";
|
|
4
5
|
export function privateJson(body, init = {}) {
|
|
5
6
|
const headers = new Headers(init.headers);
|
|
@@ -36,7 +37,7 @@ export async function authorize(request, baseUrl, auth, runtimeContext, identity
|
|
|
36
37
|
const identity = { actor, interactive: false };
|
|
37
38
|
let access;
|
|
38
39
|
try {
|
|
39
|
-
access = parseConnectorAccess(identityConfig?.connectorAccess ? await identityConfig.connectorAccess(identity) : "all");
|
|
40
|
+
access = parseConnectorAccess(identityConfig?.connectorAccess ? await identityConfig.connectorAccess(identity) : "all", { allowReadOnly: true });
|
|
40
41
|
}
|
|
41
42
|
catch {
|
|
42
43
|
return {
|
|
@@ -83,7 +84,7 @@ export async function authorize(request, baseUrl, auth, runtimeContext, identity
|
|
|
83
84
|
? await identityConfig.activityAccess(principal)
|
|
84
85
|
: false;
|
|
85
86
|
}
|
|
86
|
-
access = parseConnectorAccess(identityConfig?.connectorAccess ? await identityConfig.connectorAccess(identity) : "all");
|
|
87
|
+
access = parseConnectorAccess(identityConfig?.connectorAccess ? await identityConfig.connectorAccess(identity) : "all", { allowReadOnly: true });
|
|
87
88
|
if (interactive) {
|
|
88
89
|
credentialAdministration = identityConfig?.credentialAdministration ? await identityConfig.credentialAdministration(identity) : "none";
|
|
89
90
|
personalConnection = principal && identityConfig?.personalConnection ? await identityConfig.personalConnection(identity) : "none";
|
|
@@ -194,6 +195,23 @@ export function validateAuthPermissions(authz, registry) {
|
|
|
194
195
|
}
|
|
195
196
|
}
|
|
196
197
|
}
|
|
198
|
+
/**
|
|
199
|
+
* Whether this identity may open artifact pages: exactly whoever may read the
|
|
200
|
+
* `artifacts` connector — it is visible to them and `get_artifact` is among
|
|
201
|
+
* its tools they were granted. Viewing grants nothing a program could not
|
|
202
|
+
* already read (ethos, "Human routes manage auth, never capability").
|
|
203
|
+
*/
|
|
204
|
+
export function mayViewArtifacts(authz, registry) {
|
|
205
|
+
if (authz.connectorIds !== "all" && !authz.connectorIds.includes("artifacts")) {
|
|
206
|
+
return false;
|
|
207
|
+
}
|
|
208
|
+
const tools = authz.toolAccess?.get("artifacts");
|
|
209
|
+
if (tools && !tools.has("get_artifact"))
|
|
210
|
+
return false;
|
|
211
|
+
if (!authz.guardedToolAccess?.get("artifacts")?.has("get_artifact"))
|
|
212
|
+
return true;
|
|
213
|
+
return registry.getConnector("artifacts")?.staticTools?.some((tool) => tool.name === "get_artifact" && isExplicitlyReadOnly(tool)) === true;
|
|
214
|
+
}
|
|
197
215
|
export function mayManageConnector(authz, connector) {
|
|
198
216
|
if (!authz.identity.interactive)
|
|
199
217
|
return false;
|
package/dist/routes/ui-data.js
CHANGED
|
@@ -10,6 +10,7 @@ import { resolveDiscoveryConcurrency } from "../concurrency.js";
|
|
|
10
10
|
import { credentialTestRule, describeUndeclaredCredentialFields, storedCredentialShape, } from "../credential-rules.js";
|
|
11
11
|
import { closeScope } from "../runtime/connector-scope.js";
|
|
12
12
|
import { withDeadlineEffect } from "../runtime/run.js";
|
|
13
|
+
import { isApprovalExempt, NO_EXEMPTIONS, } from "../tool-safety.js";
|
|
13
14
|
import { uiProblemFor, uiToolSafety } from "../ui.js";
|
|
14
15
|
import { CONNECTA_VERSION } from "../version.js";
|
|
15
16
|
const DETAIL_TIMEOUT_MS = 30_000;
|
|
@@ -59,7 +60,7 @@ export function uiData(registry, baseUrl, options) {
|
|
|
59
60
|
name: t.name,
|
|
60
61
|
address: `${c.id}.${t.name}`,
|
|
61
62
|
...(t.description ? { description: t.description } : {}),
|
|
62
|
-
safety: uiToolSafety(t),
|
|
63
|
+
safety: uiToolSafety(t, isApprovalExempt(options.approval ?? NO_EXEMPTIONS, c, t.name, t)),
|
|
63
64
|
}))).pipe(Effect.map((tools) => ({ status, tools, catalogFailed: false })),
|
|
64
65
|
// The registry owns the failed catalog observation; the page only
|
|
65
66
|
// needs to know there was one.
|
package/dist/routes/ui.js
CHANGED
|
@@ -3,7 +3,7 @@ import { CONNECTA_VERSION } from "../version.js";
|
|
|
3
3
|
import { CONNECTA_FAVICON_ICO } from "../favicon.js";
|
|
4
4
|
import { CONNECTA_FAVICON_SVG, operatorPageForPath, renderUiHtml, } from "../ui.js";
|
|
5
5
|
import { authorized, refuse, scopeFor, serveOperator, visibleRegistry, } from "./operator.js";
|
|
6
|
-
import { mayManageConnector, privateJson, } from "./shared.js";
|
|
6
|
+
import { mayManageConnector, mayViewArtifacts, privateJson, } from "./shared.js";
|
|
7
7
|
import { uiData } from "./ui-data.js";
|
|
8
8
|
/**
|
|
9
9
|
* Headers that make an operator-supplied favicon body inert on this origin.
|
|
@@ -71,7 +71,10 @@ export async function routeUi(context) {
|
|
|
71
71
|
});
|
|
72
72
|
}
|
|
73
73
|
const operatorPage = operatorPageForPath(path);
|
|
74
|
-
|
|
74
|
+
const artifactPage = operatorPage === "artifacts" || operatorPage === "artifact";
|
|
75
|
+
if (operatorPage &&
|
|
76
|
+
(operatorPage !== "activity" || opts.activity?.list) &&
|
|
77
|
+
(!artifactPage || opts.artifactsModule)) {
|
|
75
78
|
if (request.method !== "GET" && request.method !== "HEAD") {
|
|
76
79
|
return privateJson({ error: "method not allowed" }, { status: 405 });
|
|
77
80
|
}
|
|
@@ -89,14 +92,18 @@ export async function routeUi(context) {
|
|
|
89
92
|
// style/font/network needs and the page's inline <style> stay unrestricted
|
|
90
93
|
// — only script execution, the XSS sink, is gated.
|
|
91
94
|
const nonce = uiScriptNonce();
|
|
95
|
+
// An artifact shell frames the sandboxed page from its own origin and
|
|
96
|
+
// nothing else; Connections stays on the deployment's public origin.
|
|
97
|
+
const homeUrl = opts.publicUrl ? new URL("/", opts.publicUrl).toString() : "/";
|
|
92
98
|
return new Response(request.method === "HEAD"
|
|
93
99
|
? null
|
|
94
|
-
: renderUiHtml(uiAuth, mcpUrl, opts.branding, nonce, operatorPage), {
|
|
100
|
+
: renderUiHtml(uiAuth, mcpUrl, opts.branding, nonce, operatorPage, { homeUrl }), {
|
|
95
101
|
status: 200,
|
|
96
102
|
headers: {
|
|
97
103
|
"Content-Type": "text/html; charset=utf-8",
|
|
98
104
|
"Content-Security-Policy": `script-src 'nonce-${nonce}' 'strict-dynamic' https: 'unsafe-inline'; ` +
|
|
99
|
-
"object-src 'none'; base-uri 'none'; frame-ancestors 'none'"
|
|
105
|
+
"object-src 'none'; base-uri 'none'; frame-ancestors 'none'" +
|
|
106
|
+
(artifactPage ? "; frame-src 'self'" : ""),
|
|
100
107
|
"X-Content-Type-Options": "nosniff",
|
|
101
108
|
},
|
|
102
109
|
});
|
|
@@ -146,6 +153,7 @@ function connectorDetail(context, id) {
|
|
|
146
153
|
mayManage,
|
|
147
154
|
timeoutMs: opts.probeTimeoutMs ?? 30_000,
|
|
148
155
|
signal: request.signal,
|
|
156
|
+
approval: opts.approval,
|
|
149
157
|
});
|
|
150
158
|
return privateJson({ ...data.connectors[0], permissions: permissions(connector) });
|
|
151
159
|
});
|
|
@@ -173,6 +181,7 @@ function summary(context) {
|
|
|
173
181
|
serverInfo: opts.serverInfo,
|
|
174
182
|
connectaVersion: CONNECTA_VERSION,
|
|
175
183
|
activityEnabled,
|
|
184
|
+
...(opts.artifactsModule && mayViewArtifacts(authz, opts.registry) ? { artifactsEnabled: true } : {}),
|
|
176
185
|
credentialManagement,
|
|
177
186
|
oauthManagement: visible.some(c => mayManage(c.id)),
|
|
178
187
|
connectors: visible.map(c => ({ id: c.id, ...(c.title ? { title: c.title } : {}), ...(c.description ? { description: c.description } : {}), authScope: c.authScope ?? "shared", status: "loading", toolCount: 0, tools: [], oauth: Boolean(c.startAuth && c.disconnectAuth), permissions: permissions(c) })),
|
|
@@ -0,0 +1,347 @@
|
|
|
1
|
+
// The paused-run journal: layout, tokens, and the storage it lives in.
|
|
2
|
+
//
|
|
3
|
+
// A paused `execute_code` run is not a held program. It is data — the
|
|
4
|
+
// program's source, every host call it made and what each returned, and one
|
|
5
|
+
// header that says where the run stands — written to the same subject-
|
|
6
|
+
// partitioned storage the result stash uses, at the same trust level, and
|
|
7
|
+
// expiring on its own. Resuming replays the program from the top against
|
|
8
|
+
// those records. Nothing request-bound is in here: a journal is data, not a
|
|
9
|
+
// request (ethos, "Nothing request-bound survives a request").
|
|
10
|
+
//
|
|
11
|
+
// Web-API only, like everything reachable from the root entry.
|
|
12
|
+
import { withDeadline } from "./timeout.js";
|
|
13
|
+
/** Entry chunks are the result stash's size (`RESULT_CHUNK_BYTES`), in characters. */
|
|
14
|
+
const JOURNAL_CHUNK_CHARS = 49_152;
|
|
15
|
+
/**
|
|
16
|
+
* The most one run may journal: its source plus every recorded host call.
|
|
17
|
+
* Internal rather than configurable — it bounds what one caller can park in
|
|
18
|
+
* shared storage for half an hour, and nothing about a deployment makes a
|
|
19
|
+
* larger notebook the right answer. L6 bounds each host call's result at 256
|
|
20
|
+
* KiB on QuickJS, so a run near the host-call budget can reach it.
|
|
21
|
+
*/
|
|
22
|
+
export const MAX_JOURNAL_BYTES = 4 * 1024 * 1024;
|
|
23
|
+
/**
|
|
24
|
+
* The largest pending write a pause will hold, serialized. The arguments are
|
|
25
|
+
* echoed whole in the pause result and must be repeated whole to resume, so
|
|
26
|
+
* this is a bound on what a human is asked to read in a prompt, not a storage
|
|
27
|
+
* limit. A bigger write goes through `call_destructive_tool`.
|
|
28
|
+
*/
|
|
29
|
+
export const MAX_PENDING_ARGS_BYTES = 16 * 1024;
|
|
30
|
+
/** How long an ended run keeps its answer for a repeated resume, in ms. */
|
|
31
|
+
const FINAL_HOLD_MS = 30 * 60 * 1_000;
|
|
32
|
+
/** A completed run keeps at most this much of its answer for a retried resume. */
|
|
33
|
+
export const MAX_FINAL_CHARS = 24_000;
|
|
34
|
+
const encoder = new TextEncoder();
|
|
35
|
+
export function utf8Bytes(text) {
|
|
36
|
+
return encoder.encode(text).byteLength;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* JSON with object keys sorted at every depth, so two argument objects that
|
|
40
|
+
* differ only in key order serialize identically. Arrays keep their order and
|
|
41
|
+
* a type change is a different string: `1` and `"1"` never match.
|
|
42
|
+
*
|
|
43
|
+
* Written out directly rather than by building sorted objects and handing
|
|
44
|
+
* them to `JSON.stringify`: assigning an own `__proto__` key onto a plain
|
|
45
|
+
* object sets its prototype instead, so a key that `JSON.parse` kept would
|
|
46
|
+
* vanish from the canonical form, and two different writes would compare
|
|
47
|
+
* equal.
|
|
48
|
+
*/
|
|
49
|
+
export function canonicalJson(value) {
|
|
50
|
+
return canonical(value) ?? "null";
|
|
51
|
+
}
|
|
52
|
+
function canonical(input) {
|
|
53
|
+
let value = input;
|
|
54
|
+
if (value !== null &&
|
|
55
|
+
typeof value === "object" &&
|
|
56
|
+
typeof value.toJSON === "function") {
|
|
57
|
+
value = value.toJSON();
|
|
58
|
+
}
|
|
59
|
+
// Boxed primitives serialize as their value, as JSON.stringify has them.
|
|
60
|
+
if (value instanceof Number ||
|
|
61
|
+
value instanceof String ||
|
|
62
|
+
value instanceof Boolean) {
|
|
63
|
+
value = value.valueOf();
|
|
64
|
+
}
|
|
65
|
+
if (value === null || typeof value !== "object")
|
|
66
|
+
return JSON.stringify(value);
|
|
67
|
+
if (Array.isArray(value)) {
|
|
68
|
+
// Indexed, not mapped: `map` skips a hole, and JSON has none — a hole is
|
|
69
|
+
// `null`, as JSON.stringify writes it.
|
|
70
|
+
const items = [];
|
|
71
|
+
for (let index = 0; index < value.length; index++) {
|
|
72
|
+
items.push(canonical(value[index]) ?? "null");
|
|
73
|
+
}
|
|
74
|
+
return `[${items.join(",")}]`;
|
|
75
|
+
}
|
|
76
|
+
const parts = [];
|
|
77
|
+
for (const key of Object.keys(value).sort()) {
|
|
78
|
+
const item = canonical(value[key]);
|
|
79
|
+
if (item !== undefined)
|
|
80
|
+
parts.push(`${JSON.stringify(key)}:${item}`);
|
|
81
|
+
}
|
|
82
|
+
return `{${parts.join(",")}}`;
|
|
83
|
+
}
|
|
84
|
+
/** Whether a `__proto__` key appears anywhere in a JSON value. */
|
|
85
|
+
export function hasProtoKey(value) {
|
|
86
|
+
if (value === null || typeof value !== "object")
|
|
87
|
+
return false;
|
|
88
|
+
if (Array.isArray(value))
|
|
89
|
+
return value.some(hasProtoKey);
|
|
90
|
+
return Object.keys(value).some((key) => key === "__proto__" ||
|
|
91
|
+
hasProtoKey(value[key]));
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* How a replayed host call finds its record: the operation, the address as
|
|
95
|
+
* the program wrote it, and its canonical arguments. Several calls may share a
|
|
96
|
+
* key; they answer in the order they were first issued.
|
|
97
|
+
*/
|
|
98
|
+
export function journalKey(op, address, args) {
|
|
99
|
+
return `${op}\n${address}\n${canonicalJson(args ?? {})}`;
|
|
100
|
+
}
|
|
101
|
+
const TOKEN_RE = /^r1\.([0-9a-f]{32})\.([0-9a-f]{32})\.(\d{1,15})$/;
|
|
102
|
+
export function formatToken(token) {
|
|
103
|
+
return `r1.${token.runId}.${token.nonce}.${token.expiresAt}`;
|
|
104
|
+
}
|
|
105
|
+
export function parseToken(value) {
|
|
106
|
+
const match = TOKEN_RE.exec(value);
|
|
107
|
+
if (!match)
|
|
108
|
+
return undefined;
|
|
109
|
+
return {
|
|
110
|
+
runId: match[1] ?? "",
|
|
111
|
+
nonce: match[2] ?? "",
|
|
112
|
+
expiresAt: Number(match[3]),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/** 128 random bits as hex: a run id, a pause nonce, or a claim id. */
|
|
116
|
+
export function randomId() {
|
|
117
|
+
const words = new Uint32Array(4);
|
|
118
|
+
crypto.getRandomValues(words);
|
|
119
|
+
return Array.from(words, (word) => word.toString(16).padStart(8, "0")).join("");
|
|
120
|
+
}
|
|
121
|
+
export async function sha256Hex(text) {
|
|
122
|
+
const digest = await crypto.subtle.digest("SHA-256", encoder.encode(text));
|
|
123
|
+
return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, "0")).join("");
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Whether a consequential call landed, as far as anyone can know.
|
|
127
|
+
*
|
|
128
|
+
* `unknown` is the one outcome replay must never paper over: the call left
|
|
129
|
+
* connecta and no answer came back, so sending it again could do it twice and
|
|
130
|
+
* not sending it could leave it undone. That is a dispatched call that timed
|
|
131
|
+
* out, was cancelled, found the service unavailable, or failed any way that
|
|
132
|
+
* is not a verdict. A refusal code (`REFUSALS`) or a downstream tool's own
|
|
133
|
+
* `isError` is an answer, so it is `failed`, except that an `isError` whose text
|
|
134
|
+
* classifies as a timeout is a gateway reporting that it gave up, and stays
|
|
135
|
+
* unknown. A call that was never dispatched is `failed`: nothing was sent.
|
|
136
|
+
* `result_processing_failed` means the downstream call completed.
|
|
137
|
+
*/
|
|
138
|
+
export function classifyWriteOutcome(outcome) {
|
|
139
|
+
if (outcome.ok)
|
|
140
|
+
return "ok";
|
|
141
|
+
if (!outcome.dispatched)
|
|
142
|
+
return "failed";
|
|
143
|
+
const code = outcome.error?.code;
|
|
144
|
+
if (code === "result_processing_failed")
|
|
145
|
+
return "ok";
|
|
146
|
+
if (code === "timeout" || code === "cancelled" || code === "unavailable") {
|
|
147
|
+
return "unknown";
|
|
148
|
+
}
|
|
149
|
+
if (code !== undefined && REFUSALS.has(code))
|
|
150
|
+
return "failed";
|
|
151
|
+
return outcome.answered === true ? "failed" : "unknown";
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Codes that say the other side refused the call rather than acted on it:
|
|
155
|
+
* the credential, the arguments, the resource, the rate, or a base version
|
|
156
|
+
* someone else already moved past (`conflict`). Anything else a
|
|
157
|
+
* connector reports after dispatch — `connector_call_failed` from a response
|
|
158
|
+
* too large to read, a redirect it would not follow, a body it could not
|
|
159
|
+
* parse, a 5xx — may come after the write landed, so only a downstream
|
|
160
|
+
* tool's own `isError` answer makes it a known failure.
|
|
161
|
+
*/
|
|
162
|
+
const REFUSALS = new Set([
|
|
163
|
+
"auth_required",
|
|
164
|
+
"invalid_args",
|
|
165
|
+
"not_found",
|
|
166
|
+
"rate_limited",
|
|
167
|
+
"input_required_unsupported",
|
|
168
|
+
"conflict",
|
|
169
|
+
]);
|
|
170
|
+
function isRecord(value) {
|
|
171
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
172
|
+
}
|
|
173
|
+
/** A header read back from storage, or undefined when it is not one. */
|
|
174
|
+
function parseHeader(raw) {
|
|
175
|
+
let value;
|
|
176
|
+
try {
|
|
177
|
+
value = JSON.parse(raw);
|
|
178
|
+
}
|
|
179
|
+
catch {
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
if (!isRecord(value) ||
|
|
183
|
+
value.v !== 1 ||
|
|
184
|
+
!["paused", "running", "completed", "failed"].includes(String(value.state)) ||
|
|
185
|
+
typeof value.nonce !== "string" ||
|
|
186
|
+
typeof value.expiresAt !== "number" ||
|
|
187
|
+
typeof value.clock !== "number" ||
|
|
188
|
+
!Array.isArray(value.seed) ||
|
|
189
|
+
value.seed.length !== 4 ||
|
|
190
|
+
!Array.isArray(value.approvals) ||
|
|
191
|
+
!Array.isArray(value.writes) ||
|
|
192
|
+
typeof value.entries !== "number" ||
|
|
193
|
+
typeof value.bytes !== "number" ||
|
|
194
|
+
typeof value.playFrom !== "number" ||
|
|
195
|
+
typeof value.programHash !== "string") {
|
|
196
|
+
return undefined;
|
|
197
|
+
}
|
|
198
|
+
return value;
|
|
199
|
+
}
|
|
200
|
+
/**
|
|
201
|
+
* One run's keys inside the caller's result storage.
|
|
202
|
+
*
|
|
203
|
+
* `run:<id>` is the header and the only key ever compare-and-set.
|
|
204
|
+
* `run:<id>:src` is the program, written once. `run:<id>:e:<n>` is entry `n`,
|
|
205
|
+
* split into `#<k>` continuation keys past one chunk, written before the
|
|
206
|
+
* header that counts it. Every key expires with the run.
|
|
207
|
+
*
|
|
208
|
+
* Every storage call here answers within `timeoutMs` or rejects, as a
|
|
209
|
+
* failing store would: a play reserving or settling a write cannot be
|
|
210
|
+
* interrupted, and a store that never answers must not hold it — or the
|
|
211
|
+
* request waiting on it — forever. A write that timed out may still land;
|
|
212
|
+
* each caller treats the call as failed, and every write here is one that is
|
|
213
|
+
* safe to find landed later (a compare-and-set that landed after all only
|
|
214
|
+
* makes the next one lose).
|
|
215
|
+
*/
|
|
216
|
+
export class RunJournal {
|
|
217
|
+
storage;
|
|
218
|
+
runId;
|
|
219
|
+
timeoutMs;
|
|
220
|
+
constructor(storage, runId, timeoutMs) {
|
|
221
|
+
this.storage = storage;
|
|
222
|
+
this.runId = runId;
|
|
223
|
+
this.timeoutMs = timeoutMs;
|
|
224
|
+
}
|
|
225
|
+
bounded(operation) {
|
|
226
|
+
if (this.timeoutMs === undefined)
|
|
227
|
+
return operation();
|
|
228
|
+
return withDeadline(() => operation(), {
|
|
229
|
+
timeoutMs: this.timeoutMs,
|
|
230
|
+
timeoutError: new Error(`paused-run storage did not answer within ${this.timeoutMs}ms`),
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
get headerKey() {
|
|
234
|
+
return `run:${this.runId}`;
|
|
235
|
+
}
|
|
236
|
+
/** Seconds until `expiresAt`, never less than one. */
|
|
237
|
+
static ttlSeconds(expiresAt, now = Date.now()) {
|
|
238
|
+
return Math.max(1, Math.ceil((expiresAt - now) / 1_000));
|
|
239
|
+
}
|
|
240
|
+
async readHeader() {
|
|
241
|
+
const raw = await this.bounded(() => this.storage.get(this.headerKey));
|
|
242
|
+
if (raw === null)
|
|
243
|
+
return undefined;
|
|
244
|
+
const header = parseHeader(raw);
|
|
245
|
+
return header ? { raw, header } : undefined;
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Write `next` only if the header is still exactly `expected` (`null`:
|
|
249
|
+
* absent). Returns the stored string on success, `undefined` when another
|
|
250
|
+
* writer got there first.
|
|
251
|
+
*/
|
|
252
|
+
async casHeader(expected, next) {
|
|
253
|
+
const cas = this.storage.compareAndSet;
|
|
254
|
+
if (!cas)
|
|
255
|
+
throw new Error("resumable writes need KVStorage.compareAndSet");
|
|
256
|
+
const raw = JSON.stringify(next);
|
|
257
|
+
// The header outlives `expiresAt` while a claim is live, so a play that
|
|
258
|
+
// runs past it can still record what it sent, and after the run ends, so
|
|
259
|
+
// a repeated resume gets the same answer rather than "expired, run it
|
|
260
|
+
// again" — which would repeat every write the run made. For the same
|
|
261
|
+
// reason a run that has sent writes keeps its header past its deadline
|
|
262
|
+
// and past a lapsed claim, as a tombstone with their counts.
|
|
263
|
+
const sent = next.writes.length > 0;
|
|
264
|
+
const keepUntil = Math.max(next.expiresAt, next.claim?.until ?? 0, next.final ? Date.now() + FINAL_HOLD_MS : 0, sent ? next.expiresAt + FINAL_HOLD_MS : 0, sent && next.claim ? next.claim.until + FINAL_HOLD_MS : 0);
|
|
265
|
+
const written = await this.bounded(() => cas.call(this.storage, this.headerKey, expected, raw, {
|
|
266
|
+
ttlSeconds: RunJournal.ttlSeconds(keepUntil),
|
|
267
|
+
}));
|
|
268
|
+
return written ? raw : undefined;
|
|
269
|
+
}
|
|
270
|
+
async writeSource(source, expiresAt) {
|
|
271
|
+
await this.bounded(() => this.storage.set(`${this.headerKey}:src`, source, {
|
|
272
|
+
ttlSeconds: RunJournal.ttlSeconds(expiresAt),
|
|
273
|
+
}));
|
|
274
|
+
}
|
|
275
|
+
async readSource(programHash) {
|
|
276
|
+
const source = await this.bounded(() => this.storage.get(`${this.headerKey}:src`));
|
|
277
|
+
if (source === null)
|
|
278
|
+
return undefined;
|
|
279
|
+
return (await sha256Hex(source)) === programHash ? source : undefined;
|
|
280
|
+
}
|
|
281
|
+
/** Serialized size of an entry as journaled, for the run's byte bound. */
|
|
282
|
+
static entryText(entry) {
|
|
283
|
+
return JSON.stringify(entry);
|
|
284
|
+
}
|
|
285
|
+
writeEntry(index, entry, expiresAt) {
|
|
286
|
+
return this.bounded(() => this.writeEntryUnbounded(index, entry, expiresAt));
|
|
287
|
+
}
|
|
288
|
+
async writeEntryUnbounded(index, entry, expiresAt) {
|
|
289
|
+
const text = RunJournal.entryText(entry);
|
|
290
|
+
const chunks = [];
|
|
291
|
+
for (let at = 0; at < text.length; at += JOURNAL_CHUNK_CHARS) {
|
|
292
|
+
chunks.push(text.slice(at, at + JOURNAL_CHUNK_CHARS));
|
|
293
|
+
}
|
|
294
|
+
if (chunks.length === 0)
|
|
295
|
+
chunks.push("");
|
|
296
|
+
const ttlSeconds = RunJournal.ttlSeconds(expiresAt);
|
|
297
|
+
const key = `${this.headerKey}:e:${index}`;
|
|
298
|
+
// Continuations first, the head last: a head that exists names chunks
|
|
299
|
+
// that all exist.
|
|
300
|
+
for (let part = 1; part < chunks.length; part++) {
|
|
301
|
+
await this.storage.set(`${key}#${part}`, chunks[part] ?? "", { ttlSeconds });
|
|
302
|
+
}
|
|
303
|
+
await this.storage.set(key, `${chunks.length}\n${chunks[0] ?? ""}`, {
|
|
304
|
+
ttlSeconds,
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Entries `0 … count - 1`, or undefined when one is missing or unreadable:
|
|
309
|
+
* a journal with a hole cannot be replayed. (A slot a `sending` write
|
|
310
|
+
* reserved and never filled is such a hole, and a run holding one has
|
|
311
|
+
* failed rather than paused, so it is never read.)
|
|
312
|
+
*/
|
|
313
|
+
readEntries(count) {
|
|
314
|
+
return this.bounded(() => this.readEntriesUnbounded(count));
|
|
315
|
+
}
|
|
316
|
+
async readEntriesUnbounded(count) {
|
|
317
|
+
const entries = [];
|
|
318
|
+
for (let index = 0; index < count; index++) {
|
|
319
|
+
const key = `${this.headerKey}:e:${index}`;
|
|
320
|
+
const head = await this.storage.get(key);
|
|
321
|
+
if (head === null)
|
|
322
|
+
return undefined;
|
|
323
|
+
const newline = head.indexOf("\n");
|
|
324
|
+
const parts = Number(head.slice(0, newline));
|
|
325
|
+
if (newline < 1 || !Number.isSafeInteger(parts) || parts < 1)
|
|
326
|
+
return undefined;
|
|
327
|
+
let text = head.slice(newline + 1);
|
|
328
|
+
for (let part = 1; part < parts; part++) {
|
|
329
|
+
const chunk = await this.storage.get(`${key}#${part}`);
|
|
330
|
+
if (chunk === null)
|
|
331
|
+
return undefined;
|
|
332
|
+
text += chunk;
|
|
333
|
+
}
|
|
334
|
+
try {
|
|
335
|
+
const entry = JSON.parse(text);
|
|
336
|
+
if (!isRecord(entry) || typeof entry.key !== "string" || !isRecord(entry.outcome)) {
|
|
337
|
+
return undefined;
|
|
338
|
+
}
|
|
339
|
+
entries.push(entry);
|
|
340
|
+
}
|
|
341
|
+
catch {
|
|
342
|
+
return undefined;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
return entries;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// One read that several readers in one request wait on, each on its own terms.
|
|
2
|
+
//
|
|
3
|
+
// Discovery and the registry both coalesce a connector's catalog read inside
|
|
4
|
+
// one request, so a search, a describe, and the call after them cost one
|
|
5
|
+
// downstream listing. The read used to run under whichever reader asked
|
|
6
|
+
// first: its signal was the read's signal, so a short-deadline call that
|
|
7
|
+
// happened to start the read failed a search that joined it with the call's
|
|
8
|
+
// timeout, and a cancelled search cancelled a read others were still waiting
|
|
9
|
+
// on (#571).
|
|
10
|
+
//
|
|
11
|
+
// So the read owns its signal. Each reader waits under its own cancellation,
|
|
12
|
+
// and the read is cancelled only when no reader is left to want it. Nothing
|
|
13
|
+
// here is shared across requests: a SharedRead lives in one request's map and
|
|
14
|
+
// is waited on only by that request's fibers, which is why a plain count of
|
|
15
|
+
// readers is enough and no edge is needed between them.
|
|
16
|
+
import { Deferred, Effect, Exit } from "effect";
|
|
17
|
+
export class SharedRead {
|
|
18
|
+
onEnd;
|
|
19
|
+
controller = new AbortController();
|
|
20
|
+
outcome = Deferred.makeUnsafe();
|
|
21
|
+
readers = 0;
|
|
22
|
+
ended = false;
|
|
23
|
+
/**
|
|
24
|
+
* Start `read` at once, under a signal the read owns. `onEnd` runs once:
|
|
25
|
+
* when the read settles, with whether it succeeded, or when it is cancelled
|
|
26
|
+
* first, with `false`. It runs before any reader hears the outcome, so an
|
|
27
|
+
* owner that evicts the read there sends a reader who retries on hearing of
|
|
28
|
+
* a failure to a fresh read, never back to this one.
|
|
29
|
+
*/
|
|
30
|
+
constructor(read, onEnd = () => { }) {
|
|
31
|
+
this.onEnd = onEnd;
|
|
32
|
+
// The executor turns a synchronous throw from `read` into a rejection.
|
|
33
|
+
new Promise((resolve) => resolve(read(this.controller.signal))).then((value) => this.settle(Exit.succeed(value)), (cause) => this.settle(Exit.fail(cause)));
|
|
34
|
+
}
|
|
35
|
+
settle(exit) {
|
|
36
|
+
this.end(Exit.isSuccess(exit));
|
|
37
|
+
Deferred.doneUnsafe(this.outcome, exit);
|
|
38
|
+
}
|
|
39
|
+
end(succeeded) {
|
|
40
|
+
if (this.ended)
|
|
41
|
+
return;
|
|
42
|
+
this.ended = true;
|
|
43
|
+
this.onEnd(succeeded);
|
|
44
|
+
}
|
|
45
|
+
// Cancel the read for good. After this it is no longer joinable: whatever
|
|
46
|
+
// it eventually settles with still reaches the readers already waiting, but
|
|
47
|
+
// its owner has evicted it and a newcomer starts afresh.
|
|
48
|
+
cancel(reason) {
|
|
49
|
+
if (this.ended)
|
|
50
|
+
return;
|
|
51
|
+
this.controller.abort(reason);
|
|
52
|
+
this.end(false);
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Wait for the read under this reader's own cancellation.
|
|
56
|
+
*
|
|
57
|
+
* A reader whose signal aborts while others still wait fails at once with
|
|
58
|
+
* its own reason and leaves the read to them. The last reader's abort is the
|
|
59
|
+
* read's: it is forwarded to the read's signal and that reader waits for the
|
|
60
|
+
* read's answer, as a lone reader always has, so a connector that finishes
|
|
61
|
+
* its listing anyway still hands it over. A reader that is interrupted — its
|
|
62
|
+
* deadline, as `withDeadlineEffect` ends it — simply leaves, and when it was
|
|
63
|
+
* the last one the read is cancelled with its reason.
|
|
64
|
+
*/
|
|
65
|
+
join(signal) {
|
|
66
|
+
return Effect.suspend(() => {
|
|
67
|
+
this.readers++;
|
|
68
|
+
let waiting = true;
|
|
69
|
+
const leave = Effect.sync(() => {
|
|
70
|
+
if (!waiting)
|
|
71
|
+
return;
|
|
72
|
+
waiting = false;
|
|
73
|
+
this.readers--;
|
|
74
|
+
if (this.readers === 0)
|
|
75
|
+
this.cancel(signal?.reason);
|
|
76
|
+
});
|
|
77
|
+
const answer = Deferred.await(this.outcome);
|
|
78
|
+
const own = signal
|
|
79
|
+
? Effect.raceFirst(answer, this.leftBy(signal))
|
|
80
|
+
: answer;
|
|
81
|
+
return Effect.ensuring(own, leave);
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
// Fails with the reader's abort reason when other readers remain; when it is
|
|
85
|
+
// the last, forwards the abort to the read and never completes, leaving the
|
|
86
|
+
// read's own answer to win the race.
|
|
87
|
+
leftBy(signal) {
|
|
88
|
+
return Effect.callback((resume) => {
|
|
89
|
+
const onAbort = () => {
|
|
90
|
+
signal.removeEventListener("abort", onAbort);
|
|
91
|
+
if (this.readers > 1) {
|
|
92
|
+
resume(Effect.fail(signal.reason));
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
this.cancel(signal.reason);
|
|
96
|
+
}
|
|
97
|
+
};
|
|
98
|
+
if (signal.aborted) {
|
|
99
|
+
onAbort();
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
103
|
+
return Effect.sync(() => signal.removeEventListener("abort", onAbort));
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|