@chengxianglibra/dsh-data-analysis 0.1.1 → 0.1.2

package/lib/compatibility.js

@@ -46,7 +46,7 @@ function parsePackageManifest(value) {
                 packageSpec: requiredString(marivo.packageSpec, 'package compatibility marivo.packageSpec'),
             }),
             contracts: Object.freeze({
-                runtimeInstallation: requiredLiteral(contracts.runtimeInstallation, 'dsh-data-analysis-runtime/v2', 'package compatibility contracts.runtimeInstallation'),
+                runtimeInstallation: requiredLiteral(contracts.runtimeInstallation, 'dsh-data-analysis-runtime/v3', 'package compatibility contracts.runtimeInstallation'),
                 subprocessPolicy: requiredLiteral(contracts.subprocessPolicy, 'direct-argv-inherited-env-snapshot-overlay-v2', 'package compatibility contracts.subprocessPolicy'),
             }),
         }),

package/lib/datasource/authoring.js

@@ -0,0 +1,70 @@
+import { z } from 'zod';
+export const datasourceAuthoringSchema = z.object({
+    fingerprint: z.string(),
+    backends: z.array(z.object({
+        name: z.string(),
+        fields: z.array(z.object({
+            name: z.string(),
+            type: z.enum(['string', 'number', 'boolean', 'json']),
+            required: z.boolean(),
+            description: z.string(),
+            default: z.unknown().optional(),
+        })),
+    })),
+});
+// The public DatasourceSpec union and dataclass fields are read from the checked Runtime.
+// No backend registry or field schema is duplicated in the plugin.
+export const DATASOURCE_AUTHORING_PROGRAM = String.raw `
+import contextlib
+import dataclasses
+import json
+import sys
+import typing
+import marivo.datasource as md
+
+class Discard:
+    def write(self, value): return len(value)
+    def flush(self): pass
+
+def spec_fields(cls):
+    hints = typing.get_type_hints(cls)
+    result = []
+    for field in dataclasses.fields(cls):
+        if not field.init or field.name == "ai_context":
+            continue
+        annotation = hints[field.name]
+        types = set(typing.get_args(annotation) or [annotation]) - {type(None)}
+        kind = "string" if types == {str} else "boolean" if types == {bool} else "number" if types <= {int, float} else "json"
+        item = {"name": field.name, "type": kind,
+            "required": field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING,
+            "description": field.metadata.get("description", "")}
+        if field.default is not dataclasses.MISSING:
+            item["default"] = field.default
+        result.append(item)
+    return result
+
+try:
+    payload = json.load(sys.stdin)
+    with contextlib.redirect_stdout(Discard()), contextlib.redirect_stderr(Discard()):
+        specs = {cls.backend_type: cls for cls in typing.get_args(md.DatasourceSpec)}
+        if payload["action"] == "schema":
+            result = {"backends": [{"name": key, "fields": spec_fields(cls)} for key, cls in specs.items()]}
+        else:
+            cls = specs[payload["backend"]]
+            fields = payload["fields"]
+            allowed = {field["name"] for field in spec_fields(cls)}
+            if set(fields) - allowed:
+                raise ValueError("unknown-field")
+            spec = cls(**fields)
+            if any(item.name == spec.name for item in md.list()):
+                result = {"error": "datasource-already-exists"}
+            else:
+                # Validate references before writing; the Host applies its own reserved-name check too.
+                if sorted(spec.env_refs.values()) != sorted(payload["refs"]):
+                    raise ValueError("credential-reference-mismatch")
+                md.register(spec)
+                result = {"name": spec.name}
+except Exception:
+    result = {"error": "datasource-definition-invalid"}
+print(json.dumps(result))
+`.trim();

package/lib/datasource/bridge-programs.js

@@ -1,3 +1,4 @@
+import { RESOLVER_PROGRAM } from "./resolver-program.js";
 export const MARIVO_DATASOURCE_DESCRIBE_PROGRAM = String.raw `
 import json
 import os
@@ -5,11 +6,17 @@ import sys
 import marivo
 import marivo.datasource as md
 
-description = md.describe(sys.argv[1])
-print(json.dumps({
-    "name": description.name,
-    "refs": list(description.env_refs.values()),
-}, sort_keys=True))
+import hashlib
+
+def project_description(description):
+    payload = {"name": description.name, "backend": description.backend_type,
+        "fields": description.literal_fields, "refs": description.env_refs}
+    return {"name": description.name, "backend": description.backend_type,
+        "properties": description.literal_fields, "refs": list(description.env_refs.values()),
+        "fields": description.env_refs,
+        "definition": hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()}
+
+print(json.dumps(project_description(md.describe(sys.argv[1])), sort_keys=True))
 `.trim();
 export const MARIVO_DATASOURCE_INVENTORY_PROGRAM = String.raw `
 import json
@@ -18,14 +25,21 @@ import sys
 import marivo
 import marivo.datasource as md
 
+import hashlib
+
+def project_description(description):
+    payload = {"name": description.name, "backend": description.backend_type,
+        "fields": description.literal_fields, "refs": description.env_refs}
+    return {"name": description.name, "backend": description.backend_type,
+        "properties": description.literal_fields, "refs": list(description.env_refs.values()),
+        "fields": description.env_refs,
+        "definition": hashlib.sha256(json.dumps(payload, sort_keys=True, default=str).encode()).hexdigest()}
+
 print(json.dumps({
-    "datasources": [{
-        "name": description.name,
-        "refs": list(description.env_refs.values()),
-    } for description in (md.describe(item.name) for item in md.list())],
+    "datasources": [project_description(md.describe(item.name)) for item in md.list()],
 }, sort_keys=True))
 `.trim();
-export const MARIVO_DATASOURCE_TEST_PROGRAM = String.raw `
+export const MARIVO_DATASOURCE_TEST_PROGRAM = `${RESOLVER_PROGRAM}\n${String.raw `
 import contextlib
 import io
 import json
@@ -34,17 +48,15 @@ import sys
 import marivo
 import marivo.datasource as md
 
+payload = json.load(sys.stdin)
+resolver = SnapshotResolver(payload)
+secret_values = list(payload["values"].values())
 captured_stdout = io.StringIO()
 captured_stderr = io.StringIO()
 try:
     with contextlib.redirect_stdout(captured_stdout), contextlib.redirect_stderr(captured_stderr):
-        description = md.describe(sys.argv[1])
-        secret_values = [
-            value
-            for ref in description.env_refs.values()
-            if (value := os.environ.get(ref))
-        ]
-        result = md.test(sys.argv[1])
+        with md.credential_scope(resolver=resolver):
+            result = md.test(sys.argv[1])
 except Exception as exc:
     print(json.dumps({"kind": "datasource-test-failed", "exception_type": type(exc).__name__}), file=sys.stderr)
     raise SystemExit(70)
@@ -77,4 +89,4 @@ print(json.dumps(redact({
     "failure": failure,
     "repair": repair,
 }), sort_keys=True))
-`.trim();
+`.trim()}`;

package/lib/datasource/bridge.js

@@ -1,7 +1,9 @@
 import { isCredentialRefName } from '@deepseek-ai/dsh-credentials';
 import { MarivoEnvironmentError } from "../environment/errors.js";
 import { resolveMarivoBridgeSource } from "../environment/source.js";
+import { DATASOURCE_AUTHORING_PROGRAM, datasourceAuthoringSchema, } from "./authoring.js";
 import { MARIVO_DATASOURCE_DESCRIBE_PROGRAM, MARIVO_DATASOURCE_INVENTORY_PROGRAM, MARIVO_DATASOURCE_TEST_PROGRAM, } from "./bridge-programs.js";
+import { marivoCredentialStorageRef } from "./shell-env.js";
 const DATASOURCE_LIMITS = Object.freeze({
     timeoutMs: 30_000,
     stdoutMaxBytes: 262_144,
@@ -24,6 +26,16 @@ function string(value) {
 function nullableString(value) {
     return value === null || typeof value === 'string';
 }
+function jsonValue(value) {
+    if (value === null || typeof value === 'string' || typeof value === 'boolean')
+        return true;
+    if (typeof value === 'number')
+        return Number.isFinite(value);
+    if (Array.isArray(value))
+        return value.every(jsonValue);
+    const fields = object(value);
+    return fields !== undefined && Object.values(fields).every(jsonValue);
+}
 function parseJsonObject(stdout, phase) {
     try {
         const parsed = JSON.parse(stdout.toString('utf8'));
@@ -41,15 +53,24 @@ function parseDescription(value, phase) {
     if (source === undefined) {
         throw new MarivoEnvironmentError('subprocess-output-invalid', `Marivo datasource ${phase} returned invalid fields`, { phase });
     }
-    exactKeys(source, ['name', 'refs'], phase);
+    exactKeys(source, ['name', 'backend', 'properties', 'refs', 'fields', 'definition'], phase);
     const name = string(source.name);
-    if (name === undefined || !Array.isArray(source.refs)) {
+    const backend = string(source.backend);
+    const properties = object(source.properties);
+    if (name === undefined ||
+        name.length > 256 ||
+        backend === undefined ||
+        backend.length > 256 ||
+        properties === undefined ||
+        !jsonValue(properties) ||
+        !Array.isArray(source.refs) ||
+        source.refs.length > 128) {
         throw new MarivoEnvironmentError('subprocess-output-invalid', `Marivo datasource ${phase} returned invalid fields`, { phase });
     }
     const refs = [];
     const seen = new Set();
     for (const ref of source.refs) {
-        if (typeof ref !== 'string' || !isCredentialRefName(ref)) {
+        if (typeof ref !== 'string' || ref.length > 256 || !isCredentialRefName(ref)) {
             throw new MarivoEnvironmentError('subprocess-output-invalid', `Marivo datasource ${phase} returned an invalid credential reference`, { phase });
         }
         if (!seen.has(ref)) {
@@ -57,7 +78,21 @@ function parseDescription(value, phase) {
             refs.push(ref);
         }
     }
-    return { name, refs };
+    const fields = object(source.fields);
+    if (!fields ||
+        typeof source.definition !== 'string' ||
+        !/^[a-f0-9]{64}$/.test(source.definition) ||
+        Object.values(fields).some((ref) => typeof ref !== 'string' || !refs.includes(ref)) ||
+        refs.some((ref) => !Object.values(fields).includes(ref)))
+        throw new Error('Invalid datasource definition projection');
+    return {
+        name,
+        backend,
+        properties: properties,
+        refs,
+        fields: fields,
+        definition: source.definition,
+    };
 }
 function parseInventory(stdout) {
     const value = parseJsonObject(stdout, 'inventory');
@@ -87,7 +122,9 @@ function parseRepair(value) {
         throw new TypeError('repair must be an object');
     exactKeys(source, ['kind', 'help_target', 'action', 'snippet', 'candidates', 'preserves_evidence'], 'test');
     if (string(source.kind) === undefined ||
-        string(source.help_target) === undefined ||
+        object(source.help_target) === undefined ||
+        string(object(source.help_target)?.surface) === undefined ||
+        !nullableString(object(source.help_target)?.canonical_id) ||
         typeof source.action !== 'string' ||
         !nullableString(source.snippet) ||
         !Array.isArray(source.candidates) ||
@@ -160,6 +197,55 @@ export class MarivoDatasourceBridge {
         this.#runner = runner;
         this.binding = runner.binding;
     }
+    async authoring(signal) {
+        const result = await this.#runner.runChecked({
+            program: DATASOURCE_AUTHORING_PROGRAM,
+            stdin: JSON.stringify({ action: 'schema' }),
+            limits: DATASOURCE_LIMITS,
+            signal,
+        });
+        this.#assertSuccess(result, 'inventory');
+        return datasourceAuthoringSchema.parse({
+            ...parseJsonObject(result.stdout, 'inventory'),
+            fingerprint: this.binding.fingerprint,
+        });
+    }
+    async create(input, signal) {
+        const refs = Object.entries(input.fields)
+            .filter(([key]) => key.endsWith('_env'))
+            .flatMap(([, value]) => typeof value === 'string'
+            ? [value]
+            : value && typeof value === 'object' && !Array.isArray(value)
+                ? Object.values(value)
+                : []);
+        for (const ref of refs) {
+            if (typeof ref !== 'string')
+                return { error: 'datasource-credential-ref-invalid' };
+            try {
+                marivoCredentialStorageRef(ref);
+            }
+            catch (error) {
+                if (error instanceof MarivoEnvironmentError &&
+                    error.code === 'datasource-credential-ref-invalid')
+                    return { error: 'datasource-credential-ref-invalid' };
+                throw error;
+            }
+        }
+        const result = await this.#runner.runChecked({
+            program: DATASOURCE_AUTHORING_PROGRAM,
+            stdin: JSON.stringify({ action: 'create', ...input, refs }),
+            limits: DATASOURCE_LIMITS,
+            signal,
+        });
+        this.#assertSuccess(result, 'inventory');
+        const value = parseJsonObject(result.stdout, 'inventory');
+        if (value.error === 'datasource-already-exists' ||
+            value.error === 'datasource-definition-invalid')
+            return { error: value.error };
+        if (typeof value.name !== 'string')
+            throw new Error('Invalid datasource creation result');
+        return { name: value.name };
+    }
     async describe(name, signal) {
         const result = await this.#runner.runChecked({
             program: MARIVO_DATASOURCE_DESCRIBE_PROGRAM,
@@ -179,16 +265,22 @@ export class MarivoDatasourceBridge {
         this.#assertSuccess(result, 'inventory');
         return parseInventory(result.stdout);
     }
-    async test(name, environmentOverlay, signal) {
+    async test(description, values, signal) {
+        const { name } = description;
         const result = await this.#runner.runChecked({
             program: MARIVO_DATASOURCE_TEST_PROGRAM,
             args: [name],
-            environmentOverlay,
+            stdin: JSON.stringify({
+                project_root: this.binding.projectRoot,
+                grants: { [name]: description },
+                values,
+            }),
+            secretValues: Object.values(values),
             limits: DATASOURCE_LIMITS,
             signal,
         });
         this.#assertSuccess(result, 'test');
-        return parseTest(result.stdout, Object.values(environmentOverlay).filter((value) => Boolean(value)));
+        return parseTest(result.stdout, Object.values(values));
     }
     #assertSuccess(result, phase) {
         if (result.exitCode === 0)

package/lib/datasource/index.js

@@ -1,4 +1,5 @@
-export { createMarivoDatasourceAccessTool, MARIVO_DATASOURCE_ACCESS_TOOL_NAME, registerMarivoDatasourceAccessTool, } from "./access.js";
 export { MarivoDatasourceBridge, resolveMarivoDatasourceBridge, resolveMarivoDatasourceInventoryBridge, } from "./bridge.js";
-export { assertMarivoCredentialReferences, DEFAULT_MARIVO_CREDENTIAL_LEASE_MAX_USES, DEFAULT_MARIVO_CREDENTIAL_LEASE_TTL_MS, MARIVO_CREDENTIAL_LEASE_PREFIX, MARIVO_CREDENTIAL_STORAGE_PREFIX, MarivoShellCredentialLeases, marivoCredentialStorageRef, } from "./shell-env.js";
+export { registerMarivoPythonTool } from "./python.js";
+export { MarivoCredentialService } from "./service.js";
+export { assertMarivoCredentialReferences, MARIVO_CREDENTIAL_STORAGE_PREFIX, marivoCredentialStorageRef, } from "./shell-env.js";
 export { createMarivoDatasourceTestTool, MARIVO_DATASOURCE_TEST_TOOL_NAME, registerMarivoDatasourceTestTool, } from "./test.js";

package/lib/datasource/python.js

@@ -0,0 +1,115 @@
+import process from 'node:process';
+import { defineTool } from '@deepseek-ai/dsh-tools';
+import { savePythonExecution } from "../python-execution.js";
+import { resolveMarivoDatasourceBridge } from "./bridge.js";
+import { PYTHON_LAUNCHER, PYTHON_WORKER } from "./resolver-program.js";
+import { datasourceToolValue } from "./test.js";
+function quote(value) {
+    return process.platform === 'win32'
+        ? `'${value.replaceAll("'", "''")}'`
+        : `'${value.replaceAll("'", "'\\''")}'`;
+}
+export function registerMarivoPythonTool(ctx, source, service) {
+    return ctx.tools.register(defineTool({
+        name: 'marivo_python',
+        description: 'Execute foreground Python in the bound Marivo Workspace using Host-injected credentials. This call waits for missing credentials for every listed datasource before starting Python once; configured credentials do not trigger a connection test. Create/resume Sessions and readers inside this execution; close Sessions in finally. No background execution or secret environment variables. Nonzero exits are reported without replay. Successful execution saves the exact submitted code and returns codeRef; explicitly associate it with presentation datasets through draft codeRefs. A codeCaptureError does not change the execution outcome and must not trigger replay.',
+        parameters: {
+            code: {
+                type: 'string',
+                required: true,
+                description: 'Python code, never credential values. Use try/finally for per-call Session resource cleanup; consult current Runtime Help for Session APIs.',
+            },
+            datasources: {
+                type: 'array',
+                required: true,
+                items: { type: 'string' },
+                description: 'All exact datasource names this execution may access through data or metadata connections, including inspection and datasources without passwords; pass [] only when no datasource will be accessed. Catalog-only definition reads need no datasource connection.',
+            },
+        },
+        output: {
+            schema: { type: 'json' },
+            render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }],
+        },
+        async execute(args, exec) {
+            const code = args.code;
+            if (!code.trim() || Buffer.byteLength(code) > 131072)
+                throw new Error('Invalid Python code size');
+            const shell = ctx.get('shell');
+            if (!shell)
+                throw new Error('DSH Shell execution service is required');
+            const shellEnv = ctx.get('shellEnv');
+            if (!shellEnv)
+                throw new Error('DSH Shell environment service is required');
+            const policyService = ctx.get('sandboxPolicy');
+            if (shell.sandboxMode !== undefined && !policyService)
+                throw new Error('DSH sandbox policy is required');
+            const prepared = await service.track(service.prepareExecution(exec, () => resolveMarivoDatasourceBridge(source), args.datasources));
+            if (!('status' in prepared) || prepared.status !== 'ready')
+                return datasourceToolValue(prepared);
+            try {
+                const { binding } = prepared.bridge;
+                const policy = policyService?.resolve(exec.agent ? { session: exec.agent.session } : {});
+                const spec = shell.resolve({
+                    command: `${process.platform === 'win32' ? '& ' : ''}${quote(binding.pythonExecutable)} -c ${quote(PYTHON_LAUNCHER)}`,
+                    workdir: binding.projectRoot,
+                    timeoutMs: 120_000,
+                    signal: prepared.signal,
+                    stdin: JSON.stringify({
+                        identity: binding,
+                        project_root: binding.projectRoot,
+                        grants: prepared.grants,
+                        values: prepared.values,
+                        code,
+                        worker: PYTHON_WORKER,
+                    }),
+                    env: { MARIVO_PERSIST_CREDENTIALS: '0' },
+                    dshEnv: shellEnv.collect(exec),
+                    ...(policy ? { sandboxPolicy: policy } : {}),
+                });
+                // Keep this guard adjacent to the only launch, after synchronous Host hooks.
+                prepared.assertCurrent();
+                const startedAt = new Date().toISOString();
+                const result = await service.track(shell.run(spec));
+                const finishedAt = new Date().toISOString();
+                // Defense at the result seam as well as before Harness collection/spill.
+                const redact = (text) => Object.values(prepared.values)
+                    .filter(Boolean)
+                    .sort((a, b) => b.length - a.length)
+                    .reduce((out, value) => out.split(value).join('[REDACTED]'), text);
+                let codeRef;
+                let codeCaptureError;
+                if (result.exitCode === 0 && !result.timedOut && !result.aborted) {
+                    try {
+                        prepared.assertCurrent();
+                        if (Object.values(prepared.values).some((value) => value && code.includes(value)))
+                            throw new Error('Host credential in submitted code');
+                        codeRef = await service.track(savePythonExecution(binding.projectRoot, { text: code, startedAt, finishedAt }, prepared.signal));
+                        prepared.assertCurrent();
+                    }
+                    catch {
+                        codeRef = undefined;
+                        codeCaptureError =
+                            'Python completed successfully, but its source code could not be saved. Do not rerun the code to recover this snapshot; inspect its existing effects.';
+                    }
+                }
+                return JSON.parse(JSON.stringify({
+                    exitCode: result.exitCode,
+                    timedOut: result.timedOut,
+                    aborted: result.aborted,
+                    stdout: redact(result.stdout.text),
+                    stderr: redact(result.stderr.text),
+                    truncated: result.stdout.truncated || result.stderr.truncated,
+                    sandbox: result.sandbox ?? null,
+                    ...(codeRef ? { codeRef } : {}),
+                    ...(codeCaptureError ? { codeCaptureError } : {}),
+                }));
+            }
+            catch {
+                throw new Error('Marivo Python execution failed or was cancelled; inspect the execution policy and retry only after checking effects');
+            }
+            finally {
+                prepared.release();
+            }
+        },
+    }));
+}

package/lib/datasource/resolver-program.js

@@ -0,0 +1,112 @@
+/** Shared Python adapter. All inputs below arrive on a Host-owned pipe. */
+export const RESOLVER_PROGRAM = String.raw `
+import hashlib
+import json
+import os
+import sys
+from pathlib import Path
+import marivo
+import marivo.datasource as md
+
+if not all(hasattr(md, name) for name in ("credential_scope", "CredentialRequest", "SecretValue", "DatasourceCredentialError")):
+    raise RuntimeError("Marivo credential_scope capability is required; upgrade the bound Runtime")
+
+def definition(description):
+    return hashlib.sha256(json.dumps({
+        "name": description.name, "backend": description.backend_type,
+        "fields": description.literal_fields, "refs": description.env_refs,
+    }, sort_keys=True, default=str).encode()).hexdigest()
+
+class SnapshotResolver:
+    def __init__(self, payload):
+        self.root = Path(payload["project_root"]).resolve()
+        self.grants = payload["grants"]
+        self.values = payload["values"]
+
+    def resolve(self, request):
+        grant = self.grants.get(request.datasource)
+        if request.cancelled:
+            reason = "timeout"
+        elif Path(request.project_root).resolve() != self.root or grant is None:
+            reason = "denied"
+        elif any(grant["fields"].get(field) != request.reference for field in request.fields):
+            reason = "denied"
+        elif not request.fields or request.reference not in grant["fields"].values():
+            reason = "denied"
+        elif definition(md.describe(request.datasource)) != grant["definition"]:
+            reason = "denied"
+        elif not self.values.get(request.reference):
+            reason = "missing"
+        else:
+            return md.SecretValue(self.values[request.reference])
+        raise md.DatasourceCredentialError(reason=reason, reference=request.reference,
+            datasource=request.datasource, fields=request.fields)
+`.trim();
+/** The worker checks the exact binding and enters scope before any user code. */
+export const PYTHON_WORKER = `${RESOLVER_PROGRAM}\n${String.raw `
+payload = json.load(sys.stdin)
+expected = payload["identity"]
+if (os.path.abspath(sys.executable) != expected["pythonExecutable"] or
+    marivo.__version__ != expected["marivoVersion"] or
+    os.path.abspath(marivo.__file__) != expected["packagePath"] or
+    str(Path.cwd().resolve()) != payload["project_root"]):
+    raise RuntimeError("Marivo execution identity changed; rebind required")
+resolver = SnapshotResolver(payload)
+code = payload.pop("code")
+sys.stdin = open(os.devnull)
+try:
+    with md.credential_scope(resolver=resolver):
+        for name, grant in resolver.grants.items():
+            description = md.describe(name)
+            if grant["name"] != name or description.name != name:
+                raise RuntimeError("Marivo datasource identity changed; execution cancelled")
+            if definition(description) != grant["definition"]:
+                raise RuntimeError("Marivo datasource definition changed; execution cancelled")
+        exec(compile(code, "<marivo_python>", "exec"), {"__name__": "__main__"})
+finally:
+    resolver.values.clear()
+    payload.clear()
+`}`;
+/** Capture raw fd output before Harness collection/spill, including driver writes. */
+export const PYTHON_LAUNCHER = String.raw `
+import json, subprocess, sys, threading
+payload = json.load(sys.stdin)
+worker = payload.pop("worker")
+values = sorted(set(payload["values"].values()), key=len, reverse=True)
+child = subprocess.Popen([sys.executable, "-c", worker], stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+streams = [bytearray(), bytearray()]
+overflow = threading.Event()
+def collect(pipe, target):
+    while True:
+        chunk = pipe.read(8192)
+        if not chunk:
+            break
+        if len(target) + len(chunk) > 1048576:
+            overflow.set()
+            child.kill()
+        elif not overflow.is_set():
+            target.extend(chunk)
+threads = [threading.Thread(target=collect, args=(pipe, buf), daemon=True)
+    for pipe, buf in zip((child.stdout, child.stderr), streams)]
+for thread in threads:
+    thread.start()
+try:
+    child.stdin.write(json.dumps(payload).encode())
+    child.stdin.close()
+except BrokenPipeError:
+    pass
+child.wait()
+for thread in threads:
+    thread.join()
+if overflow.is_set():
+    print("Marivo Python output limit exceeded", file=sys.stderr)
+    raise SystemExit(74)
+for stream, raw in zip((sys.stdout, sys.stderr), streams):
+    text = raw.decode("utf-8", errors="replace")
+    for value in values:
+        if value:
+            text = text.replace(value, "[REDACTED]")
+    stream.write(text)
+raise SystemExit(child.returncode if child.returncode >= 0 else 1)
+`.trim();

package/lib/datasource/rpc.js

@@ -0,0 +1,88 @@
+import { z } from 'zod';
+import { CREDENTIAL_CHANNEL, credentialError } from "./service.js";
+const text = z.string().min(1).max(256);
+const handle = { generation: z.string().uuid(), id: z.string().uuid(), scope: z.string().uuid() };
+const mutation = z
+    .object({
+    ...handle,
+    action: z.enum(['save', 'update', 'delete', 'test', 'submit', 'diagnose']),
+    version: z.string().max(65536),
+    requestId: z.string().uuid().optional(),
+    reference: text.optional(),
+    changes: z.record(text, z.string().min(1).max(65536)).optional(),
+})
+    .strict();
+export function registerCredentialRpc(connection, service, workspace) {
+    const unregister = connection.rpc.handle(CREDENTIAL_CHANNEL, async (endpoint, payload, signal) => {
+        try {
+            if (Buffer.byteLength(JSON.stringify(payload) ?? '') > 1_048_576)
+                throw new Error('invalid-request');
+            let value;
+            if (endpoint === 'overview') {
+                const input = z.object({ workspaceId: text }).strict().parse(payload);
+                value = {
+                    generation: service.generation,
+                    datasources: await service.overview(input.workspaceId, () => workspace(input.workspaceId), signal),
+                };
+            }
+            else if (endpoint === 'authoring') {
+                const input = z.object({ workspaceId: text }).strict().parse(payload);
+                const bridge = await workspace(input.workspaceId);
+                if (!bridge.authoring)
+                    throw new Error('authoring-unavailable');
+                value = { generation: service.generation, ...(await bridge.authoring(signal)) };
+            }
+            else if (endpoint === 'create-datasource') {
+                const input = z
+                    .object({
+                    workspaceId: text,
+                    generation: z.string().uuid(),
+                    fingerprint: z.string().min(1).max(256),
+                    backend: text,
+                    fields: z.record(text, z.unknown()),
+                })
+                    .strict()
+                    .parse(payload);
+                value = await service.createDatasource(input.generation, input.fingerprint, input, () => workspace(input.workspaceId), signal);
+            }
+            else if (endpoint === 'watch') {
+                const input = z
+                    .object({ sessionId: text, cursor: z.string().max(256).optional() })
+                    .strict()
+                    .parse(payload);
+                value = await service.waitWatch(input.sessionId, input.cursor, signal);
+            }
+            else if (endpoint === 'start') {
+                value = service.start(mutation.parse(payload));
+            }
+            else if (endpoint === 'operation') {
+                const input = z.object(handle).strict().parse(payload);
+                value = service.operation(input.generation, input.id, input.scope) ?? null;
+            }
+            else if (endpoint === 'cancel-operation') {
+                const input = z.object(handle).strict().parse(payload);
+                service.cancelOperation(input.generation, input.id, input.scope);
+                value = { cancelled: true };
+            }
+            else if (endpoint === 'cancel-request') {
+                const input = z.object({ requestId: z.string().uuid() }).strict().parse(payload);
+                service.cancelRequest(input.requestId);
+                value = { cancelled: true };
+            }
+            else
+                throw new Error('unknown-endpoint');
+            return { ok: true, value };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: { code: 'internal', message: credentialError(error), details: {} },
+            };
+        }
+    }, { authority: 'trusted-host' });
+    return async () => {
+        const draining = unregister();
+        await service.close();
+        await draining;
+    };
+}