@chengxianglibra/dsh-data-analysis 0.1.1 → 0.2.0

package/lib/datasource/python.js

@@ -0,0 +1,206 @@
+import { performance } from 'node:perf_hooks';
+import process from 'node:process';
+import { defineTool } from '@deepseek-ai/dsh-tools';
+import { savePythonExecution } from "../python-execution.js";
+import { registerMarivoTool } from "../tool-lifecycle.js";
+import { resolveMarivoDatasourceBridge } from "./bridge.js";
+import { pythonTimeout, resolvePythonOptions } from "./python-options.js";
+import { PYTHON_LAUNCHER, PYTHON_WORKER } from "./resolver-program.js";
+import { CredentialServiceError, } from "./service.js";
+import { datasourceToolValue } from "./test.js";
+/** The message survives Harness error normalization; it contains only safe adapter facts. */
+export class MarivoPythonExecutionError extends Error {
+    execution;
+    constructor(message, execution) {
+        super(`${message}; execution=${JSON.stringify(execution)}`);
+        this.name = 'MarivoPythonExecutionError';
+        this.execution = execution;
+    }
+}
+const CHECK_EFFECTS = 'Inspect existing effects and the intended Session through current Runtime Help before retrying; do not automatically replay. Saved results and remote query cancellation are not confirmed.';
+const PREPARATION_CODES = new Set([
+    'agent-required',
+    'invalid-datasources',
+    'call-ended',
+    'context-changed',
+    'credentials-changed',
+    'execution-ended',
+    'disposed',
+]);
+function quote(value) {
+    return process.platform === 'win32'
+        ? `'${value.replaceAll("'", "''")}'`
+        : `'${value.replaceAll("'", "'\\''")}'`;
+}
+export function registerMarivoPythonTool(ctx, source, service, options = {}) {
+    const limits = resolvePythonOptions(options);
+    return registerMarivoTool(ctx, defineTool({
+        name: 'marivo_python',
+        description: 'Execute foreground Python in the bound Runtime and Workspace, including local pandas and native DuckDB file analysis. This call waits for missing credentials for every listed Marivo datasource before starting Python once; configured credentials do not trigger a connection test. When using Marivo Sessions or readers, create/resume them inside this execution and close Sessions in finally. No background execution or secret environment variables. Nonzero exits are reported without replay. Successful execution saves the exact submitted code and returns codeRef; explicitly associate it with presentation datasets through draft codeRefs. A codeCaptureError does not change the execution outcome and must not trigger replay. The execution summary reports adapter phase, outcome and effective Shell budget, not query progress or saved Artifacts. Credential waiting precedes the Shell budget; outer Code Mode deadlines and cancellation still apply. Timeout does not confirm remote query cancellation or absence of saved results.',
+        parameters: {
+            code: {
+                type: 'string',
+                required: true,
+                description: 'Python code, never credential values. Close resources within this call. When using Marivo Session APIs, consult current Runtime Help and close Sessions in finally.',
+            },
+            datasources: {
+                type: 'array',
+                required: true,
+                items: { type: 'string' },
+                description: 'All exact Marivo datasource names this execution may access through data or metadata connections, including inspection and datasources without passwords. Pass [] for local pandas or native DuckDB file analysis without Marivo datasource access. Catalog-only definition reads need no datasource connection.',
+            },
+            timeoutMs: {
+                type: 'number',
+                description: `Optional foreground Shell timeout in milliseconds, a positive integer at most 2147483647. Defaults to ${limits.pythonTimeoutMs}; capped at ${limits.pythonMaxTimeoutMs} and then by Harness Shell. Outer Code Mode deadlines remain independent.`,
+            },
+        },
+        output: {
+            schema: { type: 'json' },
+            render: (_args, value) => [{ type: 'text', text: JSON.stringify(value) }],
+        },
+        async execute(args, exec) {
+            const calledAt = performance.now();
+            let phase = 'preparing';
+            let requestedTimeoutMs = null;
+            let effectiveTimeoutMs = null;
+            let shellStartedAt = null;
+            let executionElapsedMs = null;
+            let prepared;
+            const summary = (reason, nextAction) => {
+                const now = performance.now();
+                return {
+                    phase,
+                    reason,
+                    requestedTimeoutMs,
+                    effectiveTimeoutMs,
+                    elapsedMs: now - calledAt,
+                    executionElapsedMs: executionElapsedMs ?? (shellStartedAt === null ? null : now - shellStartedAt),
+                    nextAction,
+                };
+            };
+            const rejectInput = (message) => {
+                throw new MarivoPythonExecutionError(message, summary('not-started', 'Correct the input or Host configuration before retrying.'));
+            };
+            try {
+                try {
+                    requestedTimeoutMs = pythonTimeout(args.timeoutMs === undefined ? limits.pythonTimeoutMs : args.timeoutMs, 'timeoutMs');
+                }
+                catch {
+                    rejectInput('timeoutMs must be a positive integer no greater than 2147483647');
+                }
+                const code = args.code;
+                if (typeof code !== 'string' || !code.trim() || Buffer.byteLength(code) > 131072)
+                    rejectInput('Invalid Python code size');
+                const shell = ctx.get('shell');
+                if (!shell)
+                    rejectInput('DSH Shell execution service is required');
+                const shellEnv = ctx.get('shellEnv');
+                if (!shellEnv)
+                    rejectInput('DSH Shell environment service is required');
+                const policyService = ctx.get('sandboxPolicy');
+                if (shell.sandboxMode !== undefined && !policyService)
+                    rejectInput('DSH sandbox policy is required');
+                const admission = await service.track(service.prepareExecution(exec, () => resolveMarivoDatasourceBridge(source), args.datasources));
+                if (!('status' in admission) || admission.status !== 'ready')
+                    return {
+                        ...datasourceToolValue(admission),
+                        execution: summary('not-started', 'Resolve the datasource credential requirement or reported connection failure before retrying.'),
+                    };
+                prepared = admission;
+                const { binding } = prepared.bridge;
+                const policy = policyService?.resolve(exec.agent ? { session: exec.agent.session } : {});
+                const spec = shell.resolve({
+                    command: `${process.platform === 'win32' ? '& ' : ''}${quote(binding.pythonExecutable)} -c ${quote(PYTHON_LAUNCHER)}`,
+                    workdir: binding.projectRoot,
+                    timeoutMs: Math.min(requestedTimeoutMs, limits.pythonMaxTimeoutMs),
+                    signal: prepared.signal,
+                    stdin: JSON.stringify({
+                        identity: binding,
+                        project_root: binding.projectRoot,
+                        grants: prepared.grants,
+                        values: prepared.values,
+                        code,
+                        worker: PYTHON_WORKER,
+                    }),
+                    env: { MARIVO_PERSIST_CREDENTIALS: '0' },
+                    dshEnv: shellEnv.collect(exec),
+                    ...(policy ? { sandboxPolicy: policy } : {}),
+                });
+                effectiveTimeoutMs = spec.timeoutMs;
+                // Keep this guard adjacent to the only launch, after synchronous Host hooks.
+                prepared.assertCurrent();
+                const startedAt = new Date().toISOString();
+                phase = 'executing';
+                shellStartedAt = performance.now();
+                const result = await service.track(shell.run(spec));
+                executionElapsedMs = performance.now() - shellStartedAt;
+                const finishedAt = new Date().toISOString();
+                const credentialValues = prepared.values;
+                // Defense at the result seam as well as before Harness collection/spill.
+                const redact = (text) => Object.values(credentialValues)
+                    .filter(Boolean)
+                    .sort((a, b) => b.length - a.length)
+                    .reduce((out, value) => out.split(value).join('[REDACTED]'), text);
+                let codeRef;
+                let codeCaptureError;
+                if (result.exitCode === 0 && !result.timedOut && !result.aborted) {
+                    phase = 'capturing-code';
+                    try {
+                        prepared.assertCurrent();
+                        if (Object.values(prepared.values).some((value) => value && code.includes(value)))
+                            throw new Error('Host credential in submitted code');
+                        codeRef = await service.track(savePythonExecution(binding.projectRoot, { text: code, startedAt, finishedAt }, prepared.signal));
+                        prepared.assertCurrent();
+                    }
+                    catch {
+                        codeRef = undefined;
+                        codeCaptureError =
+                            'Python completed successfully, but its source code could not be saved. Do not rerun the code to recover this snapshot; inspect its existing effects.';
+                    }
+                }
+                const reason = result.timedOut
+                    ? 'timed-out'
+                    : result.aborted
+                        ? 'cancelled'
+                        : result.exitCode === 0
+                            ? 'succeeded'
+                            : result.exitCode === null
+                                ? 'unknown'
+                                : 'nonzero-exit';
+                return JSON.parse(JSON.stringify({
+                    exitCode: result.exitCode,
+                    timedOut: result.timedOut,
+                    aborted: result.aborted,
+                    stdout: redact(result.stdout.text),
+                    stderr: redact(result.stderr.text),
+                    truncated: result.stdout.truncated || result.stderr.truncated,
+                    sandbox: result.sandbox ?? null,
+                    ...(codeRef ? { codeRef } : {}),
+                    ...(codeCaptureError ? { codeCaptureError } : {}),
+                    execution: summary(reason, codeCaptureError ?? (reason === 'succeeded' ? null : CHECK_EFFECTS)),
+                }));
+            }
+            catch (error) {
+                if (error instanceof MarivoPythonExecutionError)
+                    throw error;
+                const cancelled = (prepared?.signal ?? exec.signal).aborted;
+                const reason = cancelled
+                    ? 'cancelled'
+                    : shellStartedAt === null
+                        ? 'not-started'
+                        : 'unknown';
+                const code = error instanceof CredentialServiceError && PREPARATION_CODES.has(error.code)
+                    ? ` (${error.code})`
+                    : '';
+                throw new MarivoPythonExecutionError(shellStartedAt === null
+                    ? `Marivo Python preparation failed or was cancelled; Python was not started${code}`
+                    : 'Marivo Python execution outcome is unknown after an execution service failure or cancellation', summary(reason, shellStartedAt === null
+                    ? 'Check credentials, Workspace binding and Host execution policy before retrying.'
+                    : CHECK_EFFECTS));
+            }
+            finally {
+                prepared?.release();
+            }
+        },
+    }));
+}

package/lib/datasource/resolver-program.js

@@ -0,0 +1,112 @@
+/** Shared Python adapter. All inputs below arrive on a Host-owned pipe. */
+export const RESOLVER_PROGRAM = String.raw `
+import hashlib
+import json
+import os
+import sys
+from pathlib import Path
+import marivo
+import marivo.datasource as md
+
+if not all(hasattr(md, name) for name in ("credential_scope", "CredentialRequest", "SecretValue", "DatasourceCredentialError")):
+    raise RuntimeError("Marivo credential_scope capability is required; upgrade the bound Runtime")
+
+def definition(description):
+    return hashlib.sha256(json.dumps({
+        "name": description.name, "backend": description.backend_type,
+        "fields": description.literal_fields, "refs": description.env_refs,
+    }, sort_keys=True, default=str).encode()).hexdigest()
+
+class SnapshotResolver:
+    def __init__(self, payload):
+        self.root = Path(payload["project_root"]).resolve()
+        self.grants = payload["grants"]
+        self.values = payload["values"]
+
+    def resolve(self, request):
+        grant = self.grants.get(request.datasource)
+        if request.cancelled:
+            reason = "timeout"
+        elif Path(request.project_root).resolve() != self.root or grant is None:
+            reason = "denied"
+        elif any(grant["fields"].get(field) != request.reference for field in request.fields):
+            reason = "denied"
+        elif not request.fields or request.reference not in grant["fields"].values():
+            reason = "denied"
+        elif definition(md.describe(request.datasource)) != grant["definition"]:
+            reason = "denied"
+        elif not self.values.get(request.reference):
+            reason = "missing"
+        else:
+            return md.SecretValue(self.values[request.reference])
+        raise md.DatasourceCredentialError(reason=reason, reference=request.reference,
+            datasource=request.datasource, fields=request.fields)
+`.trim();
+/** The worker checks the exact binding and enters scope before any user code. */
+export const PYTHON_WORKER = `${RESOLVER_PROGRAM}\n${String.raw `
+payload = json.load(sys.stdin)
+expected = payload["identity"]
+if (os.path.abspath(sys.executable) != expected["pythonExecutable"] or
+    marivo.__version__ != expected["marivoVersion"] or
+    os.path.abspath(marivo.__file__) != expected["packagePath"] or
+    str(Path.cwd().resolve()) != payload["project_root"]):
+    raise RuntimeError("Marivo execution identity changed; rebind required")
+resolver = SnapshotResolver(payload)
+code = payload.pop("code")
+sys.stdin = open(os.devnull)
+try:
+    with md.credential_scope(resolver=resolver):
+        for name, grant in resolver.grants.items():
+            description = md.describe(name)
+            if grant["name"] != name or description.name != name:
+                raise RuntimeError("Marivo datasource identity changed; execution cancelled")
+            if definition(description) != grant["definition"]:
+                raise RuntimeError("Marivo datasource definition changed; execution cancelled")
+        exec(compile(code, "<marivo_python>", "exec"), {"__name__": "__main__"})
+finally:
+    resolver.values.clear()
+    payload.clear()
+`}`;
+/** Capture raw fd output before Harness collection/spill, including driver writes. */
+export const PYTHON_LAUNCHER = String.raw `
+import json, subprocess, sys, threading
+payload = json.load(sys.stdin)
+worker = payload.pop("worker")
+values = sorted(set(payload["values"].values()), key=len, reverse=True)
+child = subprocess.Popen([sys.executable, "-c", worker], stdin=subprocess.PIPE,
+    stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+streams = [bytearray(), bytearray()]
+overflow = threading.Event()
+def collect(pipe, target):
+    while True:
+        chunk = pipe.read(8192)
+        if not chunk:
+            break
+        if len(target) + len(chunk) > 1048576:
+            overflow.set()
+            child.kill()
+        elif not overflow.is_set():
+            target.extend(chunk)
+threads = [threading.Thread(target=collect, args=(pipe, buf), daemon=True)
+    for pipe, buf in zip((child.stdout, child.stderr), streams)]
+for thread in threads:
+    thread.start()
+try:
+    child.stdin.write(json.dumps(payload).encode())
+    child.stdin.close()
+except BrokenPipeError:
+    pass
+child.wait()
+for thread in threads:
+    thread.join()
+if overflow.is_set():
+    print("Marivo Python output limit exceeded", file=sys.stderr)
+    raise SystemExit(74)
+for stream, raw in zip((sys.stdout, sys.stderr), streams):
+    text = raw.decode("utf-8", errors="replace")
+    for value in values:
+        if value:
+            text = text.replace(value, "[REDACTED]")
+    stream.write(text)
+raise SystemExit(child.returncode if child.returncode >= 0 else 1)
+`.trim();

package/lib/datasource/rpc.js

@@ -0,0 +1,150 @@
+import { z } from 'zod';
+import { registerPluginRpc } from "../rpc.js";
+import { withDatasourceDefaults } from "./defaults.js";
+import { CREDENTIAL_CHANNEL, credentialError } from "./service.js";
+const text = z.string().min(1).max(256);
+const handle = { generation: z.string().uuid(), id: z.string().uuid(), scope: z.string().uuid() };
+const mutation = z
+    .object({
+    ...handle,
+    action: z.enum(['save', 'update', 'delete', 'delete-datasource', 'test', 'submit', 'diagnose']),
+    deleteCredentials: z.boolean().optional(),
+    version: z.string().max(65536),
+    requestId: z.string().uuid().optional(),
+    reference: text.optional(),
+    changes: z.record(text, z.string().min(1).max(65536)).optional(),
+})
+    .strict();
+export function registerCredentialRpc(connection, service, workspace, datasourceDefaults) {
+    const unregister = registerPluginRpc(connection, CREDENTIAL_CHANNEL, [
+        'overview',
+        'authoring',
+        'create-datasource',
+        'configuration',
+        'update-datasource',
+        'select-configuration',
+        'watch',
+        'start',
+        'operation',
+        'cancel-operation',
+        'cancel-request',
+    ], async (endpoint, payload, signal) => {
+        try {
+            if (Buffer.byteLength(JSON.stringify(payload) ?? '') > 1_048_576)
+                throw new Error('invalid-request');
+            let value;
+            if (endpoint === 'overview') {
+                const input = z.object({ workspaceId: text }).strict().parse(payload);
+                value = {
+                    generation: service.generation,
+                    datasources: await service.overview(input.workspaceId, () => workspace(input.workspaceId), signal),
+                };
+            }
+            else if (endpoint === 'authoring') {
+                const input = z
+                    .object({ workspaceId: text, mode: z.enum(['create', 'edit']).default('create') })
+                    .strict()
+                    .parse(payload);
+                const bridge = await workspace(input.workspaceId);
+                if (!bridge.authoring)
+                    throw new Error('authoring-unavailable');
+                const schema = await bridge.authoring(signal);
+                value = {
+                    generation: service.generation,
+                    ...(input.mode === 'edit'
+                        ? schema
+                        : withDatasourceDefaults(schema, datasourceDefaults)),
+                };
+            }
+            else if (endpoint === 'configuration') {
+                const input = z.object({ workspaceId: text, name: text }).strict().parse(payload);
+                const bridge = await workspace(input.workspaceId);
+                if (!bridge.configuration)
+                    throw new Error('authoring-unavailable');
+                value = await bridge.configuration(input.name, signal);
+            }
+            else if (endpoint === 'select-configuration') {
+                const input = z
+                    .object({ workspaceId: text, requestId: z.string().uuid(), name: text })
+                    .strict()
+                    .parse(payload);
+                value = await service.selectConfiguration(input.requestId, input.workspaceId, input.name, () => workspace(input.workspaceId), signal);
+            }
+            else if (endpoint === 'create-datasource' || endpoint === 'update-datasource') {
+                const input = z
+                    .object({
+                    workspaceId: text,
+                    generation: z.string().uuid(),
+                    fingerprint: z.string().min(1).max(256),
+                    backend: text,
+                    name: text.optional(),
+                    version: text.optional(),
+                    requestId: z.string().uuid().optional(),
+                    fields: z.record(text, z.unknown()),
+                })
+                    .strict()
+                    .parse(payload);
+                const request = input.requestId
+                    ? await service.configurationRequest(input.requestId, input.workspaceId, () => workspace(input.workspaceId), signal)
+                    : undefined;
+                const operationSignal = request ? AbortSignal.any([signal, request.signal]) : signal;
+                const resolve = async () => {
+                    if (input.requestId)
+                        await service.configurationRequest(input.requestId, input.workspaceId, () => workspace(input.workspaceId), operationSignal);
+                    return workspace(input.workspaceId);
+                };
+                if (endpoint === 'update-datasource') {
+                    const { name, version } = z.object({ name: text, version: text }).parse(input);
+                    if (request?.view.configuration.mode === 'edit' &&
+                        request.view.configuration.name !== name)
+                        throw new Error('invalid-request');
+                    value = await service.updateDatasource(input.generation, input.fingerprint, { backend: input.backend, fields: input.fields, name, version }, resolve, operationSignal);
+                }
+                else {
+                    if (input.name || input.version || request?.view.configuration.mode === 'edit')
+                        throw new Error('invalid-request');
+                    value = await service.createDatasource(input.generation, input.fingerprint, { backend: input.backend, fields: input.fields }, resolve, operationSignal);
+                }
+                if (input.requestId) {
+                    const configured = await service.selectConfiguration(input.requestId, input.workspaceId, value.name, () => workspace(input.workspaceId), operationSignal);
+                    value = { ...value, request: configured };
+                }
+            }
+            else if (endpoint === 'watch') {
+                const input = z
+                    .object({ sessionId: text, cursor: z.string().max(256).optional() })
+                    .strict()
+                    .parse(payload);
+                value = await service.waitWatch(input.sessionId, input.cursor, signal);
+            }
+            else if (endpoint === 'start') {
+                value = service.start(mutation.parse(payload));
+            }
+            else if (endpoint === 'operation') {
+                const input = z.object(handle).strict().parse(payload);
+                value = service.operation(input.generation, input.id, input.scope) ?? null;
+            }
+            else if (endpoint === 'cancel-operation') {
+                const input = z.object(handle).strict().parse(payload);
+                service.cancelOperation(input.generation, input.id, input.scope);
+                value = { cancelled: true };
+            }
+            else if (endpoint === 'cancel-request') {
+                const input = z.object({ requestId: z.string().uuid() }).strict().parse(payload);
+                service.cancelRequest(input.requestId);
+                value = { cancelled: true };
+            }
+            else
+                throw new Error('unknown-endpoint');
+            return { ok: true, value };
+        }
+        catch (error) {
+            return {
+                ok: false,
+                error: { code: 'internal', message: credentialError(error), details: {} },
+            };
+        }
+    });
+    // The profile owns this shared service; withdrawing its Web entry must not close it.
+    return unregister;
+}