@chengxianglibra/dsh-data-analysis 0.1.1 → 0.1.2

package/lib/python-execution.js

@@ -0,0 +1,232 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { constants } from 'node:fs';
+import { link, lstat, mkdir, open, realpath, unlink } from 'node:fs/promises';
+import path from 'node:path';
+import { resolveDshHome } from '@deepseek-ai/dsh-home-paths';
+const CODE_BYTES = 131072;
+const RECORD_BYTES = 262144;
+const EXECUTION_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
+const DIGEST = /^[0-9a-f]{64}$/;
+function invalid() {
+    throw new Error('Python execution source is unavailable or invalid');
+}
+function same(left, right) {
+    return left.dev === right.dev && left.ino === right.ino;
+}
+async function directory(filename) {
+    const stat = await lstat(filename);
+    if (!stat.isDirectory() || stat.isSymbolicLink() || (await realpath(filename)) !== filename)
+        invalid();
+    return stat;
+}
+async function directories(workspaceRoot, create, signal) {
+    signal?.throwIfAborted();
+    const root = await realpath(workspaceRoot);
+    if (root !== path.resolve(workspaceRoot))
+        invalid();
+    const workspaceIdentity = await directory(root);
+    const hostRoot = path.resolve(resolveDshHome());
+    const paths = [
+        hostRoot,
+        path.join(hostRoot, 'dsh-data-analysis'),
+        path.join(hostRoot, 'dsh-data-analysis', 'python-executions'),
+        path.join(hostRoot, 'dsh-data-analysis', 'python-executions', sha256(Buffer.from(root))),
+    ];
+    const identities = [];
+    const check = async (checkSignal = true) => {
+        if (checkSignal)
+            signal?.throwIfAborted();
+        if (!same(await directory(root), workspaceIdentity) ||
+            path.resolve(resolveDshHome()) !== hostRoot)
+            invalid();
+        for (const [index, expected] of identities.entries())
+            if (!same(await directory(paths[index]), expected))
+                invalid();
+        if (checkSignal)
+            signal?.throwIfAborted();
+    };
+    for (const [index, filename] of paths.entries()) {
+        await check();
+        if (create && index > 0) {
+            try {
+                await mkdir(filename, { mode: 0o700 });
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST')
+                    throw error;
+            }
+        }
+        identities.push(await directory(filename));
+    }
+    await check();
+    return { root, parent: paths[3], check };
+}
+function sourceText(text) {
+    return (typeof text === 'string' &&
+        !!text.trim() &&
+        !text.includes('\0') &&
+        !/[\uD800-\uDFFF]/u.test(text) &&
+        Buffer.byteLength(text) <= CODE_BYTES);
+}
+function timestamp(value) {
+    return (typeof value === 'string' &&
+        Number.isFinite(Date.parse(value)) &&
+        new Date(value).toISOString() === value);
+}
+function sha256(bytes) {
+    return createHash('sha256').update(bytes).digest('hex');
+}
+/** Publish a complete record once. Its final name never exposes a partial write or overwrites. */
+export async function savePythonExecution(workspaceRoot, input, signal) {
+    try {
+        signal?.throwIfAborted();
+        const { text, startedAt, finishedAt } = input;
+        if (!sourceText(text) ||
+            !timestamp(startedAt) ||
+            !timestamp(finishedAt) ||
+            finishedAt < startedAt)
+            invalid();
+        const workspace = await directories(workspaceRoot, true, signal);
+        const executionId = randomUUID();
+        const bytes = Buffer.from(JSON.stringify({
+            schemaVersion: 1,
+            executionId,
+            projectRoot: workspace.root,
+            language: 'python',
+            text,
+            startedAt,
+            finishedAt,
+            exitCode: 0,
+        }));
+        if (bytes.length > RECORD_BYTES)
+            invalid();
+        const ref = { executionId, sha256: sha256(bytes) };
+        const temporary = path.join(workspace.parent, `.${executionId}.pending`);
+        const final = path.join(workspace.parent, `${executionId}.json`);
+        await workspace.check();
+        const file = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
+        const identity = await file.stat();
+        let published = false;
+        try {
+            await file.writeFile(bytes, { signal });
+            await file.sync();
+            await workspace.check();
+            if (!same(await lstat(temporary), identity))
+                invalid();
+            // Hard-link publication is atomic and fails if a final path already exists.
+            await link(temporary, final);
+            published = true;
+            await workspace.check();
+            if (!same(await lstat(final), identity))
+                invalid();
+            const verified = await readPythonExecution(workspace.root, ref, signal);
+            if (verified.text !== text)
+                invalid();
+            await workspace.check();
+            return ref;
+        }
+        catch (error) {
+            if (published) {
+                try {
+                    await workspace.check(false);
+                    if (same(await lstat(final), identity))
+                        await unlink(final);
+                }
+                catch {
+                    /* Do not remove a replaced path or hide the original error. */
+                }
+            }
+            throw error;
+        }
+        finally {
+            await file.close();
+            try {
+                await workspace.check(false);
+                if (same(await lstat(temporary), identity))
+                    await unlink(temporary);
+            }
+            catch {
+                /* Cleanup cannot override successful execution or follow a replaced parent. */
+            }
+        }
+    }
+    catch {
+        signal?.throwIfAborted();
+        invalid();
+    }
+}
+/** Resolve Host-stored execution identities for the bound Workspace. Never executes code. */
+export async function readPythonExecution(workspaceRoot, ref, signal) {
+    try {
+        signal?.throwIfAborted();
+        if (!ref || !EXECUTION_ID.test(ref.executionId) || !DIGEST.test(ref.sha256))
+            invalid();
+        const { executionId, sha256: expectedDigest } = ref;
+        const workspace = await directories(workspaceRoot, false, signal);
+        const filename = path.join(workspace.parent, `${executionId}.json`);
+        const expected = await lstat(filename);
+        if (!expected.isFile() || expected.isSymbolicLink() || (await realpath(filename)) !== filename)
+            invalid();
+        const file = await open(filename, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+        try {
+            const before = await file.stat();
+            if (!before.isFile() || !same(before, expected) || before.size > RECORD_BYTES)
+                invalid();
+            const bytes = Buffer.alloc(RECORD_BYTES + 1);
+            let length = 0;
+            while (length < bytes.length) {
+                signal?.throwIfAborted();
+                const { bytesRead } = await file.read(bytes, length, bytes.length - length, null);
+                if (!bytesRead)
+                    break;
+                length += bytesRead;
+            }
+            const after = await file.stat();
+            const current = await lstat(filename);
+            await workspace.check();
+            if (length > RECORD_BYTES ||
+                length !== after.size ||
+                !same(current, after) ||
+                !current.isFile() ||
+                before.size !== after.size ||
+                before.mtimeMs !== after.mtimeMs ||
+                before.ctimeMs !== after.ctimeMs ||
+                (await realpath(filename)) !== filename)
+                invalid();
+            const raw = bytes.subarray(0, length);
+            if (sha256(raw) !== expectedDigest)
+                invalid();
+            const record = JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(raw));
+            if (!record ||
+                typeof record !== 'object' ||
+                Array.isArray(record) ||
+                Object.keys(record).sort().join(',') !==
+                    'executionId,exitCode,finishedAt,language,projectRoot,schemaVersion,startedAt,text' ||
+                record.schemaVersion !== 1 ||
+                record.executionId !== executionId ||
+                record.projectRoot !== workspace.root ||
+                record.language !== 'python' ||
+                record.exitCode !== 0 ||
+                !sourceText(record.text) ||
+                !timestamp(record.startedAt) ||
+                !timestamp(record.finishedAt) ||
+                record.finishedAt < record.startedAt)
+                invalid();
+            signal?.throwIfAborted();
+            return {
+                language: 'python',
+                text: record.text,
+                provenance: 'execution',
+                executionId,
+                sha256: expectedDigest,
+            };
+        }
+        finally {
+            await file.close();
+        }
+    }
+    catch {
+        signal?.throwIfAborted();
+        invalid();
+    }
+}

package/lib/semantic-browser/contracts.js

@@ -0,0 +1,80 @@
+import { boundedText, closed, parseRef, refKey, } from "../semantic-reference/contracts.js";
+import { parseComputation } from "./definition.js";
+export const BROWSER_ENDPOINT = 'semantic-browser/catalog';
+export const CATALOG_MAX_BYTES = 32 * 1024 * 1024;
+function array(value, max) {
+    if (!Array.isArray(value) || value.length > max)
+        throw new Error('invalid-catalog');
+    return value;
+}
+function text(value) {
+    return boundedText(value, CATALOG_MAX_BYTES, true);
+}
+function nullable(value) {
+    return value === null ? null : text(value);
+}
+export function parseCatalogProjection(value) {
+    const r = closed(value, ['fingerprint', 'kinds', 'objects']);
+    const kinds = array(r.kinds, 128).map((item) => boundedText(item, 128));
+    const objects = array(r.objects, 100_000).map((item) => {
+        const row = closed(item, [
+            'ref',
+            'computation',
+            'name',
+            'domain',
+            'definition',
+            'guardrails',
+            'source',
+            'fields',
+            'relations',
+        ]);
+        const ref = parseRef(row.ref);
+        if (!kinds.includes(ref.kind))
+            throw new Error('invalid-catalog');
+        const source = closed(row.source, ['file', 'line', 'symbol']);
+        if (!Number.isSafeInteger(source.line) || source.line < 0)
+            throw new Error('invalid-catalog');
+        return {
+            ref,
+            computation: parseComputation(row.computation, ref, boundedText(r.fingerprint, 256)),
+            name: text(row.name),
+            domain: nullable(row.domain),
+            definition: nullable(row.definition),
+            guardrails: array(row.guardrails, 100_000).map(text),
+            source: { file: text(source.file), line: source.line, symbol: text(source.symbol) },
+            fields: array(row.fields, 128).map((value) => {
+                const field = closed(value, ['name', 'value']);
+                return { name: boundedText(field.name, 128), value: text(field.value) };
+            }),
+            relations: array(row.relations, 100_000).map((value) => {
+                const relation = closed(value, ['field', 'ref']);
+                return { field: boundedText(relation.field, 128), ref: parseRef(relation.ref) };
+            }),
+        };
+    });
+    if (new Set(kinds).size !== kinds.length ||
+        new Set(objects.map((x) => refKey(x.ref))).size !== objects.length)
+        throw new Error('invalid-catalog');
+    return { fingerprint: boundedText(r.fingerprint, 256), kinds, objects };
+}
+export function parseCatalogSnapshot(value) {
+    const r = closed(value, [
+        'workspaceId',
+        'projectRoot',
+        'environmentFingerprint',
+        'loadedAt',
+        'fingerprint',
+        'kinds',
+        'objects',
+    ]);
+    const loadedAt = boundedText(r.loadedAt, 64);
+    if (!Number.isFinite(Date.parse(loadedAt)))
+        throw new Error('invalid-catalog');
+    return {
+        ...parseCatalogProjection({ fingerprint: r.fingerprint, kinds: r.kinds, objects: r.objects }),
+        workspaceId: boundedText(r.workspaceId, 256),
+        projectRoot: boundedText(r.projectRoot, 8192),
+        environmentFingerprint: boundedText(r.environmentFingerprint, 256),
+        loadedAt,
+    };
+}

package/lib/semantic-browser/definition.js

@@ -0,0 +1,64 @@
+import { closed, parseRef, refKey } from "../semantic-reference/contracts.js";
+// Transport allowlist for the public definition payload, not a semantic validator.
+const keys = new Set('kind operation q target target_kind filter dimension operator values value weight numerator denominator terms sign metric base over selection ref resolution anchor count unit grain calendar level status expression reason entity name value_type operand data_type left right condition when_true when_false declared override effective source fold schema path display language form text bindings alias redacted_literals'.split(' '));
+function value(input, depth = 0, budget = { left: 10000 }) {
+    if (--budget.left < 0 || depth > 64)
+        throw new Error('definition-too-large');
+    if (input === null || typeof input === 'boolean')
+        return input;
+    if (typeof input === 'string' && input.length <= 1024 * 1024)
+        return input;
+    if (typeof input === 'number' && Number.isFinite(input))
+        return input;
+    if (Array.isArray(input))
+        return input.map((x) => value(x, depth + 1, budget));
+    if (typeof input !== 'object' || input === null)
+        throw new Error('invalid-definition');
+    const result = {};
+    for (const [key, item] of Object.entries(input)) {
+        if (!keys.has(key))
+            throw new Error('invalid-definition-field');
+        result[key] = value(item, depth + 1, budget);
+    }
+    if (result.schema === 'marivo.semantic_ref/v1')
+        parseRef(result);
+    return result;
+}
+export function parseComputation(input, ref, fingerprint) {
+    if (input === null)
+        return null;
+    const r = closed(input, [
+        'schema',
+        'ref',
+        'catalog_definition_fingerprint',
+        'node',
+        'temporal',
+        'source_location',
+    ]);
+    if (r.schema !== 'marivo.semantic_definition/v1' ||
+        refKey(parseRef(r.ref)) !== refKey(ref) ||
+        r.catalog_definition_fingerprint !== fingerprint)
+        throw new Error('definition-identity-mismatch');
+    const source = closed(r.source_location, ['file', 'line']);
+    if (typeof source.file !== 'string' ||
+        !Number.isSafeInteger(source.line) ||
+        source.line < 0)
+        throw new Error('invalid-definition-source');
+    const node = value(r.node), temporal = value(r.temporal);
+    if (!node ||
+        typeof node !== 'object' ||
+        Array.isArray(node) ||
+        typeof node.kind !== 'string' ||
+        !temporal ||
+        typeof temporal !== 'object' ||
+        Array.isArray(temporal))
+        throw new Error('invalid-definition');
+    return {
+        schema: r.schema,
+        ref,
+        catalog_definition_fingerprint: fingerprint,
+        node,
+        temporal,
+        source_location: { file: source.file, line: source.line },
+    };
+}

package/lib/semantic-browser/program.js

@@ -0,0 +1,119 @@
+/** Explicit public Details projection. No object dump, source-code read or data operation. */
+export const BROWSER_CATALOG_PROGRAM = String.raw `
+import contextlib, json, sys
+from datetime import date, datetime
+from enum import Enum
+from collections.abc import Mapping
+
+class DiscardOutput:
+    def write(self, text):
+        return len(text)
+    def flush(self):
+        pass
+
+with contextlib.redirect_stdout(DiscardOutput()), contextlib.redirect_stderr(DiscardOutput()):
+    import marivo.semantic as ms
+    from marivo.refs import Ref, RefPayloadV1, SemanticKind
+
+    FIELDS = {
+        "domain": "owner default",
+        "datasource": "backend_type",
+        "entity": "datasource primary_key",
+        "dimension": "entity",
+        "measure": "entity unit additivity",
+        "time_dimension": "entity parse_kind data_type granularity format timezone is_default sample_interval",
+        "metric": "metric_type entities root_entity aggregation aggregation_target aggregation_target_kind measure unit additivity fold status_time_dimension fanout_policy filter effective_entities candidate_dimensions candidate_time_dimensions measure_lineage weighted_mean_value weighted_mean_weight composition components linear_terms required_relationships parity_status",
+        "relationship": "from_entity to_entity from_keys to_keys",
+        "event": "source_entity identity occurred_at participants predicate_kind definition_fingerprint",
+        "state_model": "subject states inceptions transitions definition_fingerprint",
+        "period_calendar": "date boundary_timezone coverage correspondences snapshot_status snapshot_digest",
+        "temporal_set": "boundary_timezone coverage occurrence_id start end category occurrence_count snapshot_status",
+        "work_schedule": "boundary_timezone coverage date is_working snapshot_status",
+    }
+
+    def ref_payload(value):
+        return RefPayloadV1.from_ref(value).to_dict()
+
+    # Traversal is restricted to values of the explicitly selected public fields.
+    def display(value, field, relations):
+        if isinstance(value, Ref):
+            relations.append({"field": field, "ref": ref_payload(value)})
+            return value.key
+        if isinstance(value, Enum):
+            return str(value.value)
+        if isinstance(value, (date, datetime)):
+            return value.isoformat()
+        if value is None:
+            return "—"
+        if isinstance(value, bool):
+            return "true" if value else "false"
+        if isinstance(value, (str, int, float)):
+            return str(value)
+        if isinstance(value, (tuple, list)):
+            return "\n".join(display(v, field, relations) for v in value)
+        if isinstance(value, Mapping):
+            return "\n".join(str(k) + ": " + display(v, field, relations) for k, v in value.items())
+        # Public temporal value, not a generic repr of a Python object.
+        if field == "sample_interval":
+            return value.to_token()
+        raise TypeError("unsupported display field: " + field)
+
+    def project(entry):
+        d = entry.details()
+        relations, fields = [], []
+        def add(name, value):
+            if value is not None:
+                fields.append({"name": name, "value": display(value, name, relations)})
+        for name in FIELDS.get(entry.kind.value, "").split():
+            if hasattr(d, name):
+                add(name, getattr(d, name))
+        for name in ("parents", "children", "dependents"):
+            for ref in getattr(d, name):
+                relations.append({"field": name, "ref": ref_payload(ref)})
+        if entry.kind.value == "entity":
+            source = d.source
+            add("source_kind", source.kind)
+            # Remote file URLs may contain credentials. Only table identity is projected.
+            if source.kind == "table":
+                add("source_table", source.table)
+                add("source_database", source.database)
+            version = d.versioning
+            if version is not None:
+                for name in ("kind", "partition_field", "grain", "timezone", "format", "valid_from", "valid_to", "interval", "open_end"):
+                    if hasattr(version, name):
+                        add("version_" + name, getattr(version, name))
+        if entry.kind.value == "period_calendar":
+            for level in d.levels:
+                add("level", level.name)
+                for name in ("key_ref", "period_count", "direct_finer_levels", "direct_coarser_levels", "rollup_targets"):
+                    add("level_" + name, getattr(level, name))
+        computation = d.definition.to_dict() if entry.kind.value in ("metric", "measure", "dimension", "time_dimension") else None
+        def definition_refs(value, role="definition"):
+            if isinstance(value, dict):
+                if value.get("schema") == "marivo.semantic_ref/v1":
+                    relations.append({"field": role, "ref": value})
+                else:
+                    for key, item in value.items():
+                        definition_refs(item, role + "." + key)
+            elif isinstance(value, list):
+                for item in value:
+                    definition_refs(item, role)
+        if computation is not None:
+            definition_refs(computation["node"])
+            definition_refs(computation["temporal"], "temporal")
+        unique = {(r["field"], r["ref"]["kind"], r["ref"]["path"]): r for r in relations}
+        return {
+            "computation": computation, "ref": ref_payload(entry.ref), "name": entry.name, "domain": d.domain,
+            "definition": d.context.business_definition, "guardrails": list(d.context.guardrails),
+            "source": {"file": d.source_location.file, "line": d.source_location.line, "symbol": d.python_symbol},
+            "fields": fields, "relations": list(unique.values()),
+        }
+
+    catalog = ms.load(workspace_dir=sys.argv[1])
+    result = {
+        "fingerprint": catalog.definition_fingerprint,
+        "kinds": [kind.value for kind in SemanticKind],
+        "objects": [project(entry) for kind in SemanticKind for entry in catalog.items(kind).items],
+    }
+print(json.dumps(result, ensure_ascii=True))
+`.trim();

package/lib/semantic-browser/service.js

@@ -0,0 +1,72 @@
+import { realpath } from 'node:fs/promises';
+import { boundedText, closed } from "../semantic-reference/contracts.js";
+import { abortable } from "../semantic-reference/rpc.js";
+import { CATALOG_MAX_BYTES, parseCatalogProjection } from "./contracts.js";
+import { BROWSER_CATALOG_PROGRAM } from "./program.js";
+/** Workspace-addressed read service; intentionally has no Agent or credential dependency. */
+export class SemanticBrowserService {
+    #host;
+    #lifetime = new AbortController();
+    constructor(host) {
+        this.#host = host;
+    }
+    #assertOwner(id, workspacePath, root) {
+        const current = this.#host.getWorkspace(id);
+        if (!current || current.path !== workspacePath || this.#host.projectRoot(current) !== root)
+            throw new Error('workspace-changed');
+    }
+    async read(payload, caller) {
+        const signal = AbortSignal.any([caller, this.#lifetime.signal]);
+        signal.throwIfAborted();
+        const request = closed(payload, ['workspaceId']);
+        const id = boundedText(request.workspaceId, 256);
+        const workspace = this.#host.getWorkspace(id);
+        if (!workspace)
+            throw new Error('workspace-unavailable');
+        const root = this.#host.projectRoot(workspace);
+        const runner = await abortable(this.#host.resolve(root), signal);
+        this.#assertOwner(id, workspace.path, root);
+        if (runner.status !== 'ready')
+            throw new Error('environment-failed');
+        const result = await runner.runChecked({
+            program: BROWSER_CATALOG_PROGRAM,
+            args: [runner.binding.projectRoot],
+            signal,
+            environmentOverlay: { MARIVO_TELEMETRY: 'off', PYTHONDONTWRITEBYTECODE: '1' },
+            limits: { timeoutMs: 30_000, stdoutMaxBytes: CATALOG_MAX_BYTES, stderrMaxBytes: 8192 },
+        });
+        signal.throwIfAborted();
+        if ((await realpath(root)) !== runner.binding.projectRoot)
+            throw new Error('workspace-changed');
+        this.#assertOwner(id, workspace.path, root);
+        signal.throwIfAborted();
+        if (result.exitCode !== 0)
+            throw new Error('catalog-load-failed');
+        if (result.stdout.byteLength > CATALOG_MAX_BYTES)
+            throw new Error('catalog-too-large');
+        const projection = parseCatalogProjection(JSON.parse(result.stdout.toString('utf8')));
+        return {
+            ...projection,
+            workspaceId: id,
+            projectRoot: runner.binding.projectRoot,
+            environmentFingerprint: runner.binding.fingerprint,
+            loadedAt: new Date().toISOString(),
+        };
+    }
+    dispose() {
+        this.#lifetime.abort();
+    }
+}
+/** Fixed diagnostics only: Python errors may contain model code or datasource secrets. */
+export function browserFailure(error) {
+    const code = error instanceof Error && 'code' in error ? error.code : undefined;
+    if (code === 'subprocess-timeout')
+        return '语义层加载超时，请检查模型后重试。';
+    if (code === 'subprocess-output-limit' ||
+        (error instanceof Error && error.message === 'catalog-too-large'))
+        return '语义层超过读取大小上限，未加载不完整目录。';
+    if (error instanceof Error &&
+        ['workspace-unavailable', 'workspace-changed'].includes(error.message))
+        return 'Workspace 已不可用或绑定发生变化，请重新选择。';
+    return '无法加载语义层，请检查项目模型与 Marivo Runtime 后重试。';
+}