@chengxianglibra/dsh-data-analysis 0.1.1 → 0.2.0

package/lib/report-publishing/config.js

@@ -0,0 +1,69 @@
+import { isCredentialRefName } from '@deepseek-ai/dsh-credentials';
+import { z } from 'zod';
+const reference = z.string().refine(isCredentialRefName);
+const url = z
+    .string()
+    .url()
+    .refine((value) => {
+    let parsed;
+    try {
+        parsed = new URL(value);
+    }
+    catch {
+        return false;
+    }
+    return (['http:', 'https:'].includes(parsed.protocol) &&
+        !parsed.username &&
+        !parsed.password &&
+        !parsed.search &&
+        !parsed.hash);
+});
+export const publishingConfigSchema = z
+    .object({
+    enabled: z.boolean().default(false),
+    storage: z
+        .object({
+        name: z.string().min(1).max(100),
+        protocol: z.literal('s3').default('s3'),
+        endpoint: url,
+        region: z.string().min(1),
+        bucket: z
+            .string()
+            .min(1)
+            .max(255)
+            .regex(/^[a-zA-Z0-9][a-zA-Z0-9.-]*$/),
+        forcePathStyle: z.boolean().default(false),
+        accessKeyIdRef: reference,
+        secretAccessKeyRef: reference,
+        sessionTokenRef: reference.optional(),
+    })
+        .strict(),
+    pathPrefix: z
+        .string()
+        .default('reports')
+        .refine((value) => value === '' ||
+        value.split('/').every((part) => /^[a-zA-Z0-9_-][a-zA-Z0-9._-]*$/.test(part))),
+    publicBaseUrl: url,
+})
+    .strict()
+    .superRefine((config, ctx) => {
+    const refs = [
+        config.storage.accessKeyIdRef,
+        config.storage.secretAccessKeyRef,
+        config.storage.sessionTokenRef,
+    ].filter(Boolean);
+    if (new Set(refs).size !== refs.length)
+        ctx.addIssue({ code: 'custom', message: 'distinct-credential-references-required' });
+});
+export function resolvePublishingConfig(value) {
+    if (value === undefined)
+        return undefined;
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error('report-publishing-config-invalid');
+    if (value.enabled === false || value.enabled === undefined)
+        return undefined;
+    const result = publishingConfigSchema.safeParse(value);
+    if (!result.success)
+        throw new Error('report-publishing-config-invalid');
+    return result.data;
+}

package/lib/report-publishing/service.js

@@ -0,0 +1,180 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
+import { credentialRef } from '@deepseek-ai/dsh-credentials';
+import { PRESENTATION_BUDGETS } from "../presentation/contracts/index.js";
+export function publishingPathName(value, fallback) {
+    const name = Array.from(value.normalize('NFKC').replace(/[^\p{L}\p{N}_-]+/gu, '-'))
+        .slice(0, 48)
+        .join('')
+        .replace(/^-+|-+$/g, '');
+    return name || fallback;
+}
+export async function uploadReport(input) {
+    const { config, credentials, key, bytes, signal } = input;
+    const client = new S3Client({
+        endpoint: config.storage.endpoint,
+        region: config.storage.region,
+        forcePathStyle: config.storage.forcePathStyle,
+        credentials,
+        maxAttempts: 3,
+        requestChecksumCalculation: 'WHEN_REQUIRED',
+    });
+    try {
+        await client.send(new PutObjectCommand({
+            Bucket: config.storage.bucket,
+            Key: key,
+            Body: bytes,
+            ContentType: 'text/html; charset=utf-8',
+            ContentDisposition: 'inline',
+            CacheControl: 'public, max-age=31536000, immutable',
+        }), { abortSignal: signal });
+    }
+    finally {
+        client.destroy();
+    }
+}
+export class ReportPublishingService {
+    configId = randomUUID();
+    config;
+    store;
+    files;
+    upload;
+    workspaceName;
+    constructor(config, store, files, upload = uploadReport, workspaceName = () => 'workspace') {
+        this.workspaceName = workspaceName;
+        this.config = config
+            ? Object.freeze({ ...config, storage: Object.freeze({ ...config.storage }) })
+            : undefined;
+        this.store = store;
+        this.files = files;
+        this.upload = upload;
+    }
+    assertConfig(configId) {
+        if (configId !== this.configId)
+            throw new Error('report-publishing-config-changed');
+    }
+    references() {
+        const storage = this.config?.storage;
+        if (!storage)
+            return [];
+        return [
+            { field: 'accessKeyId', reference: storage.accessKeyIdRef, required: true },
+            { field: 'secretAccessKey', reference: storage.secretAccessKeyRef, required: true },
+            ...(storage.sessionTokenRef
+                ? [{ field: 'sessionToken', reference: storage.sessionTokenRef, required: true }]
+                : []),
+        ];
+    }
+    async describe() {
+        if (!this.config)
+            return { enabled: false, fields: [] };
+        const fields = await Promise.all(this.references().map(async (field) => {
+            const info = await this.store.describe(credentialRef(field.reference));
+            return {
+                ...field,
+                configured: info.configured,
+                writable: info.writable,
+                ...(info.source ? { source: info.source } : {}),
+            };
+        }));
+        return {
+            enabled: true,
+            configId: this.configId,
+            name: this.config.storage.name,
+            bucket: this.config.storage.bucket,
+            fields,
+        };
+    }
+    async change(field, value, signal) {
+        const item = this.references().find((item) => item.field === field);
+        if (!item)
+            throw new Error('report-publishing-disabled-or-field-unavailable');
+        signal.throwIfAborted();
+        if (value === undefined)
+            await this.store.unset(credentialRef(item.reference));
+        else
+            await this.store.set(credentialRef(item.reference), value);
+    }
+    async publish(scope, reportId, buildId, signal, viewHtml) {
+        const config = this.config;
+        if (!config)
+            throw new Error('report-publishing-disabled');
+        const history = await this.files.catalog('reports/history', { ...scope, reportId }, signal);
+        if (!('versions' in history))
+            throw new Error('report-publishing-build-unavailable');
+        const receipt = history.versions.find((entry) => entry.receipt.buildId === buildId)?.receipt;
+        if (!receipt)
+            throw new Error('report-publishing-build-unavailable');
+        const credentials = {};
+        for (const field of this.references()) {
+            signal.throwIfAborted();
+            const hit = await this.store.resolve(credentialRef(field.reference));
+            if (!hit?.value)
+                throw new Error('report-publishing-credentials-missing');
+            credentials[field.field] = hit.value;
+        }
+        const asset = await this.files.read({ ...scope, receipt, asset: viewHtml === undefined ? 'index.html' : 'presentation.json' }, signal);
+        let bytes;
+        if (viewHtml === undefined)
+            bytes = Buffer.from(asset.bodyBase64, 'base64');
+        else {
+            if (Buffer.byteLength(viewHtml) > PRESENTATION_BUDGETS.htmlBytes ||
+                !viewHtml.startsWith('<!doctype html><html lang="zh-CN"><head>'))
+                throw new Error('report-publishing-view-invalid');
+            // Browser-authored snapshot: force a restrictive policy before any client-supplied markup.
+            bytes = Buffer.from(viewHtml.replace('<head>', `<head><meta http-equiv="Content-Security-Policy" content="default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'">`));
+            if (bytes.length > PRESENTATION_BUDGETS.htmlBytes)
+                throw new Error('report-publishing-view-invalid');
+        }
+        const sha256 = createHash('sha256').update(bytes).digest('hex');
+        const identity = createHash('sha256')
+            .update(JSON.stringify([
+            receipt.workspaceId,
+            reportId,
+            buildId,
+            viewHtml === undefined ? 'report' : 'view',
+            sha256,
+        ]))
+            .digest('hex')
+            .slice(0, 32);
+        const key = [
+            config.pathPrefix,
+            publishingPathName(this.workspaceName(receipt.workspaceId), 'workspace'),
+            publishingPathName(receipt.title, 'report'),
+            identity,
+            'index.html',
+        ]
+            .filter(Boolean)
+            .join('/');
+        signal.throwIfAborted();
+        try {
+            await this.upload({
+                config,
+                credentials: credentials,
+                key,
+                bytes,
+                signal,
+            });
+        }
+        catch {
+            throw new Error('report-publishing-upload-unconfirmed');
+        }
+        finally {
+            for (const key of Object.keys(credentials))
+                delete credentials[key];
+        }
+        return {
+            storage: config.storage.name,
+            bucket: config.storage.bucket,
+            key,
+            url: `${config.publicBaseUrl.replace(/\/+$/, '')}/${key.split('/').map(encodeURIComponent).join('/')}`,
+            workspaceId: receipt.workspaceId,
+            reportId,
+            buildId,
+            sha256,
+            bytes: bytes.length,
+            kind: viewHtml === undefined ? 'report' : 'view',
+            publishedAt: new Date().toISOString(),
+        };
+    }
+}

package/lib/rpc.js

@@ -0,0 +1,49 @@
+/** Dedicated plugin RPC envelopes over Harness-owned authenticated exact Fetch routes. */
+import { clientRequestSchema, } from '@deepseek-ai/dsh-client-connection';
+import { finishCleanup, PendingTasks } from "./lifecycle.js";
+export function registerPluginRpc(connection, channel, endpoints, handler) {
+    const lifetime = new AbortController();
+    const tasks = new PendingTasks();
+    let closing;
+    const disposers = [];
+    const close = () => {
+        if (closing)
+            return closing;
+        lifetime.abort();
+        closing = finishCleanup([...disposers.reverse(), () => tasks.drain()]);
+        return closing;
+    };
+    try {
+        for (const endpoint of endpoints) {
+            const method = `${channel.slice(1)}/${endpoint}`;
+            disposers.push(connection.fetch.register({
+                path: `/api/${method}`,
+                methods: ['POST'],
+                requestBody: 'buffered',
+                fetch(request) {
+                    if (lifetime.signal.aborted)
+                        return Promise.resolve(new Response('Plugin disposed', { status: 503 }));
+                    return tasks.track((async () => {
+                        if (request.headers.get('content-type')?.split(';')[0]?.trim().toLowerCase() !==
+                            'application/json')
+                            return new Response('Expected application/json', { status: 415 });
+                        const body = await request.json().catch(() => undefined);
+                        const parsed = clientRequestSchema.safeParse(body);
+                        if (!parsed.success || parsed.data.method !== method)
+                            return new Response('Invalid plugin request', { status: 400 });
+                        // A request admitted before withdrawal may still be decoding its body.
+                        if (lifetime.signal.aborted)
+                            return new Response('Plugin disposed', { status: 503 });
+                        const result = await handler(endpoint, parsed.data.payload, AbortSignal.any([request.signal, lifetime.signal]));
+                        return Response.json({ type: 'server-response', rpcId: parsed.data.rpcId, result });
+                    })());
+                },
+            }));
+        }
+    }
+    catch (error) {
+        void close();
+        throw error;
+    }
+    return close;
+}

package/lib/semantic-browser/contracts.js

@@ -0,0 +1,97 @@
+import { boundedText, closed, parseRef, refKey, } from "../semantic-reference/contracts.js";
+import { parseComputation } from "./definition.js";
+export const BROWSER_ENDPOINT = 'semantic-browser/catalog';
+export const CATALOG_MAX_BYTES = 32 * 1024 * 1024;
+export const CATALOG_ERROR_SCHEMA = 'marivo.semantic_catalog_error/v1';
+const CATALOG_ERROR_MAX_CHARS = 16 * 1024;
+function array(value, max) {
+    if (!Array.isArray(value) || value.length > max)
+        throw new Error('invalid-catalog');
+    return value;
+}
+function text(value) {
+    return boundedText(value, CATALOG_MAX_BYTES, true);
+}
+function nullable(value) {
+    return value === null ? null : text(value);
+}
+export function parseCatalogProjection(value) {
+    const r = closed(value, ['fingerprint', 'kinds', 'objects']);
+    const kinds = array(r.kinds, 128).map((item) => boundedText(item, 128));
+    const objects = array(r.objects, 100_000).map((item) => {
+        const row = closed(item, [
+            'ref',
+            'computation',
+            'name',
+            'domain',
+            'definition',
+            'guardrails',
+            'source',
+            'fields',
+            'relations',
+        ]);
+        const ref = parseRef(row.ref);
+        if (!kinds.includes(ref.kind))
+            throw new Error('invalid-catalog');
+        const source = closed(row.source, ['file', 'line', 'symbol']);
+        if (!Number.isSafeInteger(source.line) || source.line < 0)
+            throw new Error('invalid-catalog');
+        return {
+            ref,
+            computation: parseComputation(row.computation, ref, boundedText(r.fingerprint, 256)),
+            name: text(row.name),
+            domain: nullable(row.domain),
+            definition: nullable(row.definition),
+            guardrails: array(row.guardrails, 100_000).map(text),
+            source: { file: text(source.file), line: source.line, symbol: text(source.symbol) },
+            fields: array(row.fields, 128).map((value) => {
+                const field = closed(value, ['name', 'value']);
+                return { name: boundedText(field.name, 128), value: text(field.value) };
+            }),
+            relations: array(row.relations, 100_000).map((value) => {
+                const relation = closed(value, ['field', 'ref']);
+                return { field: boundedText(relation.field, 128), ref: parseRef(relation.ref) };
+            }),
+        };
+    });
+    if (new Set(kinds).size !== kinds.length ||
+        new Set(objects.map((x) => refKey(x.ref))).size !== objects.length)
+        throw new Error('invalid-catalog');
+    return { fingerprint: boundedText(r.fingerprint, 256), kinds, objects };
+}
+export function parseCatalogFailure(value) {
+    try {
+        const r = closed(value, ['schema', 'kind', 'message']);
+        if (r.schema !== CATALOG_ERROR_SCHEMA || r.kind !== 'semantic')
+            return undefined;
+        return {
+            schema: CATALOG_ERROR_SCHEMA,
+            kind: 'semantic',
+            message: boundedText(r.message, CATALOG_ERROR_MAX_CHARS),
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+export function parseCatalogSnapshot(value) {
+    const r = closed(value, [
+        'workspaceId',
+        'projectRoot',
+        'environmentFingerprint',
+        'loadedAt',
+        'fingerprint',
+        'kinds',
+        'objects',
+    ]);
+    const loadedAt = boundedText(r.loadedAt, 64);
+    if (!Number.isFinite(Date.parse(loadedAt)))
+        throw new Error('invalid-catalog');
+    return {
+        ...parseCatalogProjection({ fingerprint: r.fingerprint, kinds: r.kinds, objects: r.objects }),
+        workspaceId: boundedText(r.workspaceId, 256),
+        projectRoot: boundedText(r.projectRoot, 8192),
+        environmentFingerprint: boundedText(r.environmentFingerprint, 256),
+        loadedAt,
+    };
+}

package/lib/semantic-browser/definition.js

@@ -0,0 +1,64 @@
+import { closed, parseRef, refKey } from "../semantic-reference/contracts.js";
+// Transport allowlist for the public definition payload, not a semantic validator.
+const keys = new Set('kind operation q target target_kind filter dimension operator values value weight numerator denominator terms sign metric base over selection ref resolution anchor count unit grain calendar level status expression reason entity name value_type operand data_type left right condition when_true when_false declared override effective source fold schema path display language form text bindings alias redacted_literals'.split(' '));
+function value(input, depth = 0, budget = { left: 10000 }) {
+    if (--budget.left < 0 || depth > 64)
+        throw new Error('definition-too-large');
+    if (input === null || typeof input === 'boolean')
+        return input;
+    if (typeof input === 'string' && input.length <= 1024 * 1024)
+        return input;
+    if (typeof input === 'number' && Number.isFinite(input))
+        return input;
+    if (Array.isArray(input))
+        return input.map((x) => value(x, depth + 1, budget));
+    if (typeof input !== 'object' || input === null)
+        throw new Error('invalid-definition');
+    const result = {};
+    for (const [key, item] of Object.entries(input)) {
+        if (!keys.has(key))
+            throw new Error('invalid-definition-field');
+        result[key] = value(item, depth + 1, budget);
+    }
+    if (result.schema === 'marivo.semantic_ref/v1')
+        parseRef(result);
+    return result;
+}
+export function parseComputation(input, ref, fingerprint) {
+    if (input === null)
+        return null;
+    const r = closed(input, [
+        'schema',
+        'ref',
+        'catalog_definition_fingerprint',
+        'node',
+        'temporal',
+        'source_location',
+    ]);
+    if (r.schema !== 'marivo.semantic_definition/v1' ||
+        refKey(parseRef(r.ref)) !== refKey(ref) ||
+        r.catalog_definition_fingerprint !== fingerprint)
+        throw new Error('definition-identity-mismatch');
+    const source = closed(r.source_location, ['file', 'line']);
+    if (typeof source.file !== 'string' ||
+        !Number.isSafeInteger(source.line) ||
+        source.line < 0)
+        throw new Error('invalid-definition-source');
+    const node = value(r.node), temporal = value(r.temporal);
+    if (!node ||
+        typeof node !== 'object' ||
+        Array.isArray(node) ||
+        typeof node.kind !== 'string' ||
+        !temporal ||
+        typeof temporal !== 'object' ||
+        Array.isArray(temporal))
+        throw new Error('invalid-definition');
+    return {
+        schema: r.schema,
+        ref,
+        catalog_definition_fingerprint: fingerprint,
+        node,
+        temporal,
+        source_location: { file: source.file, line: source.line },
+    };
+}

package/lib/semantic-browser/program.js

@@ -0,0 +1,129 @@
+/** Explicit public Details projection. No object dump, source-code read or data operation. */
+export const BROWSER_CATALOG_PROGRAM = String.raw `
+import contextlib, json, sys
+from datetime import date, datetime
+from enum import Enum
+from collections.abc import Mapping
+
+class DiscardOutput:
+    def write(self, text):
+        return len(text)
+    def flush(self):
+        pass
+
+with contextlib.redirect_stdout(DiscardOutput()), contextlib.redirect_stderr(DiscardOutput()):
+    import marivo.semantic as ms
+    from marivo.refs import Ref, RefPayloadV1, SemanticKind
+    from marivo.semantic.errors import SemanticError, SemanticLoadFailed
+
+    FIELDS = {
+        "domain": "owner default",
+        "datasource": "backend_type",
+        "entity": "datasource primary_key",
+        "dimension": "entity",
+        "measure": "entity unit additivity",
+        "time_dimension": "entity parse_kind data_type granularity format timezone is_default sample_interval",
+        "metric": "metric_type entities root_entity aggregation aggregation_target aggregation_target_kind measure unit additivity fold status_time_dimension fanout_policy filter effective_entities candidate_dimensions candidate_time_dimensions measure_lineage weighted_mean_value weighted_mean_weight composition components linear_terms required_relationships parity_status",
+        "relationship": "from_entity to_entity from_keys to_keys",
+        "event": "source_entity identity occurred_at participants predicate_kind definition_fingerprint",
+        "state_model": "subject states inceptions transitions definition_fingerprint",
+        "period_calendar": "date boundary_timezone coverage correspondences snapshot_status snapshot_digest",
+        "temporal_set": "boundary_timezone coverage occurrence_id start end category occurrence_count snapshot_status",
+        "work_schedule": "boundary_timezone coverage date is_working snapshot_status",
+    }
+
+    def ref_payload(value):
+        return RefPayloadV1.from_ref(value).to_dict()
+
+    # Traversal is restricted to values of the explicitly selected public fields.
+    def display(value, field, relations):
+        if isinstance(value, Ref):
+            relations.append({"field": field, "ref": ref_payload(value)})
+            return value.key
+        if isinstance(value, Enum):
+            return str(value.value)
+        if isinstance(value, (date, datetime)):
+            return value.isoformat()
+        if value is None:
+            return "—"
+        if isinstance(value, bool):
+            return "true" if value else "false"
+        if isinstance(value, (str, int, float)):
+            return str(value)
+        if isinstance(value, (tuple, list)):
+            return "\n".join(display(v, field, relations) for v in value)
+        if isinstance(value, Mapping):
+            return "\n".join(str(k) + ": " + display(v, field, relations) for k, v in value.items())
+        # Public temporal value, not a generic repr of a Python object.
+        if field == "sample_interval":
+            return value.to_token()
+        raise TypeError("unsupported display field: " + field)
+
+    def project(entry):
+        d = entry.details()
+        relations, fields = [], []
+        def add(name, value):
+            if value is not None:
+                fields.append({"name": name, "value": display(value, name, relations)})
+        for name in FIELDS.get(entry.kind.value, "").split():
+            if hasattr(d, name):
+                add(name, getattr(d, name))
+        for name in ("parents", "children", "dependents"):
+            for ref in getattr(d, name):
+                relations.append({"field": name, "ref": ref_payload(ref)})
+        if entry.kind.value == "entity":
+            source = d.source
+            add("source_kind", source.kind)
+            # Remote file URLs may contain credentials. Only table identity is projected.
+            if source.kind == "table":
+                add("source_table", source.table)
+                add("source_database", source.database)
+            version = d.versioning
+            if version is not None:
+                for name in ("kind", "partition_field", "grain", "timezone", "format", "valid_from", "valid_to", "interval", "open_end"):
+                    if hasattr(version, name):
+                        add("version_" + name, getattr(version, name))
+        if entry.kind.value == "period_calendar":
+            for level in d.levels:
+                add("level", level.name)
+                for name in ("key_ref", "period_count", "direct_finer_levels", "direct_coarser_levels", "rollup_targets"):
+                    add("level_" + name, getattr(level, name))
+        computation = d.definition.to_dict() if entry.kind.value in ("metric", "measure", "dimension", "time_dimension") else None
+        def definition_refs(value, role="definition"):
+            if isinstance(value, dict):
+                if value.get("schema") == "marivo.semantic_ref/v1":
+                    relations.append({"field": role, "ref": value})
+                else:
+                    for key, item in value.items():
+                        definition_refs(item, role + "." + key)
+            elif isinstance(value, list):
+                for item in value:
+                    definition_refs(item, role)
+        if computation is not None:
+            definition_refs(computation["node"])
+            definition_refs(computation["temporal"], "temporal")
+        unique = {(r["field"], r["ref"]["kind"], r["ref"]["path"]): r for r in relations}
+        return {
+            "computation": computation, "ref": ref_payload(entry.ref), "name": entry.name, "domain": d.domain,
+            "definition": d.context.business_definition, "guardrails": list(d.context.guardrails),
+            "source": {"file": d.source_location.file, "line": d.source_location.line, "symbol": d.python_symbol},
+            "fields": fields, "relations": list(unique.values()),
+        }
+
+try:
+    catalog = ms.load(workspace_dir=sys.argv[1])
+    result = {
+        "fingerprint": catalog.definition_fingerprint,
+        "kinds": [kind.value for kind in SemanticKind],
+        "objects": [project(entry) for kind in SemanticKind for entry in catalog.items(kind).items],
+    }
+except (SemanticLoadFailed, SemanticError) as error:
+    print(json.dumps({
+        "schema": "marivo.semantic_catalog_error/v1",
+        "kind": "semantic",
+        "message": str(error),
+    }, ensure_ascii=True))
+    raise SystemExit(1)
+
+print(json.dumps(result, ensure_ascii=True))
+`.trim();