@deepexi/datasense-connector 3.0.1 → 3.1.0

package/testing/engine.js

@@ -1,339 +0,0 @@
-import { fork } from 'node:child_process';
-import path from 'node:path';
-import { IS_SOURCE, PACKAGE_ROOT } from '../internal/location.js';
-import { loadGuestRuntime } from '../runtime/guest-runtime.js';
-import { parseChildMessage, undeclaredEmitTable, } from '../runtime/protocol.js';
-import { createInProcessGuestSession } from '../runtime/session.js';
-import { summarizeDefinition } from '../validate.js';
-import { LocalRunError } from './types.js';
-function isRecord(value) {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-function errorMessage(error) {
-    return error instanceof Error ? error.message : String(error);
-}
-function emptyResult() {
-    return { records: [], removals: [], checkpoints: [], logs: [], rejected: [], declaration: null, stopped: false, error: null };
-}
-function hostCallError(error) {
-    const code = isRecord(error) && typeof error.code === 'string' ? error.code : 'HOST_CALL_FAILED';
-    const result = { code: code === 'HOST_DENIED' ? 'EGRESS_HOST_DENIED' : code, message: errorMessage(error) };
-    if (isRecord(error) && typeof error.status === 'number' && Number.isInteger(error.status))
-        result.status = error.status;
-    if (isRecord(error) && typeof error.retryAfterMs === 'number' && error.retryAfterMs >= 0)
-        result.retryAfterMs = error.retryAfterMs;
-    return result;
-}
-function decode(raw) {
-    if (typeof raw !== 'string')
-        return raw;
-    try {
-        return JSON.parse(raw);
-    }
-    catch {
-        return null;
-    }
-}
-function drive(transportFactory, input) {
-    const { execute, host } = input;
-    const job = execute.job;
-    const redact = (text) => (host.redact ? host.redact(text) : text);
-    const result = emptyResult();
-    const limit = input.limit !== undefined && Number.isFinite(input.limit) && input.limit >= 0 ? Math.floor(input.limit) : null;
-    let declaredTables = [];
-    let declared = false;
-    let seq = 0;
-    let finished = false;
-    const addLogs = (logs) => {
-        for (const log of logs)
-            result.logs.push({ at: log.at, level: log.level, message: redact(log.message) });
-    };
-    return new Promise((resolve) => {
-        let timer;
-        let transport = null;
-        const finish = (error) => {
-            if (finished)
-                return;
-            finished = true;
-            if (timer !== undefined)
-                clearTimeout(timer);
-            if (error)
-                result.error = { code: error.code, message: redact(error.message), stack: error.stack === null ? null : redact(error.stack) };
-            const stopping = transport ? transport.stop() : Promise.resolve();
-            void stopping.then(() => resolve(result), () => resolve(result));
-        };
-        const send = (message) => {
-            if (finished || !transport)
-                return;
-            transport.send(message);
-        };
-        const onEmit = (id, payload) => {
-            const batch = payload;
-            seq += 1;
-            addLogs(batch.logs);
-            let records = batch.records;
-            let stop = false;
-            if (limit !== null) {
-                const room = Math.max(0, limit - result.records.length);
-                if (records.length >= room) {
-                    stop = true;
-                    records = records.slice(0, room);
-                }
-            }
-            const truncated = records.length < batch.records.length;
-            result.records.push(...records);
-            result.removals.push(...batch.removals);
-            if (batch.checkpoint && !truncated)
-                result.checkpoints.push({ seq, state: batch.checkpoint.state });
-            if (stop)
-                result.stopped = true;
-            send({ type: 'reply', id, ok: true, value: { seq, duplicate: false, rejected: 0, ...(stop ? { stop: true } : {}) } });
-        };
-        const onCall = (message) => {
-            switch (message.method) {
-                case 'definition':
-                    try {
-                        host.onDefinition?.(message.payload);
-                    }
-                    catch (error) {
-                        finish({ code: 'RUNNER_CRASHED', message: `the host rejected the connector definition: ${errorMessage(error)}`, stack: null });
-                        return;
-                    }
-                    send({ type: 'reply', id: message.id, ok: true, value: null });
-                    return;
-                case 'declaration': {
-                    const report = message.payload;
-                    result.declaration = { summary: report.summary, discover: report.discover };
-                    declared = true;
-                    const stream = job.kind === 'pull' && Object.hasOwn(report.discover.streams, job.stream) ? report.discover.streams[job.stream] : undefined;
-                    declaredTables = stream ? Object.keys(stream.tables) : [];
-                    send({ type: 'reply', id: message.id, ok: true, value: null });
-                    return;
-                }
-                case 'emit':
-                    onEmit(message.id, message.payload);
-                    return;
-                default:
-                    host.call(message.method, message.payload).then((value) => send({ type: 'reply', id: message.id, ok: true, value: value ?? null }), (error) => {
-                        if (error instanceof LocalRunError && error.code === 'DB_DRIVER_MISSING') {
-                            finish({ code: error.code, message: error.message, stack: null });
-                            return;
-                        }
-                        send({ type: 'reply', id: message.id, ok: false, error: hostCallError(error) });
-                    });
-            }
-        };
-        const onResult = (outcome) => {
-            switch (outcome.kind) {
-                case 'push':
-                    for (const record of outcome.records)
-                        result.records.push({ table: record.table, row: record.row });
-                    for (const removal of outcome.removals)
-                        result.removals.push({ table: removal.table, key: removal.key });
-                    result.rejected.push(...outcome.rejected.map((item) => ({ requestId: item.requestId, code: item.code, message: redact(item.message) })));
-                    break;
-                case 'discover':
-                    result.declaration = { summary: outcome.summary, discover: outcome.discover };
-                    break;
-                case 'check':
-                    break;
-                case 'pull':
-                    if (outcome.stopped === true)
-                        result.stopped = true;
-                    break;
-            }
-            addLogs(outcome.logs);
-            finish(null);
-        };
-        const onMessage = (raw) => {
-            if (finished)
-                return;
-            const decoded = decode(raw);
-            const message = parseChildMessage(decoded, job, { tables: declaredTables, declared });
-            if (!message) {
-                const table = undeclaredEmitTable(decoded, job, { tables: declaredTables });
-                const call = decoded;
-                if (table !== null) {
-                    finish({ code: 'SCHEMA_ERROR', message: `connector emitted records for undeclared table ${table}`, stack: null });
-                }
-                else if (declared && typeof call === 'object' && call !== null && call.type === 'call' && call.method === 'declaration') {
-                    finish({ code: 'RUNNER_CRASHED', message: 'connector sandbox reported the declaration more than once (protocol error)', stack: null });
-                }
-                else {
-                    finish({ code: 'RUNNER_CRASHED', message: 'connector sandbox sent a malformed message', stack: null });
-                }
-                return;
-            }
-            switch (message.type) {
-                case 'ready':
-                    send(execute);
-                    return;
-                case 'call':
-                    onCall(message);
-                    return;
-                case 'result':
-                    onResult(message.result);
-                    return;
-                case 'failure':
-                    addLogs(message.logs);
-                    finish({ code: message.code, message: message.message, stack: message.stack ?? null });
-                    return;
-            }
-        };
-        const onExit = (description) => {
-            finish({ code: 'RUNNER_CRASHED', message: `connector sandbox exited without a result (${description})`, stack: null });
-        };
-        try {
-            transport = transportFactory(onMessage, onExit);
-        }
-        catch (error) {
-            finish({ code: 'RUNNER_CRASHED', message: `connector sandbox failed to start: ${errorMessage(error)}`, stack: null });
-            return;
-        }
-        if (input.timeoutMs !== undefined && Number.isFinite(input.timeoutMs)) {
-            timer = setTimeout(() => {
-                finish({ code: 'TIMEOUT', message: `connector execution exceeded ${input.timeoutMs} ms`, stack: null });
-            }, Math.max(0, input.timeoutMs));
-        }
-        if (!input.waitForReady)
-            send(execute);
-    });
-}
-export function createInProcessExecutor(definition) {
-    return {
-        async run(job, host, options = {}) {
-            const guest = await loadGuestRuntime();
-            const named = isRecord(definition) ? definition : {};
-            const expect = { name: String(named.name ?? ''), version: String(named.version ?? '') };
-            const execute = { type: 'execute', guest, code: '', expect, job };
-            return drive((onMessage) => {
-                let active = true;
-                const session = createInProcessGuestSession((json) => {
-                    if (active)
-                        queueMicrotask(() => onMessage(json));
-                }, definition);
-                return {
-                    send(message) {
-                        if (active)
-                            session.handle(JSON.stringify(message));
-                    },
-                    async stop() {
-                        if (!active)
-                            return;
-                        session.handle(JSON.stringify({ type: 'cancel' }));
-                        active = false;
-                    },
-                };
-            }, { execute, host, limit: options.limit, timeoutMs: options.timeoutMs, waitForReady: false });
-        },
-    };
-}
-export const VM_CHILD_ENTRY = path.join(PACKAGE_ROOT, 'testing', IS_SOURCE ? 'child.ts' : 'child.cjs');
-const KILL_WAIT_MS = 5000;
-function forkTransport(entry) {
-    return (onMessage, onExit) => {
-        const child = fork(entry, [], {
-            env: {},
-            execArgv: [],
-            stdio: ['ignore', 'pipe', 'inherit', 'ipc'],
-            serialization: 'json',
-        });
-        child.stdout?.on('data', (chunk) => process.stderr.write(chunk));
-        let exited = false;
-        const exitedPromise = new Promise((resolve) => {
-            child.once('exit', (code, signal) => {
-                exited = true;
-                resolve();
-                onExit(signal ? `signal ${signal}` : `exit code ${String(code)}`);
-            });
-        });
-        child.on('message', (raw) => onMessage(raw));
-        child.on('error', (error) => onExit(`spawn error: ${error.message}`));
-        return {
-            send(message) {
-                if (exited || !child.connected)
-                    return;
-                try {
-                    child.send(JSON.stringify(message));
-                }
-                catch {
-                }
-            },
-            async stop() {
-                if (!exited)
-                    child.kill('SIGKILL');
-                await Promise.race([exitedPromise, new Promise((resolve) => setTimeout(resolve, KILL_WAIT_MS))]);
-            },
-        };
-    };
-}
-export function createVmExecutor(source, options = {}) {
-    const entry = options.childEntry ?? VM_CHILD_ENTRY;
-    return {
-        async run(job, host, runOptions = {}) {
-            const guest = options.guestRuntime ?? await loadGuestRuntime();
-            const execute = {
-                type: 'execute', guest, code: source.code, expect: { name: source.name, version: source.version }, job,
-            };
-            return drive(forkTransport(entry), { execute, host, limit: runOptions.limit, timeoutMs: runOptions.timeoutMs, waitForReady: true });
-        },
-    };
-}
-const INSPECT_ABORT = 'inspect-definition: stop after the definition';
-const MISMATCH_PREFIX = 'connector code defines ';
-export async function inspectDefinition(executor, options = {}) {
-    let summary = null;
-    const host = {
-        call: async (method) => {
-            throw new LocalRunError('HOST_DENIED', `${method} is not available while only loading the definition`);
-        },
-        close: async () => { },
-        onDefinition(value) {
-            summary = value;
-            throw new Error(INSPECT_ABORT);
-        },
-    };
-    const result = await executor.run({ kind: 'check', config: {}, secretFields: [] }, host, { timeoutMs: options.timeoutMs ?? 30_000 });
-    if (isRecord(summary) && typeof summary.name === 'string' && typeof summary.version === 'string') {
-        return { ok: true, summary: summary };
-    }
-    const error = result.error ?? { code: 'CONNECTOR_LOAD_FAILED', message: 'the connector did not report its definition' };
-    return { ok: false, code: error.code, message: error.message, mismatch: error.message.startsWith(MISMATCH_PREFIX) };
-}
-function secretFieldsOf(definition) {
-    try {
-        return summarizeDefinition(definition).secretFields;
-    }
-    catch {
-        return [];
-    }
-}
-export function runStream(definition, input) {
-    const job = {
-        kind: 'pull',
-        config: input.config,
-        secretFields: secretFieldsOf(definition),
-        stream: input.stream,
-        mode: input.mode ?? 'full',
-        state: input.state ?? null,
-        streamConfig: input.streamConfig ?? {},
-        tables: [],
-        discoverFirst: true,
-    };
-    return createInProcessExecutor(definition).run(job, input.host, input.limit !== undefined ? { limit: input.limit } : {});
-}
-export function runDiscover(definition, input) {
-    return createInProcessExecutor(definition).run({ kind: 'discover', config: input.config, secretFields: secretFieldsOf(definition) }, input.host);
-}
-export function runCheck(definition, input) {
-    return createInProcessExecutor(definition).run({ kind: 'check', config: input.config, secretFields: secretFieldsOf(definition) }, input.host);
-}
-export function runPush(definition, input) {
-    const job = {
-        kind: 'push',
-        config: input.config,
-        secretFields: secretFieldsOf(definition),
-        requests: input.requests.map((request) => ({ ...request, push: input.push })),
-    };
-    return createInProcessExecutor(definition).run(job, input.host);
-}

package/testing/fixtures.d.ts

@@ -1,145 +0,0 @@
-/**
- * 夹具格式（CLI `read --record` 写、`connector test` 读、runner 服务端回放读，共享值域）。
- *
- * 源码目录下 `fixtures/<用例名>/`：
- * - `case.json`（必有）：一个 `FixtureCase`；
- * - `exchanges.ndjson`（可选）：每行一个 `FixtureExchange`，`seq` 严格递增；
- * - `requests.ndjson`（推送用例必有，其他用例不能有）：每行一个 `FixtureRequest`。
- *
- * 用例名 `^[a-z0-9][a-z0-9_-]{0,63}$`。夹具里长度 ≥ 4 的 Secret 明文在录制时已替换为 `***`，凭证句柄
- * `ontos:secret:<field>` 保持原样（见 `recorder.ts`）。NDJSON 每行以 `\n` 结尾，键按插入顺序输出（便于 diff）。
- *
- * 本文件除 `readFixtureDir`（动态导入 `node:fs`）外不依赖 Node 模块。
- */
-import type { CryptoRequest } from '../runtime/protocol.js';
-/** 夹具目录里的一个文件；`path` 相对源码根目录，如 `fixtures/basic/case.json`（与 shared 的同名类型同形）。 */
-export interface ConnectorFixtureFile {
-    path: string;
-    content: string;
-}
-/**
- * 用例的期望（全部可选，给出的每一项都必须精确相等，否则 `EXPECTATION_FAILED`）：
- * - `records` / `removals`：表 → 条数（只比较给出的表）；
- * - `rejected`：推送拒收条数；`checkpoints`：断点个数；
- * - `tables`：用例范围内声明的表（拉取取该数据流、推送取该推送、discover 取全部），按集合比较。
- */
-export interface FixtureExpectation {
-    records?: Record<string, number>;
-    removals?: Record<string, number>;
-    rejected?: number;
-    checkpoints?: number;
-    tables?: string[];
-}
-/** `case.json`；`secrets` 是字段 → 值（录制后长度 ≥ 4 的值为 `***`），回放时 `secret.reveal` 照给。 */
-export type FixtureCase = {
-    kind: 'pull';
-    stream: string;
-    mode: 'full' | 'incremental';
-    state?: unknown;
-    streamConfig?: Record<string, unknown>;
-    config: Record<string, unknown>;
-    secrets?: Record<string, string>;
-    limit?: number;
-    expect?: FixtureExpectation;
-} | {
-    kind: 'push';
-    push: string;
-    config: Record<string, unknown>;
-    secrets?: Record<string, string>;
-    expect?: FixtureExpectation;
-} | {
-    kind: 'discover';
-    config: Record<string, unknown>;
-    secrets?: Record<string, string>;
-    expect?: FixtureExpectation;
-};
-/**
- * 一次宿主往来：
- * - `http`：`request.url` 已并入 query 并按键排序、含句柄的 query 值为 `***`；凭证类头为 `***`；body 里的句柄原样保留；
- * - `db`：一次查询的全部分页合成一条，`rows` 为全部行；
- * - `crypto`：请求（key / data 里的句柄原样）与结果。
- */
-export type FixtureExchange = {
-    seq: number;
-    kind: 'http';
-    request: {
-        method: string;
-        url: string;
-        headers: Record<string, string>;
-        body: string | null;
-        bodyEncoding: 'utf8' | 'base64';
-    };
-    response: {
-        status: number;
-        headers: Record<string, string>;
-        body: string;
-        bodyEncoding: 'utf8' | 'base64';
-    } | null;
-    error: {
-        code: string;
-        message: string;
-    } | null;
-} | {
-    seq: number;
-    kind: 'db';
-    sql: string;
-    params: unknown[];
-    fields: Array<{
-        name: string;
-        dataTypeId: number;
-    }>;
-    rows: Array<Record<string, unknown>>;
-    error: {
-        code: string;
-        message: string;
-    } | null;
-} | {
-    seq: number;
-    kind: 'crypto';
-    request: CryptoRequest;
-    value: string;
-};
-/** 推送用例重放的一条收件箱请求。 */
-export interface FixtureRequest {
-    requestId: string;
-    receivedAt: string;
-    headers: Record<string, string>;
-    body: string;
-}
-export interface LoadedFixture {
-    name: string;
-    case: FixtureCase;
-    exchanges: FixtureExchange[];
-    requests: FixtureRequest[];
-}
-/** 夹具格式错误：`file` 为相对源码根目录的路径，`line` 为 NDJSON 的行号（从 1 起；与整个文件相关时为 null）。 */
-export declare class FixtureFormatError extends Error {
-    readonly file: string;
-    readonly line: number | null;
-    constructor(message: string, file: string, line?: number | null);
-}
-export declare const FIXTURE_CASE_NAME_PATTERN: RegExp;
-export declare const FIXTURES_DIR = "fixtures";
-/**
- * 校验并解析夹具文件（只看 `fixtures/` 下的文件，其余路径报错）；结果按用例名排序。
- * 格式错误抛 `FixtureFormatError`（含 file、line）。以 `.` 开头的文件（如 `.DS_Store`）忽略。
- */
-export declare function parseFixtures(files: ConnectorFixtureFile[]): LoadedFixture[];
-/**
- * 把一个用例写成夹具文件：`case.json`（两格缩进）、`exchanges.ndjson`（有往来时）、`requests.ndjson`（推送用例）。
- * `case.secrets` 里长度 ≥ 4 的值写成 `***`。与 `parseFixtures` 往返无损（`secrets` 已遮蔽时）。
- */
-export declare function serializeFixture(name: string, input: {
-    case: FixtureCase;
-    exchanges: FixtureExchange[];
-    requests?: FixtureRequest[];
-}): ConnectorFixtureFile[];
-/**
- * 读 `<dir>/fixtures/**` 下的全部文件（`path` 相对 `dir`、用 `/` 分隔、按路径排序）；没有 `fixtures/` 时返回空数组。
- * 以 `.` 开头的文件与目录跳过；遇到符号链接抛 `FixtureFormatError`（夹具是租户数据，不跟随链接）。
- */
-export declare function readFixtureDir(dir: string): Promise<ConnectorFixtureFile[]>;
-/** http 请求的规范 URL：query 并入 URL，再按键排序（同名键保持相对顺序）。解析不了时原样返回。 */
-export declare function canonicalRequestUrl(url: string, query: Record<string, string | string[]> | undefined): string;
-/** 键排序后的 JSON（db 参数、crypto 请求的比较用）。 */
-export declare function canonicalJson(value: unknown): string;