@deepexi/datasense-connector 3.0.1 → 3.1.0

package/testing/compliance.js

@@ -1,291 +0,0 @@
-import { parseIsoInstant } from './schema.js';
-export const COMPLIANCE_CODES = Object.freeze([
-    'PRIMARY_KEY_DUPLICATE',
-    'RECORD_SCHEMA_MISMATCH',
-    'UNDECLARED_TABLE',
-    'CHECKPOINT_NOT_MONOTONIC',
-    'FULL_READ_NOT_REPEATABLE',
-    'FIXTURE_EXCHANGE_UNUSED',
-    'FIXTURE_EXCHANGE_MISMATCH',
-    'EXPECTATION_FAILED',
-]);
-function isRecord(value) {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-function tablesOf(value) {
-    const tables = {};
-    if (!isRecord(value))
-        return tables;
-    for (const [name, table] of Object.entries(value)) {
-        if (!isRecord(table) || !isRecord(table.columns))
-            continue;
-        const columns = {};
-        for (const [column, type] of Object.entries(table.columns)) {
-            if (isRecord(type) && typeof type.kind === 'string')
-                columns[column] = type;
-        }
-        const primaryKey = Array.isArray(table.primaryKey) ? table.primaryKey.filter((item) => typeof item === 'string') : [];
-        tables[name] = { columns, primaryKey };
-    }
-    return tables;
-}
-function entriesOf(value) {
-    return isRecord(value) ? Object.entries(value).filter((entry) => isRecord(entry[1])) : [];
-}
-export function scopedTables(declaration, scope) {
-    if (!declaration || !isRecord(declaration.discover))
-        return null;
-    const { streams, pushes } = declaration.discover;
-    if (scope.stream !== undefined) {
-        const stream = entriesOf(streams).find(([name]) => name === scope.stream)?.[1];
-        return tablesOf(stream?.tables);
-    }
-    if (scope.push !== undefined) {
-        const push = entriesOf(pushes).find(([name]) => name === scope.push)?.[1];
-        return tablesOf(push?.tables);
-    }
-    const all = {};
-    for (const [, item] of [...entriesOf(streams), ...entriesOf(pushes)])
-        Object.assign(all, tablesOf(item.tables));
-    return all;
-}
-const INSTANT_PATTERN = /^(\d{4})-(\d{2})-(\d{2})[T ](\d{2}):(\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?(Z|[+-]\d{2}(?::?\d{2})?)$/i;
-const DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/;
-const INTEGER_PATTERN = /^-?\d+$/;
-const DECIMAL_PATTERN = /^-?(\d+)(?:\.(\d+))?$/;
-function validCalendarDate(year, month, day) {
-    return month >= 1 && month <= 12 && day >= 1 && day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
-}
-export function columnValueProblem(type, value) {
-    if (value === null || value === undefined)
-        return null;
-    switch (type.kind) {
-        case 'string':
-            return typeof value === 'string' ? null : 'is not a valid string';
-        case 'integer':
-            if (typeof value === 'number' && Number.isSafeInteger(value))
-                return null;
-            if (typeof value === 'bigint')
-                return null;
-            if (typeof value === 'string' && INTEGER_PATTERN.test(value))
-                return null;
-            return 'is not a valid integer';
-        case 'decimal': {
-            let text;
-            if (typeof value === 'number' && Number.isFinite(value))
-                text = String(value);
-            else if (typeof value === 'string')
-                text = value;
-            else
-                return 'is not a valid decimal';
-            const match = DECIMAL_PATTERN.exec(text);
-            if (!match)
-                return 'is not a valid decimal';
-            if (type.precision !== undefined && type.scale !== undefined) {
-                const integerDigits = (match[1] ?? '').replace(/^0+(?=\d)/, '');
-                if (integerDigits !== '0' && integerDigits.length > type.precision - type.scale)
-                    return `exceeds decimal precision ${type.precision},${type.scale}`;
-            }
-            return null;
-        }
-        case 'boolean':
-            return typeof value === 'boolean' ? null : 'is not a valid boolean';
-        case 'date': {
-            if (typeof value !== 'string')
-                return 'is not a valid date';
-            const match = DATE_PATTERN.exec(value);
-            return match && validCalendarDate(Number(match[1]), Number(match[2]), Number(match[3])) ? null : 'is not a valid date';
-        }
-        case 'instant': {
-            if (typeof value !== 'string')
-                return 'is not a valid instant';
-            const match = INSTANT_PATTERN.exec(value);
-            if (!match)
-                return 'is not an ISO 8601 timestamp with a time zone offset';
-            const [, year, month, day, hour, minute, second] = match;
-            const valid = validCalendarDate(Number(year), Number(month), Number(day)) && Number(hour) <= 23 && Number(minute) <= 59 && Number(second ?? '0') <= 59;
-            return valid && parseIsoInstant(value) !== null ? null : 'is not a valid point in time';
-        }
-        case 'json':
-            try {
-                return JSON.stringify(value) === undefined ? 'is not valid JSON' : null;
-            }
-            catch {
-                return 'is not valid JSON';
-            }
-        default:
-            return `has an unsupported column type ${type.kind}`;
-    }
-}
-class FindingSet {
-    items = new Map();
-    ordered = [];
-    add(key, build) {
-        const existing = this.items.get(key);
-        if (existing) {
-            existing.count += 1;
-            existing.finding.detail.count = existing.count;
-            return;
-        }
-        const { code, message, table, sample, extra } = build();
-        const finding = { code, message, ...(table !== undefined ? { table } : {}), detail: { count: 1, ...(sample !== undefined ? { sample } : {}), ...extra } };
-        this.items.set(key, { finding, count: 1 });
-        this.ordered.push(finding);
-    }
-    push(finding) {
-        this.ordered.push(finding);
-    }
-    list() {
-        return this.ordered;
-    }
-}
-function checkRows(input, tables, findings) {
-    const keysSeen = new Map();
-    const undeclared = (table, sample) => {
-        findings.add(`undeclared\0${table}`, () => ({
-            code: 'UNDECLARED_TABLE', table, sample,
-            message: `table "${table}" is not declared${input.stream !== undefined ? ` by stream "${input.stream}"` : input.push !== undefined ? ` by push "${input.push}"` : ''}`,
-        }));
-    };
-    const mismatch = (table, column, reason, sample) => {
-        findings.add(`schema\0${table}\0${column}\0${reason}`, () => ({
-            code: 'RECORD_SCHEMA_MISMATCH', table, sample, extra: { column },
-            message: `column "${column}" of table "${table}" ${reason}`,
-        }));
-    };
-    for (const record of input.result.records) {
-        const declared = Object.hasOwn(tables, record.table) ? tables[record.table] : undefined;
-        if (!declared) {
-            undeclared(record.table, record.row);
-            continue;
-        }
-        for (const column of Object.keys(record.row)) {
-            if (!Object.hasOwn(declared.columns, column))
-                mismatch(record.table, column, 'is not declared', record.row);
-        }
-        for (const [column, type] of Object.entries(declared.columns)) {
-            const problem = columnValueProblem(type, record.row[column]);
-            if (problem)
-                mismatch(record.table, column, problem, record.row[column]);
-        }
-        const missing = declared.primaryKey.filter((column) => record.row[column] === null || record.row[column] === undefined);
-        for (const column of missing)
-            mismatch(record.table, column, 'is a primary key column and is missing', record.row);
-        if (declared.primaryKey.length === 0 || missing.length > 0)
-            continue;
-        const key = JSON.stringify(declared.primaryKey.map((column) => record.row[column]));
-        const seen = keysSeen.get(record.table) ?? new Set();
-        if (seen.has(key)) {
-            const sample = Object.fromEntries(declared.primaryKey.map((column) => [column, record.row[column]]));
-            findings.add(`pk\0${record.table}`, () => ({
-                code: 'PRIMARY_KEY_DUPLICATE', table: record.table, sample,
-                message: `table "${record.table}" received more than one record with the same primary key (${declared.primaryKey.join(', ')})`,
-            }));
-        }
-        seen.add(key);
-        keysSeen.set(record.table, seen);
-    }
-    for (const removal of input.result.removals) {
-        const declared = Object.hasOwn(tables, removal.table) ? tables[removal.table] : undefined;
-        if (!declared) {
-            undeclared(removal.table, removal.key);
-            continue;
-        }
-        for (const column of Object.keys(removal.key)) {
-            if (!declared.primaryKey.includes(column))
-                mismatch(removal.table, column, 'is not part of the primary key (removal)', removal.key);
-        }
-        for (const column of declared.primaryKey) {
-            const value = removal.key[column];
-            if (value === null || value === undefined) {
-                mismatch(removal.table, column, 'is a primary key column and is missing (removal)', removal.key);
-                continue;
-            }
-            const type = declared.columns[column];
-            const problem = type ? columnValueProblem(type, value) : 'is not declared';
-            if (problem)
-                mismatch(removal.table, column, `${problem} (removal)`, value);
-        }
-    }
-}
-function comparable(value) {
-    if (typeof value === 'number' && Number.isFinite(value))
-        return { kind: 'number', at: value };
-    if (typeof value === 'string') {
-        const at = parseIsoInstant(value);
-        if (at !== null)
-            return { kind: 'time', at };
-    }
-    return null;
-}
-function checkCheckpoints(checkpoints, findings) {
-    for (let index = 1; index < checkpoints.length; index += 1) {
-        const previous = checkpoints[index - 1];
-        const next = checkpoints[index];
-        if (!previous || !next || !isRecord(previous.state) || !isRecord(next.state))
-            continue;
-        for (const [field, before] of Object.entries(previous.state)) {
-            if (!Object.hasOwn(next.state, field))
-                continue;
-            const after = next.state[field];
-            const a = comparable(before);
-            const b = comparable(after);
-            if (!a || !b || a.kind !== b.kind || b.at >= a.at)
-                continue;
-            findings.push({
-                code: 'CHECKPOINT_NOT_MONOTONIC',
-                message: `checkpoint field "${field}" went backwards from ${JSON.stringify(before)} to ${JSON.stringify(after)} (checkpoint ${next.seq})`,
-                detail: { field, previous: before, next: after, seq: next.seq },
-            });
-        }
-    }
-}
-function canonical(value) {
-    return JSON.stringify(value, (_key, item) => (isRecord(item) ? Object.fromEntries(Object.keys(item).sort().map((key) => [key, item[key]])) : item));
-}
-function recordSet(records, tables) {
-    return records
-        .map((record) => {
-        const primaryKey = tables && Object.hasOwn(tables, record.table) ? tables[record.table]?.primaryKey ?? [] : [];
-        return `${record.table}\0${canonical(primaryKey.map((column) => record.row[column]))}\0${canonical(record.row)}`;
-    })
-        .sort();
-}
-function checkRepeatable(input, tables, findings) {
-    const rerun = input.rerun;
-    if (input.mode !== 'full' || !rerun)
-        return;
-    if (rerun.error) {
-        findings.push({
-            code: 'FULL_READ_NOT_REPEATABLE',
-            message: `the second full read failed (${rerun.error.code}: ${rerun.error.message})`,
-            detail: { error: rerun.error },
-        });
-        return;
-    }
-    const first = recordSet(input.result.records, tables);
-    const second = recordSet(rerun.records, tables);
-    if (first.length === second.length && first.every((item, index) => item === second[index]))
-        return;
-    const firstSet = new Set(first);
-    const secondSet = new Set(second);
-    findings.push({
-        code: 'FULL_READ_NOT_REPEATABLE',
-        message: `two full reads returned different records (${first.length} vs ${second.length} record(s))`,
-        detail: {
-            firstCount: first.length,
-            secondCount: second.length,
-            onlyInFirst: first.filter((item) => !secondSet.has(item)).length,
-            onlyInSecond: second.filter((item) => !firstSet.has(item)).length,
-        },
-    });
-}
-export function checkCompliance(input) {
-    const findings = new FindingSet();
-    const tables = scopedTables(input.declaration, { ...(input.stream !== undefined ? { stream: input.stream } : {}), ...(input.push !== undefined ? { push: input.push } : {}) });
-    if (tables)
-        checkRows(input, tables, findings);
-    checkCheckpoints(input.result.checkpoints, findings);
-    checkRepeatable(input, tables, findings);
-    return findings.list();
-}

package/testing/engine.d.ts

@@ -1,66 +0,0 @@
-import type { LoadedSource } from './source-loader.js';
-import { type GuestExecutor, type LocalHost, type LocalRunResult } from './types.js';
-/** vitest：当前进程直接执行已导入的定义对象，不过 vm（复用 guest 运行时的 `ctx` 实现）。 */
-export declare function createInProcessExecutor(definition: unknown): GuestExecutor;
-/** 子进程入口：源码下是 `src/testing/child.ts`（Bun 直接执行），dist 下是构建产出的 `testing/child.cjs`。 */
-export declare const VM_CHILD_ENTRY: string;
-export interface VmExecutorOptions {
-    /** guest 运行时源码；缺省 `loadGuestRuntime()`。CLI 传构建时内联的 `guest.cjs` 文本。 */
-    guestRuntime?: string;
-    /** 子进程入口；缺省 `VM_CHILD_ENTRY`（CLI 打成单文件时传自己发布的路径，测试用来换替身）。 */
-    childEntry?: string;
-}
-/** vm 内执行（CLI 用）：空环境变量的子进程 + 与 runner 同一个 guest 会话。 */
-export declare function createVmExecutor(source: LoadedSource, options?: VmExecutorOptions): GuestExecutor;
-/** `inspectDefinition` 的结果：成功时是 guest 上报的定义摘要（`summarizeDefinition`）。 */
-export type DefinitionInspection = {
-    ok: true;
-    summary: {
-        name: string;
-        version: string;
-    } & Record<string, unknown>;
-} | {
-    ok: false;
-    code: string;
-    message: string;
-    mismatch: boolean;
-};
-/**
- * 只加载连接器、不执行它的任何作业：发一个 `check` 作业，guest 加载代码、校验定义、核对期望的名字与版本
- * （vm 执行器取 `package.json` 的 name / version）后上报定义摘要，宿主在 `onDefinition` 里记下摘要并中止，
- * 连接器的 `check` 与出站调用都不会执行。加载失败时 `code` 为 guest 的失败码（通常 `CONNECTOR_LOAD_FAILED`），
- * `mismatch` 表示定义的名字或版本与 `package.json` 不同。
- */
-export declare function inspectDefinition(executor: GuestExecutor, options?: {
-    timeoutMs?: number;
-}): Promise<DefinitionInspection>;
-export interface StreamRunInput {
-    stream: string;
-    mode?: 'full' | 'incremental';
-    state?: unknown;
-    streamConfig?: Record<string, unknown>;
-    config: Record<string, unknown>;
-    limit?: number;
-    host: LocalHost;
-}
-/** 拉取一个数据流（先 check + discover，再按声明里该数据流的表拉取，与预览相同）。 */
-export declare function runStream(definition: unknown, input: StreamRunInput): Promise<LocalRunResult>;
-export declare function runDiscover(definition: unknown, input: {
-    config: Record<string, unknown>;
-    host: LocalHost;
-}): Promise<LocalRunResult>;
-export declare function runCheck(definition: unknown, input: {
-    config: Record<string, unknown>;
-    host: LocalHost;
-}): Promise<LocalRunResult>;
-export declare function runPush(definition: unknown, input: {
-    push: string;
-    requests: Array<{
-        requestId: string;
-        receivedAt: string;
-        headers: Record<string, string>;
-        body: string;
-    }>;
-    config: Record<string, unknown>;
-    host: LocalHost;
-}): Promise<LocalRunResult>;