@deepexi/datasense-connector 3.0.1 → 3.1.0

package/testing/fixtures.js

@@ -1,407 +0,0 @@
-import { REDACT_MIN_LENGTH, REDACTED } from './redaction.js';
-export class FixtureFormatError extends Error {
-    file;
-    line;
-    constructor(message, file, line = null) {
-        super(line === null ? `${file}: ${message}` : `${file}:${line}: ${message}`);
-        this.name = 'FixtureFormatError';
-        this.file = file;
-        this.line = line;
-    }
-}
-export const FIXTURE_CASE_NAME_PATTERN = /^[a-z0-9][a-z0-9_-]{0,63}$/;
-export const FIXTURES_DIR = 'fixtures';
-const CASE_FILE = 'case.json';
-const EXCHANGES_FILE = 'exchanges.ndjson';
-const REQUESTS_FILE = 'requests.ndjson';
-const CASE_FILES = new Set([CASE_FILE, EXCHANGES_FILE, REQUESTS_FILE]);
-class ShapeError extends Error {
-}
-function isRecord(value) {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-function fail(message) {
-    throw new ShapeError(message);
-}
-function onlyKeys(value, allowed, where) {
-    for (const key of Object.keys(value)) {
-        if (!allowed.includes(key))
-            fail(`${where} has an unknown field "${key}"`);
-    }
-}
-function requireString(value, where) {
-    if (typeof value !== 'string')
-        fail(`${where} must be a string`);
-    return value;
-}
-function requireNonEmptyString(value, where) {
-    const text = requireString(value, where);
-    if (text === '')
-        fail(`${where} must not be empty`);
-    return text;
-}
-function requireRecord(value, where) {
-    if (!isRecord(value))
-        fail(`${where} must be an object`);
-    return value;
-}
-function requireStringRecord(value, where) {
-    const record = requireRecord(value, where);
-    for (const [key, item] of Object.entries(record)) {
-        if (typeof item !== 'string')
-            fail(`${where}.${key} must be a string`);
-    }
-    return record;
-}
-function requireCount(value, where) {
-    if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0)
-        fail(`${where} must be a non-negative integer`);
-    return value;
-}
-function requireError(value, where) {
-    if (value === null)
-        return;
-    const error = requireRecord(value, where);
-    onlyKeys(error, ['code', 'message'], where);
-    requireString(error.code, `${where}.code`);
-    requireString(error.message, `${where}.message`);
-}
-function requireEncoding(value, where) {
-    if (value !== 'utf8' && value !== 'base64')
-        fail(`${where} must be "utf8" or "base64"`);
-}
-function validateExpectation(value) {
-    const expectation = requireRecord(value, 'expect');
-    onlyKeys(expectation, ['records', 'removals', 'rejected', 'checkpoints', 'tables'], 'expect');
-    for (const field of ['records', 'removals']) {
-        if (expectation[field] === undefined)
-            continue;
-        for (const [table, count] of Object.entries(requireRecord(expectation[field], `expect.${field}`)))
-            requireCount(count, `expect.${field}.${table}`);
-    }
-    if (expectation.rejected !== undefined)
-        requireCount(expectation.rejected, 'expect.rejected');
-    if (expectation.checkpoints !== undefined)
-        requireCount(expectation.checkpoints, 'expect.checkpoints');
-    if (expectation.tables !== undefined) {
-        if (!Array.isArray(expectation.tables) || !expectation.tables.every((table) => typeof table === 'string'))
-            fail('expect.tables must be an array of strings');
-    }
-}
-function validateCase(value) {
-    const item = requireRecord(value, 'case');
-    const common = ['kind', 'config', 'secrets', 'expect'];
-    switch (item.kind) {
-        case 'pull':
-            onlyKeys(item, [...common, 'stream', 'mode', 'state', 'streamConfig', 'limit'], 'case');
-            requireNonEmptyString(item.stream, 'stream');
-            if (item.mode !== 'full' && item.mode !== 'incremental')
-                fail('mode must be "full" or "incremental"');
-            if (item.streamConfig !== undefined)
-                requireRecord(item.streamConfig, 'streamConfig');
-            if (item.limit !== undefined && (typeof item.limit !== 'number' || !Number.isSafeInteger(item.limit) || item.limit <= 0))
-                fail('limit must be a positive integer');
-            break;
-        case 'push':
-            onlyKeys(item, [...common, 'push'], 'case');
-            requireNonEmptyString(item.push, 'push');
-            break;
-        case 'discover':
-            onlyKeys(item, common, 'case');
-            break;
-        default:
-            fail('kind must be one of "pull", "push", "discover"');
-    }
-    requireRecord(item.config, 'config');
-    if (item.secrets !== undefined)
-        requireStringRecord(item.secrets, 'secrets');
-    if (item.expect !== undefined)
-        validateExpectation(item.expect);
-    return item;
-}
-function validateCryptoRequest(value) {
-    const request = requireRecord(value, 'request');
-    requireString(request.data, 'request.data');
-    if (request.encoding !== 'hex' && request.encoding !== 'base64')
-        fail('request.encoding must be "hex" or "base64"');
-    switch (request.op) {
-        case 'hash':
-            onlyKeys(request, ['op', 'algorithm', 'data', 'encoding'], 'request');
-            if (!['md5', 'sha1', 'sha256', 'sha512'].includes(String(request.algorithm)))
-                fail('request.algorithm is not a hash algorithm');
-            return;
-        case 'hmac':
-            onlyKeys(request, ['op', 'algorithm', 'key', 'data', 'encoding'], 'request');
-            requireString(request.key, 'request.key');
-            if (!['sha1', 'sha256', 'sha512'].includes(String(request.algorithm)))
-                fail('request.algorithm is not an hmac algorithm');
-            return;
-        case 'sign':
-            onlyKeys(request, ['op', 'algorithm', 'key', 'data', 'encoding'], 'request');
-            requireString(request.key, 'request.key');
-            if (request.algorithm !== 'rsa-sha256')
-                fail('request.algorithm must be "rsa-sha256"');
-            return;
-        default:
-            fail('request.op must be one of "hmac", "sign", "hash"');
-    }
-}
-function validateExchange(value) {
-    const item = requireRecord(value, 'exchange');
-    if (typeof item.seq !== 'number' || !Number.isSafeInteger(item.seq) || item.seq <= 0)
-        fail('seq must be a positive integer');
-    switch (item.kind) {
-        case 'http': {
-            onlyKeys(item, ['seq', 'kind', 'request', 'response', 'error'], 'exchange');
-            const request = requireRecord(item.request, 'request');
-            onlyKeys(request, ['method', 'url', 'headers', 'body', 'bodyEncoding'], 'request');
-            requireNonEmptyString(request.method, 'request.method');
-            requireNonEmptyString(request.url, 'request.url');
-            requireStringRecord(request.headers, 'request.headers');
-            if (request.body !== null)
-                requireString(request.body, 'request.body');
-            requireEncoding(request.bodyEncoding, 'request.bodyEncoding');
-            if (item.response !== null) {
-                const response = requireRecord(item.response, 'response');
-                onlyKeys(response, ['status', 'headers', 'body', 'bodyEncoding'], 'response');
-                if (typeof response.status !== 'number' || !Number.isInteger(response.status))
-                    fail('response.status must be an integer');
-                requireStringRecord(response.headers, 'response.headers');
-                requireString(response.body, 'response.body');
-                requireEncoding(response.bodyEncoding, 'response.bodyEncoding');
-            }
-            requireError(item.error, 'error');
-            if ((item.response === null) === (item.error === null))
-                fail('exactly one of response and error must be null');
-            break;
-        }
-        case 'db': {
-            onlyKeys(item, ['seq', 'kind', 'sql', 'params', 'fields', 'rows', 'error'], 'exchange');
-            requireString(item.sql, 'sql');
-            if (!Array.isArray(item.params))
-                fail('params must be an array');
-            if (!Array.isArray(item.fields))
-                fail('fields must be an array');
-            for (const field of item.fields) {
-                const entry = requireRecord(field, 'fields[]');
-                requireString(entry.name, 'fields[].name');
-                if (typeof entry.dataTypeId !== 'number' || !Number.isInteger(entry.dataTypeId))
-                    fail('fields[].dataTypeId must be an integer');
-            }
-            if (!Array.isArray(item.rows) || !item.rows.every(isRecord))
-                fail('rows must be an array of objects');
-            requireError(item.error, 'error');
-            break;
-        }
-        case 'crypto':
-            onlyKeys(item, ['seq', 'kind', 'request', 'value'], 'exchange');
-            validateCryptoRequest(item.request);
-            requireString(item.value, 'value');
-            break;
-        default:
-            fail('kind must be one of "http", "db", "crypto"');
-    }
-    return item;
-}
-function validateRequest(value) {
-    const item = requireRecord(value, 'request');
-    onlyKeys(item, ['requestId', 'receivedAt', 'headers', 'body'], 'request');
-    requireNonEmptyString(item.requestId, 'requestId');
-    requireNonEmptyString(item.receivedAt, 'receivedAt');
-    requireStringRecord(item.headers, 'headers');
-    requireString(item.body, 'body');
-    return item;
-}
-function parseJsonFile(file, validate) {
-    let value;
-    try {
-        value = JSON.parse(file.content);
-    }
-    catch (error) {
-        throw new FixtureFormatError(`not valid JSON (${error instanceof Error ? error.message : String(error)})`, file.path);
-    }
-    try {
-        return validate(value);
-    }
-    catch (error) {
-        if (error instanceof ShapeError)
-            throw new FixtureFormatError(error.message, file.path);
-        throw error;
-    }
-}
-function parseNdjson(file, validate) {
-    const items = [];
-    const lines = file.content.split('\n');
-    for (let index = 0; index < lines.length; index += 1) {
-        const text = (lines[index] ?? '').replace(/\r$/, '');
-        if (text.trim() === '')
-            continue;
-        const line = index + 1;
-        let value;
-        try {
-            value = JSON.parse(text);
-        }
-        catch {
-            throw new FixtureFormatError('not valid JSON', file.path, line);
-        }
-        try {
-            items.push(validate(value, line));
-        }
-        catch (error) {
-            if (error instanceof ShapeError)
-                throw new FixtureFormatError(error.message, file.path, line);
-            throw error;
-        }
-    }
-    return items;
-}
-function normalizePath(filePath) {
-    return filePath.replace(/\\/g, '/').replace(/^\.\//, '');
-}
-export function parseFixtures(files) {
-    const groups = new Map();
-    for (const raw of files) {
-        const file = { path: normalizePath(raw.path), content: raw.content };
-        const parts = file.path.split('/');
-        if (parts.some((part) => part === '' || part === '.' || part === '..'))
-            throw new FixtureFormatError('fixture paths must be plain relative paths', file.path);
-        if (parts.some((part) => part.startsWith('.')))
-            continue;
-        if (parts.length !== 3 || parts[0] !== FIXTURES_DIR) {
-            throw new FixtureFormatError(`unexpected file: fixture files live at ${FIXTURES_DIR}/<case>/{${[...CASE_FILES].join(',')}}`, file.path);
-        }
-        const name = parts[1] ?? '';
-        const fileName = parts[2] ?? '';
-        if (!FIXTURE_CASE_NAME_PATTERN.test(name)) {
-            throw new FixtureFormatError(`case name "${name}" must match ${FIXTURE_CASE_NAME_PATTERN.source}`, file.path);
-        }
-        if (!CASE_FILES.has(fileName)) {
-            throw new FixtureFormatError(`unexpected file in case "${name}": expected ${[...CASE_FILES].join(', ')}`, file.path);
-        }
-        const group = groups.get(name) ?? new Map();
-        if (group.has(fileName))
-            throw new FixtureFormatError('duplicate file', file.path);
-        group.set(fileName, file);
-        groups.set(name, group);
-    }
-    const fixtures = [];
-    for (const name of [...groups.keys()].sort()) {
-        const group = groups.get(name) ?? new Map();
-        const casePath = `${FIXTURES_DIR}/${name}/${CASE_FILE}`;
-        const caseFile = group.get(CASE_FILE);
-        if (!caseFile)
-            throw new FixtureFormatError(`case "${name}" has no ${CASE_FILE}`, casePath);
-        const fixtureCase = parseJsonFile(caseFile, validateCase);
-        const exchangesFile = group.get(EXCHANGES_FILE);
-        let previousSeq = 0;
-        const exchanges = exchangesFile
-            ? parseNdjson(exchangesFile, (value) => {
-                const exchange = validateExchange(value);
-                if (exchange.seq <= previousSeq)
-                    fail(`seq ${exchange.seq} must be greater than the previous seq ${previousSeq}`);
-                previousSeq = exchange.seq;
-                return exchange;
-            })
-            : [];
-        const requestsFile = group.get(REQUESTS_FILE);
-        if (fixtureCase.kind === 'push' && !requestsFile) {
-            throw new FixtureFormatError(`push case "${name}" needs ${REQUESTS_FILE} next to ${CASE_FILE}`, casePath);
-        }
-        if (fixtureCase.kind !== 'push' && requestsFile) {
-            throw new FixtureFormatError(`${REQUESTS_FILE} only belongs to push cases`, requestsFile.path);
-        }
-        const seen = new Set();
-        const requests = requestsFile
-            ? parseNdjson(requestsFile, (value) => {
-                const request = validateRequest(value);
-                if (seen.has(request.requestId))
-                    fail(`requestId "${request.requestId}" appears more than once`);
-                seen.add(request.requestId);
-                return request;
-            })
-            : [];
-        fixtures.push({ name, case: fixtureCase, exchanges, requests });
-    }
-    return fixtures;
-}
-function redactCaseSecrets(fixtureCase) {
-    if (!fixtureCase.secrets)
-        return fixtureCase;
-    const secrets = Object.fromEntries(Object.entries(fixtureCase.secrets).map(([field, value]) => [field, value.length >= REDACT_MIN_LENGTH ? REDACTED : value]));
-    return { ...fixtureCase, secrets };
-}
-function ndjson(items) {
-    return items.map((item) => `${JSON.stringify(item)}\n`).join('');
-}
-export function serializeFixture(name, input) {
-    const dir = `${FIXTURES_DIR}/${name}`;
-    if (!FIXTURE_CASE_NAME_PATTERN.test(name))
-        throw new FixtureFormatError(`case name "${name}" must match ${FIXTURE_CASE_NAME_PATTERN.source}`, `${dir}/${CASE_FILE}`);
-    const requests = input.requests ?? [];
-    if (input.case.kind !== 'push' && requests.length > 0)
-        throw new FixtureFormatError(`${REQUESTS_FILE} only belongs to push cases`, `${dir}/${REQUESTS_FILE}`);
-    const files = [{ path: `${dir}/${CASE_FILE}`, content: `${JSON.stringify(redactCaseSecrets(input.case), null, 2)}\n` }];
-    if (input.exchanges.length > 0)
-        files.push({ path: `${dir}/${EXCHANGES_FILE}`, content: ndjson(input.exchanges) });
-    if (input.case.kind === 'push')
-        files.push({ path: `${dir}/${REQUESTS_FILE}`, content: ndjson(requests) });
-    return files;
-}
-export async function readFixtureDir(dir) {
-    const fs = await import('node:fs/promises');
-    const path = await import('node:path');
-    const root = path.join(dir, FIXTURES_DIR);
-    const files = [];
-    const walk = async (absolute, relative) => {
-        const entries = await fs.readdir(absolute, { withFileTypes: true });
-        for (const entry of entries) {
-            if (entry.name.startsWith('.'))
-                continue;
-            const childAbsolute = path.join(absolute, entry.name);
-            const childRelative = `${relative}/${entry.name}`;
-            if (entry.isSymbolicLink())
-                throw new FixtureFormatError('symbolic links are not allowed in fixtures', childRelative);
-            if (entry.isDirectory())
-                await walk(childAbsolute, childRelative);
-            else if (entry.isFile())
-                files.push({ path: childRelative, content: await fs.readFile(childAbsolute, 'utf8') });
-        }
-    };
-    let stat;
-    try {
-        stat = await fs.lstat(root);
-    }
-    catch (error) {
-        if (isRecord(error) && error.code === 'ENOENT')
-            return [];
-        throw error;
-    }
-    if (stat.isSymbolicLink())
-        throw new FixtureFormatError('symbolic links are not allowed in fixtures', FIXTURES_DIR);
-    if (!stat.isDirectory())
-        throw new FixtureFormatError('must be a directory', FIXTURES_DIR);
-    await walk(root, FIXTURES_DIR);
-    return files.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
-}
-export function canonicalRequestUrl(url, query) {
-    let parsed;
-    try {
-        parsed = new URL(url);
-    }
-    catch {
-        return url;
-    }
-    for (const [name, value] of Object.entries(query ?? {})) {
-        for (const item of Array.isArray(value) ? value : [value])
-            parsed.searchParams.append(name, item);
-    }
-    parsed.searchParams.sort();
-    return parsed.toString();
-}
-export function canonicalJson(value) {
-    return JSON.stringify(value, (_key, item) => {
-        if (!isRecord(item))
-            return item;
-        return Object.fromEntries(Object.keys(item).sort().map((key) => [key, item[key]]));
-    });
-}

package/testing/gateway.d.ts

@@ -1,65 +0,0 @@
-import type { LookupAddress } from 'node:dns';
-import { LocalDbError } from './local-db.js';
-import { type LocalHost } from './types.js';
-export interface LocalGatewayOptions {
-    /** 连接器声明 ∩ 本地配置；空数组 = 全部拒绝。 */
-    hosts: string[];
-    rateLimit?: {
-        perSecond: number;
-        concurrency: number;
-    };
-    /** 字段 → 明文，用于句柄替换与遮蔽。 */
-    secrets: Record<string, string>;
-    db?: {
-        host: string;
-        port: number;
-        database: string;
-        user: string;
-        password: string;
-        ssl?: boolean;
-    } | null;
-    /** 测试注入；给了就不走真实网络（不解析主机名、不固定连接地址）。 */
-    fetch?: typeof fetch;
-    /** 测试注入：真实网络下的主机名解析，缺省 `dns.lookup(host, { all: true })`。 */
-    resolve?: (host: string) => Promise<LookupAddress[]>;
-    /** 注入 postgres 驱动（CLI 用）；缺省动态 `import('postgres')`。 */
-    postgres?: unknown;
-    /** 录制器挂这里（Phase 3）。 */
-    onExchange?: (exchange: LocalExchange) => void;
-}
-/** 未遮蔽的原始往来，录制器负责遮蔽。 */
-export interface LocalExchange {
-    kind: 'http' | 'db' | 'crypto';
-    request: unknown;
-    response: unknown;
-    error: {
-        code: string;
-        message: string;
-    } | null;
-    durationMs: number;
-}
-/** 本地网关自己的宿主调用失败（码与平台网关一致）。 */
-export declare class LocalGatewayError extends Error {
-    readonly code: string;
-    readonly status: number | undefined;
-    constructor(code: string, message: string, status?: number);
-}
-/** 去掉 IPv6 方括号与末尾的点，转小写；IPv4 映射的 IPv6（`::ffff:a9fe:a9fe`）还原成 IPv4。 */
-export declare function normalizeHost(host: string): string;
-/**
- * 限流：同时在途不超过 `concurrency`，放行间隔 `1 / perSecond` 秒。
- * 释放槽位时若有人排队，槽位直接交给队头（`active` 不减），避免刚进来的请求在交接间隙抢占同一个槽位。
- */
-export declare class LocalLimiter {
-    private readonly perSecond;
-    private readonly concurrency;
-    private nextAt;
-    private active;
-    private readonly waiting;
-    constructor(perSecond: number, concurrency: number);
-    acquire(): Promise<() => void>;
-}
-export declare function createLocalGateway(options: LocalGatewayOptions): LocalHost & {
-    redact(text: string): string;
-};
-export { LocalDbError };