@deepexi/datasense-connector 3.0.1 → 3.1.0

package/src/testing/engine.ts

@@ -0,0 +1,445 @@
+/**
+ * 本地执行器：同一个 guest 作业交给两种执行方式，宿主调用一律交给传入的 `LocalHost`（本地模拟网关或夹具回放）。
+ *
+ * - `createVmExecutor(source)`：在空环境变量的子进程里执行（Node 用 `child_process.fork`，Bun 下同样 fork 当前运行时），
+ *   子进程只建 `createGuestSession`（与 runner 同一个 guest 会话、同一个 vm 上下文），宿主调用经 IPC 回到父进程；
+ *   超时或取消时杀子进程（ADR-0133 决策 8）。给 CLI 的 `read` / `check` / `test` 与 `runTests`。
+ * - `createInProcessExecutor(module)`：在当前进程里执行已导入的连接器模块（`{ descriptor, implementations }`，与打包产物同形），
+ *   复用同一份 guest 运行时（`ctx` 实现、批次、失败归类都与 vm 内一致），只是不过 vm。给 vitest。
+ *
+ * 两种方式共用同一个消息驱动（`drive`）：父进程侧按 `parseChildMessage` 校验每条消息，`emit` 累积记录与断点并实现 `limit`
+ * （累计记录数达到 limit 时截掉超出部分、回 `{ stop: true }`），其余方法交给 `host.call`。
+ */
+import { fork, type ChildProcess } from 'node:child_process';
+import path from 'node:path';
+import { IS_SOURCE, PACKAGE_ROOT } from '../internal/location';
+import { loadGuestRuntime } from '../runtime/guest-runtime';
+import {
+  parseChildMessage, undeclaredEmitTable, type ChildMessage, type GuestFailureCode, type GuestJob, type GuestLog, type GuestOperationPhase,
+  type GuestOperationResult, type GuestResult, type HostCallError, type ParentMessage,
+} from '../runtime/protocol';
+import { createInProcessGuestSession } from '../runtime/session';
+import type { ConnectorDescriptor } from '../descriptor';
+import type { LocalConnector } from './source-loader';
+import { LocalRunError, type GuestExecutor, type LocalHost, type LocalRunResult } from './types';
+
+/** 执行器的传输层：把父进程消息交给 guest，guest 的消息经 `onMessage` 回来。 */
+interface Transport {
+  send(message: ParentMessage): void;
+  /** 结束执行（杀子进程 / 丢弃进程内会话），等它真正退出。 */
+  stop(): Promise<void>;
+}
+
+type TransportFactory = (onMessage: (raw: unknown) => void, onExit: (description: string) => void) => Transport;
+
+interface DriveInput {
+  execute: Extract<ParentMessage, { type: 'execute' }>;
+  host: LocalHost;
+  limit?: number;
+  timeoutMs?: number;
+  /** vm 子进程先发 `ready` 再收 `execute`；进程内会话直接收 `execute`。 */
+  waitForReady: boolean;
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function emptyResult(): LocalRunResult {
+  return { records: [], removals: [], checkpoints: [], logs: [], rejected: [], stopped: false, error: null };
+}
+
+/** 宿主抛出的错误 → 回给 guest 的宿主调用失败。本地网关的 `HOST_DENIED` 回线上同码 `EGRESS_HOST_DENIED`。 */
+function hostCallError(error: unknown): HostCallError {
+  const code = isRecord(error) && typeof error.code === 'string' ? error.code : 'HOST_CALL_FAILED';
+  const result: HostCallError = { code: code === 'HOST_DENIED' ? 'EGRESS_HOST_DENIED' : code, message: errorMessage(error) };
+  if (isRecord(error) && typeof error.status === 'number' && Number.isInteger(error.status)) result.status = error.status;
+  if (isRecord(error) && typeof error.retryAfterMs === 'number' && error.retryAfterMs >= 0) result.retryAfterMs = error.retryAfterMs;
+  return result;
+}
+
+function decode(raw: unknown): unknown {
+  if (typeof raw !== 'string') return raw;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
+/** 操作结果里的文案与日志过一遍遮蔽（输出是外部系统的业务数据，原样保留）。 */
+function redactOperation(outcome: GuestOperationResult, redact: (text: string) => string): GuestOperationResult {
+  const logs = outcome.logs.map((log) => ({ ...log, message: redact(log.message) }));
+  if (outcome.ok) return { ...outcome, logs };
+  return {
+    ...outcome,
+    message: redact(outcome.message),
+    ...(outcome.external ? { external: { ...outcome.external, message: redact(outcome.external.message) } } : {}),
+    logs,
+  };
+}
+
+/** 一次作业的消息驱动（两种执行器共用）。 */
+function drive(transportFactory: TransportFactory, input: DriveInput): Promise<LocalRunResult> {
+  const { execute, host } = input;
+  const job = execute.job;
+  const redact = (text: string): string => (host.redact ? host.redact(text) : text);
+  const result = emptyResult();
+  const limit = input.limit !== undefined && Number.isFinite(input.limit) && input.limit >= 0 ? Math.floor(input.limit) : null;
+  let seq = 0;
+  let finished = false;
+
+  const addLogs = (logs: readonly GuestLog[]): void => {
+    for (const log of logs) result.logs.push({ at: log.at, level: log.level, message: redact(log.message) });
+  };
+
+  return new Promise<LocalRunResult>((resolve) => {
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    let transport: Transport | null = null;
+
+    const finish = (error: LocalRunResult['error']): void => {
+      if (finished) return;
+      finished = true;
+      if (timer !== undefined) clearTimeout(timer);
+      if (error) result.error = { code: error.code, message: redact(error.message), stack: error.stack === null ? null : redact(error.stack) };
+      const stopping = transport ? transport.stop() : Promise.resolve();
+      void stopping.then(() => resolve(result), () => resolve(result));
+    };
+
+    const send = (message: ParentMessage): void => {
+      if (finished || !transport) return;
+      transport.send(message);
+    };
+
+    const onEmit = (id: number, payload: Extract<ChildMessage, { type: 'call' }>['payload']): void => {
+      const batch = payload as { records: LocalRunResult['records']; removals: LocalRunResult['removals']; checkpoint?: { state: unknown }; logs: GuestLog[] };
+      seq += 1;
+      addLogs(batch.logs);
+      let records = batch.records;
+      let stop = false;
+      if (limit !== null) {
+        const room = Math.max(0, limit - result.records.length);
+        if (records.length >= room) {
+          stop = true;
+          records = records.slice(0, room);
+        }
+      }
+      const truncated = records.length < batch.records.length;
+      result.records.push(...records);
+      result.removals.push(...batch.removals);
+      // 截掉了记录时，这一批的断点跨过了被截掉的记录，不记。
+      if (batch.checkpoint && !truncated) result.checkpoints.push({ seq, state: batch.checkpoint.state });
+      if (stop) result.stopped = true;
+      send({ type: 'reply', id, ok: true, value: { seq, duplicate: false, rejected: 0, ...(stop ? { stop: true } : {}) } });
+    };
+
+    const onCall = (message: Extract<ChildMessage, { type: 'call' }>): void => {
+      switch (message.method) {
+        case 'emit':
+          onEmit(message.id, message.payload);
+          return;
+        default:
+          host.call(message.method, message.payload).then(
+            (value) => send({ type: 'reply', id: message.id, ok: true, value: value ?? null }),
+            (error: unknown) => {
+              if (error instanceof LocalRunError && error.code === 'DB_DRIVER_MISSING') {
+                finish({ code: error.code, message: error.message, stack: null });
+                return;
+              }
+              send({ type: 'reply', id: message.id, ok: false, error: hostCallError(error) });
+            },
+          );
+      }
+    };
+
+    const onResult = (outcome: GuestResult): void => {
+      switch (outcome.kind) {
+        case 'push':
+          for (const record of outcome.records) result.records.push({ table: record.table, row: record.row });
+          for (const removal of outcome.removals) result.removals.push({ table: removal.table, key: removal.key });
+          result.rejected.push(...outcome.rejected.map((item) => ({ requestId: item.requestId, code: item.code, message: redact(item.message) })));
+          break;
+        case 'check':
+          break;
+        case 'suggest':
+          result.suggest = outcome.columns.map((column) => ({ ...column }));
+          break;
+        case 'pull':
+          if (outcome.stopped === true) result.stopped = true;
+          break;
+        case 'operation':
+          result.operation = redactOperation(outcome, redact);
+          break;
+      }
+      addLogs(outcome.logs);
+      finish(null);
+    };
+
+    const onMessage = (raw: unknown): void => {
+      if (finished) return;
+      const decoded = decode(raw);
+      const message = parseChildMessage(decoded, job);
+      if (!message) {
+        // 与 runner 一致：写了没声明的表按 SCHEMA_ERROR，其余按崩溃。
+        const table = undeclaredEmitTable(decoded, job);
+        if (table !== null) {
+          finish({ code: 'SCHEMA_ERROR', message: `connector emitted records for undeclared table ${table}`, stack: null });
+        } else {
+          finish({ code: 'RUNNER_CRASHED', message: 'connector sandbox sent a malformed message', stack: null });
+        }
+        return;
+      }
+      switch (message.type) {
+        case 'ready':
+          send(execute);
+          return;
+        case 'call':
+          onCall(message);
+          return;
+        case 'result':
+          onResult(message.result);
+          return;
+        case 'failure':
+          addLogs(message.logs);
+          finish({ code: message.code, message: message.message, stack: message.stack ?? null });
+          return;
+      }
+    };
+
+    const onExit = (description: string): void => {
+      finish({ code: 'RUNNER_CRASHED', message: `connector sandbox exited without a result (${description})`, stack: null });
+    };
+
+    try {
+      transport = transportFactory(onMessage, onExit);
+    } catch (error) {
+      finish({ code: 'RUNNER_CRASHED', message: `connector sandbox failed to start: ${errorMessage(error)}`, stack: null });
+      return;
+    }
+    if (input.timeoutMs !== undefined && Number.isFinite(input.timeoutMs)) {
+      timer = setTimeout(() => {
+        finish({ code: 'TIMEOUT', message: `connector execution exceeded ${input.timeoutMs} ms`, stack: null });
+      }, Math.max(0, input.timeoutMs));
+    }
+    if (!input.waitForReady) send(execute);
+  });
+}
+
+// ---------------------------------------------------------------------------
+// 进程内
+
+/** 连接器模块里的描述（结构不对时为 undefined，由 guest 加载时报错）。 */
+function descriptorOf(module: unknown): ConnectorDescriptor | undefined {
+  if (!isRecord(module)) return undefined;
+  const descriptor = module.descriptor;
+  return isRecord(descriptor) && typeof descriptor.name === 'string' && typeof descriptor.version === 'string' ? descriptor as unknown as ConnectorDescriptor : undefined;
+}
+
+/** vitest：当前进程直接执行已导入的连接器模块（`{ descriptor, implementations }`），不过 vm（复用 guest 运行时的 `ctx` 实现）。 */
+export function createInProcessExecutor(module: unknown): GuestExecutor {
+  const descriptor = descriptorOf(module);
+  return {
+    ...(descriptor ? { descriptor } : {}),
+    async run(job, host, options = {}) {
+      const guest = await loadGuestRuntime();
+      const expect = { name: descriptor?.name ?? '', version: descriptor?.version ?? '' };
+      const execute: Extract<ParentMessage, { type: 'execute' }> = { type: 'execute', guest, code: '', expect, job };
+      return drive((onMessage) => {
+        let active = true;
+        const session = createInProcessGuestSession((json) => {
+          // guest 的消息在会话的定时器回调里发出；交给驱动前切到下一个微任务，避免在 guest 调用栈里重入。
+          if (active) queueMicrotask(() => onMessage(json));
+        }, module);
+        return {
+          send(message) {
+            if (active) session.handle(JSON.stringify(message));
+          },
+          async stop() {
+            if (!active) return;
+            // 进程内无法强行打断连接器代码：发 cancel 让 guest 尽快停下，之后的消息一律丢弃。
+            session.handle(JSON.stringify({ type: 'cancel' }));
+            active = false;
+          },
+        };
+      }, { execute, host, limit: options.limit, timeoutMs: options.timeoutMs, waitForReady: false });
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// vm 子进程
+
+/** 子进程入口：源码下是 `src/testing/child.ts`（Bun 直接执行），dist 下是构建产出的 `testing/child.cjs`。 */
+export const VM_CHILD_ENTRY: string = path.join(PACKAGE_ROOT, 'testing', IS_SOURCE ? 'child.ts' : 'child.cjs');
+
+export interface VmExecutorOptions {
+  /** guest 运行时源码；缺省 `loadGuestRuntime()`。CLI 传构建时内联的 `guest.cjs` 文本。 */
+  guestRuntime?: string;
+  /** 子进程入口；缺省 `VM_CHILD_ENTRY`（CLI 打成单文件时传自己发布的路径，测试用来换替身）。 */
+  childEntry?: string;
+}
+
+/** `cancel` / 超时后等子进程退出的上限；到时 SIGKILL。 */
+const KILL_WAIT_MS = 5000;
+
+function forkTransport(entry: string): TransportFactory {
+  return (onMessage, onExit) => {
+    const child: ChildProcess = fork(entry, [], {
+      env: {},
+      execArgv: [],
+      // 子进程的 stdout 转到父进程 stderr：连接器或它的依赖往 stdout 写的东西不能混进 CLI 的 NDJSON / JSON 输出。
+      stdio: ['ignore', 'pipe', 'inherit', 'ipc'],
+      serialization: 'json',
+    });
+    child.stdout?.on('data', (chunk: Buffer) => process.stderr.write(chunk));
+    let exited = false;
+    const exitedPromise = new Promise<void>((resolve) => {
+      child.once('exit', (code, signal) => {
+        exited = true;
+        resolve();
+        onExit(signal ? `signal ${signal}` : `exit code ${String(code)}`);
+      });
+    });
+    child.on('message', (raw: unknown) => onMessage(raw));
+    child.on('error', (error) => onExit(`spawn error: ${error.message}`));
+    return {
+      send(message) {
+        if (exited || !child.connected) return;
+        try {
+          child.send(JSON.stringify(message));
+        } catch {
+          // 子进程已退出：由 exit 处理
+        }
+      },
+      async stop() {
+        if (!exited) child.kill('SIGKILL');
+        await Promise.race([exitedPromise, new Promise<void>((resolve) => setTimeout(resolve, KILL_WAIT_MS))]);
+      },
+    };
+  };
+}
+
+/** vm 内执行（CLI 与 `runTests` 用）：空环境变量的子进程 + 与 runner 同一个 guest 会话。 */
+export function createVmExecutor(connector: LocalConnector, options: VmExecutorOptions = {}): GuestExecutor {
+  const entry = options.childEntry ?? VM_CHILD_ENTRY;
+  return {
+    descriptor: connector.descriptor,
+    async run(job, host, runOptions = {}) {
+      const guest = options.guestRuntime ?? await loadGuestRuntime();
+      const execute: Extract<ParentMessage, { type: 'execute' }> = {
+        type: 'execute', guest, code: connector.code, expect: { name: connector.name, version: connector.version }, job,
+      };
+      return drive(forkTransport(entry), { execute, host, limit: runOptions.limit, timeoutMs: runOptions.timeoutMs, waitForReady: true });
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// 进程内便捷包装（vitest 用）
+
+export interface StreamRunInput {
+  stream: string;
+  mode?: 'full' | 'incremental';
+  state?: unknown;
+  streamConfig?: Record<string, unknown>;
+  config: Record<string, unknown>;
+  limit?: number;
+  host: LocalHost;
+}
+
+/** 作业的 `secretFields`：描述里的全部凭证配置项（脚本里拿到句柄，由本地网关替换）。 */
+function secretFieldsOf(descriptor: ConnectorDescriptor | undefined): string[] {
+  if (!descriptor || !isRecord(descriptor.config)) return [];
+  return Object.entries(descriptor.config).filter(([, field]) => isRecord(field) && field.type === 'secret').map(([name]) => name);
+}
+
+/** 拉取一个数据流（表取描述里该数据流的表，与线上相同）。`module` 是 `{ descriptor, implementations }`。 */
+export function runStream(module: unknown, input: StreamRunInput): Promise<LocalRunResult> {
+  const descriptor = descriptorOf(module);
+  const stream = descriptor && Object.hasOwn(descriptor.streams, input.stream) ? descriptor.streams[input.stream] : undefined;
+  const job: GuestJob = {
+    kind: 'pull',
+    config: input.config,
+    secretFields: secretFieldsOf(descriptor),
+    stream: input.stream,
+    mode: input.mode ?? 'full',
+    state: input.state ?? null,
+    streamConfig: input.streamConfig ?? {},
+    tables: stream && isRecord(stream.tables) ? Object.keys(stream.tables) : [],
+  };
+  return createInProcessExecutor(module).run(job, input.host, input.limit !== undefined ? { limit: input.limit } : {});
+}
+
+export function runCheck(module: unknown, input: { config: Record<string, unknown>; host: LocalHost }): Promise<LocalRunResult> {
+  return createInProcessExecutor(module).run({ kind: 'check', config: input.config, secretFields: secretFieldsOf(descriptorOf(module)) }, input.host);
+}
+
+/** 跑一次建议列（`suggestColumns`），结果在 `LocalRunResult.suggest`。 */
+export function runSuggestColumns(module: unknown, input: { stream: string; table: string; config: Record<string, unknown>; host: LocalHost }): Promise<LocalRunResult> {
+  return createInProcessExecutor(module).run({
+    kind: 'suggest', config: input.config, secretFields: secretFieldsOf(descriptorOf(module)), stream: input.stream, table: input.table,
+  }, input.host);
+}
+
+export interface OperationRunInput {
+  config: Record<string, unknown>;
+  operation: string;
+  params: Record<string, unknown>;
+  idempotencyKey: string;
+  /** 缺省 `run`。 */
+  phase?: GuestOperationPhase;
+  /** 宿主调用（本地模拟网关或夹具回放）。 */
+  host: LocalHost;
+  /** 作业的凭证字段；缺省取执行器描述里的全部凭证配置项。 */
+  secretFields?: string[];
+  timeoutMs?: number;
+}
+
+/** 执行器层面的失败码 → 操作结果码：超时按 `CANCELED`（与 runner 超过截止时间杀进程一致），其余本地码按 `FATAL_ERROR`。 */
+function operationFailureCode(code: string): Extract<GuestOperationResult, { ok: false }>['code'] {
+  if (code === 'TIMEOUT') return 'CANCELED';
+  const known: readonly string[] = [
+    'CONNECTOR_LOAD_FAILED', 'CONNECTOR_LOOKUP_REQUIRED', 'CONFIG_INVALID', 'SCHEMA_ERROR', 'FATAL_ERROR', 'AUTH_ERROR', 'TRANSIENT_ERROR', 'PUSH_UNKNOWN',
+    'SUGGEST_INVALID', 'CANCELED',
+  ] satisfies readonly GuestFailureCode[];
+  return known.includes(code) ? code as GuestFailureCode : 'FATAL_ERROR';
+}
+
+/**
+ * 执行一次连接器操作（`run` / `lookup` / `simulate`），返回 `GuestOperationResult`。宿主调用交给 `input.host`
+ * （本地模拟网关时请求经网关、可挂录制器）。加载失败、超时等执行器层面的失败也折成 `ok: false` 的结果。
+ */
+export async function runOperation(executor: GuestExecutor, input: OperationRunInput): Promise<GuestOperationResult> {
+  const phase = input.phase ?? 'run';
+  const secretFields = input.secretFields ?? (phase === 'simulate' ? [] : secretFieldsOf(executor.descriptor));
+  const job: GuestJob = {
+    kind: 'operation', phase, config: input.config, secretFields,
+    operation: input.operation, params: input.params, idempotencyKey: input.idempotencyKey,
+  };
+  const result = await executor.run(job, input.host, input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {});
+  if (result.operation) return result.operation;
+  const error = result.error ?? { code: 'FATAL_ERROR', message: 'the connector returned no operation result' };
+  return { kind: 'operation', phase, ok: false, code: operationFailureCode(error.code), message: error.message, logs: result.logs.map((log) => ({ ...log })) };
+}
+
+export function runPush(
+  module: unknown,
+  input: {
+    push: string;
+    requests: Array<{ requestId: string; receivedAt: string; headers: Record<string, string>; body: string }>;
+    config: Record<string, unknown>;
+    host: LocalHost;
+  },
+): Promise<LocalRunResult> {
+  const job: GuestJob = {
+    kind: 'push',
+    config: input.config,
+    secretFields: secretFieldsOf(descriptorOf(module)),
+    requests: input.requests.map((request) => ({ ...request, push: input.push })),
+  };
+  return createInProcessExecutor(module).run(job, input.host);
+}