@deepexi/datasense-connector 3.0.1 → 3.1.0

package/src/runtime/guest/push.ts

@@ -0,0 +1,92 @@
+/**
+ * 推送作业：对每条请求按推送名找处理器，先 `verify` 再 `map`。行为与第 1 期一致：
+ * - `ctx.record` / `ctx.remove` 直接记录，请求成功后才并入结果；请求结束后再用该 `ctx` 抛错；
+ * - 单条请求抛错只拒收该条（code 取错误的 `code`，普通异常为 `CONNECTOR_ERROR`），已产生的记录丢弃；
+ * - `SCHEMA_ERROR` / `FATAL_ERROR` 让整次执行失败；
+ * - 推送名在描述里不存在的请求拒收，code `PUSH_UNKNOWN`。
+ *
+ * 记录里不属于该推送的表或未声明的字段照常交给宿主，由宿主（connector）丢弃并告警。
+ */
+import type { PushContext, PushRequest } from '../../types';
+import type { GuestJob, GuestPushResult } from '../protocol';
+import { jsonParse, jsonStringify } from './bridge';
+import { deepFreeze } from './ctx';
+import { connectorStack, errorInfo, failure, type FailureMessage } from './failure';
+import type { LoadedConnector } from './loader';
+import { createLogBuffer } from './log';
+import { isoNow } from './time';
+
+type PushJob = Extract<GuestJob, { kind: 'push' }>;
+
+const WHOLE_RUN_FAILURES: readonly string[] = ['SCHEMA_ERROR', 'FATAL_ERROR'];
+
+/** 把连接器给的行复制成纯 JSON 数据；不可序列化时抛错，按该请求的连接器异常处理。 */
+function copyRecord(value: unknown, what: string): Record<string, unknown> {
+  if (value === null || typeof value !== 'object' || Array.isArray(value)) throw new TypeError(`${what} must be an object`);
+  const text = jsonStringify(value);
+  if (typeof text !== 'string') throw new TypeError(`${what} is not serializable`);
+  return jsonParse(text) as Record<string, unknown>;
+}
+
+export async function runPush(connector: LoadedConnector, job: PushJob): Promise<{ type: 'result'; result: GuestPushResult } | FailureMessage> {
+  const handlers = connector.pushes;
+  if (Object.keys(handlers).length === 0) return failure('CONNECTOR_LOAD_FAILED', 'connector does not support push', []);
+  const config = deepFreeze(job.config);
+  const logs = createLogBuffer();
+  const result: GuestPushResult = { kind: 'push', records: [], removals: [], rejected: [], logs: [] };
+
+  for (const request of job.requests) {
+    const requestId = request.requestId;
+    const handler = Object.hasOwn(handlers, request.push) ? handlers[request.push] : undefined;
+    if (!handler) {
+      result.rejected.push({ requestId, code: 'PUSH_UNKNOWN', message: `connector has no push named ${request.push}` });
+      continue;
+    }
+    const pending: { records: GuestPushResult['records']; removals: GuestPushResult['removals']; open: boolean } = { records: [], removals: [], open: true };
+    const assertOpen = () => {
+      if (!pending.open) throw new TypeError('ctx is no longer usable after the request finished');
+    };
+    const ctx: PushContext = Object.freeze({
+      config,
+      record(table: string, row: Record<string, unknown>) {
+        assertOpen();
+        if (typeof table !== 'string') throw new TypeError('table must be a string');
+        pending.records.push({ requestId, table, row: copyRecord(row, 'row') });
+      },
+      remove(table: string, key: Record<string, unknown>) {
+        assertOpen();
+        if (typeof table !== 'string') throw new TypeError('table must be a string');
+        pending.removals.push({ requestId, table, key: copyRecord(key, 'key') });
+      },
+      log: logs.logger(requestId, () => pending.open),
+      time: Object.freeze({ now: isoNow }),
+    });
+    const body = request.body;
+    const req: PushRequest = Object.freeze({
+      id: requestId,
+      receivedAt: request.receivedAt,
+      headers: deepFreeze({ ...request.headers }),
+      body,
+      json<T = unknown>(): T {
+        return jsonParse(body) as T;
+      },
+    });
+
+    try {
+      if (typeof handler.verify === 'function') await Reflect.apply(handler.verify, handler, [ctx, req]);
+      await Reflect.apply(handler.map, handler, [ctx, req]);
+      pending.open = false;
+      result.records.push(...pending.records);
+      result.removals.push(...pending.removals);
+    } catch (error) {
+      pending.open = false;
+      const info = errorInfo(error);
+      if (info.code !== null && WHOLE_RUN_FAILURES.includes(info.code)) {
+        return failure(info.code as 'SCHEMA_ERROR' | 'FATAL_ERROR', info.message, logs.drain(), connectorStack(info.stack));
+      }
+      result.rejected.push({ requestId, code: info.code ?? 'CONNECTOR_ERROR', message: info.message });
+    }
+  }
+  result.logs = logs.drain();
+  return { type: 'result', result };
+}

package/src/runtime/guest/signal.ts

@@ -0,0 +1,14 @@
+/** `ctx.signal`：父进程发 `cancel` 时置位并调用 `onAbort` 回调。 */
+import type { ConnectorSignal } from '../../types';
+import { isAborted, onAbort } from './bridge';
+
+export function createSignal(): ConnectorSignal {
+  const signal = {
+    onAbort(fn: () => void): void {
+      if (typeof fn !== 'function') throw new TypeError('ctx.signal.onAbort expects a function');
+      onAbort(fn);
+    },
+  };
+  Object.defineProperty(signal, 'aborted', { get: isAborted, enumerable: true });
+  return Object.freeze(signal) as ConnectorSignal;
+}

package/src/runtime/guest/suggest.ts

@@ -0,0 +1,51 @@
+/**
+ * 建议列作业：调数据流的 `suggestColumns(ctx, { table })`，从外部系统读出可扩展表的可用字段。
+ * - 数据流没有实现建议列、或 `table` 不是该数据流的可扩展表：`CONNECTOR_LOAD_FAILED`；
+ * - 返回值复制成纯 JSON 后经 `validateSuggestedColumns` 检查，不合法时 `SUGGEST_INVALID`；
+ * - 与该表基础列同名的建议去掉（记 warn 日志），其余原样作为结果。
+ */
+import { validateSuggestedColumns } from '../../validate';
+import type { GuestJob, GuestResult } from '../protocol';
+import { jsonParse, jsonStringify, throwIfAborted } from './bridge';
+import { createCheckContext, type ContextOptions } from './ctx';
+import { failure, failureFromError, type FailureMessage } from './failure';
+import type { LoadedConnector } from './loader';
+
+type SuggestJob = Extract<GuestJob, { kind: 'suggest' }>;
+
+function plain(value: unknown): unknown {
+  let text: unknown;
+  try {
+    text = jsonStringify(value);
+  } catch {
+    return undefined;
+  }
+  return typeof text === 'string' ? jsonParse(text) : undefined;
+}
+
+export async function runSuggest(connector: LoadedConnector, job: SuggestJob, options: ContextOptions): Promise<{ type: 'result'; result: GuestResult } | FailureMessage> {
+  const suggest = Object.hasOwn(connector.suggestColumns, job.stream) ? connector.suggestColumns[job.stream] : undefined;
+  if (!suggest) return failure('CONNECTOR_LOAD_FAILED', `stream ${job.stream} does not implement suggestColumns`, options.logs.drain());
+  const stream = connector.descriptor.streams[job.stream];
+  const table = stream && Object.hasOwn(stream.tables, job.table) ? stream.tables[job.table] : undefined;
+  if (!table || table.extensible !== true) {
+    return failure('CONNECTOR_LOAD_FAILED', `table ${job.table} is not an extensible table of stream ${job.stream}`, options.logs.drain());
+  }
+  const ctx = createCheckContext(options);
+  let value: unknown;
+  try {
+    value = plain(await Reflect.apply(suggest, undefined, [ctx, Object.freeze({ table: job.table })]));
+    throwIfAborted();
+  } catch (error) {
+    return failureFromError(error, options.logs.drain());
+  }
+  const checked = validateSuggestedColumns(value);
+  if (!checked.ok) {
+    const issues = checked.issues.map((issue) => `${issue.path || '(root)'}: ${issue.message}`).join('; ');
+    return failure('SUGGEST_INVALID', `suggestColumns returned invalid columns: ${issues}`, options.logs.drain());
+  }
+  const base = checked.columns.filter((column) => Object.hasOwn(table.columns, column.name)).map((column) => column.name);
+  if (base.length > 0) options.logs.logger().warn(`suggestColumns returned base column(s) of table ${job.table}, ignored: ${base.join(', ')}`);
+  const columns = checked.columns.filter((column) => !Object.hasOwn(table.columns, column.name));
+  return { type: 'result', result: { kind: 'suggest', columns, logs: options.logs.drain() } };
+}

package/src/runtime/guest/time.ts

@@ -0,0 +1,137 @@
+/** `ctx.time`：`now()` 与 `toUtc(本地时间文本, 时区, 格式?)`。时区换算用上下文自己的 `Intl.DateTimeFormat`。 */
+import type { ConnectorTime } from '../../types';
+
+const DateCtor = Date;
+const dateUtc = Date.UTC;
+const dateParse = Date.parse;
+const reflectApply = Reflect.apply;
+const toISOString = Date.prototype.toISOString;
+const DateTimeFormat = Intl.DateTimeFormat;
+
+export function isoNow(): string {
+  return reflectApply(toISOString, new DateCtor(), []) as string;
+}
+
+function iso(ms: number): string {
+  return reflectApply(toISOString, new DateCtor(ms), []) as string;
+}
+
+const DEFAULT_PATTERN = /^(\d{4})[-/](\d{1,2})[-/](\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?)?$/;
+const ZONED_PATTERN = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})$/i;
+
+interface WallClock { year: number; month: number; day: number; hour: number; minute: number; second: number; millisecond: number }
+
+type Field = keyof WallClock;
+const TOKENS: ReadonlyArray<{ token: string; field: Field; pattern: string }> = [
+  { token: 'YYYY', field: 'year', pattern: '(\\d{4})' },
+  { token: 'SSS', field: 'millisecond', pattern: '(\\d{1,3})' },
+  { token: 'MM', field: 'month', pattern: '(\\d{2})' },
+  { token: 'DD', field: 'day', pattern: '(\\d{2})' },
+  { token: 'HH', field: 'hour', pattern: '(\\d{2})' },
+  { token: 'mm', field: 'minute', pattern: '(\\d{2})' },
+  { token: 'ss', field: 'second', pattern: '(\\d{2})' },
+  { token: 'M', field: 'month', pattern: '(\\d{1,2})' },
+  { token: 'D', field: 'day', pattern: '(\\d{1,2})' },
+  { token: 'H', field: 'hour', pattern: '(\\d{1,2})' },
+];
+
+function escapeRegExp(text: string): string {
+  return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+}
+
+function parseWithFormat(local: string, format: string): WallClock | null {
+  const fields: Field[] = [];
+  let source = '';
+  let i = 0;
+  while (i < format.length) {
+    const match = TOKENS.find((entry) => format.startsWith(entry.token, i));
+    if (match) {
+      fields.push(match.field);
+      source += match.pattern;
+      i += match.token.length;
+    } else {
+      source += escapeRegExp(format.charAt(i));
+      i += 1;
+    }
+  }
+  const result = new RegExp(`^${source}$`).exec(local);
+  if (!result || !fields.includes('year') || !fields.includes('month') || !fields.includes('day')) return null;
+  const clock: WallClock = { year: 0, month: 0, day: 0, hour: 0, minute: 0, second: 0, millisecond: 0 };
+  fields.forEach((field, index) => {
+    const text = result[index + 1] ?? '0';
+    clock[field] = field === 'millisecond' ? Number(text.padEnd(3, '0')) : Number(text);
+  });
+  return clock;
+}
+
+function parseDefault(local: string): WallClock | null {
+  const match = DEFAULT_PATTERN.exec(local);
+  if (!match) return null;
+  return {
+    year: Number(match[1]), month: Number(match[2]), day: Number(match[3]),
+    hour: Number(match[4] ?? 0), minute: Number(match[5] ?? 0), second: Number(match[6] ?? 0),
+    millisecond: Number((match[7] ?? '0').slice(0, 3).padEnd(3, '0')),
+  };
+}
+
+/** 墙上时间按 UTC 解释的毫秒数；字段越界（如 2 月 30 日）返回 null。 */
+function wallMillis(clock: WallClock): number | null {
+  const ms = dateUtc(clock.year, clock.month - 1, clock.day, clock.hour, clock.minute, clock.second, clock.millisecond);
+  if (Number.isNaN(ms)) return null;
+  const check = new DateCtor(ms);
+  if (check.getUTCFullYear() !== clock.year || check.getUTCMonth() !== clock.month - 1 || check.getUTCDate() !== clock.day
+    || check.getUTCHours() !== clock.hour || check.getUTCMinutes() !== clock.minute || check.getUTCSeconds() !== clock.second) return null;
+  return ms;
+}
+
+const formatters = new Map<string, Intl.DateTimeFormat>();
+
+function formatter(timeZone: string): Intl.DateTimeFormat {
+  let cached = formatters.get(timeZone);
+  if (!cached) {
+    try {
+      cached = new DateTimeFormat('en-US', {
+        timeZone, hourCycle: 'h23', year: 'numeric', month: 'numeric', day: 'numeric', hour: 'numeric', minute: 'numeric', second: 'numeric',
+      });
+    } catch {
+      throw new TypeError(`unknown time zone ${timeZone}`);
+    }
+    formatters.set(timeZone, cached);
+  }
+  return cached;
+}
+
+/** 时区在 `utcMs` 时刻相对 UTC 的偏移（毫秒）。 */
+function zoneOffset(timeZone: string, utcMs: number): number {
+  const base = utcMs - (((utcMs % 1000) + 1000) % 1000);
+  const parts: Record<string, number> = {};
+  for (const part of formatter(timeZone).formatToParts(new DateCtor(base))) {
+    if (part.type !== 'literal') parts[part.type] = Number(part.value);
+  }
+  const asUtc = dateUtc(parts.year ?? 0, (parts.month ?? 1) - 1, parts.day ?? 1, parts.hour ?? 0, parts.minute ?? 0, parts.second ?? 0);
+  return asUtc - base;
+}
+
+export function toUtc(local: string, timeZone: string, format?: string): string {
+  if (typeof local !== 'string') throw new TypeError('ctx.time.toUtc expects a date-time string');
+  if (typeof timeZone !== 'string' || timeZone.length === 0) throw new TypeError('ctx.time.toUtc expects a time zone like Asia/Shanghai');
+  const text = local.trim();
+  if (format === undefined && ZONED_PATTERN.test(text)) {
+    // 文本自带时区偏移：直接按它解析，忽略 timeZone。
+    const ms = dateParse(text.replace(' ', 'T'));
+    if (Number.isNaN(ms)) throw new TypeError(`invalid date-time ${local}`);
+    return iso(ms);
+  }
+  const clock = format === undefined ? parseDefault(text) : parseWithFormat(text, format);
+  const wall = clock ? wallMillis(clock) : null;
+  if (wall === null) throw new TypeError(`invalid date-time ${local}${format === undefined ? '' : ` for format ${format}`}`);
+  const first = zoneOffset(timeZone, wall);
+  let utc = wall - first;
+  const second = zoneOffset(timeZone, utc);
+  if (second !== first) utc = wall - second;
+  return iso(utc);
+}
+
+export function createTime(): ConnectorTime {
+  return Object.freeze({ now: isoNow, toUtc });
+}

package/src/runtime/guest-runtime.ts

@@ -0,0 +1,75 @@
+/**
+ * guest 运行时的位置与源码串（宿主侧）。
+ *
+ * - `GUEST_ENTRY_PATH`：源码下指向 `src/runtime/guest/index.ts`（runner 用 `bundleTrustedEntry` 自己打包）；
+ *   dist 下指向编译后的 `runtime/guest/index.js`，它与预打包的 `guest.cjs` 出自同一份源码。
+ * - `loadGuestRuntime()`：dist 下读 `guest.cjs`（构建时打好，Node 可用）；源码下用 `Bun.build` 打包，参数与 shared 的
+ *   `bundleTrustedEntry` 一致（cjs、browser、SDK 内联、无外部依赖、`macros: false`）。结果进程内缓存，失败不缓存。
+ */
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+import { IS_SOURCE, moduleFile, PACKAGE_ROOT } from '../internal/location';
+
+export const GUEST_ENTRY_PATH: string = moduleFile('runtime/guest/index');
+
+/** dist 根目录下预打包的 guest 运行时文件名。 */
+export const GUEST_BUNDLE_FILE = 'guest.cjs';
+
+type BunBuild = typeof Bun.build;
+type BunBuildConfig = Parameters<BunBuild>[0];
+
+const CODE_FILE = /\.[cm]?[jt]sx?$/;
+
+/**
+ * Bun 1.3 起：当前进程已 import 过的源文件再交给 `Bun.build` 会报「Unexpected reading file」，经插件自己读文件绕开
+ * （与 shared `connector-bundle.ts` 的 `readSourcePlugin` 相同）。
+ */
+function readSourcePlugin(bun: typeof Bun): NonNullable<BunBuildConfig['plugins']>[number] {
+  return {
+    name: 'datasense-connector-read-source',
+    setup(build) {
+      build.onLoad({ filter: CODE_FILE }, async (args) => {
+        const extension = args.path.slice(args.path.lastIndexOf('.') + 1);
+        const loader = extension === 'tsx' ? 'tsx' : extension === 'jsx' ? 'jsx' : /^[cm]?js$/.test(extension) ? 'js' : 'ts';
+        return { contents: await bun.file(args.path).text(), loader };
+      });
+    },
+  };
+}
+
+/** `Bun.build` 的 `macros: false`（bun-types 未声明）；与 shared 的 `MACROS_OFF` 相同。 */
+const MACROS_OFF = { macros: false } as unknown as Partial<BunBuildConfig>;
+
+/** 用 Bun 把 guest 入口打成 CJS 字符串（SDK 内联）。构建脚本与源码下的 `loadGuestRuntime` 共用。 */
+export async function bundleGuestWithBun(entry: string = GUEST_ENTRY_PATH): Promise<string> {
+  const bun = (globalThis as { Bun?: typeof Bun }).Bun;
+  if (!bun) throw new Error('the guest runtime can only be bundled from source under Bun; use the built package (dist) under Node.js');
+  const result = await bun.build({
+    entrypoints: [entry],
+    format: 'cjs',
+    target: 'browser',
+    minify: false,
+    external: [],
+    plugins: [readSourcePlugin(bun)],
+    throw: false,
+    ...MACROS_OFF,
+  } as BunBuildConfig);
+  const output = result.outputs.find((item) => item.kind === 'entry-point');
+  if (!result.success || !output) {
+    throw new Error(`guest runtime bundle failed: ${result.logs.map((log) => log.message).join('; ') || 'no output'}`);
+  }
+  return output.text();
+}
+
+let cached: Promise<string> | null = null;
+
+/** 可放进 `execute.guest` 的 guest 运行时 CJS 字符串（进程内缓存）。 */
+export function loadGuestRuntime(): Promise<string> {
+  if (cached) return cached;
+  const pending = IS_SOURCE ? bundleGuestWithBun() : readFile(path.join(PACKAGE_ROOT, GUEST_BUNDLE_FILE), 'utf8');
+  cached = pending;
+  pending.catch(() => {
+    if (cached === pending) cached = null;
+  });
+  return pending;
+}

package/src/runtime/index.ts

@@ -0,0 +1,53 @@
+/**
+ * `@deepexi/datasense-connector/runtime`：宿主侧入口（runner 的执行子进程与父进程、`./testing` 用），不在 vm 里加载。
+ *
+ * - `GUEST_ENTRY_PATH`：guest 运行时入口（源码下是 `src/runtime/guest/index.ts`，dist 下是编译后的 `runtime/guest/index.js`），
+ *   runner 用 `bundleTrustedEntry` 打成 CJS 字符串，放进 `execute.guest`；
+ * - `loadGuestRuntime`：直接拿到可放进 `execute.guest` 的 CJS 字符串：dist 下读预打包的 `guest.cjs`，源码下用 `Bun.build` 打包（进程内缓存）；
+ * - `createGuestSession`：子进程里建 vm 上下文、加载 guest 与连接器代码，之后只转发字符串消息；
+ * - `parseParentMessage` / `parseChildMessage`：两侧解析对方消息，结构不合法返回 null；`undeclaredEmitTable` 区分
+ *   「`emit` 写了没声明的表」（宿主按 `SCHEMA_ERROR` 结束）与其他不合法消息。
+ *
+ * 连接器打包产物导出 `{ descriptor, implementations }`（虚拟入口契约）。guest 加载后先核对 `expect` 的名字与版本、
+ * 实现与描述是否一一对应，再执行作业；宿主需要的信息（出站主机、生效限流、该数据流的表）由宿主从描述派生，guest 不上报。
+ *
+ * 连接器操作作业（`kind: 'operation'`）的结果总是 `GuestOperationResult`（失败也走 `result`），只有加载失败走 `failure`；
+ * `simulate` 阶段不允许任何宿主调用，其余阶段不允许 `db.*` / `emit`。`suggest` 作业不允许 `emit`。
+ * `emit` 的 payload 是 `GuestPullBatch`（不带 `seq`，由宿主编号），回复 `EmitReply`（`{ stop: true }` 时提前结束）。
+ */
+export { GUEST_ENTRY_PATH, loadGuestRuntime } from './guest-runtime';
+export { createGuestSession } from './session';
+export { GUEST_FAILURE_CODES, GUEST_OPERATION_FAILURE_CODES, HOST_METHODS, parseChildMessage, parseParentMessage, undeclaredEmitTable } from './protocol';
+export type {
+  CacheEntry,
+  CacheGetRequest,
+  CacheSetRequest,
+  ChildMessage,
+  CryptoRequest,
+  CryptoResponse,
+  DbCursorRequest,
+  DbNextResponse,
+  DbQueryRequest,
+  DbQueryResponse,
+  EgressHttpRequest,
+  EgressHttpResponse,
+  EmitReply,
+  GuestFailureCode,
+  GuestJob,
+  GuestLog,
+  GuestOperationFailureCode,
+  GuestOperationJob,
+  GuestOperationPhase,
+  GuestOperationResult,
+  GuestPullBatch,
+  GuestPushRequest,
+  GuestPushResult,
+  GuestResult,
+  HostCallError,
+  HostMethod,
+  JobRateLimit,
+  LookupAnswer,
+  ParentMessage,
+  SecretRevealRequest,
+  SecretRevealResponse,
+} from './protocol';