@deepexi/datasense-connector 3.0.1 → 3.1.0

package/src/testing/gateway.ts

@@ -0,0 +1,482 @@
+/**
+ * 本地模拟网关：在作者机器上应答 guest 的宿主调用，语义照抄平台出站网关（connector `src/egress/`）：
+ *
+ * - 主机求交：请求主机必须匹配 `hosts` 里的一项（精确主机名，或 `*.后缀` 匹配一级或多级子域）；调用方传入的 `hosts`
+ *   已是「连接器声明 ∩ 本地配置」，空数组即全部拒绝。不匹配时 `LocalRunError('HOST_DENIED')`，回给连接器的码与线上相同
+ *   （`EGRESS_HOST_DENIED`）。本地开发要连本机，所以不做私网拦截，但 `169.254.169.254`、`100.100.100.200`、`fd00:ec2::254`
+ *   这三个云元数据地址仍拦截（含 IPv4 映射写法）。用真实网络时每一跳（含重定向）都先解析主机名，解析到它们同样拦截，
+ *   连接只连这次检查过的地址（`pinned-http.ts`），不给 DNS 重绑定留第二次解析的窗口。
+ * - 凭证句柄 `ontos:secret:<字段>` 在 URL（含 URL 编码写法）、请求头、utf8 body 与 `crypto` 的 key / data 里替换成明文；
+ *   回给连接器的响应 url、头与 body 里出现的明文（含 URL 编码与表单编码写法，见 `secretRedactionTexts`）替换为 `***`。`secret.reveal` 直接回明文（只有 `exposeToScript` 字段会来取）。
+ * - 限流：`rateLimit` 给出时按 `1 / perSecond` 秒的间隔逐个放行（不攒额度），同时在途不超过 `concurrency`；
+ *   超出的请求排队等待，不回 429（本地没有并发的其他运行）。
+ * - 超时缺省 30 秒、上限 120 秒（`EGRESS_TIMEOUT`）；响应超过 10 MiB `EGRESS_RESPONSE_TOO_LARGE`；同主机重定向最多跟 5 次。
+ * - `ctx.db` 连 `db` 配置的库（见 `local-db.ts`），`ctx.cache` 存在进程内存里。
+ *
+ * 每次 http / db / crypto 调用都经 `onExchange` 交出**未遮蔽**的往来（请求是句柄形态，响应是上游原文），录制器负责遮蔽。
+ */
+import { createHash, createHmac, sign as rsaSign } from 'node:crypto';
+import type { LookupAddress } from 'node:dns';
+import { lookup } from 'node:dns/promises';
+import type {
+  CacheEntry, CryptoRequest, DbQueryRequest, EgressHttpRequest, EgressHttpResponse,
+} from '../runtime/protocol';
+import { createLocalDb, LocalDbError, type LocalDb } from './local-db';
+import { PinnedBodyTooLargeError, pinnedRequest } from './pinned-http';
+import { redactSecretValues, secretRedactionTexts } from './redaction';
+import { LocalRunError, type HostMethod, type LocalHost } from './types';
+
+export interface LocalGatewayOptions {
+  /** 连接器声明 ∩ 本地配置；空数组 = 全部拒绝。 */
+  hosts: string[];
+  rateLimit?: { perSecond: number; concurrency: number };
+  /** 字段 → 明文，用于句柄替换与遮蔽。 */
+  secrets: Record<string, string>;
+  db?: { host: string; port: number; database: string; user: string; password: string; ssl?: boolean } | null;
+  /** 测试注入；给了就不走真实网络（不解析主机名、不固定连接地址）。 */
+  fetch?: typeof fetch;
+  /** 测试注入：真实网络下的主机名解析，缺省 `dns.lookup(host, { all: true })`。 */
+  resolve?: (host: string) => Promise<LookupAddress[]>;
+  /** 注入 postgres 驱动（CLI 用）；缺省动态 `import('postgres')`。 */
+  postgres?: unknown;
+  /** 录制器挂这里（Phase 3）。 */
+  onExchange?: (exchange: LocalExchange) => void;
+}
+
+/** 未遮蔽的原始往来，录制器负责遮蔽。 */
+export interface LocalExchange {
+  kind: 'http' | 'db' | 'crypto';
+  request: unknown;
+  response: unknown;
+  error: { code: string; message: string } | null;
+  durationMs: number;
+}
+
+/** 本地网关自己的宿主调用失败（码与平台网关一致）。 */
+export class LocalGatewayError extends Error {
+  readonly code: string;
+  readonly status: number | undefined;
+
+  constructor(code: string, message: string, status?: number) {
+    super(message);
+    this.name = 'LocalGatewayError';
+    this.code = code;
+    this.status = status;
+  }
+}
+
+/** 与 shared `connector-limits.ts` 同值。 */
+const DEFAULT_TIMEOUT_MS = 30_000;
+const MAX_TIMEOUT_MS = 120_000;
+const RESPONSE_MAX_BYTES = 10 * 1024 * 1024;
+const MAX_REDIRECTS = 5;
+const REDIRECT_STATUSES: ReadonlySet<number> = new Set([301, 302, 303, 307, 308]);
+/** 发请求前去掉的头（由 fetch 自己管）。 */
+const DROPPED_REQUEST_HEADERS: ReadonlySet<string> = new Set(['host', 'content-length', 'connection', 'transfer-encoding', 'keep-alive', 'upgrade']);
+
+/** 云元数据地址：本地网关也拦截。 */
+const METADATA_ADDRESSES: ReadonlySet<string> = new Set(['169.254.169.254', '100.100.100.200', 'fd00:ec2::254']);
+
+const SECRET_FIELD = '[A-Za-z_][A-Za-z0-9_]*';
+const PLAIN_HANDLE = new RegExp(`ontos:secret:(${SECRET_FIELD})`, 'g');
+const URL_HANDLE = new RegExp(`ontos(?::|%3[Aa])secret(?::|%3[Aa])(${SECRET_FIELD})`, 'g');
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+// ---------------------------------------------------------------------------
+// 主机
+
+/** 去掉 IPv6 方括号与末尾的点，转小写；IPv4 映射的 IPv6（`::ffff:a9fe:a9fe`）还原成 IPv4。 */
+export function normalizeHost(host: string): string {
+  const value = host.trim().replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase();
+  const mapped = /^::ffff:(?:0:)?([0-9a-f]{1,4}):([0-9a-f]{1,4})$/.exec(value);
+  if (mapped) {
+    const high = Number.parseInt(mapped[1] ?? '0', 16);
+    const low = Number.parseInt(mapped[2] ?? '0', 16);
+    return `${high >> 8}.${high & 255}.${low >> 8}.${low & 255}`;
+  }
+  const dotted = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/.exec(value);
+  if (dotted?.[1]) return dotted[1];
+  return value;
+}
+
+/** 单项匹配：精确主机名，或 `*.后缀` 匹配后缀下一级或多级子域（不含后缀本身）。与 connector `hosts.ts` 相同。 */
+function matchHostPattern(host: string, pattern: string): boolean {
+  const normalized = normalizeHost(pattern);
+  if (normalized.startsWith('*.')) {
+    const suffix = normalized.slice(1);
+    return suffix.length > 1 && host.length > suffix.length && host.endsWith(suffix);
+  }
+  return normalized !== '' && host === normalized;
+}
+
+function isMetadataAddress(host: string): boolean {
+  return METADATA_ADDRESSES.has(normalizeHost(host));
+}
+
+function isIpLiteral(host: string): boolean {
+  return /^\d+\.\d+\.\d+\.\d+$/.test(host) || host.includes(':');
+}
+
+// ---------------------------------------------------------------------------
+// 句柄
+
+function secretUnavailable(field: string): LocalGatewayError {
+  return new LocalGatewayError('EGRESS_SECRET_UNAVAILABLE', `Secret '${field}' is unavailable: it isn't configured locally`, 422);
+}
+
+function lookupSecret(values: Record<string, string>, field: string): string {
+  const value = Object.hasOwn(values, field) ? values[field] : undefined;
+  if (typeof value !== 'string') throw secretUnavailable(field);
+  return value;
+}
+
+function substituteHandles(text: string, values: Record<string, string>): string {
+  for (const match of text.matchAll(PLAIN_HANDLE)) lookupSecret(values, match[1] ?? '');
+  return text.replace(PLAIN_HANDLE, (_match, field: string) => lookupSecret(values, field));
+}
+
+function substituteUrlHandles(text: string, values: Record<string, string>): string {
+  for (const match of text.matchAll(URL_HANDLE)) lookupSecret(values, match[1] ?? '');
+  return text.replace(URL_HANDLE, (_match, field: string) => encodeURIComponent(lookupSecret(values, field)));
+}
+
+// ---------------------------------------------------------------------------
+// 限流
+
+/**
+ * 限流：同时在途不超过 `concurrency`，放行间隔 `1 / perSecond` 秒。
+ * 释放槽位时若有人排队，槽位直接交给队头（`active` 不减），避免刚进来的请求在交接间隙抢占同一个槽位。
+ */
+export class LocalLimiter {
+  private nextAt = 0;
+  private active = 0;
+  private readonly waiting: Array<() => void> = [];
+
+  constructor(private readonly perSecond: number, private readonly concurrency: number) {}
+
+  async acquire(): Promise<() => void> {
+    if (this.active < this.concurrency && this.waiting.length === 0) {
+      this.active += 1;
+    } else {
+      // 槽位由释放方交接过来，`active` 已经算上了这一个。
+      await new Promise<void>((resolve) => this.waiting.push(resolve));
+    }
+    const now = Date.now();
+    const at = Math.max(now, this.nextAt);
+    this.nextAt = at + 1000 / this.perSecond;
+    if (at > now) await sleep(at - now);
+    let released = false;
+    return () => {
+      if (released) return;
+      released = true;
+      const next = this.waiting.shift();
+      if (next) next();
+      else this.active -= 1;
+    };
+  }
+}
+
+// ---------------------------------------------------------------------------
+// http
+
+function parseHttpRequest(payload: unknown): EgressHttpRequest {
+  if (!isRecord(payload) || typeof payload.url !== 'string' || typeof payload.method !== 'string') {
+    throw new LocalGatewayError('BAD_REQUEST', 'http request must have a method and a url', 400);
+  }
+  return payload as unknown as EgressHttpRequest;
+}
+
+/** 脚本视角的 URL（句柄未替换）加上 query。 */
+function scriptUrlOf(request: EgressHttpRequest): URL {
+  let url: URL;
+  try {
+    url = new URL(request.url);
+  } catch {
+    throw new LocalGatewayError('BAD_REQUEST', 'url must be an absolute URL', 400);
+  }
+  for (const [name, value] of Object.entries(request.query ?? {})) {
+    for (const item of Array.isArray(value) ? value : [value]) url.searchParams.append(name, item);
+  }
+  return url;
+}
+
+const utf8 = new TextDecoder('utf-8', { fatal: true });
+
+function encodeBody(bytes: Uint8Array): { body: string; bodyEncoding: 'utf8' | 'base64' } {
+  try {
+    return { body: utf8.decode(bytes), bodyEncoding: 'utf8' };
+  } catch {
+    return { body: Buffer.from(bytes).toString('base64'), bodyEncoding: 'base64' };
+  }
+}
+
+function tooLarge(): LocalGatewayError {
+  return new LocalGatewayError('EGRESS_RESPONSE_TOO_LARGE', `The external system's response exceeds the ${RESPONSE_MAX_BYTES / 1024 / 1024} MiB limit`, 502);
+}
+
+function hostDenied(host: string): LocalRunError {
+  return new LocalRunError('HOST_DENIED', `Access to host '${host}' is denied: add it to the connector's hosts and the local configuration`);
+}
+
+// ---------------------------------------------------------------------------
+// crypto
+
+function computeCrypto(request: CryptoRequest, values: Record<string, string>): { value: string } {
+  const data = substituteHandles(request.data, values);
+  switch (request.op) {
+    case 'hash':
+      return { value: createHash(request.algorithm).update(data, 'utf8').digest(request.encoding) };
+    case 'hmac':
+      return { value: createHmac(request.algorithm, substituteHandles(request.key, values)).update(data, 'utf8').digest(request.encoding) };
+    case 'sign': {
+      try {
+        return { value: rsaSign('sha256', Buffer.from(data, 'utf8'), substituteHandles(request.key, values)).toString(request.encoding) };
+      } catch (error) {
+        if (error instanceof LocalGatewayError) throw error;
+        throw new LocalGatewayError('BAD_REQUEST', 'key is not a valid RSA private key', 400);
+      }
+    }
+    default:
+      throw new LocalGatewayError('BAD_REQUEST', 'op must be one of hmac, sign, hash', 400);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// 网关
+
+export function createLocalGateway(options: LocalGatewayOptions): LocalHost & { redact(text: string): string } {
+  const secrets = { ...options.secrets };
+  const secretTexts = secretRedactionTexts([...Object.values(secrets), ...(options.db?.password ? [options.db.password] : [])]);
+  const redact = (text: string): string => redactSecretValues(text, secretTexts);
+  const hostPatterns = [...options.hosts];
+  const doFetch = options.fetch ?? fetch;
+  const realNetwork = options.fetch === undefined;
+  const resolveHost = options.resolve ?? ((host: string) => lookup(host, { all: true, verbatim: true }));
+  const limiter = options.rateLimit && options.rateLimit.perSecond > 0
+    ? new LocalLimiter(options.rateLimit.perSecond, Math.max(1, Math.floor(options.rateLimit.concurrency || 1)))
+    : null;
+  const cache = new Map<string, CacheEntry>();
+  let db: LocalDb | null = null;
+
+  const exchange = (entry: LocalExchange): void => {
+    try {
+      options.onExchange?.(entry);
+    } catch {
+      // 录制器出错不影响请求本身
+    }
+  };
+
+  /**
+   * 主机检查（每一跳都做）：主机求交、元数据地址拦截；真实网络下解析主机名，解析结果含元数据地址时拦截，
+   * 返回检查过的地址供连接固定使用（注入 fetch 时返回 null）。
+   */
+  const checkHost = async (url: URL): Promise<LookupAddress[] | null> => {
+    const host = normalizeHost(url.hostname);
+    if (isMetadataAddress(host)) throw hostDenied(host);
+    const allowed = hostPatterns.some((pattern) => matchHostPattern(host, pattern) || normalizeHost(pattern) === url.host.toLowerCase());
+    if (!allowed) throw hostDenied(host);
+    if (!realNetwork) return null;
+    if (isIpLiteral(host)) return [{ address: host, family: host.includes(':') ? 6 : 4 }];
+    let addresses: LookupAddress[];
+    try {
+      addresses = await resolveHost(host);
+    } catch (error) {
+      throw new LocalGatewayError('EGRESS_UPSTREAM_FAILED', redact(`The outbound request failed: cannot resolve ${host}: ${errorMessage(error)}`), 502);
+    }
+    if (addresses.length === 0) throw new LocalGatewayError('EGRESS_UPSTREAM_FAILED', `The outbound request failed: cannot resolve ${host}`, 502);
+    if (addresses.some((record) => isMetadataAddress(record.address))) throw hostDenied(host);
+    return addresses;
+  };
+
+  const sendOnce = async (
+    url: URL, method: string, headers: Record<string, string>, body: Uint8Array | null, addresses: LookupAddress[] | null, signal: AbortSignal,
+  ): Promise<Response> => {
+    try {
+      if (addresses !== null) return await pinnedRequest({ url, method, headers, body, addresses, signal, maxBytes: RESPONSE_MAX_BYTES });
+      return await doFetch(url.toString(), {
+        method,
+        headers,
+        // DOM lib 下 `Uint8Array<ArrayBufferLike>` 不算 BodyInit（CLI 带 DOM lib 编译本文件），运行时 fetch 照收。
+        ...(body !== null && method !== 'GET' && method !== 'HEAD' ? { body: body as unknown as NonNullable<RequestInit['body']> } : {}),
+        redirect: 'manual',
+        signal,
+      });
+    } catch (error) {
+      if (error instanceof PinnedBodyTooLargeError) throw tooLarge();
+      if (signal.aborted) throw new LocalGatewayError('EGRESS_TIMEOUT', 'The outbound request timed out', 504);
+      throw new LocalGatewayError('EGRESS_UPSTREAM_FAILED', redact(`The outbound request failed: ${errorMessage(error)}`), 502);
+    }
+  };
+
+  const http = async (payload: unknown): Promise<EgressHttpResponse> => {
+    const request = parseHttpRequest(payload);
+    const scriptUrl = scriptUrlOf(request);
+    let url: URL;
+    try {
+      url = new URL(substituteUrlHandles(scriptUrl.toString(), secrets));
+    } catch (error) {
+      if (error instanceof LocalGatewayError) throw error;
+      throw new LocalGatewayError('BAD_REQUEST', 'url is not valid after substituting secrets', 400);
+    }
+    if (url.protocol !== 'https:' && url.protocol !== 'http:') throw new LocalGatewayError('EGRESS_SCHEME_DENIED', 'Only http(s) outbound requests are allowed', 403);
+    let addresses = await checkHost(url);
+    let headers: Record<string, string> = {};
+    for (const [name, value] of Object.entries(request.headers ?? {})) {
+      if (DROPPED_REQUEST_HEADERS.has(name.toLowerCase())) continue;
+      headers[name] = substituteHandles(value, secrets);
+    }
+    let body: Uint8Array | null = null;
+    if (request.body !== null && request.body !== undefined) {
+      body = request.bodyEncoding === 'base64' ? Buffer.from(request.body, 'base64') : Buffer.from(substituteHandles(request.body, secrets), 'utf8');
+    }
+    const timeoutMs = Math.min(request.timeoutMs ?? DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS);
+    const release = limiter ? await limiter.acquire() : () => undefined;
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      let method: string = request.method;
+      let redirects = 0;
+      for (;;) {
+        const response = await sendOnce(url, method, headers, body, addresses, controller.signal);
+        const location = response.headers.get('location');
+        if (REDIRECT_STATUSES.has(response.status) && location !== null && redirects < MAX_REDIRECTS) {
+          let next: URL | null = null;
+          try {
+            next = new URL(location, url);
+          } catch {
+            next = null;
+          }
+          if (next && next.host === url.host && (next.protocol === 'https:' || next.protocol === 'http:')) {
+            redirects += 1;
+            if (response.status === 303 ? method !== 'HEAD' : (response.status === 301 || response.status === 302) && method !== 'GET' && method !== 'HEAD') {
+              method = 'GET';
+              body = null;
+              headers = Object.fromEntries(Object.entries(headers).filter(([name]) => name.toLowerCase() !== 'content-type'));
+            }
+            url = next;
+            // 重定向的每一跳重新解析、重新检查，连接固定到这一跳检查过的地址。
+            addresses = await checkHost(url);
+            continue;
+          }
+        }
+        let bytes: Uint8Array;
+        try {
+          bytes = method === 'HEAD' ? new Uint8Array() : new Uint8Array(await response.arrayBuffer());
+        } catch (error) {
+          if (controller.signal.aborted) throw new LocalGatewayError('EGRESS_TIMEOUT', 'The outbound request timed out', 504);
+          throw new LocalGatewayError('EGRESS_UPSTREAM_FAILED', redact(`Reading the response failed: ${errorMessage(error)}`), 502);
+        }
+        if (bytes.byteLength > RESPONSE_MAX_BYTES) throw tooLarge();
+        const responseHeaders: Record<string, string> = {};
+        response.headers.forEach((value, name) => {
+          const lower = name.toLowerCase();
+          // fetch 已经解压：编码与长度头不再对应正文。
+          if (lower === 'content-encoding' || lower === 'content-length') return;
+          responseHeaders[lower] = value;
+        });
+        return {
+          status: response.status,
+          url: redirects === 0 ? scriptUrl.toString() : url.toString(),
+          headers: responseHeaders,
+          ...encodeBody(bytes),
+        };
+      }
+    } finally {
+      clearTimeout(timer);
+      release();
+    }
+  };
+
+  /** 回给连接器的响应：明文替换为 `***`（与平台网关相同）。 */
+  const redactResponse = (response: EgressHttpResponse): EgressHttpResponse => ({
+    ...response,
+    url: redact(response.url),
+    headers: Object.fromEntries(Object.entries(response.headers).map(([name, value]) => [name, redact(value)])),
+    body: response.bodyEncoding === 'utf8' ? redact(response.body) : response.body,
+  });
+
+  const dbOf = (): LocalDb => {
+    if (!options.db) throw new LocalGatewayError('DB_PROXY_NOT_DECLARED', 'ctx.db is not configured locally: set db in the local configuration', 403);
+    if (isMetadataAddress(options.db.host)) throw hostDenied(normalizeHost(options.db.host));
+    db ??= createLocalDb(options.db, { driver: options.postgres, secrets: Object.values(secrets) });
+    return db;
+  };
+
+  /** 计时并把往来交给 `onExchange`；失败时同样交出（带 error）后再抛。 */
+  const traced = async <T>(kind: LocalExchange['kind'], request: unknown, run: () => Promise<T>): Promise<T> => {
+    const startedAt = Date.now();
+    try {
+      const response = await run();
+      exchange({ kind, request, response, error: null, durationMs: Date.now() - startedAt });
+      return response;
+    } catch (error) {
+      const code = isRecord(error) && typeof error.code === 'string' ? error.code : 'HOST_CALL_FAILED';
+      exchange({ kind, request, response: null, error: { code, message: errorMessage(error) }, durationMs: Date.now() - startedAt });
+      throw error;
+    }
+  };
+
+  const call = async (method: HostMethod, payload: unknown): Promise<unknown> => {
+    switch (method) {
+      case 'http':
+        return redactResponse(await traced('http', payload, () => http(payload)));
+      case 'db.query': {
+        const request = payload as DbQueryRequest;
+        return traced('db', request, () => dbOf().query({ sql: request.sql, params: request.params ?? [] }));
+      }
+      case 'db.next': {
+        const cursorId = isRecord(payload) && typeof payload.cursorId === 'string' ? payload.cursorId : '';
+        return traced('db', { cursorId }, () => dbOf().next(cursorId));
+      }
+      case 'db.close': {
+        const cursorId = isRecord(payload) && typeof payload.cursorId === 'string' ? payload.cursorId : '';
+        await dbOf().close(cursorId);
+        return null;
+      }
+      case 'crypto':
+        return traced('crypto', payload, async () => computeCrypto(payload as CryptoRequest, secrets));
+      case 'cache.get': {
+        const key = isRecord(payload) && typeof payload.key === 'string' ? payload.key : '';
+        const entry = cache.get(key);
+        if (!entry || Date.parse(entry.expiresAt) <= Date.now()) return null;
+        return entry;
+      }
+      case 'cache.set': {
+        if (!isRecord(payload) || typeof payload.key !== 'string' || typeof payload.value !== 'string' || typeof payload.ttlSeconds !== 'number') {
+          throw new LocalGatewayError('BAD_REQUEST', 'cache.set needs key, value and ttlSeconds', 400);
+        }
+        cache.set(payload.key, { value: payload.value, expiresAt: new Date(Date.now() + payload.ttlSeconds * 1000).toISOString() });
+        return null;
+      }
+      case 'secret.reveal': {
+        const field = isRecord(payload) && typeof payload.field === 'string' ? payload.field : '';
+        return { value: lookupSecret(secrets, field) };
+      }
+      default:
+        throw new LocalGatewayError('HOST_METHOD_UNSUPPORTED', `the local gateway does not answer ${method}`);
+    }
+  };
+
+  return {
+    call,
+    redact,
+    async close() {
+      const current = db;
+      db = null;
+      if (current) await current.end();
+    },
+  };
+}
+
+export { LocalDbError };