@deepexi/datasense-connector 3.0.1 → 3.1.0

package/src/runtime/guest/map.ts

@@ -0,0 +1,45 @@
+/** `ctx.map`：受控并发，上限取 `options.concurrency` 与生效并发（连接器 `rateLimit.concurrency`，缺省 4，再与连接覆盖取小）的较小者；结果保持输入顺序。 */
+import { throwIfAborted } from './bridge';
+import { DEFAULT_CONCURRENCY } from './limits';
+
+export type MapFunction = <T, R>(items: readonly T[], fn: (item: T) => Promise<R>, options?: { concurrency?: number }) => Promise<R[]>;
+
+function positiveInteger(value: unknown, what: string): number {
+  if (typeof value !== 'number' || !Number.isInteger(value) || value < 1) throw new TypeError(`${what} must be a positive integer`);
+  return value;
+}
+
+export function createMap(concurrency: number | null): MapFunction {
+  const cap = concurrency ?? DEFAULT_CONCURRENCY;
+  return async <T, R>(items: readonly T[], fn: (item: T) => Promise<R>, options?: { concurrency?: number }): Promise<R[]> => {
+    if (!Array.isArray(items)) throw new TypeError('ctx.map expects an array');
+    if (typeof fn !== 'function') throw new TypeError('ctx.map expects a function');
+    const requested = options?.concurrency === undefined ? cap : positiveInteger(options.concurrency, 'ctx.map concurrency');
+    const limit = Math.min(requested, cap);
+    const results: R[] = new Array<R>(items.length);
+    let next = 0;
+    let failed = false;
+    let firstError: unknown;
+
+    const worker = async (): Promise<void> => {
+      while (!failed && next < items.length) {
+        const index = next;
+        next += 1;
+        try {
+          throwIfAborted();
+          results[index] = await fn(items[index] as T);
+        } catch (error) {
+          if (!failed) {
+            failed = true;
+            firstError = error;
+          }
+        }
+      }
+    };
+    const workers: Array<Promise<void>> = [];
+    for (let i = 0; i < Math.min(limit, items.length); i += 1) workers.push(worker());
+    await Promise.all(workers);
+    if (failed) throw firstError;
+    return results;
+  };
+}

package/src/runtime/guest/operation.ts

@@ -0,0 +1,193 @@
+/**
+ * 连接器操作作业（在 vm 上下文**里**执行）。结果总是 `GuestOperationResult`，失败也走 `result`：
+ * - `run`：校验连接配置、准备凭证，按 `input` 声明校验入参（不合格 `INPUT_INVALID`，不调 `run`），
+ *   调 `run(ctx, params, { idempotencyKey })`，按 `output` 声明校验返回值（不合格 `OUTPUT_INVALID`）；
+ * - `lookup`：同样的准备与入参校验后调 `lookup(ctx, { idempotencyKey, params })`；抛错、回答不合法或 `applied` 的输出不合格一律按
+ *   `{ status: 'unknown' }`；没有实现 `lookup` 时是带 `notImplemented: true` 的 `unknown`；
+ * - `simulate`：不取凭证、不给 `ctx`，期间任何宿主调用都被拒；只校验入参并调 `simulate(params)`，
+ *   没有实现 `simulate` 时 `FATAL_ERROR`（宿主据此回退缺省的模拟结果）。
+ *
+ * 错误归类按 `code`：`REJECTED`（带外部系统的业务码）、`INDETERMINATE`、`AUTH_ERROR`、`TRANSIENT_ERROR`（含 `RATE_LIMITED`）、
+ * `SCHEMA_ERROR`、`FATAL_ERROR`；其余按 `FATAL_ERROR`，运行已取消时一律 `CANCELED`。
+ */
+import type { ConnectorDescriptor, ValueField } from '../../descriptor';
+import type { OperationContext } from '../../types';
+import { validateConfig, validateOperationValue, type DeclarationIssue } from '../../validate';
+import type { GuestOperationFailureCode, GuestOperationJob, GuestOperationResult, LookupAnswer } from '../protocol';
+import { denyHostCalls, isAborted, jsonParse, jsonStringify, throwIfAborted } from './bridge';
+import { createCache } from './cache';
+import { deepFreeze, defaultBaseUrl, prepareSecrets } from './ctx';
+import { createCrypto } from './crypto';
+import { errorInfo } from './failure';
+import { createHttp } from './http';
+import type { LoadedConnector } from './loader';
+import { createLogBuffer, type LogBuffer } from './log';
+import { createTime } from './time';
+
+type Outcome = { type: 'result'; result: GuestOperationResult };
+
+/** 成功结果去掉 `kind` 与 `logs`（按联合逐个去掉）。 */
+type Success = GuestOperationResult extends infer R ? R extends { ok: true } ? Omit<R, 'kind' | 'logs'> : never : never;
+
+interface Failure { code: GuestOperationFailureCode; message: string; external?: { code?: string; message: string } }
+
+/** 连接器抛出的值里能直接沿用的码；`RATE_LIMITED` 归到 `TRANSIENT_ERROR`。 */
+const CODE_MAP: Readonly<Record<string, GuestOperationFailureCode>> = Object.freeze({
+  REJECTED: 'REJECTED',
+  INDETERMINATE: 'INDETERMINATE',
+  AUTH_ERROR: 'AUTH_ERROR',
+  TRANSIENT_ERROR: 'TRANSIENT_ERROR',
+  RATE_LIMITED: 'TRANSIENT_ERROR',
+  SCHEMA_ERROR: 'SCHEMA_ERROR',
+  FATAL_ERROR: 'FATAL_ERROR',
+  CANCELED: 'CANCELED',
+});
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function issuesText(issues: readonly DeclarationIssue[]): string {
+  return issues.map((issue) => `${issue.path || '(root)'}: ${issue.message}`).join('; ');
+}
+
+/** 复制成纯 JSON 数据，切断与连接器对象（getter、原型）的联系；不可序列化时返回 undefined。 */
+function plain(value: unknown): unknown {
+  let text: unknown;
+  try {
+    text = jsonStringify(value);
+  } catch {
+    return undefined;
+  }
+  return typeof text === 'string' ? jsonParse(text) : undefined;
+}
+
+/** `RejectedError` 的 `details.code`（外部系统的业务码）；读属性出错时不带。 */
+function externalCode(error: unknown): string | undefined {
+  if (error === null || typeof error !== 'object') return undefined;
+  try {
+    const details = (error as { details?: unknown }).details;
+    if (isRecord(details) && typeof details.code === 'string') return details.code;
+  } catch {
+    return undefined;
+  }
+  return undefined;
+}
+
+function failureOf(error: unknown): Failure {
+  if (isAborted()) return { code: 'CANCELED', message: 'run was canceled' };
+  const info = errorInfo(error);
+  const mapped = info.code !== null && Object.hasOwn(CODE_MAP, info.code) ? CODE_MAP[info.code] : undefined;
+  if (mapped === 'REJECTED') {
+    const code = externalCode(error);
+    return { code: 'REJECTED', message: info.message, external: { ...(code !== undefined ? { code } : {}), message: info.message } };
+  }
+  if (mapped) return { code: mapped, message: info.message };
+  return { code: 'FATAL_ERROR', message: info.code === null ? info.message : `${info.code}: ${info.message}` };
+}
+
+/** 操作的 `ctx`：没有 `record`、`db`、`paginate`、`map`、`signal`。 */
+function createOperationContext(config: Record<string, unknown>, secrets: Readonly<Record<string, string>>, descriptor: ConnectorDescriptor, logs: LogBuffer): OperationContext {
+  const frozen = deepFreeze(config);
+  return Object.freeze({
+    config: frozen,
+    secrets,
+    http: createHttp(defaultBaseUrl(descriptor, frozen)),
+    crypto: createCrypto(),
+    cache: createCache(),
+    time: createTime(),
+    log: logs.logger(),
+  });
+}
+
+/** 按描述的字段校验并复制一个值；不合格时返回问题。 */
+function checked(schema: Record<string, ValueField>, value: unknown, path: 'input' | 'output'): { ok: true; value: Record<string, unknown> } | { ok: false; message: string } {
+  const copy = plain(value);
+  if (copy === undefined) return { ok: false, message: `${path} is not serializable` };
+  const issues = validateOperationValue(schema, copy, path);
+  if (issues.length > 0) return { ok: false, message: `${path} does not match the declaration: ${issuesText(issues)}` };
+  return { ok: true, value: copy as Record<string, unknown> };
+}
+
+/** lookup 的回答规整：只认三种状态，`applied` 的输出按 `output` 声明校验，其余一律 `unknown`。 */
+function lookupAnswer(output: Record<string, ValueField>, value: unknown, logs: LogBuffer): LookupAnswer {
+  const answer = plain(value);
+  if (!isRecord(answer)) return { status: 'unknown' };
+  if (answer.status === 'not_applied') return { status: 'not_applied' };
+  if (answer.status !== 'applied') return { status: 'unknown' };
+  const checkedOutput = checked(output, answer.output, 'output');
+  if (!checkedOutput.ok) {
+    logs.logger().warn(`lookup answered applied but ${checkedOutput.message}; treated as unknown`);
+    return { status: 'unknown' };
+  }
+  return { status: 'applied', output: checkedOutput.value };
+}
+
+export async function runOperation(connector: LoadedConnector, job: GuestOperationJob): Promise<Outcome> {
+  const phase = job.phase;
+  const logs = createLogBuffer();
+  const done = (result: Success): Outcome =>
+    ({ type: 'result', result: { kind: 'operation', ...result, logs: logs.drain() } as GuestOperationResult });
+  const fail = (failure: Failure): Outcome =>
+    ({ type: 'result', result: { kind: 'operation', phase, ok: false, ...failure, logs: logs.drain() } as GuestOperationResult });
+
+  const operation = Object.hasOwn(connector.operations, job.operation) ? connector.operations[job.operation] : undefined;
+  const contract = Object.hasOwn(connector.descriptor.operations, job.operation) ? connector.descriptor.operations[job.operation] : undefined;
+  if (!operation || !contract) return fail({ code: 'CONNECTOR_LOAD_FAILED', message: `connector has no operation named ${job.operation}` });
+
+  if (phase === 'simulate') {
+    // 纯函数：不取凭证；从这里起任何宿主调用都被拒。
+    denyHostCalls();
+    if (typeof operation.simulate !== 'function') return fail({ code: 'FATAL_ERROR', message: `operation ${job.operation} does not implement simulate()` });
+    const params = checked(contract.input, job.params, 'input');
+    if (!params.ok) return fail({ code: 'INPUT_INVALID', message: params.message });
+    let value: unknown;
+    try {
+      value = await Reflect.apply(operation.simulate, operation, [deepFreeze(params.value)]);
+    } catch (error) {
+      return fail(failureOf(error));
+    }
+    const output = checked(contract.output, value, 'output');
+    if (!output.ok) return fail({ code: 'OUTPUT_INVALID', message: `simulate() returned an invalid result: ${output.message}` });
+    return done({ phase: 'simulate', ok: true, output: output.value });
+  }
+
+  const configIssues = validateConfig(connector.descriptor.config, job.config);
+  if (configIssues.length > 0) return fail({ code: 'CONFIG_INVALID', message: `invalid connection config: ${issuesText(configIssues)}` });
+  let secrets: Readonly<Record<string, string>>;
+  try {
+    secrets = await prepareSecrets(connector.descriptor, job.secretFields);
+  } catch (error) {
+    return fail(failureOf(error));
+  }
+  const params = checked(contract.input, job.params, 'input');
+  if (!params.ok) return fail({ code: 'INPUT_INVALID', message: params.message });
+  const ctx = createOperationContext(job.config, secrets, connector.descriptor, logs);
+  const idempotencyKey = job.idempotencyKey;
+
+  if (phase === 'lookup') {
+    if (typeof operation.lookup !== 'function') return done({ phase: 'lookup', ok: true, lookup: { status: 'unknown', notImplemented: true } });
+    let value: unknown;
+    try {
+      value = await Reflect.apply(operation.lookup, operation, [ctx, deepFreeze({ idempotencyKey, params: params.value })]);
+      throwIfAborted();
+    } catch (error) {
+      if (isAborted()) return fail({ code: 'CANCELED', message: 'run was canceled' });
+      const info = errorInfo(error);
+      logs.logger().warn(`lookup threw ${info.code ?? 'an error'}: ${info.message}; treated as unknown`);
+      return done({ phase: 'lookup', ok: true, lookup: { status: 'unknown' } });
+    }
+    return done({ phase: 'lookup', ok: true, lookup: lookupAnswer(contract.output, value, logs) });
+  }
+
+  let value: unknown;
+  try {
+    value = await Reflect.apply(operation.run, operation, [ctx, deepFreeze(params.value), Object.freeze({ idempotencyKey })]);
+    throwIfAborted();
+  } catch (error) {
+    return fail(failureOf(error));
+  }
+  const output = checked(contract.output, value, 'output');
+  if (!output.ok) return fail({ code: 'OUTPUT_INVALID', message: `run() returned an invalid result: ${output.message}` });
+  return done({ phase: 'run', ok: true, output: output.value });
+}

package/src/runtime/guest/paginate.ts

@@ -0,0 +1,169 @@
+/** `ctx.paginate`：四种 HTTP 分页与数据库按键分页，都是产出 `Page` 的异步迭代器；取到空页即结束（空页不产出）。 */
+import type {
+  CursorPaginateOptions, HttpClient, HttpResponse, KeysetPaginateOptions, LinkPaginateOptions, OffsetPaginateOptions, Page,
+  PagePaginateOptions, Paginate,
+} from '../../types';
+import { throwIfAborted } from './bridge';
+
+function compare(a: unknown, b: unknown): number {
+  if (typeof a === 'number' && typeof b === 'number') return a - b;
+  if (typeof a === 'bigint' && typeof b === 'bigint') return a < b ? -1 : a > b ? 1 : 0;
+  if (a instanceof Date && b instanceof Date) return a.getTime() - b.getTime();
+  const left = String(a);
+  const right = String(b);
+  return left < right ? -1 : left > right ? 1 : 0;
+}
+
+export function makePage<Row>(rows: Row[]): Page<Row> {
+  return Object.freeze({
+    rows,
+    maxOf<V>(fn: (row: Row) => V): V | undefined {
+      let max: V | undefined;
+      for (const row of rows) {
+        const value = fn(row);
+        if (value === undefined || value === null) continue;
+        if (max === undefined || compare(value, max) > 0) max = value;
+      }
+      return max;
+    },
+  });
+}
+
+function rowsOf<Row>(value: unknown, what: string): Row[] {
+  if (!Array.isArray(value)) throw new TypeError(`${what} must return an array of rows`);
+  return value as Row[];
+}
+
+function requireFunction(value: unknown, what: string): void {
+  if (typeof value !== 'function') throw new TypeError(`${what} must be a function`);
+}
+
+async function* pageIterator<Res, Row>(options: PagePaginateOptions<Res, Row>): AsyncGenerator<Page<Row>> {
+  requireFunction(options.request, 'paginate.page request');
+  requireFunction(options.rows, 'paginate.page rows');
+  const first = options.firstPage ?? 1;
+  for (let page = first; ; page += 1) {
+    throwIfAborted();
+    const res = await options.request(page);
+    const rows = rowsOf<Row>(options.rows(res), 'paginate.page rows');
+    if (rows.length === 0) return;
+    yield makePage(rows);
+    if (options.totalPages) {
+      const total = options.totalPages(res);
+      if (typeof total !== 'number' || !Number.isFinite(total) || page - first + 1 >= total) return;
+    }
+  }
+}
+
+async function* offsetIterator<Res, Row>(options: OffsetPaginateOptions<Res, Row>): AsyncGenerator<Page<Row>> {
+  requireFunction(options.request, 'paginate.offset request');
+  requireFunction(options.rows, 'paginate.offset rows');
+  const limit = options.limit;
+  if (typeof limit !== 'number' || !Number.isInteger(limit) || limit < 1) throw new TypeError('paginate.offset limit must be a positive integer');
+  for (let offset = options.startOffset ?? 0; ; offset += limit) {
+    throwIfAborted();
+    const res = await options.request(offset, limit);
+    const rows = rowsOf<Row>(options.rows(res), 'paginate.offset rows');
+    if (rows.length === 0) return;
+    yield makePage(rows);
+    if (options.total) {
+      const total = options.total(res);
+      if (typeof total !== 'number' || !Number.isFinite(total) || offset + limit >= total) return;
+    } else if (rows.length < limit) {
+      return;
+    }
+  }
+}
+
+async function* cursorIterator<Res, Row>(options: CursorPaginateOptions<Res, Row>): AsyncGenerator<Page<Row>> {
+  requireFunction(options.request, 'paginate.cursor request');
+  requireFunction(options.rows, 'paginate.cursor rows');
+  requireFunction(options.nextCursor, 'paginate.cursor nextCursor');
+  let cursor: string | undefined;
+  const seen = new Set<string>();
+  for (;;) {
+    throwIfAborted();
+    const res = await options.request(cursor);
+    const rows = rowsOf<Row>(options.rows(res), 'paginate.cursor rows');
+    if (rows.length > 0) yield makePage(rows);
+    const next = options.nextCursor(res);
+    if (next === null || next === undefined || next === '') return;
+    const text = String(next);
+    // 来源回同一个游标会死循环。
+    if (seen.has(text)) throw new TypeError(`paginate.cursor got the same cursor twice: ${text}`);
+    seen.add(text);
+    cursor = text;
+  }
+}
+
+/** 解析 `Link` 头里 `rel="next"` 的地址。 */
+export function nextLink(header: string | undefined): string | null {
+  if (!header) return null;
+  for (const part of header.split(',')) {
+    const match = /^\s*<([^>]*)>(.*)$/.exec(part);
+    if (!match) continue;
+    for (const param of (match[2] ?? '').split(';')) {
+      const rel = /^\s*rel\s*=\s*"?([^"]*)"?\s*$/i.exec(param);
+      if (rel && (rel[1] ?? '').toLowerCase().split(/\s+/).includes('next')) return match[1] ?? null;
+    }
+  }
+  return null;
+}
+
+function resolveAgainst(link: string, current: string): string {
+  if (/^https?:\/\//i.test(link)) return link;
+  const origin = /^(https?:\/\/[^/?#]+)/i.exec(current)?.[1];
+  if (!origin) return link;
+  if (link.startsWith('//')) return `${current.slice(0, current.indexOf(':') + 1)}${link}`;
+  if (link.startsWith('/')) return `${origin}${link}`;
+  const path = current.slice(origin.length).replace(/[?#].*$/, '');
+  const dir = path.slice(0, path.lastIndexOf('/') + 1) || '/';
+  return `${origin}${dir}${link}`;
+}
+
+function linkIterator<Res, Row>(http: HttpClient) {
+  return async function* (options: LinkPaginateOptions<Res, Row>): AsyncGenerator<Page<Row>> {
+    requireFunction(options.rows, 'paginate.link rows');
+    if (typeof options.url !== 'string') throw new TypeError('paginate.link url must be a string');
+    let url: string | null = options.url;
+    const seen = new Set<string>();
+    while (url !== null) {
+      throwIfAborted();
+      seen.add(url);
+      const res: HttpResponse<Res> = await http.get<Res>(url, options.options);
+      const rows = rowsOf<Row>(options.rows(res), 'paginate.link rows');
+      if (rows.length > 0) yield makePage(rows);
+      const next = options.next ? options.next(res) : nextLink(res.headers.link);
+      if (next === null || next === undefined || next === '') return;
+      const resolved = resolveAgainst(String(next), res.url || url);
+      if (seen.has(resolved)) throw new TypeError(`paginate.link got the same next link twice: ${resolved}`);
+      url = resolved;
+    }
+  };
+}
+
+async function* keysetIterator<Row, Key>(options: KeysetPaginateOptions<Row, Key>): AsyncGenerator<Page<Row>> {
+  requireFunction(options.query, 'paginate.keyset query');
+  requireFunction(options.key, 'paginate.keyset key');
+  let last: Key | undefined;
+  for (;;) {
+    throwIfAborted();
+    const rows: Row[] = [];
+    for await (const row of options.query(last)) rows.push(row);
+    if (rows.length === 0) return;
+    yield makePage(rows);
+    const key = options.key(rows[rows.length - 1] as Row);
+    if (key === undefined || key === null) throw new TypeError('paginate.keyset key returned no value for the last row');
+    last = key;
+  }
+}
+
+export function createPaginate(http: HttpClient): Paginate {
+  return Object.freeze({
+    page: pageIterator,
+    offset: offsetIterator,
+    cursor: cursorIterator,
+    link: linkIterator(http),
+    keyset: keysetIterator,
+  }) as Paginate;
+}

package/src/runtime/guest/pull.ts

@@ -0,0 +1,177 @@
+/**
+ * 拉取作业：逐项拉取数据流生成器的产出，只认 `yield` 出来的 `ctx.record` / `ctx.remove` / `ctx.checkpoint` 标记。
+ * 记录按批经宿主调用 `emit` 交给宿主：每批最多 1000 条或 4 MiB（先到为准，字节数与服务端同口径，见 `itemBytes`）；
+ * 单条就超过 4 MiB 的记录 / 删除标记发不出去，按条拒收（丢弃并记 warn 日志），条数随下一批的 `dropped` 上报、由服务端计入
+ * `stats.rejected`（流结束时只剩丢弃条数也发一批空批次）；累计超过 `PULL_REJECT_THRESHOLD` 条时运行失败。
+ * 产出断点时把当前这一批连同断点立即发出，
+ * 宿主先写入断点之前的记录再提交断点（设计 §4.7），之后失败只丢断点之后的记录。
+ * 每次 `emit` 等宿主回复后才继续拉下一项（背压）。批次不带 `seq`，由宿主编号。
+ * 回复带 `{ stop: true }` 时（预览达到条数上限）触发 `ctx.signal`、不再拉取，结果为 `{ kind: 'pull', stopped: true, … }`。
+ */
+import { errors } from '../../errors';
+import type { StreamInput } from '../../types';
+import type { GuestJob, GuestPullBatch, GuestResult } from '../protocol';
+import { call, isAborted, isStopped, jsonParse, jsonStringify, requestStop } from './bridge';
+import { utf8Length } from './crypto';
+import { createPullContext, deepFreeze, type ContextOptions } from './ctx';
+import { failure, failureFromError, type FailureMessage } from './failure';
+import { PULL_BATCH_MAX_BYTES, PULL_BATCH_MAX_RECORDS, PULL_REJECT_THRESHOLD } from './limits';
+import type { LoadedConnector } from './loader';
+
+type PullJob = Extract<GuestJob, { kind: 'pull' }>;
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/** 复制成纯 JSON 数据（连接器之后再改原对象也不影响已产出的记录），同时得到序列化文本。 */
+function copy(value: unknown, what: string): { value: unknown; text: string } {
+  let text: unknown;
+  try {
+    text = jsonStringify(value);
+  } catch (error) {
+    throw new errors.SchemaError(`${what} is not serializable: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  if (typeof text !== 'string') throw new errors.SchemaError(`${what} is not serializable`);
+  return { value: jsonParse(text), text };
+}
+
+interface PendingBatch {
+  records: GuestPullBatch['records'];
+  removals: GuestPullBatch['removals'];
+  checkpoint: { state: unknown } | undefined;
+  bytes: number;
+  /** 本批发出前丢弃的超大条数。 */
+  dropped: number;
+}
+
+/**
+ * 服务端按 `Buffer.byteLength(JSON.stringify([records, removals]))` 校验一批（≤ `PULL_BATCH_MAX_BYTES`，
+ * 见 connector 的 `runner-protocol/routes.ts`），这里按同一形状计数：
+ * 外层 `[[…],[…]]` 固定 5 字节；每项 `{"table":<表名>,"row":<值>}` / `{"table":<表名>,"key":<值>}` 的框架 17 字节，
+ * 加表名与值的 JSON 字节数，再按每项 1 个逗号计（每个数组最后一项其实没有逗号，最多多算 2 字节，只会偏保守）。
+ */
+const BATCH_FRAME_BYTES = 5;
+const ITEM_FRAME_BYTES = 17 + 1;
+
+function itemBytes(table: string, valueText: string): number {
+  return ITEM_FRAME_BYTES + utf8Length(jsonStringify(table)) + utf8Length(valueText);
+}
+
+function emptyBatch(): PendingBatch {
+  return { records: [], removals: [], checkpoint: undefined, bytes: BATCH_FRAME_BYTES, dropped: 0 };
+}
+
+function closeIterator(iterator: AsyncIterator<unknown>): void {
+  try {
+    const closing = iterator.return?.();
+    if (closing && typeof closing.then === 'function') closing.then(undefined, () => undefined);
+  } catch {
+    // 生成器收尾出错不影响失败回报
+  }
+}
+
+export async function runPull(connector: LoadedConnector, job: PullJob, options: ContextOptions): Promise<{ type: 'result'; result: GuestResult } | FailureMessage> {
+  const streams = connector.streams;
+  const stream = streams[job.stream];
+  if (!stream) return failure('CONNECTOR_LOAD_FAILED', `connector has no stream named ${job.stream}`, options.logs.drain());
+  const ctx = createPullContext(options);
+  const input: StreamInput = Object.freeze({
+    mode: job.mode,
+    state: job.state === null ? undefined : deepFreeze(job.state),
+    streamConfig: deepFreeze(job.streamConfig),
+  });
+  const tables = new Set(job.tables);
+  let batch = emptyBatch();
+  let emitted = 0;
+
+  const flush = async (): Promise<void> => {
+    const payload: GuestPullBatch = {
+      records: batch.records,
+      removals: batch.removals,
+      ...(batch.checkpoint ? { checkpoint: batch.checkpoint } : {}),
+      logs: options.logs.drain(),
+      ...(batch.dropped > 0 ? { dropped: batch.dropped } : {}),
+    };
+    batch = emptyBatch();
+    const reply = await call('emit', payload);
+    emitted += 1;
+    if (isRecord(reply) && reply.stop === true) requestStop();
+  };
+  const result = (): { type: 'result'; result: GuestResult } => ({
+    type: 'result',
+    result: { kind: 'pull', emittedBatches: emitted, ...(isStopped() ? { stopped: true } : {}), logs: options.logs.drain() },
+  });
+  let oversized = 0;
+  const log = options.logs.logger();
+  const add = async (what: 'record' | 'removal', table: string, size: number, push: () => void): Promise<void> => {
+    // 单条放进空批次也超限：服务端会整批拒绝（400），只能按条拒收，不让整次运行失败。
+    if (BATCH_FRAME_BYTES + size > PULL_BATCH_MAX_BYTES) {
+      oversized += 1;
+      batch.dropped += 1;
+      log.warn(`RECORD_TOO_LARGE: ${what} for table "${table}" is ${size} bytes, more than the ${PULL_BATCH_MAX_BYTES}-byte batch limit; it was dropped`);
+      if (oversized > PULL_REJECT_THRESHOLD) {
+        throw new errors.SchemaError(`stream ${job.stream} produced ${oversized} records larger than ${PULL_BATCH_MAX_BYTES} bytes, more than ${PULL_REJECT_THRESHOLD}`);
+      }
+      return;
+    }
+    if (batch.records.length + batch.removals.length > 0 && batch.bytes + size > PULL_BATCH_MAX_BYTES) await flush();
+    push();
+    batch.bytes += size;
+    if (batch.records.length + batch.removals.length >= PULL_BATCH_MAX_RECORDS) await flush();
+  };
+  const checkTable = (table: unknown): string => {
+    if (typeof table !== 'string' || !tables.has(table)) {
+      throw new errors.SchemaError(`stream ${job.stream} produced a record for table ${String(table)}, which is not declared for this stream`);
+    }
+    return table;
+  };
+
+  let iterator: AsyncIterator<unknown>;
+  try {
+    const iterable = Reflect.apply(stream, undefined, [ctx, input]) as unknown;
+    const factory = isRecord(iterable) || typeof iterable === 'function' ? (iterable as Partial<AsyncIterable<unknown>>)[Symbol.asyncIterator] : undefined;
+    if (typeof factory !== 'function') throw new errors.SchemaError(`stream ${job.stream} must be an async generator function`);
+    iterator = Reflect.apply(factory, iterable, []) as AsyncIterator<unknown>;
+  } catch (error) {
+    return failureFromError(error, options.logs.drain());
+  }
+
+  try {
+    for (;;) {
+      if (isAborted()) {
+        closeIterator(iterator);
+        return isStopped() ? result() : failure('CANCELED', 'run was canceled', options.logs.drain());
+      }
+      const step = await iterator.next();
+      if (step.done) break;
+      const item: unknown = step.value;
+      const kind = isRecord(item) ? item.__ontos : undefined;
+      if (kind === 'record' && isRecord(item)) {
+        const table = checkTable(item.table);
+        if (!isRecord(item.row)) throw new errors.SchemaError(`stream ${job.stream} produced a record whose row is not an object`);
+        const row = copy(item.row, 'row');
+        await add('record', table, itemBytes(table, row.text), () => batch.records.push({ table, row: row.value as Record<string, unknown> }));
+      } else if (kind === 'remove' && isRecord(item)) {
+        const table = checkTable(item.table);
+        if (!isRecord(item.key)) throw new errors.SchemaError(`stream ${job.stream} produced a removal whose key is not an object`);
+        const key = copy(item.key, 'key');
+        await add('removal', table, itemBytes(table, key.text), () => batch.removals.push({ table, key: key.value as Record<string, unknown> }));
+      } else if (kind === 'checkpoint' && isRecord(item)) {
+        batch.checkpoint = { state: copy(item.state, 'checkpoint state').value };
+        // 断点之前的记录随断点一起发出，不等凑满一批：页小于一批时也能逐页提交断点。
+        await flush();
+      } else {
+        throw new errors.SchemaError(`stream ${job.stream} yielded a value that is not ctx.record(), ctx.remove() or ctx.checkpoint()`);
+      }
+    }
+    if (isAborted()) return isStopped() ? result() : failure('CANCELED', 'run was canceled', options.logs.drain());
+    if (batch.records.length + batch.removals.length > 0 || batch.dropped > 0) await flush();
+  } catch (error) {
+    closeIterator(iterator);
+    // 提前结束之后生成器收尾抛出的错误（如在途宿主调用被拒）不算失败。
+    if (isStopped()) return result();
+    return failureFromError(error, options.logs.drain());
+  }
+  return result();
+}