@deepexi/datasense-connector 3.0.1 → 3.1.0

package/runtime/guest/pull.d.ts

@@ -1,12 +0,0 @@
-import type { ConnectorDefinition } from '../../types.js';
-import type { GuestJob, GuestResult } from '../protocol.js';
-import { type ContextOptions } from './ctx.js';
-import { type FailureMessage } from './failure.js';
-type PullJob = Extract<GuestJob, {
-    kind: 'pull';
-}>;
-export declare function runPull(definition: ConnectorDefinition, job: PullJob, options: ContextOptions): Promise<{
-    type: 'result';
-    result: GuestResult;
-} | FailureMessage>;
-export {};

package/runtime/guest/pull.js

@@ -1,150 +0,0 @@
-import { errors } from '../../errors.js';
-import { call, isAborted, isStopped, jsonParse, jsonStringify, requestStop } from './bridge.js';
-import { utf8Length } from './crypto.js';
-import { createPullContext, deepFreeze } from './ctx.js';
-import { failure, failureFromError } from './failure.js';
-import { PULL_BATCH_MAX_BYTES, PULL_BATCH_MAX_RECORDS, PULL_REJECT_THRESHOLD } from './limits.js';
-function isRecord(value) {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-function copy(value, what) {
-    let text;
-    try {
-        text = jsonStringify(value);
-    }
-    catch (error) {
-        throw new errors.SchemaError(`${what} is not serializable: ${error instanceof Error ? error.message : String(error)}`);
-    }
-    if (typeof text !== 'string')
-        throw new errors.SchemaError(`${what} is not serializable`);
-    return { value: jsonParse(text), text };
-}
-const BATCH_FRAME_BYTES = 5;
-const ITEM_FRAME_BYTES = 17 + 1;
-function itemBytes(table, valueText) {
-    return ITEM_FRAME_BYTES + utf8Length(jsonStringify(table)) + utf8Length(valueText);
-}
-function emptyBatch() {
-    return { records: [], removals: [], checkpoint: undefined, bytes: BATCH_FRAME_BYTES, dropped: 0 };
-}
-function closeIterator(iterator) {
-    try {
-        const closing = iterator.return?.();
-        if (closing && typeof closing.then === 'function')
-            closing.then(undefined, () => undefined);
-    }
-    catch {
-    }
-}
-export async function runPull(definition, job, options) {
-    const streams = definition.streams ?? {};
-    const stream = streams[job.stream];
-    const ctx = createPullContext(options);
-    const input = Object.freeze({
-        mode: job.mode,
-        state: job.state === null ? undefined : deepFreeze(job.state),
-        streamConfig: deepFreeze(job.streamConfig),
-    });
-    const tables = new Set(job.tables);
-    let batch = emptyBatch();
-    let emitted = 0;
-    const flush = async () => {
-        const payload = {
-            records: batch.records,
-            removals: batch.removals,
-            ...(batch.checkpoint ? { checkpoint: batch.checkpoint } : {}),
-            logs: options.logs.drain(),
-            ...(batch.dropped > 0 ? { dropped: batch.dropped } : {}),
-        };
-        batch = emptyBatch();
-        const reply = await call('emit', payload);
-        emitted += 1;
-        if (isRecord(reply) && reply.stop === true)
-            requestStop();
-    };
-    const result = () => ({
-        type: 'result',
-        result: { kind: 'pull', emittedBatches: emitted, ...(isStopped() ? { stopped: true } : {}), logs: options.logs.drain() },
-    });
-    let oversized = 0;
-    const log = options.logs.logger();
-    const add = async (what, table, size, push) => {
-        if (BATCH_FRAME_BYTES + size > PULL_BATCH_MAX_BYTES) {
-            oversized += 1;
-            batch.dropped += 1;
-            log.warn(`RECORD_TOO_LARGE: ${what} for table "${table}" is ${size} bytes, more than the ${PULL_BATCH_MAX_BYTES}-byte batch limit; it was dropped`);
-            if (oversized > PULL_REJECT_THRESHOLD) {
-                throw new errors.SchemaError(`stream ${job.stream} produced ${oversized} records larger than ${PULL_BATCH_MAX_BYTES} bytes, more than ${PULL_REJECT_THRESHOLD}`);
-            }
-            return;
-        }
-        if (batch.records.length + batch.removals.length > 0 && batch.bytes + size > PULL_BATCH_MAX_BYTES)
-            await flush();
-        push();
-        batch.bytes += size;
-        if (batch.records.length + batch.removals.length >= PULL_BATCH_MAX_RECORDS)
-            await flush();
-    };
-    const checkTable = (table) => {
-        if (typeof table !== 'string' || !tables.has(table)) {
-            throw new errors.SchemaError(`stream ${job.stream} produced a record for table ${String(table)}, which is not declared for this stream`);
-        }
-        return table;
-    };
-    let iterator;
-    try {
-        const iterable = Reflect.apply(stream, streams, [ctx, input]);
-        const factory = isRecord(iterable) || typeof iterable === 'function' ? iterable[Symbol.asyncIterator] : undefined;
-        if (typeof factory !== 'function')
-            throw new errors.SchemaError(`stream ${job.stream} must be an async generator function`);
-        iterator = Reflect.apply(factory, iterable, []);
-    }
-    catch (error) {
-        return failureFromError(error, options.logs.drain());
-    }
-    try {
-        for (;;) {
-            if (isAborted()) {
-                closeIterator(iterator);
-                return isStopped() ? result() : failure('CANCELED', 'run was canceled', options.logs.drain());
-            }
-            const step = await iterator.next();
-            if (step.done)
-                break;
-            const item = step.value;
-            const kind = isRecord(item) ? item.__ontos : undefined;
-            if (kind === 'record' && isRecord(item)) {
-                const table = checkTable(item.table);
-                if (!isRecord(item.row))
-                    throw new errors.SchemaError(`stream ${job.stream} produced a record whose row is not an object`);
-                const row = copy(item.row, 'row');
-                await add('record', table, itemBytes(table, row.text), () => batch.records.push({ table, row: row.value }));
-            }
-            else if (kind === 'remove' && isRecord(item)) {
-                const table = checkTable(item.table);
-                if (!isRecord(item.key))
-                    throw new errors.SchemaError(`stream ${job.stream} produced a removal whose key is not an object`);
-                const key = copy(item.key, 'key');
-                await add('removal', table, itemBytes(table, key.text), () => batch.removals.push({ table, key: key.value }));
-            }
-            else if (kind === 'checkpoint' && isRecord(item)) {
-                batch.checkpoint = { state: copy(item.state, 'checkpoint state').value };
-                await flush();
-            }
-            else {
-                throw new errors.SchemaError(`stream ${job.stream} yielded a value that is not ctx.record(), ctx.remove() or ctx.checkpoint()`);
-            }
-        }
-        if (isAborted())
-            return isStopped() ? result() : failure('CANCELED', 'run was canceled', options.logs.drain());
-        if (batch.records.length + batch.removals.length > 0 || batch.dropped > 0)
-            await flush();
-    }
-    catch (error) {
-        closeIterator(iterator);
-        if (isStopped())
-            return result();
-        return failureFromError(error, options.logs.drain());
-    }
-    return result();
-}

package/runtime/guest/push.d.ts

@@ -1,18 +0,0 @@
-/**
- * 推送作业：对每条请求按推送名找处理器，先 `verify` 再 `map`。行为与第 1 期一致：
- * - `ctx.record` / `ctx.remove` 直接记录，请求成功后才并入结果；请求结束后再用该 `ctx` 抛错；
- * - 单条请求抛错只拒收该条（code 取错误的 `code`，普通异常为 `CONNECTOR_ERROR`），已产生的记录丢弃；
- * - `SCHEMA_ERROR` / `FATAL_ERROR` 让整次执行失败；
- * - 推送名在定义里不存在的请求拒收，code `PUSH_UNKNOWN`（定义只有按名字的写法 `push: { <名>: { verify?, map } }`）。
- */
-import type { ConnectorDefinition } from '../../types.js';
-import type { GuestJob, GuestPushResult } from '../protocol.js';
-import { type FailureMessage } from './failure.js';
-type PushJob = Extract<GuestJob, {
-    kind: 'push';
-}>;
-export declare function runPush(definition: ConnectorDefinition, job: PushJob): Promise<{
-    type: 'result';
-    result: GuestPushResult;
-} | FailureMessage>;
-export {};

package/runtime/guest/push.js

@@ -1,80 +0,0 @@
-import { jsonParse, jsonStringify } from './bridge.js';
-import { deepFreeze } from './ctx.js';
-import { connectorStack, errorInfo, failure } from './failure.js';
-import { createLogBuffer } from './log.js';
-import { isoNow } from './time.js';
-const WHOLE_RUN_FAILURES = ['SCHEMA_ERROR', 'FATAL_ERROR'];
-function copyRecord(value, what) {
-    if (value === null || typeof value !== 'object' || Array.isArray(value))
-        throw new TypeError(`${what} must be an object`);
-    const text = jsonStringify(value);
-    if (typeof text !== 'string')
-        throw new TypeError(`${what} is not serializable`);
-    return jsonParse(text);
-}
-export async function runPush(definition, job) {
-    const handlers = definition.push !== null && typeof definition.push === 'object' ? definition.push : {};
-    if (Object.keys(handlers).length === 0)
-        return failure('CONNECTOR_LOAD_FAILED', 'connector does not support push', []);
-    const config = deepFreeze(job.config);
-    const logs = createLogBuffer();
-    const result = { kind: 'push', records: [], removals: [], rejected: [], logs: [] };
-    for (const request of job.requests) {
-        const requestId = request.requestId;
-        const handler = Object.hasOwn(handlers, request.push) ? handlers[request.push] : undefined;
-        if (!handler) {
-            result.rejected.push({ requestId, code: 'PUSH_UNKNOWN', message: `connector has no push named ${request.push}` });
-            continue;
-        }
-        const pending = { records: [], removals: [], open: true };
-        const assertOpen = () => {
-            if (!pending.open)
-                throw new TypeError('ctx is no longer usable after the request finished');
-        };
-        const ctx = Object.freeze({
-            config,
-            record(table, row) {
-                assertOpen();
-                if (typeof table !== 'string')
-                    throw new TypeError('table must be a string');
-                pending.records.push({ requestId, table, row: copyRecord(row, 'row') });
-            },
-            remove(table, key) {
-                assertOpen();
-                if (typeof table !== 'string')
-                    throw new TypeError('table must be a string');
-                pending.removals.push({ requestId, table, key: copyRecord(key, 'key') });
-            },
-            log: logs.logger(requestId, () => pending.open),
-            time: Object.freeze({ now: isoNow }),
-        });
-        const body = request.body;
-        const req = Object.freeze({
-            id: requestId,
-            receivedAt: request.receivedAt,
-            headers: deepFreeze({ ...request.headers }),
-            body,
-            json() {
-                return jsonParse(body);
-            },
-        });
-        try {
-            if (typeof handler.verify === 'function')
-                await Reflect.apply(handler.verify, handler, [ctx, req]);
-            await Reflect.apply(handler.map, handler, [ctx, req]);
-            pending.open = false;
-            result.records.push(...pending.records);
-            result.removals.push(...pending.removals);
-        }
-        catch (error) {
-            pending.open = false;
-            const info = errorInfo(error);
-            if (info.code !== null && WHOLE_RUN_FAILURES.includes(info.code)) {
-                return failure(info.code, info.message, logs.drain(), connectorStack(info.stack));
-            }
-            result.rejected.push({ requestId, code: info.code ?? 'CONNECTOR_ERROR', message: info.message });
-        }
-    }
-    result.logs = logs.drain();
-    return { type: 'result', result };
-}

package/runtime/guest/signal.d.ts

@@ -1,3 +0,0 @@
-/** `ctx.signal`：父进程发 `cancel` 时置位并调用 `onAbort` 回调。 */
-import type { ConnectorSignal } from '../../types.js';
-export declare function createSignal(): ConnectorSignal;

package/runtime/guest/signal.js

@@ -1,12 +0,0 @@
-import { isAborted, onAbort } from './bridge.js';
-export function createSignal() {
-    const signal = {
-        onAbort(fn) {
-            if (typeof fn !== 'function')
-                throw new TypeError('ctx.signal.onAbort expects a function');
-            onAbort(fn);
-        },
-    };
-    Object.defineProperty(signal, 'aborted', { get: isAborted, enumerable: true });
-    return Object.freeze(signal);
-}

package/runtime/guest/time.d.ts

@@ -1,5 +0,0 @@
-/** `ctx.time`：`now()` 与 `toUtc(本地时间文本, 时区, 格式?)`。时区换算用上下文自己的 `Intl.DateTimeFormat`。 */
-import type { ConnectorTime } from '../../types.js';
-export declare function isoNow(): string;
-export declare function toUtc(local: string, timeZone: string, format?: string): string;
-export declare function createTime(): ConnectorTime;

package/runtime/guest/time.js

@@ -1,127 +0,0 @@
-const DateCtor = Date;
-const dateUtc = Date.UTC;
-const dateParse = Date.parse;
-const reflectApply = Reflect.apply;
-const toISOString = Date.prototype.toISOString;
-const DateTimeFormat = Intl.DateTimeFormat;
-export function isoNow() {
-    return reflectApply(toISOString, new DateCtor(), []);
-}
-function iso(ms) {
-    return reflectApply(toISOString, new DateCtor(ms), []);
-}
-const DEFAULT_PATTERN = /^(\d{4})[-/](\d{1,2})[-/](\d{1,2})(?:[ T](\d{1,2}):(\d{2})(?::(\d{2})(?:\.(\d{1,9}))?)?)?$/;
-const ZONED_PATTERN = /^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})$/i;
-const TOKENS = [
-    { token: 'YYYY', field: 'year', pattern: '(\\d{4})' },
-    { token: 'SSS', field: 'millisecond', pattern: '(\\d{1,3})' },
-    { token: 'MM', field: 'month', pattern: '(\\d{2})' },
-    { token: 'DD', field: 'day', pattern: '(\\d{2})' },
-    { token: 'HH', field: 'hour', pattern: '(\\d{2})' },
-    { token: 'mm', field: 'minute', pattern: '(\\d{2})' },
-    { token: 'ss', field: 'second', pattern: '(\\d{2})' },
-    { token: 'M', field: 'month', pattern: '(\\d{1,2})' },
-    { token: 'D', field: 'day', pattern: '(\\d{1,2})' },
-    { token: 'H', field: 'hour', pattern: '(\\d{1,2})' },
-];
-function escapeRegExp(text) {
-    return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-}
-function parseWithFormat(local, format) {
-    const fields = [];
-    let source = '';
-    let i = 0;
-    while (i < format.length) {
-        const match = TOKENS.find((entry) => format.startsWith(entry.token, i));
-        if (match) {
-            fields.push(match.field);
-            source += match.pattern;
-            i += match.token.length;
-        }
-        else {
-            source += escapeRegExp(format.charAt(i));
-            i += 1;
-        }
-    }
-    const result = new RegExp(`^${source}$`).exec(local);
-    if (!result || !fields.includes('year') || !fields.includes('month') || !fields.includes('day'))
-        return null;
-    const clock = { year: 0, month: 0, day: 0, hour: 0, minute: 0, second: 0, millisecond: 0 };
-    fields.forEach((field, index) => {
-        const text = result[index + 1] ?? '0';
-        clock[field] = field === 'millisecond' ? Number(text.padEnd(3, '0')) : Number(text);
-    });
-    return clock;
-}
-function parseDefault(local) {
-    const match = DEFAULT_PATTERN.exec(local);
-    if (!match)
-        return null;
-    return {
-        year: Number(match[1]), month: Number(match[2]), day: Number(match[3]),
-        hour: Number(match[4] ?? 0), minute: Number(match[5] ?? 0), second: Number(match[6] ?? 0),
-        millisecond: Number((match[7] ?? '0').slice(0, 3).padEnd(3, '0')),
-    };
-}
-function wallMillis(clock) {
-    const ms = dateUtc(clock.year, clock.month - 1, clock.day, clock.hour, clock.minute, clock.second, clock.millisecond);
-    if (Number.isNaN(ms))
-        return null;
-    const check = new DateCtor(ms);
-    if (check.getUTCFullYear() !== clock.year || check.getUTCMonth() !== clock.month - 1 || check.getUTCDate() !== clock.day
-        || check.getUTCHours() !== clock.hour || check.getUTCMinutes() !== clock.minute || check.getUTCSeconds() !== clock.second)
-        return null;
-    return ms;
-}
-const formatters = new Map();
-function formatter(timeZone) {
-    let cached = formatters.get(timeZone);
-    if (!cached) {
-        try {
-            cached = new DateTimeFormat('en-US', {
-                timeZone, hourCycle: 'h23', year: 'numeric', month: 'numeric', day: 'numeric', hour: 'numeric', minute: 'numeric', second: 'numeric',
-            });
-        }
-        catch {
-            throw new TypeError(`unknown time zone ${timeZone}`);
-        }
-        formatters.set(timeZone, cached);
-    }
-    return cached;
-}
-function zoneOffset(timeZone, utcMs) {
-    const base = utcMs - (((utcMs % 1000) + 1000) % 1000);
-    const parts = {};
-    for (const part of formatter(timeZone).formatToParts(new DateCtor(base))) {
-        if (part.type !== 'literal')
-            parts[part.type] = Number(part.value);
-    }
-    const asUtc = dateUtc(parts.year ?? 0, (parts.month ?? 1) - 1, parts.day ?? 1, parts.hour ?? 0, parts.minute ?? 0, parts.second ?? 0);
-    return asUtc - base;
-}
-export function toUtc(local, timeZone, format) {
-    if (typeof local !== 'string')
-        throw new TypeError('ctx.time.toUtc expects a date-time string');
-    if (typeof timeZone !== 'string' || timeZone.length === 0)
-        throw new TypeError('ctx.time.toUtc expects a time zone like Asia/Shanghai');
-    const text = local.trim();
-    if (format === undefined && ZONED_PATTERN.test(text)) {
-        const ms = dateParse(text.replace(' ', 'T'));
-        if (Number.isNaN(ms))
-            throw new TypeError(`invalid date-time ${local}`);
-        return iso(ms);
-    }
-    const clock = format === undefined ? parseDefault(text) : parseWithFormat(text, format);
-    const wall = clock ? wallMillis(clock) : null;
-    if (wall === null)
-        throw new TypeError(`invalid date-time ${local}${format === undefined ? '' : ` for format ${format}`}`);
-    const first = zoneOffset(timeZone, wall);
-    let utc = wall - first;
-    const second = zoneOffset(timeZone, utc);
-    if (second !== first)
-        utc = wall - second;
-    return iso(utc);
-}
-export function createTime() {
-    return Object.freeze({ now: isoNow, toUtc });
-}

package/runtime/guest-runtime.d.ts

@@ -1,7 +0,0 @@
-export declare const GUEST_ENTRY_PATH: string;
-/** dist 根目录下预打包的 guest 运行时文件名。 */
-export declare const GUEST_BUNDLE_FILE = "guest.cjs";
-/** 用 Bun 把 guest 入口打成 CJS 字符串（SDK 内联）。构建脚本与源码下的 `loadGuestRuntime` 共用。 */
-export declare function bundleGuestWithBun(entry?: string): Promise<string>;
-/** 可放进 `execute.guest` 的 guest 运行时 CJS 字符串（进程内缓存）。 */
-export declare function loadGuestRuntime(): Promise<string>;

package/runtime/guest-runtime.js

@@ -1,51 +0,0 @@
-import { readFile } from 'node:fs/promises';
-import path from 'node:path';
-import { IS_SOURCE, moduleFile, PACKAGE_ROOT } from '../internal/location.js';
-export const GUEST_ENTRY_PATH = moduleFile('runtime/guest/index');
-export const GUEST_BUNDLE_FILE = 'guest.cjs';
-const CODE_FILE = /\.[cm]?[jt]sx?$/;
-function readSourcePlugin(bun) {
-    return {
-        name: 'datasense-connector-read-source',
-        setup(build) {
-            build.onLoad({ filter: CODE_FILE }, async (args) => {
-                const extension = args.path.slice(args.path.lastIndexOf('.') + 1);
-                const loader = extension === 'tsx' ? 'tsx' : extension === 'jsx' ? 'jsx' : /^[cm]?js$/.test(extension) ? 'js' : 'ts';
-                return { contents: await bun.file(args.path).text(), loader };
-            });
-        },
-    };
-}
-const MACROS_OFF = { macros: false };
-export async function bundleGuestWithBun(entry = GUEST_ENTRY_PATH) {
-    const bun = globalThis.Bun;
-    if (!bun)
-        throw new Error('the guest runtime can only be bundled from source under Bun; use the built package (dist) under Node.js');
-    const result = await bun.build({
-        entrypoints: [entry],
-        format: 'cjs',
-        target: 'browser',
-        minify: false,
-        external: [],
-        plugins: [readSourcePlugin(bun)],
-        throw: false,
-        ...MACROS_OFF,
-    });
-    const output = result.outputs.find((item) => item.kind === 'entry-point');
-    if (!result.success || !output) {
-        throw new Error(`guest runtime bundle failed: ${result.logs.map((log) => log.message).join('; ') || 'no output'}`);
-    }
-    return output.text();
-}
-let cached = null;
-export function loadGuestRuntime() {
-    if (cached)
-        return cached;
-    const pending = IS_SOURCE ? bundleGuestWithBun() : readFile(path.join(PACKAGE_ROOT, GUEST_BUNDLE_FILE), 'utf8');
-    cached = pending;
-    pending.catch(() => {
-        if (cached === pending)
-            cached = null;
-    });
-    return pending;
-}

package/runtime/index.d.ts

@@ -1,18 +0,0 @@
-/**
- * `@deepexi/datasense-connector/runtime`：宿主侧入口（runner 的执行子进程与父进程、`./testing` 用），不在 vm 里加载。
- *
- * - `GUEST_ENTRY_PATH`：guest 运行时入口（源码下是 `src/runtime/guest/index.ts`，dist 下是编译后的 `runtime/guest/index.js`），
- *   runner 用 `bundleTrustedEntry` 打成 CJS 字符串，放进 `execute.guest`；
- * - `loadGuestRuntime`：直接拿到可放进 `execute.guest` 的 CJS 字符串：dist 下读预打包的 `guest.cjs`，源码下用 `Bun.build` 打包（进程内缓存）；
- * - `createGuestSession`：子进程里建 vm 上下文、加载 guest 与连接器代码，之后只转发字符串消息；
- * - `parseParentMessage` / `parseChildMessage`：两侧解析对方消息，结构不合法返回 null；`undeclaredEmitTable` 区分
- *   「`emit` 写了没声明的表」（宿主按 `SCHEMA_ERROR` 结束）与其他不合法消息。
- *
- * 宿主调用 `definition` 的 payload 是 `summarizeDefinition` 的结果：guest 在加载并核对 `expect` 之后、执行作业之前发一次
- * （push 作业不发），宿主回 ok 后才继续。`emit` 的 payload 是 `GuestPullBatch`（不带 `seq`，由宿主编号），回复 `EmitReply`
- * （`{ stop: true }` 时提前结束）。`declaration` 只在 `discoverFirst` 的拉取作业里发一次（再发按协议错误），payload 是 `DeclarationReport`，回复 null。
- */
-export { GUEST_ENTRY_PATH, loadGuestRuntime } from './guest-runtime.js';
-export { createGuestSession } from './session.js';
-export { GUEST_FAILURE_CODES, HOST_METHODS, parseChildMessage, parseParentMessage, undeclaredEmitTable } from './protocol.js';
-export type { CacheEntry, CacheGetRequest, CacheSetRequest, ChildMessage, CryptoRequest, CryptoResponse, DbCursorRequest, DbNextResponse, DbQueryRequest, DbQueryResponse, DeclarationReport, DefinitionReply, DefinitionSummary, EgressHttpRequest, EgressHttpResponse, EmitReply, GuestFailureCode, GuestJob, GuestLog, GuestPullBatch, GuestPushRequest, GuestPushResult, GuestResult, HostCallError, HostMethod, NormalizedDiscover, ParentMessage, ParseChildOptions, SecretRevealRequest, SecretRevealResponse, } from './protocol.js';

package/runtime/index.js

@@ -1,3 +0,0 @@
-export { GUEST_ENTRY_PATH, loadGuestRuntime } from './guest-runtime.js';
-export { createGuestSession } from './session.js';
-export { GUEST_FAILURE_CODES, HOST_METHODS, parseChildMessage, parseParentMessage, undeclaredEmitTable } from './protocol.js';