@deepexi/datasense-connector 0.0.0-stage → 3.0.1

package/runtime/guest/http.js

@@ -0,0 +1,204 @@
+import { errors } from '../../errors.js';
+import { HostCallFailure, call, jsonParse, jsonStringify, now, sleep, throwIfAborted } from './bridge.js';
+import { bytesToBase64 } from './crypto.js';
+import { MAX_RETRY_WAIT_MS, RATE_LIMIT_BACKOFF_MS, RATE_LIMIT_RETRIES, TRANSIENT_BACKOFF_MS, TRANSIENT_RETRIES } from './limits.js';
+const METHODS = ['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD'];
+const ABSOLUTE_URL = /^https?:\/\//i;
+const TRANSIENT_STATUS = [502, 503, 504];
+const IDEMPOTENT_METHODS = ['GET', 'HEAD', 'PUT', 'DELETE'];
+const JSON_CONTENT_TYPE = /(?:^|[/+])json\s*(?:;|$)/i;
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function stringify(value) {
+    if (typeof value === 'string')
+        return value;
+    if (value instanceof Date)
+        return value.toISOString();
+    if (typeof value === 'object' && value !== null)
+        return jsonStringify(value);
+    return String(value);
+}
+export function resolveUrl(url, baseUrl) {
+    if (ABSOLUTE_URL.test(url))
+        return url;
+    if (!baseUrl)
+        throw new TypeError(`relative URL ${url} needs auth.baseUrl or a fixed host in the connector's hosts`);
+    const base = baseUrl.replace(/\/+$/, '');
+    return `${base}${url.startsWith('/') ? '' : '/'}${url}`;
+}
+function buildHeaders(options) {
+    const headers = {};
+    for (const source of [options.auth?.headers, options.headers]) {
+        if (source === undefined)
+            continue;
+        if (!isRecord(source))
+            throw new TypeError('headers must be an object');
+        for (const [name, value] of Object.entries(source)) {
+            if (value === undefined || value === null)
+                continue;
+            headers[name.toLowerCase()] = stringify(value);
+        }
+    }
+    return headers;
+}
+function buildQuery(query) {
+    const out = {};
+    if (query === undefined)
+        return out;
+    if (!isRecord(query))
+        throw new TypeError('query must be an object');
+    for (const [name, value] of Object.entries(query)) {
+        if (value === undefined || value === null)
+            continue;
+        out[name] = Array.isArray(value) ? value.filter((item) => item !== undefined && item !== null).map(stringify) : stringify(value);
+    }
+    return out;
+}
+function buildBody(body, headers) {
+    if (body === undefined || body === null)
+        return { body: null, bodyEncoding: 'utf8' };
+    if (typeof body === 'string')
+        return { body, bodyEncoding: 'utf8' };
+    if (body instanceof Uint8Array)
+        return { body: bytesToBase64(Array.from(body)), bodyEncoding: 'base64' };
+    if (body instanceof ArrayBuffer)
+        return { body: bytesToBase64(Array.from(new Uint8Array(body))), bodyEncoding: 'base64' };
+    const text = jsonStringify(body);
+    if (typeof text !== 'string')
+        throw new TypeError('request body is not serializable');
+    if (headers['content-type'] === undefined)
+        headers['content-type'] = 'application/json';
+    return { body: text, bodyEncoding: 'utf8' };
+}
+function buildRequest(method, url, options, defaultBaseUrl) {
+    const upper = typeof method === 'string' ? method.toUpperCase() : '';
+    if (!METHODS.includes(upper))
+        throw new TypeError(`unsupported HTTP method ${String(method)}`);
+    if (typeof url !== 'string' || url.length === 0)
+        throw new TypeError('url must be a non-empty string');
+    if (!isRecord(options))
+        throw new TypeError('options must be an object');
+    const baseUrl = options.auth?.baseUrl ?? defaultBaseUrl;
+    const headers = buildHeaders(options);
+    const request = {
+        method: upper,
+        url: resolveUrl(url, typeof baseUrl === 'string' ? baseUrl : null),
+        headers,
+        query: buildQuery(options.query),
+        ...buildBody(options.body, headers),
+    };
+    if (options.retry !== undefined && typeof options.retry !== 'boolean')
+        throw new TypeError('retry must be a boolean');
+    if (options.timeoutMs !== undefined) {
+        if (typeof options.timeoutMs !== 'number' || !(options.timeoutMs > 0))
+            throw new TypeError('timeoutMs must be a positive number');
+        request.timeoutMs = options.timeoutMs;
+    }
+    return request;
+}
+function toResponse(value, parseJson) {
+    if (!isRecord(value) || typeof value.status !== 'number' || typeof value.body !== 'string')
+        throw new TypeError('http host call returned a malformed response');
+    const raw = value;
+    const headers = {};
+    if (isRecord(raw.headers)) {
+        for (const [name, item] of Object.entries(raw.headers))
+            if (typeof item === 'string')
+                headers[name.toLowerCase()] = item;
+    }
+    const text = raw.body;
+    const url = typeof raw.url === 'string' ? raw.url : '';
+    let data = text;
+    if (parseJson && raw.bodyEncoding !== 'base64' && JSON_CONTENT_TYPE.test(headers['content-type'] ?? '') && text.length > 0) {
+        try {
+            data = jsonParse(text);
+        }
+        catch {
+            throw new errors.FatalError(`response from ${url} is not valid JSON`);
+        }
+    }
+    return { status: raw.status, url, headers, data, text };
+}
+function retryAfterMs(headers) {
+    const value = headers['retry-after'];
+    if (value === undefined)
+        return undefined;
+    const trimmed = value.trim();
+    if (/^\d+(?:\.\d+)?$/.test(trimmed))
+        return Number(trimmed) * 1000;
+    const at = Date.parse(trimmed);
+    return Number.isNaN(at) ? undefined : Math.max(0, at - now());
+}
+function clampWait(ms) {
+    return Math.min(Math.max(0, ms), MAX_RETRY_WAIT_MS);
+}
+export function createHttp(defaultBaseUrl) {
+    async function request(method, url, options = {}) {
+        const req = buildRequest(method, url, options, defaultBaseUrl);
+        const raw = options.raw === true;
+        const label = `${req.method} ${req.url}`;
+        const retryTransient = options.retry ?? IDEMPOTENT_METHODS.includes(req.method);
+        const transientLimit = retryTransient ? TRANSIENT_RETRIES : 0;
+        let rateRetries = 0;
+        let transientRetries = 0;
+        for (;;) {
+            throwIfAborted();
+            let value;
+            try {
+                value = await call('http', req);
+            }
+            catch (error) {
+                if (error instanceof HostCallFailure && error.code === 'EGRESS_RATE_LIMITED') {
+                    const wait = error.retryAfterMs ?? RATE_LIMIT_BACKOFF_MS * 2 ** rateRetries;
+                    if (rateRetries >= RATE_LIMIT_RETRIES)
+                        throw new errors.RateLimited(wait, `${label} is still rate limited after ${RATE_LIMIT_RETRIES} retries`);
+                    rateRetries += 1;
+                    await sleep(clampWait(wait));
+                    continue;
+                }
+                if (error instanceof HostCallFailure && error.code === 'EGRESS_TIMEOUT') {
+                    if (transientRetries >= transientLimit)
+                        throw new errors.TransientError(`${label} timed out after ${transientRetries} retries`);
+                    await sleep(TRANSIENT_BACKOFF_MS * 2 ** transientRetries);
+                    transientRetries += 1;
+                    continue;
+                }
+                throw error;
+            }
+            const response = toResponse(value, !raw);
+            if (raw)
+                return response;
+            const status = response.status;
+            if (status === 429) {
+                const wait = retryAfterMs(response.headers) ?? RATE_LIMIT_BACKOFF_MS * 2 ** rateRetries;
+                if (rateRetries >= RATE_LIMIT_RETRIES)
+                    throw new errors.RateLimited(wait, `${label} returned 429 after ${RATE_LIMIT_RETRIES} retries`);
+                rateRetries += 1;
+                await sleep(clampWait(wait));
+                continue;
+            }
+            if (TRANSIENT_STATUS.includes(status)) {
+                if (transientRetries >= transientLimit)
+                    throw new errors.TransientError(`${label} returned ${status} after ${transientRetries} retries`);
+                await sleep(TRANSIENT_BACKOFF_MS * 2 ** transientRetries);
+                transientRetries += 1;
+                continue;
+            }
+            if (status === 401)
+                throw new errors.AuthError(`${label} returned 401`);
+            if (status >= 400)
+                throw new errors.FatalError(`${label} returned ${status}`);
+            return response;
+        }
+    }
+    const verb = (method) => (url, options) => request(method, url, options);
+    return Object.freeze({
+        get: verb('GET'),
+        post: verb('POST'),
+        put: verb('PUT'),
+        patch: verb('PATCH'),
+        delete: verb('DELETE'),
+        request,
+    });
+}

package/runtime/guest/index.d.ts

@@ -0,0 +1,12 @@
+export interface GuestApi {
+    /** 只生效一次：`json` 为 `{ expect, job }`，`factory` 为上下文里编译出的 `(function (module, exports, require) {…})`。 */
+    start(json: string, factory: unknown): void;
+    handle(json: string): void;
+    poll(): string;
+}
+export declare function bootGuest(): GuestApi;
+/**
+ * 进程内执行（`./testing` 的 `createInProcessExecutor`，给 vitest 用）：每次执行都重新求值一份 guest 运行时，
+ * 这里的全局对象是宿主进程自己的，所以不拿掉任何全局；其余与 `bootGuest` 相同。
+ */
+export declare function bootLocalGuest(): GuestApi;

package/runtime/guest/index.js

@@ -0,0 +1,89 @@
+import { summarizeDefinition, validateConfig } from '../../validate.js';
+import { call, finish, handle, jsonParse, jsonStringify, poll } from './bridge.js';
+import { concurrencyFromDefinitionReply, prepareSecrets } from './ctx.js';
+import { discoverDeclaration, runCheck, runDiscover } from './discover.js';
+import { connectorStack, errorInfo, failure, failureFromError } from './failure.js';
+import { loadDefinition } from './loader.js';
+import { createLogBuffer } from './log.js';
+import { runPull } from './pull.js';
+import { runPush } from './push.js';
+const STRIPPED_GLOBALS = ['WebAssembly', 'ShadowRealm', 'console'];
+function issuesText(issues) {
+    return issues.map((issue) => `${issue.path || '(root)'}: ${issue.message}`).join('; ');
+}
+async function run(definition, job) {
+    if (job.kind === 'push')
+        return runPush(definition, job);
+    if (job.kind === 'pull' && !(definition.streams && Object.hasOwn(definition.streams, job.stream))) {
+        return failure('CONNECTOR_LOAD_FAILED', `connector has no stream named ${job.stream}`, []);
+    }
+    const summary = jsonParse(jsonStringify(summarizeDefinition(definition)));
+    const logs = createLogBuffer();
+    let effectiveConcurrency;
+    try {
+        effectiveConcurrency = concurrencyFromDefinitionReply(await call('definition', summary));
+    }
+    catch (error) {
+        return failureFromError(error, logs.drain());
+    }
+    const issues = validateConfig(definition.config, job.config);
+    if (issues.length > 0)
+        return failure('CONFIG_INVALID', `invalid connection config: ${issuesText(issues)}`, logs.drain());
+    let secrets;
+    try {
+        secrets = await prepareSecrets(summary, job.secretFields);
+    }
+    catch (error) {
+        return failureFromError(error, logs.drain());
+    }
+    const options = { config: job.config, secrets, summary, logs, effectiveConcurrency };
+    if (job.kind === 'discover')
+        return runDiscover(definition, options);
+    if (job.kind === 'check')
+        return runCheck(definition, options);
+    if (job.discoverFirst !== true)
+        return runPull(definition, job, options);
+    const declared = await discoverDeclaration(definition, options);
+    if (!declared.ok)
+        return declared.failure;
+    try {
+        await call('declaration', { summary, discover: declared.discover });
+    }
+    catch (error) {
+        return failureFromError(error, logs.drain());
+    }
+    const stream = Object.hasOwn(declared.discover.streams, job.stream) ? declared.discover.streams[job.stream] : undefined;
+    if (!stream)
+        return failure('SCHEMA_ERROR', `stream ${job.stream} is not in the discover result`, logs.drain());
+    return runPull(definition, { ...job, tables: Object.keys(stream.tables) }, options);
+}
+async function execute(json, factory) {
+    const { expect, job } = jsonParse(json);
+    const loaded = loadDefinition(factory, expect);
+    if (!loaded.ok)
+        return failure('CONNECTOR_LOAD_FAILED', loaded.message, [], connectorStack(loaded.stack));
+    return run(loaded.definition, job);
+}
+let started = false;
+function start(json, factory) {
+    if (started || typeof json !== 'string')
+        return;
+    started = true;
+    let pending;
+    try {
+        pending = execute(json, factory);
+    }
+    catch (error) {
+        finish(failure('FATAL_ERROR', errorInfo(error).message, []));
+        return;
+    }
+    pending.then((outcome) => finish(outcome), (error) => finish(failure('FATAL_ERROR', `connector runtime failed: ${errorInfo(error).message}`, [])));
+}
+export function bootGuest() {
+    for (const name of STRIPPED_GLOBALS)
+        Reflect.deleteProperty(globalThis, name);
+    return Object.freeze({ start, handle, poll });
+}
+export function bootLocalGuest() {
+    return Object.freeze({ start, handle, poll });
+}

package/runtime/guest/limits.d.ts

@@ -0,0 +1,20 @@
+/** guest 侧限额；与 shared 的 connector-limits 同值（SDK 不依赖 shared）。 */
+export declare const PULL_BATCH_MAX_RECORDS = 1000;
+export declare const PULL_BATCH_MAX_BYTES: number;
+/** 一次拉取运行里被拒收的记录超过这个数时运行失败（与 shared 的 `PULL_REJECT_THRESHOLD` 同值）。 */
+export declare const PULL_REJECT_THRESHOLD = 1000;
+export declare const MAX_LOGS = 1000;
+export declare const MAX_LOG_MESSAGE = 4096;
+/** 连接器与连接都没声明限流时网关用的并发（`EGRESS_DEFAULT_RATE.concurrency`）。 */
+export declare const DEFAULT_CONCURRENCY = 4;
+/** 429 / 网关限流的最多重试次数。 */
+export declare const RATE_LIMIT_RETRIES = 5;
+/** 上游 502 / 503 / 504 与网关超时的最多重试次数，退避从 200 ms 起、每次翻倍。 */
+export declare const TRANSIENT_RETRIES = 3;
+export declare const TRANSIENT_BACKOFF_MS = 200;
+/** 429 没给 `Retry-After` 时的等待起点（每次翻倍）。 */
+export declare const RATE_LIMIT_BACKOFF_MS = 1000;
+/** 单次等待上限，避免来源给出离谱的 `Retry-After`。 */
+export declare const MAX_RETRY_WAIT_MS = 600000;
+/** 连接器代码在 vm 里的文件名；失败回报的调用栈只保留这个文件的帧。 */
+export declare const CONNECTOR_FILENAME = "connector.js";

package/runtime/guest/limits.js

@@ -0,0 +1,12 @@
+export const PULL_BATCH_MAX_RECORDS = 1000;
+export const PULL_BATCH_MAX_BYTES = 4 * 1024 * 1024;
+export const PULL_REJECT_THRESHOLD = 1000;
+export const MAX_LOGS = 1000;
+export const MAX_LOG_MESSAGE = 4096;
+export const DEFAULT_CONCURRENCY = 4;
+export const RATE_LIMIT_RETRIES = 5;
+export const TRANSIENT_RETRIES = 3;
+export const TRANSIENT_BACKOFF_MS = 200;
+export const RATE_LIMIT_BACKOFF_MS = 1000;
+export const MAX_RETRY_WAIT_MS = 600_000;
+export const CONNECTOR_FILENAME = 'connector.js';

package/runtime/guest/loader.d.ts

@@ -0,0 +1,15 @@
+import type { ConnectorDefinition } from '../../types.js';
+export declare const SDK_SPECIFIER = "@deepexi/datasense-connector";
+export type LoadOutcome = {
+    ok: true;
+    definition: ConnectorDefinition;
+} | {
+    ok: false;
+    message: string;
+    stack: string | undefined;
+};
+/** 执行连接器代码、取默认导出、校验定义并核对期望的名字与版本。 */
+export declare function loadDefinition(factory: unknown, expect: {
+    name: string;
+    version: string;
+}): LoadOutcome;

package/runtime/guest/loader.js

@@ -0,0 +1,42 @@
+import { ConnectorError } from '../../errors.js';
+import * as sdk from '../../index.js';
+import { validateConnectorDefinition } from '../../validate.js';
+import { errorInfo } from './failure.js';
+export const SDK_SPECIFIER = '@deepexi/datasense-connector';
+const sdkModule = Object.freeze({ ...sdk, __esModule: true });
+function connectorRequire(specifier) {
+    if (specifier === SDK_SPECIFIER)
+        return sdkModule;
+    throw new ConnectorError('CONNECTOR_LOAD_FAILED', `cannot require ${String(specifier)}: connectors can only import ${SDK_SPECIFIER} and relative files`);
+}
+function resolveExport(exported) {
+    if (exported !== null && (typeof exported === 'object' || typeof exported === 'function')) {
+        const fallback = exported.default;
+        if (fallback !== undefined && fallback !== null)
+            return fallback;
+    }
+    return exported;
+}
+export function loadDefinition(factory, expect) {
+    if (typeof factory !== 'function')
+        return { ok: false, message: 'connector code did not compile to a module', stack: undefined };
+    let definition;
+    try {
+        const module = { exports: {} };
+        Reflect.apply(factory, undefined, [module, module.exports, connectorRequire]);
+        definition = resolveExport(module.exports);
+    }
+    catch (error) {
+        const info = errorInfo(error);
+        return { ok: false, message: `connector code failed to load: ${info.message}`, stack: info.stack };
+    }
+    const issues = validateConnectorDefinition(definition);
+    if (issues.length > 0) {
+        return { ok: false, message: `invalid connector definition: ${issues.map((issue) => `${issue.path || '(root)'}: ${issue.message}`).join('; ')}`, stack: undefined };
+    }
+    const loaded = definition;
+    if (loaded.name !== expect.name || loaded.version !== expect.version) {
+        return { ok: false, message: `connector code defines ${loaded.name}@${loaded.version}, expected ${expect.name}@${expect.version}`, stack: undefined };
+    }
+    return { ok: true, definition: loaded };
+}

package/runtime/guest/log.d.ts

@@ -0,0 +1,10 @@
+/** `ctx.log`：一次运行最多保留 1000 条，单条超过 4096 字符截断；拉取作业每批 `emit` 带走已有的日志。 */
+import type { ConnectorLog } from '../../types.js';
+import type { GuestLog } from '../protocol.js';
+export interface LogBuffer {
+    /** 取走尚未上报的日志。 */
+    drain(): GuestLog[];
+    /** `isOpen` 返回 false 时丢弃（推送请求结束后再写日志无效）。 */
+    logger(requestId?: string, isOpen?: () => boolean): ConnectorLog;
+}
+export declare function createLogBuffer(): LogBuffer;

package/runtime/guest/log.js

@@ -0,0 +1,36 @@
+import { jsonStringify } from './bridge.js';
+import { MAX_LOG_MESSAGE, MAX_LOGS } from './limits.js';
+import { isoNow } from './time.js';
+function render(message, data) {
+    let text = typeof message === 'string' ? message : 'non-string log message';
+    if (data !== undefined) {
+        try {
+            const extra = jsonStringify(data);
+            if (typeof extra === 'string')
+                text = `${text} ${extra}`;
+        }
+        catch {
+        }
+    }
+    return text.length > MAX_LOG_MESSAGE ? text.slice(0, MAX_LOG_MESSAGE) : text;
+}
+export function createLogBuffer() {
+    let entries = [];
+    let total = 0;
+    const write = (level, requestId, isOpen) => (message, data) => {
+        if (total >= MAX_LOGS || (isOpen && !isOpen()))
+            return;
+        total += 1;
+        entries[entries.length] = { at: isoNow(), level, message: render(message, data), ...(requestId !== undefined ? { requestId } : {}) };
+    };
+    return {
+        drain() {
+            const out = entries;
+            entries = [];
+            return out;
+        },
+        logger(requestId, isOpen) {
+            return Object.freeze({ info: write('info', requestId, isOpen), warn: write('warn', requestId, isOpen), error: write('error', requestId, isOpen) });
+        },
+    };
+}

package/runtime/guest/map.d.ts

@@ -0,0 +1,4 @@
+export type MapFunction = <T, R>(items: readonly T[], fn: (item: T) => Promise<R>, options?: {
+    concurrency?: number;
+}) => Promise<R[]>;
+export declare function createMap(concurrency: number | null): MapFunction;

package/runtime/guest/map.js

@@ -0,0 +1,45 @@
+import { throwIfAborted } from './bridge.js';
+import { DEFAULT_CONCURRENCY } from './limits.js';
+function positiveInteger(value, what) {
+    if (typeof value !== 'number' || !Number.isInteger(value) || value < 1)
+        throw new TypeError(`${what} must be a positive integer`);
+    return value;
+}
+export function createMap(concurrency) {
+    const cap = concurrency ?? DEFAULT_CONCURRENCY;
+    return async (items, fn, options) => {
+        if (!Array.isArray(items))
+            throw new TypeError('ctx.map expects an array');
+        if (typeof fn !== 'function')
+            throw new TypeError('ctx.map expects a function');
+        const requested = options?.concurrency === undefined ? cap : positiveInteger(options.concurrency, 'ctx.map concurrency');
+        const limit = Math.min(requested, cap);
+        const results = new Array(items.length);
+        let next = 0;
+        let failed = false;
+        let firstError;
+        const worker = async () => {
+            while (!failed && next < items.length) {
+                const index = next;
+                next += 1;
+                try {
+                    throwIfAborted();
+                    results[index] = await fn(items[index]);
+                }
+                catch (error) {
+                    if (!failed) {
+                        failed = true;
+                        firstError = error;
+                    }
+                }
+            }
+        };
+        const workers = [];
+        for (let i = 0; i < Math.min(limit, items.length); i += 1)
+            workers.push(worker());
+        await Promise.all(workers);
+        if (failed)
+            throw firstError;
+        return results;
+    };
+}

package/runtime/guest/paginate.d.ts

@@ -0,0 +1,6 @@
+/** `ctx.paginate`：四种 HTTP 分页与数据库按键分页，都是产出 `Page` 的异步迭代器；取到空页即结束（空页不产出）。 */
+import type { HttpClient, Page, Paginate } from '../../types.js';
+export declare function makePage<Row>(rows: Row[]): Page<Row>;
+/** 解析 `Link` 头里 `rel="next"` 的地址。 */
+export declare function nextLink(header: string | undefined): string | null;
+export declare function createPaginate(http: HttpClient): Paginate;