@deepexi/datasense-connector 3.0.1 → 3.1.0

package/runtime/guest/crypto.js

@@ -1,177 +0,0 @@
-import { call } from './bridge.js';
-const HMAC_ALGORITHMS = ['sha1', 'sha256', 'sha512'];
-const HASH_ALGORITHMS = ['md5', 'sha1', 'sha256', 'sha512'];
-const ENCODINGS = ['hex', 'base64'];
-const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
-function encoding(value) {
-    if (value === undefined)
-        return 'hex';
-    if (typeof value !== 'string' || !ENCODINGS.includes(value))
-        throw new TypeError("encoding must be 'hex' or 'base64'");
-    return value;
-}
-function text(value, what) {
-    if (typeof value !== 'string')
-        throw new TypeError(`${what} must be a string`);
-    return value;
-}
-async function digest(request) {
-    const response = await call('crypto', request);
-    if (!response || typeof response.value !== 'string')
-        throw new TypeError('crypto host call returned no value');
-    return response.value;
-}
-export function utf8Encode(input) {
-    const bytes = [];
-    for (let i = 0; i < input.length; i += 1) {
-        let code = input.charCodeAt(i);
-        if (code >= 0xd800 && code <= 0xdbff && i + 1 < input.length) {
-            const low = input.charCodeAt(i + 1);
-            if (low >= 0xdc00 && low <= 0xdfff) {
-                code = 0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00);
-                i += 1;
-            }
-        }
-        if (code >= 0xd800 && code <= 0xdfff)
-            code = 0xfffd;
-        if (code < 0x80)
-            bytes.push(code);
-        else if (code < 0x800)
-            bytes.push(0xc0 | (code >> 6), 0x80 | (code & 63));
-        else if (code < 0x10000)
-            bytes.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
-        else
-            bytes.push(0xf0 | (code >> 18), 0x80 | ((code >> 12) & 63), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63));
-    }
-    return bytes;
-}
-export function utf8Length(input) {
-    let length = 0;
-    for (let i = 0; i < input.length; i += 1) {
-        const code = input.charCodeAt(i);
-        if (code < 0x80)
-            length += 1;
-        else if (code < 0x800)
-            length += 2;
-        else if (code >= 0xd800 && code <= 0xdbff && i + 1 < input.length) {
-            const low = input.charCodeAt(i + 1);
-            if (low >= 0xdc00 && low <= 0xdfff) {
-                length += 4;
-                i += 1;
-            }
-            else {
-                length += 3;
-            }
-        }
-        else
-            length += 3;
-    }
-    return length;
-}
-function utf8Decode(bytes) {
-    let out = '';
-    let i = 0;
-    while (i < bytes.length) {
-        const b0 = bytes[i] ?? 0;
-        let code = 0xfffd;
-        let size = 1;
-        const cont = (offset) => {
-            const b = bytes[i + offset];
-            return b !== undefined && (b & 0xc0) === 0x80 ? b & 63 : -1;
-        };
-        if (b0 < 0x80) {
-            code = b0;
-        }
-        else if (b0 >= 0xc2 && b0 < 0xe0) {
-            const b1 = cont(1);
-            if (b1 >= 0) {
-                code = ((b0 & 31) << 6) | b1;
-                size = 2;
-            }
-        }
-        else if (b0 >= 0xe0 && b0 < 0xf0) {
-            const b1 = cont(1);
-            const b2 = cont(2);
-            if (b1 >= 0 && b2 >= 0) {
-                const value = ((b0 & 15) << 12) | (b1 << 6) | b2;
-                if (value >= 0x800 && (value < 0xd800 || value > 0xdfff)) {
-                    code = value;
-                    size = 3;
-                }
-            }
-        }
-        else if (b0 >= 0xf0 && b0 < 0xf5) {
-            const b1 = cont(1);
-            const b2 = cont(2);
-            const b3 = cont(3);
-            if (b1 >= 0 && b2 >= 0 && b3 >= 0) {
-                const value = ((b0 & 7) << 18) | (b1 << 12) | (b2 << 6) | b3;
-                if (value >= 0x10000 && value <= 0x10ffff) {
-                    code = value;
-                    size = 4;
-                }
-            }
-        }
-        out += String.fromCodePoint(code);
-        i += size;
-    }
-    return out;
-}
-export function bytesToBase64(bytes) {
-    let out = '';
-    for (let i = 0; i < bytes.length; i += 3) {
-        const b0 = bytes[i] ?? 0;
-        const b1 = bytes[i + 1];
-        const b2 = bytes[i + 2];
-        const chunk = (b0 << 16) | ((b1 ?? 0) << 8) | (b2 ?? 0);
-        out += ALPHABET.charAt((chunk >> 18) & 63) + ALPHABET.charAt((chunk >> 12) & 63);
-        out += b1 === undefined ? '=' : ALPHABET.charAt((chunk >> 6) & 63);
-        out += b2 === undefined ? '=' : ALPHABET.charAt(chunk & 63);
-    }
-    return out;
-}
-export function base64ToBytes(input) {
-    const clean = input.replace(/[\s=]/g, '').replace(/-/g, '+').replace(/_/g, '/');
-    const bytes = [];
-    let buffer = 0;
-    let bits = 0;
-    for (let i = 0; i < clean.length; i += 1) {
-        const value = ALPHABET.indexOf(clean.charAt(i));
-        if (value < 0)
-            throw new TypeError('invalid base64 text');
-        buffer = (buffer << 6) | value;
-        bits += 6;
-        if (bits >= 8) {
-            bits -= 8;
-            bytes.push((buffer >> bits) & 0xff);
-        }
-    }
-    return bytes;
-}
-export function base64Encode(input) {
-    return bytesToBase64(utf8Encode(text(input, 'base64Encode input')));
-}
-export function base64Decode(input) {
-    return utf8Decode(base64ToBytes(text(input, 'base64Decode input')));
-}
-export function createCrypto() {
-    return Object.freeze({
-        hmac(algorithm, key, data, enc) {
-            if (!HMAC_ALGORITHMS.includes(algorithm))
-                return Promise.reject(new TypeError(`unsupported hmac algorithm ${String(algorithm)}`));
-            return Promise.resolve().then(() => digest({ op: 'hmac', algorithm, key: text(key, 'hmac key'), data: text(data, 'hmac data'), encoding: encoding(enc) }));
-        },
-        sign(algorithm, key, data, enc) {
-            if (algorithm !== 'rsa-sha256')
-                return Promise.reject(new TypeError(`unsupported sign algorithm ${String(algorithm)}`));
-            return Promise.resolve().then(() => digest({ op: 'sign', algorithm, key: text(key, 'sign key'), data: text(data, 'sign data'), encoding: encoding(enc) }));
-        },
-        hash(algorithm, data, enc) {
-            if (!HASH_ALGORITHMS.includes(algorithm))
-                return Promise.reject(new TypeError(`unsupported hash algorithm ${String(algorithm)}`));
-            return Promise.resolve().then(() => digest({ op: 'hash', algorithm, data: text(data, 'hash data'), encoding: encoding(enc) }));
-        },
-        base64Encode,
-        base64Decode,
-    });
-}

package/runtime/guest/ctx.d.ts

@@ -1,30 +0,0 @@
-import type { CheckContext, PullContext } from '../../types.js';
-import type { DefinitionSummary } from '../../validate.js';
-import type { LogBuffer } from './log.js';
-/** 凭证句柄的前缀，与 shared 的 `SECRET_HANDLE_PREFIX` 同值。 */
-export declare const SECRET_HANDLE_PREFIX = "ontos:secret:";
-export declare function deepFreeze<T>(value: T): T;
-/** 相对路径的缺省基址：连接器 `hosts` 里按配置展开后的第一个确定主机（不含 `*`）；没有则为 null。 */
-export declare function defaultBaseUrl(hosts: readonly string[], config: Record<string, unknown>): string | null;
-/**
- * `ctx.secrets`：每个已配置的凭证字段给句柄 `ontos:secret:<字段>`；声明了 `exposeToScript` 的字段经 `secret.reveal`
- * 取明文。`ctx.secrets` 是同步读取的普通对象，所以这些字段在作业开始前各取一次，缓存到本次运行结束。
- */
-export declare function prepareSecrets(summary: DefinitionSummary, secretFields: readonly string[]): Promise<Readonly<Record<string, string>>>;
-export interface ContextOptions {
-    config: Record<string, unknown>;
-    secrets: Readonly<Record<string, string>>;
-    summary: DefinitionSummary;
-    logs: LogBuffer;
-    /** 宿主在 `definition` 回复里给出的生效并发（连接覆盖与连接器声明取小）；没给时为 null，只按声明。 */
-    effectiveConcurrency?: number | null;
-}
-/**
- * `definition` 宿主调用的回复里取生效并发：`{ rateLimit: { concurrency } }`。
- * 网关按「连接器声明与连接覆盖取小」限流，`ctx.map` 的上限要跟它一致，否则覆盖更小时会一直撞 429。
- * 回复为空（204）或不带这一项时返回 null。
- */
-export declare function concurrencyFromDefinitionReply(reply: unknown): number | null;
-export declare function createCheckContext(options: ContextOptions): CheckContext;
-/** 拉取的 `ctx`：`record` / `remove` / `checkpoint` 只产出标记，没有副作用；只有 `yield` 出来的标记算数。 */
-export declare function createPullContext(options: ContextOptions): PullContext;

package/runtime/guest/ctx.js

@@ -1,124 +0,0 @@
-import { errors } from '../../errors.js';
-import { call } from './bridge.js';
-import { createCache } from './cache.js';
-import { createCrypto } from './crypto.js';
-import { createDb } from './db.js';
-import { createHttp } from './http.js';
-import { DEFAULT_CONCURRENCY } from './limits.js';
-import { createMap } from './map.js';
-import { createPaginate } from './paginate.js';
-import { createSignal } from './signal.js';
-import { createTime } from './time.js';
-export const SECRET_HANDLE_PREFIX = 'ontos:secret:';
-const HOST_PLACEHOLDER = /\{config\.([^{}]*)\}/g;
-function isRecord(value) {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-export function deepFreeze(value) {
-    if (value === null || typeof value !== 'object')
-        return value;
-    for (const key of Object.keys(value))
-        deepFreeze(value[key]);
-    return Object.freeze(value);
-}
-function configString(config, path) {
-    let current = config;
-    for (const segment of path.split('.')) {
-        if (!isRecord(current) || !Object.hasOwn(current, segment))
-            return undefined;
-        current = current[segment];
-    }
-    return typeof current === 'string' && current.length > 0 ? current : undefined;
-}
-export function defaultBaseUrl(hosts, config) {
-    for (const host of hosts) {
-        let missing = false;
-        const expanded = host.replace(HOST_PLACEHOLDER, (_match, path) => {
-            const value = configString(config, path);
-            if (value === undefined)
-                missing = true;
-            return value ?? '';
-        });
-        if (!missing && !expanded.includes('*'))
-            return `https://${expanded}`;
-    }
-    return null;
-}
-export async function prepareSecrets(summary, secretFields) {
-    const secrets = {};
-    for (const field of secretFields) {
-        if (!summary.secretFields.includes(field))
-            continue;
-        if (summary.exposeToScript.includes(field)) {
-            const response = await call('secret.reveal', { field });
-            if (!isRecord(response) || typeof response.value !== 'string')
-                throw new TypeError(`secret.reveal returned no value for ${field}`);
-            secrets[field] = response.value;
-        }
-        else {
-            secrets[field] = `${SECRET_HANDLE_PREFIX}${field}`;
-        }
-    }
-    return Object.freeze(secrets);
-}
-export function concurrencyFromDefinitionReply(reply) {
-    if (!isRecord(reply) || !isRecord(reply.rateLimit))
-        return null;
-    const concurrency = reply.rateLimit.concurrency;
-    if (typeof concurrency !== 'number' || !Number.isFinite(concurrency) || concurrency < 1)
-        return null;
-    return Math.floor(concurrency);
-}
-function mapConcurrency(options) {
-    const declared = options.summary.rateLimit?.concurrency ?? DEFAULT_CONCURRENCY;
-    const effective = options.effectiveConcurrency ?? null;
-    return effective === null ? declared : Math.min(declared, effective);
-}
-function tableName(table) {
-    if (typeof table !== 'string' || table.length === 0)
-        throw new TypeError('table must be a non-empty string');
-    return table;
-}
-function plainObject(value, what) {
-    if (!isRecord(value))
-        throw new TypeError(`${what} must be an object`);
-    return value;
-}
-export function createCheckContext(options) {
-    const config = deepFreeze(options.config);
-    const http = createHttp(defaultBaseUrl(options.summary.hosts, config));
-    const ctx = {
-        config,
-        secrets: options.secrets,
-        http,
-        paginate: createPaginate(http),
-        db: createDb(),
-        map: createMap(mapConcurrency(options)),
-        cache: createCache(),
-        crypto: createCrypto(),
-        time: createTime(),
-        log: options.logs.logger(),
-        signal: createSignal(),
-    };
-    Object.defineProperty(ctx, 'files', {
-        get() {
-            throw new errors.NotAvailable('ctx.files');
-        },
-    });
-    return Object.freeze(ctx);
-}
-export function createPullContext(options) {
-    const base = createCheckContext(options);
-    const ctx = Object.create(null);
-    for (const key of Object.keys(base))
-        ctx[key] = base[key];
-    Object.defineProperty(ctx, 'files', {
-        get() {
-            throw new errors.NotAvailable('ctx.files');
-        },
-    });
-    ctx.record = (table, row) => Object.freeze({ __ontos: 'record', table: tableName(table), row: plainObject(row, 'row') });
-    ctx.remove = (table, key) => Object.freeze({ __ontos: 'remove', table: tableName(table), key: plainObject(key, 'key') });
-    ctx.checkpoint = (state) => Object.freeze({ __ontos: 'checkpoint', state });
-    return Object.freeze(ctx);
-}

package/runtime/guest/db.d.ts

@@ -1,6 +0,0 @@
-/**
- * `ctx.db`：经宿主的数据库代理执行只读查询。`db.query` 回第一批行与游标，未完时逐批 `db.next`；
- * 迭代中途退出（`break` / 抛错）时发 `db.close` 释放游标。参数在这里规整成 JSON 值（日期转 ISO 文本，bigint 转字符串）。
- */
-import type { ConnectorDb } from '../../types.js';
-export declare function createDb(): ConnectorDb;

package/runtime/guest/db.js

@@ -1,59 +0,0 @@
-import { call, isAborted, throwIfAborted } from './bridge.js';
-function isRecord(value) {
-    return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
-function param(value) {
-    if (value === undefined)
-        return null;
-    if (typeof value === 'bigint')
-        return value.toString();
-    if (value instanceof Date)
-        return value.toISOString();
-    return value;
-}
-function rowsOf(value) {
-    if (!isRecord(value) || !Array.isArray(value.rows) || typeof value.done !== 'boolean')
-        throw new TypeError('db host call returned a malformed response');
-    return value.rows;
-}
-async function* run(sql, params) {
-    throwIfAborted();
-    const first = await call('db.query', { sql, params: params.map(param) });
-    let rows = rowsOf(first);
-    let done = first.done;
-    const cursorId = typeof first.cursorId === 'string' ? first.cursorId : null;
-    if (!done && cursorId === null)
-        throw new TypeError('db host call returned more rows without a cursor');
-    try {
-        for (;;) {
-            for (const row of rows)
-                yield row;
-            if (done || cursorId === null)
-                return;
-            throwIfAborted();
-            const next = await call('db.next', { cursorId });
-            rows = rowsOf(next);
-            done = next.done;
-        }
-    }
-    finally {
-        if (!done && cursorId !== null) {
-            const closing = call('db.close', { cursorId });
-            if (isAborted())
-                closing.catch(() => undefined);
-            else
-                await closing;
-        }
-    }
-}
-export function createDb() {
-    return Object.freeze({
-        query(sql, params = []) {
-            if (typeof sql !== 'string' || sql.trim().length === 0)
-                throw new TypeError('ctx.db.query expects a SQL string');
-            if (!Array.isArray(params))
-                throw new TypeError('ctx.db.query params must be an array');
-            return run(sql, [...params]);
-        },
-    });
-}

package/runtime/guest/discover.d.ts

@@ -1,29 +0,0 @@
-/**
- * discover 与 check 作业：
- * - discover：有 `check` 时先跑 `check`，再跑 `discover`；结果经 `validateDiscoverResult` 校验后规整
- *   （缺省 `snapshot: 'partial'`，推送按名字），连同定义摘要一起回报。`discoverFirst` 的拉取作业复用同一段（`discoverDeclaration`）。
- * - check：只跑 `check`（没有时直接成功），结果 `{ kind: 'check', ok: true, logs }`。
- */
-import type { ConnectorDefinition } from '../../types.js';
-import { type DefinitionSummary, type NormalizedDiscoverResult } from '../../validate.js';
-import type { GuestResult } from '../protocol.js';
-import { type ContextOptions } from './ctx.js';
-import { type FailureMessage } from './failure.js';
-/** 跑 check（若有）+ discover，返回规整后的声明；连接器出错或结果不合法时返回失败回报。 */
-export declare function discoverDeclaration(definition: ConnectorDefinition, options: ContextOptions): Promise<{
-    ok: true;
-    discover: NormalizedDiscoverResult;
-} | {
-    ok: false;
-    failure: FailureMessage;
-}>;
-export declare function runDiscover(definition: ConnectorDefinition, options: ContextOptions & {
-    summary: DefinitionSummary;
-}): Promise<{
-    type: 'result';
-    result: GuestResult;
-} | FailureMessage>;
-export declare function runCheck(definition: ConnectorDefinition, options: ContextOptions): Promise<{
-    type: 'result';
-    result: GuestResult;
-} | FailureMessage>;

package/runtime/guest/discover.js

@@ -1,51 +0,0 @@
-import { normalizeDiscoverResult, validateDiscoverResult } from '../../validate.js';
-import { jsonParse, jsonStringify, throwIfAborted } from './bridge.js';
-import { createCheckContext } from './ctx.js';
-import { failure, failureFromError } from './failure.js';
-function plain(value) {
-    const text = jsonStringify(value);
-    if (typeof text !== 'string')
-        throw new TypeError('discover result is not serializable');
-    return jsonParse(text);
-}
-export async function discoverDeclaration(definition, options) {
-    const ctx = createCheckContext(options);
-    let raw;
-    try {
-        if (typeof definition.check === 'function')
-            await Reflect.apply(definition.check, definition, [ctx]);
-        throwIfAborted();
-        raw = plain(await Reflect.apply(definition.discover, definition, [ctx]));
-        throwIfAborted();
-    }
-    catch (error) {
-        return { ok: false, failure: failureFromError(error, options.logs.drain()) };
-    }
-    const issues = validateDiscoverResult(raw, definition);
-    if (issues.length > 0) {
-        const message = `invalid discover result: ${issues.map((issue) => `${issue.path || '(root)'}: ${issue.message}`).join('; ')}`;
-        return { ok: false, failure: failure('SCHEMA_ERROR', message, options.logs.drain()) };
-    }
-    return { ok: true, discover: normalizeDiscoverResult(raw) };
-}
-export async function runDiscover(definition, options) {
-    const outcome = await discoverDeclaration(definition, options);
-    if (!outcome.ok)
-        return outcome.failure;
-    return {
-        type: 'result',
-        result: { kind: 'discover', summary: options.summary, discover: outcome.discover, logs: options.logs.drain() },
-    };
-}
-export async function runCheck(definition, options) {
-    const ctx = createCheckContext(options);
-    try {
-        if (typeof definition.check === 'function')
-            await Reflect.apply(definition.check, definition, [ctx]);
-        throwIfAborted();
-    }
-    catch (error) {
-        return failureFromError(error, options.logs.drain());
-    }
-    return { type: 'result', result: { kind: 'check', ok: true, logs: options.logs.drain() } };
-}

package/runtime/guest/failure.d.ts

@@ -1,24 +0,0 @@
-/** 把连接器抛出的值归成失败回报。错误类别只按 `code` 字段判断（连接器可能抛任意值）。 */
-import type { GuestFailureCode, GuestLog } from '../protocol.js';
-export interface ErrorInfo {
-    code: string | null;
-    message: string;
-    stack: string | undefined;
-}
-/** 只取 code / message / stack 三个原始值；读属性抛错或类型不对时按普通异常处理。 */
-export declare function errorInfo(error: unknown): ErrorInfo;
-/**
- * 调用栈只留第一行（错误文案）与连接器代码自己的帧：宿主帧带 runner 的本地路径，guest 运行时的帧对作者没有意义。
- * 连接器代码以 `connector.js` 为文件名加载、包装函数与第一行同行，行号直接对应打包产物。
- */
-export declare function connectorStack(stack: string | undefined): string | undefined;
-export interface FailureMessage {
-    type: 'failure';
-    code: GuestFailureCode;
-    message: string;
-    stack?: string;
-    logs: GuestLog[];
-}
-export declare function failure(code: GuestFailureCode, message: string, logs: GuestLog[], stack?: string): FailureMessage;
-/** 连接器异常 → 失败回报：认得的 code 原样映射，其余（含宿主调用的错误码）按 FATAL_ERROR，文案带上原 code；运行已取消时一律 CANCELED。 */
-export declare function failureFromError(error: unknown, logs: GuestLog[]): FailureMessage;

package/runtime/guest/failure.js

@@ -1,63 +0,0 @@
-import { isAborted } from './bridge.js';
-import { CONNECTOR_FILENAME } from './limits.js';
-export function errorInfo(error) {
-    let code = null;
-    let message = 'connector threw a non-error value';
-    let stack;
-    if (error !== null && (typeof error === 'object' || typeof error === 'function')) {
-        const value = error;
-        try {
-            if (typeof value.code === 'string')
-                code = value.code;
-        }
-        catch {
-            code = null;
-        }
-        try {
-            if (typeof value.message === 'string')
-                message = value.message;
-        }
-        catch { }
-        try {
-            if (typeof value.stack === 'string')
-                stack = value.stack;
-        }
-        catch {
-            stack = undefined;
-        }
-    }
-    else if (typeof error === 'string') {
-        message = error;
-    }
-    return { code, message, stack };
-}
-export function connectorStack(stack) {
-    if (stack === undefined)
-        return undefined;
-    const lines = stack.split('\n');
-    const frames = lines.slice(1).filter((line) => line.includes(`${CONNECTOR_FILENAME}:`));
-    if (frames.length === 0)
-        return undefined;
-    return [lines[0] ?? '', ...frames].join('\n');
-}
-const CODE_MAP = Object.freeze({
-    AUTH_ERROR: 'AUTH_ERROR',
-    TRANSIENT_ERROR: 'TRANSIENT_ERROR',
-    RATE_LIMITED: 'TRANSIENT_ERROR',
-    SCHEMA_ERROR: 'SCHEMA_ERROR',
-    FATAL_ERROR: 'FATAL_ERROR',
-    CANCELED: 'CANCELED',
-});
-export function failure(code, message, logs, stack) {
-    return { type: 'failure', code, message, ...(stack !== undefined ? { stack } : {}), logs };
-}
-export function failureFromError(error, logs) {
-    const info = errorInfo(error);
-    const stack = connectorStack(info.stack);
-    if (isAborted())
-        return failure('CANCELED', 'run was canceled', logs, stack);
-    const mapped = info.code !== null && Object.hasOwn(CODE_MAP, info.code) ? CODE_MAP[info.code] : undefined;
-    if (mapped)
-        return failure(mapped, info.message, logs, stack);
-    return failure('FATAL_ERROR', info.code === null ? info.message : `${info.code}: ${info.message}`, logs, stack);
-}

package/runtime/guest/http.d.ts

@@ -1,4 +0,0 @@
-import type { HttpClient } from '../../types.js';
-/** 相对路径拼在 `auth.baseUrl` 上；没有时拼在缺省基址（连接器 `hosts` 里第一个确定的主机）上。 */
-export declare function resolveUrl(url: string, baseUrl: string | null): string;
-export declare function createHttp(defaultBaseUrl: string | null): HttpClient;