@deepexi/datasense-connector 0.0.0-stage → 3.0.1

package/testing/types.d.ts

@@ -0,0 +1,79 @@
+/**
+ * `./testing` 的公共类型与错误类（本地运行、本地模拟网关、源码加载器共用）。
+ */
+import type { GuestJob, HostMethod } from '../runtime/protocol.js';
+export type { GuestJob, HostMethod };
+/**
+ * 宿主：guest 的宿主调用（`http` / `db.*` / `crypto` / `cache.*` / `secret.reveal`）交给它应答。
+ * `definition` / `declaration` / `emit` 由执行器自己处理，不交给宿主。实现有本地模拟网关（`createLocalGateway`）与夹具回放。
+ * `call` 抛出的错误回给 guest 作为宿主调用失败（`code` 取错误的 `code`）；抛 `LocalRunError('DB_DRIVER_MISSING')` 时整次运行以该码结束。
+ */
+export interface LocalHost {
+    call(method: HostMethod, payload: unknown): Promise<unknown>;
+    close(): Promise<void>;
+    /** 可选：遮蔽文本里的 Secret 明文（执行器用它遮蔽结果里的日志与错误文案）；本地模拟网关提供。 */
+    redact?(text: string): string;
+    /**
+     * 可选：guest 上报定义摘要（`definition` 调用，push 作业没有）时由执行器先交给宿主，再回复 guest。
+     * 摘要在任何出站调用之前上报；CLI 用它按声明的 `hosts` 与本地配置展开出站范围（与平台 discover 运行取摘要相同）。
+     */
+    onDefinition?(summary: unknown): void;
+}
+/** 本地运行的结果；`error` 非 null 时运行失败（code 为 guest 的失败码，或 `TIMEOUT` / `DB_DRIVER_MISSING` 等本地码）。 */
+export interface LocalRunResult {
+    records: Array<{
+        table: string;
+        row: Record<string, unknown>;
+    }>;
+    removals: Array<{
+        table: string;
+        key: Record<string, unknown>;
+    }>;
+    checkpoints: Array<{
+        seq: number;
+        state: unknown;
+    }>;
+    logs: Array<{
+        at: string;
+        level: 'info' | 'warn' | 'error';
+        message: string;
+    }>;
+    rejected: Array<{
+        requestId: string;
+        code: string;
+        message: string;
+    }>;
+    declaration: {
+        summary: unknown;
+        discover: unknown;
+    } | null;
+    stopped: boolean;
+    error: {
+        code: string;
+        message: string;
+        stack: string | null;
+    } | null;
+}
+/**
+ * 执行器：同一个作业交给不同的执行方式；宿主调用一律交给传入的 host。
+ * `limit` 由执行器实现：累计记录数达到 limit 时对 `emit` 回 `{ stop: true }`，超出部分截掉。
+ */
+export interface GuestExecutor {
+    run(job: GuestJob, host: LocalHost, options?: {
+        limit?: number;
+        timeoutMs?: number;
+    }): Promise<LocalRunResult>;
+}
+export type LocalRunErrorCode = 'DB_DRIVER_MISSING' | 'HOST_DENIED' | 'TIMEOUT';
+/** 本地运行环境的问题（不是连接器代码的问题）。 */
+export declare class LocalRunError extends Error {
+    readonly code: LocalRunErrorCode;
+    constructor(code: LocalRunErrorCode, message: string);
+}
+export type SourceLoadErrorCode = 'IMPORT_FORBIDDEN' | 'PATH_ESCAPE' | 'SYMLINK' | 'MANIFEST_INVALID' | 'SYNTAX_ERROR' | 'TOO_LARGE';
+/** 源码目录加载失败；`file` 为相对源码根目录的路径（与整个目录相关时为 null）。 */
+export declare class SourceLoadError extends Error {
+    readonly code: SourceLoadErrorCode;
+    readonly file: string | null;
+    constructor(code: SourceLoadErrorCode, message: string, file?: string | null);
+}

package/testing/types.js

@@ -0,0 +1,18 @@
+export class LocalRunError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.name = 'LocalRunError';
+        this.code = code;
+    }
+}
+export class SourceLoadError extends Error {
+    code;
+    file;
+    constructor(code, message, file = null) {
+        super(message);
+        this.name = 'SourceLoadError';
+        this.code = code;
+        this.file = file;
+    }
+}

package/types.d.ts

@@ -0,0 +1,302 @@
+/**
+ * 连接器 SDK 的声明类型。
+ *
+ * 这些类型描述连接器作者写出的定义；运行时它们只是普通对象。连接器打包后在 `node:vm` 里执行，
+ * `ctx` 由 SDK 的 guest 运行时提供，所以这里不引用任何 Node / Bun 内置模块。
+ *
+ * 外部写操作（`writes`）与文件来源（`ctx.files`）不在本期：类型里不出现 `writes`，
+ * `ctx.files` 的类型是 `never`，访问时抛 `errors.NotAvailable('ctx.files')`。
+ */
+export type ScalarKind = 'string' | 'integer' | 'decimal' | 'boolean' | 'date' | 'instant' | 'json';
+/**
+ * 字段的展示元数据，`t.*()` 末尾的可选参数。随声明上报，界面据此显示表单标签、说明与占位提示。
+ * `placeholder` 只对标量与 `t.secret()` 有意义；长度上限见 `PRESENTATION_LIMITS`。
+ */
+export interface FieldMeta {
+    title?: string;
+    description?: string;
+    placeholder?: string;
+}
+export interface ScalarType {
+    readonly kind: ScalarKind;
+    readonly precision?: number;
+    readonly scale?: number;
+    readonly title?: string;
+    /** 作为表列时是列说明。 */
+    readonly description?: string;
+    readonly placeholder?: string;
+}
+/** 凭证字段。只能出现在连接配置的顶层；脚本里拿到的是句柄，`exposeToScript` 时可取明文。 */
+export interface SecretType {
+    readonly kind: 'secret';
+    readonly exposeToScript?: boolean;
+    readonly title?: string;
+    readonly description?: string;
+    readonly placeholder?: string;
+}
+export interface ObjectType {
+    readonly kind: 'object';
+    readonly shape: Readonly<Record<string, SdkType>>;
+    readonly title?: string;
+    readonly description?: string;
+}
+export interface ArrayType {
+    readonly kind: 'array';
+    readonly item: SdkType;
+    readonly title?: string;
+    readonly description?: string;
+}
+export type SdkType = ScalarType | SecretType | ObjectType | ArrayType;
+export interface TableDeclaration {
+    columns: Record<string, ScalarType>;
+    primaryKey: string[];
+    retention?: {
+        column: string;
+        days: number;
+    };
+    purgeColumns?: {
+        tenant?: string;
+        user?: string;
+    };
+    indexes?: string[][];
+    /** 表的显示名，≤ 40 字符。 */
+    title?: string;
+    /** 表说明，≤ 200 字符。 */
+    description?: string;
+}
+export type SyncMode = 'full' | 'incremental';
+/** discover 返回的单个数据流：输出的表、支持的同步方式与数据流参数。 */
+export interface StreamDiscovery {
+    tables: Record<string, TableDeclaration>;
+    syncModes: SyncMode[];
+    /** 全量是否是完整快照（未出现的主键视为删除）；缺省 `'partial'`。 */
+    snapshot?: 'complete' | 'partial';
+    /** 同步配置上可填写的数据流参数，作为 `input.streamConfig` 传入。 */
+    params?: ObjectType;
+}
+export interface PushDiscovery {
+    tables: Record<string, TableDeclaration>;
+}
+export interface DiscoverResult {
+    streams?: Record<string, StreamDiscovery>;
+    /** 按推送名声明输出的表；只接受按名字的写法。 */
+    push?: Record<string, PushDiscovery>;
+}
+export interface StreamInput<S = unknown> {
+    mode: SyncMode;
+    /** 上次提交的断点；首次为 undefined。 */
+    state: S | undefined;
+    streamConfig: Readonly<Record<string, unknown>>;
+}
+export interface RecordItem {
+    readonly __ontos: 'record';
+    table: string;
+    row: Record<string, unknown>;
+}
+export interface RemoveItem {
+    readonly __ontos: 'remove';
+    table: string;
+    key: Record<string, unknown>;
+}
+export interface CheckpointItem {
+    readonly __ontos: 'checkpoint';
+    state: unknown;
+}
+export type StreamItem = RecordItem | RemoveItem | CheckpointItem;
+export type StreamFunction = (ctx: PullContext, input: StreamInput) => AsyncIterable<StreamItem>;
+export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE' | 'HEAD';
+export interface HttpResponse<T = unknown> {
+    status: number;
+    /** 最终地址（跟随重定向之后）。 */
+    url: string;
+    headers: Record<string, string>;
+    /** `content-type` 为 JSON 时是解析后的值，否则同 `text`。 */
+    data: T;
+    text: string;
+}
+/** 登录换来的会话：附加的请求头与相对路径的基址。 */
+export interface HttpAuth {
+    headers?: Record<string, string>;
+    baseUrl?: string;
+}
+export interface HttpOptions {
+    headers?: Record<string, string>;
+    query?: Record<string, unknown>;
+    body?: unknown;
+    auth?: HttpAuth;
+    timeoutMs?: number;
+    /** 为 true 时不按状态码抛错、不解析 JSON，原样返回。 */
+    raw?: boolean;
+    /**
+     * 上游 502 / 503 / 504 与网关超时是否自动重试（请求可能已被上游处理）。
+     * 缺省只对幂等方法（GET / HEAD / PUT / DELETE）重试；POST / PATCH 需显式传 `true`，传 `false` 则一律不重试。
+     * 网关限流与上游 429（请求未被处理）不受影响，总会等待后重发。
+     */
+    retry?: boolean;
+}
+export interface HttpClient {
+    get<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
+    post<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
+    put<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
+    patch<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
+    delete<T = unknown>(url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
+    request<T = unknown>(method: HttpMethod, url: string, options?: HttpOptions): Promise<HttpResponse<T>>;
+}
+/** 分页迭代产出的一页。 */
+export interface Page<Row = unknown> {
+    rows: Row[];
+    /** 本页各行取值的最大值（字符串按字典序、数字按大小）；空页为 undefined。 */
+    maxOf<V>(fn: (row: Row) => V): V | undefined;
+}
+/** 按页码翻页。没有 `totalPages` 时翻到空页为止。 */
+export interface PagePaginateOptions<Res, Row> {
+    /** 缺省 1。 */
+    firstPage?: number;
+    request(page: number): Promise<HttpResponse<Res>>;
+    rows(res: HttpResponse<Res>): Row[];
+    totalPages?(res: HttpResponse<Res>): number;
+}
+/** 按偏移量翻页。没有 `total` 时翻到不满一页为止。 */
+export interface OffsetPaginateOptions<Res, Row> {
+    limit: number;
+    /** 缺省 0。 */
+    startOffset?: number;
+    request(offset: number, limit: number): Promise<HttpResponse<Res>>;
+    rows(res: HttpResponse<Res>): Row[];
+    total?(res: HttpResponse<Res>): number;
+}
+/** 按游标翻页。`nextCursor` 返回空值时结束。 */
+export interface CursorPaginateOptions<Res, Row> {
+    request(cursor: string | undefined): Promise<HttpResponse<Res>>;
+    rows(res: HttpResponse<Res>): Row[];
+    nextCursor(res: HttpResponse<Res>): string | null | undefined;
+}
+/** 按下一页链接翻页。缺省取响应头 `Link` 里 `rel="next"` 的地址。 */
+export interface LinkPaginateOptions<Res, Row> {
+    url: string;
+    options?: HttpOptions;
+    rows(res: HttpResponse<Res>): Row[];
+    next?(res: HttpResponse<Res>): string | null | undefined;
+}
+/** 数据库按键分页：`query` 以上一页最后一行的键（首页为 undefined）查下一页，返回空页时结束。 */
+export interface KeysetPaginateOptions<Row, Key> {
+    query(last: Key | undefined): AsyncIterable<Row>;
+    key(row: Row): Key;
+}
+export interface Paginate {
+    page<Res = unknown, Row = unknown>(options: PagePaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
+    offset<Res = unknown, Row = unknown>(options: OffsetPaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
+    cursor<Res = unknown, Row = unknown>(options: CursorPaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
+    link<Res = unknown, Row = unknown>(options: LinkPaginateOptions<Res, Row>): AsyncIterable<Page<Row>>;
+    keyset<Row = Record<string, unknown>, Key = unknown>(options: KeysetPaginateOptions<Row, Key>): AsyncIterable<Page<Row>>;
+}
+export interface ConnectorDb {
+    /** 一次查询一个只读事务；只接受参数化查询（`$1`、`$2`…），一条语句。 */
+    query<R = Record<string, unknown>>(sql: string, params?: unknown[]): AsyncIterable<R>;
+}
+export interface ConnectorCache {
+    get(key: string): Promise<string | undefined>;
+    set(key: string, value: string, ttlSeconds: number): Promise<void>;
+    getOrSet(key: string, ttlSeconds: number, fn: () => Promise<string>): Promise<string>;
+}
+export type HmacAlgorithm = 'sha1' | 'sha256' | 'sha512';
+export type SignAlgorithm = 'rsa-sha256';
+export type HashAlgorithm = 'md5' | 'sha1' | 'sha256' | 'sha512';
+export type DigestEncoding = 'hex' | 'base64';
+/** `key` 可以是凭证句柄（`ctx.secrets.<字段>`），由网关代算。 */
+export interface ConnectorCrypto {
+    hmac(algorithm: HmacAlgorithm, key: string, data: string, encoding?: DigestEncoding): Promise<string>;
+    sign(algorithm: SignAlgorithm, key: string, data: string, encoding?: DigestEncoding): Promise<string>;
+    hash(algorithm: HashAlgorithm, data: string, encoding?: DigestEncoding): Promise<string>;
+    base64Encode(text: string): string;
+    base64Decode(b64: string): string;
+}
+export interface ConnectorTime {
+    /** 当前时间，ISO 8601 UTC。 */
+    now(): string;
+    /** 把来源给的本地时间文本按时区换成 ISO 8601 UTC。 */
+    toUtc(local: string, timeZone: string, format?: string): string;
+}
+export interface ConnectorSignal {
+    readonly aborted: boolean;
+    onAbort(fn: () => void): void;
+}
+export interface ConnectorLog {
+    info(message: string, data?: Record<string, unknown>): void;
+    warn(message: string, data?: Record<string, unknown>): void;
+    error(message: string, data?: Record<string, unknown>): void;
+}
+export interface PullContext {
+    readonly config: Readonly<Record<string, unknown>>;
+    /** 凭证句柄（`ontos:secret:<字段>`）；`exposeToScript` 字段是明文。 */
+    readonly secrets: Readonly<Record<string, string>>;
+    readonly http: HttpClient;
+    readonly paginate: Paginate;
+    readonly db: ConnectorDb;
+    /** 保留名字；访问即抛 `errors.NotAvailable('ctx.files')`。 */
+    readonly files: never;
+    map<T, R>(items: readonly T[], fn: (item: T) => Promise<R>, options?: {
+        concurrency?: number;
+    }): Promise<R[]>;
+    readonly cache: ConnectorCache;
+    readonly crypto: ConnectorCrypto;
+    readonly time: ConnectorTime;
+    record(table: string, row: Record<string, unknown>): RecordItem;
+    remove(table: string, key: Record<string, unknown>): RemoveItem;
+    checkpoint(state: unknown): CheckpointItem;
+    readonly log: ConnectorLog;
+    readonly signal: ConnectorSignal;
+}
+export type CheckContext = Omit<PullContext, 'record' | 'remove' | 'checkpoint'>;
+export type DiscoverContext = CheckContext;
+export interface PushRequest {
+    readonly id: string;
+    readonly receivedAt: string;
+    readonly headers: Readonly<Record<string, string>>;
+    readonly body: string;
+    json<T = unknown>(): T;
+}
+export interface PushContext {
+    readonly config: Readonly<Record<string, unknown>>;
+    record(table: string, row: Record<string, unknown>): void;
+    remove(table: string, key: Record<string, unknown>): void;
+    readonly log: ConnectorLog;
+    readonly time: {
+        now(): string;
+    };
+}
+export interface PushHandler {
+    verify?(ctx: PushContext, req: PushRequest): void | Promise<void>;
+    map(ctx: PushContext, req: PushRequest): void | Promise<void>;
+}
+export interface ConnectorRateLimit {
+    perSecond: number;
+    concurrency: number;
+}
+export interface ConnectorDefinition {
+    name: string;
+    /** semver，形如 `1.2.3` */
+    version: string;
+    config: ObjectType;
+    /** 出站主机范围：精确主机名或 `*.后缀`，可含 `{config.<路径>}`。数据流用 `ctx.http` 时必填。 */
+    hosts?: string[];
+    /** 数据流用 `ctx.db` 时声明；本期只支持 PostgreSQL。 */
+    db?: {
+        engine: 'postgresql';
+    };
+    rateLimit?: ConnectorRateLimit;
+    check?(ctx: CheckContext): void | Promise<void>;
+    discover(ctx: DiscoverContext): DiscoverResult | Promise<DiscoverResult>;
+    streams?: Record<string, StreamFunction>;
+    /** 按推送名声明：`push: { default: { verify?, map } }`。第 1 期的单对象写法 `push: { verify?, map }` 已删除。 */
+    push?: Record<string, PushHandler>;
+}
+/** 第 1 期的单对象推送写法：`map` / `verify` 直接是函数（按名字的写法里每个值都是对象）。 */
+export declare function isSingleObjectPush(push: unknown): boolean;
+/** 单对象推送写法的校验文案（`defineConnector` 抛错与 `validateConnectorDefinition` 共用）。 */
+export declare const SINGLE_OBJECT_PUSH_MESSAGE = "push must be declared by name, e.g. push: { default: { map } }; the single-object form push: { map } is not supported";
+/**
+ * 给连接器定义提供类型推断，原样返回定义。只做一项检查：单对象推送写法 `push: { map }` 直接抛 `TypeError`
+ * （其余校验由平台加载时的 `validateConnectorDefinition` 做）。
+ */
+export declare function defineConnector<T extends ConnectorDefinition>(definition: T): T;

package/types.js

@@ -0,0 +1,13 @@
+export function isSingleObjectPush(push) {
+    if (push === null || typeof push !== 'object' || Array.isArray(push))
+        return false;
+    const value = push;
+    return typeof value.map === 'function' || typeof value.verify === 'function';
+}
+export const SINGLE_OBJECT_PUSH_MESSAGE = 'push must be declared by name, e.g. push: { default: { map } }; the single-object form push: { map } is not supported';
+export function defineConnector(definition) {
+    if (definition !== null && typeof definition === 'object' && isSingleObjectPush(definition.push)) {
+        throw new TypeError(SINGLE_OBJECT_PUSH_MESSAGE);
+    }
+    return definition;
+}

package/validate.d.ts

@@ -0,0 +1,78 @@
+/**
+ * 连接器定义、discover 结果与连接配置的校验。平台（meta、connector、runner）与连接器作者共用同一套规则；
+ * 输入可能来自 vm 里的不可信代码，所以按 unknown 逐层检查，只返回问题列表、不抛异常。
+ */
+import { type ConnectorDefinition, type DiscoverResult, type ObjectType, type SyncMode, type TableDeclaration } from './types.js';
+/** 表名、连接器名、数据流名与推送名的规则，与 shared 的 DATA_FOUNDATION_NAME_PATTERN 一致。 */
+export declare const SDK_NAME_PATTERN: RegExp;
+/** 保留的推送名：`dev` 会和接收地址的 `/dev` 后缀冲突。与 shared 同值，SDK 不依赖 shared，自己定义。 */
+export declare const RESERVED_PUSH_NAMES: readonly string[];
+/** 缺省推送名（接收地址不带推送名后缀时对应它）。 */
+export declare const DEFAULT_PUSH_NAME = "default";
+/**
+ * 展示元数据的长度上限（字符数按 Unicode 码点计，字节数按解码后的字节计）。
+ * `title` / `description` / `readmeBytes` / `iconBytes` / `iconMinPx` 是连接器清单（package.json、README.md、图标）的上限；
+ * 其余是 `t.*()` 字段与表声明的上限。
+ */
+export declare const PRESENTATION_LIMITS: {
+    readonly title: 40;
+    readonly description: 120;
+    readonly readmeBytes: 65536;
+    readonly iconBytes: 262144;
+    readonly iconMinPx: 128;
+    readonly fieldTitle: 40;
+    readonly fieldDescription: 200;
+    readonly placeholder: 100;
+    readonly tableTitle: 40;
+    readonly tableDescription: 200;
+};
+export interface DeclarationIssue {
+    path: string;
+    message: string;
+}
+/** 定义摘要：discover 运行由 guest 上报，平台据此判断出站主机范围、凭证字段与推送 / 数据流列表。 */
+export interface DefinitionSummary {
+    name: string;
+    version: string;
+    hosts: string[];
+    db: {
+        engine: 'postgresql';
+    } | null;
+    rateLimit: {
+        perSecond: number;
+        concurrency: number;
+    } | null;
+    pushes: string[];
+    streams: string[];
+    secretFields: string[];
+    exposeToScript: string[];
+    /** `config` 声明（`t.*()` 产出的普通对象），平台保存连接配置时按它校验；旧声明没有这个字段。 */
+    config?: Record<string, unknown>;
+}
+/** 规整后的 discover 结果：`snapshot` 补缺省值，与 shared 的 `ConnectorDeclaration` 去掉 `definition` 同形。 */
+export interface NormalizedDiscoverResult {
+    streams: Record<string, {
+        tables: Record<string, TableDeclaration>;
+        syncModes: SyncMode[];
+        snapshot: 'complete' | 'partial';
+        params?: ObjectType;
+    }>;
+    pushes: Record<string, {
+        tables: Record<string, TableDeclaration>;
+    }>;
+}
+export declare function validateConnectorDefinition(definition: unknown): DeclarationIssue[];
+/**
+ * 校验 discover 结果：数据流与推送都要和定义一一对应，表名在一个连接下唯一。
+ * `push` 只接受按名字的写法 `{ <推送名>: { tables } }`。
+ */
+export declare function validateDiscoverResult(result: unknown, definition: ConnectorDefinition): DeclarationIssue[];
+/** 规整一份已通过 `validateDiscoverResult` 的结果：缺省 `snapshot` 为 `'partial'`，缺省的 `streams` / `push` 为空。 */
+export declare function normalizeDiscoverResult(result: DiscoverResult): NormalizedDiscoverResult;
+/** 定义摘要；只含可 JSON 序列化的值，用于跨 vm 边界上报与声明差异。各列表按声明顺序。 */
+export declare function summarizeDefinition(definition: ConnectorDefinition): DefinitionSummary;
+/**
+ * 按连接器的 `config` 声明校验连接配置的值。所有非 secret 字段必填；
+ * 顶层 `t.secret()` 字段的值走 Secret 引用，不在配置值里校验，出现在配置值里也忽略。
+ */
+export declare function validateConfig(schema: ObjectType, config: unknown): DeclarationIssue[];