@meshagent/meshagent 0.38.4 → 0.39.0

package/dist/esm/room-client.d.ts

@@ -1,4 +1,4 @@
-import { DatabaseClient } from "./database-client";
+import { DatasetsClient } from "./datasets-client";
 import { DeveloperClient } from "./developer-client";
 import { type EventHandler, type EventName } from "./event-emitter";
 import { MessagingClient } from "./messaging-client";
@@ -52,7 +52,7 @@ export declare class RoomClient {
     readonly developer: DeveloperClient;
     readonly messaging: MessagingClient;
     readonly queues: QueuesClient;
-    readonly database: DatabaseClient;
+    readonly datasets: DatasetsClient;
     readonly agents: AgentsClient;
     readonly secrets: SecretsClient;
     readonly containers: ContainersClient;
@@ -111,7 +111,9 @@ export declare class RoomClient {
     on(eventName: EventName, callback: EventHandler<RoomEvent>): void;
     off(eventName: EventName, callback: EventHandler<RoomEvent>): void;
     emit(event: RoomEvent): void;
-    listen(): AsyncIterable<RoomEvent>;
+    listen({ abortSignal }?: {
+        abortSignal?: AbortSignal;
+    }): AsyncIterable<RoomEvent>;
     waitForClose(): Promise<void>;
     waitUntilConnected(): Promise<void>;
     _waitUntilConnectedForMessages(): Promise<void>;

package/dist/esm/room-client.js

@@ -1,5 +1,5 @@
 import { Completer } from "./completer";
-import { DatabaseClient } from "./database-client";
+import { DatasetsClient } from "./datasets-client";
 import { DeveloperClient } from "./developer-client";
 import { EventEmitter } from "./event-emitter";
 import { MessagingClient } from "./messaging-client";
@@ -244,7 +244,7 @@ export class RoomClient {
         this.developer = new DeveloperClient({ room: this });
         this.messaging = new MessagingClient({ room: this });
         this.queues = new QueuesClient({ room: this });
-        this.database = new DatabaseClient({ room: this });
+        this.datasets = new DatasetsClient({ room: this });
         this.agents = new AgentsClient({ room: this });
         this.secrets = new SecretsClient({
             room: this,
@@ -298,8 +298,74 @@ export class RoomClient {
         this._eventsController.add(event);
         this._eventEmitter.emit(event.name, event);
     }
-    listen() {
-        return this._eventsController.stream;
+    listen({ abortSignal } = {}) {
+        if (abortSignal === undefined || abortSignal === null) {
+            return this._eventsController.stream;
+        }
+        const source = this._eventsController.stream;
+        return {
+            [Symbol.asyncIterator]() {
+                const eventIterator = source[Symbol.asyncIterator]();
+                let abortReject = null;
+                let cleanedUp = false;
+                const abortError = () => {
+                    if (abortSignal.reason != null) {
+                        return abortSignal.reason;
+                    }
+                    const error = new Error("Aborted");
+                    error.name = "AbortError";
+                    return error;
+                };
+                const cleanup = async () => {
+                    if (cleanedUp) {
+                        return;
+                    }
+                    cleanedUp = true;
+                    abortSignal.removeEventListener("abort", onAbort);
+                    await eventIterator.return?.();
+                };
+                const onAbort = () => {
+                    const reject = abortReject;
+                    abortReject = null;
+                    reject?.(abortError());
+                    void cleanup();
+                };
+                abortSignal.addEventListener("abort", onAbort, { once: true });
+                return {
+                    async next() {
+                        if (abortSignal.aborted) {
+                            await cleanup();
+                            return Promise.reject(abortError());
+                        }
+                        const abortPromise = new Promise((_, reject) => {
+                            abortReject = reject;
+                        });
+                        try {
+                            const result = await Promise.race([eventIterator.next(), abortPromise]);
+                            if (result.done) {
+                                await cleanup();
+                            }
+                            return result;
+                        }
+                        catch (error) {
+                            await cleanup();
+                            throw error;
+                        }
+                        finally {
+                            abortReject = null;
+                        }
+                    },
+                    async return() {
+                        await cleanup();
+                        return { done: true, value: undefined };
+                    },
+                    async throw(e) {
+                        await cleanup();
+                        return Promise.reject(e);
+                    },
+                };
+            },
+        };
     }
     async waitForClose() {
         await this._roomClosed.fut;

package/dist/node/datasets-client.d.ts

@@ -0,0 +1,415 @@
+import { Schema, Table } from "apache-arrow";
+import { RoomClient } from "./room-client";
+export type CreateMode = "create" | "overwrite" | "create_if_not_exists";
+export interface TableRef {
+    name: string;
+    namespace?: string[];
+    alias?: string;
+    branch?: string;
+    version?: number;
+}
+export interface TableVersion {
+    version: number;
+    timestamp: Date;
+    metadata: Record<string, unknown>;
+}
+export interface TableIndex {
+    name: string;
+    columns: string[];
+    type: string;
+    fields: number[];
+    typeUrl: string | null;
+    numRowsIndexed: number | null;
+    numSegments: number | null;
+    totalSizeBytes: number | null;
+    details: Record<string, unknown>;
+    statistics: Record<string, unknown>;
+}
+export type DatasetVectorIndexType = "IVF_PQ" | "IVF_HNSW_PQ" | "IVF_HNSW_SQ" | "IVF_RQ";
+export type DatasetScalarIndexType = "BTREE" | "BITMAP" | "LABEL_LIST" | "NGRAM" | "ZONEMAP" | "INVERTED" | "FTS" | "BLOOMFILTER" | "RTREE";
+export type DatasetIndexType = DatasetVectorIndexType | DatasetScalarIndexType;
+export interface DatasetIndexConfig {
+    column: string | string[];
+    index_type: DatasetIndexType;
+    name?: string;
+    metric?: string;
+    replace?: boolean;
+    num_partitions?: number;
+    ivf_centroids?: number[][];
+    pq_codebook?: number[][];
+    num_sub_vectors?: number;
+    accelerator?: string;
+    index_cache_size?: number;
+    shuffle_partition_batches?: number;
+    shuffle_partition_concurrency?: number;
+    ivf_centroids_file?: string;
+    precomputed_partition_dataset?: string;
+    filter_nan?: boolean;
+    train?: boolean;
+    fragment_ids?: number[];
+    index_uuid?: string;
+    target_partition_size?: number;
+    skip_transpose?: boolean;
+    num_bits?: number;
+    index_file_version?: string;
+    max_level?: number;
+    m?: number;
+    ef_construction?: number;
+    with_position?: boolean;
+    memory_limit?: number;
+    num_workers?: number;
+    skip_merge?: boolean;
+    base_tokenizer?: string;
+    language?: string;
+    max_token_length?: number;
+    lower_case?: boolean;
+    stem?: boolean;
+    remove_stop_words?: boolean;
+    custom_stop_words?: string[];
+    ascii_folding?: boolean;
+}
+export interface DatasetOptimizeConfig {
+    compact_files?: boolean;
+    optimize_indices?: boolean;
+    cleanup_old_versions?: boolean;
+    target_rows_per_fragment?: number;
+    max_rows_per_group?: number;
+    max_bytes_per_file?: number;
+    materialize_deletions?: boolean;
+    materialize_deletions_threshold?: number;
+    defer_index_remap?: boolean;
+    num_threads?: number;
+    batch_size?: number;
+    compaction_mode?: "reencode" | "try_binary_copy" | "force_binary_copy";
+    binary_copy_read_batch_bytes?: number;
+    num_indices_to_merge?: number;
+    index_names?: string[];
+    retrain?: boolean;
+    older_than_seconds?: number;
+    retain_versions?: number;
+    delete_unverified?: boolean;
+    error_if_tagged_old_versions?: boolean;
+    delete_rate_limit?: number;
+}
+export interface DatasetOptimizeResult {
+    compaction: Record<string, unknown> | null;
+    optimizedIndices: boolean;
+    cleanup: Record<string, unknown> | null;
+}
+export interface DatasetTableStats {
+    dataset: Record<string, unknown>;
+    data: Record<string, unknown>;
+}
+export interface TableBranch {
+    name: string;
+    parentBranch: string | null;
+    parentVersion: number | null;
+    createdAt: Date | null;
+    manifestSize: number | null;
+}
+export interface DatasetSqlQuery {
+    kind: "query";
+    schema: Schema;
+    queryId: string;
+}
+export interface DatasetSqlStatement {
+    kind: "statement";
+    rowsAffected: number;
+}
+export type DatasetSqlExecution = DatasetSqlQuery | DatasetSqlStatement;
+export type DatasetSqlCancelStatus = "cancelled" | "cancelling" | "not_cancellable";
+export interface DatasetSqlCancelResult {
+    status: DatasetSqlCancelStatus;
+}
+export declare abstract class DatasetValueEncoder {
+    abstract encodeDatasetValue(): unknown;
+}
+export declare class DatasetExpression extends DatasetValueEncoder {
+    readonly expression: string;
+    constructor(expression: string);
+    encodeDatasetValue(): Record<string, string>;
+    toString(): string;
+}
+export declare class DatasetDate extends DatasetValueEncoder {
+    readonly value: string;
+    constructor(value: string);
+    encodeDatasetValue(): Record<string, string>;
+    toString(): string;
+}
+export type DatasetJsonScalarValue = null | boolean | number | string;
+export type DatasetJsonValue = DatasetJsonScalarValue | DatasetJsonValue[] | {
+    [key: string]: DatasetJsonValue;
+};
+export declare class DatasetStruct extends DatasetValueEncoder {
+    readonly fields: Record<string, DatasetValue>;
+    constructor(fields: Record<string, DatasetValue>);
+    toJson(): Record<string, unknown>;
+    encodeDatasetValue(): Record<string, Record<string, unknown>>;
+}
+export declare class DatasetJson extends DatasetValueEncoder {
+    readonly value: DatasetJsonValue;
+    constructor(value: DatasetJsonValue);
+    toJson(): DatasetJsonValue;
+    encodeDatasetValue(): Record<string, DatasetJsonValue>;
+}
+export type DatasetScalarValue = null | boolean | number | string | Uint8Array | DatasetUuid | Date;
+export type DatasetValue = DatasetScalarValue | DatasetValueEncoder | DatasetValue[];
+export type DatasetRecord = Record<string, DatasetValue>;
+export type DatasetRows = DatasetRecord[];
+export type DatasetRowChunks = AsyncIterable<DatasetRows> | Iterable<DatasetRows>;
+export type DatasetWhere = string | Record<string, DatasetValue>;
+type DatasetRoomInvoker = Pick<RoomClient, "invoke" | "invokeStream">;
+export declare class DatasetUuid {
+    readonly value: string;
+    constructor(value: string);
+    toString(): string;
+}
+type ArrowTableChunks = Iterable<Table> | AsyncIterable<Table>;
+export declare class DatasetsClient {
+    private room;
+    constructor({ room }: {
+        room: DatasetRoomInvoker;
+    });
+    private _unexpectedResponseError;
+    private invoke;
+    private invokeContent;
+    private invokeStream;
+    private drainWriteStream;
+    private streamRows;
+    private streamArrow;
+    listTables({ namespace, branch }?: {
+        namespace?: string[];
+        branch?: string;
+    }): Promise<string[]>;
+    private createTable;
+    createTableWithSchema({ name, schema, data, mode, namespace, branch, metadata }: {
+        name: string;
+        schema?: Schema;
+        data?: Iterable<Table> | Table;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    createTableFromData({ name, data, mode, namespace, branch, metadata }: {
+        name: string;
+        data?: Iterable<Table> | Table;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    createTableFromDataStream({ name, chunks, schema, mode, namespace, branch, metadata }: {
+        name: string;
+        chunks: ArrowTableChunks;
+        schema?: Schema;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    createTableFromJsonData({ name, data, mode, namespace, branch, metadata }: {
+        name: string;
+        data?: DatasetRows;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    dropTable({ name, ignoreMissing, namespace, branch }: {
+        name: string;
+        ignoreMissing?: boolean;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    dropIndex({ table, name, namespace, branch }: {
+        table: string;
+        name: string;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    addColumns({ table, newColumns, namespace, branch }: {
+        table: string;
+        newColumns: Record<string, string> | Schema;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    dropColumns({ table, columns, namespace, branch }: {
+        table: string;
+        columns: string[];
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    insert({ table, records, namespace, branch }: {
+        table: string;
+        records: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    insertStream({ table, chunks, namespace, branch }: {
+        table: string;
+        chunks: ArrowTableChunks;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    update({ table, where, values, namespace, branch }: {
+        table: string;
+        where: string;
+        values: DatasetRecord;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    delete({ table, where, namespace, branch }: {
+        table: string;
+        where: string;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    merge({ table, on, records, namespace, branch }: {
+        table: string;
+        on: string;
+        records: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    mergeStream({ table, on, chunks, namespace, branch }: {
+        table: string;
+        on: string;
+        chunks: ArrowTableChunks;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    sql({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<Table[]>;
+    openSqlQuery({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<DatasetSqlQuery>;
+    executeSql({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<DatasetSqlExecution>;
+    sqlStream({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): AsyncIterable<Table>;
+    readSqlQuery({ queryId }: {
+        queryId: string;
+    }): AsyncIterable<Table>;
+    closeSqlQuery({ queryId }: {
+        queryId: string;
+    }): Promise<void>;
+    cancelSqlQuery({ queryId }: {
+        queryId: string;
+    }): Promise<DatasetSqlCancelResult>;
+    executeSqlStatement({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<number>;
+    search({ table, text, vector, where, offset, limit, select, namespace, branch, version }: {
+        table: string;
+        text?: string;
+        vector?: number[];
+        where?: DatasetWhere;
+        offset?: number;
+        limit?: number;
+        select?: string[];
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<Table[]>;
+    searchStream({ table, text, vector, where, offset, limit, select, namespace, branch, version }: {
+        table: string;
+        text?: string;
+        vector?: number[];
+        where?: DatasetWhere;
+        offset?: number;
+        limit?: number;
+        select?: string[];
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): AsyncIterable<Table>;
+    count({ table, text, vector, where, namespace, branch, version }: {
+        table: string;
+        text?: string;
+        vector?: number[];
+        where?: DatasetWhere;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<number>;
+    inspect({ table, namespace, branch, version }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<Schema>;
+    optimize(table: string): Promise<DatasetOptimizeResult>;
+    optimize(params: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        config?: DatasetOptimizeConfig;
+    }): Promise<DatasetOptimizeResult>;
+    stats({ table, namespace, branch, version, maxRowsPerGroup }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+        maxRowsPerGroup?: number;
+    }): Promise<DatasetTableStats>;
+    restore({ table, version, namespace, branch }: {
+        table: string;
+        version: number;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    listVersions({ table, namespace, branch }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<TableVersion[]>;
+    createIndex({ table, config, namespace, branch }: {
+        table: string;
+        config: DatasetIndexConfig;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    listIndexes({ table, namespace, branch, version }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<TableIndex[]>;
+    listBranches({ namespace }?: {
+        namespace?: string[];
+    }): Promise<TableBranch[]>;
+    createBranch({ branch, fromBranch, namespace }: {
+        branch: string;
+        fromBranch?: string;
+        namespace?: string[];
+    }): Promise<void>;
+    deleteBranch({ branch, namespace }: {
+        branch: string;
+        namespace?: string[];
+    }): Promise<void>;
+}
+export {};