@meshagent/meshagent 0.38.4 → 0.39.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+## [0.39.0]
+- Expanded TypeScript datasets client support for dataset index management (index configuration/remapping and index metadata).
+- Added TypeScript datasets SQL cancellation API with typed cancel status/results.
+- Applied “database” -> “datasets” terminology and toolkit/client updates across Node SDK clients (breaking for prior database-named usage).
+- Updated Node SDK/client surfaces for improved LLM proxy “pipes”, custom LLM usage tracking, and pricing/usage reporting (including gpt-5.5 pricing).
+- Updated a Node example dependency: `uuid` bumped to `^14.0.0` (from `^11.1.0`).
+
 ## [0.38.4]
 - Stability
 
@@ -33,7 +40,7 @@
 - Meshagent TS client now normalizes binary request bodies for fetch and includes TypeScript declarations for runtime entrypoint functions.
 
 ## [0.37.0]
-- Breaking: Database client now supports `json`, `uuid`, `list`, and `struct` types with typed wrappers (DatabaseJson/DatabaseStruct/DatabaseExpression/DatabaseDate/DatabaseUuid); list/struct values must be wrapped and update now takes `values` only.
+- Breaking: Datasets client now supports `json`, `uuid`, `list`, and `struct` types with typed wrappers (DatasetJson/DatasetStruct/DatasetExpression/DatasetDate/DatasetUuid); list/struct values must be wrapped and update now takes `values` only.
 - Breaking: Containers build now streams build contexts (start/data chunks) with `mountPath`/`chunks` and removes `start_build`.
 - Breaking: Toolkit/hosting refactor replaces RemoteToolkit with startHostedToolkit/HostedToolkit, removes ToolkitConfiguration and `supports_context`, and updates React/Tailwind helpers to start hosted toolkits.
 - Participant tokens now include LLM grants and richer grant serialization (including allowed toolkits and extra payload preservation), and schema helpers add `json`/`uuid` data types.
@@ -136,7 +143,7 @@
 - Breaking: tool invocation now uses toolkit-based `room.invoke`/`room.*` events with streaming tool-call chunks, and `RemoteToolkit` registration follows the new room-scoped protocol.
 - Added a Containers client with image listing/pulling, container lifecycle operations, and exec/log streaming.
 - Storage client replaced handle-based writes with streaming upload/download, download URLs, and size metadata.
-- Database and Sync clients now stream inserts/queries/search and sync updates with typed value handling; messaging/queues/developer clients updated to toolkit invocation.
+- Datasets and Sync clients now stream inserts/queries/search and sync updates with typed value handling; messaging/queues/developer clients updated to toolkit invocation.
 
 ## [0.29.4]
 - Stability

package/dist/browser/datasets-client.d.ts

@@ -0,0 +1,415 @@
+import { Schema, Table } from "apache-arrow";
+import { RoomClient } from "./room-client";
+export type CreateMode = "create" | "overwrite" | "create_if_not_exists";
+export interface TableRef {
+    name: string;
+    namespace?: string[];
+    alias?: string;
+    branch?: string;
+    version?: number;
+}
+export interface TableVersion {
+    version: number;
+    timestamp: Date;
+    metadata: Record<string, unknown>;
+}
+export interface TableIndex {
+    name: string;
+    columns: string[];
+    type: string;
+    fields: number[];
+    typeUrl: string | null;
+    numRowsIndexed: number | null;
+    numSegments: number | null;
+    totalSizeBytes: number | null;
+    details: Record<string, unknown>;
+    statistics: Record<string, unknown>;
+}
+export type DatasetVectorIndexType = "IVF_PQ" | "IVF_HNSW_PQ" | "IVF_HNSW_SQ" | "IVF_RQ";
+export type DatasetScalarIndexType = "BTREE" | "BITMAP" | "LABEL_LIST" | "NGRAM" | "ZONEMAP" | "INVERTED" | "FTS" | "BLOOMFILTER" | "RTREE";
+export type DatasetIndexType = DatasetVectorIndexType | DatasetScalarIndexType;
+export interface DatasetIndexConfig {
+    column: string | string[];
+    index_type: DatasetIndexType;
+    name?: string;
+    metric?: string;
+    replace?: boolean;
+    num_partitions?: number;
+    ivf_centroids?: number[][];
+    pq_codebook?: number[][];
+    num_sub_vectors?: number;
+    accelerator?: string;
+    index_cache_size?: number;
+    shuffle_partition_batches?: number;
+    shuffle_partition_concurrency?: number;
+    ivf_centroids_file?: string;
+    precomputed_partition_dataset?: string;
+    filter_nan?: boolean;
+    train?: boolean;
+    fragment_ids?: number[];
+    index_uuid?: string;
+    target_partition_size?: number;
+    skip_transpose?: boolean;
+    num_bits?: number;
+    index_file_version?: string;
+    max_level?: number;
+    m?: number;
+    ef_construction?: number;
+    with_position?: boolean;
+    memory_limit?: number;
+    num_workers?: number;
+    skip_merge?: boolean;
+    base_tokenizer?: string;
+    language?: string;
+    max_token_length?: number;
+    lower_case?: boolean;
+    stem?: boolean;
+    remove_stop_words?: boolean;
+    custom_stop_words?: string[];
+    ascii_folding?: boolean;
+}
+export interface DatasetOptimizeConfig {
+    compact_files?: boolean;
+    optimize_indices?: boolean;
+    cleanup_old_versions?: boolean;
+    target_rows_per_fragment?: number;
+    max_rows_per_group?: number;
+    max_bytes_per_file?: number;
+    materialize_deletions?: boolean;
+    materialize_deletions_threshold?: number;
+    defer_index_remap?: boolean;
+    num_threads?: number;
+    batch_size?: number;
+    compaction_mode?: "reencode" | "try_binary_copy" | "force_binary_copy";
+    binary_copy_read_batch_bytes?: number;
+    num_indices_to_merge?: number;
+    index_names?: string[];
+    retrain?: boolean;
+    older_than_seconds?: number;
+    retain_versions?: number;
+    delete_unverified?: boolean;
+    error_if_tagged_old_versions?: boolean;
+    delete_rate_limit?: number;
+}
+export interface DatasetOptimizeResult {
+    compaction: Record<string, unknown> | null;
+    optimizedIndices: boolean;
+    cleanup: Record<string, unknown> | null;
+}
+export interface DatasetTableStats {
+    dataset: Record<string, unknown>;
+    data: Record<string, unknown>;
+}
+export interface TableBranch {
+    name: string;
+    parentBranch: string | null;
+    parentVersion: number | null;
+    createdAt: Date | null;
+    manifestSize: number | null;
+}
+export interface DatasetSqlQuery {
+    kind: "query";
+    schema: Schema;
+    queryId: string;
+}
+export interface DatasetSqlStatement {
+    kind: "statement";
+    rowsAffected: number;
+}
+export type DatasetSqlExecution = DatasetSqlQuery | DatasetSqlStatement;
+export type DatasetSqlCancelStatus = "cancelled" | "cancelling" | "not_cancellable";
+export interface DatasetSqlCancelResult {
+    status: DatasetSqlCancelStatus;
+}
+export declare abstract class DatasetValueEncoder {
+    abstract encodeDatasetValue(): unknown;
+}
+export declare class DatasetExpression extends DatasetValueEncoder {
+    readonly expression: string;
+    constructor(expression: string);
+    encodeDatasetValue(): Record<string, string>;
+    toString(): string;
+}
+export declare class DatasetDate extends DatasetValueEncoder {
+    readonly value: string;
+    constructor(value: string);
+    encodeDatasetValue(): Record<string, string>;
+    toString(): string;
+}
+export type DatasetJsonScalarValue = null | boolean | number | string;
+export type DatasetJsonValue = DatasetJsonScalarValue | DatasetJsonValue[] | {
+    [key: string]: DatasetJsonValue;
+};
+export declare class DatasetStruct extends DatasetValueEncoder {
+    readonly fields: Record<string, DatasetValue>;
+    constructor(fields: Record<string, DatasetValue>);
+    toJson(): Record<string, unknown>;
+    encodeDatasetValue(): Record<string, Record<string, unknown>>;
+}
+export declare class DatasetJson extends DatasetValueEncoder {
+    readonly value: DatasetJsonValue;
+    constructor(value: DatasetJsonValue);
+    toJson(): DatasetJsonValue;
+    encodeDatasetValue(): Record<string, DatasetJsonValue>;
+}
+export type DatasetScalarValue = null | boolean | number | string | Uint8Array | DatasetUuid | Date;
+export type DatasetValue = DatasetScalarValue | DatasetValueEncoder | DatasetValue[];
+export type DatasetRecord = Record<string, DatasetValue>;
+export type DatasetRows = DatasetRecord[];
+export type DatasetRowChunks = AsyncIterable<DatasetRows> | Iterable<DatasetRows>;
+export type DatasetWhere = string | Record<string, DatasetValue>;
+type DatasetRoomInvoker = Pick<RoomClient, "invoke" | "invokeStream">;
+export declare class DatasetUuid {
+    readonly value: string;
+    constructor(value: string);
+    toString(): string;
+}
+type ArrowTableChunks = Iterable<Table> | AsyncIterable<Table>;
+export declare class DatasetsClient {
+    private room;
+    constructor({ room }: {
+        room: DatasetRoomInvoker;
+    });
+    private _unexpectedResponseError;
+    private invoke;
+    private invokeContent;
+    private invokeStream;
+    private drainWriteStream;
+    private streamRows;
+    private streamArrow;
+    listTables({ namespace, branch }?: {
+        namespace?: string[];
+        branch?: string;
+    }): Promise<string[]>;
+    private createTable;
+    createTableWithSchema({ name, schema, data, mode, namespace, branch, metadata }: {
+        name: string;
+        schema?: Schema;
+        data?: Iterable<Table> | Table;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    createTableFromData({ name, data, mode, namespace, branch, metadata }: {
+        name: string;
+        data?: Iterable<Table> | Table;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    createTableFromDataStream({ name, chunks, schema, mode, namespace, branch, metadata }: {
+        name: string;
+        chunks: ArrowTableChunks;
+        schema?: Schema;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    createTableFromJsonData({ name, data, mode, namespace, branch, metadata }: {
+        name: string;
+        data?: DatasetRows;
+        mode?: CreateMode;
+        namespace?: string[];
+        branch?: string;
+        metadata?: Record<string, unknown>;
+    }): Promise<void>;
+    dropTable({ name, ignoreMissing, namespace, branch }: {
+        name: string;
+        ignoreMissing?: boolean;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    dropIndex({ table, name, namespace, branch }: {
+        table: string;
+        name: string;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    addColumns({ table, newColumns, namespace, branch }: {
+        table: string;
+        newColumns: Record<string, string> | Schema;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    dropColumns({ table, columns, namespace, branch }: {
+        table: string;
+        columns: string[];
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    insert({ table, records, namespace, branch }: {
+        table: string;
+        records: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    insertStream({ table, chunks, namespace, branch }: {
+        table: string;
+        chunks: ArrowTableChunks;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    update({ table, where, values, namespace, branch }: {
+        table: string;
+        where: string;
+        values: DatasetRecord;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    delete({ table, where, namespace, branch }: {
+        table: string;
+        where: string;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    merge({ table, on, records, namespace, branch }: {
+        table: string;
+        on: string;
+        records: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    mergeStream({ table, on, chunks, namespace, branch }: {
+        table: string;
+        on: string;
+        chunks: ArrowTableChunks;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    sql({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<Table[]>;
+    openSqlQuery({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<DatasetSqlQuery>;
+    executeSql({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<DatasetSqlExecution>;
+    sqlStream({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): AsyncIterable<Table>;
+    readSqlQuery({ queryId }: {
+        queryId: string;
+    }): AsyncIterable<Table>;
+    closeSqlQuery({ queryId }: {
+        queryId: string;
+    }): Promise<void>;
+    cancelSqlQuery({ queryId }: {
+        queryId: string;
+    }): Promise<DatasetSqlCancelResult>;
+    executeSqlStatement({ query, tables, params, namespace, branch }: {
+        query: string;
+        tables?: Array<TableRef | string>;
+        params?: Table;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<number>;
+    search({ table, text, vector, where, offset, limit, select, namespace, branch, version }: {
+        table: string;
+        text?: string;
+        vector?: number[];
+        where?: DatasetWhere;
+        offset?: number;
+        limit?: number;
+        select?: string[];
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<Table[]>;
+    searchStream({ table, text, vector, where, offset, limit, select, namespace, branch, version }: {
+        table: string;
+        text?: string;
+        vector?: number[];
+        where?: DatasetWhere;
+        offset?: number;
+        limit?: number;
+        select?: string[];
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): AsyncIterable<Table>;
+    count({ table, text, vector, where, namespace, branch, version }: {
+        table: string;
+        text?: string;
+        vector?: number[];
+        where?: DatasetWhere;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<number>;
+    inspect({ table, namespace, branch, version }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<Schema>;
+    optimize(table: string): Promise<DatasetOptimizeResult>;
+    optimize(params: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        config?: DatasetOptimizeConfig;
+    }): Promise<DatasetOptimizeResult>;
+    stats({ table, namespace, branch, version, maxRowsPerGroup }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+        maxRowsPerGroup?: number;
+    }): Promise<DatasetTableStats>;
+    restore({ table, version, namespace, branch }: {
+        table: string;
+        version: number;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    listVersions({ table, namespace, branch }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<TableVersion[]>;
+    createIndex({ table, config, namespace, branch }: {
+        table: string;
+        config: DatasetIndexConfig;
+        namespace?: string[];
+        branch?: string;
+    }): Promise<void>;
+    listIndexes({ table, namespace, branch, version }: {
+        table: string;
+        namespace?: string[];
+        branch?: string;
+        version?: number;
+    }): Promise<TableIndex[]>;
+    listBranches({ namespace }?: {
+        namespace?: string[];
+    }): Promise<TableBranch[]>;
+    createBranch({ branch, fromBranch, namespace }: {
+        branch: string;
+        fromBranch?: string;
+        namespace?: string[];
+    }): Promise<void>;
+    deleteBranch({ branch, namespace }: {
+        branch: string;
+        namespace?: string[];
+    }): Promise<void>;
+}
+export {};