@meshagent/meshagent 0.38.4 → 0.39.0

package/dist/esm/{database-client.js → datasets-client.js}

@@ -1,18 +1,19 @@
-import { DataType } from "./data-types";
+import { Schema, Table, tableFromIPC, tableToIPC } from "apache-arrow";
 import { RoomServerException } from "./room-server-client";
-import { ControlContent, ErrorContent, JsonContent } from "./response";
-export class DatabaseValueEncoder {
+import { BinaryContent, ControlContent, EmptyContent, ErrorContent, JsonContent } from "./response";
+const ARROW_IPC_STREAM_MIME_TYPE = "application/vnd.apache.arrow.stream";
+export class DatasetValueEncoder {
 }
-export class DatabaseExpression extends DatabaseValueEncoder {
+export class DatasetExpression extends DatasetValueEncoder {
     constructor(expression) {
         super();
         const normalized = expression.trim();
         if (normalized === "") {
-            throw new TypeError("database expression must not be empty");
+            throw new TypeError("dataset expression must not be empty");
         }
         this.expression = normalized;
     }
-    encodeDatabaseValue() {
+    encodeDatasetValue() {
         return {
             expression: this.expression,
         };
@@ -21,17 +22,17 @@ export class DatabaseExpression extends DatabaseValueEncoder {
         return this.expression;
     }
 }
-export class DatabaseDate extends DatabaseValueEncoder {
+export class DatasetDate extends DatasetValueEncoder {
     constructor(value) {
         super();
         const normalized = value.trim();
         const parsed = new Date(`${normalized}T00:00:00Z`);
         if (!ISO_DATE_REGEX.test(normalized) || Number.isNaN(parsed.getTime()) || parsed.toISOString().slice(0, 10) !== normalized) {
-            throw new TypeError("invalid database date format");
+            throw new TypeError("invalid dataset date format");
         }
         this.value = normalized;
     }
-    encodeDatabaseValue() {
+    encodeDatasetValue() {
         return {
             date: this.value,
         };
@@ -40,12 +41,12 @@ export class DatabaseDate extends DatabaseValueEncoder {
         return this.value;
     }
 }
-export class DatabaseStruct extends DatabaseValueEncoder {
+export class DatasetStruct extends DatasetValueEncoder {
     constructor(fields) {
         super();
         this.fields = Object.fromEntries(Object.entries(fields).map(([key, value]) => {
             if (typeof key !== "string") {
-                throw new TypeError("database struct keys must be strings");
+                throw new TypeError("dataset struct keys must be strings");
             }
             return [key, value];
         }));
@@ -53,21 +54,21 @@ export class DatabaseStruct extends DatabaseValueEncoder {
     toJson() {
         return Object.fromEntries(Object.entries(this.fields).map(([key, value]) => [key, encodeRecordValue(value)]));
     }
-    encodeDatabaseValue() {
+    encodeDatasetValue() {
         return {
             struct: this.toJson(),
         };
     }
 }
-export class DatabaseJson extends DatabaseValueEncoder {
+export class DatasetJson extends DatasetValueEncoder {
     constructor(value) {
         super();
-        this.value = normalizeDatabaseJsonValue(value);
+        this.value = normalizeDatasetJsonValue(value);
     }
     toJson() {
         return this.value;
     }
-    encodeDatabaseValue() {
+    encodeDatasetValue() {
         return {
             json: this.value,
         };
@@ -90,7 +91,7 @@ function formatUuidHex(value) {
         `${value.substring(16, 20)}-` +
         `${value.substring(20)}`);
 }
-export class DatabaseUuid {
+export class DatasetUuid {
     constructor(value) {
         this.value = formatUuidHex(normalizeUuidHex(value));
     }
@@ -132,7 +133,7 @@ function isPlainRecord(value) {
     return isRecord(value) && (Object.getPrototypeOf(value) === Object.prototype
         || Object.getPrototypeOf(value) === null);
 }
-function normalizeDatabaseJsonValue(value) {
+function normalizeDatasetJsonValue(value) {
     if (value === null ||
         typeof value === "boolean" ||
         typeof value === "number" ||
@@ -140,12 +141,12 @@ function normalizeDatabaseJsonValue(value) {
         return value;
     }
     if (Array.isArray(value)) {
-        return value.map((item) => normalizeDatabaseJsonValue(item));
+        return value.map((item) => normalizeDatasetJsonValue(item));
     }
     if (isPlainRecord(value)) {
-        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, normalizeDatabaseJsonValue(item)]));
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, normalizeDatasetJsonValue(item)]));
     }
-    throw new TypeError("database json values must be valid JSON");
+    throw new TypeError("dataset json values must be valid JSON");
 }
 function metadataEntries(metadata) {
     if (metadata == null) {
@@ -156,98 +157,11 @@ function metadataEntries(metadata) {
         value: typeof value === "string" ? value : JSON.stringify(value),
     }));
 }
-function toolkitDataTypeJson(dataType) {
-    const json = dataType.toJson();
-    const payload = {
-        type: json.type,
-        nullable: json.nullable ?? null,
-        metadata: metadataEntries(json.metadata),
-    };
-    if (json.type === "vector" || json.type === "list") {
-        payload.element_type = toolkitDataTypeJson(DataType.fromJson(json.element_type));
-    }
-    else if (json.type === "struct") {
-        const fields = json.fields;
-        if (!Array.isArray(fields)) {
-            throw new RoomServerException("unexpected return type from database.inspect");
-        }
-        payload.fields = fields.map((field) => {
-            if (!isRecord(field) || typeof field.name !== "string" || !isRecord(field.data_type)) {
-                throw new RoomServerException("unexpected return type from database.inspect");
-            }
-            return {
-                name: field.name,
-                data_type: toolkitDataTypeJson(DataType.fromJson(field.data_type)),
-            };
-        });
-    }
-    if (json.type === "vector") {
-        payload.size = json.size;
-    }
-    return payload;
-}
-function schemaEntries(schema) {
-    if (schema == null) {
-        return null;
-    }
-    return Object.entries(schema).map(([name, dataType]) => ({
-        name,
-        data_type: toolkitDataTypeJson(dataType),
-    }));
-}
-function publicDataTypeJson(value) {
-    if (!isRecord(value)) {
-        throw new RoomServerException("unexpected return type from database.inspect");
-    }
-    const type = value.type;
-    if (typeof type !== "string") {
-        throw new RoomServerException("unexpected return type from database.inspect");
-    }
-    const metadataList = value.metadata;
-    let metadata;
-    if (metadataList != null) {
-        if (!Array.isArray(metadataList)) {
-            throw new RoomServerException("unexpected return type from database.inspect");
-        }
-        metadata = {};
-        for (const entry of metadataList) {
-            if (!isRecord(entry) || typeof entry.key !== "string" || typeof entry.value !== "string") {
-                throw new RoomServerException("unexpected return type from database.inspect");
-            }
-            metadata[entry.key] = entry.value;
-        }
-    }
-    const payload = {
-        type,
-        nullable: value.nullable,
-        metadata,
-    };
-    if (type === "vector") {
-        payload.size = value.size;
-        payload.element_type = publicDataTypeJson(value.element_type);
-    }
-    else if (type === "list") {
-        payload.element_type = publicDataTypeJson(value.element_type);
-    }
-    else if (type === "struct") {
-        const rawFields = value.fields;
-        if (!Array.isArray(rawFields)) {
-            throw new RoomServerException("unexpected return type from database.inspect");
-        }
-        payload.fields = Object.fromEntries(rawFields.map((field) => {
-            if (!isRecord(field) || typeof field.name !== "string") {
-                throw new RoomServerException("unexpected return type from database.inspect");
-            }
-            return [field.name, publicDataTypeJson(field.data_type)];
-        }));
-    }
-    return payload;
-}
 function encodeRecordValue(value) {
-    if (value instanceof DatabaseValueEncoder) {
-        return value.encodeDatabaseValue();
+    if (value instanceof DatasetValueEncoder) {
+        return value.encodeDatasetValue();
     }
-    if (value instanceof DatabaseUuid) {
+    if (value instanceof DatasetUuid) {
         return {
             uuid: value.toString(),
         };
@@ -268,90 +182,90 @@ function encodeRecordValue(value) {
         };
     }
     if (isRecord(value)) {
-        throw new RoomServerException("database object values must use DatabaseStruct or DatabaseJson");
+        throw new RoomServerException("dataset object values must use DatasetStruct or DatasetJson");
     }
     return value;
 }
-function databaseSqlLiteral(value) {
-    if (value instanceof DatabaseUuid) {
+function datasetSqlLiteral(value) {
+    if (value instanceof DatasetUuid) {
         return `X'${normalizeUuidHex(value.toString())}'`;
     }
-    if (value instanceof DatabaseDate) {
+    if (value instanceof DatasetDate) {
         return JSON.stringify(value.toString());
     }
     if (value instanceof Date) {
         return JSON.stringify(value.toISOString().replace("+00:00", "Z"));
     }
-    if (value instanceof DatabaseJson) {
+    if (value instanceof DatasetJson) {
         return JSON.stringify(JSON.stringify(value.toJson()));
     }
-    if (value instanceof DatabaseStruct) {
-        const fields = Object.entries(value.fields).map(([key, fieldValue]) => (`${JSON.stringify(key)}, ${databaseSqlLiteral(fieldValue)}`));
+    if (value instanceof DatasetStruct) {
+        const fields = Object.entries(value.fields).map(([key, fieldValue]) => (`${JSON.stringify(key)}, ${datasetSqlLiteral(fieldValue)}`));
         return `named_struct(${fields.join(", ")})`;
     }
     return JSON.stringify(encodeRecordValue(value));
 }
 function decodeRecordValue(value) {
     if (Array.isArray(value)) {
-        throw new RoomServerException("database list values must use a {'list': [...]} wrapper");
+        throw new RoomServerException("dataset list values must use a {'list': [...]} wrapper");
     }
     if (!isRecord(value)) {
         return value;
     }
     const entries = Object.entries(value);
     if (entries.length !== 1) {
-        throw new RoomServerException("database object values must use a single-key type wrapper");
+        throw new RoomServerException("dataset object values must use a single-key type wrapper");
     }
     const [wrapper, payload] = entries[0];
     switch (wrapper) {
         case "binary":
             if (typeof payload !== "string") {
-                throw new RoomServerException("database binary values must be base64 strings");
+                throw new RoomServerException("dataset binary values must be base64 strings");
             }
             return base64ToBytes(payload);
         case "uuid":
             if (typeof payload !== "string") {
-                throw new RoomServerException("database uuid values must be strings");
+                throw new RoomServerException("dataset uuid values must be strings");
             }
-            return new DatabaseUuid(payload);
+            return new DatasetUuid(payload);
         case "expression":
             if (typeof payload !== "string") {
-                throw new RoomServerException("database expression values must be strings");
+                throw new RoomServerException("dataset expression values must be strings");
             }
-            return new DatabaseExpression(payload);
+            return new DatasetExpression(payload);
         case "date":
             if (typeof payload !== "string") {
-                throw new RoomServerException("database date values must be strings");
+                throw new RoomServerException("dataset date values must be strings");
             }
-            return new DatabaseDate(payload);
+            return new DatasetDate(payload);
         case "timestamp":
             if (typeof payload !== "string") {
-                throw new RoomServerException("database timestamp values must be strings");
+                throw new RoomServerException("dataset timestamp values must be strings");
             }
             {
                 const parsed = new Date(payload);
                 if (Number.isNaN(parsed.getTime())) {
-                    throw new RoomServerException("database timestamp value is not valid");
+                    throw new RoomServerException("dataset timestamp value is not valid");
                 }
                 return parsed;
             }
         case "list":
             if (!Array.isArray(payload)) {
-                throw new RoomServerException("database list values must be arrays");
+                throw new RoomServerException("dataset list values must be arrays");
             }
             return payload.map((item) => decodeRecordValue(item));
         case "struct":
             if (!isRecord(payload)) {
-                throw new RoomServerException("database struct values must be objects");
+                throw new RoomServerException("dataset struct values must be objects");
             }
-            return new DatabaseStruct(Object.fromEntries(Object.entries(payload).map(([key, item]) => [key, decodeRecordValue(item)])));
+            return new DatasetStruct(Object.fromEntries(Object.entries(payload).map(([key, item]) => [key, decodeRecordValue(item)])));
         case "json":
-            return new DatabaseJson(normalizeDatabaseJsonValue(payload));
+            return new DatasetJson(normalizeDatasetJsonValue(payload));
         default:
-            throw new RoomServerException(`unsupported database value wrapper '${wrapper}'`);
+            throw new RoomServerException(`unsupported dataset value wrapper '${wrapper}'`);
     }
 }
-function encodeDatabaseRecord(record) {
+function encodeDatasetRecord(record) {
     return Object.fromEntries(Object.entries(record).map(([key, value]) => [key, encodeRecordValue(value)]));
 }
 function rowsChunk(records) {
@@ -367,21 +281,21 @@ function rowsChunk(records) {
 }
 function recordsFromRowsChunk(payload, operation) {
     if (!isRecord(payload) || payload.kind !== "rows" || !Array.isArray(payload.rows)) {
-        throw new RoomServerException(`unexpected return type from database.${operation}`);
+        throw new RoomServerException(`unexpected return type from datasets.${operation}`);
     }
     return payload.rows.map((row) => {
         if (!isRecord(row) || !Array.isArray(row.columns)) {
-            throw new RoomServerException(`unexpected return type from database.${operation}`);
+            throw new RoomServerException(`unexpected return type from datasets.${operation}`);
         }
         return Object.fromEntries(row.columns.map((column) => {
             if (!isRecord(column) || typeof column.name !== "string") {
-                throw new RoomServerException(`unexpected return type from database.${operation}`);
+                throw new RoomServerException(`unexpected return type from datasets.${operation}`);
             }
             try {
                 return [column.name, decodeRecordValue(column.value)];
             }
             catch {
-                throw new RoomServerException(`unexpected return type from database.${operation}`);
+                throw new RoomServerException(`unexpected return type from datasets.${operation}`);
             }
         }));
     });
@@ -410,7 +324,7 @@ async function* toAsyncIterable(chunks) {
 function buildWhereClause(where) {
     if (where != null && typeof where === "object" && !Array.isArray(where)) {
         return Object.entries(where)
-            .map(([key, value]) => `${key} = ${databaseSqlLiteral(value)}`)
+            .map(([key, value]) => `${key} = ${datasetSqlLiteral(value)}`)
             .join(" AND ");
     }
     if (typeof where === "string") {
@@ -423,36 +337,64 @@ function normalizeTableRefs(tables) {
 }
 function tableIndexFromJson(value) {
     if (!isRecord(value) || typeof value.name !== "string" || typeof value.type !== "string" || !Array.isArray(value.columns)) {
-        throw new RoomServerException("unexpected return type from database.list_indexes");
+        throw new RoomServerException("unexpected return type from datasets.list_indexes");
     }
     return {
         name: value.name,
         type: value.type,
         columns: value.columns.map((column) => {
             if (typeof column !== "string") {
-                throw new RoomServerException("unexpected return type from database.list_indexes");
+                throw new RoomServerException("unexpected return type from datasets.list_indexes");
             }
             return column;
         }),
+        fields: Array.isArray(value.fields) ? value.fields.map((field) => Number(field)) : [],
+        typeUrl: typeof value.type_url === "string" ? value.type_url : null,
+        numRowsIndexed: typeof value.num_rows_indexed === "number" ? value.num_rows_indexed : null,
+        numSegments: typeof value.num_segments === "number" ? value.num_segments : null,
+        totalSizeBytes: typeof value.total_size_bytes === "number" ? value.total_size_bytes : null,
+        details: typeof value.details_json === "string" ? JSON.parse(value.details_json) : isRecord(value.details) ? value.details : {},
+        statistics: typeof value.statistics_json === "string" ? JSON.parse(value.statistics_json) : isRecord(value.statistics) ? value.statistics : {},
     };
 }
+function optimizeResultFromJson(value) {
+    if (!isRecord(value)) {
+        throw new RoomServerException("unexpected return type from datasets.optimize");
+    }
+    return {
+        compaction: typeof value.compaction_json === "string" ? JSON.parse(value.compaction_json) : isRecord(value.compaction) ? value.compaction : null,
+        optimizedIndices: value.optimized_indices === true,
+        cleanup: typeof value.cleanup_json === "string" ? JSON.parse(value.cleanup_json) : isRecord(value.cleanup) ? value.cleanup : null,
+    };
+}
+function tableStatsFromJson(value) {
+    if (!isRecord(value)) {
+        throw new RoomServerException("unexpected return type from datasets.stats");
+    }
+    const dataset = typeof value.dataset_json === "string" ? JSON.parse(value.dataset_json) : value.dataset;
+    const data = typeof value.data_json === "string" ? JSON.parse(value.data_json) : value.data;
+    if (!isRecord(dataset) || !isRecord(data)) {
+        throw new RoomServerException("unexpected return type from datasets.stats");
+    }
+    return { dataset, data };
+}
 function tableVersionFromJson(value) {
     if (!isRecord(value) || typeof value.metadata_json !== "string" || typeof value.timestamp !== "string" || typeof value.version !== "number") {
-        throw new RoomServerException("unexpected return type from database.list_versions");
+        throw new RoomServerException("unexpected return type from datasets.list_versions");
     }
     const timestamp = new Date(value.timestamp);
     if (Number.isNaN(timestamp.getTime())) {
-        throw new RoomServerException("unexpected return type from database.list_versions");
+        throw new RoomServerException("unexpected return type from datasets.list_versions");
     }
     let metadata;
     try {
         metadata = JSON.parse(value.metadata_json);
     }
     catch (_) {
-        throw new RoomServerException("unexpected return type from database.list_versions");
+        throw new RoomServerException("unexpected return type from datasets.list_versions");
     }
     if (!isRecord(metadata)) {
-        throw new RoomServerException("unexpected return type from database.list_versions");
+        throw new RoomServerException("unexpected return type from datasets.list_versions");
     }
     return {
         version: value.version,
@@ -462,27 +404,27 @@ function tableVersionFromJson(value) {
 }
 function tableBranchFromJson(value) {
     if (!isRecord(value) || typeof value.name !== "string") {
-        throw new RoomServerException("unexpected return type from database.list_branches");
+        throw new RoomServerException("unexpected return type from datasets.list_branches");
     }
     if (value.parent_branch != null && typeof value.parent_branch !== "string") {
-        throw new RoomServerException("unexpected return type from database.list_branches");
+        throw new RoomServerException("unexpected return type from datasets.list_branches");
     }
     if (value.parent_version != null
         && (typeof value.parent_version !== "number" || !Number.isInteger(value.parent_version))) {
-        throw new RoomServerException("unexpected return type from database.list_branches");
+        throw new RoomServerException("unexpected return type from datasets.list_branches");
     }
     if (value.manifest_size != null
         && (typeof value.manifest_size !== "number" || !Number.isInteger(value.manifest_size))) {
-        throw new RoomServerException("unexpected return type from database.list_branches");
+        throw new RoomServerException("unexpected return type from datasets.list_branches");
     }
     let createdAt = null;
     if (value.created_at != null) {
         if (typeof value.created_at !== "string") {
-            throw new RoomServerException("unexpected return type from database.list_branches");
+            throw new RoomServerException("unexpected return type from datasets.list_branches");
         }
         createdAt = new Date(value.created_at);
         if (Number.isNaN(createdAt.getTime())) {
-            throw new RoomServerException("unexpected return type from database.list_branches");
+            throw new RoomServerException("unexpected return type from datasets.list_branches");
         }
     }
     return {
@@ -493,7 +435,7 @@ function tableBranchFromJson(value) {
         manifestSize: value.manifest_size ?? null,
     };
 }
-class DatabaseWriteInputStream {
+class DatasetWriteInputStream {
     constructor(start, chunks) {
         this.start = start;
         this.pulls = [];
@@ -553,7 +495,98 @@ class DatabaseWriteInputStream {
         }
     }
 }
-class DatabaseReadInputStream {
+async function* toAsyncArrowIterable(chunks) {
+    if (Symbol.asyncIterator in chunks) {
+        for await (const chunk of chunks) {
+            yield chunk;
+        }
+        return;
+    }
+    for (const chunk of chunks) {
+        yield chunk;
+    }
+}
+function schemaToIPC(schema) {
+    return tableToIPC(new Table(schema, []), "stream");
+}
+function tableFromIPCBytes(data) {
+    const table = tableFromIPC(data);
+    if (table instanceof Promise) {
+        throw new RoomServerException("unexpected async Arrow IPC result");
+    }
+    return table;
+}
+function schemaFromIPCBytes(data) {
+    return tableFromIPCBytes(data).schema;
+}
+class DatasetArrowWriteInputStream {
+    constructor(start, chunks, schema) {
+        this.start = start;
+        this.schema = schema;
+        this.pulls = [];
+        this.closed = false;
+        this.source = toAsyncArrowIterable(chunks)[Symbol.asyncIterator]();
+    }
+    requestNext() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        this.pulls.push(() => undefined);
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        while (this.pulls.length > 0) {
+            const waiter = this.pulls.shift();
+            waiter?.();
+        }
+        void this.source.return?.();
+    }
+    async waitForPull() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        await new Promise((resolve) => {
+            this.pulls.push(resolve);
+        });
+    }
+    async *stream() {
+        yield new BinaryContent({
+            data: this.schema == null ? new Uint8Array() : schemaToIPC(this.schema),
+            headers: this.start,
+        });
+        while (!this.closed) {
+            await this.waitForPull();
+            if (this.closed) {
+                return;
+            }
+            const nextChunk = await this.source.next();
+            if (nextChunk.done) {
+                return;
+            }
+            if (nextChunk.value.numRows === 0) {
+                continue;
+            }
+            yield new BinaryContent({
+                data: tableToIPC(nextChunk.value, "stream"),
+                headers: { kind: "data", content_type: ARROW_IPC_STREAM_MIME_TYPE },
+            });
+        }
+    }
+}
+class DatasetReadInputStream {
     constructor(start) {
         this.start = start;
         this.pulls = [];
@@ -604,15 +637,66 @@ class DatabaseReadInputStream {
         }
     }
 }
-export class DatabaseClient {
+class DatasetArrowReadInputStream {
+    constructor(start) {
+        this.start = start;
+        this.pulls = [];
+        this.closed = false;
+    }
+    requestNext() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        this.pulls.push(() => undefined);
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        while (this.pulls.length > 0) {
+            const waiter = this.pulls.shift();
+            waiter?.();
+        }
+    }
+    async waitForPull() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        await new Promise((resolve) => {
+            this.pulls.push(resolve);
+        });
+    }
+    async *stream() {
+        yield new BinaryContent({ data: new Uint8Array(), headers: this.start });
+        while (!this.closed) {
+            await this.waitForPull();
+            if (this.closed) {
+                return;
+            }
+            yield new BinaryContent({ data: new Uint8Array(), headers: { kind: "pull" } });
+        }
+    }
+}
+export class DatasetsClient {
     constructor({ room }) {
         this.room = room;
     }
     _unexpectedResponseError(operation) {
-        return new RoomServerException(`unexpected return type from database.${operation}`);
+        return new RoomServerException(`unexpected return type from datasets.${operation}`);
     }
     async invoke(operation, input) {
-        const response = await this.room.invoke({ toolkit: "database", tool: operation, input });
+        const response = await this.room.invoke({ toolkit: "dataset", tool: operation, input });
         if (response instanceof JsonContent) {
             return response;
         }
@@ -621,8 +705,11 @@ export class DatabaseClient {
         }
         return null;
     }
+    async invokeContent(operation, input) {
+        return await this.room.invoke({ toolkit: "dataset", tool: operation, input });
+    }
     async invokeStream(operation, input) {
-        return await this.room.invokeStream({ toolkit: "database", tool: operation, input });
+        return await this.room.invokeStream({ toolkit: "dataset", tool: operation, input });
     }
     async drainWriteStream(operation, input) {
         const response = await this.invokeStream(operation, input.stream());
@@ -637,6 +724,13 @@ export class DatabaseClient {
                     }
                     throw this._unexpectedResponseError(operation);
                 }
+                if (chunk instanceof BinaryContent) {
+                    if (chunk.headers.kind !== "pull") {
+                        throw this._unexpectedResponseError(operation);
+                    }
+                    input.requestNext();
+                    continue;
+                }
                 if (!(chunk instanceof JsonContent) || chunk.json.kind !== "pull") {
                     throw this._unexpectedResponseError(operation);
                 }
@@ -648,7 +742,7 @@ export class DatabaseClient {
         }
     }
     async *streamRows(operation, start) {
-        const input = new DatabaseReadInputStream(start);
+        const input = new DatasetReadInputStream(start);
         const response = await this.invokeStream(operation, input.stream());
         input.requestNext();
         try {
@@ -673,6 +767,32 @@ export class DatabaseClient {
             input.close();
         }
     }
+    async *streamArrow(operation, start) {
+        const input = new DatasetArrowReadInputStream(start);
+        const response = await this.invokeStream(operation, input.stream());
+        input.requestNext();
+        try {
+            for await (const chunk of response) {
+                if (chunk instanceof ErrorContent) {
+                    throw new RoomServerException(chunk.text, chunk.code);
+                }
+                if (chunk instanceof ControlContent) {
+                    if (chunk.method === "close") {
+                        return;
+                    }
+                    throw this._unexpectedResponseError(operation);
+                }
+                if (!(chunk instanceof BinaryContent) || chunk.headers.kind !== "data") {
+                    throw this._unexpectedResponseError(operation);
+                }
+                yield tableFromIPCBytes(chunk.data);
+                input.requestNext();
+            }
+        }
+        finally {
+            input.close();
+        }
+    }
     async listTables({ namespace, branch } = {}) {
         const response = await this.invoke("list_tables", {
             namespace: namespace ?? null,
@@ -684,22 +804,21 @@ export class DatabaseClient {
         return Array.isArray(response.json.tables) ? response.json.tables : [];
     }
     async createTable({ name, data, schema, mode = "create", namespace, branch, metadata, }) {
-        const input = new DatabaseWriteInputStream({
+        const input = new DatasetArrowWriteInputStream({
             kind: "start",
             name,
-            fields: schemaEntries(schema),
             mode,
             namespace: namespace ?? null,
             branch: branch ?? null,
             metadata: metadataEntries(metadata),
-        }, data ?? []);
+        }, data ?? [], schema);
         await this.drainWriteStream("create_table", input);
     }
     async createTableWithSchema({ name, schema, data, mode = "create", namespace, branch, metadata }) {
         return this.createTable({
             name,
             schema,
-            data: data == null ? undefined : rowChunkList(data),
+            data: data == null ? undefined : data instanceof Table ? [data] : data,
             mode,
             namespace,
             branch,
@@ -709,7 +828,7 @@ export class DatabaseClient {
     async createTableFromData({ name, data, mode = "create", namespace, branch, metadata }) {
         return this.createTable({
             name,
-            data: data == null ? undefined : rowChunkList(data),
+            data: data == null ? undefined : data instanceof Table ? [data] : data,
             mode,
             namespace,
             branch,
@@ -719,9 +838,20 @@ export class DatabaseClient {
     async createTableFromDataStream({ name, chunks, schema, mode = "create", namespace, branch, metadata }) {
         return this.createTable({ name, data: chunks, schema, mode, namespace, branch, metadata });
     }
+    async createTableFromJsonData({ name, data, mode = "create", namespace, branch, metadata }) {
+        const input = new DatasetWriteInputStream({
+            kind: "start",
+            name,
+            mode,
+            namespace: namespace ?? null,
+            branch: branch ?? null,
+            metadata: metadataEntries(metadata),
+        }, data == null ? [] : rowChunkList(data));
+        await this.drainWriteStream("create_table", input);
+    }
     async dropTable({ name, ignoreMissing = false, namespace, branch }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "drop_table",
             input: {
                 name,
@@ -733,20 +863,30 @@ export class DatabaseClient {
     }
     async dropIndex({ table, name, namespace, branch }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "drop_index",
             input: { table, name, namespace: namespace ?? null, branch: branch ?? null },
         });
     }
     async addColumns({ table, newColumns, namespace, branch }) {
+        if (newColumns instanceof Schema) {
+            await this.invokeContent("add_columns", new BinaryContent({
+                data: schemaToIPC(newColumns),
+                headers: {
+                    table,
+                    namespace: namespace ?? null,
+                    branch: branch ?? null,
+                    content_type: ARROW_IPC_STREAM_MIME_TYPE,
+                },
+            }));
+            return;
+        }
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "add_columns",
             input: {
                 table,
-                columns: Object.entries(newColumns).map(([name, value]) => (value instanceof DataType
-                    ? { name, value_sql: null, data_type: toolkitDataTypeJson(value) }
-                    : { name, value_sql: value, data_type: null })),
+                columns: Object.entries(newColumns).map(([name, value]) => ({ name, value_sql: value })),
                 namespace: namespace ?? null,
                 branch: branch ?? null,
             },
@@ -754,16 +894,16 @@ export class DatabaseClient {
     }
     async dropColumns({ table, columns, namespace, branch }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "drop_columns",
             input: { table, columns, namespace: namespace ?? null, branch: branch ?? null },
         });
     }
     async insert({ table, records, namespace, branch }) {
-        await this.insertStream({ table, chunks: rowChunkList(records), namespace, branch });
+        await this.insertStream({ table, chunks: [records], namespace, branch });
     }
     async insertStream({ table, chunks, namespace, branch }) {
-        const input = new DatabaseWriteInputStream({
+        const input = new DatasetArrowWriteInputStream({
             kind: "start",
             table,
             namespace: namespace ?? null,
@@ -773,7 +913,7 @@ export class DatabaseClient {
     }
     async update({ table, where, values, namespace, branch }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "update",
             input: {
                 table,
@@ -786,16 +926,16 @@ export class DatabaseClient {
     }
     async delete({ table, where, namespace, branch }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "delete",
             input: { table, where, namespace: namespace ?? null, branch: branch ?? null },
         });
     }
     async merge({ table, on, records, namespace, branch }) {
-        await this.mergeStream({ table, on, chunks: rowChunkList(records), namespace, branch });
+        await this.mergeStream({ table, on, chunks: [records], namespace, branch });
     }
     async mergeStream({ table, on, chunks, namespace, branch }) {
-        const input = new DatabaseWriteInputStream({
+        const input = new DatasetArrowWriteInputStream({
             kind: "start",
             table,
             on,
@@ -804,23 +944,125 @@ export class DatabaseClient {
         }, chunks);
         await this.drainWriteStream("merge", input);
     }
-    async sql({ query, tables, params }) {
-        const rows = [];
-        for await (const chunk of this.sqlStream({ query, tables, params })) {
-            rows.push(...chunk);
+    async sql({ query, tables, params, namespace, branch }) {
+        const results = [];
+        for await (const chunk of this.sqlStream({ query, tables, params, namespace, branch })) {
+            results.push(chunk);
+        }
+        return results;
+    }
+    async openSqlQuery({ query, tables, params, namespace, branch }) {
+        const response = await this.invokeContent("open_sql_query", new BinaryContent({
+            data: params == null ? new Uint8Array() : tableToIPC(params, "stream"),
+            headers: {
+                query,
+                tables: normalizeTableRefs(tables ?? []),
+                namespace: namespace ?? null,
+                branch: branch ?? null,
+            },
+        }));
+        if (!(response instanceof BinaryContent)) {
+            throw this._unexpectedResponseError("open_sql_query");
+        }
+        const queryId = response.headers.query_id;
+        if (typeof queryId !== "string" || queryId === "") {
+            throw this._unexpectedResponseError("open_sql_query");
+        }
+        return {
+            schema: schemaFromIPCBytes(response.data),
+            kind: "query",
+            queryId,
+        };
+    }
+    async executeSql({ query, tables, params, namespace, branch }) {
+        const response = await this.invokeContent("execute_sql", new BinaryContent({
+            data: params == null ? new Uint8Array() : tableToIPC(params, "stream"),
+            headers: {
+                query,
+                tables: normalizeTableRefs(tables ?? []),
+                namespace: namespace ?? null,
+                branch: branch ?? null,
+            },
+        }));
+        if (response instanceof BinaryContent) {
+            if (response.headers.kind !== "query") {
+                throw this._unexpectedResponseError("execute_sql");
+            }
+            const queryId = response.headers.query_id;
+            if (typeof queryId !== "string" || queryId === "") {
+                throw this._unexpectedResponseError("execute_sql");
+            }
+            return {
+                kind: "query",
+                schema: schemaFromIPCBytes(response.data),
+                queryId,
+            };
+        }
+        if (response instanceof JsonContent) {
+            if (response.json.kind !== "statement"
+                || typeof response.json.rows_affected !== "number"
+                || !Number.isInteger(response.json.rows_affected)) {
+                throw this._unexpectedResponseError("execute_sql");
+            }
+            return {
+                kind: "statement",
+                rowsAffected: response.json.rows_affected,
+            };
+        }
+        throw this._unexpectedResponseError("execute_sql");
+    }
+    async *sqlStream({ query, tables, params, namespace, branch }) {
+        const result = await this.executeSql({ query, tables, params, namespace, branch });
+        if (result.kind === "statement") {
+            throw new RoomServerException(`SQL statement did not return rows; rows_affected=${result.rowsAffected}`);
+        }
+        const opened = result;
+        try {
+            yield* this.readSqlQuery({ queryId: opened.queryId });
+        }
+        finally {
+            await this.closeSqlQuery({ queryId: opened.queryId });
         }
-        return rows;
     }
-    async *sqlStream({ query, tables, params }) {
-        yield* this.streamRows("sql", {
+    async *readSqlQuery({ queryId }) {
+        yield* this.streamArrow("read_sql_query", {
             kind: "start",
-            query,
-            tables: normalizeTableRefs(tables),
-            params_json: params == null ? null : JSON.stringify(encodeDatabaseRecord(params)),
+            query_id: queryId,
         });
     }
+    async closeSqlQuery({ queryId }) {
+        const response = await this.room.invoke({ toolkit: "dataset", tool: "close_sql_query", input: { query_id: queryId } });
+        if (!(response instanceof EmptyContent)) {
+            throw this._unexpectedResponseError("close_sql_query");
+        }
+    }
+    async cancelSqlQuery({ queryId }) {
+        const response = await this.room.invoke({ toolkit: "dataset", tool: "cancel_sql_query", input: { query_id: queryId } });
+        if (!(response instanceof JsonContent)
+            || !["cancelled", "cancelling", "not_cancellable"].includes(response.json.status)) {
+            throw this._unexpectedResponseError("cancel_sql_query");
+        }
+        return { status: response.json.status };
+    }
+    async executeSqlStatement({ query, tables, params, namespace, branch }) {
+        const response = await this.invokeContent("execute_sql_statement", new BinaryContent({
+            data: params == null ? new Uint8Array() : tableToIPC(params, "stream"),
+            headers: {
+                query,
+                tables: normalizeTableRefs(tables ?? []),
+                namespace: namespace ?? null,
+                branch: branch ?? null,
+            },
+        }));
+        if (!(response instanceof JsonContent)
+            || typeof response.json.rows_affected !== "number"
+            || !Number.isInteger(response.json.rows_affected)) {
+            throw this._unexpectedResponseError("execute_sql_statement");
+        }
+        return response.json.rows_affected;
+    }
     async search({ table, text, vector, where, offset, limit, select, namespace, branch, version }) {
-        const rows = [];
+        const results = [];
         for await (const chunk of this.searchStream({
             table,
             text,
@@ -833,12 +1075,12 @@ export class DatabaseClient {
             branch,
             version,
         })) {
-            rows.push(...chunk);
+            results.push(chunk);
         }
-        return rows;
+        return results;
     }
     async *searchStream({ table, text, vector, where, offset, limit, select, namespace, branch, version }) {
-        yield* this.streamRows("search", {
+        yield* this.streamArrow("search", {
             kind: "start",
             table,
             text: text ?? null,
@@ -870,35 +1112,53 @@ export class DatabaseClient {
         return response.json.count;
     }
     async inspect({ table, namespace, branch, version }) {
-        const response = await this.invoke("inspect", {
-            table,
-            namespace: namespace ?? null,
-            branch: branch ?? null,
-            version: version ?? null,
+        const response = await this.room.invoke({
+            toolkit: "dataset",
+            tool: "inspect",
+            input: {
+                table,
+                namespace: namespace ?? null,
+                branch: branch ?? null,
+                version: version ?? null,
+            },
         });
-        if (!(response instanceof JsonContent) || !Array.isArray(response.json.fields)) {
+        if (!(response instanceof BinaryContent)) {
             throw this._unexpectedResponseError("inspect");
         }
-        return Object.fromEntries(response.json.fields.map((field) => {
-            if (!isRecord(field) || typeof field.name !== "string") {
-                throw this._unexpectedResponseError("inspect");
-            }
-            return [field.name, DataType.fromJson(publicDataTypeJson(field.data_type))];
-        }));
+        return tableFromIPCBytes(response.data).schema;
     }
     async optimize(tableOrParams) {
         const table = typeof tableOrParams === "string" ? tableOrParams : tableOrParams.table;
         const namespace = typeof tableOrParams === "string" ? undefined : tableOrParams.namespace;
         const branch = typeof tableOrParams === "string" ? undefined : tableOrParams.branch;
-        await this.room.invoke({
-            toolkit: "database",
-            tool: "optimize",
-            input: { table, namespace: namespace ?? null, branch: branch ?? null },
+        const config = typeof tableOrParams === "string" ? undefined : tableOrParams.config;
+        const response = await this.invoke("optimize", {
+            table,
+            namespace: namespace ?? null,
+            branch: branch ?? null,
+            config: config ?? null,
         });
+        if (!(response instanceof JsonContent)) {
+            throw this._unexpectedResponseError("optimize");
+        }
+        return optimizeResultFromJson(response.json);
+    }
+    async stats({ table, namespace, branch, version, maxRowsPerGroup }) {
+        const response = await this.invoke("stats", {
+            table,
+            namespace: namespace ?? null,
+            branch: branch ?? null,
+            version: version ?? null,
+            max_rows_per_group: maxRowsPerGroup ?? null,
+        });
+        if (!(response instanceof JsonContent)) {
+            throw this._unexpectedResponseError("stats");
+        }
+        return tableStatsFromJson(response.json);
     }
     async restore({ table, version, namespace, branch }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "restore",
             input: { table, version, namespace: namespace ?? null, branch: branch ?? null },
         });
@@ -914,25 +1174,11 @@ export class DatabaseClient {
         }
         return response.json.versions.map((version) => tableVersionFromJson(version));
     }
-    async createVectorIndex({ table, column, replace = false, namespace, branch }) {
-        await this.room.invoke({
-            toolkit: "database",
-            tool: "create_vector_index",
-            input: { table, column, replace, namespace: namespace ?? null, branch: branch ?? null },
-        });
-    }
-    async createScalarIndex({ table, column, replace = false, namespace, branch }) {
-        await this.room.invoke({
-            toolkit: "database",
-            tool: "create_scalar_index",
-            input: { table, column, replace, namespace: namespace ?? null, branch: branch ?? null },
-        });
-    }
-    async createFullTextSearchIndex({ table, column, replace = false, namespace, branch }) {
+    async createIndex({ table, config, namespace, branch }) {
         await this.room.invoke({
-            toolkit: "database",
-            tool: "create_full_text_search_index",
-            input: { table, column, replace, namespace: namespace ?? null, branch: branch ?? null },
+            toolkit: "dataset",
+            tool: "create_index",
+            input: { table, config, namespace: namespace ?? null, branch: branch ?? null },
         });
     }
     async listIndexes({ table, namespace, branch, version }) {
@@ -958,7 +1204,7 @@ export class DatabaseClient {
     }
     async createBranch({ branch, fromBranch, namespace }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "create_branch",
             input: {
                 branch,
@@ -969,7 +1215,7 @@ export class DatabaseClient {
     }
     async deleteBranch({ branch, namespace }) {
         await this.room.invoke({
-            toolkit: "database",
+            toolkit: "dataset",
             tool: "delete_branch",
             input: { branch, namespace: namespace ?? null },
         });