@meshagent/meshagent 0.29.4 → 0.30.1

package/dist/esm/database-client.js

@@ -1,123 +1,656 @@
-import { JsonContent } from "./response";
+import { DataType } from "./data-types";
+import { RoomServerException } from "./room-server-client";
+import { ControlContent, ErrorContent, JsonContent } from "./response";
+const globalScope = globalThis;
+function bytesToBase64(bytes) {
+    if (globalScope.Buffer) {
+        return globalScope.Buffer.from(bytes).toString("base64");
+    }
+    if (!globalScope.btoa) {
+        throw new Error("base64 encoding is not available in this runtime");
+    }
+    let binary = "";
+    for (const byte of bytes) {
+        binary += String.fromCharCode(byte);
+    }
+    return globalScope.btoa(binary);
+}
+function base64ToBytes(base64) {
+    if (globalScope.Buffer) {
+        return Uint8Array.from(globalScope.Buffer.from(base64, "base64"));
+    }
+    if (!globalScope.atob) {
+        throw new Error("base64 decoding is not available in this runtime");
+    }
+    const binary = globalScope.atob(base64);
+    const bytes = new Uint8Array(binary.length);
+    for (let index = 0; index < binary.length; index += 1) {
+        bytes[index] = binary.charCodeAt(index);
+    }
+    return bytes;
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function metadataEntries(metadata) {
+    if (metadata == null) {
+        return null;
+    }
+    return Object.entries(metadata).map(([key, value]) => ({
+        key,
+        value: typeof value === "string" ? value : JSON.stringify(encodeLegacyValue(value)),
+    }));
+}
+function toolkitDataTypeJson(dataType) {
+    const json = dataType.toJson();
+    const payload = {
+        type: json.type,
+        nullable: json.nullable ?? null,
+        metadata: metadataEntries(json.metadata),
+    };
+    if (json.type === "vector" || json.type === "list") {
+        payload.element_type = toolkitDataTypeJson(DataType.fromJson(json.element_type));
+    }
+    else if (json.type === "struct") {
+        const fields = json.fields;
+        if (!Array.isArray(fields)) {
+            throw new RoomServerException("unexpected return type from database.inspect");
+        }
+        payload.fields = fields.map((field) => {
+            if (!isRecord(field) || typeof field.name !== "string" || !isRecord(field.data_type)) {
+                throw new RoomServerException("unexpected return type from database.inspect");
+            }
+            return {
+                name: field.name,
+                data_type: toolkitDataTypeJson(DataType.fromJson(field.data_type)),
+            };
+        });
+    }
+    if (json.type === "vector") {
+        payload.size = json.size;
+    }
+    return payload;
+}
+function schemaEntries(schema) {
+    if (schema == null) {
+        return null;
+    }
+    return Object.entries(schema).map(([name, dataType]) => ({
+        name,
+        data_type: toolkitDataTypeJson(dataType),
+    }));
+}
+function publicDataTypeJson(value) {
+    if (!isRecord(value)) {
+        throw new RoomServerException("unexpected return type from database.inspect");
+    }
+    const type = value.type;
+    if (typeof type !== "string") {
+        throw new RoomServerException("unexpected return type from database.inspect");
+    }
+    const metadataList = value.metadata;
+    let metadata;
+    if (metadataList != null) {
+        if (!Array.isArray(metadataList)) {
+            throw new RoomServerException("unexpected return type from database.inspect");
+        }
+        metadata = {};
+        for (const entry of metadataList) {
+            if (!isRecord(entry) || typeof entry.key !== "string" || typeof entry.value !== "string") {
+                throw new RoomServerException("unexpected return type from database.inspect");
+            }
+            metadata[entry.key] = entry.value;
+        }
+    }
+    const payload = {
+        type,
+        nullable: value.nullable,
+        metadata,
+    };
+    if (type === "vector") {
+        payload.size = value.size;
+        payload.element_type = publicDataTypeJson(value.element_type);
+    }
+    else if (type === "list") {
+        payload.element_type = publicDataTypeJson(value.element_type);
+    }
+    else if (type === "struct") {
+        const rawFields = value.fields;
+        if (!Array.isArray(rawFields)) {
+            throw new RoomServerException("unexpected return type from database.inspect");
+        }
+        payload.fields = Object.fromEntries(rawFields.map((field) => {
+            if (!isRecord(field) || typeof field.name !== "string") {
+                throw new RoomServerException("unexpected return type from database.inspect");
+            }
+            return [field.name, publicDataTypeJson(field.data_type)];
+        }));
+    }
+    return payload;
+}
+function encodeLegacyValue(value) {
+    if (value instanceof Uint8Array) {
+        return {
+            encoding: "base64",
+            data: bytesToBase64(value),
+        };
+    }
+    if (value instanceof Date) {
+        return value.toISOString().replace("+00:00", "Z");
+    }
+    if (Array.isArray(value)) {
+        return value.map((item) => encodeLegacyValue(item));
+    }
+    if (isRecord(value)) {
+        return Object.fromEntries(Object.entries(value).map(([key, entryValue]) => [key, encodeLegacyValue(entryValue)]));
+    }
+    return value;
+}
+function encodeStreamValue(value) {
+    if (value == null) {
+        return { type: "null" };
+    }
+    if (typeof value === "boolean") {
+        return { type: "bool", value };
+    }
+    if (typeof value === "number") {
+        if (Number.isInteger(value)) {
+            return { type: "int", value };
+        }
+        return { type: "float", value };
+    }
+    if (typeof value === "string") {
+        return { type: "text", value };
+    }
+    if (value instanceof Uint8Array) {
+        return {
+            type: "binary",
+            data: bytesToBase64(value),
+        };
+    }
+    if (value instanceof Date) {
+        return {
+            type: "timestamp",
+            value: value.toISOString().replace("+00:00", "Z"),
+        };
+    }
+    if (Array.isArray(value)) {
+        return {
+            type: "list",
+            items: value.map((item) => encodeStreamValue(item)),
+        };
+    }
+    if (isRecord(value)) {
+        return {
+            type: "struct",
+            fields: Object.entries(value).map(([name, fieldValue]) => ({
+                name,
+                value: encodeStreamValue(fieldValue),
+            })),
+        };
+    }
+    throw new RoomServerException(`database stream does not support value type ${typeof value}`);
+}
+function decodeStreamValue(value, operation) {
+    if (!isRecord(value) || typeof value.type !== "string") {
+        throw new RoomServerException(`unexpected return type from database.${operation}`);
+    }
+    switch (value.type) {
+        case "null":
+            return null;
+        case "bool":
+            if (typeof value.value !== "boolean") {
+                throw new RoomServerException(`unexpected return type from database.${operation}`);
+            }
+            return value.value;
+        case "int":
+        case "float":
+            if (typeof value.value !== "number") {
+                throw new RoomServerException(`unexpected return type from database.${operation}`);
+            }
+            return value.value;
+        case "text":
+        case "date":
+        case "timestamp":
+            if (typeof value.value !== "string") {
+                throw new RoomServerException(`unexpected return type from database.${operation}`);
+            }
+            return value.value;
+        case "binary":
+            if (typeof value.data !== "string") {
+                throw new RoomServerException(`unexpected return type from database.${operation}`);
+            }
+            return base64ToBytes(value.data);
+        case "list":
+            if (!Array.isArray(value.items)) {
+                throw new RoomServerException(`unexpected return type from database.${operation}`);
+            }
+            return value.items.map((item) => decodeStreamValue(item, operation));
+        case "struct":
+            if (!Array.isArray(value.fields)) {
+                throw new RoomServerException(`unexpected return type from database.${operation}`);
+            }
+            return Object.fromEntries(value.fields.map((field) => {
+                if (!isRecord(field) || typeof field.name !== "string") {
+                    throw new RoomServerException(`unexpected return type from database.${operation}`);
+                }
+                return [field.name, decodeStreamValue(field.value, operation)];
+            }));
+        default:
+            throw new RoomServerException(`unexpected return type from database.${operation}`);
+    }
+}
+function rowsChunk(records) {
+    return {
+        kind: "rows",
+        rows: records.map((record) => ({
+            columns: Object.entries(record).map(([name, value]) => ({
+                name,
+                value: encodeStreamValue(value),
+            })),
+        })),
+    };
+}
+function recordsFromRowsChunk(payload, operation) {
+    if (!isRecord(payload) || payload.kind !== "rows" || !Array.isArray(payload.rows)) {
+        throw new RoomServerException(`unexpected return type from database.${operation}`);
+    }
+    return payload.rows.map((row) => {
+        if (!isRecord(row) || !Array.isArray(row.columns)) {
+            throw new RoomServerException(`unexpected return type from database.${operation}`);
+        }
+        return Object.fromEntries(row.columns.map((column) => {
+            if (!isRecord(column) || typeof column.name !== "string") {
+                throw new RoomServerException(`unexpected return type from database.${operation}`);
+            }
+            return [column.name, decodeStreamValue(column.value, operation)];
+        }));
+    });
+}
+function rowChunkList(records, rowsPerChunk = 128) {
+    if (rowsPerChunk <= 0) {
+        throw new RoomServerException("rowsPerChunk must be greater than zero");
+    }
+    const chunks = [];
+    for (let index = 0; index < records.length; index += rowsPerChunk) {
+        chunks.push(records.slice(index, index + rowsPerChunk));
+    }
+    return chunks;
+}
+async function* toAsyncIterable(chunks) {
+    if (Symbol.asyncIterator in Object(chunks)) {
+        for await (const chunk of chunks) {
+            yield chunk;
+        }
+        return;
+    }
+    for (const chunk of chunks) {
+        yield chunk;
+    }
+}
+function buildWhereClause(where) {
+    if (where != null && typeof where === "object" && !Array.isArray(where)) {
+        return Object.entries(where)
+            .map(([key, value]) => `${key} = ${JSON.stringify(encodeLegacyValue(value))}`)
+            .join(" AND ");
+    }
+    if (typeof where === "string") {
+        return where;
+    }
+    return null;
+}
+class DatabaseWriteInputStream {
+    constructor(start, chunks) {
+        this.start = start;
+        this.pulls = [];
+        this.closed = false;
+        this.source = toAsyncIterable(chunks)[Symbol.asyncIterator]();
+    }
+    requestNext() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        this.pulls.push(() => undefined);
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        while (this.pulls.length > 0) {
+            const waiter = this.pulls.shift();
+            waiter?.();
+        }
+        void this.source.return?.();
+    }
+    async waitForPull() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        await new Promise((resolve) => {
+            this.pulls.push(resolve);
+        });
+    }
+    async *stream() {
+        yield new JsonContent({ json: this.start });
+        while (!this.closed) {
+            await this.waitForPull();
+            if (this.closed) {
+                return;
+            }
+            const nextChunk = await this.source.next();
+            if (nextChunk.done) {
+                return;
+            }
+            if (nextChunk.value.length === 0) {
+                continue;
+            }
+            yield new JsonContent({ json: rowsChunk(nextChunk.value) });
+        }
+    }
+}
+class DatabaseReadInputStream {
+    constructor(start) {
+        this.start = start;
+        this.pulls = [];
+        this.closed = false;
+    }
+    requestNext() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        this.pulls.push(() => undefined);
+    }
+    close() {
+        if (this.closed) {
+            return;
+        }
+        this.closed = true;
+        while (this.pulls.length > 0) {
+            const waiter = this.pulls.shift();
+            waiter?.();
+        }
+    }
+    async waitForPull() {
+        if (this.closed) {
+            return;
+        }
+        const waiter = this.pulls.shift();
+        if (waiter) {
+            waiter();
+            return;
+        }
+        await new Promise((resolve) => {
+            this.pulls.push(resolve);
+        });
+    }
+    async *stream() {
+        yield new JsonContent({ json: this.start });
+        while (!this.closed) {
+            await this.waitForPull();
+            if (this.closed) {
+                return;
+            }
+            yield new JsonContent({ json: { kind: "pull" } });
+        }
+    }
+}
 export class DatabaseClient {
     constructor({ room }) {
         this.room = room;
     }
-    async listTables() {
-        const response = await this.room.sendRequest("database.list_tables", {});
-        return response?.json?.tables ?? [];
+    _unexpectedResponseError(operation) {
+        return new RoomServerException(`unexpected return type from database.${operation}`);
+    }
+    async invoke(operation, input) {
+        const response = await this.room.invoke({ toolkit: "database", tool: operation, input });
+        if (response instanceof JsonContent) {
+            return response;
+        }
+        if (response == null) {
+            return null;
+        }
+        return null;
     }
-    async createTable({ name, data, schema, mode = "create" }) {
-        let schemaDict;
-        if (schema) {
-            schemaDict = {};
-            for (const [key, value] of Object.entries(schema)) {
-                schemaDict[key] = value.toJson();
+    async invokeStream(operation, input) {
+        return await this.room.invokeStream({ toolkit: "database", tool: operation, input });
+    }
+    async drainWriteStream(operation, input) {
+        const response = await this.invokeStream(operation, input.stream());
+        try {
+            for await (const chunk of response) {
+                if (chunk instanceof ErrorContent) {
+                    throw new RoomServerException(chunk.text, chunk.code);
+                }
+                if (chunk instanceof ControlContent) {
+                    if (chunk.method === "close") {
+                        return;
+                    }
+                    throw this._unexpectedResponseError(operation);
+                }
+                if (!(chunk instanceof JsonContent) || chunk.json.kind !== "pull") {
+                    throw this._unexpectedResponseError(operation);
+                }
+                input.requestNext();
+            }
+        }
+        finally {
+            input.close();
+        }
+    }
+    async *streamRows(operation, start) {
+        const input = new DatabaseReadInputStream(start);
+        const response = await this.invokeStream(operation, input.stream());
+        input.requestNext();
+        try {
+            for await (const chunk of response) {
+                if (chunk instanceof ErrorContent) {
+                    throw new RoomServerException(chunk.text, chunk.code);
+                }
+                if (chunk instanceof ControlContent) {
+                    if (chunk.method === "close") {
+                        return;
+                    }
+                    throw this._unexpectedResponseError(operation);
+                }
+                if (!(chunk instanceof JsonContent)) {
+                    throw this._unexpectedResponseError(operation);
+                }
+                yield recordsFromRowsChunk(chunk.json, operation);
+                input.requestNext();
             }
         }
-        const payload = {
+        finally {
+            input.close();
+        }
+    }
+    async listTables() {
+        const response = await this.invoke("list_tables", { namespace: null });
+        if (!(response instanceof JsonContent)) {
+            throw this._unexpectedResponseError("list_tables");
+        }
+        return Array.isArray(response.json.tables) ? response.json.tables : [];
+    }
+    async createTable({ name, data, schema, mode = "create", }) {
+        const input = new DatabaseWriteInputStream({
+            kind: "start",
             name,
-            data,
-            schema: schemaDict,
+            fields: schemaEntries(schema),
             mode,
-        };
-        await this.room.sendRequest("database.create_table", payload);
+            namespace: null,
+            metadata: null,
+        }, data ?? []);
+        await this.drainWriteStream("create_table", input);
     }
     async createTableWithSchema({ name, schema, data, mode = "create" }) {
-        return this.createTable({ name, schema, data, mode });
+        return this.createTable({
+            name,
+            schema,
+            data: data == null ? undefined : rowChunkList(data),
+            mode,
+        });
     }
     async createTableFromData({ name, data, mode = "create" }) {
-        return this.createTable({ name, data, mode });
+        return this.createTable({
+            name,
+            data: data == null ? undefined : rowChunkList(data),
+            mode,
+        });
+    }
+    async createTableFromDataStream({ name, chunks, schema, mode = "create" }) {
+        return this.createTable({ name, data: chunks, schema, mode });
     }
     async dropTable({ name, ignoreMissing = false }) {
-        await this.room.sendRequest("database.drop_table", { name, ignoreMissing });
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "drop_table",
+            input: { name, ignore_missing: ignoreMissing, namespace: null },
+        });
     }
     async addColumns({ table, newColumns }) {
-        await this.room.sendRequest("database.add_columns", {
-            table,
-            new_columns: newColumns
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "add_columns",
+            input: {
+                table,
+                columns: Object.entries(newColumns).map(([name, valueSql]) => ({ name, value_sql: valueSql, data_type: null })),
+                namespace: null,
+            },
         });
     }
     async dropColumns({ table, columns }) {
-        await this.room.sendRequest("database.drop_columns", { table, columns });
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "drop_columns",
+            input: { table, columns, namespace: null },
+        });
     }
     async insert({ table, records }) {
-        await this.room.sendRequest("database.insert", { table, records });
+        await this.insertStream({ table, chunks: rowChunkList(records) });
     }
-    async update({ table, where, values, valuesSql }) {
-        const payload = {
+    async insertStream({ table, chunks }) {
+        const input = new DatabaseWriteInputStream({
+            kind: "start",
             table,
-            where,
-            values,
-            valuesSql,
-        };
-        await this.room.sendRequest("database.update", payload);
+            namespace: null,
+        }, chunks);
+        await this.drainWriteStream("insert", input);
+    }
+    async update({ table, where, values, valuesSql }) {
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "update",
+            input: {
+                table,
+                where,
+                values: values == null ? null : Object.entries(values).map(([column, value]) => ({ column, value_json: JSON.stringify(encodeLegacyValue(value)) })),
+                values_sql: valuesSql == null ? null : Object.entries(valuesSql).map(([column, expression]) => ({ column, expression })),
+                namespace: null,
+            },
+        });
     }
     async delete({ table, where }) {
-        await this.room.sendRequest("database.delete", { table, where });
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "delete",
+            input: { table, where, namespace: null },
+        });
     }
     async merge({ table, on, records }) {
-        await this.room.sendRequest("database.merge", { table, on, records });
+        await this.mergeStream({ table, on, chunks: rowChunkList(records) });
+    }
+    async mergeStream({ table, on, chunks }) {
+        const input = new DatabaseWriteInputStream({
+            kind: "start",
+            table,
+            on,
+            namespace: null,
+        }, chunks);
+        await this.drainWriteStream("merge", input);
     }
     async sql({ query, tables, params }) {
-        const payload = {
+        const rows = [];
+        for await (const chunk of this.sqlStream({ query, tables, params })) {
+            rows.push(...chunk);
+        }
+        return rows;
+    }
+    async *sqlStream({ query, tables, params }) {
+        yield* this.streamRows("sql", {
+            kind: "start",
             query,
             tables,
-            params,
-        };
-        const response = await this.room.sendRequest("database.sql", payload);
-        if (response instanceof JsonContent) {
-            if (response?.json?.results) {
-                return response.json.results;
-            }
-        }
-        return [];
+            params_json: params == null ? null : JSON.stringify(encodeLegacyValue(params)),
+        });
     }
     async search({ table, text, vector, where, limit, select }) {
-        let whereClause = where;
-        if (where && typeof where === "object" && !Array.isArray(where)) {
-            const parts = [];
-            for (const [key, value] of Object.entries(where)) {
-                parts.push(`${key} = ${JSON.stringify(value)}`);
-            }
-            whereClause = parts.join(" AND ");
+        const rows = [];
+        for await (const chunk of this.searchStream({ table, text, vector, where, limit, select })) {
+            rows.push(...chunk);
         }
-        const payload = {
+        return rows;
+    }
+    async *searchStream({ table, text, vector, where, limit, select }) {
+        yield* this.streamRows("search", {
+            kind: "start",
             table,
-            where: whereClause,
-            text,
-        };
-        if (limit !== undefined) {
-            payload.limit = limit;
-        }
-        if (select !== undefined) {
-            payload.select = select;
-        }
-        if (vector !== undefined) {
-            payload.vector = vector;
-        }
-        const response = await this.room.sendRequest("database.search", payload);
-        if (response instanceof JsonContent) {
-            if (response?.json?.results) {
-                return response.json.results;
-            }
-        }
-        return [];
+            text: text ?? null,
+            vector: vector ?? null,
+            text_columns: null,
+            where: buildWhereClause(where),
+            offset: null,
+            limit: limit ?? null,
+            select: select ?? null,
+            namespace: null,
+        });
     }
     async optimize(table) {
-        await this.room.sendRequest("database.optimize", { table });
+        await this.room.invoke({ toolkit: "database", tool: "optimize", input: { table, namespace: null } });
     }
-    async createVectorIndex({ table, column }) {
-        await this.room.sendRequest("database.create_vector_index", { table, column });
+    async createVectorIndex({ table, column, replace = false }) {
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "create_vector_index",
+            input: { table, column, replace, namespace: null },
+        });
     }
-    async createScalarIndex({ table, column }) {
-        await this.room.sendRequest("database.create_scalar_index", { table, column });
+    async createScalarIndex({ table, column, replace = false }) {
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "create_scalar_index",
+            input: { table, column, replace, namespace: null },
+        });
     }
-    async createFullTextSearchIndex({ table, column }) {
-        await this.room.sendRequest("database.create_full_text_search_index", { table, column });
+    async createFullTextSearchIndex({ table, column, replace = false }) {
+        await this.room.invoke({
+            toolkit: "database",
+            tool: "create_full_text_search_index",
+            input: { table, column, replace, namespace: null },
+        });
     }
     async listIndexes({ table }) {
-        const response = await this.room.sendRequest("database.list_indexes", { table });
-        return response?.json ?? {};
+        const response = await this.invoke("list_indexes", { table, namespace: null });
+        if (!(response instanceof JsonContent)) {
+            throw this._unexpectedResponseError("list_indexes");
+        }
+        if (!Array.isArray(response.json.indexes)) {
+            throw this._unexpectedResponseError("list_indexes");
+        }
+        return response.json.indexes;
     }
 }