@lancedb/lancedb 0.21.2-beta.0 → 0.21.2

package/CLAUDE.md

@@ -0,0 +1,13 @@
+These are the typescript bindings of LanceDB.
+The core Rust library is in the `../rust/lancedb` directory, the rust binding
+code is in the `src/` directory and the typescript bindings are in
+the `lancedb/` directory.
+
+Whenever you change the Rust code, you will need to recompile: `npm run build`.
+
+Common commands:
+* Build: `npm run build`
+* Lint: `npm run lint`
+* Fix lints: `npm run lint-fix`
+* Test: `npm test`
+* Run single test file: `npm test __test__/arrow.test.ts`

package/dist/arrow.d.ts

@@ -36,6 +36,9 @@ export type TableLike = ArrowTable | {
     batches: RecordBatchLike[];
 };
 export type IntoVector = Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
+export type MultiVector = IntoVector[];
+export declare function isMultiVector(value: unknown): value is MultiVector;
+export declare function isIntoVector(value: unknown): value is IntoVector;
 export declare function isArrowTable(value: object): value is TableLike;
 export declare function isNull(value: unknown): value is Null;
 export declare function isInt(value: unknown): value is Int;
@@ -257,6 +260,11 @@ export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?:
  * Create an empty table with the given schema
  */
 export declare function createEmptyTable(schema: Schema): ArrowTable;
+/**
+ * Ensures that all nested fields defined in the schema exist in the data,
+ * filling missing fields with null values.
+ */
+export declare function ensureNestedFieldsExist(data: Array<Record<string, unknown>>, schema: Schema): Array<Record<string, unknown>>;
 interface JsonDataType {
     type: string;
     fields?: JsonField[];

package/dist/arrow.js

@@ -17,6 +17,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MakeArrowTableOptions = exports.VectorColumnOptions = void 0;
+exports.isMultiVector = isMultiVector;
+exports.isIntoVector = isIntoVector;
 exports.isArrowTable = isArrowTable;
 exports.isNull = isNull;
 exports.isInt = isInt;
@@ -49,11 +51,20 @@ exports.fromBufferToRecordBatch = fromBufferToRecordBatch;
 exports.fromRecordBatchToBuffer = fromRecordBatchToBuffer;
 exports.fromTableToStreamBuffer = fromTableToStreamBuffer;
 exports.createEmptyTable = createEmptyTable;
+exports.ensureNestedFieldsExist = ensureNestedFieldsExist;
 exports.dataTypeToJson = dataTypeToJson;
 const apache_arrow_1 = require("apache-arrow");
 const registry_1 = require("./embedding/registry");
 const sanitize_1 = require("./sanitize");
 __exportStar(require("apache-arrow"), exports);
+function isMultiVector(value) {
+    return Array.isArray(value) && isIntoVector(value[0]);
+}
+function isIntoVector(value) {
+    return (value instanceof Float32Array ||
+        value instanceof Float64Array ||
+        (Array.isArray(value) && !Array.isArray(value[0])));
+}
 function isArrowTable(value) {
     if (value instanceof apache_arrow_1.Table)
         return true;
@@ -711,6 +722,13 @@ async function applyEmbeddingsFromMetadata(table, schema) {
         const vector = makeVector(vectors, destType);
         columns[destColumn] = vector;
     }
+    // Add any missing columns from the schema as null vectors
+    for (const field of schema.fields) {
+        if (!(field.name in columns)) {
+            const nullValues = new Array(table.numRows).fill(null);
+            columns[field.name] = makeVector(nullValues, field.type);
+        }
+    }
     const newTable = new apache_arrow_1.Table(columns);
     return alignTable(newTable, schema);
 }
@@ -824,7 +842,14 @@ async function applyEmbeddings(table, embeddings, schema) {
  * be placed at the end of the table, after all of the input columns.
  */
 async function convertToTable(data, embeddings, makeTableOptions) {
-    const table = makeArrowTable(data, makeTableOptions);
+    let processedData = data;
+    // If we have a schema with embedding metadata, we need to preprocess the data
+    // to ensure all nested fields are present
+    if (makeTableOptions?.schema &&
+        makeTableOptions.schema.metadata?.has("embedding_functions")) {
+        processedData = ensureNestedFieldsExist(data, makeTableOptions.schema);
+    }
+    const table = makeArrowTable(processedData, makeTableOptions);
     return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
 }
 /** Creates the Arrow Type for a Vector column with dimension `dim` */
@@ -893,7 +918,17 @@ async function fromDataToBuffer(data, embeddings, schema) {
         schema = (0, sanitize_1.sanitizeSchema)(schema);
     }
     if (isArrowTable(data)) {
-        return fromTableToBuffer((0, sanitize_1.sanitizeTable)(data), embeddings, schema);
+        const table = (0, sanitize_1.sanitizeTable)(data);
+        // If we have a schema with embedding functions, we need to ensure all columns exist
+        // before applying embeddings, since applyEmbeddingsFromMetadata expects all columns
+        // to be present in the table
+        if (schema && schema.metadata?.has("embedding_functions")) {
+            const alignedTable = alignTableToSchema(table, schema);
+            return fromTableToBuffer(alignedTable, embeddings, schema);
+        }
+        else {
+            return fromTableToBuffer(table, embeddings, schema);
+        }
     }
     else {
         const table = await convertToTable(data, embeddings, { schema });
@@ -1004,6 +1039,61 @@ function validateSchemaEmbeddings(schema, data, embeddings) {
     }
     return new apache_arrow_1.Schema(fields, schema.metadata);
 }
+/**
+ * Ensures that all nested fields defined in the schema exist in the data,
+ * filling missing fields with null values.
+ */
+function ensureNestedFieldsExist(data, schema) {
+    return data.map((row) => {
+        const completeRow = {};
+        for (const field of schema.fields) {
+            if (field.name in row) {
+                if (field.type.constructor.name === "Struct" &&
+                    row[field.name] !== null &&
+                    row[field.name] !== undefined) {
+                    // Handle nested struct
+                    const nestedValue = row[field.name];
+                    completeRow[field.name] = ensureStructFieldsExist(nestedValue, field.type);
+                }
+                else {
+                    // Non-struct field or null struct value
+                    completeRow[field.name] = row[field.name];
+                }
+            }
+            else {
+                // Field is missing from the data - set to null
+                completeRow[field.name] = null;
+            }
+        }
+        return completeRow;
+    });
+}
+/**
+ * Recursively ensures that all fields in a struct type exist in the data,
+ * filling missing fields with null values.
+ */
+function ensureStructFieldsExist(data, structType) {
+    const completeStruct = {};
+    for (const childField of structType.children) {
+        if (childField.name in data) {
+            if (childField.type.constructor.name === "Struct" &&
+                data[childField.name] !== null &&
+                data[childField.name] !== undefined) {
+                // Recursively handle nested struct
+                completeStruct[childField.name] = ensureStructFieldsExist(data[childField.name], childField.type);
+            }
+            else {
+                // Non-struct field or null struct value
+                completeStruct[childField.name] = data[childField.name];
+            }
+        }
+        else {
+            // Field is missing - set to null
+            completeStruct[childField.name] = null;
+        }
+    }
+    return completeStruct;
+}
 // Matches format of https://github.com/lancedb/lance/blob/main/rust/lance/src/arrow/json.rs
 function dataTypeToJson(dataType) {
     switch (dataType.typeId) {
@@ -1123,3 +1213,51 @@ function fieldToJson(field) {
         metadata: field.metadata,
     };
 }
+function alignTableToSchema(table, targetSchema) {
+    const existingColumns = new Map();
+    // Map existing columns
+    for (const field of table.schema.fields) {
+        existingColumns.set(field.name, table.getChild(field.name));
+    }
+    // Create vectors for all fields in target schema
+    const alignedColumns = {};
+    for (const field of targetSchema.fields) {
+        if (existingColumns.has(field.name)) {
+            // Column exists, use it
+            alignedColumns[field.name] = existingColumns.get(field.name);
+        }
+        else {
+            // Column missing, create null vector
+            alignedColumns[field.name] = createNullVector(field, table.numRows);
+        }
+    }
+    // Create new table with aligned schema and columns
+    return new apache_arrow_1.Table(targetSchema, alignedColumns);
+}
+function createNullVector(field, numRows) {
+    if (field.type.constructor.name === "Struct") {
+        // For struct types, create a struct with null fields
+        const structType = field.type;
+        const childVectors = structType.children.map((childField) => createNullVector(childField, numRows));
+        // Create struct data
+        const structData = (0, apache_arrow_1.makeData)({
+            type: structType,
+            length: numRows,
+            nullCount: 0,
+            children: childVectors.map((v) => v.data[0]),
+        });
+        return (0, apache_arrow_1.makeVector)(structData);
+    }
+    else {
+        // For other types, create a vector of nulls
+        const nullBitmap = new Uint8Array(Math.ceil(numRows / 8));
+        // All bits are 0, meaning all values are null
+        const data = (0, apache_arrow_1.makeData)({
+            type: field.type,
+            length: numRows,
+            nullCount: numRows,
+            nullBitmap,
+        });
+        return (0, apache_arrow_1.makeVector)(data);
+    }
+}

package/dist/connection.d.ts

@@ -62,6 +62,9 @@ export interface OpenTableOptions {
     /**
      * Set the size of the index cache, specified as a number of entries
      *
+     * @deprecated Use session-level cache configuration instead.
+     * Create a Session with custom cache sizes and pass it to the connect() function.
+     *
      * The exact meaning of an "entry" will depend on the type of index:
      * - IVF: there is one entry for each IVF partition
      * - BTREE: there is one entry for the entire index

package/dist/index.d.ts

@@ -1,15 +1,16 @@
 import { Connection } from "./connection";
-import { ConnectionOptions } from "./native.js";
+import { ConnectionOptions, Session } from "./native.js";
 export { AddColumnsSql, ConnectionOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, MergeResult, AddResult, AddColumnsResult, AlterColumnsResult, DeleteResult, DropColumnsResult, UpdateResult, } from "./native.js";
 export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
 export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, } from "./connection";
+export { Session } from "./native.js";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, QueryExecutionOptions, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
 export { Index, IndexOptions, IvfPqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, } from "./indices";
 export { Table, AddDataOptions, UpdateOptions, OptimizeOptions, Version, ColumnAlteration, } from "./table";
 export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
 export * as embedding from "./embedding";
 export * as rerankers from "./rerankers";
-export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, } from "./arrow";
+export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
 export { IntoSql, packBits } from "./util";
 /**
  * Connect to a LanceDB instance at the given URI.
@@ -35,7 +36,7 @@ export { IntoSql, packBits } from "./util";
  * });
  * ```
  */
-export declare function connect(uri: string, options?: Partial<ConnectionOptions>): Promise<Connection>;
+export declare function connect(uri: string, options?: Partial<ConnectionOptions>, session?: Session): Promise<Connection>;
 /**
  * Connect to a LanceDB instance at the given URI.
  *
@@ -53,6 +54,15 @@ export declare function connect(uri: string, options?: Partial<ConnectionOptions
  *   storageOptions: {timeout: "60s"}
  * });
  * ```
+ *
+ * @example
+ * ```ts
+ * const session = Session.default();
+ * const conn = await connect({
+ *   uri: "/path/to/database",
+ *   session: session
+ * });
+ * ```
  */
 export declare function connect(options: Partial<ConnectionOptions> & {
     uri: string;

package/dist/index.js

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.packBits = exports.rerankers = exports.embedding = exports.MergeInsertBuilder = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.TagContents = exports.Tags = void 0;
+exports.packBits = exports.rerankers = exports.embedding = exports.MergeInsertBuilder = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Session = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.TagContents = exports.Tags = void 0;
 exports.connect = connect;
 const connection_1 = require("./connection");
 const native_js_1 = require("./native.js");
@@ -15,6 +15,8 @@ Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get:
 Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: function () { return arrow_1.VectorColumnOptions; } });
 var connection_2 = require("./connection");
 Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
+var native_js_3 = require("./native.js");
+Object.defineProperty(exports, "Session", { enumerable: true, get: function () { return native_js_3.Session; } });
 var query_1 = require("./query");
 Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
 Object.defineProperty(exports, "QueryBase", { enumerable: true, get: function () { return query_1.QueryBase; } });
@@ -38,21 +40,23 @@ exports.embedding = require("./embedding");
 exports.rerankers = require("./rerankers");
 var util_1 = require("./util");
 Object.defineProperty(exports, "packBits", { enumerable: true, get: function () { return util_1.packBits; } });
-async function connect(uriOrOptions, options = {}) {
+async function connect(uriOrOptions, options) {
     let uri;
+    let finalOptions = {};
     if (typeof uriOrOptions !== "string") {
         const { uri: uri_, ...opts } = uriOrOptions;
         uri = uri_;
-        options = opts;
+        finalOptions = opts;
     }
     else {
         uri = uriOrOptions;
+        finalOptions = options || {};
     }
     if (!uri) {
         throw new Error("uri is required");
     }
-    options = options ?? {};
-    options.storageOptions = (0, connection_1.cleanseStorageOptions)(options.storageOptions);
-    const nativeConn = await native_js_1.Connection.new(uri, options);
+    finalOptions = finalOptions ?? {};
+    finalOptions.storageOptions = (0, connection_1.cleanseStorageOptions)(finalOptions.storageOptions);
+    const nativeConn = await native_js_1.Connection.new(uri, finalOptions);
     return new connection_1.LocalConnection(nativeConn);
 }

package/dist/native.d.ts

@@ -274,6 +274,11 @@ export interface ConnectionOptions {
    * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
    */
   storageOptions?: Record<string, string>
+  /**
+   * (For LanceDB OSS only): the session to use for this connection. Holds
+   * shared caches and other session-specific state.
+   */
+  session?: Session
   /** (For LanceDB cloud only): configuration for the remote HTTP client. */
   clientConfig?: ClientConfig
   /**
@@ -403,6 +408,40 @@ export class RrfReranker {
   static tryNew(k: Float32Array): Promise<RrfReranker>
   rerankHybrid(query: string, vecResults: Buffer, ftsResults: Buffer): Promise<Buffer>
 }
+/**
+ * A session for managing caches and object stores across LanceDB operations.
+ *
+ * Sessions allow you to configure cache sizes for index and metadata caches,
+ * which can significantly impact memory use and performance. They can
+ * also be re-used across multiple connections to share the same cache state.
+ */
+export class Session {
+  /**
+   * Create a new session with custom cache sizes.
+   *
+   * # Parameters
+   *
+   * - `index_cache_size_bytes`: The size of the index cache in bytes.
+   *   Index data is stored in memory in this cache to speed up queries.
+   *   Defaults to 6GB if not specified.
+   * - `metadata_cache_size_bytes`: The size of the metadata cache in bytes.
+   *   The metadata cache stores file metadata and schema information in memory.
+   *   This cache improves scan and write performance.
+   *   Defaults to 1GB if not specified.
+   */
+  constructor(indexCacheSizeBytes?: bigint | undefined | null, metadataCacheSizeBytes?: bigint | undefined | null)
+  /**
+   * Create a session with default cache sizes.
+   *
+   * This is equivalent to creating a session with 6GB index cache
+   * and 1GB metadata cache.
+   */
+  static default(): Session
+  /** Get the current size of the session caches in bytes. */
+  sizeBytes(): bigint
+  /** Get the approximate number of items cached in the session. */
+  approxNumItems(): number
+}
 export class Table {
   name: string
   display(): string

package/dist/native.js

@@ -319,7 +319,7 @@ if (!nativeBinding) {
     }
     throw new Error(`Failed to load native binding`);
 }
-const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, JsFullTextQuery, Reranker, RrfReranker, Table, TagContents, Tags } = nativeBinding;
+const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, JsFullTextQuery, Reranker, RrfReranker, Session, Table, TagContents, Tags } = nativeBinding;
 module.exports.Connection = Connection;
 module.exports.Index = Index;
 module.exports.RecordBatchIterator = RecordBatchIterator;
@@ -329,6 +329,7 @@ module.exports.VectorQuery = VectorQuery;
 module.exports.JsFullTextQuery = JsFullTextQuery;
 module.exports.Reranker = Reranker;
 module.exports.RrfReranker = RrfReranker;
+module.exports.Session = Session;
 module.exports.Table = Table;
 module.exports.TagContents = TagContents;
 module.exports.Tags = Tags;

package/dist/table.d.ts

@@ -1,4 +1,4 @@
-import { Table as ArrowTable, Data, DataType, IntoVector, Schema } from "./arrow";
+import { Table as ArrowTable, Data, DataType, IntoVector, MultiVector, Schema } from "./arrow";
 import { IndexOptions } from "./indices";
 import { MergeInsertBuilder } from "./merge";
 import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, OptimizeStats, TableStatistics, Tags, UpdateResult, Table as _NativeTable } from "./native";
@@ -280,7 +280,7 @@ export declare abstract class Table {
      * when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
      * if the query is a string and no embedding function is defined, it will be treated as a full text search query
      */
-    abstract search(query: string | IntoVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
     /**
      * Search the table with a given query vector.
      *
@@ -288,7 +288,7 @@ export declare abstract class Table {
      * is the same thing as calling `nearestTo` on the builder returned
      * by `query`.  @see {@link Query#nearestTo} for more details.
      */
-    abstract vectorSearch(vector: IntoVector): VectorQuery;
+    abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
     /**
      * Add new columns with defined values.
      * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
@@ -463,8 +463,8 @@ export declare class LocalTable extends Table {
     prewarmIndex(name: string): Promise<void>;
     waitForIndex(indexNames: string[], timeoutSeconds: number): Promise<void>;
     query(): Query;
-    search(query: string | IntoVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
-    vectorSearch(vector: IntoVector): VectorQuery;
+    search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
     addColumns(newColumnTransforms: AddColumnsSql[]): Promise<AddColumnsResult>;
     alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
     dropColumns(columnNames: string[]): Promise<DropColumnsResult>;

package/dist/table.js

@@ -171,6 +171,13 @@ class LocalTable extends Table {
         return this.query().nearestTo(queryPromise);
     }
     vectorSearch(vector) {
+        if ((0, arrow_1.isMultiVector)(vector)) {
+            const query = this.query().nearestTo(vector[0]);
+            for (const v of vector.slice(1)) {
+                query.addQueryVector(v);
+            }
+            return query;
+        }
         return this.query().nearestTo(vector);
     }
     // TODO: Support BatchUDF

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.21.2-beta.0",
+  "version": "0.21.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -100,14 +100,14 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.21.2-beta.0",
-    "@lancedb/lancedb-darwin-arm64": "0.21.2-beta.0",
-    "@lancedb/lancedb-linux-x64-gnu": "0.21.2-beta.0",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.21.2-beta.0",
-    "@lancedb/lancedb-linux-x64-musl": "0.21.2-beta.0",
-    "@lancedb/lancedb-linux-arm64-musl": "0.21.2-beta.0",
-    "@lancedb/lancedb-win32-x64-msvc": "0.21.2-beta.0",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.21.2-beta.0"
+    "@lancedb/lancedb-darwin-x64": "0.21.2",
+    "@lancedb/lancedb-darwin-arm64": "0.21.2",
+    "@lancedb/lancedb-linux-x64-gnu": "0.21.2",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.21.2",
+    "@lancedb/lancedb-linux-x64-musl": "0.21.2",
+    "@lancedb/lancedb-linux-arm64-musl": "0.21.2",
+    "@lancedb/lancedb-win32-x64-msvc": "0.21.2",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.21.2"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"