@lancedb/lancedb 0.21.1 → 0.21.2-beta.1

package/dist/arrow.d.ts

@@ -36,6 +36,9 @@ export type TableLike = ArrowTable | {
     batches: RecordBatchLike[];
 };
 export type IntoVector = Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
+export type MultiVector = IntoVector[];
+export declare function isMultiVector(value: unknown): value is MultiVector;
+export declare function isIntoVector(value: unknown): value is IntoVector;
 export declare function isArrowTable(value: object): value is TableLike;
 export declare function isNull(value: unknown): value is Null;
 export declare function isInt(value: unknown): value is Int;
@@ -257,6 +260,11 @@ export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?:
  * Create an empty table with the given schema
  */
 export declare function createEmptyTable(schema: Schema): ArrowTable;
+/**
+ * Ensures that all nested fields defined in the schema exist in the data,
+ * filling missing fields with null values.
+ */
+export declare function ensureNestedFieldsExist(data: Array<Record<string, unknown>>, schema: Schema): Array<Record<string, unknown>>;
 interface JsonDataType {
     type: string;
     fields?: JsonField[];

package/dist/arrow.js

@@ -17,6 +17,8 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MakeArrowTableOptions = exports.VectorColumnOptions = void 0;
+exports.isMultiVector = isMultiVector;
+exports.isIntoVector = isIntoVector;
 exports.isArrowTable = isArrowTable;
 exports.isNull = isNull;
 exports.isInt = isInt;
@@ -49,11 +51,20 @@ exports.fromBufferToRecordBatch = fromBufferToRecordBatch;
 exports.fromRecordBatchToBuffer = fromRecordBatchToBuffer;
 exports.fromTableToStreamBuffer = fromTableToStreamBuffer;
 exports.createEmptyTable = createEmptyTable;
+exports.ensureNestedFieldsExist = ensureNestedFieldsExist;
 exports.dataTypeToJson = dataTypeToJson;
 const apache_arrow_1 = require("apache-arrow");
 const registry_1 = require("./embedding/registry");
 const sanitize_1 = require("./sanitize");
 __exportStar(require("apache-arrow"), exports);
+function isMultiVector(value) {
+    return Array.isArray(value) && isIntoVector(value[0]);
+}
+function isIntoVector(value) {
+    return (value instanceof Float32Array ||
+        value instanceof Float64Array ||
+        (Array.isArray(value) && !Array.isArray(value[0])));
+}
 function isArrowTable(value) {
     if (value instanceof apache_arrow_1.Table)
         return true;
@@ -711,6 +722,13 @@ async function applyEmbeddingsFromMetadata(table, schema) {
         const vector = makeVector(vectors, destType);
         columns[destColumn] = vector;
     }
+    // Add any missing columns from the schema as null vectors
+    for (const field of schema.fields) {
+        if (!(field.name in columns)) {
+            const nullValues = new Array(table.numRows).fill(null);
+            columns[field.name] = makeVector(nullValues, field.type);
+        }
+    }
     const newTable = new apache_arrow_1.Table(columns);
     return alignTable(newTable, schema);
 }
@@ -824,7 +842,14 @@ async function applyEmbeddings(table, embeddings, schema) {
  * be placed at the end of the table, after all of the input columns.
  */
 async function convertToTable(data, embeddings, makeTableOptions) {
-    const table = makeArrowTable(data, makeTableOptions);
+    let processedData = data;
+    // If we have a schema with embedding metadata, we need to preprocess the data
+    // to ensure all nested fields are present
+    if (makeTableOptions?.schema &&
+        makeTableOptions.schema.metadata?.has("embedding_functions")) {
+        processedData = ensureNestedFieldsExist(data, makeTableOptions.schema);
+    }
+    const table = makeArrowTable(processedData, makeTableOptions);
     return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
 }
 /** Creates the Arrow Type for a Vector column with dimension `dim` */
@@ -893,7 +918,17 @@ async function fromDataToBuffer(data, embeddings, schema) {
         schema = (0, sanitize_1.sanitizeSchema)(schema);
     }
     if (isArrowTable(data)) {
-        return fromTableToBuffer((0, sanitize_1.sanitizeTable)(data), embeddings, schema);
+        const table = (0, sanitize_1.sanitizeTable)(data);
+        // If we have a schema with embedding functions, we need to ensure all columns exist
+        // before applying embeddings, since applyEmbeddingsFromMetadata expects all columns
+        // to be present in the table
+        if (schema && schema.metadata?.has("embedding_functions")) {
+            const alignedTable = alignTableToSchema(table, schema);
+            return fromTableToBuffer(alignedTable, embeddings, schema);
+        }
+        else {
+            return fromTableToBuffer(table, embeddings, schema);
+        }
     }
     else {
         const table = await convertToTable(data, embeddings, { schema });
@@ -1004,6 +1039,61 @@ function validateSchemaEmbeddings(schema, data, embeddings) {
     }
     return new apache_arrow_1.Schema(fields, schema.metadata);
 }
+/**
+ * Ensures that all nested fields defined in the schema exist in the data,
+ * filling missing fields with null values.
+ */
+function ensureNestedFieldsExist(data, schema) {
+    return data.map((row) => {
+        const completeRow = {};
+        for (const field of schema.fields) {
+            if (field.name in row) {
+                if (field.type.constructor.name === "Struct" &&
+                    row[field.name] !== null &&
+                    row[field.name] !== undefined) {
+                    // Handle nested struct
+                    const nestedValue = row[field.name];
+                    completeRow[field.name] = ensureStructFieldsExist(nestedValue, field.type);
+                }
+                else {
+                    // Non-struct field or null struct value
+                    completeRow[field.name] = row[field.name];
+                }
+            }
+            else {
+                // Field is missing from the data - set to null
+                completeRow[field.name] = null;
+            }
+        }
+        return completeRow;
+    });
+}
+/**
+ * Recursively ensures that all fields in a struct type exist in the data,
+ * filling missing fields with null values.
+ */
+function ensureStructFieldsExist(data, structType) {
+    const completeStruct = {};
+    for (const childField of structType.children) {
+        if (childField.name in data) {
+            if (childField.type.constructor.name === "Struct" &&
+                data[childField.name] !== null &&
+                data[childField.name] !== undefined) {
+                // Recursively handle nested struct
+                completeStruct[childField.name] = ensureStructFieldsExist(data[childField.name], childField.type);
+            }
+            else {
+                // Non-struct field or null struct value
+                completeStruct[childField.name] = data[childField.name];
+            }
+        }
+        else {
+            // Field is missing - set to null
+            completeStruct[childField.name] = null;
+        }
+    }
+    return completeStruct;
+}
 // Matches format of https://github.com/lancedb/lance/blob/main/rust/lance/src/arrow/json.rs
 function dataTypeToJson(dataType) {
     switch (dataType.typeId) {
@@ -1123,3 +1213,51 @@ function fieldToJson(field) {
         metadata: field.metadata,
     };
 }
+function alignTableToSchema(table, targetSchema) {
+    const existingColumns = new Map();
+    // Map existing columns
+    for (const field of table.schema.fields) {
+        existingColumns.set(field.name, table.getChild(field.name));
+    }
+    // Create vectors for all fields in target schema
+    const alignedColumns = {};
+    for (const field of targetSchema.fields) {
+        if (existingColumns.has(field.name)) {
+            // Column exists, use it
+            alignedColumns[field.name] = existingColumns.get(field.name);
+        }
+        else {
+            // Column missing, create null vector
+            alignedColumns[field.name] = createNullVector(field, table.numRows);
+        }
+    }
+    // Create new table with aligned schema and columns
+    return new apache_arrow_1.Table(targetSchema, alignedColumns);
+}
+function createNullVector(field, numRows) {
+    if (field.type.constructor.name === "Struct") {
+        // For struct types, create a struct with null fields
+        const structType = field.type;
+        const childVectors = structType.children.map((childField) => createNullVector(childField, numRows));
+        // Create struct data
+        const structData = (0, apache_arrow_1.makeData)({
+            type: structType,
+            length: numRows,
+            nullCount: 0,
+            children: childVectors.map((v) => v.data[0]),
+        });
+        return (0, apache_arrow_1.makeVector)(structData);
+    }
+    else {
+        // For other types, create a vector of nulls
+        const nullBitmap = new Uint8Array(Math.ceil(numRows / 8));
+        // All bits are 0, meaning all values are null
+        const data = (0, apache_arrow_1.makeData)({
+            type: field.type,
+            length: numRows,
+            nullCount: numRows,
+            nullBitmap,
+        });
+        return (0, apache_arrow_1.makeVector)(data);
+    }
+}

package/dist/index.d.ts

@@ -9,7 +9,7 @@ export { Table, AddDataOptions, UpdateOptions, OptimizeOptions, Version, ColumnA
 export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
 export * as embedding from "./embedding";
 export * as rerankers from "./rerankers";
-export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, } from "./arrow";
+export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
 export { IntoSql, packBits } from "./util";
 /**
  * Connect to a LanceDB instance at the given URI.

package/dist/indices.d.ts

@@ -410,7 +410,7 @@ export interface FtsOptions {
      *
      * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
      */
-    baseTokenizer?: "simple" | "whitespace" | "raw";
+    baseTokenizer?: "simple" | "whitespace" | "raw" | "ngram";
     /**
      * language for stemming and stop words
      * this is only used when `stem` or `remove_stop_words` is true
@@ -437,6 +437,18 @@ export interface FtsOptions {
      * whether to remove punctuation
      */
     asciiFolding?: boolean;
+    /**
+     * ngram min length
+     */
+    ngramMinLength?: number;
+    /**
+     * ngram max length
+     */
+    ngramMaxLength?: number;
+    /**
+     * whether to only index the prefix of the token for ngram tokenizer
+     */
+    prefixOnly?: boolean;
 }
 export declare class Index {
     private readonly inner;

package/dist/indices.js

@@ -107,7 +107,7 @@ class Index {
      * You can combine filters with full text search.
      */
     static fts(options) {
-        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding));
+        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding, options?.ngramMinLength, options?.ngramMaxLength, options?.prefixOnly));
     }
     /**
      *

package/dist/native.d.ts

@@ -325,7 +325,7 @@ export class Index {
   static btree(): Index
   static bitmap(): Index
   static labelList(): Index
-  static fts(withPosition?: boolean | undefined | null, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null): Index
+  static fts(withPosition?: boolean | undefined | null, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null): Index
   static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
   static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
 }

package/dist/table.d.ts

@@ -1,4 +1,4 @@
-import { Table as ArrowTable, Data, DataType, IntoVector, Schema } from "./arrow";
+import { Table as ArrowTable, Data, DataType, IntoVector, MultiVector, Schema } from "./arrow";
 import { IndexOptions } from "./indices";
 import { MergeInsertBuilder } from "./merge";
 import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, OptimizeStats, TableStatistics, Tags, UpdateResult, Table as _NativeTable } from "./native";
@@ -38,10 +38,10 @@ export interface OptimizeOptions {
      * // Delete all versions older than 1 day
      * const olderThan = new Date();
      * olderThan.setDate(olderThan.getDate() - 1));
-     * tbl.cleanupOlderVersions(olderThan);
+     * tbl.optimize({cleanupOlderThan: olderThan});
      *
      * // Delete all versions except the current version
-     * tbl.cleanupOlderVersions(new Date());
+     * tbl.optimize({cleanupOlderThan: new Date()});
      */
     cleanupOlderThan: Date;
     deleteUnverified: boolean;
@@ -280,7 +280,7 @@ export declare abstract class Table {
      * when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
      * if the query is a string and no embedding function is defined, it will be treated as a full text search query
      */
-    abstract search(query: string | IntoVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
     /**
      * Search the table with a given query vector.
      *
@@ -288,7 +288,7 @@ export declare abstract class Table {
      * is the same thing as calling `nearestTo` on the builder returned
      * by `query`.  @see {@link Query#nearestTo} for more details.
      */
-    abstract vectorSearch(vector: IntoVector): VectorQuery;
+    abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
     /**
      * Add new columns with defined values.
      * @param {AddColumnsSql[]} newColumnTransforms pairs of column names and
@@ -463,8 +463,8 @@ export declare class LocalTable extends Table {
     prewarmIndex(name: string): Promise<void>;
     waitForIndex(indexNames: string[], timeoutSeconds: number): Promise<void>;
     query(): Query;
-    search(query: string | IntoVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
-    vectorSearch(vector: IntoVector): VectorQuery;
+    search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
+    vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
     addColumns(newColumnTransforms: AddColumnsSql[]): Promise<AddColumnsResult>;
     alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
     dropColumns(columnNames: string[]): Promise<DropColumnsResult>;

package/dist/table.js

@@ -171,6 +171,13 @@ class LocalTable extends Table {
         return this.query().nearestTo(queryPromise);
     }
     vectorSearch(vector) {
+        if ((0, arrow_1.isMultiVector)(vector)) {
+            const query = this.query().nearestTo(vector[0]);
+            for (const v of vector.slice(1)) {
+                query.addQueryVector(v);
+            }
+            return query;
+        }
         return this.query().nearestTo(vector);
     }
     // TODO: Support BatchUDF

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.21.1",
+  "version": "0.21.2-beta.1",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -100,14 +100,14 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.21.1",
-    "@lancedb/lancedb-darwin-arm64": "0.21.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.21.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.21.1",
-    "@lancedb/lancedb-linux-x64-musl": "0.21.1",
-    "@lancedb/lancedb-linux-arm64-musl": "0.21.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.21.1",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.21.1"
+    "@lancedb/lancedb-darwin-x64": "0.21.2-beta.1",
+    "@lancedb/lancedb-darwin-arm64": "0.21.2-beta.1",
+    "@lancedb/lancedb-linux-x64-gnu": "0.21.2-beta.1",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.21.2-beta.1",
+    "@lancedb/lancedb-linux-x64-musl": "0.21.2-beta.1",
+    "@lancedb/lancedb-linux-arm64-musl": "0.21.2-beta.1",
+    "@lancedb/lancedb-win32-x64-msvc": "0.21.2-beta.1",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.21.2-beta.1"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"