@lancedb/lancedb 0.37.1 → 0.38.0-beta.12

package/dist/query.js

@@ -2,8 +2,9 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.Query = exports.TakeQuery = exports.VectorQuery = exports.StandardQueryBase = exports.QueryBase = void 0;
+exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.Query = exports.AutoQuery = exports.TakeQuery = exports.VectorQuery = exports.StandardQueryBase = exports.QueryBase = void 0;
 exports.RecordBatchIterator = RecordBatchIterator;
+exports.createAutoQuery = createAutoQuery;
 exports.instanceOfFullTextQuery = instanceOfFullTextQuery;
 const arrow_1 = require("./arrow");
 const native_1 = require("./native");
@@ -32,6 +33,22 @@ class RecordBatchIterable {
         return RecordBatchIterator(this.inner.execute(this.options?.maxBatchLength, this.options?.timeoutMs));
     }
 }
+function nearestToNative(inner, vector) {
+    const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
+    if (raw) {
+        return inner.nearestToRaw(raw.data, raw.dtype);
+    }
+    return inner.nearestTo(Float32Array.from(vector));
+}
+function addQueryVectorToNative(inner, vector) {
+    const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
+    if (raw) {
+        inner.addQueryVectorRaw(raw.data, raw.dtype);
+    }
+    else {
+        inner.addQueryVector(Float32Array.from(vector));
+    }
+}
 /** Common methods supported by all query types
  *
  * @see {@link Query}
@@ -45,8 +62,9 @@ class QueryBase {
      * @hidden
      */
     constructor(inner) {
-        this.inner = inner;
-        // intentionally empty
+        if (inner !== undefined) {
+            this.inner = inner;
+        }
     }
     // call a function on the inner (either a promise or the actual object)
     /**
@@ -63,6 +81,14 @@ class QueryBase {
             fn(this.inner);
         }
     }
+    /**
+     * Return the native query used by the next terminal operation.
+     *
+     * @hidden
+     */
+    async getInner() {
+        return this.inner;
+    }
     /**
      * Return only the specified columns.
      *
@@ -132,13 +158,9 @@ class QueryBase {
     /**
      * @hidden
      */
-    nativeExecute(options) {
-        if (this.inner instanceof Promise) {
-            return this.inner.then((inner) => inner.execute(options?.maxBatchLength, options?.timeoutMs));
-        }
-        else {
-            return this.inner.execute(options?.maxBatchLength, options?.timeoutMs);
-        }
+    async nativeExecute(options) {
+        const inner = await this.getInner();
+        return inner.execute(options?.maxBatchLength, options?.timeoutMs);
     }
     /**
      * Execute the query and return the results as an @see {@link AsyncIterator}
@@ -164,13 +186,7 @@ class QueryBase {
     /** Collect the results as an Arrow @see {@link ArrowTable}. */
     async toArrow(options) {
         const batches = [];
-        let inner;
-        if (this.inner instanceof Promise) {
-            inner = await this.inner;
-        }
-        else {
-            inner = this.inner;
-        }
+        const inner = await this.getInner();
         for await (const batch of new RecordBatchIterable(inner, options)) {
             batches.push(batch);
         }
@@ -197,12 +213,8 @@ class QueryBase {
      * @returns A Promise that resolves to a string containing the query execution plan explanation.
      */
     async explainPlan(verbose = false) {
-        if (this.inner instanceof Promise) {
-            return this.inner.then((inner) => inner.explainPlan(verbose));
-        }
-        else {
-            return this.inner.explainPlan(verbose);
-        }
+        const inner = await this.getInner();
+        return inner.explainPlan(verbose);
     }
     /**
      * Executes the query and returns the physical query plan annotated with runtime metrics.
@@ -237,12 +249,8 @@ class QueryBase {
      */
     async analyzePlan(distributedMetrics) {
         const distributedMetricsMode = distributedMetrics ?? "aggregate";
-        if (this.inner instanceof Promise) {
-            return this.inner.then((inner) => inner.analyzePlan(distributedMetricsMode));
-        }
-        else {
-            return this.inner.analyzePlan(distributedMetricsMode);
-        }
+        const inner = await this.getInner();
+        return inner.analyzePlan(distributedMetricsMode);
     }
     /**
      * Returns the schema of the output that will be returned by this query.
@@ -253,13 +261,8 @@ class QueryBase {
      * @returns An Arrow Schema describing the output columns.
      */
     async outputSchema() {
-        let schemaBuffer;
-        if (this.inner instanceof Promise) {
-            schemaBuffer = await this.inner.then((inner) => inner.outputSchema());
-        }
-        else {
-            schemaBuffer = await this.inner.outputSchema();
-        }
+        const inner = await this.getInner();
+        const schemaBuffer = await inner.outputSchema();
         const schema = (0, arrow_1.tableFromIPC)(schemaBuffer).schema;
         return schema;
     }
@@ -403,6 +406,12 @@ class VectorQuery extends StandardQueryBase {
     constructor(inner) {
         super(inner);
     }
+    /**
+     * @hidden
+     */
+    doVectorCall(fn) {
+        super.doCall(fn);
+    }
     /**
      * Set the number of partitions to search (probe)
      *
@@ -430,7 +439,7 @@ class VectorQuery extends StandardQueryBase {
      * the minimum and maximum to the same value.
      */
     nprobes(nprobes) {
-        super.doCall((inner) => inner.nprobes(nprobes));
+        this.doVectorCall((inner) => inner.nprobes(nprobes));
         return this;
     }
     /**
@@ -442,7 +451,7 @@ class VectorQuery extends StandardQueryBase {
      * but will also increase latency.
      */
     minimumNprobes(minimumNprobes) {
-        super.doCall((inner) => inner.minimumNprobes(minimumNprobes));
+        this.doVectorCall((inner) => inner.minimumNprobes(minimumNprobes));
         return this;
     }
     /**
@@ -455,7 +464,7 @@ class VectorQuery extends StandardQueryBase {
      * potential false negatives.
      */
     maximumNprobes(maximumNprobes) {
-        super.doCall((inner) => inner.maximumNprobes(maximumNprobes));
+        this.doVectorCall((inner) => inner.maximumNprobes(maximumNprobes));
         return this;
     }
     /*
@@ -467,7 +476,7 @@ class VectorQuery extends StandardQueryBase {
      * `undefined` means no lower or upper bound.
      */
     distanceRange(lowerBound, upperBound) {
-        super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
+        this.doVectorCall((inner) => inner.distanceRange(lowerBound, upperBound));
         return this;
     }
     /**
@@ -480,7 +489,7 @@ class VectorQuery extends StandardQueryBase {
      * also increase the latency of your query. The default value is 1.5*limit.
      */
     ef(ef) {
-        super.doCall((inner) => inner.ef(ef));
+        this.doVectorCall((inner) => inner.ef(ef));
         return this;
     }
     /**
@@ -493,7 +502,7 @@ class VectorQuery extends StandardQueryBase {
      * whose data type is a fixed-size-list of floats.
      */
     column(column) {
-        super.doCall((inner) => inner.column(column));
+        this.doVectorCall((inner) => inner.column(column));
         return this;
     }
     /**
@@ -511,7 +520,7 @@ class VectorQuery extends StandardQueryBase {
      * By default "l2" is used.
      */
     distanceType(distanceType) {
-        super.doCall((inner) => inner.distanceType(distanceType));
+        this.doVectorCall((inner) => inner.distanceType(distanceType));
         return this;
     }
     /**
@@ -544,7 +553,7 @@ class VectorQuery extends StandardQueryBase {
      * distance between the query vector and the actual uncompressed vector.
      */
     refineFactor(refineFactor) {
-        super.doCall((inner) => inner.refineFactor(refineFactor));
+        this.doVectorCall((inner) => inner.refineFactor(refineFactor));
         return this;
     }
     /**
@@ -568,7 +577,7 @@ class VectorQuery extends StandardQueryBase {
      * factor can often help restore some of the results lost by post filtering.
      */
     postfilter() {
-        super.doCall((inner) => inner.postfilter());
+        this.doVectorCall((inner) => inner.postfilter());
         return this;
     }
     /**
@@ -581,50 +590,43 @@ class VectorQuery extends StandardQueryBase {
      * calculate your recall to select an appropriate value for nprobes.
      */
     bypassVectorIndex() {
-        super.doCall((inner) => inner.bypassVectorIndex());
+        this.doVectorCall((inner) => inner.bypassVectorIndex());
         return this;
     }
     /*
      * Add a query vector to the search
      *
      * This method can be called multiple times to add multiple query vectors
-     * to the search. If multiple query vectors are added, then they will be searched
-     * in parallel, and the results will be concatenated. A column called `query_index`
-     * will be added to indicate the index of the query vector that produced the result.
-     *
-     * Performance wise, this is equivalent to running multiple queries concurrently.
+     * to the search. A column called `query_index` will be added to indicate the index
+     * of the query vector that produced the result. Flat searches share one table scan
+     * across the query vectors, avoiding the scan and memory amplification of running
+     * multiple queries concurrently. Indexed searches may still perform per-vector
+     * index work.
      */
     addQueryVector(vector) {
         if (vector instanceof Promise) {
+            // Observe the promise as soon as it is accepted. The existing native
+            // query may still be pending, and delaying observation until it resolves
+            // can otherwise surface a fast rejection as unhandled.
+            const settledVector = vector.then((value) => ({ status: "fulfilled", value }), (reason) => ({ status: "rejected", reason }));
             const res = (async () => {
-                try {
-                    const v = await vector;
-                    // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
-                    const value = this.addQueryVector(v);
-                    const inner = value.inner;
-                    return inner;
-                }
-                catch (e) {
-                    return Promise.reject(e);
+                const inner = await this.getInner();
+                const outcome = await settledVector;
+                if (outcome.status === "rejected") {
+                    throw outcome.reason;
                 }
+                addQueryVectorToNative(inner, outcome.value);
+                return inner;
             })();
             return new VectorQuery(res);
         }
         else {
-            super.doCall((inner) => {
-                const raw = Array.isArray(vector) ? null : (0, arrow_1.extractVectorBuffer)(vector);
-                if (raw) {
-                    inner.addQueryVectorRaw(raw.data, raw.dtype);
-                }
-                else {
-                    inner.addQueryVector(Float32Array.from(vector));
-                }
-            });
+            this.doVectorCall((inner) => addQueryVectorToNative(inner, vector));
             return this;
         }
     }
     rerank(reranker) {
-        super.doCall((inner) => inner.rerank(async (args) => {
+        this.doVectorCall((inner) => inner.rerank(async (args) => {
             const vecResults = await (0, arrow_1.fromBufferToRecordBatch)(args.vecResults);
             const ftsResults = await (0, arrow_1.fromBufferToRecordBatch)(args.ftsResults);
             const result = await reranker.rerankHybrid(args.query, vecResults, ftsResults);
@@ -635,6 +637,51 @@ class VectorQuery extends StandardQueryBase {
     }
 }
 exports.VectorQuery = VectorQuery;
+/**
+ * Create a string query whose vector/FTS routing is resolved against the active
+ * table schema when the query executes.
+ *
+ * @hidden
+ */
+function createAutoQuery(table, query, columns, getVector) {
+    let cachedPreparation;
+    const snapshotRoute = async () => {
+        const snapshot = await table.querySnapshot();
+        const schema = (0, arrow_1.tableFromIPC)(await snapshot.schema()).schema;
+        return {
+            table: snapshot,
+            embeddingMetadata: schema.metadata.get("embedding_functions"),
+        };
+    };
+    const createInner = async () => {
+        const route = await snapshotRoute();
+        if (route.embeddingMetadata === undefined) {
+            const inner = route.table.query();
+            inner.fullTextSearch({ query, columns });
+            return inner;
+        }
+        const metadata = route.embeddingMetadata;
+        if (cachedPreparation?.metadata !== metadata) {
+            cachedPreparation = {
+                metadata,
+                vector: Promise.resolve().then(() => getVector(metadata)),
+            };
+        }
+        const preparation = cachedPreparation;
+        let vector;
+        try {
+            vector = await preparation.vector;
+        }
+        catch (error) {
+            if (cachedPreparation === preparation) {
+                cachedPreparation = undefined;
+            }
+            throw error;
+        }
+        return nearestToNative(route.table.query(), vector);
+    };
+    return new AutoQuery(createInner);
+}
 /**
  * A query that returns a subset of the rows in the table.
  *
@@ -659,6 +706,38 @@ class TakeQuery extends QueryBase {
     }
 }
 exports.TakeQuery = TakeQuery;
+/**
+ * A builder for automatic string searches.
+ *
+ * Automatic search determines whether to use full-text or vector search from
+ * the table revision selected for each execution. This builder exposes the
+ * common operations supported by both query families.
+ *
+ * @hideconstructor
+ */
+class AutoQuery extends StandardQueryBase {
+    createInner;
+    calls = [];
+    /** @hidden */
+    constructor(createInner) {
+        super();
+        this.createInner = createInner;
+    }
+    /** @hidden */
+    doCall(fn) {
+        this.calls.push(fn);
+    }
+    /** @hidden */
+    async getInner() {
+        const calls = [...this.calls];
+        const inner = await this.createInner();
+        for (const call of calls) {
+            call(inner);
+        }
+        return inner;
+    }
+}
+exports.AutoQuery = AutoQuery;
 /** A builder for LanceDB queries.
  *
  * @see {@link Table#query}, {@link Table#search}
@@ -710,41 +789,15 @@ class Query extends StandardQueryBase {
      * a default `limit` of 10 will be used.  @see {@link Query#limit}
      */
     nearestTo(vector) {
-        const callNearestTo = (inner, resolved) => {
-            const raw = Array.isArray(resolved)
-                ? null
-                : (0, arrow_1.extractVectorBuffer)(resolved);
-            if (raw) {
-                return inner.nearestToRaw(raw.data, raw.dtype);
-            }
-            return inner.nearestTo(Float32Array.from(resolved));
-        };
-        if (this.inner instanceof Promise) {
-            const nativeQuery = this.inner.then(async (inner) => {
-                const resolved = vector instanceof Promise ? await vector : vector;
-                return callNearestTo(inner, resolved);
-            });
+        const inner = this.inner;
+        if (inner instanceof Promise) {
+            const nativeQuery = inner.then(async (resolvedInner) => nearestToNative(resolvedInner, await vector));
             return new VectorQuery(nativeQuery);
         }
         if (vector instanceof Promise) {
-            const res = (async () => {
-                try {
-                    const v = await vector;
-                    // biome-ignore lint/suspicious/noExplicitAny: we need to get the `inner`, but js has no package scoping
-                    const value = this.nearestTo(v);
-                    const inner = value.inner;
-                    return inner;
-                }
-                catch (e) {
-                    return Promise.reject(e);
-                }
-            })();
-            return new VectorQuery(res);
-        }
-        else {
-            const vectorQuery = callNearestTo(this.inner, vector);
-            return new VectorQuery(vectorQuery);
+            return new VectorQuery(vector.then((resolvedVector) => nearestToNative(inner, resolvedVector)));
         }
+        return new VectorQuery(nearestToNative(inner, vector));
     }
     nearestToText(query, columns) {
         this.doCall((inner) => {

package/dist/sanitize.js

@@ -45,15 +45,21 @@ function sanitizeMetadata(metadataLike) {
     if (metadataLike === undefined || metadataLike === null) {
         return undefined;
     }
-    if (!(metadataLike instanceof Map)) {
+    let entries;
+    try {
+        entries = Map.prototype.entries.call(metadataLike);
+    }
+    catch {
         throw Error("Expected metadata, if present, to be a Map<string, string>");
     }
-    for (const item of metadataLike) {
-        if (typeof item[0] !== "string" || typeof item[1] !== "string") {
+    const metadata = new Map();
+    for (const [key, value] of entries) {
+        if (typeof key !== "string" || typeof value !== "string") {
             throw Error("Expected metadata, if present, to be a Map<string, string> but it had non-string keys or values");
         }
+        metadata.set(key, value);
     }
-    return metadataLike;
+    return metadata;
 }
 function sanitizeInt(typeLike) {
     if (!("bitWidth" in typeLike) ||

package/dist/schema.d.ts

@@ -0,0 +1,16 @@
+import { Schema } from "apache-arrow";
+type InferenceOptions = {
+    dictionaryEncodeStrings: boolean;
+    vectorColumns: Record<string, {
+        type: unknown;
+    }>;
+};
+/**
+ * Infer the Arrow schema represented by a set of records.
+ *
+ * This is the intentionally small interface to schema inference. The stateful
+ * details of combining partial type evidence are encapsulated below so callers
+ * only need to provide records, an optional schema, and inference options.
+ */
+export declare function inferSchema(data: Array<Record<string, unknown>>, schema: Schema | undefined, options: InferenceOptions): Schema;
+export {};