@lancedb/lancedb 0.14.1-beta.5 → 0.14.2-beta.0

package/dist/arrow.d.ts

@@ -233,6 +233,16 @@ export declare function fromTableToBuffer(table: ArrowTable, embeddings?: Embedd
  * `schema` is required if the table is empty
  */
 export declare function fromDataToBuffer(data: Data, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
+/**
+ * Read a single record batch from a buffer.
+ *
+ * Returns null if the buffer does not contain a record batch
+ */
+export declare function fromBufferToRecordBatch(data: Buffer): Promise<RecordBatch | null>;
+/**
+ * Create a buffer containing a single record batch
+ */
+export declare function fromRecordBatchToBuffer(batch: RecordBatch): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

package/dist/arrow.js

@@ -56,6 +56,8 @@ exports.fromRecordsToBuffer = fromRecordsToBuffer;
 exports.fromRecordsToStreamBuffer = fromRecordsToStreamBuffer;
 exports.fromTableToBuffer = fromTableToBuffer;
 exports.fromDataToBuffer = fromDataToBuffer;
+exports.fromBufferToRecordBatch = fromBufferToRecordBatch;
+exports.fromRecordBatchToBuffer = fromRecordBatchToBuffer;
 exports.fromTableToStreamBuffer = fromTableToStreamBuffer;
 exports.createEmptyTable = createEmptyTable;
 const apache_arrow_1 = require("apache-arrow");
@@ -655,6 +657,24 @@ async function fromDataToBuffer(data, embeddings, schema) {
         return fromTableToBuffer(table);
     }
 }
+/**
+ * Read a single record batch from a buffer.
+ *
+ * Returns null if the buffer does not contain a record batch
+ */
+async function fromBufferToRecordBatch(data) {
+    const iter = await apache_arrow_1.RecordBatchFileReader.readAll(Buffer.from(data)).next()
+        .value;
+    const recordBatch = iter?.next().value;
+    return recordBatch || null;
+}
+/**
+ * Create a buffer containing a single record batch
+ */
+async function fromRecordBatchToBuffer(batch) {
+    const writer = new apache_arrow_1.RecordBatchFileWriter().writeAll([batch]);
+    return Buffer.from(await writer.toUint8Array());
+}
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

package/dist/index.d.ts

@@ -7,6 +7,7 @@ export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, }
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
 export { Table, AddDataOptions, UpdateOptions, OptimizeOptions } from "./table";
 export * as embedding from "./embedding";
+export * as rerankers from "./rerankers";
 /**
  * Connect to a LanceDB instance at the given URI.
  *

package/dist/index.js

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.WriteMode = void 0;
+exports.rerankers = exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.WriteMode = void 0;
 exports.connect = connect;
 const connection_1 = require("./connection");
 const native_js_1 = require("./native.js");
@@ -35,6 +35,7 @@ Object.defineProperty(exports, "Index", { enumerable: true, get: function () { r
 var table_1 = require("./table");
 Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
 exports.embedding = require("./embedding");
+exports.rerankers = require("./rerankers");
 async function connect(uriOrOptions, opts = {}) {
     let uri;
     if (typeof uriOrOptions !== "string") {

package/dist/native.d.ts

@@ -78,6 +78,14 @@ export interface ClientConfig {
   retryConfig?: RetryConfig
   timeoutConfig?: TimeoutConfig
 }
+export interface RerankerCallbacks {
+  rerankHybrid: (...args: any[]) => any
+}
+export interface RerankHybridCallbackArgs {
+  query: string
+  vecResults: Array<number>
+  ftsResults: Array<number>
+}
 /** A description of an index currently configured on a column */
 export interface IndexConfig {
   /** The name of the index */
@@ -306,9 +314,25 @@ export class VectorQuery {
   offset(offset: number): void
   fastSearch(): void
   withRowId(): void
+  rerank(callbacks: RerankerCallbacks): void
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
   explainPlan(verbose: boolean): Promise<string>
 }
+/**
+ * Reranker implementation that "wraps" a NodeJS Reranker implementation.
+ * This contains references to the callbacks that can be used to invoke the
+ * reranking methods on the NodeJS implementation and handles serializing the
+ * record batches to Arrow IPC buffers.
+ */
+export class Reranker {
+  static new(callbacks: RerankerCallbacks): Reranker
+}
+export type RRFReranker = RrfReranker
+/** Wrapper around rust RRFReranker */
+export class RrfReranker {
+  static tryNew(k: Float32Array): Promise<RrfReranker>
+  rerankHybrid(query: string, vecResults: Buffer, ftsResults: Buffer): Promise<Buffer>
+}
 export class Table {
   name: string
   display(): string

package/dist/native.js

@@ -319,12 +319,14 @@ if (!nativeBinding) {
     }
     throw new Error(`Failed to load native binding`);
 }
-const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, Table, WriteMode } = nativeBinding;
+const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, Reranker, RrfReranker, Table, WriteMode } = nativeBinding;
 module.exports.Connection = Connection;
 module.exports.Index = Index;
 module.exports.RecordBatchIterator = RecordBatchIterator;
 module.exports.NativeMergeInsertBuilder = NativeMergeInsertBuilder;
 module.exports.Query = Query;
 module.exports.VectorQuery = VectorQuery;
+module.exports.Reranker = Reranker;
+module.exports.RrfReranker = RrfReranker;
 module.exports.Table = Table;
 module.exports.WriteMode = WriteMode;

package/dist/query.d.ts

@@ -1,6 +1,7 @@
 import { Table as ArrowTable, type IntoVector, RecordBatch } from "./arrow";
 import { type IvfPqOptions } from "./indices";
 import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
+import { Reranker } from "./rerankers";
 export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     private promisedInner?;
     private inner?;
@@ -278,6 +279,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery> {
      */
     bypassVectorIndex(): VectorQuery;
     addQueryVector(vector: IntoVector): VectorQuery;
+    rerank(reranker: Reranker): VectorQuery;
 }
 /** A builder for LanceDB queries. */
 export declare class Query extends QueryBase<NativeQuery> {

package/dist/query.js

@@ -457,6 +457,18 @@ class VectorQuery extends QueryBase {
             return this;
         }
     }
+    rerank(reranker) {
+        super.doCall((inner) => inner.rerank({
+            rerankHybrid: async (_, args) => {
+                const vecResults = await (0, arrow_1.fromBufferToRecordBatch)(args.vecResults);
+                const ftsResults = await (0, arrow_1.fromBufferToRecordBatch)(args.ftsResults);
+                const result = await reranker.rerankHybrid(args.query, vecResults, ftsResults);
+                const buffer = (0, arrow_1.fromRecordBatchToBuffer)(result);
+                return buffer;
+            },
+        }));
+        return this;
+    }
 }
 exports.VectorQuery = VectorQuery;
 /** A builder for LanceDB queries. */

package/dist/rerankers/index.d.ts

@@ -0,0 +1,5 @@
+import { RecordBatch } from "apache-arrow";
+export * from "./rrf";
+export interface Reranker {
+    rerankHybrid(query: string, vecResults: RecordBatch, ftsResults: RecordBatch): Promise<RecordBatch>;
+}

package/dist/rerankers/index.js

@@ -0,0 +1,19 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./rrf"), exports);

package/dist/rerankers/rrf.d.ts

@@ -0,0 +1,13 @@
+import { RecordBatch } from "apache-arrow";
+import { RrfReranker as NativeRRFReranker } from "../native";
+/**
+ * Reranks the results using the Reciprocal Rank Fusion (RRF) algorithm.
+ *
+ * Internally this uses the Rust implementation
+ */
+export declare class RRFReranker {
+    private inner;
+    constructor(inner: NativeRRFReranker);
+    static create(k?: number): Promise<RRFReranker>;
+    rerankHybrid(query: string, vecResults: RecordBatch, ftsResults: RecordBatch): Promise<RecordBatch>;
+}

package/dist/rerankers/rrf.js

@@ -0,0 +1,27 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RRFReranker = void 0;
+const arrow_1 = require("../arrow");
+const native_1 = require("../native");
+/**
+ * Reranks the results using the Reciprocal Rank Fusion (RRF) algorithm.
+ *
+ * Internally this uses the Rust implementation
+ */
+class RRFReranker {
+    inner;
+    constructor(inner) {
+        this.inner = inner;
+    }
+    static async create(k = 60) {
+        return new RRFReranker(await native_1.RrfReranker.tryNew(new Float32Array([k])));
+    }
+    async rerankHybrid(query, vecResults, ftsResults) {
+        const buffer = await this.inner.rerankHybrid(query, await (0, arrow_1.fromRecordBatchToBuffer)(vecResults), await (0, arrow_1.fromRecordBatchToBuffer)(ftsResults));
+        const recordBatch = await (0, arrow_1.fromBufferToRecordBatch)(buffer);
+        return recordBatch;
+    }
+}
+exports.RRFReranker = RRFReranker;

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.14.1-beta.5",
+  "version": "0.14.2-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -98,14 +98,14 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.14.1-beta.5",
-    "@lancedb/lancedb-darwin-arm64": "0.14.1-beta.5",
-    "@lancedb/lancedb-linux-x64-gnu": "0.14.1-beta.5",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.14.1-beta.5",
-    "@lancedb/lancedb-linux-x64-musl": "0.14.1-beta.5",
-    "@lancedb/lancedb-linux-arm64-musl": "0.14.1-beta.5",
-    "@lancedb/lancedb-win32-x64-msvc": "0.14.1-beta.5",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.14.1-beta.5"
+    "@lancedb/lancedb-darwin-x64": "0.14.2-beta.0",
+    "@lancedb/lancedb-darwin-arm64": "0.14.2-beta.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.14.2-beta.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.14.2-beta.0",
+    "@lancedb/lancedb-linux-x64-musl": "0.14.2-beta.0",
+    "@lancedb/lancedb-linux-arm64-musl": "0.14.2-beta.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.14.2-beta.0",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.14.2-beta.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"