@lancedb/lancedb 0.14.1 → 0.15.0-beta.0

package/CONTRIBUTING.md

@@ -0,0 +1,76 @@
+# Contributing to LanceDB Typescript
+
+This document outlines the process for contributing to LanceDB Typescript.
+For general contribution guidelines, see [CONTRIBUTING.md](../CONTRIBUTING.md).
+
+## Project layout
+
+The Typescript package is a wrapper around the Rust library, `lancedb`. We use
+the [napi-rs](https://napi.rs/) library to create the bindings between Rust and
+Typescript.
+
+* `src/`: Rust bindings source code
+* `lancedb/`: Typescript package source code
+* `__test__/`: Unit tests
+* `examples/`: An npm package with the examples shown in the documentation
+
+## Development environment
+
+To set up your development environment, you will need to install the following:
+
+1. Node.js 14 or later
+2. Rust's package manager, Cargo. Use [rustup](https://rustup.rs/) to install.
+3. [protoc](https://grpc.io/docs/protoc-installation/) (Protocol Buffers compiler)
+
+Initial setup:
+
+```shell
+npm install
+```
+
+### Commit Hooks
+
+It is **highly recommended** to install the [pre-commit](https://pre-commit.com/) hooks to ensure that your
+code is formatted correctly and passes basic checks before committing:
+
+```shell
+pre-commit install
+```
+
+## Development
+
+Most common development commands can be run using the npm scripts.
+
+Build the package
+
+```shell
+npm install
+npm run build
+```
+
+Lint:
+
+```shell
+npm run lint
+```
+
+Format and fix lints:
+
+```shell
+npm run lint-fix
+```
+
+Run tests:
+
+```shell
+npm test
+```
+
+To run a single test:
+
+```shell
+# Single file: table.test.ts
+npm test -- table.test.ts
+# Single test: 'merge insert' in table.test.ts
+npm test -- table.test.ts --testNamePattern=merge\ insert
+```

package/README.md

@@ -36,37 +36,4 @@ The [quickstart](../basic.md) contains a more complete example.
 
 ## Development
 
-```sh
-npm run build
-npm run test
-```
-
-### Running lint / format
-
-LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
-To manually lint your code you can run:
-
-```sh
-npm run lint
-```
-
-to automatically fix all fixable issues:
-
-```sh
-npm run lint-fix
-```
-
-If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
-
-### Generating docs
-
-```sh
-npm run docs
-
-cd ../docs
-# Asssume the virtual environment was created
-# python3 -m venv venv
-# pip install -r requirements.txt
-. ./venv/bin/activate
-mkdocs build
-```
+See [CONTRIBUTING.md](./CONTRIBUTING.md) for information on how to contribute to LanceDB.

package/dist/arrow.d.ts

@@ -233,6 +233,16 @@ export declare function fromTableToBuffer(table: ArrowTable, embeddings?: Embedd
  * `schema` is required if the table is empty
  */
 export declare function fromDataToBuffer(data: Data, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
+/**
+ * Read a single record batch from a buffer.
+ *
+ * Returns null if the buffer does not contain a record batch
+ */
+export declare function fromBufferToRecordBatch(data: Buffer): Promise<RecordBatch | null>;
+/**
+ * Create a buffer containing a single record batch
+ */
+export declare function fromRecordBatchToBuffer(batch: RecordBatch): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

package/dist/arrow.js

@@ -56,6 +56,8 @@ exports.fromRecordsToBuffer = fromRecordsToBuffer;
 exports.fromRecordsToStreamBuffer = fromRecordsToStreamBuffer;
 exports.fromTableToBuffer = fromTableToBuffer;
 exports.fromDataToBuffer = fromDataToBuffer;
+exports.fromBufferToRecordBatch = fromBufferToRecordBatch;
+exports.fromRecordBatchToBuffer = fromRecordBatchToBuffer;
 exports.fromTableToStreamBuffer = fromTableToStreamBuffer;
 exports.createEmptyTable = createEmptyTable;
 const apache_arrow_1 = require("apache-arrow");
@@ -655,6 +657,24 @@ async function fromDataToBuffer(data, embeddings, schema) {
         return fromTableToBuffer(table);
     }
 }
+/**
+ * Read a single record batch from a buffer.
+ *
+ * Returns null if the buffer does not contain a record batch
+ */
+async function fromBufferToRecordBatch(data) {
+    const iter = await apache_arrow_1.RecordBatchFileReader.readAll(Buffer.from(data)).next()
+        .value;
+    const recordBatch = iter?.next().value;
+    return recordBatch || null;
+}
+/**
+ * Create a buffer containing a single record batch
+ */
+async function fromRecordBatchToBuffer(batch) {
+    const writer = new apache_arrow_1.RecordBatchFileWriter().writeAll([batch]);
+    return Buffer.from(await writer.toUint8Array());
+}
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *

package/dist/index.d.ts

@@ -7,6 +7,7 @@ export { ExecutableQuery, Query, QueryBase, VectorQuery, RecordBatchIterator, }
 export { Index, IndexOptions, IvfPqOptions } from "./indices";
 export { Table, AddDataOptions, UpdateOptions, OptimizeOptions } from "./table";
 export * as embedding from "./embedding";
+export * as rerankers from "./rerankers";
 /**
  * Connect to a LanceDB instance at the given URI.
  *

package/dist/index.js

@@ -13,7 +13,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.WriteMode = void 0;
+exports.rerankers = exports.embedding = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.WriteMode = void 0;
 exports.connect = connect;
 const connection_1 = require("./connection");
 const native_js_1 = require("./native.js");
@@ -35,6 +35,7 @@ Object.defineProperty(exports, "Index", { enumerable: true, get: function () { r
 var table_1 = require("./table");
 Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
 exports.embedding = require("./embedding");
+exports.rerankers = require("./rerankers");
 async function connect(uriOrOptions, opts = {}) {
     let uri;
     if (typeof uriOrOptions !== "string") {

package/dist/native.d.ts

@@ -78,6 +78,14 @@ export interface ClientConfig {
   retryConfig?: RetryConfig
   timeoutConfig?: TimeoutConfig
 }
+export interface RerankerCallbacks {
+  rerankHybrid: (...args: any[]) => any
+}
+export interface RerankHybridCallbackArgs {
+  query: string
+  vecResults: Array<number>
+  ftsResults: Array<number>
+}
 /** A description of an index currently configured on a column */
 export interface IndexConfig {
   /** The name of the index */
@@ -296,6 +304,7 @@ export class VectorQuery {
   postfilter(): void
   refineFactor(refineFactor: number): void
   nprobes(nprobe: number): void
+  distanceRange(lowerBound?: number | undefined | null, upperBound?: number | undefined | null): void
   ef(ef: number): void
   bypassVectorIndex(): void
   onlyIf(predicate: string): void
@@ -306,9 +315,25 @@ export class VectorQuery {
   offset(offset: number): void
   fastSearch(): void
   withRowId(): void
+  rerank(callbacks: RerankerCallbacks): void
   execute(maxBatchLength?: number | undefined | null): Promise<RecordBatchIterator>
   explainPlan(verbose: boolean): Promise<string>
 }
+/**
+ * Reranker implementation that "wraps" a NodeJS Reranker implementation.
+ * This contains references to the callbacks that can be used to invoke the
+ * reranking methods on the NodeJS implementation and handles serializing the
+ * record batches to Arrow IPC buffers.
+ */
+export class Reranker {
+  static new(callbacks: RerankerCallbacks): Reranker
+}
+export type RRFReranker = RrfReranker
+/** Wrapper around rust RRFReranker */
+export class RrfReranker {
+  static tryNew(k: Float32Array): Promise<RrfReranker>
+  rerankHybrid(query: string, vecResults: Buffer, ftsResults: Buffer): Promise<Buffer>
+}
 export class Table {
   name: string
   display(): string

package/dist/native.js

@@ -319,12 +319,14 @@ if (!nativeBinding) {
     }
     throw new Error(`Failed to load native binding`);
 }
-const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, Table, WriteMode } = nativeBinding;
+const { Connection, Index, RecordBatchIterator, NativeMergeInsertBuilder, Query, VectorQuery, Reranker, RrfReranker, Table, WriteMode } = nativeBinding;
 module.exports.Connection = Connection;
 module.exports.Index = Index;
 module.exports.RecordBatchIterator = RecordBatchIterator;
 module.exports.NativeMergeInsertBuilder = NativeMergeInsertBuilder;
 module.exports.Query = Query;
 module.exports.VectorQuery = VectorQuery;
+module.exports.Reranker = Reranker;
+module.exports.RrfReranker = RrfReranker;
 module.exports.Table = Table;
 module.exports.WriteMode = WriteMode;

package/dist/query.d.ts

@@ -1,6 +1,7 @@
 import { Table as ArrowTable, type IntoVector, RecordBatch } from "./arrow";
 import { type IvfPqOptions } from "./indices";
 import { RecordBatchIterator as NativeBatchIterator, Query as NativeQuery, Table as NativeTable, VectorQuery as NativeVectorQuery } from "./native";
+import { Reranker } from "./rerankers";
 export declare class RecordBatchIterator implements AsyncIterator<RecordBatch> {
     private promisedInner?;
     private inner?;
@@ -181,6 +182,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery> {
      * you the desired recall.
      */
     nprobes(nprobes: number): VectorQuery;
+    distanceRange(lowerBound?: number, upperBound?: number): VectorQuery;
     /**
      * Set the number of candidates to consider during the search
      *
@@ -278,6 +280,7 @@ export declare class VectorQuery extends QueryBase<NativeVectorQuery> {
      */
     bypassVectorIndex(): VectorQuery;
     addQueryVector(vector: IntoVector): VectorQuery;
+    rerank(reranker: Reranker): VectorQuery;
 }
 /** A builder for LanceDB queries. */
 export declare class Query extends QueryBase<NativeQuery> {

package/dist/query.js

@@ -308,6 +308,18 @@ class VectorQuery extends QueryBase {
         super.doCall((inner) => inner.nprobes(nprobes));
         return this;
     }
+    /*
+     * Set the distance range to use
+     *
+     * Only rows with distances within range [lower_bound, upper_bound)
+     * will be returned.
+     *
+     * `undefined` means no lower or upper bound.
+     */
+    distanceRange(lowerBound, upperBound) {
+        super.doCall((inner) => inner.distanceRange(lowerBound, upperBound));
+        return this;
+    }
     /**
      * Set the number of candidates to consider during the search
      *
@@ -457,6 +469,18 @@ class VectorQuery extends QueryBase {
             return this;
         }
     }
+    rerank(reranker) {
+        super.doCall((inner) => inner.rerank({
+            rerankHybrid: async (_, args) => {
+                const vecResults = await (0, arrow_1.fromBufferToRecordBatch)(args.vecResults);
+                const ftsResults = await (0, arrow_1.fromBufferToRecordBatch)(args.ftsResults);
+                const result = await reranker.rerankHybrid(args.query, vecResults, ftsResults);
+                const buffer = (0, arrow_1.fromRecordBatchToBuffer)(result);
+                return buffer;
+            },
+        }));
+        return this;
+    }
 }
 exports.VectorQuery = VectorQuery;
 /** A builder for LanceDB queries. */

package/dist/rerankers/index.d.ts

@@ -0,0 +1,5 @@
+import { RecordBatch } from "apache-arrow";
+export * from "./rrf";
+export interface Reranker {
+    rerankHybrid(query: string, vecResults: RecordBatch, ftsResults: RecordBatch): Promise<RecordBatch>;
+}

package/dist/rerankers/index.js

@@ -0,0 +1,19 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./rrf"), exports);

package/dist/rerankers/rrf.d.ts

@@ -0,0 +1,13 @@
+import { RecordBatch } from "apache-arrow";
+import { RrfReranker as NativeRRFReranker } from "../native";
+/**
+ * Reranks the results using the Reciprocal Rank Fusion (RRF) algorithm.
+ *
+ * Internally this uses the Rust implementation
+ */
+export declare class RRFReranker {
+    private inner;
+    constructor(inner: NativeRRFReranker);
+    static create(k?: number): Promise<RRFReranker>;
+    rerankHybrid(query: string, vecResults: RecordBatch, ftsResults: RecordBatch): Promise<RecordBatch>;
+}

package/dist/rerankers/rrf.js

@@ -0,0 +1,27 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.RRFReranker = void 0;
+const arrow_1 = require("../arrow");
+const native_1 = require("../native");
+/**
+ * Reranks the results using the Reciprocal Rank Fusion (RRF) algorithm.
+ *
+ * Internally this uses the Rust implementation
+ */
+class RRFReranker {
+    inner;
+    constructor(inner) {
+        this.inner = inner;
+    }
+    static async create(k = 60) {
+        return new RRFReranker(await native_1.RrfReranker.tryNew(new Float32Array([k])));
+    }
+    async rerankHybrid(query, vecResults, ftsResults) {
+        const buffer = await this.inner.rerankHybrid(query, await (0, arrow_1.fromRecordBatchToBuffer)(vecResults), await (0, arrow_1.fromRecordBatchToBuffer)(ftsResults));
+        const recordBatch = await (0, arrow_1.fromBufferToRecordBatch)(buffer);
+        return recordBatch;
+    }
+}
+exports.RRFReranker = RRFReranker;

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.14.1",
+  "version": "0.15.0-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -98,14 +98,14 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.14.1",
-    "@lancedb/lancedb-darwin-arm64": "0.14.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.14.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.14.1",
-    "@lancedb/lancedb-linux-x64-musl": "0.14.1",
-    "@lancedb/lancedb-linux-arm64-musl": "0.14.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.14.1",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.14.1"
+    "@lancedb/lancedb-darwin-x64": "0.15.0-beta.0",
+    "@lancedb/lancedb-darwin-arm64": "0.15.0-beta.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.15.0-beta.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.15.0-beta.0",
+    "@lancedb/lancedb-linux-x64-musl": "0.15.0-beta.0",
+    "@lancedb/lancedb-linux-arm64-musl": "0.15.0-beta.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.15.0-beta.0",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.15.0-beta.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"