@lancedb/lancedb 0.28.0-beta.9 → 0.29.1-beta.0

package/dist/scannable.js

@@ -0,0 +1,200 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Scannable = void 0;
+const arrow_1 = require("./arrow");
+const native_js_1 = require("./native.js");
+/**
+ * A data source that can be scanned as a stream of Arrow `RecordBatch`es.
+ *
+ * `Scannable` wraps the schema + optional row count + rescannable flag and
+ * a callback that yields batches one at a time. It is passed to consumers
+ * (e.g. `Table.add`, `createTable`, `mergeInsert` — follow-up work) that
+ * need to pull data without materializing the full dataset in JS memory.
+ *
+ * Batches cross the JS↔Rust boundary as Arrow IPC Stream messages; a fresh
+ * writer serializes each batch, and the Rust side decodes it with
+ * `arrow_ipc::reader::StreamReader`. One batch is in flight at a time.
+ */
+class Scannable {
+    schema;
+    numRows;
+    rescannable;
+    /** @hidden */
+    native;
+    constructor(native, schema, numRows, rescannable) {
+        this.native = native;
+        this.schema = schema;
+        this.numRows = numRows;
+        this.rescannable = rescannable;
+    }
+    /** @hidden Access the native handle for passing through to Rust consumers. */
+    get inner() {
+        return this.native;
+    }
+    /**
+     * Build a Scannable from an explicit schema and a factory that returns a
+     * fresh batch iterator on each call.
+     *
+     * The factory is invoked once per scan. Each iterator yields
+     * `RecordBatch`es matching the declared schema. Use this when you need
+     * direct control over the pull loop — for example, to wrap a streaming
+     * source whose batches are produced lazily.
+     *
+     * @param schema - The Arrow schema of the produced batches.
+     * @param factory - Called at the start of each scan to produce a batch
+     *   iterator. Must be idempotent when `rescannable` is true.
+     * @param opts - Optional hints. `rescannable` defaults to `true`; set to
+     *   `false` if calling `factory()` twice would not reproduce the same data.
+     */
+    static async fromFactory(schema, factory, opts = {}) {
+        const numRows = opts.numRows ?? null;
+        if (numRows != null && !Number.isInteger(numRows)) {
+            throw new TypeError("numRows must be an integer");
+        }
+        const rescannable = opts.rescannable ?? true;
+        let iter = null;
+        const getNextBatch = async (isStart) => {
+            // `isStart` is true on the first pull of every new scan_as_stream.
+            // Drop any cached iterator so factory() is re-invoked for the next scan
+            if (isStart) {
+                iter = null;
+            }
+            if (iter === null) {
+                iter = normalizeIterator(factory());
+            }
+            const result = await iter.next();
+            if (result.done) {
+                iter = null;
+                return null;
+            }
+            return (0, arrow_1.fromRecordBatchToStreamBuffer)(result.value);
+        };
+        const schemaBuf = await (0, arrow_1.fromTableToBuffer)((0, arrow_1.makeEmptyTable)(schema));
+        const native = new native_js_1.NapiScannable(schemaBuf, numRows, rescannable, getNextBatch);
+        return new Scannable(native, schema, numRows, rescannable);
+    }
+    /**
+     * Build a Scannable from an in-memory Arrow `Table`. Always rescannable;
+     * the table's batches are replayed on each scan.
+     *
+     * The table's row count is authoritative: `opts.numRows` must either be
+     * omitted or equal to `table.numRows`. `opts.rescannable` of `false` is
+     * rejected because in-memory Tables are always rescannable.
+     */
+    static async fromTable(table, opts = {}) {
+        if (opts.numRows != null && opts.numRows !== table.numRows) {
+            throw new TypeError(`opts.numRows (${opts.numRows}) does not match table.numRows (${table.numRows}). ` +
+                `The table's row count is authoritative; omit numRows or pass the matching value.`);
+        }
+        if (opts.rescannable === false) {
+            throw new TypeError(`fromTable does not accept rescannable: false. ` +
+                `In-memory Arrow Tables are always rescannable; omit the option or pass true.`);
+        }
+        return Scannable.fromFactory(table.schema, () => table.batches, {
+            numRows: table.numRows,
+            rescannable: true,
+        });
+    }
+    /**
+     * Build a Scannable from an iterable of `RecordBatch`es. `rescannable`
+     * defaults to `false`. Pass an explicit schema so the consumer can
+     * validate before any batch is pulled.
+     *
+     * `opts.rescannable: true` is honest for replayable iterables (Arrays,
+     * Sets, or custom iterables whose `[Symbol.iterator]()` returns a fresh
+     * iterator each call). It is rejected for one-shot iterables (generators,
+     * async generators, or already-an-iterator inputs) because their
+     * `[Symbol.iterator]()` returns the same exhausted object on the second
+     * scan. For replayable sources outside this shape, use
+     * `fromFactory(schema, () => createIter(), { rescannable: true })`.
+     *
+     * Note: when `opts.rescannable` is `true`, the constructor calls
+     * `[Symbol.iterator]()` once on the input to perform the structural check.
+     */
+    static async fromIterable(schema, iter, opts = {}) {
+        if (opts.rescannable === true && isOneShotIterable(iter)) {
+            throw new TypeError(`fromIterable: rescannable: true is not honest for one-shot iterables ` +
+                `(generators, async generators, or iterators where [Symbol.iterator]() ` +
+                `returns the same object). The source would be exhausted after the first scan. ` +
+                `Use fromFactory(schema, () => createIter(), { rescannable: true }) for sources ` +
+                `where each call mints a fresh iterator.`);
+        }
+        return Scannable.fromFactory(schema, () => iter, {
+            numRows: opts.numRows,
+            rescannable: opts.rescannable ?? false,
+        });
+    }
+    /**
+     * Build a Scannable from an Arrow `RecordBatchReader`. A reader can only
+     * be consumed once; `rescannable` defaults to `false`.
+     *
+     * The reader must already be opened (via `.open()`) so its `.schema` is
+     * populated. `RecordBatchReader.from(...)` returns an unopened reader.
+     *
+     * `opts.rescannable: true` is rejected because `RecordBatchReader` is a
+     * self-iterator (its `[Symbol.iterator]()` returns itself), and this
+     * constructor does not call `reader.reset()` between scans, so a second
+     * scan would always see an exhausted reader. For genuinely replayable
+     * sources, use
+     * `fromFactory(schema, () => openReader(), { rescannable: true })`,
+     * which mints a fresh reader on each scan.
+     */
+    static async fromRecordBatchReader(reader, opts = {}) {
+        if (opts.rescannable === true) {
+            throw new TypeError(`fromRecordBatchReader does not accept rescannable: true. ` +
+                `RecordBatchReader is a self-iterator (its [Symbol.iterator]() ` +
+                `returns itself) and would be exhausted after the first scan. ` +
+                `Use fromFactory(schema, () => openReader(), { rescannable: true }) ` +
+                `for sources where each call mints a fresh reader.`);
+        }
+        return Scannable.fromFactory(reader.schema, () => reader, {
+            numRows: opts.numRows,
+            rescannable: false,
+        });
+    }
+}
+exports.Scannable = Scannable;
+function normalizeIterator(source) {
+    if (source == null) {
+        throw new TypeError("Scannable factory returned null/undefined");
+    }
+    if (typeof source[Symbol.asyncIterator] === "function") {
+        return source[Symbol.asyncIterator]();
+    }
+    if (typeof source[Symbol.iterator] === "function") {
+        return source[Symbol.iterator]();
+    }
+    // Already an iterator (has `.next`).
+    if (typeof source.next === "function") {
+        return source;
+    }
+    throw new TypeError("Scannable factory returned a non-iterable value");
+}
+// A "self-iterator" returns the same object from `[Symbol.iterator]()` /
+// `[Symbol.asyncIterator]()`. Generators behave this way, so they exhaust
+// after one pass. Replayable iterables (Array, Set, custom) return a fresh
+// iterator each call. Detection mirrors `normalizeIterator`'s ordering so
+// classification matches scan-time behavior.
+function isOneShotIterable(source) {
+    // null/undefined are not one-shot in any meaningful sense; let
+    // `normalizeIterator` raise the actual error at scan time.
+    if (source == null)
+        return false;
+    const ref = source;
+    if (typeof source[Symbol.asyncIterator] ===
+        "function") {
+        const it = source[Symbol.asyncIterator]();
+        return it === ref;
+    }
+    if (typeof source[Symbol.iterator] === "function") {
+        const it = source[Symbol.iterator]();
+        return it === ref;
+    }
+    // Already-an-iterator (has `.next` but no `Symbol.iterator`) is by
+    // definition one-shot.
+    if (typeof source.next === "function")
+        return true;
+    return false;
+}

package/dist/table.d.ts

@@ -61,6 +61,26 @@ export interface Version {
     timestamp: Date;
     metadata: Record<string, string>;
 }
+/**
+ * Specification selecting Lance's MemWAL LSM-style write path for
+ * `mergeInsert`.
+ *
+ * `specType` is `"bucket"`, `"identity"`, or `"unsharded"`. For `"bucket"`,
+ * `column` and `numBuckets` are required; for `"identity"`, `column` is
+ * required.
+ */
+export interface LsmWriteSpec {
+    /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
+    specType: "bucket" | "identity" | "unsharded";
+    /** Bucket and identity variants: the sharding column. */
+    column?: string;
+    /** Bucket variant: the number of buckets, in `[1, 1024]`. */
+    numBuckets?: number;
+    /** Names of indexes the MemWAL should keep up to date during writes. */
+    maintainedIndexes?: string[];
+    /** Default `ShardWriter` configuration recorded in the MemWAL index. */
+    writerConfigDefaults?: Record<string, string>;
+}
 /**
  * A Table is a collection of Records in a LanceDB Database.
  *
@@ -219,6 +239,24 @@ export declare abstract class Table {
      * wasteful.
      */
     abstract prewarmIndex(name: string): Promise<void>;
+    /**
+     * Prewarm one or more columns of data in the table.
+     *
+     * @param columns The columns to prewarm. If undefined, all columns are prewarmed.
+     *
+     * This will load the column data into the page cache so that future queries that
+     * read those columns avoid the initial cold-start latency.  This call initiates
+     * prewarming and returns once the request is accepted; the warming itself may
+     * continue in the background.  Calling it on already-prewarmed columns is a
+     * no-op on the server.
+     *
+     * Prewarming is generally useful for columns used in filters or projections.
+     * Large columns (e.g. high-dimensional vectors or binary data) may not be
+     * practical to prewarm.
+     *
+     * This feature is currently only supported on remote tables.
+     */
+    abstract prewarmData(columns?: string[]): Promise<void>;
     /**
      * Waits for asynchronous indexing to complete on the table.
      *
@@ -348,6 +386,54 @@ export declare abstract class Table {
      * containing the new version number of the table after dropping the columns.
      */
     abstract dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
+    /**
+     * Set the unenforced primary key for this table to a single column.
+     *
+     * "Unenforced" means LanceDB does not check uniqueness on writes; the
+     * column is recorded in the schema as the primary key for use by features
+     * such as `merge_insert`. Only single-column primary keys are supported,
+     * and the key cannot be changed once set.
+     * @param {string | string[]} columns The primary key column. A one-element
+     * array is also accepted; passing more than one column is rejected.
+     * @returns {Promise<void>}
+     */
+    abstract setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
+    /**
+     * Install an {@link LsmWriteSpec} on this table, selecting Lance's MemWAL
+     * LSM-style write path for future `mergeInsert` calls.
+     *
+     * `LsmWriteSpec` chooses one of three sharding strategies via `specType`:
+     *
+     * - `"bucket"` — hash-bucket writes by the single-column unenforced primary
+     *   key (`column` and `numBuckets` required).
+     * - `"identity"` — shard by the raw value of a scalar `column`.
+     * - `"unsharded"` — route every write to a single shard.
+     *
+     * All variants require the table to have an unenforced primary key
+     * ({@link Table#setUnenforcedPrimaryKey}); bucket sharding additionally
+     * requires it to be the single column being bucketed.
+     * @param {LsmWriteSpec} spec The sharding spec to install.
+     * @returns {Promise<void>}
+     * @example
+     * ```ts
+     * await table.setUnenforcedPrimaryKey("id");
+     * await table.setLsmWriteSpec({
+     *   specType: "bucket",
+     *   column: "id",
+     *   numBuckets: 16,
+     *   maintainedIndexes: ["id_idx"],
+     * });
+     * ```
+     */
+    abstract setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>;
+    /**
+     * Remove the {@link LsmWriteSpec} from this table, reverting to the standard
+     * `mergeInsert` write path.
+     *
+     * Errors if no spec is currently set.
+     * @returns {Promise<void>}
+     */
+    abstract unsetLsmWriteSpec(): Promise<void>;
     /** Retrieve the version of the table */
     abstract version(): Promise<number>;
     /**
@@ -499,6 +585,7 @@ export declare class LocalTable extends Table {
     createIndex(column: string, options?: Partial<IndexOptions>): Promise<void>;
     dropIndex(name: string): Promise<void>;
     prewarmIndex(name: string): Promise<void>;
+    prewarmData(columns?: string[]): Promise<void>;
     waitForIndex(indexNames: string[], timeoutSeconds: number): Promise<void>;
     takeOffsets(offsets: number[]): TakeQuery;
     takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
@@ -508,6 +595,9 @@ export declare class LocalTable extends Table {
     addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema): Promise<AddColumnsResult>;
     alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
     dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
+    setUnenforcedPrimaryKey(columns: string | string[]): Promise<void>;
+    setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>;
+    unsetLsmWriteSpec(): Promise<void>;
     version(): Promise<number>;
     checkout(version: number | string): Promise<void>;
     checkoutLatest(): Promise<void>;

package/dist/table.js

@@ -131,6 +131,9 @@ class LocalTable extends Table {
     async prewarmIndex(name) {
         await this.inner.prewarmIndex(name);
     }
+    async prewarmData(columns) {
+        await this.inner.prewarmData(columns);
+    }
     async waitForIndex(indexNames, timeoutSeconds) {
         await this.inner.waitForIndex(indexNames, timeoutSeconds);
     }
@@ -255,6 +258,16 @@ class LocalTable extends Table {
     async dropColumns(columnNames) {
         return await this.inner.dropColumns(columnNames);
     }
+    async setUnenforcedPrimaryKey(columns) {
+        const cols = typeof columns === "string" ? [columns] : columns;
+        return await this.inner.setUnenforcedPrimaryKey(cols);
+    }
+    async setLsmWriteSpec(spec) {
+        return await this.inner.setLsmWriteSpec(spec);
+    }
+    async unsetLsmWriteSpec() {
+        return await this.inner.unsetLsmWriteSpec();
+    }
     async version() {
         return await this.inner.version();
     }

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.28.0-beta.9",
+  "version": "0.29.1-beta.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -38,15 +38,15 @@
     "url": "https://github.com/lancedb/lancedb"
   },
   "devDependencies": {
-    "@aws-sdk/client-dynamodb": "^3.33.0",
-    "@aws-sdk/client-kms": "^3.33.0",
-    "@aws-sdk/client-s3": "^3.33.0",
+    "@aws-sdk/client-dynamodb": "3.1003.0",
+    "@aws-sdk/client-kms": "3.1003.0",
+    "@aws-sdk/client-s3": "3.1003.0",
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
-    "@napi-rs/cli": "^3.5.1",
+    "@napi-rs/cli": "3.5.1",
     "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
-    "@types/node": "^22.7.4",
+    "@types/node": "22.7.4",
     "@types/tmp": "^0.2.6",
     "apache-arrow-15": "npm:apache-arrow@15.0.0",
     "apache-arrow-16": "npm:apache-arrow@16.0.0",
@@ -57,9 +57,9 @@
     "shx": "^0.3.4",
     "tmp": "^0.2.3",
     "ts-jest": "^29.1.2",
-    "typedoc": "^0.26.4",
-    "typedoc-plugin-markdown": "^4.2.1",
-    "typescript": "^5.5.4",
+    "typedoc": "0.26.4",
+    "typedoc-plugin-markdown": "4.2.1",
+    "typescript": "5.5.4",
     "typescript-eslint": "^7.1.0"
   },
   "ava": {
@@ -68,6 +68,7 @@
   "engines": {
     "node": ">= 18"
   },
+  "packageManager": "pnpm@11.1.1",
   "cpu": [
     "x64",
     "arm64"
@@ -80,11 +81,10 @@
   "scripts": {
     "artifacts": "napi artifacts",
     "build:debug": "napi build --platform --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir lancedb",
-    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
+    "postbuild:debug": "shx mkdir -p dist && shx cp lancedb/*.node dist/ && node -e \"require('fs').writeFileSync('dist/package.json', JSON.stringify({name:'@lancedb/lancedb',type:'commonjs'}))\"",
     "build:release": "napi build --platform --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js --output-dir dist",
-    "postbuild:release": "shx mkdir -p dist && shx cp lancedb/*.node dist/",
-    "build": "npm run build:debug && npm run tsc",
-    "build-release": "npm run build:release && npm run tsc",
+    "build": "pnpm build:debug && pnpm tsc",
+    "build-release": "pnpm build:release && pnpm tsc",
     "tsc": "tsc -b",
     "posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
     "lint-ci": "biome ci .",
@@ -94,7 +94,7 @@
     "lint-fix": "biome check --write . && biome format --write .",
     "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
-    "integration": "S3_TEST=1 npm run test",
+    "integration": "S3_TEST=1 pnpm test",
     "universal": "napi universalize",
     "version": "napi version"
   },
@@ -102,13 +102,13 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.28.0-beta.9",
-    "@lancedb/lancedb-linux-x64-gnu": "0.28.0-beta.9",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.28.0-beta.9",
-    "@lancedb/lancedb-linux-x64-musl": "0.28.0-beta.9",
-    "@lancedb/lancedb-linux-arm64-musl": "0.28.0-beta.9",
-    "@lancedb/lancedb-win32-x64-msvc": "0.28.0-beta.9",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.28.0-beta.9"
+    "@lancedb/lancedb-darwin-arm64": "0.29.1-beta.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.29.1-beta.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.29.1-beta.0",
+    "@lancedb/lancedb-linux-x64-musl": "0.29.1-beta.0",
+    "@lancedb/lancedb-linux-arm64-musl": "0.29.1-beta.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.29.1-beta.0",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.29.1-beta.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"

package/pnpm-workspace.yaml

@@ -0,0 +1,18 @@
+# Flat node_modules layout. The @napi-rs/cli build step fails to locate
+# the cdylib artifact under pnpm's isolated layout; the hoisted linker
+# mirrors npm's structure and unblocks the native build.
+nodeLinker: hoisted
+
+# Block resolution of versions less than 24h old (Shai-Hulud window).
+# This is the pnpm 11 default but pinned here so it's visible to
+# reviewers and survives a future pnpm major flipping the default.
+minimumReleaseAge: 1440
+
+# Fail install if a transitive dep tries to run an unapproved script.
+strictDepBuilds: true
+
+allowBuilds:
+  '@biomejs/biome': true
+  onnxruntime-node: true
+  protobufjs: true
+  sharp: true