@lancedb/lancedb 0.40.0-beta.11 → 0.40.0-beta.13

package/NODEJS_THIRD_PARTY_LICENSES.md

@@ -275,6 +275,7 @@
 [@types/node@20.16.10](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
 [@types/node@20.17.9](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
 [@types/node@22.7.4](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
+[@types/node@25.9.8](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
 [@types/semver@7.5.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
 [@types/stack-utils@2.0.3](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
 [@types/tmp@0.2.6](https://github.com/DefinitelyTyped/DefinitelyTyped) - MIT
@@ -306,6 +307,9 @@
 [apache-arrow@16.0.0](https://github.com/apache/arrow) - Apache-2.0
 [apache-arrow@17.0.0](https://github.com/apache/arrow) - Apache-2.0
 [apache-arrow@18.0.0](https://github.com/apache/arrow) - Apache-2.0
+[apache-arrow@19.0.1](https://github.com/apache/arrow) - Apache-2.0
+[apache-arrow@20.0.0](https://github.com/apache/arrow) - Apache-2.0
+[apache-arrow@21.2.0](https://github.com/apache/arrow-js) - Apache-2.0
 [argparse@1.0.10](https://github.com/nodeca/argparse) - MIT
 [argparse@2.0.1](https://github.com/nodeca/argparse) - Python-2.0
 [array-back@3.1.0](https://github.com/75lb/array-back) - MIT
@@ -348,6 +352,7 @@
 [color@4.2.3](https://github.com/Qix-/color) - MIT
 [combined-stream@1.0.8](https://github.com/felixge/node-combined-stream) - MIT
 [command-line-args@5.2.1](https://github.com/75lb/command-line-args) - MIT
+[command-line-args@6.0.2](https://github.com/75lb/command-line-args) - MIT
 [command-line-usage@7.0.1](https://github.com/75lb/command-line-usage) - MIT
 [concat-map@0.0.1](https://github.com/substack/node-concat-map) - MIT
 [convert-source-map@2.0.0](https://github.com/thlorenz/convert-source-map) - MIT
@@ -398,12 +403,14 @@
 [file-entry-cache@6.0.1](https://github.com/royriojas/file-entry-cache) - MIT
 [fill-range@7.1.1](https://github.com/jonschlinkert/fill-range) - MIT
 [find-replace@3.0.0](https://github.com/75lb/find-replace) - MIT
+[find-replace@5.0.2](https://github.com/75lb/find-replace) - MIT
 [find-up@4.1.0](https://github.com/sindresorhus/find-up) - MIT
 [find-up@5.0.0](https://github.com/sindresorhus/find-up) - MIT
 [flat-cache@3.2.0](https://github.com/jaredwray/flat-cache) - MIT
 [flatbuffers@1.12.0](https://github.com/google/flatbuffers) - Apache*
 [flatbuffers@23.5.26](https://github.com/google/flatbuffers) - Apache*
 [flatbuffers@24.3.25](https://github.com/google/flatbuffers) - Apache-2.0
+[flatbuffers@25.9.23](https://github.com/google/flatbuffers) - Apache-2.0
 [flatted@3.2.9](https://github.com/WebReflection/flatted) - ISC
 [follow-redirects@1.15.6](https://github.com/follow-redirects/follow-redirects) - MIT
 [foreground-child@3.3.0](https://github.com/tapjs/foreground-child) - ISC
@@ -644,6 +651,7 @@
 [uc.micro@2.1.0](https://github.com/markdown-it/uc.micro) - MIT
 [undici-types@5.26.5](https://github.com/nodejs/undici) - MIT
 [undici-types@6.19.8](https://github.com/nodejs/undici) - MIT
+[undici-types@7.24.6](https://github.com/nodejs/undici) - MIT
 [update-browserslist-db@1.0.13](https://github.com/browserslist/update-db) - MIT
 [uri-js@4.4.1](https://github.com/garycourt/uri-js) - BSD-2-Clause
 [uuid@9.0.1](https://github.com/uuidjs/uuid) - MIT

package/dist/arrow.d.ts

@@ -9,11 +9,22 @@ export type SchemaLike = Schema | {
     get names(): unknown[];
 };
 export type FieldLike = Field | {
-    type: string;
+    type: string | DataTypeLike;
     name: string;
     nullable: boolean;
     metadata?: Map<string, string>;
 };
+/**
+ * A `DataType` from any copy or version of apache-arrow.
+ *
+ * Arrow 21 brands its classes with `unique symbol` properties, so a type
+ * object from a second copy of the library no longer satisfies the `DataType`
+ * type of this one even though it is structurally identical. Inputs that only
+ * need to be sanitized accept this looser shape instead.
+ */
+export type DataTypeLike = DataType | {
+    readonly typeId: number;
+};
 /**
  * Create an Arrow field backed by LanceDB's JSON extension type.
  *
@@ -205,6 +216,12 @@ export declare class MakeArrowTableOptions {
  * ```
  */
 export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>, metadata?: Map<string, string>): ArrowTable;
+/**
+ * Reads `Blob` / `File` values in the blob columns of `schema` into bytes so
+ * the synchronous conversion in {@link makeArrowTable} can accept them.
+ * Returns `data` itself when there is nothing to read.
+ */
+export declare function resolveBlobInputs(data: Array<Record<string, unknown>>, schema?: SchemaLike): Promise<Array<Record<string, unknown>>>;
 /**
  * Create an empty Arrow table with the provided schema
  */

package/dist/arrow.js

@@ -42,6 +42,7 @@ exports.isUnion = isUnion;
 exports.isFixedSizeBinary = isFixedSizeBinary;
 exports.isFixedSizeList = isFixedSizeList;
 exports.makeArrowTable = makeArrowTable;
+exports.resolveBlobInputs = resolveBlobInputs;
 exports.makeEmptyTable = makeEmptyTable;
 exports.convertToTable = convertToTable;
 exports.newVectorType = newVectorType;
@@ -354,6 +355,22 @@ function makeArrowTable(data, options, metadata) {
         schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
         schema = validateSchemaEmbeddings(schema, data, options?.embeddingFunction);
     }
+    if (schema !== undefined) {
+        // Validate blob values up front and give every one the full
+        // `{ data, uri }` shape, so inference sees the same struct whether a row
+        // passed bytes, a URI, or a partial struct.
+        data = mapBlobInputs(data, schema, (value, field, row) => {
+            if (value === undefined) {
+                return value;
+            }
+            try {
+                return (0, blob_1.coerceBlobValue)(value);
+            }
+            catch (e) {
+                throw new Error(`Invalid value for blob field ${field} at row ${row}: ${e.message}`);
+            }
+        });
+    }
     let schemaMetadata = schema?.metadata || new Map();
     if (metadata !== undefined) {
         schemaMetadata = new Map([...schemaMetadata, ...metadata]);
@@ -401,6 +418,87 @@ function containsBlobField(field) {
     }
     return (field.type.children ?? []).some((child) => containsBlobField(child));
 }
+/**
+ * Calls `visit` on every value that lands in a blob column of `schema`,
+ * including blobs nested in structs and lists, and replaces it with the
+ * result. Records and arrays are copied only where a value changed, so the
+ * caller's data is never mutated.
+ */
+function mapBlobInputs(data, schema, visit) {
+    if (!schema.fields.some(containsBlobField)) {
+        return data;
+    }
+    return data.map((record, row) => isObject(record)
+        ? mapBlobFields(record, schema.fields, "", row, visit)
+        : record);
+}
+function mapBlobFields(record, fields, prefix, row, visit) {
+    let out;
+    for (const field of fields) {
+        if (!containsBlobField(field) || !Object.hasOwn(record, field.name)) {
+            continue;
+        }
+        const value = record[field.name];
+        const mapped = mapBlobField(value, field, `${prefix}${field.name}`, row, visit);
+        if (mapped !== value) {
+            out ??= { ...record };
+            out[field.name] = mapped;
+        }
+    }
+    return out ?? record;
+}
+function mapBlobField(value, field, label, row, visit) {
+    if ((0, blob_1.isBlobField)(field)) {
+        return visit(value, label, row);
+    }
+    if (field.type instanceof apache_arrow_1.Struct && isObject(value)) {
+        return mapBlobFields(value, field.type.children, `${label}.`, row, visit);
+    }
+    if (isList(field.type) && Array.isArray(value)) {
+        const child = field.type.children[0];
+        let out;
+        for (const [index, element] of value.entries()) {
+            const mapped = mapBlobField(element, child, `${label}[${index}]`, row, visit);
+            if (mapped !== element) {
+                out ??= [...value];
+                out[index] = mapped;
+            }
+        }
+        return out ?? value;
+    }
+    return value;
+}
+/**
+ * Reads `Blob` / `File` values in the blob columns of `schema` into bytes so
+ * the synchronous conversion in {@link makeArrowTable} can accept them.
+ * Returns `data` itself when there is nothing to read.
+ */
+async function resolveBlobInputs(data, schema) {
+    if (schema === undefined || schema === null) {
+        return data;
+    }
+    const sanitized = (0, sanitize_1.sanitizeSchema)(schema);
+    // Keyed by the Blob itself, so rows sharing one Blob (bare or as
+    // `{ data }`) read it once and share the bytes.
+    const reads = new Map();
+    mapBlobInputs(data, sanitized, (value) => {
+        const source = (0, blob_1.blobToRead)(value);
+        if (source !== undefined && !reads.has(source)) {
+            reads.set(source, source.arrayBuffer().then((buffer) => new Uint8Array(buffer)));
+        }
+        return value;
+    });
+    if (reads.size === 0) {
+        return data;
+    }
+    const bytes = new Map(await Promise.all([...reads].map(async ([source, read]) => [source, await read])));
+    return mapBlobInputs(data, sanitized, (value) => {
+        const source = (0, blob_1.blobToRead)(value);
+        return source === undefined
+            ? value
+            : (0, blob_1.withBlobBytes)(value, bytes.get(source));
+    });
+}
 function isObject(value) {
     return (typeof value === "object" &&
         value !== null &&
@@ -795,6 +893,7 @@ async function convertToTable(data, embeddings, makeTableOptions) {
         makeTableOptions.schema.metadata?.has("embedding_functions")) {
         processedData = ensureNestedFieldsExist(data, makeTableOptions.schema);
     }
+    processedData = await resolveBlobInputs(processedData, makeTableOptions?.schema);
     const table = makeArrowTable(processedData, makeTableOptions);
     return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
 }
@@ -835,6 +934,69 @@ async function fromRecordsToStreamBuffer(data, embeddings, schema) {
     const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(table);
     return Buffer.from(await writer.toUint8Array());
 }
+// `Type.Utf8View` / `Type.BinaryView` as numbers: the enum members only exist
+// in Arrow 21+, and this module compiles against every supported release.
+const UTF8_VIEW_TYPE_ID = 24;
+const BINARY_VIEW_TYPE_ID = 23;
+/**
+ * Copy a Utf8View / BinaryView `Data` into a single `Data` of `type`.
+ */
+function materializeViewData(child, type) {
+    const builder = (0, apache_arrow_1.makeBuilder)({ type, nullValues: [null] });
+    for (const value of new apache_arrow_1.Vector([child])) {
+        builder.append(value);
+    }
+    return builder.finish().flush();
+}
+/**
+ * Rebuild any top-level Utf8View / BinaryView column as Utf8 / Binary.
+ *
+ * Lance stores the view types as their offset-based equivalents anyway, so
+ * nothing is lost. Doing it here also sidesteps an Arrow JS 21 bug: its IPC
+ * writer emits a truncated views buffer for a *sliced* view array, which the
+ * Rust reader rejects with "Need at least N bytes in buffers[0]".
+ *
+ * The record batches are rebuilt positionally rather than through a
+ * `Record<string, Vector>`: JavaScript enumerates integer-like keys first, so
+ * a field named e.g. `"1"` would otherwise be paired with the wrong column.
+ *
+ * Tables without view columns are returned as-is.
+ */
+function materializeViewColumns(table) {
+    const replacements = new Map();
+    table.schema.fields.forEach((field, i) => {
+        if (field.type.typeId === UTF8_VIEW_TYPE_ID) {
+            replacements.set(i, new apache_arrow_1.Utf8());
+        }
+        else if (field.type.typeId === BINARY_VIEW_TYPE_ID) {
+            replacements.set(i, new apache_arrow_1.Binary());
+        }
+    });
+    if (replacements.size === 0) {
+        return table;
+    }
+    const fields = table.schema.fields.map((field, i) => {
+        const type = replacements.get(i);
+        return type === undefined
+            ? field
+            : new apache_arrow_1.Field(field.name, type, field.nullable, field.metadata);
+    });
+    const schema = new apache_arrow_1.Schema(fields, table.schema.metadata);
+    const batches = table.batches.map((batch) => {
+        const children = batch.data.children.map((child, i) => {
+            const type = replacements.get(i);
+            return type === undefined ? child : materializeViewData(child, type);
+        });
+        const data = (0, apache_arrow_1.makeData)({
+            type: new apache_arrow_1.Struct(fields),
+            length: batch.numRows,
+            nullCount: 0,
+            children,
+        });
+        return new apache_arrow_1.RecordBatch(schema, data);
+    });
+    return new apache_arrow_1.Table(schema, batches);
+}
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
  *
@@ -847,7 +1009,7 @@ async function fromTableToBuffer(table, embeddings, schema) {
     if (schema !== undefined && schema !== null) {
         schema = (0, sanitize_1.sanitizeSchema)(schema);
     }
-    const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
+    const tableWithEmbeddings = materializeViewColumns(await applyEmbeddings(table, embeddings, schema));
     validateBlobSchema(tableWithEmbeddings.schema);
     const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(tableWithEmbeddings);
     return Buffer.from(await writer.toUint8Array());
@@ -906,7 +1068,8 @@ async function fromRecordBatchToBuffer(batch) {
  * batch + EOS) suitable for incremental decode by `arrow_ipc::reader::StreamReader`.
  */
 async function fromRecordBatchToStreamBuffer(batch) {
-    const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll([batch]);
+    const table = materializeViewColumns(new apache_arrow_1.Table([batch]));
+    const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(table);
     return Buffer.from(await writer.toUint8Array());
 }
 /**
@@ -918,7 +1081,7 @@ async function fromRecordBatchToStreamBuffer(batch) {
  * `schema` is required if the table is empty
  */
 async function fromTableToStreamBuffer(table, embeddings, schema) {
-    const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
+    const tableWithEmbeddings = materializeViewColumns(await applyEmbeddings(table, embeddings, schema));
     const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
     return Buffer.from(await writer.toUint8Array());
 }
@@ -1019,7 +1182,7 @@ function ensureNestedFieldsExist(data, schema) {
         const completeRow = {};
         for (const field of schema.fields) {
             if (field.name in row) {
-                if (field.type.constructor.name === "Struct" &&
+                if (isPlainStructField(field) &&
                     row[field.name] !== null &&
                     row[field.name] !== undefined) {
                     // Handle nested struct
@@ -1034,15 +1197,22 @@ function ensureNestedFieldsExist(data, schema) {
             else {
                 // Keep a missing struct valid while filling each of its children with
                 // null. This is distinct from an explicitly null struct value.
-                completeRow[field.name] =
-                    field.type.constructor.name === "Struct"
-                        ? ensureStructFieldsExist({}, field.type)
-                        : null;
+                completeRow[field.name] = isPlainStructField(field)
+                    ? ensureStructFieldsExist({}, field.type)
+                    : null;
             }
         }
         return completeRow;
     });
 }
+/**
+ * Blob fields are Arrow structs, but their values are bytes, URIs, or
+ * `{ data } | { uri }` inputs that the blob coercion in `makeArrowTable`
+ * handles, so they must not be filled in like ordinary structs.
+ */
+function isPlainStructField(field) {
+    return field.type.constructor.name === "Struct" && !(0, blob_1.isBlobField)(field);
+}
 /**
  * Recursively ensures that all fields in a struct type exist in the data,
  * filling missing fields with null values.
@@ -1051,7 +1221,7 @@ function ensureStructFieldsExist(data, structType) {
     const completeStruct = {};
     for (const childField of structType.children) {
         if (childField.name in data) {
-            if (childField.type.constructor.name === "Struct" &&
+            if (isPlainStructField(childField) &&
                 data[childField.name] !== null &&
                 data[childField.name] !== undefined) {
                 // Recursively handle nested struct
@@ -1065,10 +1235,9 @@ function ensureStructFieldsExist(data, structType) {
         else {
             // Keep a missing struct valid while filling each of its children with
             // null. This is distinct from an explicitly null struct value.
-            completeStruct[childField.name] =
-                childField.type.constructor.name === "Struct"
-                    ? ensureStructFieldsExist({}, childField.type)
-                    : null;
+            completeStruct[childField.name] = isPlainStructField(childField)
+                ? ensureStructFieldsExist({}, childField.type)
+                : null;
         }
     }
     return completeStruct;

package/dist/blob.d.ts

@@ -1,6 +1,42 @@
 import { Field } from "apache-arrow";
 import { BlobFile as NativeBlobFile } from "./native";
-export type BlobInput = {
+/**
+ * Bytes accepted for a blob value. `ArrayBuffer` is wrapped without copying.
+ * `Blob` and `File` are read with `arrayBuffer()`, which only the async write
+ * paths ({@link Connection.createTable}, {@link Table.add},
+ * {@link Table.mergeInsert}) can do.
+ */
+export type BlobData = Buffer | Uint8Array | ArrayBuffer | Blob;
+/** A URI accepted for a blob value. A `URL` is stored as its `href`. */
+export type BlobUri = string | URL;
+/**
+ * A value accepted for a `lance.blob.v2` column (see {@link blob}).
+ *
+ * Either inline bytes, a URI pointing at external bytes, or a struct that sets
+ * exactly one of `data` or `uri`. `null` and `undefined` write a null blob.
+ *
+ * @example
+ * ```ts
+ * import { pathToFileURL } from "node:url";
+ * import type { BlobInput } from "@lancedb/lancedb";
+ *
+ * const rows: { id: bigint; image: BlobInput }[] = [
+ *   { id: 1n, image: await (await fetch(url)).arrayBuffer() },
+ *   { id: 2n, image: pathToFileURL("/data/cat.png") },
+ *   { id: 3n, image: { data: new Blob(["hello"]) } },
+ * ];
+ * await table.add(rows);
+ * ```
+ */
+export type BlobInput = BlobData | BlobUri | {
+    data: BlobData;
+    uri?: null;
+} | {
+    data?: null;
+    uri: BlobUri;
+} | null | undefined;
+/** A blob value normalized to the columns of the blob struct. */
+export type BlobValue = {
     data: Buffer | Uint8Array | null;
     uri: string | null;
 };
@@ -76,17 +112,69 @@ export declare class BlobFile {
     /** Returns the blob size in bytes. */
     size(): bigint;
     /**
-     * Reads from the cursor to the end and advances the cursor.
+     * Reads from the cursor and advances the cursor.
      *
-     * A second call returns an empty buffer. {@link BlobFile.readRange} does
-     * not move the cursor.
+     * Reads to the end when `maxBytes` is omitted, or at most `maxBytes` bytes
+     * otherwise. Returns an empty buffer at the end of the blob.
+     * {@link BlobFile.readRange} does not move the cursor.
      */
-    read(): Promise<Buffer>;
+    read(maxBytes?: bigint, options?: BlobReadOptions): Promise<Buffer>;
     /**
      * Reads the half-open byte range `[start, end)`.
      *
      * Fails when `end` is past the blob size. Does not move the cursor.
      */
-    readRange(start: bigint, end: bigint): Promise<Buffer>;
+    readRange(start: bigint, end: bigint, options?: BlobReadOptions): Promise<Buffer>;
+    /**
+     * Reads several half-open byte ranges. Returns one buffer per range, in
+     * the order given.
+     *
+     * Fails when any `end` is past the blob size. Does not move the cursor.
+     */
+    readRanges(ranges: BlobRange[], options?: BlobReadOptions): Promise<Buffer[]>;
+    /** Moves the cursor to `position`, in bytes from the start of the blob. */
+    seek(position: bigint): Promise<void>;
+    /** Returns the cursor position in bytes. */
+    tell(): Promise<bigint>;
+    /**
+     * Releases the handle. Reads after `close()` fail. Calling it again does
+     * nothing.
+     */
+    close(): Promise<void>;
+    /** Returns true after {@link BlobFile.close}. */
+    isClosed(): boolean;
 }
-export declare function coerceBlobValue(value: unknown): BlobInput | null;
+/** Options for blob reads. */
+export type BlobReadOptions = {
+    /**
+     * Cancels the read. The call rejects with `signal.reason`, and the native
+     * read stops, including in-flight requests to a remote table.
+     */
+    signal?: AbortSignal;
+};
+/** @ignore */
+export declare function runWithSignal<T>(signal: AbortSignal | undefined, run: (signal: AbortSignal | undefined) => Promise<T>): Promise<T>;
+/** A half-open byte range `[start, end)` for {@link BlobFile.readRanges}. */
+export type BlobRange = {
+    start: bigint;
+    end: bigint;
+};
+/**
+ * Rewrites the synchronous-only widenings of {@link BlobInput} (`ArrayBuffer`,
+ * `URL`) into `Uint8Array` and URI strings, keeping the value's shape. Other
+ * values are returned unchanged and validated later by {@link coerceBlobValue}.
+ *
+ * Throws on `Blob` / `File`, which must be read into bytes first.
+ */
+export declare function normalizeBlobInput(value: unknown): unknown;
+/**
+ * Returns the `Blob` / `File` a blob value needs read, either the value itself
+ * or its `data` field, or `undefined` when there is nothing to read.
+ */
+export declare function blobToRead(value: unknown): Blob | undefined;
+/**
+ * Replaces the `Blob` that {@link blobToRead} found in `value` with `bytes`,
+ * keeping the value's shape.
+ */
+export declare function withBlobBytes(value: unknown, bytes: Uint8Array): unknown;
+export declare function coerceBlobValue(input: unknown): BlobValue | null;