@lancedb/lancedb 0.15.1-beta.2 → 0.16.0

package/README.md

@@ -32,7 +32,7 @@ const results = await table.vectorSearch([0.1, 0.3]).limit(20).toArray();
 console.log(results);
 ```
 
-The [quickstart](../basic.md) contains a more complete example.
+The [quickstart](https://lancedb.github.io/lancedb/basic/) contains a more complete example.
 
 ## Development
 

package/dist/arrow.d.ts

@@ -91,8 +91,6 @@ export declare class MakeArrowTableOptions {
  * This function converts an array of Record<String, any> (row-major JS objects)
  * to an Arrow Table (a columnar structure)
  *
- * Note that it currently does not support nulls.
- *
  * If a schema is provided then it will be used to determine the resulting array
  * types.  Fields will also be reordered to fit the order defined by the schema.
  *
@@ -100,6 +98,9 @@ export declare class MakeArrowTableOptions {
  * will be controlled by the order of properties in the first record.  If a type
  * is inferred it will always be nullable.
  *
+ * If not all fields are found in the data, then a subset of the schema will be
+ * returned.
+ *
  * If the input is empty then a schema must be provided to create an empty table.
  *
  * When a schema is not specified then data types will be inferred.  The inference
@@ -107,11 +108,13 @@ export declare class MakeArrowTableOptions {
  *
  *  - boolean => Bool
  *  - number => Float64
+ *  - bigint => Int64
  *  - String => Utf8
  *  - Buffer => Binary
  *  - Record<String, any> => Struct
  *  - Array<any> => List
  * @example
+ * ```ts
  * import { fromTableToBuffer, makeArrowTable } from "../arrow";
  * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
  *
@@ -133,43 +136,41 @@ export declare class MakeArrowTableOptions {
  * names and data types.
  *
  * ```ts
- *
  * const schema = new Schema([
-    new Field("a", new Float64()),
-    new Field("b", new Float64()),
-    new Field(
-      "vector",
-      new FixedSizeList(3, new Field("item", new Float32()))
-    ),
-  ]);
-  const table = makeArrowTable([
-    { a: 1, b: 2, vector: [1, 2, 3] },
-    { a: 4, b: 5, vector: [4, 5, 6] },
-    { a: 7, b: 8, vector: [7, 8, 9] },
-  ]);
-  assert.deepEqual(table.schema, schema);
+ *   new Field("a", new Float64()),
+ *   new Field("b", new Float64()),
+ *   new Field(
+ *     "vector",
+ *     new FixedSizeList(3, new Field("item", new Float32()))
+ *   ),
+ * ]);
+ * const table = makeArrowTable([
+ *   { a: 1, b: 2, vector: [1, 2, 3] },
+ *   { a: 4, b: 5, vector: [4, 5, 6] },
+ *   { a: 7, b: 8, vector: [7, 8, 9] },
+ * ]);
+ * assert.deepEqual(table.schema, schema);
  * ```
  *
  * You can specify the vector column types and names using the options as well
  *
- * ```typescript
- *
+ * ```ts
  * const schema = new Schema([
-    new Field('a', new Float64()),
-    new Field('b', new Float64()),
-    new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-    new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
-  ]);
+ *   new Field('a', new Float64()),
+ *   new Field('b', new Float64()),
+ *   new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+ *   new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ * ]);
  * const table = makeArrowTable([
-    { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-    { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-    { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
-  ], {
-    vectorColumns: {
-      vec1: { type: new Float16() },
-      vec2: { type: new Float16() }
-    }
-  }
+ *   { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+ *   { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+ *   { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ * ], {
+ *   vectorColumns: {
+ *     vec1: { type: new Float16() },
+ *     vec2: { type: new Float16() }
+ *   }
+ * }
  * assert.deepEqual(table.schema, schema)
  * ```
  */

package/dist/arrow.js

@@ -186,8 +186,6 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
  * This function converts an array of Record<String, any> (row-major JS objects)
  * to an Arrow Table (a columnar structure)
  *
- * Note that it currently does not support nulls.
- *
  * If a schema is provided then it will be used to determine the resulting array
  * types.  Fields will also be reordered to fit the order defined by the schema.
  *
@@ -195,6 +193,9 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
  * will be controlled by the order of properties in the first record.  If a type
  * is inferred it will always be nullable.
  *
+ * If not all fields are found in the data, then a subset of the schema will be
+ * returned.
+ *
  * If the input is empty then a schema must be provided to create an empty table.
  *
  * When a schema is not specified then data types will be inferred.  The inference
@@ -202,11 +203,13 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
  *
  *  - boolean => Bool
  *  - number => Float64
+ *  - bigint => Int64
  *  - String => Utf8
  *  - Buffer => Binary
  *  - Record<String, any> => Struct
  *  - Array<any> => List
  * @example
+ * ```ts
  * import { fromTableToBuffer, makeArrowTable } from "../arrow";
  * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
  *
@@ -228,149 +231,325 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
  * names and data types.
  *
  * ```ts
- *
  * const schema = new Schema([
-    new Field("a", new Float64()),
-    new Field("b", new Float64()),
-    new Field(
-      "vector",
-      new FixedSizeList(3, new Field("item", new Float32()))
-    ),
-  ]);
-  const table = makeArrowTable([
-    { a: 1, b: 2, vector: [1, 2, 3] },
-    { a: 4, b: 5, vector: [4, 5, 6] },
-    { a: 7, b: 8, vector: [7, 8, 9] },
-  ]);
-  assert.deepEqual(table.schema, schema);
+ *   new Field("a", new Float64()),
+ *   new Field("b", new Float64()),
+ *   new Field(
+ *     "vector",
+ *     new FixedSizeList(3, new Field("item", new Float32()))
+ *   ),
+ * ]);
+ * const table = makeArrowTable([
+ *   { a: 1, b: 2, vector: [1, 2, 3] },
+ *   { a: 4, b: 5, vector: [4, 5, 6] },
+ *   { a: 7, b: 8, vector: [7, 8, 9] },
+ * ]);
+ * assert.deepEqual(table.schema, schema);
  * ```
  *
  * You can specify the vector column types and names using the options as well
  *
- * ```typescript
- *
+ * ```ts
  * const schema = new Schema([
-    new Field('a', new Float64()),
-    new Field('b', new Float64()),
-    new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-    new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
-  ]);
+ *   new Field('a', new Float64()),
+ *   new Field('b', new Float64()),
+ *   new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
+ *   new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
+ * ]);
  * const table = makeArrowTable([
-    { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-    { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-    { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
-  ], {
-    vectorColumns: {
-      vec1: { type: new Float16() },
-      vec2: { type: new Float16() }
-    }
-  }
+ *   { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
+ *   { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
+ *   { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
+ * ], {
+ *   vectorColumns: {
+ *     vec1: { type: new Float16() },
+ *     vec2: { type: new Float16() }
+ *   }
+ * }
  * assert.deepEqual(table.schema, schema)
  * ```
  */
 function makeArrowTable(data, options, metadata) {
+    const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
+    let schema = undefined;
+    if (opt.schema !== undefined && opt.schema !== null) {
+        schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
+        schema = validateSchemaEmbeddings(schema, data, options?.embeddingFunction);
+    }
+    let schemaMetadata = schema?.metadata || new Map();
+    if (metadata !== undefined) {
+        schemaMetadata = new Map([...schemaMetadata, ...metadata]);
+    }
     if (data.length === 0 &&
         (options?.schema === undefined || options?.schema === null)) {
         throw new Error("At least one record or a schema needs to be provided");
     }
-    const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
-    if (opt.schema !== undefined && opt.schema !== null) {
-        opt.schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
-        opt.schema = validateSchemaEmbeddings(opt.schema, data, options?.embeddingFunction);
-    }
-    const columns = {};
-    // TODO: sample dataset to find missing columns
-    // Prefer the field ordering of the schema, if present
-    const columnNames = opt.schema != null ? opt.schema.names : Object.keys(data[0]);
-    for (const colName of columnNames) {
-        if (data.length !== 0 &&
-            !Object.prototype.hasOwnProperty.call(data[0], colName)) {
-            // The field is present in the schema, but not in the data, skip it
-            continue;
-        }
-        // Extract a single column from the records (transpose from row-major to col-major)
-        let values = data.map((datum) => datum[colName]);
-        // By default (type === undefined) arrow will infer the type from the JS type
-        let type;
-        if (opt.schema !== undefined) {
-            // If there is a schema provided, then use that for the type instead
-            type = opt.schema?.fields.filter((f) => f.name === colName)[0]?.type;
-            if (apache_arrow_1.DataType.isInt(type) && type.bitWidth === 64) {
-                // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
-                values = values.map((v) => {
-                    if (v === null) {
-                        return v;
+    else if (data.length === 0) {
+        if (schema === undefined) {
+            throw new Error("A schema must be provided if data is empty");
+        }
+        else {
+            schema = new apache_arrow_1.Schema(schema.fields, schemaMetadata);
+            return new apache_arrow_1.Table(schema);
+        }
+    }
+    let inferredSchema = inferSchema(data, schema, opt);
+    inferredSchema = new apache_arrow_1.Schema(inferredSchema.fields, schemaMetadata);
+    const finalColumns = {};
+    for (const field of inferredSchema.fields) {
+        finalColumns[field.name] = transposeData(data, field);
+    }
+    return new apache_arrow_1.Table(inferredSchema, finalColumns);
+}
+function inferSchema(data, schema, opts) {
+    // We will collect all fields we see in the data.
+    const pathTree = new PathTree();
+    for (const [rowI, row] of data.entries()) {
+        for (const [path, value] of rowPathsAndValues(row)) {
+            if (!pathTree.has(path)) {
+                // First time seeing this field.
+                if (schema !== undefined) {
+                    const field = getFieldForPath(schema, path);
+                    if (field === undefined) {
+                        throw new Error(`Found field not in schema: ${path.join(".")} at row ${rowI}`);
                     }
-                    if (typeof v === "bigint") {
-                        return v;
+                    else {
+                        pathTree.set(path, field.type);
                     }
-                    if (typeof v === "number") {
-                        return BigInt(v);
+                }
+                else {
+                    const inferredType = inferType(value, path, opts);
+                    if (inferredType === undefined) {
+                        throw new Error(`Failed to infer data type for field ${path.join(".")} at row ${rowI}. \
+                             Consider providing an explicit schema.`);
                     }
-                    throw new Error(`Expected BigInt or number for column ${colName}, got ${typeof v}`);
-                });
+                    pathTree.set(path, inferredType);
+                }
+            }
+            else if (schema === undefined) {
+                const currentType = pathTree.get(path);
+                const newType = inferType(value, path, opts);
+                if (currentType !== newType) {
+                    new Error(`Failed to infer schema for data. Previously inferred type \
+                     ${currentType} but found ${newType} at row ${rowI}. Consider \
+                     providing an explicit schema.`);
+                }
             }
         }
-        else {
-            // Otherwise, check to see if this column is one of the vector columns
-            // defined by opt.vectorColumns and, if so, use the fixed size list type
-            const vectorColumnOptions = opt.vectorColumns[colName];
-            if (vectorColumnOptions !== undefined) {
-                const firstNonNullValue = values.find((v) => v !== null);
-                if (Array.isArray(firstNonNullValue)) {
-                    type = newVectorType(firstNonNullValue.length, vectorColumnOptions.type);
+    }
+    if (schema === undefined) {
+        function fieldsFromPathTree(pathTree) {
+            const fields = [];
+            for (const [name, value] of pathTree.map.entries()) {
+                if (value instanceof PathTree) {
+                    const children = fieldsFromPathTree(value);
+                    fields.push(new apache_arrow_1.Field(name, new apache_arrow_1.Struct(children), true));
                 }
                 else {
-                    throw new Error(`Column ${colName} is expected to be a vector column but first non-null value is not an array.  Could not determine size of vector column`);
+                    fields.push(new apache_arrow_1.Field(name, value, true));
                 }
             }
+            return fields;
         }
-        try {
-            // Convert an Array of JS values to an arrow vector
-            columns[colName] = makeVector(values, type, opt.dictionaryEncodeStrings);
-        }
-        catch (error) {
-            // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-            throw Error(`Could not convert column "${colName}" to Arrow: ${error}`);
-        }
-    }
-    if (opt.schema != null) {
-        // `new ArrowTable(columns)` infers a schema which may sometimes have
-        // incorrect nullability (it assumes nullable=true always)
-        //
-        // `new ArrowTable(schema, columns)` will also fail because it will create a
-        // batch with an inferred schema and then complain that the batch schema
-        // does not match the provided schema.
-        //
-        // To work around this we first create a table with the wrong schema and
-        // then patch the schema of the batches so we can use
-        // `new ArrowTable(schema, batches)` which does not do any schema inference
-        const firstTable = new apache_arrow_1.Table(columns);
-        const batchesFixed = firstTable.batches.map((batch) => new apache_arrow_1.RecordBatch(opt.schema, batch.data));
-        let schema;
-        if (metadata !== undefined) {
-            let schemaMetadata = opt.schema.metadata;
-            if (schemaMetadata.size === 0) {
-                schemaMetadata = metadata;
-            }
-            else {
-                for (const [key, entry] of schemaMetadata.entries()) {
-                    schemaMetadata.set(key, entry);
+        const fields = fieldsFromPathTree(pathTree);
+        return new apache_arrow_1.Schema(fields);
+    }
+    else {
+        function takeMatchingFields(fields, pathTree) {
+            const outFields = [];
+            for (const field of fields) {
+                if (pathTree.map.has(field.name)) {
+                    const value = pathTree.get([field.name]);
+                    if (value instanceof PathTree) {
+                        const struct = field.type;
+                        const children = takeMatchingFields(struct.children, value);
+                        outFields.push(new apache_arrow_1.Field(field.name, new apache_arrow_1.Struct(children), field.nullable));
+                    }
+                    else {
+                        outFields.push(new apache_arrow_1.Field(field.name, value, field.nullable));
+                    }
                 }
             }
-            schema = new apache_arrow_1.Schema(opt.schema.fields, schemaMetadata);
+            return outFields;
+        }
+        const fields = takeMatchingFields(schema.fields, pathTree);
+        return new apache_arrow_1.Schema(fields);
+    }
+}
+function* rowPathsAndValues(row, basePath = []) {
+    for (const [key, value] of Object.entries(row)) {
+        if (isObject(value)) {
+            yield* rowPathsAndValues(value, [...basePath, key]);
         }
         else {
-            schema = opt.schema;
+            yield [[...basePath, key], value];
         }
-        return new apache_arrow_1.Table(schema, batchesFixed);
     }
-    const tbl = new apache_arrow_1.Table(columns);
-    if (metadata !== undefined) {
-        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-        tbl.schema.metadata = metadata;
+}
+function isObject(value) {
+    return (typeof value === "object" &&
+        value !== null &&
+        !Array.isArray(value) &&
+        !(value instanceof RegExp) &&
+        !(value instanceof Date) &&
+        !(value instanceof Set) &&
+        !(value instanceof Map) &&
+        !(value instanceof Buffer));
+}
+function getFieldForPath(schema, path) {
+    let current = schema;
+    for (const key of path) {
+        if (current instanceof apache_arrow_1.Schema) {
+            const field = current.fields.find((f) => f.name === key);
+            if (field === undefined) {
+                return undefined;
+            }
+            current = field;
+        }
+        else if (current instanceof apache_arrow_1.Field && apache_arrow_1.DataType.isStruct(current.type)) {
+            const struct = current.type;
+            const field = struct.children.find((f) => f.name === key);
+            if (field === undefined) {
+                return undefined;
+            }
+            current = field;
+        }
+        else {
+            return undefined;
+        }
+    }
+    if (current instanceof apache_arrow_1.Field) {
+        return current;
+    }
+    else {
+        return undefined;
+    }
+}
+/**
+ * Try to infer which Arrow type to use for a given value.
+ *
+ * May return undefined if the type cannot be inferred.
+ */
+function inferType(value, path, opts) {
+    if (typeof value === "bigint") {
+        return new apache_arrow_1.Int64();
+    }
+    else if (typeof value === "number") {
+        // Even if it's an integer, it's safer to assume Float64. Users can
+        // always provide an explicit schema or use BigInt if they mean integer.
+        return new apache_arrow_1.Float64();
+    }
+    else if (typeof value === "string") {
+        if (opts.dictionaryEncodeStrings) {
+            return new apache_arrow_1.Dictionary(new apache_arrow_1.Utf8(), new apache_arrow_1.Int32());
+        }
+        else {
+            return new apache_arrow_1.Utf8();
+        }
+    }
+    else if (typeof value === "boolean") {
+        return new apache_arrow_1.Bool();
+    }
+    else if (value instanceof Buffer) {
+        return new apache_arrow_1.Binary();
+    }
+    else if (Array.isArray(value)) {
+        if (value.length === 0) {
+            return undefined; // Without any values we can't infer the type
+        }
+        if (path.length === 1 && Object.hasOwn(opts.vectorColumns, path[0])) {
+            const floatType = (0, sanitize_1.sanitizeType)(opts.vectorColumns[path[0]].type);
+            return new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", floatType, true));
+        }
+        const valueType = inferType(value[0], path, opts);
+        if (valueType === undefined) {
+            return undefined;
+        }
+        // Try to automatically detect embedding columns.
+        if (valueType instanceof apache_arrow_1.Float && path[path.length - 1] === "vector") {
+            // We default to Float32 for vectors.
+            const child = new apache_arrow_1.Field("item", new apache_arrow_1.Float32(), true);
+            return new apache_arrow_1.FixedSizeList(value.length, child);
+        }
+        else {
+            const child = new apache_arrow_1.Field("item", valueType, true);
+            return new apache_arrow_1.List(child);
+        }
+    }
+    else {
+        // TODO: timestamp
+        return undefined;
+    }
+}
+class PathTree {
+    map;
+    constructor(entries) {
+        this.map = new Map();
+        if (entries !== undefined) {
+            for (const [path, value] of entries) {
+                this.set(path, value);
+            }
+        }
+    }
+    has(path) {
+        let ref = this;
+        for (const part of path) {
+            if (!(ref instanceof PathTree) || !ref.map.has(part)) {
+                return false;
+            }
+            ref = ref.map.get(part);
+        }
+        return true;
+    }
+    get(path) {
+        let ref = this;
+        for (const part of path) {
+            if (!(ref instanceof PathTree) || !ref.map.has(part)) {
+                return undefined;
+            }
+            ref = ref.map.get(part);
+        }
+        return ref;
+    }
+    set(path, value) {
+        let ref = this;
+        for (const part of path.slice(0, path.length - 1)) {
+            if (!ref.map.has(part)) {
+                ref.map.set(part, new PathTree());
+            }
+            ref = ref.map.get(part);
+        }
+        ref.map.set(path[path.length - 1], value);
+    }
+}
+function transposeData(data, field, path = []) {
+    if (field.type instanceof apache_arrow_1.Struct) {
+        const childFields = field.type.children;
+        const childVectors = childFields.map((child) => {
+            return transposeData(data, child, [...path, child.name]);
+        });
+        const structData = (0, apache_arrow_1.makeData)({
+            type: field.type,
+            children: childVectors,
+        });
+        return (0, apache_arrow_1.makeVector)(structData);
+    }
+    else {
+        const valuesPath = [...path, field.name];
+        const values = data.map((datum) => {
+            let current = datum;
+            for (const key of valuesPath) {
+                if (isObject(current) && Object.hasOwn(current, key)) {
+                    current = current[key];
+                }
+                else {
+                    return null;
+                }
+            }
+            return current;
+        });
+        return makeVector(values, field.type);
     }
-    return tbl;
 }
 /**
  * Create an empty Arrow table with the provided schema
@@ -409,6 +588,36 @@ function makeListVector(lists) {
 function makeVector(values, type, stringAsDictionary) {
     if (type !== undefined) {
         // No need for inference, let Arrow create it
+        if (type instanceof apache_arrow_1.Int) {
+            if (apache_arrow_1.DataType.isInt(type) && type.bitWidth === 64) {
+                // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
+                values = values.map((v) => {
+                    if (v === null) {
+                        return v;
+                    }
+                    else if (typeof v === "bigint") {
+                        return v;
+                    }
+                    else if (typeof v === "number") {
+                        return BigInt(v);
+                    }
+                    else {
+                        return v;
+                    }
+                });
+            }
+            else {
+                // Similarly, bigint isn't supported for 16 or 32-bit ints.
+                values = values.map((v) => {
+                    if (typeof v == "bigint") {
+                        return Number(v);
+                    }
+                    else {
+                        return v;
+                    }
+                });
+            }
+        }
         return (0, apache_arrow_1.vectorFromArray)(values, type);
     }
     if (values.length === 0) {