@lancedb/lancedb 0.7.0 → 0.8.0

package/nodejs-artifacts/arrow.js

@@ -1,768 +0,0 @@
-"use strict";
-// Copyright 2023 Lance Developers.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createEmptyTable = exports.fromTableToStreamBuffer = exports.fromDataToBuffer = exports.fromTableToBuffer = exports.fromRecordsToStreamBuffer = exports.fromRecordsToBuffer = exports.newVectorType = exports.convertToTable = exports.makeEmptyTable = exports.makeArrowTable = exports.MakeArrowTableOptions = exports.VectorColumnOptions = exports.isFixedSizeList = exports.isFixedSizeBinary = exports.isUnion = exports.isStruct = exports.isList = exports.isDuration = exports.isInterval = exports.isTimestamp = exports.isTime = exports.isDate = exports.isDecimal = exports.isBool = exports.isLargeUtf8 = exports.isUtf8 = exports.isLargeBinary = exports.isBinary = exports.isFloat = exports.isInt = exports.isNull = exports.isDataType = exports.isArrowTable = void 0;
-const apache_arrow_1 = require("apache-arrow");
-const registry_1 = require("./embedding/registry");
-const sanitize_1 = require("./sanitize");
-__exportStar(require("apache-arrow"), exports);
-function isArrowTable(value) {
-    if (value instanceof apache_arrow_1.Table)
-        return true;
-    return "schema" in value && "batches" in value;
-}
-exports.isArrowTable = isArrowTable;
-function isDataType(value) {
-    return (value instanceof apache_arrow_1.DataType ||
-        apache_arrow_1.DataType.isNull(value) ||
-        apache_arrow_1.DataType.isInt(value) ||
-        apache_arrow_1.DataType.isFloat(value) ||
-        apache_arrow_1.DataType.isBinary(value) ||
-        apache_arrow_1.DataType.isLargeBinary(value) ||
-        apache_arrow_1.DataType.isUtf8(value) ||
-        apache_arrow_1.DataType.isLargeUtf8(value) ||
-        apache_arrow_1.DataType.isBool(value) ||
-        apache_arrow_1.DataType.isDecimal(value) ||
-        apache_arrow_1.DataType.isDate(value) ||
-        apache_arrow_1.DataType.isTime(value) ||
-        apache_arrow_1.DataType.isTimestamp(value) ||
-        apache_arrow_1.DataType.isInterval(value) ||
-        apache_arrow_1.DataType.isDuration(value) ||
-        apache_arrow_1.DataType.isList(value) ||
-        apache_arrow_1.DataType.isStruct(value) ||
-        apache_arrow_1.DataType.isUnion(value) ||
-        apache_arrow_1.DataType.isFixedSizeBinary(value) ||
-        apache_arrow_1.DataType.isFixedSizeList(value) ||
-        apache_arrow_1.DataType.isMap(value) ||
-        apache_arrow_1.DataType.isDictionary(value));
-}
-exports.isDataType = isDataType;
-function isNull(value) {
-    return value instanceof apache_arrow_1.Null || apache_arrow_1.DataType.isNull(value);
-}
-exports.isNull = isNull;
-function isInt(value) {
-    return value instanceof apache_arrow_1.Int || apache_arrow_1.DataType.isInt(value);
-}
-exports.isInt = isInt;
-function isFloat(value) {
-    return value instanceof apache_arrow_1.Float || apache_arrow_1.DataType.isFloat(value);
-}
-exports.isFloat = isFloat;
-function isBinary(value) {
-    return value instanceof apache_arrow_1.Binary || apache_arrow_1.DataType.isBinary(value);
-}
-exports.isBinary = isBinary;
-function isLargeBinary(value) {
-    return value instanceof apache_arrow_1.LargeBinary || apache_arrow_1.DataType.isLargeBinary(value);
-}
-exports.isLargeBinary = isLargeBinary;
-function isUtf8(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isUtf8(value);
-}
-exports.isUtf8 = isUtf8;
-function isLargeUtf8(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isLargeUtf8(value);
-}
-exports.isLargeUtf8 = isLargeUtf8;
-function isBool(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isBool(value);
-}
-exports.isBool = isBool;
-function isDecimal(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isDecimal(value);
-}
-exports.isDecimal = isDecimal;
-function isDate(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isDate(value);
-}
-exports.isDate = isDate;
-function isTime(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isTime(value);
-}
-exports.isTime = isTime;
-function isTimestamp(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isTimestamp(value);
-}
-exports.isTimestamp = isTimestamp;
-function isInterval(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isInterval(value);
-}
-exports.isInterval = isInterval;
-function isDuration(value) {
-    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isDuration(value);
-}
-exports.isDuration = isDuration;
-function isList(value) {
-    return value instanceof apache_arrow_1.List || apache_arrow_1.DataType.isList(value);
-}
-exports.isList = isList;
-function isStruct(value) {
-    return value instanceof apache_arrow_1.Struct || apache_arrow_1.DataType.isStruct(value);
-}
-exports.isStruct = isStruct;
-function isUnion(value) {
-    return value instanceof apache_arrow_1.Struct || apache_arrow_1.DataType.isUnion(value);
-}
-exports.isUnion = isUnion;
-function isFixedSizeBinary(value) {
-    return value instanceof apache_arrow_1.FixedSizeBinary || apache_arrow_1.DataType.isFixedSizeBinary(value);
-}
-exports.isFixedSizeBinary = isFixedSizeBinary;
-function isFixedSizeList(value) {
-    return value instanceof apache_arrow_1.FixedSizeList || apache_arrow_1.DataType.isFixedSizeList(value);
-}
-exports.isFixedSizeList = isFixedSizeList;
-/*
- * Options to control how a column should be converted to a vector array
- */
-class VectorColumnOptions {
-    /** Vector column type. */
-    type = new apache_arrow_1.Float32();
-    constructor(values) {
-        Object.assign(this, values);
-    }
-}
-exports.VectorColumnOptions = VectorColumnOptions;
-/** Options to control the makeArrowTable call. */
-class MakeArrowTableOptions {
-    /*
-     * Schema of the data.
-     *
-     * If this is not provided then the data type will be inferred from the
-     * JS type.  Integer numbers will become int64, floating point numbers
-     * will become float64 and arrays will become variable sized lists with
-     * the data type inferred from the first element in the array.
-     *
-     * The schema must be specified if there are no records (e.g. to make
-     * an empty table)
-     */
-    schema;
-    /*
-     * Mapping from vector column name to expected type
-     *
-     * Lance expects vector columns to be fixed size list arrays (i.e. tensors)
-     * However, `makeArrowTable` will not infer this by default (it creates
-     * variable size list arrays).  This field can be used to indicate that a column
-     * should be treated as a vector column and converted to a fixed size list.
-     *
-     * The keys should be the names of the vector columns.  The value specifies the
-     * expected data type of the vector columns.
-     *
-     * If `schema` is provided then this field is ignored.
-     *
-     * By default, the column named "vector" will be assumed to be a float32
-     * vector column.
-     */
-    vectorColumns = {
-        vector: new VectorColumnOptions(),
-    };
-    embeddings;
-    embeddingFunction;
-    /**
-     * If true then string columns will be encoded with dictionary encoding
-     *
-     * Set this to true if your string columns tend to repeat the same values
-     * often.  For more precise control use the `schema` property to specify the
-     * data type for individual columns.
-     *
-     * If `schema` is provided then this property is ignored.
-     */
-    dictionaryEncodeStrings = false;
-    constructor(values) {
-        Object.assign(this, values);
-    }
-}
-exports.MakeArrowTableOptions = MakeArrowTableOptions;
-/**
- * An enhanced version of the {@link makeTable} function from Apache Arrow
- * that supports nested fields and embeddings columns.
- *
- * (typically you do not need to call this function.  It will be called automatically
- * when creating a table or adding data to it)
- *
- * This function converts an array of Record<String, any> (row-major JS objects)
- * to an Arrow Table (a columnar structure)
- *
- * Note that it currently does not support nulls.
- *
- * If a schema is provided then it will be used to determine the resulting array
- * types.  Fields will also be reordered to fit the order defined by the schema.
- *
- * If a schema is not provided then the types will be inferred and the field order
- * will be controlled by the order of properties in the first record.  If a type
- * is inferred it will always be nullable.
- *
- * If the input is empty then a schema must be provided to create an empty table.
- *
- * When a schema is not specified then data types will be inferred.  The inference
- * rules are as follows:
- *
- *  - boolean => Bool
- *  - number => Float64
- *  - String => Utf8
- *  - Buffer => Binary
- *  - Record<String, any> => Struct
- *  - Array<any> => List
- * @example
- * import { fromTableToBuffer, makeArrowTable } from "../arrow";
- * import { Field, FixedSizeList, Float16, Float32, Int32, Schema } from "apache-arrow";
- *
- * const schema = new Schema([
- *   new Field("a", new Int32()),
- *   new Field("b", new Float32()),
- *   new Field("c", new FixedSizeList(3, new Field("item", new Float16()))),
- *  ]);
- *  const table = makeArrowTable([
- *    { a: 1, b: 2, c: [1, 2, 3] },
- *    { a: 4, b: 5, c: [4, 5, 6] },
- *    { a: 7, b: 8, c: [7, 8, 9] },
- *  ], { schema });
- * ```
- *
- * By default it assumes that the column named `vector` is a vector column
- * and it will be converted into a fixed size list array of type float32.
- * The `vectorColumns` option can be used to support other vector column
- * names and data types.
- *
- * ```ts
- *
- * const schema = new Schema([
-    new Field("a", new Float64()),
-    new Field("b", new Float64()),
-    new Field(
-      "vector",
-      new FixedSizeList(3, new Field("item", new Float32()))
-    ),
-  ]);
-  const table = makeArrowTable([
-    { a: 1, b: 2, vector: [1, 2, 3] },
-    { a: 4, b: 5, vector: [4, 5, 6] },
-    { a: 7, b: 8, vector: [7, 8, 9] },
-  ]);
-  assert.deepEqual(table.schema, schema);
- * ```
- *
- * You can specify the vector column types and names using the options as well
- *
- * ```typescript
- *
- * const schema = new Schema([
-    new Field('a', new Float64()),
-    new Field('b', new Float64()),
-    new Field('vec1', new FixedSizeList(3, new Field('item', new Float16()))),
-    new Field('vec2', new FixedSizeList(3, new Field('item', new Float16())))
-  ]);
- * const table = makeArrowTable([
-    { a: 1, b: 2, vec1: [1, 2, 3], vec2: [2, 4, 6] },
-    { a: 4, b: 5, vec1: [4, 5, 6], vec2: [8, 10, 12] },
-    { a: 7, b: 8, vec1: [7, 8, 9], vec2: [14, 16, 18] }
-  ], {
-    vectorColumns: {
-      vec1: { type: new Float16() },
-      vec2: { type: new Float16() }
-    }
-  }
- * assert.deepEqual(table.schema, schema)
- * ```
- */
-function makeArrowTable(data, options, metadata) {
-    if (data.length === 0 &&
-        (options?.schema === undefined || options?.schema === null)) {
-        throw new Error("At least one record or a schema needs to be provided");
-    }
-    const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
-    if (opt.schema !== undefined && opt.schema !== null) {
-        opt.schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
-        opt.schema = validateSchemaEmbeddings(opt.schema, data, options?.embeddingFunction);
-    }
-    const columns = {};
-    // TODO: sample dataset to find missing columns
-    // Prefer the field ordering of the schema, if present
-    const columnNames = opt.schema != null ? opt.schema.names : Object.keys(data[0]);
-    for (const colName of columnNames) {
-        if (data.length !== 0 &&
-            !Object.prototype.hasOwnProperty.call(data[0], colName)) {
-            // The field is present in the schema, but not in the data, skip it
-            continue;
-        }
-        // Extract a single column from the records (transpose from row-major to col-major)
-        let values = data.map((datum) => datum[colName]);
-        // By default (type === undefined) arrow will infer the type from the JS type
-        let type;
-        if (opt.schema !== undefined) {
-            // If there is a schema provided, then use that for the type instead
-            type = opt.schema?.fields.filter((f) => f.name === colName)[0]?.type;
-            if (apache_arrow_1.DataType.isInt(type) && type.bitWidth === 64) {
-                // wrap in BigInt to avoid bug: https://github.com/apache/arrow/issues/40051
-                values = values.map((v) => {
-                    if (v === null) {
-                        return v;
-                    }
-                    if (typeof v === "bigint") {
-                        return v;
-                    }
-                    if (typeof v === "number") {
-                        return BigInt(v);
-                    }
-                    throw new Error(`Expected BigInt or number for column ${colName}, got ${typeof v}`);
-                });
-            }
-        }
-        else {
-            // Otherwise, check to see if this column is one of the vector columns
-            // defined by opt.vectorColumns and, if so, use the fixed size list type
-            const vectorColumnOptions = opt.vectorColumns[colName];
-            if (vectorColumnOptions !== undefined) {
-                const firstNonNullValue = values.find((v) => v !== null);
-                if (Array.isArray(firstNonNullValue)) {
-                    type = newVectorType(firstNonNullValue.length, vectorColumnOptions.type);
-                }
-                else {
-                    throw new Error(`Column ${colName} is expected to be a vector column but first non-null value is not an array.  Could not determine size of vector column`);
-                }
-            }
-        }
-        try {
-            // Convert an Array of JS values to an arrow vector
-            columns[colName] = makeVector(values, type, opt.dictionaryEncodeStrings);
-        }
-        catch (error) {
-            // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-            throw Error(`Could not convert column "${colName}" to Arrow: ${error}`);
-        }
-    }
-    if (opt.schema != null) {
-        // `new ArrowTable(columns)` infers a schema which may sometimes have
-        // incorrect nullability (it assumes nullable=true always)
-        //
-        // `new ArrowTable(schema, columns)` will also fail because it will create a
-        // batch with an inferred schema and then complain that the batch schema
-        // does not match the provided schema.
-        //
-        // To work around this we first create a table with the wrong schema and
-        // then patch the schema of the batches so we can use
-        // `new ArrowTable(schema, batches)` which does not do any schema inference
-        const firstTable = new apache_arrow_1.Table(columns);
-        const batchesFixed = firstTable.batches.map((batch) => new apache_arrow_1.RecordBatch(opt.schema, batch.data));
-        let schema;
-        if (metadata !== undefined) {
-            let schemaMetadata = opt.schema.metadata;
-            if (schemaMetadata.size === 0) {
-                schemaMetadata = metadata;
-            }
-            else {
-                for (const [key, entry] of schemaMetadata.entries()) {
-                    schemaMetadata.set(key, entry);
-                }
-            }
-            schema = new apache_arrow_1.Schema(opt.schema.fields, schemaMetadata);
-        }
-        else {
-            schema = opt.schema;
-        }
-        return new apache_arrow_1.Table(schema, batchesFixed);
-    }
-    const tbl = new apache_arrow_1.Table(columns);
-    if (metadata !== undefined) {
-        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-        tbl.schema.metadata = metadata;
-    }
-    return tbl;
-}
-exports.makeArrowTable = makeArrowTable;
-/**
- * Create an empty Arrow table with the provided schema
- */
-function makeEmptyTable(schema, metadata) {
-    return makeArrowTable([], { schema }, metadata);
-}
-exports.makeEmptyTable = makeEmptyTable;
-/**
- * Helper function to convert Array<Array<any>> to a variable sized list array
- */
-// @ts-expect-error (Vector<unknown> is not assignable to Vector<any>)
-function makeListVector(lists) {
-    if (lists.length === 0 || lists[0].length === 0) {
-        throw Error("Cannot infer list vector from empty array or empty list");
-    }
-    const sampleList = lists[0];
-    // biome-ignore lint/suspicious/noExplicitAny: skip
-    let inferredType;
-    try {
-        const sampleVector = makeVector(sampleList);
-        inferredType = sampleVector.type;
-    }
-    catch (error) {
-        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
-        throw Error(`Cannot infer list vector.  Cannot infer inner type: ${error}`);
-    }
-    const listBuilder = (0, apache_arrow_1.makeBuilder)({
-        type: new apache_arrow_1.List(new apache_arrow_1.Field("item", inferredType, true)),
-    });
-    for (const list of lists) {
-        listBuilder.append(list);
-    }
-    return listBuilder.finish().toVector();
-}
-/** Helper function to convert an Array of JS values to an Arrow Vector */
-function makeVector(values, type, stringAsDictionary) {
-    if (type !== undefined) {
-        // No need for inference, let Arrow create it
-        return (0, apache_arrow_1.vectorFromArray)(values, type);
-    }
-    if (values.length === 0) {
-        throw Error("makeVector requires at least one value or the type must be specfied");
-    }
-    const sampleValue = values.find((val) => val !== null && val !== undefined);
-    if (sampleValue === undefined) {
-        throw Error("makeVector cannot infer the type if all values are null or undefined");
-    }
-    if (Array.isArray(sampleValue)) {
-        // Default Arrow inference doesn't handle list types
-        return makeListVector(values);
-    }
-    else if (Buffer.isBuffer(sampleValue)) {
-        // Default Arrow inference doesn't handle Buffer
-        return (0, apache_arrow_1.vectorFromArray)(values, new apache_arrow_1.Binary());
-    }
-    else if (!(stringAsDictionary ?? false) &&
-        (typeof sampleValue === "string" || sampleValue instanceof String)) {
-        // If the type is string then don't use Arrow's default inference unless dictionaries are requested
-        // because it will always use dictionary encoding for strings
-        return (0, apache_arrow_1.vectorFromArray)(values, new apache_arrow_1.Utf8());
-    }
-    else {
-        // Convert a JS array of values to an arrow vector
-        return (0, apache_arrow_1.vectorFromArray)(values);
-    }
-}
-/** Helper function to apply embeddings from metadata to an input table */
-async function applyEmbeddingsFromMetadata(table, schema) {
-    const registry = (0, registry_1.getRegistry)();
-    const functions = registry.parseFunctions(schema.metadata);
-    const columns = Object.fromEntries(table.schema.fields.map((field) => [
-        field.name,
-        table.getChild(field.name),
-    ]));
-    for (const functionEntry of functions.values()) {
-        const sourceColumn = columns[functionEntry.sourceColumn];
-        const destColumn = functionEntry.vectorColumn ?? "vector";
-        if (sourceColumn === undefined) {
-            throw new Error(`Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`);
-        }
-        if (columns[destColumn] !== undefined) {
-            throw new Error(`Attempt to apply embeddings to table failed because column ${destColumn} already existed`);
-        }
-        if (table.batches.length > 1) {
-            throw new Error("Internal error: `makeArrowTable` unexpectedly created a table with more than one batch");
-        }
-        const values = sourceColumn.toArray();
-        const vectors = await functionEntry.function.computeSourceEmbeddings(values);
-        if (vectors.length !== values.length) {
-            throw new Error("Embedding function did not return an embedding for each input element");
-        }
-        let destType;
-        const dtype = schema.fields.find((f) => f.name === destColumn).type;
-        if (isFixedSizeList(dtype)) {
-            destType = (0, sanitize_1.sanitizeType)(dtype);
-        }
-        else {
-            throw new Error("Expected FixedSizeList as datatype for vector field, instead got: " +
-                dtype);
-        }
-        const vector = makeVector(vectors, destType);
-        columns[destColumn] = vector;
-    }
-    const newTable = new apache_arrow_1.Table(columns);
-    return alignTable(newTable, schema);
-}
-/** Helper function to apply embeddings to an input table */
-async function applyEmbeddings(table, embeddings, schema) {
-    if (schema !== undefined && schema !== null) {
-        schema = (0, sanitize_1.sanitizeSchema)(schema);
-    }
-    if (schema?.metadata.has("embedding_functions")) {
-        return applyEmbeddingsFromMetadata(table, schema);
-    }
-    else if (embeddings == null || embeddings === undefined) {
-        return table;
-    }
-    // Convert from ArrowTable to Record<String, Vector>
-    const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
-        const name = table.schema.fields[idx].name;
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        const vec = table.getChildAt(idx);
-        return [name, vec];
-    });
-    const newColumns = Object.fromEntries(colEntries);
-    const sourceColumn = newColumns[embeddings.sourceColumn];
-    const destColumn = embeddings.vectorColumn ?? "vector";
-    const innerDestType = embeddings.function.embeddingDataType() ?? new apache_arrow_1.Float32();
-    if (sourceColumn === undefined) {
-        throw new Error(`Cannot apply embedding function because the source column '${embeddings.sourceColumn}' was not present in the data`);
-    }
-    if (table.numRows === 0) {
-        if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
-            // We have an empty table and it already has the embedding column so no work needs to be done
-            // Note: we don't return an error like we did below because this is a common occurrence.  For example,
-            // if we call convertToTable with 0 records and a schema that includes the embedding
-            return table;
-        }
-        const dimensions = embeddings.function.ndims();
-        if (dimensions !== undefined) {
-            const destType = newVectorType(dimensions, innerDestType);
-            newColumns[destColumn] = makeVector([], destType);
-        }
-        else if (schema != null) {
-            const destField = schema.fields.find((f) => f.name === destColumn);
-            if (destField != null) {
-                newColumns[destColumn] = makeVector([], destField.type);
-            }
-            else {
-                throw new Error(`Attempt to apply embeddings to an empty table failed because schema was missing embedding column '${destColumn}'`);
-            }
-        }
-        else {
-            throw new Error("Attempt to apply embeddings to an empty table when the embeddings function does not specify `embeddingDimension`");
-        }
-    }
-    else {
-        if (Object.prototype.hasOwnProperty.call(newColumns, destColumn)) {
-            throw new Error(`Attempt to apply embeddings to table failed because column ${destColumn} already existed`);
-        }
-        if (table.batches.length > 1) {
-            throw new Error("Internal error: `makeArrowTable` unexpectedly created a table with more than one batch");
-        }
-        const values = sourceColumn.toArray();
-        const vectors = await embeddings.function.computeSourceEmbeddings(values);
-        if (vectors.length !== values.length) {
-            throw new Error("Embedding function did not return an embedding for each input element");
-        }
-        const destType = newVectorType(vectors[0].length, innerDestType);
-        newColumns[destColumn] = makeVector(vectors, destType);
-    }
-    const newTable = new apache_arrow_1.Table(newColumns);
-    if (schema != null) {
-        if (schema.fields.find((f) => f.name === destColumn) === undefined) {
-            throw new Error(`When using embedding functions and specifying a schema the schema should include the embedding column but the column ${destColumn} was missing`);
-        }
-        return alignTable(newTable, schema);
-    }
-    return newTable;
-}
-/**
- * Convert an Array of records into an Arrow Table, optionally applying an
- * embeddings function to it.
- *
- * This function calls `makeArrowTable` first to create the Arrow Table.
- * Any provided `makeTableOptions` (e.g. a schema) will be passed on to
- * that call.
- *
- * The embedding function will be passed a column of values (based on the
- * `sourceColumn` of the embedding function) and expects to receive back
- * number[][] which will be converted into a fixed size list column.  By
- * default this will be a fixed size list of Float32 but that can be
- * customized by the `embeddingDataType` property of the embedding function.
- *
- * If a schema is provided in `makeTableOptions` then it should include the
- * embedding columns.  If no schema is provded then embedding columns will
- * be placed at the end of the table, after all of the input columns.
- */
-async function convertToTable(data, embeddings, makeTableOptions) {
-    const table = makeArrowTable(data, makeTableOptions);
-    return await applyEmbeddings(table, embeddings, makeTableOptions?.schema);
-}
-exports.convertToTable = convertToTable;
-/** Creates the Arrow Type for a Vector column with dimension `dim` */
-function newVectorType(dim, innerType) {
-    // in Lance we always default to have the elements nullable, so we need to set it to true
-    // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
-    const children = new apache_arrow_1.Field("item", (0, sanitize_1.sanitizeType)(innerType), true);
-    return new apache_arrow_1.FixedSizeList(dim, children);
-}
-exports.newVectorType = newVectorType;
-/**
- * Serialize an Array of records into a buffer using the Arrow IPC File serialization
- *
- * This function will call `convertToTable` and pass on `embeddings` and `schema`
- *
- * `schema` is required if data is empty
- */
-async function fromRecordsToBuffer(data, embeddings, schema) {
-    if (schema !== undefined && schema !== null) {
-        schema = (0, sanitize_1.sanitizeSchema)(schema);
-    }
-    const table = await convertToTable(data, embeddings, { schema });
-    const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(table);
-    return Buffer.from(await writer.toUint8Array());
-}
-exports.fromRecordsToBuffer = fromRecordsToBuffer;
-/**
- * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
- *
- * This function will call `convertToTable` and pass on `embeddings` and `schema`
- *
- * `schema` is required if data is empty
- */
-async function fromRecordsToStreamBuffer(data, embeddings, schema) {
-    if (schema !== undefined && schema !== null) {
-        schema = (0, sanitize_1.sanitizeSchema)(schema);
-    }
-    const table = await convertToTable(data, embeddings, { schema });
-    const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(table);
-    return Buffer.from(await writer.toUint8Array());
-}
-exports.fromRecordsToStreamBuffer = fromRecordsToStreamBuffer;
-/**
- * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
- *
- * This function will apply `embeddings` to the table in a manner similar to
- * `convertToTable`.
- *
- * `schema` is required if the table is empty
- */
-async function fromTableToBuffer(table, embeddings, schema) {
-    if (schema !== undefined && schema !== null) {
-        schema = (0, sanitize_1.sanitizeSchema)(schema);
-    }
-    const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
-    const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(tableWithEmbeddings);
-    return Buffer.from(await writer.toUint8Array());
-}
-exports.fromTableToBuffer = fromTableToBuffer;
-/**
- * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
- *
- * This function will apply `embeddings` to the table in a manner similar to
- * `convertToTable`.
- *
- * `schema` is required if the table is empty
- */
-async function fromDataToBuffer(data, embeddings, schema) {
-    if (schema !== undefined && schema !== null) {
-        schema = (0, sanitize_1.sanitizeSchema)(schema);
-    }
-    if (isArrowTable(data)) {
-        return fromTableToBuffer((0, sanitize_1.sanitizeTable)(data), embeddings, schema);
-    }
-    else {
-        const table = await convertToTable(data, embeddings, { schema });
-        return fromTableToBuffer(table);
-    }
-}
-exports.fromDataToBuffer = fromDataToBuffer;
-/**
- * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
- *
- * This function will apply `embeddings` to the table in a manner similar to
- * `convertToTable`.
- *
- * `schema` is required if the table is empty
- */
-async function fromTableToStreamBuffer(table, embeddings, schema) {
-    const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
-    const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
-    return Buffer.from(await writer.toUint8Array());
-}
-exports.fromTableToStreamBuffer = fromTableToStreamBuffer;
-/**
- * Reorder the columns in `batch` so that they agree with the field order in `schema`
- */
-function alignBatch(batch, schema) {
-    const alignedChildren = [];
-    for (const field of schema.fields) {
-        const indexInBatch = batch.schema.fields?.findIndex((f) => f.name === field.name);
-        if (indexInBatch < 0) {
-            throw new Error(`The column ${field.name} was not found in the Arrow Table`);
-        }
-        alignedChildren.push(batch.data.children[indexInBatch]);
-    }
-    const newData = (0, apache_arrow_1.makeData)({
-        type: new apache_arrow_1.Struct(schema.fields),
-        length: batch.numRows,
-        nullCount: batch.nullCount,
-        children: alignedChildren,
-    });
-    return new apache_arrow_1.RecordBatch(schema, newData);
-}
-/**
- * Reorder the columns in `table` so that they agree with the field order in `schema`
- */
-function alignTable(table, schema) {
-    const alignedBatches = table.batches.map((batch) => alignBatch(batch, schema));
-    return new apache_arrow_1.Table(schema, alignedBatches);
-}
-/**
- * Create an empty table with the given schema
- */
-function createEmptyTable(schema) {
-    return new apache_arrow_1.Table((0, sanitize_1.sanitizeSchema)(schema));
-}
-exports.createEmptyTable = createEmptyTable;
-function validateSchemaEmbeddings(schema, data, embeddings) {
-    const fields = [];
-    const missingEmbeddingFields = [];
-    // First we check if the field is a `FixedSizeList`
-    // Then we check if the data contains the field
-    // if it does not, we add it to the list of missing embedding fields
-    // Finally, we check if those missing embedding fields are `this._embeddings`
-    // if they are not, we throw an error
-    for (let field of schema.fields) {
-        if (isFixedSizeList(field.type)) {
-            field = (0, sanitize_1.sanitizeField)(field);
-            if (data.length !== 0 && data?.[0]?.[field.name] === undefined) {
-                if (schema.metadata.has("embedding_functions")) {
-                    const embeddings = JSON.parse(schema.metadata.get("embedding_functions"));
-                    if (
-                    // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
-                    embeddings.find((f) => f["vectorColumn"] === field.name) ===
-                        undefined) {
-                        missingEmbeddingFields.push(field);
-                    }
-                }
-                else {
-                    missingEmbeddingFields.push(field);
-                }
-            }
-            else {
-                fields.push(field);
-            }
-        }
-        else {
-            fields.push(field);
-        }
-    }
-    if (missingEmbeddingFields.length > 0 && embeddings === undefined) {
-        throw new Error(`Table has embeddings: "${missingEmbeddingFields
-            .map((f) => f.name)
-            .join(",")}", but no embedding function was provided`);
-    }
-    return new apache_arrow_1.Schema(fields, schema.metadata);
-}