@lancedb/lancedb 0.4.20 → 0.5.1

package/README.md

@@ -43,29 +43,20 @@ npm run test
 
 ### Running lint / format
 
-LanceDb uses eslint for linting. VSCode does not need any plugins to use eslint. However, it
-may need some additional configuration. Make sure that eslint.experimental.useFlatConfig is
-set to true. Also, if your vscode root folder is the repo root then you will need to set
-the eslint.workingDirectories to ["nodejs"]. To manually lint your code you can run:
+LanceDb uses [biome](https://biomejs.dev/) for linting and formatting. if you are using VSCode you will need to install the official [Biome](https://marketplace.visualstudio.com/items?itemName=biomejs.biome) extension.
+To manually lint your code you can run:
 
 ```sh
 npm run lint
 ```
 
-LanceDb uses prettier for formatting. If you are using VSCode you will need to install the
-"Prettier - Code formatter" extension. You should then configure it to be the default formatter
-for typescript and you should enable format on save. To manually check your code's format you
-can run:
+to automatically fix all fixable issues:
 
 ```sh
-npm run chkformat
+npm run lint-fix
 ```
 
-If you need to manually format your code you can run:
-
-```sh
-npx prettier --write .
-```
+If you do not have your workspace root set to the `nodejs` directory, unfortunately the extension will not work. You can still run the linting and formatting commands manually.
 
 ### Generating docs
 

package/biome.json

@@ -0,0 +1,142 @@
+{
+  "$schema": "https://biomejs.dev/schemas/1.7.3/schema.json",
+  "organizeImports": {
+    "enabled": true
+  },
+  "files": {
+    "ignore": [
+      "**/dist/**/*",
+      "**/native.js",
+      "**/native.d.ts",
+      "**/npm/**/*",
+      "**/.vscode/**"
+    ]
+  },
+  "formatter": {
+    "indentStyle": "space"
+  },
+  "linter": {
+    "enabled": true,
+    "rules": {
+      "recommended": false,
+      "complexity": {
+        "noBannedTypes": "error",
+        "noExtraBooleanCast": "error",
+        "noMultipleSpacesInRegularExpressionLiterals": "error",
+        "noUselessCatch": "error",
+        "noUselessThisAlias": "error",
+        "noUselessTypeConstraint": "error",
+        "noWith": "error"
+      },
+      "correctness": {
+        "noConstAssign": "error",
+        "noConstantCondition": "error",
+        "noEmptyCharacterClassInRegex": "error",
+        "noEmptyPattern": "error",
+        "noGlobalObjectCalls": "error",
+        "noInnerDeclarations": "error",
+        "noInvalidConstructorSuper": "error",
+        "noNewSymbol": "error",
+        "noNonoctalDecimalEscape": "error",
+        "noPrecisionLoss": "error",
+        "noSelfAssign": "error",
+        "noSetterReturn": "error",
+        "noSwitchDeclarations": "error",
+        "noUndeclaredVariables": "error",
+        "noUnreachable": "error",
+        "noUnreachableSuper": "error",
+        "noUnsafeFinally": "error",
+        "noUnsafeOptionalChaining": "error",
+        "noUnusedLabels": "error",
+        "noUnusedVariables": "warn",
+        "useIsNan": "error",
+        "useValidForDirection": "error",
+        "useYield": "error"
+      },
+      "style": {
+        "noNamespace": "error",
+        "useAsConstAssertion": "error",
+        "useBlockStatements": "off",
+        "useNamingConvention": {
+          "level": "error",
+          "options": {
+            "strictCase": false
+          }
+        }
+      },
+      "suspicious": {
+        "noAssignInExpressions": "error",
+        "noAsyncPromiseExecutor": "error",
+        "noCatchAssign": "error",
+        "noClassAssign": "error",
+        "noCompareNegZero": "error",
+        "noControlCharactersInRegex": "error",
+        "noDebugger": "error",
+        "noDuplicateCase": "error",
+        "noDuplicateClassMembers": "error",
+        "noDuplicateObjectKeys": "error",
+        "noDuplicateParameters": "error",
+        "noEmptyBlockStatements": "error",
+        "noExplicitAny": "error",
+        "noExtraNonNullAssertion": "error",
+        "noFallthroughSwitchClause": "error",
+        "noFunctionAssign": "error",
+        "noGlobalAssign": "error",
+        "noImportAssign": "error",
+        "noMisleadingCharacterClass": "error",
+        "noMisleadingInstantiator": "error",
+        "noPrototypeBuiltins": "error",
+        "noRedeclare": "error",
+        "noShadowRestrictedNames": "error",
+        "noUnsafeDeclarationMerging": "error",
+        "noUnsafeNegation": "error",
+        "useGetterReturn": "error",
+        "useValidTypeof": "error"
+      }
+    },
+    "ignore": ["**/dist/**/*", "**/native.js", "**/native.d.ts"]
+  },
+  "javascript": {
+    "globals": []
+  },
+  "overrides": [
+    {
+      "include": [
+        "**/*.ts",
+        "**/*.tsx",
+        "**/*.mts",
+        "**/*.cts",
+        "__test__/*.test.ts"
+      ],
+      "linter": {
+        "rules": {
+          "correctness": {
+            "noConstAssign": "off",
+            "noGlobalObjectCalls": "off",
+            "noInvalidConstructorSuper": "off",
+            "noNewSymbol": "off",
+            "noSetterReturn": "off",
+            "noUndeclaredVariables": "off",
+            "noUnreachable": "off",
+            "noUnreachableSuper": "off"
+          },
+          "style": {
+            "noArguments": "error",
+            "noVar": "error",
+            "useConst": "error"
+          },
+          "suspicious": {
+            "noDuplicateClassMembers": "off",
+            "noDuplicateObjectKeys": "off",
+            "noDuplicateParameters": "off",
+            "noFunctionAssign": "off",
+            "noImportAssign": "off",
+            "noRedeclare": "off",
+            "noUnsafeNegation": "off",
+            "useGetterReturn": "off"
+          }
+        }
+      }
+    }
+  ]
+}

package/dist/arrow.d.ts

@@ -1,6 +1,29 @@
 /// <reference types="node" />
-import { type Schema, Table as ArrowTable, type Float } from "apache-arrow";
+import { Table as ArrowTable, Binary, DataType, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, Schema, Struct, Utf8 } from "apache-arrow";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
+import { EmbeddingFunctionConfig } from "./embedding/registry";
+export * from "apache-arrow";
+export declare function isArrowTable(value: object): value is ArrowTable;
+export declare function isDataType(value: unknown): value is DataType;
+export declare function isNull(value: unknown): value is Null;
+export declare function isInt(value: unknown): value is Int;
+export declare function isFloat(value: unknown): value is Float;
+export declare function isBinary(value: unknown): value is Binary;
+export declare function isLargeBinary(value: unknown): value is LargeBinary;
+export declare function isUtf8(value: unknown): value is Utf8;
+export declare function isLargeUtf8(value: unknown): value is Utf8;
+export declare function isBool(value: unknown): value is Utf8;
+export declare function isDecimal(value: unknown): value is Utf8;
+export declare function isDate(value: unknown): value is Utf8;
+export declare function isTime(value: unknown): value is Utf8;
+export declare function isTimestamp(value: unknown): value is Utf8;
+export declare function isInterval(value: unknown): value is Utf8;
+export declare function isDuration(value: unknown): value is Utf8;
+export declare function isList(value: unknown): value is List;
+export declare function isStruct(value: unknown): value is Struct;
+export declare function isUnion(value: unknown): value is Struct;
+export declare function isFixedSizeBinary(value: unknown): value is FixedSizeBinary;
+export declare function isFixedSizeList(value: unknown): value is FixedSizeList;
 /** Data type accepted by NodeJS SDK */
 export type Data = Record<string, unknown>[] | ArrowTable;
 export declare class VectorColumnOptions {
@@ -12,6 +35,7 @@ export declare class VectorColumnOptions {
 export declare class MakeArrowTableOptions {
     schema?: Schema;
     vectorColumns: Record<string, VectorColumnOptions>;
+    embeddings?: EmbeddingFunction<unknown>;
     /**
      * If true then string columns will be encoded with dictionary encoding
      *
@@ -116,11 +140,11 @@ export declare class MakeArrowTableOptions {
  * assert.deepEqual(table.schema, schema)
  * ```
  */
-export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>): ArrowTable;
+export declare function makeArrowTable(data: Array<Record<string, unknown>>, options?: Partial<MakeArrowTableOptions>, metadata?: Map<string, string>): ArrowTable;
 /**
  * Create an empty Arrow table with the provided schema
  */
-export declare function makeEmptyTable(schema: Schema): ArrowTable;
+export declare function makeEmptyTable(schema: Schema, metadata?: Map<string, string>): ArrowTable;
 /**
  * Convert an Array of records into an Arrow Table, optionally applying an
  * embeddings function to it.
@@ -139,7 +163,9 @@ export declare function makeEmptyTable(schema: Schema): ArrowTable;
  * embedding columns.  If no schema is provded then embedding columns will
  * be placed at the end of the table, after all of the input columns.
  */
-export declare function convertToTable<T>(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
+export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
+/** Creates the Arrow Type for a Vector column with dimension `dim` */
+export declare function newVectorType<T extends Float>(dim: number, innerType: T): FixedSizeList<T>;
 /**
  * Serialize an Array of records into a buffer using the Arrow IPC File serialization
  *
@@ -147,7 +173,7 @@ export declare function convertToTable<T>(data: Array<Record<string, unknown>>,
  *
  * `schema` is required if data is empty
  */
-export declare function fromRecordsToBuffer<T>(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromRecordsToBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Array of records into a buffer using the Arrow IPC Stream serialization
  *
@@ -155,7 +181,7 @@ export declare function fromRecordsToBuffer<T>(data: Array<Record<string, unknow
  *
  * `schema` is required if data is empty
  */
-export declare function fromRecordsToStreamBuffer<T>(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromRecordsToStreamBuffer(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
  *
@@ -164,7 +190,7 @@ export declare function fromRecordsToStreamBuffer<T>(data: Array<Record<string,
  *
  * `schema` is required if the table is empty
  */
-export declare function fromTableToBuffer<T>(table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromTableToBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC File serialization
  *
@@ -173,7 +199,7 @@ export declare function fromTableToBuffer<T>(table: ArrowTable, embeddings?: Emb
  *
  * `schema` is required if the table is empty
  */
-export declare function fromDataToBuffer<T>(data: Data, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromDataToBuffer(data: Data, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Serialize an Arrow Table into a buffer using the Arrow IPC Stream serialization
  *
@@ -182,7 +208,7 @@ export declare function fromDataToBuffer<T>(data: Data, embeddings?: EmbeddingFu
  *
  * `schema` is required if the table is empty
  */
-export declare function fromTableToStreamBuffer<T>(table: ArrowTable, embeddings?: EmbeddingFunction<T>, schema?: Schema): Promise<Buffer>;
+export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?: EmbeddingFunctionConfig, schema?: Schema): Promise<Buffer>;
 /**
  * Create an empty table with the given schema
  */

package/dist/arrow.js

@@ -12,10 +12,133 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createEmptyTable = exports.fromTableToStreamBuffer = exports.fromDataToBuffer = exports.fromTableToBuffer = exports.fromRecordsToStreamBuffer = exports.fromRecordsToBuffer = exports.convertToTable = exports.makeEmptyTable = exports.makeArrowTable = exports.MakeArrowTableOptions = exports.VectorColumnOptions = void 0;
+exports.createEmptyTable = exports.fromTableToStreamBuffer = exports.fromDataToBuffer = exports.fromTableToBuffer = exports.fromRecordsToStreamBuffer = exports.fromRecordsToBuffer = exports.newVectorType = exports.convertToTable = exports.makeEmptyTable = exports.makeArrowTable = exports.MakeArrowTableOptions = exports.VectorColumnOptions = exports.isFixedSizeList = exports.isFixedSizeBinary = exports.isUnion = exports.isStruct = exports.isList = exports.isDuration = exports.isInterval = exports.isTimestamp = exports.isTime = exports.isDate = exports.isDecimal = exports.isBool = exports.isLargeUtf8 = exports.isUtf8 = exports.isLargeBinary = exports.isBinary = exports.isFloat = exports.isInt = exports.isNull = exports.isDataType = exports.isArrowTable = void 0;
 const apache_arrow_1 = require("apache-arrow");
+const registry_1 = require("./embedding/registry");
 const sanitize_1 = require("./sanitize");
+__exportStar(require("apache-arrow"), exports);
+function isArrowTable(value) {
+    if (value instanceof apache_arrow_1.Table)
+        return true;
+    return "schema" in value && "batches" in value;
+}
+exports.isArrowTable = isArrowTable;
+function isDataType(value) {
+    return (value instanceof apache_arrow_1.DataType ||
+        apache_arrow_1.DataType.isNull(value) ||
+        apache_arrow_1.DataType.isInt(value) ||
+        apache_arrow_1.DataType.isFloat(value) ||
+        apache_arrow_1.DataType.isBinary(value) ||
+        apache_arrow_1.DataType.isLargeBinary(value) ||
+        apache_arrow_1.DataType.isUtf8(value) ||
+        apache_arrow_1.DataType.isLargeUtf8(value) ||
+        apache_arrow_1.DataType.isBool(value) ||
+        apache_arrow_1.DataType.isDecimal(value) ||
+        apache_arrow_1.DataType.isDate(value) ||
+        apache_arrow_1.DataType.isTime(value) ||
+        apache_arrow_1.DataType.isTimestamp(value) ||
+        apache_arrow_1.DataType.isInterval(value) ||
+        apache_arrow_1.DataType.isDuration(value) ||
+        apache_arrow_1.DataType.isList(value) ||
+        apache_arrow_1.DataType.isStruct(value) ||
+        apache_arrow_1.DataType.isUnion(value) ||
+        apache_arrow_1.DataType.isFixedSizeBinary(value) ||
+        apache_arrow_1.DataType.isFixedSizeList(value) ||
+        apache_arrow_1.DataType.isMap(value) ||
+        apache_arrow_1.DataType.isDictionary(value));
+}
+exports.isDataType = isDataType;
+function isNull(value) {
+    return value instanceof apache_arrow_1.Null || apache_arrow_1.DataType.isNull(value);
+}
+exports.isNull = isNull;
+function isInt(value) {
+    return value instanceof apache_arrow_1.Int || apache_arrow_1.DataType.isInt(value);
+}
+exports.isInt = isInt;
+function isFloat(value) {
+    return value instanceof apache_arrow_1.Float || apache_arrow_1.DataType.isFloat(value);
+}
+exports.isFloat = isFloat;
+function isBinary(value) {
+    return value instanceof apache_arrow_1.Binary || apache_arrow_1.DataType.isBinary(value);
+}
+exports.isBinary = isBinary;
+function isLargeBinary(value) {
+    return value instanceof apache_arrow_1.LargeBinary || apache_arrow_1.DataType.isLargeBinary(value);
+}
+exports.isLargeBinary = isLargeBinary;
+function isUtf8(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isUtf8(value);
+}
+exports.isUtf8 = isUtf8;
+function isLargeUtf8(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isLargeUtf8(value);
+}
+exports.isLargeUtf8 = isLargeUtf8;
+function isBool(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isBool(value);
+}
+exports.isBool = isBool;
+function isDecimal(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isDecimal(value);
+}
+exports.isDecimal = isDecimal;
+function isDate(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isDate(value);
+}
+exports.isDate = isDate;
+function isTime(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isTime(value);
+}
+exports.isTime = isTime;
+function isTimestamp(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isTimestamp(value);
+}
+exports.isTimestamp = isTimestamp;
+function isInterval(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isInterval(value);
+}
+exports.isInterval = isInterval;
+function isDuration(value) {
+    return value instanceof apache_arrow_1.Utf8 || apache_arrow_1.DataType.isDuration(value);
+}
+exports.isDuration = isDuration;
+function isList(value) {
+    return value instanceof apache_arrow_1.List || apache_arrow_1.DataType.isList(value);
+}
+exports.isList = isList;
+function isStruct(value) {
+    return value instanceof apache_arrow_1.Struct || apache_arrow_1.DataType.isStruct(value);
+}
+exports.isStruct = isStruct;
+function isUnion(value) {
+    return value instanceof apache_arrow_1.Struct || apache_arrow_1.DataType.isUnion(value);
+}
+exports.isUnion = isUnion;
+function isFixedSizeBinary(value) {
+    return value instanceof apache_arrow_1.FixedSizeBinary || apache_arrow_1.DataType.isFixedSizeBinary(value);
+}
+exports.isFixedSizeBinary = isFixedSizeBinary;
+function isFixedSizeList(value) {
+    return value instanceof apache_arrow_1.FixedSizeList || apache_arrow_1.DataType.isFixedSizeList(value);
+}
+exports.isFixedSizeList = isFixedSizeList;
 /*
  * Options to control how a column should be converted to a vector array
  */
@@ -60,6 +183,7 @@ class MakeArrowTableOptions {
     vectorColumns = {
         vector: new VectorColumnOptions(),
     };
+    embeddings;
     /**
      * If true then string columns will be encoded with dictionary encoding
      *
@@ -167,7 +291,7 @@ exports.MakeArrowTableOptions = MakeArrowTableOptions;
  * assert.deepEqual(table.schema, schema)
  * ```
  */
-function makeArrowTable(data, options) {
+function makeArrowTable(data, options, metadata) {
     if (data.length === 0 &&
         (options?.schema === undefined || options?.schema === null)) {
         throw new Error("At least one record or a schema needs to be provided");
@@ -175,6 +299,7 @@ function makeArrowTable(data, options) {
     const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
     if (opt.schema !== undefined && opt.schema !== null) {
         opt.schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
+        opt.schema = validateSchemaEmbeddings(opt.schema, data, opt.embeddings);
     }
     const columns = {};
     // TODO: sample dataset to find missing columns
@@ -244,20 +369,38 @@ function makeArrowTable(data, options) {
         // then patch the schema of the batches so we can use
         // `new ArrowTable(schema, batches)` which does not do any schema inference
         const firstTable = new apache_arrow_1.Table(columns);
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         const batchesFixed = firstTable.batches.map((batch) => new apache_arrow_1.RecordBatch(opt.schema, batch.data));
-        return new apache_arrow_1.Table(opt.schema, batchesFixed);
+        let schema;
+        if (metadata !== undefined) {
+            let schemaMetadata = opt.schema.metadata;
+            if (schemaMetadata.size === 0) {
+                schemaMetadata = metadata;
+            }
+            else {
+                for (const [key, entry] of schemaMetadata.entries()) {
+                    schemaMetadata.set(key, entry);
+                }
+            }
+            schema = new apache_arrow_1.Schema(opt.schema.fields, schemaMetadata);
+        }
+        else {
+            schema = opt.schema;
+        }
+        return new apache_arrow_1.Table(schema, batchesFixed);
     }
-    else {
-        return new apache_arrow_1.Table(columns);
+    const tbl = new apache_arrow_1.Table(columns);
+    if (metadata !== undefined) {
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        tbl.schema.metadata = metadata;
     }
+    return tbl;
 }
 exports.makeArrowTable = makeArrowTable;
 /**
  * Create an empty Arrow table with the provided schema
  */
-function makeEmptyTable(schema) {
-    return makeArrowTable([], { schema });
+function makeEmptyTable(schema, metadata) {
+    return makeArrowTable([], { schema }, metadata);
 }
 exports.makeEmptyTable = makeEmptyTable;
 /**
@@ -269,7 +412,7 @@ function makeListVector(lists) {
         throw Error("Cannot infer list vector from empty array or empty list");
     }
     const sampleList = lists[0];
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    // biome-ignore lint/suspicious/noExplicitAny: skip
     let inferredType;
     try {
         const sampleVector = makeVector(sampleList);
@@ -319,9 +462,52 @@ function makeVector(values, type, stringAsDictionary) {
         return (0, apache_arrow_1.vectorFromArray)(values);
     }
 }
+/** Helper function to apply embeddings from metadata to an input table */
+async function applyEmbeddingsFromMetadata(table, schema) {
+    const registry = (0, registry_1.getRegistry)();
+    const functions = registry.parseFunctions(schema.metadata);
+    const columns = Object.fromEntries(table.schema.fields.map((field) => [
+        field.name,
+        table.getChild(field.name),
+    ]));
+    for (const functionEntry of functions.values()) {
+        const sourceColumn = columns[functionEntry.sourceColumn];
+        const destColumn = functionEntry.vectorColumn ?? "vector";
+        if (sourceColumn === undefined) {
+            throw new Error(`Cannot apply embedding function because the source column '${functionEntry.sourceColumn}' was not present in the data`);
+        }
+        if (columns[destColumn] !== undefined) {
+            throw new Error(`Attempt to apply embeddings to table failed because column ${destColumn} already existed`);
+        }
+        if (table.batches.length > 1) {
+            throw new Error("Internal error: `makeArrowTable` unexpectedly created a table with more than one batch");
+        }
+        const values = sourceColumn.toArray();
+        const vectors = await functionEntry.function.computeSourceEmbeddings(values);
+        if (vectors.length !== values.length) {
+            throw new Error("Embedding function did not return an embedding for each input element");
+        }
+        let destType;
+        const dtype = schema.fields.find((f) => f.name === destColumn).type;
+        if (isFixedSizeList(dtype)) {
+            destType = (0, sanitize_1.sanitizeType)(dtype);
+        }
+        else {
+            throw new Error("Expected FixedSizeList as datatype for vector field, instead got: " +
+                dtype);
+        }
+        const vector = makeVector(vectors, destType);
+        columns[destColumn] = vector;
+    }
+    const newTable = new apache_arrow_1.Table(columns);
+    return alignTable(newTable, schema);
+}
 /** Helper function to apply embeddings to an input table */
 async function applyEmbeddings(table, embeddings, schema) {
-    if (embeddings == null) {
+    if (schema?.metadata.has("embedding_functions")) {
+        return applyEmbeddingsFromMetadata(table, schema);
+    }
+    else if (embeddings == null || embeddings === undefined) {
         return table;
     }
     if (schema !== undefined && schema !== null) {
@@ -336,8 +522,8 @@ async function applyEmbeddings(table, embeddings, schema) {
     });
     const newColumns = Object.fromEntries(colEntries);
     const sourceColumn = newColumns[embeddings.sourceColumn];
-    const destColumn = embeddings.destColumn ?? "vector";
-    const innerDestType = embeddings.embeddingDataType ?? new apache_arrow_1.Float32();
+    const destColumn = embeddings.vectorColumn ?? "vector";
+    const innerDestType = embeddings.function.embeddingDataType() ?? new apache_arrow_1.Float32();
     if (sourceColumn === undefined) {
         throw new Error(`Cannot apply embedding function because the source column '${embeddings.sourceColumn}' was not present in the data`);
     }
@@ -348,8 +534,9 @@ async function applyEmbeddings(table, embeddings, schema) {
             // if we call convertToTable with 0 records and a schema that includes the embedding
             return table;
         }
-        if (embeddings.embeddingDimension !== undefined) {
-            const destType = newVectorType(embeddings.embeddingDimension, innerDestType);
+        const dimensions = embeddings.function.ndims();
+        if (dimensions !== undefined) {
+            const destType = newVectorType(dimensions, innerDestType);
             newColumns[destColumn] = makeVector([], destType);
         }
         else if (schema != null) {
@@ -373,7 +560,7 @@ async function applyEmbeddings(table, embeddings, schema) {
             throw new Error("Internal error: `makeArrowTable` unexpectedly created a table with more than one batch");
         }
         const values = sourceColumn.toArray();
-        const vectors = await embeddings.embed(values);
+        const vectors = await embeddings.function.computeSourceEmbeddings(values);
         if (vectors.length !== values.length) {
             throw new Error("Embedding function did not return an embedding for each input element");
         }
@@ -416,9 +603,10 @@ exports.convertToTable = convertToTable;
 function newVectorType(dim, innerType) {
     // in Lance we always default to have the elements nullable, so we need to set it to true
     // otherwise we often get schema mismatches because the stored data always has schema with nullable elements
-    const children = new apache_arrow_1.Field("item", innerType, true);
+    const children = new apache_arrow_1.Field("item", (0, sanitize_1.sanitizeType)(innerType), true);
     return new apache_arrow_1.FixedSizeList(dim, children);
 }
+exports.newVectorType = newVectorType;
 /**
  * Serialize an Array of records into a buffer using the Arrow IPC File serialization
  *
@@ -480,12 +668,12 @@ async function fromDataToBuffer(data, embeddings, schema) {
     if (schema !== undefined && schema !== null) {
         schema = (0, sanitize_1.sanitizeSchema)(schema);
     }
-    if (data instanceof apache_arrow_1.Table) {
+    if (isArrowTable(data)) {
         return fromTableToBuffer(data, embeddings, schema);
     }
     else {
-        const table = await convertToTable(data);
-        return fromTableToBuffer(table, embeddings, schema);
+        const table = await convertToTable(data, embeddings, { schema });
+        return fromTableToBuffer(table);
     }
 }
 exports.fromDataToBuffer = fromDataToBuffer;
@@ -537,3 +725,43 @@ function createEmptyTable(schema) {
     return new apache_arrow_1.Table((0, sanitize_1.sanitizeSchema)(schema));
 }
 exports.createEmptyTable = createEmptyTable;
+function validateSchemaEmbeddings(schema, data, embeddings) {
+    const fields = [];
+    const missingEmbeddingFields = [];
+    // First we check if the field is a `FixedSizeList`
+    // Then we check if the data contains the field
+    // if it does not, we add it to the list of missing embedding fields
+    // Finally, we check if those missing embedding fields are `this._embeddings`
+    // if they are not, we throw an error
+    for (let field of schema.fields) {
+        if (isFixedSizeList(field.type)) {
+            field = (0, sanitize_1.sanitizeField)(field);
+            if (data.length !== 0 && data?.[0]?.[field.name] === undefined) {
+                if (schema.metadata.has("embedding_functions")) {
+                    const embeddings = JSON.parse(schema.metadata.get("embedding_functions"));
+                    if (
+                    // biome-ignore lint/suspicious/noExplicitAny: we don't know the type of `f`
+                    embeddings.find((f) => f["vectorColumn"] === field.name) ===
+                        undefined) {
+                        missingEmbeddingFields.push(field);
+                    }
+                }
+                else {
+                    missingEmbeddingFields.push(field);
+                }
+            }
+            else {
+                fields.push(field);
+            }
+        }
+        else {
+            fields.push(field);
+        }
+    }
+    if (missingEmbeddingFields.length > 0 && embeddings === undefined) {
+        throw new Error(`Table has embeddings: "${missingEmbeddingFields
+            .map((f) => f.name)
+            .join(",")}", but no embedding function was provided`);
+    }
+    return new apache_arrow_1.Schema(fields, schema.metadata);
+}

package/dist/connection.d.ts

@@ -1,6 +1,7 @@
+import { Table as ArrowTable, Schema } from "./arrow";
+import { EmbeddingFunctionConfig } from "./embedding/registry";
 import { ConnectionOptions, Connection as LanceDbConnection } from "./native";
 import { Table } from "./table";
-import { Table as ArrowTable, Schema } from "apache-arrow";
 /**
  * Connect to a LanceDB instance at the given URI.
  *
@@ -39,6 +40,8 @@ export interface CreateTableOptions {
      * The available options are described at https://lancedb.github.io/lancedb/guides/storage/
      */
     storageOptions?: Record<string, string>;
+    schema?: Schema;
+    embeddingFunction?: EmbeddingFunctionConfig;
 }
 export interface OpenTableOptions {
     /**