@lancedb/lancedb 0.18.1 → 0.18.2

package/README.md

@@ -11,11 +11,9 @@ npm install @lancedb/lancedb
 This will download the appropriate native library for your platform. We currently
 support:
 
-- Linux (x86_64 and aarch64)
+- Linux (x86_64 and aarch64 on glibc and musl)
 - MacOS (Intel and ARM/M1/M2)
-- Windows (x86_64 only)
-
-We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
+- Windows (x86_64 and aarch64)
 
 ## Usage
 

package/dist/arrow.d.ts

@@ -1,4 +1,4 @@
-import { Table as ArrowTable, Binary, BufferType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
+import { Table as ArrowTable, Binary, BufferType, DataType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
 import { type EmbeddingFunction } from "./embedding/embedding_function";
 import { EmbeddingFunctionConfig } from "./embedding/registry";
@@ -257,3 +257,15 @@ export declare function fromTableToStreamBuffer(table: ArrowTable, embeddings?:
  * Create an empty table with the given schema
  */
 export declare function createEmptyTable(schema: Schema): ArrowTable;
+interface JsonDataType {
+    type: string;
+    fields?: JsonField[];
+    length?: number;
+}
+interface JsonField {
+    name: string;
+    type: JsonDataType;
+    nullable: boolean;
+    metadata: Map<string, string>;
+}
+export declare function dataTypeToJson(dataType: DataType): JsonDataType;

package/dist/arrow.js

@@ -49,6 +49,7 @@ exports.fromBufferToRecordBatch = fromBufferToRecordBatch;
 exports.fromRecordBatchToBuffer = fromRecordBatchToBuffer;
 exports.fromTableToStreamBuffer = fromTableToStreamBuffer;
 exports.createEmptyTable = createEmptyTable;
+exports.dataTypeToJson = dataTypeToJson;
 const apache_arrow_1 = require("apache-arrow");
 const registry_1 = require("./embedding/registry");
 const sanitize_1 = require("./sanitize");
@@ -126,6 +127,22 @@ class VectorColumnOptions {
     }
 }
 exports.VectorColumnOptions = VectorColumnOptions;
+// biome-ignore lint/suspicious/noExplicitAny: skip
+function vectorFromArray(data, type) {
+    // Workaround for: https://github.com/apache/arrow/issues/45862
+    // If FSL type with float
+    if (apache_arrow_1.DataType.isFixedSizeList(type) && apache_arrow_1.DataType.isFloat(type.valueType)) {
+        const extendedData = [...data, new Array(type.listSize).fill(0.0)];
+        const array = (0, apache_arrow_1.vectorFromArray)(extendedData, type);
+        return array.slice(0, data.length);
+    }
+    else if (type === undefined) {
+        return (0, apache_arrow_1.vectorFromArray)(data);
+    }
+    else {
+        return (0, apache_arrow_1.vectorFromArray)(data, type);
+    }
+}
 /** Options to control the makeArrowTable call. */
 class MakeArrowTableOptions {
     /*
@@ -618,7 +635,7 @@ function makeVector(values, type, stringAsDictionary) {
                 });
             }
         }
-        return (0, apache_arrow_1.vectorFromArray)(values, type);
+        return vectorFromArray(values, type);
     }
     if (values.length === 0) {
         throw Error("makeVector requires at least one value or the type must be specfied");
@@ -633,17 +650,17 @@ function makeVector(values, type, stringAsDictionary) {
     }
     else if (Buffer.isBuffer(sampleValue)) {
         // Default Arrow inference doesn't handle Buffer
-        return (0, apache_arrow_1.vectorFromArray)(values, new apache_arrow_1.Binary());
+        return vectorFromArray(values, new apache_arrow_1.Binary());
     }
     else if (!(stringAsDictionary ?? false) &&
         (typeof sampleValue === "string" || sampleValue instanceof String)) {
         // If the type is string then don't use Arrow's default inference unless dictionaries are requested
         // because it will always use dictionary encoding for strings
-        return (0, apache_arrow_1.vectorFromArray)(values, new apache_arrow_1.Utf8());
+        return vectorFromArray(values, new apache_arrow_1.Utf8());
     }
     else {
         // Convert a JS array of values to an arrow vector
-        return (0, apache_arrow_1.vectorFromArray)(values);
+        return vectorFromArray(values);
     }
 }
 /** Helper function to apply embeddings from metadata to an input table */
@@ -966,3 +983,122 @@ function validateSchemaEmbeddings(schema, data, embeddings) {
     }
     return new apache_arrow_1.Schema(fields, schema.metadata);
 }
+// Matches format of https://github.com/lancedb/lance/blob/main/rust/lance/src/arrow/json.rs
+function dataTypeToJson(dataType) {
+    switch (dataType.typeId) {
+        // For primitives, matches https://github.com/lancedb/lance/blob/e12bb9eff2a52f753668d4b62c52e4d72b10d294/rust/lance-core/src/datatypes.rs#L185
+        case apache_arrow_1.Type.Null:
+            return { type: "null" };
+        case apache_arrow_1.Type.Bool:
+            return { type: "bool" };
+        case apache_arrow_1.Type.Int8:
+            return { type: "int8" };
+        case apache_arrow_1.Type.Int16:
+            return { type: "int16" };
+        case apache_arrow_1.Type.Int32:
+            return { type: "int32" };
+        case apache_arrow_1.Type.Int64:
+            return { type: "int64" };
+        case apache_arrow_1.Type.Uint8:
+            return { type: "uint8" };
+        case apache_arrow_1.Type.Uint16:
+            return { type: "uint16" };
+        case apache_arrow_1.Type.Uint32:
+            return { type: "uint32" };
+        case apache_arrow_1.Type.Uint64:
+            return { type: "uint64" };
+        case apache_arrow_1.Type.Int: {
+            const bitWidth = dataType.bitWidth;
+            const signed = dataType.isSigned;
+            const prefix = signed ? "" : "u";
+            return { type: `${prefix}int${bitWidth}` };
+        }
+        case apache_arrow_1.Type.Float: {
+            switch (dataType.precision) {
+                case apache_arrow_1.Precision.HALF:
+                    return { type: "halffloat" };
+                case apache_arrow_1.Precision.SINGLE:
+                    return { type: "float" };
+                case apache_arrow_1.Precision.DOUBLE:
+                    return { type: "double" };
+            }
+            throw Error("Unsupported float precision");
+        }
+        case apache_arrow_1.Type.Float16:
+            return { type: "halffloat" };
+        case apache_arrow_1.Type.Float32:
+            return { type: "float" };
+        case apache_arrow_1.Type.Float64:
+            return { type: "double" };
+        case apache_arrow_1.Type.Utf8:
+            return { type: "string" };
+        case apache_arrow_1.Type.Binary:
+            return { type: "binary" };
+        case apache_arrow_1.Type.LargeUtf8:
+            return { type: "large_string" };
+        case apache_arrow_1.Type.LargeBinary:
+            return { type: "large_binary" };
+        case apache_arrow_1.Type.List:
+            return {
+                type: "list",
+                fields: [fieldToJson(dataType.children[0])],
+            };
+        case apache_arrow_1.Type.FixedSizeList: {
+            const fixedSizeList = dataType;
+            return {
+                type: "fixed_size_list",
+                fields: [fieldToJson(fixedSizeList.children[0])],
+                length: fixedSizeList.listSize,
+            };
+        }
+        case apache_arrow_1.Type.Struct:
+            return {
+                type: "struct",
+                fields: dataType.children.map(fieldToJson),
+            };
+        case apache_arrow_1.Type.Date: {
+            const unit = dataType.unit;
+            return {
+                type: unit === apache_arrow_1.DateUnit.DAY ? "date32:day" : "date64:ms",
+            };
+        }
+        case apache_arrow_1.Type.Timestamp: {
+            const timestamp = dataType;
+            const timezone = timestamp.timezone || "-";
+            return {
+                type: `timestamp:${timestamp.unit}:${timezone}`,
+            };
+        }
+        case apache_arrow_1.Type.Decimal: {
+            const decimal = dataType;
+            return {
+                type: `decimal:${decimal.bitWidth}:${decimal.precision}:${decimal.scale}`,
+            };
+        }
+        case apache_arrow_1.Type.Duration: {
+            const duration = dataType;
+            return { type: `duration:${duration.unit}` };
+        }
+        case apache_arrow_1.Type.FixedSizeBinary: {
+            const byteWidth = dataType.byteWidth;
+            return { type: `fixed_size_binary:${byteWidth}` };
+        }
+        case apache_arrow_1.Type.Dictionary: {
+            const dict = dataType;
+            const indexType = dataTypeToJson(dict.indices);
+            const valueType = dataTypeToJson(dict.valueType);
+            return {
+                type: `dict:${valueType.type}:${indexType.type}:false`,
+            };
+        }
+    }
+    throw new Error("Unsupported data type");
+}
+function fieldToJson(field) {
+    return {
+        name: field.name,
+        type: dataTypeToJson(field.type),
+        nullable: field.nullable,
+        metadata: field.metadata,
+    };
+}

package/dist/index.d.ts

@@ -1,16 +1,16 @@
 import { Connection } from "./connection";
 import { ConnectionOptions } from "./native.js";
-export { AddColumnsSql, ColumnAlteration, ConnectionOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, OptimizeStats, CompactionStats, RemovalStats, } from "./native.js";
+export { AddColumnsSql, ConnectionOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, OptimizeStats, CompactionStats, RemovalStats, } from "./native.js";
 export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
 export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, } from "./connection";
 export { ExecutableQuery, Query, QueryBase, VectorQuery, QueryExecutionOptions, FullTextSearchOptions, RecordBatchIterator, } from "./query";
-export { Index, IndexOptions, IvfPqOptions, HnswPqOptions, HnswSqOptions, FtsOptions, } from "./indices";
-export { Table, AddDataOptions, UpdateOptions, OptimizeOptions, Version, } from "./table";
+export { Index, IndexOptions, IvfPqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, } from "./indices";
+export { Table, AddDataOptions, UpdateOptions, OptimizeOptions, Version, ColumnAlteration, } from "./table";
 export { MergeInsertBuilder } from "./merge";
 export * as embedding from "./embedding";
 export * as rerankers from "./rerankers";
 export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, } from "./arrow";
-export { IntoSql } from "./util";
+export { IntoSql, packBits } from "./util";
 /**
  * Connect to a LanceDB instance at the given URI.
  *

package/dist/index.js

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.rerankers = exports.embedding = exports.MergeInsertBuilder = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
+exports.packBits = exports.rerankers = exports.embedding = exports.MergeInsertBuilder = exports.Table = exports.Index = exports.RecordBatchIterator = exports.VectorQuery = exports.QueryBase = exports.Query = exports.Connection = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = void 0;
 exports.connect = connect;
 const connection_1 = require("./connection");
 const native_js_1 = require("./native.js");
@@ -25,6 +25,8 @@ var merge_1 = require("./merge");
 Object.defineProperty(exports, "MergeInsertBuilder", { enumerable: true, get: function () { return merge_1.MergeInsertBuilder; } });
 exports.embedding = require("./embedding");
 exports.rerankers = require("./rerankers");
+var util_1 = require("./util");
+Object.defineProperty(exports, "packBits", { enumerable: true, get: function () { return util_1.packBits; } });
 async function connect(uriOrOptions, options = {}) {
     let uri;
     if (typeof uriOrOptions !== "string") {

package/dist/indices.d.ts

@@ -54,13 +54,13 @@ export interface IvfPqOptions {
      *
      * "l2" - Euclidean distance. This is a very common distance metric that
      * accounts for both magnitude and direction when determining the distance
-     * between vectors. L2 distance has a range of [0, ∞).
+     * between vectors. l2 distance has a range of [0, ∞).
      *
      * "cosine" - Cosine distance.  Cosine distance is a distance metric
      * calculated from the cosine similarity between two vectors. Cosine
      * similarity is a measure of similarity between two non-zero vectors of an
      * inner product space. It is defined to equal the cosine of the angle
-     * between them.  Unlike L2, the cosine distance is not affected by the
+     * between them.  Unlike l2, the cosine distance is not affected by the
      * magnitude of the vectors.  Cosine distance has a range of [0, 2].
      *
      * Note: the cosine distance is undefined when one (or both) of the vectors
@@ -69,7 +69,7 @@ export interface IvfPqOptions {
      *
      * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
      * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     * l2 norm is 1), then dot distance is equivalent to the cosine distance.
      */
     distanceType?: "l2" | "cosine" | "dot";
     /**
@@ -114,18 +114,18 @@ export interface HnswPqOptions {
      *
      * "l2" - Euclidean distance. This is a very common distance metric that
      * accounts for both magnitude and direction when determining the distance
-     * between vectors. L2 distance has a range of [0, ∞).
+     * between vectors. l2 distance has a range of [0, ∞).
      *
      * "cosine" - Cosine distance.  Cosine distance is a distance metric
      * calculated from the cosine similarity between two vectors. Cosine
      * similarity is a measure of similarity between two non-zero vectors of an
      * inner product space. It is defined to equal the cosine of the angle
-     * between them.  Unlike L2, the cosine distance is not affected by the
+     * between them.  Unlike l2, the cosine distance is not affected by the
      * magnitude of the vectors.  Cosine distance has a range of [0, 2].
      *
      * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
      * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     * l2 norm is 1), then dot distance is equivalent to the cosine distance.
      */
     distanceType?: "l2" | "cosine" | "dot";
     /**
@@ -223,18 +223,18 @@ export interface HnswSqOptions {
      *
      * "l2" - Euclidean distance. This is a very common distance metric that
      * accounts for both magnitude and direction when determining the distance
-     * between vectors. L2 distance has a range of [0, ∞).
+     * between vectors. l2 distance has a range of [0, ∞).
      *
      * "cosine" - Cosine distance.  Cosine distance is a distance metric
      * calculated from the cosine similarity between two vectors. Cosine
      * similarity is a measure of similarity between two non-zero vectors of an
      * inner product space. It is defined to equal the cosine of the angle
-     * between them.  Unlike L2, the cosine distance is not affected by the
+     * between them.  Unlike l2, the cosine distance is not affected by the
      * magnitude of the vectors.  Cosine distance has a range of [0, 2].
      *
      * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
      * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
-     * L2 norm is 1), then dot distance is equivalent to the cosine distance.
+     * l2 norm is 1), then dot distance is equivalent to the cosine distance.
      */
     distanceType?: "l2" | "cosine" | "dot";
     /**
@@ -303,6 +303,90 @@ export interface HnswSqOptions {
      */
     efConstruction?: number;
 }
+/**
+ * Options to create an `IVF_FLAT` index
+ */
+export interface IvfFlatOptions {
+    /**
+     * The number of IVF partitions to create.
+     *
+     * This value should generally scale with the number of rows in the dataset.
+     * By default the number of partitions is the square root of the number of
+     * rows.
+     *
+     * If this value is too large then the first part of the search (picking the
+     * right partition) will be slow.  If this value is too small then the second
+     * part of the search (searching within a partition) will be slow.
+     */
+    numPartitions?: number;
+    /**
+     * Distance type to use to build the index.
+     *
+     * Default value is "l2".
+     *
+     * This is used when training the index to calculate the IVF partitions
+     * (vectors are grouped in partitions with similar vectors according to this
+     * distance type).
+     *
+     * The distance type used to train an index MUST match the distance type used
+     * to search the index.  Failure to do so will yield inaccurate results.
+     *
+     * The following distance types are available:
+     *
+     * "l2" - Euclidean distance. This is a very common distance metric that
+     * accounts for both magnitude and direction when determining the distance
+     * between vectors. l2 distance has a range of [0, ∞).
+     *
+     * "cosine" - Cosine distance.  Cosine distance is a distance metric
+     * calculated from the cosine similarity between two vectors. Cosine
+     * similarity is a measure of similarity between two non-zero vectors of an
+     * inner product space. It is defined to equal the cosine of the angle
+     * between them.  Unlike l2, the cosine distance is not affected by the
+     * magnitude of the vectors.  Cosine distance has a range of [0, 2].
+     *
+     * Note: the cosine distance is undefined when one (or both) of the vectors
+     * are all zeros (there is no direction).  These vectors are invalid and may
+     * never be returned from a vector search.
+     *
+     * "dot" - Dot product. Dot distance is the dot product of two vectors. Dot
+     * distance has a range of (-∞, ∞). If the vectors are normalized (i.e. their
+     * l2 norm is 1), then dot distance is equivalent to the cosine distance.
+     *
+     * "hamming" - Hamming distance. Hamming distance is a distance metric
+     * calculated from the number of bits that are different between two vectors.
+     * Hamming distance has a range of [0, dimension]. Note that the hamming distance
+     * is only valid for binary vectors.
+     */
+    distanceType?: "l2" | "cosine" | "dot" | "hamming";
+    /**
+     * Max iteration to train IVF kmeans.
+     *
+     * When training an IVF FLAT index we use kmeans to calculate the partitions.  This parameter
+     * controls how many iterations of kmeans to run.
+     *
+     * Increasing this might improve the quality of the index but in most cases these extra
+     * iterations have diminishing returns.
+     *
+     * The default value is 50.
+     */
+    maxIterations?: number;
+    /**
+     * The number of vectors, per partition, to sample when training IVF kmeans.
+     *
+     * When an IVF FLAT index is trained, we need to calculate partitions.  These are groups
+     * of vectors that are similar to each other.  To do this we use an algorithm called kmeans.
+     *
+     * Running kmeans on a large dataset can be slow.  To speed this up we run kmeans on a
+     * random sample of the data.  This parameter controls the size of the sample.  The total
+     * number of vectors used to train the index is `sample_rate * num_partitions`.
+     *
+     * Increasing this value might improve the quality of the index but in most cases the
+     * default should be sufficient.
+     *
+     * The default value is 256.
+     */
+    sampleRate?: number;
+}
 /**
  * Options to create a full text search index
  */
@@ -380,6 +464,23 @@ export declare class Index {
      * currently is also a memory intensive operation.
      */
     static ivfPq(options?: Partial<IvfPqOptions>): Index;
+    /**
+     * Create an IvfFlat index
+     *
+     * This index groups vectors into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF FLAT index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfFlat(options?: Partial<IvfFlatOptions>): Index;
     /**
      * Create a btree index
      *

package/dist/indices.js

@@ -34,6 +34,25 @@ class Index {
     static ivfPq(options) {
         return new Index(native_1.Index.ivfPq(options?.distanceType, options?.numPartitions, options?.numSubVectors, options?.maxIterations, options?.sampleRate));
     }
+    /**
+     * Create an IvfFlat index
+     *
+     * This index groups vectors into partitions of similar vectors.  Each partition keeps track of
+     * a centroid which is the average value of all vectors in the group.
+     *
+     * During a query the centroids are compared with the query vector to find the closest
+     * partitions.  The vectors in these partitions are then searched to find
+     * the closest vectors.
+     *
+     * The partitioning process is called IVF and the `num_partitions` parameter controls how
+     * many groups to create.
+     *
+     * Note that training an IVF FLAT index on a large dataset is a slow operation and
+     * currently is also a memory intensive operation.
+     */
+    static ivfFlat(options) {
+        return new Index(native_1.Index.ivfFlat(options?.distanceType, options?.numPartitions, options?.maxIterations, options?.sampleRate));
+    }
     /**
      * Create a btree index
      *

package/dist/native.d.ts

@@ -185,6 +185,11 @@ export interface IndexStatistics {
   distanceType?: string
   /** The number of parts this index is split into. */
   numIndices?: number
+  /**
+   * The KMeans loss value of the index,
+   * it is only present for vector indices.
+   */
+  loss?: number
 }
 export interface Version {
   version: number
@@ -257,6 +262,7 @@ export class Connection {
 }
 export class Index {
   static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
+  static ivfFlat(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
   static btree(): Index
   static bitmap(): Index
   static labelList(): Index

package/dist/table.d.ts

@@ -1,7 +1,7 @@
-import { Table as ArrowTable, Data, IntoVector, Schema } from "./arrow";
+import { Table as ArrowTable, Data, DataType, IntoVector, Schema } from "./arrow";
 import { IndexOptions } from "./indices";
 import { MergeInsertBuilder } from "./merge";
-import { AddColumnsSql, ColumnAlteration, IndexConfig, IndexStatistics, OptimizeStats, Table as _NativeTable } from "./native";
+import { AddColumnsSql, IndexConfig, IndexStatistics, OptimizeStats, Table as _NativeTable } from "./native";
 import { Query, VectorQuery } from "./query";
 import { IntoSql } from "./util";
 export { IndexConfig } from "./native";
@@ -439,3 +439,37 @@ export declare class LocalTable extends Table {
      */
     migrateManifestPathsV2(): Promise<void>;
 }
+/**
+ *  A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+    /**
+     * The path to the column to alter. This is a dot-separated path to the column.
+     * If it is a top-level column then it is just the name of the column. If it is
+     * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+     * `c` nested inside a column `b` nested inside a column `a`.
+     */
+    path: string;
+    /**
+     * The new name of the column. If not provided then the name will not be changed.
+     * This must be distinct from the names of all other columns in the table.
+     */
+    rename?: string;
+    /**
+     * A new data type for the column. If not provided then the data type will not be changed.
+     * Changing data types is limited to casting to the same general type. For example, these
+     * changes are valid:
+     * * `int32` -> `int64` (integers)
+     * * `double` -> `float` (floats)
+     * * `string` -> `large_string` (strings)
+     * But these changes are not:
+     * * `int32` -> `double` (mix integers and floats)
+     * * `string` -> `int32` (mix strings and integers)
+     */
+    dataType?: string | DataType;
+    /** Set the new nullability. Note that a nullable column cannot be made non-nullable. */
+    nullable?: boolean;
+}

package/dist/table.js

@@ -7,6 +7,7 @@ const arrow_1 = require("./arrow");
 const registry_1 = require("./embedding/registry");
 const merge_1 = require("./merge");
 const query_1 = require("./query");
+const sanitize_1 = require("./sanitize");
 const util_1 = require("./util");
 /**
  * A Table is a collection of Records in a LanceDB Database.
@@ -170,7 +171,28 @@ class LocalTable extends Table {
         await this.inner.addColumns(newColumnTransforms);
     }
     async alterColumns(columnAlterations) {
-        await this.inner.alterColumns(columnAlterations);
+        const processedAlterations = columnAlterations.map((alteration) => {
+            if (typeof alteration.dataType === "string") {
+                return {
+                    ...alteration,
+                    dataType: JSON.stringify({ type: alteration.dataType }),
+                };
+            }
+            else if (alteration.dataType === undefined) {
+                return {
+                    ...alteration,
+                    dataType: undefined,
+                };
+            }
+            else {
+                const dataType = (0, sanitize_1.sanitizeType)(alteration.dataType);
+                return {
+                    ...alteration,
+                    dataType: JSON.stringify((0, arrow_1.dataTypeToJson)(dataType)),
+                };
+            }
+        });
+        await this.inner.alterColumns(processedAlterations);
     }
     async dropColumns(columnNames) {
         await this.inner.dropColumns(columnNames);

package/dist/util.d.ts

@@ -1,5 +1,6 @@
 export type IntoSql = string | number | boolean | null | Date | ArrayBufferLike | Buffer | IntoSql[];
 export declare function toSQL(value: IntoSql): string;
+export declare function packBits(data: Array<number>): Array<number>;
 export declare class TTLCache {
     private readonly ttl;
     private readonly cache;

package/dist/util.js

@@ -4,6 +4,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.TTLCache = void 0;
 exports.toSQL = toSQL;
+exports.packBits = packBits;
 function toSQL(value) {
     if (typeof value === "string") {
         return `'${value.replace(/'/g, "''")}'`;
@@ -33,6 +34,15 @@ function toSQL(value) {
         throw new Error(`Unsupported value type: ${typeof value} value: (${value})`);
     }
 }
+function packBits(data) {
+    const packed = Array(data.length >> 3).fill(0);
+    for (let i = 0; i < data.length; i++) {
+        const byte = i >> 3;
+        const bit = i & 7;
+        packed[byte] |= data[i] << bit;
+    }
+    return packed;
+}
 class TTLCache {
     ttl;
     // biome-ignore lint/suspicious/noExplicitAny: <explanation>

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.18.1",
+  "version": "0.18.2",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -29,7 +29,6 @@
         "aarch64-apple-darwin",
         "x86_64-unknown-linux-gnu",
         "aarch64-unknown-linux-gnu",
-        "x86_64-unknown-linux-musl",
         "aarch64-unknown-linux-musl",
         "x86_64-pc-windows-msvc",
         "aarch64-pc-windows-msvc"
@@ -81,8 +80,10 @@
     "artifacts": "napi artifacts",
     "build:debug": "napi build --platform --no-const-enum --dts ../lancedb/native.d.ts --js ../lancedb/native.js lancedb",
     "build:release": "napi build --platform --no-const-enum --release --dts ../lancedb/native.d.ts --js ../lancedb/native.js dist/",
-    "build": "npm run build:debug && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts && shx cp lancedb/*.node dist/",
-    "build-release": "npm run build:release && tsc -b && shx cp lancedb/native.d.ts dist/native.d.ts",
+    "build": "npm run build:debug && npm run tsc && shx cp lancedb/*.node dist/",
+    "build-release": "npm run build:release && npm run tsc",
+    "tsc": "tsc -b",
+    "posttsc": "shx cp lancedb/native.d.ts dist/native.d.ts",
     "lint-ci": "biome ci .",
     "docs": "typedoc --plugin typedoc-plugin-markdown --treatWarningsAsErrors --out ../docs/src/js lancedb/index.ts",
     "postdocs": "node typedoc_post_process.js",
@@ -98,14 +99,13 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.18.1",
-    "@lancedb/lancedb-darwin-arm64": "0.18.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.18.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.18.1",
-    "@lancedb/lancedb-linux-x64-musl": "0.18.1",
-    "@lancedb/lancedb-linux-arm64-musl": "0.18.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.18.1",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.18.1"
+    "@lancedb/lancedb-darwin-x64": "0.18.2",
+    "@lancedb/lancedb-darwin-arm64": "0.18.2",
+    "@lancedb/lancedb-linux-x64-gnu": "0.18.2",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.18.2",
+    "@lancedb/lancedb-linux-arm64-musl": "0.18.2",
+    "@lancedb/lancedb-win32-x64-msvc": "0.18.2",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.18.2"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"

package/typedoc_post_process.js

@@ -1,68 +0,0 @@
-const fs = require("fs");
-const path = require("path");
-
-// Read all files in the directory
-function processDirectory(directoryPath) {
-  fs.readdir(directoryPath, { withFileTypes: true }, (err, files) => {
-    if (err) {
-      return console.error("Unable to scan directory: " + err);
-    }
-
-    files.forEach((file) => {
-      const filePath = path.join(directoryPath, file.name);
-
-      if (file.isDirectory()) {
-        // Recursively process subdirectory
-        processDirectory(filePath);
-      } else if (file.isFile()) {
-        // Read each file
-        fs.readFile(filePath, "utf8", (err, data) => {
-          if (err) {
-            return console.error("Unable to read file: " + err);
-          }
-
-          // Process the file content
-          const processedData = processContents(data);
-
-          // Write the processed content back to the file
-          fs.writeFile(filePath, processedData, "utf8", (err) => {
-            if (err) {
-              return console.error("Unable to write file: " + err);
-            }
-            console.log(`Processed file: ${filePath}`);
-          });
-        });
-      }
-    });
-  });
-}
-
-function processContents(contents) {
-  // This changes the parameters section to put the parameter description on
-  // the same line as the bullet with the parameter name and type.
-  return (
-    contents
-      .replace(/(## Parameters[\s\S]*?)(?=##|$)/g, (match) => {
-        let lines = match
-          .split("\n")
-          .map((line) => line.trim())
-
-          .filter((line) => line !== "")
-          .map((line) => {
-            if (line.startsWith("##")) {
-              return line;
-            } else if (line.startsWith("•")) {
-              return "\n*" + line.substring(1);
-            } else {
-              return "    " + line;
-            }
-          });
-        return lines.join("\n") + "\n\n";
-      })
-      // Also trim trailing whitespace
-      .replace(/([^ \t])[ \t]+\n/g, "$1\n")
-  );
-}
-
-// Start processing from the root directory
-processDirectory("../docs/src/js");