@dengxifeng/lancedb 0.26.2

package/dist/embedding/transformers.d.ts

@@ -0,0 +1,36 @@
+import { Float } from "../arrow";
+import { EmbeddingFunction } from "./embedding_function";
+export type XenovaTransformerOptions = {
+    /** The wasm compatible model to use */
+    model: string;
+    /**
+     * The wasm compatible tokenizer to use
+     * If not provided, it will use the default tokenizer for the model
+     */
+    tokenizer?: string;
+    /**
+     * The number of dimensions of the embeddings
+     *
+     * We will attempt to infer this from the model config if not provided.
+     * Since there isn't a standard way to get this information from the model,
+     * you may need to manually specify this if using a model that doesn't have a 'hidden_size' in the config.
+     * */
+    ndims?: number;
+    /** Options for the tokenizer */
+    tokenizerOptions?: {
+        textPair?: string | string[];
+        padding?: boolean | "max_length";
+        addSpecialTokens?: boolean;
+        truncation?: boolean;
+        maxLength?: number;
+    };
+};
+export declare class TransformersEmbeddingFunction extends EmbeddingFunction<string, Partial<XenovaTransformerOptions>> {
+    #private;
+    constructor(optionsRaw?: Partial<XenovaTransformerOptions>);
+    init(): Promise<void>;
+    ndims(): number;
+    embeddingDataType(): Float;
+    computeSourceEmbeddings(data: string[]): Promise<number[][]>;
+    computeQueryEmbeddings(data: string): Promise<number[]>;
+}

package/dist/embedding/transformers.js

@@ -0,0 +1,122 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TransformersEmbeddingFunction = void 0;
+const arrow_1 = require("../arrow");
+const embedding_function_1 = require("./embedding_function");
+const registry_1 = require("./registry");
+let TransformersEmbeddingFunction = class TransformersEmbeddingFunction extends embedding_function_1.EmbeddingFunction {
+    #model;
+    #tokenizer;
+    #modelName;
+    #initialized = false;
+    #tokenizerOptions;
+    #ndims;
+    constructor(optionsRaw = {
+        model: "Xenova/all-MiniLM-L6-v2",
+    }) {
+        super();
+        const options = this.resolveVariables(optionsRaw);
+        const modelName = options?.model ?? "Xenova/all-MiniLM-L6-v2";
+        this.#tokenizerOptions = {
+            padding: true,
+            ...options.tokenizerOptions,
+        };
+        this.#ndims = options.ndims;
+        this.#modelName = modelName;
+    }
+    async init() {
+        let transformers;
+        try {
+            // SAFETY:
+            // since typescript transpiles `import` to `require`, we need to do this in an unsafe way
+            // We can't use `require` because `@huggingface/transformers` is an ESM module
+            // and we can't use `import` directly because typescript will transpile it to `require`.
+            // and we want to remain compatible with both ESM and CJS modules
+            // so we use `eval` to bypass typescript for this specific import.
+            transformers = await eval('import("@huggingface/transformers")');
+        }
+        catch (e) {
+            throw new Error(`error loading @huggingface/transformers\nReason: ${e}`);
+        }
+        try {
+            this.#model = await transformers.AutoModel.from_pretrained(this.#modelName, { dtype: "fp32" });
+        }
+        catch (e) {
+            throw new Error(`error loading model ${this.#modelName}. Make sure you are using a wasm compatible model.\nReason: ${e}`);
+        }
+        try {
+            this.#tokenizer = await transformers.AutoTokenizer.from_pretrained(this.#modelName);
+        }
+        catch (e) {
+            throw new Error(`error loading tokenizer for ${this.#modelName}. Make sure you are using a wasm compatible model:\nReason: ${e}`);
+        }
+        this.#initialized = true;
+    }
+    ndims() {
+        if (this.#ndims) {
+            return this.#ndims;
+        }
+        else {
+            const config = this.#model.config;
+            // biome-ignore lint/style/useNamingConvention: we don't control this name.
+            const ndims = config.hidden_size;
+            if (!ndims) {
+                throw new Error("hidden_size not found in model config, you may need to manually specify the embedding dimensions. ");
+            }
+            return ndims;
+        }
+    }
+    embeddingDataType() {
+        return new arrow_1.Float32();
+    }
+    async computeSourceEmbeddings(data) {
+        // this should only happen if the user is trying to use the function directly.
+        // Anything going through the registry should already be initialized.
+        if (!this.#initialized) {
+            return Promise.reject(new Error("something went wrong: embedding function not initialized. Please call init()"));
+        }
+        const tokenizer = this.#tokenizer;
+        const model = this.#model;
+        const inputs = await tokenizer(data, this.#tokenizerOptions);
+        let tokens = await model.forward(inputs);
+        tokens = tokens[Object.keys(tokens)[0]];
+        const [nItems, nTokens] = tokens.dims;
+        tokens = tensorDiv(tokens.sum(1), nTokens);
+        // TODO: support other data types
+        const tokenData = tokens.data;
+        const stride = this.ndims();
+        const embeddings = [];
+        for (let i = 0; i < nItems; i++) {
+            const start = i * stride;
+            const end = start + stride;
+            const slice = tokenData.slice(start, end);
+            embeddings.push(Array.from(slice)); // TODO: Avoid copy here
+        }
+        return embeddings;
+    }
+    async computeQueryEmbeddings(data) {
+        return (await this.computeSourceEmbeddings([data]))[0];
+    }
+};
+exports.TransformersEmbeddingFunction = TransformersEmbeddingFunction;
+exports.TransformersEmbeddingFunction = TransformersEmbeddingFunction = __decorate([
+    (0, registry_1.register)("huggingface"),
+    __metadata("design:paramtypes", [Object])
+], TransformersEmbeddingFunction);
+const tensorDiv = (src, divBy) => {
+    for (let i = 0; i < src.data.length; ++i) {
+        src.data[i] /= divBy;
+    }
+    return src;
+};

package/dist/header.d.ts

@@ -0,0 +1,162 @@
+/**
+ * Header providers for LanceDB remote connections.
+ *
+ * This module provides a flexible header management framework for LanceDB remote
+ * connections, allowing users to implement custom header strategies for
+ * authentication, request tracking, custom metadata, or any other header-based
+ * requirements.
+ *
+ * @module header
+ */
+/**
+ * Abstract base class for providing custom headers for each request.
+ *
+ * Users can implement this interface to provide dynamic headers for various purposes
+ * such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
+ * custom metadata, or any other header-based requirements. The provider is called
+ * before each request to ensure fresh header values are always used.
+ *
+ * @example
+ * Simple JWT token provider:
+ * ```typescript
+ * class JWTProvider extends HeaderProvider {
+ *   constructor(private token: string) {
+ *     super();
+ *   }
+ *
+ *   getHeaders(): Record<string, string> {
+ *     return { authorization: `Bearer ${this.token}` };
+ *   }
+ * }
+ * ```
+ *
+ * @example
+ * Provider with request tracking:
+ * ```typescript
+ * class RequestTrackingProvider extends HeaderProvider {
+ *   constructor(private sessionId: string) {
+ *     super();
+ *   }
+ *
+ *   getHeaders(): Record<string, string> {
+ *     return {
+ *       "X-Session-Id": this.sessionId,
+ *       "X-Request-Id": `req-${Date.now()}`
+ *     };
+ *   }
+ * }
+ * ```
+ */
+export declare abstract class HeaderProvider {
+    /**
+     * Get the latest headers to be added to requests.
+     *
+     * This method is called before each request to the remote LanceDB server.
+     * Implementations should return headers that will be merged with existing headers.
+     *
+     * @returns Dictionary of header names to values to add to the request.
+     * @throws If unable to fetch headers, the exception will be propagated and the request will fail.
+     */
+    abstract getHeaders(): Record<string, string>;
+}
+/**
+ * Example implementation: A simple header provider that returns static headers.
+ *
+ * This is an example implementation showing how to create a HeaderProvider
+ * for cases where headers don't change during the session.
+ *
+ * @example
+ * ```typescript
+ * const provider = new StaticHeaderProvider({
+ *   authorization: "Bearer my-token",
+ *   "X-Custom-Header": "custom-value"
+ * });
+ * const headers = provider.getHeaders();
+ * // Returns: {authorization: 'Bearer my-token', 'X-Custom-Header': 'custom-value'}
+ * ```
+ */
+export declare class StaticHeaderProvider extends HeaderProvider {
+    private _headers;
+    /**
+     * Initialize with static headers.
+     * @param headers - Headers to return for every request.
+     */
+    constructor(headers: Record<string, string>);
+    /**
+     * Return the static headers.
+     * @returns Copy of the static headers.
+     */
+    getHeaders(): Record<string, string>;
+}
+/**
+ * Token response from OAuth provider.
+ * @public
+ */
+export interface TokenResponse {
+    accessToken: string;
+    expiresIn?: number;
+}
+/**
+ * Example implementation: OAuth token provider with automatic refresh.
+ *
+ * This is an example implementation showing how to manage OAuth tokens
+ * with automatic refresh when they expire.
+ *
+ * @example
+ * ```typescript
+ * async function fetchToken(): Promise<TokenResponse> {
+ *   const response = await fetch("https://oauth.example.com/token", {
+ *     method: "POST",
+ *     body: JSON.stringify({
+ *       grant_type: "client_credentials",
+ *       client_id: "your-client-id",
+ *       client_secret: "your-client-secret"
+ *     }),
+ *     headers: { "Content-Type": "application/json" }
+ *   });
+ *   const data = await response.json();
+ *   return {
+ *     accessToken: data.access_token,
+ *     expiresIn: data.expires_in
+ *   };
+ * }
+ *
+ * const provider = new OAuthHeaderProvider(fetchToken);
+ * const headers = provider.getHeaders();
+ * // Returns: {"authorization": "Bearer <your-token>"}
+ * ```
+ */
+export declare class OAuthHeaderProvider extends HeaderProvider {
+    private _tokenFetcher;
+    private _refreshBufferSeconds;
+    private _currentToken;
+    private _tokenExpiresAt;
+    private _refreshPromise;
+    /**
+     * Initialize the OAuth provider.
+     * @param tokenFetcher - Function to fetch new tokens. Should return object with 'accessToken' and optionally 'expiresIn'.
+     * @param refreshBufferSeconds - Seconds before expiry to refresh token. Default 300 (5 minutes).
+     */
+    constructor(tokenFetcher: () => Promise<TokenResponse> | TokenResponse, refreshBufferSeconds?: number);
+    /**
+     * Check if token needs refresh.
+     */
+    private _needsRefresh;
+    /**
+     * Refresh the token if it's expired or close to expiring.
+     */
+    private _refreshTokenIfNeeded;
+    /**
+     * Get OAuth headers, refreshing token if needed.
+     * Note: This is synchronous for now as the Rust implementation expects sync.
+     * In a real implementation, this would need to handle async properly.
+     * @returns Headers with Bearer token authorization.
+     * @throws If unable to fetch or refresh token.
+     */
+    getHeaders(): Record<string, string>;
+    /**
+     * Manually refresh the token.
+     * Call this before using getHeaders() to ensure token is available.
+     */
+    refreshToken(): Promise<void>;
+}

package/dist/header.js

@@ -0,0 +1,217 @@
+"use strict";
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright The LanceDB Authors
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = void 0;
+/**
+ * Header providers for LanceDB remote connections.
+ *
+ * This module provides a flexible header management framework for LanceDB remote
+ * connections, allowing users to implement custom header strategies for
+ * authentication, request tracking, custom metadata, or any other header-based
+ * requirements.
+ *
+ * @module header
+ */
+/**
+ * Abstract base class for providing custom headers for each request.
+ *
+ * Users can implement this interface to provide dynamic headers for various purposes
+ * such as authentication (OAuth tokens, API keys), request tracking (correlation IDs),
+ * custom metadata, or any other header-based requirements. The provider is called
+ * before each request to ensure fresh header values are always used.
+ *
+ * @example
+ * Simple JWT token provider:
+ * ```typescript
+ * class JWTProvider extends HeaderProvider {
+ *   constructor(private token: string) {
+ *     super();
+ *   }
+ *
+ *   getHeaders(): Record<string, string> {
+ *     return { authorization: `Bearer ${this.token}` };
+ *   }
+ * }
+ * ```
+ *
+ * @example
+ * Provider with request tracking:
+ * ```typescript
+ * class RequestTrackingProvider extends HeaderProvider {
+ *   constructor(private sessionId: string) {
+ *     super();
+ *   }
+ *
+ *   getHeaders(): Record<string, string> {
+ *     return {
+ *       "X-Session-Id": this.sessionId,
+ *       "X-Request-Id": `req-${Date.now()}`
+ *     };
+ *   }
+ * }
+ * ```
+ */
+class HeaderProvider {
+}
+exports.HeaderProvider = HeaderProvider;
+/**
+ * Example implementation: A simple header provider that returns static headers.
+ *
+ * This is an example implementation showing how to create a HeaderProvider
+ * for cases where headers don't change during the session.
+ *
+ * @example
+ * ```typescript
+ * const provider = new StaticHeaderProvider({
+ *   authorization: "Bearer my-token",
+ *   "X-Custom-Header": "custom-value"
+ * });
+ * const headers = provider.getHeaders();
+ * // Returns: {authorization: 'Bearer my-token', 'X-Custom-Header': 'custom-value'}
+ * ```
+ */
+class StaticHeaderProvider extends HeaderProvider {
+    _headers;
+    /**
+     * Initialize with static headers.
+     * @param headers - Headers to return for every request.
+     */
+    constructor(headers) {
+        super();
+        this._headers = { ...headers };
+    }
+    /**
+     * Return the static headers.
+     * @returns Copy of the static headers.
+     */
+    getHeaders() {
+        return { ...this._headers };
+    }
+}
+exports.StaticHeaderProvider = StaticHeaderProvider;
+/**
+ * Example implementation: OAuth token provider with automatic refresh.
+ *
+ * This is an example implementation showing how to manage OAuth tokens
+ * with automatic refresh when they expire.
+ *
+ * @example
+ * ```typescript
+ * async function fetchToken(): Promise<TokenResponse> {
+ *   const response = await fetch("https://oauth.example.com/token", {
+ *     method: "POST",
+ *     body: JSON.stringify({
+ *       grant_type: "client_credentials",
+ *       client_id: "your-client-id",
+ *       client_secret: "your-client-secret"
+ *     }),
+ *     headers: { "Content-Type": "application/json" }
+ *   });
+ *   const data = await response.json();
+ *   return {
+ *     accessToken: data.access_token,
+ *     expiresIn: data.expires_in
+ *   };
+ * }
+ *
+ * const provider = new OAuthHeaderProvider(fetchToken);
+ * const headers = provider.getHeaders();
+ * // Returns: {"authorization": "Bearer <your-token>"}
+ * ```
+ */
+class OAuthHeaderProvider extends HeaderProvider {
+    _tokenFetcher;
+    _refreshBufferSeconds;
+    _currentToken = null;
+    _tokenExpiresAt = null;
+    _refreshPromise = null;
+    /**
+     * Initialize the OAuth provider.
+     * @param tokenFetcher - Function to fetch new tokens. Should return object with 'accessToken' and optionally 'expiresIn'.
+     * @param refreshBufferSeconds - Seconds before expiry to refresh token. Default 300 (5 minutes).
+     */
+    constructor(tokenFetcher, refreshBufferSeconds = 300) {
+        super();
+        this._tokenFetcher = tokenFetcher;
+        this._refreshBufferSeconds = refreshBufferSeconds;
+    }
+    /**
+     * Check if token needs refresh.
+     */
+    _needsRefresh() {
+        if (this._currentToken === null) {
+            return true;
+        }
+        if (this._tokenExpiresAt === null) {
+            // No expiration info, assume token is valid
+            return false;
+        }
+        // Refresh if we're within the buffer time of expiration
+        const now = Date.now() / 1000;
+        return now >= this._tokenExpiresAt - this._refreshBufferSeconds;
+    }
+    /**
+     * Refresh the token if it's expired or close to expiring.
+     */
+    async _refreshTokenIfNeeded() {
+        if (!this._needsRefresh()) {
+            return;
+        }
+        // If refresh is already in progress, wait for it
+        if (this._refreshPromise) {
+            await this._refreshPromise;
+            return;
+        }
+        // Start refresh
+        this._refreshPromise = (async () => {
+            try {
+                const tokenData = await this._tokenFetcher();
+                this._currentToken = tokenData.accessToken;
+                if (!this._currentToken) {
+                    throw new Error("Token fetcher did not return 'accessToken'");
+                }
+                // Set expiration if provided
+                if (tokenData.expiresIn) {
+                    this._tokenExpiresAt = Date.now() / 1000 + tokenData.expiresIn;
+                }
+                else {
+                    // Token doesn't expire or expiration unknown
+                    this._tokenExpiresAt = null;
+                }
+            }
+            finally {
+                this._refreshPromise = null;
+            }
+        })();
+        await this._refreshPromise;
+    }
+    /**
+     * Get OAuth headers, refreshing token if needed.
+     * Note: This is synchronous for now as the Rust implementation expects sync.
+     * In a real implementation, this would need to handle async properly.
+     * @returns Headers with Bearer token authorization.
+     * @throws If unable to fetch or refresh token.
+     */
+    getHeaders() {
+        // For simplicity in this example, we assume the token is already fetched
+        // In a real implementation, this would need to handle the async nature properly
+        if (!this._currentToken && !this._refreshPromise) {
+            // Synchronously trigger refresh - this is a limitation of the current implementation
+            throw new Error("Token not initialized. Call refreshToken() first or use async initialization.");
+        }
+        if (!this._currentToken) {
+            throw new Error("Failed to obtain OAuth token");
+        }
+        return { authorization: `Bearer ${this._currentToken}` };
+    }
+    /**
+     * Manually refresh the token.
+     * Call this before using getHeaders() to ensure token is available.
+     */
+    async refreshToken() {
+        this._currentToken = null; // Force refresh
+        await this._refreshTokenIfNeeded();
+    }
+}
+exports.OAuthHeaderProvider = OAuthHeaderProvider;

package/dist/index.d.ts

@@ -0,0 +1,85 @@
+import { Connection } from "./connection";
+import { ConnectionOptions, Session } from "./native.js";
+import { HeaderProvider } from "./header";
+export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
+export { AddColumnsSql, ConnectionOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, MergeResult, AddResult, AddColumnsResult, AlterColumnsResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, } from "./native.js";
+export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
+export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, } from "./connection";
+export { Session } from "./native.js";
+export { ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, QueryExecutionOptions, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
+export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, } from "./indices";
+export { Table, AddDataOptions, UpdateOptions, OptimizeOptions, Version, ColumnAlteration, } from "./table";
+export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
+export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
+export * as embedding from "./embedding";
+export { permutationBuilder, PermutationBuilder } from "./permutation";
+export * as rerankers from "./rerankers";
+export { SchemaLike, TableLike, FieldLike, RecordBatchLike, DataLike, IntoVector, MultiVector, } from "./arrow";
+export { IntoSql, packBits } from "./util";
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param {string} uri - The uri of the database. If the database uri starts
+ * with `db://` then it connects to a remote database.
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @param  options - The options to use when connecting to the database
+ * @example
+ * ```ts
+ * const conn = await connect("/path/to/database");
+ * ```
+ * @example
+ * ```ts
+ * const conn = await connect(
+ *   "s3://bucket/path/to/database",
+ *   {storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ * @example
+ * Using with a header provider for per-request authentication:
+ * ```ts
+ * const provider = new StaticHeaderProvider({
+ *   "X-API-Key": "my-key"
+ * });
+ * const conn = await connectWithHeaderProvider(
+ *   "db://host:port",
+ *   options,
+ *   provider
+ * );
+ * ```
+ */
+export declare function connect(uri: string, options?: Partial<ConnectionOptions>, session?: Session, headerProvider?: HeaderProvider | (() => Record<string, string>) | (() => Promise<Record<string, string>>)): Promise<Connection>;
+/**
+ * Connect to a LanceDB instance at the given URI.
+ *
+ * Accepted formats:
+ *
+ * - `/path/to/database` - local database
+ * - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud storage
+ * - `db://host:port` - remote database (LanceDB cloud)
+ * @param  options - The options to use when connecting to the database
+ * @see {@link ConnectionOptions} for more details on the URI format.
+ * @example
+ * ```ts
+ * const conn = await connect({
+ *   uri: "/path/to/database",
+ *   storageOptions: {timeout: "60s"}
+ * });
+ * ```
+ *
+ * @example
+ * ```ts
+ * const session = Session.default();
+ * const conn = await connect({
+ *   uri: "/path/to/database",
+ *   session: session
+ * });
+ * ```
+ */
+export declare function connect(options: Partial<ConnectionOptions> & {
+    uri: string;
+}): Promise<Connection>;