@lancedb/lancedb 0.16.1-beta.3 → 0.17.0

package/dist/embedding/embedding_function.d.ts

@@ -11,39 +11,52 @@ export interface EmbeddingFunctionConstructor<T extends EmbeddingFunction = Embe
 }
 /**
  * An embedding function that automatically creates vector representation for a given column.
+ *
+ * It's important subclasses pass the **original** options to the super constructor
+ * and then pass those options to `resolveVariables` to resolve any variables before
+ * using them.
+ *
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ *   constructor(options: {model: string, timeout: number}) {
+ *     super(optionsRaw);
+ *     const options = this.resolveVariables(optionsRaw);
+ *     this.model = options.model;
+ *     this.timeout = options.timeout;
+ *   }
+ * }
+ * ```
  */
 export declare abstract class EmbeddingFunction<T = any, M extends FunctionOptions = FunctionOptions> {
+    #private;
     /**
      * @ignore
      *  This is only used for associating the options type with the class for type checking
      */
     readonly TOptions: M;
     /**
-     * Convert the embedding function to a JSON object
-     * It is used to serialize the embedding function to the schema
-     * It's important that any object returned by this method contains all the necessary
-     * information to recreate the embedding function
-     *
-     * It should return the same object that was passed to the constructor
-     * If it does not, the embedding function will not be able to be recreated, or could be recreated incorrectly
+     * Get the original arguments to the constructor, to serialize them so they
+     * can be used to recreate the embedding function later.
+     */
+    toJSON(): Record<string, any>;
+    constructor();
+    /**
+     * Provide a list of keys in the function options that should be treated as
+     * sensitive. If users pass raw values for these keys, they will be rejected.
+     */
+    protected getSensitiveKeys(): string[];
+    /**
+     * Apply variables to the config.
+     */
+    protected resolveVariables(config: Partial<M>): Partial<M>;
+    /**
+     * Optionally load any resources needed for the embedding function.
      *
-     * @example
-     * ```ts
-     * class MyEmbeddingFunction extends EmbeddingFunction {
-     *   constructor(options: {model: string, timeout: number}) {
-     *     super();
-     *     this.model = options.model;
-     *     this.timeout = options.timeout;
-     *   }
-     *   toJSON() {
-     *     return {
-     *       model: this.model,
-     *       timeout: this.timeout,
-     *     };
-     * }
-     * ```
+     * This method is called after the embedding function has been initialized
+     * but before any embeddings are computed. It is useful for loading local models
+     * or other resources that are needed for the embedding function to work.
      */
-    abstract toJSON(): Partial<M>;
     init?(): Promise<void>;
     /**
      * sourceField is used in combination with `LanceSchema` to provide a declarative data model

package/dist/embedding/embedding_function.js

@@ -6,8 +6,25 @@ exports.TextEmbeddingFunction = exports.EmbeddingFunction = void 0;
 require("reflect-metadata");
 const arrow_1 = require("../arrow");
 const sanitize_1 = require("../sanitize");
+const registry_1 = require("./registry");
 /**
  * An embedding function that automatically creates vector representation for a given column.
+ *
+ * It's important subclasses pass the **original** options to the super constructor
+ * and then pass those options to `resolveVariables` to resolve any variables before
+ * using them.
+ *
+ * @example
+ * ```ts
+ * class MyEmbeddingFunction extends EmbeddingFunction {
+ *   constructor(options: {model: string, timeout: number}) {
+ *     super(optionsRaw);
+ *     const options = this.resolveVariables(optionsRaw);
+ *     this.model = options.model;
+ *     this.timeout = options.timeout;
+ *   }
+ * }
+ * ```
  */
 class EmbeddingFunction {
     /**
@@ -16,6 +33,59 @@ class EmbeddingFunction {
      */
     // biome-ignore lint/style/useNamingConvention: we want to keep the name as it is
     TOptions;
+    #config;
+    /**
+     * Get the original arguments to the constructor, to serialize them so they
+     * can be used to recreate the embedding function later.
+     */
+    // biome-ignore lint/suspicious/noExplicitAny :
+    toJSON() {
+        return JSON.parse(JSON.stringify(this.#config));
+    }
+    constructor() {
+        this.#config = {};
+    }
+    /**
+     * Provide a list of keys in the function options that should be treated as
+     * sensitive. If users pass raw values for these keys, they will be rejected.
+     */
+    getSensitiveKeys() {
+        return [];
+    }
+    /**
+     * Apply variables to the config.
+     */
+    resolveVariables(config) {
+        this.#config = config;
+        const registry = (0, registry_1.getRegistry)();
+        const newConfig = { ...config };
+        for (const [key_, value] of Object.entries(newConfig)) {
+            if (this.getSensitiveKeys().includes(key_) &&
+                !value.startsWith("$var:")) {
+                throw new Error(`The key "${key_}" is sensitive and cannot be set directly. Please use the $var: syntax to set it.`);
+            }
+            // Makes TS happy (https://stackoverflow.com/a/78391854)
+            const key = key_;
+            if (typeof value === "string" && value.startsWith("$var:")) {
+                const [name, defaultValue] = value.slice(5).split(":", 2);
+                const variableValue = registry.getVar(name);
+                if (!variableValue) {
+                    if (defaultValue) {
+                        // biome-ignore lint/suspicious/noExplicitAny:
+                        newConfig[key] = defaultValue;
+                    }
+                    else {
+                        throw new Error(`Variable "${name}" not found`);
+                    }
+                }
+                else {
+                    // biome-ignore lint/suspicious/noExplicitAny:
+                    newConfig[key] = variableValue;
+                }
+            }
+        }
+        return newConfig;
+    }
     /**
      * sourceField is used in combination with `LanceSchema` to provide a declarative data model
      *

package/dist/embedding/openai.d.ts

@@ -7,10 +7,8 @@ export type OpenAIOptions = {
 };
 export declare class OpenAIEmbeddingFunction extends EmbeddingFunction<string, Partial<OpenAIOptions>> {
     #private;
-    constructor(options?: Partial<OpenAIOptions>);
-    toJSON(): {
-        model: (string & {}) | "text-embedding-ada-002" | "text-embedding-3-small" | "text-embedding-3-large";
-    };
+    constructor(optionsRaw?: Partial<OpenAIOptions>);
+    protected getSensitiveKeys(): string[];
     ndims(): number;
     embeddingDataType(): Float;
     computeSourceEmbeddings(data: string[]): Promise<number[][]>;

package/dist/embedding/openai.js

@@ -18,10 +18,11 @@ const registry_1 = require("./registry");
 let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_function_1.EmbeddingFunction {
     #openai;
     #modelName;
-    constructor(options = {
+    constructor(optionsRaw = {
         model: "text-embedding-ada-002",
     }) {
         super();
+        const options = this.resolveVariables(optionsRaw);
         const openAIKey = options?.apiKey ?? process.env.OPENAI_API_KEY;
         if (!openAIKey) {
             throw new Error("OpenAI API key is required");
@@ -45,10 +46,8 @@ let OpenAIEmbeddingFunction = class OpenAIEmbeddingFunction extends embedding_fu
         this.#openai = new Openai(configuration);
         this.#modelName = modelName;
     }
-    toJSON() {
-        return {
-            model: this.#modelName,
-        };
+    getSensitiveKeys() {
+        return ["apiKey"];
     }
     ndims() {
         switch (this.#modelName) {

package/dist/embedding/registry.d.ts

@@ -34,6 +34,28 @@ export declare class EmbeddingFunctionRegistry {
     parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Promise<Map<string, EmbeddingFunctionConfig>>;
     functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
     getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;
+    /**
+     * Set a variable. These can be accessed in the embedding function
+     * configuration using the syntax `$var:variable_name`. If they are not
+     * set, an error will be thrown letting you know which key is unset. If you
+     * want to supply a default value, you can add an additional part in the
+     * configuration like so: `$var:variable_name:default_value`. Default values
+     * can be used for runtime configurations that are not sensitive, such as
+     * whether to use a GPU for inference.
+     *
+     * The name must not contain colons. The default value can contain colons.
+     *
+     * @param name
+     * @param value
+     */
+    setVar(name: string, value: string): void;
+    /**
+     * Get a variable.
+     * @param name
+     * @returns
+     * @see {@link setVar}
+     */
+    getVar(name: string): string | undefined;
 }
 export declare function register(name?: string): (ctor: EmbeddingFunctionConstructor<EmbeddingFunction<any, import("./embedding_function").FunctionOptions>>) => any;
 /**

package/dist/embedding/registry.js

@@ -14,6 +14,7 @@ require("reflect-metadata");
  */
 class EmbeddingFunctionRegistry {
     #functions = new Map();
+    #variables = new Map();
     /**
      * Get the number of registered functions
      */
@@ -60,10 +61,7 @@ class EmbeddingFunctionRegistry {
         }
         else {
             // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-            create = function (options) {
-                const instance = new factory(options);
-                return instance;
-            };
+            create = (options) => new factory(options);
         }
         return {
             create,
@@ -119,6 +117,35 @@ class EmbeddingFunctionRegistry {
         metadata.set("embedding_functions", JSON.stringify(jsonData));
         return metadata;
     }
+    /**
+     * Set a variable. These can be accessed in the embedding function
+     * configuration using the syntax `$var:variable_name`. If they are not
+     * set, an error will be thrown letting you know which key is unset. If you
+     * want to supply a default value, you can add an additional part in the
+     * configuration like so: `$var:variable_name:default_value`. Default values
+     * can be used for runtime configurations that are not sensitive, such as
+     * whether to use a GPU for inference.
+     *
+     * The name must not contain colons. The default value can contain colons.
+     *
+     * @param name
+     * @param value
+     */
+    setVar(name, value) {
+        if (name.includes(":")) {
+            throw new Error("Variable names cannot contain colons");
+        }
+        this.#variables.set(name, value);
+    }
+    /**
+     * Get a variable.
+     * @param name
+     * @returns
+     * @see {@link setVar}
+     */
+    getVar(name) {
+        return this.#variables.get(name);
+    }
 }
 exports.EmbeddingFunctionRegistry = EmbeddingFunctionRegistry;
 const _REGISTRY = new EmbeddingFunctionRegistry();

package/dist/embedding/transformers.d.ts

@@ -27,8 +27,7 @@ export type XenovaTransformerOptions = {
 };
 export declare class TransformersEmbeddingFunction extends EmbeddingFunction<string, Partial<XenovaTransformerOptions>> {
     #private;
-    constructor(options?: Partial<XenovaTransformerOptions>);
-    toJSON(): Record<string, any>;
+    constructor(optionsRaw?: Partial<XenovaTransformerOptions>);
     init(): Promise<void>;
     ndims(): number;
     embeddingDataType(): Float;

package/dist/embedding/transformers.js

@@ -22,10 +22,11 @@ let TransformersEmbeddingFunction = class TransformersEmbeddingFunction extends
     #initialized = false;
     #tokenizerOptions;
     #ndims;
-    constructor(options = {
+    constructor(optionsRaw = {
         model: "Xenova/all-MiniLM-L6-v2",
     }) {
         super();
+        const options = this.resolveVariables(optionsRaw);
         const modelName = options?.model ?? "Xenova/all-MiniLM-L6-v2";
         this.#tokenizerOptions = {
             padding: true,
@@ -34,22 +35,6 @@ let TransformersEmbeddingFunction = class TransformersEmbeddingFunction extends
         this.#ndims = options.ndims;
         this.#modelName = modelName;
     }
-    toJSON() {
-        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-        const obj = {
-            model: this.#modelName,
-        };
-        if (this.#ndims) {
-            obj["ndims"] = this.#ndims;
-        }
-        if (this.#tokenizerOptions) {
-            obj["tokenizerOptions"] = this.#tokenizerOptions;
-        }
-        if (this.#tokenizer) {
-            obj["tokenizer"] = this.#tokenizer.name;
-        }
-        return obj;
-    }
     async init() {
         let transformers;
         try {

package/package.json

@@ -11,7 +11,7 @@
     "ann"
   ],
   "private": false,
-  "version": "0.16.1-beta.3",
+  "version": "0.17.0",
   "main": "dist/index.js",
   "exports": {
     ".": "./dist/index.js",
@@ -98,14 +98,14 @@
     "reflect-metadata": "^0.2.2"
   },
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.16.1-beta.3",
-    "@lancedb/lancedb-darwin-arm64": "0.16.1-beta.3",
-    "@lancedb/lancedb-linux-x64-gnu": "0.16.1-beta.3",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.16.1-beta.3",
-    "@lancedb/lancedb-linux-x64-musl": "0.16.1-beta.3",
-    "@lancedb/lancedb-linux-arm64-musl": "0.16.1-beta.3",
-    "@lancedb/lancedb-win32-x64-msvc": "0.16.1-beta.3",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.16.1-beta.3"
+    "@lancedb/lancedb-darwin-x64": "0.17.0",
+    "@lancedb/lancedb-darwin-arm64": "0.17.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.17.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.17.0",
+    "@lancedb/lancedb-linux-x64-musl": "0.17.0",
+    "@lancedb/lancedb-linux-arm64-musl": "0.17.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.17.0",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.17.0"
   },
   "peerDependencies": {
     "apache-arrow": ">=15.0.0 <=18.1.0"