@lancedb/lancedb 0.4.3

package/.eslintignore

@@ -0,0 +1,3 @@
+**/dist/**/*
+**/native.js
+**/native.d.ts

package/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "lancedb-nodejs"
+edition.workspace = true
+version = "0.0.0"
+license.workspace = true
+description.workspace = true
+repository.workspace = true
+keywords.workspace = true
+categories.workspace = true
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+arrow-ipc.workspace = true
+futures.workspace = true
+lancedb = { path = "../rust/lancedb" }
+napi = { version = "2.15", default-features = false, features = [
+    "napi7",
+    "async",
+] }
+napi-derive = "2"
+
+# Prevent dynamic linking of lzma, which comes from datafusion
+lzma-sys = { version = "*", features = ["static"] }
+
+[build-dependencies]
+napi-build = "2.1"

package/README.md

@@ -0,0 +1,49 @@
+# (New) LanceDB NodeJS SDK
+
+It will replace the NodeJS SDK when it is ready.
+
+## Development
+
+```sh
+npm run build
+npm t
+```
+
+### Running lint / format
+
+LanceDb uses eslint for linting. VSCode does not need any plugins to use eslint. However, it
+may need some additional configuration. Make sure that eslint.experimental.useFlatConfig is
+set to true. Also, if your vscode root folder is the repo root then you will need to set
+the eslint.workingDirectories to ["nodejs"]. To manually lint your code you can run:
+
+```sh
+npm run lint
+```
+
+LanceDb uses prettier for formatting. If you are using VSCode you will need to install the
+"Prettier - Code formatter" extension. You should then configure it to be the default formatter
+for typescript and you should enable format on save. To manually check your code's format you
+can run:
+
+```sh
+npm run chkformat
+```
+
+If you need to manually format your code you can run:
+
+```sh
+npx prettier --write .
+```
+
+### Generating docs
+
+```sh
+npm run docs
+
+cd ../docs
+# Asssume the virtual environment was created
+# python3 -m venv venv
+# pip install -r requirements.txt
+. ./venv/bin/activate
+mkdocs build
+```

package/build.rs

@@ -0,0 +1,5 @@
+extern crate napi_build;
+
+fn main() {
+    napi_build::setup();
+}

package/eslint.config.js

@@ -0,0 +1,28 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+// @ts-check
+
+const eslint = require("@eslint/js");
+const tseslint = require("typescript-eslint");
+const eslintConfigPrettier = require("eslint-config-prettier");
+const jsdoc = require("eslint-plugin-jsdoc");
+
+module.exports = tseslint.config(
+  eslint.configs.recommended,
+  jsdoc.configs["flat/recommended"],
+  eslintConfigPrettier,
+  ...tseslint.configs.recommended,
+  {
+    rules: {
+      "@typescript-eslint/naming-convention": "error",
+      "jsdoc/require-returns": "off",
+      "jsdoc/require-param": "off",
+      "jsdoc/require-jsdoc": [
+        "error",
+        {
+          publicOnly: true,
+        },
+      ],
+    },
+    plugins: jsdoc,
+  },
+);

package/examples/js/index.mjs

@@ -0,0 +1,40 @@
+// Copyright 2024 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global console */
+
+"use strict";
+
+import * as lancedb from "@lancedb/lancedb";
+
+async function example() {
+  const db = await lancedb.connect("data/sample-lancedb");
+
+  const data = [
+    { id: 1, vector: [0.1, 0.2], price: 10 },
+    { id: 2, vector: [1.1, 1.2], price: 50 },
+  ];
+
+  const table = await db.createTable("vectors", data, { existOk: true });
+  console.log(await db.tableNames());
+
+  const results = await table.vectorSearch([0.1, 0.3]).limit(20);
+  for await (const batch of results) {
+    console.log(batch.toArray());
+  }
+}
+
+example()
+  .then(() => console.log("All done!"))
+  .catch(console.error);

package/examples/js/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "lancedb-example-js",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.mjs",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@lancedb/lancedb": "file:../.."
+  }
+}

package/examples/js-openai/index.mjs

@@ -0,0 +1,43 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/* global console, process */
+
+"use strict";
+
+import * as lancedb from "@lancedb/lancedb";
+
+async function example() {
+  // You need to provide an OpenAI API key, here we read it from the OPENAI_API_KEY environment variable
+  const apiKey = process.env.OPENAI_API_KEY;
+  // The embedding function will create embeddings for the 'text' column(text in this case)
+  const embedding = new lancedb.OpenAIEmbeddingFunction("text", apiKey);
+
+  const db = await lancedb.connect("data/sample-lancedb");
+
+  const data = [
+    { id: 1, text: "Black T-Shirt", price: 10 },
+    { id: 2, text: "Leather Jacket", price: 50 },
+  ];
+
+  const table = await db.createTable("vectors", data, embedding);
+  console.log(await db.tableNames());
+
+  const results = await table.search("keeps me warm").limit(1).toArray();
+  console.log(results[0].text);
+}
+
+example()
+  .then(() => console.log("All done!"))
+  .catch(console.error);

package/examples/js-openai/package-lock.json

@@ -0,0 +1,256 @@
+{
+  "name": "lancedb-example-js-openai",
+  "version": "1.0.0",
+  "lockfileVersion": 2,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "lancedb-example-js-openai",
+      "version": "1.0.0",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@lancedb/lancedb": "file:../..",
+        "openai": "^3.2.1"
+      }
+    },
+    "../..": {
+      "version": "0.4.3",
+      "cpu": [
+        "x64",
+        "arm64"
+      ],
+      "license": "Apache 2.0",
+      "os": [
+        "darwin",
+        "linux",
+        "win32"
+      ],
+      "dependencies": {
+        "openai": "^4.29.2"
+      },
+      "devDependencies": {
+        "@napi-rs/cli": "^2.18.0",
+        "@types/jest": "^29.1.2",
+        "@types/tmp": "^0.2.6",
+        "@typescript-eslint/eslint-plugin": "^6.19.0",
+        "@typescript-eslint/parser": "^6.19.0",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
+        "eslint": "^8.57.0",
+        "eslint-config-prettier": "^9.1.0",
+        "eslint-plugin-jsdoc": "^48.2.1",
+        "jest": "^29.7.0",
+        "prettier": "^3.1.0",
+        "shx": "^0.3.4",
+        "tmp": "^0.2.3",
+        "ts-jest": "^29.1.2",
+        "typedoc": "^0.25.7",
+        "typedoc-plugin-markdown": "^3.17.1",
+        "typescript": "^5.3.3",
+        "typescript-eslint": "^7.1.0"
+      },
+      "engines": {
+        "node": ">= 18"
+      },
+      "optionalDependencies": {
+        "@lancedb/lancedb-darwin-arm64": "0.4.3",
+        "@lancedb/lancedb-darwin-x64": "0.4.3",
+        "@lancedb/lancedb-linux-arm64-gnu": "0.4.3",
+        "@lancedb/lancedb-linux-x64-gnu": "0.4.3",
+        "@lancedb/lancedb-win32-x64-msvc": "0.4.3"
+      },
+      "peerDependencies": {
+        "apache-arrow": "^15.0.0"
+      }
+    },
+    "node_modules/@lancedb/lancedb": {
+      "resolved": "../..",
+      "link": true
+    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "node_modules/axios": {
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "dependencies": {
+        "follow-redirects": "^1.14.8"
+      }
+    },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/openai": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz",
+      "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==",
+      "dependencies": {
+        "axios": "^0.26.0",
+        "form-data": "^4.0.0"
+      }
+    }
+  },
+  "dependencies": {
+    "@lancedb/lancedb": {
+      "version": "file:../..",
+      "requires": {
+        "@lancedb/lancedb-darwin-arm64": "0.4.3",
+        "@lancedb/lancedb-darwin-x64": "0.4.3",
+        "@lancedb/lancedb-linux-arm64-gnu": "0.4.3",
+        "@lancedb/lancedb-linux-x64-gnu": "0.4.3",
+        "@lancedb/lancedb-win32-x64-msvc": "0.4.3",
+        "@napi-rs/cli": "^2.18.0",
+        "@types/jest": "^29.1.2",
+        "@types/tmp": "^0.2.6",
+        "@typescript-eslint/eslint-plugin": "^6.19.0",
+        "@typescript-eslint/parser": "^6.19.0",
+        "apache-arrow-old": "npm:apache-arrow@13.0.0",
+        "eslint": "^8.57.0",
+        "eslint-config-prettier": "^9.1.0",
+        "eslint-plugin-jsdoc": "^48.2.1",
+        "jest": "^29.7.0",
+        "openai": "^4.29.2",
+        "prettier": "^3.1.0",
+        "shx": "^0.3.4",
+        "tmp": "^0.2.3",
+        "ts-jest": "^29.1.2",
+        "typedoc": "^0.25.7",
+        "typedoc-plugin-markdown": "^3.17.1",
+        "typescript": "^5.3.3",
+        "typescript-eslint": "^7.1.0"
+      }
+    },
+    "asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
+    },
+    "axios": {
+      "version": "0.26.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz",
+      "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==",
+      "requires": {
+        "follow-redirects": "^1.14.8"
+      }
+    },
+    "combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "requires": {
+        "delayed-stream": "~1.0.0"
+      }
+    },
+    "delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
+    },
+    "follow-redirects": {
+      "version": "1.15.6",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz",
+      "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA=="
+    },
+    "form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "requires": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      }
+    },
+    "mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
+    },
+    "mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "requires": {
+        "mime-db": "1.52.0"
+      }
+    },
+    "openai": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz",
+      "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==",
+      "requires": {
+        "axios": "^0.26.0",
+        "form-data": "^4.0.0"
+      }
+    }
+  }
+}

package/examples/js-openai/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "lancedb-example-js-openai",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.mjs",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Lance Devs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@lancedb/lancedb": "file:../..",
+    "openai": "^3.2.1"
+  }
+}

package/examples/js-transformers/index.mjs

@@ -0,0 +1,65 @@
+// Copyright 2023 Lance Developers.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+"use strict";
+
+/* global console */
+
+import * as lancedb from "@lancedb/lancedb";
+
+async function example() {
+  // Import transformers and the all-MiniLM-L6-v2 model (https://huggingface.co/Xenova/all-MiniLM-L6-v2)
+  const { pipeline } = await import("@xenova/transformers");
+  const pipe = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2");
+
+  // Create embedding function from pipeline which returns a list of vectors from batch
+  // sourceColumn is the name of the column in the data to be embedded
+  //
+  // Output of pipe is a Tensor { data: Float32Array(384) }, so filter for the vector
+  const embedFun = {};
+  embedFun.sourceColumn = "text";
+  embedFun.embed = async function (batch) {
+    let result = [];
+    for (let text of batch) {
+      const res = await pipe(text, { pooling: "mean", normalize: true });
+      result.push(Array.from(res["data"]));
+    }
+    return result;
+  };
+
+  // Link a folder and create a table with data
+  const db = await lancedb.connect("data/sample-lancedb");
+
+  const data = [
+    { id: 1, text: "Cherry", type: "fruit" },
+    { id: 2, text: "Carrot", type: "vegetable" },
+    { id: 3, text: "Potato", type: "vegetable" },
+    { id: 4, text: "Apple", type: "fruit" },
+    { id: 5, text: "Banana", type: "fruit" },
+  ];
+
+  const table = await db.createTable("food_table", data, embedFun);
+
+  // Query the table
+  const results = await table
+    .search("a sweet fruit to eat")
+    .metricType("cosine")
+    .limit(2)
+    .execute();
+  console.log(results.map((r) => r.text));
+}
+
+example()
+  .then(() => console.log("All done!"))
+  .catch(console.error);