lance-csr 0.0.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +55 -1
- package/dist/config.d.ts +73 -0
- package/dist/config.js +120 -0
- package/dist/cypher.d.ts +339 -0
- package/dist/cypher.js +436 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +23 -0
- package/dist/namespace.d.ts +16 -0
- package/dist/namespace.js +21 -0
- package/dist/sql.d.ts +109 -0
- package/dist/sql.js +138 -0
- package/dist/types.d.ts +93 -0
- package/dist/types.js +145 -0
- package/dist/vector.d.ts +64 -0
- package/dist/vector.js +94 -0
- package/native.d.ts +163 -0
- package/native.js +796 -0
- package/package.json +65 -4
package/dist/sql.js
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running SQL queries: one at a time, or many through a cached engine.
|
|
3
|
+
*
|
|
4
|
+
* @module
|
|
5
|
+
*/
|
|
6
|
+
import * as native from "../native.js";
|
|
7
|
+
import { GraphConfig } from "./config.js";
|
|
8
|
+
import { call, datasetInputs, parametersJson, timeoutOption, toTable, } from "./types.js";
|
|
9
|
+
/**
|
|
10
|
+
* The SQL dialect `CypherQuery.toSql` writes: `Default` (generic DataFusion SQL),
|
|
11
|
+
* `Spark`, `PostgreSql`, `MySql` or `Sqlite`.
|
|
12
|
+
*/
|
|
13
|
+
export const SqlDialect = native.SqlDialect;
|
|
14
|
+
/**
|
|
15
|
+
* One SQL query, run against the datasets passed to each call.
|
|
16
|
+
*
|
|
17
|
+
* Each dataset is a table under its lower-cased name, read into memory for the call.
|
|
18
|
+
* Use {@link SqlEngine} to register datasets once and run many queries.
|
|
19
|
+
*/
|
|
20
|
+
export class SqlQuery {
|
|
21
|
+
#sql;
|
|
22
|
+
/** @param sql - The SQL query. */
|
|
23
|
+
constructor(sql) {
|
|
24
|
+
if (typeof sql !== "string") {
|
|
25
|
+
throw new TypeError("sql must be a string");
|
|
26
|
+
}
|
|
27
|
+
this.#sql = sql;
|
|
28
|
+
}
|
|
29
|
+
/** The query text. */
|
|
30
|
+
sql() {
|
|
31
|
+
return this.#sql;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Run the query.
|
|
35
|
+
*
|
|
36
|
+
* @param datasets - The tables, keyed by the names the query uses.
|
|
37
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if the query is invalid, or
|
|
38
|
+
* `Runtime` if execution fails.
|
|
39
|
+
*/
|
|
40
|
+
async execute(datasets) {
|
|
41
|
+
const inputs = datasetInputs(datasets);
|
|
42
|
+
return toTable(await call(() => native.sqlQueryExecute(this.#sql, inputs)));
|
|
43
|
+
}
|
|
44
|
+
/** The logical and physical plans for the query, without running it. */
|
|
45
|
+
async explain(datasets) {
|
|
46
|
+
const inputs = datasetInputs(datasets);
|
|
47
|
+
return call(() => native.sqlQueryExplain(this.#sql, inputs));
|
|
48
|
+
}
|
|
49
|
+
toString() {
|
|
50
|
+
return `SqlQuery(${JSON.stringify(this.#sql)})`;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Run many SQL queries on the same datasets, with graph patterns given a graph.
|
|
55
|
+
*
|
|
56
|
+
* Each dataset is a table under its lower-cased name. Lance datasets are scanned in
|
|
57
|
+
* place, pinned to the version given, with column and filter pushdown; Arrow inputs are
|
|
58
|
+
* held in memory. With a `config`, a `GRAPH_TABLE` clause embeds a Cypher graph pattern,
|
|
59
|
+
* and with `useCsr` its hops run on the CSR. See
|
|
60
|
+
* [Graph patterns in SQL](https://lance-csr.hdclabs.ai/sql/#graph-patterns-in-sql).
|
|
61
|
+
*
|
|
62
|
+
* Plans are cached per snapshot, as `CypherEngine` caches Cypher templates.
|
|
63
|
+
*
|
|
64
|
+
* @example
|
|
65
|
+
* ```ts
|
|
66
|
+
* const engine = await SqlEngine.create({
|
|
67
|
+
* person: tableFromArrays({ id: Int32Array.from([1, 2]), name: ["Alice", "Bob"] }),
|
|
68
|
+
* knows: tableFromArrays({ src: Int32Array.from([1]), dst: Int32Array.from([2]) }),
|
|
69
|
+
* });
|
|
70
|
+
* const result = await engine.execute(
|
|
71
|
+
* "SELECT p.name FROM person p JOIN knows k ON p.id = k.src WHERE k.dst = $dst",
|
|
72
|
+
* { dst: 2 },
|
|
73
|
+
* );
|
|
74
|
+
* ```
|
|
75
|
+
*/
|
|
76
|
+
export class SqlEngine {
|
|
77
|
+
#inner;
|
|
78
|
+
constructor(inner) {
|
|
79
|
+
this.#inner = inner;
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Open the datasets and build the engine.
|
|
83
|
+
*
|
|
84
|
+
* @param datasets - The tables, keyed by the names queries use; with a `config`, also
|
|
85
|
+
* by node label or relationship type, as for `CypherEngine`.
|
|
86
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if `datasets` is empty,
|
|
87
|
+
* `useCsr` is set without a `config`, or a configured column is missing.
|
|
88
|
+
*/
|
|
89
|
+
static async create(datasets, options = {}) {
|
|
90
|
+
const { config, useCsr = false } = options;
|
|
91
|
+
if (config !== undefined && !(config instanceof GraphConfig)) {
|
|
92
|
+
throw new TypeError("config must be a GraphConfig or undefined");
|
|
93
|
+
}
|
|
94
|
+
const inputs = datasetInputs(datasets);
|
|
95
|
+
return new SqlEngine(await call(() => native.openSqlEngine(inputs, config?._spec(), useCsr)));
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* SQL over a `CypherEngine`'s snapshot, sharing its datasets, CSR and statistics.
|
|
99
|
+
*
|
|
100
|
+
* `checkoutLatest` on either engine moves only that engine: the other keeps the
|
|
101
|
+
* snapshot it holds.
|
|
102
|
+
*/
|
|
103
|
+
static fromEngine(engine) {
|
|
104
|
+
// Checked structurally: importing CypherEngine here would make the modules circular.
|
|
105
|
+
if (typeof engine?._inner !== "function") {
|
|
106
|
+
throw new TypeError("engine must be a CypherEngine");
|
|
107
|
+
}
|
|
108
|
+
return new SqlEngine(native.SqlEngine.fromEngine(engine._inner()));
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Run a SQL query and return the whole result.
|
|
112
|
+
*
|
|
113
|
+
* @param sql - The SQL query.
|
|
114
|
+
* @param params - Values for `$name` parameters, as for `CypherEngine.execute`.
|
|
115
|
+
* @param options - `timeoutMs`: milliseconds for the whole query.
|
|
116
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if the query is invalid,
|
|
117
|
+
* `Timeout` past `timeoutMs`, or `Runtime` if execution fails.
|
|
118
|
+
*/
|
|
119
|
+
async execute(sql, params, options = {}) {
|
|
120
|
+
const json = parametersJson(params);
|
|
121
|
+
const timeout = timeoutOption(options.timeoutMs);
|
|
122
|
+
return toTable(await call(() => this.#inner.execute(sql, json, timeout)));
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* The plans for a SQL query, without running it unless `analyze` is set.
|
|
126
|
+
*
|
|
127
|
+
* @returns The logical and physical plans; with `analyze`, the physical plan with
|
|
128
|
+
* each operator's metrics.
|
|
129
|
+
*/
|
|
130
|
+
async explain(sql, params, options = {}) {
|
|
131
|
+
const json = parametersJson(params);
|
|
132
|
+
return call(() => this.#inner.explain(sql, json, options.analyze ?? false));
|
|
133
|
+
}
|
|
134
|
+
/** Move every Lance dataset to its latest version, as `CypherEngine.checkoutLatest` does. */
|
|
135
|
+
async checkoutLatest() {
|
|
136
|
+
await call(() => this.#inner.checkoutLatest());
|
|
137
|
+
}
|
|
138
|
+
}
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types shared by the public classes, and the helpers that convert them for the native module.
|
|
3
|
+
*
|
|
4
|
+
* @module
|
|
5
|
+
*/
|
|
6
|
+
import { RecordBatch, Table } from "apache-arrow";
|
|
7
|
+
import type * as native from "../native.js";
|
|
8
|
+
/**
|
|
9
|
+
* A Lance dataset on local disk or object storage, opened by URI.
|
|
10
|
+
*
|
|
11
|
+
* `CypherEngine` and `SqlEngine` read it lazily, pinned to `version` (the latest when
|
|
12
|
+
* left out). `CypherQuery` and `SqlQuery` read it into memory for each call.
|
|
13
|
+
*/
|
|
14
|
+
export interface LanceDataset {
|
|
15
|
+
/** Where the dataset is, such as `"data/person.lance"` or `"s3://bucket/person.lance"`. */
|
|
16
|
+
uri: string;
|
|
17
|
+
/** The version to read; the latest when left out. */
|
|
18
|
+
version?: number;
|
|
19
|
+
/** Lance storage options, such as object-store credentials and region. */
|
|
20
|
+
storageOptions?: Record<string, string>;
|
|
21
|
+
}
|
|
22
|
+
/** A table to query: an Arrow `Table` or `RecordBatch` held in memory, or a Lance dataset. */
|
|
23
|
+
export type Dataset = Table | RecordBatch | LanceDataset;
|
|
24
|
+
/**
|
|
25
|
+
* A value for a `$name` query parameter.
|
|
26
|
+
*
|
|
27
|
+
* Whole numbers are sent as integers and other numbers as floats. A `bigint` must fit
|
|
28
|
+
* in a signed 64-bit integer. A `Date` is sent as ISO 8601 text, which compares with
|
|
29
|
+
* timestamp and date columns.
|
|
30
|
+
*/
|
|
31
|
+
export type Parameter = number | bigint | string | boolean | Date | null | Parameter[] | {
|
|
32
|
+
[key: string]: Parameter;
|
|
33
|
+
};
|
|
34
|
+
/** A bound parameter value, as `CypherQuery.parameters()` returns it. */
|
|
35
|
+
export type Scalar = number | string | boolean | null | Scalar[] | {
|
|
36
|
+
[key: string]: Scalar;
|
|
37
|
+
};
|
|
38
|
+
/** Statistics for one node index, from `CypherEngine.nodeIndexStats()`. */
|
|
39
|
+
export interface NodeIndexStats {
|
|
40
|
+
/** Indexed node rows. */
|
|
41
|
+
rows: number;
|
|
42
|
+
/** The node dataset version it reflects; absent for Arrow input. */
|
|
43
|
+
version?: number;
|
|
44
|
+
/** Wall time of the build, in milliseconds. */
|
|
45
|
+
buildMs: number;
|
|
46
|
+
}
|
|
47
|
+
/** Statistics for one CSR adjacency index, from `CypherEngine.csrStats()`. */
|
|
48
|
+
export interface CsrStats {
|
|
49
|
+
/** Live indexed edges (null endpoints are skipped). */
|
|
50
|
+
edges: number;
|
|
51
|
+
/** Edges appended since the last full build, held outside the base CSR. */
|
|
52
|
+
deltaEdges: number;
|
|
53
|
+
/** Edges removed since the last full build, held outside the base CSR. */
|
|
54
|
+
tombstones: number;
|
|
55
|
+
/** Live edges outside the base CSR: appended since its last full build and not since removed. */
|
|
56
|
+
unindexedEdges: number;
|
|
57
|
+
/** How the last build, `checkoutLatest()` or `optimizeCsr()` produced it. */
|
|
58
|
+
update: "build" | "rebuild" | "loaded" | "delta" | "reused";
|
|
59
|
+
/** The relationship dataset version it reflects; absent for Arrow input. */
|
|
60
|
+
version?: number;
|
|
61
|
+
/** The relationship dataset version of the last full build; absent for Arrow input. */
|
|
62
|
+
baseVersion?: number;
|
|
63
|
+
/** Wall time of that build or delta, both directions, in milliseconds. */
|
|
64
|
+
buildMs: number;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* The kinds of {@link LanceCsrError}, matching the Python package's exceptions:
|
|
68
|
+
*
|
|
69
|
+
* - `InvalidArgument` (Python `ValueError`): an invalid query, config, parameter or option.
|
|
70
|
+
* - `NotImplemented` (Python `NotImplementedError`): a Cypher feature that parses but isn't supported.
|
|
71
|
+
* - `Timeout` (Python `TimeoutError`): the query ran past its `timeoutMs`.
|
|
72
|
+
* - `Runtime` (Python `RuntimeError`): a dataset couldn't be read, or execution failed.
|
|
73
|
+
*/
|
|
74
|
+
export type ErrorCode = "InvalidArgument" | "NotImplemented" | "Timeout" | "Runtime";
|
|
75
|
+
/** An error from the query engine. Arguments of the wrong type raise `TypeError` instead. */
|
|
76
|
+
export declare class LanceCsrError extends Error {
|
|
77
|
+
/** What went wrong; see {@link ErrorCode}. */
|
|
78
|
+
readonly code: ErrorCode;
|
|
79
|
+
constructor(code: ErrorCode, message: string, options?: ErrorOptions);
|
|
80
|
+
}
|
|
81
|
+
/** Turn the native module's coded errors into `LanceCsrError`; rethrow anything else. */
|
|
82
|
+
export declare function rethrow(err: unknown): never;
|
|
83
|
+
/** Run a native call, converting its errors. */
|
|
84
|
+
export declare function call<T>(run: () => Promise<T>): Promise<T>;
|
|
85
|
+
/** Run a synchronous native call, converting its errors. */
|
|
86
|
+
export declare function callSync<T>(run: () => T): T;
|
|
87
|
+
export declare function toTable(ipc: Uint8Array): Table;
|
|
88
|
+
export declare function arrowToIpc(value: Table | RecordBatch, name: string): Buffer;
|
|
89
|
+
/** Convert the `datasets` argument for the native module. */
|
|
90
|
+
export declare function datasetInputs(datasets: Record<string, Dataset>): Record<string, native.DatasetInput>;
|
|
91
|
+
/** Serialize `$name` parameter values as JSON for the native module, checking each value. */
|
|
92
|
+
export declare function parametersJson(params: Record<string, Parameter> | undefined | null): string | undefined;
|
|
93
|
+
export declare function timeoutOption(timeoutMs: number | undefined | null): number | undefined;
|
package/dist/types.js
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Types shared by the public classes, and the helpers that convert them for the native module.
|
|
3
|
+
*
|
|
4
|
+
* @module
|
|
5
|
+
*/
|
|
6
|
+
import { RecordBatch, Table, tableFromIPC, tableToIPC } from "apache-arrow";
|
|
7
|
+
/** An error from the query engine. Arguments of the wrong type raise `TypeError` instead. */
|
|
8
|
+
export class LanceCsrError extends Error {
|
|
9
|
+
/** What went wrong; see {@link ErrorCode}. */
|
|
10
|
+
code;
|
|
11
|
+
constructor(code, message, options) {
|
|
12
|
+
super(message, options);
|
|
13
|
+
this.name = "LanceCsrError";
|
|
14
|
+
this.code = code;
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
const CODED = /^\[(InvalidArgument|NotImplemented|Timeout|Runtime)\] ([\s\S]*)$/;
|
|
18
|
+
/** Turn the native module's coded errors into `LanceCsrError`; rethrow anything else. */
|
|
19
|
+
export function rethrow(err) {
|
|
20
|
+
const match = err instanceof Error ? CODED.exec(err.message) : null;
|
|
21
|
+
if (match) {
|
|
22
|
+
throw new LanceCsrError(match[1], match[2], { cause: err });
|
|
23
|
+
}
|
|
24
|
+
throw err;
|
|
25
|
+
}
|
|
26
|
+
/** Run a native call, converting its errors. */
|
|
27
|
+
export async function call(run) {
|
|
28
|
+
try {
|
|
29
|
+
return await run();
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
rethrow(err);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
/** Run a synchronous native call, converting its errors. */
|
|
36
|
+
export function callSync(run) {
|
|
37
|
+
try {
|
|
38
|
+
return run();
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
rethrow(err);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
export function toTable(ipc) {
|
|
45
|
+
return tableFromIPC(ipc);
|
|
46
|
+
}
|
|
47
|
+
export function arrowToIpc(value, name) {
|
|
48
|
+
if (value instanceof Table) {
|
|
49
|
+
return Buffer.from(tableToIPC(value, "stream"));
|
|
50
|
+
}
|
|
51
|
+
if (value instanceof RecordBatch) {
|
|
52
|
+
return Buffer.from(tableToIPC(new Table(value), "stream"));
|
|
53
|
+
}
|
|
54
|
+
throw new TypeError(`${name} must be an apache-arrow Table or RecordBatch`);
|
|
55
|
+
}
|
|
56
|
+
function isLanceDataset(value) {
|
|
57
|
+
return (typeof value === "object" &&
|
|
58
|
+
value !== null &&
|
|
59
|
+
typeof value.uri === "string");
|
|
60
|
+
}
|
|
61
|
+
/** Convert the `datasets` argument for the native module. */
|
|
62
|
+
export function datasetInputs(datasets) {
|
|
63
|
+
if (typeof datasets !== "object" || datasets === null) {
|
|
64
|
+
throw new TypeError("datasets must be an object keyed by node label or relationship type");
|
|
65
|
+
}
|
|
66
|
+
const inputs = {};
|
|
67
|
+
for (const [name, value] of Object.entries(datasets)) {
|
|
68
|
+
if (value instanceof Table || value instanceof RecordBatch) {
|
|
69
|
+
inputs[name] = { ipc: arrowToIpc(value, `datasets.${name}`) };
|
|
70
|
+
}
|
|
71
|
+
else if (isLanceDataset(value)) {
|
|
72
|
+
const { uri, version, storageOptions } = value;
|
|
73
|
+
inputs[name] = { uri, version, storageOptions };
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
throw new TypeError(`datasets.${name} must be an apache-arrow Table or RecordBatch, or a LanceDataset ({ uri })` +
|
|
77
|
+
" (if it is a Table, check that only one copy of apache-arrow is installed)");
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return inputs;
|
|
81
|
+
}
|
|
82
|
+
const MIN_I64 = -(2n ** 63n);
|
|
83
|
+
const MAX_I64 = 2n ** 63n - 1n;
|
|
84
|
+
// `JSON.rawJSON` (Node 21+) keeps a bigint's digits exact in the JSON text.
|
|
85
|
+
const rawJSON = JSON.rawJSON;
|
|
86
|
+
function invalidParameter(message) {
|
|
87
|
+
return new LanceCsrError("InvalidArgument", message);
|
|
88
|
+
}
|
|
89
|
+
/** Serialize `$name` parameter values as JSON for the native module, checking each value. */
|
|
90
|
+
export function parametersJson(params) {
|
|
91
|
+
if (params === undefined || params === null) {
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
if (typeof params !== "object" || Array.isArray(params)) {
|
|
95
|
+
throw new TypeError("params must be an object keyed by parameter name");
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
// `Date` values reach the replacer already converted to ISO text by their toJSON.
|
|
99
|
+
return JSON.stringify(params, (_key, value) => {
|
|
100
|
+
switch (typeof value) {
|
|
101
|
+
case "bigint":
|
|
102
|
+
if (value < MIN_I64 || value > MAX_I64) {
|
|
103
|
+
throw invalidParameter("Integer parameters must fit signed 64-bit values");
|
|
104
|
+
}
|
|
105
|
+
return rawJSON(value.toString());
|
|
106
|
+
case "number":
|
|
107
|
+
if (!Number.isFinite(value)) {
|
|
108
|
+
throw invalidParameter("Invalid float value");
|
|
109
|
+
}
|
|
110
|
+
return value;
|
|
111
|
+
case "undefined":
|
|
112
|
+
return null;
|
|
113
|
+
case "function":
|
|
114
|
+
case "symbol":
|
|
115
|
+
throw invalidParameter("Unsupported parameter type");
|
|
116
|
+
case "object":
|
|
117
|
+
if (value !== null && !Array.isArray(value)) {
|
|
118
|
+
const proto = Object.getPrototypeOf(value);
|
|
119
|
+
if (proto !== Object.prototype && proto !== null) {
|
|
120
|
+
throw invalidParameter("Unsupported parameter type");
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return value;
|
|
124
|
+
default:
|
|
125
|
+
return value;
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
catch (err) {
|
|
130
|
+
if (err instanceof LanceCsrError) {
|
|
131
|
+
throw err;
|
|
132
|
+
}
|
|
133
|
+
// JSON.stringify throws a TypeError on cycles.
|
|
134
|
+
throw invalidParameter(`Parameters can't be serialized (possibly a cyclic container): ${err.message}`);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
export function timeoutOption(timeoutMs) {
|
|
138
|
+
if (timeoutMs === undefined || timeoutMs === null) {
|
|
139
|
+
return undefined;
|
|
140
|
+
}
|
|
141
|
+
if (typeof timeoutMs !== "number") {
|
|
142
|
+
throw new TypeError("timeoutMs must be a number or undefined");
|
|
143
|
+
}
|
|
144
|
+
return timeoutMs;
|
|
145
|
+
}
|
package/dist/vector.d.ts
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ranking query results by vector similarity.
|
|
3
|
+
*
|
|
4
|
+
* @module
|
|
5
|
+
*/
|
|
6
|
+
import type { RecordBatch, Table } from "apache-arrow";
|
|
7
|
+
import * as native from "../native.js";
|
|
8
|
+
/**
|
|
9
|
+
* The distance {@link VectorSearch} ranks rows by; smaller is closer. `L2` is Euclidean
|
|
10
|
+
* distance, `Cosine` is 1 − cosine similarity and `Dot` is the negative dot product.
|
|
11
|
+
*/
|
|
12
|
+
export declare const DistanceMetric: typeof native.DistanceMetric;
|
|
13
|
+
export type DistanceMetric = native.DistanceMetric;
|
|
14
|
+
/**
|
|
15
|
+
* An immutable builder for vector similarity search.
|
|
16
|
+
*
|
|
17
|
+
* Every method except {@link VectorSearch.search} returns a new builder. Pass the
|
|
18
|
+
* builder to `search` to rank an Arrow table, or to `executeWithVectorRerank` on
|
|
19
|
+
* `CypherEngine` or `CypherQuery` to rerank a Cypher result. This is separate from the
|
|
20
|
+
* `vector_distance()` and `vector_similarity()` Cypher functions. See
|
|
21
|
+
* [Vector search](https://lance-csr.hdclabs.ai/vector-search/).
|
|
22
|
+
*
|
|
23
|
+
* @example
|
|
24
|
+
* ```ts
|
|
25
|
+
* const search = new VectorSearch("d.embedding")
|
|
26
|
+
* .queryVector([0.1, 0.2, 0.3])
|
|
27
|
+
* .metric(DistanceMetric.Cosine)
|
|
28
|
+
* .topK(5);
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
export declare class VectorSearch {
|
|
32
|
+
#private;
|
|
33
|
+
/** @param column - The vector column to rank by, such as `"embedding"` or `"d.embedding"`. */
|
|
34
|
+
constructor(column: string);
|
|
35
|
+
/** Set the vector to compare with. Required before searching. */
|
|
36
|
+
queryVector(vector: ArrayLike<number>): VectorSearch;
|
|
37
|
+
/** Set the distance metric; the default is `DistanceMetric.L2`. */
|
|
38
|
+
metric(metric: DistanceMetric): VectorSearch;
|
|
39
|
+
/** Keep at most `k` rows; the default is 10. */
|
|
40
|
+
topK(k: number): VectorSearch;
|
|
41
|
+
/** Add the distance as a column (the default), or leave it out. */
|
|
42
|
+
includeDistance(include: boolean): VectorSearch;
|
|
43
|
+
/** Name the distance column; the default is `"_distance"`. */
|
|
44
|
+
distanceColumnName(name: string): VectorSearch;
|
|
45
|
+
/**
|
|
46
|
+
* Set whether reranking queries a Lance ANN index first; `false` by default.
|
|
47
|
+
*
|
|
48
|
+
* When the datasets are Lance datasets, `CypherQuery.executeWithVectorRerank`
|
|
49
|
+
* searches the Lance approximate-nearest-neighbour index first and then runs the
|
|
50
|
+
* Cypher query on the top-k rows. This can be much faster on large datasets, but it
|
|
51
|
+
* can change the results when the query also filters, because the filter then applies
|
|
52
|
+
* to the top-k rows only.
|
|
53
|
+
*/
|
|
54
|
+
useLanceIndex(enabled: boolean): VectorSearch;
|
|
55
|
+
/**
|
|
56
|
+
* Brute-force search `table`.
|
|
57
|
+
*
|
|
58
|
+
* @param table - The rows to search, with the vector column.
|
|
59
|
+
* @returns At most `topK` rows, sorted by distance, closest first.
|
|
60
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if no query vector is set or
|
|
61
|
+
* the column doesn't exist, or `Runtime` if the search fails.
|
|
62
|
+
*/
|
|
63
|
+
search(table: Table | RecordBatch): Promise<Table>;
|
|
64
|
+
}
|
package/dist/vector.js
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Ranking query results by vector similarity.
|
|
3
|
+
*
|
|
4
|
+
* @module
|
|
5
|
+
*/
|
|
6
|
+
var _a;
|
|
7
|
+
import * as native from "../native.js";
|
|
8
|
+
import { arrowToIpc, call, toTable } from "./types.js";
|
|
9
|
+
/**
|
|
10
|
+
* The distance {@link VectorSearch} ranks rows by; smaller is closer. `L2` is Euclidean
|
|
11
|
+
* distance, `Cosine` is 1 − cosine similarity and `Dot` is the negative dot product.
|
|
12
|
+
*/
|
|
13
|
+
export const DistanceMetric = native.DistanceMetric;
|
|
14
|
+
/**
|
|
15
|
+
* An immutable builder for vector similarity search.
|
|
16
|
+
*
|
|
17
|
+
* Every method except {@link VectorSearch.search} returns a new builder. Pass the
|
|
18
|
+
* builder to `search` to rank an Arrow table, or to `executeWithVectorRerank` on
|
|
19
|
+
* `CypherEngine` or `CypherQuery` to rerank a Cypher result. This is separate from the
|
|
20
|
+
* `vector_distance()` and `vector_similarity()` Cypher functions. See
|
|
21
|
+
* [Vector search](https://lance-csr.hdclabs.ai/vector-search/).
|
|
22
|
+
*
|
|
23
|
+
* @example
|
|
24
|
+
* ```ts
|
|
25
|
+
* const search = new VectorSearch("d.embedding")
|
|
26
|
+
* .queryVector([0.1, 0.2, 0.3])
|
|
27
|
+
* .metric(DistanceMetric.Cosine)
|
|
28
|
+
* .topK(5);
|
|
29
|
+
* ```
|
|
30
|
+
*/
|
|
31
|
+
export class VectorSearch {
|
|
32
|
+
#spec;
|
|
33
|
+
/** @param column - The vector column to rank by, such as `"embedding"` or `"d.embedding"`. */
|
|
34
|
+
constructor(column) {
|
|
35
|
+
if (typeof column !== "string") {
|
|
36
|
+
throw new TypeError("column must be a string");
|
|
37
|
+
}
|
|
38
|
+
this.#spec = { column };
|
|
39
|
+
}
|
|
40
|
+
#with(change) {
|
|
41
|
+
const search = new _a(this.#spec.column);
|
|
42
|
+
Object.assign(search.#spec, this.#spec, change);
|
|
43
|
+
return search;
|
|
44
|
+
}
|
|
45
|
+
/** @internal The spec the native module rebuilds the search from. */
|
|
46
|
+
_spec() {
|
|
47
|
+
return { ...this.#spec };
|
|
48
|
+
}
|
|
49
|
+
/** Set the vector to compare with. Required before searching. */
|
|
50
|
+
queryVector(vector) {
|
|
51
|
+
return this.#with({ queryVector: Array.from(vector) });
|
|
52
|
+
}
|
|
53
|
+
/** Set the distance metric; the default is `DistanceMetric.L2`. */
|
|
54
|
+
metric(metric) {
|
|
55
|
+
return this.#with({ metric });
|
|
56
|
+
}
|
|
57
|
+
/** Keep at most `k` rows; the default is 10. */
|
|
58
|
+
topK(k) {
|
|
59
|
+
return this.#with({ topK: k });
|
|
60
|
+
}
|
|
61
|
+
/** Add the distance as a column (the default), or leave it out. */
|
|
62
|
+
includeDistance(include) {
|
|
63
|
+
return this.#with({ includeDistance: include });
|
|
64
|
+
}
|
|
65
|
+
/** Name the distance column; the default is `"_distance"`. */
|
|
66
|
+
distanceColumnName(name) {
|
|
67
|
+
return this.#with({ distanceColumnName: name });
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Set whether reranking queries a Lance ANN index first; `false` by default.
|
|
71
|
+
*
|
|
72
|
+
* When the datasets are Lance datasets, `CypherQuery.executeWithVectorRerank`
|
|
73
|
+
* searches the Lance approximate-nearest-neighbour index first and then runs the
|
|
74
|
+
* Cypher query on the top-k rows. This can be much faster on large datasets, but it
|
|
75
|
+
* can change the results when the query also filters, because the filter then applies
|
|
76
|
+
* to the top-k rows only.
|
|
77
|
+
*/
|
|
78
|
+
useLanceIndex(enabled) {
|
|
79
|
+
return this.#with({ useLanceIndex: enabled });
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Brute-force search `table`.
|
|
83
|
+
*
|
|
84
|
+
* @param table - The rows to search, with the vector column.
|
|
85
|
+
* @returns At most `topK` rows, sorted by distance, closest first.
|
|
86
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if no query vector is set or
|
|
87
|
+
* the column doesn't exist, or `Runtime` if the search fails.
|
|
88
|
+
*/
|
|
89
|
+
async search(table) {
|
|
90
|
+
const ipc = arrowToIpc(table, "table");
|
|
91
|
+
return toTable(await call(() => native.vectorSearchSearch(this._spec(), ipc)));
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
_a = VectorSearch;
|