lance-csr 0.0.0 → 0.13.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +55 -1
- package/dist/config.d.ts +73 -0
- package/dist/config.js +120 -0
- package/dist/cypher.d.ts +339 -0
- package/dist/cypher.js +436 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +23 -0
- package/dist/namespace.d.ts +16 -0
- package/dist/namespace.js +21 -0
- package/dist/sql.d.ts +109 -0
- package/dist/sql.js +138 -0
- package/dist/types.d.ts +93 -0
- package/dist/types.js +145 -0
- package/dist/vector.d.ts +64 -0
- package/dist/vector.js +94 -0
- package/native.d.ts +163 -0
- package/native.js +796 -0
- package/package.json +65 -4
package/dist/cypher.d.ts
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Running Cypher queries: one at a time, or many through a cached engine.
|
|
3
|
+
*
|
|
4
|
+
* @module
|
|
5
|
+
*/
|
|
6
|
+
import { type RecordBatch, type Schema, type Table } from "apache-arrow";
|
|
7
|
+
import * as native from "../native.js";
|
|
8
|
+
import { GraphConfig } from "./config.ts";
|
|
9
|
+
import type { SqlDialect } from "./sql.ts";
|
|
10
|
+
import { DirNamespace } from "./namespace.ts";
|
|
11
|
+
import { type CsrStats, type Dataset, type NodeIndexStats, type Parameter, type Scalar } from "./types.ts";
|
|
12
|
+
import { VectorSearch } from "./vector.ts";
|
|
13
|
+
/**
|
|
14
|
+
* How `CypherQuery.execute` runs a query. `DataFusion` (the default and only implemented
|
|
15
|
+
* one) plans and runs it with DataFusion; `LanceNative` is reserved and fails with
|
|
16
|
+
* `NotImplemented`.
|
|
17
|
+
*/
|
|
18
|
+
export declare const ExecutionStrategy: typeof native.ExecutionStrategy;
|
|
19
|
+
export type ExecutionStrategy = native.ExecutionStrategy;
|
|
20
|
+
/** Options for {@link CypherEngine.create}. */
|
|
21
|
+
export interface CypherEngineOptions {
|
|
22
|
+
/**
|
|
23
|
+
* Build the experimental [CSR index](https://lance-csr.hdclabs.ai/csr/) for relationship
|
|
24
|
+
* types with integer endpoints, and node indexes for labels with unique integer keys.
|
|
25
|
+
*/
|
|
26
|
+
useCsr?: boolean;
|
|
27
|
+
/**
|
|
28
|
+
* A target from 1 to 1024 for DataFusion query partitioning; left out, DataFusion's
|
|
29
|
+
* default. `checkoutLatest` preserves it. It is not a thread or memory limit.
|
|
30
|
+
*/
|
|
31
|
+
targetPartitions?: number;
|
|
32
|
+
/**
|
|
33
|
+
* Experimental, needs `useCsr`. Store each relationship dataset's CSR inside the
|
|
34
|
+
* dataset's own directory, at `_lance_csr/<version>.arrow`, and load it instead of
|
|
35
|
+
* building when an engine starts. Changes since the stored version apply as a delta
|
|
36
|
+
* under the same 25% threshold as `checkoutLatest`; past it, the CSR is rebuilt and
|
|
37
|
+
* stored again. Arrow inputs aren't stored, and a dataset that can't be written is
|
|
38
|
+
* built as without it.
|
|
39
|
+
*/
|
|
40
|
+
persistCsr?: boolean;
|
|
41
|
+
}
|
|
42
|
+
/** Options for running a query. */
|
|
43
|
+
export interface ExecuteOptions {
|
|
44
|
+
/** Milliseconds for the whole query; left out, no deadline. */
|
|
45
|
+
timeoutMs?: number;
|
|
46
|
+
}
|
|
47
|
+
/** Options for explaining a query. */
|
|
48
|
+
export interface ExplainOptions {
|
|
49
|
+
/**
|
|
50
|
+
* Run the query, like SQL's `EXPLAIN ANALYZE`, and show each physical operator's
|
|
51
|
+
* metrics (output rows, elapsed compute time and more) in place of the plain physical
|
|
52
|
+
* plan. The query runs to completion; its rows are discarded.
|
|
53
|
+
*/
|
|
54
|
+
analyze?: boolean;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* A query result read one batch at a time: an async iterable of Arrow record batches.
|
|
58
|
+
*
|
|
59
|
+
* Batches are computed as they are read. Scans, filters and projections stream with
|
|
60
|
+
* bounded memory; joins, sorts and aggregations may still buffer. The stream keeps its
|
|
61
|
+
* snapshot alive, so a concurrent `checkoutLatest` doesn't affect it. Breaking out of
|
|
62
|
+
* a `for await` loop, or calling {@link RecordBatchStream.close}, stops the query.
|
|
63
|
+
*
|
|
64
|
+
* @example
|
|
65
|
+
* ```ts
|
|
66
|
+
* const stream = await engine.executeBatches("MATCH (p:Person) RETURN p.name");
|
|
67
|
+
* for await (const batch of stream) {
|
|
68
|
+
* console.log(batch.numRows);
|
|
69
|
+
* }
|
|
70
|
+
* ```
|
|
71
|
+
*/
|
|
72
|
+
export declare class RecordBatchStream implements AsyncIterable<RecordBatch> {
|
|
73
|
+
#private;
|
|
74
|
+
/** The schema of every batch. */
|
|
75
|
+
readonly schema: Schema;
|
|
76
|
+
/**
|
|
77
|
+
* The next batch, or `null` at the end.
|
|
78
|
+
*
|
|
79
|
+
* @throws {@link LanceCsrError} with code `Timeout` once the query's deadline has
|
|
80
|
+
* passed, or `Runtime` if execution fails.
|
|
81
|
+
*/
|
|
82
|
+
next(): Promise<RecordBatch | null>;
|
|
83
|
+
/** Stop the query and release what it holds. Reading afterwards returns `null`. */
|
|
84
|
+
close(): Promise<void>;
|
|
85
|
+
[Symbol.asyncIterator](): AsyncIterator<RecordBatch>;
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Run many Cypher queries over the same datasets.
|
|
89
|
+
*
|
|
90
|
+
* The engine registers `datasets` once and reuses the catalog, the DataFusion session
|
|
91
|
+
* and (with `useCsr`) the CSR index across queries. Queries can run concurrently.
|
|
92
|
+
*
|
|
93
|
+
* Lance datasets are read lazily, pinned to the version given (the latest when left
|
|
94
|
+
* out): later writes, by this or another process, aren't visible until
|
|
95
|
+
* {@link CypherEngine.checkoutLatest}. Each dataset is pinned separately, so versions
|
|
96
|
+
* across tables are not a transactional snapshot. Arrow inputs are held in memory and
|
|
97
|
+
* never change; create a new engine to replace them.
|
|
98
|
+
*
|
|
99
|
+
* `execute` and `executeBatches` reuse up to 128 unoptimized query templates per
|
|
100
|
+
* snapshot, keyed by the exact query text and the names and types of scalar
|
|
101
|
+
* parameters. Values are bound before optimization on every call. A successful
|
|
102
|
+
* `checkoutLatest` clears the cache. See
|
|
103
|
+
* [Repeated queries](https://lance-csr.hdclabs.ai/lance/#repeated-queries).
|
|
104
|
+
*
|
|
105
|
+
* @example
|
|
106
|
+
* ```ts
|
|
107
|
+
* import { tableFromArrays } from "apache-arrow";
|
|
108
|
+
* import { CypherEngine, GraphConfig } from "lance-csr";
|
|
109
|
+
*
|
|
110
|
+
* const config = GraphConfig.builder()
|
|
111
|
+
* .nodeTable("Person", "id")
|
|
112
|
+
* .relTable("KNOWS", "Person", "Person", "src_id", "dst_id")
|
|
113
|
+
* .build();
|
|
114
|
+
* const engine = await CypherEngine.create(config, {
|
|
115
|
+
* Person: tableFromArrays({ id: Int32Array.from([1, 2]), name: ["Alice", "Bob"] }),
|
|
116
|
+
* KNOWS: tableFromArrays({ src_id: Int32Array.from([1]), dst_id: Int32Array.from([2]) }),
|
|
117
|
+
* });
|
|
118
|
+
* const result = await engine.execute(
|
|
119
|
+
* "MATCH (a:Person)-[:KNOWS]->(b:Person) RETURN a.name, b.name",
|
|
120
|
+
* );
|
|
121
|
+
* result.toArray().map((row) => row.toJSON()); // [{ "a.name": "Alice", "b.name": "Bob" }]
|
|
122
|
+
* ```
|
|
123
|
+
*/
|
|
124
|
+
export declare class CypherEngine {
|
|
125
|
+
#private;
|
|
126
|
+
private constructor();
|
|
127
|
+
/**
|
|
128
|
+
* Open the datasets and build the engine. Lance datasets are pinned, not read; with
|
|
129
|
+
* `useCsr` the indexes are built here, on a background thread.
|
|
130
|
+
*
|
|
131
|
+
* @param config - The graph mapping.
|
|
132
|
+
* @param datasets - The tables, keyed by node label or relationship type.
|
|
133
|
+
* @param options - CSR and partitioning options.
|
|
134
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if `datasets` is empty, a
|
|
135
|
+
* configured key or endpoint column is missing, `targetPartitions` is outside 1 to
|
|
136
|
+
* 1024, or `persistCsr` is set without `useCsr`; `Runtime` if a dataset can't be
|
|
137
|
+
* opened.
|
|
138
|
+
* @throws `TypeError` if `config` isn't a `GraphConfig` or a dataset has an
|
|
139
|
+
* unsupported type.
|
|
140
|
+
*/
|
|
141
|
+
static create(config: GraphConfig, datasets: Record<string, Dataset>, options?: CypherEngineOptions): Promise<CypherEngine>;
|
|
142
|
+
/**
|
|
143
|
+
* Run a Cypher query and return the whole result.
|
|
144
|
+
*
|
|
145
|
+
* The result is held in memory; use {@link CypherEngine.executeBatches} for large
|
|
146
|
+
* outputs.
|
|
147
|
+
*
|
|
148
|
+
* @param query - The Cypher query.
|
|
149
|
+
* @param params - Values for `$name` parameters; see {@link Parameter}.
|
|
150
|
+
* @param options - `timeoutMs`: milliseconds for the whole query.
|
|
151
|
+
* @returns The query result.
|
|
152
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if the query is invalid,
|
|
153
|
+
* refers to an unknown label, compares or combines incompatible types, or a parameter
|
|
154
|
+
* is unsupported, nested beyond 128 levels or cyclic; `NotImplemented` if the query
|
|
155
|
+
* uses a Cypher feature that parses but isn't supported; `Timeout` if it runs past
|
|
156
|
+
* `timeoutMs`; `Runtime` if execution fails.
|
|
157
|
+
*/
|
|
158
|
+
execute(query: string, params?: Record<string, Parameter> | null, options?: ExecuteOptions): Promise<Table>;
|
|
159
|
+
/**
|
|
160
|
+
* Run a Cypher query and stream the result.
|
|
161
|
+
*
|
|
162
|
+
* Planning errors reject the returned promise; execution errors reject while reading.
|
|
163
|
+
*
|
|
164
|
+
* @param query - The Cypher query.
|
|
165
|
+
* @param params - Values for `$name` parameters, as for {@link CypherEngine.execute}.
|
|
166
|
+
* @param options - `timeoutMs`: milliseconds for the whole query, including pauses
|
|
167
|
+
* between reads.
|
|
168
|
+
* @returns The result, one batch at a time.
|
|
169
|
+
*/
|
|
170
|
+
executeBatches(query: string, params?: Record<string, Parameter> | null, options?: ExecuteOptions): Promise<RecordBatchStream>;
|
|
171
|
+
/**
|
|
172
|
+
* The plans for a query, without running it unless `analyze` is set.
|
|
173
|
+
*
|
|
174
|
+
* @param query - The Cypher query.
|
|
175
|
+
* @param params - Values for `$name` parameters, as for {@link CypherEngine.execute}.
|
|
176
|
+
* @param options - `analyze`: run the query and show each operator's metrics.
|
|
177
|
+
* @returns The graph logical plan, the DataFusion logical plan and the physical plan,
|
|
178
|
+
* including pushed-down scans. With `analyze`, the physical plan's row is labelled
|
|
179
|
+
* `physical_plan_with_metrics`.
|
|
180
|
+
*/
|
|
181
|
+
explain(query: string, params?: Record<string, Parameter> | null, options?: ExplainOptions): Promise<string>;
|
|
182
|
+
/**
|
|
183
|
+
* Move every Lance dataset to its latest version.
|
|
184
|
+
*
|
|
185
|
+
* Updates the CSR, node indexes and statistics to match. Datasets whose version and
|
|
186
|
+
* commit time are unchanged keep their open handle. The replacement is validated in
|
|
187
|
+
* full before it's swapped in: on error nothing changes and the previous snapshot
|
|
188
|
+
* stays usable. Queries already running finish on the snapshot they started with.
|
|
189
|
+
* Arrow inputs are unchanged. Concurrent calls run one at a time, and
|
|
190
|
+
* `targetPartitions` is preserved. See
|
|
191
|
+
* [Lance datasets](https://lance-csr.hdclabs.ai/lance/#checkout-latest).
|
|
192
|
+
*/
|
|
193
|
+
checkoutLatest(): Promise<void>;
|
|
194
|
+
/**
|
|
195
|
+
* Rebuild each CSR that carries a delta (experimental; needs `useCsr`).
|
|
196
|
+
*
|
|
197
|
+
* `checkoutLatest` applies Lance writes as a delta over the base CSR, and rebuilds on
|
|
198
|
+
* its own only once pending changes pass 25% of the base. This rebuilds every CSR that
|
|
199
|
+
* carries a delta now, at the version the engine is pinned to. It scans every edge
|
|
200
|
+
* again, so call it after a bulk load or at a quiet time. It doesn't move to newer
|
|
201
|
+
* versions: call `checkoutLatest` first for that. Query results never change. See
|
|
202
|
+
* [Rebuilding the CSR](https://lance-csr.hdclabs.ai/csr/#rebuilding-the-csr).
|
|
203
|
+
*/
|
|
204
|
+
optimizeCsr(): Promise<void>;
|
|
205
|
+
/**
|
|
206
|
+
* Delete the CSR files stored with `persistCsr` (experimental).
|
|
207
|
+
*
|
|
208
|
+
* Removes the `_lance_csr` folder of every Lance relationship dataset this engine
|
|
209
|
+
* reads. The engine keeps the CSR it already holds in memory, so queries aren't
|
|
210
|
+
* affected. See [Storing the CSR](https://lance-csr.hdclabs.ai/csr/#storing-the-csr).
|
|
211
|
+
*
|
|
212
|
+
* @returns The number of files deleted; 0 when nothing was stored.
|
|
213
|
+
*/
|
|
214
|
+
deleteStoredCsr(): Promise<number>;
|
|
215
|
+
/**
|
|
216
|
+
* Statistics for each node index (experimental; needs `useCsr`).
|
|
217
|
+
*
|
|
218
|
+
* A label is missing when its keys aren't unique integers; relationships that end at
|
|
219
|
+
* it then use a join.
|
|
220
|
+
*/
|
|
221
|
+
nodeIndexStats(): Record<string, NodeIndexStats>;
|
|
222
|
+
/**
|
|
223
|
+
* Statistics for each CSR adjacency index (experimental; needs `useCsr`), keyed by
|
|
224
|
+
* relationship type; empty without `useCsr`. An identical result after
|
|
225
|
+
* `checkoutLatest` means the snapshot itself didn't change. See
|
|
226
|
+
* [Inspecting the CSR](https://lance-csr.hdclabs.ai/csr/#inspecting-the-csr).
|
|
227
|
+
*/
|
|
228
|
+
csrStats(): Record<string, CsrStats>;
|
|
229
|
+
/** The pinned version of each Lance dataset, keyed by dataset name. Arrow inputs aren't listed. */
|
|
230
|
+
datasetVersions(): Record<string, number>;
|
|
231
|
+
/** The engine's graph config. */
|
|
232
|
+
config(): GraphConfig;
|
|
233
|
+
/**
|
|
234
|
+
* Run a Cypher query, then rank its rows by vector similarity.
|
|
235
|
+
*
|
|
236
|
+
* @param query - The Cypher query. It must return the column `vectorSearch` ranks by.
|
|
237
|
+
* @param vectorSearch - The ranking.
|
|
238
|
+
* @returns At most `topK` rows, closest first.
|
|
239
|
+
*/
|
|
240
|
+
executeWithVectorRerank(query: string, vectorSearch: VectorSearch): Promise<Table>;
|
|
241
|
+
}
|
|
242
|
+
/**
|
|
243
|
+
* One parsed Cypher query, run against the datasets passed to each call.
|
|
244
|
+
*
|
|
245
|
+
* Builder methods return a new query. Datasets are read into memory for each call; use
|
|
246
|
+
* {@link CypherEngine} to register them once and run many queries.
|
|
247
|
+
*
|
|
248
|
+
* @example
|
|
249
|
+
* ```ts
|
|
250
|
+
* const result = await new CypherQuery("MATCH (p:Person) WHERE p.age > $min RETURN p.name")
|
|
251
|
+
* .withConfig(config)
|
|
252
|
+
* .withParameter("min", 30)
|
|
253
|
+
* .execute({ Person: people });
|
|
254
|
+
* ```
|
|
255
|
+
*/
|
|
256
|
+
export declare class CypherQuery {
|
|
257
|
+
#private;
|
|
258
|
+
/**
|
|
259
|
+
* @param queryText - The Cypher query.
|
|
260
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if `queryText` doesn't parse.
|
|
261
|
+
*/
|
|
262
|
+
constructor(queryText: string);
|
|
263
|
+
/**
|
|
264
|
+
* Set the graph config. Required before running the query.
|
|
265
|
+
*
|
|
266
|
+
* @returns A new query with the config set.
|
|
267
|
+
*/
|
|
268
|
+
withConfig(config: GraphConfig): CypherQuery;
|
|
269
|
+
/**
|
|
270
|
+
* Bind `$key` to `value`.
|
|
271
|
+
*
|
|
272
|
+
* @param key - The parameter name, without `$`.
|
|
273
|
+
* @param value - The value, of a type that {@link CypherEngine.execute} accepts in `params`.
|
|
274
|
+
* @returns A new query with the parameter bound.
|
|
275
|
+
*/
|
|
276
|
+
withParameter(key: string, value: Parameter): CypherQuery;
|
|
277
|
+
/** The query text. */
|
|
278
|
+
queryText(): string;
|
|
279
|
+
/** The bound parameters, as the engine receives them (a `Date` becomes ISO text). */
|
|
280
|
+
parameters(): Record<string, Scalar>;
|
|
281
|
+
/**
|
|
282
|
+
* Run the query.
|
|
283
|
+
*
|
|
284
|
+
* @param datasets - The tables, keyed by node label or relationship type. They are
|
|
285
|
+
* read into memory for this call.
|
|
286
|
+
* @param options - `strategy` (default `ExecutionStrategy.DataFusion`) and `timeoutMs`.
|
|
287
|
+
* @returns The query result.
|
|
288
|
+
* @throws {@link LanceCsrError} with code `InvalidArgument` if the query is invalid or a
|
|
289
|
+
* dataset is missing; `NotImplemented` for an unsupported Cypher feature or
|
|
290
|
+
* `ExecutionStrategy.LanceNative`; `Timeout` past `timeoutMs`; `Runtime` if execution
|
|
291
|
+
* fails.
|
|
292
|
+
*/
|
|
293
|
+
execute(datasets: Record<string, Dataset>, options?: ExecuteOptions & {
|
|
294
|
+
strategy?: ExecutionStrategy;
|
|
295
|
+
}): Promise<Table>;
|
|
296
|
+
/**
|
|
297
|
+
* Run the query, reading each label from `<baseUri>/<label>.lance`.
|
|
298
|
+
*
|
|
299
|
+
* @param namespace - Where the datasets are.
|
|
300
|
+
* @param strategy - How to run the query; the default is `ExecutionStrategy.DataFusion`.
|
|
301
|
+
*/
|
|
302
|
+
executeWithNamespace(namespace: DirNamespace, strategy?: ExecutionStrategy): Promise<Table>;
|
|
303
|
+
/**
|
|
304
|
+
* The plans for the query, without running it.
|
|
305
|
+
*
|
|
306
|
+
* @returns The graph logical plan, the DataFusion logical plan and the physical plan.
|
|
307
|
+
*/
|
|
308
|
+
explain(datasets: Record<string, Dataset>): Promise<string>;
|
|
309
|
+
/**
|
|
310
|
+
* Export the query as SQL joins. Only supported expressions are exported. See
|
|
311
|
+
* [From Cypher to SQL](https://lance-csr.hdclabs.ai/sql/#from-cypher-to-sql).
|
|
312
|
+
*
|
|
313
|
+
* @param datasets - The tables, keyed by node label or relationship type; their
|
|
314
|
+
* schemas shape the SQL.
|
|
315
|
+
* @param dialect - The SQL dialect; the default is `SqlDialect.Default`.
|
|
316
|
+
* @throws {@link LanceCsrError} with code `NotImplemented` if a runtime-only Cypher
|
|
317
|
+
* function has no faithful SQL translation.
|
|
318
|
+
*/
|
|
319
|
+
toSql(datasets: Record<string, Dataset>, dialect?: SqlDialect): Promise<string>;
|
|
320
|
+
/**
|
|
321
|
+
* Run the query, then rank its rows by vector similarity.
|
|
322
|
+
*
|
|
323
|
+
* This is the common GraphRAG pattern: find candidates by graph traversal, then rerank
|
|
324
|
+
* them by similarity. With `VectorSearch.useLanceIndex(true)` and Lance datasets, the
|
|
325
|
+
* Lance index is searched first instead. See
|
|
326
|
+
* [Vector search](https://lance-csr.hdclabs.ai/vector-search/).
|
|
327
|
+
*
|
|
328
|
+
* @param vectorSearch - The ranking. Its column is a returned column, such as `"d.embedding"`.
|
|
329
|
+
* @returns At most `topK` rows, closest first.
|
|
330
|
+
*/
|
|
331
|
+
executeWithVectorRerank(datasets: Record<string, Dataset>, vectorSearch: VectorSearch): Promise<Table>;
|
|
332
|
+
/** The variables the query uses, including `UNION` branches, unordered. */
|
|
333
|
+
variables(): string[];
|
|
334
|
+
/** The node labels the query uses, including `UNION` branches, unordered. */
|
|
335
|
+
nodeLabels(): string[];
|
|
336
|
+
/** The relationship types the query uses, with `UNION` branches, unordered. */
|
|
337
|
+
relationshipTypes(): string[];
|
|
338
|
+
toString(): string;
|
|
339
|
+
}
|