@lancedb/lancedb 0.42.0-beta.2 → 0.42.0-beta.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RUST_THIRD_PARTY_LICENSES.html +11637 -6176
- package/dist/arrow.js +3 -1
- package/dist/blob.d.ts +11 -0
- package/dist/blob.js +27 -0
- package/dist/cleanup.d.ts +31 -0
- package/dist/cleanup.js +80 -0
- package/dist/compaction.d.ts +23 -0
- package/dist/compaction.js +21 -0
- package/dist/connection.d.ts +2 -2
- package/dist/index.d.ts +13 -8
- package/dist/index.js +3 -2
- package/dist/indices.d.ts +44 -0
- package/dist/indices.js +42 -0
- package/dist/job.d.ts +117 -4
- package/dist/job.js +190 -12
- package/dist/merge.d.ts +17 -0
- package/dist/merge.js +7 -1
- package/dist/native.d.ts +296 -9
- package/dist/native.js +53 -52
- package/dist/prewarm.d.ts +40 -0
- package/dist/prewarm.js +37 -0
- package/dist/profiling.d.ts +44 -0
- package/dist/profiling.js +37 -0
- package/dist/query.d.ts +12 -2
- package/dist/query.js +55 -4
- package/dist/sql.d.ts +11 -0
- package/dist/sql.js +61 -0
- package/dist/table.d.ts +439 -12
- package/dist/table.js +461 -18
- package/package.json +9 -9
- package/pnpm-workspace.yaml +0 -7
package/dist/job.js
CHANGED
|
@@ -3,7 +3,12 @@
|
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
5
|
exports.Job = void 0;
|
|
6
|
+
const promises_1 = require("node:fs/promises");
|
|
6
7
|
const apache_arrow_1 = require("apache-arrow");
|
|
8
|
+
const cleanup_1 = require("./cleanup");
|
|
9
|
+
const compaction_1 = require("./compaction");
|
|
10
|
+
const prewarm_1 = require("./prewarm");
|
|
11
|
+
const profiling_1 = require("./profiling");
|
|
7
12
|
/**
|
|
8
13
|
* A handle to an operation that may still be running.
|
|
9
14
|
*
|
|
@@ -23,8 +28,18 @@ const apache_arrow_1 = require("apache-arrow");
|
|
|
23
28
|
*/
|
|
24
29
|
class Job {
|
|
25
30
|
inner;
|
|
26
|
-
|
|
31
|
+
resultDecoder;
|
|
32
|
+
constructor(inner, resultDecoder) {
|
|
27
33
|
this.inner = inner;
|
|
34
|
+
this.resultDecoder =
|
|
35
|
+
resultDecoder ??
|
|
36
|
+
(inner.jobType === "prewarm_page_cache"
|
|
37
|
+
? prewarm_1.decodePrewarmDataResult
|
|
38
|
+
: inner.jobType === "compact"
|
|
39
|
+
? compaction_1.decodeCompactionResult
|
|
40
|
+
: inner.jobType === "cleanup"
|
|
41
|
+
? cleanup_1.decodeCleanupResult
|
|
42
|
+
: undefined);
|
|
28
43
|
}
|
|
29
44
|
/**
|
|
30
45
|
* Identifies the operation on the server that is running it.
|
|
@@ -37,6 +52,52 @@ class Job {
|
|
|
37
52
|
get id() {
|
|
38
53
|
return this.inner.id ?? null;
|
|
39
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Identity of the selected attempt last observed by this handle.
|
|
57
|
+
* For operation handles, `refresh()` or `status()` can select a different
|
|
58
|
+
* attempt while `id` stays unchanged. Save this value after refreshing and
|
|
59
|
+
* use it throughout paged diagnostics. Local IDs cannot be reopened on a server.
|
|
60
|
+
*/
|
|
61
|
+
get jobId() {
|
|
62
|
+
return this.inner.jobId ?? null;
|
|
63
|
+
}
|
|
64
|
+
/** Root of the explicit retry lineage, when reported. */
|
|
65
|
+
get operationId() {
|
|
66
|
+
return this.inner.operationId ?? null;
|
|
67
|
+
}
|
|
68
|
+
/** One-based attempt ordinal. Worker replacement does not increment it. */
|
|
69
|
+
get attempt() {
|
|
70
|
+
return this.inner.attempt ?? null;
|
|
71
|
+
}
|
|
72
|
+
/** Immediate predecessor attempt, when reported. */
|
|
73
|
+
get retryOf() {
|
|
74
|
+
return this.inner.retryOf ?? null;
|
|
75
|
+
}
|
|
76
|
+
/** Detailed state last observed; unknown values never establish success. */
|
|
77
|
+
get lifecycleState() {
|
|
78
|
+
return this.inner.lifecycleState ?? null;
|
|
79
|
+
}
|
|
80
|
+
/** Source snapshot pinned for this attempt, when reported. */
|
|
81
|
+
get sourceTableVersion() {
|
|
82
|
+
return this.inner.sourceTableVersion ?? null;
|
|
83
|
+
}
|
|
84
|
+
/** Most recent execution's fencing token, when reported. */
|
|
85
|
+
get executionEpoch() {
|
|
86
|
+
return this.inner.executionEpoch ?? null;
|
|
87
|
+
}
|
|
88
|
+
/** Refresh and return the detailed lifecycle state. */
|
|
89
|
+
async statusTyped() {
|
|
90
|
+
const state = await this.status();
|
|
91
|
+
if (this.lifecycleState !== null)
|
|
92
|
+
return this.lifecycleState;
|
|
93
|
+
if (state === "running")
|
|
94
|
+
return "in_progress";
|
|
95
|
+
if (state === "finished")
|
|
96
|
+
return "succeeded";
|
|
97
|
+
if (state === "cancelled")
|
|
98
|
+
return "canceled";
|
|
99
|
+
return state;
|
|
100
|
+
}
|
|
40
101
|
/** The last observed lifecycle state, without contacting the backend. */
|
|
41
102
|
get state() {
|
|
42
103
|
return this.inner.state ?? null;
|
|
@@ -100,24 +161,62 @@ class Job {
|
|
|
100
161
|
* synchronous observer: its return value is ignored and not awaited. If it
|
|
101
162
|
* throws, the error is logged with `console.warn` and waiting continues; a
|
|
102
163
|
* callback exception does not reject `wait`.
|
|
164
|
+
*
|
|
165
|
+
* Pass an options object to set a client-side timeout. Expiry rejects with an
|
|
166
|
+
* error and stops this wait, including pending status requests and backoff.
|
|
167
|
+
* The operation continues; use the same handle to wait again. Registry jobs
|
|
168
|
+
* can also be reopened with `Connection.openJob`; query IDs cannot.
|
|
169
|
+
*
|
|
170
|
+
* @example
|
|
171
|
+
* ```ts
|
|
172
|
+
* const result = await job.wait({ timeoutMs: 60_000, onProgress: console.log });
|
|
173
|
+
* ```
|
|
103
174
|
*/
|
|
104
|
-
async wait(
|
|
105
|
-
|
|
106
|
-
|
|
175
|
+
async wait(optionsOrProgress) {
|
|
176
|
+
const options = typeof optionsOrProgress === "function"
|
|
177
|
+
? { onProgress: optionsOrProgress }
|
|
178
|
+
: (optionsOrProgress ?? {});
|
|
179
|
+
if (typeof options !== "object" || Array.isArray(options)) {
|
|
180
|
+
throw new TypeError("wait expects an options object or a progress callback");
|
|
107
181
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
}
|
|
112
|
-
catch (error) {
|
|
113
|
-
console.warn("Job progress callback threw:", error);
|
|
182
|
+
for (const key of Object.keys(options)) {
|
|
183
|
+
if (key !== "timeoutMs" && key !== "onProgress") {
|
|
184
|
+
throw new TypeError(`Unknown job wait option: ${key}`);
|
|
114
185
|
}
|
|
115
|
-
}
|
|
186
|
+
}
|
|
187
|
+
const { timeoutMs, onProgress: progressCallback } = options;
|
|
188
|
+
if (timeoutMs !== undefined &&
|
|
189
|
+
(!Number.isSafeInteger(timeoutMs) || timeoutMs < 0)) {
|
|
190
|
+
throw new RangeError("timeoutMs must be a nonnegative safe integer");
|
|
191
|
+
}
|
|
192
|
+
if (progressCallback !== undefined &&
|
|
193
|
+
typeof progressCallback !== "function") {
|
|
194
|
+
throw new TypeError("onProgress must be a function");
|
|
195
|
+
}
|
|
196
|
+
const callback = progressCallback === undefined
|
|
197
|
+
? undefined
|
|
198
|
+
: (progress) => {
|
|
199
|
+
try {
|
|
200
|
+
progressCallback(this.enrichProgress(progress));
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
console.warn("Job progress callback threw:", error);
|
|
204
|
+
}
|
|
205
|
+
};
|
|
206
|
+
if (this.resultDecoder !== undefined) {
|
|
207
|
+
return this.resultDecoder(await this.inner.waitResult(callback, timeoutMs));
|
|
208
|
+
}
|
|
209
|
+
await this.inner.wait(callback, timeoutMs);
|
|
210
|
+
return undefined;
|
|
116
211
|
}
|
|
117
212
|
/** Request cancellation. Cancelling a finished operation is a no-op. */
|
|
118
213
|
async cancel() {
|
|
119
214
|
return this.inner.cancel();
|
|
120
215
|
}
|
|
216
|
+
/** Select this exact attempt's pinned successor. Requires backend support. */
|
|
217
|
+
async retry() {
|
|
218
|
+
return new Job(await this.inner.retry(), this.resultDecoder);
|
|
219
|
+
}
|
|
121
220
|
/**
|
|
122
221
|
* Ask the backend for this job's current state, and for a server-side job
|
|
123
222
|
* its full record, then cache it for the getters above.
|
|
@@ -140,12 +239,91 @@ class Job {
|
|
|
140
239
|
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
|
141
240
|
*/
|
|
142
241
|
async events(options) {
|
|
143
|
-
const buf = await this.inner.events(options?.limit, options?.filter);
|
|
242
|
+
const buf = await this.inner.events(options?.limit, options?.filter, options?.pageToken);
|
|
144
243
|
if (buf.length === 0) {
|
|
145
244
|
return new apache_arrow_1.Table();
|
|
146
245
|
}
|
|
147
246
|
return (0, apache_arrow_1.tableFromIPC)(buf);
|
|
148
247
|
}
|
|
248
|
+
/** Walk retained history one page at a time using the server's continuation. */
|
|
249
|
+
async *history(options) {
|
|
250
|
+
let pageToken = options?.pageToken;
|
|
251
|
+
for (;;) {
|
|
252
|
+
const page = await this.events({ ...options, pageToken });
|
|
253
|
+
yield page;
|
|
254
|
+
const next = page.schema.metadata.get("lancedb.job_coordinator.page_token");
|
|
255
|
+
if (next === undefined)
|
|
256
|
+
return;
|
|
257
|
+
if (next === pageToken)
|
|
258
|
+
throw new Error("Job history repeated its page token");
|
|
259
|
+
pageToken = next;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Request a profile of a running Enterprise Function worker session.
|
|
264
|
+
* Returns the accepted identity without waiting for recording to finish.
|
|
265
|
+
* Requires System Operate permission and server profiling support. The server
|
|
266
|
+
* pins the worker, Python session and execution epoch. One capture per job
|
|
267
|
+
* can be active. Captures add workload-dependent overhead.
|
|
268
|
+
*
|
|
269
|
+
* GPU tracing requires initialized PyTorch CUDA/CUPTI. CPU sampling requires
|
|
270
|
+
* readable per-thread procfs and is unavailable in dedicated sandboxes.
|
|
271
|
+
*
|
|
272
|
+
* @example
|
|
273
|
+
* ```ts
|
|
274
|
+
* const profile = await job.captureProfile("worker-id", {
|
|
275
|
+
* kind: "gpu", durationSeconds: 2,
|
|
276
|
+
* });
|
|
277
|
+
* await job.waitForProfile(profile.profileId, { timeoutMs: 60_000 });
|
|
278
|
+
* await job.exportProfile(profile.profileId, "gpu.trace.json");
|
|
279
|
+
* ```
|
|
280
|
+
*/
|
|
281
|
+
async captureProfile(workerId, options = {}) {
|
|
282
|
+
(0, profiling_1.validateProfileOptions)(options, ["kind", "durationSeconds", "sessionSlot"]);
|
|
283
|
+
const kind = options.kind ?? "cpu";
|
|
284
|
+
if (!["cpu", "gpu", "memory"].includes(kind)) {
|
|
285
|
+
throw new TypeError("kind must be cpu, memory or gpu");
|
|
286
|
+
}
|
|
287
|
+
return (0, profiling_1.decodeFunctionProfile)(await this.inner.captureProfile(workerId, kind, options.durationSeconds, options.sessionSlot));
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Fetch capture identity and status, including retained lookup after refresh.
|
|
291
|
+
* Failed captures return status "failed" with the server's error reason.
|
|
292
|
+
* This never changes the parent job's cached state.
|
|
293
|
+
*/
|
|
294
|
+
async profileStatus(profileId) {
|
|
295
|
+
return (0, profiling_1.decodeFunctionProfile)(await this.inner.profileStatus(profileId));
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Wait for a capture with a configurable timeout and polling cadence.
|
|
299
|
+
* Defaults to 60 seconds with two seconds between requests. A timeout stops
|
|
300
|
+
* only this wait; the capture and Function refresh continue. A failed capture
|
|
301
|
+
* rejects with its server-reported reason. Keep its ID to wait again.
|
|
302
|
+
*/
|
|
303
|
+
async waitForProfile(profileId, options = {}) {
|
|
304
|
+
(0, profiling_1.validateProfileOptions)(options, ["timeoutMs", "pollIntervalMs"]);
|
|
305
|
+
return (0, profiling_1.decodeFunctionProfile)(await this.inner.waitForProfile(profileId, options.timeoutMs, options.pollIntervalMs));
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Retrieve a completed capture's actual JSON: speedscope for CPU,
|
|
309
|
+
* tracemalloc for Python allocations, or Chrome Trace Event JSON for GPU.
|
|
310
|
+
* Truncation flags are preserved. Retention is at most one hour and eight
|
|
311
|
+
* captures per job, until earlier scratch reclamation or archival.
|
|
312
|
+
* Permission, expired-artifact and unsupported-server errors propagate.
|
|
313
|
+
*/
|
|
314
|
+
async profileArtifact(profileId) {
|
|
315
|
+
return JSON.parse(await this.inner.profileArtifact(profileId));
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Write a completed capture's JSON after successful retrieval. Open CPU
|
|
319
|
+
* exports in speedscope and GPU exports in Perfetto. GPU activity durations
|
|
320
|
+
* can overlap and are not utilization; Python operator attribution and GPU
|
|
321
|
+
* allocation stacks are excluded. Filesystem errors propagate to the caller.
|
|
322
|
+
*/
|
|
323
|
+
async exportProfile(profileId, path) {
|
|
324
|
+
const artifact = await this.inner.profileArtifact(profileId);
|
|
325
|
+
await (0, promises_1.writeFile)(path, artifact, "utf8");
|
|
326
|
+
}
|
|
149
327
|
lastProgress = null;
|
|
150
328
|
enrichProgress(progress) {
|
|
151
329
|
const enriched = withProgressEta(this.lastProgress, progress);
|
package/dist/merge.d.ts
CHANGED
|
@@ -99,6 +99,23 @@ export interface WriteExecutionOptions {
|
|
|
99
99
|
* first attempt will not be cancelled.
|
|
100
100
|
*
|
|
101
101
|
* When this is set, the timeout is enforced on all attempts, including the first.
|
|
102
|
+
* For remote tables this is sent as a server execution timeout. It does not
|
|
103
|
+
* extend HTTP read or proxy idle timeouts, or prove that a failed request did
|
|
104
|
+
* not commit.
|
|
102
105
|
*/
|
|
103
106
|
timeoutMs?: number;
|
|
107
|
+
/**
|
|
108
|
+
* Override the remote retry budget for this execution only (an integer from
|
|
109
|
+
* 0 to 255). Use `execute(data, { retries: 0 })` to disable even retries after
|
|
110
|
+
* precommit conflicts (409) or rate limits (429). This does not change
|
|
111
|
+
* connection-wide/read budgets or local transaction conflict retries.
|
|
112
|
+
*
|
|
113
|
+
* By default, remote calls use the connection's `clientConfig.retryConfig`
|
|
114
|
+
* ({@link RetryConfig}). Its `statuses` (also set through
|
|
115
|
+
* `LANCE_CLIENT_RETRY_STATUSES`) selects candidates within the retry budget;
|
|
116
|
+
* it never permits replay of ambiguous merge outcomes. In particular, 502,
|
|
117
|
+
* ordinary 503, and 504 are never replayed, even if configured. Reconcile table
|
|
118
|
+
* contents before manually repeating a merge whose commit outcome is unknown.
|
|
119
|
+
*/
|
|
120
|
+
retries?: number;
|
|
104
121
|
}
|
package/dist/merge.js
CHANGED
|
@@ -102,6 +102,12 @@ class MergeInsertBuilder {
|
|
|
102
102
|
* @returns {Promise<MergeResult>} the merge result
|
|
103
103
|
*/
|
|
104
104
|
async execute(data, execOptions) {
|
|
105
|
+
if (execOptions?.retries !== undefined &&
|
|
106
|
+
(!Number.isInteger(execOptions.retries) ||
|
|
107
|
+
execOptions.retries < 0 ||
|
|
108
|
+
execOptions.retries > 255)) {
|
|
109
|
+
throw new Error("retries must be an integer between 0 and 255");
|
|
110
|
+
}
|
|
105
111
|
let schema;
|
|
106
112
|
if (this.#schema instanceof Promise) {
|
|
107
113
|
schema = await this.#schema;
|
|
@@ -114,7 +120,7 @@ class MergeInsertBuilder {
|
|
|
114
120
|
this.#native.setTimeout(execOptions.timeoutMs);
|
|
115
121
|
}
|
|
116
122
|
const buffer = await (0, arrow_1.fromDataToBuffer)(data, undefined, schema);
|
|
117
|
-
return await this.#native.execute(buffer);
|
|
123
|
+
return await this.#native.execute(buffer, execOptions?.retries);
|
|
118
124
|
}
|
|
119
125
|
}
|
|
120
126
|
exports.MergeInsertBuilder = MergeInsertBuilder;
|