@lancedb/lancedb 0.42.0-beta.20 → 0.42.0-beta.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/RUST_THIRD_PARTY_LICENSES.html +11637 -6176
- package/dist/cleanup.d.ts +31 -0
- package/dist/cleanup.js +80 -0
- package/dist/connection.d.ts +10 -0
- package/dist/connection.js +19 -1
- package/dist/index.d.ts +11 -8
- package/dist/index.js +3 -2
- package/dist/indices.d.ts +44 -0
- package/dist/indices.js +42 -0
- package/dist/job.d.ts +81 -3
- package/dist/job.js +138 -4
- package/dist/materialized_view.d.ts +5 -0
- package/dist/materialized_view.js +17 -1
- package/dist/native.d.ts +90 -27
- package/dist/native.js +53 -52
- package/dist/prewarm.d.ts +21 -0
- package/dist/prewarm.js +18 -0
- package/dist/profiling.d.ts +44 -0
- package/dist/profiling.js +37 -0
- package/dist/sql.d.ts +11 -0
- package/dist/sql.js +61 -0
- package/dist/table.d.ts +209 -26
- package/dist/table.js +384 -18
- package/package.json +8 -8
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** Per-run retention; omission inherits selected table/branch then deployment policy. */
|
|
2
|
+
export interface CleanupOptions {
|
|
3
|
+
/** Minimum old-version age in seconds. Zero is explicit, not omitted. */
|
|
4
|
+
minAgeSeconds?: number;
|
|
5
|
+
}
|
|
6
|
+
/** Version-deletion outcome. Best-effort auxiliary sweeps are excluded. */
|
|
7
|
+
export interface CleanupResult {
|
|
8
|
+
formatVersion: number;
|
|
9
|
+
minAgeSeconds: number;
|
|
10
|
+
/** False after uncertain retry/recovery; both counts are then null. */
|
|
11
|
+
statisticsComplete: boolean;
|
|
12
|
+
oldVersions: number | null;
|
|
13
|
+
bytesRemoved: number | null;
|
|
14
|
+
/** Failed object deletions, if measured. Successful removals are retained. */
|
|
15
|
+
failedDeletes: number | null;
|
|
16
|
+
}
|
|
17
|
+
/** Read-only deletion estimates; no job or deletion snapshot is reserved. */
|
|
18
|
+
export interface CleanupPlan {
|
|
19
|
+
formatVersion: number;
|
|
20
|
+
sourceVersion: number;
|
|
21
|
+
minAgeSeconds: number;
|
|
22
|
+
cutoffMillis: number;
|
|
23
|
+
versionsToRemove: number;
|
|
24
|
+
bytesToRemove: number;
|
|
25
|
+
}
|
|
26
|
+
/** @internal */
|
|
27
|
+
export declare function decodeCleanupPlan(value: unknown): CleanupPlan;
|
|
28
|
+
/** @internal */
|
|
29
|
+
export declare function validateCleanupOptions(options?: CleanupOptions): void;
|
|
30
|
+
/** @internal */
|
|
31
|
+
export declare function decodeCleanupResult(value: unknown): CleanupResult;
|
package/dist/cleanup.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.decodeCleanupPlan = decodeCleanupPlan;
|
|
6
|
+
exports.validateCleanupOptions = validateCleanupOptions;
|
|
7
|
+
exports.decodeCleanupResult = decodeCleanupResult;
|
|
8
|
+
/** @internal */
|
|
9
|
+
function decodeCleanupPlan(value) {
|
|
10
|
+
if (typeof value !== "object" || value === null)
|
|
11
|
+
throw new Error("invalid cleanup preview");
|
|
12
|
+
const plan = value;
|
|
13
|
+
if (plan.format_version !== 1 ||
|
|
14
|
+
[
|
|
15
|
+
"source_version",
|
|
16
|
+
"min_age_seconds",
|
|
17
|
+
"cutoff_millis",
|
|
18
|
+
"versions_to_remove",
|
|
19
|
+
"bytes_to_remove",
|
|
20
|
+
].some((key) => !Number.isSafeInteger(plan[key]) ||
|
|
21
|
+
(key !== "cutoff_millis" && plan[key] < 0))) {
|
|
22
|
+
throw new Error("unsupported or invalid cleanup preview");
|
|
23
|
+
}
|
|
24
|
+
return {
|
|
25
|
+
formatVersion: 1,
|
|
26
|
+
sourceVersion: plan.source_version,
|
|
27
|
+
minAgeSeconds: plan.min_age_seconds,
|
|
28
|
+
cutoffMillis: plan.cutoff_millis,
|
|
29
|
+
versionsToRemove: plan.versions_to_remove,
|
|
30
|
+
bytesToRemove: plan.bytes_to_remove,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/** @internal */
|
|
34
|
+
function validateCleanupOptions(options) {
|
|
35
|
+
if (options !== undefined &&
|
|
36
|
+
(typeof options !== "object" || options === null || Array.isArray(options))) {
|
|
37
|
+
throw new Error("cleanup options must be an object");
|
|
38
|
+
}
|
|
39
|
+
for (const key of Object.keys(options ?? {})) {
|
|
40
|
+
if (key !== "minAgeSeconds")
|
|
41
|
+
throw new Error(`Unknown cleanup option: ${key}`);
|
|
42
|
+
}
|
|
43
|
+
const age = options?.minAgeSeconds;
|
|
44
|
+
if (age !== undefined && (!Number.isSafeInteger(age) || age < 0)) {
|
|
45
|
+
throw new Error("minAgeSeconds must be a nonnegative safe integer");
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/** @internal */
|
|
49
|
+
function decodeCleanupResult(value) {
|
|
50
|
+
if (typeof value !== "object" || value === null)
|
|
51
|
+
throw new Error("invalid cleanup result");
|
|
52
|
+
const result = value;
|
|
53
|
+
const counts = [result.old_versions ?? null, result.bytes_removed ?? null];
|
|
54
|
+
const failed = result.failed_deletes ?? null;
|
|
55
|
+
if (failed !== null &&
|
|
56
|
+
(!Number.isSafeInteger(failed) ||
|
|
57
|
+
failed < 0 ||
|
|
58
|
+
result.statistics_complete !== true)) {
|
|
59
|
+
throw new Error("invalid cleanup failed-deletion measurement");
|
|
60
|
+
}
|
|
61
|
+
if (result.format_version !== 1 ||
|
|
62
|
+
typeof result.statistics_complete !== "boolean" ||
|
|
63
|
+
!Number.isSafeInteger(result.min_age_seconds) ||
|
|
64
|
+
result.min_age_seconds < 0 ||
|
|
65
|
+
counts.some((count) => count !== null &&
|
|
66
|
+
(!Number.isSafeInteger(count) || count < 0)) ||
|
|
67
|
+
(result.statistics_complete
|
|
68
|
+
? counts.some((count) => count === null)
|
|
69
|
+
: counts.some((count) => count !== null))) {
|
|
70
|
+
throw new Error("unsupported or inconsistent cleanup result");
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
formatVersion: 1,
|
|
74
|
+
minAgeSeconds: result.min_age_seconds,
|
|
75
|
+
statisticsComplete: result.statistics_complete,
|
|
76
|
+
oldVersions: counts[0],
|
|
77
|
+
bytesRemoved: counts[1],
|
|
78
|
+
failedDeletes: result.failed_deletes ?? null,
|
|
79
|
+
};
|
|
80
|
+
}
|
package/dist/connection.d.ts
CHANGED
|
@@ -226,6 +226,11 @@ export declare abstract class Connection {
|
|
|
226
226
|
where?: string;
|
|
227
227
|
limit?: number;
|
|
228
228
|
withNoData?: boolean;
|
|
229
|
+
totalCpuMillis?: number;
|
|
230
|
+
totalMemoryBytes?: number;
|
|
231
|
+
totalGpuCount?: number;
|
|
232
|
+
gpuClass?: string;
|
|
233
|
+
gpuCount?: number;
|
|
229
234
|
}): Promise<MaterializedView>;
|
|
230
235
|
/**
|
|
231
236
|
* Open the materialized view named `name`.
|
|
@@ -485,6 +490,11 @@ export declare class LocalConnection extends Connection {
|
|
|
485
490
|
where?: string;
|
|
486
491
|
limit?: number;
|
|
487
492
|
withNoData?: boolean;
|
|
493
|
+
totalCpuMillis?: number;
|
|
494
|
+
totalMemoryBytes?: number;
|
|
495
|
+
totalGpuCount?: number;
|
|
496
|
+
gpuClass?: string;
|
|
497
|
+
gpuCount?: number;
|
|
488
498
|
}): Promise<MaterializedView>;
|
|
489
499
|
openMaterializedView(name: string): Promise<MaterializedView>;
|
|
490
500
|
listMaterializedViews(options?: ListingOptions): Listing<string>;
|
package/dist/connection.js
CHANGED
|
@@ -73,7 +73,25 @@ class LocalConnection extends Connection {
|
|
|
73
73
|
}
|
|
74
74
|
async createMaterializedView(name, source, options) {
|
|
75
75
|
(0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
|
|
76
|
-
const
|
|
76
|
+
for (const [name, value] of [
|
|
77
|
+
["totalCpuMillis", options?.totalCpuMillis],
|
|
78
|
+
["totalMemoryBytes", options?.totalMemoryBytes],
|
|
79
|
+
["totalGpuCount", options?.totalGpuCount],
|
|
80
|
+
["gpuCount", options?.gpuCount],
|
|
81
|
+
]) {
|
|
82
|
+
(0, materialized_view_1.validateNonNegativeInteger)(value, name);
|
|
83
|
+
if (value === 0 && name !== "totalGpuCount") {
|
|
84
|
+
throw new Error(`${name} must be positive`);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
if (options?.totalGpuCount !== undefined &&
|
|
88
|
+
options.totalGpuCount > 4294967295) {
|
|
89
|
+
throw new Error("totalGpuCount exceeds 4294967295");
|
|
90
|
+
}
|
|
91
|
+
if (options?.gpuCount !== undefined && options.gpuCount > 4294967295) {
|
|
92
|
+
throw new Error("gpuCount exceeds 4294967295");
|
|
93
|
+
}
|
|
94
|
+
const innerTable = await this.inner.createMaterializedView(name, source, (0, materialized_view_1.normalizeSelect)(options?.select), options?.where, options?.limit, options?.withNoData ?? false, options?.totalCpuMillis, options?.totalMemoryBytes, options?.totalGpuCount, options?.gpuClass, options?.gpuCount);
|
|
77
95
|
return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
|
|
78
96
|
}
|
|
79
97
|
async openMaterializedView(name) {
|
package/dist/index.d.ts
CHANGED
|
@@ -1,23 +1,24 @@
|
|
|
1
|
-
import { Connection } from "./connection";
|
|
2
|
-
import { ConnectNamespaceOptions, ConnectionOptions, Session } from "./native.js";
|
|
3
|
-
import { HeaderProvider } from "./header";
|
|
1
|
+
import { type Connection } from "./connection";
|
|
2
|
+
import { type ConnectNamespaceOptions, type ConnectionOptions, type Session } from "./native.js";
|
|
3
|
+
import type { HeaderProvider } from "./header";
|
|
4
4
|
import type { BaseTokenizer } from "./indices";
|
|
5
5
|
import type { FtsToken } from "./table";
|
|
6
6
|
export { MaterializedView, MaterializedViewDefinition, MaterializedViewSelect, } from "./materialized_view";
|
|
7
7
|
export { ViewDescription } from "./view";
|
|
8
8
|
export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
|
9
9
|
export { instrumentLanceDbMetrics } from "./otel";
|
|
10
|
-
export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnOptions, FunctionRefreshPlanningConfig, FunctionRefreshPodResources, FunctionRetryPolicy,
|
|
10
|
+
export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnOptions, FunctionRefreshPlanningConfig, FunctionRefreshPodResources, FunctionRetryPolicy, RefreshColumnResult, FunctionErrors, FunctionErrorsOptions, FunctionErrorRecord, FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
|
|
11
11
|
export { makeArrowTable, makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
12
12
|
export { blob, isBlobField, BlobFile } from "./blob";
|
|
13
13
|
export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobRangeRequest, BlobReadOptions, BlobUri, } from "./blob";
|
|
14
14
|
export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, CreateNamespaceOptions, DropNamespaceOptions, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
|
|
15
15
|
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
|
16
16
|
export { Job, JobEventsOptions, } from "./job";
|
|
17
|
-
export type { JobProgress, JobProgressCallback, JobWaitOptions } from "./job";
|
|
17
|
+
export type { JobProgress, JobProgressCallback, JobState, JobWaitOptions, } from "./job";
|
|
18
|
+
export type { FunctionProfile, FunctionProfileKind, FunctionProfileOptions, FunctionProfileStatus, FunctionProfileWaitOptions, } from "./profiling";
|
|
18
19
|
export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
|
|
19
|
-
export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
|
|
20
|
-
export { Table, FunctionColumnAutoRefreshStatus, FunctionColumnLastRefreshJob, FunctionColumnStatus, FunctionColumnVersionStatus, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, CompactionOptions, PrewarmDataOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
|
|
20
|
+
export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, RTreeOptions, BaseTokenizer, } from "./indices";
|
|
21
|
+
export { Table, TableDelta, TableDeltaReader, TableDeltaTransaction, DeltaVersion, FunctionColumnAutoRefreshStatus, FunctionColumnLastRefreshJob, FunctionColumnStatus, FunctionColumnVersionStatus, Branches, CreateBranchOptions, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, CompactionOptions, PrewarmDataOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
|
|
21
22
|
export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
|
|
22
23
|
export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
|
|
23
24
|
export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
|
|
@@ -254,4 +255,6 @@ export declare function connectNamespace(implName: string, properties: Record<st
|
|
|
254
255
|
export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
|
|
255
256
|
export { Listing, ListingOptions } from "./listing";
|
|
256
257
|
export type { CompactionResult } from "./compaction";
|
|
257
|
-
export type {
|
|
258
|
+
export type { CleanupOptions, CleanupPlan, CleanupResult } from "./cleanup";
|
|
259
|
+
export type { PrewarmDataPlan, PrewarmDataResult, PrewarmPruningSummary, } from "./prewarm";
|
|
260
|
+
export type { QueryParameters, QueryParameterValue } from "./sql";
|
package/dist/index.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.
|
|
6
|
-
exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.func = exports.litInt = void 0;
|
|
5
|
+
exports.lit = exports.col = exports.Expr = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.TableDelta = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
6
|
+
exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.func = exports.litInt = exports.litBool = void 0;
|
|
7
7
|
exports.tokenize = tokenize;
|
|
8
8
|
exports.connect = connect;
|
|
9
9
|
exports.connectNamespace = connectNamespace;
|
|
@@ -57,6 +57,7 @@ var indices_1 = require("./indices");
|
|
|
57
57
|
Object.defineProperty(exports, "Index", { enumerable: true, get: function () { return indices_1.Index; } });
|
|
58
58
|
var table_1 = require("./table");
|
|
59
59
|
Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
|
|
60
|
+
Object.defineProperty(exports, "TableDelta", { enumerable: true, get: function () { return table_1.TableDelta; } });
|
|
60
61
|
Object.defineProperty(exports, "Branches", { enumerable: true, get: function () { return table_1.Branches; } });
|
|
61
62
|
var header_1 = require("./header");
|
|
62
63
|
Object.defineProperty(exports, "HeaderProvider", { enumerable: true, get: function () { return header_1.HeaderProvider; } });
|
package/dist/indices.d.ts
CHANGED
|
@@ -617,6 +617,17 @@ export interface FtsOptions {
|
|
|
617
617
|
*/
|
|
618
618
|
blockSize?: 128 | 256;
|
|
619
619
|
}
|
|
620
|
+
/**
|
|
621
|
+
* Options to create an `RTREE` index
|
|
622
|
+
*/
|
|
623
|
+
export interface RTreeOptions {
|
|
624
|
+
/**
|
|
625
|
+
* The maximum number of entries in each R-tree page.
|
|
626
|
+
*
|
|
627
|
+
* Must be at least 2. The default is 4096.
|
|
628
|
+
*/
|
|
629
|
+
pageSize?: number;
|
|
630
|
+
}
|
|
620
631
|
export declare class Index {
|
|
621
632
|
private readonly inner;
|
|
622
633
|
private constructor();
|
|
@@ -722,6 +733,39 @@ export declare class Index {
|
|
|
722
733
|
* full-text-search index, it matches arbitrary substrings of the raw bytes.
|
|
723
734
|
*/
|
|
724
735
|
static fm(): Index;
|
|
736
|
+
/**
|
|
737
|
+
* Create an RTree index.
|
|
738
|
+
*
|
|
739
|
+
* An RTree index is a scalar index on a GeoArrow geometry column, that is, a
|
|
740
|
+
* column whose Arrow field metadata sets `ARROW:extension:name` to a GeoArrow
|
|
741
|
+
* type such as `geoarrow.point`, `geoarrow.polygon` or `geoarrow.wkb`. It
|
|
742
|
+
* stores the bounding box of each geometry and accelerates spatial filters.
|
|
743
|
+
*
|
|
744
|
+
* The index accelerates `ST_Intersects`, `ST_Contains`, `ST_Within`,
|
|
745
|
+
* `ST_Touches`, `ST_Crosses`, `ST_Overlaps`, `ST_Covers` and `ST_CoveredBy`
|
|
746
|
+
* between the indexed column and a constant geometry, in either argument
|
|
747
|
+
* order, for example
|
|
748
|
+
* `ST_Within(geom, ST_GeomFromText('POLYGON ((...))'))`, as well as
|
|
749
|
+
* `geom IS NULL`. These filters can be combined with other conditions using
|
|
750
|
+
* `AND`.
|
|
751
|
+
*
|
|
752
|
+
* See the R-tree section of https://docs.lancedb.com/indexing/scalar-index
|
|
753
|
+
* for storing geometry, the other spatial functions, and spatial filters on
|
|
754
|
+
* vector search.
|
|
755
|
+
*
|
|
756
|
+
* Remote tables send the `RTREE` index type to the server, which must
|
|
757
|
+
* support it.
|
|
758
|
+
*
|
|
759
|
+
* @example
|
|
760
|
+
* ```ts
|
|
761
|
+
* await table.createIndex("geom", { config: Index.rtree() });
|
|
762
|
+
* await table
|
|
763
|
+
* .query()
|
|
764
|
+
* .where("ST_Intersects(geom, ST_GeomFromText('POLYGON ((0 0, 4 0, 0 4, 0 0))'))")
|
|
765
|
+
* .toArray();
|
|
766
|
+
* ```
|
|
767
|
+
*/
|
|
768
|
+
static rtree(options?: Partial<RTreeOptions>): Index;
|
|
725
769
|
/**
|
|
726
770
|
* Create a full text search index
|
|
727
771
|
*
|
package/dist/indices.js
CHANGED
|
@@ -125,6 +125,48 @@ class Index {
|
|
|
125
125
|
static fm() {
|
|
126
126
|
return new Index(native_1.Index.fm());
|
|
127
127
|
}
|
|
128
|
+
/**
|
|
129
|
+
* Create an RTree index.
|
|
130
|
+
*
|
|
131
|
+
* An RTree index is a scalar index on a GeoArrow geometry column, that is, a
|
|
132
|
+
* column whose Arrow field metadata sets `ARROW:extension:name` to a GeoArrow
|
|
133
|
+
* type such as `geoarrow.point`, `geoarrow.polygon` or `geoarrow.wkb`. It
|
|
134
|
+
* stores the bounding box of each geometry and accelerates spatial filters.
|
|
135
|
+
*
|
|
136
|
+
* The index accelerates `ST_Intersects`, `ST_Contains`, `ST_Within`,
|
|
137
|
+
* `ST_Touches`, `ST_Crosses`, `ST_Overlaps`, `ST_Covers` and `ST_CoveredBy`
|
|
138
|
+
* between the indexed column and a constant geometry, in either argument
|
|
139
|
+
* order, for example
|
|
140
|
+
* `ST_Within(geom, ST_GeomFromText('POLYGON ((...))'))`, as well as
|
|
141
|
+
* `geom IS NULL`. These filters can be combined with other conditions using
|
|
142
|
+
* `AND`.
|
|
143
|
+
*
|
|
144
|
+
* See the R-tree section of https://docs.lancedb.com/indexing/scalar-index
|
|
145
|
+
* for storing geometry, the other spatial functions, and spatial filters on
|
|
146
|
+
* vector search.
|
|
147
|
+
*
|
|
148
|
+
* Remote tables send the `RTREE` index type to the server, which must
|
|
149
|
+
* support it.
|
|
150
|
+
*
|
|
151
|
+
* @example
|
|
152
|
+
* ```ts
|
|
153
|
+
* await table.createIndex("geom", { config: Index.rtree() });
|
|
154
|
+
* await table
|
|
155
|
+
* .query()
|
|
156
|
+
* .where("ST_Intersects(geom, ST_GeomFromText('POLYGON ((0 0, 4 0, 0 4, 0 0))'))")
|
|
157
|
+
* .toArray();
|
|
158
|
+
* ```
|
|
159
|
+
*/
|
|
160
|
+
static rtree(options) {
|
|
161
|
+
const pageSize = options?.pageSize;
|
|
162
|
+
// napi converts a number to u32 by wrapping, so -1 would arrive as
|
|
163
|
+
// 4294967295; reject anything that is not a valid page size first.
|
|
164
|
+
if (pageSize !== undefined &&
|
|
165
|
+
!(Number.isInteger(pageSize) && pageSize >= 2 && pageSize <= 0xffffffff)) {
|
|
166
|
+
throw new Error(`RTree page_size must be at least 2, at most 4294967295, and an integer, got ${pageSize}`);
|
|
167
|
+
}
|
|
168
|
+
return new Index(native_1.Index.rtree(pageSize));
|
|
169
|
+
}
|
|
128
170
|
/**
|
|
129
171
|
* Create a full text search index
|
|
130
172
|
*
|
package/dist/job.d.ts
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { Table as ArrowTable } from "apache-arrow";
|
|
2
|
-
import { JobFailureInfo, Job as NativeJob } from "./native";
|
|
2
|
+
import type { JobFailureInfo, Job as NativeJob } from "./native";
|
|
3
|
+
import { type FunctionProfile, type FunctionProfileOptions, type FunctionProfileWaitOptions } from "./profiling";
|
|
3
4
|
/** Which of a job's events {@link Job.events} returns. */
|
|
4
5
|
export interface JobEventsOptions {
|
|
5
6
|
/** Maximum event rows to return, up to the server maximum of 10,000. */
|
|
6
7
|
limit?: number;
|
|
7
8
|
/** SQL-like filter over the event columns. */
|
|
8
9
|
filter?: string;
|
|
10
|
+
/** Opaque continuation from the preceding page's schema metadata. */
|
|
11
|
+
pageToken?: string;
|
|
9
12
|
}
|
|
10
13
|
/**
|
|
11
14
|
* A forward-compatible structured progress snapshot for a server-side job.
|
|
@@ -63,6 +66,8 @@ export interface JobWaitOptions {
|
|
|
63
66
|
/** Observe progress using the same poller that waits for completion. */
|
|
64
67
|
onProgress?: JobProgressCallback;
|
|
65
68
|
}
|
|
69
|
+
/** Detailed lifecycle state. Unrecognized future values remain strings. */
|
|
70
|
+
export type JobState = "created" | "queued" | "in_progress" | "succeeded" | "failed" | "canceled" | "timed_out" | (string & {});
|
|
66
71
|
/**
|
|
67
72
|
* A handle to an operation that may still be running.
|
|
68
73
|
*
|
|
@@ -93,6 +98,27 @@ export declare class Job<T = void> {
|
|
|
93
98
|
* durable resume tokens.
|
|
94
99
|
*/
|
|
95
100
|
get id(): string | null;
|
|
101
|
+
/**
|
|
102
|
+
* Identity of the selected attempt last observed by this handle.
|
|
103
|
+
* For operation handles, `refresh()` or `status()` can select a different
|
|
104
|
+
* attempt while `id` stays unchanged. Save this value after refreshing and
|
|
105
|
+
* use it throughout paged diagnostics. Local IDs cannot be reopened on a server.
|
|
106
|
+
*/
|
|
107
|
+
get jobId(): string | null;
|
|
108
|
+
/** Root of the explicit retry lineage, when reported. */
|
|
109
|
+
get operationId(): string | null;
|
|
110
|
+
/** One-based attempt ordinal. Worker replacement does not increment it. */
|
|
111
|
+
get attempt(): number | null;
|
|
112
|
+
/** Immediate predecessor attempt, when reported. */
|
|
113
|
+
get retryOf(): string | null;
|
|
114
|
+
/** Detailed state last observed; unknown values never establish success. */
|
|
115
|
+
get lifecycleState(): JobState | null;
|
|
116
|
+
/** Source snapshot pinned for this attempt, when reported. */
|
|
117
|
+
get sourceTableVersion(): number | null;
|
|
118
|
+
/** Most recent execution's fencing token, when reported. */
|
|
119
|
+
get executionEpoch(): number | null;
|
|
120
|
+
/** Refresh and return the detailed lifecycle state. */
|
|
121
|
+
statusTyped(): Promise<JobState>;
|
|
96
122
|
/** The last observed lifecycle state, without contacting the backend. */
|
|
97
123
|
get state(): string | null;
|
|
98
124
|
/**
|
|
@@ -113,8 +139,8 @@ export declare class Job<T = void> {
|
|
|
113
139
|
/** Why the job failed, when it failed and the server reports a reason. */
|
|
114
140
|
get failure(): JobFailureInfo | null;
|
|
115
141
|
/**
|
|
116
|
-
* The
|
|
117
|
-
*
|
|
142
|
+
* The current status, usually "running", "finished", "failed", or "cancelled".
|
|
143
|
+
* A Function refresh waiting for execution workers reports "waiting_for_capacity".
|
|
118
144
|
*
|
|
119
145
|
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
|
|
120
146
|
* terminal failure state. Also refreshes the getters above.
|
|
@@ -151,6 +177,8 @@ export declare class Job<T = void> {
|
|
|
151
177
|
wait(optionsOrProgress?: JobWaitOptions | JobProgressCallback): Promise<T>;
|
|
152
178
|
/** Request cancellation. Cancelling a finished operation is a no-op. */
|
|
153
179
|
cancel(): Promise<void>;
|
|
180
|
+
/** Select this exact attempt's pinned successor. Requires backend support. */
|
|
181
|
+
retry(): Promise<Job<T>>;
|
|
154
182
|
/**
|
|
155
183
|
* Ask the backend for this job's current state, and for a server-side job
|
|
156
184
|
* its full record, then cache it for the getters above.
|
|
@@ -171,6 +199,56 @@ export declare class Job<T = void> {
|
|
|
171
199
|
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
|
172
200
|
*/
|
|
173
201
|
events(options?: JobEventsOptions): Promise<ArrowTable>;
|
|
202
|
+
/** Walk retained history one page at a time using the server's continuation. */
|
|
203
|
+
history(options?: JobEventsOptions): AsyncGenerator<ArrowTable>;
|
|
204
|
+
/**
|
|
205
|
+
* Request a profile of a running Enterprise Function worker session.
|
|
206
|
+
* Returns the accepted identity without waiting for recording to finish.
|
|
207
|
+
* Requires System Operate permission and server profiling support. The server
|
|
208
|
+
* pins the worker, Python session and execution epoch. One capture per job
|
|
209
|
+
* can be active. Captures add workload-dependent overhead.
|
|
210
|
+
*
|
|
211
|
+
* GPU tracing requires initialized PyTorch CUDA/CUPTI. CPU sampling requires
|
|
212
|
+
* readable per-thread procfs and is unavailable in dedicated sandboxes.
|
|
213
|
+
*
|
|
214
|
+
* @example
|
|
215
|
+
* ```ts
|
|
216
|
+
* const profile = await job.captureProfile("worker-id", {
|
|
217
|
+
* kind: "gpu", durationSeconds: 2,
|
|
218
|
+
* });
|
|
219
|
+
* await job.waitForProfile(profile.profileId, { timeoutMs: 60_000 });
|
|
220
|
+
* await job.exportProfile(profile.profileId, "gpu.trace.json");
|
|
221
|
+
* ```
|
|
222
|
+
*/
|
|
223
|
+
captureProfile(workerId: string, options?: FunctionProfileOptions): Promise<FunctionProfile>;
|
|
224
|
+
/**
|
|
225
|
+
* Fetch capture identity and status, including retained lookup after refresh.
|
|
226
|
+
* Failed captures return status "failed" with the server's error reason.
|
|
227
|
+
* This never changes the parent job's cached state.
|
|
228
|
+
*/
|
|
229
|
+
profileStatus(profileId: string): Promise<FunctionProfile>;
|
|
230
|
+
/**
|
|
231
|
+
* Wait for a capture with a configurable timeout and polling cadence.
|
|
232
|
+
* Defaults to 60 seconds with two seconds between requests. A timeout stops
|
|
233
|
+
* only this wait; the capture and Function refresh continue. A failed capture
|
|
234
|
+
* rejects with its server-reported reason. Keep its ID to wait again.
|
|
235
|
+
*/
|
|
236
|
+
waitForProfile(profileId: string, options?: FunctionProfileWaitOptions): Promise<FunctionProfile>;
|
|
237
|
+
/**
|
|
238
|
+
* Retrieve a completed capture's actual JSON: speedscope for CPU,
|
|
239
|
+
* tracemalloc for Python allocations, or Chrome Trace Event JSON for GPU.
|
|
240
|
+
* Truncation flags are preserved. Retention is at most one hour and eight
|
|
241
|
+
* captures per job, until earlier scratch reclamation or archival.
|
|
242
|
+
* Permission, expired-artifact and unsupported-server errors propagate.
|
|
243
|
+
*/
|
|
244
|
+
profileArtifact(profileId: string): Promise<Record<string, unknown>>;
|
|
245
|
+
/**
|
|
246
|
+
* Write a completed capture's JSON after successful retrieval. Open CPU
|
|
247
|
+
* exports in speedscope and GPU exports in Perfetto. GPU activity durations
|
|
248
|
+
* can overlap and are not utilization; Python operator attribution and GPU
|
|
249
|
+
* allocation stacks are excluded. Filesystem errors propagate to the caller.
|
|
250
|
+
*/
|
|
251
|
+
exportProfile(profileId: string, path: string): Promise<void>;
|
|
174
252
|
private lastProgress;
|
|
175
253
|
private enrichProgress;
|
|
176
254
|
/**
|
package/dist/job.js
CHANGED
|
@@ -3,9 +3,12 @@
|
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
5
|
exports.Job = void 0;
|
|
6
|
+
const promises_1 = require("node:fs/promises");
|
|
6
7
|
const apache_arrow_1 = require("apache-arrow");
|
|
8
|
+
const cleanup_1 = require("./cleanup");
|
|
7
9
|
const compaction_1 = require("./compaction");
|
|
8
10
|
const prewarm_1 = require("./prewarm");
|
|
11
|
+
const profiling_1 = require("./profiling");
|
|
9
12
|
/**
|
|
10
13
|
* A handle to an operation that may still be running.
|
|
11
14
|
*
|
|
@@ -34,7 +37,9 @@ class Job {
|
|
|
34
37
|
? prewarm_1.decodePrewarmDataResult
|
|
35
38
|
: inner.jobType === "compact"
|
|
36
39
|
? compaction_1.decodeCompactionResult
|
|
37
|
-
:
|
|
40
|
+
: inner.jobType === "cleanup"
|
|
41
|
+
? cleanup_1.decodeCleanupResult
|
|
42
|
+
: undefined);
|
|
38
43
|
}
|
|
39
44
|
/**
|
|
40
45
|
* Identifies the operation on the server that is running it.
|
|
@@ -47,6 +52,52 @@ class Job {
|
|
|
47
52
|
get id() {
|
|
48
53
|
return this.inner.id ?? null;
|
|
49
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* Identity of the selected attempt last observed by this handle.
|
|
57
|
+
* For operation handles, `refresh()` or `status()` can select a different
|
|
58
|
+
* attempt while `id` stays unchanged. Save this value after refreshing and
|
|
59
|
+
* use it throughout paged diagnostics. Local IDs cannot be reopened on a server.
|
|
60
|
+
*/
|
|
61
|
+
get jobId() {
|
|
62
|
+
return this.inner.jobId ?? null;
|
|
63
|
+
}
|
|
64
|
+
/** Root of the explicit retry lineage, when reported. */
|
|
65
|
+
get operationId() {
|
|
66
|
+
return this.inner.operationId ?? null;
|
|
67
|
+
}
|
|
68
|
+
/** One-based attempt ordinal. Worker replacement does not increment it. */
|
|
69
|
+
get attempt() {
|
|
70
|
+
return this.inner.attempt ?? null;
|
|
71
|
+
}
|
|
72
|
+
/** Immediate predecessor attempt, when reported. */
|
|
73
|
+
get retryOf() {
|
|
74
|
+
return this.inner.retryOf ?? null;
|
|
75
|
+
}
|
|
76
|
+
/** Detailed state last observed; unknown values never establish success. */
|
|
77
|
+
get lifecycleState() {
|
|
78
|
+
return this.inner.lifecycleState ?? null;
|
|
79
|
+
}
|
|
80
|
+
/** Source snapshot pinned for this attempt, when reported. */
|
|
81
|
+
get sourceTableVersion() {
|
|
82
|
+
return this.inner.sourceTableVersion ?? null;
|
|
83
|
+
}
|
|
84
|
+
/** Most recent execution's fencing token, when reported. */
|
|
85
|
+
get executionEpoch() {
|
|
86
|
+
return this.inner.executionEpoch ?? null;
|
|
87
|
+
}
|
|
88
|
+
/** Refresh and return the detailed lifecycle state. */
|
|
89
|
+
async statusTyped() {
|
|
90
|
+
const state = await this.status();
|
|
91
|
+
if (this.lifecycleState !== null)
|
|
92
|
+
return this.lifecycleState;
|
|
93
|
+
if (state === "running")
|
|
94
|
+
return "in_progress";
|
|
95
|
+
if (state === "finished")
|
|
96
|
+
return "succeeded";
|
|
97
|
+
if (state === "cancelled")
|
|
98
|
+
return "canceled";
|
|
99
|
+
return state;
|
|
100
|
+
}
|
|
50
101
|
/** The last observed lifecycle state, without contacting the backend. */
|
|
51
102
|
get state() {
|
|
52
103
|
return this.inner.state ?? null;
|
|
@@ -81,8 +132,8 @@ class Job {
|
|
|
81
132
|
return this.inner.failure ?? null;
|
|
82
133
|
}
|
|
83
134
|
/**
|
|
84
|
-
* The
|
|
85
|
-
*
|
|
135
|
+
* The current status, usually "running", "finished", "failed", or "cancelled".
|
|
136
|
+
* A Function refresh waiting for execution workers reports "waiting_for_capacity".
|
|
86
137
|
*
|
|
87
138
|
* A point snapshot; unlike {@link Job.wait} it does not block or reject on a
|
|
88
139
|
* terminal failure state. Also refreshes the getters above.
|
|
@@ -162,6 +213,10 @@ class Job {
|
|
|
162
213
|
async cancel() {
|
|
163
214
|
return this.inner.cancel();
|
|
164
215
|
}
|
|
216
|
+
/** Select this exact attempt's pinned successor. Requires backend support. */
|
|
217
|
+
async retry() {
|
|
218
|
+
return new Job(await this.inner.retry(), this.resultDecoder);
|
|
219
|
+
}
|
|
165
220
|
/**
|
|
166
221
|
* Ask the backend for this job's current state, and for a server-side job
|
|
167
222
|
* its full record, then cache it for the getters above.
|
|
@@ -184,12 +239,91 @@ class Job {
|
|
|
184
239
|
* `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
|
|
185
240
|
*/
|
|
186
241
|
async events(options) {
|
|
187
|
-
const buf = await this.inner.events(options?.limit, options?.filter);
|
|
242
|
+
const buf = await this.inner.events(options?.limit, options?.filter, options?.pageToken);
|
|
188
243
|
if (buf.length === 0) {
|
|
189
244
|
return new apache_arrow_1.Table();
|
|
190
245
|
}
|
|
191
246
|
return (0, apache_arrow_1.tableFromIPC)(buf);
|
|
192
247
|
}
|
|
248
|
+
/** Walk retained history one page at a time using the server's continuation. */
|
|
249
|
+
async *history(options) {
|
|
250
|
+
let pageToken = options?.pageToken;
|
|
251
|
+
for (;;) {
|
|
252
|
+
const page = await this.events({ ...options, pageToken });
|
|
253
|
+
yield page;
|
|
254
|
+
const next = page.schema.metadata.get("lancedb.job_coordinator.page_token");
|
|
255
|
+
if (next === undefined)
|
|
256
|
+
return;
|
|
257
|
+
if (next === pageToken)
|
|
258
|
+
throw new Error("Job history repeated its page token");
|
|
259
|
+
pageToken = next;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Request a profile of a running Enterprise Function worker session.
|
|
264
|
+
* Returns the accepted identity without waiting for recording to finish.
|
|
265
|
+
* Requires System Operate permission and server profiling support. The server
|
|
266
|
+
* pins the worker, Python session and execution epoch. One capture per job
|
|
267
|
+
* can be active. Captures add workload-dependent overhead.
|
|
268
|
+
*
|
|
269
|
+
* GPU tracing requires initialized PyTorch CUDA/CUPTI. CPU sampling requires
|
|
270
|
+
* readable per-thread procfs and is unavailable in dedicated sandboxes.
|
|
271
|
+
*
|
|
272
|
+
* @example
|
|
273
|
+
* ```ts
|
|
274
|
+
* const profile = await job.captureProfile("worker-id", {
|
|
275
|
+
* kind: "gpu", durationSeconds: 2,
|
|
276
|
+
* });
|
|
277
|
+
* await job.waitForProfile(profile.profileId, { timeoutMs: 60_000 });
|
|
278
|
+
* await job.exportProfile(profile.profileId, "gpu.trace.json");
|
|
279
|
+
* ```
|
|
280
|
+
*/
|
|
281
|
+
async captureProfile(workerId, options = {}) {
|
|
282
|
+
(0, profiling_1.validateProfileOptions)(options, ["kind", "durationSeconds", "sessionSlot"]);
|
|
283
|
+
const kind = options.kind ?? "cpu";
|
|
284
|
+
if (!["cpu", "gpu", "memory"].includes(kind)) {
|
|
285
|
+
throw new TypeError("kind must be cpu, memory or gpu");
|
|
286
|
+
}
|
|
287
|
+
return (0, profiling_1.decodeFunctionProfile)(await this.inner.captureProfile(workerId, kind, options.durationSeconds, options.sessionSlot));
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Fetch capture identity and status, including retained lookup after refresh.
|
|
291
|
+
* Failed captures return status "failed" with the server's error reason.
|
|
292
|
+
* This never changes the parent job's cached state.
|
|
293
|
+
*/
|
|
294
|
+
async profileStatus(profileId) {
|
|
295
|
+
return (0, profiling_1.decodeFunctionProfile)(await this.inner.profileStatus(profileId));
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* Wait for a capture with a configurable timeout and polling cadence.
|
|
299
|
+
* Defaults to 60 seconds with two seconds between requests. A timeout stops
|
|
300
|
+
* only this wait; the capture and Function refresh continue. A failed capture
|
|
301
|
+
* rejects with its server-reported reason. Keep its ID to wait again.
|
|
302
|
+
*/
|
|
303
|
+
async waitForProfile(profileId, options = {}) {
|
|
304
|
+
(0, profiling_1.validateProfileOptions)(options, ["timeoutMs", "pollIntervalMs"]);
|
|
305
|
+
return (0, profiling_1.decodeFunctionProfile)(await this.inner.waitForProfile(profileId, options.timeoutMs, options.pollIntervalMs));
|
|
306
|
+
}
|
|
307
|
+
/**
|
|
308
|
+
* Retrieve a completed capture's actual JSON: speedscope for CPU,
|
|
309
|
+
* tracemalloc for Python allocations, or Chrome Trace Event JSON for GPU.
|
|
310
|
+
* Truncation flags are preserved. Retention is at most one hour and eight
|
|
311
|
+
* captures per job, until earlier scratch reclamation or archival.
|
|
312
|
+
* Permission, expired-artifact and unsupported-server errors propagate.
|
|
313
|
+
*/
|
|
314
|
+
async profileArtifact(profileId) {
|
|
315
|
+
return JSON.parse(await this.inner.profileArtifact(profileId));
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* Write a completed capture's JSON after successful retrieval. Open CPU
|
|
319
|
+
* exports in speedscope and GPU exports in Perfetto. GPU activity durations
|
|
320
|
+
* can overlap and are not utilization; Python operator attribution and GPU
|
|
321
|
+
* allocation stacks are excluded. Filesystem errors propagate to the caller.
|
|
322
|
+
*/
|
|
323
|
+
async exportProfile(profileId, path) {
|
|
324
|
+
const artifact = await this.inner.profileArtifact(profileId);
|
|
325
|
+
await (0, promises_1.writeFile)(path, artifact, "utf8");
|
|
326
|
+
}
|
|
193
327
|
lastProgress = null;
|
|
194
328
|
enrichProgress(progress) {
|
|
195
329
|
const enriched = withProgressEta(this.lastProgress, progress);
|