@lancedb/lancedb 0.42.0-beta.20 → 0.42.0-beta.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,31 @@
1
+ /** Per-run retention; omission inherits selected table/branch then deployment policy. */
2
+ export interface CleanupOptions {
3
+ /** Minimum old-version age in seconds. Zero is explicit, not omitted. */
4
+ minAgeSeconds?: number;
5
+ }
6
+ /** Version-deletion outcome. Best-effort auxiliary sweeps are excluded. */
7
+ export interface CleanupResult {
8
+ formatVersion: number;
9
+ minAgeSeconds: number;
10
+ /** False after uncertain retry/recovery; both counts are then null. */
11
+ statisticsComplete: boolean;
12
+ oldVersions: number | null;
13
+ bytesRemoved: number | null;
14
+ /** Failed object deletions, if measured. Successful removals are retained. */
15
+ failedDeletes: number | null;
16
+ }
17
+ /** Read-only deletion estimates; no job or deletion snapshot is reserved. */
18
+ export interface CleanupPlan {
19
+ formatVersion: number;
20
+ sourceVersion: number;
21
+ minAgeSeconds: number;
22
+ cutoffMillis: number;
23
+ versionsToRemove: number;
24
+ bytesToRemove: number;
25
+ }
26
+ /** @internal */
27
+ export declare function decodeCleanupPlan(value: unknown): CleanupPlan;
28
+ /** @internal */
29
+ export declare function validateCleanupOptions(options?: CleanupOptions): void;
30
+ /** @internal */
31
+ export declare function decodeCleanupResult(value: unknown): CleanupResult;
@@ -0,0 +1,80 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.decodeCleanupPlan = decodeCleanupPlan;
6
+ exports.validateCleanupOptions = validateCleanupOptions;
7
+ exports.decodeCleanupResult = decodeCleanupResult;
8
+ /** @internal */
9
+ function decodeCleanupPlan(value) {
10
+ if (typeof value !== "object" || value === null)
11
+ throw new Error("invalid cleanup preview");
12
+ const plan = value;
13
+ if (plan.format_version !== 1 ||
14
+ [
15
+ "source_version",
16
+ "min_age_seconds",
17
+ "cutoff_millis",
18
+ "versions_to_remove",
19
+ "bytes_to_remove",
20
+ ].some((key) => !Number.isSafeInteger(plan[key]) ||
21
+ (key !== "cutoff_millis" && plan[key] < 0))) {
22
+ throw new Error("unsupported or invalid cleanup preview");
23
+ }
24
+ return {
25
+ formatVersion: 1,
26
+ sourceVersion: plan.source_version,
27
+ minAgeSeconds: plan.min_age_seconds,
28
+ cutoffMillis: plan.cutoff_millis,
29
+ versionsToRemove: plan.versions_to_remove,
30
+ bytesToRemove: plan.bytes_to_remove,
31
+ };
32
+ }
33
+ /** @internal */
34
+ function validateCleanupOptions(options) {
35
+ if (options !== undefined &&
36
+ (typeof options !== "object" || options === null || Array.isArray(options))) {
37
+ throw new Error("cleanup options must be an object");
38
+ }
39
+ for (const key of Object.keys(options ?? {})) {
40
+ if (key !== "minAgeSeconds")
41
+ throw new Error(`Unknown cleanup option: ${key}`);
42
+ }
43
+ const age = options?.minAgeSeconds;
44
+ if (age !== undefined && (!Number.isSafeInteger(age) || age < 0)) {
45
+ throw new Error("minAgeSeconds must be a nonnegative safe integer");
46
+ }
47
+ }
48
+ /** @internal */
49
+ function decodeCleanupResult(value) {
50
+ if (typeof value !== "object" || value === null)
51
+ throw new Error("invalid cleanup result");
52
+ const result = value;
53
+ const counts = [result.old_versions ?? null, result.bytes_removed ?? null];
54
+ const failed = result.failed_deletes ?? null;
55
+ if (failed !== null &&
56
+ (!Number.isSafeInteger(failed) ||
57
+ failed < 0 ||
58
+ result.statistics_complete !== true)) {
59
+ throw new Error("invalid cleanup failed-deletion measurement");
60
+ }
61
+ if (result.format_version !== 1 ||
62
+ typeof result.statistics_complete !== "boolean" ||
63
+ !Number.isSafeInteger(result.min_age_seconds) ||
64
+ result.min_age_seconds < 0 ||
65
+ counts.some((count) => count !== null &&
66
+ (!Number.isSafeInteger(count) || count < 0)) ||
67
+ (result.statistics_complete
68
+ ? counts.some((count) => count === null)
69
+ : counts.some((count) => count !== null))) {
70
+ throw new Error("unsupported or inconsistent cleanup result");
71
+ }
72
+ return {
73
+ formatVersion: 1,
74
+ minAgeSeconds: result.min_age_seconds,
75
+ statisticsComplete: result.statistics_complete,
76
+ oldVersions: counts[0],
77
+ bytesRemoved: counts[1],
78
+ failedDeletes: result.failed_deletes ?? null,
79
+ };
80
+ }
@@ -226,6 +226,11 @@ export declare abstract class Connection {
226
226
  where?: string;
227
227
  limit?: number;
228
228
  withNoData?: boolean;
229
+ totalCpuMillis?: number;
230
+ totalMemoryBytes?: number;
231
+ totalGpuCount?: number;
232
+ gpuClass?: string;
233
+ gpuCount?: number;
229
234
  }): Promise<MaterializedView>;
230
235
  /**
231
236
  * Open the materialized view named `name`.
@@ -485,6 +490,11 @@ export declare class LocalConnection extends Connection {
485
490
  where?: string;
486
491
  limit?: number;
487
492
  withNoData?: boolean;
493
+ totalCpuMillis?: number;
494
+ totalMemoryBytes?: number;
495
+ totalGpuCount?: number;
496
+ gpuClass?: string;
497
+ gpuCount?: number;
488
498
  }): Promise<MaterializedView>;
489
499
  openMaterializedView(name: string): Promise<MaterializedView>;
490
500
  listMaterializedViews(options?: ListingOptions): Listing<string>;
@@ -73,7 +73,25 @@ class LocalConnection extends Connection {
73
73
  }
74
74
  async createMaterializedView(name, source, options) {
75
75
  (0, materialized_view_1.validateNonNegativeInteger)(options?.limit, "limit");
76
- const innerTable = await this.inner.createMaterializedView(name, source, (0, materialized_view_1.normalizeSelect)(options?.select), options?.where, options?.limit, options?.withNoData ?? false);
76
+ for (const [name, value] of [
77
+ ["totalCpuMillis", options?.totalCpuMillis],
78
+ ["totalMemoryBytes", options?.totalMemoryBytes],
79
+ ["totalGpuCount", options?.totalGpuCount],
80
+ ["gpuCount", options?.gpuCount],
81
+ ]) {
82
+ (0, materialized_view_1.validateNonNegativeInteger)(value, name);
83
+ if (value === 0 && name !== "totalGpuCount") {
84
+ throw new Error(`${name} must be positive`);
85
+ }
86
+ }
87
+ if (options?.totalGpuCount !== undefined &&
88
+ options.totalGpuCount > 4294967295) {
89
+ throw new Error("totalGpuCount exceeds 4294967295");
90
+ }
91
+ if (options?.gpuCount !== undefined && options.gpuCount > 4294967295) {
92
+ throw new Error("gpuCount exceeds 4294967295");
93
+ }
94
+ const innerTable = await this.inner.createMaterializedView(name, source, (0, materialized_view_1.normalizeSelect)(options?.select), options?.where, options?.limit, options?.withNoData ?? false, options?.totalCpuMillis, options?.totalMemoryBytes, options?.totalGpuCount, options?.gpuClass, options?.gpuCount);
77
95
  return new materialized_view_1.MaterializedView(new table_1.LocalTable(innerTable));
78
96
  }
79
97
  async openMaterializedView(name) {
package/dist/index.d.ts CHANGED
@@ -1,23 +1,24 @@
1
- import { Connection } from "./connection";
2
- import { ConnectNamespaceOptions, ConnectionOptions, Session } from "./native.js";
3
- import { HeaderProvider } from "./header";
1
+ import { type Connection } from "./connection";
2
+ import { type ConnectNamespaceOptions, type ConnectionOptions, type Session } from "./native.js";
3
+ import type { HeaderProvider } from "./header";
4
4
  import type { BaseTokenizer } from "./indices";
5
5
  import type { FtsToken } from "./table";
6
6
  export { MaterializedView, MaterializedViewDefinition, MaterializedViewSelect, } from "./materialized_view";
7
7
  export { ViewDescription } from "./view";
8
8
  export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
9
9
  export { instrumentLanceDbMetrics } from "./otel";
10
- export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnOptions, FunctionRefreshPlanningConfig, FunctionRefreshPodResources, FunctionRetryPolicy, FunctionWorkerResourceBudget, RefreshColumnResult, FunctionErrors, FunctionErrorsOptions, FunctionErrorRecord, FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
10
+ export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnOptions, FunctionRefreshPlanningConfig, FunctionRefreshPodResources, FunctionRetryPolicy, RefreshColumnResult, FunctionErrors, FunctionErrorsOptions, FunctionErrorRecord, FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
11
11
  export { makeArrowTable, makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
12
12
  export { blob, isBlobField, BlobFile } from "./blob";
13
13
  export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobRangeRequest, BlobReadOptions, BlobUri, } from "./blob";
14
14
  export { Connection, CreateTableOptions, TableNamesOptions, OpenTableOptions, CreateNamespaceOptions, DropNamespaceOptions, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
15
15
  export { JobFailureInfo, JobInfo, Session } from "./native.js";
16
16
  export { Job, JobEventsOptions, } from "./job";
17
- export type { JobProgress, JobProgressCallback, JobWaitOptions } from "./job";
17
+ export type { JobProgress, JobProgressCallback, JobState, JobWaitOptions, } from "./job";
18
+ export type { FunctionProfile, FunctionProfileKind, FunctionProfileOptions, FunctionProfileStatus, FunctionProfileWaitOptions, } from "./profiling";
18
19
  export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
19
- export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
20
- export { Table, FunctionColumnAutoRefreshStatus, FunctionColumnLastRefreshJob, FunctionColumnStatus, FunctionColumnVersionStatus, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, CompactionOptions, PrewarmDataOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
20
+ export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, RTreeOptions, BaseTokenizer, } from "./indices";
21
+ export { Table, TableDelta, TableDeltaReader, TableDeltaTransaction, DeltaVersion, FunctionColumnAutoRefreshStatus, FunctionColumnLastRefreshJob, FunctionColumnStatus, FunctionColumnVersionStatus, Branches, CreateBranchOptions, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, CompactionOptions, PrewarmDataOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, RTEWriteSpec, RTEStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
21
22
  export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
22
23
  export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
23
24
  export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
@@ -254,4 +255,6 @@ export declare function connectNamespace(implName: string, properties: Record<st
254
255
  export { Catalog, CatalogOptions, DatabaseNames, connectCatalog, } from "./catalog";
255
256
  export { Listing, ListingOptions } from "./listing";
256
257
  export type { CompactionResult } from "./compaction";
257
- export type { PrewarmDataResult } from "./prewarm";
258
+ export type { CleanupOptions, CleanupPlan, CleanupResult } from "./cleanup";
259
+ export type { PrewarmDataPlan, PrewarmDataResult, PrewarmPruningSummary, } from "./prewarm";
260
+ export type { QueryParameters, QueryParameterValue } from "./sql";
package/dist/index.js CHANGED
@@ -2,8 +2,8 @@
2
2
  // SPDX-License-Identifier: Apache-2.0
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
- exports.litBool = exports.lit = exports.col = exports.Expr = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
6
- exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.func = exports.litInt = void 0;
5
+ exports.lit = exports.col = exports.Expr = exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthSession = exports.OAuthFlowType = exports.ClientAuthMethod = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.TableDelta = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeJsonField = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
6
+ exports.Listing = exports.connectCatalog = exports.DatabaseNames = exports.Catalog = exports.func = exports.litInt = exports.litBool = void 0;
7
7
  exports.tokenize = tokenize;
8
8
  exports.connect = connect;
9
9
  exports.connectNamespace = connectNamespace;
@@ -57,6 +57,7 @@ var indices_1 = require("./indices");
57
57
  Object.defineProperty(exports, "Index", { enumerable: true, get: function () { return indices_1.Index; } });
58
58
  var table_1 = require("./table");
59
59
  Object.defineProperty(exports, "Table", { enumerable: true, get: function () { return table_1.Table; } });
60
+ Object.defineProperty(exports, "TableDelta", { enumerable: true, get: function () { return table_1.TableDelta; } });
60
61
  Object.defineProperty(exports, "Branches", { enumerable: true, get: function () { return table_1.Branches; } });
61
62
  var header_1 = require("./header");
62
63
  Object.defineProperty(exports, "HeaderProvider", { enumerable: true, get: function () { return header_1.HeaderProvider; } });
package/dist/indices.d.ts CHANGED
@@ -617,6 +617,17 @@ export interface FtsOptions {
617
617
  */
618
618
  blockSize?: 128 | 256;
619
619
  }
620
+ /**
621
+ * Options to create an `RTREE` index
622
+ */
623
+ export interface RTreeOptions {
624
+ /**
625
+ * The maximum number of entries in each R-tree page.
626
+ *
627
+ * Must be at least 2. The default is 4096.
628
+ */
629
+ pageSize?: number;
630
+ }
620
631
  export declare class Index {
621
632
  private readonly inner;
622
633
  private constructor();
@@ -722,6 +733,39 @@ export declare class Index {
722
733
  * full-text-search index, it matches arbitrary substrings of the raw bytes.
723
734
  */
724
735
  static fm(): Index;
736
+ /**
737
+ * Create an RTree index.
738
+ *
739
+ * An RTree index is a scalar index on a GeoArrow geometry column, that is, a
740
+ * column whose Arrow field metadata sets `ARROW:extension:name` to a GeoArrow
741
+ * type such as `geoarrow.point`, `geoarrow.polygon` or `geoarrow.wkb`. It
742
+ * stores the bounding box of each geometry and accelerates spatial filters.
743
+ *
744
+ * The index accelerates `ST_Intersects`, `ST_Contains`, `ST_Within`,
745
+ * `ST_Touches`, `ST_Crosses`, `ST_Overlaps`, `ST_Covers` and `ST_CoveredBy`
746
+ * between the indexed column and a constant geometry, in either argument
747
+ * order, for example
748
+ * `ST_Within(geom, ST_GeomFromText('POLYGON ((...))'))`, as well as
749
+ * `geom IS NULL`. These filters can be combined with other conditions using
750
+ * `AND`.
751
+ *
752
+ * See the R-tree section of https://docs.lancedb.com/indexing/scalar-index
753
+ * for storing geometry, the other spatial functions, and spatial filters on
754
+ * vector search.
755
+ *
756
+ * Remote tables send the `RTREE` index type to the server, which must
757
+ * support it.
758
+ *
759
+ * @example
760
+ * ```ts
761
+ * await table.createIndex("geom", { config: Index.rtree() });
762
+ * await table
763
+ * .query()
764
+ * .where("ST_Intersects(geom, ST_GeomFromText('POLYGON ((0 0, 4 0, 0 4, 0 0))'))")
765
+ * .toArray();
766
+ * ```
767
+ */
768
+ static rtree(options?: Partial<RTreeOptions>): Index;
725
769
  /**
726
770
  * Create a full text search index
727
771
  *
package/dist/indices.js CHANGED
@@ -125,6 +125,48 @@ class Index {
125
125
  static fm() {
126
126
  return new Index(native_1.Index.fm());
127
127
  }
128
+ /**
129
+ * Create an RTree index.
130
+ *
131
+ * An RTree index is a scalar index on a GeoArrow geometry column, that is, a
132
+ * column whose Arrow field metadata sets `ARROW:extension:name` to a GeoArrow
133
+ * type such as `geoarrow.point`, `geoarrow.polygon` or `geoarrow.wkb`. It
134
+ * stores the bounding box of each geometry and accelerates spatial filters.
135
+ *
136
+ * The index accelerates `ST_Intersects`, `ST_Contains`, `ST_Within`,
137
+ * `ST_Touches`, `ST_Crosses`, `ST_Overlaps`, `ST_Covers` and `ST_CoveredBy`
138
+ * between the indexed column and a constant geometry, in either argument
139
+ * order, for example
140
+ * `ST_Within(geom, ST_GeomFromText('POLYGON ((...))'))`, as well as
141
+ * `geom IS NULL`. These filters can be combined with other conditions using
142
+ * `AND`.
143
+ *
144
+ * See the R-tree section of https://docs.lancedb.com/indexing/scalar-index
145
+ * for storing geometry, the other spatial functions, and spatial filters on
146
+ * vector search.
147
+ *
148
+ * Remote tables send the `RTREE` index type to the server, which must
149
+ * support it.
150
+ *
151
+ * @example
152
+ * ```ts
153
+ * await table.createIndex("geom", { config: Index.rtree() });
154
+ * await table
155
+ * .query()
156
+ * .where("ST_Intersects(geom, ST_GeomFromText('POLYGON ((0 0, 4 0, 0 4, 0 0))'))")
157
+ * .toArray();
158
+ * ```
159
+ */
160
+ static rtree(options) {
161
+ const pageSize = options?.pageSize;
162
+ // napi converts a number to u32 by wrapping, so -1 would arrive as
163
+ // 4294967295; reject anything that is not a valid page size first.
164
+ if (pageSize !== undefined &&
165
+ !(Number.isInteger(pageSize) && pageSize >= 2 && pageSize <= 0xffffffff)) {
166
+ throw new Error(`RTree page_size must be at least 2, at most 4294967295, and an integer, got ${pageSize}`);
167
+ }
168
+ return new Index(native_1.Index.rtree(pageSize));
169
+ }
128
170
  /**
129
171
  * Create a full text search index
130
172
  *
package/dist/job.d.ts CHANGED
@@ -1,11 +1,14 @@
1
1
  import { Table as ArrowTable } from "apache-arrow";
2
- import { JobFailureInfo, Job as NativeJob } from "./native";
2
+ import type { JobFailureInfo, Job as NativeJob } from "./native";
3
+ import { type FunctionProfile, type FunctionProfileOptions, type FunctionProfileWaitOptions } from "./profiling";
3
4
  /** Which of a job's events {@link Job.events} returns. */
4
5
  export interface JobEventsOptions {
5
6
  /** Maximum event rows to return, up to the server maximum of 10,000. */
6
7
  limit?: number;
7
8
  /** SQL-like filter over the event columns. */
8
9
  filter?: string;
10
+ /** Opaque continuation from the preceding page's schema metadata. */
11
+ pageToken?: string;
9
12
  }
10
13
  /**
11
14
  * A forward-compatible structured progress snapshot for a server-side job.
@@ -63,6 +66,8 @@ export interface JobWaitOptions {
63
66
  /** Observe progress using the same poller that waits for completion. */
64
67
  onProgress?: JobProgressCallback;
65
68
  }
69
+ /** Detailed lifecycle state. Unrecognized future values remain strings. */
70
+ export type JobState = "created" | "queued" | "in_progress" | "succeeded" | "failed" | "canceled" | "timed_out" | (string & {});
66
71
  /**
67
72
  * A handle to an operation that may still be running.
68
73
  *
@@ -93,6 +98,27 @@ export declare class Job<T = void> {
93
98
  * durable resume tokens.
94
99
  */
95
100
  get id(): string | null;
101
+ /**
102
+ * Identity of the selected attempt last observed by this handle.
103
+ * For operation handles, `refresh()` or `status()` can select a different
104
+ * attempt while `id` stays unchanged. Save this value after refreshing and
105
+ * use it throughout paged diagnostics. Local IDs cannot be reopened on a server.
106
+ */
107
+ get jobId(): string | null;
108
+ /** Root of the explicit retry lineage, when reported. */
109
+ get operationId(): string | null;
110
+ /** One-based attempt ordinal. Worker replacement does not increment it. */
111
+ get attempt(): number | null;
112
+ /** Immediate predecessor attempt, when reported. */
113
+ get retryOf(): string | null;
114
+ /** Detailed state last observed; unknown values never establish success. */
115
+ get lifecycleState(): JobState | null;
116
+ /** Source snapshot pinned for this attempt, when reported. */
117
+ get sourceTableVersion(): number | null;
118
+ /** Most recent execution's fencing token, when reported. */
119
+ get executionEpoch(): number | null;
120
+ /** Refresh and return the detailed lifecycle state. */
121
+ statusTyped(): Promise<JobState>;
96
122
  /** The last observed lifecycle state, without contacting the backend. */
97
123
  get state(): string | null;
98
124
  /**
@@ -113,8 +139,8 @@ export declare class Job<T = void> {
113
139
  /** Why the job failed, when it failed and the server reports a reason. */
114
140
  get failure(): JobFailureInfo | null;
115
141
  /**
116
- * The operation's current lifecycle state: "running", "finished", "failed",
117
- * or "cancelled".
142
+ * The current status, usually "running", "finished", "failed", or "cancelled".
143
+ * A Function refresh waiting for execution workers reports "waiting_for_capacity".
118
144
  *
119
145
  * A point snapshot; unlike {@link Job.wait} it does not block or reject on a
120
146
  * terminal failure state. Also refreshes the getters above.
@@ -151,6 +177,8 @@ export declare class Job<T = void> {
151
177
  wait(optionsOrProgress?: JobWaitOptions | JobProgressCallback): Promise<T>;
152
178
  /** Request cancellation. Cancelling a finished operation is a no-op. */
153
179
  cancel(): Promise<void>;
180
+ /** Select this exact attempt's pinned successor. Requires backend support. */
181
+ retry(): Promise<Job<T>>;
154
182
  /**
155
183
  * Ask the backend for this job's current state, and for a server-side job
156
184
  * its full record, then cache it for the getters above.
@@ -171,6 +199,56 @@ export declare class Job<T = void> {
171
199
  * `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
172
200
  */
173
201
  events(options?: JobEventsOptions): Promise<ArrowTable>;
202
+ /** Walk retained history one page at a time using the server's continuation. */
203
+ history(options?: JobEventsOptions): AsyncGenerator<ArrowTable>;
204
+ /**
205
+ * Request a profile of a running Enterprise Function worker session.
206
+ * Returns the accepted identity without waiting for recording to finish.
207
+ * Requires System Operate permission and server profiling support. The server
208
+ * pins the worker, Python session and execution epoch. One capture per job
209
+ * can be active. Captures add workload-dependent overhead.
210
+ *
211
+ * GPU tracing requires initialized PyTorch CUDA/CUPTI. CPU sampling requires
212
+ * readable per-thread procfs and is unavailable in dedicated sandboxes.
213
+ *
214
+ * @example
215
+ * ```ts
216
+ * const profile = await job.captureProfile("worker-id", {
217
+ * kind: "gpu", durationSeconds: 2,
218
+ * });
219
+ * await job.waitForProfile(profile.profileId, { timeoutMs: 60_000 });
220
+ * await job.exportProfile(profile.profileId, "gpu.trace.json");
221
+ * ```
222
+ */
223
+ captureProfile(workerId: string, options?: FunctionProfileOptions): Promise<FunctionProfile>;
224
+ /**
225
+ * Fetch capture identity and status, including retained lookup after refresh.
226
+ * Failed captures return status "failed" with the server's error reason.
227
+ * This never changes the parent job's cached state.
228
+ */
229
+ profileStatus(profileId: string): Promise<FunctionProfile>;
230
+ /**
231
+ * Wait for a capture with a configurable timeout and polling cadence.
232
+ * Defaults to 60 seconds with two seconds between requests. A timeout stops
233
+ * only this wait; the capture and Function refresh continue. A failed capture
234
+ * rejects with its server-reported reason. Keep its ID to wait again.
235
+ */
236
+ waitForProfile(profileId: string, options?: FunctionProfileWaitOptions): Promise<FunctionProfile>;
237
+ /**
238
+ * Retrieve a completed capture's actual JSON: speedscope for CPU,
239
+ * tracemalloc for Python allocations, or Chrome Trace Event JSON for GPU.
240
+ * Truncation flags are preserved. Retention is at most one hour and eight
241
+ * captures per job, until earlier scratch reclamation or archival.
242
+ * Permission, expired-artifact and unsupported-server errors propagate.
243
+ */
244
+ profileArtifact(profileId: string): Promise<Record<string, unknown>>;
245
+ /**
246
+ * Write a completed capture's JSON after successful retrieval. Open CPU
247
+ * exports in speedscope and GPU exports in Perfetto. GPU activity durations
248
+ * can overlap and are not utilization; Python operator attribution and GPU
249
+ * allocation stacks are excluded. Filesystem errors propagate to the caller.
250
+ */
251
+ exportProfile(profileId: string, path: string): Promise<void>;
174
252
  private lastProgress;
175
253
  private enrichProgress;
176
254
  /**
package/dist/job.js CHANGED
@@ -3,9 +3,12 @@
3
3
  // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
4
  Object.defineProperty(exports, "__esModule", { value: true });
5
5
  exports.Job = void 0;
6
+ const promises_1 = require("node:fs/promises");
6
7
  const apache_arrow_1 = require("apache-arrow");
8
+ const cleanup_1 = require("./cleanup");
7
9
  const compaction_1 = require("./compaction");
8
10
  const prewarm_1 = require("./prewarm");
11
+ const profiling_1 = require("./profiling");
9
12
  /**
10
13
  * A handle to an operation that may still be running.
11
14
  *
@@ -34,7 +37,9 @@ class Job {
34
37
  ? prewarm_1.decodePrewarmDataResult
35
38
  : inner.jobType === "compact"
36
39
  ? compaction_1.decodeCompactionResult
37
- : undefined);
40
+ : inner.jobType === "cleanup"
41
+ ? cleanup_1.decodeCleanupResult
42
+ : undefined);
38
43
  }
39
44
  /**
40
45
  * Identifies the operation on the server that is running it.
@@ -47,6 +52,52 @@ class Job {
47
52
  get id() {
48
53
  return this.inner.id ?? null;
49
54
  }
55
+ /**
56
+ * Identity of the selected attempt last observed by this handle.
57
+ * For operation handles, `refresh()` or `status()` can select a different
58
+ * attempt while `id` stays unchanged. Save this value after refreshing and
59
+ * use it throughout paged diagnostics. Local IDs cannot be reopened on a server.
60
+ */
61
+ get jobId() {
62
+ return this.inner.jobId ?? null;
63
+ }
64
+ /** Root of the explicit retry lineage, when reported. */
65
+ get operationId() {
66
+ return this.inner.operationId ?? null;
67
+ }
68
+ /** One-based attempt ordinal. Worker replacement does not increment it. */
69
+ get attempt() {
70
+ return this.inner.attempt ?? null;
71
+ }
72
+ /** Immediate predecessor attempt, when reported. */
73
+ get retryOf() {
74
+ return this.inner.retryOf ?? null;
75
+ }
76
+ /** Detailed state last observed; unknown values never establish success. */
77
+ get lifecycleState() {
78
+ return this.inner.lifecycleState ?? null;
79
+ }
80
+ /** Source snapshot pinned for this attempt, when reported. */
81
+ get sourceTableVersion() {
82
+ return this.inner.sourceTableVersion ?? null;
83
+ }
84
+ /** Most recent execution's fencing token, when reported. */
85
+ get executionEpoch() {
86
+ return this.inner.executionEpoch ?? null;
87
+ }
88
+ /** Refresh and return the detailed lifecycle state. */
89
+ async statusTyped() {
90
+ const state = await this.status();
91
+ if (this.lifecycleState !== null)
92
+ return this.lifecycleState;
93
+ if (state === "running")
94
+ return "in_progress";
95
+ if (state === "finished")
96
+ return "succeeded";
97
+ if (state === "cancelled")
98
+ return "canceled";
99
+ return state;
100
+ }
50
101
  /** The last observed lifecycle state, without contacting the backend. */
51
102
  get state() {
52
103
  return this.inner.state ?? null;
@@ -81,8 +132,8 @@ class Job {
81
132
  return this.inner.failure ?? null;
82
133
  }
83
134
  /**
84
- * The operation's current lifecycle state: "running", "finished", "failed",
85
- * or "cancelled".
135
+ * The current status, usually "running", "finished", "failed", or "cancelled".
136
+ * A Function refresh waiting for execution workers reports "waiting_for_capacity".
86
137
  *
87
138
  * A point snapshot; unlike {@link Job.wait} it does not block or reject on a
88
139
  * terminal failure state. Also refreshes the getters above.
@@ -162,6 +213,10 @@ class Job {
162
213
  async cancel() {
163
214
  return this.inner.cancel();
164
215
  }
216
+ /** Select this exact attempt's pinned successor. Requires backend support. */
217
+ async retry() {
218
+ return new Job(await this.inner.retry(), this.resultDecoder);
219
+ }
165
220
  /**
166
221
  * Ask the backend for this job's current state, and for a server-side job
167
222
  * its full record, then cache it for the getters above.
@@ -184,12 +239,91 @@ class Job {
184
239
  * `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
185
240
  */
186
241
  async events(options) {
187
- const buf = await this.inner.events(options?.limit, options?.filter);
242
+ const buf = await this.inner.events(options?.limit, options?.filter, options?.pageToken);
188
243
  if (buf.length === 0) {
189
244
  return new apache_arrow_1.Table();
190
245
  }
191
246
  return (0, apache_arrow_1.tableFromIPC)(buf);
192
247
  }
248
+ /** Walk retained history one page at a time using the server's continuation. */
249
+ async *history(options) {
250
+ let pageToken = options?.pageToken;
251
+ for (;;) {
252
+ const page = await this.events({ ...options, pageToken });
253
+ yield page;
254
+ const next = page.schema.metadata.get("lancedb.job_coordinator.page_token");
255
+ if (next === undefined)
256
+ return;
257
+ if (next === pageToken)
258
+ throw new Error("Job history repeated its page token");
259
+ pageToken = next;
260
+ }
261
+ }
262
+ /**
263
+ * Request a profile of a running Enterprise Function worker session.
264
+ * Returns the accepted identity without waiting for recording to finish.
265
+ * Requires System Operate permission and server profiling support. The server
266
+ * pins the worker, Python session and execution epoch. One capture per job
267
+ * can be active. Captures add workload-dependent overhead.
268
+ *
269
+ * GPU tracing requires initialized PyTorch CUDA/CUPTI. CPU sampling requires
270
+ * readable per-thread procfs and is unavailable in dedicated sandboxes.
271
+ *
272
+ * @example
273
+ * ```ts
274
+ * const profile = await job.captureProfile("worker-id", {
275
+ * kind: "gpu", durationSeconds: 2,
276
+ * });
277
+ * await job.waitForProfile(profile.profileId, { timeoutMs: 60_000 });
278
+ * await job.exportProfile(profile.profileId, "gpu.trace.json");
279
+ * ```
280
+ */
281
+ async captureProfile(workerId, options = {}) {
282
+ (0, profiling_1.validateProfileOptions)(options, ["kind", "durationSeconds", "sessionSlot"]);
283
+ const kind = options.kind ?? "cpu";
284
+ if (!["cpu", "gpu", "memory"].includes(kind)) {
285
+ throw new TypeError("kind must be cpu, memory or gpu");
286
+ }
287
+ return (0, profiling_1.decodeFunctionProfile)(await this.inner.captureProfile(workerId, kind, options.durationSeconds, options.sessionSlot));
288
+ }
289
+ /**
290
+ * Fetch capture identity and status, including retained lookup after refresh.
291
+ * Failed captures return status "failed" with the server's error reason.
292
+ * This never changes the parent job's cached state.
293
+ */
294
+ async profileStatus(profileId) {
295
+ return (0, profiling_1.decodeFunctionProfile)(await this.inner.profileStatus(profileId));
296
+ }
297
+ /**
298
+ * Wait for a capture with a configurable timeout and polling cadence.
299
+ * Defaults to 60 seconds with two seconds between requests. A timeout stops
300
+ * only this wait; the capture and Function refresh continue. A failed capture
301
+ * rejects with its server-reported reason. Keep its ID to wait again.
302
+ */
303
+ async waitForProfile(profileId, options = {}) {
304
+ (0, profiling_1.validateProfileOptions)(options, ["timeoutMs", "pollIntervalMs"]);
305
+ return (0, profiling_1.decodeFunctionProfile)(await this.inner.waitForProfile(profileId, options.timeoutMs, options.pollIntervalMs));
306
+ }
307
+ /**
308
+ * Retrieve a completed capture's actual JSON: speedscope for CPU,
309
+ * tracemalloc for Python allocations, or Chrome Trace Event JSON for GPU.
310
+ * Truncation flags are preserved. Retention is at most one hour and eight
311
+ * captures per job, until earlier scratch reclamation or archival.
312
+ * Permission, expired-artifact and unsupported-server errors propagate.
313
+ */
314
+ async profileArtifact(profileId) {
315
+ return JSON.parse(await this.inner.profileArtifact(profileId));
316
+ }
317
+ /**
318
+ * Write a completed capture's JSON after successful retrieval. Open CPU
319
+ * exports in speedscope and GPU exports in Perfetto. GPU activity durations
320
+ * can overlap and are not utilization; Python operator attribution and GPU
321
+ * allocation stacks are excluded. Filesystem errors propagate to the caller.
322
+ */
323
+ async exportProfile(profileId, path) {
324
+ const artifact = await this.inner.profileArtifact(profileId);
325
+ await (0, promises_1.writeFile)(path, artifact, "utf8");
326
+ }
193
327
  lastProgress = null;
194
328
  enrichProgress(progress) {
195
329
  const enriched = withProgressEta(this.lastProgress, progress);