@lancedb/lancedb 0.41.0 → 0.42.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/job.d.ts CHANGED
@@ -7,6 +7,38 @@ export interface JobEventsOptions {
7
7
  /** SQL-like filter over the event columns. */
8
8
  filter?: string;
9
9
  }
10
+ /**
11
+ * A forward-compatible structured progress snapshot for a server-side job.
12
+ *
13
+ * Fields are optional because older servers and different job types report
14
+ * different counters. A missing counter is unknown; it is not zero.
15
+ */
16
+ export interface JobProgress {
17
+ /** Lifecycle state, including states from newer servers. */
18
+ state?: string;
19
+ /** Coarse server-derived phase, including phases from newer servers. */
20
+ phase?: string | null;
21
+ /** Whether an operator has parked the job. */
22
+ paused?: boolean;
23
+ /**
24
+ * Planned, computed, skipped, remaining, and failed row counters, when the
25
+ * server reports them.
26
+ */
27
+ rows_planned?: number | null;
28
+ rows_computed?: number | null;
29
+ rows_skipped?: number | null;
30
+ rows_remaining?: number | null;
31
+ rows_failed?: number | null;
32
+ /** Execution identity and sample time used for client-side ETA derivation. */
33
+ execution_epoch?: number | null;
34
+ sample_at_millis?: number | null;
35
+ /** Best-effort remaining milliseconds, present only for a reliable sample. */
36
+ estimated_remaining_time_millis?: number;
37
+ /** Retain job-type-specific and newer-server fields without interpretation. */
38
+ [field: string]: unknown;
39
+ }
40
+ /** Callback for progress snapshots observed while waiting for a job. */
41
+ export type JobProgressCallback = (progress: JobProgress) => void;
10
42
  /**
11
43
  * A handle to an operation that may still be running.
12
44
  *
@@ -63,8 +95,25 @@ export declare class Job {
63
95
  * terminal failure state. Also refreshes the getters above.
64
96
  */
65
97
  status(): Promise<string>;
66
- /** Wait until the operation reaches a terminal state. */
67
- wait(): Promise<void>;
98
+ /**
99
+ * Fetch a structured progress snapshot.
100
+ *
101
+ * Registry jobs include lifecycle and queue metadata even when their job
102
+ * type does not report counters. The returned object is forward-compatible
103
+ * with newer servers and job types. In-process operations return null.
104
+ */
105
+ progress(): Promise<JobProgress | null>;
106
+ /**
107
+ * Wait until the operation reaches a terminal state.
108
+ *
109
+ * The optional callback observes the same snapshots fetched by the backend
110
+ * poller that drives completion; it does not start another poller. Without a
111
+ * callback, this preserves the original wait behavior. The callback is a
112
+ * synchronous observer: its return value is ignored and not awaited. If it
113
+ * throws, the error is logged with `console.warn` and waiting continues; a
114
+ * callback exception does not reject `wait`.
115
+ */
116
+ wait(progressCallback?: JobProgressCallback): Promise<void>;
68
117
  /** Request cancellation. Cancelling a finished operation is a no-op. */
69
118
  cancel(): Promise<void>;
70
119
  /**
@@ -87,6 +136,8 @@ export declare class Job {
87
136
  * `updated_by`, `emitted_from`, `emitted_by`, and `claim_entity` columns.
88
137
  */
89
138
  events(options?: JobEventsOptions): Promise<ArrowTable>;
139
+ private lastProgress;
140
+ private enrichProgress;
90
141
  /**
91
142
  * Every field the handle currently knows, one per line, with the JSON
92
143
  * payloads indented -- a refresh job's spec and result are the point of
package/dist/job.js CHANGED
@@ -80,9 +80,39 @@ class Job {
80
80
  async status() {
81
81
  return this.inner.status();
82
82
  }
83
- /** Wait until the operation reaches a terminal state. */
84
- async wait() {
85
- return this.inner.wait();
83
+ /**
84
+ * Fetch a structured progress snapshot.
85
+ *
86
+ * Registry jobs include lifecycle and queue metadata even when their job
87
+ * type does not report counters. The returned object is forward-compatible
88
+ * with newer servers and job types. In-process operations return null.
89
+ */
90
+ async progress() {
91
+ const progress = await this.inner.progress();
92
+ return progress === null ? null : this.enrichProgress(progress);
93
+ }
94
+ /**
95
+ * Wait until the operation reaches a terminal state.
96
+ *
97
+ * The optional callback observes the same snapshots fetched by the backend
98
+ * poller that drives completion; it does not start another poller. Without a
99
+ * callback, this preserves the original wait behavior. The callback is a
100
+ * synchronous observer: its return value is ignored and not awaited. If it
101
+ * throws, the error is logged with `console.warn` and waiting continues; a
102
+ * callback exception does not reject `wait`.
103
+ */
104
+ async wait(progressCallback) {
105
+ if (progressCallback === undefined) {
106
+ return this.inner.wait();
107
+ }
108
+ return this.inner.wait((progress) => {
109
+ try {
110
+ progressCallback(this.enrichProgress(progress));
111
+ }
112
+ catch (error) {
113
+ console.warn("Job progress callback threw:", error);
114
+ }
115
+ });
86
116
  }
87
117
  /** Request cancellation. Cancelling a finished operation is a no-op. */
88
118
  async cancel() {
@@ -116,6 +146,12 @@ class Job {
116
146
  }
117
147
  return (0, apache_arrow_1.tableFromIPC)(buf);
118
148
  }
149
+ lastProgress = null;
150
+ enrichProgress(progress) {
151
+ const enriched = withProgressEta(this.lastProgress, progress);
152
+ this.lastProgress = enriched;
153
+ return enriched;
154
+ }
119
155
  /**
120
156
  * Every field the handle currently knows, one per line, with the JSON
121
157
  * payloads indented -- a refresh job's spec and result are the point of
@@ -156,6 +192,85 @@ class Job {
156
192
  }
157
193
  exports.Job = Job;
158
194
  const REPR_INDENT = " ";
195
+ const ETA_MAX_SAMPLE_INTERVAL_MILLIS = 10 * 60 * 1_000;
196
+ function withProgressEta(previous, current) {
197
+ const snapshot = { ...current };
198
+ const estimate = progressEta(previous, current);
199
+ if (estimate !== null) {
200
+ snapshot.estimated_remaining_time_millis = estimate;
201
+ }
202
+ return snapshot;
203
+ }
204
+ function progressEta(previous, current) {
205
+ if (previous === null ||
206
+ (current.state !== "running" && current.state !== "finished")) {
207
+ return null;
208
+ }
209
+ const remaining = integerValue(current.rows_remaining);
210
+ if (current.state === "finished") {
211
+ return remaining === 0 ? 0 : null;
212
+ }
213
+ if (current.paused === true ||
214
+ previous.paused === true ||
215
+ isTransitionalPhase(current.phase) ||
216
+ isTransitionalPhase(previous.phase)) {
217
+ return null;
218
+ }
219
+ const epoch = integerValue(current.execution_epoch);
220
+ const previousEpoch = integerValue(previous.execution_epoch);
221
+ if (epoch === null || previousEpoch !== epoch) {
222
+ return null;
223
+ }
224
+ const sampleAt = integerValue(current.sample_at_millis);
225
+ const previousAt = integerValue(previous.sample_at_millis);
226
+ if (sampleAt === null || previousAt === null) {
227
+ return null;
228
+ }
229
+ const elapsed = sampleAt - previousAt;
230
+ if (elapsed <= 0 ||
231
+ elapsed > ETA_MAX_SAMPLE_INTERVAL_MILLIS ||
232
+ Date.now() - sampleAt > ETA_MAX_SAMPLE_INTERVAL_MILLIS) {
233
+ return null;
234
+ }
235
+ const planned = integerValue(current.rows_planned);
236
+ const currentRemaining = integerValue(current.rows_remaining);
237
+ const previousRemaining = integerValue(previous.rows_remaining);
238
+ if (planned === null ||
239
+ currentRemaining === null ||
240
+ previousRemaining === null) {
241
+ return null;
242
+ }
243
+ if (currentRemaining === 0) {
244
+ return 0;
245
+ }
246
+ if (planned !== previous.rows_planned) {
247
+ return null;
248
+ }
249
+ if (current.fragments_planned !== previous.fragments_planned) {
250
+ return null;
251
+ }
252
+ const currentComputed = integerValue(current.rows_computed);
253
+ const previousComputed = integerValue(previous.rows_computed);
254
+ if (currentComputed === null || previousComputed === null) {
255
+ return null;
256
+ }
257
+ const currentAccounted = currentComputed + (integerValue(current.rows_skipped) ?? 0);
258
+ const previousAccounted = previousComputed + (integerValue(previous.rows_skipped) ?? 0);
259
+ const advanced = currentAccounted - previousAccounted;
260
+ if (advanced <= 0 || currentRemaining > previousRemaining) {
261
+ return null;
262
+ }
263
+ return Math.floor((currentRemaining * elapsed) / advanced);
264
+ }
265
+ function integerValue(value) {
266
+ return typeof value === "number" && Number.isInteger(value) ? value : null;
267
+ }
268
+ function isTransitionalPhase(phase) {
269
+ return (phase === "queued" ||
270
+ phase === "starting" ||
271
+ phase === "initializing" ||
272
+ phase === "paused");
273
+ }
159
274
  // biome-ignore lint/suspicious/noExplicitAny: shape varies by job type
160
275
  function indentJson(value) {
161
276
  return JSON.stringify(value, null, 4).replace(/\n/g, `\n${REPR_INDENT}`);
package/dist/native.d.ts CHANGED
@@ -135,17 +135,39 @@ export declare class DatabaseNames {
135
135
  next(): Promise<string | null>
136
136
  }
137
137
 
138
+ export declare class Expr {
139
+ eq(other: Expr): Expr
140
+ ne(other: Expr): Expr
141
+ lt(other: Expr): Expr
142
+ lte(other: Expr): Expr
143
+ gt(other: Expr): Expr
144
+ gte(other: Expr): Expr
145
+ and(other: Expr): Expr
146
+ or(other: Expr): Expr
147
+ not(): Expr
148
+ add(other: Expr): Expr
149
+ sub(other: Expr): Expr
150
+ mul(other: Expr): Expr
151
+ div(other: Expr): Expr
152
+ lower(): Expr
153
+ upper(): Expr
154
+ contains(substr: Expr): Expr
155
+ isIn(list: Array<Expr>): Expr
156
+ cast(dataType: string): Expr
157
+ toSql(): string
158
+ }
159
+
138
160
  export declare class Index {
139
- static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
140
- static ivfRq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
141
- static ivfFlat(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
161
+ static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, targetPartitionSize?: number | undefined | null): Index
162
+ static ivfRq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, targetPartitionSize?: number | undefined | null): Index
163
+ static ivfFlat(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, targetPartitionSize?: number | undefined | null): Index
142
164
  static btree(): Index
143
165
  static bitmap(): Index
144
166
  static labelList(): Index
145
167
  static fm(): Index
146
168
  static fts(withPosition?: boolean | undefined | null, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, customStopWords?: Array<string> | undefined | null, asciiFolding?: boolean | undefined | null, ngramMinLength?: number | undefined | null, ngramMaxLength?: number | undefined | null, prefixOnly?: boolean | undefined | null, blockSize?: number | undefined | null): Index
147
- static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
148
- static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
169
+ static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null, targetPartitionSize?: number | undefined | null): Index
170
+ static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null, targetPartitionSize?: number | undefined | null): Index
149
171
  }
150
172
 
151
173
  /** A handle to an operation that may still be running. */
@@ -164,8 +186,18 @@ export declare class Job {
164
186
  * client version does not know pass through as-is.
165
187
  */
166
188
  status(): Promise<string>
167
- /** Wait until the operation reaches a terminal state. */
168
- wait(): Promise<void>
189
+ /**
190
+ * The current structured progress snapshot, or null for an in-process
191
+ * operation. Registry snapshots include lifecycle and queue metadata even
192
+ * when the job type or server does not report counters.
193
+ */
194
+ progress(): Promise<Record<string, unknown> | null>
195
+ /**
196
+ * Wait until the operation reaches a terminal state. When a callback is
197
+ * supplied, observe the same progress snapshots fetched by the existing
198
+ * backend poller without starting another polling loop.
199
+ */
200
+ wait(progressCallback?: (progress: Record<string, unknown>) => void): Promise<void>
169
201
  /** Request cancellation. Cancelling a finished operation is a no-op. */
170
202
  cancel(): Promise<void>
171
203
  /**
@@ -506,6 +538,7 @@ export declare class VectorQuery {
506
538
  maximumNprobes(maximumNprobes: number): void
507
539
  distanceRange(lowerBound?: number | undefined | null, upperBound?: number | undefined | null): void
508
540
  ef(ef: number): void
541
+ approxMode(approxMode: string): void
509
542
  bypassVectorIndex(): void
510
543
  onlyIf(predicate: string): void
511
544
  fullTextSearch(query: object): void
@@ -582,7 +615,7 @@ export interface ClientConfig {
582
615
  /**
583
616
  * User identifier for tracking purposes.
584
617
  *
585
- * This is sent as the `x-lancedb-user-id` header in requests to LanceDB Cloud/Enterprise.
618
+ * This is sent as the `x-lancedb-user-id` header in requests to LanceDB Enterprise.
586
619
  * It can be set directly, or via the `LANCEDB_USER_ID` environment variable.
587
620
  * Alternatively, set `LANCEDB_USER_ID_ENV_KEY` to specify another environment
588
621
  * variable that contains the user ID value.
@@ -590,6 +623,14 @@ export interface ClientConfig {
590
623
  userId?: string
591
624
  }
592
625
 
626
+ /**
627
+ * Create a column reference expression.
628
+ *
629
+ * The column name is preserved exactly as given (case-sensitive), so
630
+ * `col("firstName")` correctly references a field named `firstName`.
631
+ */
632
+ export declare function col(name: string): Expr
633
+
593
634
  /**
594
635
  * A definition of a column alteration. The alteration changes the column at
595
636
  * `path` to have the new name `name`, to be nullable if `nullable` is true,
@@ -661,47 +702,47 @@ export interface ConnectionOptions {
661
702
  */
662
703
  readConsistencyInterval?: number
663
704
  /**
664
- * (For LanceDB OSS only): configuration for object storage.
705
+ * (For LanceDB Embedded only): configuration for object storage.
665
706
  *
666
707
  * The available options are described at https://docs.lancedb.com/storage/
667
708
  */
668
709
  storageOptions?: Record<string, string>
669
710
  /**
670
- * (For LanceDB OSS only): use directory namespace manifests as the source
711
+ * (For LanceDB Embedded only): use directory namespace manifests as the source
671
712
  * of truth for table metadata. Existing directory-listed root tables are
672
713
  * migrated into the manifest on access.
673
714
  */
674
715
  manifestEnabled?: boolean
675
716
  /**
676
- * (For LanceDB OSS only): extra properties for the backing namespace
717
+ * (For LanceDB Embedded only): extra properties for the backing namespace
677
718
  * client used by manifest-enabled native connections.
678
719
  */
679
720
  namespaceClientProperties?: Record<string, string>
680
721
  /**
681
- * (For LanceDB OSS only): the session to use for this connection. Holds
722
+ * (For LanceDB Embedded only): the session to use for this connection. Holds
682
723
  * shared caches and other session-specific state.
683
724
  */
684
725
  session?: Session
685
- /** (For LanceDB cloud only): configuration for the remote HTTP client. */
726
+ /** (For LanceDB Enterprise or LanceDB Cloud only): configuration for the HTTP client. */
686
727
  clientConfig?: ClientConfig
687
728
  /**
688
- * (For LanceDB cloud only): the API key to use with LanceDB Cloud.
729
+ * (For LanceDB Enterprise or LanceDB Cloud only): the API key to use.
689
730
  *
690
731
  * Can also be set via the environment variable `LANCEDB_API_KEY`.
691
732
  */
692
733
  apiKey?: string
693
734
  /**
694
- * (For LanceDB cloud only): the region to use for LanceDB cloud.
735
+ * (For LanceDB Cloud only): the region to use for LanceDB Cloud (legacy).
695
736
  * Defaults to 'us-east-1'.
696
737
  */
697
738
  region?: string
698
739
  /**
699
- * (For LanceDB cloud only): the host to use for LanceDB cloud. Used
700
- * for testing purposes.
740
+ * (For LanceDB Enterprise only): the host of the LanceDB Enterprise
741
+ * deployment to connect to.
701
742
  */
702
743
  hostOverride?: string
703
744
  /**
704
- * (For LanceDB cloud only): OAuth configuration for IdP-based
745
+ * (For LanceDB Enterprise only): OAuth configuration for IdP-based
705
746
  * authentication (e.g., Azure Entra ID). When set, token acquisition
706
747
  * and refresh are handled entirely in Rust. TypeScript users should pass
707
748
  * the public `OAuthConfig` type exported from `@lancedb/lancedb`.
@@ -807,6 +848,9 @@ export interface FtsToken {
807
848
  position: number
808
849
  }
809
850
 
851
+ /** Call an arbitrary registered SQL function by name. */
852
+ export declare function func(name: string, args: Array<Expr>): Expr
853
+
810
854
  /**
811
855
  * A fragment whose per-row error detail was capped: `rowsSkipped` rows
812
856
  * failed, of which only `rowsRecorded` have a record of their own.
@@ -872,56 +916,56 @@ export interface IndexConfig {
872
916
  /**
873
917
  * The UUID of the first segment of the index.
874
918
  *
875
- * `undefined` for remote tables, which do not yet surface this.
919
+ * `undefined` in LanceDB Enterprise, which does not yet surface this.
876
920
  */
877
921
  indexUuid?: string
878
922
  /**
879
923
  * The protobuf type URL, a precise type identifier for the index.
880
924
  *
881
- * `undefined` for remote tables.
925
+ * `undefined` in LanceDB Enterprise.
882
926
  */
883
927
  typeUrl?: string
884
928
  /**
885
929
  * When the index was created.
886
930
  *
887
- * `undefined` for remote tables or indices created before timestamps were tracked.
931
+ * `undefined` in LanceDB Enterprise or indices created before timestamps were tracked.
888
932
  */
889
933
  createdAt?: Date
890
934
  /**
891
935
  * The number of rows indexed, across all segments.
892
936
  *
893
- * `undefined` for remote tables.
937
+ * May be `undefined` in LanceDB Enterprise if the server does not report it.
894
938
  */
895
939
  numIndexedRows?: number
896
940
  /**
897
941
  * The number of rows not yet covered by this index.
898
942
  *
899
- * `undefined` for remote tables.
943
+ * May be `undefined` in LanceDB Enterprise if the server does not report it.
900
944
  */
901
945
  numUnindexedRows?: number
902
946
  /**
903
947
  * The total size in bytes of all index files across all segments.
904
948
  *
905
- * `undefined` for remote tables or indices without size tracking.
949
+ * `undefined` in LanceDB Enterprise or indices without size tracking.
906
950
  */
907
951
  sizeBytes?: number
908
952
  /**
909
953
  * The number of segments that make up the index.
910
954
  *
911
- * `undefined` for remote tables.
955
+ * `undefined` in LanceDB Enterprise.
912
956
  */
913
957
  numSegments?: number
914
958
  /**
915
959
  * The on-disk index format version.
916
960
  *
917
- * `undefined` for remote tables.
961
+ * `undefined` in LanceDB Enterprise.
918
962
  */
919
963
  indexVersion?: number
920
964
  /**
921
965
  * Index-type-specific details parsed as a JavaScript object.
922
966
  *
923
- * Falls back to a raw string if JSON parsing fails. `undefined` for
924
- * remote tables or when details are unavailable.
967
+ * Falls back to a raw string if JSON parsing fails. `undefined` in
968
+ * LanceDB Enterprise or when details are unavailable.
925
969
  */
926
970
  indexDetails?: any
927
971
  }
@@ -964,11 +1008,24 @@ export interface JobInfo {
964
1008
  state: string
965
1009
  /** When the job was created, in milliseconds since the epoch. */
966
1010
  createdAtMillis: number
1011
+ /** Structured lifecycle, queue, pause, and job-type-specific progress. */
1012
+ progress: Record<string, unknown>
1013
+ /** Whether an operator has parked this job. */
1014
+ paused: boolean
967
1015
  }
968
1016
 
969
1017
  /** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
970
1018
  export declare function lancedbMetricsCatalog(): Array<MetricDescription>
971
1019
 
1020
+ /** Create a literal (constant) value expression. */
1021
+ export declare function lit(value: string | number): Expr
1022
+
1023
+ /** Create a literal boolean value expression. */
1024
+ export declare function litBool(value: boolean): Expr
1025
+
1026
+ /** Create a literal integer value expression. */
1027
+ export declare function litInt(value: number): Expr
1028
+
972
1029
  /** One in-memory memtable. */
973
1030
  export interface MemtableStats {
974
1031
  /** The generation this memtable will become once frozen. */
@@ -1107,11 +1164,11 @@ export interface RefreshColumnResult {
1107
1164
  }
1108
1165
 
1109
1166
  export interface RefreshMaterializedViewResult {
1110
- /** How the view was brought up to date: "rebuild", or absent for remote SQL refreshes. */
1167
+ /** How the view was brought up to date: "rebuild", or absent for LanceDB Enterprise SQL refreshes. */
1111
1168
  mode?: string
1112
- /** Rows written by this refresh, or absent for remote SQL refreshes. */
1169
+ /** Rows written by this refresh, or absent for LanceDB Enterprise SQL refreshes. */
1113
1170
  rowsWritten?: number
1114
- /** Source version refreshed to, or absent for remote SQL refreshes. */
1171
+ /** Source version refreshed to, or absent for LanceDB Enterprise SQL refreshes. */
1115
1172
  sourceVersion?: number
1116
1173
  version: number
1117
1174
  }
@@ -1180,7 +1237,7 @@ export interface RetryConfig {
1180
1237
  backoffJitter?: number
1181
1238
  /**
1182
1239
  * The HTTP status codes for which to retry the request. Default is
1183
- * [429, 500, 502, 503].
1240
+ * [409, 429, 500, 502, 503, 504].
1184
1241
  *
1185
1242
  * You can also set this via the environment variable
1186
1243
  * `LANCE_CLIENT_RETRY_STATUSES`. Use a comma-separated list of integers.