@lancedb/lancedb 0.40.0 → 0.41.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,41 @@
1
+ /** Pagination controls for lazy resource listings. */
2
+ export interface ListingOptions {
3
+ /** Starting continuation token; omitted starts at the beginning. */
4
+ pageToken?: string;
5
+ /** Maximum results per request, not a limit on the whole listing. */
6
+ pageLimit?: number;
7
+ }
8
+ /** @internal */
9
+ export declare function validateListingOptions(options: ListingOptions): void;
10
+ /**
11
+ * A lazy async iterator over database resources with pagination state.
12
+ * Requests occur during iteration; request errors terminate the iterator.
13
+ *
14
+ * @example
15
+ * ```ts
16
+ * for await (const name of connection.listViews([], { pageLimit: 20 })) {
17
+ * console.log(name);
18
+ * }
19
+ * ```
20
+ */
21
+ export declare class Listing<T> implements AsyncIterableIterator<T> {
22
+ private readonly inner;
23
+ /** @hidden */
24
+ constructor(inner: {
25
+ next(): Promise<T | null | undefined>;
26
+ numPageResults(): number;
27
+ pageToken(): string | null | undefined;
28
+ });
29
+ [Symbol.asyncIterator](): AsyncIterableIterator<T>;
30
+ /** Fetch the next item, requesting another page only when needed. */
31
+ next(): Promise<IteratorResult<T>>;
32
+ /** Number of cached items available without another request. */
33
+ numPageResults(): number;
34
+ /**
35
+ * Token for the next request. Initially this is the supplied starting token.
36
+ * Returns undefined after fetching the final page, even with cached items.
37
+ * Drain the cache before saving a token to avoid skipping items on resumption.
38
+ * Failed requests terminate iteration and retain their token for a new iterator.
39
+ */
40
+ pageToken(): string | undefined;
41
+ }
@@ -0,0 +1,57 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.Listing = void 0;
6
+ exports.validateListingOptions = validateListingOptions;
7
+ /** @internal */
8
+ function validateListingOptions(options) {
9
+ if (options.pageLimit !== undefined &&
10
+ (!Number.isInteger(options.pageLimit) ||
11
+ options.pageLimit <= 0 ||
12
+ options.pageLimit > 2147483647)) {
13
+ throw new Error("Listing page limit must be an integer between 1 and 2147483647");
14
+ }
15
+ }
16
+ /**
17
+ * A lazy async iterator over database resources with pagination state.
18
+ * Requests occur during iteration; request errors terminate the iterator.
19
+ *
20
+ * @example
21
+ * ```ts
22
+ * for await (const name of connection.listViews([], { pageLimit: 20 })) {
23
+ * console.log(name);
24
+ * }
25
+ * ```
26
+ */
27
+ class Listing {
28
+ inner;
29
+ /** @hidden */
30
+ constructor(inner) {
31
+ this.inner = inner;
32
+ }
33
+ [Symbol.asyncIterator]() {
34
+ return this;
35
+ }
36
+ /** Fetch the next item, requesting another page only when needed. */
37
+ async next() {
38
+ const value = await this.inner.next();
39
+ return value == null
40
+ ? { done: true, value: undefined }
41
+ : { done: false, value };
42
+ }
43
+ /** Number of cached items available without another request. */
44
+ numPageResults() {
45
+ return this.inner.numPageResults();
46
+ }
47
+ /**
48
+ * Token for the next request. Initially this is the supplied starting token.
49
+ * Returns undefined after fetching the final page, even with cached items.
50
+ * Drain the cache before saving a token to avoid skipping items on resumption.
51
+ * Failed requests terminate iteration and retain their token for a new iterator.
52
+ */
53
+ pageToken() {
54
+ return this.inner.pageToken() ?? undefined;
55
+ }
56
+ }
57
+ exports.Listing = Listing;
@@ -4,10 +4,9 @@ import { Table } from "./table";
4
4
  export declare const DEFINITION_META_KEY = "mv.definition";
5
5
  /**
6
6
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
7
- * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
8
- * native dedup results.
7
+ * format 2 being a query with `GROUP BY`.
9
8
  */
10
- export declare const DEFINITION_FORMAT = 4;
9
+ export declare const DEFINITION_FORMAT = 2;
11
10
  /**
12
11
  * The query that defines a materialized view, as stored:
13
12
  * `SELECT columns FROM [ns.]table [, function(args) AS alias | , UNNEST(column) AS alias]
@@ -58,17 +57,11 @@ export declare class MaterializedView {
58
57
  /**
59
58
  * Recompute the view from its source.
60
59
  *
61
- * The refresh is incremental when the source's changes can be reconciled
62
- * into the view -- rows added, changed or removed since the last one --
63
- * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
64
- * refreshes to that source version instead of the latest.
65
- *
66
- * Concurrent refreshes of one view do not duplicate its rows. Two that
67
- * plan the same source rows conflict on commit, and the loser throws
68
- * rather than writing them a second time.
60
+ * Every refresh rebuilds the complete view. For local views,
61
+ * `sourceVersion` refreshes to that source version instead of the latest;
62
+ * remote SQL refreshes do not support source-version pinning.
69
63
  */
70
64
  refresh(options?: {
71
- full?: boolean;
72
65
  sourceVersion?: number;
73
66
  }): Promise<RefreshMaterializedViewResult>;
74
67
  }
@@ -11,10 +11,9 @@ exports.definitionFromJson = definitionFromJson;
11
11
  exports.DEFINITION_META_KEY = "mv.definition";
12
12
  /**
13
13
  * The newest stored layout this version reads: `{"format": N, "query": "<SQL>"}`,
14
- * format 2 being a query with `GROUP BY`, format 3 native pairs and format 4
15
- * native dedup results.
14
+ * format 2 being a query with `GROUP BY`.
16
15
  */
17
- exports.DEFINITION_FORMAT = 4;
16
+ exports.DEFINITION_FORMAT = 2;
18
17
  /**
19
18
  * @internal Reject a numeric option N-API would otherwise silently coerce:
20
19
  * `Infinity` reaches Rust as 0, `1.5` as 1.
@@ -133,18 +132,13 @@ class MaterializedView {
133
132
  /**
134
133
  * Recompute the view from its source.
135
134
  *
136
- * The refresh is incremental when the source's changes can be reconciled
137
- * into the view -- rows added, changed or removed since the last one --
138
- * and otherwise rebuilds. `full` forces a rebuild; `sourceVersion`
139
- * refreshes to that source version instead of the latest.
140
- *
141
- * Concurrent refreshes of one view do not duplicate its rows. Two that
142
- * plan the same source rows conflict on commit, and the loser throws
143
- * rather than writing them a second time.
135
+ * Every refresh rebuilds the complete view. For local views,
136
+ * `sourceVersion` refreshes to that source version instead of the latest;
137
+ * remote SQL refreshes do not support source-version pinning.
144
138
  */
145
139
  async refresh(options) {
146
140
  validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
147
- return await this.inner.refreshMaterializedView(options?.full, options?.sourceVersion);
141
+ return await this.inner.refreshMaterializedView(options?.sourceVersion);
148
142
  }
149
143
  }
150
144
  exports.MaterializedView = MaterializedView;
package/dist/merge.d.ts CHANGED
@@ -57,25 +57,25 @@ export declare class MergeInsertBuilder {
57
57
  */
58
58
  useIndex(useIndex: boolean): MergeInsertBuilder;
59
59
  /**
60
- * Control MemWAL routing for this merge.
60
+ * Control RTE routing for this merge.
61
61
  *
62
- * By default (unset), a `mergeInsert` on a table with an LSM write spec is
62
+ * By default (unset), a `mergeInsert` on a table with an RTE write spec is
63
63
  * routed through Lance's MemWAL shard writer, and a table without one uses the
64
64
  * standard path.
65
65
  *
66
- * @param enable - `true` forces MemWAL routing and errors if the table has no
67
- * LSM write spec. `false` forces the standard write path even when a spec is set.
66
+ * @param enable - `true` forces RTE routing and errors if the table has no
67
+ * RTE write spec. `false` forces the standard write path even when a spec is set.
68
68
  */
69
- useLsm(enable: boolean): MergeInsertBuilder;
69
+ useRTE(enable: boolean): MergeInsertBuilder;
70
70
  /**
71
71
  * Controls how an LSM merge checks that its input targets a single shard.
72
72
  *
73
- * When a table has an LSM write spec, every row in a `mergeInsert` call must
73
+ * When a table has an RTE write spec, every row in a `mergeInsert` call must
74
74
  * route to the same shard. When `true` (the default), every row is inspected
75
75
  * to verify this. When `false`, only the first row is inspected and the
76
76
  * shard it routes to is used for the whole input — a faster path for callers
77
77
  * that have already pre-sharded their input. Has no effect on tables without
78
- * an LSM write spec.
78
+ * an RTE write spec.
79
79
  *
80
80
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
81
81
  */
package/dist/merge.js CHANGED
@@ -69,27 +69,27 @@ class MergeInsertBuilder {
69
69
  return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
70
70
  }
71
71
  /**
72
- * Control MemWAL routing for this merge.
72
+ * Control RTE routing for this merge.
73
73
  *
74
- * By default (unset), a `mergeInsert` on a table with an LSM write spec is
74
+ * By default (unset), a `mergeInsert` on a table with an RTE write spec is
75
75
  * routed through Lance's MemWAL shard writer, and a table without one uses the
76
76
  * standard path.
77
77
  *
78
- * @param enable - `true` forces MemWAL routing and errors if the table has no
79
- * LSM write spec. `false` forces the standard write path even when a spec is set.
78
+ * @param enable - `true` forces RTE routing and errors if the table has no
79
+ * RTE write spec. `false` forces the standard write path even when a spec is set.
80
80
  */
81
- useLsm(enable) {
82
- return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
81
+ useRTE(enable) {
82
+ return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema);
83
83
  }
84
84
  /**
85
85
  * Controls how an LSM merge checks that its input targets a single shard.
86
86
  *
87
- * When a table has an LSM write spec, every row in a `mergeInsert` call must
87
+ * When a table has an RTE write spec, every row in a `mergeInsert` call must
88
88
  * route to the same shard. When `true` (the default), every row is inspected
89
89
  * to verify this. When `false`, only the first row is inspected and the
90
90
  * shard it routes to is used for the whole input — a faster path for callers
91
91
  * that have already pre-sharded their input. Has no effect on tables without
92
- * an LSM write spec.
92
+ * an RTE write spec.
93
93
  *
94
94
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
95
95
  */
package/dist/native.d.ts CHANGED
@@ -32,7 +32,7 @@ export declare class Catalog {
32
32
  createDatabase(name: string, existOk?: boolean | undefined | null): Promise<Connection>
33
33
  connectDatabase(name: string): Promise<Connection>
34
34
  dropDatabase(name: string, ignoreMissing?: boolean | undefined | null): Promise<void>
35
- listDatabases(limit?: number | undefined | null, pageToken?: string | undefined | null): Promise<ListDatabasesResponse>
35
+ listDatabases(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): DatabaseNames
36
36
  }
37
37
 
38
38
  export declare class Connection {
@@ -58,7 +58,7 @@ export declare class Connection {
58
58
  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
59
59
  createMaterializedView(name: string, source: string, projections: Array<Array<string>> | undefined | null, filter: string | undefined | null, limit: number | undefined | null, withNoData: boolean): Promise<Table>
60
60
  openMaterializedView(name: string): Promise<Table>
61
- listMaterializedViews(): Promise<Array<string>>
61
+ listMaterializedViews(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
62
62
  /** Drop a materialized view. */
63
63
  dropMaterializedView(name: string, namespacePath?: Array<string> | undefined | null): Promise<void>
64
64
  /** Create a view: a named query planned on every read. */
@@ -73,7 +73,7 @@ export declare class Connection {
73
73
  /** Start dropping a view and return the job deleting its definition. */
74
74
  dropViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
75
75
  /** The names of the views in one namespace. */
76
- listViews(namespacePath?: Array<string> | undefined | null): Promise<Array<string>>
76
+ listViews(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
77
77
  /** Start dropping a materialized view and return its cleanup job. */
78
78
  dropMaterializedViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
79
79
  openTable(name: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
@@ -93,7 +93,7 @@ export declare class Connection {
93
93
  */
94
94
  openJob(jobId: string): Promise<Job>
95
95
  /** List server-side jobs across the database's tables. */
96
- listJobs(): Promise<Array<JobInfo>>
96
+ listJobs(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): JobListing
97
97
  /**
98
98
  * Request cancellation of a server-side job by id. Returns true if the
99
99
  * server accepted the cancellation, false if no such job exists.
@@ -125,6 +125,16 @@ export declare class Connection {
125
125
  renameTable(currentName: string, newName: string, currentNamespacePath?: Array<string> | undefined | null, newNamespacePath?: Array<string> | undefined | null): Promise<void>
126
126
  }
127
127
 
128
+ /** A lazy iterator over database names with inspectable pagination state. */
129
+ export declare class DatabaseNames {
130
+ /** Number of names cached without another REST request. */
131
+ numPageResults(): number
132
+ /** Continuation token for the next REST request. */
133
+ pageToken(): string | null
134
+ /** Fetch the next name, returning None when exhausted. */
135
+ next(): Promise<string | null>
136
+ }
137
+
128
138
  export declare class Index {
129
139
  static ivfPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
130
140
  static ivfRq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numBits?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null): Index
@@ -193,6 +203,12 @@ export declare class Job {
193
203
  events(limit?: number | undefined | null, filter?: string | undefined | null): Promise<Buffer>
194
204
  }
195
205
 
206
+ export declare class JobListing {
207
+ numPageResults(): number
208
+ pageToken(): string | null
209
+ next(): Promise<JobInfo | null>
210
+ }
211
+
196
212
  export declare class JsFullTextQuery {
197
213
  static matchQuery(query: string, column: string, boost: number, fuzziness: number | undefined | null, maxExpansions: number, operator: string, prefixLength: number): JsFullTextQuery
198
214
  static phraseQuery(query: string, column: string, slop: number): JsFullTextQuery
@@ -212,6 +228,12 @@ export declare class JsHeaderProvider {
212
228
  constructor(getHeadersCallback: () => Promise<Record<string, string>>)
213
229
  }
214
230
 
231
+ export declare class NameListing {
232
+ numPageResults(): number
233
+ pageToken(): string | null
234
+ next(): Promise<string | null>
235
+ }
236
+
215
237
  /**
216
238
  * A Rust-side view of a JS-constructed `Scannable`.
217
239
  *
@@ -243,7 +265,7 @@ export declare class NativeMergeInsertBuilder {
243
265
  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
244
266
  setTimeout(timeout: number): void
245
267
  useIndex(useIndex: boolean): NativeMergeInsertBuilder
246
- useLsm(enable: boolean): NativeMergeInsertBuilder
268
+ useRTE(enable: boolean): NativeMergeInsertBuilder
247
269
  validateSingleShard(validateSingleShard: boolean): NativeMergeInsertBuilder
248
270
  execute(buf: Buffer): Promise<MergeResult>
249
271
  }
@@ -321,16 +343,18 @@ export declare class Query {
321
343
  nearestToRaw(data: Uint8Array, dtype: string): VectorQuery
322
344
  fastSearch(): void
323
345
  withRowId(): void
324
- useLsm(enable: boolean): void
346
+ useRTE(enable: boolean): void
325
347
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
326
348
  outputSchema(): Promise<Buffer>
327
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
349
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
328
350
  explainPlan(verbose: boolean): Promise<string>
329
351
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
330
352
  }
331
353
 
332
354
  /** Typescript-style Async Iterator over RecordBatches */
333
355
  export declare class RecordBatchIterator {
356
+ /** Return the executed stream's output schema as an Arrow IPC file. */
357
+ schema(): Buffer
334
358
  next(): Promise<Buffer | null>
335
359
  }
336
360
 
@@ -410,21 +434,21 @@ export declare class Table {
410
434
  refreshColumn(column: string): Promise<RefreshColumnResult>
411
435
  refreshColumnAsync(column: string): Promise<Job>
412
436
  functionErrors(options?: FunctionErrorsOptions | undefined | null): Promise<FunctionErrors>
413
- refreshMaterializedView(full?: boolean | undefined | null, sourceVersion?: number | undefined | null): Promise<RefreshMaterializedViewResult>
437
+ refreshMaterializedView(sourceVersion?: number | undefined | null): Promise<RefreshMaterializedViewResult>
414
438
  materializedViewDefinition(): Promise<string>
415
439
  addColumnsWithSchema(schemaBuf: Buffer): Promise<AddColumnsResult>
416
440
  alterColumns(alterations: Array<ColumnAlteration>): Promise<AlterColumnsResult>
417
441
  updateFieldMetadata(updates: Array<FieldMetadataUpdate>): Promise<UpdateFieldMetadataResult>
418
442
  dropColumns(columns: Array<string>): Promise<DropColumnsResult>
419
443
  setUnenforcedPrimaryKey(columns: Array<string>): Promise<void>
420
- setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>
421
- unsetLsmWriteSpec(): Promise<void>
422
- getLsmWriteSpec(): Promise<LsmWriteSpec | null>
423
- closeLsmWriters(): Promise<void>
424
- flushLsm(): Promise<void>
425
- compactLsm(): Promise<void>
426
- checkpointLsm(): Promise<void>
427
- getLsmStats(includeGenerationRows: boolean): Promise<LsmStats | null>
444
+ enableRTE(spec: RTEWriteSpec): Promise<void>
445
+ disableRTE(): Promise<void>
446
+ getRTESpec(): Promise<RTEWriteSpec | null>
447
+ closeRTEWriters(): Promise<void>
448
+ flushRTE(): Promise<void>
449
+ compactRTE(): Promise<void>
450
+ checkpointRTE(): Promise<void>
451
+ getRTEStats(includeSstableRows: boolean): Promise<RTEStats | null>
428
452
  version(): Promise<number>
429
453
  checkoutCurrent(): Promise<Table>
430
454
  checkout(version: number): Promise<void>
@@ -462,9 +486,9 @@ export declare class TakeQuery {
462
486
  select(columns: Array<[string, string]>): void
463
487
  selectColumns(columns: Array<string>): void
464
488
  withRowId(): void
465
- useLsm(enable: boolean): void
489
+ useRTE(enable: boolean): void
466
490
  outputSchema(): Promise<Buffer>
467
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
491
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
468
492
  explainPlan(verbose: boolean): Promise<string>
469
493
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
470
494
  }
@@ -477,6 +501,7 @@ export declare class VectorQuery {
477
501
  postfilter(): void
478
502
  refineFactor(refineFactor: number): void
479
503
  nprobes(nprobe: number): void
504
+ searchEffort(searchEffort?: number | undefined | null): void
480
505
  minimumNprobes(minimumNprobe: number): void
481
506
  maximumNprobes(maximumNprobes: number): void
482
507
  distanceRange(lowerBound?: number | undefined | null, upperBound?: number | undefined | null): void
@@ -490,11 +515,11 @@ export declare class VectorQuery {
490
515
  offset(offset: number): void
491
516
  fastSearch(): void
492
517
  withRowId(): void
493
- useLsm(enable: boolean): void
518
+ useRTE(enable: boolean): void
494
519
  rerank(rerankHybrid: (arg: RerankHybridCallbackArgs) => Promise<Buffer>): void
495
520
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
496
521
  outputSchema(): Promise<Buffer>
497
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
522
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
498
523
  explainPlan(verbose: boolean): Promise<string>
499
524
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
500
525
  }
@@ -527,45 +552,6 @@ export interface BlobRange {
527
552
  end: bigint
528
553
  }
529
554
 
530
- /**
531
- * Live state of one bucket. A table is N buckets on one node; flattening to a
532
- * single number hides the one hot bucket that is usually why someone opened
533
- * this endpoint.
534
- */
535
- export interface BucketStats {
536
- /** The shard this bucket writes. */
537
- shardId: string
538
- /** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
539
- status: string
540
- /** Epoch of the writer that currently owns the shard. */
541
- writerEpoch: number
542
- /** Version of the shard manifest these numbers were read from. */
543
- manifestVersion: number
544
- /** The generation the active memtable will become. */
545
- currentGeneration: number
546
- /** WAL position replay resumes from. */
547
- replayAfterWalEntryPosition: number
548
- /**
549
- * Highest WAL position the writer has seen. The difference against
550
- * `replayAfterWalEntryPosition` is the WAL lag.
551
- */
552
- walEntryPositionLastSeen: number
553
- /** Flushed L0 generations not yet merged into the base table. */
554
- generations: Array<GenerationStats>
555
- /**
556
- * Whether a pass owns this bucket's compaction latch right now. Says *a*
557
- * driver is running, not *whose*, and the latch is held from dispatch —
558
- * including while the pass queues for a pod-wide compactor permit. Read it
559
- * as "do not pile on", never as "mine is progressing".
560
- */
561
- compacting: boolean
562
- /**
563
- * Oldest first, active last. Absent for a `"Sealed"` bucket, whose
564
- * in-memory state is torn down.
565
- */
566
- memtables?: Array<MemtableStats>
567
- }
568
-
569
555
  export interface CatalogOptions {
570
556
  apiKey?: string
571
557
  clientConfig?: ClientConfig
@@ -579,6 +565,12 @@ export interface ClientConfig {
579
565
  userAgent?: string
580
566
  retryConfig?: RetryConfig
581
567
  timeoutConfig?: TimeoutConfig
568
+ /**
569
+ * Maximum number of concurrent blob HTTP requests across all tables and
570
+ * blob handles on this connection. Defaults to 8. Can also be set with
571
+ * `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
572
+ */
573
+ blobRequestConcurrency?: number
582
574
  extraHeaders?: Record<string, string>
583
575
  /**
584
576
  * The delimiter joining a namespace path and a name into one object
@@ -861,19 +853,6 @@ export interface FunctionErrorsOptions {
861
853
  limit?: number
862
854
  }
863
855
 
864
- /** One flushed L0 generation. */
865
- export interface GenerationStats {
866
- /** The generation number. Increases as memtables are sealed into L0. */
867
- generation: number
868
- /** On-disk size of the generation. */
869
- bytes: number
870
- /**
871
- * Present only when `includeGenerationRows` was requested. Off by default
872
- * because each count opens an uncached Lance dataset.
873
- */
874
- rows?: number
875
- }
876
-
877
856
  /** A description of an index currently configured on a column */
878
857
  export interface IndexConfig {
879
858
  /** The name of the index */
@@ -987,11 +966,6 @@ export interface JobInfo {
987
966
  /** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
988
967
  export declare function lancedbMetricsCatalog(): Array<MetricDescription>
989
968
 
990
- export interface ListDatabasesResponse {
991
- databases: Array<string>
992
- pageToken?: string
993
- }
994
-
995
969
  export interface ListNamespacesResponse {
996
970
  namespaces: Array<string>
997
971
  pageToken?: string
@@ -1002,44 +976,9 @@ export interface ListTablesResponse {
1002
976
  pageToken?: string
1003
977
  }
1004
978
 
1005
- /**
1006
- * Live per-bucket LSM state, as returned by `Table#getLsmStats`.
1007
- *
1008
- * Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
1009
- * the caller's to compute.
1010
- */
1011
- export interface LsmStats {
1012
- /** One entry per bucket backing this table. */
1013
- buckets: Array<BucketStats>
1014
- }
1015
-
1016
- /**
1017
- * Specification selecting Lance's MemWAL LSM-style write path for
1018
- * `mergeInsert`.
1019
- *
1020
- * `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
1021
- * `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
1022
- * `column` is required.
1023
- */
1024
- export interface LsmWriteSpec {
1025
- /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
1026
- specType: string
1027
- /** Bucket and identity variants: the sharding column. */
1028
- column?: string
1029
- /** Bucket variant: the number of buckets, in `[1, 1024]`. */
1030
- numBuckets?: number
1031
- /**
1032
- * Indexes the MemWAL keeps up to date. Omitted resolves every
1033
- * maintainable index on install; an empty array means none.
1034
- */
1035
- maintainedIndexes?: Array<string>
1036
- /** Default `ShardWriter` configuration recorded in the MemWAL index. */
1037
- writerConfigDefaults?: Record<string, string>
1038
- }
1039
-
1040
979
  /** One in-memory memtable. */
1041
980
  export interface MemtableStats {
1042
- /** The generation this memtable will become once sealed. */
981
+ /** The generation this memtable will become once frozen. */
1043
982
  generation: number
1044
983
  /** Rows currently buffered. */
1045
984
  rows: number
@@ -1175,7 +1114,7 @@ export interface RefreshColumnResult {
1175
1114
  }
1176
1115
 
1177
1116
  export interface RefreshMaterializedViewResult {
1178
- /** How the view was brought up to date: "rebuild", "incremental" or "no_op". */
1117
+ /** How the view was brought up to date: "rebuild". */
1179
1118
  mode: string
1180
1119
  rowsWritten: number
1181
1120
  sourceVersion: number
@@ -1254,6 +1193,41 @@ export interface RetryConfig {
1254
1193
  statuses?: Array<number>
1255
1194
  }
1256
1195
 
1196
+ /**
1197
+ * Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
1198
+ *
1199
+ * Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
1200
+ * the caller's to compute.
1201
+ */
1202
+ export interface RTEStats {
1203
+ /** One entry per table shard backing this table. */
1204
+ tableShards: Array<TableShardStats>
1205
+ }
1206
+
1207
+ /**
1208
+ * Specification enabling the realtime engine (RTE), which routes
1209
+ * `mergeInsert` through Lance's MemWAL LSM-style write path.
1210
+ *
1211
+ * `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
1212
+ * `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
1213
+ * `column` is required.
1214
+ */
1215
+ export interface RTEWriteSpec {
1216
+ /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
1217
+ specType: string
1218
+ /** Bucket and identity variants: the sharding column. */
1219
+ column?: string
1220
+ /** Bucket variant: the number of buckets, in `[1, 1024]`. */
1221
+ numBuckets?: number
1222
+ /**
1223
+ * Indexes the RTE keeps up to date. Omitted resolves every
1224
+ * maintainable index on install; an empty array means none.
1225
+ */
1226
+ maintainedIndexes?: Array<string>
1227
+ /** Default `ShardWriter` configuration recorded in the MemWAL index. */
1228
+ writerConfigDefaults?: Record<string, string>
1229
+ }
1230
+
1257
1231
  /** Result of `OAuthSession.logout()`. */
1258
1232
  export interface SessionLogout {
1259
1233
  /**
@@ -1328,6 +1302,58 @@ export interface SplitSequentialOptions {
1328
1302
  splitNames?: Array<string>
1329
1303
  }
1330
1304
 
1305
+ /** One SSTable. */
1306
+ export interface SsTableStats {
1307
+ /** The generation number. Increases as memtables are frozen into SSTables. */
1308
+ generation: number
1309
+ /** On-disk size of the SSTable. */
1310
+ bytes: number
1311
+ /**
1312
+ * Present only when `includeSstableRows` was requested. Off by default
1313
+ * because each count opens an uncached Lance dataset.
1314
+ */
1315
+ rows?: number
1316
+ }
1317
+
1318
+ /**
1319
+ * Live state of one table shard. A table is N table shards on one node; flattening to a
1320
+ * single number hides the one hot table shard that is usually why someone opened
1321
+ * this endpoint.
1322
+ */
1323
+ export interface TableShardStats {
1324
+ /** The id of this table shard. */
1325
+ shardId: string
1326
+ /** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
1327
+ status: string
1328
+ /** Epoch of the writer that currently owns the shard. */
1329
+ writerEpoch: number
1330
+ /** Version of the shard manifest these numbers were read from. */
1331
+ manifestVersion: number
1332
+ /** The generation the active memtable will become. */
1333
+ currentGeneration: number
1334
+ /** WAL position replay resumes from. */
1335
+ replayAfterWalEntryPosition: number
1336
+ /**
1337
+ * Highest WAL position the writer has seen. The difference against
1338
+ * `replayAfterWalEntryPosition` is the WAL lag.
1339
+ */
1340
+ walEntryPositionLastSeen: number
1341
+ /** SSTables not yet merged into the base table. */
1342
+ sstables: Array<SsTableStats>
1343
+ /**
1344
+ * Whether a pass owns this table shard's compaction latch right now. Says *a*
1345
+ * driver is running, not *whose*, and the latch is held from dispatch —
1346
+ * including while the pass queues for a pod-wide compactor permit. Read it
1347
+ * as "do not pile on", never as "mine is progressing".
1348
+ */
1349
+ compacting: boolean
1350
+ /**
1351
+ * Oldest first, active last. Absent for a `"Sealed"` table shard, whose
1352
+ * in-memory state is torn down.
1353
+ */
1354
+ memtables?: Array<MemtableStats>
1355
+ }
1356
+
1331
1357
  export interface TableStatistics {
1332
1358
  /**
1333
1359
  * The total size, in bytes, of the table's data files, index files, and
@@ -1338,6 +1364,17 @@ export interface TableStatistics {
1338
1364
  totalBytes: number
1339
1365
  /** The number of rows in the table */
1340
1366
  numRows: number
1367
+ /**
1368
+ * The number of rows marked as deleted across all fragments of the table
1369
+ *
1370
+ * These rows are not included in `numRows`, but still occupy space on disk
1371
+ * until the table is compacted, so a large value here indicates that the
1372
+ * table should be optimized. Fragments in which every row was deleted are
1373
+ * dropped outright, so their rows are not counted here.
1374
+ *
1375
+ * Absent (`undefined`) when the backend does not report deletion counts.
1376
+ */
1377
+ numDeletedRows?: number
1341
1378
  /** The number of indices in the table */
1342
1379
  numIndices: number
1343
1380
  /** Statistics on table fragments */