@lancedb/lancedb 0.41.0-beta.0 → 0.41.0-beta.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/native.d.ts CHANGED
@@ -58,7 +58,7 @@ export declare class Connection {
58
58
  createEmptyTable(name: string, schemaBuf: Buffer, mode: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null): Promise<Table>
59
59
  createMaterializedView(name: string, source: string, projections: Array<Array<string>> | undefined | null, filter: string | undefined | null, limit: number | undefined | null, withNoData: boolean): Promise<Table>
60
60
  openMaterializedView(name: string): Promise<Table>
61
- listMaterializedViews(): Promise<Array<string>>
61
+ listMaterializedViews(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
62
62
  /** Drop a materialized view. */
63
63
  dropMaterializedView(name: string, namespacePath?: Array<string> | undefined | null): Promise<void>
64
64
  /** Create a view: a named query planned on every read. */
@@ -73,7 +73,7 @@ export declare class Connection {
73
73
  /** Start dropping a view and return the job deleting its definition. */
74
74
  dropViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
75
75
  /** The names of the views in one namespace. */
76
- listViews(namespacePath?: Array<string> | undefined | null): Promise<Array<string>>
76
+ listViews(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, pageLimit?: number | undefined | null): NameListing
77
77
  /** Start dropping a materialized view and return its cleanup job. */
78
78
  dropMaterializedViewAsync(name: string, namespacePath?: Array<string> | undefined | null): Promise<Job>
79
79
  openTable(name: string, namespacePath?: Array<string> | undefined | null, storageOptions?: Record<string, string> | undefined | null, indexCacheSize?: number | undefined | null): Promise<Table>
@@ -93,7 +93,7 @@ export declare class Connection {
93
93
  */
94
94
  openJob(jobId: string): Promise<Job>
95
95
  /** List server-side jobs across the database's tables. */
96
- listJobs(): Promise<Array<JobInfo>>
96
+ listJobs(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): JobListing
97
97
  /**
98
98
  * Request cancellation of a server-side job by id. Returns true if the
99
99
  * server accepted the cancellation, false if no such job exists.
@@ -203,6 +203,12 @@ export declare class Job {
203
203
  events(limit?: number | undefined | null, filter?: string | undefined | null): Promise<Buffer>
204
204
  }
205
205
 
206
+ export declare class JobListing {
207
+ numPageResults(): number
208
+ pageToken(): string | null
209
+ next(): Promise<JobInfo | null>
210
+ }
211
+
206
212
  export declare class JsFullTextQuery {
207
213
  static matchQuery(query: string, column: string, boost: number, fuzziness: number | undefined | null, maxExpansions: number, operator: string, prefixLength: number): JsFullTextQuery
208
214
  static phraseQuery(query: string, column: string, slop: number): JsFullTextQuery
@@ -222,6 +228,12 @@ export declare class JsHeaderProvider {
222
228
  constructor(getHeadersCallback: () => Promise<Record<string, string>>)
223
229
  }
224
230
 
231
+ export declare class NameListing {
232
+ numPageResults(): number
233
+ pageToken(): string | null
234
+ next(): Promise<string | null>
235
+ }
236
+
225
237
  /**
226
238
  * A Rust-side view of a JS-constructed `Scannable`.
227
239
  *
@@ -253,7 +265,7 @@ export declare class NativeMergeInsertBuilder {
253
265
  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
254
266
  setTimeout(timeout: number): void
255
267
  useIndex(useIndex: boolean): NativeMergeInsertBuilder
256
- useLsm(enable: boolean): NativeMergeInsertBuilder
268
+ useRTE(enable: boolean): NativeMergeInsertBuilder
257
269
  validateSingleShard(validateSingleShard: boolean): NativeMergeInsertBuilder
258
270
  execute(buf: Buffer): Promise<MergeResult>
259
271
  }
@@ -331,10 +343,10 @@ export declare class Query {
331
343
  nearestToRaw(data: Uint8Array, dtype: string): VectorQuery
332
344
  fastSearch(): void
333
345
  withRowId(): void
334
- useLsm(enable: boolean): void
346
+ useRTE(enable: boolean): void
335
347
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
336
348
  outputSchema(): Promise<Buffer>
337
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
349
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
338
350
  explainPlan(verbose: boolean): Promise<string>
339
351
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
340
352
  }
@@ -429,14 +441,14 @@ export declare class Table {
429
441
  updateFieldMetadata(updates: Array<FieldMetadataUpdate>): Promise<UpdateFieldMetadataResult>
430
442
  dropColumns(columns: Array<string>): Promise<DropColumnsResult>
431
443
  setUnenforcedPrimaryKey(columns: Array<string>): Promise<void>
432
- setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>
433
- unsetLsmWriteSpec(): Promise<void>
434
- getLsmWriteSpec(): Promise<LsmWriteSpec | null>
435
- closeLsmWriters(): Promise<void>
436
- flushLsm(): Promise<void>
437
- compactLsm(): Promise<void>
438
- checkpointLsm(): Promise<void>
439
- getLsmStats(includeGenerationRows: boolean): Promise<LsmStats | null>
444
+ enableRTE(spec: RTEWriteSpec): Promise<void>
445
+ disableRTE(): Promise<void>
446
+ getRTESpec(): Promise<RTEWriteSpec | null>
447
+ closeRTEWriters(): Promise<void>
448
+ flushRTE(): Promise<void>
449
+ compactRTE(): Promise<void>
450
+ checkpointRTE(): Promise<void>
451
+ getRTEStats(includeSstableRows: boolean): Promise<RTEStats | null>
440
452
  version(): Promise<number>
441
453
  checkoutCurrent(): Promise<Table>
442
454
  checkout(version: number): Promise<void>
@@ -474,9 +486,9 @@ export declare class TakeQuery {
474
486
  select(columns: Array<[string, string]>): void
475
487
  selectColumns(columns: Array<string>): void
476
488
  withRowId(): void
477
- useLsm(enable: boolean): void
489
+ useRTE(enable: boolean): void
478
490
  outputSchema(): Promise<Buffer>
479
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
491
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
480
492
  explainPlan(verbose: boolean): Promise<string>
481
493
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
482
494
  }
@@ -489,6 +501,7 @@ export declare class VectorQuery {
489
501
  postfilter(): void
490
502
  refineFactor(refineFactor: number): void
491
503
  nprobes(nprobe: number): void
504
+ searchEffort(searchEffort?: number | undefined | null): void
492
505
  minimumNprobes(minimumNprobe: number): void
493
506
  maximumNprobes(maximumNprobes: number): void
494
507
  distanceRange(lowerBound?: number | undefined | null, upperBound?: number | undefined | null): void
@@ -502,11 +515,11 @@ export declare class VectorQuery {
502
515
  offset(offset: number): void
503
516
  fastSearch(): void
504
517
  withRowId(): void
505
- useLsm(enable: boolean): void
518
+ useRTE(enable: boolean): void
506
519
  rerank(rerankHybrid: (arg: RerankHybridCallbackArgs) => Promise<Buffer>): void
507
520
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
508
521
  outputSchema(): Promise<Buffer>
509
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
522
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
510
523
  explainPlan(verbose: boolean): Promise<string>
511
524
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
512
525
  }
@@ -539,45 +552,6 @@ export interface BlobRange {
539
552
  end: bigint
540
553
  }
541
554
 
542
- /**
543
- * Live state of one bucket. A table is N buckets on one node; flattening to a
544
- * single number hides the one hot bucket that is usually why someone opened
545
- * this endpoint.
546
- */
547
- export interface BucketStats {
548
- /** The shard this bucket writes. */
549
- shardId: string
550
- /** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
551
- status: string
552
- /** Epoch of the writer that currently owns the shard. */
553
- writerEpoch: number
554
- /** Version of the shard manifest these numbers were read from. */
555
- manifestVersion: number
556
- /** The generation the active memtable will become. */
557
- currentGeneration: number
558
- /** WAL position replay resumes from. */
559
- replayAfterWalEntryPosition: number
560
- /**
561
- * Highest WAL position the writer has seen. The difference against
562
- * `replayAfterWalEntryPosition` is the WAL lag.
563
- */
564
- walEntryPositionLastSeen: number
565
- /** Flushed L0 generations not yet merged into the base table. */
566
- generations: Array<GenerationStats>
567
- /**
568
- * Whether a pass owns this bucket's compaction latch right now. Says *a*
569
- * driver is running, not *whose*, and the latch is held from dispatch —
570
- * including while the pass queues for a pod-wide compactor permit. Read it
571
- * as "do not pile on", never as "mine is progressing".
572
- */
573
- compacting: boolean
574
- /**
575
- * Oldest first, active last. Absent for a `"Sealed"` bucket, whose
576
- * in-memory state is torn down.
577
- */
578
- memtables?: Array<MemtableStats>
579
- }
580
-
581
555
  export interface CatalogOptions {
582
556
  apiKey?: string
583
557
  clientConfig?: ClientConfig
@@ -591,6 +565,12 @@ export interface ClientConfig {
591
565
  userAgent?: string
592
566
  retryConfig?: RetryConfig
593
567
  timeoutConfig?: TimeoutConfig
568
+ /**
569
+ * Maximum number of concurrent blob HTTP requests across all tables and
570
+ * blob handles on this connection. Defaults to 8. Can also be set with
571
+ * `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
572
+ */
573
+ blobRequestConcurrency?: number
594
574
  extraHeaders?: Record<string, string>
595
575
  /**
596
576
  * The delimiter joining a namespace path and a name into one object
@@ -873,19 +853,6 @@ export interface FunctionErrorsOptions {
873
853
  limit?: number
874
854
  }
875
855
 
876
- /** One flushed L0 generation. */
877
- export interface GenerationStats {
878
- /** The generation number. Increases as memtables are sealed into L0. */
879
- generation: number
880
- /** On-disk size of the generation. */
881
- bytes: number
882
- /**
883
- * Present only when `includeGenerationRows` was requested. Off by default
884
- * because each count opens an uncached Lance dataset.
885
- */
886
- rows?: number
887
- }
888
-
889
856
  /** A description of an index currently configured on a column */
890
857
  export interface IndexConfig {
891
858
  /** The name of the index */
@@ -1009,44 +976,9 @@ export interface ListTablesResponse {
1009
976
  pageToken?: string
1010
977
  }
1011
978
 
1012
- /**
1013
- * Live per-bucket LSM state, as returned by `Table#getLsmStats`.
1014
- *
1015
- * Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
1016
- * the caller's to compute.
1017
- */
1018
- export interface LsmStats {
1019
- /** One entry per bucket backing this table. */
1020
- buckets: Array<BucketStats>
1021
- }
1022
-
1023
- /**
1024
- * Specification selecting Lance's MemWAL LSM-style write path for
1025
- * `mergeInsert`.
1026
- *
1027
- * `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
1028
- * `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
1029
- * `column` is required.
1030
- */
1031
- export interface LsmWriteSpec {
1032
- /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
1033
- specType: string
1034
- /** Bucket and identity variants: the sharding column. */
1035
- column?: string
1036
- /** Bucket variant: the number of buckets, in `[1, 1024]`. */
1037
- numBuckets?: number
1038
- /**
1039
- * Indexes the MemWAL keeps up to date. Omitted resolves every
1040
- * maintainable index on install; an empty array means none.
1041
- */
1042
- maintainedIndexes?: Array<string>
1043
- /** Default `ShardWriter` configuration recorded in the MemWAL index. */
1044
- writerConfigDefaults?: Record<string, string>
1045
- }
1046
-
1047
979
  /** One in-memory memtable. */
1048
980
  export interface MemtableStats {
1049
- /** The generation this memtable will become once sealed. */
981
+ /** The generation this memtable will become once frozen. */
1050
982
  generation: number
1051
983
  /** Rows currently buffered. */
1052
984
  rows: number
@@ -1261,6 +1193,41 @@ export interface RetryConfig {
1261
1193
  statuses?: Array<number>
1262
1194
  }
1263
1195
 
1196
+ /**
1197
+ * Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
1198
+ *
1199
+ * Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
1200
+ * the caller's to compute.
1201
+ */
1202
+ export interface RTEStats {
1203
+ /** One entry per table shard backing this table. */
1204
+ tableShards: Array<TableShardStats>
1205
+ }
1206
+
1207
+ /**
1208
+ * Specification enabling the realtime engine (RTE), which routes
1209
+ * `mergeInsert` through Lance's MemWAL LSM-style write path.
1210
+ *
1211
+ * `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
1212
+ * `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
1213
+ * `column` is required.
1214
+ */
1215
+ export interface RTEWriteSpec {
1216
+ /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
1217
+ specType: string
1218
+ /** Bucket and identity variants: the sharding column. */
1219
+ column?: string
1220
+ /** Bucket variant: the number of buckets, in `[1, 1024]`. */
1221
+ numBuckets?: number
1222
+ /**
1223
+ * Indexes the RTE keeps up to date. Omitted resolves every
1224
+ * maintainable index on install; an empty array means none.
1225
+ */
1226
+ maintainedIndexes?: Array<string>
1227
+ /** Default `ShardWriter` configuration recorded in the MemWAL index. */
1228
+ writerConfigDefaults?: Record<string, string>
1229
+ }
1230
+
1264
1231
  /** Result of `OAuthSession.logout()`. */
1265
1232
  export interface SessionLogout {
1266
1233
  /**
@@ -1335,6 +1302,58 @@ export interface SplitSequentialOptions {
1335
1302
  splitNames?: Array<string>
1336
1303
  }
1337
1304
 
1305
+ /** One SSTable. */
1306
+ export interface SsTableStats {
1307
+ /** The generation number. Increases as memtables are frozen into SSTables. */
1308
+ generation: number
1309
+ /** On-disk size of the SSTable. */
1310
+ bytes: number
1311
+ /**
1312
+ * Present only when `includeSstableRows` was requested. Off by default
1313
+ * because each count opens an uncached Lance dataset.
1314
+ */
1315
+ rows?: number
1316
+ }
1317
+
1318
+ /**
1319
+ * Live state of one table shard. A table is N table shards on one node; flattening to a
1320
+ * single number hides the one hot table shard that is usually why someone opened
1321
+ * this endpoint.
1322
+ */
1323
+ export interface TableShardStats {
1324
+ /** The id of this table shard. */
1325
+ shardId: string
1326
+ /** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
1327
+ status: string
1328
+ /** Epoch of the writer that currently owns the shard. */
1329
+ writerEpoch: number
1330
+ /** Version of the shard manifest these numbers were read from. */
1331
+ manifestVersion: number
1332
+ /** The generation the active memtable will become. */
1333
+ currentGeneration: number
1334
+ /** WAL position replay resumes from. */
1335
+ replayAfterWalEntryPosition: number
1336
+ /**
1337
+ * Highest WAL position the writer has seen. The difference against
1338
+ * `replayAfterWalEntryPosition` is the WAL lag.
1339
+ */
1340
+ walEntryPositionLastSeen: number
1341
+ /** SSTables not yet merged into the base table. */
1342
+ sstables: Array<SsTableStats>
1343
+ /**
1344
+ * Whether a pass owns this table shard's compaction latch right now. Says *a*
1345
+ * driver is running, not *whose*, and the latch is held from dispatch —
1346
+ * including while the pass queues for a pod-wide compactor permit. Read it
1347
+ * as "do not pile on", never as "mine is progressing".
1348
+ */
1349
+ compacting: boolean
1350
+ /**
1351
+ * Oldest first, active last. Absent for a `"Sealed"` table shard, whose
1352
+ * in-memory state is torn down.
1353
+ */
1354
+ memtables?: Array<MemtableStats>
1355
+ }
1356
+
1338
1357
  export interface TableStatistics {
1339
1358
  /**
1340
1359
  * The total size, in bytes, of the table's data files, index files, and
@@ -1345,6 +1364,17 @@ export interface TableStatistics {
1345
1364
  totalBytes: number
1346
1365
  /** The number of rows in the table */
1347
1366
  numRows: number
1367
+ /**
1368
+ * The number of rows marked as deleted across all fragments of the table
1369
+ *
1370
+ * These rows are not included in `numRows`, but still occupy space on disk
1371
+ * until the table is compacted, so a large value here indicates that the
1372
+ * table should be optimized. Fragments in which every row was deleted are
1373
+ * dropped outright, so their rows are not counted here.
1374
+ *
1375
+ * Absent (`undefined`) when the backend does not report deletion counts.
1376
+ */
1377
+ numDeletedRows?: number
1348
1378
  /** The number of indices in the table */
1349
1379
  numIndices: number
1350
1380
  /** Statistics on table fragments */