@lancedb/lancedb 0.41.0-beta.6 → 0.41.0-beta.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/blob.d.ts CHANGED
@@ -64,6 +64,8 @@ export type BlobOptions = {
64
64
  *
65
65
  * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
66
66
  * or {@link Table.fetchBlobFiles} to read bytes.
67
+ * Blobs inside lists can be written and queried as descriptors, but cannot
68
+ * currently be fetched because a row ID does not identify a list element.
67
69
  *
68
70
  * @example
69
71
  * ```ts
package/dist/blob.js CHANGED
@@ -21,6 +21,8 @@ const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-thresho
21
21
  *
22
22
  * Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
23
23
  * or {@link Table.fetchBlobFiles} to read bytes.
24
+ * Blobs inside lists can be written and queried as descriptors, but cannot
25
+ * currently be fetched because a row ID does not identify a list element.
24
26
  *
25
27
  * @example
26
28
  * ```ts
package/dist/index.d.ts CHANGED
@@ -14,9 +14,9 @@ export type { BlobData, BlobInput, BlobOptions, BlobRange, BlobReadOptions, Blob
14
14
  export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
15
15
  export { JobFailureInfo, JobInfo, Session } from "./native.js";
16
16
  export { Job, JobEventsOptions } from "./job";
17
- export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
17
+ export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, BlobMode, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
18
18
  export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
19
- export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, LsmWriteSpec, LsmStats, BucketStats, GenerationStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
19
+ export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, MemWalWriteSpec, MemWalStats, TableShardStats, SsTableStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
20
20
  export { HeaderProvider, StaticHeaderProvider, OAuthHeaderProvider, TokenResponse, } from "./header";
21
21
  export { ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions, } from "./oauth";
22
22
  export { MergeInsertBuilder, WriteExecutionOptions } from "./merge";
package/dist/merge.d.ts CHANGED
@@ -59,23 +59,23 @@ export declare class MergeInsertBuilder {
59
59
  /**
60
60
  * Control MemWAL routing for this merge.
61
61
  *
62
- * By default (unset), a `mergeInsert` on a table with an LSM write spec is
62
+ * By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
63
63
  * routed through Lance's MemWAL shard writer, and a table without one uses the
64
64
  * standard path.
65
65
  *
66
66
  * @param enable - `true` forces MemWAL routing and errors if the table has no
67
- * LSM write spec. `false` forces the standard write path even when a spec is set.
67
+ * MemWAL write spec. `false` forces the standard write path even when a spec is set.
68
68
  */
69
- useLsm(enable: boolean): MergeInsertBuilder;
69
+ useMemWal(enable: boolean): MergeInsertBuilder;
70
70
  /**
71
71
  * Controls how an LSM merge checks that its input targets a single shard.
72
72
  *
73
- * When a table has an LSM write spec, every row in a `mergeInsert` call must
73
+ * When a table has a MemWAL write spec, every row in a `mergeInsert` call must
74
74
  * route to the same shard. When `true` (the default), every row is inspected
75
75
  * to verify this. When `false`, only the first row is inspected and the
76
76
  * shard it routes to is used for the whole input — a faster path for callers
77
77
  * that have already pre-sharded their input. Has no effect on tables without
78
- * an LSM write spec.
78
+ * a MemWAL write spec.
79
79
  *
80
80
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
81
81
  */
package/dist/merge.js CHANGED
@@ -71,25 +71,25 @@ class MergeInsertBuilder {
71
71
  /**
72
72
  * Control MemWAL routing for this merge.
73
73
  *
74
- * By default (unset), a `mergeInsert` on a table with an LSM write spec is
74
+ * By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
75
75
  * routed through Lance's MemWAL shard writer, and a table without one uses the
76
76
  * standard path.
77
77
  *
78
78
  * @param enable - `true` forces MemWAL routing and errors if the table has no
79
- * LSM write spec. `false` forces the standard write path even when a spec is set.
79
+ * MemWAL write spec. `false` forces the standard write path even when a spec is set.
80
80
  */
81
- useLsm(enable) {
82
- return new MergeInsertBuilder(this.#native.useLsm(enable), this.#schema);
81
+ useMemWal(enable) {
82
+ return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema);
83
83
  }
84
84
  /**
85
85
  * Controls how an LSM merge checks that its input targets a single shard.
86
86
  *
87
- * When a table has an LSM write spec, every row in a `mergeInsert` call must
87
+ * When a table has a MemWAL write spec, every row in a `mergeInsert` call must
88
88
  * route to the same shard. When `true` (the default), every row is inspected
89
89
  * to verify this. When `false`, only the first row is inspected and the
90
90
  * shard it routes to is used for the whole input — a faster path for callers
91
91
  * that have already pre-sharded their input. Has no effect on tables without
92
- * an LSM write spec.
92
+ * a MemWAL write spec.
93
93
  *
94
94
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
95
95
  */
package/dist/native.d.ts CHANGED
@@ -265,7 +265,7 @@ export declare class NativeMergeInsertBuilder {
265
265
  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
266
266
  setTimeout(timeout: number): void
267
267
  useIndex(useIndex: boolean): NativeMergeInsertBuilder
268
- useLsm(enable: boolean): NativeMergeInsertBuilder
268
+ useMemWal(enable: boolean): NativeMergeInsertBuilder
269
269
  validateSingleShard(validateSingleShard: boolean): NativeMergeInsertBuilder
270
270
  execute(buf: Buffer): Promise<MergeResult>
271
271
  }
@@ -343,10 +343,10 @@ export declare class Query {
343
343
  nearestToRaw(data: Uint8Array, dtype: string): VectorQuery
344
344
  fastSearch(): void
345
345
  withRowId(): void
346
- useLsm(enable: boolean): void
346
+ useMemWal(enable: boolean): void
347
347
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
348
348
  outputSchema(): Promise<Buffer>
349
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
349
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
350
350
  explainPlan(verbose: boolean): Promise<string>
351
351
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
352
352
  }
@@ -441,14 +441,14 @@ export declare class Table {
441
441
  updateFieldMetadata(updates: Array<FieldMetadataUpdate>): Promise<UpdateFieldMetadataResult>
442
442
  dropColumns(columns: Array<string>): Promise<DropColumnsResult>
443
443
  setUnenforcedPrimaryKey(columns: Array<string>): Promise<void>
444
- setLsmWriteSpec(spec: LsmWriteSpec): Promise<void>
445
- unsetLsmWriteSpec(): Promise<void>
446
- getLsmWriteSpec(): Promise<LsmWriteSpec | null>
447
- closeLsmWriters(): Promise<void>
448
- flushLsm(): Promise<void>
449
- compactLsm(): Promise<void>
450
- checkpointLsm(): Promise<void>
451
- getLsmStats(includeGenerationRows: boolean): Promise<LsmStats | null>
444
+ enableMemWal(spec: MemWalWriteSpec): Promise<void>
445
+ disableMemWal(): Promise<void>
446
+ getMemWalSpec(): Promise<MemWalWriteSpec | null>
447
+ closeMemWalWriters(): Promise<void>
448
+ flushMemWal(): Promise<void>
449
+ compactMemWal(): Promise<void>
450
+ checkpointMemWal(): Promise<void>
451
+ getMemWalStats(includeSstableRows: boolean): Promise<MemWalStats | null>
452
452
  version(): Promise<number>
453
453
  checkoutCurrent(): Promise<Table>
454
454
  checkout(version: number): Promise<void>
@@ -486,9 +486,9 @@ export declare class TakeQuery {
486
486
  select(columns: Array<[string, string]>): void
487
487
  selectColumns(columns: Array<string>): void
488
488
  withRowId(): void
489
- useLsm(enable: boolean): void
489
+ useMemWal(enable: boolean): void
490
490
  outputSchema(): Promise<Buffer>
491
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
491
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
492
492
  explainPlan(verbose: boolean): Promise<string>
493
493
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
494
494
  }
@@ -515,11 +515,11 @@ export declare class VectorQuery {
515
515
  offset(offset: number): void
516
516
  fastSearch(): void
517
517
  withRowId(): void
518
- useLsm(enable: boolean): void
518
+ useMemWal(enable: boolean): void
519
519
  rerank(rerankHybrid: (arg: RerankHybridCallbackArgs) => Promise<Buffer>): void
520
520
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
521
521
  outputSchema(): Promise<Buffer>
522
- execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null): Promise<RecordBatchIterator>
522
+ execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
523
523
  explainPlan(verbose: boolean): Promise<string>
524
524
  analyzePlan(distributedMetrics?: string | undefined | null): Promise<string>
525
525
  }
@@ -552,45 +552,6 @@ export interface BlobRange {
552
552
  end: bigint
553
553
  }
554
554
 
555
- /**
556
- * Live state of one bucket. A table is N buckets on one node; flattening to a
557
- * single number hides the one hot bucket that is usually why someone opened
558
- * this endpoint.
559
- */
560
- export interface BucketStats {
561
- /** The shard this bucket writes. */
562
- shardId: string
563
- /** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
564
- status: string
565
- /** Epoch of the writer that currently owns the shard. */
566
- writerEpoch: number
567
- /** Version of the shard manifest these numbers were read from. */
568
- manifestVersion: number
569
- /** The generation the active memtable will become. */
570
- currentGeneration: number
571
- /** WAL position replay resumes from. */
572
- replayAfterWalEntryPosition: number
573
- /**
574
- * Highest WAL position the writer has seen. The difference against
575
- * `replayAfterWalEntryPosition` is the WAL lag.
576
- */
577
- walEntryPositionLastSeen: number
578
- /** Flushed L0 generations not yet merged into the base table. */
579
- generations: Array<GenerationStats>
580
- /**
581
- * Whether a pass owns this bucket's compaction latch right now. Says *a*
582
- * driver is running, not *whose*, and the latch is held from dispatch —
583
- * including while the pass queues for a pod-wide compactor permit. Read it
584
- * as "do not pile on", never as "mine is progressing".
585
- */
586
- compacting: boolean
587
- /**
588
- * Oldest first, active last. Absent for a `"Sealed"` bucket, whose
589
- * in-memory state is torn down.
590
- */
591
- memtables?: Array<MemtableStats>
592
- }
593
-
594
555
  export interface CatalogOptions {
595
556
  apiKey?: string
596
557
  clientConfig?: ClientConfig
@@ -604,6 +565,12 @@ export interface ClientConfig {
604
565
  userAgent?: string
605
566
  retryConfig?: RetryConfig
606
567
  timeoutConfig?: TimeoutConfig
568
+ /**
569
+ * Maximum number of concurrent blob HTTP requests across all tables and
570
+ * blob handles on this connection. Defaults to 8. Can also be set with
571
+ * `LANCE_CLIENT_BLOB_REQUEST_CONCURRENCY`. Must be greater than zero.
572
+ */
573
+ blobRequestConcurrency?: number
607
574
  extraHeaders?: Record<string, string>
608
575
  /**
609
576
  * The delimiter joining a namespace path and a name into one object
@@ -886,19 +853,6 @@ export interface FunctionErrorsOptions {
886
853
  limit?: number
887
854
  }
888
855
 
889
- /** One flushed L0 generation. */
890
- export interface GenerationStats {
891
- /** The generation number. Increases as memtables are sealed into L0. */
892
- generation: number
893
- /** On-disk size of the generation. */
894
- bytes: number
895
- /**
896
- * Present only when `includeGenerationRows` was requested. Off by default
897
- * because each count opens an uncached Lance dataset.
898
- */
899
- rows?: number
900
- }
901
-
902
856
  /** A description of an index currently configured on a column */
903
857
  export interface IndexConfig {
904
858
  /** The name of the index */
@@ -1022,15 +976,32 @@ export interface ListTablesResponse {
1022
976
  pageToken?: string
1023
977
  }
1024
978
 
979
+ /** One in-memory memtable. */
980
+ export interface MemtableStats {
981
+ /** The generation this memtable will become once frozen. */
982
+ generation: number
983
+ /** Rows currently buffered. */
984
+ rows: number
985
+ /** Estimated in-memory size. */
986
+ bytes: number
987
+ /** Record batches currently buffered. */
988
+ batches: number
989
+ /**
990
+ * Names of the indexes this memtable carries. An absent name is the whole
991
+ * answer to "why is my fresh-tier search on that column brute-force".
992
+ */
993
+ indexes: Array<string>
994
+ }
995
+
1025
996
  /**
1026
- * Live per-bucket LSM state, as returned by `Table#getLsmStats`.
997
+ * Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
1027
998
  *
1028
- * Nothing here is derived: sums and differences (total L0 bytes, WAL lag) are
999
+ * Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
1029
1000
  * the caller's to compute.
1030
1001
  */
1031
- export interface LsmStats {
1032
- /** One entry per bucket backing this table. */
1033
- buckets: Array<BucketStats>
1002
+ export interface MemWalStats {
1003
+ /** One entry per table shard backing this table. */
1004
+ tableShards: Array<TableShardStats>
1034
1005
  }
1035
1006
 
1036
1007
  /**
@@ -1041,7 +1012,7 @@ export interface LsmStats {
1041
1012
  * `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
1042
1013
  * `column` is required.
1043
1014
  */
1044
- export interface LsmWriteSpec {
1015
+ export interface MemWalWriteSpec {
1045
1016
  /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
1046
1017
  specType: string
1047
1018
  /** Bucket and identity variants: the sharding column. */
@@ -1057,23 +1028,6 @@ export interface LsmWriteSpec {
1057
1028
  writerConfigDefaults?: Record<string, string>
1058
1029
  }
1059
1030
 
1060
- /** One in-memory memtable. */
1061
- export interface MemtableStats {
1062
- /** The generation this memtable will become once sealed. */
1063
- generation: number
1064
- /** Rows currently buffered. */
1065
- rows: number
1066
- /** Estimated in-memory size. */
1067
- bytes: number
1068
- /** Record batches currently buffered. */
1069
- batches: number
1070
- /**
1071
- * Names of the indexes this memtable carries. An absent name is the whole
1072
- * answer to "why is my fresh-tier search on that column brute-force".
1073
- */
1074
- indexes: Array<string>
1075
- }
1076
-
1077
1031
  export interface MergeResult {
1078
1032
  version: number
1079
1033
  numInsertedRows: number
@@ -1348,6 +1302,58 @@ export interface SplitSequentialOptions {
1348
1302
  splitNames?: Array<string>
1349
1303
  }
1350
1304
 
1305
+ /** One SSTable. */
1306
+ export interface SsTableStats {
1307
+ /** The generation number. Increases as memtables are frozen into SSTables. */
1308
+ generation: number
1309
+ /** On-disk size of the SSTable. */
1310
+ bytes: number
1311
+ /**
1312
+ * Present only when `includeSstableRows` was requested. Off by default
1313
+ * because each count opens an uncached Lance dataset.
1314
+ */
1315
+ rows?: number
1316
+ }
1317
+
1318
+ /**
1319
+ * Live state of one table shard. A table is N table shards on one node; flattening to a
1320
+ * single number hides the one hot table shard that is usually why someone opened
1321
+ * this endpoint.
1322
+ */
1323
+ export interface TableShardStats {
1324
+ /** The id of this table shard. */
1325
+ shardId: string
1326
+ /** `"Active"` or `"Sealed"` (drop-table 2PC in flight). */
1327
+ status: string
1328
+ /** Epoch of the writer that currently owns the shard. */
1329
+ writerEpoch: number
1330
+ /** Version of the shard manifest these numbers were read from. */
1331
+ manifestVersion: number
1332
+ /** The generation the active memtable will become. */
1333
+ currentGeneration: number
1334
+ /** WAL position replay resumes from. */
1335
+ replayAfterWalEntryPosition: number
1336
+ /**
1337
+ * Highest WAL position the writer has seen. The difference against
1338
+ * `replayAfterWalEntryPosition` is the WAL lag.
1339
+ */
1340
+ walEntryPositionLastSeen: number
1341
+ /** SSTables not yet merged into the base table. */
1342
+ sstables: Array<SsTableStats>
1343
+ /**
1344
+ * Whether a pass owns this table shard's compaction latch right now. Says *a*
1345
+ * driver is running, not *whose*, and the latch is held from dispatch —
1346
+ * including while the pass queues for a pod-wide compactor permit. Read it
1347
+ * as "do not pile on", never as "mine is progressing".
1348
+ */
1349
+ compacting: boolean
1350
+ /**
1351
+ * Oldest first, active last. Absent for a `"Sealed"` table shard, whose
1352
+ * in-memory state is torn down.
1353
+ */
1354
+ memtables?: Array<MemtableStats>
1355
+ }
1356
+
1351
1357
  export interface TableStatistics {
1352
1358
  /**
1353
1359
  * The total size, in bytes, of the table's data files, index files, and