@lancedb/lancedb 0.41.0-beta.8 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/job.d.ts CHANGED
@@ -19,7 +19,8 @@ export interface JobEventsOptions {
19
19
  * - {@link Job.refresh} and {@link Job.status} fetch the whole record.
20
20
  * - {@link Job.wait} records the terminal state it establishes, but not the
21
21
  * rest of the record.
22
- * - Everything is null until one of those runs.
22
+ * - Registry details are null until one of those runs. Query handles report
23
+ * jobType "query" immediately.
23
24
  *
24
25
  * @hideconstructor
25
26
  */
@@ -29,15 +30,17 @@ export declare class Job {
29
30
  /**
30
31
  * Identifies the operation on the server that is running it.
31
32
  *
32
- * Operations that run in this process have no server id. The value is
33
- * opaque: parsing it or storing it to resume the job later is not supported.
33
+ * Registry handles expose a registry job ID; query handles expose a
34
+ * connection-scoped SQL query UUID string. Local handles have no ID.
35
+ * Query handles report jobType "query". Their IDs are not registry IDs or
36
+ * durable resume tokens.
34
37
  */
35
38
  get id(): string | null;
36
39
  /** The last observed lifecycle state, without contacting the backend. */
37
40
  get state(): string | null;
38
41
  /**
39
- * The job's type, as the server names it. Null for an in-process job, which
40
- * has no server-side record.
42
+ * "query" immediately for SQL-backed handles; otherwise the cached
43
+ * server-reported job type. Null for in-process handles.
41
44
  */
42
45
  get jobType(): string | null;
43
46
  /** When the job was created, in milliseconds since the epoch. */
package/dist/job.js CHANGED
@@ -16,7 +16,8 @@ const apache_arrow_1 = require("apache-arrow");
16
16
  * - {@link Job.refresh} and {@link Job.status} fetch the whole record.
17
17
  * - {@link Job.wait} records the terminal state it establishes, but not the
18
18
  * rest of the record.
19
- * - Everything is null until one of those runs.
19
+ * - Registry details are null until one of those runs. Query handles report
20
+ * jobType "query" immediately.
20
21
  *
21
22
  * @hideconstructor
22
23
  */
@@ -28,8 +29,10 @@ class Job {
28
29
  /**
29
30
  * Identifies the operation on the server that is running it.
30
31
  *
31
- * Operations that run in this process have no server id. The value is
32
- * opaque: parsing it or storing it to resume the job later is not supported.
32
+ * Registry handles expose a registry job ID; query handles expose a
33
+ * connection-scoped SQL query UUID string. Local handles have no ID.
34
+ * Query handles report jobType "query". Their IDs are not registry IDs or
35
+ * durable resume tokens.
33
36
  */
34
37
  get id() {
35
38
  return this.inner.id ?? null;
@@ -39,8 +42,8 @@ class Job {
39
42
  return this.inner.state ?? null;
40
43
  }
41
44
  /**
42
- * The job's type, as the server names it. Null for an in-process job, which
43
- * has no server-side record.
45
+ * "query" immediately for SQL-backed handles; otherwise the cached
46
+ * server-reported job type. Null for in-process handles.
44
47
  */
45
48
  get jobType() {
46
49
  return this.inner.jobType ?? null;
@@ -56,12 +56,16 @@ export declare class MaterializedView {
56
56
  definition(): Promise<MaterializedViewDefinition>;
57
57
  /**
58
58
  * Recompute the view from its source.
59
+ * Local refresh requires a source build with `experimental-functions`.
59
60
  *
60
- * Every refresh rebuilds the complete view. For local views,
61
- * `sourceVersion` refreshes to that source version instead of the latest;
62
- * remote SQL refreshes do not support source-version pinning.
61
+ * `full: true` forces a remote rebuild; otherwise the server chooses the
62
+ * refresh mode. Local views always rebuild, and `sourceVersion` selects a
63
+ * source version instead of the latest. Remote SQL refreshes do not support
64
+ * source-version pinning. Their results leave `mode`, `rowsWritten`, and
65
+ * `sourceVersion` absent because the SQL job adapter does not expose these metrics.
63
66
  */
64
67
  refresh(options?: {
68
+ full?: boolean;
65
69
  sourceVersion?: number;
66
70
  }): Promise<RefreshMaterializedViewResult>;
67
71
  }
@@ -131,14 +131,17 @@ class MaterializedView {
131
131
  }
132
132
  /**
133
133
  * Recompute the view from its source.
134
+ * Local refresh requires a source build with `experimental-functions`.
134
135
  *
135
- * Every refresh rebuilds the complete view. For local views,
136
- * `sourceVersion` refreshes to that source version instead of the latest;
137
- * remote SQL refreshes do not support source-version pinning.
136
+ * `full: true` forces a remote rebuild; otherwise the server chooses the
137
+ * refresh mode. Local views always rebuild, and `sourceVersion` selects a
138
+ * source version instead of the latest. Remote SQL refreshes do not support
139
+ * source-version pinning. Their results leave `mode`, `rowsWritten`, and
140
+ * `sourceVersion` absent because the SQL job adapter does not expose these metrics.
138
141
  */
139
142
  async refresh(options) {
140
143
  validateNonNegativeInteger(options?.sourceVersion, "sourceVersion");
141
- return await this.inner.refreshMaterializedView(options?.sourceVersion);
144
+ return await this.inner.refreshMaterializedView(options?.sourceVersion, options?.full);
142
145
  }
143
146
  }
144
147
  exports.MaterializedView = MaterializedView;
package/dist/merge.d.ts CHANGED
@@ -57,25 +57,25 @@ export declare class MergeInsertBuilder {
57
57
  */
58
58
  useIndex(useIndex: boolean): MergeInsertBuilder;
59
59
  /**
60
- * Control MemWAL routing for this merge.
60
+ * Control RTE routing for this merge.
61
61
  *
62
- * By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
62
+ * By default (unset), a `mergeInsert` on a table with an RTE write spec is
63
63
  * routed through Lance's MemWAL shard writer, and a table without one uses the
64
64
  * standard path.
65
65
  *
66
- * @param enable - `true` forces MemWAL routing and errors if the table has no
67
- * MemWAL write spec. `false` forces the standard write path even when a spec is set.
66
+ * @param enable - `true` forces RTE routing and errors if the table has no
67
+ * RTE write spec. `false` forces the standard write path even when a spec is set.
68
68
  */
69
- useMemWal(enable: boolean): MergeInsertBuilder;
69
+ useRTE(enable: boolean): MergeInsertBuilder;
70
70
  /**
71
71
  * Controls how an LSM merge checks that its input targets a single shard.
72
72
  *
73
- * When a table has a MemWAL write spec, every row in a `mergeInsert` call must
73
+ * When a table has an RTE write spec, every row in a `mergeInsert` call must
74
74
  * route to the same shard. When `true` (the default), every row is inspected
75
75
  * to verify this. When `false`, only the first row is inspected and the
76
76
  * shard it routes to is used for the whole input — a faster path for callers
77
77
  * that have already pre-sharded their input. Has no effect on tables without
78
- * a MemWAL write spec.
78
+ * an RTE write spec.
79
79
  *
80
80
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
81
81
  */
package/dist/merge.js CHANGED
@@ -69,27 +69,27 @@ class MergeInsertBuilder {
69
69
  return new MergeInsertBuilder(this.#native.useIndex(useIndex), this.#schema);
70
70
  }
71
71
  /**
72
- * Control MemWAL routing for this merge.
72
+ * Control RTE routing for this merge.
73
73
  *
74
- * By default (unset), a `mergeInsert` on a table with a MemWAL write spec is
74
+ * By default (unset), a `mergeInsert` on a table with an RTE write spec is
75
75
  * routed through Lance's MemWAL shard writer, and a table without one uses the
76
76
  * standard path.
77
77
  *
78
- * @param enable - `true` forces MemWAL routing and errors if the table has no
79
- * MemWAL write spec. `false` forces the standard write path even when a spec is set.
78
+ * @param enable - `true` forces RTE routing and errors if the table has no
79
+ * RTE write spec. `false` forces the standard write path even when a spec is set.
80
80
  */
81
- useMemWal(enable) {
82
- return new MergeInsertBuilder(this.#native.useMemWal(enable), this.#schema);
81
+ useRTE(enable) {
82
+ return new MergeInsertBuilder(this.#native.useRTE(enable), this.#schema);
83
83
  }
84
84
  /**
85
85
  * Controls how an LSM merge checks that its input targets a single shard.
86
86
  *
87
- * When a table has a MemWAL write spec, every row in a `mergeInsert` call must
87
+ * When a table has an RTE write spec, every row in a `mergeInsert` call must
88
88
  * route to the same shard. When `true` (the default), every row is inspected
89
89
  * to verify this. When `false`, only the first row is inspected and the
90
90
  * shard it routes to is used for the whole input — a faster path for callers
91
91
  * that have already pre-sharded their input. Has no effect on tables without
92
- * a MemWAL write spec.
92
+ * an RTE write spec.
93
93
  *
94
94
  * @param validateSingleShard - Whether to check every row routes to one shard. Defaults to `true`.
95
95
  */
package/dist/native.d.ts CHANGED
@@ -31,7 +31,7 @@ export declare class Catalog {
31
31
  get uri(): string
32
32
  createDatabase(name: string, existOk?: boolean | undefined | null): Promise<Connection>
33
33
  connectDatabase(name: string): Promise<Connection>
34
- dropDatabase(name: string, ignoreMissing?: boolean | undefined | null): Promise<void>
34
+ dropDatabase(name: string, ignoreMissing?: boolean | undefined | null, cascade?: boolean | undefined | null): Promise<void>
35
35
  listDatabases(pageToken?: string | undefined | null, pageLimit?: number | undefined | null): DatabaseNames
36
36
  }
37
37
 
@@ -45,8 +45,8 @@ export declare class Connection {
45
45
  close(): void
46
46
  /** List all tables in the dataset. */
47
47
  tableNames(namespacePath?: Array<string> | undefined | null, startAfter?: string | undefined | null, limit?: number | undefined | null): Promise<Array<string>>
48
- /** List a page of tables in the database. */
49
- listTables(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, limit?: number | undefined | null): Promise<ListTablesResponse>
48
+ /** Lazily list tables in the database. */
49
+ listTables(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, limit?: number | undefined | null): NameListing
50
50
  /**
51
51
  * Create table from a Apache Arrow IPC (file) buffer.
52
52
  *
@@ -111,8 +111,8 @@ export declare class Connection {
111
111
  resumeJob(jobId: string): Promise<string>
112
112
  /** Describe a namespace and return its properties. */
113
113
  describeNamespace(namespacePath: Array<string>): Promise<DescribeNamespaceResponse>
114
- /** List child namespaces under the given namespace path */
115
- listNamespaces(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, limit?: number | undefined | null): Promise<ListNamespacesResponse>
114
+ /** Lazily list child namespaces under the given namespace path */
115
+ listNamespaces(namespacePath?: Array<string> | undefined | null, pageToken?: string | undefined | null, limit?: number | undefined | null): NameListing
116
116
  /** Create a new namespace with optional properties. */
117
117
  createNamespace(namespacePath: Array<string>, mode?: string | undefined | null, properties?: Record<string, string> | undefined | null): Promise<CreateNamespaceResponse>
118
118
  /** Drop a namespace. */
@@ -151,8 +151,8 @@ export declare class Index {
151
151
  /** A handle to an operation that may still be running. */
152
152
  export declare class Job {
153
153
  /**
154
- * Identifies the operation on the server that is running it. Operations
155
- * that run in this process have no server id. The value is opaque.
154
+ * The operation's opaque ID. Interpret registry and query IDs using job_type().
155
+ * Local handles have no ID.
156
156
  */
157
157
  get id(): string | null
158
158
  /**
@@ -172,16 +172,16 @@ export declare class Job {
172
172
  * Ask the backend for this job's current state, and for a server-side job
173
173
  * its full record, then cache it for the getters below.
174
174
  *
175
- * They are all null until this runs, because submitting an operation
176
- * returns only a job id. {@link Job.status} fetches the whole record too;
175
+ * Registry details are null until this runs; query handles already expose
176
+ * job_type "query". {@link Job.status} fetches the whole registry record too;
177
177
  * {@link Job.wait} records only the terminal state it establishes.
178
178
  */
179
179
  refresh(): Promise<void>
180
180
  /** The last observed lifecycle state, without contacting the backend. */
181
181
  get state(): string | null
182
182
  /**
183
- * The job's type, as the server names it. Null for an in-process job,
184
- * which has no server-side record.
183
+ * "query" immediately for SQL-backed handles; otherwise the cached
184
+ * server-reported job type. Null for in-process handles.
185
185
  */
186
186
  get jobType(): string | null
187
187
  /** When the job was created, in milliseconds since the epoch. */
@@ -265,7 +265,7 @@ export declare class NativeMergeInsertBuilder {
265
265
  whenNotMatchedBySourceDelete(filter?: string | undefined | null): NativeMergeInsertBuilder
266
266
  setTimeout(timeout: number): void
267
267
  useIndex(useIndex: boolean): NativeMergeInsertBuilder
268
- useMemWal(enable: boolean): NativeMergeInsertBuilder
268
+ useRTE(enable: boolean): NativeMergeInsertBuilder
269
269
  validateSingleShard(validateSingleShard: boolean): NativeMergeInsertBuilder
270
270
  execute(buf: Buffer): Promise<MergeResult>
271
271
  }
@@ -343,7 +343,7 @@ export declare class Query {
343
343
  nearestToRaw(data: Uint8Array, dtype: string): VectorQuery
344
344
  fastSearch(): void
345
345
  withRowId(): void
346
- useMemWal(enable: boolean): void
346
+ useRTE(enable: boolean): void
347
347
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
348
348
  outputSchema(): Promise<Buffer>
349
349
  execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
@@ -434,21 +434,21 @@ export declare class Table {
434
434
  refreshColumn(column: string): Promise<RefreshColumnResult>
435
435
  refreshColumnAsync(column: string): Promise<Job>
436
436
  functionErrors(options?: FunctionErrorsOptions | undefined | null): Promise<FunctionErrors>
437
- refreshMaterializedView(sourceVersion?: number | undefined | null): Promise<RefreshMaterializedViewResult>
437
+ refreshMaterializedView(sourceVersion?: number | undefined | null, full?: boolean | undefined | null): Promise<RefreshMaterializedViewResult>
438
438
  materializedViewDefinition(): Promise<string>
439
439
  addColumnsWithSchema(schemaBuf: Buffer): Promise<AddColumnsResult>
440
440
  alterColumns(alterations: Array<ColumnAlteration>): Promise<AlterColumnsResult>
441
441
  updateFieldMetadata(updates: Array<FieldMetadataUpdate>): Promise<UpdateFieldMetadataResult>
442
442
  dropColumns(columns: Array<string>): Promise<DropColumnsResult>
443
443
  setUnenforcedPrimaryKey(columns: Array<string>): Promise<void>
444
- enableMemWal(spec: MemWalWriteSpec): Promise<void>
445
- disableMemWal(): Promise<void>
446
- getMemWalSpec(): Promise<MemWalWriteSpec | null>
447
- closeMemWalWriters(): Promise<void>
448
- flushMemWal(): Promise<void>
449
- compactMemWal(): Promise<void>
450
- checkpointMemWal(): Promise<void>
451
- getMemWalStats(includeSstableRows: boolean): Promise<MemWalStats | null>
444
+ enableRTE(spec: RTEWriteSpec): Promise<void>
445
+ disableRTE(): Promise<void>
446
+ getRTESpec(): Promise<RTEWriteSpec | null>
447
+ closeRTEWriters(): Promise<void>
448
+ flushRTE(): Promise<void>
449
+ compactRTE(): Promise<void>
450
+ checkpointRTE(): Promise<void>
451
+ getRTEStats(includeSstableRows: boolean): Promise<RTEStats | null>
452
452
  version(): Promise<number>
453
453
  checkoutCurrent(): Promise<Table>
454
454
  checkout(version: number): Promise<void>
@@ -486,7 +486,7 @@ export declare class TakeQuery {
486
486
  select(columns: Array<[string, string]>): void
487
487
  selectColumns(columns: Array<string>): void
488
488
  withRowId(): void
489
- useMemWal(enable: boolean): void
489
+ useRTE(enable: boolean): void
490
490
  outputSchema(): Promise<Buffer>
491
491
  execute(maxBatchLength?: number | undefined | null, timeoutMs?: number | undefined | null, withRowId?: boolean | undefined | null): Promise<RecordBatchIterator>
492
492
  explainPlan(verbose: boolean): Promise<string>
@@ -515,7 +515,7 @@ export declare class VectorQuery {
515
515
  offset(offset: number): void
516
516
  fastSearch(): void
517
517
  withRowId(): void
518
- useMemWal(enable: boolean): void
518
+ useRTE(enable: boolean): void
519
519
  rerank(rerankHybrid: (arg: RerankHybridCallbackArgs) => Promise<Buffer>): void
520
520
  orderBy(ordering?: Array<ColumnOrdering> | undefined | null): void
521
521
  outputSchema(): Promise<Buffer>
@@ -756,6 +756,9 @@ export interface DropNamespaceResponse {
756
756
  transactionId?: Array<string>
757
757
  }
758
758
 
759
+ /** Whether this native binding was built with local experimental Functions. */
760
+ export declare function experimentalFunctionsEnabled(): boolean
761
+
759
762
  /**
760
763
  * A per-field metadata update, addressed by dot-path. Merges into the field's
761
764
  * existing metadata by default; a `null` value deletes a key, and `replace`
@@ -966,16 +969,6 @@ export interface JobInfo {
966
969
  /** The catalog of described LanceDB metrics. Empty until the recorder is installed. */
967
970
  export declare function lancedbMetricsCatalog(): Array<MetricDescription>
968
971
 
969
- export interface ListNamespacesResponse {
970
- namespaces: Array<string>
971
- pageToken?: string
972
- }
973
-
974
- export interface ListTablesResponse {
975
- tables: Array<string>
976
- pageToken?: string
977
- }
978
-
979
972
  /** One in-memory memtable. */
980
973
  export interface MemtableStats {
981
974
  /** The generation this memtable will become once frozen. */
@@ -993,41 +986,6 @@ export interface MemtableStats {
993
986
  indexes: Array<string>
994
987
  }
995
988
 
996
- /**
997
- * Live per-table-shard MemWAL state, as returned by `Table#getMemWalStats`.
998
- *
999
- * Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
1000
- * the caller's to compute.
1001
- */
1002
- export interface MemWalStats {
1003
- /** One entry per table shard backing this table. */
1004
- tableShards: Array<TableShardStats>
1005
- }
1006
-
1007
- /**
1008
- * Specification selecting Lance's MemWAL LSM-style write path for
1009
- * `mergeInsert`.
1010
- *
1011
- * `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
1012
- * `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
1013
- * `column` is required.
1014
- */
1015
- export interface MemWalWriteSpec {
1016
- /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
1017
- specType: string
1018
- /** Bucket and identity variants: the sharding column. */
1019
- column?: string
1020
- /** Bucket variant: the number of buckets, in `[1, 1024]`. */
1021
- numBuckets?: number
1022
- /**
1023
- * Indexes the MemWAL keeps up to date. Omitted resolves every
1024
- * maintainable index on install; an empty array means none.
1025
- */
1026
- maintainedIndexes?: Array<string>
1027
- /** Default `ShardWriter` configuration recorded in the MemWAL index. */
1028
- writerConfigDefaults?: Record<string, string>
1029
- }
1030
-
1031
989
  export interface MergeResult {
1032
990
  version: number
1033
991
  numInsertedRows: number
@@ -1149,10 +1107,12 @@ export interface RefreshColumnResult {
1149
1107
  }
1150
1108
 
1151
1109
  export interface RefreshMaterializedViewResult {
1152
- /** How the view was brought up to date: "rebuild". */
1153
- mode: string
1154
- rowsWritten: number
1155
- sourceVersion: number
1110
+ /** How the view was brought up to date: "rebuild", or absent for remote SQL refreshes. */
1111
+ mode?: string
1112
+ /** Rows written by this refresh, or absent for remote SQL refreshes. */
1113
+ rowsWritten?: number
1114
+ /** Source version refreshed to, or absent for remote SQL refreshes. */
1115
+ sourceVersion?: number
1156
1116
  version: number
1157
1117
  }
1158
1118
 
@@ -1228,6 +1188,41 @@ export interface RetryConfig {
1228
1188
  statuses?: Array<number>
1229
1189
  }
1230
1190
 
1191
+ /**
1192
+ * Live per-table-shard RTE state, as returned by `Table#getRTEStats`.
1193
+ *
1194
+ * Nothing here is derived: sums and differences (total SSTable bytes, WAL lag) are
1195
+ * the caller's to compute.
1196
+ */
1197
+ export interface RTEStats {
1198
+ /** One entry per table shard backing this table. */
1199
+ tableShards: Array<TableShardStats>
1200
+ }
1201
+
1202
+ /**
1203
+ * Specification enabling the realtime engine (RTE), which routes
1204
+ * `mergeInsert` through Lance's MemWAL LSM-style write path.
1205
+ *
1206
+ * `specType` must be `"bucket"`, `"identity"`, or `"unsharded"`. For
1207
+ * `"bucket"`, `column` and `numBuckets` are required; for `"identity"`,
1208
+ * `column` is required.
1209
+ */
1210
+ export interface RTEWriteSpec {
1211
+ /** One of `"bucket"`, `"identity"`, or `"unsharded"`. */
1212
+ specType: string
1213
+ /** Bucket and identity variants: the sharding column. */
1214
+ column?: string
1215
+ /** Bucket variant: the number of buckets, in `[1, 1024]`. */
1216
+ numBuckets?: number
1217
+ /**
1218
+ * Indexes the RTE keeps up to date. Omitted resolves every
1219
+ * maintainable index on install; an empty array means none.
1220
+ */
1221
+ maintainedIndexes?: Array<string>
1222
+ /** Default `ShardWriter` configuration recorded in the MemWAL index. */
1223
+ writerConfigDefaults?: Record<string, string>
1224
+ }
1225
+
1231
1226
  /** Result of `OAuthSession.logout()`. */
1232
1227
  export interface SessionLogout {
1233
1228
  /**
@@ -1401,6 +1396,10 @@ export interface TimeoutConfig {
1401
1396
  * The timeout for reading data from the server in seconds. Default is 300
1402
1397
  * seconds (5 minutes). This can also be set via the environment variable
1403
1398
  * `LANCE_CLIENT_READ_TIMEOUT`, as an integer number of seconds.
1399
+ *
1400
+ * This also applies while waiting for response headers during an upload.
1401
+ * A large or slowly produced request body (for example, when creating a
1402
+ * table) may need a longer timeout even if data is still being uploaded.
1404
1403
  */
1405
1404
  readTimeout?: number
1406
1405
  /**