@lancedb/lancedb 0.38.0-beta.0 → 0.38.0-beta.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/schema.js ADDED
@@ -0,0 +1,387 @@
1
+ "use strict";
2
+ // SPDX-License-Identifier: Apache-2.0
3
+ // SPDX-FileCopyrightText: Copyright The LanceDB Authors
4
+ Object.defineProperty(exports, "__esModule", { value: true });
5
+ exports.inferSchema = inferSchema;
6
+ const apache_arrow_1 = require("apache-arrow");
7
+ const arrow_type_1 = require("./arrow_type");
8
+ const sanitize_1 = require("./sanitize");
9
+ /**
10
+ * Infer the Arrow schema represented by a set of records.
11
+ *
12
+ * This is the intentionally small interface to schema inference. The stateful
13
+ * details of combining partial type evidence are encapsulated below so callers
14
+ * only need to provide records, an optional schema, and inference options.
15
+ */
16
+ function inferSchema(data, schema, options) {
17
+ return new SchemaInferrer(schema, options).infer(data);
18
+ }
19
+ class SchemaInferrer {
20
+ providedSchema;
21
+ options;
22
+ fields = new FieldTree();
23
+ constructor(providedSchema, options) {
24
+ this.providedSchema = providedSchema;
25
+ this.options = options;
26
+ }
27
+ infer(data) {
28
+ for (const [row, record] of data.entries()) {
29
+ for (const [path, value] of recordPathsAndValues(record)) {
30
+ this.observe(path, value, row);
31
+ }
32
+ }
33
+ return this.providedSchema === undefined
34
+ ? new apache_arrow_1.Schema(fieldsFromTree(this.fields))
35
+ : new apache_arrow_1.Schema(matchingFields(this.providedSchema.fields, this.fields));
36
+ }
37
+ observe(path, value, row) {
38
+ const current = this.fields.get(path);
39
+ if (current === undefined) {
40
+ this.addField(path, value, row);
41
+ }
42
+ else if (this.providedSchema === undefined) {
43
+ this.updateInferredField(path, value, row, current);
44
+ }
45
+ }
46
+ addField(path, value, row) {
47
+ if (this.providedSchema !== undefined) {
48
+ this.addSchemaField(this.providedSchema, path, row);
49
+ return;
50
+ }
51
+ const evidence = this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
52
+ if (evidence === undefined) {
53
+ throw typeInferenceError(path, row);
54
+ }
55
+ const conflict = this.fields.set(path, evidence, (existing) => existing instanceof DeferredTypeEvidence && existing.isOnlyNulls());
56
+ if (conflict !== undefined) {
57
+ throw branchConflictError(conflict, row, "Struct");
58
+ }
59
+ }
60
+ addSchemaField(schema, path, row) {
61
+ const field = fieldAtPath(schema, path);
62
+ if (field === undefined) {
63
+ throw new Error(`Found field not in schema: ${path.join(".")} at row ${row}`);
64
+ }
65
+ const conflict = this.fields.set(path, field.type);
66
+ if (conflict !== undefined) {
67
+ throw branchConflictError(conflict, row, "Struct");
68
+ }
69
+ }
70
+ updateInferredField(path, value, row, current) {
71
+ const newType = this.inferType(value, path);
72
+ const deferred = DeferredTypeEvidence.from(value, row);
73
+ if (current instanceof FieldTree) {
74
+ if (deferred?.isOnlyNulls()) {
75
+ return;
76
+ }
77
+ throw schemaInferenceError(path, row, "Struct", describeEvidence(newType ?? deferred));
78
+ }
79
+ if (current instanceof DeferredTypeEvidence) {
80
+ this.resolveDeferredField(path, row, current, newType, deferred);
81
+ return;
82
+ }
83
+ if (newType !== undefined) {
84
+ if (!inferredTypesEqual(current, newType)) {
85
+ throw schemaInferenceError(path, row, describeEvidence(current), describeEvidence(newType));
86
+ }
87
+ return;
88
+ }
89
+ if (deferred === undefined || !deferred.matches(current)) {
90
+ throw schemaInferenceError(path, row, describeEvidence(current), describeEvidence(deferred));
91
+ }
92
+ }
93
+ resolveDeferredField(path, row, current, newType, deferred) {
94
+ if (newType !== undefined) {
95
+ if (!current.matches(newType)) {
96
+ throw schemaInferenceError(path, row, current.describe(), describeEvidence(newType));
97
+ }
98
+ this.fields.set(path, newType);
99
+ return;
100
+ }
101
+ if (deferred !== undefined) {
102
+ this.fields.set(path, current.merge(deferred));
103
+ return;
104
+ }
105
+ throw schemaInferenceError(path, row, current.describe(), describeEvidence(newType));
106
+ }
107
+ inferType(value, path) {
108
+ if (typeof value === "bigint") {
109
+ return new apache_arrow_1.Int64();
110
+ }
111
+ if (typeof value === "number") {
112
+ return new apache_arrow_1.Float64();
113
+ }
114
+ if (typeof value === "string") {
115
+ return this.options.dictionaryEncodeStrings
116
+ ? new apache_arrow_1.Dictionary(new apache_arrow_1.Utf8(), new apache_arrow_1.Int32())
117
+ : new apache_arrow_1.Utf8();
118
+ }
119
+ if (typeof value === "boolean") {
120
+ return new apache_arrow_1.Bool();
121
+ }
122
+ if (value instanceof Buffer) {
123
+ return new apache_arrow_1.Binary();
124
+ }
125
+ if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
126
+ const typedArray = (0, arrow_type_1.typedArrayToArrowType)(value);
127
+ return typedArray === undefined
128
+ ? undefined
129
+ : new apache_arrow_1.FixedSizeList(typedArray.length, new apache_arrow_1.Field("item", typedArray.elementType, true));
130
+ }
131
+ if (!Array.isArray(value) || value.length === 0) {
132
+ return undefined;
133
+ }
134
+ const configuredVector = path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
135
+ if (configuredVector !== undefined) {
136
+ return new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", (0, sanitize_1.sanitizeType)(configuredVector.type), true));
137
+ }
138
+ const itemType = this.inferArrayItemType(value, path);
139
+ if (itemType === undefined) {
140
+ return undefined;
141
+ }
142
+ return nameSuggestsVectorColumn(path[path.length - 1])
143
+ ? new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", new apache_arrow_1.Float32(), true))
144
+ : new apache_arrow_1.List(new apache_arrow_1.Field("item", itemType, true));
145
+ }
146
+ inferArrayItemType(values, path) {
147
+ let itemType;
148
+ const deferredItems = [];
149
+ for (const value of values) {
150
+ const candidate = this.inferType(value, path);
151
+ if (candidate === undefined) {
152
+ if (!isDeferredValue(value)) {
153
+ return undefined;
154
+ }
155
+ deferredItems.push(value);
156
+ }
157
+ else if (itemType === undefined) {
158
+ itemType = candidate;
159
+ }
160
+ else if (!inferredTypesEqual(itemType, candidate)) {
161
+ return undefined;
162
+ }
163
+ }
164
+ if (itemType === undefined) {
165
+ return undefined;
166
+ }
167
+ return deferredItems.every((value) => deferredValueMatchesType(value, itemType))
168
+ ? itemType
169
+ : undefined;
170
+ }
171
+ }
172
+ /** Nulls and empty/all-null lists that do not determine a type by themselves. */
173
+ class DeferredTypeEvidence {
174
+ values;
175
+ constructor(values) {
176
+ this.values = values;
177
+ }
178
+ static from(value, row) {
179
+ return isDeferredValue(value)
180
+ ? new DeferredTypeEvidence([{ value, row }])
181
+ : undefined;
182
+ }
183
+ isOnlyNulls() {
184
+ return this.values.every(({ value }) => value == null);
185
+ }
186
+ matches(type) {
187
+ return this.values.every(({ value }) => deferredValueMatchesType(value, type));
188
+ }
189
+ merge(other) {
190
+ return new DeferredTypeEvidence([...this.values, ...other.values]);
191
+ }
192
+ describe() {
193
+ const list = this.values.find(({ value }) => Array.isArray(value));
194
+ return list === undefined
195
+ ? "null"
196
+ : `List[${list.value.length}]`;
197
+ }
198
+ firstRow() {
199
+ return this.values[0].row;
200
+ }
201
+ }
202
+ /** Nested field state, kept separate from Arrow's eventual Struct types. */
203
+ class FieldTree {
204
+ children = new Map();
205
+ get(path) {
206
+ let current = this;
207
+ for (const part of path) {
208
+ if (!(current instanceof FieldTree)) {
209
+ return undefined;
210
+ }
211
+ const child = current.children.get(part);
212
+ if (child === undefined) {
213
+ return undefined;
214
+ }
215
+ current = child;
216
+ }
217
+ return current;
218
+ }
219
+ set(path, value, canReplaceLeaf = () => false) {
220
+ let branch = this;
221
+ for (const [index, part] of path.slice(0, -1).entries()) {
222
+ const child = branch.children.get(part);
223
+ if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
224
+ const nextBranch = new FieldTree();
225
+ branch.children.set(part, nextBranch);
226
+ branch = nextBranch;
227
+ }
228
+ else if (child instanceof FieldTree) {
229
+ branch = child;
230
+ }
231
+ else {
232
+ return { path: path.slice(0, index + 1), value: child };
233
+ }
234
+ }
235
+ const name = path[path.length - 1];
236
+ const current = branch.children.get(name);
237
+ if (current instanceof FieldTree) {
238
+ return { path, value: current };
239
+ }
240
+ branch.children.set(name, value);
241
+ return undefined;
242
+ }
243
+ entries() {
244
+ return this.children.entries();
245
+ }
246
+ has(name) {
247
+ return this.children.has(name);
248
+ }
249
+ }
250
+ function isLeaf(value) {
251
+ return !(value instanceof FieldTree);
252
+ }
253
+ function fieldsFromTree(tree, path = []) {
254
+ const fields = [];
255
+ for (const [name, value] of tree.entries()) {
256
+ if (value instanceof FieldTree) {
257
+ fields.push(new apache_arrow_1.Field(name, new apache_arrow_1.Struct(fieldsFromTree(value, [...path, name])), true));
258
+ }
259
+ else if (value instanceof DeferredTypeEvidence) {
260
+ throw typeInferenceError([...path, name], value.firstRow());
261
+ }
262
+ else {
263
+ fields.push(new apache_arrow_1.Field(name, value, true));
264
+ }
265
+ }
266
+ return fields;
267
+ }
268
+ function matchingFields(fields, tree) {
269
+ const matches = [];
270
+ for (const field of fields) {
271
+ if (!tree.has(field.name)) {
272
+ continue;
273
+ }
274
+ const value = tree.get([field.name]);
275
+ if (value instanceof FieldTree) {
276
+ const struct = field.type;
277
+ matches.push(new apache_arrow_1.Field(field.name, new apache_arrow_1.Struct(matchingFields(struct.children, value)), field.nullable, field.metadata));
278
+ }
279
+ else {
280
+ matches.push(field);
281
+ }
282
+ }
283
+ return matches;
284
+ }
285
+ function* recordPathsAndValues(record, path = []) {
286
+ for (const [name, value] of Object.entries(record)) {
287
+ if (isRecord(value)) {
288
+ yield* recordPathsAndValues(value, [...path, name]);
289
+ }
290
+ else if (value !== undefined) {
291
+ yield [[...path, name], value];
292
+ }
293
+ }
294
+ }
295
+ function isRecord(value) {
296
+ return (typeof value === "object" &&
297
+ value !== null &&
298
+ !Array.isArray(value) &&
299
+ !(value instanceof RegExp) &&
300
+ !(value instanceof Date) &&
301
+ !(value instanceof Set) &&
302
+ !(value instanceof Map) &&
303
+ !(value instanceof Buffer) &&
304
+ !ArrayBuffer.isView(value));
305
+ }
306
+ function fieldAtPath(schema, path) {
307
+ let fields = schema.fields;
308
+ let field;
309
+ for (const [index, name] of path.entries()) {
310
+ field = fields.find((candidate) => candidate.name === name);
311
+ if (field === undefined || index === path.length - 1) {
312
+ return field;
313
+ }
314
+ if (!apache_arrow_1.DataType.isStruct(field.type)) {
315
+ return undefined;
316
+ }
317
+ fields = field.type.children;
318
+ }
319
+ return field;
320
+ }
321
+ function isDeferredValue(value) {
322
+ return (value == null || (Array.isArray(value) && value.every(isDeferredValue)));
323
+ }
324
+ function deferredValueMatchesType(value, type) {
325
+ if (value == null) {
326
+ return true;
327
+ }
328
+ if (!Array.isArray(value)) {
329
+ return false;
330
+ }
331
+ if (apache_arrow_1.DataType.isList(type)) {
332
+ return value.every((item) => deferredValueMatchesType(item, type.valueType));
333
+ }
334
+ if (apache_arrow_1.DataType.isFixedSizeList(type)) {
335
+ return (value.length === type.listSize &&
336
+ value.every((item) => deferredValueMatchesType(item, type.valueType)));
337
+ }
338
+ return false;
339
+ }
340
+ function inferredTypesEqual(current, candidate) {
341
+ if (apache_arrow_1.DataType.isDictionary(current)) {
342
+ return (apache_arrow_1.DataType.isDictionary(candidate) &&
343
+ current.isOrdered === candidate.isOrdered &&
344
+ inferredTypesEqual(current.indices, candidate.indices) &&
345
+ inferredTypesEqual(current.dictionary, candidate.dictionary));
346
+ }
347
+ if (apache_arrow_1.DataType.isList(current)) {
348
+ return (apache_arrow_1.DataType.isList(candidate) &&
349
+ current.valueField.name === candidate.valueField.name &&
350
+ current.valueField.nullable === candidate.valueField.nullable &&
351
+ inferredTypesEqual(current.valueType, candidate.valueType));
352
+ }
353
+ if (apache_arrow_1.DataType.isFixedSizeList(current)) {
354
+ return (apache_arrow_1.DataType.isFixedSizeList(candidate) &&
355
+ current.listSize === candidate.listSize &&
356
+ current.valueField.name === candidate.valueField.name &&
357
+ current.valueField.nullable === candidate.valueField.nullable &&
358
+ inferredTypesEqual(current.valueType, candidate.valueType));
359
+ }
360
+ return apache_arrow_1.util.compareTypes(current, candidate);
361
+ }
362
+ function describeEvidence(evidence) {
363
+ if (evidence === undefined) {
364
+ return "an unsupported value";
365
+ }
366
+ return evidence instanceof DeferredTypeEvidence
367
+ ? evidence.describe()
368
+ : evidence.toString();
369
+ }
370
+ function branchConflictError(conflict, row, candidate) {
371
+ return schemaInferenceError(conflict.path, row, conflict.value instanceof FieldTree
372
+ ? "Struct"
373
+ : describeEvidence(conflict.value), candidate);
374
+ }
375
+ function schemaInferenceError(path, row, currentType, newType) {
376
+ return new Error(`Failed to infer schema for data. Previously inferred type ${currentType} ` +
377
+ `but found ${newType} for field ${path.join(".")} at row ${row}. ` +
378
+ "Consider providing an explicit schema.");
379
+ }
380
+ function typeInferenceError(path, row) {
381
+ return new Error(`Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
382
+ "Consider providing an explicit schema.");
383
+ }
384
+ function nameSuggestsVectorColumn(name) {
385
+ const normalized = name.toLowerCase();
386
+ return normalized.includes("vector") || normalized.includes("embedding");
387
+ }
package/dist/table.d.ts CHANGED
@@ -1,10 +1,11 @@
1
1
  import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, Schema } from "./arrow";
2
2
  import { IndexOptions } from "./indices";
3
3
  import { MergeInsertBuilder } from "./merge";
4
- import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, Branches as NativeBranches, OptimizeStats, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
5
- import { FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
4
+ import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, LsmStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
5
+ import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
6
6
  import { IntoSql } from "./util";
7
7
  export { IndexConfig } from "./native";
8
+ export { BucketStats, GenerationStats, LsmStats, MemtableStats, } from "./native";
8
9
  /**
9
10
  * Progress snapshot for a write operation, delivered to the `progress`
10
11
  * callback passed to {@link Table.add}.
@@ -421,7 +422,7 @@ export declare abstract class Table {
421
422
  * when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
422
423
  * if the query is a string and no embedding function is defined, it will be treated as a full text search query
423
424
  */
424
- abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
425
+ abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
425
426
  /**
426
427
  * Search the table with a given query vector.
427
428
  *
@@ -432,15 +433,75 @@ export declare abstract class Table {
432
433
  abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
433
434
  /**
434
435
  * Add new columns with defined values.
436
+ *
437
+ * The `{ computed }` form stores the expression rather than evaluating it
438
+ * now: the column is committed with no values, and rows get them from
439
+ * {@link Table#refreshColumn}. Declaring one therefore costs the same on a
440
+ * large table as on an empty one.
441
+ *
442
+ * A refresh does not revisit rows it has already filled, so mutating an
443
+ * input leaves the value computed at fill time; recomputing means dropping
444
+ * the column and declaring it again. While a declaration reads a column,
445
+ * that column cannot be renamed, retyped or dropped.
446
+ *
447
+ * On LanceDB Cloud and Enterprise the expression is planned by the
448
+ * server, and the refresh runs as a server job -- see
449
+ * {@link Table#refreshColumnAsync}.
435
450
  * @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
436
451
  * - An array of objects with column names and SQL expressions to calculate values
437
452
  * - A single Arrow Field defining one column with its data type (column will be initialized with null values)
438
453
  * - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
439
454
  * - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
455
+ * - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
440
456
  * @returns {Promise<AddColumnsResult>} A promise that resolves to an object
441
457
  * containing the new version number of the table after adding the columns.
458
+ * @example
459
+ * ```ts
460
+ * await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
461
+ * const { rowsFilled } = await table.refreshColumn("doubled");
462
+ * ```
463
+ */
464
+ abstract addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
465
+ computed: AddColumnsSql[];
466
+ }): Promise<AddColumnsResult>;
467
+ /**
468
+ * Fill the rows of a computed column that hold no value yet.
469
+ *
470
+ * Rows appended since the last refresh are filled by the next one; rows
471
+ * already filled are left as they are, so the call is idempotent and does
472
+ * not observe a mutated input. Local tables only: a remote refresh runs
473
+ * as a server job, through {@link Table#refreshColumnAsync}.
474
+ * @param {string} column The name of the computed column to fill.
475
+ * @returns {Promise<RefreshColumnResult>} A promise that resolves to the
476
+ * number of rows filled and the new version number of the table.
442
477
  */
443
- abstract addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema): Promise<AddColumnsResult>;
478
+ abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
479
+ /**
480
+ * Like {@link Table#refreshColumn}, but returns a handle to the refresh
481
+ * job instead of blocking until it completes.
482
+ *
483
+ * The job may already be complete when returned; callers must not assume
484
+ * the column is filled until {@link Job.wait} resolves. Invalid input --
485
+ * an unknown column, or one that is not computed -- rejects here rather
486
+ * than failing the job. On local tables the job runs in-process; on
487
+ * LanceDB Cloud and Enterprise it is the server's backfill job.
488
+ * @param {string} column The name of the computed column to fill.
489
+ * @example
490
+ * ```ts
491
+ * const job = await table.refreshColumnAsync("doubled");
492
+ * await job.wait();
493
+ * console.log(await job.status()); // "finished"
494
+ * ```
495
+ */
496
+ abstract refreshColumnAsync(column: string): Promise<Job>;
497
+ /**
498
+ * Recompute this table's contents from its materialized-view definition.
499
+ *
500
+ * Plumbing for {@link MaterializedView.refresh}, which is the way to call
501
+ * it: rejects tables that carry no view definition. Local tables only.
502
+ * @ignore
503
+ */
504
+ abstract refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
444
505
  /**
445
506
  * Alter the name or nullability of columns.
446
507
  * @param {ColumnAlteration[]} columnAlterations One or more alterations to
@@ -451,6 +512,18 @@ export declare abstract class Table {
451
512
  abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
452
513
  /**
453
514
  * Update per-field (column) metadata.
515
+ *
516
+ * The following keys are treated specially, by convention, and should be
517
+ * used when appropriate:
518
+ *
519
+ * - `lancedb:description`: for a human-readable description of a field.
520
+ * - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
521
+ * names the tag category; e.g. `lancedb:tag:model: "clip"`.
522
+ * - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
523
+ * `feature_v2` might be in the same logical column.
524
+ * - `lancedb:status`: for status options (`production`, `candidate`,
525
+ * `deprecated`, `archived`) to designate the current life cycle state of
526
+ * this column.
454
527
  * @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
455
528
  * update's metadata is merged into the field's existing metadata by default;
456
529
  * a value of `null` deletes that key, and `replace: true` swaps the whole map.
@@ -546,6 +619,57 @@ export declare abstract class Table {
546
619
  * @returns {Promise<void>}
547
620
  */
548
621
  abstract closeLsmWriters(): Promise<void>;
622
+ /**
623
+ * Seal every bucket's active memtable into a new L0 generation.
624
+ *
625
+ * Returns once the seal is committed. Sealing an empty memtable is a no-op,
626
+ * so this is safe to call repeatedly.
627
+ * @returns {Promise<void>}
628
+ */
629
+ abstract flushLsm(): Promise<void>;
630
+ /**
631
+ * Trigger a background L0 → base compaction pass per bucket.
632
+ *
633
+ * Returns once the passes are *dispatched*, not once they finish — watch
634
+ * {@link Table#getLsmStats} for progress, or use
635
+ * {@link Table#checkpointLsm} to wait for convergence.
636
+ * @returns {Promise<void>}
637
+ */
638
+ abstract compactLsm(): Promise<void>;
639
+ /**
640
+ * Converge this table's LSM write path into its base table.
641
+ *
642
+ * Seals once, then triggers compaction and polls until the L0 that existed
643
+ * at the start is gone. The target set is fixed at the start, so
644
+ * generations created *during* the checkpoint are ignored — that is what
645
+ * lets it terminate under write load, and what makes it best-effort: it
646
+ * converges the fresh tier as of some instant. Idempotent, abandonable at
647
+ * any point, and safe to run on a cadence.
648
+ *
649
+ * There is no liveness bound — the compactor pool is shared across tables,
650
+ * so a checkpoint queued behind unrelated work looks exactly like one that
651
+ * is merging. The caller owns the deadline.
652
+ * @returns {Promise<void>}
653
+ * @example
654
+ * ```ts
655
+ * const before = await table.getLsmStats();
656
+ * await table.checkpointLsm();
657
+ * const after = await table.getLsmStats();
658
+ * ```
659
+ */
660
+ abstract checkpointLsm(): Promise<void>;
661
+ /**
662
+ * Read live per-bucket LSM state.
663
+ *
664
+ * Answers "how far behind is my fresh tier", "which bucket is hot", and
665
+ * "why is my fresh-tier vector search brute-force". Mutates no table state.
666
+ *
667
+ * Resolves to `undefined` only when the LSM write path is not enabled.
668
+ * @param {boolean} includeGenerationRows Also count rows per L0 generation.
669
+ * Off by default because each count opens an uncached Lance dataset.
670
+ * @returns {Promise<LsmStats | undefined>}
671
+ */
672
+ abstract getLsmStats(includeGenerationRows?: boolean): Promise<LsmStats | undefined>;
549
673
  /** Retrieve the version of the table */
550
674
  abstract version(): Promise<number>;
551
675
  /**
@@ -727,9 +851,14 @@ export declare class LocalTable extends Table {
727
851
  takeOffsets(offsets: number[]): TakeQuery;
728
852
  takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
729
853
  query(): Query;
730
- search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
854
+ search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
731
855
  vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
732
- addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema): Promise<AddColumnsResult>;
856
+ addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
857
+ computed: AddColumnsSql[];
858
+ }): Promise<AddColumnsResult>;
859
+ refreshColumn(column: string): Promise<RefreshColumnResult>;
860
+ refreshColumnAsync(column: string): Promise<Job>;
861
+ refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
733
862
  alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
734
863
  updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
735
864
  dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
@@ -738,6 +867,10 @@ export declare class LocalTable extends Table {
738
867
  unsetLsmWriteSpec(): Promise<void>;
739
868
  getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
740
869
  closeLsmWriters(): Promise<void>;
870
+ flushLsm(): Promise<void>;
871
+ compactLsm(): Promise<void>;
872
+ checkpointLsm(): Promise<void>;
873
+ getLsmStats(includeGenerationRows?: boolean): Promise<LsmStats | undefined>;
741
874
  version(): Promise<number>;
742
875
  checkout(version: number | string): Promise<void>;
743
876
  checkoutLatest(): Promise<void>;
@@ -820,7 +953,8 @@ export interface FieldMetadataUpdate {
820
953
  path: string;
821
954
  /**
822
955
  * Metadata key/value pairs. Merged into the field's existing metadata by
823
- * default; a value of `null` deletes that key.
956
+ * default; a value of `null` deletes that key. See
957
+ * {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
824
958
  */
825
959
  metadata: Record<string, string | null>;
826
960
  /** If true, replace the field's entire metadata map instead of merging. */
@@ -854,8 +988,8 @@ export interface BranchRowCountSummary {
854
988
  inputsChanged: number;
855
989
  deltaAvailable: boolean;
856
990
  }
857
- /** A reason why a branch cannot currently be merged. */
858
- export interface MergeBlocker {
991
+ /** A reason why a cherry-pick cannot currently land. */
992
+ export interface CherryPickError {
859
993
  code: string;
860
994
  message: string;
861
995
  }
@@ -874,18 +1008,17 @@ export interface BranchDiff {
874
1008
  changedColumns: BranchColumnChange[];
875
1009
  addedIndexes: BranchIndexSummary[];
876
1010
  removedIndexes: BranchIndexSummary[];
877
- mergeable: boolean;
878
- mergeBlockers: MergeBlocker[];
1011
+ errors: CherryPickError[];
879
1012
  }
880
- /** Changes that would be, or were, promoted by a branch merge. */
881
- export interface MergePreview {
1013
+ /** Changes that would be, or were, promoted by a cherry-pick. */
1014
+ export interface CherryPickPreview {
882
1015
  promotedColumns: string[];
883
1016
  }
884
- /** Result of previewing or attempting a branch merge. */
885
- export interface MergeBranchResult {
886
- status: "ready" | "rejected" | "notImplemented" | "merged" | "unknown";
1017
+ /** Result of previewing or attempting a cherry-pick. */
1018
+ export interface CherryPickResult {
1019
+ status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
887
1020
  diff: BranchDiff;
888
- preview: MergePreview;
1021
+ preview: CherryPickPreview;
889
1022
  mainVersionAfter?: number;
890
1023
  }
891
1024
  /**
@@ -924,13 +1057,13 @@ export declare class Branches {
924
1057
  /** Compare a branch against main without modifying either branch. */
925
1058
  diff(fromBranch: string): Promise<BranchDiff>;
926
1059
  /**
927
- * Merge a branch into main.
1060
+ * Cherry-pick a branch onto main.
928
1061
  *
929
- * Set `dryRun` to `true` to preview the merge. A rejected merge resolves
930
- * with `status: "rejected"` instead of throwing.
1062
+ * Set `dryRun` to `true` to preview. A failed cherry-pick resolves
1063
+ * with `status: "failed"` instead of throwing.
931
1064
  *
932
- * @param fromBranch Branch to merge from.
933
- * @param dryRun When true, only preview the merge. Defaults to false.
1065
+ * @param fromBranch Branch to cherry-pick from.
1066
+ * @param dryRun When true, only preview. Defaults to false.
934
1067
  */
935
- merge(fromBranch: string, dryRun?: boolean): Promise<MergeBranchResult>;
1068
+ cherryPick(fromBranch: string, dryRun?: boolean): Promise<CherryPickResult>;
936
1069
  }