@lancedb/lancedb 0.38.0-beta.0 → 0.38.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/arrow.d.ts +4 -1
- package/dist/arrow.js +35 -282
- package/dist/arrow_type.d.ts +9 -0
- package/dist/arrow_type.js +29 -0
- package/dist/connection.d.ts +96 -2
- package/dist/connection.js +23 -0
- package/dist/embedding/index.d.ts +11 -1
- package/dist/embedding/index.js +24 -17
- package/dist/embedding/openai.js +3 -15
- package/dist/embedding/registry.d.ts +21 -1
- package/dist/embedding/registry.js +76 -20
- package/dist/embedding/transformers.js +3 -15
- package/dist/index.d.ts +5 -4
- package/dist/index.js +4 -1
- package/dist/materialized_view.d.ts +69 -0
- package/dist/materialized_view.js +112 -0
- package/dist/native.d.ts +115 -1
- package/dist/native.js +52 -52
- package/dist/query.d.ts +38 -2
- package/dist/query.js +154 -101
- package/dist/sanitize.js +10 -4
- package/dist/schema.d.ts +16 -0
- package/dist/schema.js +387 -0
- package/dist/table.d.ts +156 -23
- package/dist/table.js +55 -15
- package/package.json +10 -10
package/dist/schema.js
ADDED
|
@@ -0,0 +1,387 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.inferSchema = inferSchema;
|
|
6
|
+
const apache_arrow_1 = require("apache-arrow");
|
|
7
|
+
const arrow_type_1 = require("./arrow_type");
|
|
8
|
+
const sanitize_1 = require("./sanitize");
|
|
9
|
+
/**
|
|
10
|
+
* Infer the Arrow schema represented by a set of records.
|
|
11
|
+
*
|
|
12
|
+
* This is the intentionally small interface to schema inference. The stateful
|
|
13
|
+
* details of combining partial type evidence are encapsulated below so callers
|
|
14
|
+
* only need to provide records, an optional schema, and inference options.
|
|
15
|
+
*/
|
|
16
|
+
function inferSchema(data, schema, options) {
|
|
17
|
+
return new SchemaInferrer(schema, options).infer(data);
|
|
18
|
+
}
|
|
19
|
+
class SchemaInferrer {
|
|
20
|
+
providedSchema;
|
|
21
|
+
options;
|
|
22
|
+
fields = new FieldTree();
|
|
23
|
+
constructor(providedSchema, options) {
|
|
24
|
+
this.providedSchema = providedSchema;
|
|
25
|
+
this.options = options;
|
|
26
|
+
}
|
|
27
|
+
infer(data) {
|
|
28
|
+
for (const [row, record] of data.entries()) {
|
|
29
|
+
for (const [path, value] of recordPathsAndValues(record)) {
|
|
30
|
+
this.observe(path, value, row);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return this.providedSchema === undefined
|
|
34
|
+
? new apache_arrow_1.Schema(fieldsFromTree(this.fields))
|
|
35
|
+
: new apache_arrow_1.Schema(matchingFields(this.providedSchema.fields, this.fields));
|
|
36
|
+
}
|
|
37
|
+
observe(path, value, row) {
|
|
38
|
+
const current = this.fields.get(path);
|
|
39
|
+
if (current === undefined) {
|
|
40
|
+
this.addField(path, value, row);
|
|
41
|
+
}
|
|
42
|
+
else if (this.providedSchema === undefined) {
|
|
43
|
+
this.updateInferredField(path, value, row, current);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
addField(path, value, row) {
|
|
47
|
+
if (this.providedSchema !== undefined) {
|
|
48
|
+
this.addSchemaField(this.providedSchema, path, row);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
const evidence = this.inferType(value, path) ?? DeferredTypeEvidence.from(value, row);
|
|
52
|
+
if (evidence === undefined) {
|
|
53
|
+
throw typeInferenceError(path, row);
|
|
54
|
+
}
|
|
55
|
+
const conflict = this.fields.set(path, evidence, (existing) => existing instanceof DeferredTypeEvidence && existing.isOnlyNulls());
|
|
56
|
+
if (conflict !== undefined) {
|
|
57
|
+
throw branchConflictError(conflict, row, "Struct");
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
addSchemaField(schema, path, row) {
|
|
61
|
+
const field = fieldAtPath(schema, path);
|
|
62
|
+
if (field === undefined) {
|
|
63
|
+
throw new Error(`Found field not in schema: ${path.join(".")} at row ${row}`);
|
|
64
|
+
}
|
|
65
|
+
const conflict = this.fields.set(path, field.type);
|
|
66
|
+
if (conflict !== undefined) {
|
|
67
|
+
throw branchConflictError(conflict, row, "Struct");
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
updateInferredField(path, value, row, current) {
|
|
71
|
+
const newType = this.inferType(value, path);
|
|
72
|
+
const deferred = DeferredTypeEvidence.from(value, row);
|
|
73
|
+
if (current instanceof FieldTree) {
|
|
74
|
+
if (deferred?.isOnlyNulls()) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
throw schemaInferenceError(path, row, "Struct", describeEvidence(newType ?? deferred));
|
|
78
|
+
}
|
|
79
|
+
if (current instanceof DeferredTypeEvidence) {
|
|
80
|
+
this.resolveDeferredField(path, row, current, newType, deferred);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
if (newType !== undefined) {
|
|
84
|
+
if (!inferredTypesEqual(current, newType)) {
|
|
85
|
+
throw schemaInferenceError(path, row, describeEvidence(current), describeEvidence(newType));
|
|
86
|
+
}
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
if (deferred === undefined || !deferred.matches(current)) {
|
|
90
|
+
throw schemaInferenceError(path, row, describeEvidence(current), describeEvidence(deferred));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
resolveDeferredField(path, row, current, newType, deferred) {
|
|
94
|
+
if (newType !== undefined) {
|
|
95
|
+
if (!current.matches(newType)) {
|
|
96
|
+
throw schemaInferenceError(path, row, current.describe(), describeEvidence(newType));
|
|
97
|
+
}
|
|
98
|
+
this.fields.set(path, newType);
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
if (deferred !== undefined) {
|
|
102
|
+
this.fields.set(path, current.merge(deferred));
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
throw schemaInferenceError(path, row, current.describe(), describeEvidence(newType));
|
|
106
|
+
}
|
|
107
|
+
inferType(value, path) {
|
|
108
|
+
if (typeof value === "bigint") {
|
|
109
|
+
return new apache_arrow_1.Int64();
|
|
110
|
+
}
|
|
111
|
+
if (typeof value === "number") {
|
|
112
|
+
return new apache_arrow_1.Float64();
|
|
113
|
+
}
|
|
114
|
+
if (typeof value === "string") {
|
|
115
|
+
return this.options.dictionaryEncodeStrings
|
|
116
|
+
? new apache_arrow_1.Dictionary(new apache_arrow_1.Utf8(), new apache_arrow_1.Int32())
|
|
117
|
+
: new apache_arrow_1.Utf8();
|
|
118
|
+
}
|
|
119
|
+
if (typeof value === "boolean") {
|
|
120
|
+
return new apache_arrow_1.Bool();
|
|
121
|
+
}
|
|
122
|
+
if (value instanceof Buffer) {
|
|
123
|
+
return new apache_arrow_1.Binary();
|
|
124
|
+
}
|
|
125
|
+
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
|
126
|
+
const typedArray = (0, arrow_type_1.typedArrayToArrowType)(value);
|
|
127
|
+
return typedArray === undefined
|
|
128
|
+
? undefined
|
|
129
|
+
: new apache_arrow_1.FixedSizeList(typedArray.length, new apache_arrow_1.Field("item", typedArray.elementType, true));
|
|
130
|
+
}
|
|
131
|
+
if (!Array.isArray(value) || value.length === 0) {
|
|
132
|
+
return undefined;
|
|
133
|
+
}
|
|
134
|
+
const configuredVector = path.length === 1 ? this.options.vectorColumns[path[0]] : undefined;
|
|
135
|
+
if (configuredVector !== undefined) {
|
|
136
|
+
return new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", (0, sanitize_1.sanitizeType)(configuredVector.type), true));
|
|
137
|
+
}
|
|
138
|
+
const itemType = this.inferArrayItemType(value, path);
|
|
139
|
+
if (itemType === undefined) {
|
|
140
|
+
return undefined;
|
|
141
|
+
}
|
|
142
|
+
return nameSuggestsVectorColumn(path[path.length - 1])
|
|
143
|
+
? new apache_arrow_1.FixedSizeList(value.length, new apache_arrow_1.Field("item", new apache_arrow_1.Float32(), true))
|
|
144
|
+
: new apache_arrow_1.List(new apache_arrow_1.Field("item", itemType, true));
|
|
145
|
+
}
|
|
146
|
+
inferArrayItemType(values, path) {
|
|
147
|
+
let itemType;
|
|
148
|
+
const deferredItems = [];
|
|
149
|
+
for (const value of values) {
|
|
150
|
+
const candidate = this.inferType(value, path);
|
|
151
|
+
if (candidate === undefined) {
|
|
152
|
+
if (!isDeferredValue(value)) {
|
|
153
|
+
return undefined;
|
|
154
|
+
}
|
|
155
|
+
deferredItems.push(value);
|
|
156
|
+
}
|
|
157
|
+
else if (itemType === undefined) {
|
|
158
|
+
itemType = candidate;
|
|
159
|
+
}
|
|
160
|
+
else if (!inferredTypesEqual(itemType, candidate)) {
|
|
161
|
+
return undefined;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (itemType === undefined) {
|
|
165
|
+
return undefined;
|
|
166
|
+
}
|
|
167
|
+
return deferredItems.every((value) => deferredValueMatchesType(value, itemType))
|
|
168
|
+
? itemType
|
|
169
|
+
: undefined;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
/** Nulls and empty/all-null lists that do not determine a type by themselves. */
|
|
173
|
+
class DeferredTypeEvidence {
|
|
174
|
+
values;
|
|
175
|
+
constructor(values) {
|
|
176
|
+
this.values = values;
|
|
177
|
+
}
|
|
178
|
+
static from(value, row) {
|
|
179
|
+
return isDeferredValue(value)
|
|
180
|
+
? new DeferredTypeEvidence([{ value, row }])
|
|
181
|
+
: undefined;
|
|
182
|
+
}
|
|
183
|
+
isOnlyNulls() {
|
|
184
|
+
return this.values.every(({ value }) => value == null);
|
|
185
|
+
}
|
|
186
|
+
matches(type) {
|
|
187
|
+
return this.values.every(({ value }) => deferredValueMatchesType(value, type));
|
|
188
|
+
}
|
|
189
|
+
merge(other) {
|
|
190
|
+
return new DeferredTypeEvidence([...this.values, ...other.values]);
|
|
191
|
+
}
|
|
192
|
+
describe() {
|
|
193
|
+
const list = this.values.find(({ value }) => Array.isArray(value));
|
|
194
|
+
return list === undefined
|
|
195
|
+
? "null"
|
|
196
|
+
: `List[${list.value.length}]`;
|
|
197
|
+
}
|
|
198
|
+
firstRow() {
|
|
199
|
+
return this.values[0].row;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
/** Nested field state, kept separate from Arrow's eventual Struct types. */
|
|
203
|
+
class FieldTree {
|
|
204
|
+
children = new Map();
|
|
205
|
+
get(path) {
|
|
206
|
+
let current = this;
|
|
207
|
+
for (const part of path) {
|
|
208
|
+
if (!(current instanceof FieldTree)) {
|
|
209
|
+
return undefined;
|
|
210
|
+
}
|
|
211
|
+
const child = current.children.get(part);
|
|
212
|
+
if (child === undefined) {
|
|
213
|
+
return undefined;
|
|
214
|
+
}
|
|
215
|
+
current = child;
|
|
216
|
+
}
|
|
217
|
+
return current;
|
|
218
|
+
}
|
|
219
|
+
set(path, value, canReplaceLeaf = () => false) {
|
|
220
|
+
let branch = this;
|
|
221
|
+
for (const [index, part] of path.slice(0, -1).entries()) {
|
|
222
|
+
const child = branch.children.get(part);
|
|
223
|
+
if (child === undefined || (isLeaf(child) && canReplaceLeaf(child))) {
|
|
224
|
+
const nextBranch = new FieldTree();
|
|
225
|
+
branch.children.set(part, nextBranch);
|
|
226
|
+
branch = nextBranch;
|
|
227
|
+
}
|
|
228
|
+
else if (child instanceof FieldTree) {
|
|
229
|
+
branch = child;
|
|
230
|
+
}
|
|
231
|
+
else {
|
|
232
|
+
return { path: path.slice(0, index + 1), value: child };
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
const name = path[path.length - 1];
|
|
236
|
+
const current = branch.children.get(name);
|
|
237
|
+
if (current instanceof FieldTree) {
|
|
238
|
+
return { path, value: current };
|
|
239
|
+
}
|
|
240
|
+
branch.children.set(name, value);
|
|
241
|
+
return undefined;
|
|
242
|
+
}
|
|
243
|
+
entries() {
|
|
244
|
+
return this.children.entries();
|
|
245
|
+
}
|
|
246
|
+
has(name) {
|
|
247
|
+
return this.children.has(name);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
function isLeaf(value) {
|
|
251
|
+
return !(value instanceof FieldTree);
|
|
252
|
+
}
|
|
253
|
+
function fieldsFromTree(tree, path = []) {
|
|
254
|
+
const fields = [];
|
|
255
|
+
for (const [name, value] of tree.entries()) {
|
|
256
|
+
if (value instanceof FieldTree) {
|
|
257
|
+
fields.push(new apache_arrow_1.Field(name, new apache_arrow_1.Struct(fieldsFromTree(value, [...path, name])), true));
|
|
258
|
+
}
|
|
259
|
+
else if (value instanceof DeferredTypeEvidence) {
|
|
260
|
+
throw typeInferenceError([...path, name], value.firstRow());
|
|
261
|
+
}
|
|
262
|
+
else {
|
|
263
|
+
fields.push(new apache_arrow_1.Field(name, value, true));
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
return fields;
|
|
267
|
+
}
|
|
268
|
+
function matchingFields(fields, tree) {
|
|
269
|
+
const matches = [];
|
|
270
|
+
for (const field of fields) {
|
|
271
|
+
if (!tree.has(field.name)) {
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
274
|
+
const value = tree.get([field.name]);
|
|
275
|
+
if (value instanceof FieldTree) {
|
|
276
|
+
const struct = field.type;
|
|
277
|
+
matches.push(new apache_arrow_1.Field(field.name, new apache_arrow_1.Struct(matchingFields(struct.children, value)), field.nullable, field.metadata));
|
|
278
|
+
}
|
|
279
|
+
else {
|
|
280
|
+
matches.push(field);
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return matches;
|
|
284
|
+
}
|
|
285
|
+
function* recordPathsAndValues(record, path = []) {
|
|
286
|
+
for (const [name, value] of Object.entries(record)) {
|
|
287
|
+
if (isRecord(value)) {
|
|
288
|
+
yield* recordPathsAndValues(value, [...path, name]);
|
|
289
|
+
}
|
|
290
|
+
else if (value !== undefined) {
|
|
291
|
+
yield [[...path, name], value];
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
function isRecord(value) {
|
|
296
|
+
return (typeof value === "object" &&
|
|
297
|
+
value !== null &&
|
|
298
|
+
!Array.isArray(value) &&
|
|
299
|
+
!(value instanceof RegExp) &&
|
|
300
|
+
!(value instanceof Date) &&
|
|
301
|
+
!(value instanceof Set) &&
|
|
302
|
+
!(value instanceof Map) &&
|
|
303
|
+
!(value instanceof Buffer) &&
|
|
304
|
+
!ArrayBuffer.isView(value));
|
|
305
|
+
}
|
|
306
|
+
function fieldAtPath(schema, path) {
|
|
307
|
+
let fields = schema.fields;
|
|
308
|
+
let field;
|
|
309
|
+
for (const [index, name] of path.entries()) {
|
|
310
|
+
field = fields.find((candidate) => candidate.name === name);
|
|
311
|
+
if (field === undefined || index === path.length - 1) {
|
|
312
|
+
return field;
|
|
313
|
+
}
|
|
314
|
+
if (!apache_arrow_1.DataType.isStruct(field.type)) {
|
|
315
|
+
return undefined;
|
|
316
|
+
}
|
|
317
|
+
fields = field.type.children;
|
|
318
|
+
}
|
|
319
|
+
return field;
|
|
320
|
+
}
|
|
321
|
+
function isDeferredValue(value) {
|
|
322
|
+
return (value == null || (Array.isArray(value) && value.every(isDeferredValue)));
|
|
323
|
+
}
|
|
324
|
+
function deferredValueMatchesType(value, type) {
|
|
325
|
+
if (value == null) {
|
|
326
|
+
return true;
|
|
327
|
+
}
|
|
328
|
+
if (!Array.isArray(value)) {
|
|
329
|
+
return false;
|
|
330
|
+
}
|
|
331
|
+
if (apache_arrow_1.DataType.isList(type)) {
|
|
332
|
+
return value.every((item) => deferredValueMatchesType(item, type.valueType));
|
|
333
|
+
}
|
|
334
|
+
if (apache_arrow_1.DataType.isFixedSizeList(type)) {
|
|
335
|
+
return (value.length === type.listSize &&
|
|
336
|
+
value.every((item) => deferredValueMatchesType(item, type.valueType)));
|
|
337
|
+
}
|
|
338
|
+
return false;
|
|
339
|
+
}
|
|
340
|
+
function inferredTypesEqual(current, candidate) {
|
|
341
|
+
if (apache_arrow_1.DataType.isDictionary(current)) {
|
|
342
|
+
return (apache_arrow_1.DataType.isDictionary(candidate) &&
|
|
343
|
+
current.isOrdered === candidate.isOrdered &&
|
|
344
|
+
inferredTypesEqual(current.indices, candidate.indices) &&
|
|
345
|
+
inferredTypesEqual(current.dictionary, candidate.dictionary));
|
|
346
|
+
}
|
|
347
|
+
if (apache_arrow_1.DataType.isList(current)) {
|
|
348
|
+
return (apache_arrow_1.DataType.isList(candidate) &&
|
|
349
|
+
current.valueField.name === candidate.valueField.name &&
|
|
350
|
+
current.valueField.nullable === candidate.valueField.nullable &&
|
|
351
|
+
inferredTypesEqual(current.valueType, candidate.valueType));
|
|
352
|
+
}
|
|
353
|
+
if (apache_arrow_1.DataType.isFixedSizeList(current)) {
|
|
354
|
+
return (apache_arrow_1.DataType.isFixedSizeList(candidate) &&
|
|
355
|
+
current.listSize === candidate.listSize &&
|
|
356
|
+
current.valueField.name === candidate.valueField.name &&
|
|
357
|
+
current.valueField.nullable === candidate.valueField.nullable &&
|
|
358
|
+
inferredTypesEqual(current.valueType, candidate.valueType));
|
|
359
|
+
}
|
|
360
|
+
return apache_arrow_1.util.compareTypes(current, candidate);
|
|
361
|
+
}
|
|
362
|
+
function describeEvidence(evidence) {
|
|
363
|
+
if (evidence === undefined) {
|
|
364
|
+
return "an unsupported value";
|
|
365
|
+
}
|
|
366
|
+
return evidence instanceof DeferredTypeEvidence
|
|
367
|
+
? evidence.describe()
|
|
368
|
+
: evidence.toString();
|
|
369
|
+
}
|
|
370
|
+
function branchConflictError(conflict, row, candidate) {
|
|
371
|
+
return schemaInferenceError(conflict.path, row, conflict.value instanceof FieldTree
|
|
372
|
+
? "Struct"
|
|
373
|
+
: describeEvidence(conflict.value), candidate);
|
|
374
|
+
}
|
|
375
|
+
function schemaInferenceError(path, row, currentType, newType) {
|
|
376
|
+
return new Error(`Failed to infer schema for data. Previously inferred type ${currentType} ` +
|
|
377
|
+
`but found ${newType} for field ${path.join(".")} at row ${row}. ` +
|
|
378
|
+
"Consider providing an explicit schema.");
|
|
379
|
+
}
|
|
380
|
+
function typeInferenceError(path, row) {
|
|
381
|
+
return new Error(`Failed to infer data type for field ${path.join(".")} at row ${row}. ` +
|
|
382
|
+
"Consider providing an explicit schema.");
|
|
383
|
+
}
|
|
384
|
+
function nameSuggestsVectorColumn(name) {
|
|
385
|
+
const normalized = name.toLowerCase();
|
|
386
|
+
return normalized.includes("vector") || normalized.includes("embedding");
|
|
387
|
+
}
|
package/dist/table.d.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { Table as ArrowTable, Data, DataType, Field, IntoVector, MultiVector, Schema } from "./arrow";
|
|
2
2
|
import { IndexOptions } from "./indices";
|
|
3
3
|
import { MergeInsertBuilder } from "./merge";
|
|
4
|
-
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, Branches as NativeBranches, OptimizeStats, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
5
|
-
import { FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
4
|
+
import { AddColumnsResult, AddColumnsSql, AddResult, AlterColumnsResult, BranchContents, DeleteResult, DropColumnsResult, IndexConfig, IndexStatistics, Job, LsmStats, Branches as NativeBranches, OptimizeStats, RefreshColumnResult, RefreshMaterializedViewResult, TableStatistics, Tags, UpdateFieldMetadataResult, UpdateResult, Table as _NativeTable } from "./native";
|
|
5
|
+
import { AutoQuery, FullTextQuery, Query, TakeQuery, VectorQuery } from "./query";
|
|
6
6
|
import { IntoSql } from "./util";
|
|
7
7
|
export { IndexConfig } from "./native";
|
|
8
|
+
export { BucketStats, GenerationStats, LsmStats, MemtableStats, } from "./native";
|
|
8
9
|
/**
|
|
9
10
|
* Progress snapshot for a write operation, delivered to the `progress`
|
|
10
11
|
* callback passed to {@link Table.add}.
|
|
@@ -421,7 +422,7 @@ export declare abstract class Table {
|
|
|
421
422
|
* when "auto" is used, if the query is a string and an embedding function is defined, it will be treated as a vector query
|
|
422
423
|
* if the query is a string and no embedding function is defined, it will be treated as a full text search query
|
|
423
424
|
*/
|
|
424
|
-
abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
|
|
425
|
+
abstract search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
|
|
425
426
|
/**
|
|
426
427
|
* Search the table with a given query vector.
|
|
427
428
|
*
|
|
@@ -432,15 +433,75 @@ export declare abstract class Table {
|
|
|
432
433
|
abstract vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
|
|
433
434
|
/**
|
|
434
435
|
* Add new columns with defined values.
|
|
436
|
+
*
|
|
437
|
+
* The `{ computed }` form stores the expression rather than evaluating it
|
|
438
|
+
* now: the column is committed with no values, and rows get them from
|
|
439
|
+
* {@link Table#refreshColumn}. Declaring one therefore costs the same on a
|
|
440
|
+
* large table as on an empty one.
|
|
441
|
+
*
|
|
442
|
+
* A refresh does not revisit rows it has already filled, so mutating an
|
|
443
|
+
* input leaves the value computed at fill time; recomputing means dropping
|
|
444
|
+
* the column and declaring it again. While a declaration reads a column,
|
|
445
|
+
* that column cannot be renamed, retyped or dropped.
|
|
446
|
+
*
|
|
447
|
+
* On LanceDB Cloud and Enterprise the expression is planned by the
|
|
448
|
+
* server, and the refresh runs as a server job -- see
|
|
449
|
+
* {@link Table#refreshColumnAsync}.
|
|
435
450
|
* @param {AddColumnsSql[] | Field | Field[] | Schema} newColumnTransforms Either:
|
|
436
451
|
* - An array of objects with column names and SQL expressions to calculate values
|
|
437
452
|
* - A single Arrow Field defining one column with its data type (column will be initialized with null values)
|
|
438
453
|
* - An array of Arrow Fields defining columns with their data types (columns will be initialized with null values)
|
|
439
454
|
* - An Arrow Schema defining columns with their data types (columns will be initialized with null values)
|
|
455
|
+
* - `{ computed }`, declaring columns defined by a SQL expression whose type and inputs are derived from it
|
|
440
456
|
* @returns {Promise<AddColumnsResult>} A promise that resolves to an object
|
|
441
457
|
* containing the new version number of the table after adding the columns.
|
|
458
|
+
* @example
|
|
459
|
+
* ```ts
|
|
460
|
+
* await table.addColumns({ computed: [{ name: "doubled", valueSql: "x * 2" }] });
|
|
461
|
+
* const { rowsFilled } = await table.refreshColumn("doubled");
|
|
462
|
+
* ```
|
|
463
|
+
*/
|
|
464
|
+
abstract addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
|
|
465
|
+
computed: AddColumnsSql[];
|
|
466
|
+
}): Promise<AddColumnsResult>;
|
|
467
|
+
/**
|
|
468
|
+
* Fill the rows of a computed column that hold no value yet.
|
|
469
|
+
*
|
|
470
|
+
* Rows appended since the last refresh are filled by the next one; rows
|
|
471
|
+
* already filled are left as they are, so the call is idempotent and does
|
|
472
|
+
* not observe a mutated input. Local tables only: a remote refresh runs
|
|
473
|
+
* as a server job, through {@link Table#refreshColumnAsync}.
|
|
474
|
+
* @param {string} column The name of the computed column to fill.
|
|
475
|
+
* @returns {Promise<RefreshColumnResult>} A promise that resolves to the
|
|
476
|
+
* number of rows filled and the new version number of the table.
|
|
442
477
|
*/
|
|
443
|
-
abstract
|
|
478
|
+
abstract refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
479
|
+
/**
|
|
480
|
+
* Like {@link Table#refreshColumn}, but returns a handle to the refresh
|
|
481
|
+
* job instead of blocking until it completes.
|
|
482
|
+
*
|
|
483
|
+
* The job may already be complete when returned; callers must not assume
|
|
484
|
+
* the column is filled until {@link Job.wait} resolves. Invalid input --
|
|
485
|
+
* an unknown column, or one that is not computed -- rejects here rather
|
|
486
|
+
* than failing the job. On local tables the job runs in-process; on
|
|
487
|
+
* LanceDB Cloud and Enterprise it is the server's backfill job.
|
|
488
|
+
* @param {string} column The name of the computed column to fill.
|
|
489
|
+
* @example
|
|
490
|
+
* ```ts
|
|
491
|
+
* const job = await table.refreshColumnAsync("doubled");
|
|
492
|
+
* await job.wait();
|
|
493
|
+
* console.log(await job.status()); // "finished"
|
|
494
|
+
* ```
|
|
495
|
+
*/
|
|
496
|
+
abstract refreshColumnAsync(column: string): Promise<Job>;
|
|
497
|
+
/**
|
|
498
|
+
* Recompute this table's contents from its materialized-view definition.
|
|
499
|
+
*
|
|
500
|
+
* Plumbing for {@link MaterializedView.refresh}, which is the way to call
|
|
501
|
+
* it: rejects tables that carry no view definition. Local tables only.
|
|
502
|
+
* @ignore
|
|
503
|
+
*/
|
|
504
|
+
abstract refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
|
|
444
505
|
/**
|
|
445
506
|
* Alter the name or nullability of columns.
|
|
446
507
|
* @param {ColumnAlteration[]} columnAlterations One or more alterations to
|
|
@@ -451,6 +512,18 @@ export declare abstract class Table {
|
|
|
451
512
|
abstract alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
|
|
452
513
|
/**
|
|
453
514
|
* Update per-field (column) metadata.
|
|
515
|
+
*
|
|
516
|
+
* The following keys are treated specially, by convention, and should be
|
|
517
|
+
* used when appropriate:
|
|
518
|
+
*
|
|
519
|
+
* - `lancedb:description`: for a human-readable description of a field.
|
|
520
|
+
* - `lancedb:tag:<name>`: for a user-defined key-value tag, where the suffix
|
|
521
|
+
* names the tag category; e.g. `lancedb:tag:model: "clip"`.
|
|
522
|
+
* - `lancedb:logical-column`: for a column grouping; e.g. `feature_v1` and
|
|
523
|
+
* `feature_v2` might be in the same logical column.
|
|
524
|
+
* - `lancedb:status`: for status options (`production`, `candidate`,
|
|
525
|
+
* `deprecated`, `archived`) to designate the current life cycle state of
|
|
526
|
+
* this column.
|
|
454
527
|
* @param {FieldMetadataUpdate[]} updates One or more per-field updates. Each
|
|
455
528
|
* update's metadata is merged into the field's existing metadata by default;
|
|
456
529
|
* a value of `null` deletes that key, and `replace: true` swaps the whole map.
|
|
@@ -546,6 +619,57 @@ export declare abstract class Table {
|
|
|
546
619
|
* @returns {Promise<void>}
|
|
547
620
|
*/
|
|
548
621
|
abstract closeLsmWriters(): Promise<void>;
|
|
622
|
+
/**
|
|
623
|
+
* Seal every bucket's active memtable into a new L0 generation.
|
|
624
|
+
*
|
|
625
|
+
* Returns once the seal is committed. Sealing an empty memtable is a no-op,
|
|
626
|
+
* so this is safe to call repeatedly.
|
|
627
|
+
* @returns {Promise<void>}
|
|
628
|
+
*/
|
|
629
|
+
abstract flushLsm(): Promise<void>;
|
|
630
|
+
/**
|
|
631
|
+
* Trigger a background L0 → base compaction pass per bucket.
|
|
632
|
+
*
|
|
633
|
+
* Returns once the passes are *dispatched*, not once they finish — watch
|
|
634
|
+
* {@link Table#getLsmStats} for progress, or use
|
|
635
|
+
* {@link Table#checkpointLsm} to wait for convergence.
|
|
636
|
+
* @returns {Promise<void>}
|
|
637
|
+
*/
|
|
638
|
+
abstract compactLsm(): Promise<void>;
|
|
639
|
+
/**
|
|
640
|
+
* Converge this table's LSM write path into its base table.
|
|
641
|
+
*
|
|
642
|
+
* Seals once, then triggers compaction and polls until the L0 that existed
|
|
643
|
+
* at the start is gone. The target set is fixed at the start, so
|
|
644
|
+
* generations created *during* the checkpoint are ignored — that is what
|
|
645
|
+
* lets it terminate under write load, and what makes it best-effort: it
|
|
646
|
+
* converges the fresh tier as of some instant. Idempotent, abandonable at
|
|
647
|
+
* any point, and safe to run on a cadence.
|
|
648
|
+
*
|
|
649
|
+
* There is no liveness bound — the compactor pool is shared across tables,
|
|
650
|
+
* so a checkpoint queued behind unrelated work looks exactly like one that
|
|
651
|
+
* is merging. The caller owns the deadline.
|
|
652
|
+
* @returns {Promise<void>}
|
|
653
|
+
* @example
|
|
654
|
+
* ```ts
|
|
655
|
+
* const before = await table.getLsmStats();
|
|
656
|
+
* await table.checkpointLsm();
|
|
657
|
+
* const after = await table.getLsmStats();
|
|
658
|
+
* ```
|
|
659
|
+
*/
|
|
660
|
+
abstract checkpointLsm(): Promise<void>;
|
|
661
|
+
/**
|
|
662
|
+
* Read live per-bucket LSM state.
|
|
663
|
+
*
|
|
664
|
+
* Answers "how far behind is my fresh tier", "which bucket is hot", and
|
|
665
|
+
* "why is my fresh-tier vector search brute-force". Mutates no table state.
|
|
666
|
+
*
|
|
667
|
+
* Resolves to `undefined` only when the LSM write path is not enabled.
|
|
668
|
+
* @param {boolean} includeGenerationRows Also count rows per L0 generation.
|
|
669
|
+
* Off by default because each count opens an uncached Lance dataset.
|
|
670
|
+
* @returns {Promise<LsmStats | undefined>}
|
|
671
|
+
*/
|
|
672
|
+
abstract getLsmStats(includeGenerationRows?: boolean): Promise<LsmStats | undefined>;
|
|
549
673
|
/** Retrieve the version of the table */
|
|
550
674
|
abstract version(): Promise<number>;
|
|
551
675
|
/**
|
|
@@ -727,9 +851,14 @@ export declare class LocalTable extends Table {
|
|
|
727
851
|
takeOffsets(offsets: number[]): TakeQuery;
|
|
728
852
|
takeRowIds(rowIds: readonly (bigint | number)[]): TakeQuery;
|
|
729
853
|
query(): Query;
|
|
730
|
-
search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query;
|
|
854
|
+
search(query: string | IntoVector | MultiVector | FullTextQuery, queryType?: string, ftsColumns?: string | string[]): VectorQuery | Query | AutoQuery;
|
|
731
855
|
vectorSearch(vector: IntoVector | MultiVector): VectorQuery;
|
|
732
|
-
addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema
|
|
856
|
+
addColumns(newColumnTransforms: AddColumnsSql[] | Field | Field[] | Schema | {
|
|
857
|
+
computed: AddColumnsSql[];
|
|
858
|
+
}): Promise<AddColumnsResult>;
|
|
859
|
+
refreshColumn(column: string): Promise<RefreshColumnResult>;
|
|
860
|
+
refreshColumnAsync(column: string): Promise<Job>;
|
|
861
|
+
refreshMaterializedView(full?: boolean, sourceVersion?: number): Promise<RefreshMaterializedViewResult>;
|
|
733
862
|
alterColumns(columnAlterations: ColumnAlteration[]): Promise<AlterColumnsResult>;
|
|
734
863
|
updateFieldMetadata(updates: FieldMetadataUpdate[]): Promise<UpdateFieldMetadataResult>;
|
|
735
864
|
dropColumns(columnNames: string[]): Promise<DropColumnsResult>;
|
|
@@ -738,6 +867,10 @@ export declare class LocalTable extends Table {
|
|
|
738
867
|
unsetLsmWriteSpec(): Promise<void>;
|
|
739
868
|
getLsmWriteSpec(): Promise<LsmWriteSpec | undefined>;
|
|
740
869
|
closeLsmWriters(): Promise<void>;
|
|
870
|
+
flushLsm(): Promise<void>;
|
|
871
|
+
compactLsm(): Promise<void>;
|
|
872
|
+
checkpointLsm(): Promise<void>;
|
|
873
|
+
getLsmStats(includeGenerationRows?: boolean): Promise<LsmStats | undefined>;
|
|
741
874
|
version(): Promise<number>;
|
|
742
875
|
checkout(version: number | string): Promise<void>;
|
|
743
876
|
checkoutLatest(): Promise<void>;
|
|
@@ -820,7 +953,8 @@ export interface FieldMetadataUpdate {
|
|
|
820
953
|
path: string;
|
|
821
954
|
/**
|
|
822
955
|
* Metadata key/value pairs. Merged into the field's existing metadata by
|
|
823
|
-
* default; a value of `null` deletes that key.
|
|
956
|
+
* default; a value of `null` deletes that key. See
|
|
957
|
+
* {@link Table.updateFieldMetadata} for the conventional `lancedb:*` keys.
|
|
824
958
|
*/
|
|
825
959
|
metadata: Record<string, string | null>;
|
|
826
960
|
/** If true, replace the field's entire metadata map instead of merging. */
|
|
@@ -854,8 +988,8 @@ export interface BranchRowCountSummary {
|
|
|
854
988
|
inputsChanged: number;
|
|
855
989
|
deltaAvailable: boolean;
|
|
856
990
|
}
|
|
857
|
-
/** A reason why a
|
|
858
|
-
export interface
|
|
991
|
+
/** A reason why a cherry-pick cannot currently land. */
|
|
992
|
+
export interface CherryPickError {
|
|
859
993
|
code: string;
|
|
860
994
|
message: string;
|
|
861
995
|
}
|
|
@@ -874,18 +1008,17 @@ export interface BranchDiff {
|
|
|
874
1008
|
changedColumns: BranchColumnChange[];
|
|
875
1009
|
addedIndexes: BranchIndexSummary[];
|
|
876
1010
|
removedIndexes: BranchIndexSummary[];
|
|
877
|
-
|
|
878
|
-
mergeBlockers: MergeBlocker[];
|
|
1011
|
+
errors: CherryPickError[];
|
|
879
1012
|
}
|
|
880
|
-
/** Changes that would be, or were, promoted by a
|
|
881
|
-
export interface
|
|
1013
|
+
/** Changes that would be, or were, promoted by a cherry-pick. */
|
|
1014
|
+
export interface CherryPickPreview {
|
|
882
1015
|
promotedColumns: string[];
|
|
883
1016
|
}
|
|
884
|
-
/** Result of previewing or attempting a
|
|
885
|
-
export interface
|
|
886
|
-
status: "ready" | "
|
|
1017
|
+
/** Result of previewing or attempting a cherry-pick. */
|
|
1018
|
+
export interface CherryPickResult {
|
|
1019
|
+
status: "ready" | "failed" | "notImplemented" | "cherryPicked" | "unknown";
|
|
887
1020
|
diff: BranchDiff;
|
|
888
|
-
preview:
|
|
1021
|
+
preview: CherryPickPreview;
|
|
889
1022
|
mainVersionAfter?: number;
|
|
890
1023
|
}
|
|
891
1024
|
/**
|
|
@@ -924,13 +1057,13 @@ export declare class Branches {
|
|
|
924
1057
|
/** Compare a branch against main without modifying either branch. */
|
|
925
1058
|
diff(fromBranch: string): Promise<BranchDiff>;
|
|
926
1059
|
/**
|
|
927
|
-
*
|
|
1060
|
+
* Cherry-pick a branch onto main.
|
|
928
1061
|
*
|
|
929
|
-
* Set `dryRun` to `true` to preview
|
|
930
|
-
* with `status: "
|
|
1062
|
+
* Set `dryRun` to `true` to preview. A failed cherry-pick resolves
|
|
1063
|
+
* with `status: "failed"` instead of throwing.
|
|
931
1064
|
*
|
|
932
|
-
* @param fromBranch Branch to
|
|
933
|
-
* @param dryRun When true, only preview
|
|
1065
|
+
* @param fromBranch Branch to cherry-pick from.
|
|
1066
|
+
* @param dryRun When true, only preview. Defaults to false.
|
|
934
1067
|
*/
|
|
935
|
-
|
|
1068
|
+
cherryPick(fromBranch: string, dryRun?: boolean): Promise<CherryPickResult>;
|
|
936
1069
|
}
|