grepmax 0.26.52 → 0.26.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/NOTICE +1 -1
- package/README.md +1 -1
- package/dist/vendor/lancedb/arrow.d.ts +1 -1
- package/dist/vendor/lancedb/arrow.js +81 -3
- package/dist/vendor/lancedb/blob.d.ts +92 -0
- package/dist/vendor/lancedb/blob.js +165 -0
- package/dist/vendor/lancedb/connection.d.ts +10 -22
- package/dist/vendor/lancedb/connection.js +4 -14
- package/dist/vendor/lancedb/index.d.ts +4 -1
- package/dist/vendor/lancedb/index.js +7 -2
- package/dist/vendor/lancedb/indices.d.ts +3 -3
- package/dist/vendor/lancedb/job.d.ts +93 -0
- package/dist/vendor/lancedb/job.js +163 -0
- package/dist/vendor/lancedb/materialized_view.d.ts +2 -0
- package/dist/vendor/lancedb/materialized_view.js +3 -1
- package/dist/vendor/lancedb/merge.d.ts +1 -1
- package/dist/vendor/lancedb/merge.js +1 -1
- package/dist/vendor/lancedb/native.d.ts +50 -34
- package/dist/vendor/lancedb/native.js +53 -52
- package/dist/vendor/lancedb/provenance.json +2 -2
- package/dist/vendor/lancedb/sanitize.js +1 -1
- package/dist/vendor/lancedb/table.d.ts +47 -11
- package/dist/vendor/lancedb/table.js +36 -25
- package/package.json +9 -9
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.53",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/NOTICE
CHANGED
|
@@ -33,7 +33,7 @@ This software depends on various open-source libraries. See package.json for
|
|
|
33
33
|
a complete list of dependencies and their respective licenses.
|
|
34
34
|
|
|
35
35
|
The distribution includes the unmodified JavaScript runtime from
|
|
36
|
-
@lancedb/lancedb 0.
|
|
36
|
+
@lancedb/lancedb 0.39.0 (https://github.com/lancedb/lancedb), licensed under
|
|
37
37
|
Apache-2.0. Its license and build provenance are included in dist/vendor/lancedb/.
|
|
38
38
|
Its optional provider dependency manifest is not distributed. Platform native
|
|
39
39
|
binaries are installed as separately pinned @lancedb packages.
|
package/README.md
CHANGED
|
@@ -484,7 +484,7 @@ whether you need to act.
|
|
|
484
484
|
|
|
485
485
|
| You see | What it means |
|
|
486
486
|
|---|---|
|
|
487
|
-
| `Optimize panicked ... inverted/builder.rs` | The historical FTS merge defect is fixed in the shipped LanceDB 0.
|
|
487
|
+
| `Optimize panicked ... inverted/builder.rs` | The historical FTS merge defect is fixed in the shipped LanceDB 0.39.0 runtime. If it recurs on a current release, preserve the version and logs and report a regression; the rebuild guard remains a recovery tripwire. |
|
|
488
488
|
| `disabling auto-rebuild until an optimize succeeds` | Transient, not a wedge. The next successful optimize clears it. Search stays available. |
|
|
489
489
|
| `ANN: vector index not built` | Normal — exact search is the default. |
|
|
490
490
|
| `cannot reach the daemon socket from this sandbox` (exit 2) | The shell is sandboxed. Add the two keys in [Running under the Claude Code sandbox](#running-under-the-claude-code-sandbox); `gmax doctor` warns about the same gap. |
|
|
@@ -208,7 +208,7 @@ export declare function makeEmptyTable(schema: SchemaLike, metadata?: Map<string
|
|
|
208
208
|
* customized by the `embeddingDataType` property of the embedding function.
|
|
209
209
|
*
|
|
210
210
|
* If a schema is provided in `makeTableOptions` then it should include the
|
|
211
|
-
* embedding columns. If no schema is
|
|
211
|
+
* embedding columns. If no schema is provided then embedding columns will
|
|
212
212
|
* be placed at the end of the table, after all of the input columns.
|
|
213
213
|
*/
|
|
214
214
|
export declare function convertToTable(data: Array<Record<string, unknown>>, embeddings?: EmbeddingFunctionConfig, makeTableOptions?: Partial<MakeArrowTableOptions>): Promise<ArrowTable>;
|
|
@@ -57,6 +57,7 @@ exports.ensureNestedFieldsExist = ensureNestedFieldsExist;
|
|
|
57
57
|
exports.dataTypeToJson = dataTypeToJson;
|
|
58
58
|
const apache_arrow_1 = require("apache-arrow");
|
|
59
59
|
const arrow_type_1 = require("./arrow_type");
|
|
60
|
+
const blob_1 = require("./blob");
|
|
60
61
|
const registry_1 = require("./embedding/registry");
|
|
61
62
|
const sanitize_1 = require("./sanitize");
|
|
62
63
|
const schema_1 = require("./schema");
|
|
@@ -348,17 +349,39 @@ function makeArrowTable(data, options, metadata) {
|
|
|
348
349
|
}
|
|
349
350
|
else {
|
|
350
351
|
schema = new apache_arrow_1.Schema(schema.fields, schemaMetadata);
|
|
352
|
+
validateBlobSchema(schema);
|
|
351
353
|
return new apache_arrow_1.Table(schema);
|
|
352
354
|
}
|
|
353
355
|
}
|
|
354
356
|
let inferredSchema = (0, schema_1.inferSchema)(data, schema, opt);
|
|
355
357
|
inferredSchema = new apache_arrow_1.Schema(inferredSchema.fields, schemaMetadata);
|
|
358
|
+
validateBlobSchema(inferredSchema);
|
|
356
359
|
const finalColumns = {};
|
|
357
360
|
for (const field of inferredSchema.fields) {
|
|
358
361
|
finalColumns[field.name] = transposeData(data, field);
|
|
359
362
|
}
|
|
360
363
|
return new apache_arrow_1.Table(inferredSchema, finalColumns);
|
|
361
364
|
}
|
|
365
|
+
function validateBlobSchema(schema) {
|
|
366
|
+
for (const field of schema.fields) {
|
|
367
|
+
validateBlobField(field);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
function validateBlobField(field) {
|
|
371
|
+
if (isFixedSizeList(field.type) &&
|
|
372
|
+
containsBlobField(field.type.children[0])) {
|
|
373
|
+
throw new Error("Blob fields inside FixedSizeList are not supported. Use List instead.");
|
|
374
|
+
}
|
|
375
|
+
for (const child of field.type.children ?? []) {
|
|
376
|
+
validateBlobField(child);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
function containsBlobField(field) {
|
|
380
|
+
if ((0, blob_1.isBlobField)(field)) {
|
|
381
|
+
return true;
|
|
382
|
+
}
|
|
383
|
+
return (field.type.children ?? []).some((child) => containsBlobField(child));
|
|
384
|
+
}
|
|
362
385
|
function isObject(value) {
|
|
363
386
|
return (typeof value === "object" &&
|
|
364
387
|
value !== null &&
|
|
@@ -387,6 +410,27 @@ function valueAtPath(datum, path) {
|
|
|
387
410
|
}
|
|
388
411
|
function transposeData(data, field, path = []) {
|
|
389
412
|
const valuesPath = [...path, field.name];
|
|
413
|
+
if ((0, blob_1.isBlobField)(field) && field.type instanceof apache_arrow_1.Struct) {
|
|
414
|
+
const blobRows = data.map((datum) => (0, blob_1.coerceBlobValue)(valueAtPath(datum, valuesPath)));
|
|
415
|
+
const childVectors = field.type.children.map((child) => {
|
|
416
|
+
const values = blobRows.map((row) => row == null ? null : (row[child.name] ?? null));
|
|
417
|
+
return makeVector(values, child.type, undefined, child.nullable);
|
|
418
|
+
});
|
|
419
|
+
const nullCount = blobRows.filter((row) => row === null).length;
|
|
420
|
+
const structData = (0, apache_arrow_1.makeData)({
|
|
421
|
+
type: field.type,
|
|
422
|
+
length: blobRows.length,
|
|
423
|
+
nullCount,
|
|
424
|
+
nullBitmap: nullCount > 0
|
|
425
|
+
? apache_arrow_1.util.packBools(blobRows.map((row) => row !== null))
|
|
426
|
+
: undefined,
|
|
427
|
+
children: childVectors.map((v) => v.data[0]),
|
|
428
|
+
});
|
|
429
|
+
return (0, apache_arrow_1.makeVector)(structData);
|
|
430
|
+
}
|
|
431
|
+
if (isList(field.type) && containsBlobField(field.type.children[0])) {
|
|
432
|
+
return transposeListData(data, field, valuesPath);
|
|
433
|
+
}
|
|
390
434
|
const values = data.map((datum) => valueAtPath(datum, valuesPath));
|
|
391
435
|
if (field.type instanceof apache_arrow_1.Struct) {
|
|
392
436
|
const childFields = field.type.children;
|
|
@@ -401,7 +445,7 @@ function transposeData(data, field, path = []) {
|
|
|
401
445
|
nullBitmap: nullCount > 0
|
|
402
446
|
? apache_arrow_1.util.packBools(values.map((value) => value !== null))
|
|
403
447
|
: undefined,
|
|
404
|
-
children: childVectors,
|
|
448
|
+
children: childVectors.map((v) => v.data[0]),
|
|
405
449
|
});
|
|
406
450
|
return (0, apache_arrow_1.makeVector)(structData);
|
|
407
451
|
}
|
|
@@ -409,6 +453,39 @@ function transposeData(data, field, path = []) {
|
|
|
409
453
|
return makeVector(values, field.type, undefined, field.nullable);
|
|
410
454
|
}
|
|
411
455
|
}
|
|
456
|
+
function transposeListData(data, field, valuesPath) {
|
|
457
|
+
const listType = field.type;
|
|
458
|
+
const childField = listType.children[0];
|
|
459
|
+
const lists = data.map((datum) => valueAtPath(datum, valuesPath));
|
|
460
|
+
const flattened = [];
|
|
461
|
+
const validity = [];
|
|
462
|
+
const offsets = [0];
|
|
463
|
+
for (const list of lists) {
|
|
464
|
+
if (list == null) {
|
|
465
|
+
validity.push(false);
|
|
466
|
+
offsets.push(flattened.length);
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
if (!Array.isArray(list)) {
|
|
470
|
+
throw new Error(`expected an array for list field '${field.name}'`);
|
|
471
|
+
}
|
|
472
|
+
validity.push(true);
|
|
473
|
+
for (const element of list) {
|
|
474
|
+
flattened.push({ [childField.name]: element });
|
|
475
|
+
}
|
|
476
|
+
offsets.push(flattened.length);
|
|
477
|
+
}
|
|
478
|
+
const childVector = transposeData(flattened, childField, []);
|
|
479
|
+
const nullCount = validity.filter((valid) => !valid).length;
|
|
480
|
+
return (0, apache_arrow_1.makeVector)((0, apache_arrow_1.makeData)({
|
|
481
|
+
type: listType,
|
|
482
|
+
length: lists.length,
|
|
483
|
+
nullCount,
|
|
484
|
+
nullBitmap: nullCount > 0 ? apache_arrow_1.util.packBools(validity) : undefined,
|
|
485
|
+
valueOffsets: Int32Array.from(offsets),
|
|
486
|
+
child: childVector.data[0],
|
|
487
|
+
}));
|
|
488
|
+
}
|
|
412
489
|
/**
|
|
413
490
|
* Create an empty Arrow table with the provided schema
|
|
414
491
|
*/
|
|
@@ -497,7 +574,7 @@ function makeVector(values, type, stringAsDictionary, nullable) {
|
|
|
497
574
|
return vectorFromArray(values, type);
|
|
498
575
|
}
|
|
499
576
|
if (values.length === 0) {
|
|
500
|
-
throw Error("makeVector requires at least one value or the type must be
|
|
577
|
+
throw Error("makeVector requires at least one value or the type must be specified");
|
|
501
578
|
}
|
|
502
579
|
const sampleValue = values.find((val) => val !== null && val !== undefined);
|
|
503
580
|
if (sampleValue === undefined) {
|
|
@@ -688,7 +765,7 @@ async function applyEmbeddings(table, embeddings, schema) {
|
|
|
688
765
|
* customized by the `embeddingDataType` property of the embedding function.
|
|
689
766
|
*
|
|
690
767
|
* If a schema is provided in `makeTableOptions` then it should include the
|
|
691
|
-
* embedding columns. If no schema is
|
|
768
|
+
* embedding columns. If no schema is provided then embedding columns will
|
|
692
769
|
* be placed at the end of the table, after all of the input columns.
|
|
693
770
|
*/
|
|
694
771
|
async function convertToTable(data, embeddings, makeTableOptions) {
|
|
@@ -752,6 +829,7 @@ async function fromTableToBuffer(table, embeddings, schema) {
|
|
|
752
829
|
schema = (0, sanitize_1.sanitizeSchema)(schema);
|
|
753
830
|
}
|
|
754
831
|
const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
|
|
832
|
+
validateBlobSchema(tableWithEmbeddings.schema);
|
|
755
833
|
const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(tableWithEmbeddings);
|
|
756
834
|
return Buffer.from(await writer.toUint8Array());
|
|
757
835
|
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { Field } from "apache-arrow";
|
|
2
|
+
import { BlobFile as NativeBlobFile } from "./native";
|
|
3
|
+
export type BlobInput = {
|
|
4
|
+
data: Buffer | Uint8Array | null;
|
|
5
|
+
uri: string | null;
|
|
6
|
+
};
|
|
7
|
+
export type BlobOptions = {
|
|
8
|
+
/** Defaults to true. */
|
|
9
|
+
nullable?: boolean;
|
|
10
|
+
/**
|
|
11
|
+
* Max payload bytes kept inline in the data file. Zero is allowed. Must be a
|
|
12
|
+
* safe integer.
|
|
13
|
+
*/
|
|
14
|
+
inlineSizeThreshold?: number;
|
|
15
|
+
/**
|
|
16
|
+
* Max payload bytes stored in a packed sidecar before a dedicated file. Must
|
|
17
|
+
* be a positive safe integer.
|
|
18
|
+
*/
|
|
19
|
+
dedicatedSizeThreshold?: number;
|
|
20
|
+
/**
|
|
21
|
+
* Max bytes in one packed sidecar before starting another. Must be a positive
|
|
22
|
+
* safe integer.
|
|
23
|
+
*/
|
|
24
|
+
packFileSizeThreshold?: number;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Declares a `lance.blob.v2` column.
|
|
28
|
+
*
|
|
29
|
+
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
30
|
+
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
31
|
+
*
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* import { readFile } from "node:fs/promises";
|
|
35
|
+
* import { Field, Int64, Schema } from "apache-arrow";
|
|
36
|
+
* import { blob, connect } from "@lancedb/lancedb";
|
|
37
|
+
*
|
|
38
|
+
* const db = await connect("./data");
|
|
39
|
+
* const video = await readFile("clip.mp4");
|
|
40
|
+
* const table = await db.createTable(
|
|
41
|
+
* "videos",
|
|
42
|
+
* [{ id: 1n, video }],
|
|
43
|
+
* {
|
|
44
|
+
* schema: new Schema([
|
|
45
|
+
* new Field("id", new Int64()),
|
|
46
|
+
* blob("video"),
|
|
47
|
+
* ]),
|
|
48
|
+
* },
|
|
49
|
+
* );
|
|
50
|
+
*
|
|
51
|
+
* const rows = await table.query().select(["id"]).withRowId().toArray();
|
|
52
|
+
* const rowIds = rows.map((row) => row._rowid as bigint);
|
|
53
|
+
* const bytes = await table.fetchBlobs("video", rowIds);
|
|
54
|
+
*
|
|
55
|
+
* const [handle] = await table.fetchBlobFiles("video", rowIds);
|
|
56
|
+
* const size = handle!.size();
|
|
57
|
+
* const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
|
|
58
|
+
* ```
|
|
59
|
+
*/
|
|
60
|
+
export declare function blob(name: string, options?: BlobOptions): Field;
|
|
61
|
+
/**
|
|
62
|
+
* Checks for the `lance.blob.v2` extension marker. Does not validate the
|
|
63
|
+
* field's storage type.
|
|
64
|
+
*/
|
|
65
|
+
export declare function isBlobField(field: Field): boolean;
|
|
66
|
+
/**
|
|
67
|
+
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
|
|
68
|
+
*
|
|
69
|
+
* @hideconstructor
|
|
70
|
+
*/
|
|
71
|
+
export declare class BlobFile {
|
|
72
|
+
private readonly inner;
|
|
73
|
+
private constructor();
|
|
74
|
+
/** @ignore */
|
|
75
|
+
static fromNative(inner: NativeBlobFile): BlobFile;
|
|
76
|
+
/** Returns the blob size in bytes. */
|
|
77
|
+
size(): bigint;
|
|
78
|
+
/**
|
|
79
|
+
* Reads from the cursor to the end and advances the cursor.
|
|
80
|
+
*
|
|
81
|
+
* A second call returns an empty buffer. {@link BlobFile.readRange} does
|
|
82
|
+
* not move the cursor.
|
|
83
|
+
*/
|
|
84
|
+
read(): Promise<Buffer>;
|
|
85
|
+
/**
|
|
86
|
+
* Reads the half-open byte range `[start, end)`.
|
|
87
|
+
*
|
|
88
|
+
* Fails when `end` is past the blob size. Does not move the cursor.
|
|
89
|
+
*/
|
|
90
|
+
readRange(start: bigint, end: bigint): Promise<Buffer>;
|
|
91
|
+
}
|
|
92
|
+
export declare function coerceBlobValue(value: unknown): BlobInput | null;
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
+
exports.BlobFile = void 0;
|
|
6
|
+
exports.blob = blob;
|
|
7
|
+
exports.isBlobField = isBlobField;
|
|
8
|
+
exports.coerceBlobValue = coerceBlobValue;
|
|
9
|
+
const apache_arrow_1 = require("apache-arrow");
|
|
10
|
+
const native_1 = require("./native");
|
|
11
|
+
const BLOB_V2_EXTENSION_NAME = "lance.blob.v2";
|
|
12
|
+
const INLINE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-inline-size-threshold";
|
|
13
|
+
const DEDICATED_SIZE_THRESHOLD_KEY = "lance-encoding:blob-dedicated-size-threshold";
|
|
14
|
+
const PACK_FILE_SIZE_THRESHOLD_KEY = "lance-encoding:blob-pack-file-size-threshold";
|
|
15
|
+
/**
|
|
16
|
+
* Declares a `lance.blob.v2` column.
|
|
17
|
+
*
|
|
18
|
+
* Query results are descriptors, not payload bytes. Use {@link Table.fetchBlobs}
|
|
19
|
+
* or {@link Table.fetchBlobFiles} to read bytes.
|
|
20
|
+
*
|
|
21
|
+
* @example
|
|
22
|
+
* ```ts
|
|
23
|
+
* import { readFile } from "node:fs/promises";
|
|
24
|
+
* import { Field, Int64, Schema } from "apache-arrow";
|
|
25
|
+
* import { blob, connect } from "@lancedb/lancedb";
|
|
26
|
+
*
|
|
27
|
+
* const db = await connect("./data");
|
|
28
|
+
* const video = await readFile("clip.mp4");
|
|
29
|
+
* const table = await db.createTable(
|
|
30
|
+
* "videos",
|
|
31
|
+
* [{ id: 1n, video }],
|
|
32
|
+
* {
|
|
33
|
+
* schema: new Schema([
|
|
34
|
+
* new Field("id", new Int64()),
|
|
35
|
+
* blob("video"),
|
|
36
|
+
* ]),
|
|
37
|
+
* },
|
|
38
|
+
* );
|
|
39
|
+
*
|
|
40
|
+
* const rows = await table.query().select(["id"]).withRowId().toArray();
|
|
41
|
+
* const rowIds = rows.map((row) => row._rowid as bigint);
|
|
42
|
+
* const bytes = await table.fetchBlobs("video", rowIds);
|
|
43
|
+
*
|
|
44
|
+
* const [handle] = await table.fetchBlobFiles("video", rowIds);
|
|
45
|
+
* const size = handle!.size();
|
|
46
|
+
* const header = await handle!.readRange(0n, size < 65536n ? size : 65536n);
|
|
47
|
+
* ```
|
|
48
|
+
*/
|
|
49
|
+
function blob(name, options = {}) {
|
|
50
|
+
const metadata = new Map([
|
|
51
|
+
["ARROW:extension:name", BLOB_V2_EXTENSION_NAME],
|
|
52
|
+
]);
|
|
53
|
+
setThreshold(metadata, INLINE_SIZE_THRESHOLD_KEY, "inlineSizeThreshold", options.inlineSizeThreshold, 0);
|
|
54
|
+
setThreshold(metadata, DEDICATED_SIZE_THRESHOLD_KEY, "dedicatedSizeThreshold", options.dedicatedSizeThreshold, 1);
|
|
55
|
+
setThreshold(metadata, PACK_FILE_SIZE_THRESHOLD_KEY, "packFileSizeThreshold", options.packFileSizeThreshold, 1);
|
|
56
|
+
return new apache_arrow_1.Field(name, new apache_arrow_1.Struct([
|
|
57
|
+
new apache_arrow_1.Field("data", new apache_arrow_1.LargeBinary(), true),
|
|
58
|
+
new apache_arrow_1.Field("uri", new apache_arrow_1.Utf8(), true),
|
|
59
|
+
]), options.nullable ?? true, metadata);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Checks for the `lance.blob.v2` extension marker. Does not validate the
|
|
63
|
+
* field's storage type.
|
|
64
|
+
*/
|
|
65
|
+
function isBlobField(field) {
|
|
66
|
+
return field.metadata?.get("ARROW:extension:name") === BLOB_V2_EXTENSION_NAME;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* A lazy handle to blob bytes. Create one with {@link Table.fetchBlobFiles}.
|
|
70
|
+
*
|
|
71
|
+
* @hideconstructor
|
|
72
|
+
*/
|
|
73
|
+
class BlobFile {
|
|
74
|
+
inner;
|
|
75
|
+
constructor(inner) {
|
|
76
|
+
if (!(inner instanceof native_1.BlobFile)) {
|
|
77
|
+
throw new Error("BlobFile handles come from Table.fetchBlobFiles");
|
|
78
|
+
}
|
|
79
|
+
this.inner = inner;
|
|
80
|
+
}
|
|
81
|
+
/** @ignore */
|
|
82
|
+
static fromNative(inner) {
|
|
83
|
+
return new BlobFile(inner);
|
|
84
|
+
}
|
|
85
|
+
/** Returns the blob size in bytes. */
|
|
86
|
+
size() {
|
|
87
|
+
return this.inner.size();
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Reads from the cursor to the end and advances the cursor.
|
|
91
|
+
*
|
|
92
|
+
* A second call returns an empty buffer. {@link BlobFile.readRange} does
|
|
93
|
+
* not move the cursor.
|
|
94
|
+
*/
|
|
95
|
+
read() {
|
|
96
|
+
return this.inner.read();
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* Reads the half-open byte range `[start, end)`.
|
|
100
|
+
*
|
|
101
|
+
* Fails when `end` is past the blob size. Does not move the cursor.
|
|
102
|
+
*/
|
|
103
|
+
readRange(start, end) {
|
|
104
|
+
return this.inner.readRange(start, end);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
exports.BlobFile = BlobFile;
|
|
108
|
+
function coerceBlobValue(value) {
|
|
109
|
+
if (value == null) {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
if (isBlobBytes(value)) {
|
|
113
|
+
return { data: value, uri: null };
|
|
114
|
+
}
|
|
115
|
+
if (ArrayBuffer.isView(value)) {
|
|
116
|
+
throw new Error("Blob data must be Buffer or Uint8Array");
|
|
117
|
+
}
|
|
118
|
+
if (typeof value === "string") {
|
|
119
|
+
if (value === "") {
|
|
120
|
+
throw new Error("Blob uri cannot be empty");
|
|
121
|
+
}
|
|
122
|
+
return { data: null, uri: value };
|
|
123
|
+
}
|
|
124
|
+
if (typeof value === "object") {
|
|
125
|
+
const record = value;
|
|
126
|
+
if (!("data" in record) && !("uri" in record)) {
|
|
127
|
+
throw new Error("Blob struct values must include a 'data' or 'uri' field");
|
|
128
|
+
}
|
|
129
|
+
const uri = record.uri;
|
|
130
|
+
if (uri === "") {
|
|
131
|
+
throw new Error("Blob uri cannot be empty");
|
|
132
|
+
}
|
|
133
|
+
if (uri != null && typeof uri !== "string") {
|
|
134
|
+
throw new Error(`Blob uri must be a string or null, got ${typeof uri}`);
|
|
135
|
+
}
|
|
136
|
+
const data = record.data;
|
|
137
|
+
if (data != null && !isBlobBytes(data)) {
|
|
138
|
+
throw new Error("Blob data must be Buffer, Uint8Array, or null");
|
|
139
|
+
}
|
|
140
|
+
const bytes = data ?? null;
|
|
141
|
+
const uriValue = uri ?? null;
|
|
142
|
+
if ((bytes == null) === (uriValue == null)) {
|
|
143
|
+
throw new Error("Blob struct values must set exactly one of 'data' or 'uri'");
|
|
144
|
+
}
|
|
145
|
+
return { data: bytes, uri: uriValue };
|
|
146
|
+
}
|
|
147
|
+
throw new Error("Blob column values must be Buffer, Uint8Array, a URI string, null, or { data?, uri? }");
|
|
148
|
+
}
|
|
149
|
+
function isBlobBytes(value) {
|
|
150
|
+
return Buffer.isBuffer(value) || value instanceof Uint8Array;
|
|
151
|
+
}
|
|
152
|
+
function setThreshold(metadata, key, optionName, value, minimum) {
|
|
153
|
+
if (value === undefined) {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
if (!Number.isSafeInteger(value)) {
|
|
157
|
+
throw new Error(`${optionName} must be a safe integer`);
|
|
158
|
+
}
|
|
159
|
+
if (value < minimum) {
|
|
160
|
+
throw new Error(minimum <= 0
|
|
161
|
+
? `${optionName} must be non-negative`
|
|
162
|
+
: `${optionName} must be positive`);
|
|
163
|
+
}
|
|
164
|
+
metadata.set(key, String(value));
|
|
165
|
+
}
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { Data, SchemaLike, TableLike } from "./arrow";
|
|
2
|
-
import { Table as ArrowTable } from "./arrow";
|
|
3
2
|
import { EmbeddingFunctionConfig } from "./embedding/registry";
|
|
3
|
+
import { Job } from "./job";
|
|
4
4
|
import { MaterializedView, MaterializedViewSelect } from "./materialized_view";
|
|
5
5
|
import { Connection as LanceDbConnection } from "./native";
|
|
6
|
-
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse,
|
|
6
|
+
import type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, JobInfo, ListNamespacesResponse, ListTablesResponse } from "./native";
|
|
7
7
|
export type { CreateNamespaceResponse, DescribeNamespaceResponse, DropNamespaceResponse, ListNamespacesResponse, ListTablesResponse, };
|
|
8
8
|
import { Table } from "./table";
|
|
9
9
|
export interface CreateTableOptions {
|
|
@@ -428,21 +428,17 @@ export declare abstract class Connection {
|
|
|
428
428
|
*/
|
|
429
429
|
abstract renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
|
|
430
430
|
/**
|
|
431
|
-
*
|
|
431
|
+
* Open a server-side job by id, returning a handle with its record already
|
|
432
|
+
* populated. Rejects when the server has no such job, the way
|
|
433
|
+
* {@link Connection.openTable} does for a missing table.
|
|
432
434
|
*
|
|
433
|
-
* The
|
|
434
|
-
*
|
|
435
|
-
*
|
|
435
|
+
* The returned {@link Job} answers for its own state, specification,
|
|
436
|
+
* result, failure and event history, so there is no separate
|
|
437
|
+
* connection-level call for any of them.
|
|
436
438
|
*/
|
|
437
|
-
abstract
|
|
439
|
+
abstract openJob(jobId: string): Promise<Job>;
|
|
438
440
|
/** List server-side jobs across the database's tables. */
|
|
439
441
|
abstract listJobs(): Promise<JobInfo[]>;
|
|
440
|
-
/**
|
|
441
|
-
* Describe a single server-side job by id.
|
|
442
|
-
*
|
|
443
|
-
* Resolves to `null` when the server has no such job.
|
|
444
|
-
*/
|
|
445
|
-
abstract getJob(jobId: string): Promise<JobDescription | null>;
|
|
446
442
|
/**
|
|
447
443
|
* Request cancellation of a server-side job by id.
|
|
448
444
|
*
|
|
@@ -450,12 +446,6 @@ export declare abstract class Connection {
|
|
|
450
446
|
* such job exists. Cancelling an already-terminal job is a no-op success.
|
|
451
447
|
*/
|
|
452
448
|
abstract cancelJob(jobId: string): Promise<boolean>;
|
|
453
|
-
/**
|
|
454
|
-
* The lifecycle event history of a server-side job, as an Arrow table.
|
|
455
|
-
*
|
|
456
|
-
* Lists history across all jobs when `jobId` is omitted.
|
|
457
|
-
*/
|
|
458
|
-
abstract jobHistory(jobId?: string): Promise<ArrowTable>;
|
|
459
449
|
}
|
|
460
450
|
/** @hideconstructor */
|
|
461
451
|
export declare class LocalConnection extends Connection {
|
|
@@ -496,11 +486,9 @@ export declare class LocalConnection extends Connection {
|
|
|
496
486
|
createNamespace(namespacePath: string[], options?: Partial<CreateNamespaceOptions>): Promise<CreateNamespaceResponse>;
|
|
497
487
|
dropNamespace(namespacePath: string[], options?: Partial<DropNamespaceOptions>): Promise<DropNamespaceResponse>;
|
|
498
488
|
renameTable(currentName: string, newName: string, options?: RenameTableOptions): Promise<void>;
|
|
499
|
-
|
|
489
|
+
openJob(jobId: string): Promise<Job>;
|
|
500
490
|
listJobs(): Promise<JobInfo[]>;
|
|
501
|
-
getJob(jobId: string): Promise<JobDescription | null>;
|
|
502
491
|
cancelJob(jobId: string): Promise<boolean>;
|
|
503
|
-
jobHistory(jobId?: string): Promise<ArrowTable>;
|
|
504
492
|
}
|
|
505
493
|
/**
|
|
506
494
|
* Takes storage options and makes all the keys snake case.
|
|
@@ -4,10 +4,10 @@
|
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
5
|
exports.LocalConnection = exports.Connection = void 0;
|
|
6
6
|
exports.cleanseStorageOptions = cleanseStorageOptions;
|
|
7
|
-
const apache_arrow_1 = require("apache-arrow");
|
|
8
7
|
const arrow_1 = require("./arrow");
|
|
9
8
|
const arrow_2 = require("./arrow");
|
|
10
9
|
const registry_1 = require("./embedding/registry");
|
|
10
|
+
const job_1 = require("./job");
|
|
11
11
|
const materialized_view_1 = require("./materialized_view");
|
|
12
12
|
const sanitize_1 = require("./sanitize");
|
|
13
13
|
const table_1 = require("./table");
|
|
@@ -198,7 +198,7 @@ class LocalConnection extends Connection {
|
|
|
198
198
|
return this.inner.dropTable(name, namespacePath ?? []);
|
|
199
199
|
}
|
|
200
200
|
async dropTableAsync(name, namespacePath) {
|
|
201
|
-
return this.inner.dropTableAsync(name, namespacePath ?? []);
|
|
201
|
+
return new job_1.Job(await this.inner.dropTableAsync(name, namespacePath ?? []));
|
|
202
202
|
}
|
|
203
203
|
async dropAllTables(namespacePath) {
|
|
204
204
|
return this.inner.dropAllTables(namespacePath ?? []);
|
|
@@ -218,25 +218,15 @@ class LocalConnection extends Connection {
|
|
|
218
218
|
async renameTable(currentName, newName, options) {
|
|
219
219
|
return this.inner.renameTable(currentName, newName, options?.namespacePath ?? [], options?.newNamespacePath);
|
|
220
220
|
}
|
|
221
|
-
|
|
222
|
-
return this.inner.
|
|
221
|
+
async openJob(jobId) {
|
|
222
|
+
return new job_1.Job(await this.inner.openJob(jobId));
|
|
223
223
|
}
|
|
224
224
|
async listJobs() {
|
|
225
225
|
return this.inner.listJobs();
|
|
226
226
|
}
|
|
227
|
-
async getJob(jobId) {
|
|
228
|
-
return this.inner.getJob(jobId);
|
|
229
|
-
}
|
|
230
227
|
async cancelJob(jobId) {
|
|
231
228
|
return this.inner.cancelJob(jobId);
|
|
232
229
|
}
|
|
233
|
-
async jobHistory(jobId) {
|
|
234
|
-
const buf = await this.inner.jobHistory(jobId);
|
|
235
|
-
if (buf.length === 0) {
|
|
236
|
-
return new arrow_2.Table();
|
|
237
|
-
}
|
|
238
|
-
return (0, apache_arrow_1.tableFromIPC)(buf);
|
|
239
|
-
}
|
|
240
230
|
}
|
|
241
231
|
exports.LocalConnection = LocalConnection;
|
|
242
232
|
/**
|
|
@@ -8,8 +8,11 @@ export { JsHeaderProvider as NativeJsHeaderProvider } from "./native.js";
|
|
|
8
8
|
export { instrumentLanceDbMetrics } from "./otel";
|
|
9
9
|
export { AddColumnsSql, ConnectionOptions, ConnectNamespaceOptions, IndexStatistics, IndexConfig, ClientConfig, TimeoutConfig, RetryConfig, TlsConfig, OptimizeStats, CompactionStats, RemovalStats, TableStatistics, FragmentStatistics, FragmentSummaryStats, Tags, TagContents, BranchContents, MergeResult, AddResult, AddColumnsResult, RefreshColumnResult, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, DeleteResult, DropColumnsResult, UpdateResult, SplitCalculatedOptions, SplitRandomOptions, SplitHashOptions, SplitSequentialOptions, ShuffleOptions, OAuthConfig as NativeOAuthConfig, } from "./native.js";
|
|
10
10
|
export { makeArrowTable, MakeArrowTableOptions, Data, VectorColumnOptions, } from "./arrow";
|
|
11
|
+
export { blob, isBlobField, BlobFile } from "./blob";
|
|
12
|
+
export type { BlobOptions } from "./blob";
|
|
11
13
|
export { Connection, CreateTableOptions, TableNamesOptions, ListTablesOptions, OpenTableOptions, ListNamespacesOptions, CreateNamespaceOptions, DropNamespaceOptions, ListNamespacesResponse, ListTablesResponse, CreateNamespaceResponse, DropNamespaceResponse, DescribeNamespaceResponse, RenameTableOptions, } from "./connection";
|
|
12
|
-
export {
|
|
14
|
+
export { JobFailureInfo, JobInfo, Session } from "./native.js";
|
|
15
|
+
export { Job, JobEventsOptions } from "./job";
|
|
13
16
|
export { AutoQuery, ExecutableQuery, Query, QueryBase, VectorQuery, TakeQuery, AnalyzePlanDistributedMetrics, QueryExecutionOptions, ColumnOrdering, FullTextSearchOptions, RecordBatchIterator, FullTextQuery, MatchQuery, PhraseQuery, BoostQuery, MultiMatchQuery, BooleanQuery, FullTextQueryType, Operator, Occur, } from "./query";
|
|
14
17
|
export { Index, IndexOptions, IvfPqOptions, IvfRqOptions, IvfFlatOptions, HnswPqOptions, HnswSqOptions, FtsOptions, BaseTokenizer, } from "./indices";
|
|
15
18
|
export { Table, Branches, BranchColumnSummary, BranchColumnChange, BranchIndexSummary, BranchRowCountSummary, CherryPickError, BranchDiff, CherryPickPreview, CherryPickResult, AddDataOptions, UpdateOptions, OptimizeOptions, Version, WriteProgress, FtsToken, TokenizeTableOptions, LsmWriteSpec, LsmStats, BucketStats, GenerationStats, MemtableStats, ColumnAlteration, FieldMetadataUpdate, } from "./table";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// SPDX-License-Identifier: Apache-2.0
|
|
3
3
|
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
|
|
4
4
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
5
|
-
exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthFlowType = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.
|
|
5
|
+
exports.packBits = exports.rerankers = exports.Scannable = exports.PermutationBuilder = exports.permutationBuilder = exports.embedding = exports.MergeInsertBuilder = exports.OAuthFlowType = exports.OAuthHeaderProvider = exports.StaticHeaderProvider = exports.HeaderProvider = exports.Branches = exports.Table = exports.Index = exports.Occur = exports.Operator = exports.FullTextQueryType = exports.BooleanQuery = exports.MultiMatchQuery = exports.BoostQuery = exports.PhraseQuery = exports.MatchQuery = exports.RecordBatchIterator = exports.TakeQuery = exports.VectorQuery = exports.QueryBase = exports.Query = exports.AutoQuery = exports.Job = exports.Session = exports.Connection = exports.BlobFile = exports.isBlobField = exports.blob = exports.VectorColumnOptions = exports.MakeArrowTableOptions = exports.makeArrowTable = exports.BranchContents = exports.TagContents = exports.Tags = exports.instrumentLanceDbMetrics = exports.NativeJsHeaderProvider = exports.MaterializedView = void 0;
|
|
6
6
|
exports.tokenize = tokenize;
|
|
7
7
|
exports.connect = connect;
|
|
8
8
|
exports.connectNamespace = connectNamespace;
|
|
@@ -26,11 +26,16 @@ var arrow_1 = require("./arrow");
|
|
|
26
26
|
Object.defineProperty(exports, "makeArrowTable", { enumerable: true, get: function () { return arrow_1.makeArrowTable; } });
|
|
27
27
|
Object.defineProperty(exports, "MakeArrowTableOptions", { enumerable: true, get: function () { return arrow_1.MakeArrowTableOptions; } });
|
|
28
28
|
Object.defineProperty(exports, "VectorColumnOptions", { enumerable: true, get: function () { return arrow_1.VectorColumnOptions; } });
|
|
29
|
+
var blob_1 = require("./blob");
|
|
30
|
+
Object.defineProperty(exports, "blob", { enumerable: true, get: function () { return blob_1.blob; } });
|
|
31
|
+
Object.defineProperty(exports, "isBlobField", { enumerable: true, get: function () { return blob_1.isBlobField; } });
|
|
32
|
+
Object.defineProperty(exports, "BlobFile", { enumerable: true, get: function () { return blob_1.BlobFile; } });
|
|
29
33
|
var connection_2 = require("./connection");
|
|
30
34
|
Object.defineProperty(exports, "Connection", { enumerable: true, get: function () { return connection_2.Connection; } });
|
|
31
35
|
var native_js_4 = require("./native.js");
|
|
32
|
-
Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return native_js_4.Job; } });
|
|
33
36
|
Object.defineProperty(exports, "Session", { enumerable: true, get: function () { return native_js_4.Session; } });
|
|
37
|
+
var job_1 = require("./job");
|
|
38
|
+
Object.defineProperty(exports, "Job", { enumerable: true, get: function () { return job_1.Job; } });
|
|
34
39
|
var query_1 = require("./query");
|
|
35
40
|
Object.defineProperty(exports, "AutoQuery", { enumerable: true, get: function () { return query_1.AutoQuery; } });
|
|
36
41
|
Object.defineProperty(exports, "Query", { enumerable: true, get: function () { return query_1.Query; } });
|
|
@@ -20,7 +20,7 @@ export interface IvfPqOptions {
|
|
|
20
20
|
* This value controls how much the vector is compressed during the quantization step.
|
|
21
21
|
* The more sub vectors there are the less the vector is compressed. The default is
|
|
22
22
|
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
|
|
23
|
-
* by 16 we use the dimension
|
|
23
|
+
* by 16 we use the dimension divided by 8.
|
|
24
24
|
*
|
|
25
25
|
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
|
|
26
26
|
* us to use efficient SIMD instructions.
|
|
@@ -210,7 +210,7 @@ export interface HnswPqOptions {
|
|
|
210
210
|
* This value controls how much the vector is compressed during the quantization step.
|
|
211
211
|
* The more sub vectors there are the less the vector is compressed. The default is
|
|
212
212
|
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
|
|
213
|
-
* by 16 we use the dimension
|
|
213
|
+
* by 16 we use the dimension divided by 8.
|
|
214
214
|
*
|
|
215
215
|
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
|
|
216
216
|
* us to use efficient SIMD instructions.
|
|
@@ -678,7 +678,7 @@ export interface IndexOptions {
|
|
|
678
678
|
/**
|
|
679
679
|
* Advanced index configuration
|
|
680
680
|
*
|
|
681
|
-
* This option allows you to specify a
|
|
681
|
+
* This option allows you to specify a specific index to create and also
|
|
682
682
|
* allows you to pass in configuration for training the index.
|
|
683
683
|
*
|
|
684
684
|
* See the static methods on Index for details on the various index types.
|