@infino-ai/infino 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +348 -0
- package/infino/index.d.ts +146 -0
- package/infino/index.js +271 -0
- package/infino/native.d.ts +167 -0
- package/infino/native.js +319 -0
- package/package.json +102 -0
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
import * as arrow from "apache-arrow";
|
|
2
|
+
import { IndexSpec } from "./native.js";
|
|
3
|
+
export { IndexSpec };
|
|
4
|
+
/** Infino's build identifier (version + build hash). */
|
|
5
|
+
export declare const BUILDER_ID: string;
|
|
6
|
+
/** Vector distance metric. */
|
|
7
|
+
export type Metric = "cosine" | "l2sq" | "negdot";
|
|
8
|
+
/** Boolean mode for multi-term FTS queries. */
|
|
9
|
+
export type BoolMode = "or" | "and";
|
|
10
|
+
/** A row from a query/search when not materializing to Arrow. */
|
|
11
|
+
export type RowRecord = Record<string, unknown>;
|
|
12
|
+
/** A plain `{ column: type }` schema descriptor for `createTable`. */
|
|
13
|
+
export type SchemaDescriptor = Record<string, string | {
|
|
14
|
+
vector: number;
|
|
15
|
+
}>;
|
|
16
|
+
/** Accepted shapes for `Table.append`. */
|
|
17
|
+
export type AppendData = RowRecord[] | arrow.Table | arrow.RecordBatch | Buffer | Uint8Array;
|
|
18
|
+
/** Storage and cache config the `connect` URI can't carry. All optional. */
|
|
19
|
+
export interface ConnectOptions {
|
|
20
|
+
/** S3-compatible endpoint; requires `region`, `accessKey`, `secretKey`. */
|
|
21
|
+
endpoint?: string;
|
|
22
|
+
region?: string;
|
|
23
|
+
accessKey?: string;
|
|
24
|
+
secretKey?: string;
|
|
25
|
+
/** Local disk-cache directory for remote-backed tables. */
|
|
26
|
+
cacheDir?: string;
|
|
27
|
+
/** Disk-cache budget in bytes. */
|
|
28
|
+
cacheBudgetBytes?: number;
|
|
29
|
+
/** How cold misses are serviced. */
|
|
30
|
+
coldFetchMode?: "hybrid_with_prefetch" | "range_only" | "lazy_foreground_with_background_fill";
|
|
31
|
+
}
|
|
32
|
+
/** Row counts returned by `update` / `delete`. */
|
|
33
|
+
export interface MutationStats {
|
|
34
|
+
/** Rows the predicate matched. */
|
|
35
|
+
matched: number;
|
|
36
|
+
/** Rows tombstoned (removed from the live set). */
|
|
37
|
+
nTombstoned: number;
|
|
38
|
+
/** Matched rows not found in any live segment. */
|
|
39
|
+
nNotFound: number;
|
|
40
|
+
}
|
|
41
|
+
/** Tuning for `optimize`; all fields optional (omitted ⇒ engine default). */
|
|
42
|
+
export interface OptimizeOptions {
|
|
43
|
+
/** Build-time memory budget, in MB. */
|
|
44
|
+
maxMemoryMb?: number;
|
|
45
|
+
/** Only compact superfiles below this fill percent (0–100). */
|
|
46
|
+
minFillPercent?: number;
|
|
47
|
+
/** Target merged-superfile size, in MB. */
|
|
48
|
+
targetSuperfileSizeMb?: number;
|
|
49
|
+
}
|
|
50
|
+
export interface Bm25SearchOptions {
|
|
51
|
+
mode?: BoolMode;
|
|
52
|
+
/** Columns to return, e.g. `["_id", "score"]`; omit for full rows. */
|
|
53
|
+
projection?: string[];
|
|
54
|
+
arrow?: boolean;
|
|
55
|
+
}
|
|
56
|
+
/** Text-predicate filter for `vectorSearch` (a pushdown pre-filter, not a
|
|
57
|
+
* post-filter): kNN ranks only among rows whose FTS-indexed `column` matches
|
|
58
|
+
* `query`. */
|
|
59
|
+
export interface VectorFilter {
|
|
60
|
+
/** FTS-indexed column the predicate applies to. */
|
|
61
|
+
column: string;
|
|
62
|
+
/** Query terms, tokenized by the index tokenizer. */
|
|
63
|
+
query: string;
|
|
64
|
+
/** `"or"` (default) or `"and"`. */
|
|
65
|
+
mode?: BoolMode;
|
|
66
|
+
}
|
|
67
|
+
export interface VectorSearchOptions {
|
|
68
|
+
/** IVF partitions to probe (higher = better recall, more work). */
|
|
69
|
+
nprobe?: number;
|
|
70
|
+
/** Over-fetch multiplier for the exact-rerank stage (higher = better recall). */
|
|
71
|
+
rerankMult?: number;
|
|
72
|
+
projection?: string[];
|
|
73
|
+
arrow?: boolean;
|
|
74
|
+
/** Restrict the kNN to rows matching a text predicate (pushdown pre-filter). */
|
|
75
|
+
filter?: VectorFilter;
|
|
76
|
+
}
|
|
77
|
+
export interface TokenMatchOptions {
|
|
78
|
+
mode?: BoolMode;
|
|
79
|
+
projection?: string[];
|
|
80
|
+
arrow?: boolean;
|
|
81
|
+
}
|
|
82
|
+
export interface MatchOptions {
|
|
83
|
+
projection?: string[];
|
|
84
|
+
arrow?: boolean;
|
|
85
|
+
}
|
|
86
|
+
export interface QueryOptions {
|
|
87
|
+
arrow?: boolean;
|
|
88
|
+
}
|
|
89
|
+
export declare class Table {
|
|
90
|
+
private inner;
|
|
91
|
+
constructor(inner: any);
|
|
92
|
+
/** The table's Arrow schema. */
|
|
93
|
+
schema(): arrow.Schema;
|
|
94
|
+
/**
|
|
95
|
+
* Append rows. Accepts an array of objects, an apache-arrow
|
|
96
|
+
* Table/RecordBatch, or raw Arrow IPC bytes. Durable on return; one
|
|
97
|
+
* append == one commit.
|
|
98
|
+
*/
|
|
99
|
+
append(data: AppendData): void;
|
|
100
|
+
/** Ranked BM25 search; rows as records (or an Arrow `Table`). */
|
|
101
|
+
bm25Search(column: string, query: string, k: number, opts: Bm25SearchOptions & {
|
|
102
|
+
arrow: true;
|
|
103
|
+
}): arrow.Table;
|
|
104
|
+
bm25Search(column: string, query: string, k: number, opts?: Bm25SearchOptions): RowRecord[];
|
|
105
|
+
/** Vector kNN; rows as records (or an Arrow `Table`). */
|
|
106
|
+
vectorSearch(column: string, query: number[] | Float32Array, k: number, opts: VectorSearchOptions & {
|
|
107
|
+
arrow: true;
|
|
108
|
+
}): arrow.Table;
|
|
109
|
+
vectorSearch(column: string, query: number[] | Float32Array, k: number, opts?: VectorSearchOptions): RowRecord[];
|
|
110
|
+
/** Unranked token match; matching rows as records (or an Arrow `Table`). */
|
|
111
|
+
tokenMatch(column: string, query: string, opts: TokenMatchOptions & {
|
|
112
|
+
arrow: true;
|
|
113
|
+
}): arrow.Table;
|
|
114
|
+
tokenMatch(column: string, query: string, opts?: TokenMatchOptions): RowRecord[];
|
|
115
|
+
/** Unranked exact match; matching rows as records (or an Arrow `Table`). */
|
|
116
|
+
exactMatch(column: string, value: string, opts: MatchOptions & {
|
|
117
|
+
arrow: true;
|
|
118
|
+
}): arrow.Table;
|
|
119
|
+
exactMatch(column: string, value: string, opts?: MatchOptions): RowRecord[];
|
|
120
|
+
/** Replace rows matching a SQL predicate (e.g. `"status = 'spam'"`) with
|
|
121
|
+
* `data` (same shapes as `append`), 1:1 — the matched count must equal the
|
|
122
|
+
* replacement-row count. Requires durable storage (not `memory://`). */
|
|
123
|
+
update(predicate: string, data: AppendData): MutationStats;
|
|
124
|
+
/** Delete rows matching a SQL predicate (e.g. `"status = 'spam'"`).
|
|
125
|
+
* Requires durable storage (not `memory://`). */
|
|
126
|
+
delete(predicate: string): MutationStats;
|
|
127
|
+
/** Merge small / underfilled superfiles into larger ones (omit `settings`
|
|
128
|
+
* for engine defaults). */
|
|
129
|
+
optimize(settings?: OptimizeOptions): void;
|
|
130
|
+
}
|
|
131
|
+
export declare class Connection {
|
|
132
|
+
private inner;
|
|
133
|
+
constructor(inner: any);
|
|
134
|
+
/** Create a table from an apache-arrow `Schema` or `{ column: type }`. */
|
|
135
|
+
createTable(name: string, schema: arrow.Schema | SchemaDescriptor | Buffer, indexes: IndexSpec): Table;
|
|
136
|
+
openTable(name: string): Table;
|
|
137
|
+
dropTable(name: string, purge?: boolean): void;
|
|
138
|
+
listTables(): string[];
|
|
139
|
+
/** SQL across the catalog; rows as records (or an Arrow `Table`). */
|
|
140
|
+
querySql(sql: string, opts: QueryOptions & {
|
|
141
|
+
arrow: true;
|
|
142
|
+
}): arrow.Table;
|
|
143
|
+
querySql(sql: string, opts?: QueryOptions): RowRecord[];
|
|
144
|
+
}
|
|
145
|
+
/** Open (or create) a catalog rooted at `uri`. */
|
|
146
|
+
export declare function connect(uri: string, options?: ConnectOptions): Connection;
|
package/infino/index.js
ADDED
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
// SPDX-FileCopyrightText: Copyright The Infino Authors
|
|
4
|
+
//
|
|
5
|
+
// Public Node.js API for infino. Pass arrays of objects (or apache-arrow
|
|
6
|
+
// Tables) in; get plain records out. `{ arrow: true }` on a search or
|
|
7
|
+
// query returns an apache-arrow `Table` instead of records.
|
|
8
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
+
if (k2 === undefined) k2 = k;
|
|
10
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
+
}
|
|
14
|
+
Object.defineProperty(o, k2, desc);
|
|
15
|
+
}) : (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
o[k2] = m[k];
|
|
18
|
+
}));
|
|
19
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
20
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
21
|
+
}) : function(o, v) {
|
|
22
|
+
o["default"] = v;
|
|
23
|
+
});
|
|
24
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
25
|
+
var ownKeys = function(o) {
|
|
26
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
27
|
+
var ar = [];
|
|
28
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
29
|
+
return ar;
|
|
30
|
+
};
|
|
31
|
+
return ownKeys(o);
|
|
32
|
+
};
|
|
33
|
+
return function (mod) {
|
|
34
|
+
if (mod && mod.__esModule) return mod;
|
|
35
|
+
var result = {};
|
|
36
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
37
|
+
__setModuleDefault(result, mod);
|
|
38
|
+
return result;
|
|
39
|
+
};
|
|
40
|
+
})();
|
|
41
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
42
|
+
exports.Connection = exports.Table = exports.BUILDER_ID = exports.IndexSpec = void 0;
|
|
43
|
+
exports.connect = connect;
|
|
44
|
+
const arrow = __importStar(require("apache-arrow"));
|
|
45
|
+
const native_js_1 = require("./native.js");
|
|
46
|
+
Object.defineProperty(exports, "IndexSpec", { enumerable: true, get: function () { return native_js_1.IndexSpec; } });
|
|
47
|
+
/** Infino's build identifier (version + build hash). */
|
|
48
|
+
exports.BUILDER_ID = (0, native_js_1.builderId)();
|
|
49
|
+
const STREAM = "stream";
|
|
50
|
+
// --- Arrow <-> IPC helpers (the boundary this layer hides) ---
|
|
51
|
+
// Rebuild an arrow type in our instance from the consumer's type. When
|
|
52
|
+
// apache-arrow is loaded as two module instances, our `makeData` can't
|
|
53
|
+
// dispatch on the consumer's type object, but its numeric `typeId` reads
|
|
54
|
+
// across instances.
|
|
55
|
+
function nativeTypeFromForeign(t) {
|
|
56
|
+
switch (t.typeId) {
|
|
57
|
+
case arrow.Type.Utf8: return new arrow.Utf8();
|
|
58
|
+
case arrow.Type.LargeUtf8: return new arrow.LargeUtf8();
|
|
59
|
+
case arrow.Type.Bool: return new arrow.Bool();
|
|
60
|
+
case arrow.Type.Int: return new arrow.Int(t.isSigned, t.bitWidth);
|
|
61
|
+
case arrow.Type.Float: return new arrow.Float(t.precision);
|
|
62
|
+
case arrow.Type.FixedSizeList:
|
|
63
|
+
return new arrow.FixedSizeList(t.listSize, new arrow.Field("item", nativeTypeFromForeign(t.children[0].type), true));
|
|
64
|
+
default:
|
|
65
|
+
throw new TypeError(`createTable: unsupported column type (typeId ${t.typeId})`);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
// Build an arrow type from a descriptor value: a type-name string, or
|
|
69
|
+
// `{ vector: dim }` for a FixedSizeList<Float32, dim> column.
|
|
70
|
+
function nativeTypeFromSpec(spec) {
|
|
71
|
+
if (spec && typeof spec === "object" && typeof spec.vector === "number") {
|
|
72
|
+
return new arrow.FixedSizeList(spec.vector, new arrow.Field("item", new arrow.Float32(), true));
|
|
73
|
+
}
|
|
74
|
+
switch (String(spec).toLowerCase()) {
|
|
75
|
+
case "utf8":
|
|
76
|
+
case "string": return new arrow.Utf8();
|
|
77
|
+
case "large_utf8":
|
|
78
|
+
case "largeutf8": return new arrow.LargeUtf8();
|
|
79
|
+
case "bool":
|
|
80
|
+
case "boolean": return new arrow.Bool();
|
|
81
|
+
case "int32": return new arrow.Int32();
|
|
82
|
+
case "int64": return new arrow.Int64();
|
|
83
|
+
case "float32": return new arrow.Float32();
|
|
84
|
+
case "float64":
|
|
85
|
+
case "double": return new arrow.Float64();
|
|
86
|
+
default:
|
|
87
|
+
throw new TypeError(`createTable: unknown column type ${JSON.stringify(spec)}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
// An apache-arrow `Schema`, a plain `{ column: type }` descriptor, or raw
|
|
91
|
+
// IPC bytes -> the IPC the addon's createTable wants (an empty table that
|
|
92
|
+
// carries just the schema). Types are rebuilt in OUR arrow instance.
|
|
93
|
+
function schemaToIpc(schema) {
|
|
94
|
+
if (Buffer.isBuffer(schema))
|
|
95
|
+
return schema;
|
|
96
|
+
if (schema instanceof Uint8Array)
|
|
97
|
+
return Buffer.from(schema);
|
|
98
|
+
let fields;
|
|
99
|
+
if (schema && Array.isArray(schema.fields)) {
|
|
100
|
+
fields = schema.fields.map((f) => new arrow.Field(f.name, nativeTypeFromForeign(f.type), f.nullable));
|
|
101
|
+
}
|
|
102
|
+
else if (schema && typeof schema === "object") {
|
|
103
|
+
fields = Object.entries(schema).map(([name, spec]) => new arrow.Field(name, nativeTypeFromSpec(spec), false));
|
|
104
|
+
}
|
|
105
|
+
else {
|
|
106
|
+
throw new TypeError("createTable: schema must be an apache-arrow Schema or a { column: type } descriptor");
|
|
107
|
+
}
|
|
108
|
+
const nativeSchema = new arrow.Schema(fields);
|
|
109
|
+
const children = fields.map((f) => arrow.makeData({ type: f.type, length: 0 }));
|
|
110
|
+
const structData = arrow.makeData({
|
|
111
|
+
type: new arrow.Struct(fields),
|
|
112
|
+
length: 0,
|
|
113
|
+
nullCount: 0,
|
|
114
|
+
children,
|
|
115
|
+
});
|
|
116
|
+
const empty = new arrow.Table(new arrow.RecordBatch(nativeSchema, structData));
|
|
117
|
+
return Buffer.from(arrow.tableToIPC(empty, STREAM));
|
|
118
|
+
}
|
|
119
|
+
// Build one typed Arrow column from row objects. `vectorFromArray` handles
|
|
120
|
+
// scalars; FixedSizeList<Float32> (vector columns) need the nested Data
|
|
121
|
+
// built by hand. The schema here is ours (from the addon), so its types
|
|
122
|
+
// are same-instance.
|
|
123
|
+
function buildColumn(field, rows) {
|
|
124
|
+
const values = rows.map((r) => r[field.name]);
|
|
125
|
+
const t = field.type;
|
|
126
|
+
if (t && typeof t.listSize === "number") {
|
|
127
|
+
const flat = Float32Array.from(values.flat());
|
|
128
|
+
const child = arrow.makeData({ type: t.children[0].type, length: flat.length, data: flat });
|
|
129
|
+
const data = arrow.makeData({ type: t, length: rows.length, nullCount: 0, child });
|
|
130
|
+
return arrow.makeVector(data);
|
|
131
|
+
}
|
|
132
|
+
return arrow.vectorFromArray(values, field.type);
|
|
133
|
+
}
|
|
134
|
+
// Normalize append input -> IPC bytes. An array of objects, or an
|
|
135
|
+
// apache-arrow Table / RecordBatch (normalized to rows via its own
|
|
136
|
+
// `toArray()`/`toJSON()`); either way the columns are rebuilt in our arrow
|
|
137
|
+
// instance from the declared schema. (We can't feed the consumer's Table
|
|
138
|
+
// straight into our `tableToIPC` — a different module instance isn't
|
|
139
|
+
// recognized.)
|
|
140
|
+
function dataToIpc(data, getSchema) {
|
|
141
|
+
if (Buffer.isBuffer(data))
|
|
142
|
+
return data;
|
|
143
|
+
if (data instanceof Uint8Array)
|
|
144
|
+
return Buffer.from(data);
|
|
145
|
+
let rows;
|
|
146
|
+
const d = data;
|
|
147
|
+
if (Array.isArray(data)) {
|
|
148
|
+
rows = data;
|
|
149
|
+
}
|
|
150
|
+
else if (d && (Array.isArray(d.batches) || (d.schema && typeof d.numRows === "number"))) {
|
|
151
|
+
rows = Array.from(d).map((r) => r.toJSON());
|
|
152
|
+
}
|
|
153
|
+
else {
|
|
154
|
+
throw new TypeError("append: expected an array of objects, an apache-arrow Table / RecordBatch, or an Arrow IPC Buffer");
|
|
155
|
+
}
|
|
156
|
+
const schema = getSchema();
|
|
157
|
+
const cols = {};
|
|
158
|
+
for (const field of schema.fields)
|
|
159
|
+
cols[field.name] = buildColumn(field, rows);
|
|
160
|
+
return Buffer.from(arrow.tableToIPC(new arrow.Table(cols), STREAM));
|
|
161
|
+
}
|
|
162
|
+
// A Decimal128 value renders as a 4×u32 little-endian array in records.
|
|
163
|
+
// Convert (scale-0 -> integer) to a `bigint`, matching token/exact match.
|
|
164
|
+
function decimalToBigInt(words) {
|
|
165
|
+
let v = 0n;
|
|
166
|
+
for (let i = words.length - 1; i >= 0; i--)
|
|
167
|
+
v = (v << 32n) | BigInt(words[i] >>> 0);
|
|
168
|
+
const bits = BigInt(words.length * 32);
|
|
169
|
+
if (v >= 1n << (bits - 1n))
|
|
170
|
+
v -= 1n << bits; // two's-complement sign
|
|
171
|
+
return v;
|
|
172
|
+
}
|
|
173
|
+
// IPC result bytes -> records (default) or an apache-arrow Table. In record
|
|
174
|
+
// form, scale-0 Decimal columns (notably `_id`) become `bigint`.
|
|
175
|
+
function decode(buf, asArrow) {
|
|
176
|
+
const table = arrow.tableFromIPC(buf);
|
|
177
|
+
if (asArrow)
|
|
178
|
+
return table;
|
|
179
|
+
const intCols = table.schema.fields
|
|
180
|
+
.filter((f) => f.type.typeId === arrow.Type.Decimal && f.type.scale === 0)
|
|
181
|
+
.map((f) => f.name);
|
|
182
|
+
return table.toArray().map((row) => {
|
|
183
|
+
const obj = row.toJSON();
|
|
184
|
+
for (const name of intCols) {
|
|
185
|
+
const cell = obj[name];
|
|
186
|
+
if (cell != null && typeof cell !== "bigint")
|
|
187
|
+
obj[name] = decimalToBigInt(cell);
|
|
188
|
+
}
|
|
189
|
+
return obj;
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
// --- friendly handles ---
|
|
193
|
+
class Table {
|
|
194
|
+
inner;
|
|
195
|
+
constructor(inner) {
|
|
196
|
+
this.inner = inner;
|
|
197
|
+
}
|
|
198
|
+
/** The table's Arrow schema. */
|
|
199
|
+
schema() {
|
|
200
|
+
return arrow.tableFromIPC(this.inner.schema()).schema;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Append rows. Accepts an array of objects, an apache-arrow
|
|
204
|
+
* Table/RecordBatch, or raw Arrow IPC bytes. Durable on return; one
|
|
205
|
+
* append == one commit.
|
|
206
|
+
*/
|
|
207
|
+
append(data) {
|
|
208
|
+
this.inner.append(dataToIpc(data, () => this.schema()));
|
|
209
|
+
}
|
|
210
|
+
bm25Search(column, query, k, opts = {}) {
|
|
211
|
+
const buf = this.inner.bm25Search(column, query, k, opts.mode, opts.projection);
|
|
212
|
+
return decode(buf, opts.arrow);
|
|
213
|
+
}
|
|
214
|
+
vectorSearch(column, query, k, opts = {}) {
|
|
215
|
+
const q = query instanceof Float32Array ? query : Float32Array.from(query);
|
|
216
|
+
const buf = this.inner.vectorSearch(column, q, k, opts.nprobe, opts.rerankMult, opts.projection, opts.filter);
|
|
217
|
+
return decode(buf, opts.arrow);
|
|
218
|
+
}
|
|
219
|
+
tokenMatch(column, query, opts = {}) {
|
|
220
|
+
const buf = this.inner.tokenMatch(column, query, opts.mode, opts.projection);
|
|
221
|
+
return decode(buf, opts.arrow);
|
|
222
|
+
}
|
|
223
|
+
exactMatch(column, value, opts = {}) {
|
|
224
|
+
const buf = this.inner.exactMatch(column, value, opts.projection);
|
|
225
|
+
return decode(buf, opts.arrow);
|
|
226
|
+
}
|
|
227
|
+
/** Replace rows matching a SQL predicate (e.g. `"status = 'spam'"`) with
|
|
228
|
+
* `data` (same shapes as `append`), 1:1 — the matched count must equal the
|
|
229
|
+
* replacement-row count. Requires durable storage (not `memory://`). */
|
|
230
|
+
update(predicate, data) {
|
|
231
|
+
return this.inner.update(predicate, dataToIpc(data, () => this.schema()));
|
|
232
|
+
}
|
|
233
|
+
/** Delete rows matching a SQL predicate (e.g. `"status = 'spam'"`).
|
|
234
|
+
* Requires durable storage (not `memory://`). */
|
|
235
|
+
delete(predicate) {
|
|
236
|
+
return this.inner.delete(predicate);
|
|
237
|
+
}
|
|
238
|
+
/** Merge small / underfilled superfiles into larger ones (omit `settings`
|
|
239
|
+
* for engine defaults). */
|
|
240
|
+
optimize(settings) {
|
|
241
|
+
this.inner.optimize(settings);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
exports.Table = Table;
|
|
245
|
+
class Connection {
|
|
246
|
+
inner;
|
|
247
|
+
constructor(inner) {
|
|
248
|
+
this.inner = inner;
|
|
249
|
+
}
|
|
250
|
+
/** Create a table from an apache-arrow `Schema` or `{ column: type }`. */
|
|
251
|
+
createTable(name, schema, indexes) {
|
|
252
|
+
return new Table(this.inner.createTable(name, schemaToIpc(schema), indexes));
|
|
253
|
+
}
|
|
254
|
+
openTable(name) {
|
|
255
|
+
return new Table(this.inner.openTable(name));
|
|
256
|
+
}
|
|
257
|
+
dropTable(name, purge) {
|
|
258
|
+
this.inner.dropTable(name, purge);
|
|
259
|
+
}
|
|
260
|
+
listTables() {
|
|
261
|
+
return this.inner.listTables();
|
|
262
|
+
}
|
|
263
|
+
querySql(sql, opts = {}) {
|
|
264
|
+
return decode(this.inner.querySql(sql), opts.arrow);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
exports.Connection = Connection;
|
|
268
|
+
/** Open (or create) a catalog rooted at `uri`. */
|
|
269
|
+
function connect(uri, options) {
|
|
270
|
+
return new Connection((0, native_js_1.connect)(uri, options));
|
|
271
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/* tslint:disable */
|
|
2
|
+
/* eslint-disable */
|
|
3
|
+
|
|
4
|
+
/* auto-generated by NAPI-RS */
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Storage and cache config the `connect` URI can't carry. All fields are
|
|
8
|
+
* optional; omit for local / `memory://` / ambient-credential S3 with no
|
|
9
|
+
* disk cache.
|
|
10
|
+
*/
|
|
11
|
+
export interface ConnectOptions {
|
|
12
|
+
/** S3-compatible endpoint; requires `region`, `accessKey`, `secretKey`. */
|
|
13
|
+
endpoint?: string
|
|
14
|
+
region?: string
|
|
15
|
+
accessKey?: string
|
|
16
|
+
secretKey?: string
|
|
17
|
+
/** Local disk-cache directory for remote-backed tables. */
|
|
18
|
+
cacheDir?: string
|
|
19
|
+
/** Disk-cache budget in bytes (a JS number; up to 2^53). */
|
|
20
|
+
cacheBudgetBytes?: number
|
|
21
|
+
/**
|
|
22
|
+
* Cold-miss strategy: `"hybrid_with_prefetch"` | `"range_only"` |
|
|
23
|
+
* `"lazy_foreground_with_background_fill"`.
|
|
24
|
+
*/
|
|
25
|
+
coldFetchMode?: string
|
|
26
|
+
}
|
|
27
|
+
/** Tuning for `optimize`; all fields optional (omitted ⇒ engine default). */
|
|
28
|
+
export interface OptimizeOptions {
|
|
29
|
+
/** Build-time memory budget, in MB. */
|
|
30
|
+
maxMemoryMb?: number
|
|
31
|
+
/** Only compact superfiles below this fill percent (0–100). */
|
|
32
|
+
minFillPercent?: number
|
|
33
|
+
/** Target merged-superfile size, in MB. */
|
|
34
|
+
targetSuperfileSizeMb?: number
|
|
35
|
+
}
|
|
36
|
+
/** Row counts from an `update` / `delete`. */
|
|
37
|
+
export interface MutationStats {
|
|
38
|
+
/** Rows the predicate matched. */
|
|
39
|
+
matched: number
|
|
40
|
+
/** Rows tombstoned (removed from the live set). */
|
|
41
|
+
nTombstoned: number
|
|
42
|
+
/** Matched rows that were not found in any live segment. */
|
|
43
|
+
nNotFound: number
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Text-predicate filter for `vectorSearch` — a pushdown pre-filter, not a
|
|
47
|
+
* post-filter: kNN ranks only among rows whose FTS-indexed `column` matches
|
|
48
|
+
* `query`. `mode` is `"or"` (default) or `"and"`.
|
|
49
|
+
*/
|
|
50
|
+
export interface VectorFilter {
|
|
51
|
+
/** FTS-indexed column the predicate applies to. */
|
|
52
|
+
column: string
|
|
53
|
+
/** Query terms, tokenized by the index tokenizer. */
|
|
54
|
+
query: string
|
|
55
|
+
/** Token matching mode: `"or"` (default) or `"and"`. */
|
|
56
|
+
mode?: string
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Open (or create) a catalog rooted at `uri` (local dir, `memory://`, or
|
|
60
|
+
* object-store prefix). S3-compatible static credentials are passed via
|
|
61
|
+
* `options` (the JS-idiomatic form of the Rust `ConnectOptions`).
|
|
62
|
+
*/
|
|
63
|
+
export declare function connect(uri: string, options?: ConnectOptions | undefined | null): Connection
|
|
64
|
+
/**
|
|
65
|
+
* Infino's build identifier (version + build hash) from the core crate.
|
|
66
|
+
* Re-exported on the JS side as the `BUILDER_ID` string constant.
|
|
67
|
+
*/
|
|
68
|
+
export declare function builderId(): string
|
|
69
|
+
/**
|
|
70
|
+
* Declares which columns are full-text (BM25) and which are vector (IVF
|
|
71
|
+
* kNN) indexed. Built fluently:
|
|
72
|
+
* `new IndexSpec().fts("body").vector("emb", 384, 256, "cosine")`.
|
|
73
|
+
*/
|
|
74
|
+
export declare class IndexSpec {
|
|
75
|
+
constructor()
|
|
76
|
+
/** Mark `column` (a UTF-8 string column) as full-text indexed. */
|
|
77
|
+
fts(column: string): IndexSpec
|
|
78
|
+
/**
|
|
79
|
+
* Mark `column` (a `fixed_size_list<float32, dim>`) as vector
|
|
80
|
+
* indexed. `nCent` is the IVF centroid count (size it to the table's
|
|
81
|
+
* scale); `metric` is `"cosine"` / `"l2sq"` / `"negdot"`.
|
|
82
|
+
*/
|
|
83
|
+
vector(column: string, dim: number, nCent: number, metric: string): IndexSpec
|
|
84
|
+
}
|
|
85
|
+
/** A catalog connection. `const db = connect(uri)`. */
|
|
86
|
+
export declare class Connection {
|
|
87
|
+
/**
|
|
88
|
+
* Create a table from an Arrow `Schema` (sent as an IPC `Buffer` —
|
|
89
|
+
* an empty `apache-arrow` table built with the schema) and an
|
|
90
|
+
* `IndexSpec`.
|
|
91
|
+
*/
|
|
92
|
+
createTable(name: string, schema: Buffer, indexes: IndexSpec): Table
|
|
93
|
+
/** Open an existing table by name. */
|
|
94
|
+
openTable(name: string): Table
|
|
95
|
+
/** Drop (unregister) a table. */
|
|
96
|
+
dropTable(name: string, purge?: boolean | undefined | null): void
|
|
97
|
+
/** List the catalog's table names. */
|
|
98
|
+
listTables(): Array<string>
|
|
99
|
+
/**
|
|
100
|
+
* Run SQL across the catalog's tables; returns an Arrow IPC `Buffer`
|
|
101
|
+
* the JS side reads with `tableFromIPC`. Search is available in SQL
|
|
102
|
+
* via the TVFs, e.g.
|
|
103
|
+
* `SELECT _id, score FROM bm25_search('docs', 'body', 'q', 10)`.
|
|
104
|
+
*/
|
|
105
|
+
querySql(sql: string): Buffer
|
|
106
|
+
}
|
|
107
|
+
/** A single-table handle. */
|
|
108
|
+
export declare class Table {
|
|
109
|
+
/**
|
|
110
|
+
* Append data, sent as an Arrow IPC `Buffer` (`tableToIPC` on the JS
|
|
111
|
+
* side). Durable when this returns — one `append` == one commit ==
|
|
112
|
+
* one sealed segment, so batch rows per call. Multi-batch streams are
|
|
113
|
+
* concatenated into one commit; an empty stream is a no-op.
|
|
114
|
+
*/
|
|
115
|
+
append(data: Buffer): void
|
|
116
|
+
/**
|
|
117
|
+
* BM25 search over one FTS column. Returns matching rows as an Arrow
|
|
118
|
+
* IPC `Buffer` (read with `tableFromIPC`). `mode` is `"or"` (default)
|
|
119
|
+
* or `"and"`. `projection` selects the returned columns — pass
|
|
120
|
+
* `["_id", "score"]` for just id + score, or omit for full rows.
|
|
121
|
+
*/
|
|
122
|
+
bm25Search(column: string, query: string, k: number, mode?: string | undefined | null, projection?: Array<string> | undefined | null): Buffer
|
|
123
|
+
/**
|
|
124
|
+
* Vector kNN over one vector column. `query` is a `Float32Array`
|
|
125
|
+
* (crosses by reference — no copy). Returns matching rows as an Arrow
|
|
126
|
+
* IPC `Buffer` (read with `tableFromIPC`). `projection` selects the
|
|
127
|
+
* returned columns (`["_id", "score"]` for just id + score, or omit
|
|
128
|
+
* for full rows).
|
|
129
|
+
*/
|
|
130
|
+
vectorSearch(column: string, query: Float32Array, k: number, nprobe?: number | undefined | null, rerankMult?: number | undefined | null, projection?: Array<string> | undefined | null, filter?: VectorFilter | undefined | null): Buffer
|
|
131
|
+
/**
|
|
132
|
+
* Unranked token match over one FTS column — every row whose `column`
|
|
133
|
+
* matches the query's tokens under `mode` (`"or"` default, `"and"`).
|
|
134
|
+
* Returns Arrow rows like [`Table::bm25_search`], with `score` = 0.0.
|
|
135
|
+
* `projection` selects columns (omit for full rows).
|
|
136
|
+
*/
|
|
137
|
+
tokenMatch(column: string, query: string, mode?: string | undefined | null, projection?: Array<string> | undefined | null): Buffer
|
|
138
|
+
/**
|
|
139
|
+
* Unranked exact match of `value` against `column`. Returns Arrow rows
|
|
140
|
+
* like [`Table::bm25_search`], with `score` = 0.0. `projection` selects
|
|
141
|
+
* columns (omit for full rows).
|
|
142
|
+
*/
|
|
143
|
+
exactMatch(column: string, value: string, projection?: Array<string> | undefined | null): Buffer
|
|
144
|
+
/**
|
|
145
|
+
* Delete every row matching a SQL `predicate` (e.g. `"status = 'spam'"`),
|
|
146
|
+
* returning the mutation counts. Requires durable storage — a `memory://`
|
|
147
|
+
* table surfaces a clear error.
|
|
148
|
+
*/
|
|
149
|
+
delete(predicate: string): MutationStats
|
|
150
|
+
/**
|
|
151
|
+
* Replace every row matching a SQL `predicate` with `rows` (an Arrow IPC
|
|
152
|
+
* `Buffer`, like `append`), 1:1 — the matched count must equal the
|
|
153
|
+
* replacement-row count or the engine errors. Requires durable storage.
|
|
154
|
+
*/
|
|
155
|
+
update(predicate: string, rows: Buffer): MutationStats
|
|
156
|
+
/**
|
|
157
|
+
* Merge small / underfilled superfiles into larger ones. `settings` tunes
|
|
158
|
+
* the memory budget, fill threshold, and target size (omit for engine
|
|
159
|
+
* defaults).
|
|
160
|
+
*/
|
|
161
|
+
optimize(settings?: OptimizeOptions | undefined | null): void
|
|
162
|
+
/**
|
|
163
|
+
* The user-facing Arrow schema, as an Arrow IPC `Buffer` (an empty
|
|
164
|
+
* table carrying the schema; read with `tableFromIPC`).
|
|
165
|
+
*/
|
|
166
|
+
schema(): Buffer
|
|
167
|
+
}
|