@infino-ai/infino 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -273
- package/infino/index.d.ts +51 -5
- package/infino/index.js +16 -0
- package/infino/native.d.ts +47 -7
- package/package.json +8 -8
package/README.md
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# infino
|
|
2
2
|
|
|
3
|
+
[](https://www.npmjs.com/package/@infino-ai/infino)
|
|
4
|
+
[](https://www.npmjs.com/package/@infino-ai/infino)
|
|
5
|
+
[](https://www.npmjs.com/package/@infino-ai/infino)
|
|
6
|
+
[](https://www.apache.org/licenses/LICENSE-2.0)
|
|
7
|
+
|
|
3
8
|
**SQL, full-text, and vector search over your data on object storage — one engine, no server to run.**
|
|
4
9
|
|
|
5
10
|
Infino keeps your data in Apache Parquet on object storage (local disk, Amazon
|
|
@@ -9,14 +14,9 @@ and vector indexes embedded directly inside it; a table composes many such files
|
|
|
9
14
|
with snapshot-isolated reads, append-only writes, and atomic commits. It runs in
|
|
10
15
|
your process — there is no daemon, no cluster, and no managed service to operate.
|
|
11
16
|
|
|
12
|
-
Use it for **RAG**, **agent memory**, **hybrid search**, and **semantic
|
|
13
|
-
|
|
14
|
-
**SQL** query engine in one library
|
|
15
|
-
server to run.
|
|
16
|
-
|
|
17
|
-
Synchronous, Arrow at the boundary: pass arrays of objects (or apache-arrow
|
|
18
|
-
`Table`s) in, get plain records out; pass `{ arrow: true }` to a search or query
|
|
19
|
-
for an apache-arrow `Table` instead.
|
|
17
|
+
Use it for **RAG**, **agent memory**, **hybrid search**, and **semantic
|
|
18
|
+
search**: an embedded **vector database**, **full-text (BM25)** search engine,
|
|
19
|
+
and **SQL** query engine in one library.
|
|
20
20
|
|
|
21
21
|
## Install
|
|
22
22
|
|
|
@@ -33,8 +33,8 @@ toolchain required. Supported platforms:
|
|
|
33
33
|
| Linux (glibc) | x64, arm64 |
|
|
34
34
|
| Linux (musl / Alpine) | x64, arm64 |
|
|
35
35
|
|
|
36
|
-
`apache-arrow` is installed as a dependency and used at
|
|
37
|
-
`Table`s, or `{ arrow: true }` results).
|
|
36
|
+
Requires Node.js >= 18. `apache-arrow` is installed as a dependency and used at
|
|
37
|
+
the boundary (passing in `Table`s, or `{ arrow: true }` results).
|
|
38
38
|
|
|
39
39
|
## Quickstart
|
|
40
40
|
|
|
@@ -64,285 +64,45 @@ docs.append([
|
|
|
64
64
|
{ source: "blog", body: "Enable dark mode under Settings then Appearance.", embedding: embed(1) },
|
|
65
65
|
]);
|
|
66
66
|
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
const
|
|
70
|
-
const
|
|
67
|
+
// Retrieve context to ground an agent's next answer — keyword, vector,
|
|
68
|
+
// hybrid (BM25 + vector fused in one pass), or SQL:
|
|
69
|
+
const keyword = docs.bm25Search("body", "cancel subscription", 5); // BM25
|
|
70
|
+
const semantic = docs.vectorSearch("embedding", embed(0), 5); // vector kNN
|
|
71
|
+
const hybrid = docs.hybridSearch("body", "cancel subscription", "embedding", embed(0), 5); // fused
|
|
72
|
+
const billing = db.querySql("SELECT body FROM docs WHERE source = 'help-center'"); // SQL filter
|
|
71
73
|
```
|
|
72
74
|
|
|
73
75
|
CommonJS works too — `const { connect, IndexSpec } = require("@infino-ai/infino");`.
|
|
74
76
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
with a README; build the addon first, then `npm install && node index.mjs`):
|
|
79
|
-
|
|
80
|
-
- [`agent-memory/`](examples/agent-memory) — infino as an AI agent's long-term
|
|
81
|
-
memory: load a real multi-session conversation, then recall it with hybrid
|
|
82
|
-
search, query it with SQL (`GROUP BY`, filters), and forget parts of it.
|
|
83
|
-
- [`hybrid-search-api/`](examples/hybrid-search-api) — an embedded HTTP search
|
|
84
|
-
service over a real product catalog, ranked by native `hybrid_search` — with
|
|
85
|
-
no separate search server to run.
|
|
86
|
-
|
|
87
|
-
## Core concepts
|
|
88
|
-
|
|
89
|
-
- **Connection** — a handle to a catalog (a set of tables under one URI). Open
|
|
90
|
-
it with `connect(uri)`.
|
|
91
|
-
- **Table** — an append-only, snapshot-isolated collection of rows. Each table
|
|
92
|
-
carries an auto-generated `_id` column.
|
|
93
|
-
- **IndexSpec** — declares which columns are full-text (BM25) and which are
|
|
94
|
-
vector indexed. Columns without an index are still stored, filterable in SQL,
|
|
95
|
-
and returnable via projection.
|
|
96
|
-
- **Commits** — every `append`, `update`, and `delete` is a single atomic
|
|
97
|
-
commit. Readers see a consistent snapshot and are never torn by a concurrent
|
|
98
|
-
write.
|
|
99
|
-
- **Arrow at the boundary** — searches return plain records (or an apache-arrow
|
|
100
|
-
`Table` with `{ arrow: true }`); `append` and `update` accept an array of
|
|
101
|
-
objects or an apache-arrow `Table` / `RecordBatch`.
|
|
102
|
-
|
|
103
|
-
## Full-text search
|
|
104
|
-
|
|
105
|
-
```javascript
|
|
106
|
-
const docs = db.createTable("docs", { title: "large_utf8" }, new IndexSpec().fts("title"));
|
|
107
|
-
docs.append([{ title: "the quick brown fox" }, { title: "a lazy dog" }]);
|
|
108
|
-
|
|
109
|
-
// Ranked BM25 — higher score is a better match.
|
|
110
|
-
docs.bm25Search("title", "quick fox", 10); // OR by default
|
|
111
|
-
docs.bm25Search("title", "quick fox", 10, { mode: "and" }); // require all terms
|
|
112
|
-
|
|
113
|
-
// Unranked matching (score is 0): every row containing the term(s),
|
|
114
|
-
// or an exact whole-value match.
|
|
115
|
-
docs.tokenMatch("title", "fox");
|
|
116
|
-
docs.exactMatch("title", "the quick brown fox");
|
|
117
|
-
```
|
|
118
|
-
|
|
119
|
-
## Vector search
|
|
120
|
-
|
|
121
|
-
Vector columns are `FixedSizeList<Float32, dim>` with `dim` in `[16, 4096]`. The
|
|
122
|
-
distance metric is fixed when you declare the index (`"cosine"`, `"l2sq"`, or
|
|
123
|
-
`"negdot"`); for vector results a smaller score is nearer. The query vector is a
|
|
124
|
-
`number[]` or `Float32Array`.
|
|
125
|
-
|
|
126
|
-
```javascript
|
|
127
|
-
const spec = new IndexSpec().vector("emb", 384, 256, "cosine"); // (column, dim, nCent, metric)
|
|
128
|
-
const vecs = db.createTable("vecs", { emb: { vector: 384 } }, spec);
|
|
129
|
-
|
|
130
|
-
vecs.vectorSearch("emb", queryVector, 10); // top-10 nearest
|
|
131
|
-
vecs.vectorSearch("emb", queryVector, 10, { nprobe: 32 }); // probe more partitions (recall)
|
|
132
|
-
vecs.vectorSearch("emb", queryVector, 10, { rerankMult: 4 }); // wider exact-rerank pool (recall)
|
|
133
|
-
```
|
|
134
|
-
|
|
135
|
-
**Filtered vector search.** Restrict the kNN to rows matching a text predicate —
|
|
136
|
-
a pushdown *pre-filter*, so you get the nearest *matching* rows (not a
|
|
137
|
-
post-filter over the global top-k). The filter `column` must be FTS-indexed.
|
|
138
|
-
|
|
139
|
-
```javascript
|
|
140
|
-
vecs.vectorSearch("emb", queryVector, 10, {
|
|
141
|
-
filter: { column: "title", query: "billing", mode: "or" },
|
|
142
|
-
});
|
|
143
|
-
```
|
|
144
|
-
|
|
145
|
-
## Hybrid search
|
|
146
|
-
|
|
147
|
-
Combine BM25 and vector search in **one query** with the `hybrid_search` table
|
|
148
|
-
function — a single pass over both indexes, fused inside the engine (no separate
|
|
149
|
-
reranker service, no two round-trips). Keyword-only search misses paraphrases;
|
|
150
|
-
vector-only search misses exact terms — hybrid gets both. Results come back
|
|
151
|
-
best-first with a fused `score`.
|
|
152
|
-
|
|
153
|
-
```javascript
|
|
154
|
-
const spec = new IndexSpec().fts("body").vector("emb", 384, 256, "cosine");
|
|
155
|
-
const docs = db.createTable("docs", { body: "large_utf8", emb: { vector: 384 } }, spec);
|
|
156
|
-
docs.append([{ body: "To cancel a subscription, open Settings then Billing.", emb: embed(/* … */) }]);
|
|
157
|
-
|
|
158
|
-
// hybrid_search(table, text_col, query_text, vec_col, query_vec, k)
|
|
159
|
-
const qvec = embed("how do I stop my plan?").join(",");
|
|
160
|
-
db.querySql(
|
|
161
|
-
`SELECT _id, score FROM hybrid_search('docs', 'body', 'cancel subscription', 'emb', '${qvec}', 10)`,
|
|
162
|
-
);
|
|
163
|
-
```
|
|
164
|
-
|
|
165
|
-
For a complete, runnable hybrid search service see the
|
|
166
|
-
[`hybrid-search-api` example](examples/hybrid-search-api).
|
|
167
|
-
|
|
168
|
-
## SQL
|
|
169
|
-
|
|
170
|
-
Run SQL across the catalog's tables for analytics and filtering; the search
|
|
171
|
-
functions are also available as SQL table functions. Results come back as plain
|
|
172
|
-
records (or an apache-arrow `Table` with `{ arrow: true }`).
|
|
173
|
-
|
|
174
|
-
```javascript
|
|
175
|
-
db.querySql("SELECT COUNT(*) AS n FROM docs");
|
|
176
|
-
db.querySql("SELECT title FROM docs WHERE title = 'a lazy dog'");
|
|
177
|
-
|
|
178
|
-
// The search methods are also SQL table functions — bm25_search, vector_search,
|
|
179
|
-
// and hybrid_search (see "Hybrid search" above) — so you can filter, join, and
|
|
180
|
-
// aggregate over search results.
|
|
181
|
-
db.querySql("SELECT _id, score FROM bm25_search('docs', 'title', 'fox', 10)");
|
|
182
|
-
```
|
|
183
|
-
|
|
184
|
-
## Projections
|
|
185
|
-
|
|
186
|
-
By default a search returns just `_id` and `score` — no row data is decoded.
|
|
187
|
-
Name the columns you want to materialize:
|
|
188
|
-
|
|
189
|
-
```javascript
|
|
190
|
-
docs.bm25Search("title", "fox", 10); // _id + score only
|
|
191
|
-
docs.bm25Search("title", "fox", 10, { projection: ["_id", "title", "score"] });
|
|
192
|
-
```
|
|
193
|
-
|
|
194
|
-
## Updates and deletes
|
|
195
|
-
|
|
196
|
-
Mutations require durable storage (a local path or object store, not
|
|
197
|
-
`memory://`). The predicate is a SQL boolean expression — the same thing you'd
|
|
198
|
-
write after `WHERE` — evaluated against the table's columns.
|
|
199
|
-
|
|
200
|
-
```javascript
|
|
201
|
-
docs.append([{ title: "draft post" }, { title: "spam" }]);
|
|
202
|
-
|
|
203
|
-
// Delete every row matching the predicate.
|
|
204
|
-
docs.delete("title = 'spam'");
|
|
205
|
-
|
|
206
|
-
// Replace matched rows 1:1 with new rows (same input shapes as append).
|
|
207
|
-
const stats = docs.update("title = 'draft post'", [{ title: "published post" }]);
|
|
208
|
-
console.log(stats.matched, stats.nTombstoned, stats.nNotFound);
|
|
209
|
-
```
|
|
210
|
-
|
|
211
|
-
`update` is a one-to-one replacement: the number of matched rows must equal the
|
|
212
|
-
number you supply, otherwise it throws. Both methods return `{ matched,
|
|
213
|
-
nTombstoned, nNotFound }`.
|
|
77
|
+
> The API is synchronous. In a long-running server, run calls in a
|
|
78
|
+
> [`worker_thread`](https://nodejs.org/api/worker_threads.html) so a query
|
|
79
|
+
> doesn't block the event loop.
|
|
214
80
|
|
|
215
|
-
##
|
|
81
|
+
## Documentation
|
|
216
82
|
|
|
217
|
-
|
|
218
|
-
merging small or underfilled files into larger ones — which keeps reads efficient.
|
|
219
|
-
|
|
220
|
-
```javascript
|
|
221
|
-
docs.optimize(); // engine defaults
|
|
222
|
-
docs.optimize({ targetSuperfileSizeMb: 256, minFillPercent: 50 });
|
|
223
|
-
```
|
|
224
|
-
|
|
225
|
-
## Storage backends
|
|
226
|
-
|
|
227
|
-
`connect` selects the backend from the URI:
|
|
228
|
-
|
|
229
|
-
| URI | Backend |
|
|
230
|
-
| --------------------- | ---------------------------------------- |
|
|
231
|
-
| `./data`, `/abs/path` | Local filesystem |
|
|
232
|
-
| `s3://bucket/prefix` | Amazon S3 / S3-compatible object storage |
|
|
233
|
-
| `memory://` | In-process, ephemeral (testing) |
|
|
234
|
-
|
|
235
|
-
For S3-compatible stores that need an explicit endpoint and static credentials,
|
|
236
|
-
pass them in `options` (omit to use ambient AWS credentials):
|
|
237
|
-
|
|
238
|
-
```javascript
|
|
239
|
-
const db = connect("s3://bucket/prefix", {
|
|
240
|
-
endpoint: "https://s3.example.com",
|
|
241
|
-
region: "us-east-1",
|
|
242
|
-
accessKey: "…",
|
|
243
|
-
secretKey: "…",
|
|
244
|
-
});
|
|
245
|
-
```
|
|
83
|
+
Full docs, guides, and the API reference live at **[docs.infino.ai](https://docs.infino.ai)**:
|
|
246
84
|
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
cacheBudgetBytes: 64 * 1024 ** 3,
|
|
258
|
-
coldFetchMode: "lazy_foreground_with_background_fill",
|
|
259
|
-
});
|
|
260
|
-
```
|
|
261
|
-
|
|
262
|
-
## Schema and type requirements
|
|
263
|
-
|
|
264
|
-
- Full-text columns must be Arrow `LargeUtf8` (`"large_utf8"` in a descriptor).
|
|
265
|
-
- Vector columns must be `FixedSizeList<Float32, dim>` (`{ vector: dim }`) with
|
|
266
|
-
`dim` in `[16, 4096]`.
|
|
267
|
-
- The `_id` column is generated by the engine; do not declare it. It comes back
|
|
268
|
-
as a JavaScript `bigint`.
|
|
269
|
-
- `createTable` accepts an apache-arrow `Schema` or a plain `{ column: type }`
|
|
270
|
-
descriptor; `append` / `update` accept an array of objects or an apache-arrow
|
|
271
|
-
`Table` / `RecordBatch`, coerced against the table's declared schema.
|
|
272
|
-
|
|
273
|
-
## API reference
|
|
274
|
-
|
|
275
|
-
- `connect(uri, options?)` — backend from the URI scheme. `options`:
|
|
276
|
-
S3-compatible credentials (`endpoint`, `region`, `accessKey`, `secretKey` —
|
|
277
|
-
`endpoint` requires the other three) and, for remote-backed tables, a local
|
|
278
|
-
disk cache (`cacheDir`, `cacheBudgetBytes`, `coldFetchMode`).
|
|
279
|
-
- `Connection`
|
|
280
|
-
- `createTable(name, schema, IndexSpec)` / `openTable(name)` /
|
|
281
|
-
`dropTable(name, purge?)` (`purge = true` also deletes the data) /
|
|
282
|
-
`listTables()` / `querySql(sql, { arrow? })`.
|
|
283
|
-
- `Table`
|
|
284
|
-
- `append(data)` — one `append` is one commit.
|
|
285
|
-
- `bm25Search(col, q, k, { mode?, projection?, arrow? })` — ranked BM25.
|
|
286
|
-
- `vectorSearch(col, query, k, { nprobe?, rerankMult?, filter?, projection?, arrow? })`
|
|
287
|
-
— ranked kNN; `filter` (`{ column, query, mode? }`, `column` FTS-indexed) is
|
|
288
|
-
a pushdown pre-filter.
|
|
289
|
-
- `tokenMatch(col, q, { mode?, projection?, arrow? })` /
|
|
290
|
-
`exactMatch(col, value, { projection?, arrow? })` — unranked (`score` is `0`).
|
|
291
|
-
- `update(predicate, data)` / `delete(predicate)` — mutate rows matching a SQL
|
|
292
|
-
predicate; return `{ matched, nTombstoned, nNotFound }`; require durable
|
|
293
|
-
storage.
|
|
294
|
-
- `optimize({ maxMemoryMb?, minFillPercent?, targetSuperfileSizeMb? })`.
|
|
295
|
-
- `schema()` — the table's apache-arrow `Schema`.
|
|
296
|
-
- `IndexSpec().fts(col).vector(col, dim, nCent, metric)`.
|
|
297
|
-
- `BUILDER_ID` (named export) — the engine's build identifier string.
|
|
298
|
-
|
|
299
|
-
Search results default to `_id` + `score`; name columns in `projection` to
|
|
300
|
-
materialize row data.
|
|
85
|
+
- [Quickstart](https://docs.infino.ai/quickstart) — install to first query
|
|
86
|
+
- [Core concepts](https://docs.infino.ai/core-concepts) — superfiles, commits, and indexes
|
|
87
|
+
- Guides — [Tables & indexing](https://docs.infino.ai/guides/tables) ·
|
|
88
|
+
[Search: BM25, vector, hybrid](https://docs.infino.ai/guides/search) ·
|
|
89
|
+
[Embeddings](https://docs.infino.ai/guides/embeddings) ·
|
|
90
|
+
[Storage & credentials](https://docs.infino.ai/guides/storage)
|
|
91
|
+
- [SQL reference](https://docs.infino.ai/sql-reference) — query tables and the search table-valued functions
|
|
92
|
+
- [API reference](https://docs.infino.ai/api-reference) — the full Node surface, generated from the package
|
|
93
|
+
- [Integrations](https://docs.infino.ai/integrations) — LangChain, CrewAI, Vercel AI SDK, MCP
|
|
94
|
+
- [Examples](examples) — runnable agent-memory and hybrid-search-service demos
|
|
301
95
|
|
|
302
96
|
## Building from source
|
|
303
97
|
|
|
304
|
-
The binding is built with [napi-rs](https://napi.rs/)
|
|
305
|
-
toolchain
|
|
98
|
+
The binding is built with [napi-rs](https://napi.rs/) and requires a Rust
|
|
99
|
+
toolchain.
|
|
306
100
|
|
|
307
101
|
```sh
|
|
308
102
|
cd infino-node
|
|
309
103
|
npm install && npm run build && npm test
|
|
310
104
|
```
|
|
311
105
|
|
|
312
|
-
## Notes
|
|
313
|
-
|
|
314
|
-
- The API is **synchronous**. In a long-running server, run calls in a
|
|
315
|
-
`worker_thread` so a query doesn't block the event loop.
|
|
316
|
-
|
|
317
|
-
## FAQ
|
|
318
|
-
|
|
319
|
-
**Is infino a vector database?** It does vector search, but it's more than that —
|
|
320
|
-
an embedded engine that runs vector search *and* full-text (BM25) *and* SQL over
|
|
321
|
-
one copy of your data. Reach for it wherever you'd use a vector database, plus the
|
|
322
|
-
cases a vector store alone can't cover: keyword search, filtering, joins, and
|
|
323
|
-
aggregates.
|
|
324
|
-
|
|
325
|
-
**Does it need a server?** No. It runs in your Node.js process — no daemon, no
|
|
326
|
-
cluster, no managed service. Your data is Parquet on local disk or S3.
|
|
327
|
-
|
|
328
|
-
**Can it do hybrid (keyword + vector) search?** Yes, natively — BM25 and vector
|
|
329
|
-
fused in a single pass via `hybrid_search` (see [Hybrid search](#hybrid-search)),
|
|
330
|
-
not a client-side rerank.
|
|
331
|
-
|
|
332
|
-
**Where is my data stored?** As Apache Parquet files on local disk or any
|
|
333
|
-
S3-compatible object store; each file embeds its own BM25 and vector indexes.
|
|
334
|
-
|
|
335
|
-
**Does it work with TypeScript?** Yes — the package ships type definitions and the
|
|
336
|
-
API is identical from JavaScript and TypeScript. Both ESM `import` and CommonJS
|
|
337
|
-
`require` work.
|
|
338
|
-
|
|
339
|
-
**Do I need a Rust toolchain to install it?** No — a prebuilt native binary is
|
|
340
|
-
selected automatically at install (macOS and Linux, x64 and arm64).
|
|
341
|
-
|
|
342
|
-
**Is it a good fit for RAG or agent memory?** Yes, that's a primary use case:
|
|
343
|
-
store documents or conversation history once, retrieve with hybrid search, and
|
|
344
|
-
filter/aggregate with SQL. See the runnable [examples](examples).
|
|
345
|
-
|
|
346
106
|
## License
|
|
347
107
|
|
|
348
108
|
Apache-2.0.
|
package/infino/index.d.ts
CHANGED
|
@@ -17,17 +17,23 @@ export type SchemaDescriptor = Record<string, string | {
|
|
|
17
17
|
export type AppendData = RowRecord[] | arrow.Table | arrow.RecordBatch | Buffer | Uint8Array;
|
|
18
18
|
/** Storage and cache config the `connect` URI can't carry. All optional. */
|
|
19
19
|
export interface ConnectOptions {
|
|
20
|
-
/**
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Credentials/tuning for the URI-selected backend, keyed by `object_store`
|
|
22
|
+
* config strings (`aws_*` / `azure_*`). An unknown key is rejected at
|
|
23
|
+
* `connect`.
|
|
24
|
+
*/
|
|
25
|
+
storageOptions?: Record<string, string>;
|
|
25
26
|
/** Local disk-cache directory for remote-backed tables. */
|
|
26
27
|
cacheDir?: string;
|
|
27
28
|
/** Disk-cache budget in bytes. */
|
|
28
29
|
cacheBudgetBytes?: number;
|
|
29
30
|
/** How cold misses are serviced. */
|
|
30
31
|
coldFetchMode?: "hybrid_with_prefetch" | "range_only" | "lazy_foreground_with_background_fill";
|
|
32
|
+
/**
|
|
33
|
+
* Probe the object store at `connect` (default `false`). `true` fails fast
|
|
34
|
+
* on bad credentials instead of on the first table operation.
|
|
35
|
+
*/
|
|
36
|
+
validate?: boolean;
|
|
31
37
|
}
|
|
32
38
|
/** Row counts returned by `update` / `delete`. */
|
|
33
39
|
export interface MutationStats {
|
|
@@ -38,6 +44,19 @@ export interface MutationStats {
|
|
|
38
44
|
/** Matched rows not found in any live segment. */
|
|
39
45
|
nNotFound: number;
|
|
40
46
|
}
|
|
47
|
+
/** Counts from a `gc` sweep. */
|
|
48
|
+
export interface GcReport {
|
|
49
|
+
/** Bytes reclaimed by deleting orphaned objects. */
|
|
50
|
+
bytesFreed: number;
|
|
51
|
+
/** Orphaned objects deleted. */
|
|
52
|
+
objectsDeleted: number;
|
|
53
|
+
/** Objects kept because they are still referenced by the live set. */
|
|
54
|
+
objectsSkippedLive: number;
|
|
55
|
+
/** Objects kept because they are younger than the grace period. */
|
|
56
|
+
objectsSkippedTooNew: number;
|
|
57
|
+
/** Objects that failed to delete (left for the next sweep). */
|
|
58
|
+
deleteErrors: number;
|
|
59
|
+
}
|
|
41
60
|
/** Tuning for `optimize`; all fields optional (omitted ⇒ engine default). */
|
|
42
61
|
export interface OptimizeOptions {
|
|
43
62
|
/** Build-time memory budget, in MB. */
|
|
@@ -74,6 +93,16 @@ export interface VectorSearchOptions {
|
|
|
74
93
|
/** Restrict the kNN to rows matching a text predicate (pushdown pre-filter). */
|
|
75
94
|
filter?: VectorFilter;
|
|
76
95
|
}
|
|
96
|
+
/** Options for `hybridSearch`. `mode` applies to the BM25 side; `nprobe` to
|
|
97
|
+
* the vector side. */
|
|
98
|
+
export interface HybridSearchOptions {
|
|
99
|
+
/** BM25 boolean mode: `"or"` (default) or `"and"`. */
|
|
100
|
+
mode?: BoolMode;
|
|
101
|
+
/** IVF partitions to probe on the vector side (higher = better recall). */
|
|
102
|
+
nprobe?: number;
|
|
103
|
+
projection?: string[];
|
|
104
|
+
arrow?: boolean;
|
|
105
|
+
}
|
|
77
106
|
export interface TokenMatchOptions {
|
|
78
107
|
mode?: BoolMode;
|
|
79
108
|
projection?: string[];
|
|
@@ -83,6 +112,9 @@ export interface MatchOptions {
|
|
|
83
112
|
projection?: string[];
|
|
84
113
|
arrow?: boolean;
|
|
85
114
|
}
|
|
115
|
+
export interface CountOptions {
|
|
116
|
+
mode?: BoolMode;
|
|
117
|
+
}
|
|
86
118
|
export interface QueryOptions {
|
|
87
119
|
arrow?: boolean;
|
|
88
120
|
}
|
|
@@ -107,6 +139,13 @@ export declare class Table {
|
|
|
107
139
|
arrow: true;
|
|
108
140
|
}): arrow.Table;
|
|
109
141
|
vectorSearch(column: string, query: number[] | Float32Array, k: number, opts?: VectorSearchOptions): RowRecord[];
|
|
142
|
+
/** Hybrid BM25 + vector search, fused with reciprocal-rank fusion; rows as
|
|
143
|
+
* records (or an Arrow `Table`). `score` is the fused RRF score (higher is
|
|
144
|
+
* better). */
|
|
145
|
+
hybridSearch(textColumn: string, textQuery: string, vectorColumn: string, vectorQuery: number[] | Float32Array, k: number, opts: HybridSearchOptions & {
|
|
146
|
+
arrow: true;
|
|
147
|
+
}): arrow.Table;
|
|
148
|
+
hybridSearch(textColumn: string, textQuery: string, vectorColumn: string, vectorQuery: number[] | Float32Array, k: number, opts?: HybridSearchOptions): RowRecord[];
|
|
110
149
|
/** Unranked token match; matching rows as records (or an Arrow `Table`). */
|
|
111
150
|
tokenMatch(column: string, query: string, opts: TokenMatchOptions & {
|
|
112
151
|
arrow: true;
|
|
@@ -117,6 +156,9 @@ export declare class Table {
|
|
|
117
156
|
arrow: true;
|
|
118
157
|
}): arrow.Table;
|
|
119
158
|
exactMatch(column: string, value: string, opts?: MatchOptions): RowRecord[];
|
|
159
|
+
/** Count rows matching a BM25 keyword `query` over `column`, without
|
|
160
|
+
* fetching them. `mode` is `"or"` (default) or `"and"`. */
|
|
161
|
+
count(column: string, query: string, opts?: CountOptions): number;
|
|
120
162
|
/** Replace rows matching a SQL predicate (e.g. `"status = 'spam'"`) with
|
|
121
163
|
* `data` (same shapes as `append`), 1:1 — the matched count must equal the
|
|
122
164
|
* replacement-row count. Requires durable storage (not `memory://`). */
|
|
@@ -127,6 +169,10 @@ export declare class Table {
|
|
|
127
169
|
/** Merge small / underfilled superfiles into larger ones (omit `settings`
|
|
128
170
|
* for engine defaults). */
|
|
129
171
|
optimize(settings?: OptimizeOptions): void;
|
|
172
|
+
/** Delete orphaned storage objects left by compaction or interrupted writes.
|
|
173
|
+
* Only objects older than `graceSecs` (a safety window against racing
|
|
174
|
+
* readers/writers) are removed. Requires durable storage (not `memory://`). */
|
|
175
|
+
gc(graceSecs: number): GcReport;
|
|
130
176
|
}
|
|
131
177
|
export declare class Connection {
|
|
132
178
|
private inner;
|
package/infino/index.js
CHANGED
|
@@ -216,6 +216,11 @@ class Table {
|
|
|
216
216
|
const buf = this.inner.vectorSearch(column, q, k, opts.nprobe, opts.rerankMult, opts.projection, opts.filter);
|
|
217
217
|
return decode(buf, opts.arrow);
|
|
218
218
|
}
|
|
219
|
+
hybridSearch(textColumn, textQuery, vectorColumn, vectorQuery, k, opts = {}) {
|
|
220
|
+
const q = vectorQuery instanceof Float32Array ? vectorQuery : Float32Array.from(vectorQuery);
|
|
221
|
+
const buf = this.inner.hybridSearch(textColumn, textQuery, vectorColumn, q, k, opts.mode, opts.nprobe, opts.projection);
|
|
222
|
+
return decode(buf, opts.arrow);
|
|
223
|
+
}
|
|
219
224
|
tokenMatch(column, query, opts = {}) {
|
|
220
225
|
const buf = this.inner.tokenMatch(column, query, opts.mode, opts.projection);
|
|
221
226
|
return decode(buf, opts.arrow);
|
|
@@ -224,6 +229,11 @@ class Table {
|
|
|
224
229
|
const buf = this.inner.exactMatch(column, value, opts.projection);
|
|
225
230
|
return decode(buf, opts.arrow);
|
|
226
231
|
}
|
|
232
|
+
/** Count rows matching a BM25 keyword `query` over `column`, without
|
|
233
|
+
* fetching them. `mode` is `"or"` (default) or `"and"`. */
|
|
234
|
+
count(column, query, opts = {}) {
|
|
235
|
+
return this.inner.count(column, query, opts.mode);
|
|
236
|
+
}
|
|
227
237
|
/** Replace rows matching a SQL predicate (e.g. `"status = 'spam'"`) with
|
|
228
238
|
* `data` (same shapes as `append`), 1:1 — the matched count must equal the
|
|
229
239
|
* replacement-row count. Requires durable storage (not `memory://`). */
|
|
@@ -240,6 +250,12 @@ class Table {
|
|
|
240
250
|
optimize(settings) {
|
|
241
251
|
this.inner.optimize(settings);
|
|
242
252
|
}
|
|
253
|
+
/** Delete orphaned storage objects left by compaction or interrupted writes.
|
|
254
|
+
* Only objects older than `graceSecs` (a safety window against racing
|
|
255
|
+
* readers/writers) are removed. Requires durable storage (not `memory://`). */
|
|
256
|
+
gc(graceSecs) {
|
|
257
|
+
return this.inner.gc(graceSecs);
|
|
258
|
+
}
|
|
243
259
|
}
|
|
244
260
|
exports.Table = Table;
|
|
245
261
|
class Connection {
|
package/infino/native.d.ts
CHANGED
|
@@ -9,11 +9,12 @@
|
|
|
9
9
|
* disk cache.
|
|
10
10
|
*/
|
|
11
11
|
export interface ConnectOptions {
|
|
12
|
-
/**
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
/**
|
|
13
|
+
* Credentials/tuning for the URI-selected backend, keyed by
|
|
14
|
+
* `object_store` config strings (`aws_*` / `azure_*`). An unknown key
|
|
15
|
+
* is rejected at `connect`.
|
|
16
|
+
*/
|
|
17
|
+
storageOptions?: Record<string, string>
|
|
17
18
|
/** Local disk-cache directory for remote-backed tables. */
|
|
18
19
|
cacheDir?: string
|
|
19
20
|
/** Disk-cache budget in bytes (a JS number; up to 2^53). */
|
|
@@ -23,6 +24,11 @@ export interface ConnectOptions {
|
|
|
23
24
|
* `"lazy_foreground_with_background_fill"`.
|
|
24
25
|
*/
|
|
25
26
|
coldFetchMode?: string
|
|
27
|
+
/**
|
|
28
|
+
* Probe the object store at `connect` (default `false`). `true` fails
|
|
29
|
+
* fast on bad credentials instead of on first use.
|
|
30
|
+
*/
|
|
31
|
+
validate?: boolean
|
|
26
32
|
}
|
|
27
33
|
/** Tuning for `optimize`; all fields optional (omitted ⇒ engine default). */
|
|
28
34
|
export interface OptimizeOptions {
|
|
@@ -42,6 +48,19 @@ export interface MutationStats {
|
|
|
42
48
|
/** Matched rows that were not found in any live segment. */
|
|
43
49
|
nNotFound: number
|
|
44
50
|
}
|
|
51
|
+
/** Counts from a `gc` sweep. */
|
|
52
|
+
export interface GcReport {
|
|
53
|
+
/** Bytes reclaimed by deleting orphaned objects. */
|
|
54
|
+
bytesFreed: number
|
|
55
|
+
/** Orphaned objects deleted. */
|
|
56
|
+
objectsDeleted: number
|
|
57
|
+
/** Objects kept because they are still referenced by the live set. */
|
|
58
|
+
objectsSkippedLive: number
|
|
59
|
+
/** Objects kept because they are younger than the grace period. */
|
|
60
|
+
objectsSkippedTooNew: number
|
|
61
|
+
/** Objects that failed to delete (left for the next sweep). */
|
|
62
|
+
deleteErrors: number
|
|
63
|
+
}
|
|
45
64
|
/**
|
|
46
65
|
* Text-predicate filter for `vectorSearch` — a pushdown pre-filter, not a
|
|
47
66
|
* post-filter: kNN ranks only among rows whose FTS-indexed `column` matches
|
|
@@ -57,8 +76,10 @@ export interface VectorFilter {
|
|
|
57
76
|
}
|
|
58
77
|
/**
|
|
59
78
|
* Open (or create) a catalog rooted at `uri` (local dir, `memory://`, or
|
|
60
|
-
* object-store prefix).
|
|
61
|
-
*
|
|
79
|
+
* object-store prefix). Credentials are passed via `options.storageOptions`
|
|
80
|
+
* (the JS-idiomatic form of the Rust `ConnectOptions`). Pass `validate: true`
|
|
81
|
+
* to probe object stores at connect (off by default) so bad credentials fail
|
|
82
|
+
* there rather than on the first table operation.
|
|
62
83
|
*/
|
|
63
84
|
export declare function connect(uri: string, options?: ConnectOptions | undefined | null): Connection
|
|
64
85
|
/**
|
|
@@ -141,6 +162,19 @@ export declare class Table {
|
|
|
141
162
|
* columns (omit for full rows).
|
|
142
163
|
*/
|
|
143
164
|
exactMatch(column: string, value: string, projection?: Array<string> | undefined | null): Buffer
|
|
165
|
+
/**
|
|
166
|
+
* Count rows matching a BM25 keyword `query` over `column`, without
|
|
167
|
+
* fetching them. `mode` is `"or"` (default) or `"and"`.
|
|
168
|
+
*/
|
|
169
|
+
count(column: string, query: string, mode?: string | undefined | null): number
|
|
170
|
+
/**
|
|
171
|
+
* Hybrid BM25 + vector search fused with reciprocal-rank fusion.
|
|
172
|
+
* `text_column`/`text_query` (under `mode`) drive BM25; `vector_column`/
|
|
173
|
+
* `vector_query` (a `Float32Array`, with optional `nprobe`) drive vector
|
|
174
|
+
* kNN. Returns Arrow rows like [`Table::bm25_search`], with `score` the
|
|
175
|
+
* fused RRF score (higher is better); `projection` selects columns.
|
|
176
|
+
*/
|
|
177
|
+
hybridSearch(textColumn: string, textQuery: string, vectorColumn: string, vectorQuery: Float32Array, k: number, mode?: string | undefined | null, nprobe?: number | undefined | null, projection?: Array<string> | undefined | null): Buffer
|
|
144
178
|
/**
|
|
145
179
|
* Delete every row matching a SQL `predicate` (e.g. `"status = 'spam'"`),
|
|
146
180
|
* returning the mutation counts. Requires durable storage — a `memory://`
|
|
@@ -159,6 +193,12 @@ export declare class Table {
|
|
|
159
193
|
* defaults).
|
|
160
194
|
*/
|
|
161
195
|
optimize(settings?: OptimizeOptions | undefined | null): void
|
|
196
|
+
/**
|
|
197
|
+
* Delete orphaned storage objects left by compaction or interrupted
|
|
198
|
+
* writes. Only objects older than `graceSecs` (a safety window against
|
|
199
|
+
* racing readers/writers) are removed. Requires durable storage.
|
|
200
|
+
*/
|
|
201
|
+
gc(graceSecs: number): GcReport
|
|
162
202
|
/**
|
|
163
203
|
* The user-facing Arrow schema, as an Arrow IPC `Buffer` (an empty
|
|
164
204
|
* table carrying the schema; read with `tableFromIPC`).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@infino-ai/infino",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2",
|
|
4
4
|
"description": "Fast search on object storage — SQL, full-text, and vector search.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"publishConfig": {
|
|
@@ -82,12 +82,12 @@
|
|
|
82
82
|
"apache-arrow": "^17"
|
|
83
83
|
},
|
|
84
84
|
"optionalDependencies": {
|
|
85
|
-
"infx-darwin-x64": "0.1.
|
|
86
|
-
"infx-darwin-arm64": "0.1.
|
|
87
|
-
"infx-linux-x64-gnu": "0.1.
|
|
88
|
-
"infx-linux-arm64-gnu": "0.1.
|
|
89
|
-
"infx-linux-x64-musl": "0.1.
|
|
90
|
-
"infx-linux-arm64-musl": "0.1.
|
|
85
|
+
"infx-darwin-x64": "0.1.2",
|
|
86
|
+
"infx-darwin-arm64": "0.1.2",
|
|
87
|
+
"infx-linux-x64-gnu": "0.1.2",
|
|
88
|
+
"infx-linux-arm64-gnu": "0.1.2",
|
|
89
|
+
"infx-linux-x64-musl": "0.1.2",
|
|
90
|
+
"infx-linux-arm64-musl": "0.1.2"
|
|
91
91
|
},
|
|
92
92
|
"devDependencies": {
|
|
93
93
|
"@napi-rs/cli": "^2.18.0",
|
|
@@ -99,4 +99,4 @@
|
|
|
99
99
|
"infino/native.js",
|
|
100
100
|
"infino/native.d.ts"
|
|
101
101
|
]
|
|
102
|
-
}
|
|
102
|
+
}
|