@omgbase/core 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/core/ingest.js +15 -12
- package/dist/src/core/read/document.js +6 -1
- package/dist/src/core/read/nodes.js +5 -1
- package/dist/src/core/store/fts.d.ts +38 -7
- package/dist/src/core/store/fts.js +60 -9
- package/dist/src/core/store/rebuild.js +5 -4
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.js +1 -0
- package/dist/src/mcp/server.js +20 -9
- package/dist/src/oqx-js/context.d.ts +11 -0
- package/dist/src/oqx-js/context.js +16 -0
- package/dist/src/oqx-js/planner.d.ts +20 -1
- package/dist/src/oqx-js/planner.js +95 -2
- package/dist/src/oqx-js/run.js +27 -2
- package/dist/src/oqx-js/sql/translate.d.ts +71 -1
- package/dist/src/oqx-js/sql/translate.js +165 -70
- package/dist/src/search/tasks.d.ts +6 -1
- package/dist/src/search/tasks.js +8 -2
- package/dist/src/sync/external-source.js +20 -7
- package/dist/src/sync/plugin.d.ts +30 -5
- package/dist/src/sync/plugin.js +13 -1
- package/dist/src/sync/watcher.d.ts +2 -0
- package/dist/src/sync/watcher.js +10 -2
- package/package.json +1 -1
package/dist/src/core/ingest.js
CHANGED
|
@@ -6,7 +6,7 @@ import { assignIds, writeBlockTree, putBlob, newCommit, writeRevision } from "./
|
|
|
6
6
|
import { sha256, childQuoteDepth, visibleText } from "./hash.js";
|
|
7
7
|
import { mintId } from "./ids.js";
|
|
8
8
|
import { keyBetween } from "./order-key.js";
|
|
9
|
-
import { ftsDeleteDoc, ftsIndexDoc } from "./store/fts.js";
|
|
9
|
+
import { ftsBeforeEvictRows, ftsDeleteDoc, ftsIndexDoc } from "./store/fts.js";
|
|
10
10
|
import { rebuildSections } from "./store/sections.js";
|
|
11
11
|
import { writeDocNodes, projectSectionNodes, toByteSpans } from "./store/nodes.js";
|
|
12
12
|
import { flattenFrontmatter, flattenComputed, writeDocProperties } from "./store/properties.js";
|
|
@@ -288,25 +288,28 @@ export function ingestFile(store, repoId, path, content, opts = {}) {
|
|
|
288
288
|
});
|
|
289
289
|
}
|
|
290
290
|
// Remove another document's `blocks` row for each id in `ids` (a cross-doc move
|
|
291
|
-
// or a resurrection re-homes the id into `docId`). A live foreign row still
|
|
292
|
-
// an FTS entry (external-content index: delete with the original text); a
|
|
293
|
-
// tombstoned
|
|
291
|
+
// or a resurrection re-homes the id into `docId`). A live foreign LEAF row still
|
|
292
|
+
// has an FTS entry (external-content index: delete with the original text); a
|
|
293
|
+
// tombstoned row or a container does not (spec/search §1.1) — ftsBeforeEvictRows
|
|
294
|
+
// decides leaf-ness over the table before any row goes and re-indexes a parent
|
|
295
|
+
// left childless. The source document, if it commits later in the same
|
|
294
296
|
// checkpoint, rewrites its rows wholesale and never lists a carried id as
|
|
295
297
|
// deleted, so nothing there depends on the evicted row.
|
|
296
298
|
function evictForeignBlockRows(db, docId, ids) {
|
|
297
|
-
const sel = db.prepare("SELECT rowid, text, deleted_commit FROM blocks WHERE block_id = ? AND doc_id != ?");
|
|
298
|
-
const ftsDel = db.prepare("INSERT INTO blocks_fts(blocks_fts, rowid, text) VALUES('delete', ?, ?)");
|
|
299
|
+
const sel = db.prepare("SELECT rowid, block_id, doc_id, parent_block, text, deleted_commit FROM blocks WHERE block_id = ? AND doc_id != ?");
|
|
299
300
|
const del = db.prepare("DELETE FROM blocks WHERE block_id = ? AND doc_id != ?");
|
|
300
301
|
const unpool = db.prepare("DELETE FROM resurrection_pool WHERE block_id = ?");
|
|
302
|
+
const rows = [];
|
|
301
303
|
for (const id of ids) {
|
|
302
304
|
const row = sel.get(id, docId);
|
|
303
|
-
if (row)
|
|
304
|
-
|
|
305
|
-
ftsDel.run(row.rowid, row.text);
|
|
306
|
-
del.run(id, docId);
|
|
307
|
-
}
|
|
308
|
-
unpool.run(id);
|
|
305
|
+
if (row)
|
|
306
|
+
rows.push(row);
|
|
309
307
|
}
|
|
308
|
+
ftsBeforeEvictRows(db, rows);
|
|
309
|
+
for (const r of rows)
|
|
310
|
+
del.run(r.block_id, docId);
|
|
311
|
+
for (const id of ids)
|
|
312
|
+
unpool.run(id);
|
|
310
313
|
}
|
|
311
314
|
// Zip assigned block ids onto the parsed RawBlocks for node projection. The
|
|
312
315
|
// parsed `rest` and `assigned` trees are the same blocks in the same order and
|
|
@@ -116,8 +116,13 @@ export function docsReadMany(store, repoId, refs, opts = {}) {
|
|
|
116
116
|
errors.push({ ref, error: "doc_not_found" });
|
|
117
117
|
continue;
|
|
118
118
|
}
|
|
119
|
+
// The budget is checked before the SECOND item onward (spec/surface §2, 1.2):
|
|
120
|
+
// the first item is always emitted even when it alone exceeds the budget —
|
|
121
|
+
// the same "at least one row" contract docs_list/docs_tree keep, so a
|
|
122
|
+
// budgeted client always makes progress. `truncated` stays honest: it is
|
|
123
|
+
// set only when the budget stopped an item that would otherwise have shipped.
|
|
119
124
|
const cost = Math.ceil(JSON.stringify(read).length / 4);
|
|
120
|
-
if (tokens + cost > budget) {
|
|
125
|
+
if (items.length > 0 && tokens + cost > budget) {
|
|
121
126
|
truncated = true;
|
|
122
127
|
break;
|
|
123
128
|
}
|
|
@@ -89,8 +89,12 @@ export function nodesGetMany(store, docId, blockIds, opts = {}) {
|
|
|
89
89
|
continue;
|
|
90
90
|
}
|
|
91
91
|
const projected = project(store, node, resolution, false);
|
|
92
|
+
// At least one resolved node is always returned (spec/surface §2, 1.2): the
|
|
93
|
+
// budget is checked before the second node onward, as docsReadMany does, so
|
|
94
|
+
// a first node that alone exceeds it still ships and `truncated` only says
|
|
95
|
+
// that more remained.
|
|
92
96
|
const cost = Math.ceil(JSON.stringify(projected).length / 4);
|
|
93
|
-
if (tokens + cost > budget) {
|
|
97
|
+
if (nodes.length > 0 && tokens + cost > budget) {
|
|
94
98
|
truncated = true;
|
|
95
99
|
break;
|
|
96
100
|
}
|
|
@@ -1,10 +1,41 @@
|
|
|
1
1
|
import type { Database } from "better-sqlite3";
|
|
2
|
-
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
|
|
7
|
-
|
|
2
|
+
/**
|
|
3
|
+
* SQL predicate: the `blocks` row aliased `b` is a live leaf. `c.doc_id =
|
|
4
|
+
* b.doc_id` lets the NOT EXISTS use `idx_blocks_doc` (block ids are unique, so
|
|
5
|
+
* it never changes the answer).
|
|
6
|
+
*/
|
|
7
|
+
export declare const LIVE_LEAF_SQL = "b.deleted_commit IS NULL AND NOT EXISTS (SELECT 1 FROM blocks c WHERE c.doc_id = b.doc_id AND c.parent_block = b.block_id AND c.deleted_commit IS NULL)";
|
|
8
|
+
/** Remove a document's live leaf rows from the FTS index (call before deleting
|
|
9
|
+
* or tombstoning blocks). Only live leaves are indexed (see ftsIndexDoc), so the
|
|
10
|
+
* delete must target the SAME set: an already-tombstoned block or a container has
|
|
11
|
+
* no FTS row, and re-issuing a 'delete' for it corrupts the external-content
|
|
12
|
+
* index's statistics. Keeping this symmetric with ftsIndexDoc makes a tombstone
|
|
13
|
+
* followed by a later re-ingest of the same doc id (observed delete → recreate)
|
|
14
|
+
* idempotent. */
|
|
8
15
|
export declare function ftsDeleteDoc(db: Database, docId: string): void;
|
|
9
|
-
/** Index a document's current
|
|
16
|
+
/** Index a document's current live leaf rows (call after inserting blocks). */
|
|
10
17
|
export declare function ftsIndexDoc(db: Database, docId: string): void;
|
|
18
|
+
/**
|
|
19
|
+
* Rebuild the whole index: `'delete-all'`, then every live leaf of every live
|
|
20
|
+
* document. (FTS5's own `'rebuild'` command reads the content table wholesale,
|
|
21
|
+
* containers included, so it is not used.)
|
|
22
|
+
*/
|
|
23
|
+
export declare function ftsRebuild(db: Database): void;
|
|
24
|
+
/**
|
|
25
|
+
* Remove single `blocks` rows (by id) that are about to be deleted out of
|
|
26
|
+
* another document's row set — the cross-document eviction of spec/store §5.4
|
|
27
|
+
* step 8 — keeping the index equal to the table's live leaves. Every row's
|
|
28
|
+
* leaf-ness is decided over the table BEFORE any of them goes (so a list moving
|
|
29
|
+
* with its items is order-independent: the items are leaves and lose their
|
|
30
|
+
* entries, the list never had one), and a live parent that keeps its row but
|
|
31
|
+
* loses its last live child becomes a leaf and gains an entry. The caller owns
|
|
32
|
+
* the DELETE of the rows, which must follow this call.
|
|
33
|
+
*/
|
|
34
|
+
export declare function ftsBeforeEvictRows(db: Database, rows: {
|
|
35
|
+
rowid: number;
|
|
36
|
+
block_id: string;
|
|
37
|
+
doc_id: string;
|
|
38
|
+
parent_block: string | null;
|
|
39
|
+
text: string;
|
|
40
|
+
deleted_commit: string | null;
|
|
41
|
+
}[]): void;
|
|
@@ -1,22 +1,73 @@
|
|
|
1
|
-
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
/**
|
|
2
|
+
* SQL predicate: the `blocks` row aliased `b` is a live leaf. `c.doc_id =
|
|
3
|
+
* b.doc_id` lets the NOT EXISTS use `idx_blocks_doc` (block ids are unique, so
|
|
4
|
+
* it never changes the answer).
|
|
5
|
+
*/
|
|
6
|
+
export const LIVE_LEAF_SQL = "b.deleted_commit IS NULL AND NOT EXISTS (SELECT 1 FROM blocks c WHERE c.doc_id = b.doc_id AND c.parent_block = b.block_id AND c.deleted_commit IS NULL)";
|
|
7
|
+
/** Remove a document's live leaf rows from the FTS index (call before deleting
|
|
8
|
+
* or tombstoning blocks). Only live leaves are indexed (see ftsIndexDoc), so the
|
|
9
|
+
* delete must target the SAME set: an already-tombstoned block or a container has
|
|
10
|
+
* no FTS row, and re-issuing a 'delete' for it corrupts the external-content
|
|
11
|
+
* index's statistics. Keeping this symmetric with ftsIndexDoc makes a tombstone
|
|
12
|
+
* followed by a later re-ingest of the same doc id (observed delete → recreate)
|
|
13
|
+
* idempotent. */
|
|
7
14
|
export function ftsDeleteDoc(db, docId) {
|
|
8
|
-
const rows = db.prepare(
|
|
15
|
+
const rows = db.prepare(`SELECT b.rowid AS rowid, b.text AS text FROM blocks b WHERE b.doc_id = ? AND ${LIVE_LEAF_SQL}`).all(docId);
|
|
9
16
|
const del = db.prepare("INSERT INTO blocks_fts(blocks_fts, rowid, text) VALUES('delete', ?, ?)");
|
|
10
17
|
for (const r of rows)
|
|
11
18
|
del.run(r.rowid, r.text);
|
|
12
19
|
}
|
|
13
|
-
/** Index a document's current
|
|
20
|
+
/** Index a document's current live leaf rows (call after inserting blocks). */
|
|
14
21
|
export function ftsIndexDoc(db, docId) {
|
|
15
22
|
const rows = db
|
|
16
|
-
.prepare(
|
|
23
|
+
.prepare(`SELECT b.rowid AS rowid, b.text AS text FROM blocks b WHERE b.doc_id = ? AND ${LIVE_LEAF_SQL}`)
|
|
17
24
|
.all(docId);
|
|
18
25
|
const ins = db.prepare("INSERT INTO blocks_fts(rowid, text) VALUES(?, ?)");
|
|
19
26
|
for (const r of rows)
|
|
20
27
|
ins.run(r.rowid, r.text);
|
|
21
28
|
}
|
|
29
|
+
/**
|
|
30
|
+
* Rebuild the whole index: `'delete-all'`, then every live leaf of every live
|
|
31
|
+
* document. (FTS5's own `'rebuild'` command reads the content table wholesale,
|
|
32
|
+
* containers included, so it is not used.)
|
|
33
|
+
*/
|
|
34
|
+
export function ftsRebuild(db) {
|
|
35
|
+
db.exec("INSERT INTO blocks_fts(blocks_fts) VALUES('delete-all')");
|
|
36
|
+
const docIds = db.prepare("SELECT doc_id FROM docs WHERE deleted_commit IS NULL").all().map((r) => r.doc_id);
|
|
37
|
+
for (const docId of docIds)
|
|
38
|
+
ftsIndexDoc(db, docId);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Remove single `blocks` rows (by id) that are about to be deleted out of
|
|
42
|
+
* another document's row set — the cross-document eviction of spec/store §5.4
|
|
43
|
+
* step 8 — keeping the index equal to the table's live leaves. Every row's
|
|
44
|
+
* leaf-ness is decided over the table BEFORE any of them goes (so a list moving
|
|
45
|
+
* with its items is order-independent: the items are leaves and lose their
|
|
46
|
+
* entries, the list never had one), and a live parent that keeps its row but
|
|
47
|
+
* loses its last live child becomes a leaf and gains an entry. The caller owns
|
|
48
|
+
* the DELETE of the rows, which must follow this call.
|
|
49
|
+
*/
|
|
50
|
+
export function ftsBeforeEvictRows(db, rows) {
|
|
51
|
+
if (rows.length === 0)
|
|
52
|
+
return;
|
|
53
|
+
const evicted = new Set(rows.map((r) => r.block_id));
|
|
54
|
+
const hasLiveChild = db.prepare("SELECT 1 FROM blocks c WHERE c.doc_id = ? AND c.parent_block = ? AND c.deleted_commit IS NULL LIMIT 1");
|
|
55
|
+
const del = db.prepare("INSERT INTO blocks_fts(blocks_fts, rowid, text) VALUES('delete', ?, ?)");
|
|
56
|
+
const leaves = rows.filter((r) => r.deleted_commit === null && !hasLiveChild.get(r.doc_id, r.block_id));
|
|
57
|
+
for (const r of leaves)
|
|
58
|
+
del.run(r.rowid, r.text);
|
|
59
|
+
// Parents that stay live but are left childless are leaves from now on.
|
|
60
|
+
const parentRow = db.prepare(`SELECT b.rowid AS rowid, b.text AS text FROM blocks b WHERE b.block_id = ? AND b.doc_id = ? AND b.deleted_commit IS NULL
|
|
61
|
+
AND NOT EXISTS (SELECT 1 FROM blocks c WHERE c.doc_id = b.doc_id AND c.parent_block = b.block_id AND c.deleted_commit IS NULL AND c.block_id NOT IN (${rows.map(() => "?").join(",")}))`);
|
|
62
|
+
const ins = db.prepare("INSERT INTO blocks_fts(rowid, text) VALUES(?, ?)");
|
|
63
|
+
const seen = new Set();
|
|
64
|
+
for (const r of rows) {
|
|
65
|
+
if (r.deleted_commit !== null || !r.parent_block || evicted.has(r.parent_block) || seen.has(r.parent_block))
|
|
66
|
+
continue;
|
|
67
|
+
seen.add(r.parent_block);
|
|
68
|
+
const p = parentRow.get(r.parent_block, r.doc_id, ...rows.map((x) => x.block_id));
|
|
69
|
+
if (p)
|
|
70
|
+
ins.run(p.rowid, p.text);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
22
73
|
//# sourceMappingURL=fts.js.map
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { rebuildSections } from "./sections.js";
|
|
2
2
|
import { rebuildDocEdges } from "./edges.js";
|
|
3
|
-
import {
|
|
3
|
+
import { ftsRebuild } from "./fts.js";
|
|
4
4
|
export function rebuildIndex(store, target = "all") {
|
|
5
5
|
store.write((db) => {
|
|
6
6
|
const docIds = db.prepare("SELECT doc_id FROM docs WHERE deleted_commit IS NULL").all().map((r) => r.doc_id);
|
|
@@ -14,8 +14,10 @@ export function rebuildIndex(store, target = "all") {
|
|
|
14
14
|
rebuildDocEdges(db, docId);
|
|
15
15
|
}
|
|
16
16
|
if (target === "fts" || target === "all") {
|
|
17
|
-
//
|
|
18
|
-
|
|
17
|
+
// spec/search §1.1: 'delete-all', then every live leaf of every live doc
|
|
18
|
+
// (FTS5's 'rebuild' would index the content table wholesale, containers
|
|
19
|
+
// included).
|
|
20
|
+
ftsRebuild(db);
|
|
19
21
|
}
|
|
20
22
|
if (target === "block_changes" || target === "all") {
|
|
21
23
|
// block_changes is a projection of dispositions across commits: rebuild
|
|
@@ -24,6 +26,5 @@ export function rebuildIndex(store, target = "all") {
|
|
|
24
26
|
db.exec("INSERT OR IGNORE INTO block_changes (block_id, commit_id, kind) SELECT block_id, commit_id, kind FROM dispositions");
|
|
25
27
|
}
|
|
26
28
|
});
|
|
27
|
-
void ftsIndexDoc; // fts rebuild uses the FTS5 'rebuild' command directly
|
|
28
29
|
}
|
|
29
30
|
//# sourceMappingURL=rebuild.js.map
|
package/dist/src/index.d.ts
CHANGED
|
@@ -19,7 +19,8 @@ export { ingestDirectory, type IngestDirResult } from "./sync/attach.js";
|
|
|
19
19
|
export { ensureRepo } from "./core/attach.js";
|
|
20
20
|
export { walkMarkdown, walkMarkdownAsync, nodeFs, type SyncFs, type FileStat } from "./sync/fs-util.js";
|
|
21
21
|
export { Watcher, type WatcherOptions } from "./sync/watcher.js";
|
|
22
|
-
export type { SyncSource, SourceCapabilities, SourceIdentity, SourceEntry, SourceItem, SourceWatch, WatchListener, } from "./sync/plugin.js";
|
|
22
|
+
export type { SyncSource, SourceCapabilities, SourceIdentity, SourceEntry, SourceItem, SourceWatch, WatchEvent, WatchListener, } from "./sync/plugin.js";
|
|
23
|
+
export { awaitReady, WATCH_READY_PATIENCE_MS } from "./sync/plugin.js";
|
|
23
24
|
export { createExternalSource, type ExternalSourceSpec } from "./sync/external-source.js";
|
|
24
25
|
export { ensureAdapter, listAdapters, createSource, deleteSource, listSources, getSourceByName, attachSourceToRepo, detachSourceFromRepo, sourcesForRepo, renderConfigFlags, type AdapterRow, type SourceRow, } from "./sync/sources.js";
|
|
25
26
|
export { reconcileChanges, attachSource } from "./sync/driver.js";
|
package/dist/src/index.js
CHANGED
|
@@ -20,6 +20,7 @@ export { ingestDirectory } from "./sync/attach.js";
|
|
|
20
20
|
export { ensureRepo } from "./core/attach.js";
|
|
21
21
|
export { walkMarkdown, walkMarkdownAsync, nodeFs } from "./sync/fs-util.js";
|
|
22
22
|
export { Watcher } from "./sync/watcher.js";
|
|
23
|
+
export { awaitReady, WATCH_READY_PATIENCE_MS } from "./sync/plugin.js";
|
|
23
24
|
export { createExternalSource } from "./sync/external-source.js";
|
|
24
25
|
export { ensureAdapter, listAdapters, createSource, deleteSource, listSources, getSourceByName, attachSourceToRepo, detachSourceFromRepo, sourcesForRepo, renderConfigFlags, } from "./sync/sources.js";
|
|
25
26
|
export { reconcileChanges, attachSource } from "./sync/driver.js";
|
package/dist/src/mcp/server.js
CHANGED
|
@@ -45,6 +45,15 @@ const toSchema = z.object({
|
|
|
45
45
|
]),
|
|
46
46
|
at: atSchema,
|
|
47
47
|
});
|
|
48
|
+
// blocks_insert / blocks_move carry the op-level expectation of spec/mutate §1.2:
|
|
49
|
+
// only `parent_children_hash` (the destination parent's live direct child ids
|
|
50
|
+
// joined by "," and sha256-hexed) has a meaning there — there is no target block
|
|
51
|
+
// whose content_hash could be checked — so the schema names that one key alone
|
|
52
|
+
// (an accidental `content_hash` is dropped, not an error; the kernel would
|
|
53
|
+
// ignore it anyway).
|
|
54
|
+
const parentExpectSchema = z.object({
|
|
55
|
+
parent_children_hash: z.string().optional(),
|
|
56
|
+
});
|
|
48
57
|
const expectSchema = z.object({
|
|
49
58
|
content_hash: z.string().optional(),
|
|
50
59
|
parent_children_hash: z.string().optional(),
|
|
@@ -279,7 +288,7 @@ export function buildServer(ctx) {
|
|
|
279
288
|
}
|
|
280
289
|
});
|
|
281
290
|
server.registerTool("docs_get_many", {
|
|
282
|
-
description: "Batch whole-document read — the hydrate half of query→hydrate. The plural of docs_read: pass `docs`, a list of refs (each a doc id OR a path, same id-or-path symmetry docs_read accepts in its `doc` field), and get back one full read per ref. Returns `{ items, errors, truncated }`: each found doc is a full docs_read projection (`content` = complete file bytes verbatim, `properties` grouped by source, plus `path`/`docId`/`rev`, and — with include_ids:true — the document's ordered block `ids`, a `hashes` {block id → content hash} map for raw-`apply` CAS pinning, and a `parents` {block id → parent block id | null} map so top-level vs nested blocks are distinguishable without an outline read); a ref that resolves to no live document lands in `errors` as {ref, error:\"doc_not_found\"} WITHOUT failing the call, so one bad ref never sinks the batch. Duplicate refs collapse first-seen (a repeated ref yields a single item). Capped at " + MANY_DOCS_CAP + " refs per call; excess refs are dropped and `truncated` is set. Pass budget_tokens to cap total hydrated size — the batch stops early and flags `truncated` when the next doc would exceed it. Use this after query/text_search/resolve to pull N whole docs in ONE round-trip instead of N serial docs_read calls; for a single doc use docs_read, and for lean structure-only orientation use docs_outline.",
|
|
291
|
+
description: "Batch whole-document read — the hydrate half of query→hydrate. The plural of docs_read: pass `docs`, a list of refs (each a doc id OR a path, same id-or-path symmetry docs_read accepts in its `doc` field), and get back one full read per ref. Returns `{ items, errors, truncated }`: each found doc is a full docs_read projection (`content` = complete file bytes verbatim, `properties` grouped by source, plus `path`/`docId`/`rev`, and — with include_ids:true — the document's ordered block `ids`, a `hashes` {block id → content hash} map for raw-`apply` CAS pinning, and a `parents` {block id → parent block id | null} map so top-level vs nested blocks are distinguishable without an outline read); a ref that resolves to no live document lands in `errors` as {ref, error:\"doc_not_found\"} WITHOUT failing the call, so one bad ref never sinks the batch. Duplicate refs collapse first-seen (a repeated ref yields a single item). Capped at " + MANY_DOCS_CAP + " refs per call; excess refs are dropped and `truncated` is set. Pass budget_tokens to cap total hydrated size — the batch stops early and flags `truncated` when the next doc would exceed it; the FIRST doc always ships even when it alone exceeds the budget (the same at-least-one-row contract as docs_list/docs_tree, so a budgeted caller always makes progress). Use this after query/text_search/resolve to pull N whole docs in ONE round-trip instead of N serial docs_read calls; for a single doc use docs_read, and for lean structure-only orientation use docs_outline.",
|
|
283
292
|
inputSchema: {
|
|
284
293
|
docs: z.array(z.string()),
|
|
285
294
|
include_ids: z.boolean().optional(),
|
|
@@ -322,7 +331,7 @@ export function buildServer(ctx) {
|
|
|
322
331
|
}
|
|
323
332
|
});
|
|
324
333
|
server.registerTool("nodes_get_many", {
|
|
325
|
-
description: "Fetch up to 100 blocks by id (in request order) with budget truncation. Block ids are globally unique, so `ids` may span ANY number of documents — each id is resolved to its owning doc server-side; `doc`/`path` is an optional SCOPE (ids from other docs then count as unresolved), not a requirement. Result carries `nodes`, `truncated` (cap or `budget_tokens` hit), and `unresolved` — the requested ids that name no live block (never silently dropped).",
|
|
334
|
+
description: "Fetch up to 100 blocks by id (in request order) with budget truncation. Block ids are globally unique, so `ids` may span ANY number of documents — each id is resolved to its owning doc server-side; `doc`/`path` is an optional SCOPE (ids from other docs then count as unresolved), not a requirement. Result carries `nodes`, `truncated` (cap or `budget_tokens` hit — the first resolved node always ships even when it alone exceeds the budget, so a budgeted caller always makes progress), and `unresolved` — the requested ids that name no live block (never silently dropped).",
|
|
326
335
|
inputSchema: {
|
|
327
336
|
doc: z.string().optional(),
|
|
328
337
|
path: z.string().optional(),
|
|
@@ -540,16 +549,18 @@ export function buildServer(ctx) {
|
|
|
540
549
|
// local: the resolution that used to live in the CLI now lives next to the
|
|
541
550
|
// store, so a remote client needs no local database.
|
|
542
551
|
server.registerTool("blocks_insert", {
|
|
543
|
-
description: "Insert new block(s) parsed from `markdown` under a parent, at a position. `to` names the parent: a BLOCK ref (id or locator) nests the new blocks under that block, or a DOCUMENT ref (doc id or path) places them at the document's top level — so appending a new section to a document is `{ to: \"<path>\", markdown: \"## New\\n\\n…\" }` (the positional generalization of docs_append). `at` places among the parent's children: \"end\" (default) / \"start\" / {before|after: <block ref>}. Expands to one insert op through the kernel writer.",
|
|
544
|
-
inputSchema: { to: z.string(), markdown: z.string(), at: atSchema.optional(), dry_run: z.boolean().optional(), ...REPO_ARG },
|
|
552
|
+
description: "Insert new block(s) parsed from `markdown` under a parent, at a position. `to` names the parent: a BLOCK ref (id or locator) nests the new blocks under that block, or a DOCUMENT ref (doc id or path) places them at the document's top level — so appending a new section to a document is `{ to: \"<path>\", markdown: \"## New\\n\\n…\" }` (the positional generalization of docs_append). `at` places among the parent's children: \"end\" (default) / \"start\" / {before|after: <block ref>}. Optional `expect.parent_children_hash` is the destination-parent CAS: the parent's CURRENT direct child ids (the document's top-level ids for a document `to`) joined by \",\" and sha256-hexed — compute it from docs_read include_ids (`ids` filtered by `parents`) — and the insert fails `stale_expectation` (with `data.current.parent_children_hash`) if the siblings changed under you; `content_hash` has no meaning here (there is no target block) and is not accepted. Expands to one insert op through the kernel writer.",
|
|
553
|
+
inputSchema: { to: z.string(), markdown: z.string(), at: atSchema.optional(), expect: parentExpectSchema.optional(), dry_run: z.boolean().optional(), ...REPO_ARG },
|
|
545
554
|
}, async (args) => {
|
|
546
555
|
try {
|
|
547
556
|
const { repoId, rootPath } = repoScope(args.repo);
|
|
548
557
|
const root = requireRoot(rootPath);
|
|
549
558
|
const { parent, docId } = resolveParentRef(repoId, args.to);
|
|
550
559
|
const to = { parent, at: resolveAt(repoId, args.at) };
|
|
551
|
-
// A document parent carries no block to infer the doc from; pin it.
|
|
552
|
-
|
|
560
|
+
// A document parent carries no block to infer the doc from; pin it. The
|
|
561
|
+
// caller's expect rides through as the op-level expectation (spec/mutate
|
|
562
|
+
// §1.2: checked against the destination parent before the op mutates).
|
|
563
|
+
const ops = [{ op: "insert", ...(typeof parent === "object" ? { doc: docId } : {}), to, markdown: args.markdown, ...(args.expect ? { expect: args.expect } : {}) }];
|
|
553
564
|
return applyOps(repoId, root, ops, "blocks_insert", args.dry_run);
|
|
554
565
|
}
|
|
555
566
|
catch (e) {
|
|
@@ -583,8 +594,8 @@ export function buildServer(ctx) {
|
|
|
583
594
|
}
|
|
584
595
|
});
|
|
585
596
|
server.registerTool("blocks_move", {
|
|
586
|
-
description: "Move block(s) under a new parent at a position. `blocks` are block refs; `to` (the parent) is a block ref, or the blocks' OWN document (id or path) to move them to its top level — moving to ANOTHER document's root is not expressible (target_missing): anchor on a block in that document with `at` {before|after} instead. `at` is \"end\"/\"start\"/{before|after: <ref>}. One move op through the kernel writer.",
|
|
587
|
-
inputSchema: { blocks: z.array(z.string()), to: z.string(), at: atSchema.optional(), dry_run: z.boolean().optional(), ...REPO_ARG },
|
|
597
|
+
description: "Move block(s) under a new parent at a position. `blocks` are block refs; `to` (the parent) is a block ref, or the blocks' OWN document (id or path) to move them to its top level — moving to ANOTHER document's root is not expressible (target_missing): anchor on a block in that document with `at` {before|after} instead. `at` is \"end\"/\"start\"/{before|after: <ref>}. Optional `expect.parent_children_hash` is the destination-parent CAS (as blocks_insert: the destination parent's CURRENT direct child ids joined by \",\", sha256 hex), checked once for the whole run before anything moves — `stale_expectation` with `data.current.parent_children_hash` if the destination's children changed; `content_hash` is meaningless here and not accepted. One move op through the kernel writer.",
|
|
598
|
+
inputSchema: { blocks: z.array(z.string()), to: z.string(), at: atSchema.optional(), expect: parentExpectSchema.optional(), dry_run: z.boolean().optional(), ...REPO_ARG },
|
|
588
599
|
}, async (args) => {
|
|
589
600
|
try {
|
|
590
601
|
const { repoId, rootPath } = repoScope(args.repo);
|
|
@@ -597,7 +608,7 @@ export function buildServer(ctx) {
|
|
|
597
608
|
throw new EngineError("target_missing", `blocks_move cannot target another document's root (${args.to}); anchor on a block in that document with at.before/at.after`, { data: { to: args.to } });
|
|
598
609
|
}
|
|
599
610
|
const to = { parent, at: resolveAt(repoId, args.at) };
|
|
600
|
-
const ops = [{ op: "move", blocks, to }];
|
|
611
|
+
const ops = [{ op: "move", blocks, to, ...(args.expect ? { expect: args.expect } : {}) }];
|
|
601
612
|
return applyOps(repoId, root, ops, "blocks_move", args.dry_run);
|
|
602
613
|
}
|
|
603
614
|
catch (e) {
|
|
@@ -20,4 +20,15 @@ export declare function makeStoreContext(store: Store, repoId: string, opts?: St
|
|
|
20
20
|
/** Tag SQL rows with their target so the store context resolves them (used by the
|
|
21
21
|
* tier-3 planner to hand produced rows back for residual evaluation). */
|
|
22
22
|
export declare function tagRows(rows: Record<string, unknown>[], target: Target): unknown[];
|
|
23
|
+
/**
|
|
24
|
+
* A store row surfacing as a VALUE in a result (spec/surface §1.4, rows as
|
|
25
|
+
* values, 1.2) is rendered as `{ id, path }` — its id column and its document's
|
|
26
|
+
* path, the same two keys every hit carries — never the store row itself.
|
|
27
|
+
* Returns undefined for anything that is not a tagged row (run.ts leaves those
|
|
28
|
+
* alone).
|
|
29
|
+
*/
|
|
30
|
+
export declare function rowRef(value: unknown): {
|
|
31
|
+
id: string;
|
|
32
|
+
path: string;
|
|
33
|
+
} | undefined;
|
|
23
34
|
export { TARGET };
|
|
@@ -610,5 +610,21 @@ function hex(b) {
|
|
|
610
610
|
export function tagRows(rows, target) {
|
|
611
611
|
return tagAll(rows, target);
|
|
612
612
|
}
|
|
613
|
+
/**
|
|
614
|
+
* A store row surfacing as a VALUE in a result (spec/surface §1.4, rows as
|
|
615
|
+
* values, 1.2) is rendered as `{ id, path }` — its id column and its document's
|
|
616
|
+
* path, the same two keys every hit carries — never the store row itself.
|
|
617
|
+
* Returns undefined for anything that is not a tagged row (run.ts leaves those
|
|
618
|
+
* alone).
|
|
619
|
+
*/
|
|
620
|
+
export function rowRef(value) {
|
|
621
|
+
const t = value?.[TARGET];
|
|
622
|
+
if (!t)
|
|
623
|
+
return undefined;
|
|
624
|
+
const r = value;
|
|
625
|
+
const id = t === "docs" ? r.doc_id : t === "blocks" ? r.block_id : t === "nodes" ? r.node_id : r.edge_id;
|
|
626
|
+
const path = t === "docs" ? r.path : r.__path;
|
|
627
|
+
return { id: String(id), path: String(path ?? "") };
|
|
628
|
+
}
|
|
613
629
|
export { TARGET };
|
|
614
630
|
//# sourceMappingURL=context.js.map
|
|
@@ -1,6 +1,25 @@
|
|
|
1
|
-
import type { Query, QueryPlanner, Plan } from "@omgbase/oqx";
|
|
1
|
+
import type { Query, Where, QueryPlanner, Plan } from "@omgbase/oqx";
|
|
2
2
|
import type { Store } from "../core/store/store.js";
|
|
3
3
|
import { type StoreContextOptions } from "./context.js";
|
|
4
|
+
import { type Target } from "./sql/translate.js";
|
|
5
|
+
/**
|
|
6
|
+
* True when evaluating `where` in memory could raise an OQX error the pushed
|
|
7
|
+
* conjuncts might hide by emptying the scan. The walk is generic over the whole
|
|
8
|
+
* residual tree — every `Where` node, every nested block (its `from`, `where`,
|
|
9
|
+
* `select`, `order by`, `follow`, `limit`/`offset`) and every `Expr` — and flags:
|
|
10
|
+
*
|
|
11
|
+
* • any `call` node (a function or method: unknown name, bad regex, wrong target…)
|
|
12
|
+
* • any nested block whose consumer is `single` (raises on 0 or >1 rows)
|
|
13
|
+
* • any `^`-escaped name: an `outer` reference or a `^name:` lift
|
|
14
|
+
* • a bare reserved docs basename (`id path updated_at content_hash body`): as
|
|
15
|
+
* an `ident` at the ROOT scope when the target is `docs`, or as `doc.<name>`
|
|
16
|
+
* on any target at any depth (the reach-through row is a doc)
|
|
17
|
+
*
|
|
18
|
+
* A residual made only of comparisons, logical operators, `in`/ranges, `!`,
|
|
19
|
+
* literals, bindings and plain reads (including exists/none/count/collect/first
|
|
20
|
+
* blocks of the same) cannot raise, so the pushed conjuncts are kept.
|
|
21
|
+
*/
|
|
22
|
+
export declare function residualMayRaise(where: Where | null, target: Target): boolean;
|
|
4
23
|
export declare class SQLiteQueryPlanner implements QueryPlanner {
|
|
5
24
|
private store;
|
|
6
25
|
private repoId;
|
|
@@ -7,10 +7,16 @@
|
|
|
7
7
|
//
|
|
8
8
|
// This is deliberately conservative: only simple top-level target scans (no
|
|
9
9
|
// `follow`, no source re-projection) with at least one pushable scalar conjunct
|
|
10
|
-
// are planned; everything else declines to a full in-memory run.
|
|
10
|
+
// are planned; everything else declines to a full in-memory run. One more
|
|
11
|
+
// decline keeps the planned path invisible (spec/surface §1 decline (c)): a
|
|
12
|
+
// pushed conjunct that empties the scan would also hide an OQX eval error a
|
|
13
|
+
// residual conjunct raises in memory (`path == "x" && $path == "nope.md"` was
|
|
14
|
+
// `[]` planned, `filter_invalid` in memory), so when any residual conjunct could
|
|
15
|
+
// raise — a function or method call, a `single` block, a `^`-escaped name, or a
|
|
16
|
+
// bare reserved docs basename — the WHOLE query runs unplanned.
|
|
11
17
|
import { ROWS_ROOT, partitionPushable, residualQuery } from "@omgbase/oqx";
|
|
12
18
|
import { makeStoreContext, tagRows } from "./context.js";
|
|
13
|
-
import { translatePredicate } from "./sql/translate.js";
|
|
19
|
+
import { translatePredicate, RESERVED_DOC_BASENAMES } from "./sql/translate.js";
|
|
14
20
|
const ALIAS = {
|
|
15
21
|
docs: { self: "d", doc: "d" },
|
|
16
22
|
blocks: { self: "b", doc: "d" },
|
|
@@ -56,6 +62,91 @@ function rootTarget(source) {
|
|
|
56
62
|
}
|
|
57
63
|
return null;
|
|
58
64
|
}
|
|
65
|
+
// ---- the residual walk (decline (c)) -----------------------------------------
|
|
66
|
+
/**
|
|
67
|
+
* True when evaluating `where` in memory could raise an OQX error the pushed
|
|
68
|
+
* conjuncts might hide by emptying the scan. The walk is generic over the whole
|
|
69
|
+
* residual tree — every `Where` node, every nested block (its `from`, `where`,
|
|
70
|
+
* `select`, `order by`, `follow`, `limit`/`offset`) and every `Expr` — and flags:
|
|
71
|
+
*
|
|
72
|
+
* • any `call` node (a function or method: unknown name, bad regex, wrong target…)
|
|
73
|
+
* • any nested block whose consumer is `single` (raises on 0 or >1 rows)
|
|
74
|
+
* • any `^`-escaped name: an `outer` reference or a `^name:` lift
|
|
75
|
+
* • a bare reserved docs basename (`id path updated_at content_hash body`): as
|
|
76
|
+
* an `ident` at the ROOT scope when the target is `docs`, or as `doc.<name>`
|
|
77
|
+
* on any target at any depth (the reach-through row is a doc)
|
|
78
|
+
*
|
|
79
|
+
* A residual made only of comparisons, logical operators, `in`/ranges, `!`,
|
|
80
|
+
* literals, bindings and plain reads (including exists/none/count/collect/first
|
|
81
|
+
* blocks of the same) cannot raise, so the pushed conjuncts are kept.
|
|
82
|
+
*/
|
|
83
|
+
export function residualMayRaise(where, target) {
|
|
84
|
+
return where !== null && whereMayRaise(where, target, true);
|
|
85
|
+
}
|
|
86
|
+
function whereMayRaise(w, target, root) {
|
|
87
|
+
switch (w.kind) {
|
|
88
|
+
case "and":
|
|
89
|
+
case "or": return w.parts.some((p) => whereMayRaise(p, target, root));
|
|
90
|
+
case "not": return whereMayRaise(w.expr, target, root);
|
|
91
|
+
case "scalar": return exprMayRaise(w.expr, target, root);
|
|
92
|
+
case "op": return opMayRaise(w, target, root);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
function opMayRaise(op, target, root) {
|
|
96
|
+
if (op.op === "single")
|
|
97
|
+
return true;
|
|
98
|
+
return exprMayRaise(op.receiver, target, root) || subMayRaise(op.sub, target);
|
|
99
|
+
}
|
|
100
|
+
// Inside a block the rows are a different scope (a relation's rows), so a bare
|
|
101
|
+
// reserved name is an ordinary read there — only `doc.<reserved>` still raises.
|
|
102
|
+
function subMayRaise(s, target) {
|
|
103
|
+
if (s.from.some((e) => exprMayRaise(e, target, false)))
|
|
104
|
+
return true;
|
|
105
|
+
if (s.where && whereMayRaise(s.where, target, false))
|
|
106
|
+
return true;
|
|
107
|
+
for (const item of s.select) {
|
|
108
|
+
if (item.kind === "field") {
|
|
109
|
+
if (item.lift > 0 || exprMayRaise(item.expr, target, false))
|
|
110
|
+
return true;
|
|
111
|
+
}
|
|
112
|
+
else if (opMayRaise(item.op, target, false))
|
|
113
|
+
return true;
|
|
114
|
+
}
|
|
115
|
+
if (s.orderBy?.some((o) => exprMayRaise(o.expr, target, false)))
|
|
116
|
+
return true;
|
|
117
|
+
if (s.follow && followMayRaise(s.follow, target))
|
|
118
|
+
return true;
|
|
119
|
+
if (s.limit && exprMayRaise(s.limit, target, false))
|
|
120
|
+
return true;
|
|
121
|
+
if (s.offset && exprMayRaise(s.offset, target, false))
|
|
122
|
+
return true;
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
function followMayRaise(f, target) {
|
|
126
|
+
return [f.receiver, f.where, f.frontier, f.by].some((e) => e !== null && exprMayRaise(e, target, false));
|
|
127
|
+
}
|
|
128
|
+
function exprMayRaise(e, target, root) {
|
|
129
|
+
switch (e.kind) {
|
|
130
|
+
case "call": return true;
|
|
131
|
+
case "outer": return true;
|
|
132
|
+
case "ident": return root && target === "docs" && RESERVED_DOC_BASENAMES.has(e.name);
|
|
133
|
+
case "member":
|
|
134
|
+
if (e.recv.kind === "ident" && e.recv.name === "doc" && RESERVED_DOC_BASENAMES.has(e.name))
|
|
135
|
+
return true;
|
|
136
|
+
return exprMayRaise(e.recv, target, root);
|
|
137
|
+
case "index": return exprMayRaise(e.recv, target, root) || exprMayRaise(e.index, target, root);
|
|
138
|
+
case "unary": return exprMayRaise(e.expr, target, root);
|
|
139
|
+
case "binary":
|
|
140
|
+
case "logical":
|
|
141
|
+
case "in":
|
|
142
|
+
return exprMayRaise(e.left, target, root) || exprMayRaise(e.right, target, root);
|
|
143
|
+
case "range":
|
|
144
|
+
return (e.lo !== null && exprMayRaise(e.lo, target, root)) || (e.hi !== null && exprMayRaise(e.hi, target, root));
|
|
145
|
+
case "lit":
|
|
146
|
+
case "binding": return false;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
// ---- the planner -------------------------------------------------------------
|
|
59
150
|
export class SQLiteQueryPlanner {
|
|
60
151
|
store;
|
|
61
152
|
repoId;
|
|
@@ -76,6 +167,8 @@ export class SQLiteQueryPlanner {
|
|
|
76
167
|
const { pushed, residual } = partitionPushable(query.where, (e) => translatePredicate(e, ctx) !== null);
|
|
77
168
|
if (pushed.length === 0)
|
|
78
169
|
return null; // nothing to push — let the engine do it all
|
|
170
|
+
if (residualMayRaise(residual, target))
|
|
171
|
+
return null; // decline (c): run unplanned
|
|
79
172
|
const frags = pushed.map((e) => translatePredicate(e, ctx));
|
|
80
173
|
const g = guards(target, this.repoId);
|
|
81
174
|
const whereSql = [g.sql, ...frags.map((f) => `(${f.sql})`)].join(" AND ");
|
package/dist/src/oqx-js/run.js
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// keyset pagination, count/exists scalars). Same signature + shape as the former
|
|
5
5
|
// in-tree compiler, so every caller and the corpus are unchanged.
|
|
6
6
|
import { parse, PlannedEngine, InMemoryEngine, OqxError } from "@omgbase/oqx";
|
|
7
|
-
import { makeStoreContext } from "./context.js";
|
|
7
|
+
import { makeStoreContext, rowRef } from "./context.js";
|
|
8
8
|
import { SQLiteQueryPlanner } from "./planner.js";
|
|
9
9
|
import { FilterInvalid } from "../search/cel/parser.js";
|
|
10
10
|
import { float32ToBlob } from "../core/vec.js";
|
|
@@ -194,10 +194,35 @@ const PATH_ITEM = { kind: "field", name: "__oqx_path", expr: { kind: "ident", na
|
|
|
194
194
|
// so it rides through id/path injection, keyset paging, and distinct as an
|
|
195
195
|
// ordinary record field and is peeled off at the end.
|
|
196
196
|
const VALUE_KEY = "__oqx_value";
|
|
197
|
+
// Rows as values (spec/surface §1.4, 1.2). The engine hands back store rows
|
|
198
|
+
// wherever the projection made a row a VALUE — a nested `collect { }` /
|
|
199
|
+
// `first { }` / `single { }` with an empty projection ("the row itself",
|
|
200
|
+
// spec/oqx §12), a `values` projection of `$value`/`$self`, a field bound to a
|
|
201
|
+
// row — and a store row is not a wire shape (column names, `attrs` as a JSON
|
|
202
|
+
// string, the `__path` join column, Buffers). Walk the projected value and
|
|
203
|
+
// render every tagged row as `{ id, path }`; everything else passes through
|
|
204
|
+
// untouched (plain records and arrays are descended, scalars kept). Hits are
|
|
205
|
+
// records the engine built from the injected id/path items, never rows.
|
|
206
|
+
function renderValue(v) {
|
|
207
|
+
if (v === null || typeof v !== "object")
|
|
208
|
+
return v;
|
|
209
|
+
const ref = rowRef(v);
|
|
210
|
+
if (ref)
|
|
211
|
+
return ref;
|
|
212
|
+
if (Array.isArray(v))
|
|
213
|
+
return v.map(renderValue);
|
|
214
|
+
const proto = Object.getPrototypeOf(v);
|
|
215
|
+
if (proto !== Object.prototype && proto !== null)
|
|
216
|
+
return v; // Buffer, Map, … — not a record
|
|
217
|
+
const out = {};
|
|
218
|
+
for (const [k, x] of Object.entries(v))
|
|
219
|
+
out[k] = renderValue(x);
|
|
220
|
+
return out;
|
|
221
|
+
}
|
|
197
222
|
function toHit(row) {
|
|
198
223
|
const o = row;
|
|
199
224
|
const { __oqx_id, __oqx_path, ...rest } = o;
|
|
200
|
-
return { id: String(__oqx_id), path: String(__oqx_path ?? ""), ...rest };
|
|
225
|
+
return { id: String(__oqx_id), path: String(__oqx_path ?? ""), ...renderValue(rest) };
|
|
201
226
|
}
|
|
202
227
|
// Top-level `select distinct`: dedup hits by their USER projection (every field
|
|
203
228
|
// except the injected id/path), keeping the first (so the retained hit carries a
|