@omgbase/core 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +172 -0
- package/dist/src/cli/eval-matcher.d.ts +6 -1
- package/dist/src/cli/eval-matcher.js +14 -2
- package/dist/src/core/clock.d.ts +9 -0
- package/dist/src/core/clock.js +46 -0
- package/dist/src/core/hash.d.ts +25 -1
- package/dist/src/core/hash.js +110 -10
- package/dist/src/core/ids.d.ts +12 -0
- package/dist/src/core/ids.js +43 -1
- package/dist/src/core/ingest.d.ts +22 -2
- package/dist/src/core/ingest.js +92 -18
- package/dist/src/core/parse/parse.d.ts +1 -1
- package/dist/src/core/parse/parse.js +31 -9
- package/dist/src/core/store/nodes.d.ts +13 -0
- package/dist/src/core/store/nodes.js +24 -0
- package/dist/src/core/store/writers.d.ts +6 -0
- package/dist/src/core/store/writers.js +1 -0
- package/dist/src/core/utf8.d.ts +29 -0
- package/dist/src/core/utf8.js +96 -0
- package/dist/src/format/adapter.d.ts +7 -1
- package/dist/src/format/markdown.js +36 -41
- package/dist/src/graph/extract.d.ts +41 -1
- package/dist/src/graph/extract.js +59 -1
- package/dist/src/graph/history.d.ts +39 -1
- package/dist/src/graph/history.js +154 -20
- package/dist/src/index.d.ts +10 -6
- package/dist/src/index.js +11 -6
- package/dist/src/mcp/graph.js +8 -4
- package/dist/src/mcp/reference.d.ts +1 -1
- package/dist/src/mcp/reference.js +17 -6
- package/dist/src/mcp/server.js +15 -10
- package/dist/src/mutate/apply.d.ts +9 -0
- package/dist/src/mutate/apply.js +24 -11
- package/dist/src/mutate/docs.d.ts +2 -0
- package/dist/src/mutate/docs.js +10 -6
- package/dist/src/mutate/known-ids.js +11 -3
- package/dist/src/mutate/macros.js +9 -2
- package/dist/src/mutate/ops.d.ts +5 -4
- package/dist/src/mutate/ops.js +62 -15
- package/dist/src/mutate/plan-update.d.ts +7 -1
- package/dist/src/mutate/plan-update.js +3 -1
- package/dist/src/oqx-js/context.js +57 -7
- package/dist/src/reconcile/eval/generator.d.ts +24 -1
- package/dist/src/reconcile/eval/generator.js +248 -0
- package/dist/src/reconcile/eval/metrics.d.ts +33 -9
- package/dist/src/reconcile/eval/metrics.js +55 -32
- package/dist/src/reconcile/flatten.js +7 -5
- package/dist/src/reconcile/phase5.d.ts +2 -1
- package/dist/src/reconcile/phase5.js +27 -10
- package/dist/src/reconcile/phase6a.js +17 -5
- package/dist/src/reconcile/phases.d.ts +44 -6
- package/dist/src/reconcile/phases.js +0 -0
- package/dist/src/reconcile/reconcile.js +16 -9
- package/dist/src/reconcile/types.d.ts +4 -1
- package/dist/src/reconcile/types.js +11 -2
- package/dist/src/search/external.d.ts +10 -0
- package/dist/src/search/external.js +18 -1
- package/dist/src/search/provider.d.ts +7 -3
- package/dist/src/search/provider.js +8 -7
- package/dist/src/search/rrf.js +4 -0
- package/dist/src/search/tasks.d.ts +28 -0
- package/dist/src/search/tasks.js +36 -23
- package/dist/src/search/vector.js +30 -14
- package/dist/src/sync/attach.d.ts +7 -2
- package/dist/src/sync/attach.js +10 -8
- package/dist/src/sync/checkpoint.d.ts +18 -1
- package/dist/src/sync/checkpoint.js +31 -29
- package/dist/src/sync/driver.d.ts +15 -4
- package/dist/src/sync/driver.js +18 -43
- package/dist/src/sync/external-source.js +7 -1
- package/dist/src/sync/freshness.d.ts +73 -15
- package/dist/src/sync/freshness.js +128 -97
- package/dist/src/sync/fs-util.d.ts +19 -0
- package/dist/src/sync/fs-util.js +24 -1
- package/dist/src/sync/observe.d.ts +53 -5
- package/dist/src/sync/observe.js +162 -21
- package/dist/src/sync/reconciling-ingest.d.ts +42 -2
- package/dist/src/sync/reconciling-ingest.js +87 -13
- package/dist/src/sync/recovery.d.ts +5 -1
- package/dist/src/sync/recovery.js +8 -8
- package/dist/src/sync/workspace.d.ts +11 -3
- package/dist/src/sync/workspace.js +39 -30
- package/dist/src/sync/writer-lock.js +16 -26
- package/package.json +3 -3
package/dist/src/mutate/ops.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { mintId } from "../core/ids.js";
|
|
2
2
|
import { parseTree } from "../core/parse/tree.js";
|
|
3
3
|
import { adapterForFormat } from "../format/index.js";
|
|
4
|
+
import { codeUnitIndexOf } from "../core/utf8.js";
|
|
4
5
|
import { MutationError, locate, rawHashHex, parentChildrenHash, markContainerDirty, ownerOf } from "./tree.js";
|
|
5
6
|
function defaultTrivia(format) {
|
|
6
7
|
if (format === "json")
|
|
@@ -112,6 +113,20 @@ function checkContentHash(block, expect, opIndex) {
|
|
|
112
113
|
});
|
|
113
114
|
}
|
|
114
115
|
}
|
|
116
|
+
// §1.2 `check_parent_children_hash`: when the expectation carries a
|
|
117
|
+
// `parent_children_hash`, the LIVE direct children of the sibling list it names
|
|
118
|
+
// (the document top level for `doc.children`) must hash to it. Which list is the
|
|
119
|
+
// op's business: update/split/remove/merge pass the block's current sibling list,
|
|
120
|
+
// insert/move the destination's (resolved as the op resolves `to`). Always after
|
|
121
|
+
// the content CAS and before the tree is touched, so a stale order never lands.
|
|
122
|
+
export function checkParentChildrenHash(siblings, expect, opIndex) {
|
|
123
|
+
if (expect?.parent_children_hash === undefined)
|
|
124
|
+
return;
|
|
125
|
+
const current = parentChildrenHash(siblings);
|
|
126
|
+
if (current !== expect.parent_children_hash) {
|
|
127
|
+
throw new MutationError("stale_expectation", "parent_children_hash mismatch", { op_index: opIndex, current: { parent_children_hash: current }, retriable: true });
|
|
128
|
+
}
|
|
129
|
+
}
|
|
115
130
|
// Assign known ids onto a freshly-parsed subtree by positional key (parentKey +
|
|
116
131
|
// '/' + index), the same convention flatten()/known-ids use. Keys absent from
|
|
117
132
|
// the map leave the block's minted id in place (a genuinely new child).
|
|
@@ -139,8 +154,11 @@ function markSubtreeClean(b) {
|
|
|
139
154
|
markSubtreeClean(c);
|
|
140
155
|
}
|
|
141
156
|
// ---- the six ops ------------------------------------------------------------
|
|
142
|
-
export function opInsert(doc, to, markdown) {
|
|
157
|
+
export function opInsert(doc, to, markdown, expect, opIndex = 0) {
|
|
143
158
|
const { siblings, index } = resolveTarget(doc, to);
|
|
159
|
+
// §1.2: the destination parent's order CAS — `siblings` IS the resolved parent
|
|
160
|
+
// list (a section scope or a `{ doc: true }` anchor both name the top level).
|
|
161
|
+
checkParentChildrenHash(siblings, expect, opIndex);
|
|
144
162
|
let blocks = parseContentToBlocks(markdown, doc.format);
|
|
145
163
|
// Inserting into a list: the content must land as list ITEMS, not a nested
|
|
146
164
|
// list. Markdown `- x` parses to a `list` wrapping a `list_item`; unwrap it to
|
|
@@ -203,6 +221,8 @@ export function opUpdate(doc, blockId, opIndex, markdown, attrs, expect, trivia,
|
|
|
203
221
|
// juggling post-op content hashes across the changeset.
|
|
204
222
|
if (markdown !== undefined || expect?.content_hash !== undefined)
|
|
205
223
|
checkContentHash(found.block, expect, opIndex);
|
|
224
|
+
// §1.2: the block's CURRENT parent, after the content CAS.
|
|
225
|
+
checkParentChildrenHash(found.siblings, expect, opIndex);
|
|
206
226
|
// Blocks minted from MULTI-block content: the target takes the first parsed
|
|
207
227
|
// block (keeping its id); the rest become fresh siblings right after it.
|
|
208
228
|
const extra = [];
|
|
@@ -308,7 +328,7 @@ export function opUpdate(doc, blockId, opIndex, markdown, attrs, expect, trivia,
|
|
|
308
328
|
}
|
|
309
329
|
return { ids: [blockId, ...extra.map((b) => b.id)] };
|
|
310
330
|
}
|
|
311
|
-
export function opMove(doc, blockIds, to, opIndex) {
|
|
331
|
+
export function opMove(doc, blockIds, to, opIndex, expect) {
|
|
312
332
|
if (blockIds.length === 0)
|
|
313
333
|
throw new MutationError("not_contiguous", "move requires ≥1 block");
|
|
314
334
|
// Locate all; assert contiguous siblings.
|
|
@@ -332,6 +352,12 @@ export function opMove(doc, blockIds, to, opIndex) {
|
|
|
332
352
|
if (typeof to.parent === "string" && movedIds.has(to.parent)) {
|
|
333
353
|
throw new MutationError("cycle_move", "target is inside the moved subtree", { op_index: opIndex });
|
|
334
354
|
}
|
|
355
|
+
// §1.2: the DESTINATION parent's order CAS, resolved on the pre-removal tree
|
|
356
|
+
// (before the op mutates anything) and checked once for the whole run. When
|
|
357
|
+
// source and destination are one list this is the "same siblings reordered
|
|
358
|
+
// under me" guard: the hash covers the moved blocks' current positions too.
|
|
359
|
+
if (expect?.parent_children_hash !== undefined)
|
|
360
|
+
checkParentChildrenHash(resolveTarget(doc, to).siblings, expect, opIndex);
|
|
335
361
|
// Extract (preserve order) then re-insert at the target.
|
|
336
362
|
const moving = located.map((l) => l.block);
|
|
337
363
|
for (const m of moving) {
|
|
@@ -363,7 +389,7 @@ export function opMove(doc, blockIds, to, opIndex) {
|
|
|
363
389
|
// document is already well-formed) and never touches the last block's trailing
|
|
364
390
|
// trivia (that is the document's trailing bytes). Exact target trivia is the
|
|
365
391
|
// planner's job via `update { trivia }`; this only guarantees well-formedness.
|
|
366
|
-
function healTopLevelSeams(children, format) {
|
|
392
|
+
export function healTopLevelSeams(children, format) {
|
|
367
393
|
const sep = defaultTrivia(format);
|
|
368
394
|
for (let i = 0; i < children.length - 1; i++) {
|
|
369
395
|
const b = children[i];
|
|
@@ -394,8 +420,11 @@ export function opRemove(doc, blockIds, opIndex, expectPer) {
|
|
|
394
420
|
hint: "the id is not a live block of the targeted document — it may have been removed by an earlier op in this changeset (removing a block removes its whole subtree), or never existed",
|
|
395
421
|
});
|
|
396
422
|
}
|
|
397
|
-
|
|
398
|
-
|
|
423
|
+
const e = expectPer?.[id];
|
|
424
|
+
if (e) {
|
|
425
|
+
checkContentHash(found.block, e, opIndex);
|
|
426
|
+
checkParentChildrenHash(found.siblings, e, opIndex); // §1.2: its current parent
|
|
427
|
+
}
|
|
399
428
|
if (!hasAncestorIn(doc, found.siblings, set))
|
|
400
429
|
tops.push(id);
|
|
401
430
|
}
|
|
@@ -453,8 +482,14 @@ export function opSplit(doc, blockId, at, opIndex, expect) {
|
|
|
453
482
|
if (!found)
|
|
454
483
|
throw new MutationError("block_missing", `block ${blockId} not found`, { op_index: opIndex, block: blockId });
|
|
455
484
|
checkContentHash(found.block, expect, opIndex);
|
|
485
|
+
checkParentChildrenHash(found.siblings, expect, opIndex); // §1.2: its current parent
|
|
456
486
|
const raw = found.block.raw;
|
|
457
|
-
|
|
487
|
+
// `at` are UTF-8 byte offsets into the raw (spec/mutate §2.5) — the same
|
|
488
|
+
// language-neutral unit every persisted span uses. A JavaScript string slices
|
|
489
|
+
// at UTF-16 code units, so convert each cut first; an offset inside a
|
|
490
|
+
// multi-byte sequence rounds down to that character's start, one past the end
|
|
491
|
+
// clamps (core/utf8.ts).
|
|
492
|
+
const cuts = [0, ...at.map((b) => codeUnitIndexOf(raw, b)), raw.length].sort((a, b) => a - b);
|
|
458
493
|
const pieces = [];
|
|
459
494
|
for (let i = 1; i < cuts.length; i++)
|
|
460
495
|
pieces.push(raw.slice(cuts[i - 1], cuts[i]));
|
|
@@ -467,7 +502,23 @@ export function opSplit(doc, blockId, at, opIndex, expect) {
|
|
|
467
502
|
const newBlocks = nonEmpty.slice(1).map((p) => ({
|
|
468
503
|
id: mintId("b"), type: found.block.type, raw: p, trivia: "\n\n", attrs: { ...found.block.attrs }, children: [], dirty: true,
|
|
469
504
|
}));
|
|
505
|
+
// Seams exactly as opUpdate's extra-sibling path (spec/mutate §2.5): the
|
|
506
|
+
// target's trailing trivia is the document's tiling AFTER the whole run, so it
|
|
507
|
+
// moves to the last piece; at the top level every earlier piece (the first
|
|
508
|
+
// included) must separate blocks — a last block's lone "\n" would otherwise
|
|
509
|
+
// soft-join the pieces into one paragraph and lose the minted ids.
|
|
510
|
+
const tail = found.block.trivia;
|
|
511
|
+
if (found.siblings === doc.children) {
|
|
512
|
+
const sep = defaultTrivia(doc.format);
|
|
513
|
+
if (!separatesBlocks(found.block.trivia, doc.format))
|
|
514
|
+
found.block.trivia = sep;
|
|
515
|
+
for (const b of newBlocks.slice(0, -1))
|
|
516
|
+
if (!separatesBlocks(b.trivia, doc.format))
|
|
517
|
+
b.trivia = sep;
|
|
518
|
+
}
|
|
519
|
+
newBlocks[newBlocks.length - 1].trivia = tail;
|
|
470
520
|
found.siblings.splice(found.index + 1, 0, ...newBlocks);
|
|
521
|
+
markContainerDirty(doc, found.siblings);
|
|
471
522
|
return { ids: [blockId, ...newBlocks.map((b) => b.id)] };
|
|
472
523
|
}
|
|
473
524
|
export function opMerge(doc, blockIds, opIndex, separator = " ", expectPer) {
|
|
@@ -487,8 +538,11 @@ export function opMerge(doc, blockIds, opIndex, separator = " ", expectPer) {
|
|
|
487
538
|
throw new MutationError("not_contiguous", "merge blocks must share a parent", { op_index: opIndex });
|
|
488
539
|
if (located[i].block.type !== type)
|
|
489
540
|
throw new MutationError("type_mismatch", "merge blocks must share a type", { op_index: opIndex });
|
|
490
|
-
|
|
491
|
-
|
|
541
|
+
const e = expectPer?.[blockIds[i]];
|
|
542
|
+
if (e) {
|
|
543
|
+
checkContentHash(located[i].block, e, opIndex);
|
|
544
|
+
checkParentChildrenHash(located[i].siblings, e, opIndex); // §1.2: its current parent
|
|
545
|
+
}
|
|
492
546
|
}
|
|
493
547
|
for (let i = 1; i < indices.length; i++)
|
|
494
548
|
if (indices[i] !== indices[i - 1] + 1)
|
|
@@ -503,11 +557,4 @@ export function opMerge(doc, blockIds, opIndex, separator = " ", expectPer) {
|
|
|
503
557
|
siblings.splice(indices[i], 1);
|
|
504
558
|
return { ids: [first.id], mergedInto };
|
|
505
559
|
}
|
|
506
|
-
export function checkParentChildrenHash(doc, parentId, expected, opIndex) {
|
|
507
|
-
const list = parentId ? locate(doc, parentId)?.block.children ?? doc.children : doc.children;
|
|
508
|
-
const current = parentChildrenHash(list);
|
|
509
|
-
if (current !== expected) {
|
|
510
|
-
throw new MutationError("stale_expectation", "parent_children_hash mismatch", { op_index: opIndex, current: { parent_children_hash: current }, retriable: true });
|
|
511
|
-
}
|
|
512
|
-
}
|
|
513
560
|
//# sourceMappingURL=ops.js.map
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { Store } from "../core/store/store.js";
|
|
2
2
|
import { type ReconcileConfig } from "../reconcile/types.js";
|
|
3
3
|
import { type ApplyResult } from "./apply.js";
|
|
4
|
+
import type { DocStore } from "./doc-store.js";
|
|
4
5
|
import { type Opset } from "./opset.js";
|
|
5
6
|
export interface PlanUpdateOptions {
|
|
6
7
|
config?: ReconcileConfig;
|
|
@@ -13,7 +14,10 @@ export interface PlanUpdateOptions {
|
|
|
13
14
|
export declare function planUpdate(store: Store, repoId: string, rootPath: string, docRef: string, content: string, opts?: PlanUpdateOptions): Opset;
|
|
14
15
|
export interface ApplyOpsetRequest {
|
|
15
16
|
repoId: string;
|
|
16
|
-
|
|
17
|
+
/** Working-tree root for the default filesystem write target; omit only with an explicit `docStore`. */
|
|
18
|
+
rootPath?: string;
|
|
19
|
+
/** Write target (ADR-014 §5); defaults to a filesystem store at `rootPath`. */
|
|
20
|
+
docStore?: DocStore;
|
|
17
21
|
opset: Opset;
|
|
18
22
|
origin: {
|
|
19
23
|
actor: string;
|
|
@@ -21,6 +25,8 @@ export interface ApplyOpsetRequest {
|
|
|
21
25
|
};
|
|
22
26
|
dryRun?: boolean;
|
|
23
27
|
omgbaseDir?: string;
|
|
28
|
+
/** Commit timestamp (RFC 3339 UTC); defaults to now (see `ApplyRequest.ts`). */
|
|
29
|
+
ts?: string;
|
|
24
30
|
}
|
|
25
31
|
/**
|
|
26
32
|
* Apply an opset through the kernel write path. Validates the plan's
|
|
@@ -159,11 +159,13 @@ export function applyOpset(store, req) {
|
|
|
159
159
|
}
|
|
160
160
|
return apply(store, {
|
|
161
161
|
repoId: req.repoId,
|
|
162
|
-
rootPath: req.rootPath,
|
|
162
|
+
...(req.rootPath !== undefined ? { rootPath: req.rootPath } : {}),
|
|
163
|
+
...(req.docStore !== undefined ? { docStore: req.docStore } : {}),
|
|
163
164
|
ops: opsetKernelOps(opset),
|
|
164
165
|
origin: req.origin,
|
|
165
166
|
...(req.dryRun !== undefined ? { dryRun: req.dryRun } : {}),
|
|
166
167
|
...(req.omgbaseDir !== undefined ? { omgbaseDir: req.omgbaseDir } : {}),
|
|
168
|
+
...(req.ts !== undefined ? { ts: req.ts } : {}),
|
|
167
169
|
...(opset.frontmatter ? { setFrontmatter: [{ doc: opset.precondition.doc, raw: opset.frontmatter.raw }] } : {}),
|
|
168
170
|
});
|
|
169
171
|
}
|
|
@@ -17,6 +17,8 @@ import { docsRead } from "../core/read/document.js";
|
|
|
17
17
|
import { FilterInvalid } from "../search/cel/parser.js";
|
|
18
18
|
import { sanitizeFtsQuery } from "../search/fts-query.js";
|
|
19
19
|
import { cosineBytes } from "../core/vec.js";
|
|
20
|
+
import { ctxHashHex } from "../search/embeddings.js";
|
|
21
|
+
import { makeBlockContextResolver } from "../search/tasks.js";
|
|
20
22
|
const TARGET = Symbol("oqx.target");
|
|
21
23
|
const REPO_ROOT = Symbol("oqx.repoRoot");
|
|
22
24
|
const PROP_SOURCE = Symbol("oqx.propSource");
|
|
@@ -159,11 +161,49 @@ export function makeStoreContext(store, repoId, opts = {}) {
|
|
|
159
161
|
const r = one(`SELECT ordinal FROM blocks WHERE doc_id = ? AND block_id = ?`, block.doc_id, first);
|
|
160
162
|
return r?.ordinal ?? block.ordinal;
|
|
161
163
|
};
|
|
164
|
+
// ---- a document's live blocks in DOCUMENT order (spec/surface §1.2) --------
|
|
165
|
+
// Pre-order over the containment forest (children by parent_block, siblings by
|
|
166
|
+
// (ordinal, block_id)); each row is paired with its top-level ancestor's
|
|
167
|
+
// ordinal, which is what section ranges are expressed in.
|
|
168
|
+
const docBlocksPreorder = (docId, path) => {
|
|
169
|
+
const rows = all(`SELECT b.*, ? AS __path FROM blocks b WHERE b.doc_id = ? AND b.deleted_commit IS NULL ORDER BY b.ordinal, b.block_id`, path, docId);
|
|
170
|
+
const ids = new Set(rows.map((r) => r.block_id));
|
|
171
|
+
const byParent = new Map();
|
|
172
|
+
for (const r of rows) {
|
|
173
|
+
const parent = r.parent_block != null && ids.has(r.parent_block) ? r.parent_block : null;
|
|
174
|
+
let list = byParent.get(parent);
|
|
175
|
+
if (!list)
|
|
176
|
+
byParent.set(parent, (list = []));
|
|
177
|
+
list.push(r);
|
|
178
|
+
}
|
|
179
|
+
const out = [];
|
|
180
|
+
const walk = (parent, top) => {
|
|
181
|
+
for (const r of byParent.get(parent) ?? []) {
|
|
182
|
+
const t = top ?? r.ordinal;
|
|
183
|
+
out.push({ row: r, top: t });
|
|
184
|
+
walk(r.block_id, t);
|
|
185
|
+
}
|
|
186
|
+
};
|
|
187
|
+
walk(null, null);
|
|
188
|
+
return out;
|
|
189
|
+
};
|
|
190
|
+
// A document's nodes in DOCUMENT order: by the owning block's pre-order rank
|
|
191
|
+
// (block-less nodes first), then span_start, then node_id. Node ids are content
|
|
192
|
+
// hashes over random doc ids, so an id order would differ per ingest.
|
|
193
|
+
const docNodesInOrder = (docId, path) => {
|
|
194
|
+
const rank = new Map();
|
|
195
|
+
docBlocksPreorder(docId, path).forEach((x, i) => rank.set(x.row.block_id, i));
|
|
196
|
+
const rows = all(`SELECT n.*, ? AS __path FROM nodes n WHERE n.doc_id = ?`, path, docId);
|
|
197
|
+
const pos = (n) => (n.block_id == null ? -1 : rank.get(n.block_id) ?? Number.MAX_SAFE_INTEGER);
|
|
198
|
+
const span = (n) => (typeof n.span_start === "number" ? n.span_start : -1);
|
|
199
|
+
rows.sort((a, b) => pos(a) - pos(b) || span(a) - span(b) || (a.node_id < b.node_id ? -1 : a.node_id === b.node_id ? 0 : 1));
|
|
200
|
+
return tagAll(rows, "nodes");
|
|
201
|
+
};
|
|
162
202
|
// ---- relations (return row arrays) ----------------------------------------
|
|
163
203
|
const rel = {
|
|
164
204
|
docs: {
|
|
165
|
-
nodes: (r) =>
|
|
166
|
-
blocks: (r) => tagAll(
|
|
205
|
+
nodes: (r) => docNodesInOrder(r.doc_id, r.path),
|
|
206
|
+
blocks: (r) => tagAll(docBlocksPreorder(r.doc_id, r.path).map((x) => x.row), "blocks"),
|
|
167
207
|
out: (r) => tagAll(all(`SELECT DISTINCT d2.* FROM docs d2 JOIN edges e ON e.dst_node = d2.doc_id
|
|
168
208
|
WHERE e.src_doc = ? AND e.to_commit IS NULL AND d2.repo_id = ? AND d2.deleted_commit IS NULL ORDER BY d2.path, d2.doc_id`, r.doc_id, repoId), "docs"),
|
|
169
209
|
in: (r) => tagAll(all(`SELECT DISTINCT d2.* FROM docs d2 JOIN edges e ON e.src_doc = d2.doc_id
|
|
@@ -174,7 +214,7 @@ export function makeStoreContext(store, repoId, opts = {}) {
|
|
|
174
214
|
},
|
|
175
215
|
blocks: {
|
|
176
216
|
children: (r) => tagAll(all(`SELECT b.*, ? AS __path FROM blocks b WHERE b.parent_block = ? AND b.deleted_commit IS NULL ORDER BY b.ordinal, b.block_id`, r.__path, r.block_id), "blocks"),
|
|
177
|
-
nodes: (r) => tagAll(all(`SELECT n.*, ? AS __path FROM nodes n WHERE n.block_id =
|
|
217
|
+
nodes: (r) => tagAll(all(`SELECT n.*, ? AS __path FROM nodes n WHERE n.block_id = ? ORDER BY n.span_start, n.node_id`, r.__path, r.block_id), "nodes"),
|
|
178
218
|
out_edges: (r) => tagAll(all(`SELECT e.*, ? AS __path FROM edges e WHERE e.src_block = ? AND e.to_commit IS NULL ORDER BY e.predicate, e.edge_id`, r.__path, r.block_id), "edges"),
|
|
179
219
|
// enclosing section node(s): md:section whose range contains this block's top ordinal.
|
|
180
220
|
section: (r) => {
|
|
@@ -190,9 +230,8 @@ export function makeStoreContext(store, repoId, opts = {}) {
|
|
|
190
230
|
const f = jattr(r, "first_ordinal"), l = jattr(r, "last_ordinal");
|
|
191
231
|
if (f == null || l == null)
|
|
192
232
|
return [];
|
|
193
|
-
// blocks whose top-level ordinal falls in the section range.
|
|
194
|
-
const
|
|
195
|
-
const kept = rows.filter((b) => { const t = topOrdinal(b); return t >= f && t <= l; });
|
|
233
|
+
// blocks whose top-level ancestor's ordinal falls in the section range, in document order.
|
|
234
|
+
const kept = docBlocksPreorder(r.doc_id, r.__path).filter((x) => x.top >= f && x.top <= l).map((x) => x.row);
|
|
196
235
|
return tagAll(kept, "blocks");
|
|
197
236
|
},
|
|
198
237
|
subsections: (r) => {
|
|
@@ -433,6 +472,17 @@ export function makeStoreContext(store, repoId, opts = {}) {
|
|
|
433
472
|
// blocks
|
|
434
473
|
return !!one(`SELECT 1 FROM blocks_fts WHERE rowid = (SELECT rowid FROM blocks WHERE block_id = ?) AND blocks_fts MATCH ?`, r.block_id, match);
|
|
435
474
|
};
|
|
475
|
+
// A block's vector is the cache row for its CURRENT (raw_hash, ctx_hash)
|
|
476
|
+
// (spec/surface §1.3, spec/search §3 — since search 1.1; before, the first row
|
|
477
|
+
// for the raw_hash whatever its context). The context resolver is built once
|
|
478
|
+
// per query (one scan of titles and sections), then a per-block chain walk.
|
|
479
|
+
let blockContexts;
|
|
480
|
+
const blockVector = (r, model) => {
|
|
481
|
+
blockContexts ??= makeBlockContextResolver(store, { repoId });
|
|
482
|
+
const path = typeof r.__path === "string" ? r.__path : String(scalar(`SELECT path FROM docs WHERE doc_id = ?`, r.doc_id) ?? "");
|
|
483
|
+
const ctx = blockContexts.ctx({ doc_id: String(r.doc_id), path, ordinal: Number(r.ordinal), type: String(r.type) });
|
|
484
|
+
return one(`SELECT vec FROM embeddings WHERE content_hash = ? AND ctx_hash = ? AND model = ?`, r.raw_hash, Buffer.from(ctxHashHex(ctx), "hex"), model);
|
|
485
|
+
};
|
|
436
486
|
const semanticScore = (t, r, phrase) => {
|
|
437
487
|
if (t === "nodes" || t === "edges")
|
|
438
488
|
throw new FilterInvalid("semantic(...) is available on the docs and blocks targets", "OQX semantic");
|
|
@@ -441,7 +491,7 @@ export function makeStoreContext(store, repoId, opts = {}) {
|
|
|
441
491
|
throw new FilterInvalid(`semantic(${JSON.stringify(phrase)}) needs an embedding provider; none is configured for this query`, "OQX semantic");
|
|
442
492
|
const row = t === "docs"
|
|
443
493
|
? one(`SELECT vec FROM doc_embeddings WHERE doc_id = ? AND model = ?`, r.doc_id, resolved.model)
|
|
444
|
-
:
|
|
494
|
+
: blockVector(r, resolved.model);
|
|
445
495
|
if (!row)
|
|
446
496
|
return null;
|
|
447
497
|
return cosineBytes(row.vec, resolved.vec);
|
|
@@ -1,12 +1,22 @@
|
|
|
1
1
|
import type { MatchBlock } from "../types.js";
|
|
2
2
|
export type EditClass = "edit" | "insert" | "delete" | "move" | "reorder" | "split" | "merge" | "copy";
|
|
3
|
+
/** Structured-mode classes (list-item edits; see generateStructuredCase). */
|
|
4
|
+
export type StructuredEditClass = "item-edit-mid" | "item-edit-last" | "item-insert" | "item-delete" | "item-move" | "item-gains-nested-list" | "item-reorder"
|
|
5
|
+
/** an item deleted and an unrelated item inserted in its slot (the new item must mint; m2.3 singleton-rule cost) */
|
|
6
|
+
| "item-replace"
|
|
7
|
+
/** the list container whose items were edited (must carry) */
|
|
8
|
+
| "list"
|
|
9
|
+
/** any block the script did not touch (must carry) */
|
|
10
|
+
| "same"
|
|
11
|
+
/** the paragraph / nested list / nested items minted under an item that gained a nested list */
|
|
12
|
+
| "nested-new";
|
|
3
13
|
export interface GroundTruth {
|
|
4
14
|
/** new block key → old id it should carry (absent ⇒ should be minted) */
|
|
5
15
|
carries: Map<string, string>;
|
|
6
16
|
/** old ids expected to be deleted */
|
|
7
17
|
deleted: Set<string>;
|
|
8
18
|
/** per new key, the edit class that produced it (for per-class metrics) */
|
|
9
|
-
classOf: Map<string, EditClass>;
|
|
19
|
+
classOf: Map<string, EditClass | StructuredEditClass>;
|
|
10
20
|
}
|
|
11
21
|
export interface GeneratedCase {
|
|
12
22
|
old: MatchBlock[];
|
|
@@ -19,3 +29,16 @@ export declare function generateCase(seed: number, opts?: {
|
|
|
19
29
|
size?: number;
|
|
20
30
|
intensity?: number;
|
|
21
31
|
}): GeneratedCase;
|
|
32
|
+
export interface StructuredOpts {
|
|
33
|
+
/** top-level blocks in the base document (default 10) */
|
|
34
|
+
size?: number;
|
|
35
|
+
/** edits per case ≈ size × intensity (default 0.3 ⇒ 3) */
|
|
36
|
+
intensity?: number;
|
|
37
|
+
/** words per list item; a number fixes it, absent ⇒ 2–6 per item */
|
|
38
|
+
itemWords?: number;
|
|
39
|
+
}
|
|
40
|
+
type Cls = StructuredEditClass;
|
|
41
|
+
export declare const STRUCTURED_ITEM_CLASSES: readonly Cls[];
|
|
42
|
+
/** Generate one structured case (see the mode comment above). */
|
|
43
|
+
export declare function generateStructuredCase(seed: number, opts?: StructuredOpts): GeneratedCase;
|
|
44
|
+
export {};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { sha256, normalizeVisibleText } from "../../core/hash.js";
|
|
2
|
+
import { flatten } from "../flatten.js";
|
|
2
3
|
// Small deterministic PRNG (mulberry32).
|
|
3
4
|
export function rng(seed) {
|
|
4
5
|
let a = seed >>> 0;
|
|
@@ -126,4 +127,251 @@ export function generateCase(seed, opts = {}) {
|
|
|
126
127
|
deleted.add(b.id);
|
|
127
128
|
return { old, neu, truth: { carries, deleted, classOf } };
|
|
128
129
|
}
|
|
130
|
+
// A larger vocabulary than the flat mode's so short items stay unique within a
|
|
131
|
+
// document (ground truth needs distinguishable items; duplicates would make the
|
|
132
|
+
// "right" id a coin toss the matcher cannot be blamed for).
|
|
133
|
+
const ITEM_WORDS = ("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima mike november oscar papa " +
|
|
134
|
+
"quebec romeo sierra tango uniform victor whiskey xray yankee zulu apple banana cherry damson elder " +
|
|
135
|
+
"fig grape hazel iris jasmine kale lemon mango nectar olive peach quince radish sage thyme umber " +
|
|
136
|
+
"violet walnut yarrow zest amber birch cedar dune ember flint gorse heath ivy jade kelp lotus moss " +
|
|
137
|
+
"nickel onyx pearl quartz reed slate tundra vale willow").split(" ");
|
|
138
|
+
function itemToSource(it, withId) {
|
|
139
|
+
const own = it.words.join(" ");
|
|
140
|
+
if (it.nested === null) {
|
|
141
|
+
const s = { type: "list_item", raw: `- ${own}`, children: [] };
|
|
142
|
+
if (withId && it.id)
|
|
143
|
+
s.blockId = it.id;
|
|
144
|
+
return s;
|
|
145
|
+
}
|
|
146
|
+
// Parser shape: item raw keeps the nested lines; children are the folded
|
|
147
|
+
// paragraph and the nested list; the nested list's first line starts at content.
|
|
148
|
+
const nestedItemRaws = it.nested.map((w) => `- ${w}`);
|
|
149
|
+
const nestedList = {
|
|
150
|
+
type: "list",
|
|
151
|
+
raw: nestedItemRaws.join("\n "),
|
|
152
|
+
children: nestedItemRaws.map((raw) => ({ type: "list_item", raw, children: [] })),
|
|
153
|
+
};
|
|
154
|
+
const s = {
|
|
155
|
+
type: "list_item",
|
|
156
|
+
raw: `- ${own}\n ${nestedItemRaws.join("\n ")}`,
|
|
157
|
+
children: [{ type: "paragraph", raw: own, children: [] }, nestedList],
|
|
158
|
+
};
|
|
159
|
+
if (withId && it.id)
|
|
160
|
+
s.blockId = it.id;
|
|
161
|
+
return s;
|
|
162
|
+
}
|
|
163
|
+
function blockToSource(b, withId) {
|
|
164
|
+
if (b.kind === "list") {
|
|
165
|
+
const children = b.items.map((it) => itemToSource(it, withId));
|
|
166
|
+
const s = { type: "list", raw: children.map((c) => c.raw).join("\n"), children };
|
|
167
|
+
if (withId)
|
|
168
|
+
s.blockId = b.id;
|
|
169
|
+
return s;
|
|
170
|
+
}
|
|
171
|
+
const s = { type: b.kind, raw: b.raw, children: [] };
|
|
172
|
+
if (withId)
|
|
173
|
+
s.blockId = b.id;
|
|
174
|
+
return s;
|
|
175
|
+
}
|
|
176
|
+
/** Walk a FlatSource tree in flatten()'s order, yielding the same positional keys. */
|
|
177
|
+
function walkKeys(list, parentKey, fn) {
|
|
178
|
+
list.forEach((b, index) => {
|
|
179
|
+
const key = `${parentKey ?? ""}/${index}`;
|
|
180
|
+
fn(b, key);
|
|
181
|
+
walkKeys(b.children, key, fn);
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
export const STRUCTURED_ITEM_CLASSES = [
|
|
185
|
+
"item-edit-mid", "item-edit-last", "item-insert", "item-delete", "item-move", "item-gains-nested-list", "item-reorder",
|
|
186
|
+
"item-replace",
|
|
187
|
+
];
|
|
188
|
+
/** Generate one structured case (see the mode comment above). */
|
|
189
|
+
export function generateStructuredCase(seed, opts = {}) {
|
|
190
|
+
// Offset the seed space so structured cases never share a PRNG stream with flat ones.
|
|
191
|
+
const r = rng((seed + 0x5a5a0000) >>> 0);
|
|
192
|
+
const size = opts.size ?? 10;
|
|
193
|
+
const intensity = opts.intensity ?? 0.3;
|
|
194
|
+
const pick = (arr) => arr[Math.floor(r() * arr.length)];
|
|
195
|
+
const wordCount = () => opts.itemWords ?? 2 + Math.floor(r() * 5);
|
|
196
|
+
let nextId = 0;
|
|
197
|
+
const mint = () => `b_${nextId++}`;
|
|
198
|
+
const used = new Set(); // item texts, unique across the document
|
|
199
|
+
const freshWords = (n) => {
|
|
200
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
201
|
+
const w = Array.from({ length: n }, () => pick(ITEM_WORDS));
|
|
202
|
+
const key = w.join(" ");
|
|
203
|
+
if (!used.has(key)) {
|
|
204
|
+
used.add(key);
|
|
205
|
+
return w;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
// Vocabulary exhausted for this length (only plausible for 1-word items): accept a repeat.
|
|
209
|
+
return Array.from({ length: n }, () => pick(ITEM_WORDS));
|
|
210
|
+
};
|
|
211
|
+
const replaceWord = (words, at) => {
|
|
212
|
+
// Replace so the resulting item text is new to the document (bounded retries).
|
|
213
|
+
for (let attempt = 0; attempt < 50; attempt++) {
|
|
214
|
+
const w = pick(ITEM_WORDS);
|
|
215
|
+
if (w === words[at])
|
|
216
|
+
continue;
|
|
217
|
+
const trial = [...words];
|
|
218
|
+
trial[at] = w;
|
|
219
|
+
const key = trial.join(" ");
|
|
220
|
+
if (!used.has(key)) {
|
|
221
|
+
used.add(key);
|
|
222
|
+
words[at] = w;
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
words[at] = pick(ITEM_WORDS.filter((w) => w !== words[at]));
|
|
227
|
+
};
|
|
228
|
+
// Base document: ids are assigned in document (pre-order) order.
|
|
229
|
+
const blocks = [];
|
|
230
|
+
for (let i = 0; i < size; i++) {
|
|
231
|
+
const roll = r();
|
|
232
|
+
// Guarantee two lists so there is always something to edit.
|
|
233
|
+
const wantList = roll < 0.5 || (i >= size - 2 && blocks.filter((b) => b.kind === "list").length < 2);
|
|
234
|
+
if (wantList) {
|
|
235
|
+
const id = mint();
|
|
236
|
+
const n = 3 + Math.floor(r() * 5); // 3–7 items
|
|
237
|
+
const items = Array.from({ length: n }, () => ({ id: mint(), words: freshWords(wordCount()), cls: null, nested: null }));
|
|
238
|
+
blocks.push({ kind: "list", id, items, cls: null });
|
|
239
|
+
}
|
|
240
|
+
else if (roll < 0.65) {
|
|
241
|
+
blocks.push({ kind: "heading", id: mint(), raw: `${"#".repeat(1 + Math.floor(r() * 3))} ${sentence(r, 2 + Math.floor(r() * 3))}` });
|
|
242
|
+
}
|
|
243
|
+
else {
|
|
244
|
+
blocks.push({ kind: "paragraph", id: mint(), raw: sentence(r, 8 + Math.floor(r() * 8)) });
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
const oldTree = blocks.map((b) => blockToSource(b, true));
|
|
248
|
+
const old = flatten(oldTree);
|
|
249
|
+
// Scripted item edits.
|
|
250
|
+
const deleted = new Set();
|
|
251
|
+
const lists = blocks.filter((b) => b.kind === "list");
|
|
252
|
+
const nEdits = Math.max(1, Math.floor(size * intensity));
|
|
253
|
+
for (let e = 0; e < nEdits; e++) {
|
|
254
|
+
const list = pick(lists);
|
|
255
|
+
const cls = pick(STRUCTURED_ITEM_CLASSES);
|
|
256
|
+
const untouched = list.items.map((it, i) => [it, i]).filter(([it]) => it.id !== null && it.cls === null);
|
|
257
|
+
if (untouched.length === 0)
|
|
258
|
+
continue;
|
|
259
|
+
const [it, idx] = pick(untouched);
|
|
260
|
+
switch (cls) {
|
|
261
|
+
case "item-edit-mid": {
|
|
262
|
+
// change 1–2 words, none of them the last one
|
|
263
|
+
const positions = it.words.length - 1;
|
|
264
|
+
if (positions < 1)
|
|
265
|
+
continue;
|
|
266
|
+
const changes = Math.min(positions, 1 + Math.floor(r() * 2));
|
|
267
|
+
const chosen = new Set();
|
|
268
|
+
while (chosen.size < changes)
|
|
269
|
+
chosen.add(Math.floor(r() * positions));
|
|
270
|
+
used.delete(it.words.join(" "));
|
|
271
|
+
for (const at of chosen)
|
|
272
|
+
replaceWord(it.words, at);
|
|
273
|
+
it.cls = cls;
|
|
274
|
+
break;
|
|
275
|
+
}
|
|
276
|
+
case "item-edit-last": {
|
|
277
|
+
used.delete(it.words.join(" "));
|
|
278
|
+
replaceWord(it.words, it.words.length - 1);
|
|
279
|
+
it.cls = cls;
|
|
280
|
+
break;
|
|
281
|
+
}
|
|
282
|
+
case "item-insert": {
|
|
283
|
+
const at = Math.floor(r() * (list.items.length + 1));
|
|
284
|
+
list.items.splice(at, 0, { id: null, words: freshWords(wordCount()), cls, nested: null });
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
case "item-delete": {
|
|
288
|
+
if (list.items.length < 3)
|
|
289
|
+
continue;
|
|
290
|
+
deleted.add(it.id);
|
|
291
|
+
list.items.splice(idx, 1);
|
|
292
|
+
break;
|
|
293
|
+
}
|
|
294
|
+
case "item-move": {
|
|
295
|
+
if (list.items.length < 3)
|
|
296
|
+
continue;
|
|
297
|
+
list.items.splice(idx, 1);
|
|
298
|
+
let dest = Math.floor(r() * list.items.length);
|
|
299
|
+
if (dest >= idx)
|
|
300
|
+
dest++; // any position other than the original
|
|
301
|
+
list.items.splice(Math.min(dest, list.items.length), 0, it);
|
|
302
|
+
it.cls = cls;
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
case "item-reorder": {
|
|
306
|
+
const j = idx + 1 < list.items.length ? idx + 1 : idx - 1;
|
|
307
|
+
if (j < 0)
|
|
308
|
+
continue;
|
|
309
|
+
const other = list.items[j];
|
|
310
|
+
if (other.id === null || other.cls !== null)
|
|
311
|
+
continue;
|
|
312
|
+
list.items[idx] = other;
|
|
313
|
+
list.items[j] = it;
|
|
314
|
+
it.cls = cls;
|
|
315
|
+
other.cls = cls;
|
|
316
|
+
break;
|
|
317
|
+
}
|
|
318
|
+
case "item-replace": {
|
|
319
|
+
// Delete the item and put an unrelated fresh item in its slot: the same
|
|
320
|
+
// shape the singleton rule (spec/reconcile §5 phase 4a step 5) reads as
|
|
321
|
+
// an edit, so this class measures its cost (the new item must mint).
|
|
322
|
+
deleted.add(it.id);
|
|
323
|
+
list.items[idx] = { id: null, words: freshWords(wordCount()), cls, nested: null };
|
|
324
|
+
break;
|
|
325
|
+
}
|
|
326
|
+
case "item-gains-nested-list": {
|
|
327
|
+
it.nested = Array.from({ length: 2 + Math.floor(r() * 2) }, () => freshWords(1)[0]);
|
|
328
|
+
it.cls = cls;
|
|
329
|
+
break;
|
|
330
|
+
}
|
|
331
|
+
default:
|
|
332
|
+
continue;
|
|
333
|
+
}
|
|
334
|
+
list.cls = "list";
|
|
335
|
+
}
|
|
336
|
+
// New tree: built with the truth ids attached, harvested, then stripped.
|
|
337
|
+
const newTree = blocks.map((b) => blockToSource(b, true));
|
|
338
|
+
const carries = new Map();
|
|
339
|
+
const classOf = new Map();
|
|
340
|
+
const clsByKey = new Map();
|
|
341
|
+
{
|
|
342
|
+
// Classes by key: mirror blockToSource's shape.
|
|
343
|
+
const listIdx = new Map();
|
|
344
|
+
blocks.forEach((b, i) => listIdx.set(b, i));
|
|
345
|
+
for (const b of blocks) {
|
|
346
|
+
const bk = `/${listIdx.get(b)}`;
|
|
347
|
+
if (b.kind !== "list") {
|
|
348
|
+
clsByKey.set(bk, "same");
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
clsByKey.set(bk, b.cls ?? "same");
|
|
352
|
+
b.items.forEach((it, i) => {
|
|
353
|
+
const ik = `${bk}/${i}`;
|
|
354
|
+
clsByKey.set(ik, it.cls ?? (it.id === null ? "item-insert" : "same"));
|
|
355
|
+
if (it.nested !== null) {
|
|
356
|
+
clsByKey.set(`${ik}/0`, "nested-new");
|
|
357
|
+
clsByKey.set(`${ik}/1`, "nested-new");
|
|
358
|
+
it.nested.forEach((_, j) => clsByKey.set(`${ik}/1/${j}`, "nested-new"));
|
|
359
|
+
}
|
|
360
|
+
});
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
walkKeys(newTree, null, (b, key) => {
|
|
364
|
+
const cls = clsByKey.get(key) ?? "same";
|
|
365
|
+
classOf.set(key, cls);
|
|
366
|
+
if (b.blockId && !deleted.has(b.blockId))
|
|
367
|
+
carries.set(key, b.blockId);
|
|
368
|
+
delete b.blockId;
|
|
369
|
+
});
|
|
370
|
+
const neu = flatten(newTree);
|
|
371
|
+
const carriedIds = new Set(carries.values());
|
|
372
|
+
for (const b of old)
|
|
373
|
+
if (!carriedIds.has(b.blockId))
|
|
374
|
+
deleted.add(b.blockId);
|
|
375
|
+
return { old, neu, truth: { carries, deleted, classOf } };
|
|
376
|
+
}
|
|
129
377
|
//# sourceMappingURL=generator.js.map
|