@omgbase/core 0.1.0 → 0.3.0

package/dist/src/core/ingest.js

@@ -3,24 +3,36 @@ import { parseTree } from "./parse/tree.js";
 import { render } from "./parse/render.js";
 import { reconstructContent } from "./read/document.js";
 import { assignIds, writeBlockTree, putBlob, newCommit, writeRevision } from "./store/writers.js";
-import { sha256, normalizeVisibleText } from "./hash.js";
+import { sha256, childQuoteDepth, visibleText } from "./hash.js";
 import { mintId } from "./ids.js";
 import { keyBetween } from "./order-key.js";
 import { ftsDeleteDoc, ftsIndexDoc } from "./store/fts.js";
 import { rebuildSections } from "./store/sections.js";
-import { writeDocNodes, projectSectionNodes } from "./store/nodes.js";
+import { writeDocNodes, projectSectionNodes, toByteSpans } from "./store/nodes.js";
 import { flattenFrontmatter, flattenComputed, writeDocProperties } from "./store/properties.js";
 import { maintainEdges, adoptPhantoms } from "./store/edges.js";
 import { parse as parseYaml } from "yaml";
 import { adapterForPath } from "../format/index.js";
+/** The parse step of ingest, exported so a batch caller can parse + reconcile
+ * every document of a checkpoint BEFORE committing any (the cross-document
+ * phase needs all per-doc results first) and hand the tree back via
+ * `opts.parsed`. Frontmatter is split off: `rest` is what the IdResolver sees. */
+export function parseForIngest(path, content) {
+    const adapter = adapterForPath(path);
+    const tree = adapter ? adapter.parse(content) : parseTree(content);
+    return { tree, ...extractFrontmatter(tree.children) };
+}
 function extractFrontmatter(blocks) {
     if (blocks.length > 0 && blocks[0].type === "frontmatter") {
         return { fmBlock: blocks[0], rest: blocks.slice(1) };
     }
     return { fmBlock: null, rest: blocks };
 }
-// Parse a frontmatter block's raw (incl. --- fences) into a JSON-safe object.
-// Malformed YAML yields {} — frontmatter is queryable metadata, not load-bearing.
+// Parse a frontmatter block's raw (incl. --- fences) into a JSON-safe object:
+// the YAML text is the raw minus its first line (the opening `---`) and its last
+// line (the closing `---`, with any trailing whitespace); `\n` or `\r\n` endings
+// (spec/properties §3.1). Malformed YAML, or a document that is not a mapping,
+// yields {} — frontmatter is queryable metadata, not load-bearing.
 function parseFrontmatter(fmBlock) {
     if (!fmBlock)
         return {};
@@ -35,12 +47,14 @@ function parseFrontmatter(fmBlock) {
         return {};
     }
 }
-function flatten(blocks, parent, depth, ancestorPath, out) {
+// `quoteDepth` is the number of blockquote ancestors — the §4.1 text rule needs
+// it (spec/format), and containers derive text from their children.
+function flatten(blocks, parent, depth, ancestorPath, out, quoteDepth = 0) {
     let prevKey = null;
     blocks.forEach((b, ordinal) => {
         const orderKey = keyBetween(prevKey, null);
         prevKey = orderKey;
-        const visible = normalizeVisibleText(b.raw, b.type);
+        const visible = visibleText(b, quoteDepth);
         out.push({
             blockId: b.blockId,
             parentBlock: parent,
@@ -56,7 +70,7 @@ function flatten(blocks, parent, depth, ancestorPath, out) {
             triviaHash: b.trivia.length > 0 ? sha256(b.trivia) : null,
         });
         if (b.children.length > 0) {
-            flatten(b.children, b.blockId, depth + 1, `${ancestorPath}${b.blockId}/`, out);
+            flatten(b.children, b.blockId, depth + 1, `${ancestorPath}${b.blockId}/`, out, childQuoteDepth(b.type, quoteDepth));
         }
     });
 }
@@ -67,7 +81,7 @@ export function ingestFile(store, repoId, path, content, opts = {}) {
     const adapter = adapterForPath(path);
     const format = opts.format ?? adapter?.format ?? "markdown";
     return store.write((db) => {
-        const tree = adapter ? adapter.parse(content) : parseTree(content);
+        const tree = opts.parsed ?? (adapter ? adapter.parse(content) : parseTree(content));
         const { fmBlock, rest } = extractFrontmatter(tree.children);
         // Frontmatter blob (markdown-specific) preserved for revision history.
         const fmBlobHex = fmBlock ? putBlob(db, fmBlock.raw) : null;
@@ -76,24 +90,38 @@ export function ingestFile(store, repoId, path, content, opts = {}) {
         // reconstructs byte-for-byte instead of assuming a canonical blank line.
         // NULL when there is no frontmatter.
         const fmTrivia = fmBlock ? fmBlock.trivia : null;
-        // Document property bag: adapter-provided for non-markdown formats,
-        // frontmatter-parsed for markdown. Flattened into the properties table
-        // below (source=frontmatter); also drives frontmatter edge extraction.
-        const metadata = (adapter?.extractMetadata)
-            ? (adapter.extractMetadata(content) ?? parseFrontmatter(fmBlock))
+        // Document property bag: whole-document metadata for the formats whose
+        // adapter extracts one (YAML, JSON); for Markdown — whose adapter has no
+        // `extractMetadata` — the YAML of the `frontmatter` block and nothing else
+        // (spec/properties §3.1: a document whose first block is not `frontmatter`
+        // has no frontmatter rows, whatever else its bytes resemble). Flattened into
+        // the properties table below (source=frontmatter); also drives frontmatter
+        // edge extraction.
+        const metadata = adapter?.extractMetadata
+            ? (adapter.extractMetadata(content) ?? {})
             : parseFrontmatter(fmBlock);
         // Upsert the document row.
-        let doc = db.prepare("SELECT doc_id FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
+        let doc = db.prepare("SELECT doc_id, deleted_commit FROM docs WHERE repo_id = ? AND path = ?").get(repoId, path);
         const docId = doc?.doc_id ?? mintId("d");
         const isNew = !doc;
         if (!doc) {
             db.prepare("INSERT INTO docs (doc_id, repo_id, path, format, leading_trivia, frontmatter_trivia) VALUES (?, ?, ?, ?, ?, ?)").run(docId, repoId, path, format, tree.leadingTrivia, fmTrivia);
-            doc = { doc_id: docId };
+            doc = { doc_id: docId, deleted_commit: null };
             // Adopt phantom edges that pointed at this path so backlinks re-point.
             adoptPhantoms(db, path, docId);
         }
         else {
-            db.prepare("UPDATE docs SET format = ?, leading_trivia = ?, frontmatter_trivia = ? WHERE doc_id = ?").run(format, tree.leadingTrivia, fmTrivia, docId);
+            // A tombstoned row still owns the path's identity (spec/store §5.6
+            // "Re-creation"): bytes observed again at the path revive it — the row is
+            // live again, so reads that filter `deleted_commit IS NULL` see it and the
+            // next identical observation echo-gates instead of re-ingesting.
+            const revived = doc.deleted_commit !== null;
+            db.prepare("UPDATE docs SET format = ?, leading_trivia = ?, frontmatter_trivia = ?, deleted_commit = NULL WHERE doc_id = ?").run(format, tree.leadingTrivia, fmTrivia, docId);
+            // spec/graph §3.5: adoption happens whenever a row BECOMES live, so a
+            // revived tombstone also collects the phantom edges minted at its path
+            // while it was gone (§8 "Fixed — a revived document adopted no phantoms").
+            if (revived)
+                adoptPhantoms(db, path, docId);
         }
         // Assign block ids via the resolver when supplied (it reconciles against
         // the prior revision — for a new doc that's an empty old tree, so all blocks
@@ -103,7 +131,7 @@ export function ingestFile(store, repoId, path, content, opts = {}) {
             : { assigned: assignIds(rest), dispositions: [], deleted: [] };
         const assigned = resolved.assigned;
         const rootTreeHex = writeBlockTree(db, assigned);
-        const commit = newCommit(db, { repoId, ts, origin });
+        const commit = newCommit(db, { repoId, ts, origin, actor: opts.actor ?? null, reason: opts.reason ?? null });
         const renderedHash = sha256(content);
         const rev = writeRevision(db, {
             docId,
@@ -122,6 +150,14 @@ export function ingestFile(store, repoId, path, content, opts = {}) {
             for (const id of resolved.deleted)
                 pool.run(commit.commitId, expires, id, docId);
         }
+        // Ids arriving from ANOTHER document — a cross-document move whose source
+        // commits later in this checkpoint (or was tombstoned in it), or a
+        // resurrection out of a tombstoned doc whose rows still sit in `blocks` —
+        // must not collide on the block_id primary key. Evict the foreign row (and
+        // its FTS entry when live) and drop any pool row: the id is live here now.
+        const incoming = [...(resolved.crossDocIds ?? []), ...(resolved.consumedPool ?? [])];
+        if (incoming.length > 0)
+            evictForeignBlockRows(db, docId, incoming);
         // Refresh current-state blocks (clear + repopulate with carried/minted ids).
         // FTS is external-content: delete old index rows before dropping blocks.
         ftsDeleteDoc(db, docId);
@@ -155,7 +191,12 @@ export function ingestFile(store, repoId, path, content, opts = {}) {
                 ...n,
                 blockId: n.blockId && idSet.has(n.blockId) ? n.blockId : "",
             }));
-            writeDocNodes(db, repoId, docId, [...withIds, ...sectionNodes]);
+            // Adapters record spans as string indices; the store keeps byte offsets
+            // into the block's UTF-8 raw (spec/graph §2.3), so convert here — the one
+            // place that has both the nodes and the raws they index.
+            const rawById = collectRaws(assigned);
+            const byteSpanned = toByteSpans(withIds, (id) => rawById.get(id));
+            writeDocNodes(db, repoId, docId, [...byteSpanned, ...sectionNodes]);
             // Inline properties (key:: value) → property rows (source=inline). The
             // authored shape within the inline source drives `card`: a key that occurs
             // exactly once in the document is `scalar` (so `element == "fire"` can
@@ -246,6 +287,27 @@ export function ingestFile(store, repoId, path, content, opts = {}) {
         };
     });
 }
+// Remove another document's `blocks` row for each id in `ids` (a cross-doc move
+// or a resurrection re-homes the id into `docId`). A live foreign row still has
+// an FTS entry (external-content index: delete with the original text); a
+// tombstoned one does not. The source document, if it commits later in the same
+// checkpoint, rewrites its rows wholesale and never lists a carried id as
+// deleted, so nothing there depends on the evicted row.
+function evictForeignBlockRows(db, docId, ids) {
+    const sel = db.prepare("SELECT rowid, text, deleted_commit FROM blocks WHERE block_id = ? AND doc_id != ?");
+    const ftsDel = db.prepare("INSERT INTO blocks_fts(blocks_fts, rowid, text) VALUES('delete', ?, ?)");
+    const del = db.prepare("DELETE FROM blocks WHERE block_id = ? AND doc_id != ?");
+    const unpool = db.prepare("DELETE FROM resurrection_pool WHERE block_id = ?");
+    for (const id of ids) {
+        const row = sel.get(id, docId);
+        if (row) {
+            if (row.deleted_commit === null)
+                ftsDel.run(row.rowid, row.text);
+            del.run(id, docId);
+        }
+        unpool.run(id);
+    }
+}
 // Zip assigned block ids onto the parsed RawBlocks for node projection. The
 // parsed `rest` and `assigned` trees are the same blocks in the same order and
 // nesting (assigned is a 1:1 id-carrying projection of rest), so we walk them in
@@ -272,6 +334,18 @@ function typedInlineValue(raw) {
         return { valText: null, valNum: Number(v), valBool: null, valJson: null, type: "number" };
     return { valText: v, valNum: null, valBool: null, valJson: null, type: "string" };
 }
+function collectRaws(blocks) {
+    const raws = new Map();
+    const walk = (list) => {
+        for (const b of list) {
+            raws.set(b.blockId, b.raw);
+            if (b.children.length > 0)
+                walk(b.children);
+        }
+    };
+    walk(blocks);
+    return raws;
+}
 function collectBlockIds(blocks) {
     const ids = new Set();
     const walk = (list) => {

package/dist/src/core/parse/parse.d.ts

@@ -1,3 +1,3 @@
 import type { RawBlock } from "./types.js";
-/** Parse Markdown source into a flat list of top-level blocks with byte spans. */
+/** Parse Markdown source into a flat list of top-level blocks with spans into `source`. */
 export declare function parseBlocks(source: string): RawBlock[];

package/dist/src/core/parse/parse.js

@@ -3,7 +3,7 @@ import { gfm } from "micromark-extension-gfm";
 import { gfmFromMarkdown } from "mdast-util-gfm";
 import { frontmatter } from "micromark-extension-frontmatter";
 import { frontmatterFromMarkdown } from "mdast-util-frontmatter";
-import { normalizeVisibleText } from "../hash.js";
+import { childQuoteDepth, visibleText } from "../hash.js";
 // Nodes whose children nest as blocks (03 §1: parser nesting only).
 // Everything else at block level maps to a leaf block; unknown → opaque.
 function mapType(node) {
@@ -66,16 +66,30 @@ function childrenOf(node) {
     const kids = node.children;
     return Array.isArray(kids) ? kids : [];
 }
-function buildBlock(node, source) {
-    const start = node.position?.start.offset;
-    const end = node.position?.end.offset;
-    if (start === undefined || end === undefined) {
+function isLineEnding(code) {
+    return code === 0x0a /* \n */ || code === 0x0d /* \r */;
+}
+// `quoteDepth` = number of blockquote ancestors, which the §4.1 text rule needs
+// because a nested block's raw keeps the `> ` prefixes of its continuation lines.
+function buildBlock(node, source, shift, quoteDepth) {
+    const rawStart = node.position?.start.offset;
+    const rawEnd = node.position?.end.offset;
+    if (rawStart === undefined || rawEnd === undefined) {
         throw new Error(`node ${node.type} missing position offsets`);
     }
+    const start = rawStart + shift;
+    let end = rawEnd + shift;
+    // Spans exclude the terminating line ending (spec/format §1 inv. 4). micromark
+    // already ends most blocks before it, but a construct that runs to EOF without
+    // a closer (an unclosed fence, an unclosed HTML comment, and the list items
+    // holding one) is handed every trailing line ending, blank lines included.
+    // Those bytes belong to the trivia that follows the block.
+    while (end > start && isLineEnding(source.charCodeAt(end - 1)))
+        end--;
     const raw = source.slice(start, end);
     const mapped = mapType(node);
     const type = mapped ?? "opaque";
-    let children = mapped === null ? [] : childrenOf(node).map((c) => buildBlock(c, source));
+    let children = mapped === null ? [] : childrenOf(node).map((c) => buildBlock(c, source, shift, childQuoteDepth(type, quoteDepth)));
     // Fold a tight list item's lone paragraph: the item block carries the text
     // directly (frozen outline format, 06 §6). Loose/multi-block items keep them.
     if ((type === "list_item" || type === "task") && children.length === 1 && children[0].type === "paragraph") {
@@ -85,7 +99,7 @@ function buildBlock(node, source) {
         type,
         span: { start, end },
         raw,
-        text: normalizeVisibleText(raw, type),
+        text: visibleText({ type, raw, children }, quoteDepth),
         attrs: mapped === null ? {} : attrsFor(node),
         children,
         trivia: "",
@@ -93,12 +107,20 @@ function buildBlock(node, source) {
         outLinks: [],
     };
 }
-/** Parse Markdown source into a flat list of top-level blocks with byte spans. */
+// micromark's preprocessor (micromark/lib/preprocess.js) drops exactly one
+// leading U+FEFF before tokenizing and never emits it as a chunk, so every
+// position it reports is an index into the stripped string. Spans must index
+// the true source (spec/format §1 inv. 6: the BOM is leading trivia), so all
+// offsets move by one code unit when — and only when — the source starts with
+// a BOM. A second BOM, or one anywhere else, is ordinary content and unshifted.
+const BOM = 0xfeff;
+/** Parse Markdown source into a flat list of top-level blocks with spans into `source`. */
 export function parseBlocks(source) {
     const tree = fromMarkdown(source, {
         extensions: [gfm(), frontmatter(["yaml"])],
         mdastExtensions: [gfmFromMarkdown(), frontmatterFromMarkdown(["yaml"])],
     });
-    return tree.children.map((c) => buildBlock(c, source));
+    const shift = source.charCodeAt(0) === BOM ? 1 : 0;
+    return tree.children.map((c) => buildBlock(c, source, shift, 0));
 }
 //# sourceMappingURL=parse.js.map

package/dist/src/core/store/nodes.d.ts

@@ -1,5 +1,18 @@
 import type { Database } from "better-sqlite3";
 import type { ProjectedNode } from "../../format/adapter.js";
 export declare function projectSectionNodes(db: Database, docId: string): ProjectedNode[];
+/**
+ * Convert adapter-projected spans (JavaScript string indices — UTF-16 code
+ * units, spec/graph §2.3) to byte offsets into the block's UTF-8 `raw`, which is
+ * what the `nodes` table stores (spec/graph §8 "Fixed"). `rawOf` returns the raw
+ * of a block by id; a node whose block raw is unknown keeps no span rather than a
+ * wrong one. Nodes without a span (sections) pass through untouched.
+ */
+export declare function toByteSpans(projected: ProjectedNode[], rawOf: (blockId: string) => string | undefined): ProjectedNode[];
 export declare function deleteDocNodes(db: Database, docId: string): void;
+/**
+ * Replace a document's `nodes` rows (and their FTS rows) with `projected`, in
+ * order; `node_id` is the spec/graph §2.3 hash over (doc, block, kind, ordinal).
+ * Spans are stored as given — callers convert to bytes first (`toByteSpans`).
+ */
 export declare function writeDocNodes(db: Database, repoId: string, docId: string, projected: ProjectedNode[]): number;

package/dist/src/core/store/nodes.js

@@ -1,4 +1,5 @@
 import { hashHex } from "../hash.js";
+import { codeUnitSpanToBytes } from "../utf8.js";
 function mintNodeId(docId, blockId, kind, ordinal) {
     return "n_" + hashHex(`${docId}|${blockId}|${kind}|${ordinal}`).slice(0, 12);
 }
@@ -27,6 +28,24 @@ export function projectSectionNodes(db, docId) {
         attrs: { level: r.level, first_ordinal: r.first_ordinal, last_ordinal: r.last_ordinal },
     }));
 }
+/**
+ * Convert adapter-projected spans (JavaScript string indices — UTF-16 code
+ * units, spec/graph §2.3) to byte offsets into the block's UTF-8 `raw`, which is
+ * what the `nodes` table stores (spec/graph §8 "Fixed"). `rawOf` returns the raw
+ * of a block by id; a node whose block raw is unknown keeps no span rather than a
+ * wrong one. Nodes without a span (sections) pass through untouched.
+ */
+export function toByteSpans(projected, rawOf) {
+    return projected.map((p) => {
+        if (p.spanStart === undefined || p.spanEnd === undefined)
+            return p;
+        const raw = rawOf(p.blockId);
+        if (raw === undefined)
+            return { ...p, spanStart: undefined, spanEnd: undefined };
+        const { start, end } = codeUnitSpanToBytes(raw, p.spanStart, p.spanEnd);
+        return { ...p, spanStart: start, spanEnd: end };
+    });
+}
 export function deleteDocNodes(db, docId) {
     const rows = db.prepare("SELECT rowid, name, value FROM nodes WHERE doc_id = ?").all(docId);
     for (const r of rows) {
@@ -34,6 +53,11 @@ export function deleteDocNodes(db, docId) {
     }
     db.prepare("DELETE FROM nodes WHERE doc_id = ?").run(docId);
 }
+/**
+ * Replace a document's `nodes` rows (and their FTS rows) with `projected`, in
+ * order; `node_id` is the spec/graph §2.3 hash over (doc, block, kind, ordinal).
+ * Spans are stored as given — callers convert to bytes first (`toByteSpans`).
+ */
 export function writeDocNodes(db, repoId, docId, projected) {
     deleteDocNodes(db, docId);
     const insert = db.prepare(`INSERT INTO nodes (node_id, repo_id, doc_id, block_id, kind, name, value, span_start, span_end, attrs)

package/dist/src/core/store/writers.d.ts

@@ -9,6 +9,12 @@ export interface TreeInputBlock {
     trivia: string;
     attrs: Record<string, unknown>;
     children: TreeInputBlock[];
+    /** The parsed block's source span, when the tree came from a parse — what
+     *  `ownText` (spec/properties §3.2) uses to blank a container's children. */
+    span?: {
+        start: number;
+        end: number;
+    };
 }
 /** Assign fresh minted ids to a parsed block tree (Stage-1 re-mint path). */
 export declare function assignIds(blocks: RawBlock[]): TreeInputBlock[];

package/dist/src/core/store/writers.js

@@ -9,6 +9,7 @@ export function assignIds(blocks) {
         trivia: b.trivia,
         attrs: b.attrs,
         children: assignIds(b.children),
+        span: b.span,
     }));
 }
 /** Content-address raw text into blobs; returns hex hash. Dedups. */

package/dist/src/core/utf8.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Byte offset of every code-unit index of `s` (length + 1 entries, so the
+ * end-of-string index resolves too). One pass over the string; a surrogate
+ * pair contributes 4 bytes at the index after the pair, and the index between
+ * its two halves maps to the pair's start (no span ever lands there).
+ */
+export declare function byteOffsetTable(s: string): Uint32Array;
+/** The UTF-8 byte offset of code-unit index `index` of `s` (clamped to the string). */
+export declare function byteOffsetOf(s: string, index: number): number;
+/**
+ * Convert a `[start, end)` code-unit span of `s` to a byte span. Both ends are
+ * clamped to the string; `start <= end` is preserved.
+ */
+export declare function codeUnitSpanToBytes(s: string, start: number, end: number): {
+    start: number;
+    end: number;
+};
+/**
+ * The code-unit index whose byte offset is `byte` in `s`. A byte offset that
+ * falls inside a multi-byte sequence (never produced by this module, but a
+ * foreign writer could) rounds down to the start of that character; an offset
+ * past the end clamps to `s.length`.
+ */
+export declare function codeUnitIndexOf(s: string, byte: number): number;
+/** Convert a `[start, end)` byte span of `s` back to a code-unit span (inverse of `codeUnitSpanToBytes`). */
+export declare function byteSpanToCodeUnits(s: string, start: number, end: number): {
+    start: number;
+    end: number;
+};

package/dist/src/core/utf8.js

@@ -0,0 +1,96 @@
+// UTF-16 code-unit index ⇄ UTF-8 byte offset (spec/format §1 inv. 5, spec/graph
+// §2.3). JavaScript strings index code units; every language-neutral offset the
+// store persists is a byte offset into the UTF-8 encoding of the same text, so
+// the reference converts at the store boundary: node spans are converted to
+// bytes before `nodes` rows are written, and back to code units before an
+// editor slices a block's raw. Both directions are exact for well-formed text;
+// a lone surrogate counts as the 3 bytes `Buffer.from(s, "utf8")` emits for
+// U+FFFD, so the table always agrees with the bytes actually stored.
+/**
+ * Byte offset of every code-unit index of `s` (length + 1 entries, so the
+ * end-of-string index resolves too). One pass over the string; a surrogate
+ * pair contributes 4 bytes at the index after the pair, and the index between
+ * its two halves maps to the pair's start (no span ever lands there).
+ */
+export function byteOffsetTable(s) {
+    const table = new Uint32Array(s.length + 1);
+    let bytes = 0;
+    let i = 0;
+    while (i < s.length) {
+        table[i] = bytes;
+        const c = s.charCodeAt(i);
+        if (c < 0x80) {
+            bytes += 1;
+            i += 1;
+        }
+        else if (c < 0x800) {
+            bytes += 2;
+            i += 1;
+        }
+        else if (c >= 0xd800 && c <= 0xdbff && i + 1 < s.length) {
+            const d = s.charCodeAt(i + 1);
+            if (d >= 0xdc00 && d <= 0xdfff) {
+                table[i + 1] = bytes;
+                bytes += 4;
+                i += 2;
+            }
+            else {
+                bytes += 3; // lone high surrogate → U+FFFD
+                i += 1;
+            }
+        }
+        else {
+            bytes += 3; // BMP ≥ U+0800, or a lone surrogate → U+FFFD
+            i += 1;
+        }
+    }
+    table[s.length] = bytes;
+    return table;
+}
+/** The UTF-8 byte offset of code-unit index `index` of `s` (clamped to the string). */
+export function byteOffsetOf(s, index) {
+    return byteOffsetTable(s)[Math.max(0, Math.min(index, s.length))];
+}
+/**
+ * Convert a `[start, end)` code-unit span of `s` to a byte span. Both ends are
+ * clamped to the string; `start <= end` is preserved.
+ */
+export function codeUnitSpanToBytes(s, start, end) {
+    const table = byteOffsetTable(s);
+    const clamp = (i) => Math.max(0, Math.min(i, s.length));
+    return { start: table[clamp(start)], end: table[clamp(Math.max(start, end))] };
+}
+/**
+ * The code-unit index whose byte offset is `byte` in `s`. A byte offset that
+ * falls inside a multi-byte sequence (never produced by this module, but a
+ * foreign writer could) rounds down to the start of that character; an offset
+ * past the end clamps to `s.length`.
+ */
+export function codeUnitIndexOf(s, byte) {
+    const table = byteOffsetTable(s);
+    // `table` is non-decreasing; find the greatest index whose offset ≤ byte.
+    let lo = 0;
+    let hi = s.length;
+    while (lo < hi) {
+        const mid = (lo + hi + 1) >>> 1;
+        if (table[mid] <= byte)
+            lo = mid;
+        else
+            hi = mid - 1;
+    }
+    // The index between the halves of a surrogate pair shares the pair's start
+    // offset; step back to the high surrogate so a slice never splits the pair.
+    if (lo > 0 && lo < s.length) {
+        const c = s.charCodeAt(lo);
+        if (c >= 0xdc00 && c <= 0xdfff && table[lo] === table[lo - 1])
+            lo -= 1;
+    }
+    return lo;
+}
+/** Convert a `[start, end)` byte span of `s` back to a code-unit span (inverse of `codeUnitSpanToBytes`). */
+export function byteSpanToCodeUnits(s, start, end) {
+    const a = codeUnitIndexOf(s, start);
+    const b = codeUnitIndexOf(s, Math.max(start, end));
+    return { start: a, end: Math.max(a, b) };
+}
+//# sourceMappingURL=utf8.js.map

package/dist/src/format/adapter.d.ts

@@ -14,6 +14,12 @@ export interface ProjectedNode {
     name?: string | undefined;
     value?: string | undefined;
     blockId: string;
+    /**
+     * `[spanStart, spanEnd)` of the feature within the block's raw, as JavaScript
+     * string indices (UTF-16 code units). Ingest converts to UTF-8 byte offsets
+     * before the `nodes` row is written (spec/graph §2.3), and `node_set`
+     * converts back before an editor slices the raw.
+     */
     spanStart?: number | undefined;
     spanEnd?: number | undefined;
     attrs?: Record<string, unknown> | undefined;
@@ -27,7 +33,7 @@ export interface EmbeddingChunk {
 export interface NodeEditContext {
     /** The block's current raw bytes (what the edit rewrites). */
     blockRaw: string;
-    /** The node's byte span within blockRaw (null when unrecorded — see spans). */
+    /** The node's span within blockRaw as string indices — the stored byte span converted back (null when unrecorded). */
     span: {
         start: number;
         end: number;