openwriter 0.13.0 → 0.15.0

package/dist/server/markdown-serialize.js

@@ -1,8 +1,24 @@
 /**
  * TipTap JSON -> Markdown serialization.
  * Converts TipTap document to markdown with YAML frontmatter.
+ *
+ * Node identity persistence:
+ *   - Frontmatter `nodes` array carries the (id, fingerprint) pair for every
+ *     block in the document. On load, markdown-parse.ts runs the matcher
+ *     against this array to reassign IDs to surviving blocks.
+ *   - Markdown body is COMPLETELY UNDISTURBED — no anchors, no comment
+ *     sentinels (except the legacy `<!-- -->` empty-paragraph marker which
+ *     has no semantic ID attached anymore).
+ *   - Legacy `^id` caret anchors are no longer emitted. Old docs that still
+ *     contain them are migrated transparently on the next save: parse reads
+ *     the anchors, matcher pins them, serialize emits the new `nodes`
+ *     frontmatter and a clean body.
+ *
+ * adr: adr/node-identity-matcher.md
  */
-import { LEAF_BLOCK_TYPES } from './helpers.js';
+import { generateNodeId, LEAF_BLOCK_TYPES } from './helpers.js';
+import { tiptapToBlocks } from './node-blocks.js';
+import { fingerprintAll } from './node-fingerprint.js';
 // ============================================================================
 // TipTap -> Markdown
 // ============================================================================
@@ -57,21 +73,98 @@ function collectPendingState(doc) {
     walk(doc.content || []);
     return Object.keys(pending).length > 0 ? pending : undefined;
 }
+/**
+ * Build the `nodes` frontmatter entry — one (id, fingerprint) per block
+ * in pre-order traversal of the TipTap tree.
+ *
+ * Each block's ID comes from its TipTap node's `attrs.id`. Fingerprints are
+ * computed from the walker-style block list derived directly from the TipTap
+ * tree (no separate markdown re-parse — same source of truth that builds
+ * the visible doc).
+ */
+function collectNodesFrontmatter(doc) {
+    const blocks = tiptapToBlocks(doc);
+    const fingerprints = fingerprintAll(blocks);
+    const ids = collectBlockIds(doc);
+    // ids array is parallel to blocks array — same pre-order traversal.
+    const entries = [];
+    for (let i = 0; i < blocks.length; i++) {
+        const id = ids[i] || generateNodeId();
+        entries.push({ id, fp: fingerprints[i] });
+    }
+    return entries;
+}
+/**
+ * Cap graveyard size to avoid frontmatter bloat on docs with many edits.
+ * Newest entries (highest position) win — the ones most likely to be
+ * paste-back targets. Older entries expire silently.
+ */
+const GRAVEYARD_MAX = 50;
+/** Walk the TipTap tree in the SAME pre-order as tiptapToBlocks, collect IDs. */
+function collectBlockIds(doc) {
+    const ids = [];
+    const blockTypes = new Set([
+        'heading', 'paragraph', 'bulletList', 'orderedList', 'taskList',
+        'listItem', 'taskItem', 'blockquote', 'codeBlock', 'horizontalRule',
+        'table', 'image', 'tableRow', 'tableCell', 'tableHeader',
+    ]);
+    const containerTypes = new Set([
+        'bulletList', 'orderedList', 'taskList', 'listItem', 'taskItem', 'blockquote',
+    ]);
+    function walk(nodes) {
+        if (!nodes)
+            return;
+        for (const node of nodes) {
+            if (blockTypes.has(node.type)) {
+                ids.push(node.attrs?.id || '');
+                if (containerTypes.has(node.type) && node.content)
+                    walk(node.content);
+            }
+            else if (node.content) {
+                walk(node.content);
+            }
+        }
+    }
+    walk(doc.content || []);
+    return ids;
+}
 /**
  * Convert TipTap document to markdown with JSON frontmatter.
  * Metadata stored as minified JSON between --- delimiters (valid YAML).
  * Editor never sees frontmatter — it's stripped on load, regenerated on save.
  * Pending state is persisted in frontmatter `pending` key.
+ * Node identity persisted in frontmatter `nodes` key (id + fingerprint per block).
  */
 export function tiptapToMarkdown(doc, title, metadata) {
     const meta = { ...metadata, title };
-    // Collect pending state from node attrs into frontmatter
-    const pendingState = collectPendingState(doc);
-    if (pendingState) {
-        meta.pending = pendingState;
+    // Disk is canonical only — never emit `pending:` frontmatter. Pending
+    // state lives in the sidecar at `_pending/{docId}.json`, separated from
+    // the .md file so external markdown editors see clean canonical content.
+    //
+    // If the caller passed a doc that still has in-memory pending attrs,
+    // serialize from a reverted clone so the body is canonical. Callers
+    // that have already done the split (writeToDisk's overlay path) pass
+    // an already-canonical doc; this revert is a no-op for them.
+    // adr: adr/pending-overlay-model.md
+    delete meta.pending;
+    const canonicalDoc = revertPendingForSerialization(doc);
+    // Collect node identity graph (id + fingerprint per block) for next-load matcher
+    const nodes = collectNodesFrontmatter(canonicalDoc);
+    if (nodes.length > 0) {
+        meta.nodes = nodes;
+    }
+    else {
+        delete meta.nodes;
+    }
+    // Graveyard: recently-orphaned (id, fingerprint) entries kept across saves so
+    // paste-back/undo can restore the original ID via exact fingerprint match.
+    // The caller (writeToDisk) puts the matcher's nextGraveyard into metadata.graveyard;
+    // we cap it here to keep the file small.
+    if (Array.isArray(meta.graveyard) && meta.graveyard.length > 0) {
+        meta.graveyard = meta.graveyard.slice(0, GRAVEYARD_MAX);
     }
     else {
-        delete meta.pending;
+        delete meta.graveyard;
     }
     // Strip undefined/null values
     for (const key of Object.keys(meta)) {
@@ -79,12 +172,58 @@ export function tiptapToMarkdown(doc, title, metadata) {
             delete meta[key];
     }
     const frontmatter = `---\n${JSON.stringify(meta)}\n---\n\n`;
-    const body = nodesToMarkdown(doc.content || []);
+    // Serialize the body from the canonical (reverted) clone — never from the
+    // pending-modified live doc, otherwise the on-disk body would contain
+    // rewritten prose without the original anywhere to revert to.
+    const body = nodesToMarkdown(canonicalDoc.content || []);
     return frontmatter + body;
 }
-/** Convert TipTap document to markdown body only (no frontmatter). */
+/** Convert TipTap document to markdown body only (no frontmatter).
+ *  Like tiptapToMarkdown, the body is canonical (pending reverted). */
 export function tiptapToBody(doc) {
-    return nodesToMarkdown(doc.content || []);
+    const canonicalDoc = revertPendingForSerialization(doc);
+    return nodesToMarkdown(canonicalDoc.content || []);
+}
+/**
+ * Deep clone of `doc` with pending decorations reverted, used by the
+ * markdown serializer to ensure disk content is canonical. Mirrors
+ * state.cloneWithPendingReverted but is local to the serializer to
+ * avoid a state.ts → markdown-serialize.ts cycle.
+ *
+ * - status='insert' → drop the node
+ * - status='rewrite' → restore from pendingOriginalContent (or drop if absent)
+ * - status='delete' → keep but clear pending attrs
+ * - no status → keep, strip stray pending attrs
+ */
+const PENDING_KEYS = ['pendingStatus', 'pendingOriginalContent', 'pendingGroupId', 'pendingTextEdits', 'pendingSelectionFrom', 'pendingSelectionTo', 'pendingOriginalFrom', 'pendingOriginalTo', 'pendingOrphan', 'pendingStaleBaseline'];
+function revertPendingForSerialization(doc) {
+    function clean(node) {
+        const clone = JSON.parse(JSON.stringify(node));
+        if (clone.attrs) {
+            for (const k of PENDING_KEYS)
+                delete clone.attrs[k];
+        }
+        if (clone.content)
+            clone.content = walk(clone.content);
+        return clone;
+    }
+    function walk(nodes) {
+        const result = [];
+        for (const node of nodes || []) {
+            const status = node?.attrs?.pendingStatus;
+            if (status === 'insert')
+                continue;
+            if (status === 'rewrite') {
+                const original = node.attrs?.pendingOriginalContent;
+                if (original)
+                    result.push(clean(original));
+                continue;
+            }
+            result.push(clean(node));
+        }
+        return result;
+    }
+    return { type: 'doc', content: walk(doc?.content || []) };
 }
 function nodesToMarkdown(nodes) {
     let result = '';
@@ -98,19 +237,16 @@ function nodeToMarkdown(node, indent) {
         case 'heading': {
             const level = node.attrs?.level || 1;
             const prefix = '#'.repeat(level);
-            const idSuffix = node.attrs?.id ? ` ^${node.attrs.id}` : '';
-            return `${prefix} ${inlineToMarkdown(node.content)}${idSuffix}\n\n`;
+            // Body stays undisturbed — node ID is persisted in frontmatter `nodes`, not as a trailing anchor.
+            return `${prefix} ${inlineToMarkdown(node.content)}\n\n`;
         }
         case 'paragraph': {
             const text = inlineToMarkdown(node.content);
-            const id = node.attrs?.id;
             if (text) {
-                const idSuffix = id ? ` ^${id}` : '';
-                return `${indent}${text}${idSuffix}\n\n`;
+                return `${indent}${text}\n\n`;
             }
-            // Empty paragraph: embed id in the existing sentinel comment
-            const emptyMarker = id ? `<!-- ^${id} -->` : '<!-- -->';
-            return `${indent}${emptyMarker}\n\n`;
+            // Empty paragraph: use plain sentinel (frontmatter `nodes` carries the ID).
+            return `${indent}<!-- -->\n\n`;
         }
         case 'bulletList':
             return listToMarkdown(node.content, '- ', indent);
@@ -188,28 +324,76 @@ function taskListToMarkdown(items, indent) {
     }
     return result + '\n';
 }
+/**
+ * Serialize a TipTap table node to GFM markdown.
+ *
+ * Critical invariants (each one's absence causes silent table → paragraph
+ * loss on round-trip — observed live as `sync-check FAIL: expected table,
+ * got paragraph` on the Beat Sheet doc):
+ *
+ *   1. ALWAYS emit the header-separator row `| --- | --- |` after the first
+ *      row, regardless of whether any cell is a `tableHeader`. GFM table
+ *      recognition requires the delimiter row — without it, markdown-it
+ *      parses each `| ... |` line as a paragraph and the entire table is
+ *      dropped. (One-time consequence: a header-less table's first row
+ *      becomes `tableHeader` cells after the first round-trip. Stable
+ *      thereafter.)
+ *
+ *   2. Escape `|` inside cell text as `\|` so it doesn't terminate the cell
+ *      column.
+ *
+ *   3. Collapse multi-paragraph cells with `<br>` joiners. The inline
+ *      cell format can't represent multiple block paragraphs; without
+ *      collapsing, only the first paragraph round-trips and the rest are
+ *      silently lost.
+ *
+ *   4. Ensure a blank line precedes the table block (caller does `\n\n`
+ *      tailing on prior nodes; we keep the leading newline minimal).
+ */
 function tableToMarkdown(node) {
     const rows = node.content || [];
     if (rows.length === 0)
         return '';
+    function cellContentToText(cell) {
+        const content = cell.content || [];
+        if (content.length === 0)
+            return '';
+        // Each cell typically holds one paragraph, but a TipTap table can carry
+        // multi-paragraph cells (and arbitrary blocks). Concatenate paragraphs
+        // with <br> so no inline content is dropped.
+        const parts = [];
+        for (const child of content) {
+            if (child.type === 'paragraph') {
+                parts.push(inlineToMarkdown(child.content));
+            }
+            else if (child.content) {
+                // Non-paragraph block (rare in tables) — fall through to inline.
+                parts.push(inlineToMarkdown(child.content));
+            }
+        }
+        // Escape pipes and replace newlines with <br>.
+        return parts.join('<br>').replace(/\|/g, '\\|').replace(/\r?\n/g, '<br>');
+    }
     const lines = [];
-    let isFirstRow = true;
-    for (const row of rows) {
+    const firstRowCells = rows[0]?.content || [];
+    const columnCount = firstRowCells.length;
+    for (let r = 0; r < rows.length; r++) {
+        const row = rows[r];
         const cells = row.content || [];
-        const cellTexts = cells.map((cell) => {
-            const para = cell.content?.[0];
-            return para ? inlineToMarkdown(para.content) : '';
-        });
+        const cellTexts = cells.map(cellContentToText);
+        // Pad short rows so the markdown table has consistent column count.
+        while (cellTexts.length < columnCount)
+            cellTexts.push('');
         lines.push(`| ${cellTexts.join(' | ')} |`);
-        if (isFirstRow) {
-            const hasHeaders = cells.some((c) => c.type === 'tableHeader');
-            if (hasHeaders) {
-                lines.push(`| ${cellTexts.map(() => '---').join(' | ')} |`);
-            }
-            isFirstRow = false;
+        if (r === 0) {
+            // ALWAYS emit the separator — GFM parsing requires it for table
+            // recognition. This is the load-bearing invariant.
+            lines.push(`| ${Array(columnCount).fill('---').join(' | ')} |`);
         }
     }
-    return lines.join('\n') + '\n\n';
+    // Leading blank line ensures we're not glued to the prior block (which
+    // would cause the table to be consumed as a paragraph continuation).
+    return '\n' + lines.join('\n') + '\n\n';
 }
 // ---- Inline mark serialization ----
 const SERIALIZED_MARKS = ['bold', 'italic', 'code', 'strike', 'underline', 'highlight', 'subscript', 'superscript', 'link'];

package/dist/server/markdown.js

@@ -1,6 +1,38 @@
 /**
  * Barrel re-export for markdown serialization and parsing.
  * All existing imports from './markdown.js' continue to work unchanged.
+ *
+ * Also exposes `tiptapToMarkdownChecked` — a sync-observed wrapper that
+ * verifies the TipTap → markdown → TipTap round-trip preserves document
+ * shape. Use this when you want to catch silent drift; the unchecked
+ * `tiptapToMarkdown` stays the cheap path for hot loops.
  */
+import { tiptapToMarkdown } from './markdown-serialize.js';
+import { markdownToTiptap } from './markdown-parse.js';
+import { shapeOfTiptap, compareShapes, formatSyncReport, } from './node-sync-check.js';
 export { tiptapToMarkdown, tiptapToBody, nodeText, inlineToMarkdown } from './markdown-serialize.js';
 export { markdownToTiptap, markdownToNodes } from './markdown-parse.js';
+export { shapeOfTiptap, computeShape, compareShapes, formatSyncReport, } from './node-sync-check.js';
+/**
+ * Serialize TipTap to markdown, then re-parse and verify that the round-trip
+ * preserved document shape. If the shapes diverge, a node was misapplied —
+ * content ended up in the wrong block on save, or the parse interpreted
+ * something differently than the serialize emitted.
+ *
+ * Returns the markdown AND the sync report. Callers decide whether a failed
+ * report should block the save (throw) or just log and continue.
+ *
+ * Cost: one extra parse + two shape computations per save. Acceptable for
+ * any user-driven save; consider the unchecked path inside tight loops.
+ */
+export function tiptapToMarkdownChecked(doc, title, metadata) {
+    const markdown = tiptapToMarkdown(doc, title, metadata);
+    const expectedShape = shapeOfTiptap(doc);
+    const reparsed = markdownToTiptap(markdown);
+    const actualShape = shapeOfTiptap(reparsed.document);
+    const syncReport = compareShapes(expectedShape, actualShape);
+    if (!syncReport.ok) {
+        console.error(formatSyncReport(syncReport, `serialize:${title}`));
+    }
+    return { markdown, syncReport };
+}