npm - openwriter - Versions diffs - 0.19.0 → 0.20.0 - Mend

openwriter 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

package/dist/client/assets/{index-BZ7LCzrR.js → index-B1-K-j46.js} +52 -52
package/dist/client/index.html +1 -1
package/dist/server/backlinks.js +148 -108
package/dist/server/index.js +30 -5
package/dist/server/mcp.js +75 -109
package/dist/server/state.js +51 -17
package/package.json +1 -1

package/dist/client/index.html CHANGED Viewed

@@ -10,7 +10,7 @@
     <link rel="preconnect" href="https://fonts.googleapis.com" />
     <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
     <link href="https://fonts.googleapis.com/css2?family=Charter:ital,wght@0,400;0,700;1,400&family=Crimson+Pro:ital,wght@0,300;0,400;0,600;0,700;1,400&family=DM+Sans:ital,wght@0,400;0,500;0,600;0,700;1,400&family=DM+Serif+Display&family=IBM+Plex+Mono:wght@400;500;600&family=IBM+Plex+Sans:wght@400;500;600&family=Inter:wght@400;500;600;700&family=Libre+Baskerville:ital,wght@0,400;0,700;1,400&family=Literata:ital,opsz,wght@0,7..72,400;0,7..72,600;0,7..72,700;1,7..72,400&family=Newsreader:ital,opsz,wght@0,6..72,400;0,6..72,600;1,6..72,400&family=Playfair+Display:wght@400;600;700;900&family=Source+Serif+4:ital,opsz,wght@0,8..60,400;0,8..60,600;0,8..60,700;1,8..60,400&family=Space+Grotesk:wght@400;500;600;700&family=Space+Mono:wght@400;700&display=swap" rel="stylesheet" />
-    <script type="module" crossorigin src="/assets/index-BZ7LCzrR.js"></script>
+    <script type="module" crossorigin src="/assets/index-B1-K-j46.js"></script>
     <link rel="stylesheet" crossorigin href="/assets/index-0ttVnjRp.css">
   </head>
   <body>

package/dist/server/backlinks.js CHANGED Viewed

@@ -1,26 +1,28 @@
 /**
- * Backlinks engine: keeps each doc's frontmatter `backlinks` field in sync
- * with the forward links pointing at it from other docs.
+ * Connections engine (v0.20.0): doc-to-doc connections are structural data,
+ * stored as `references: [docId, ...]` arrays in each source's frontmatter.
+ * The inbound list on any target is computed live as the inverse of every
+ * doc's references — there is no stored derived field on disk.
  *
  * Design:
- *   - Forward links in prose = source of truth (link mark with `doc:` href).
- *   - Backlinks frontmatter = derived projection, eventually consistent.
- *   - Incremental on save: when a doc's forward links change, only the
- *     affected target docs get their backlinks refreshed.
- *   - Full rebuild via /api/rebuild-backlinks (idempotent rescue path).
+ *   - `references:` in frontmatter = source of truth (this doc connects to these).
+ *   - Backlinks = computed live (scan all docs' references, return those listing
+ *     us). Cached in memory for query-time speed; invalidated on any references
+ *     write.
+ *   - Legacy `doc:` prose links in body keep rendering (TipTap PadLink) AND
+ *     auto-populate `references` on save — backward compat.
+ *   - Legacy stored `backlinks:` frontmatter field is dropped on any save
+ *     (lazy migration). One-off `rebuildAllReferences()` does the bulk migrate.
  *
- * Frontmatter schema (lean — anchor text + refs only, no snippet/context):
- *   backlinks:
- *     - text: "the territorial imperative"
- *       from_doc: a3f2c1d4         # source docId
- *       from_node: f6c3830d        # source nodeId where link mark lives
- *       to_node: 1a2b3c4d          # optional: target nodeId being linked to
+ * The pre-v0.20 incremental backlinks pipeline (updateBacklinksForSource) is
+ * gone — it had a race that meant on-save updates didn't fire reliably (the
+ * test session that motivated this refactor caught it). Computing live
+ * removes the entire class of incremental-update bugs.
  */
 import { readFileSync, existsSync, readdirSync } from 'fs';
 import { join } from 'path';
 import matter from 'gray-matter';
 import { getDataDir, atomicWriteFileSync, resolveDocPath, isExternalDoc } from './helpers.js';
-import { filenameByDocId } from './documents.js';
 import { markdownToTiptap } from './markdown-parse.js';
 const HEX8 = /^[a-f0-9]{8}$/;
 const ANCHOR_TEXT_MAX = 80; // truncate long anchor text in backlinks frontmatter
@@ -121,7 +123,7 @@ export function extractForwardLinks(doc, sourceDocId) {
  * Read a doc's frontmatter from disk and parse it.
  * Returns null if the file doesn't exist or can't be parsed.
  */
-function readFrontmatter(filename) {
+export function readFrontmatter(filename) {
     try {
         const filePath = resolveDocPath(filename);
         if (!existsSync(filePath))
@@ -139,7 +141,7 @@ function readFrontmatter(filename) {
  * Only touches the frontmatter — does NOT re-serialize the body, which would
  * lose nodeIds and reformat. This is safe to call on non-active docs.
  */
-function writeFrontmatter(filename, newData) {
+export function writeFrontmatter(filename, newData) {
     const filePath = resolveDocPath(filename);
     const raw = readFileSync(filePath, 'utf-8');
     const parsed = matter(raw);
@@ -157,85 +159,112 @@ function writeFrontmatter(filename, newData) {
         return;
     atomicWriteFileSync(filePath, newFrontmatter);
 }
-/** Convert ForwardLinks targeting a given doc into Backlink entries for its frontmatter. */
-function toBacklinks(targetDocId, allLinks) {
-    return allLinks
-        .filter((l) => l.to_doc === targetDocId)
-        .map((l) => {
-        const entry = {
-            text: l.text,
-            from_doc: l.from_doc,
-            from_node: l.from_node,
-        };
-        if (l.to_node)
-            entry.to_node = l.to_node;
-        return entry;
-    });
+// ============================================================================
+// COMPUTE-LIVE BACKLINKS — the new v0.20 surface
+// ============================================================================
+//
+// `computeBacklinksFor(targetDocId)` returns every doc that lists targetDocId
+// in its `references:` frontmatter array. Cached in an inverse-index map keyed
+// by target docId. Any write that touches a source's references calls
+// `invalidateBacklinksCache(sourceDocId)` to wipe affected entries; the next
+// read rebuilds them lazily.
+/** Inverse index: target docId → Set of source docIds that reference it. */
+let backlinksCache = null;
+/** Build (or rebuild) the entire inverse index by scanning every .md in the
+ *  data dir. Runs O(N) over the corpus; called once on first read after an
+ *  invalidation. Personal corpora ≤ a few hundred docs make this trivial. */
+function buildBacklinksCache() {
+    const cache = new Map();
+    let files = [];
+    try {
+        files = readdirSync(getDataDir()).filter((f) => f.endsWith('.md'));
+    }
+    catch {
+        return cache;
+    }
+    for (const f of files) {
+        try {
+            const raw = readFileSync(join(getDataDir(), f), 'utf-8');
+            const parsed = matter(raw);
+            const sourceDocId = parsed.data?.docId;
+            if (!sourceDocId || typeof sourceDocId !== 'string')
+                continue;
+            const refs = parsed.data?.references;
+            if (!Array.isArray(refs))
+                continue;
+            for (const targetDocId of refs) {
+                if (typeof targetDocId !== 'string')
+                    continue;
+                if (!cache.has(targetDocId))
+                    cache.set(targetDocId, new Set());
+                cache.get(targetDocId).add(sourceDocId);
+            }
+        }
+        catch {
+            // skip unreadable
+        }
+    }
+    return cache;
+}
+/** Drop the in-memory cache. Next read rebuilds from disk. Called from
+ *  state.ts:writeToDisk after a save that may have changed references. */
+export function invalidateBacklinksCache() {
+    backlinksCache = null;
+}
+/**
+ * Return every source doc that references targetDocId. Pure read; the
+ * frontmatter `references:` arrays across the workspace are the only data
+ * consulted. Cached in memory.
+ */
+export function computeBacklinksFor(targetDocId) {
+    if (!backlinksCache)
+        backlinksCache = buildBacklinksCache();
+    const sources = backlinksCache.get(targetDocId);
+    if (!sources)
+        return [];
+    return Array.from(sources)
+        .sort()
+        .map((from_doc) => ({ from_doc }));
 }
+// ============================================================================
+// PROSE-LINK AUTO-SYNC — backward compat for legacy [text](doc:id) prose links
+// ============================================================================
 /**
- * Incremental update: source doc's forward links changed from oldLinks to newLinks.
- * Update each affected target doc's backlinks frontmatter.
+ * Scan a TipTap doc for prose `doc:` links and merge their target docIds
+ * into the source's `references:` frontmatter. Idempotent — only writes
+ * when there are new docIds to add.
  *
- * If `currentDocMetadata` is provided, it's the live in-memory metadata for the
- * source doc (the active doc). The caller is responsible for persisting it.
- * For OTHER target docs we touch their files directly.
+ * Called from state.ts:writeToDisk after the markdown body is persisted, so
+ * existing prose links (which still render as click-through internal links
+ * via the PadLink TipTap extension) automatically appear in `references:`
+ * for graph/crawl/backlinks-panel consumption.
  */
-export function updateBacklinksForSource(sourceDocId, newLinks, oldLinks) {
-    const oldTargets = new Set(oldLinks.map((l) => l.to_doc));
-    const newTargets = new Set(newLinks.map((l) => l.to_doc));
-    const affected = new Set([...oldTargets, ...newTargets]);
-    const touched = [];
-    for (const targetDocId of affected) {
-        if (targetDocId === sourceDocId)
-            continue; // Skip self-links (rare; if any, handled by caller)
-        const targetFilename = filenameByDocId(targetDocId);
-        if (!targetFilename) {
-            // Target doc not found anywhere — broken link, source-side surface added in a follow-up
-            continue;
-        }
-        const fm = readFrontmatter(targetFilename);
-        if (!fm)
-            continue;
-        // Pull all existing backlinks, drop ones from this source, then add new
-        const existing = Array.isArray(fm.data.backlinks) ? fm.data.backlinks : [];
-        const kept = existing.filter((b) => b.from_doc !== sourceDocId);
-        const fromThisSource = newLinks
-            .filter((l) => l.to_doc === targetDocId)
-            .map((l) => {
-            const entry = {
-                text: l.text,
-                from_doc: l.from_doc,
-                from_node: l.from_node,
-            };
-            if (l.to_node)
-                entry.to_node = l.to_node;
-            return entry;
-        });
-        const updated = [...kept, ...fromThisSource];
-        // Stable ordering for diff-friendliness: by from_doc, from_node
-        updated.sort((a, b) => {
-            if (a.from_doc !== b.from_doc)
-                return a.from_doc < b.from_doc ? -1 : 1;
-            return a.from_node < b.from_node ? -1 : 1;
-        });
-        const newData = { ...fm.data };
-        if (updated.length > 0)
-            newData.backlinks = updated;
-        else
-            delete newData.backlinks;
-        try {
-            writeFrontmatter(targetFilename, newData);
-            touched.push(targetDocId);
-        }
-        catch {
-            // Best-effort — skip on error
+export function syncReferencesFromProse(sourceDocId, sourceDoc, currentMetadata) {
+    const links = extractForwardLinks(sourceDoc, sourceDocId);
+    if (links.length === 0)
+        return null;
+    const proseTargets = new Set();
+    for (const l of links)
+        proseTargets.add(l.to_doc);
+    const existing = Array.isArray(currentMetadata.references) ? currentMetadata.references : [];
+    const merged = new Set(existing);
+    const added = [];
+    for (const t of proseTargets) {
+        if (!merged.has(t)) {
+            merged.add(t);
+            added.push(t);
         }
     }
-    return { touched };
+    if (added.length === 0)
+        return null;
+    return { added, newReferences: Array.from(merged) };
 }
+// ============================================================================
+// MIGRATION — bulk backfill from prose links + strip stored backlinks
+// ============================================================================
 /**
  * Read all docs in the data dir, return their parsed frontmatter + tiptap doc.
- * Used by full rebuild.
+ * Used by the migration rebuild.
  */
 function loadAllDocsForRebuild() {
     const out = [];
@@ -253,7 +282,7 @@ function loadAllDocsForRebuild() {
             const docId = parsed.metadata?.docId;
             if (!docId)
                 continue;
-            out.push({ docId, filename: f, doc: parsed.document });
+            out.push({ docId, filename: f, doc: parsed.document, metadata: parsed.metadata });
         }
         catch {
             // skip unreadable
@@ -262,37 +291,39 @@ function loadAllDocsForRebuild() {
     return out;
 }
 /**
- * Full rebuild: scan all docs, compute backlinks for each from scratch,
- * write updated frontmatter to docs whose backlinks changed.
- * Idempotent. Run via /api/rebuild-backlinks.
+ * Full rescan: for every doc, extract prose `doc:` links from body and merge
+ * their targets into `references:` frontmatter. Also strip any legacy
+ * `backlinks:` field. Idempotent — re-running produces no changes if the
+ * corpus is already migrated.
+ *
+ * Replaces the v0.19 `rebuildAllBacklinks` which built the (now-removed)
+ * derived backlinks projection. The new rescue path is `/api/rebuild-references`
+ * (with `/api/rebuild-backlinks` kept as a 308 redirect for one release cycle).
  */
-export function rebuildAllBacklinks() {
+export function rebuildAllReferences() {
     const allDocs = loadAllDocsForRebuild();
-    // Collect every forward link in the workspace
-    const allLinks = [];
-    for (const d of allDocs) {
-        allLinks.push(...extractForwardLinks(d.doc, d.docId));
-    }
-    // For each doc, compute its inbound = backlinks, write if changed
     let updated = 0;
     for (const d of allDocs) {
-        const newBacklinks = toBacklinks(d.docId, allLinks);
-        newBacklinks.sort((a, b) => {
-            if (a.from_doc !== b.from_doc)
-                return a.from_doc < b.from_doc ? -1 : 1;
-            return a.from_node < b.from_node ? -1 : 1;
-        });
         const fm = readFrontmatter(d.filename);
         if (!fm)
             continue;
-        const existing = Array.isArray(fm.data.backlinks) ? fm.data.backlinks : [];
-        if (JSON.stringify(existing) === JSON.stringify(newBacklinks))
+        // Extract prose links → docIds
+        const proseLinks = extractForwardLinks(d.doc, d.docId);
+        const proseTargets = new Set(proseLinks.map((l) => l.to_doc));
+        // Merge with existing references (dedup)
+        const existing = Array.isArray(fm.data.references) ? fm.data.references : [];
+        const merged = Array.from(new Set([...existing, ...proseTargets])).sort();
+        // Decide whether anything changed
+        const referencesChanged = JSON.stringify(existing.slice().sort()) !== JSON.stringify(merged);
+        const hadLegacyBacklinks = 'backlinks' in fm.data;
+        if (!referencesChanged && !hadLegacyBacklinks)
             continue;
         const newData = { ...fm.data };
-        if (newBacklinks.length > 0)
-            newData.backlinks = newBacklinks;
+        if (merged.length > 0)
+            newData.references = merged;
         else
-            delete newData.backlinks;
+            delete newData.references;
+        delete newData.backlinks; // lazy migration
         try {
             writeFrontmatter(d.filename, newData);
             updated++;
@@ -301,8 +332,17 @@ export function rebuildAllBacklinks() {
             // skip
         }
     }
+    invalidateBacklinksCache();
     return { scanned: allDocs.length, updated };
 }
+/**
+ * @deprecated v0.20 — kept as a no-op shim so any caller imports still work.
+ * The incremental backlinks pipeline is gone; backlinks compute live. State's
+ * writeToDisk no longer calls this.
+ */
+export function updateBacklinksForSource() {
+    return { touched: [] };
+}
 /**
  * Read previously-saved markdown from disk for a given source filename
  * and extract its forward links. Used by the save hook to compute the

package/dist/server/index.js CHANGED Viewed

@@ -286,13 +286,38 @@ export async function startHttpServer(options = {}) {
     app.get('/api/documents', (_req, res) => {
         res.json(listDocuments());
     });
-    // Backlinks: full rebuild across all docs (idempotent rescue path).
-    // The normal flow updates backlinks incrementally on each save; this endpoint
-    // exists for repair after external edits or to bootstrap an unmigrated workspace.
+    // References: get the live computed inverse for a target docId. Returns
+    // every source doc that lists this docId in its `references:` frontmatter.
+    // Cached server-side; cache invalidated on any save that touches references.
+    app.get('/api/backlinks/:docId', async (req, res) => {
+        try {
+            const { computeBacklinksFor } = await import('./backlinks.js');
+            res.json(computeBacklinksFor(req.params.docId));
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // References: full rebuild across all docs (idempotent rescue path).
+    // Walks every .md, extracts legacy prose `doc:` links from body, merges
+    // their targets into `references:`, strips any legacy `backlinks:` field.
+    // Idempotent — safe to re-run.
+    app.post('/api/rebuild-references', async (_req, res) => {
+        try {
+            const { rebuildAllReferences } = await import('./backlinks.js');
+            const result = rebuildAllReferences();
+            res.json(result);
+        }
+        catch (err) {
+            res.status(500).json({ error: err.message });
+        }
+    });
+    // Legacy alias: kept for one release cycle so existing scripts/agents
+    // pointing at the old path still work. Forwards to the new endpoint.
     app.post('/api/rebuild-backlinks', async (_req, res) => {
         try {
-            const { rebuildAllBacklinks } = await import('./backlinks.js');
-            const result = rebuildAllBacklinks();
+            const { rebuildAllReferences } = await import('./backlinks.js');
+            const result = rebuildAllReferences();
             res.json(result);
         }
         catch (err) {