@helping-ai-workflow/md2doc 3.2.1 → 3.4.0

package/lib/editor/server.js

@@ -2,6 +2,8 @@
 const fs = require('fs');
 const path = require('path');
 const http = require('http');
+const crypto = require('crypto');
+const drawioLib = require('../drawio.js');
 const { renderMarkdown } = require('../md2doc.js');
 
 const LINEOPS_SRC = fs.readFileSync(path.join(__dirname, 'lineops.js'), 'utf8');
@@ -54,6 +56,50 @@ const TURNDOWN_SRC = fs.readFileSync(
   require.resolve('turndown/lib/turndown.browser.umd.js'), 'utf8');
 const PASTE_MD_SRC = fs.readFileSync(path.join(__dirname, 'paste-md.js'), 'utf8');
 
+// v3.4.0 batch3 Task 6: the wavedrom GUI's four modules.
+//
+// wave-codec.js / wave-geometry.js / wave-store.js are plain CommonJS ON
+// PURPOSE, and rewriting them into the UMD header every other module here uses
+// is not available: test/wave-codec.test.js bans the spelling `require(` in the
+// codec outright, test/wave-geometry.test.js bans every host-object read a UMD
+// header performs (`typeof self`, `root.x = …`), and test/wave-store.test.js
+// pins the store's require list to exactly `["require('./wave-codec.js')"]`. So
+// the browser gets them through the small CommonJS shim below instead —
+// the module text itself is inlined byte for byte.
+//
+// wave-ui.js is UMD like its neighbours and needs no shim; in the page it
+// defines `window.md2docWaveUi` and takes the other three as parameters.
+const WAVE_CODEC_SRC = fs.readFileSync(path.join(__dirname, 'wave-codec.js'), 'utf8');
+const WAVE_GEOMETRY_SRC = fs.readFileSync(path.join(__dirname, 'wave-geometry.js'), 'utf8');
+const WAVE_STORE_SRC = fs.readFileSync(path.join(__dirname, 'wave-store.js'), 'utf8');
+const WAVE_UI_SRC = fs.readFileSync(path.join(__dirname, 'wave-ui.js'), 'utf8');
+
+// One CommonJS module, wrapped so a plain <script> can carry it.
+//
+// The `'use strict'` is the FIRST statement of the script, not of the wrapper
+// function, so the whole tag — module body included — is strict. That is
+// load-bearing rather than tidy: the store deep-freezes every document it
+// keeps, and a frozen object only BITES in strict mode. test/wave-store.test.js
+// T24 measured the sloppy-mode alternative — four of five writes to a frozen
+// document were silent — so a wrapper that lost the prologue would put the
+// original corruption back and make it quieter.
+//
+// `require` here resolves by basename against what has already been shimmed,
+// which is enough for the only two edges that exist (both './wave-codec.js')
+// and stays a total function for anything else: an unknown name answers
+// undefined and the module that asked fails where it asked.
+function waveModuleTag(name, src) {
+  return '<script>\'use strict\';\n' +
+    'window.__md2docWave = window.__md2docWave || {};\n' +
+    '(function () {\n' +
+    'var module = { exports: {} };\n' +
+    'var exports = module.exports;\n' +
+    'function require(p) { return window.__md2docWave[String(p).split(\'/\').pop()]; }\n' +
+    src + '\n' +
+    'window.__md2docWave[' + JSON.stringify(name) + '] = module.exports;\n' +
+    '})();</script>\n';
+}
+
 // The SAME pure module the page gets, required here for the node side of
 // /api/asset. asset.js is deliberately free of `require('fs')`/`require('path')`
 // (it is plain string arithmetic), so every filesystem decision below is made
@@ -145,15 +191,290 @@ function assertBlockRangesFit(blocks, lines) {
   }
 }
 
-async function createEditorServer({ files, idleTimeoutMs = 30000, clientJs = '', listenPort = 0 }) {
+// F13: the never-opened grace deadline is deliberately its own constant,
+// not idleTimeoutMs reused. idleTimeoutMs governs the gap since the last
+// browser ping (client.js pings every 10s; the default 30,000ms tolerates
+// a short run of missed pings once a real tab is confirmed present).
+// Before the URL is opened there is no ping to measure against — the wait
+// is instead for a human to alt-tab to a browser, or, over SSH, copy the
+// printed URL across to a local machine and load it there, which routinely
+// takes longer than idleTimeoutMs allows. Reusing idleTimeoutMs for that
+// wait would close sessions on people still reading the terminal output.
+// Five minutes is chosen as a margin generous for that human handoff while
+// still finite; test/cli-edit.test.js drives it through a shortened
+// override — an unopened session still exits on its own, and a session
+// opened partway through the grace window keeps working afterward.
+const DEFAULT_NEVER_OPENED_GRACE_MS = 5 * 60 * 1000;
+
+// T21 item 3: closing this server has to mean "stop serving now", not "stop
+// serving once every open socket happens to go away". `server.close()` on its
+// own does the latter: it stops accepting, then waits for the connections it
+// already has. A browser handed the edit URL opens a socket it may never send
+// a request on (a speculative preconnect), and such a socket is not one Node
+// retires by itself — driven against `md2doc --edit` on this box with a
+// viewer stub that opened one TCP connection to the port and sent no bytes,
+// SIGINT printed cli.js's line and the process was still alive 10 s later,
+// 3 runs of 3 — while the same runs with a viewer stub that connected and
+// sent a request, and with `--no-open`, exited in 7-10 ms. The same hang
+// reaches the idle and never-opened deadlines below, which have no second
+// keystroke to rescue them, so every exit this file owns goes through here.
+//
+// closeAllConnections() lands the sockets that close() would wait on. It is
+// Node >= 18.2 and package.json's engines floor is >= 18, so the call is
+// guarded rather than assumed; on an 18.0/18.1 runtime the behaviour is
+// exactly what it was before this function existed.
+function shutdownServer(server, reason) {
+  // F13: name the reason before closing, so cli.js's listener (the process's
+  // stdout writer) can print a line distinct from the other exits.
+  if (reason) server.emit('md2doc-auto-close', reason);
+  server.close();
+  if (typeof server.closeAllConnections === 'function') server.closeAllConnections();
+}
+
+async function createEditorServer({
+  files, idleTimeoutMs = 30000, neverOpenedGraceMs = DEFAULT_NEVER_OPENED_GRACE_MS,
+  clientJs = '', listenPort = 0,
+}) {
   const absFiles = files.map((f) => path.resolve(f));
   let idleTimer = null;
+  let neverOpenedTimer = null;
   let started = false;
 
+  // v3.4.0 batch2 Task 6 — Ruling B2-2: watch every `.drawio`/`.xml` file the
+  // CURRENTLY OPEN document references, and let /api/ping's already-firing
+  // 10s heartbeat (see its own comment below) double as the staleness check,
+  // rather than inventing a push transport server.js has never had (no
+  // WebSocket/SSE/fs.watch anywhere in this stack — see this task's
+  // investigation note).
+  //
+  // TWO maps, and the split is the whole point of fix round 1:
+  //
+  //   drawioWatch   fileId -> { refs }.  WHAT THE LAST BAKE RESOLVED.
+  //                 The list of absolute paths that render actually resolved,
+  //                 i.e. the only paths worth stat'ing for this document.
+  //                 Fix round 2 (re-review G13): this used to carry a
+  //                 `stamps` Map as well, which nothing ever read — the new
+  //                 tab's baseline comes from trackDrawioRefs()'s RETURN
+  //                 value, handed straight to registerDrawioClient() in the
+  //                 same expression. Dead state in a freshly written
+  //                 mechanism is a maintenance trap, so it is gone; the stamp
+  //                 lib/md2doc.js takes before the bake (review F4) still
+  //                 travels, just not through here.
+  //
+  //   drawioClients clientKey -> { fileId, stamps, pending, lastSeen }.
+  //                 WHAT ONE BROWSER TAB IS ACTUALLY SHOWING. Review F3: the
+  //                 baseline used to be per-file, so with two tabs open on one
+  //                 document the first to ping consumed the one signal and the
+  //                 second sat on a stale diagram looking perfectly healthy.
+  //                 A tab gets its id minted by GET /edit/:id — the same
+  //                 response that paints its DOM — so its baseline is exactly
+  //                 the bytes it is displaying, with no window in which
+  //                 another tab's render could seed it wrong.
+  //
+  // Review F2's invariant: a client's `stamps` advances ONLY when that client
+  // has told us its DOM now shows those bytes. Until then every heartbeat
+  // keeps reporting the same staleness, so every one of the client's bail
+  // paths (network error, mid-gesture DOM, focus inside the block, …) really
+  // does get retried on the next tick instead of losing the change forever.
+  const drawioWatch = new Map();
+  const drawioClients = new Map();
+  // The token that identifies a CONTENT VERSION, not a ping. Minted once per
+  // (client, observed stamp set) and repeated verbatim on every later ping
+  // that still sees the same bytes — if it were per-ping, a refresh that took
+  // longer than one 10s beat would always ack a token the server had already
+  // superseded, and the pair would never converge.
+  let drawioTokenSeq = 0;
+  let drawioLastSweep = 0;
+  // 10 minutes = 60 missed heartbeats. A tab that has not pinged in that long
+  // is closed (or its machine slept); keeping its entry would leak one small
+  // object per tab ever opened for the life of the server process.
+  const DRAWIO_CLIENT_IDLE_MS = 10 * 60 * 1000;
+
+  function drawioClientKey(fileId, clientId) {
+    if (typeof clientId !== 'string' || clientId === '') return null;
+    if (typeof fileId !== 'number' || !isFinite(fileId)) return null;
+    return String(fileId) + '\u0000' + clientId;
+  }
+
+  function sweepDrawioClients(now) {
+    if (now - drawioLastSweep < 60 * 1000) return;
+    drawioLastSweep = now;
+    for (const [k, c] of drawioClients) {
+      if (now - c.lastSeen > DRAWIO_CLIENT_IDLE_MS) drawioClients.delete(k);
+    }
+  }
+
+  // Fix round 2 (re-review G5). A tab that is still talking to us is alive by
+  // definition, so a well-formed clientId with no entry gets one back rather
+  // than being answered 204 forever with no signal and no route back but a
+  // reload. Reachable the moment a laptop sleeps or a background tab's timers
+  // are frozen for longer than the sweep window — and before this, the tab's
+  // own returning heartbeat was what deleted it (the sweep ran BEFORE the
+  // lastSeen refresh), so the failure mode was silent, permanent, and caused
+  // by the recovery attempt itself.
+  //
+  // Seeded with an EMPTY stamp map on purpose, not with the last bake's: we
+  // do not know what that tab's DOM is showing, and an empty baseline makes
+  // the next check report stale, which costs one re-render the client then
+  // acks. Wrong in the safe direction (one redundant bake) instead of the
+  // unsafe one (a tab that silently never updates again).
+  //
+  // Fix round 3 (re-review2 H4): the bound is EVICT-OLDEST, applied at BOTH
+  // doors. Round 2 capped only the ping door and refused past the cap, which
+  // was wrong twice over: `GET /edit/:id` wrote into the same map with no
+  // check at all, and refusing meant that once 64 entries accumulated, a
+  // genuinely swept live tab could never re-register — G5's own guard rail
+  // reintroducing exactly the silent deafness G5 was raised about. Dropping
+  // the least-recently-seen entry instead keeps the map at <= MAX from both
+  // doors AND never turns a live tab away: the entry we drop is by
+  // construction the one that has gone quietest.
+  //
+  // WHAT THIS BOUNDS, EXACTLY (fix round 4, re-review3 M3 — the sentence this
+  // replaces claimed more than the code delivers): **the map, not the render
+  // load.** Round 2's refuse-at-cap did cap the bakes a fresh-id-per-ping
+  // caller could provoke — past 64 ids it answered a bare 204, so no token and
+  // no `/api/render` followed. Evict-oldest removes that ceiling on purpose:
+  // a caller rotating a new id every ping now gets `{stale, token}` every
+  // time and can drive `/api/render` — and therefore headless-Chromium bakes
+  // — without bound. That trade is deliberate, because the refusal it removes
+  // is exactly what made a swept live tab permanently deaf, and availability
+  // for the real tab is worth more here than a ceiling on a caller that is
+  // already inside the loopback-only, Host-checked front door (see the
+  // DNS-rebinding note above `hostIsLoopback`). Recorded as a known
+  // limitation rather than guarded: every guard added to this mechanism so
+  // far has produced the next round's finding.
+  const DRAWIO_CLIENT_MAX = 64;
+  function makeRoomForDrawioClient() {
+    while (drawioClients.size >= DRAWIO_CLIENT_MAX) {
+      let oldestKey = null, oldestSeen = Infinity;
+      for (const [k, c] of drawioClients) {
+        if (c.lastSeen < oldestSeen) { oldestSeen = c.lastSeen; oldestKey = k; }
+      }
+      if (oldestKey === null) return;   // unreachable while size > 0
+      drawioClients.delete(oldestKey);
+    }
+  }
+  function ensureDrawioClient(fileId, clientId, now) {
+    const key = drawioClientKey(fileId, clientId);
+    if (key === null) return null;
+    let c = drawioClients.get(key);
+    if (c) { c.lastSeen = now; return c; }
+    if (!drawioWatch.has(fileId)) return null;
+    makeRoomForDrawioClient();
+    c = { fileId, stamps: new Map(), pending: null, lastSeen: now };
+    drawioClients.set(key, c);
+    return c;
+  }
+
+  // Called after every render that actually re-reads the source markdown
+  // (GET /edit/:id and POST /api/render both call renderMarkdown() fresh).
+  // Records WHAT WAS BAKED; deliberately does NOT touch any client's
+  // baseline — that is review F2's invariant, and the trap F2 names is
+  // exactly this function: /api/render runs BEFORE the client has decided
+  // whether it can apply the result, so advancing anything here would lose
+  // every post-response bail.
+  function trackDrawioRefs(fileId, refs) {
+    const list = Array.isArray(refs) ? refs : [];
+    const paths = [];
+    const stamps = new Map();
+    for (const r of list) {
+      const abs = r && typeof r.path === 'string' ? r.path : null;
+      if (!abs || stamps.has(abs)) continue;
+      paths.push(abs);
+      stamps.set(abs, r.stamp === undefined ? null : r.stamp);
+    }
+    drawioWatch.set(fileId, { refs: paths });
+    return stamps;
+  }
+
+  function registerDrawioClient(fileId, stamps) {
+    const clientId = crypto.randomBytes(12).toString('hex');
+    const key = drawioClientKey(fileId, clientId);
+    if (key === null) return clientId;   // unreachable today; never key on null
+    const now = Date.now();
+    makeRoomForDrawioClient();           // H4: this door is bounded too
+    drawioClients.set(key, {
+      fileId, stamps: new Map(stamps), pending: null, lastSeen: now,
+    });
+    sweepDrawioClients(now);             // G5: after the write, never before
+    return clientId;
+  }
+
+  function sameStamps(a, b) {
+    if (a.size !== b.size) return false;
+    for (const [k, v] of a) { if (!b.has(k) || b.get(k) !== v) return false; }
+    return true;
+  }
+
+  // (a) requirement: a document with no drawio references must cost this
+  // heartbeat NOTHING beyond the Map lookups below — zero fs.statSync calls,
+  // same as the bare 204 this replaces. Only a document that actually has
+  // `refs.length > 0` pays for the stat loop that follows.
+  //
+  // Returns the token to hand the client, or null for "nothing stale".
+  function checkDrawioStale(fileId, clientId) {
+    const state = drawioWatch.get(fileId);
+    if (!state || state.refs.length === 0) return null;
+    const key = drawioClientKey(fileId, clientId);
+    if (key === null) return null;
+    const c = drawioClients.get(key);
+    // An unknown client here is one touchDrawioClient() declined to
+    // (re-)register: a page from before this feature (no clientId at all), a
+    // malformed id, or a fileId this server never rendered. It gets the old
+    // bare 204 rather than a signal it has no baseline to interpret. Note
+    // that hitting DRAWIO_CLIENT_MAX is NOT one of those cases any more —
+    // see makeRoomForDrawioClient() (re-review2 H4).
+    if (!c) return null;
+    const now = new Map();
+    let changed = false;
+    for (const p of state.refs) {
+      const st = drawioLib.drawioStampOf(p);
+      now.set(p, st);
+      if (c.stamps.get(p) !== st) changed = true;
+    }
+    if (!changed) { c.pending = null; return null; }
+    if (c.pending && sameStamps(c.pending.stamps, now)) return c.pending.id;
+    drawioTokenSeq += 1;
+    c.pending = { id: 'd' + drawioTokenSeq, stamps: now };
+    return c.pending.id;
+  }
+
+  // Keeps a live tab out of the idle sweep above (re-registering it if the
+  // sweep already got there — G5), and is the one place the sweep is actually
+  // driven from: the heartbeat is the only thing that fires often enough to
+  // be worth hanging it on, and it is throttled to at most once a minute
+  // inside sweepDrawioClients().
+  //
+  // ORDER IS LOAD-BEARING: ensure/touch first, sweep second. The other way
+  // round, a tab returning from a >10-minute freeze is deleted by the very
+  // ping it sent to say it was back.
+  function touchDrawioClient(fileId, clientId) {
+    const now = Date.now();
+    ensureDrawioClient(fileId, clientId, now);
+    sweepDrawioClients(now);
+  }
+
+  // The client's half of F2's invariant: it sends back the token it was given
+  // ONLY after its DOM really carries that render. Anything less than a
+  // complete apply (a block skipped because the user's focus was inside it,
+  // a mid-gesture DOM, a failed fetch) sends nothing, so `stamps` stays put
+  // and the very next heartbeat reports the same staleness again.
+  function ackDrawioApplied(fileId, clientId, token) {
+    if (typeof token !== 'string' || token === '') return;
+    const key = drawioClientKey(fileId, clientId);
+    if (key === null) return;
+    const c = drawioClients.get(key);
+    if (!c || !c.pending || c.pending.id !== token) return;
+    c.stamps = c.pending.stamps;
+    c.pending = null;
+  }
+
   function bumpIdle(server) {
     if (!started) return;
     if (idleTimer) clearTimeout(idleTimer);
-    idleTimer = setTimeout(() => server.close(), idleTimeoutMs);
+    idleTimer = setTimeout(() => {
+      shutdownServer(server, 'idle');
+    }, idleTimeoutMs);
     if (idleTimer.unref) idleTimer.unref();
   }
 
@@ -235,11 +556,15 @@ async function createEditorServer({ files, idleTimeoutMs = 30000, clientJs = '',
         const eol = (crlfCount > bareLf && crlfCount > bareCr) ? '\r\n'
           : (bareCr > bareLf && bareCr > crlfCount) ? '\r'
           : '\n';
-        const { html, blocks } = await renderMarkdown(mdText, file, { editMode: true });
+        const { html, blocks, drawioRefs } = await renderMarkdown(mdText, file, { editMode: true });
+        // Fix round 1 (review F3): mint this TAB's own staleness baseline from
+        // the very render that is about to paint it. Per-tab, not per-file —
+        // two tabs on one document must each get told about an external edit.
+        const drawioClientId = registerDrawioClient(fileId, trackDrawioRefs(fileId, drawioRefs));
         const lines = mdText.split(/\r\n|\r|\n/);
         assertBlockRangesFit(blocks, lines);
         const payload = JSON.stringify({
-          fileId, mtimeMs, eol, lines, blocks,
+          fileId, mtimeMs, eol, lines, blocks, drawioClientId,
         });
         const inject =
           `<script>window.__ED__ = ${payload.replace(/</g, '\\u003c')}</script>\n` +
@@ -257,6 +582,10 @@ async function createEditorServer({ files, idleTimeoutMs = 30000, clientJs = '',
           `<script>${ASSET_SRC}</script>\n` +
           `<script>${TURNDOWN_SRC}</script>\n` +
           `<script>${PASTE_MD_SRC}</script>\n` +
+          waveModuleTag('wave-codec.js', WAVE_CODEC_SRC) +
+          waveModuleTag('wave-geometry.js', WAVE_GEOMETRY_SRC) +
+          waveModuleTag('wave-store.js', WAVE_STORE_SRC) +
+          `<script>${WAVE_UI_SRC}</script>\n` +
           `<script>${clientJs}</script>\n`;
         // Splice at the LAST "</body>" — the document's real closing tag.
         // The first occurrence can sit inside an inlined diagram bundle's JS
@@ -283,7 +612,10 @@ async function createEditorServer({ files, idleTimeoutMs = 30000, clientJs = '',
         const { fileId, content } = await readJson(req);
         const file = absFiles[fileId];
         if (!file) return send(res, 404, { error: 'unknown file' });
-        const { parts, blocks } = await renderMarkdown(content, file, { editMode: true });
+        const { parts, blocks, drawioRefs } = await renderMarkdown(content, file, { editMode: true });
+        // Refs only — NOT any client's baseline. See trackDrawioRefs()'s own
+        // comment: the client has not yet decided whether it can apply this.
+        trackDrawioRefs(fileId, drawioRefs);
         bumpIdle(server);
         return send(res, 200, { parts, blocks });
       }
@@ -412,6 +744,40 @@ async function createEditorServer({ files, idleTimeoutMs = 30000, clientJs = '',
 
       if (req.method === 'POST' && url.pathname === '/api/ping') {
         bumpIdle(server);
+        // v3.4.0 batch2 Task 6: fileId / drawioClientId / drawioAck are NEW,
+        // all optional, on the body this heartbeat already sent every 10s
+        // (old clients / the existing test in test/editor-server.test.js send
+        // `{}` — that still resolves to "no tracked state", the same
+        // zero-cost 204 as before). A body that fails to parse degrades the
+        // same way: no staleness check, plain 204 — a malformed ping body
+        // must never be the thing that breaks the idle-keepalive contract.
+        //
+        // Fix round 1 (review F8): an explicit 8 KB readJson limit, not the
+        // 50 MB default — the same precedent /api/save set for its own body,
+        // for the same reason. This body is always three short scalars, and
+        // this is the ONE route that fires unconditionally every 10 seconds
+        // from every open tab; it has no business also being the most
+        // permissive one in the server.
+        let fileId, drawioClientId, drawioAck;
+        try {
+          const body = await readJson(req, 8 * 1024);
+          if (body) {
+            fileId = body.fileId;
+            drawioClientId = body.drawioClientId;
+            drawioAck = body.drawioAck;
+          }
+        } catch (e) {
+          fileId = undefined; drawioClientId = undefined; drawioAck = undefined;
+        }
+        // Ack FIRST, then re-check: a client that has just confirmed it
+        // applied version N must get a 204 on this same beat unless the file
+        // has moved on past N since.
+        touchDrawioClient(fileId, drawioClientId);
+        ackDrawioApplied(fileId, drawioClientId, drawioAck);
+        const staleToken = checkDrawioStale(fileId, drawioClientId);
+        if (staleToken) {
+          return send(res, 200, { stale: true, token: staleToken });
+        }
         res.writeHead(204);
         return res.end();
       }
@@ -442,6 +808,28 @@ async function createEditorServer({ files, idleTimeoutMs = 30000, clientJs = '',
     server.listen(listenPort, '127.0.0.1');
   });
   const port = server.address().port;
+
+  // F13: counts from listen (this point), not from process start — listen
+  // is the moment a client could first reach the server, and that is the
+  // deadline this timer is measuring against. Unref'd like idleTimer, and
+  // for the same reason: the listening socket is what should keep the
+  // process alive, not this timer. A timer that held a ref would keep the
+  // process up on its own even if the socket already went away, which is
+  // backwards for a clock whose only job is to give up and exit.
+  neverOpenedTimer = setTimeout(() => {
+    // The guard against closing a session that has already been opened —
+    // this timer is armed once, at listen time, and left to run; it is
+    // not cancelled by the GET /edit/:id handler. Whether the URL was
+    // opened by the time the deadline arrives is decided here, by reading
+    // `started` fresh at fire time. Proven load-bearing by mutation: with
+    // this line removed, an actively-pinged session still gets closed the
+    // moment this deadline passes (see test/cli-edit.test.js's "must
+    // survive the never-opened grace deadline" case).
+    if (started) return;
+    shutdownServer(server, 'never-opened');
+  }, neverOpenedGraceMs);
+  if (neverOpenedTimer.unref) neverOpenedTimer.unref();
+
   return {
     server,
     port,
@@ -449,7 +837,11 @@ async function createEditorServer({ files, idleTimeoutMs = 30000, clientJs = '',
       const i = absFiles.indexOf(path.resolve(absPath));
       return i === -1 ? null : `http://127.0.0.1:${port}/edit/${i}`;
     },
-    close() { if (idleTimer) clearTimeout(idleTimer); server.close(); },
+    close() {
+      if (idleTimer) clearTimeout(idleTimer);
+      if (neverOpenedTimer) clearTimeout(neverOpenedTimer);
+      shutdownServer(server, null);
+    },
   };
 }