@entropicwarrior/sdoc 0.2.0 → 0.2.2

package/README.md

@@ -87,20 +87,26 @@ All `.sdoc` files are designed for progressive disclosure — read the `@about`
     {
         Unlimited nesting. Each scope is independently addressable.
 
-        {[code lang=python]
-            def hello():
-                print("Hello from SDOC")
-        }
+        ```python
+        def hello():
+            print("Hello from SDOC")
+        ```
     }
 
-    # A List
+    # Status :example
     {
-        {[.]
-            - First item
-            - Second item with **bold**
-            - Third item
+        {[table 60% center]
+            Endpoint | Status
+            /v2/api | {+Active+}
+            /v1/api | {-Deprecated-}
         }
     }
+
+    # Internal Notes :comment
+    {
+        This scope is invisible in rendered output but
+        stays in the AST for tooling and agents.
+    }
 }
 ```
 
@@ -153,7 +159,7 @@ Markdown-style images with optional width and alignment:
 
 ### Tables
 
-Pipe-delimited tables with optional `borderless` and `headerless` flags.
+Pipe-delimited tables with optional flags for appearance (`borderless`, `headerless`), width (`auto`, `60%`, `400px`), and alignment (`left`, `center`, `right`). All flags compose freely.
 
 ### Lists
 
@@ -167,6 +173,18 @@ Tag any section with `@id` and cross-reference it anywhere with `@id` — render
 
 Turn any SDOC file into an HTML slide deck with themes, layouts (center, two-column), speaker notes, and PDF export.
 
+### Scope Types
+
+Classify scopes with a `:type` annotation — `:schema`, `:warning`, `:deprecated`, `:example`, or any custom label. Types render as `data-scope-type` attributes and CSS classes for styling.
+
+### Data Blocks
+
+Tag a JSON code fence with `:data` and the parser validates and stores the parsed result on the AST node. `extractDataBlocks()` gives programmatic access. Ideal for embedding schemas, configs, and structured metadata alongside prose.
+
+### Comment Scopes
+
+A `:comment` scope is excluded from rendered output but stays in the AST — perfect for agent instructions, internal notes, and build metadata that readers shouldn't see.
+
 ### Custom Styling
 
 Per-folder `sdoc.config.json` or per-file `@meta` scope for custom CSS, headers, footers, and confidentiality banners. Configs cascade from workspace root to file.

package/docs/reference/sdoc-authoring.sdoc

@@ -286,7 +286,7 @@ Content of Section B.
                 \`\{~text~\}\` | Highlight (yellow)
             }
 
-            Links: \`[Link text](https://example.com)\`
+            Links: \`[Link text](https://example.com)\` or \`[Other doc](./other-file.sdoc)\`. Relative paths resolve from the document's directory.
 
             Images: \`![Alt text](path/to/image.png)\`
 
@@ -671,5 +671,20 @@ Content of Section B.
             }
             ```
         }
+
+        # @References Inside Link Labels @refs-in-link-labels
+        {
+            Inline \`@references\` are parsed everywhere, including inside link labels. If you mention a scope ID in a link label, escape the \`@\` to prevent it being treated as a reference to the current document:
+
+            **Wrong:** \`[See domain-model.sdoc @my-section](./domain-model.sdoc#my-section)\`
+
+            The \`@my-section\` is parsed as a reference and flagged as broken (it does not exist in *this* file).
+
+            **Right:** \`[See domain-model.sdoc \\@my-section](./domain-model.sdoc#my-section)\`
+
+            Or simply omit the \`@\` from the label — the URL fragment already carries the target:
+
+            **Also right:** \`[See domain-model.sdoc § my-section](./domain-model.sdoc#my-section)\`
+        }
     }
 }

package/lexica/specification.sdoc

@@ -395,16 +395,26 @@ Content of Section B.
                 - A reference is `@id` in text (unescaped)
                 - References link to the scope with that ID
                 - ID uniqueness is strongly recommended; tooling may warn on duplicates
+                - References are parsed inside link labels — use `\@` to include a literal `@` in a link label without triggering a reference
             }
         }
 
-        # External Links @external-links
+        # Links @links
         {
-            Markdown-style links:
+            Markdown-style links with absolute URLs or relative file paths:
 
             ```
             [label](https://example.com)
+            [other doc](./other-file.sdoc)
+            [parent doc](../guide/intro.sdoc)
             ```
+
+            {[.]
+                - Absolute URLs (any scheme) open externally
+                - Relative paths are resolved from the document's directory
+                - Fragments (`./file.sdoc#section`) and query strings are stripped for file resolution
+                - Tooling may warn on broken relative links (target file does not exist)
+            }
         }
 
         # Autolinks @autolinks

package/package.json

@@ -2,12 +2,12 @@
   "name": "@entropicwarrior/sdoc",
   "displayName": "SDOC",
   "description": "A plain-text documentation format with explicit brace scoping — deterministic parsing, AI-agent efficiency, and 10-50x token savings vs Markdown.",
-  "version": "0.2.0",
+  "version": "0.2.2",
   "publisher": "entropicwarrior",
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/entropicwarrior/sdoc"
+    "url": "git+https://github.com/entropicwarrior/sdoc.git"
   },
   "homepage": "https://github.com/entropicwarrior/sdoc",
   "bugs": {
@@ -22,7 +22,7 @@
     "ai-agent"
   ],
   "bin": {
-    "sdoc-sync-notion": "./tools/sync-notion.js"
+    "sdoc-sync-notion": "tools/sync-notion.js"
   },
   "exports": {
     ".": "./index.js",

package/src/sdoc.js

@@ -1379,6 +1379,9 @@ function renderInlineNodes(nodes) {
           return escapeHtml(node.value);
         case "ref": {
           const href = `#${escapeAttr(node.id)}`;
+          if (_renderOptions.brokenRefIds && _renderOptions.brokenRefIds.has(node.id)) {
+            return `<a class="sdoc-ref sdoc-broken-ref" href="${href}"><span class="sdoc-broken-icon">\u26A0</span>@${escapeHtml(node.id)}</a>`;
+          }
           return `<a class="sdoc-ref" href="${href}">@${escapeHtml(node.id)}</a>`;
         }
         case "code":
@@ -1404,6 +1407,11 @@ function renderInlineNodes(nodes) {
         case "mark_highlight":
           return `<mark class="sdoc-mark sdoc-mark-highlight">${renderInlineNodes(node.children)}</mark>`;
         case "link":
+          if (_renderOptions.brokenLinkHrefs && _renderOptions.brokenLinkHrefs.has(node.href)) {
+            return `<a class="sdoc-link sdoc-broken-link" href="${escapeAttr(node.href)}" target="_blank" rel="noopener noreferrer"><span class="sdoc-broken-icon">\u26A0</span>${renderInlineNodes(
+              node.children
+            )}</a>`;
+          }
           return `<a class="sdoc-link" href="${escapeAttr(node.href)}" target="_blank" rel="noopener noreferrer">${renderInlineNodes(
             node.children
           )}</a>`;
@@ -1981,6 +1989,19 @@ const DEFAULT_STYLE = `
   .sdoc-mark-negative  { background-color: rgba(210, 25, 25, 0.18); color: #a81414; }
   .sdoc-mark-highlight { background-color: rgba(255, 255, 0, 0.75); }
 
+  .sdoc-broken-ref, .sdoc-link.sdoc-broken-link {
+    color: #c33;
+    text-decoration: wavy underline #c33;
+    text-underline-offset: 2px;
+    background: rgba(204, 51, 51, 0.08);
+    border-radius: 2px;
+    padding: 0 0.15em;
+  }
+  .sdoc-broken-icon {
+    font-size: 0.75em;
+    margin-right: 0.15em;
+  }
+
   .sdoc-image {
     display: inline-block;
     max-width: 100%;
@@ -2093,11 +2114,44 @@ const PRINT_STYLE = `
       word-wrap: break-word;
     }
     .sdoc-mark { -webkit-print-color-adjust: exact; print-color-adjust: exact; }
+    .sdoc-table { break-inside: avoid; }
+    .sdoc-code { break-inside: avoid; }
+    .sdoc-blockquote { break-inside: avoid; }
   }
 `;
 
 const MERMAID_CDN = "https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.min.js";
 const KATEX_CDN_CSS = "https://cdn.jsdelivr.net/npm/katex@0.16/dist/katex.min.css";
+const HLJS_CDN = "https://cdn.jsdelivr.net/npm/@highlightjs/cdn-assets@11.11.1/highlight.min.js";
+
+// Custom highlight.js grammar for SDOC language – registered before highlightElement calls
+const HLJS_SDOC_GRAMMAR = `hljs.registerLanguage('sdoc',function(hljs){return{name:'SDOC',aliases:['sdoc'],contains:[
+hljs.COMMENT('^\\\\s*//','$'),
+{className:'section',begin:'^\\\\s*#+\\\\s+',end:'$',contains:[
+  {className:'symbol',begin:'@[A-Za-z_][A-Za-z0-9_-]*'}
+]},
+{className:'meta',begin:'^\\\\s*@[A-Za-z_][A-Za-z0-9_-]*',end:'(?=\\\\s*\\\\{|$)'},
+{className:'code',begin:'\`\`\`',end:'\`\`\`',contains:[hljs.BACKSLASH_ESCAPE]},
+{className:'code',begin:'\`[^\`]+\`'},
+{className:'string',begin:'\\\\[',end:'\\\\]\\\\([^)]*\\\\)',contains:[
+  {className:'link',begin:'\\\\(',end:'\\\\)'}
+]},
+{className:'keyword',begin:'\\\\{[!?+\\\\-=~^]',end:'[!?+\\\\-=~^]\\\\}'},
+{className:'keyword',begin:'\\\\{\\\\[(?:\\\\.|#|\\\\d+|table)\\\\]'},
+{className:'strong',begin:'\\\\*\\\\*',end:'\\\\*\\\\*'},
+{className:'emphasis',begin:'(?<!\\\\*)\\\\*(?!\\\\*)',end:'\\\\*(?!\\\\*)'},
+{className:'deletion',begin:'~~',end:'~~'},
+{className:'bullet',begin:'^\\\\s*[-]\\\\s'},
+{className:'bullet',begin:'^\\\\s*\\\\d+[.)]\\\\ '},
+{className:'quote',begin:'^\\\\s*>',end:'$'},
+{className:'symbol',begin:'(?<!\\\\\\\\)@[A-Za-z_][A-Za-z0-9_-]*'},
+{className:'attr',begin:'[A-Za-z_][A-Za-z0-9_-]*(?=\\\\s*:)',end:':',excludeEnd:true}
+]}});`;
+
+// Highlight.js GitHub light theme (inline so it works inside shadow DOM in the web viewer)
+const HLJS_LIGHT_CSS = "pre code.hljs{display:block;overflow-x:auto;padding:1em}code.hljs{padding:3px 5px}.hljs{color:#24292e;background:#fff}.hljs-doctag,.hljs-keyword,.hljs-meta .hljs-keyword,.hljs-template-tag,.hljs-template-variable,.hljs-type,.hljs-variable.language_{color:#d73a49}.hljs-title,.hljs-title.class_,.hljs-title.class_.inherited__,.hljs-title.function_{color:#6f42c1}.hljs-attr,.hljs-attribute,.hljs-literal,.hljs-meta,.hljs-number,.hljs-operator,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-variable{color:#005cc5}.hljs-meta .hljs-string,.hljs-regexp,.hljs-string{color:#032f62}.hljs-built_in,.hljs-symbol{color:#e36209}.hljs-code,.hljs-comment,.hljs-formula{color:#6a737d}.hljs-name,.hljs-quote,.hljs-selector-pseudo,.hljs-selector-tag{color:#22863a}.hljs-subst{color:#24292e}.hljs-section{color:#005cc5;font-weight:700}.hljs-bullet{color:#735c0f}.hljs-emphasis{color:#24292e;font-style:italic}.hljs-strong{color:#24292e;font-weight:700}.hljs-addition{color:#22863a;background-color:#f0fff4}.hljs-deletion{color:#b31d28;background-color:#ffeef0}";
+// Highlight.js GitHub dark theme color overrides (used in @media and VS Code dark-mode CSS)
+const HLJS_DARK_COLORS_CSS = ".hljs{color:#c9d1d9;background:transparent}.hljs-doctag,.hljs-keyword,.hljs-meta .hljs-keyword,.hljs-template-tag,.hljs-template-variable,.hljs-type,.hljs-variable.language_{color:#ff7b72}.hljs-title,.hljs-title.class_,.hljs-title.class_.inherited__,.hljs-title.function_{color:#d2a8ff}.hljs-attr,.hljs-attribute,.hljs-literal,.hljs-meta,.hljs-number,.hljs-operator,.hljs-selector-attr,.hljs-selector-class,.hljs-selector-id,.hljs-variable{color:#79c0ff}.hljs-meta .hljs-string,.hljs-regexp,.hljs-string{color:#a5d6ff}.hljs-built_in,.hljs-symbol{color:#ffa657}.hljs-code,.hljs-comment,.hljs-formula{color:#8b949e}.hljs-name,.hljs-quote,.hljs-selector-pseudo,.hljs-selector-tag{color:#7ee787}.hljs-subst{color:#c9d1d9}.hljs-section{color:#1f6feb;font-weight:700}.hljs-bullet{color:#f2cc60}.hljs-emphasis{color:#c9d1d9;font-style:italic}.hljs-strong{color:#c9d1d9;font-weight:700}.hljs-addition{color:#aff5b4;background-color:#033a16}.hljs-deletion{color:#ffdcd7;background-color:#67060c}";
 
 function hasMermaidBlocks(nodes) {
   for (const node of nodes) {
@@ -2112,9 +2166,21 @@ function hasMermaidBlocks(nodes) {
   return false;
 }
 
-function renderHtmlDocumentFromParsed(parsed, title, options = {}) {
-  _renderOptions = options.renderOptions ?? {};
-  const body = parsed.nodes
+function hasHighlightableCodeBlocks(nodes) {
+  for (const node of nodes) {
+    if (node.type === "code" && node.lang && node.lang !== "mermaid" && node.lang !== "math") return true;
+    if (node.children && hasHighlightableCodeBlocks(node.children)) return true;
+    if (node.items) {
+      for (const item of node.items) {
+        if (item.children && hasHighlightableCodeBlocks(item.children)) return true;
+      }
+    }
+  }
+  return false;
+}
+
+function renderBodyNodes(nodes) {
+  return nodes
     .map((node, index) => {
       if (node.type === "scope" && index === 0) {
         return renderScope(node, 1, true);
@@ -2122,6 +2188,17 @@ function renderHtmlDocumentFromParsed(parsed, title, options = {}) {
       return renderNode(node, 1);
     })
     .join("\n");
+}
+
+function renderHtmlBody(text) {
+  const parsed = parseSdoc(text);
+  const metaResult = extractMeta(parsed.nodes);
+  return renderBodyNodes(metaResult.nodes);
+}
+
+function renderHtmlDocumentFromParsed(parsed, title, options = {}) {
+  _renderOptions = options.renderOptions ?? {};
+  const body = renderBodyNodes(parsed.nodes);
   _renderOptions = {};
   const errorHtml = renderErrors(parsed.errors);
 
@@ -2152,6 +2229,15 @@ function renderHtmlDocumentFromParsed(parsed, title, options = {}) {
   const katexCssTag = body.includes('class="katex"')
     ? `\n<link rel="stylesheet" href="${KATEX_CDN_CSS}" />`
     : "";
+  const hasHljs = hasHighlightableCodeBlocks(parsed.nodes);
+  // Highlight.js CSS is inlined (not a <link>) so it is extracted by parseDocHtml in the web viewer
+  // and applied inside shadow DOM. The @media query handles dark mode in browsers.
+  const hljsCssInline = hasHljs
+    ? `\n${HLJS_LIGHT_CSS}\n.sdoc-code code.hljs{padding:0;background:transparent}\n@media (prefers-color-scheme:dark){${HLJS_DARK_COLORS_CSS}}`
+    : "";
+  const hljsScript = hasHljs
+    ? `\n<script src="${HLJS_CDN}"></script>\n<script>${HLJS_SDOC_GRAMMAR}\ndocument.querySelectorAll('pre.sdoc-code code[class*="language-"]').forEach(function(b){hljs.highlightElement(b);});</script>`
+    : "";
 
   return `<!DOCTYPE html>
 <html lang="en">
@@ -2160,7 +2246,7 @@ function renderHtmlDocumentFromParsed(parsed, title, options = {}) {
 <meta name="viewport" content="width=device-width, initial-scale=1.0" />
 <title>${escapeHtml(title)}</title>
 <style>
-${cssBase}${cssAppend}
+${cssBase}${cssAppend}${hljsCssInline}
 </style>${katexCssTag}
 </head>
 <body>
@@ -2174,7 +2260,7 @@ ${cssBase}${cssAppend}
       </main>
     </div>
     ${footerContent ? `<footer class="sdoc-page-footer">${footerContent}</footer>` : ""}
-  </div>${scriptTag}${mermaidScript}
+  </div>${scriptTag}${mermaidScript}${hljsScript}
 </body>
 </html>`;
 }
@@ -2430,6 +2516,127 @@ function extractAbout(nodes) {
   return null;
 }
 
+function collectAllIds(nodes) {
+  const ids = new Set();
+  function walk(nodeList) {
+    for (const node of nodeList) {
+      if (node.type === "scope") {
+        if (node.id) ids.add(node.id);
+        if (node.title) ids.add(slugify(node.title));
+      }
+      if (node.children) walk(node.children);
+      if (node.type === "list" && node.items) {
+        walk(node.items);
+      }
+    }
+  }
+  walk(nodes);
+  return ids;
+}
+
+function collectInlineRefs(nodes) {
+  const refs = [];
+  const links = [];
+
+  function walkInlineNodes(inlineNodes, lineStart, lineEnd) {
+    for (const node of inlineNodes) {
+      if (node.type === "ref") {
+        refs.push({ id: node.id, lineStart, lineEnd });
+      } else if (node.type === "link") {
+        links.push({ href: node.href, lineStart, lineEnd });
+      }
+      if (node.children) {
+        walkInlineNodes(node.children, lineStart, lineEnd);
+      }
+    }
+  }
+
+  function processText(text, lineStart, lineEnd) {
+    const inlineNodes = parseInline(text);
+    walkInlineNodes(inlineNodes, lineStart, lineEnd);
+  }
+
+  function walk(nodeList) {
+    for (const node of nodeList) {
+      if (node.type === "paragraph" && node.text) {
+        processText(node.text, node.lineStart, node.lineEnd);
+      } else if (node.type === "blockquote" && node.paragraphs) {
+        for (const para of node.paragraphs) {
+          processText(para, node.lineStart, node.lineEnd);
+        }
+      } else if (node.type === "scope") {
+        if (node.title) {
+          processText(node.title, node.lineStart, node.lineStart);
+        }
+        if (node.children) walk(node.children);
+      } else if (node.type === "list" && node.items) {
+        walk(node.items);
+      } else if (node.type === "table") {
+        if (node.headers) {
+          for (const cell of node.headers) {
+            processText(cell, node.lineStart, node.lineEnd);
+          }
+        }
+        if (node.rows) {
+          for (const row of node.rows) {
+            for (const cell of row) {
+              processText(cell, node.lineStart, node.lineEnd);
+            }
+          }
+        }
+      }
+      // Handle list items (no type field, but have title/children)
+      if (!node.type) {
+        if (node.title) processText(node.title, node.lineStart || 0, node.lineEnd || node.lineStart || 0);
+        if (node.children) walk(node.children);
+      }
+    }
+  }
+  walk(nodes);
+  return { refs, links };
+}
+
+function validateRefs(nodes, options = {}) {
+  const ids = collectAllIds(nodes);
+  const externalIds = options.externalIds || new Set();
+  const { refs, links } = collectInlineRefs(nodes);
+  const warnings = [];
+
+  for (const ref of refs) {
+    if (!ids.has(ref.id) && !externalIds.has(ref.id)) {
+      warnings.push({
+        type: "broken-ref",
+        id: ref.id,
+        message: `Broken reference: @${ref.id} does not match any scope ID or title`,
+        lineStart: ref.lineStart,
+        lineEnd: ref.lineEnd
+      });
+    }
+  }
+
+  for (const link of links) {
+    const href = link.href;
+    if (/^https?:\/\//i.test(href) || /^mailto:/i.test(href) || href.startsWith("#") || href.startsWith("data:")) {
+      continue;
+    }
+    if (options.resolveFilePath) {
+      const filePath = href.split("#")[0].split("?")[0];
+      if (!filePath) continue;
+      if (!options.resolveFilePath(filePath)) {
+        warnings.push({
+          type: "broken-link",
+          href,
+          message: `Broken link: file not found — ${href}`,
+          lineStart: link.lineStart,
+          lineEnd: link.lineEnd
+        });
+      }
+    }
+  }
+
+  return warnings;
+}
+
 async function resolveIncludes(nodes, resolverFn) {
   for (const node of nodes) {
     if (node.type === "code" && node.src) {
@@ -2466,6 +2673,7 @@ module.exports = {
   resolveIncludes,
   renderFragment,
   renderTextParagraphs,
+  renderHtmlBody,
   renderHtmlDocumentFromParsed,
   renderHtmlDocument,
   formatSdoc,
@@ -2476,6 +2684,10 @@ module.exports = {
   extractAbout,
   extractDataBlocks,
   KNOWN_SCOPE_TYPES,
+  // Validation
+  collectAllIds,
+  collectInlineRefs,
+  validateRefs,
   // Low-level helpers for custom renderers (e.g. slide-renderer)
   parseInline,
   renderKatex,

package/tools/sdoc2html

@@ -0,0 +1,32 @@
+#!/usr/bin/env node
+// SDOC → HTML fragment translator
+//
+// Usage:
+//   node tools/sdoc2html input.sdoc     # read from file
+//   cat input.sdoc | node tools/sdoc2html  # read from stdin
+//
+// Writes an HTML fragment to stdout (no <!DOCTYPE>, no <html>/<body> wrapper).
+// Designed for integration with github/markup as a translator script.
+
+const fs = require("fs");
+const { renderHtmlBody } = require("../src/sdoc");
+
+function main() {
+  let input;
+  const filename = process.argv[2];
+
+  if (filename) {
+    try {
+      input = fs.readFileSync(filename, "utf8");
+    } catch (err) {
+      process.stderr.write(`sdoc2html: ${err.message}\n`);
+      process.exit(1);
+    }
+  } else {
+    input = fs.readFileSync(0, "utf8");
+  }
+
+  process.stdout.write(renderHtmlBody(input) + "\n");
+}
+
+main();