@tishlang/tishdoc-parse 0.1.0

package/README.md

@@ -0,0 +1,55 @@
+# @tishlang/tishdoc-parse
+
+Tish-native Markdown / TishDoc parser. Frontmatter (YAML / JSON), `meta.imports`, `:::directive`, includes, and a CommonMark/GFM-ish body grammar (headings, paragraphs, fenced code, lists ordered + unordered, blockquotes, thematic breaks, tables, links, images, italics, bold, strikethrough, code spans, autolinks, hard line breaks) → AST + diagnostics.
+
+## Install
+
+```bash
+npm install @tishlang/tishdoc-parse
+```
+
+## Usage (Tish)
+
+```tish
+import { parseDocument, stringifyAst } from "@tishlang/tishdoc-parse"
+
+let result = parseDocument(source, null)
+let json = stringifyAst(result.ast)
+```
+
+## Exports
+
+- `parseDocument(source, options) -> { ast, diagnostics, bodyStartLine1 }`
+- `stringifyDiagnostics(diagnostics) -> string`
+- `stringifyAst(ast) -> string`
+
+`options` is optional and may include:
+
+| Key | Type | Notes |
+|---|---|---|
+| `readPartial` | `(path) => string` | Resolver for `:::include{path=…}` blocks. |
+| `maxIncludeDepth` | `number` | Default 16. |
+| `strictMeta` | `boolean` | Promote meta validation warnings to errors. |
+
+## AST node types (subset for README rendering)
+
+`Document`, `Heading`, `Paragraph`, `Text`, `Emphasis` (`<strong>`), `Italic` (`<em>`), `Strikethrough`, `CodeSpan`, `CodeBlock`, `Link`, `Image`, `LineBreak`, `BlockQuote`, `ThematicBreak`, `List` (unordered), `OrderedList`, `ListItem`, `Table`, `TableCell`, `DirectiveBlock`, `DirectiveLeaf`, `Include`.
+
+See `src/ast.tish` for builder signatures.
+
+## Companions
+
+- [`@tishlang/tishdoc-render-html`](https://github.com/tishlang/tishdoc-render-html) — AST → safe semantic HTML (default consumer for READMEs and docs).
+- [`@tishlang/tish-syntax-highlight`](https://github.com/tishlang/tish-syntax-highlight) — fenced-code syntax highlighter used by the HTML renderer.
+- [`@tishlang/tishdoc-render-typst`](https://github.com/tishlang/tishdoc-render-typst) — AST → Typst source (for PDF / print pipelines).
+
+## Build
+
+```bash
+tish build --target js --no-optimize src/main.tish -o dist/tishdoc-parse.js
+TISH_EXPORT_OUT=dist/tishdoc-parse.js \
+  TISH_EXPORT_NAMES=parseDocument,stringifyDiagnostics,stringifyAst \
+  node scripts/append-exports.mjs
+```
+
+The script replaces the older `tish-creator/scripts/js_append_exports.tish` so this package is now standalone.

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@tishlang/tishdoc-parse",
+  "version": "0.1.0",
+  "description": "TishDoc parse pipeline: frontmatter, includes, Markdown+directives → AST",
+  "type": "module",
+  "tish": {
+    "module": "./src/main.tish"
+  },
+  "main": "./src/main.tish",
+  "exports": {
+    ".": {
+      "tish": "./src/main.tish",
+      "import": "./dist/tishdoc-parse.js",
+      "default": "./dist/tishdoc-parse.js"
+    }
+  },
+  "devDependencies": {
+    "@tishlang/tish": "1.9.2"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/tishlang/tishdoc-parse.git"
+  },
+  "scripts": {
+    "build": "tish build --target js --no-optimize src/main.tish -o dist/tishdoc-parse.js && TISH_EXPORT_OUT=dist/tishdoc-parse.js TISH_EXPORT_NAMES=parseDocument,stringifyDiagnostics,stringifyAst node scripts/append-exports.mjs",
+    "test": "tish run test/run_tests.tish && tish run test/golden_note.tish && tish run test/markdown_blocks.tish && tish run test/markdown_document.tish && tish run test/include_resolve.tish && tish run test/meta_imports.tish"
+  },
+  "dependencies": {},
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "src",
+    "README.md"
+  ]
+}

package/src/ast.tish

@@ -0,0 +1,181 @@
+// AST node builders — schema matches docs/schema/tishdoc-ast.schema.json
+
+export fn schemaVersion() {
+  return 1
+}
+
+export fn documentNode(meta, children) {
+  let n = {}
+  n["type"] = "Document"
+  n["schemaVersion"] = schemaVersion()
+  n["meta"] = meta
+  n["children"] = children
+  return n
+}
+
+export fn headingNode(level, children) {
+  let n = {}
+  n["type"] = "Heading"
+  n["level"] = level
+  n["children"] = children
+  return n
+}
+
+export fn paragraphNode(children) {
+  let n = {}
+  n["type"] = "Paragraph"
+  n["children"] = children
+  return n
+}
+
+export fn textNode(value) {
+  let n = {}
+  n["type"] = "Text"
+  n["value"] = value
+  return n
+}
+
+export fn emphasisNode(children) {
+  let n = {}
+  n["type"] = "Emphasis"
+  n["children"] = children
+  return n
+}
+
+export fn codeSpanNode(value) {
+  let n = {}
+  n["type"] = "CodeSpan"
+  n["value"] = value
+  return n
+}
+
+export fn listNode(ordered, items) {
+  let n = {}
+  n["type"] = "List"
+  n["ordered"] = ordered
+  n["items"] = items
+  return n
+}
+
+export fn listItemNode(children) {
+  let n = {}
+  n["type"] = "ListItem"
+  n["children"] = children
+  return n
+}
+
+export fn codeBlockNode(lang, value) {
+  let n = {}
+  n["type"] = "CodeBlock"
+  n["lang"] = lang
+  n["value"] = value
+  return n
+}
+
+export fn directiveBlockNode(name, attrs, children) {
+  let n = {}
+  n["type"] = "DirectiveBlock"
+  n["name"] = name
+  n["attrs"] = attrs
+  n["children"] = children
+  return n
+}
+
+export fn directiveLeafNode(name, attrs) {
+  let n = {}
+  n["type"] = "DirectiveLeaf"
+  n["name"] = name
+  n["attrs"] = attrs
+  return n
+}
+
+export fn includeNode(path, resolved, error, children) {
+  let n = {}
+  n["type"] = "Include"
+  n["path"] = path
+  n["resolved"] = resolved
+  n["error"] = error
+  n["children"] = children
+  return n
+}
+
+// --- Extended GFM/CommonMark-ish nodes for README rendering ----------------
+// Added so the same TishDoc pipeline can render arbitrary `README.md` files
+// (e.g. VS Code extension marketplace pages). These coexist with the
+// existing TishDoc nodes; older renderers should fall through to a no-op
+// for unknown `type` values.
+
+export fn italicNode(children) {
+  let n = {}
+  n["type"] = "Italic"
+  n["children"] = children
+  return n
+}
+
+export fn strikethroughNode(children) {
+  let n = {}
+  n["type"] = "Strikethrough"
+  n["children"] = children
+  return n
+}
+
+export fn linkNode(href, title, children) {
+  let n = {}
+  n["type"] = "Link"
+  n["href"] = href
+  n["title"] = title
+  n["children"] = children
+  return n
+}
+
+export fn imageNode(src, alt, title) {
+  let n = {}
+  n["type"] = "Image"
+  n["src"] = src
+  n["alt"] = alt
+  n["title"] = title
+  return n
+}
+
+export fn lineBreakNode() {
+  let n = {}
+  n["type"] = "LineBreak"
+  return n
+}
+
+export fn blockQuoteNode(children) {
+  let n = {}
+  n["type"] = "BlockQuote"
+  n["children"] = children
+  return n
+}
+
+export fn thematicBreakNode() {
+  let n = {}
+  n["type"] = "ThematicBreak"
+  return n
+}
+
+export fn orderedListNode(start, items) {
+  let n = {}
+  n["type"] = "OrderedList"
+  n["start"] = start
+  n["items"] = items
+  return n
+}
+
+export fn tableNode(aligns, header, rows) {
+  let n = {}
+  n["type"] = "Table"
+  n["aligns"] = aligns
+  n["header"] = header
+  n["rows"] = rows
+  return n
+}
+
+export fn tableCellNode(children) {
+  let n = {}
+  n["type"] = "TableCell"
+  n["children"] = children
+  return n
+}

package/src/attrs.tish

@@ -0,0 +1,92 @@
+// Parse `{ key=value key2="v" }` into a plain object (minimal).
+
+import { trim } from "./trim.tish"
+
+export fn parseDirectiveAttrs(s) {
+  let out = {}
+  if (s === null) {
+    return out
+  }
+  let t = trim(typeof s === "string" ? s : String(s))
+  if (t === "") {
+    return out
+  }
+  if (!t.startsWith("{") || !t.endsWith("}")) {
+    return out
+  }
+  let inner = trim(t.slice(1, t.length - 1))
+  if (inner === "") {
+    return out
+  }
+  let parts = splitTopLevelSpaces(inner)
+  let pi = 0
+  while (pi < parts.length) {
+    let p = parts[pi]
+    let eq = p.indexOf("=")
+    if (eq > 0) {
+      let k = trim(p.slice(0, eq))
+      let v = trim(p.slice(eq + 1))
+      if (v.charAt(0) === "\"" && v.charAt(v.length - 1) === "\"") {
+        v = v.slice(1, v.length - 1)
+      } else if (v.charAt(0) === "'" && v.charAt(v.length - 1) === "'") {
+        v = v.slice(1, v.length - 1)
+      }
+      out[k] = v
+    }
+    pi = pi + 1
+  }
+  return out
+}
+
+fn splitTopLevelSpaces(inner) {
+  let out = []
+  let buf = ""
+  let i = 0
+  let depth = 0
+  let inStr = false
+  let sq = ""
+  while (i < inner.length) {
+    let c = inner.charAt(i)
+    if (inStr) {
+      buf = buf + c
+      if (c === sq && inner.charAt(i - 1) !== "\\") {
+        inStr = false
+      }
+      i = i + 1
+      continue
+    }
+    if (c === "\"" || c === "'") {
+      inStr = true
+      sq = c
+      buf = buf + c
+      i = i + 1
+      continue
+    }
+    if (c === "{" || c === "(" || c === "[") {
+      depth = depth + 1
+      buf = buf + c
+      i = i + 1
+      continue
+    }
+    if (c === "}" || c === ")" || c === "]") {
+      depth = depth - 1
+      buf = buf + c
+      i = i + 1
+      continue
+    }
+    if (c === " " && depth === 0) {
+      if (trim(buf) !== "") {
+        out.push(trim(buf))
+      }
+      buf = ""
+      i = i + 1
+      continue
+    }
+    buf = buf + c
+    i = i + 1
+  }
+  if (trim(buf) !== "") {
+    out.push(trim(buf))
+  }
+  return out
+}

package/src/frontmatter.tish

@@ -0,0 +1,160 @@
+import { parseSimpleYamlBlock } from "./yaml_simple.tish"
+
+fn makeFm(meta, body, metaRaw, diagnostics, language) {
+  let out = {}
+  out.meta = meta
+  out.body = body
+  out.metaRaw = metaRaw
+  out.diagnostics = diagnostics
+  out.language = language
+  return out
+}
+
+fn splitJsonFrontmatterFromStart(raw) {
+  if (!raw.startsWith("{")) {
+    return null
+  }
+  let depth = 0
+  let i = 0
+  let inStr = false
+  let strQuote = ""
+  let esc = false
+  while (i < raw.length) {
+    let c = raw.charAt(i)
+    if (inStr) {
+      if (esc) {
+        esc = false
+      } else if (c === "\\") {
+        esc = true
+      } else if (c === strQuote) {
+        inStr = false
+      }
+      i = i + 1
+      continue
+    }
+    if (c === "\"" || c === "'") {
+      inStr = true
+      strQuote = c
+      i = i + 1
+      continue
+    }
+    if (c === "{") {
+      depth = depth + 1
+    } else if (c === "}") {
+      depth = depth - 1
+      if (depth === 0) {
+        let jsonStr = raw.slice(0, i + 1)
+        let rest = raw.slice(i + 1)
+        let k = 0
+        while (k < rest.length) {
+          let ch = rest.charAt(k)
+          if (ch !== " " && ch !== "\t" && ch !== "\r" && ch !== "\n") {
+            break
+          }
+          k = k + 1
+        }
+        rest = rest.slice(k)
+        if (rest.length > 0 && rest.charAt(0) !== "\n") {
+          rest = "\n" + rest
+        }
+        let jr = {}
+        jr.jsonStr = jsonStr
+        jr.rest = rest
+        return jr
+      }
+    }
+    i = i + 1
+  }
+  return null
+}
+
+export fn matterPing() {
+  return 99
+}
+
+export fn splitMatter(raw) {
+  let diagnostics = []
+  if (raw === null) {
+    return makeFm({}, "", "", diagnostics, "")
+  }
+  let s = raw
+  if (typeof s !== "string") {
+    s = String(s)
+  }
+  let lines = s.split("\n")
+  if (lines.length === 0) {
+    return makeFm({}, s, "", diagnostics, "")
+  }
+
+  if (!lines[0].startsWith("---")) {
+    if (s.trim().startsWith("{")) {
+      let r = splitJsonFrontmatterFromStart(s)
+      if (r !== null) {
+        let metaJson = {}
+        try {
+          metaJson = JSON.parse(r.jsonStr)
+        } catch (e) {
+          let de = {}
+          de.level = "error"
+          de.message = "Invalid JSON frontmatter"
+          diagnostics.push(de)
+          return makeFm({}, s, r.jsonStr, diagnostics, "json")
+        }
+        return makeFm(metaJson, r.rest, r.jsonStr, diagnostics, "json")
+      }
+    }
+  }
+
+  if (!lines[0].startsWith("---")) {
+    return makeFm({}, s, "", diagnostics, "")
+  }
+
+  let lang = "yaml"
+  let rest0 = lines[0].trim().slice(3).trim()
+  if (rest0 !== "") {
+    lang = rest0
+  }
+
+  let i = 1
+  let found = false
+  let matterLines = []
+  while (i < lines.length) {
+    let line = lines[i]
+    if (line.trim() === "---") {
+      found = true
+      break
+    }
+    matterLines.push(line)
+    i = i + 1
+  }
+
+  if (!found) {
+    let du = {}
+    du.level = "warn"
+    du.message = "Unclosed frontmatter fence; treating whole file as body"
+    diagnostics.push(du)
+    return makeFm({}, s, "", diagnostics, "")
+  }
+
+  let matterStr = matterLines.join("\n")
+  let bodyLines = []
+  let j = i + 1
+  while (j < lines.length) {
+    bodyLines.push(lines[j])
+    j = j + 1
+  }
+  let body = bodyLines.join("\n")
+
+  let meta = {}
+  if (lang === "yaml" || lang === "yml") {
+    meta = parseSimpleYamlBlock(matterStr)
+  } else {
+    let dw = {}
+    dw.level = "warn"
+    dw.message = "Frontmatter language not supported for fence: " + lang + "; use yaml, yml, or JSON object at file start."
+    diagnostics.push(dw)
+    return makeFm({}, body, matterStr, diagnostics, lang)
+  }
+
+  return makeFm(meta, body, matterStr, diagnostics, lang)
+}

package/src/includes.tish

@@ -0,0 +1,180 @@
+// Resolve `::include{path="relative.md"}` or `::include{path=relative.md}` in source (no remote URLs).
+// Helpers must appear before `resolveOnce` / export so `tish run` on this module alone sees them.
+
+fn trimIncludePath(s) {
+  let a = 0
+  let b = s.length
+  while (a < b) {
+    let c = s.charAt(a)
+    if (c !== " " && c !== "\t" && c !== "\r" && c !== "\n") {
+      break
+    }
+    a = a + 1
+  }
+  while (b > a) {
+    let c2 = s.charAt(b - 1)
+    if (c2 !== " " && c2 !== "\t" && c2 !== "\r" && c2 !== "\n") {
+      break
+    }
+    b = b - 1
+  }
+  return s.slice(a, b)
+}
+
+fn arrayContains(arr, v) {
+  let i = 0
+  while (i < arr.length) {
+    if (arr[i] === v) {
+      return true
+    }
+    i = i + 1
+  }
+  return false
+}
+
+fn resolveOnce(text, readPartial, maxDepth, visited, depth, diagnostics) {
+  if (depth > maxDepth) {
+    let d1 = {}
+    d1["level"] = "error"
+    d1["message"] = "Include max depth exceeded (" + String(maxDepth) + ")"
+    diagnostics.push(d1)
+    let rx = {}
+    rx["body"] = text
+    rx["diagnostics"] = diagnostics
+    return rx
+  }
+
+  let marker = "::include{path="
+  let out = ""
+  let pos = 0
+  while (pos < text.length) {
+    let idx = text.indexOf(marker, pos)
+    if (idx === -1) {
+      out = out + text.slice(pos)
+      break
+    }
+    out = out + text.slice(pos, idx)
+    let pathStart = idx + marker.length
+    let path = ""
+    let endDirective = -1
+    if (pathStart >= text.length) {
+      let d0 = {}
+      d0["level"] = "error"
+      d0["message"] = "Incomplete ::include after path="
+      diagnostics.push(d0)
+      out = out + text.slice(idx)
+      break
+    }
+    if (text.charAt(pathStart) === "\"") {
+      let closeQuote = text.indexOf("\"", pathStart + 1)
+      if (closeQuote === -1) {
+        let d2 = {}
+        d2["level"] = "error"
+        d2["message"] = "Unclosed path in ::include"
+        diagnostics.push(d2)
+        out = out + text.slice(idx)
+        break
+      }
+      path = text.slice(pathStart + 1, closeQuote)
+      let afterQuote = closeQuote + 1
+      if (afterQuote >= text.length || text.charAt(afterQuote) !== "}") {
+        let d3 = {}
+        d3["level"] = "error"
+        d3["message"] = "Expected } after path in ::include"
+        diagnostics.push(d3)
+        out = out + text.slice(idx)
+        break
+      }
+      endDirective = afterQuote + 1
+    } else {
+      let endBrace = text.indexOf("}", pathStart)
+      if (endBrace === -1) {
+        let d6 = {}
+        d6["level"] = "error"
+        d6["message"] = "Unclosed ::include (missing })"
+        diagnostics.push(d6)
+        out = out + text.slice(idx)
+        break
+      }
+      path = trimIncludePath(text.slice(pathStart, endBrace))
+      if (path.length === 0) {
+        let d7 = {}
+        d7["level"] = "error"
+        d7["message"] = "Empty path in ::include"
+        diagnostics.push(d7)
+        out = out + text.slice(idx)
+        break
+      }
+      endDirective = endBrace + 1
+    }
+    if (arrayContains(visited, path)) {
+      let d4 = {}
+      d4["level"] = "error"
+      d4["message"] = "Include cycle detected: " + path
+      diagnostics.push(d4)
+      out = out + "<!-- include cycle: " + path + " -->"
+      pos = endDirective
+      continue
+    }
+
+    let content = readPartial(path)
+    if (content === null) {
+      let d5 = {}
+      d5["level"] = "error"
+      d5["message"] = "Include not found: " + path
+      diagnostics.push(d5)
+      out = out + "<!-- missing include: " + path + " -->"
+      pos = endDirective
+      continue
+    }
+
+    let nextVisited = []
+    let vi = 0
+    while (vi < visited.length) {
+      nextVisited.push(visited[vi])
+      vi = vi + 1
+    }
+    nextVisited.push(path)
+
+    let nested = resolveOnce(content, readPartial, maxDepth, nextVisited, depth + 1, diagnostics)
+    out = out + nested["body"]
+    pos = endDirective
+  }
+  let rf = {}
+  rf["body"] = out
+  rf["diagnostics"] = diagnostics
+  return rf
+}
+
+export fn resolveIncludesInBody(srcText, incOpts) {
+  let diagnostics = []
+  if (srcText === null) {
+    let r0 = {}
+    r0["body"] = ""
+    r0["diagnostics"] = diagnostics
+    return r0
+  }
+  let text = typeof srcText === "string" ? srcText : String(srcText)
+  let readPartial = incOpts["readPartial"]
+  if (readPartial === null) {
+    let r1 = {}
+    r1["body"] = text
+    r1["diagnostics"] = diagnostics
+    return r1
+  }
+
+  let maxDepth = incOpts["maxDepth"]
+  if (maxDepth === null) {
+    maxDepth = 16
+  }
+  let visited = incOpts["visited"]
+  if (visited === null) {
+    visited = []
+  }
+  let depth = incOpts["depth"]
+  if (depth === null) {
+    depth = 0
+  }
+
+  return resolveOnce(text, readPartial, maxDepth, visited, depth, diagnostics)
+}