@tishlang/tishdoc-parse 0.1.0

package/src/inline.tish

@@ -0,0 +1,380 @@
+import {
+  textNode,
+  emphasisNode,
+  italicNode,
+  strikethroughNode,
+  codeSpanNode,
+  linkNode,
+  imageNode,
+  lineBreakNode
+} from "./ast.tish"
+
+// Inline parser. Handles: `code`, **strong**, *em* / _em_, ~~strike~~,
+// [text](url), ![alt](url), <https://autolink>, hard line break (two
+// trailing spaces + \n), and backslash escapes for special characters.
+//
+// Not full CommonMark — designed for typical READMEs. Unknown markup falls
+// through as plain text.
+
+fn isAsciiAlpha(c) {
+  if (c >= "a" && c <= "z") { return true }
+  if (c >= "A" && c <= "Z") { return true }
+  return false
+}
+
+fn isAsciiAlphaNum(c) {
+  if (isAsciiAlpha(c)) { return true }
+  if (c >= "0" && c <= "9") { return true }
+  return false
+}
+
+// Match a backtick-delimited code span: `code` or ``co`de`` (matches longest
+// run of opening backticks). Returns { value, end } or null.
+fn matchCodeSpan(s, start) {
+  let n = 0
+  while ((start + n) < s.length && s.charAt(start + n) === "`") {
+    n = n + 1
+  }
+  if (n === 0) { return null }
+  let i = start + n
+  while (i < s.length) {
+    if (s.charAt(i) === "`") {
+      let m = 0
+      while ((i + m) < s.length && s.charAt(i + m) === "`") {
+        m = m + 1
+      }
+      if (m === n) {
+        let inner = s.slice(start + n, i)
+        let r = {}
+        r["value"] = inner
+        r["end"] = i + n
+        return r
+      }
+      i = i + m
+      continue
+    }
+    i = i + 1
+  }
+  return null
+}
+
+// Match `[label](href "title")` or `![label](src "title")`. `start` points at
+// the leading `[` (caller skipped any `!`). Returns { label, dest, title, end }
+// where `end` is the index AFTER the closing `)`.
+fn matchLinkOrImage(s, start) {
+  if (start >= s.length || s.charAt(start) !== "[") { return null }
+  let depth = 1
+  let i = start + 1
+  while (i < s.length) {
+    let c = s.charAt(i)
+    if (c === "\\" && (i + 1) < s.length) {
+      i = i + 2
+      continue
+    }
+    if (c === "[") { depth = depth + 1 }
+    else if (c === "]") {
+      depth = depth - 1
+      if (depth === 0) { break }
+    }
+    i = i + 1
+  }
+  if (i >= s.length) { return null }
+  let label = s.slice(start + 1, i)
+  let j = i + 1
+  if (j >= s.length || s.charAt(j) !== "(") { return null }
+  j = j + 1
+  // Skip leading whitespace inside (...)
+  while (j < s.length && (s.charAt(j) === " " || s.charAt(j) === "\t")) { j = j + 1 }
+  let dest = ""
+  if (j < s.length && s.charAt(j) === "<") {
+    j = j + 1
+    let dStart = j
+    while (j < s.length && s.charAt(j) !== ">" && s.charAt(j) !== "\n") {
+      j = j + 1
+    }
+    if (j >= s.length || s.charAt(j) !== ">") { return null }
+    dest = s.slice(dStart, j)
+    j = j + 1
+  } else {
+    let dStart = j
+    let pdepth = 0
+    while (j < s.length) {
+      let c = s.charAt(j)
+      if (c === "\\" && (j + 1) < s.length) { j = j + 2; continue }
+      if (c === "(") { pdepth = pdepth + 1 }
+      else if (c === ")") {
+        if (pdepth === 0) { break }
+        pdepth = pdepth - 1
+      } else if (c === " " || c === "\t" || c === "\n") {
+        break
+      }
+      j = j + 1
+    }
+    dest = s.slice(dStart, j)
+  }
+  // Optional title: "..." or '...' or (...)
+  let title = null
+  while (j < s.length && (s.charAt(j) === " " || s.charAt(j) === "\t")) { j = j + 1 }
+  if (j < s.length && (s.charAt(j) === "\"" || s.charAt(j) === "'")) {
+    let q = s.charAt(j)
+    j = j + 1
+    let tStart = j
+    while (j < s.length && s.charAt(j) !== q) {
+      if (s.charAt(j) === "\\" && (j + 1) < s.length) { j = j + 2; continue }
+      j = j + 1
+    }
+    if (j >= s.length) { return null }
+    title = s.slice(tStart, j)
+    j = j + 1
+    while (j < s.length && (s.charAt(j) === " " || s.charAt(j) === "\t")) { j = j + 1 }
+  }
+  if (j >= s.length || s.charAt(j) !== ")") { return null }
+  let r = {}
+  r["label"] = label
+  r["dest"] = dest
+  r["title"] = title
+  r["end"] = j + 1
+  return r
+}
+
+// Match an autolink `<https://example.com>` or `<mailto:a@b>` returning the
+// URL and the index AFTER the closing `>`. Reject anything with whitespace.
+fn matchAutolink(s, start) {
+  if (start >= s.length || s.charAt(start) !== "<") { return null }
+  let i = start + 1
+  let urlStart = i
+  while (i < s.length) {
+    let c = s.charAt(i)
+    if (c === ">") { break }
+    if (c === " " || c === "\t" || c === "\n" || c === "<") { return null }
+    i = i + 1
+  }
+  if (i >= s.length || s.charAt(i) !== ">") { return null }
+  let url = s.slice(urlStart, i)
+  let isUrl = false
+  if (url.length >= 8 && url.slice(0, 8) === "https://") { isUrl = true }
+  if (!isUrl && url.length >= 7 && url.slice(0, 7) === "http://") { isUrl = true }
+  if (!isUrl && url.length >= 7 && url.slice(0, 7) === "mailto:") { isUrl = true }
+  if (!isUrl) { return null }
+  let r = {}
+  r["url"] = url
+  r["end"] = i + 1
+  return r
+}
+
+// Match a delimiter run: count `n` consecutive copies of `ch` starting at i.
+// Used for emphasis pairing (** **, * *, ~~ ~~). Returns the matching closing
+// run such that the inner span has at least 1 char and contains no newline
+// breaks for emphasis (we forbid intra-paragraph emphasis crossing a hard
+// break for safety).
+fn matchEmphasis(s, start, ch, want) {
+  // Skip the opening run.
+  let i = start + want
+  if (i >= s.length) { return null }
+  // Inner cannot start with whitespace for proper emphasis.
+  let firstInner = s.charAt(i)
+  if (firstInner === " " || firstInner === "\t" || firstInner === "\n") { return null }
+  while (i < s.length) {
+    let c = s.charAt(i)
+    if (c === "\\" && (i + 1) < s.length) { i = i + 2; continue }
+    if (c === "`") {
+      // Skip code span so inline code doesn't swallow our delimiter.
+      let cs = matchCodeSpan(s, i)
+      if (cs !== null) { i = cs["end"]; continue }
+    }
+    if (c === ch) {
+      let n = 0
+      while ((i + n) < s.length && s.charAt(i + n) === ch) { n = n + 1 }
+      if (n >= want) {
+        // The character before the closing run must not be whitespace for
+        // valid emphasis; inner must be non-empty.
+        let prev = s.charAt(i - 1)
+        if (prev !== " " && prev !== "\t" && prev !== "\n" && i > (start + want)) {
+          let inner = s.slice(start + want, i)
+          let r = {}
+          r["inner"] = inner
+          r["end"] = i + want
+          return r
+        }
+      }
+      i = i + n
+      continue
+    }
+    i = i + 1
+  }
+  return null
+}
+
+fn flushTextRun(out, run) {
+  if (run.length > 0) {
+    out.push(textNode(run))
+  }
+}
+
+fn italicUnderscoreLeftOk(s, pos) {
+  if (pos === 0) { return true }
+  let prev = s.charAt(pos - 1)
+  if (isAsciiAlphaNum(prev)) { return false }
+  return true
+}
+
+fn italicUnderscoreRightOk(s, ch, endPos) {
+  if (ch !== "_") { return true }
+  if (endPos >= s.length) { return true }
+  let nx = s.charAt(endPos)
+  if (isAsciiAlphaNum(nx)) { return false }
+  return true
+}
+
+fn isEscapable(nx) {
+  if (nx === "\\") { return true }
+  if (nx === "`") { return true }
+  if (nx === "*") { return true }
+  if (nx === "_") { return true }
+  if (nx === "{") { return true }
+  if (nx === "}") { return true }
+  if (nx === "[") { return true }
+  if (nx === "]") { return true }
+  if (nx === "(") { return true }
+  if (nx === ")") { return true }
+  if (nx === "#") { return true }
+  if (nx === "+") { return true }
+  if (nx === "-") { return true }
+  if (nx === ".") { return true }
+  if (nx === "!") { return true }
+  if (nx === "<") { return true }
+  if (nx === ">") { return true }
+  if (nx === "~") { return true }
+  if (nx === "|") { return true }
+  return false
+}
+
+export fn parseInlineSegments(line) {
+  let out = []
+  if (line === null) {
+    return out
+  }
+  let s = typeof line === "string" ? line : String(line)
+  let pos = 0
+  let run = ""
+  while (pos < s.length) {
+    let c = s.charAt(pos)
+
+    // Backslash escape — emit next char literally.
+    // Tish JS backend block-scopes nested `let`, so we evaluate the escape
+    // condition inline (no intermediate `let esc`) to avoid scope-leak.
+    if (c === "\\" && (pos + 1) < s.length && isEscapable(s.charAt(pos + 1))) {
+      run = run + s.charAt(pos + 1)
+      pos = pos + 2
+      continue
+    }
+
+    // Hard line break (two trailing spaces + \n) inside a paragraph.
+    if (c === "\n") {
+      if (run.length >= 2 && run.slice(run.length - 2) === "  ") {
+        run = run.slice(0, run.length - 2)
+        flushTextRun(out, run)
+        run = ""
+        out.push(lineBreakNode())
+      } else {
+        flushTextRun(out, run)
+        run = ""
+        out.push(textNode("\n"))
+      }
+      pos = pos + 1
+      continue
+    }
+
+    // Inline code.
+    if (c === "`") {
+      let cs = matchCodeSpan(s, pos)
+      if (cs !== null) {
+        flushTextRun(out, run)
+        run = ""
+        out.push(codeSpanNode(cs["value"]))
+        pos = cs["end"]
+        continue
+      }
+    }
+
+    // Image: ![alt](src)
+    if (c === "!" && (pos + 1) < s.length && s.charAt(pos + 1) === "[") {
+      let m = matchLinkOrImage(s, pos + 1)
+      if (m !== null) {
+        flushTextRun(out, run)
+        run = ""
+        out.push(imageNode(m["dest"], m["label"], m["title"]))
+        pos = m["end"]
+        continue
+      }
+    }
+
+    // Link: [text](href)
+    if (c === "[") {
+      let lm = matchLinkOrImage(s, pos)
+      if (lm !== null) {
+        flushTextRun(out, run)
+        run = ""
+        let inner = parseInlineSegments(lm["label"])
+        out.push(linkNode(lm["dest"], lm["title"], inner))
+        pos = lm["end"]
+        continue
+      }
+    }
+
+    // Autolink: <https://...>
+    if (c === "<") {
+      let al = matchAutolink(s, pos)
+      if (al !== null) {
+        flushTextRun(out, run)
+        run = ""
+        out.push(linkNode(al["url"], null, [textNode(al["url"])]))
+        pos = al["end"]
+        continue
+      }
+    }
+
+    // Strong: **text**
+    if (c === "*" && (pos + 1) < s.length && s.charAt(pos + 1) === "*") {
+      let r = matchEmphasis(s, pos, "*", 2)
+      if (r !== null) {
+        flushTextRun(out, run)
+        run = ""
+        out.push(emphasisNode(parseInlineSegments(r["inner"])))
+        pos = r["end"]
+        continue
+      }
+    }
+
+    // Italic: *text* or _text_
+    // For underscore italics, require non-alphanumeric on both sides
+    // (so identifiers like `foo_bar_baz` aren't broken).
+    if ((c === "*") || (c === "_" && italicUnderscoreLeftOk(s, pos))) {
+      let r2 = matchEmphasis(s, pos, c, 1)
+      if (r2 !== null && italicUnderscoreRightOk(s, c, r2["end"])) {
+        flushTextRun(out, run)
+        run = ""
+        out.push(italicNode(parseInlineSegments(r2["inner"])))
+        pos = r2["end"]
+        continue
+      }
+    }
+
+    // Strikethrough: ~~text~~
+    if (c === "~" && (pos + 1) < s.length && s.charAt(pos + 1) === "~") {
+      let rs = matchEmphasis(s, pos, "~", 2)
+      if (rs !== null) {
+        flushTextRun(out, run)
+        run = ""
+        out.push(strikethroughNode(parseInlineSegments(rs["inner"])))
+        pos = rs["end"]
+        continue
+      }
+    }
+
+    run = run + c
+    pos = pos + 1
+  }
+  flushTextRun(out, run)
+  return out
+}

package/src/main.tish

@@ -0,0 +1,287 @@
+import { parseSimpleYamlBlock } from "./yaml_simple.tish"
+import { resolveIncludesInBody } from "./includes.tish"
+import { parseMarkdownToBlocks } from "./parse_blocks.tish"
+import { documentNode } from "./ast.tish"
+import { validateMeta } from "./validate_meta.tish"
+import { applyMetaImportsToBody } from "./meta_imports.tish"
+
+fn splitJsonFrontmatter(raw) {
+  if (!raw.startsWith("{")) {
+    return null
+  }
+  let depth = 0
+  let i = 0
+  let inStr = false
+  let strQuote = ""
+  let esc = false
+  while (i < raw.length) {
+    let c = raw.charAt(i)
+    if (inStr) {
+      if (esc) {
+        esc = false
+      } else if (c === "\\") {
+        esc = true
+      } else if (c === strQuote) {
+        inStr = false
+      }
+      i = i + 1
+      continue
+    }
+    if (c === "\"" || c === "'") {
+      inStr = true
+      strQuote = c
+      i = i + 1
+      continue
+    }
+    if (c === "{") {
+      depth = depth + 1
+    } else if (c === "}") {
+      depth = depth - 1
+      if (depth === 0) {
+        let jsonStr = raw.slice(0, i + 1)
+        let rest = raw.slice(i + 1)
+        let k = 0
+        while (k < rest.length) {
+          let ch = rest.charAt(k)
+          if (ch !== " " && ch !== "\t" && ch !== "\r" && ch !== "\n") {
+            break
+          }
+          k = k + 1
+        }
+        rest = rest.slice(k)
+        if (rest.length > 0 && rest.charAt(0) !== "\n") {
+          rest = "\n" + rest
+        }
+        let jr = {}
+        jr["jsonStr"] = jsonStr
+        jr["rest"] = rest
+        return jr
+      }
+    }
+    i = i + 1
+  }
+  return null
+}
+
+fn splitYamlFence(raw) {
+  let diagnostics = []
+  if (raw === null) {
+    let e0 = {}
+    e0["meta"] = {}
+    e0["body"] = ""
+    e0["metaRaw"] = ""
+    e0["diagnostics"] = diagnostics
+    e0["language"] = ""
+    e0["bodyStartLine1"] = 1
+    return e0
+  }
+  let s = typeof raw === "string" ? raw : String(raw)
+  let t = s.trim()
+  if (t.startsWith("{")) {
+    let rj = splitJsonFrontmatter(s)
+    if (rj !== null) {
+      let metaJ = {}
+      try {
+        metaJ = JSON.parse(rj["jsonStr"])
+      } catch (e) {
+        let de = {}
+        de["level"] = "error"
+        de["message"] = "Invalid JSON frontmatter"
+        diagnostics.push(de)
+        let ej = {}
+        ej["meta"] = {}
+        ej["body"] = s
+        ej["metaRaw"] = rj["jsonStr"]
+        ej["diagnostics"] = diagnostics
+        ej["language"] = "json"
+        ej["bodyStartLine1"] = 1
+        return ej
+      }
+      let okj = {}
+      okj["meta"] = metaJ
+      okj["body"] = rj["rest"]
+      okj["metaRaw"] = rj["jsonStr"]
+      okj["diagnostics"] = diagnostics
+      okj["language"] = "json"
+      let jsonLines = rj["jsonStr"].split("\n").length
+      okj["bodyStartLine1"] = jsonLines + 1
+      return okj
+    }
+  }
+
+  let lines = s.split("\n")
+  if (lines.length === 0) {
+    let e1 = {}
+    e1["meta"] = {}
+    e1["body"] = s
+    e1["metaRaw"] = ""
+    e1["diagnostics"] = diagnostics
+    e1["language"] = ""
+    e1["bodyStartLine1"] = 1
+    return e1
+  }
+  if (!lines[0].startsWith("---")) {
+    let e2 = {}
+    e2["meta"] = {}
+    e2["body"] = s
+    e2["metaRaw"] = ""
+    e2["diagnostics"] = diagnostics
+    e2["language"] = ""
+    e2["bodyStartLine1"] = 1
+    return e2
+  }
+  let lang = "yaml"
+  let rest0 = lines[0].trim().slice(3).trim()
+  if (rest0 !== "") {
+    lang = rest0
+  }
+  let i = 1
+  let found = false
+  let matterLines = []
+  while (i < lines.length) {
+    let line = lines[i]
+    if (line.trim() === "---") {
+      found = true
+      break
+    }
+    matterLines.push(line)
+    i = i + 1
+  }
+  if (!found) {
+    let du = {}
+    du["level"] = "warn"
+    du["message"] = "Unclosed frontmatter fence"
+    diagnostics.push(du)
+    let e3 = {}
+    e3["meta"] = {}
+    e3["body"] = s
+    e3["metaRaw"] = ""
+    e3["diagnostics"] = diagnostics
+    e3["language"] = ""
+    e3["bodyStartLine1"] = 1
+    return e3
+  }
+  let matterStr = matterLines.join("\n")
+  let bodyLines = []
+  let j = i + 1
+  while (j < lines.length) {
+    bodyLines.push(lines[j])
+    j = j + 1
+  }
+  let body = bodyLines.join("\n")
+  let meta = {}
+  if (lang === "yaml" || lang === "yml") {
+    meta = parseSimpleYamlBlock(matterStr)
+  } else {
+    let dw = {}
+    dw["level"] = "warn"
+    dw["message"] = "Unsupported frontmatter fence language: " + lang
+    diagnostics.push(dw)
+    let e4 = {}
+    e4["meta"] = {}
+    e4["body"] = body
+    e4["metaRaw"] = matterStr
+    e4["diagnostics"] = diagnostics
+    e4["language"] = lang
+    e4["bodyStartLine1"] = i + 2
+    return e4
+  }
+  let ok = {}
+  ok["meta"] = meta
+  ok["body"] = body
+  ok["metaRaw"] = matterStr
+  ok["diagnostics"] = diagnostics
+  ok["language"] = lang
+  ok["bodyStartLine1"] = i + 2
+  return ok
+}
+
+// Options may omit keys; direct `opts["readPartial"]` throws if the key is absent (Tish).
+fn normalizeParseOptions(options) {
+  let o = {}
+  o["readPartial"] = null
+  o["maxIncludeDepth"] = null
+  o["strictMeta"] = false
+  if (options === null) {
+    return o
+  }
+  let ks = Object.keys(options)
+  let ki = 0
+  while (ki < ks.length) {
+    let key = ks[ki]
+    if (key === "readPartial") {
+      o["readPartial"] = options[key]
+    }
+    if (key === "maxIncludeDepth") {
+      o["maxIncludeDepth"] = options[key]
+    }
+    if (key === "strictMeta") {
+      o["strictMeta"] = options[key]
+    }
+    ki = ki + 1
+  }
+  return o
+}
+
+export fn parseDocument(source, options) {
+  let allDiag = []
+  let opts = normalizeParseOptions(options)
+
+  let fm = splitYamlFence(source)
+  let di = 0
+  while (di < fm["diagnostics"].length) {
+    allDiag.push(fm["diagnostics"][di])
+    di = di + 1
+  }
+
+  let metaVm = validateMeta(fm["meta"], opts)
+  let mi = 0
+  while (mi < metaVm["diagnostics"].length) {
+    allDiag.push(metaVm["diagnostics"][mi])
+    mi = mi + 1
+  }
+
+  let body = fm["body"]
+  let imp = applyMetaImportsToBody(fm["meta"], body, opts["readPartial"], allDiag)
+  body = imp["body"]
+
+  let incOpts = {}
+  incOpts["readPartial"] = opts["readPartial"]
+  incOpts["maxDepth"] = 16
+  let mdOpt = opts["maxIncludeDepth"]
+  if (mdOpt !== null) {
+    incOpts["maxDepth"] = mdOpt
+  }
+  incOpts["visited"] = []
+  incOpts["depth"] = 0
+  let inc = resolveIncludesInBody(body, incOpts)
+  let ii = 0
+  while (ii < inc["diagnostics"].length) {
+    allDiag.push(inc["diagnostics"][ii])
+    ii = ii + 1
+  }
+
+  let bsl = fm["bodyStartLine1"]
+  if (bsl === null) {
+    bsl = 1
+  }
+  let blocks = parseMarkdownToBlocks(inc["body"])
+  let ast = documentNode(fm["meta"], blocks)
+  let pr = {}
+  pr["ast"] = ast
+  pr["diagnostics"] = allDiag
+  pr["bodyStartLine1"] = bsl
+  return pr
+}
+
+
+export fn stringifyDiagnostics(diagnostics) {
+  if (diagnostics === null) {
+    return JSON.stringify([])
+  }
+  return JSON.stringify(diagnostics)
+}
+
+export fn stringifyAst(ast) {
+  return JSON.stringify(ast)
+}