@abreen/tada 1.13.9 → 1.14.2

package/build/utils/markdown-partials.ts

@@ -0,0 +1,235 @@
+import fs from 'fs';
+import path from 'path';
+import _ from 'lodash';
+import type MarkdownIt from 'markdown-it';
+import type StateCore from 'markdown-it/lib/rules_core/state_core.mjs';
+import type StateBlock from 'markdown-it/lib/rules_block/state_block.mjs';
+import { extensionIsMarkdown, isPartial } from './file-types';
+import { stripHtmlComments } from './html-comments';
+import type { RenderDependencyCollector } from '../types';
+
+const MAX_INCLUDE_DEPTH = 10;
+const PARTIAL_DIRECTIVE_PATTERN = /^\{\{\{([\s\S]*?)\}\}\}$/;
+const ESCAPED_PARTIAL_DIRECTIVE_PATTERN = /^\\\{\{\{[\s\S]*\}\}\}$/;
+const PARTIAL_CONTEXT_STACK_KEY = '__tadaMarkdownPartialContextStack';
+const LOOSE_PARTIAL_LIST_META_KEY = '__tadaLoosePartialList';
+
+interface MarkdownPartialContext {
+  filePath: string;
+  depth: number;
+}
+
+interface MarkdownPartialsEnv {
+  [PARTIAL_CONTEXT_STACK_KEY]?: MarkdownPartialContext[];
+}
+
+export interface MarkdownPartialsOptions {
+  filePath: string;
+  templateParams: Record<string, unknown>;
+  dependencyCollector?: RenderDependencyCollector;
+}
+
+function getLineText(state: StateBlock, line: number): string {
+  const start = state.bMarks[line] + state.tShift[line];
+  const end = state.eMarks[line];
+  return state.src.slice(start, end);
+}
+
+function getCurrentPartialContext(
+  env: MarkdownPartialsEnv,
+  filePath: string,
+): MarkdownPartialContext {
+  const stack = env[PARTIAL_CONTEXT_STACK_KEY];
+  return stack?.[stack.length - 1] ?? { filePath, depth: 0 };
+}
+
+function renderPartialContent(
+  relativePath: string,
+  caller: MarkdownPartialContext,
+  options: MarkdownPartialsOptions,
+): { resolvedPath: string; content: string } {
+  const resolvedPath = path.resolve(
+    path.dirname(caller.filePath),
+    relativePath,
+  );
+
+  if (!fs.existsSync(resolvedPath)) {
+    throw new Error(
+      `${caller.filePath}: partial not found: ${relativePath} (resolved to ${resolvedPath})`,
+    );
+  }
+  if (!isPartial(resolvedPath)) {
+    throw new Error(
+      `${caller.filePath}: partial target must start with "_": ${relativePath}`,
+    );
+  }
+  if (!extensionIsMarkdown(path.extname(resolvedPath).toLowerCase())) {
+    throw new Error(
+      `${caller.filePath}: partial target must be a Markdown file: ${relativePath}`,
+    );
+  }
+
+  options.dependencyCollector?.partials?.add(resolvedPath);
+  if (caller.depth >= MAX_INCLUDE_DEPTH) {
+    throw new Error(
+      `${caller.filePath}: maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded`,
+    );
+  }
+
+  const raw = fs.readFileSync(resolvedPath, 'utf-8');
+  const content = stripHtmlComments(raw);
+
+  try {
+    return {
+      resolvedPath,
+      content: _.template(content)(options.templateParams),
+    };
+  } catch (err) {
+    throw new Error(
+      `${resolvedPath}: Lodash template error in partial: ${(err as Error).message}`,
+      { cause: err },
+    );
+  }
+}
+
+function tokenizePartialContent(
+  state: StateBlock,
+  content: string,
+  context: MarkdownPartialContext,
+): void {
+  const env = state.env as MarkdownPartialsEnv;
+  const stack = (env[PARTIAL_CONTEXT_STACK_KEY] ??= []);
+  stack.push(context);
+  const tokenStart = state.tokens.length;
+
+  try {
+    const childState = new state.md.block.State(
+      content,
+      state.md,
+      state.env,
+      state.tokens,
+    );
+    childState.level = state.level;
+    childState.parentType = state.parentType;
+    childState.tight = state.tight;
+    childState.ddIndent = state.ddIndent;
+    childState.listIndent = state.listIndent;
+
+    state.md.block.tokenize(childState, childState.line, childState.lineMax);
+    if (!childState.tight) {
+      markCurrentListLoose(state, tokenStart);
+    }
+  } finally {
+    stack.pop();
+  }
+}
+
+function markCurrentListLoose(state: StateBlock, tokenStart: number): void {
+  let closedLists = 0;
+  for (let i = tokenStart - 1; i >= 0; i--) {
+    const token = state.tokens[i];
+    if (
+      token.type === 'bullet_list_close' ||
+      token.type === 'ordered_list_close'
+    ) {
+      closedLists++;
+      continue;
+    }
+
+    if (
+      token.type === 'bullet_list_open' ||
+      token.type === 'ordered_list_open'
+    ) {
+      if (closedLists === 0) {
+        token.meta = {
+          ...(token.meta ?? {}),
+          [LOOSE_PARTIAL_LIST_META_KEY]: true,
+        };
+        return;
+      }
+      closedLists--;
+    }
+  }
+}
+
+function restoreLoosePartialLists(state: StateCore): void {
+  const looseListLevels: number[] = [];
+
+  for (const token of state.tokens) {
+    if (
+      token.type === 'bullet_list_open' ||
+      token.type === 'ordered_list_open'
+    ) {
+      if (token.meta?.[LOOSE_PARTIAL_LIST_META_KEY]) {
+        looseListLevels.push(token.level);
+      }
+      continue;
+    }
+
+    if (
+      token.type === 'bullet_list_close' ||
+      token.type === 'ordered_list_close'
+    ) {
+      const listIndex = looseListLevels.lastIndexOf(token.level);
+      if (listIndex !== -1) {
+        looseListLevels.splice(listIndex, 1);
+      }
+      continue;
+    }
+
+    if (token.type !== 'paragraph_open' && token.type !== 'paragraph_close') {
+      continue;
+    }
+
+    if (looseListLevels.some(level => token.level === level + 2)) {
+      token.hidden = false;
+    }
+  }
+}
+
+export default function markdownPartialsPlugin(
+  md: MarkdownIt,
+  options: MarkdownPartialsOptions,
+): void {
+  md.core.ruler.after(
+    'block',
+    'tada_markdown_partial_loose_lists',
+    restoreLoosePartialLists,
+  );
+
+  md.block.ruler.before(
+    'paragraph',
+    'tada_markdown_partial',
+    (state, startLine, _endLine, silent) => {
+      const trimmedLine = getLineText(state, startLine).trim();
+      if (ESCAPED_PARTIAL_DIRECTIVE_PATTERN.test(trimmedLine)) {
+        return false;
+      }
+
+      const match = trimmedLine.match(PARTIAL_DIRECTIVE_PATTERN);
+      if (!match) {
+        return false;
+      }
+
+      if (silent) {
+        return true;
+      }
+
+      const relativePath = match[1].trim();
+      if (!relativePath) {
+        throw new Error(`${options.filePath}: partial path is empty`);
+      }
+
+      const env = state.env as MarkdownPartialsEnv;
+      const caller = getCurrentPartialContext(env, options.filePath);
+      const partial = renderPartialContent(relativePath, caller, options);
+      tokenizePartialContent(state, partial.content, {
+        filePath: partial.resolvedPath,
+        depth: caller.depth + 1,
+      });
+      state.line = startLine + 1;
+      return true;
+    },
+    { alt: ['paragraph', 'reference', 'blockquote', 'list'] },
+  );
+}

package/build/utils/markdown.ts

@@ -18,7 +18,8 @@ import columnsPlugin from '../columns-plugin';
 import markdownItKatex from '@vscode/markdown-it-katex';
 import katex from 'katex';
 import renderA11yString from 'katex/contrib/render-a11y-string';
-import type { SiteVariables } from '../types';
+import markdownPartialsPlugin from './markdown-partials';
+import type { RenderDependencyCollector, SiteVariables } from '../types';
 
 const DETAILS_PATTERN = /^details\s+(.*)$/;
 const ALERT_PATTERN = /^(note|warning)(?:\s+"(.+)"|\s+(.+))?$/;
@@ -31,6 +32,8 @@ interface CreateMarkdownOptions {
   literateJavaOutputPaths?: Set<string>;
   sourceUrlPath?: string;
   validTargets?: Set<string>;
+  templateParams?: Record<string, unknown>;
+  dependencyCollector?: RenderDependencyCollector;
 }
 
 function capitalize(str: string): string {
@@ -102,6 +105,14 @@ export function createMarkdown(
       },
     });
 
+  if (filePath && options.templateParams) {
+    markdown.use(markdownPartialsPlugin, {
+      filePath,
+      templateParams: options.templateParams,
+      dependencyCollector: options.dependencyCollector,
+    });
+  }
+
   function createWrapperToken(
     type: string,
     nesting: -1 | 0 | 1,

package/build/utils/python-runner/trace_runner.py

@@ -5,17 +5,22 @@ import json
 import os
 import pdb
 import sys
+import traceback
 import types
 
 
 class CaptureStream(io.TextIOBase):
-    def __init__(self, sink):
+    def __init__(self, sink, stream):
         self._sink = sink
+        self._stream = stream
 
     def write(self, s):
         if not isinstance(s, str):
             s = str(s)
-        self._sink.append(s)
+        if self._sink and self._sink[-1]["stream"] == self._stream:
+            self._sink[-1]["text"] += s
+        else:
+            self._sink.append({"stream": self._stream, "text": s})
         return len(s)
 
     def flush(self):
@@ -28,9 +33,10 @@ class CaptureStream(io.TextIOBase):
 class TracePdb(pdb.Pdb):
     SYNTHETIC_FRAME_NAMES = {"<listcomp>", "<dictcomp>", "<setcomp>"}
 
-    def __init__(self, target_path, trace_writer, output_chunks):
+    def __init__(self, target_path, traced_paths, trace_writer, output_chunks):
         super().__init__(nosigint=True, readrc=False)
         self.target_path = os.path.abspath(target_path)
+        self.traced_paths = set(os.path.abspath(path) for path in traced_paths)
         self.trace_writer = trace_writer
         self.output_chunks = output_chunks
         self.pending_snapshot = None
@@ -69,18 +75,24 @@ class TracePdb(pdb.Pdb):
         if self.pending_snapshot is None:
             return
         payload = dict(self.pending_snapshot)
-        payload["stdout"] = self._drain_output()
+        payload["output"] = self._drain_output()
         self.trace_writer.write(json.dumps(payload) + "\n")
         self.trace_writer.flush()
         self.pending_snapshot = None
 
     def _drain_output(self):
         if not self.output_chunks:
-            return ""
-        output = "".join(self.output_chunks)
+            return []
+        output = list(self.output_chunks)
         self.output_chunks.clear()
         return output
 
+    def emit_crash(self, frame):
+        payload = self._capture_snapshot(frame)
+        payload["output"] = self._drain_output()
+        self.trace_writer.write(json.dumps(payload) + "\n")
+        self.trace_writer.flush()
+
     def _capture_snapshot(self, frame):
         reachable = {}
         return {
@@ -206,7 +218,7 @@ class TracePdb(pdb.Pdb):
 
     def _is_traced_frame(self, frame):
         return (
-            os.path.abspath(frame.f_code.co_filename) == self.target_path
+            os.path.abspath(frame.f_code.co_filename) in self.traced_paths
             and frame.f_code.co_name not in self.SYNTHETIC_FRAME_NAMES
         )
 
@@ -221,11 +233,14 @@ class TracePdb(pdb.Pdb):
 
 
 def main():
-    if len(sys.argv) != 2:
-        print("Usage: trace_runner.py <python_file>", file=sys.stderr)
+    if len(sys.argv) < 2:
+        print("Usage: trace_runner.py <python_file> [traced_file ...]", file=sys.stderr)
         sys.exit(1)
 
     target_path = os.path.abspath(sys.argv[1])
+    traced_paths = [os.path.abspath(path) for path in sys.argv[2:]]
+    if target_path not in traced_paths:
+        traced_paths.insert(0, target_path)
     target_dir = os.path.dirname(target_path)
     target_name = os.path.basename(target_path)
 
@@ -234,8 +249,9 @@ def main():
 
     trace_writer = sys.stdout
     output_chunks = []
-    capture = CaptureStream(output_chunks)
-    debugger = TracePdb(target_path, trace_writer, output_chunks)
+    stdout_capture = CaptureStream(output_chunks, "stdout")
+    stderr_capture = CaptureStream(output_chunks, "stderr")
+    debugger = TracePdb(target_path, traced_paths, trace_writer, output_chunks)
 
     code_globals = {
         "__name__": "__main__",
@@ -253,14 +269,21 @@ def main():
             source = f.read()
         compiled = compile(source, target_path, "exec")
 
-        sys.stdout = capture
-        sys.stderr = capture
+        sys.stdout = stdout_capture
+        sys.stderr = stderr_capture
 
         try:
             debugger.set_step()
             debugger.runcall(exec, compiled, code_globals, code_globals)
         except SystemExit:
             pass
+        except BaseException as exc:
+            debugger.flush_pending()
+            sys.stderr.write(format_target_traceback(exc, set(traced_paths)))
+            frame = find_traced_exception_frame(debugger, exc.__traceback__)
+            if frame is None:
+                raise
+            debugger.emit_crash(frame)
         finally:
             debugger.flush_pending()
     finally:
@@ -268,5 +291,27 @@ def main():
         sys.stderr = original_stderr
 
 
+def find_traced_exception_frame(debugger, tb):
+    traced = None
+    while tb is not None:
+        if debugger._is_traced_frame(tb.tb_frame):
+            traced = tb.tb_frame
+        tb = tb.tb_next
+    return traced
+
+
+def format_target_traceback(exc, traced_paths):
+    frames = [
+        frame
+        for frame in traceback.extract_tb(exc.__traceback__)
+        if os.path.abspath(frame.filename) in traced_paths
+    ]
+    return (
+        "Traceback (most recent call last):\n"
+        + "".join(traceback.format_list(frames))
+        + "".join(traceback.format_exception_only(type(exc), exc))
+    )
+
+
 if __name__ == "__main__":
     main()

package/build/utils/render.ts

@@ -25,7 +25,7 @@ import {
 } from './code';
 import { extensionIsMarkdown } from './file-types';
 import { createTraceHelpers } from './trace';
-import { createIncludeFunction } from './include';
+import { stripHtmlComments } from './html-comments';
 import { finalizeHtmlPage } from './final-html';
 import {
   createApplyBasePath,
@@ -60,6 +60,8 @@ const log = makeLogger(import.meta.url);
 
 const tadaVersion: string = pkg.version;
 
+export { stripHtmlComments } from './html-comments';
+
 const REQUIRED_FRONT_MATTER_FIELDS = ['title'];
 
 function isWatchMode(assetFiles: string[]): boolean {
@@ -433,9 +435,9 @@ function renderPlainTextContent(
       {
         manifestUrl: string;
         artifactId: string;
-        highlightedSource: string;
+        highlightedSources: { file: string; highlightedSource: string }[];
         totalSteps: number;
-        mtime: number;
+        sourceMtims: Record<string, number>;
       }
     >;
     contentDir?: string;
@@ -459,7 +461,7 @@ function renderPlainTextContent(
       `${filePath}: slides mode is only supported on Markdown pages`,
     );
   }
-  const md = createMarkdown(siteVariables, {
+  const frontMatterMd = createMarkdown(siteVariables, {
     filePath,
     slides: pageVariables.slides === true,
   });
@@ -480,8 +482,8 @@ function renderPlainTextContent(
   );
 
   // Render title and description as inline Markdown
-  renderInlineField(md, pageVariablesProcessed, 'title');
-  renderInlineField(md, pageVariablesProcessed, 'description');
+  renderInlineField(frontMatterMd, pageVariablesProcessed, 'title');
+  renderInlineField(frontMatterMd, pageVariablesProcessed, 'description');
 
   resolveAuthor(pageVariablesProcessed, filePath, dependencyCollector);
 
@@ -526,8 +528,6 @@ function renderPlainTextContent(
     params.renderTrace = helpers.renderTrace;
   }
 
-  params.include = createIncludeFunction(filePath, params, dependencyCollector);
-
   let html: string;
   try {
     html = _.template(strippedContent)(params);
@@ -541,6 +541,12 @@ function renderPlainTextContent(
   let tocItems: unknown[] | null = null;
   let alertIds: string[] = [];
   if (extensionIsMarkdown(ext)) {
+    const md = createMarkdown(siteVariables, {
+      filePath,
+      slides: pageVariables.slides === true,
+      templateParams: params,
+      dependencyCollector,
+    });
     const env: Record<string, unknown> = { alertIds: [] as string[] };
     html = md.render(html!, env);
     tocItems = (env.tocItems as unknown[] | undefined) || null;
@@ -555,10 +561,6 @@ function renderPlainTextContent(
   };
 }
 
-export function stripHtmlComments(str: string): string {
-  return str.replace(/<!---[\s\S]*?-->/g, '');
-}
-
 export function renderLiterateJavaPageAsset({
   filePath,
   contentDir,