@abreen/tada 1.8.1 → 1.10.1

package/build/types.d.ts

@@ -50,6 +50,7 @@ export interface ContentRenderResult {
   errors: Error[];
   changedHtmlAssetPaths: Set<string>;
   removedHtmlAssetPaths: Set<string>;
+  removedOutputRelPaths: Set<string>;
   htmlAssetsByPath: Map<string, string>;
   buildContentFiles: string[];
 }
@@ -75,6 +76,8 @@ export interface RenderPlainTextOptions {
   validInternalTargets: Set<string>;
   assetFiles: string[];
   literateJavaOutputPaths?: Set<string>;
+  dependencyCollector?: RenderDependencyCollector;
+  cachedTraceSourceDir?: string;
   traceCache?: Map<
     string,
     {
@@ -84,7 +87,12 @@ export interface RenderPlainTextOptions {
       mtime: number;
     }
   >;
-  javacAvailable?: boolean;
+  traceToolAvailability?: TraceToolAvailability;
+}
+
+export interface TraceToolAvailability {
+  java?: boolean;
+  python?: boolean;
 }
 
 /** Options for code page asset rendering */
@@ -94,6 +102,7 @@ export interface RenderCodePageOptions {
   distDir: string;
   siteVariables: SiteVariables;
   assetFiles: string[];
+  dependencyCollector?: RenderDependencyCollector;
 }
 
 /** Options for literate Java asset rendering */
@@ -106,6 +115,7 @@ export interface RenderLiterateJavaOptions {
   skipExecution?: boolean;
   validInternalTargets: Set<string>;
   literateJavaOutputPaths?: Set<string>;
+  dependencyCollector?: RenderDependencyCollector;
 }
 
 /** Options for copied content asset rendering */
@@ -115,6 +125,14 @@ export interface RenderCopiedContentOptions {
   siteVariables: SiteVariables;
 }
 
+export interface RenderDependencyCollector {
+  partials?: Set<string>;
+  traceFiles?: Set<string>;
+  internalTargets?: Set<string>;
+  generatedOutputPaths?: Set<string>;
+  setAuthorKey?: (authorKey: string) => void;
+}
+
 /** Front matter parse result (from parseFrontMatterAndContent) */
 export interface ParsedContent {
   pageVariables: Record<string, unknown>;

package/build/util.ts

@@ -1,6 +1,8 @@
 export {
   getBuildContentFiles,
   getContentFiles,
+  getContentOutputRelPaths,
+  getContentSourceOutputRelPaths,
   getFilesByExtensions,
   getValidInternalTargets,
   shouldSkipContentFile,
@@ -15,6 +17,7 @@ export {
   getProjectDir,
   getPublicDir,
   normalizeOutputPath,
+  toPosix,
 } from './utils/paths';
 export {
   injectAssetTags,

package/build/utils/content-files.ts

@@ -51,6 +51,104 @@ export function getBuildContentFiles(
   );
 }
 
+export function getContentSourceOutputRelPaths(
+  contentDir: string,
+  filePath: string,
+  codeExtensions: string[],
+  codeEnabled: boolean,
+): Set<string> {
+  const processedExtSet = new Set(getProcessedExtensions(codeExtensions));
+  const relPath = toPosix(path.relative(contentDir, filePath));
+  const ext = path.extname(filePath).slice(1).toLowerCase();
+  const parsed = path.parse(relPath);
+  const outputs = new Set<string>();
+
+  if (!processedExtSet.has(ext)) {
+    outputs.add(relPath);
+    return outputs;
+  }
+
+  const buildContentFiles = new Set(
+    getBuildContentFiles(contentDir, codeExtensions),
+  );
+  if (!buildContentFiles.has(filePath)) {
+    return outputs;
+  }
+
+  if (isLiterateJava(filePath)) {
+    outputs.add(toPosix(path.join(parsed.dir, `${parsed.name}.html`)));
+    outputs.add(toPosix(path.join(parsed.dir, parsed.name)));
+    return outputs;
+  }
+
+  if (
+    extensionIsMarkdown(parsed.ext.toLowerCase()) ||
+    parsed.ext.toLowerCase() === '.html'
+  ) {
+    outputs.add(toPosix(path.join(parsed.dir, `${parsed.name}.html`)));
+    return outputs;
+  }
+
+  const codeExtensionSet = new Set(
+    codeExtensions.map(value => value.toLowerCase()),
+  );
+  if (codeExtensionSet.has(ext)) {
+    outputs.add(relPath);
+    if (codeEnabled) {
+      outputs.add(`${relPath}.html`);
+    }
+  }
+
+  return outputs;
+}
+
+export function getContentOutputRelPaths(
+  contentDir: string,
+  codeExtensions: string[],
+  codeEnabled: boolean,
+): Set<string> {
+  const buildContentFiles = getBuildContentFiles(contentDir, codeExtensions);
+  const codeExtensionSet = new Set(
+    codeExtensions.map(ext => ext.toLowerCase()),
+  );
+  const outputs = new Set<string>();
+
+  for (const filePath of buildContentFiles) {
+    const relPath = toPosix(path.relative(contentDir, filePath));
+    const parsed = path.parse(relPath);
+    const ext = parsed.ext.toLowerCase();
+
+    if (isLiterateJava(filePath)) {
+      outputs.add(toPosix(path.join(parsed.dir, `${parsed.name}.html`)));
+      outputs.add(toPosix(path.join(parsed.dir, parsed.name)));
+      continue;
+    }
+
+    if (extensionIsMarkdown(ext) || ext === '.html') {
+      outputs.add(toPosix(path.join(parsed.dir, `${parsed.name}.html`)));
+      continue;
+    }
+
+    if (codeExtensionSet.has(ext.slice(1))) {
+      outputs.add(relPath);
+      if (codeEnabled) {
+        outputs.add(`${relPath}.html`);
+      }
+    }
+  }
+
+  const processedExtSet = new Set(getProcessedExtensions(codeExtensions));
+  for (const filePath of walkFiles(contentDir)) {
+    const ext = path.extname(filePath).slice(1).toLowerCase();
+    if (processedExtSet.has(ext)) {
+      continue;
+    }
+    outputs.add(toPosix(path.relative(contentDir, filePath)));
+  }
+
+  return outputs;
+}
+
 function addGeneratedRouteAliases(
   pathSet: Set<string>,
   outputPath: string,

package/build/utils/include.ts

@@ -3,12 +3,14 @@ import path from 'path';
 import _ from 'lodash';
 import { isPartial } from './file-types';
 import { stripHtmlComments } from './render';
+import type { RenderDependencyCollector } from '../types';
 
 const MAX_INCLUDE_DEPTH = 10;
 
 export function createIncludeFunction(
   callerFilePath: string,
   templateParams: Record<string, unknown>,
+  collector?: RenderDependencyCollector,
   depth: number = 0,
 ): (relativePath: string) => string {
   return (relativePath: string): string => {
@@ -25,6 +27,7 @@ export function createIncludeFunction(
         `${callerFilePath}: include target must start with "_": ${relativePath}`,
       );
     }
+    collector?.partials?.add(resolved);
     if (depth >= MAX_INCLUDE_DEPTH) {
       throw new Error(
         `${callerFilePath}: maximum include depth (${MAX_INCLUDE_DEPTH}) exceeded`,
@@ -37,6 +40,7 @@ export function createIncludeFunction(
     const nestedInclude = createIncludeFunction(
       resolved,
       templateParams,
+      collector,
       depth + 1,
     );
     const nestedParams = { ...templateParams, include: nestedInclude };

package/build/utils/python-runner/trace_runner.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+
+import io
+import json
+import os
+import pdb
+import sys
+import types
+
+
+class CaptureStream(io.TextIOBase):
+    def __init__(self, sink):
+        self._sink = sink
+
+    def write(self, s):
+        if not isinstance(s, str):
+            s = str(s)
+        self._sink.append(s)
+        return len(s)
+
+    def flush(self):
+        return None
+
+    def isatty(self):
+        return False
+
+
+class TracePdb(pdb.Pdb):
+    SYNTHETIC_FRAME_NAMES = {"<listcomp>", "<dictcomp>", "<setcomp>"}
+
+    def __init__(self, target_path, trace_writer, output_chunks):
+        super().__init__(nosigint=True, readrc=False)
+        self.target_path = os.path.abspath(target_path)
+        self.trace_writer = trace_writer
+        self.output_chunks = output_chunks
+        self.pending_snapshot = None
+        self.object_ids = {}
+        self.next_object_id = 1
+
+    def user_line(self, frame):
+        if not self._is_traced_frame(frame):
+            self.set_step()
+            return
+
+        snapshot = self._capture_snapshot(frame)
+        self._emit_pending()
+        self.pending_snapshot = snapshot
+        self.set_step()
+
+    def user_call(self, frame, argument_list):
+        if self._is_traced_frame(frame):
+            self.set_step()
+
+    def user_return(self, frame, return_value):
+        if self._is_traced_frame(frame):
+            snapshot = self._capture_snapshot(frame)
+            self._emit_pending()
+            self.pending_snapshot = snapshot
+            self.set_step()
+
+    def user_exception(self, frame, exc_info):
+        if self._is_traced_frame(frame):
+            self.set_step()
+
+    def flush_pending(self):
+        self._emit_pending()
+
+    def _emit_pending(self):
+        if self.pending_snapshot is None:
+            return
+        payload = dict(self.pending_snapshot)
+        payload["stdout"] = self._drain_output()
+        self.trace_writer.write(json.dumps(payload) + "\n")
+        self.trace_writer.flush()
+        self.pending_snapshot = None
+
+    def _drain_output(self):
+        if not self.output_chunks:
+            return ""
+        output = "".join(self.output_chunks)
+        self.output_chunks.clear()
+        return output
+
+    def _capture_snapshot(self, frame):
+        reachable = {}
+        return {
+            "line": frame.f_lineno,
+            "file": os.path.basename(frame.f_code.co_filename),
+            "stack": self._serialize_stack(frame, reachable),
+            "heap": self._serialize_heap(reachable),
+        }
+
+    def _serialize_stack(self, frame, reachable):
+        frames = []
+        current = frame
+        while current is not None:
+            if self._is_traced_frame(current):
+                locals_json = {}
+                for name, value in current.f_locals.items():
+                    if self._should_skip_local(name, value):
+                        continue
+                    locals_json[name] = self._serialize_value(value, reachable)
+                frames.append(
+                    {
+                        "method": current.f_code.co_name,
+                        "class": "Python",
+                        "line": current.f_lineno,
+                        "locals": locals_json,
+                    }
+                )
+            current = current.f_back
+        return frames
+
+    def _serialize_heap(self, reachable):
+        heap = {}
+        queue = list(reachable.items())
+        seen = set()
+        idx = 0
+        while idx < len(queue):
+            obj_id, obj = queue[idx]
+            idx += 1
+            if obj_id in seen:
+                continue
+            seen.add(obj_id)
+            heap[obj_id] = self._serialize_heap_object(obj, reachable)
+            for new_id, new_obj in reachable.items():
+                if new_id not in seen:
+                    queue.append((new_id, new_obj))
+        return heap
+
+    def _serialize_heap_object(self, obj, reachable):
+        if isinstance(obj, str):
+            return {"type": "String", "value": obj}
+
+        if isinstance(obj, (list, tuple)):
+            return {
+                "type": type(obj).__name__,
+                "elements": [self._serialize_value(value, reachable) for value in obj],
+            }
+
+        if isinstance(obj, dict) and all(isinstance(key, str) for key in obj.keys()):
+            return {
+                "type": "dict",
+                "fields": {
+                    key: self._serialize_value(value, reachable)
+                    for key, value in obj.items()
+                },
+            }
+
+        obj_dict = getattr(obj, "__dict__", None)
+        if isinstance(obj_dict, dict):
+            return {
+                "type": type(obj).__name__,
+                "fields": {
+                    key: self._serialize_value(value, reachable)
+                    for key, value in obj_dict.items()
+                    if not key.startswith("__") and not self._should_skip_local(key, value)
+                },
+            }
+
+        return {"type": type(obj).__name__, "value": self._safe_repr(obj)}
+
+    def _serialize_value(self, value, reachable):
+        if value is None:
+            return {"type": "null"}
+        if isinstance(value, bool):
+            return {"type": "boolean", "value": value}
+        if isinstance(value, int) and not isinstance(value, bool):
+            return {"type": "int", "value": value}
+        if isinstance(value, float):
+            return {"type": "double", "value": value}
+        if isinstance(value, str):
+            obj_id = self._get_object_id(value)
+            reachable.setdefault(obj_id, value)
+            return {"type": "ref", "id": obj_id}
+
+        if self._is_heap_value(value):
+            obj_id = self._get_object_id(value)
+            reachable.setdefault(obj_id, value)
+            return {"type": "ref", "id": obj_id}
+
+        return {"type": "unknown"}
+
+    def _is_heap_value(self, value):
+        if isinstance(value, str):
+            return True
+        if isinstance(value, (list, tuple)):
+            return True
+        if isinstance(value, dict):
+            return all(isinstance(key, str) for key in value.keys())
+        obj_dict = getattr(value, "__dict__", None)
+        return isinstance(obj_dict, dict)
+
+    def _get_object_id(self, obj):
+        key = id(obj)
+        if key not in self.object_ids:
+            self.object_ids[key] = f"obj_{self.next_object_id}"
+            self.next_object_id += 1
+        return self.object_ids[key]
+
+    def _safe_repr(self, value):
+        try:
+            return repr(value)
+        except Exception:
+            return f"<{type(value).__name__}>"
+
+    def _is_traced_frame(self, frame):
+        return (
+            os.path.abspath(frame.f_code.co_filename) == self.target_path
+            and frame.f_code.co_name not in self.SYNTHETIC_FRAME_NAMES
+        )
+
+    def _should_skip_local(self, name, value):
+        if name.startswith("."):
+            return True
+        if name.startswith("__"):
+            return True
+        if isinstance(value, (types.ModuleType, types.FunctionType, type)):
+            return True
+        return False
+
+
+def main():
+    if len(sys.argv) != 2:
+        print("Usage: trace_runner.py <python_file>", file=sys.stderr)
+        sys.exit(1)
+
+    target_path = os.path.abspath(sys.argv[1])
+    target_dir = os.path.dirname(target_path)
+    target_name = os.path.basename(target_path)
+
+    if target_dir and target_dir not in sys.path:
+        sys.path.insert(0, target_dir)
+
+    trace_writer = sys.stdout
+    output_chunks = []
+    capture = CaptureStream(output_chunks)
+    debugger = TracePdb(target_path, trace_writer, output_chunks)
+
+    code_globals = {
+        "__name__": "__main__",
+        "__file__": target_path,
+        "__package__": None,
+        "__cached__": None,
+        "__builtins__": __builtins__,
+    }
+
+    original_stdout = sys.stdout
+    original_stderr = sys.stderr
+
+    try:
+        with open(target_path, "r", encoding="utf-8") as f:
+            source = f.read()
+        compiled = compile(source, target_path, "exec")
+
+        sys.stdout = capture
+        sys.stderr = capture
+
+        try:
+            debugger.set_step()
+            debugger.runcall(exec, compiled, code_globals, code_globals)
+        except SystemExit:
+            pass
+        finally:
+            debugger.flush_pending()
+    finally:
+        sys.stdout = original_stdout
+        sys.stderr = original_stderr
+
+
+if __name__ == "__main__":
+    main()

package/build/utils/render.ts

@@ -24,6 +24,7 @@ import {
   toPosix,
   toUrlPath,
 } from './paths';
+import { applySourceTemplate } from './source-template';
 import pkg from '../../package.json' with { type: 'json' };
 import { parseFrontMatterAndContent } from './front-matter';
 import { createMarkdown } from './markdown';
@@ -42,6 +43,8 @@ import type {
   RenderCodePageOptions,
   RenderLiterateJavaOptions,
   RenderCopiedContentOptions,
+  RenderDependencyCollector,
+  TraceToolAvailability,
 } from '../types';
 
 const log = makeLogger(import.meta.url);
@@ -80,6 +83,7 @@ interface TemplateParametersInput {
 function resolveAuthor(
   pageVariables: Record<string, unknown>,
   filePath: string,
+  dependencyCollector?: RenderDependencyCollector,
 ): void {
   if (!pageVariables.author) {
     return;
@@ -93,6 +97,7 @@ function resolveAuthor(
     );
   }
   const authorKey = pageVariables.author as string;
+  dependencyCollector?.setAuthorKey?.(authorKey);
   const authorEntry = authors[authorKey];
   if (!authorEntry) {
     throw new Error(
@@ -130,8 +135,6 @@ function createTemplateParameters({
     vars: siteVariables.vars || {},
     ...createGlobals(pageVariables, siteVariables, subPath),
     site: siteVariables,
-    base: siteVariables.base,
-    basePath: siteVariables.basePath,
     page: pageVariables,
     content,
     applyBasePath,
@@ -206,8 +209,10 @@ export function renderPlainTextPageAsset({
   validInternalTargets,
   assetFiles,
   literateJavaOutputPaths,
+  dependencyCollector,
+  cachedTraceSourceDir,
   traceCache,
-  javacAvailable,
+  traceToolAvailability,
 }: RenderPlainTextOptions): Asset[] {
   const { dir, name, ext } = path.parse(filePath);
   const subPath = toPosix(path.relative(contentDir, path.join(dir, name)));
@@ -224,11 +229,13 @@ export function renderPlainTextPageAsset({
     watchMode,
     {
       validateInternalLinks: extensionIsMarkdown(ext.toLowerCase()),
+      dependencyCollector,
       traceCache,
       contentDir,
       distDir,
       literateJavaOutputPaths,
-      javacAvailable,
+      cachedTraceSourceDir,
+      traceToolAvailability,
     },
   );
 
@@ -282,7 +289,8 @@ export function renderCodePageAsset({
   const subPath = toPosix(path.relative(contentDir, path.join(dir, name)));
   const applyBasePath = createApplyBasePath(siteVariables);
   const lang = siteVariables.codeLanguages![ext.slice(1).toLowerCase()];
-  const sourceCode = fs.readFileSync(filePath, 'utf-8');
+  const rawSource = fs.readFileSync(filePath, 'utf-8');
+  const sourceCode = applySourceTemplate(rawSource, siteVariables, filePath);
   const pageDirPath = toPosix(path.relative(contentDir, dir));
 
   log.info`Rendering code page ${B`${subPath + ext}`}`;
@@ -346,20 +354,26 @@ export function renderCopiedContentAsset({
 
   log.info`${label} ${B`${relPath}`}`;
 
-  if (filePath.endsWith('.java') && isFeatureEnabled(siteVariables, 'code')) {
-    const sourceCode = fs.readFileSync(filePath, 'utf-8');
+  if (!isFeatureEnabled(siteVariables, 'code')) {
+    return [{ assetPath: relPath, content: fs.readFileSync(filePath) }];
+  }
+
+  const rawSource = fs.readFileSync(filePath, 'utf-8');
+  const templated = applySourceTemplate(rawSource, siteVariables, filePath);
+
+  if (filePath.endsWith('.java')) {
     const pageDirPath = toPosix(
       path.relative(contentDir, path.dirname(filePath)),
     );
     const rewrittenLines = rewriteProseLinks(
-      sourceCode.split('\n'),
+      templated.split('\n'),
       siteVariables,
       pageDirPath,
     );
     return [{ assetPath: relPath, content: rewrittenLines.join('\n') }];
   }
 
-  return [{ assetPath: relPath, content: fs.readFileSync(filePath) }];
+  return [{ assetPath: relPath, content: templated }];
 }
 
 /** Parses the file, renders using template, returns HTML & params used to generate page */
@@ -372,13 +386,16 @@ function renderPlainTextContent(
   isWatchMode: boolean,
   {
     validateInternalLinks = true,
+    dependencyCollector,
     traceCache,
     contentDir,
     distDir,
     literateJavaOutputPaths,
-    javacAvailable,
+    cachedTraceSourceDir,
+    traceToolAvailability,
   }: {
     validateInternalLinks?: boolean;
+    dependencyCollector?: RenderDependencyCollector;
     traceCache?: Map<
       string,
       {
@@ -391,7 +408,8 @@ function renderPlainTextContent(
     contentDir?: string;
     distDir?: string;
     literateJavaOutputPaths?: Set<string>;
-    javacAvailable?: boolean;
+    cachedTraceSourceDir?: string;
+    traceToolAvailability?: TraceToolAvailability;
   } = {},
 ): {
   content: string | null;
@@ -409,6 +427,9 @@ function renderPlainTextContent(
       codeExtensions: isFeatureEnabled(siteVariables, 'code')
         ? Object.keys(siteVariables.codeLanguages!)
         : [],
+      onValidLink: (targetPath: string) => {
+        dependencyCollector?.internalTargets?.add(targetPath);
+      },
     },
     literateJavaOutputPaths,
     sourceUrlPath,
@@ -440,7 +461,7 @@ function renderPlainTextContent(
   renderInlineField(md, pageVariablesProcessed, 'title');
   renderInlineField(md, pageVariablesProcessed, 'description');
 
-  resolveAuthor(pageVariablesProcessed, filePath);
+  resolveAuthor(pageVariablesProcessed, filePath, dependencyCollector);
 
   const parentError = validateParentLink(
     pageVariablesProcessed.parent,
@@ -450,6 +471,11 @@ function renderPlainTextContent(
   if (parentError) {
     throw new Error(parentError);
   }
+  if (typeof pageVariablesProcessed.parent === 'string') {
+    dependencyCollector?.internalTargets?.add(
+      normalizeOutputPath(pageVariablesProcessed.parent),
+    );
+  }
 
   const strippedContent = stripHtmlComments(content);
 
@@ -469,12 +495,14 @@ function renderPlainTextContent(
       distDir,
       applyBasePath,
       cache: traceCache,
-      javacAvailable,
+      toolAvailability: traceToolAvailability,
+      dependencyCollector,
+      cachedTraceSourceDir,
     });
     params.renderTrace = helpers.renderTrace;
   }
 
-  params.include = createIncludeFunction(filePath, params);
+  params.include = createIncludeFunction(filePath, params, dependencyCollector);
 
   let html: string;
   try {
@@ -516,6 +544,7 @@ export function renderLiterateJavaPageAsset({
   skipExecution,
   validInternalTargets,
   literateJavaOutputPaths,
+  dependencyCollector,
 }: RenderLiterateJavaOptions): Asset[] {
   const { dir, name } = path.parse(filePath);
   const className = deriveClassName(filePath);
@@ -575,6 +604,9 @@ export function renderLiterateJavaPageAsset({
       codeExtensions: isFeatureEnabled(siteVariables, 'code')
         ? Object.keys(siteVariables.codeLanguages!)
         : [],
+      onValidLink: (targetPath: string) => {
+        dependencyCollector?.internalTargets?.add(targetPath);
+      },
     },
     literateJavaOutputPaths,
     sourceUrlPath,
@@ -650,7 +682,7 @@ export function renderLiterateJavaPageAsset({
     );
   }
 
-  resolveAuthor(pageVariables, filePath);
+  resolveAuthor(pageVariables, filePath, dependencyCollector);
 
   const templateParameters = createTemplateParameters({
     pageVariables,