@abreen/tada 1.13.7 → 1.14.1

package/README.md

@@ -40,6 +40,7 @@ A static site generator. The successor to Presto.
   * `??? question ... ???` renders a Q & A section; answers hidden until click
   * `+++ ... +++ ... +++` renders a two-column layout
   * Special heading subtitles with `## Heading # A subtitle here`
+  * `{{{ _partial.md }}}` syntax for including partials
 - Automatically generated favicon
   * Text, color, font and font weight taken from config file
 
@@ -333,6 +334,51 @@ templates][lodash].
 - Custom variables from the `"vars"` property of the config are available
   under `vars` (e.g., `<%= vars.staffEmail %>`)
 
+### Markdown partials
+
+Use the `{{{ _partial.md }}}` syntax to include a partial (a fragment of
+Markdown in a separate file whose name must start with `_`) at that particular
+location in a Markdown file.
+
+This is useful for reducing duplication across multiple documents which use the
+exact same content, or for breaking up a large document into building blocks
+which are easier to edit.
+
+Partials are never built into their own pages, and they don't have front matter.
+
+The `{{{ ... }}}` syntax is not a simple string replacement; instead, the
+Markdown syntax tree of the partial is *added* to the syntax tree of the
+Markdown file that includes it. This is a powerful feature because it allows you
+to include content as a subtree.
+
+For example, this partial `_groceries.md`
+
+```
+* apples
+* oranges
+* pears
+```
+
+can be inserted as a sub-list of a larger list in a Markdown page
+
+```
+title: To do list
+
+* Grocery shopping
+  {{{ _groceries.md }}}
+* Car wash
+```
+
+which results in:
+
+```
+* Grocery shopping
+  * apples
+  * oranges
+  * pears
+* Car wash
+```
+
 
 [inter]: https://fonts.google.com/specimen/Inter
 [lodash]: https://lodash.info/doc/template

package/build/generate-content-assets.ts

@@ -6,16 +6,15 @@ import { createContentRecord } from './source-records';
 import { validateConfigLinks } from './validate-config-links';
 import { config, getConfigFileName } from './templates';
 import { initHighlighter } from './utils/shiki-highlighter';
-import { checkTraceToolAvailability, isTraceSourceFile } from './utils/trace';
+import { checkTraceToolAvailability } from './utils/trace';
 import type {
   SiteVariables,
   ContentRenderOptions,
   ContentRenderResult,
   HtmlOutputAnalysis,
   TraceToolAvailability,
-  WatchState,
 } from './types';
-import type { TadaSourceRecord } from './watch/snapshot';
+import type { TraceCache } from './watch/compiler-types';
 
 const log = makeLogger(import.meta.url);
 
@@ -25,152 +24,41 @@ function cloneHtmlOutputAnalysis(
   return { outgoingTargets: new Set(analysis.outgoingTargets) };
 }
 
+function writeRecordOutputs(
+  distDir: string,
+  outputs: Map<string, string | Buffer>,
+): void {
+  for (const [outputPath, content] of outputs) {
+    const outPath = path.join(distDir, outputPath);
+    fs.mkdirSync(path.dirname(outPath), { recursive: true });
+    fs.writeFileSync(outPath, content);
+  }
+}
+
 export class ContentRenderer {
   private siteVariables: SiteVariables;
-  private sourceFileCache: Map<string, TadaSourceRecord>;
-  private lastBuildFiles: Set<string>;
-  private javacAvailable: boolean | undefined;
   private traceToolAvailability: TraceToolAvailability | undefined;
-  private traceCache: Map<
-    string,
-    {
-      manifestUrl: string;
-      artifactId: string;
-      highlightedSource: string;
-      totalSteps: number;
-      mtime: number;
-    }
-  > = new Map();
+  private traceCache: TraceCache = new Map();
 
   constructor(siteVariables: SiteVariables) {
     this.siteVariables = siteVariables;
-    this.sourceFileCache = new Map();
-    this.lastBuildFiles = new Set();
-  }
-
-  private getCachedRecord(filePath: string): TadaSourceRecord | undefined {
-    return this.sourceFileCache.get(filePath);
   }
 
   async initHighlighter(): Promise<void> {
     await initHighlighter(getRuntimeBundledShikiLanguages(this.siteVariables));
   }
 
-  getDirtySourceFiles(
-    buildContentFiles: string[],
-    { changedContentFiles, jsonDataChanged, partialsChanged }: WatchState = {},
-  ): Set<string> {
-    if (
-      this.lastBuildFiles.size === 0 ||
-      !changedContentFiles ||
-      jsonDataChanged ||
-      partialsChanged
-    ) {
-      return new Set(buildContentFiles);
-    }
-
-    // If any trace source changed, the trace cache will be stale, so rebuild
-    // all content pages to pick up the new trace output.
-    const traceFileChanged = [...changedContentFiles].some(isTraceSourceFile);
-    if (traceFileChanged) {
-      return new Set(buildContentFiles);
-    }
-
-    const buildFileSet = new Set(buildContentFiles);
-    const dirtySourceFiles = new Set<string>();
-    for (const filePath of changedContentFiles) {
-      if (buildFileSet.has(filePath)) {
-        dirtySourceFiles.add(filePath);
-      }
-    }
-    return dirtySourceFiles;
-  }
-
-  updateSourceCache(
-    filePath: string,
-    record: TadaSourceRecord,
-    changedOutputRelPaths: Set<string>,
-    removedOutputRelPaths: Set<string>,
-    changedHtmlAssetPaths: Set<string>,
-    removedHtmlAssetPaths: Set<string>,
-  ): void {
-    const previousRecord = this.getCachedRecord(filePath);
-    const nextOutputPaths = new Set(record.outputs.keys());
-
-    for (const outputPath of previousRecord?.outputs.keys() ?? []) {
-      if (nextOutputPaths.has(outputPath)) {
-        continue;
-      }
-      removedOutputRelPaths.add(outputPath);
-      if (outputPath.endsWith('.html')) {
-        removedHtmlAssetPaths.add(outputPath);
-      }
-    }
-
-    for (const outputPath of record.outputs.keys()) {
-      changedOutputRelPaths.add(outputPath);
-      if (outputPath.endsWith('.html')) {
-        changedHtmlAssetPaths.add(outputPath);
-      }
-    }
-
-    this.sourceFileCache.set(filePath, record);
-  }
-
-  writeCachedAssets(distDir: string, buildContentFiles: string[]): void {
-    for (const filePath of buildContentFiles) {
-      const record = this.getCachedRecord(filePath);
-      if (!record) {
-        continue;
-      }
-      for (const [outputPath, content] of record.outputs) {
-        const outPath = path.join(distDir, outputPath);
-        fs.mkdirSync(path.dirname(outPath), { recursive: true });
-        fs.writeFileSync(outPath, content);
-      }
-    }
-  }
-
-  pruneRemovedSources(
-    buildFileSet: Set<string>,
-    removedOutputRelPaths: Set<string>,
-    removedHtmlAssetPaths: Set<string>,
-  ): void {
-    for (const filePath of [...this.lastBuildFiles]) {
-      if (buildFileSet.has(filePath)) {
-        continue;
-      }
-
-      const previousRecord = this.getCachedRecord(filePath);
-      for (const outputPath of previousRecord?.outputs.keys() ?? []) {
-        removedOutputRelPaths.add(outputPath);
-        if (outputPath.endsWith('.html')) {
-          removedHtmlAssetPaths.add(outputPath);
-        }
-      }
-      this.sourceFileCache.delete(filePath);
-    }
-  }
-
   processContent({
     distDir,
     assetFiles,
     scan,
-    watchState,
   }: ContentRenderOptions): ContentRenderResult {
     const buildContentFiles = [...scan.buildContentFiles];
-    const buildFileSet = new Set<string>(buildContentFiles);
-    const dirtySourceFiles = this.getDirtySourceFiles(
-      buildContentFiles,
-      watchState,
-    );
-    const changedOutputRelPaths = new Set<string>();
-    const removedOutputRelPaths = new Set<string>();
-    const changedHtmlAssetPaths = new Set<string>();
-    const removedHtmlAssetPaths = new Set<string>();
     const validInternalTargets = scan.validTargets;
 
     const errors: Error[] = [];
+    const htmlAssetsByPath = new Map<string, string>();
+    const htmlAnalysisByPath = new Map<string, HtmlOutputAnalysis>();
 
     const configLinkErrors = validateConfigLinks(
       validInternalTargets,
@@ -185,29 +73,21 @@ export class ContentRenderer {
       errors.push(new Error(msg));
     }
 
-    this.pruneRemovedSources(
-      buildFileSet,
-      removedOutputRelPaths,
-      removedHtmlAssetPaths,
-    );
-
-    if (dirtySourceFiles.size > 0) {
-      const noun = dirtySourceFiles.size === 1 ? 'file' : 'files';
-      log.info`Processing ${dirtySourceFiles.size} content ${noun}`;
+    if (buildContentFiles.length > 0) {
+      const noun = buildContentFiles.length === 1 ? 'file' : 'files';
+      log.info`Processing ${buildContentFiles.length} content ${noun}`;
     }
 
-    if (this.javacAvailable === undefined && dirtySourceFiles.size > 0) {
+    if (!this.traceToolAvailability && buildContentFiles.length > 0) {
       this.traceToolAvailability = checkTraceToolAvailability();
-      this.javacAvailable = this.traceToolAvailability.java;
-      if (!this.javacAvailable) {
+      if (!this.traceToolAvailability.java) {
         log.warn`javac was not found; literate Java pages will not include execution output`;
       }
     }
 
-    for (const filePath of dirtySourceFiles) {
-      let record: TadaSourceRecord;
+    for (const filePath of buildContentFiles) {
       try {
-        record = createContentRecord({
+        const record = createContentRecord({
           filePath,
           siteVariables: this.siteVariables,
           scan,
@@ -215,53 +95,27 @@ export class ContentRenderer {
           outputDir: distDir,
           traceCache: this.traceCache,
           traceToolAvailability: this.traceToolAvailability,
-          skipLiterateJavaExecution: !this.javacAvailable,
+          skipLiterateJavaExecution: !this.traceToolAvailability?.java,
         });
+        writeRecordOutputs(distDir, record.outputs);
+        for (const [outputPath, content] of record.outputs) {
+          if (!outputPath.endsWith('.html') || typeof content !== 'string') {
+            continue;
+          }
+          htmlAssetsByPath.set(outputPath, content);
+          const analysis = record.htmlAnalysisByOutputPath?.get(outputPath);
+          if (analysis) {
+            htmlAnalysisByPath.set(
+              outputPath,
+              cloneHtmlOutputAnalysis(analysis),
+            );
+          }
+        }
       } catch (err) {
         errors.push(err as Error);
-        continue;
-      }
-      this.updateSourceCache(
-        filePath,
-        record,
-        changedOutputRelPaths,
-        removedOutputRelPaths,
-        changedHtmlAssetPaths,
-        removedHtmlAssetPaths,
-      );
-    }
-
-    this.writeCachedAssets(distDir, buildContentFiles);
-    this.lastBuildFiles = buildFileSet;
-
-    // Collect HTML asset content and analysis for Pagefind
-    const htmlAssetsByPath = new Map<string, string>();
-    const htmlAnalysisByPath = new Map<string, HtmlOutputAnalysis>();
-    for (const filePath of buildContentFiles) {
-      const record = this.getCachedRecord(filePath);
-      if (!record) {
-        continue;
-      }
-      for (const [outputPath, content] of record.outputs) {
-        if (!outputPath.endsWith('.html') || typeof content !== 'string') {
-          continue;
-        }
-        htmlAssetsByPath.set(outputPath, content);
-        const analysis = record.htmlAnalysisByOutputPath?.get(outputPath);
-        if (analysis) {
-          htmlAnalysisByPath.set(outputPath, cloneHtmlOutputAnalysis(analysis));
-        }
       }
     }
 
-    return {
-      errors,
-      changedHtmlAssetPaths,
-      removedHtmlAssetPaths,
-      removedOutputRelPaths,
-      htmlAssetsByPath,
-      htmlAnalysisByPath,
-      buildContentFiles,
-    };
+    return { errors, htmlAssetsByPath, htmlAnalysisByPath };
   }
 }

package/build/pipeline.ts

@@ -15,7 +15,7 @@ import { makeLogger, printFlair } from './log';
 import { generateBuildManifest, getNextVersion } from './build-manifest';
 import { scanProject } from './source-model';
 import { validateConfig } from './watch/validation';
-import { applyCommitPlan } from '../watch/fs-commit';
+import { applyCommitPlan } from './watch/fs-commit';
 import path from 'path';
 
 const log = makeLogger(import.meta.url);

package/build/source-model.ts

@@ -15,7 +15,7 @@ import {
   normalizeOutputPath,
   toPosix,
 } from './utils/paths';
-import type { ChangeBatch } from '../watch/types';
+import type { ChangeBatch } from './watch/types';
 import type { SiteVariables } from './types';
 
 export interface TadaProjectScan {
@@ -34,18 +34,6 @@ export interface TadaProjectScan {
   sourceTargetPaths: Map<string, Set<string>>;
 }
 
-export interface TadaProjectScanSnapshotLike {
-  processedExts: Set<string>;
-  contentFiles: Set<string>;
-  buildContentFiles: Set<string>;
-  publicFiles: Set<string>;
-  literateJavaOutputPaths: Set<string>;
-  contentOwners: Map<string, string>;
-  publicOwners: Map<string, string>;
-  sourceOutputPaths: Map<string, Set<string>>;
-  sourceTargetPaths: Map<string, Set<string>>;
-}
-
 export function shouldSkipContentFile(filePath: string): boolean {
   const ext = path.extname(filePath).toLowerCase();
   if (!(extensionIsMarkdown(ext) || ext === '.html')) {
@@ -333,7 +321,7 @@ export function scanProject(siteVariables: SiteVariables): TadaProjectScan {
 }
 
 export function updateProjectScan(
-  snapshot: TadaProjectScanSnapshotLike,
+  snapshot: TadaProjectScan,
   batch: ChangeBatch,
 ): TadaProjectScan {
   const scan = createEmptyScan(snapshot.processedExts);

package/build/types.d.ts

@@ -49,25 +49,13 @@ export interface ContentRenderOptions {
   distDir: string;
   assetFiles: string[];
   scan: TadaProjectScan;
-  watchState?: WatchState;
-}
-
-/** State passed from watch mode to control incremental rebuilds */
-export interface WatchState {
-  changedContentFiles?: Set<string>;
-  jsonDataChanged?: boolean;
-  partialsChanged?: boolean;
 }
 
 /** Result from ContentRenderer.processContent() */
 export interface ContentRenderResult {
   errors: Error[];
-  changedHtmlAssetPaths: Set<string>;
-  removedHtmlAssetPaths: Set<string>;
-  removedOutputRelPaths: Set<string>;
   htmlAssetsByPath: Map<string, string>;
   htmlAnalysisByPath: Map<string, HtmlOutputAnalysis>;
-  buildContentFiles: string[];
 }
 
 /** Logger returned by makeLogger() */
@@ -98,9 +86,9 @@ export interface RenderPlainTextOptions {
     {
       manifestUrl: string;
       artifactId: string;
-      highlightedSource: string;
+      highlightedSources: { file: string; highlightedSource: string }[];
       totalSteps: number;
-      mtime: number;
+      sourceMtims: Record<string, number>;
     }
   >;
   traceToolAvailability?: TraceToolAvailability;
@@ -218,7 +206,13 @@ export interface TraceStep {
   file: string;
   stack: TraceStackFrame[];
   heap: Record<string, TraceHeapObject>;
-  stdout: string;
+  output?: TraceOutputEvent[];
+  transientHeapRoots?: string[];
+}
+
+export interface TraceOutputEvent {
+  stream: 'stdout' | 'stderr';
+  text: string;
 }
 
 export interface TraceStackFrame {
@@ -244,13 +238,18 @@ export type TraceValue =
 export type TraceHeapObject =
   | { type: string; elements: TraceValue[] }
   | { type: string; fields: Record<string, TraceValue> }
-  | { type: string; value: string };
+  | { type: string; value: string | number | boolean };
 
 /** Manifest written alongside trace chunks */
 export interface TraceManifest {
   totalSteps: number;
   chunkSize: number;
-  sourceFile: string;
+  primaryFile: string;
+  sources: TraceSource[];
+}
+
+export interface TraceSource {
+  file: string;
   source: string;
   lineToSteps: Record<number, number[]>;
 }
@@ -279,8 +278,9 @@ export interface TraceLayout {
 
 /** A single entry in a chunk file (new format with precomputed SVG). */
 export interface TraceChunkEntry {
+  file: string;
   line: number;
-  stdout: string;
+  output?: TraceOutputEvent[];
   svg: string;
 }
 

package/build/utils/html-comments.ts

@@ -0,0 +1,3 @@
+export function stripHtmlComments(str: string): string {
+  return str.replace(/<!---[\s\S]*?-->/g, '');
+}