@abreen/tada 1.0.2 → 1.2.0

package/build/content-watch.ts

@@ -0,0 +1,107 @@
+import path from 'path';
+import type { SiteVariables } from './types';
+import { getContentDir, getBuildContentFiles } from './util';
+import {
+  compileTemplates,
+  getHtmlTemplatesDir,
+  getJsonDataDir,
+  JSON_DATA_FILES,
+} from './templates';
+import { getProjectDir } from './utils/paths';
+import { B } from './colors';
+import { makeLogger } from './log';
+
+const log = makeLogger(__filename);
+
+interface ChangeDetectionResult {
+  templateError: Error | null;
+  needsRestart: boolean;
+  changedContentFiles?: Set<string>;
+  templatesChanged?: boolean;
+}
+
+export class ContentChangeDetector {
+  private siteVariables: SiteVariables;
+  private siteConfigPath: string;
+  private lastSig: string | null;
+
+  constructor(siteVariables: SiteVariables) {
+    this.siteVariables = siteVariables;
+    this.siteConfigPath = path.resolve(getProjectDir(), 'site.dev.json');
+    this.lastSig = null;
+  }
+
+  detectChanges(modifiedFiles: Iterable<string>): ChangeDetectionResult {
+    const resolvedFiles = new Set(
+      [...(modifiedFiles || [])].map(filePath => path.resolve(filePath)),
+    );
+
+    const htmlTemplatesDir = getHtmlTemplatesDir();
+    const jsonDataDir = getJsonDataDir();
+
+    // Try to recompile templates
+    let templateError: Error | null = null;
+    try {
+      compileTemplates(this.siteVariables);
+    } catch (err) {
+      templateError = err as Error;
+    }
+
+    if (templateError) {
+      return { templateError, needsRestart: false };
+    }
+
+    const contentDir = getContentDir();
+    const normalizedContentDir = path.resolve(contentDir) + path.sep;
+    const normalizedHtmlDir = path.resolve(htmlTemplatesDir) + path.sep;
+    const buildContentFiles = getBuildContentFiles(
+      contentDir,
+      Object.keys(this.siteVariables.codeLanguages || {}),
+    );
+
+    // Detect structural changes (files added or deleted)
+    const sig = buildContentFiles.slice().sort().join('\0');
+    let needsRestart = false;
+    if (this.lastSig !== null && sig !== this.lastSig) {
+      needsRestart = true;
+    }
+    this.lastSig = sig;
+
+    const changedContentFiles = new Set(
+      [...resolvedFiles].filter(filePath =>
+        filePath.startsWith(normalizedContentDir),
+      ),
+    );
+
+    // Check if any HTML template or JSON data file changed
+    const jsonDataPaths = JSON_DATA_FILES.map(f =>
+      path.resolve(jsonDataDir, f),
+    );
+    const changedTemplatePaths = [...resolvedFiles].filter(
+      filePath =>
+        filePath === path.resolve(htmlTemplatesDir) ||
+        filePath.startsWith(normalizedHtmlDir) ||
+        jsonDataPaths.includes(filePath),
+    );
+    const templatesChanged = changedTemplatePaths.length > 0;
+
+    for (const filePath of changedTemplatePaths) {
+      log.event`${B`${path.basename(filePath)}`} changed, rebuilding`;
+    }
+
+    // Check if site config changed
+    const siteConfigChanged = resolvedFiles.has(
+      path.resolve(this.siteConfigPath),
+    );
+    if (siteConfigChanged) {
+      needsRestart = true;
+    }
+
+    return {
+      templateError: null,
+      needsRestart,
+      changedContentFiles,
+      templatesChanged,
+    };
+  }
+}

package/build/copy.ts

@@ -0,0 +1,118 @@
+import fs from 'fs';
+import path from 'path';
+import { makeLogger } from './log';
+import { B } from './colors';
+
+const log = makeLogger(__filename);
+
+interface CollectedFile {
+  abs: string;
+  rel: string;
+}
+
+function collectFiles(dir: string): CollectedFile[] {
+  let entries: fs.Dirent[];
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true, recursive: true });
+  } catch {
+    return [];
+  }
+  return entries
+    .filter(entry => entry.isFile())
+    .map(entry => {
+      const abs = path.join(entry.parentPath, entry.name);
+      const rel = path.relative(dir, abs);
+      return { abs, rel: rel.split(path.sep).join(path.posix.sep) };
+    });
+}
+
+export function copyPublicFiles(
+  publicDir: string,
+  distDir: string,
+): Set<string> {
+  const files = collectFiles(publicDir);
+  const publicRelPaths = new Set<string>();
+  for (const { abs, rel } of files) {
+    const dest = path.join(distDir, rel);
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    fs.copyFileSync(abs, dest);
+    publicRelPaths.add(rel);
+    log.info`Copying public file ${B`${rel}`}`;
+  }
+  return publicRelPaths;
+}
+
+export function copyContentAssets(
+  contentDir: string,
+  distDir: string,
+  processedExtensions: string[],
+  publicRelPaths: Set<string>,
+): Set<string> {
+  const processedExtSet = new Set(processedExtensions);
+  const files = collectFiles(contentDir);
+  const contentAssetRelPaths = new Set<string>();
+  const conflicts: string[] = [];
+  for (const { abs, rel } of files) {
+    const ext = path.extname(abs).slice(1).toLowerCase();
+    if (processedExtSet.has(ext)) {
+      continue;
+    }
+    contentAssetRelPaths.add(rel);
+    if (publicRelPaths && publicRelPaths.has(rel)) {
+      conflicts.push(rel);
+    }
+    const dest = path.join(distDir, rel);
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    fs.copyFileSync(abs, dest);
+  }
+  if (conflicts.length > 0) {
+    for (const rel of conflicts) {
+      log.error`content/${B`${rel}`} conflicts with public/${B`${rel}`}`;
+    }
+    const noun = conflicts.length === 1 ? 'file' : 'files';
+    throw new Error(
+      `${conflicts.length} ${noun} in content/ and public/ have the same path`,
+    );
+  }
+  return contentAssetRelPaths;
+}
+
+export function copyPublicFile(
+  publicDir: string,
+  distDir: string,
+  filePath: string,
+  contentAssetRelPaths?: Set<string>,
+): void {
+  const rel = path
+    .relative(publicDir, filePath)
+    .split(path.sep)
+    .join(path.posix.sep);
+  if (contentAssetRelPaths && contentAssetRelPaths.has(rel)) {
+    log.error`public/${B`${rel}`} conflicts with content/${B`${rel}`}`;
+    throw new Error(`public/${rel} and content/${rel} have the same path`);
+  }
+  const dest = path.join(distDir, rel);
+  fs.mkdirSync(path.dirname(dest), { recursive: true });
+  fs.copyFileSync(filePath, dest);
+  log.info`Copying public file ${B`${rel}`}`;
+}
+
+export function copyContentFile(
+  contentDir: string,
+  distDir: string,
+  filePath: string,
+  publicRelPaths?: Set<string>,
+): void {
+  const rel = path
+    .relative(contentDir, filePath)
+    .split(path.sep)
+    .join(path.posix.sep);
+  if (publicRelPaths && publicRelPaths.has(rel)) {
+    log.error`content/${B`${rel}`} conflicts with public/${B`${rel}`}`;
+    throw new Error(`content/${rel} and public/${rel} have the same path`);
+  }
+  const dest = path.join(distDir, rel);
+  fs.mkdirSync(path.dirname(dest), { recursive: true });
+  fs.copyFileSync(filePath, dest);
+  log.info`Copying content file ${B`${rel}`}`;
+}

package/{webpack/deflist-id-plugin.js → build/deflist-id-plugin.ts}

@@ -1,11 +1,12 @@
-const textToId = require('./text-to-id');
+import type MarkdownIt from 'markdown-it';
+import textToId from './text-to-id';
 
-module.exports = function deflistIdPlugin(md) {
+export default function deflistIdPlugin(md: MarkdownIt): void {
   md.core.ruler.push('deflist_id_injector', function (state) {
     const tokens = state.tokens;
-    const used = new Map();
+    const used = new Map<string, number>();
 
-    function slugify(str) {
+    function slugify(str: string): string {
       let slug = textToId(str);
 
       if (!slug) {
@@ -13,7 +14,7 @@ module.exports = function deflistIdPlugin(md) {
       }
 
       if (used.has(slug)) {
-        const n = used.get(slug) + 1;
+        const n = (used.get(slug) as number) + 1;
         used.set(slug, n);
         slug = `${slug}-${n}`;
       } else {
@@ -59,4 +60,4 @@ module.exports = function deflistIdPlugin(md) {
       i++;
     }
   });
-};
+}

package/{webpack/external-links-plugin.js → build/external-links-plugin.ts}

@@ -1,15 +1,24 @@
-const { makeLogger } = require('./log');
+import type MarkdownIt from 'markdown-it';
+import type Token from 'markdown-it/lib/token.mjs';
+import { makeLogger } from './log';
+import type { SiteVariables } from './types';
 
 const log = makeLogger(__filename);
 
-module.exports = function externalLinks(md, siteVariables) {
-  function addClass(token) {
+export default function externalLinks(
+  md: MarkdownIt,
+  siteVariables: SiteVariables,
+): void {
+  function addClass(token: Token): void {
     if (token.type === 'link_open') {
       const href = token.attrGet('href');
+      if (!href) {
+        return;
+      }
 
       if (href.match(/^https?:\/\/.*$/)) {
         const url = new URL(href);
-        if (!siteVariables.internalDomains.includes(url.host)) {
+        if (!siteVariables.internalDomains?.includes(url.host)) {
           const classAttr = token.attrGet('class');
           let newClassAttr;
 
@@ -34,4 +43,4 @@ module.exports = function externalLinks(md, siteVariables) {
   md.core.ruler.push('external_links', state => {
     state.tokens.map(addClass);
   });
-};
+}

package/build/features.ts

@@ -0,0 +1,11 @@
+import type { SiteVariables } from './types';
+
+export function isFeatureEnabled(
+  siteVariables: SiteVariables,
+  featureName: string,
+): boolean {
+  return (
+    siteVariables.features?.[featureName as keyof SiteVariables['features']] !==
+    false
+  );
+}

package/build/generate-content-assets.ts

@@ -0,0 +1,320 @@
+import fs from 'fs';
+import path from 'path';
+import { makeLogger } from './log';
+import { isFeatureEnabled } from './features';
+import { initHighlighter } from './utils/shiki-highlighter';
+import {
+  getBuildContentFiles,
+  getContentDir,
+  getValidInternalTargets,
+  renderCodePageAsset,
+  renderCopiedContentAsset,
+  renderLiterateJavaPageAsset,
+  renderPlainTextPageAsset,
+} from './util';
+import { isLiterateJava } from './utils/file-types';
+import { checkJavac } from './utils/literate-java';
+import type {
+  SiteVariables,
+  Asset,
+  ContentRenderOptions,
+  ContentRenderResult,
+  WatchState,
+} from './types';
+
+const log = makeLogger(__filename);
+
+export class ContentRenderer {
+  private siteVariables: SiteVariables;
+  private loggedCodeDisabled: boolean;
+  private sourceFileCache: Map<string, Asset[]>;
+  private lastBuildFiles: Set<string>;
+  private javacAvailable: boolean | undefined;
+
+  constructor(siteVariables: SiteVariables) {
+    this.siteVariables = siteVariables || {};
+    this.loggedCodeDisabled = false;
+    this.sourceFileCache = new Map();
+    this.lastBuildFiles = new Set();
+  }
+
+  async initHighlighter(): Promise<void> {
+    const langs = [
+      ...new Set([
+        'plaintext',
+        'text',
+        ...Object.values(this.siteVariables.codeLanguages || {}),
+      ]),
+    ];
+    await initHighlighter(langs);
+  }
+
+  getDirtySourceFiles(
+    buildContentFiles: string[],
+    { changedContentFiles, templatesChanged }: WatchState = {},
+  ): Set<string> {
+    if (
+      this.lastBuildFiles.size === 0 ||
+      !changedContentFiles ||
+      templatesChanged
+    ) {
+      return new Set(buildContentFiles);
+    }
+
+    const buildFileSet = new Set(buildContentFiles);
+    const dirtySourceFiles = new Set<string>();
+    for (const filePath of changedContentFiles) {
+      if (buildFileSet.has(filePath)) {
+        dirtySourceFiles.add(filePath);
+      }
+    }
+    return dirtySourceFiles;
+  }
+
+  isCopiedAssetSource(filePath: string): boolean {
+    const ext = path.extname(filePath).toLowerCase();
+    const codeExtensions = this.siteVariables.codeLanguages || {};
+    return Object.prototype.hasOwnProperty.call(codeExtensions, ext.slice(1));
+  }
+
+  getDirtyCopiedSourceFiles(
+    buildContentFiles: string[],
+    { changedContentFiles }: WatchState = {},
+  ): string[] {
+    if (this.lastBuildFiles.size === 0 || !changedContentFiles) {
+      return buildContentFiles.filter(filePath =>
+        this.isCopiedAssetSource(filePath),
+      );
+    }
+
+    const buildFileSet = new Set(buildContentFiles);
+    const dirtyCopiedSourceFiles: string[] = [];
+    for (const filePath of changedContentFiles) {
+      if (!buildFileSet.has(filePath) || !this.isCopiedAssetSource(filePath)) {
+        continue;
+      }
+      dirtyCopiedSourceFiles.push(filePath);
+    }
+
+    return dirtyCopiedSourceFiles;
+  }
+
+  renderSourceAssets(
+    filePath: string,
+    contentDir: string,
+    distDir: string,
+    validInternalTargets: Set<string>,
+    assetFiles: string[],
+  ): Asset[] {
+    const ext = path.extname(filePath).toLowerCase();
+    const assets: Asset[] = [];
+
+    if (isLiterateJava(filePath)) {
+      assets.push(
+        ...renderLiterateJavaPageAsset({
+          filePath,
+          contentDir,
+          distDir,
+          siteVariables: this.siteVariables,
+          assetFiles,
+          skipExecution: !this.javacAvailable,
+        }),
+      );
+      return assets;
+    }
+
+    if (ext === '.html' || ext === '.md' || ext === '.markdown') {
+      assets.push(
+        ...renderPlainTextPageAsset({
+          filePath,
+          contentDir,
+          distDir,
+          siteVariables: this.siteVariables,
+          validInternalTargets,
+          assetFiles,
+        }),
+      );
+      return assets;
+    }
+
+    const codeExtensions = this.siteVariables.codeLanguages || {};
+    if (ext.slice(1) in codeExtensions) {
+      if (isFeatureEnabled(this.siteVariables, 'code')) {
+        assets.push(
+          ...renderCodePageAsset({
+            filePath,
+            contentDir,
+            distDir,
+            siteVariables: this.siteVariables,
+            assetFiles,
+          }),
+        );
+      } else if (!this.loggedCodeDisabled) {
+        log.info`Not generating source code pages due to site.features.code = false`;
+        this.loggedCodeDisabled = true;
+      }
+    }
+
+    return assets;
+  }
+
+  writeUncachedAssets(
+    distDir: string,
+    copiedSourceFiles: string[],
+    contentDir: string,
+  ): void {
+    for (const filePath of copiedSourceFiles) {
+      for (const asset of renderCopiedContentAsset({ filePath, contentDir })) {
+        const outPath = path.join(distDir, asset.assetPath);
+        fs.mkdirSync(path.dirname(outPath), { recursive: true });
+        fs.writeFileSync(outPath, asset.content);
+      }
+    }
+  }
+
+  updateSourceCache(
+    filePath: string,
+    assets: Asset[],
+    changedHtmlAssetPaths: Set<string>,
+    removedHtmlAssetPaths: Set<string>,
+  ): void {
+    const previousAssets = this.sourceFileCache.get(filePath) || [];
+    const nextAssetPaths = new Set(assets.map(asset => asset.assetPath));
+
+    for (const asset of previousAssets) {
+      if (nextAssetPaths.has(asset.assetPath)) {
+        continue;
+      }
+      if (asset.assetPath.endsWith('.html')) {
+        removedHtmlAssetPaths.add(asset.assetPath);
+      }
+    }
+
+    for (const asset of assets) {
+      if (asset.assetPath.endsWith('.html')) {
+        changedHtmlAssetPaths.add(asset.assetPath);
+      }
+    }
+
+    this.sourceFileCache.set(filePath, assets);
+  }
+
+  writeCachedAssets(distDir: string, buildContentFiles: string[]): void {
+    for (const filePath of buildContentFiles) {
+      const assets = this.sourceFileCache.get(filePath) || [];
+      for (const asset of assets) {
+        const outPath = path.join(distDir, asset.assetPath);
+        fs.mkdirSync(path.dirname(outPath), { recursive: true });
+        fs.writeFileSync(outPath, asset.content);
+      }
+    }
+  }
+
+  pruneRemovedSources(
+    buildFileSet: Set<string>,
+    removedHtmlAssetPaths: Set<string>,
+  ): void {
+    for (const filePath of [...this.lastBuildFiles]) {
+      if (buildFileSet.has(filePath)) {
+        continue;
+      }
+
+      const previousAssets = this.sourceFileCache.get(filePath) || [];
+      for (const asset of previousAssets) {
+        if (asset.assetPath.endsWith('.html')) {
+          removedHtmlAssetPaths.add(asset.assetPath);
+        }
+      }
+      this.sourceFileCache.delete(filePath);
+    }
+  }
+
+  processContent({
+    distDir,
+    assetFiles,
+    watchState,
+  }: ContentRenderOptions): ContentRenderResult {
+    const contentDir: string = getContentDir();
+    const buildContentFiles: string[] = getBuildContentFiles(
+      contentDir,
+      Object.keys(this.siteVariables.codeLanguages || {}),
+    );
+    const buildFileSet = new Set<string>(buildContentFiles);
+    const dirtySourceFiles = this.getDirtySourceFiles(
+      buildContentFiles,
+      watchState,
+    );
+    const dirtyCopiedSourceFiles = this.getDirtyCopiedSourceFiles(
+      buildContentFiles,
+      watchState,
+    );
+    const changedHtmlAssetPaths = new Set<string>();
+    const removedHtmlAssetPaths = new Set<string>();
+    const validInternalTargets = getValidInternalTargets(
+      contentDir,
+      buildContentFiles,
+      Object.keys(this.siteVariables.codeLanguages || {}),
+    );
+    const errors: Error[] = [];
+
+    this.pruneRemovedSources(buildFileSet, removedHtmlAssetPaths);
+
+    if (dirtySourceFiles.size > 0) {
+      const noun = dirtySourceFiles.size === 1 ? 'file' : 'files';
+      log.info`Processing ${dirtySourceFiles.size} content ${noun}`;
+    }
+
+    const hasLiterateJava = [...dirtySourceFiles].some(isLiterateJava);
+    if (hasLiterateJava && this.javacAvailable === undefined) {
+      this.javacAvailable = checkJavac();
+      if (!this.javacAvailable) {
+        log.warn`javac was not found; literate Java pages will not include execution output`;
+      }
+    }
+
+    for (const filePath of dirtySourceFiles) {
+      let assets: Asset[];
+      try {
+        assets = this.renderSourceAssets(
+          filePath,
+          contentDir,
+          distDir,
+          validInternalTargets,
+          assetFiles,
+        );
+      } catch (err) {
+        errors.push(err as Error);
+        continue;
+      }
+      this.updateSourceCache(
+        filePath,
+        assets,
+        changedHtmlAssetPaths,
+        removedHtmlAssetPaths,
+      );
+    }
+
+    this.writeCachedAssets(distDir, buildContentFiles);
+    this.writeUncachedAssets(distDir, dirtyCopiedSourceFiles, contentDir);
+    this.lastBuildFiles = buildFileSet;
+
+    // Collect HTML asset content for Pagefind
+    const htmlAssetsByPath = new Map<string, string>();
+    for (const filePath of buildContentFiles) {
+      const assets = this.sourceFileCache.get(filePath) || [];
+      for (const asset of assets) {
+        if (asset.assetPath.endsWith('.html')) {
+          htmlAssetsByPath.set(asset.assetPath, asset.content as string);
+        }
+      }
+    }
+
+    return {
+      errors,
+      changedHtmlAssetPaths,
+      removedHtmlAssetPaths,
+      htmlAssetsByPath,
+      buildContentFiles,
+    };
+  }
+}