muriel 1.0.4 → 1.0.5

package/README.md

@@ -17,6 +17,8 @@ No nodes. No operators. No DSL gymnastics.
 - **Everything else is series**
 - **Worker threads are optional**
 
+NOTE: The worker pool is more suited for CPU-bound pure-JS transforms where you want to avoid blocking the main thread — e.g., heavy string processing, JSON transformations, or math-intensive filters across thousands of packets.
+
 ---
 
 ## Transform API

package/examples/catpea-blog-profile.json

@@ -1,12 +1,30 @@
 {
+
+  "debug":{
+
+    "_processOnly": ["poem-2074",  "poem-2077",  "poem-2094",  "poem-2095",  "poem-2111"],
+
+
+    "mostRecent": 48,
+    "skipCovers": true,
+    "skipAudio": true
+  },
+
   "profile": "catpea_www",
   "title": "Cat Pea",
 
   "src": "examples/catpea-blog-sample-data/database/main-posts",
   "dest": "examples/catpea-blog-sample-data/dist/{profile}",
-  "theme": "examples/catpea-blog-sample-data/theme",
+
+  "theme": {
+    "src": "examples/catpea-blog-sample-data/themes/striped-dark-solarize",
+    "dest": "examples/catpea-blog-sample-data/dist/{profile}"
+
+  },
+
 
   "pagerizer": {
+    "pp": 24,
     "dest": "examples/catpea-blog-sample-data/dist/{profile}"
   },
 
@@ -20,13 +38,24 @@
     "width": 1024,
     "height": 1024,
     "quality": 80,
-    "effort": 4
+    "effort": 4,
+    "exif": {
+      "IFD0": {
+        "Copyright": "Cat Pea",
+        "ImageDescription": "Cat Pea Blog Post Cover"
+      }
+    }
   },
 
   "audio": {
-    "dest": "examples/catpea-blog-sample-data/dist/audio/{chapter}/docs/{id}.mp3",
-    "url": "https://catpea.github.io/chapter-15/{id}.mp3",
-    "preset": "balanced"
+    "dest": "examples/catpea-blog-sample-data/dist/audio/chapter-{chapter}/docs/{id}.mp3",
+    "url": "https://catpea.github.io/chapter-{chapter}/{id}.mp3",
+    "preset": "balanced",
+    "id3": {
+      "artist": "Cat Pea",
+      "album_artist": "Cat Pea",
+      "publisher": "catpea.com"
+    }
   },
 
   "text": {}

package/examples/muriel-blog-maker/blog.js

@@ -3,7 +3,7 @@ import { flow } from '../../index.js';
 import fs from 'node:fs';
 import path from 'node:path';
 
-import { setup, processedPosts, manifestUpdates, loadManifest, saveManifest, computeConfigHash } from './lib.js';
+import { setup, resolvePath, processedPosts, manifestUpdates, loadManifest, saveManifest, computeConfigHash } from './lib.js';
 
 import postScanner   from './transforms/post-scanner/index.js';
 import skipUnchanged from './transforms/skip-unchanged/index.js';
@@ -16,6 +16,7 @@ import collectPost   from './transforms/collect-post/index.js';
 import homepage      from './transforms/homepage/index.js';
 import pagerizer     from './transforms/pagerizer/index.js';
 import rssFeed       from './transforms/rss-feed/index.js';
+import useTheme      from './transforms/use-theme/index.js';
 
 // ─────────────────────────────────────────────
 // Configuration
@@ -37,7 +38,7 @@ setup(baseDir, profile);
 // Manifest
 // ─────────────────────────────────────────────
 
-const manifestPath = path.join(baseDir, profile.profile, '.muriel-manifest.json');
+const manifestPath = path.join(resolvePath(profile.dest), '.muriel-manifest.json');
 const manifest = await loadManifest(manifestPath);
 
 const configHash = computeConfigHash(profile);
@@ -58,19 +59,21 @@ console.log(`──────────────────────
 
 const blog = flow([
 
-  [postScanner(profile.src), 'scanned'],
-  ['scanned', skipUnchanged(manifest), 'post'],
+  [ postScanner({ src: profile.src }, profile.debug), skipUnchanged(manifest), 'post' ],
 
   ['post',
-    [processCover(profile.cover), processAudio(profile.audio), copyFiles()],
-    processText(),
-    verifyPost(),
-    collectPost(),
+
+    [
+      processCover(profile.cover, profile.debug),
+      processAudio(profile.audio, profile.debug),
+      copyFiles()
+    ],
+
+    processText(), verifyPost(), collectPost(),
+
   'done'],
 
-  ['done',
-    [homepage({ pp: 12 }), pagerizer({ pp: 24 }), rssFeed()],
-  'finished'],
+  ['done', [homepage(profile.pagerizer), pagerizer(profile.pagerizer), rssFeed()], useTheme({ ...profile.theme, dest: profile.dest }), 'finished'],
 
 ], { context: { profile } });
 

package/examples/muriel-blog-maker/lib.js

@@ -37,9 +37,37 @@ export function escapeXml(str) {
     .replace(/'/g, ''');
 }
 
+export function buildPager(currentPage, totalPages, radius = 5) {
+  if (totalPages <= 1) return [];
+
+  // Small page count: list all pages descending
+  const window = radius * 2 + 1;
+  if (totalPages <= window) {
+    return Array.from({ length: totalPages }, (_, i) => {
+      const pn = totalPages - i;
+      return { text: `${pn}`, url: `page-${pn}.html`, ariaCurrent: pn === currentPage, pageNum: pn };
+    });
+  }
+
+  // Large page count: circular window centered on currentPage
+  const pages = [];
+  for (let offset = -radius; offset <= radius; offset++) {
+    const pn = ((currentPage - 1 + offset + totalPages) % totalPages) + 1;
+    pages.push({ text: `${pn}`, url: `page-${pn}.html`, ariaCurrent: pn === currentPage, pageNum: pn });
+  }
+  const low = currentPage - radius;
+  const high = currentPage + radius;
+  const wrapped = pages.filter(p => p.pageNum < low || p.pageNum > high);
+  const main = pages.filter(p => p.pageNum >= low && p.pageNum <= high);
+  return [
+    ...wrapped.sort((a, b) => b.pageNum - a.pageNum),
+    ...main.sort((a, b) => b.pageNum - a.pageNum)
+  ];
+}
+
 export function renderPostCard(post) {
   return `    <article class="post">
-      ${post.coverUrl ? `<a href="${post.permalinkUrl}"><img src="${post.coverUrl}" alt="" loading="lazy"></a>` : ''}
+      ${post.coverUrl ? `<a href="${post.permalinkUrl}"><img src="${post.coverUrl}" alt="" loading="lazy"></a>` : ``}
       <div class="post-content">
         <h2><a href="${post.permalinkUrl}">${post.postData.title || post.postId}</a></h2>
         <time>${post.postData.date ? new Date(post.postData.date).toLocaleDateString() : ''}</time>
@@ -47,25 +75,6 @@ export function renderPostCard(post) {
     </article>`;
 }
 
-export const pageStyles = `
-    * { box-sizing: border-box; }
-    body { font-family: system-ui, sans-serif; max-width: 1200px; margin: 0 auto; padding: 2rem; line-height: 1.6; background: #111; color: #eee; }
-    a { color: #6af; text-decoration: none; }
-    a:hover { text-decoration: underline; }
-    h1 { border-bottom: 1px solid #333; padding-bottom: 0.5rem; }
-    .posts { display: grid; grid-template-columns: repeat(auto-fill, minmax(280px, 1fr)); gap: 1.5rem; }
-    .post { background: #1a1a1a; border-radius: 8px; overflow: hidden; transition: transform 0.2s; }
-    .post:hover { transform: translateY(-4px); }
-    .post img { width: 100%; height: 180px; object-fit: cover; }
-    .post-content { padding: 1rem; }
-    .post h2 { margin: 0 0 0.5rem; font-size: 1.1rem; }
-    .post time { color: #888; font-size: 0.85rem; }
-    .pager { display: flex; justify-content: center; gap: 0.5rem; margin: 2rem 0; flex-wrap: wrap; }
-    .pager a, .pager span { padding: 0.5rem 1rem; background: #222; border-radius: 4px; }
-    .pager span[aria-current="true"] { background: #6af; color: #000; }
-    nav.nav { display: flex; justify-content: space-between; margin-bottom: 1rem; }
-`;
-
 export const mp3Presets = {
   highQuality: (src, out) => [
     '-hide_banner', '-loglevel', 'error', '-threads', '0', '-i', src,

package/examples/muriel-blog-maker/transforms/homepage/SNIPPETS.md

@@ -0,0 +1,8 @@
+      const pagerHtml = totalPages > 1
+        ? `  <nav class="pager">
+    <a href="page-${totalPages}.html">Browse Archive</a>
+${homePager.map(p => p.ariaCurrent
+  ? `    <a aria-current="true"href="${p.url}">${p.text}</a>`
+  : `    <a href="${p.url}">${p.text}</a>`
+).join('\n')}
+  </nav>`

package/examples/muriel-blog-maker/transforms/homepage/index.js

@@ -1,6 +1,6 @@
 import path from 'node:path';
 import { mkdir } from 'node:fs/promises';
-import { resolvePath, processedPosts, renderPostCard, pageStyles, atomicWriteFile } from '../../lib.js';
+import { resolvePath, processedPosts, renderPostCard, buildPager, atomicWriteFile } from '../../lib.js';
 
 export default function homepage({ pp = 12 } = {}) {
   let expectedTotal = null;
@@ -22,33 +22,22 @@ export default function homepage({ pp = 12 } = {}) {
 
       const latestPosts = sortedNewestFirst.slice(0, pp);
 
-      // Calculate archive page count for pager links
       const archivePP = 24;
       const totalPages = Math.ceil(validPosts.length / archivePP) || 1;
 
       const destDir = resolvePath(profile.pagerizer.dest);
       await mkdir(destDir, { recursive: true });
 
-      // Pager links centered on highest page with circular wrap
-      const pagerRadius = 5;
-      const centerPage = totalPages;
-      const allPages = Array.from(
-        { length: pagerRadius * 2 + 1 },
-        (_, i) => {
-          let pageNum = ((centerPage - pagerRadius + i - 1 + totalPages) % totalPages) + 1;
-          return { text: `${pageNum}`, url: `page-${pageNum}.html`, pageNum };
-        }
-      );
-
-      const lowBound = centerPage - pagerRadius;
-      const highBound = centerPage + pagerRadius;
-      const wrapped = allPages.filter(p => p.pageNum < lowBound || p.pageNum > highBound);
-      const main = allPages.filter(p => p.pageNum >= lowBound && p.pageNum <= highBound);
+      const homePager = buildPager(totalPages, totalPages);
 
-      const homePager = [
-        ...wrapped.sort((a, b) => b.pageNum - a.pageNum),
-        ...main.sort((a, b) => b.pageNum - a.pageNum)
-      ];
+      const pagerHtml = totalPages > 1
+        ? `  <nav class="pager">
+${homePager.map(p => p.ariaCurrent
+  ? `    <a aria-current="true"href="${p.url}">${p.text}</a>`
+  : `    <a href="${p.url}">${p.text}</a>`
+).join('\n')}
+  </nav>`
+        : '';
 
       const html = `<!DOCTYPE html>
 <html lang="en">
@@ -57,24 +46,18 @@ export default function homepage({ pp = 12 } = {}) {
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <title>${profile.title}</title>
   <link rel="alternate" type="application/rss+xml" title="${profile.title} Feed" href="/feed.xml">
-  <style>${pageStyles}</style>
+  <link rel="stylesheet" href="/style.css">
 </head>
 <body>
-  <header>
-    <h1>${profile.title}</h1>
-  </header>
 
   <main class="posts">
 ${latestPosts.map(renderPostCard).join('\n')}
   </main>
 
-  <nav class="pager">
-    <a href="page-${totalPages}.html">Browse Archive</a>
-${homePager.map(p => `    <a href="${p.url}">${p.text}</a>`).join('\n')}
-  </nav>
+${pagerHtml}
 
   <footer>
-    <p><a href="/feed.xml">RSS Feed</a></p>
+    <p><a href="/feed.xml">RSS</a></p>
   </footer>
 </body>
 </html>`;

package/examples/muriel-blog-maker/transforms/pagerizer/index.js

@@ -1,5 +1,5 @@
 import { mkdir } from 'node:fs/promises';
-import { resolvePath, processedPosts, renderPostCard, pageStyles, chunk, atomicWriteFile } from '../../lib.js';
+import { resolvePath, processedPosts, renderPostCard, chunk, buildPager, atomicWriteFile } from '../../lib.js';
 
 export default function pagerizer({ pp = 24 } = {}) {
   let expectedTotal = null;
@@ -39,30 +39,19 @@ export default function pagerizer({ pp = 24 } = {}) {
         const olderPageNumber = olderChunkIndex !== null ? totalPages - olderChunkIndex : null;
         const newerPageNumber = newerChunkIndex !== null ? totalPages - newerChunkIndex : null;
 
-        // Circular pager: [wrapped pages descending] [main pages descending]
-        const pagerRadius = 5;
-        const allPages = Array.from(
-          { length: pagerRadius * 2 + 1 },
-          (_, j) => {
-            let pn = ((pageNumber - pagerRadius + j - 1 + totalPages) % totalPages) + 1;
-            return {
-              text: `${pn}`,
-              url: `page-${pn}.html`,
-              ariaCurrent: pn === pageNumber,
-              pageNum: pn
-            };
-          }
-        );
-
-        const lowBound = pageNumber - pagerRadius;
-        const highBound = pageNumber + pagerRadius;
-        const wrapped = allPages.filter(p => p.pageNum < lowBound || p.pageNum > highBound);
-        const main = allPages.filter(p => p.pageNum >= lowBound && p.pageNum <= highBound);
-
-        const pager = [
-          ...wrapped.sort((a, b) => b.pageNum - a.pageNum),
-          ...main.sort((a, b) => b.pageNum - a.pageNum)
-        ];
+        const pager = buildPager(pageNumber, totalPages);
+
+        const pagerHtml = totalPages > 1
+          ? `  <nav class="pager">
+    <a href="index.html">Home</a>
+${pager.map(p => p.ariaCurrent
+  ? `    <span aria-current="true">${p.text}</span>`
+  : `    <a href="${p.url}">${p.text}</a>`
+).join('\n')}
+  </nav>`
+          : `  <nav class="pager">
+    <a href="index.html">Home</a>
+  </nav>`;
 
         const html = `<!DOCTYPE html>
 <html lang="en">
@@ -71,31 +60,23 @@ export default function pagerizer({ pp = 24 } = {}) {
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <title>${profile.title} - Page ${pageNumber}</title>
   <link rel="alternate" type="application/rss+xml" title="${profile.title} Feed" href="/feed.xml">
-  <style>${pageStyles}</style>
+  <link rel="stylesheet" href="/style.css">
 </head>
 <body>
-  <header>
-    <h1>${profile.title}</h1>
-  </header>
 
-  <nav class="nav">
-    ${newerPageNumber ? `<a href="page-${newerPageNumber}.html">&larr; Back</a>` : '<a href="index.html">&larr; Home</a>'}
-    <span>Page ${pageNumber} of ${totalPages}</span>
-    ${olderPageNumber ? `<a href="page-${olderPageNumber}.html">Next &rarr;</a>` : '<span></span>'}
-  </nav>
 
   <main class="posts">
 ${chunkPosts.map(renderPostCard).join('\n')}
   </main>
 
-  <nav class="pager">
-    <a href="index.html">Home</a>
-${pager.map(p => p.ariaCurrent
-  ? `    <span aria-current="true">${p.text}</span>`
-  : `    <a href="${p.url}">${p.text}</a>`
-).join('\n')}
+  <nav class="nav">
+    ${newerPageNumber ? `<a href="page-${newerPageNumber}.html">&larr; Newer</a>` : '<a href="index.html">&larr; Home</a>'}
+    <span>Page ${pageNumber} of ${totalPages}</span>
+    ${olderPageNumber ? `<a href="page-${olderPageNumber}.html">Older &rarr;</a>` : '<span></span>'}
   </nav>
 
+${pagerHtml}
+
   <footer>
     <p><a href="/feed.xml">RSS Feed</a></p>
   </footer>

package/examples/muriel-blog-maker/transforms/post-scanner/index.js

@@ -2,9 +2,10 @@ import path from 'node:path';
 import { readdir, readFile } from 'node:fs/promises';
 import { resolvePath } from '../../lib.js';
 
-export default function postScanner(postsDir) {
+export default function postScanner({src}, debug) {
+
   return async send => {
-    const srcDir = resolvePath(postsDir);
+    const srcDir = resolvePath(src);
     console.log(`Scanning: ${srcDir}`);
 
     const entries = await readdir(srcDir, { withFileTypes: true });
@@ -22,13 +23,21 @@ export default function postScanner(postsDir) {
     }
 
 
-    const selectedPostDirs = validPostDirs.slice(1_000, 1_005)
+
+    let selectedPostDirs = validPostDirs;
+    if (debug?.processOnly?.length) {
+      const allowed = new Set(debug.processOnly);
+      selectedPostDirs = validPostDirs.filter(p => allowed.has(p.postId));
+    } else if (debug?.mostRecent) {
+      selectedPostDirs = validPostDirs.slice(validPostDirs.length - debug.mostRecent);
+    }
+
     const totalPosts = selectedPostDirs.length;
     console.log(`  Found ${totalPosts} posts`);
 
     for (const { postId, postDir, dirFiles } of selectedPostDirs) {
-      const postData = JSON.parse(await readFile(path.join(postDir, 'post.json'), 'utf-8'));
 
+      const postData = JSON.parse(await readFile(path.join(postDir, 'post.json'), 'utf-8'));
       const cover = dirFiles.find(f => f.startsWith('cover.'));
       const audio = dirFiles.find(f => f.startsWith('audio.'));
 

package/examples/muriel-blog-maker/transforms/process-audio/index.js

@@ -5,8 +5,9 @@ import { spawn } from 'node:child_process';
 import { once } from 'node:events';
 import { resolvePath, mp3Presets, encodingSemaphore } from '../../lib.js';
 
-export default function processAudio(config) {
+export default function processAudio(config, debug) {
   const preset = config.preset || 'balanced';
+  const id3 = config.id3 || {};
 
   return async (send, packet) => {
     if (packet._cached) {
@@ -16,7 +17,7 @@ export default function processAudio(config) {
 
     const { files, guid, postId, postData } = packet;
 
-    if (!files.audio || !fs.existsSync(files.audio)) {
+    if (!files.audio || !fs.existsSync(files.audio) || debug.skipAudio) {
       console.log(`  [audio] ${postId}: No audio file`);
       send({ ...packet, audioResult: { skipped: true } });
       return;
@@ -27,6 +28,23 @@ export default function processAudio(config) {
         .replace('{chapter}', postData.chapter || '0')
         .replace('{id}', postData.id)
     );
+
+    // If output already exists, skip encoding (delete file to force rebuild)
+    if (fs.existsSync(destPath)) {
+      const outputStats = fs.statSync(destPath);
+      console.log(`  [audio] ${postId}: exists ${(outputStats.size / 1024 / 1024).toFixed(1)}MB`);
+      send({
+        ...packet,
+        audioResult: {
+          success: true,
+          path: destPath,
+          url: config.url.replace('{chapter}', postData.chapter || '0').replace('{id}', postData.id),
+          size: outputStats.size
+        }
+      });
+      return;
+    }
+
     await mkdir(path.dirname(destPath), { recursive: true });
 
     const presetFn = mp3Presets[preset];
@@ -40,6 +58,11 @@ export default function processAudio(config) {
       const tmpPath = destPath + '.tmp';
       const args = presetFn(files.audio, tmpPath);
 
+      // Append id3 metadata flags
+      for (const [key, value] of Object.entries(id3)) {
+        args.splice(args.indexOf('-f'), 0, '-metadata', `${key}=${value}`);
+      }
+
       await encodingSemaphore.acquire();
       let code, stderr = '';
       try {
@@ -67,7 +90,7 @@ export default function processAudio(config) {
         audioResult: {
           success: true,
           path: destPath,
-          url: config.url.replace('{id}', postData.id),
+          url: config.url.replace('{chapter}', postData.chapter || '0').replace('{id}', postData.id),
           size: outputStats.size,
           reduction: parseFloat(reduction)
         }

package/examples/muriel-blog-maker/transforms/process-cover/index.js

@@ -2,12 +2,12 @@ import fs from 'node:fs';
 import path from 'node:path';
 import { mkdir, rename } from 'node:fs/promises';
 import sharp from 'sharp';
-import { resolvePath, encodingSemaphore } from '../../lib.js';
+import { resolvePath, encodingSemaphore, atomicCopyFile } from '../../lib.js';
 
 sharp.concurrency(1);
 
-export default function processCover(config) {
-  const { width = 1024, height = 1024, quality = 80, effort = 4 } = config;
+export default function processCover(config, debug) {
+  const { width = 1024, height = 1024, quality = 80, effort = 4, exif = {} } = config;
 
   return async (send, packet) => {
     if (packet._cached) {
@@ -17,27 +17,59 @@ export default function processCover(config) {
 
     const { files, guid, postId } = packet;
 
-    if (!files.cover || !fs.existsSync(files.cover)) {
+    if (!files.cover || !fs.existsSync(files.cover) || debug.skipCovers) {
       console.log(`  [cover] ${postId}: No cover image`);
       send({ ...packet, coverResult: { skipped: true } });
       return;
     }
 
     const destPath = resolvePath(config.dest.replace('{guid}', guid));
+
+    // If output already exists, skip encoding (delete file to force rebuild)
+    if (fs.existsSync(destPath)) {
+      const stats = fs.statSync(destPath);
+      console.log(`  [cover] ${postId}: exists ${(stats.size / 1024).toFixed(1)}KB`);
+      send({
+        ...packet,
+        coverResult: {
+          success: true,
+          path: destPath,
+          url: config.url.replace('{guid}', guid),
+          size: stats.size
+        }
+      });
+      return;
+    }
+
     await mkdir(path.dirname(destPath), { recursive: true });
 
     try {
-      const tmpPath = destPath + '.tmp';
-      await encodingSemaphore.acquire();
-      try {
-        await sharp(files.cover)
-          .resize(width, height, { kernel: sharp.kernel.mitchell, fit: 'inside' })
-          .avif({ quality, effort })
-          .toFile(tmpPath);
-      } finally {
-        encodingSemaphore.release();
+      if (files.cover.endsWith('.avif')) {
+        await atomicCopyFile(files.cover, destPath);
+        console.log(`  [cover] ${postId}: copied (already AVIF)`);
+      } else {
+        const tmpPath = destPath + '.tmp';
+        await encodingSemaphore.acquire();
+        try {
+          let pipeline = sharp(files.cover)
+            .resize(width, height, { kernel: sharp.kernel.mitchell, fit: 'inside' })
+            .avif({ quality, effort });
+          if (Object.keys(exif).length > 0) {
+            pipeline = pipeline.withExif(exif);
+          }
+          await pipeline.toFile(tmpPath);
+        } catch (sharpErr) {
+          if (sharpErr.message.includes('unsupported image format')) {
+            console.log(`  [cover] ${postId}: sharp can't decode, copying as-is`);
+            await atomicCopyFile(files.cover, destPath);
+          } else {
+            throw sharpErr;
+          }
+        } finally {
+          encodingSemaphore.release();
+        }
+        if (fs.existsSync(tmpPath)) await rename(tmpPath, destPath);
       }
-      await rename(tmpPath, destPath);
 
       const stats = fs.statSync(destPath);
       console.log(`  [cover] ${postId}: ${(stats.size / 1024).toFixed(1)}KB → ${path.basename(destPath)}`);

package/examples/muriel-blog-maker/transforms/skip-unchanged/index.js

@@ -7,6 +7,7 @@ import { resolvePath, manifestUpdates, hashFileContent } from '../../lib.js';
 export default function skipUnchanged(manifest) {
 
   return async (send, packet) => {
+
     const { postId, postDir, guid, profile } = packet;
     const entry = manifest.posts[postId];
 

package/examples/muriel-blog-maker/transforms/use-theme/index.js

@@ -0,0 +1,42 @@
+import path from 'node:path';
+import { readdir, mkdir } from 'node:fs/promises';
+import { resolvePath, atomicCopyFile } from '../../lib.js';
+
+export default function useTheme(config) {
+  const themeSrc = resolvePath(config.src);
+  const destDir = resolvePath(config.dest);
+
+  return async (send, packet) => {
+    const allComplete = packet.branches?.every(b => b._complete);
+    if (!allComplete) {
+      send(packet);
+      return;
+    }
+
+    try {
+      const count = await copyRecursive(themeSrc, destDir);
+      console.log(`  [theme] Installed ${count} file(s) from ${path.basename(themeSrc)}`);
+    } catch (err) {
+      console.error(`  [theme] Error - ${err.message}`);
+    }
+
+    send(packet);
+  };
+}
+
+async function copyRecursive(src, dest) {
+  await mkdir(dest, { recursive: true });
+  const entries = await readdir(src, { withFileTypes: true });
+  let count = 0;
+  for (const entry of entries) {
+    const srcPath = path.join(src, entry.name);
+    const destPath = path.join(dest, entry.name);
+    if (entry.isDirectory()) {
+      count += await copyRecursive(srcPath, destPath);
+    } else {
+      await atomicCopyFile(srcPath, destPath);
+      count++;
+    }
+  }
+  return count;
+}

package/examples/sync-database.js

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
 
 //NOTE: this file will be executed by a human
-process.exit();
 
 import { readdir, readFile, writeFile, copyFile, mkdir, access } from "fs/promises";
 import { constants as fsConstants } from "fs";

package/index.js

@@ -27,6 +27,19 @@ function createPipe(name) {
  * Worker pool (optional)
  * ───────────────────────────────────────────── */
 
+ /*
+  NOTE: About createWorkerPool
+  You do need to pass a size explicitly, but it doesn't have to be CPU count. Here's how it works:
+  - Activation: Pass workers in options — flow([...], { workers: 4 }) or flow([...], { workers: os.cpus().length }). If omitted, size is undefined and it returns null (disabled).
+  - Mechanism: Creates size Worker threads, distributes work round-robin via cursor++ % workers.length.
+  - Serialization: The filter function is sent as filter.toString() and eval()'d inside the worker. The packet is passed via postMessage (structured clone).
+  Important limitation: Because filters are serialized via toString() + eval(), the worker pool only works with pure, self-contained transforms — functions that don't import external modules or close over external variables. A function
+  like (send, packet) => { send({ ...packet, x: packet.y * 2 }) } works fine. But your blog builder transforms (importing sharp, ffmpeg, fs, etc.) would not work with it.
+  For the blog builder, the encodingSemaphore is the right approach. The heavy work (sharp, ffmpeg) happens in external native processes anyway — worker threads wouldn't add anything. The semaphore limits concurrency to os.cpus().length
+  without the serialization constraint.
+  The worker pool is more suited for CPU-bound pure-JS transforms where you want to avoid blocking the main thread — e.g., heavy string processing, JSON transformations, or math-intensive filters across thousands of packets.
+ */
+
 function createWorkerPool(size) {
   if (size === undefined) return null;
 
@@ -207,11 +220,20 @@ export function flow(definition, options = {}) {
 
   // First pass: set up all edges (listeners)
   for (const edge of definition) {
-    // Producer → pipe (defer execution)
+    // Producer → [stages] → pipe (defer execution)
     if (typeof edge[0] === 'function') {
       const producer = edge[0];
-      const pipe = getPipe(edge[1]);
-      producers.push(() => producer(packet => pipe.send({ ...packet, ...ctx })));
+      const outputName = edge[edge.length - 1];
+      const stages = compileStages(edge.slice(1, -1));
+
+      if (stages.length === 0) {
+        const pipe = getPipe(outputName);
+        producers.push(() => producer(packet => pipe.send({ ...packet, ...ctx })));
+      } else {
+        const producerPipe = createPipe(`_producer_${producers.length}`);
+        connectEdge(producerPipe, stages, getPipe(outputName), ctx, pool);
+        producers.push(() => producer(packet => producerPipe.send({ ...packet, ...ctx })));
+      }
       continue;
     }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "type": "module",
   "name": "muriel",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "description": "Lightweight Filtergraph Flow Engine",
   "main": "index.js",
   "scripts": {

package/catpea_www/.muriel-manifest.json

@@ -1,366 +0,0 @@
-{
-  "version": 1,
-  "configHash": "4334d284663296411f31a0421e4d6b6c0511ee0430a1a9db86e1cfa2872cd911",
-  "posts": {
-    "poem-1001": {
-      "compositeHash": "6715a83bc29b60037c7cf96e946a86d2b62f551d06e8e75da3f66eed281c4513",
-      "files": {
-        "audio.mp3": {
-          "mtime": 1770321388663.5708,
-          "size": 3947444,
-          "hash": "1ac432bdc39fd1fd56e60db15e6c09b7b12d2014e5e646c1757b83901783abec"
-        },
-        "cover.jpg": {
-          "mtime": 1770321388663.7854,
-          "size": 541318,
-          "hash": "d33e976401350485b33b744580cff2039defccc9fa3c4e06cf91f7d2163a1d48"
-        },
-        "post.json": {
-          "mtime": 1770321388663.9695,
-          "size": 311,
-          "hash": "25686c8858fa7350f8b2d5f72b03c64eaefd3e4e8f1c6f37350e3055c121a80c"
-        },
-        "text.md": {
-          "mtime": 1770321388663.9695,
-          "size": 1283,
-          "hash": "ff971ec162528681d1ab586f50b3ec194385809d368dae2194e2c5e9a5e25cd6"
-        }
-      },
-      "results": {
-        "coverResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/d0ee9d67-040d-41b6-8ee6-0913cd195b2d/cover.avif",
-          "url": "/permalink/d0ee9d67-040d-41b6-8ee6-0913cd195b2d/cover.avif",
-          "size": 109011
-        },
-        "audioResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/audio/5/docs/poem-1001.mp3",
-          "url": "https://catpea.github.io/chapter-15/poem-1001.mp3",
-          "size": 1325895,
-          "reduction": 66.4
-        },
-        "textResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/d0ee9d67-040d-41b6-8ee6-0913cd195b2d/index.html",
-          "htmlLength": 2238
-        },
-        "filesResult": {
-          "skipped": true,
-          "reason": "no files dir"
-        },
-        "valid": true,
-        "errors": [],
-        "collectedPost": {
-          "postId": "poem-1001",
-          "guid": "d0ee9d67-040d-41b6-8ee6-0913cd195b2d",
-          "valid": true,
-          "errors": [],
-          "postData": {
-            "id": "poem-1001",
-            "guid": "d0ee9d67-040d-41b6-8ee6-0913cd195b2d",
-            "chapter": 5,
-            "title": "Little By Little; Or, To Live Above The Common Levels Of Life",
-            "description": null,
-            "date": "2022-11-27T04:25:38.628Z",
-            "lastmod": null,
-            "artwork": [
-              "https://unsplash.com/photos/TvpTk9m5H1E"
-            ]
-          },
-          "coverUrl": "/permalink/d0ee9d67-040d-41b6-8ee6-0913cd195b2d/cover.avif",
-          "audioUrl": "https://catpea.github.io/chapter-15/poem-1001.mp3",
-          "permalinkUrl": "/permalink/d0ee9d67-040d-41b6-8ee6-0913cd195b2d/"
-        }
-      }
-    },
-    "poem-1004": {
-      "compositeHash": "0e094d783dd770333fc2ad7315e6971b3ff2925e7895555025b62a619b094f40",
-      "files": {
-        "audio.mp3": {
-          "mtime": 1770321388671.8887,
-          "size": 2972368,
-          "hash": "2c279e07095e0c1cf88dcfa82220e4976ba76380e747a66851f878f9fdf5b952"
-        },
-        "cover.jpg": {
-          "mtime": 1770321388672.2324,
-          "size": 310641,
-          "hash": "3d78f4cce735ec0cd2d4acff69cfaa070c1b2a941370781edad791017492cf51"
-        },
-        "post.json": {
-          "mtime": 1770321388672.7017,
-          "size": 353,
-          "hash": "69de9df29ae2f8a3c577dab12cae142d0317d44abfc4e7e2131de1a819786223"
-        },
-        "text.md": {
-          "mtime": 1770321388672.7017,
-          "size": 1862,
-          "hash": "6cf8f0ad86dc0299aaab5d7f0c261da9606661fb93e086bb85b814bd6545cdfa"
-        }
-      },
-      "results": {
-        "coverResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/17f655e8-1a02-4040-94da-7e9ff2da9bb1/cover.avif",
-          "url": "/permalink/17f655e8-1a02-4040-94da-7e9ff2da9bb1/cover.avif",
-          "size": 274128
-        },
-        "audioResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/audio/5/docs/poem-1004.mp3",
-          "url": "https://catpea.github.io/chapter-15/poem-1004.mp3",
-          "size": 1043753,
-          "reduction": 64.9
-        },
-        "textResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/17f655e8-1a02-4040-94da-7e9ff2da9bb1/index.html",
-          "htmlLength": 2898
-        },
-        "filesResult": {
-          "skipped": true,
-          "reason": "no files dir"
-        },
-        "valid": true,
-        "errors": [],
-        "collectedPost": {
-          "postId": "poem-1004",
-          "guid": "17f655e8-1a02-4040-94da-7e9ff2da9bb1",
-          "valid": true,
-          "errors": [],
-          "postData": {
-            "id": "poem-1004",
-            "guid": "17f655e8-1a02-4040-94da-7e9ff2da9bb1",
-            "chapter": 5,
-            "title": "Confusing Programming Can Be Pretty Colorful If You Build Everything Out Of Interesting Little Machines",
-            "description": null,
-            "date": "2022-11-30T04:11:09.289Z",
-            "lastmod": null,
-            "artwork": [
-              "https://unsplash.com/photos/vuAkRNH4sGA"
-            ]
-          },
-          "coverUrl": "/permalink/17f655e8-1a02-4040-94da-7e9ff2da9bb1/cover.avif",
-          "audioUrl": "https://catpea.github.io/chapter-15/poem-1004.mp3",
-          "permalinkUrl": "/permalink/17f655e8-1a02-4040-94da-7e9ff2da9bb1/"
-        }
-      }
-    },
-    "poem-1005": {
-      "compositeHash": "9e61d9adcdbf0f511eaf5cc44810e274e7cc57f7d99c389b095732677296f0f9",
-      "files": {
-        "audio.mp3": {
-          "mtime": 1770321388674.134,
-          "size": 7287177,
-          "hash": "861bbcc081f4a378b86184cffe4bfe7632be7ca724b84d6242774666ef51add1"
-        },
-        "cover.jpg": {
-          "mtime": 1770321388674.279,
-          "size": 509184,
-          "hash": "5e4317a39b4855cbb144fdc599dc71795770779d1f706525cb14525e479a5fb4"
-        },
-        "post.json": {
-          "mtime": 1770321388674.4119,
-          "size": 320,
-          "hash": "093c9a24e6a7b9a654958fa0b8ede2b6d8be75114336909a2cd2061146626eb7"
-        },
-        "text.md": {
-          "mtime": 1770321388674.4119,
-          "size": 2540,
-          "hash": "54588e268459278b034c697f2b29399eff2b8f1b82e0b080b80ab1e4044e7a2e"
-        }
-      },
-      "results": {
-        "coverResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/3c110103-9606-43fe-ba72-75a5045f8a0a/cover.avif",
-          "url": "/permalink/3c110103-9606-43fe-ba72-75a5045f8a0a/cover.avif",
-          "size": 54218
-        },
-        "audioResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/audio/5/docs/poem-1005.mp3",
-          "url": "https://catpea.github.io/chapter-15/poem-1005.mp3",
-          "size": 2555933,
-          "reduction": 64.9
-        },
-        "textResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/3c110103-9606-43fe-ba72-75a5045f8a0a/index.html",
-          "htmlLength": 3569
-        },
-        "filesResult": {
-          "skipped": true,
-          "reason": "no files dir"
-        },
-        "valid": true,
-        "errors": [],
-        "collectedPost": {
-          "postId": "poem-1005",
-          "guid": "3c110103-9606-43fe-ba72-75a5045f8a0a",
-          "valid": true,
-          "errors": [],
-          "postData": {
-            "id": "poem-1005",
-            "guid": "3c110103-9606-43fe-ba72-75a5045f8a0a",
-            "chapter": 5,
-            "title": "Prototyping And Coding Your First Web Operating System and Web Desktop",
-            "description": null,
-            "date": "2022-12-01T04:06:47.823Z",
-            "lastmod": null,
-            "artwork": [
-              "https://unsplash.com/photos/64YrPKiguAE"
-            ]
-          },
-          "coverUrl": "/permalink/3c110103-9606-43fe-ba72-75a5045f8a0a/cover.avif",
-          "audioUrl": "https://catpea.github.io/chapter-15/poem-1005.mp3",
-          "permalinkUrl": "/permalink/3c110103-9606-43fe-ba72-75a5045f8a0a/"
-        }
-      }
-    },
-    "poem-1003": {
-      "compositeHash": "e41d8865afd9bc0dd3d41dea6d944d56c62f8ad5f897392973521babc82ab6a0",
-      "files": {
-        "audio.mp3": {
-          "mtime": 1770321388668.4753,
-          "size": 8635341,
-          "hash": "0fd28140c8a085308635b72332a3994665805898a0f904acad7f560d0ac1f93c"
-        },
-        "cover.jpg": {
-          "mtime": 1770321388668.6865,
-          "size": 215096,
-          "hash": "37c0d2e353f85f436a877af2856b88fa1e17875efe8e13d9e7e08fb338720e24"
-        },
-        "post.json": {
-          "mtime": 1770321388668.875,
-          "size": 277,
-          "hash": "3f72a82af449174047a1338f5cd80e45f8b4da70a48e16370c6c7ac4191b2a17"
-        },
-        "text.md": {
-          "mtime": 1770321388668.875,
-          "size": 3545,
-          "hash": "acdf38771363c2eb5bcd6967d97d600b79ff2787d5efa59546f46ff39850dd36"
-        }
-      },
-      "results": {
-        "coverResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/7c262035-1b1e-4b9c-ac3e-c1f9b2583538/cover.avif",
-          "url": "/permalink/7c262035-1b1e-4b9c-ac3e-c1f9b2583538/cover.avif",
-          "size": 104642
-        },
-        "audioResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/audio/5/docs/poem-1003.mp3",
-          "url": "https://catpea.github.io/chapter-15/poem-1003.mp3",
-          "size": 2986536,
-          "reduction": 65.4
-        },
-        "textResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/7c262035-1b1e-4b9c-ac3e-c1f9b2583538/index.html",
-          "htmlLength": 4501
-        },
-        "filesResult": {
-          "skipped": true,
-          "reason": "no files dir"
-        },
-        "valid": true,
-        "errors": [],
-        "collectedPost": {
-          "postId": "poem-1003",
-          "guid": "7c262035-1b1e-4b9c-ac3e-c1f9b2583538",
-          "valid": true,
-          "errors": [],
-          "postData": {
-            "id": "poem-1003",
-            "guid": "7c262035-1b1e-4b9c-ac3e-c1f9b2583538",
-            "chapter": 5,
-            "title": "You Must Unlock Your Genius",
-            "description": null,
-            "date": "2022-11-29T04:28:54.843Z",
-            "lastmod": null,
-            "artwork": [
-              "https://unsplash.com/photos/6EdFFNU4Qlw"
-            ]
-          },
-          "coverUrl": "/permalink/7c262035-1b1e-4b9c-ac3e-c1f9b2583538/cover.avif",
-          "audioUrl": "https://catpea.github.io/chapter-15/poem-1003.mp3",
-          "permalinkUrl": "/permalink/7c262035-1b1e-4b9c-ac3e-c1f9b2583538/"
-        }
-      }
-    },
-    "poem-1002": {
-      "compositeHash": "282c701c825d371d946258a0b1bf9dbf77057fdcdc377cd2ee8d1dcd4954173e",
-      "files": {
-        "audio.mp3": {
-          "mtime": 1770321388665.9543,
-          "size": 24097197,
-          "hash": "e32492dddd818331cabbd7d5e2f8eda3d780a92d04cf6dfe99d05727440030a7"
-        },
-        "cover.jpg": {
-          "mtime": 1770321388666.1982,
-          "size": 327475,
-          "hash": "46d2aad492fa446624bf30ed89f56f888ea79ca682e2686bb7b1b27b6942e414"
-        },
-        "post.json": {
-          "mtime": 1770321388666.4639,
-          "size": 315,
-          "hash": "f2a340e7ce7b23b5ccfe53cf4cc25a98886ac8ecc2e9839ca704386869a6ab72"
-        },
-        "text.md": {
-          "mtime": 1770321388666.4639,
-          "size": 2924,
-          "hash": "545f1a5b98b1a5b4267198c14b6b3becdb4c1ea662a5fcc794d0745e9999325b"
-        }
-      },
-      "results": {
-        "coverResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/93be5330-7fc4-4bd2-b7e4-1d5efc024cca/cover.avif",
-          "url": "/permalink/93be5330-7fc4-4bd2-b7e4-1d5efc024cca/cover.avif",
-          "size": 185760
-        },
-        "audioResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/audio/5/docs/poem-1002.mp3",
-          "url": "https://catpea.github.io/chapter-15/poem-1002.mp3",
-          "size": 8285538,
-          "reduction": 65.6
-        },
-        "textResult": {
-          "success": true,
-          "path": "/home/meow/Universe/Development/npm/muriel/examples/catpea-blog-sample-data/dist/catpea_www/permalink/93be5330-7fc4-4bd2-b7e4-1d5efc024cca/index.html",
-          "htmlLength": 4023
-        },
-        "filesResult": {
-          "skipped": true,
-          "reason": "no files dir"
-        },
-        "valid": true,
-        "errors": [],
-        "collectedPost": {
-          "postId": "poem-1002",
-          "guid": "93be5330-7fc4-4bd2-b7e4-1d5efc024cca",
-          "valid": true,
-          "errors": [],
-          "postData": {
-            "id": "poem-1002",
-            "guid": "93be5330-7fc4-4bd2-b7e4-1d5efc024cca",
-            "chapter": 5,
-            "title": "We Are Star Babies; Or, The World Needs You To Unlock Your Genius",
-            "description": null,
-            "date": "2022-11-28T00:43:31.853Z",
-            "lastmod": null,
-            "artwork": [
-              "https://unsplash.com/photos/AmkpzghJ5eo"
-            ]
-          },
-          "coverUrl": "/permalink/93be5330-7fc4-4bd2-b7e4-1d5efc024cca/cover.avif",
-          "audioUrl": "https://catpea.github.io/chapter-15/poem-1002.mp3",
-          "permalinkUrl": "/permalink/93be5330-7fc4-4bd2-b7e4-1d5efc024cca/"
-        }
-      }
-    }
-  }
-}