@safetnsr/md-pipe 0.2.0 → 0.3.0

package/README.md

@@ -1,7 +1,7 @@
 # md-pipe
 
-Event-driven automation for markdown directories.
-`entr` watches files. `md-pipe` understands them.
+Markdown content pipelines.
+Write markdown → auto-publish everywhere.
 
 12 lines of YAML → your markdown folder auto-publishes, auto-indexes, and auto-archives based on frontmatter changes. Works with any `.md` directory — Obsidian vaults, agent workspaces, docs-as-code repos, digital gardens.
 
@@ -23,17 +23,97 @@ npm install -g @safetnsr/md-pipe
 # 1. Create config
 md-pipe init
 
-# 2. Edit .md-pipe.yml with your triggers
+# 2. Edit .md-pipe.yml with your pipelines
 
 # 3. Start watching
 md-pipe watch
 ```
 
-## Config (.md-pipe.yml)
+## Pipelines (v0.3+)
+
+Pipelines are multi-step content workflows that fire on frontmatter changes:
+
+```yaml
+watch: ./docs
+
+pipelines:
+  - name: publish-post
+    trigger:
+      path: "posts/**"
+      frontmatter: { status: publish }
+      frontmatter_changed: [status]
+    steps:
+      - run: "echo Publishing {{fm.title}}"
+      - copy: { to: "./_site/posts" }
+      - update-frontmatter:
+          published_at: "{{now}}"
+          published: "true"
+      - webhook:
+          url: "$WEBHOOK_URL"
+          body: { title: "{{fm.title}}", slug: "{{slug}}" }
+```
+
+Steps execute in order. If one fails, the pipeline stops (unless `continue_on_error: true`).
+
+### Step Types
+
+| Step | Description |
+|---|---|
+| `run` | Shell command. Supports template vars and `$ENV` vars |
+| `update-frontmatter` | Write fields back to the source file's YAML frontmatter |
+| `copy` | Copy file to a destination directory |
+| `webhook` | POST JSON to a URL |
+| `template` | Render a template file with context data |
+
+### Template Variables
+
+Available in all step configs:
+
+| Variable | Description |
+|---|---|
+| `{{now}}` | ISO timestamp |
+| `{{date}}` | YYYY-MM-DD |
+| `{{slug}}` | Filename without extension |
+| `{{file}}` | Absolute path |
+| `{{basename}}` | Filename |
+| `{{relative}}` | Path relative to watch dir |
+| `{{dir}}` | Parent directory |
+| `{{tags}}` | Comma-separated tags |
+| `{{fm.title}}` | Frontmatter field |
+| `{{step.0.stdout}}` | Output from step 0 |
+
+### Pipeline Context
+
+Each step's output is available to subsequent steps:
+
+```yaml
+steps:
+  - run: "deploy.sh $FILE"                    # stdout captured
+  - update-frontmatter:
+      url: "{{step.0.stdout}}"                # use output from step 0
+      deployed_at: "{{now}}"
+```
+
+### Manual Run
+
+Test a pipeline without watching:
+
+```bash
+md-pipe run publish-post posts/hello.md
+```
+
+Run against all matching files:
+
+```bash
+md-pipe run publish-post
+```
+
+## Legacy Triggers
+
+Simple triggers with a single `run` command still work (backward compatible):
 
 ```yaml
 watch: ./docs
-debounce: 500       # ms, for watch mode (default: 200)
 
 triggers:
   - name: publish
@@ -41,24 +121,11 @@ triggers:
       path: "posts/**"
       frontmatter:
         status: publish
-    cwd: project     # run from config dir instead of file dir
-    run: "./scripts/deploy.sh $FILE"
-
-  - name: reindex
-    match:
-      frontmatter_changed:
-        - title
-        - tags
-        - category
-    run: "./scripts/reindex.sh $FILE"
-
-  - name: urgent-notify
-    match:
-      tags:
-        - urgent
-    run: "curl -X POST $WEBHOOK -d '{\"file\": \"$FILE\"}'"
+    run: "echo Publishing $FILE"
 ```
 
+Triggers and pipelines can coexist in the same config.
+
 ## Commands
 
 | Command | Description |
@@ -66,7 +133,8 @@ triggers:
 | `md-pipe init` | Scaffold a `.md-pipe.yml` config file |
 | `md-pipe watch` | Start watching for changes and trigger actions |
 | `md-pipe once` | Run triggers against current files (CI/batch mode) |
-| `md-pipe test <file>` | Show which triggers match a specific file |
+| `md-pipe run <pipeline> [file]` | Manually trigger a pipeline |
+| `md-pipe test <file>` | Show which triggers/pipelines match a file |
 
 ## Flags
 
@@ -78,184 +146,141 @@ triggers:
 | `--verbose` | Show trigger + file + first line of command |
 | `--debug` | Show full interpolated commands |
 | `--state <path>` | State file for idempotent `once` mode |
-| `--version, -v` | Show version |
-| `--help, -h` | Show help |
 
 ## Trigger Matchers
 
+All matchers work in both `triggers` (via `match:`) and `pipelines` (via `trigger:`).
+
 ### `path` — glob pattern matching
 ```yaml
-match:
-  path: "posts/**/*.md"
+path: "posts/**/*.md"
 ```
 
-### `frontmatter` — match specific frontmatter values
+### `frontmatter` — match specific values
 ```yaml
-match:
-  frontmatter:
-    status: publish
-    type: blog
+frontmatter:
+  status: publish
+  type: blog
 ```
 
-### Negation — match anything except a value
+### Negation
 ```yaml
-match:
-  frontmatter:
-    status: "!draft"    # matches publish, review, etc — NOT draft
+frontmatter:
+  status: "!draft"    # matches anything except draft
 ```
 
 ### `frontmatter_changed` — fires when specific fields change
 ```yaml
-match:
-  frontmatter_changed:
-    - title
-    - tags
+frontmatter_changed:
+  - title
+  - tags
 ```
-This is the key differentiator over generic file watchers. `md-pipe` tracks frontmatter state and only triggers when the fields you care about actually change.
+This is the key differentiator. md-pipe tracks frontmatter state and only triggers when the fields you care about actually change.
 
 ### `tags` — match files with specific tags
 ```yaml
-match:
-  tags:
-    - urgent
-    - review
+tags:
+  - urgent
 ```
 
-### `content` — match body text (substring)
+### `content` / `content_regex` — match body text
 ```yaml
-match:
-  content: "TODO"       # fires on files containing "TODO" in the body
+content: "TODO"
+content_regex: "\\[ \\]"   # unchecked checkboxes
 ```
 
-### `content_regex` — match body text (regex)
-```yaml
-match:
-  content_regex: "\\[ \\]"   # unchecked checkboxes
-```
+Matchers can be combined — all conditions must match.
 
-Matchers can be combined — all conditions must match for the trigger to fire.
+## Step Details
 
-## Trigger Options
+### `update-frontmatter`
+
+Write fields back to the source markdown file:
 
-### `cwd` — working directory for commands
 ```yaml
-triggers:
-  - name: build
-    match:
-      path: "**"
-    cwd: project    # "project" = config dir, "file" = file dir (default)
-    run: bash scripts/build.sh $FILE
+- update-frontmatter:
+    published_at: "{{now}}"
+    published: "true"          # coerced to boolean
+    url: "{{step.0.stdout}}"   # from previous step
 ```
 
-## Environment Variables
-
-Scripts receive these env vars:
-
-| Variable | Description |
-|---|---|
-| `$FILE` | Absolute path to the changed file |
-| `$DIR` | Directory containing the file |
-| `$BASENAME` | Filename without directory |
-| `$RELATIVE` | Path relative to watch directory |
-| `$FRONTMATTER` | JSON string of all frontmatter |
-| `$DIFF` | JSON string of changed frontmatter fields |
-| `$TAGS` | Comma-separated list of tags |
-| `$FM_<field>` | Direct access to frontmatter fields |
-
-### `$FM_` variables
+Values are coerced: `"true"` → `true`, `"false"` → `false`, numeric strings → numbers.
 
-Every frontmatter field is exposed as `$FM_<fieldname>`:
+### `copy`
 
 ```yaml
----
-title: "Hello World"
-status: publish
-version: "2.1"
----
+- copy: { to: "./_site/posts" }           # preserves directory structure
+- copy: { to: "./_site", flatten: true }   # flat copy
 ```
 
-```bash
-echo $FM_title    # → Hello World
-echo $FM_status   # → publish
-echo $FM_version  # → 2.1
-```
-
-No more `node -pe "JSON.parse(...)"` to extract a single field!
-
-## Idempotent Mode (--state)
+### `webhook`
 
-For CI/cron use, `--state` tracks which files have been processed:
+```yaml
+- webhook:
+    url: "$WEBHOOK_URL"                     # env var expansion
+    method: POST                            # default
+    headers:
+      Authorization: "Bearer $API_KEY"
+    body:
+      title: "{{fm.title}}"
+      file: "{{relative}}"
+```
 
-```bash
-# First run: processes all matching files
-md-pipe once --state .md-pipe-state.json
+### `template`
 
-# Second run: skips unchanged files
-md-pipe once --state .md-pipe-state.json
+```yaml
+- template:
+    src: "./templates/post.html"
+    out: "./_site/{{slug}}.html"
 ```
 
-The state file tracks content hashes per trigger+file combo. Only fires when actual content changes.
-
-## Watch Mode Debouncing
+Template files use the same `{{variable}}` syntax.
 
-File saves can trigger multiple events. Configure debounce in your config:
+### Error Handling
 
 ```yaml
-debounce: 500       # milliseconds (default: 200)
-# or
-debounce: "500ms"
-debounce: "2s"
+pipelines:
+  - name: resilient
+    continue_on_error: true     # pipeline-level
+    steps:
+      - run: "might-fail.sh"
+        continue_on_error: true # step-level
+      - run: "always-runs.sh"
 ```
 
-## Agent Interface (--json)
+## Environment Variables
 
-```bash
-md-pipe once --json
-```
+Shell commands (`run` steps and legacy triggers) receive:
 
-```json
-{
-  "total": 42,
-  "matched": 3,
-  "skipped": 0,
-  "actions": [
-    {
-      "triggerName": "publish",
-      "filePath": "/docs/posts/hello.md",
-      "command": "./scripts/deploy.sh /docs/posts/hello.md",
-      "exitCode": 0,
-      "stdout": "deployed",
-      "stderr": "",
-      "durationMs": 150
-    }
-  ],
-  "errors": []
-}
-```
+| Variable | Description |
+|---|---|
+| `$FILE` | Absolute path |
+| `$DIR` | Directory |
+| `$BASENAME` | Filename |
+| `$RELATIVE` | Relative path |
+| `$SLUG` | Filename without extension |
+| `$FRONTMATTER` | JSON string of all frontmatter |
+| `$DIFF` | JSON of changed fields |
+| `$TAGS` | Comma-separated tags |
+| `$FM_<field>` | Direct frontmatter access |
+| `$STEP_OUTPUT` | stdout from the previous step |
+| `$STEP_N_STDOUT` | stdout from step N |
+
+## Idempotent Mode (--state)
 
 ```bash
-md-pipe watch --json
-# streams one JSON object per triggered action
+md-pipe once --state .md-pipe-state.json
+# Only processes files that changed since last run
 ```
 
 ## Use Cases
 
-**Docs-as-code auto-deploy** — publish pages when `status: publish` is set in frontmatter
-
-**Agent workspace automation** — trigger reindexing when an agent updates `AGENTS.md` or `NOW.md`
-
-**Obsidian vault pipeline** — auto-archive notes, generate backlinks, push to git on tag changes
-
-**Digital garden** — auto-generate RSS feed when new posts are added
-
-**CI/CD for markdown** — `md-pipe once --state .state.json` in your CI pipeline for idempotent processing
-
-**TODO tracking** — find files with `content: "TODO"` or `content_regex: "\\[ \\]"` for unchecked tasks
-
-## Pair With
-
-- [`@safetnsr/md-kit`](https://github.com/safetnsr/md-kit) — markdown workspace toolkit (wikilinks, broken links, backlink maps)
-- [`@safetnsr/ai-ready`](https://github.com/safetnsr/ai-ready) — AI-compatibility score for your codebase
+- **Content pipeline** — write markdown, auto-publish to static site + CMS
+- **Docs-as-code** — deploy pages when `status: publish` is set
+- **Agent workspace** — trigger reindexing when agents update files
+- **Obsidian vault** — auto-archive, generate backlinks, push to git
+- **Digital garden** — auto-generate RSS, index pages, deploy
+- **CI/CD** — `md-pipe once --state .state.json` for idempotent processing
 
 ## License
 

package/dist/cli.js

@@ -11,17 +11,19 @@ const config_js_1 = require("./core/config.js");
 const watcher_js_1 = require("./core/watcher.js");
 const once_js_1 = require("./core/once.js");
 const test_file_js_1 = require("./core/test-file.js");
-const VERSION = '0.2.0';
+const run_command_js_1 = require("./core/run-command.js");
+const VERSION = '0.3.0';
 function printHelp() {
     console.log(`
-${chalk_1.default.bold('md-pipe')} — event-driven automation for markdown directories
-${chalk_1.default.dim('entr watches files. md-pipe understands them.')}
+${chalk_1.default.bold('md-pipe')} — markdown content pipelines
+${chalk_1.default.dim('Write markdown → auto-publish everywhere.')}
 
 ${chalk_1.default.bold('Usage:')}
-  md-pipe init              Scaffold a .md-pipe.yml config file
-  md-pipe watch             Start watching for changes and trigger actions
-  md-pipe once              Run triggers against current files (CI/batch mode)
-  md-pipe test <file>       Show which triggers match a specific file
+  md-pipe init                       Scaffold a .md-pipe.yml config file
+  md-pipe watch                      Start watching for changes
+  md-pipe once                       Run triggers against current files (CI/batch)
+  md-pipe run <pipeline> [file]      Manually trigger a pipeline on a file
+  md-pipe test <file>                Show which triggers/pipelines match a file
 
 ${chalk_1.default.bold('Flags:')}
   --config, -c <path>       Path to config file (default: .md-pipe.yml)
@@ -35,26 +37,41 @@ ${chalk_1.default.bold('Flags:')}
 
 ${chalk_1.default.bold('Config (.md-pipe.yml):')}
   watch: ./docs
-  debounce: 500       # ms, for watch mode (default: 200)
+
+  # Legacy triggers (simple match + run)
   triggers:
     - name: publish
       match:
         path: "posts/**"
         frontmatter: { status: publish }
-        content: "TODO"            # body substring
-        content_regex: "\\\\[\\\\s\\\\]"  # body regex
-      cwd: project     # "project" or "file" (default)
       run: "echo Publishing $FILE"
 
-${chalk_1.default.bold('Env vars passed to scripts:')}
-  $FILE          Absolute path to the changed file
-  $DIR           Directory containing the file
-  $BASENAME      Filename without directory
-  $RELATIVE      Path relative to watch directory
-  $FRONTMATTER   JSON string of all frontmatter
-  $DIFF          JSON string of changed frontmatter fields
-  $TAGS          Comma-separated list of tags
-  $FM_<field>    Direct access to frontmatter (e.g. $FM_title, $FM_status)
+  # Pipelines (v0.3+) — multi-step content pipelines
+  pipelines:
+    - name: publish-post
+      trigger:
+        path: "posts/**"
+        frontmatter: { status: publish }
+        frontmatter_changed: [status]
+      steps:
+        - run: "echo Publishing {{fm.title}}"
+        - update-frontmatter: { published_at: "{{now}}" }
+        - copy: { to: "./_site/posts" }
+        - webhook: { url: "$WEBHOOK_URL" }
+
+${chalk_1.default.bold('Step types:')}
+  run                  Shell command
+  update-frontmatter   Write back to source file frontmatter
+  webhook              POST JSON to a URL
+  copy                 Copy file to destination directory
+  template             Render a template with file data
+
+${chalk_1.default.bold('Template variables:')}
+  {{now}}              ISO timestamp
+  {{date}}             YYYY-MM-DD
+  {{slug}}             Filename without extension
+  {{fm.title}}         Frontmatter field
+  {{step.0.stdout}}    Output from step 0
 
 ${chalk_1.default.bold('More:')} https://github.com/safetnsr/md-pipe
 `);
@@ -62,12 +79,14 @@ ${chalk_1.default.bold('More:')} https://github.com/safetnsr/md-pipe
 function parseArgs(args) {
     let command = '';
     let file;
+    let pipelineName;
     let config;
     let dryRun = false;
     let json = false;
     let verbose = false;
     let debug = false;
     let state;
+    let positionals = [];
     for (let i = 0; i < args.length; i++) {
         const arg = args[i];
         if (arg === '--help' || arg === '-h') {
@@ -106,12 +125,17 @@ function parseArgs(args) {
             command = arg;
             continue;
         }
-        if (command === 'test' && !file) {
-            file = arg;
-            continue;
-        }
+        positionals.push(arg);
+    }
+    // Parse positionals based on command
+    if (command === 'run') {
+        pipelineName = positionals[0];
+        file = positionals[1];
     }
-    return { command: command || 'help', file, config, dryRun, json, verbose, debug, state };
+    else if (command === 'test') {
+        file = positionals[0];
+    }
+    return { command: command || 'help', file, pipelineName, config, dryRun, json, verbose, debug, state };
 }
 function getConfig(configPath) {
     const cwd = process.cwd();
@@ -122,7 +146,6 @@ function getConfig(configPath) {
     }
     return (0, config_js_1.loadConfig)(cfgPath);
 }
-/** Format command for display: verbose shows first line, debug shows full */
 function formatCommand(command, debug) {
     if (debug)
         return command;
@@ -130,6 +153,24 @@ function formatCommand(command, debug) {
     const truncated = firstLine.length > 80 ? firstLine.slice(0, 77) + '...' : firstLine;
     return command.includes('\n') ? truncated + ' …' : truncated;
 }
+function formatPipelineResult(result, debug) {
+    const overallStatus = result.success
+        ? chalk_1.default.green('✓')
+        : chalk_1.default.red('✗');
+    console.log(`  ${overallStatus} Pipeline ${chalk_1.default.cyan(result.pipelineName)} ` +
+        chalk_1.default.dim(`(${result.durationMs}ms)`));
+    for (let i = 0; i < result.steps.length; i++) {
+        const step = result.steps[i];
+        const status = step.success
+            ? chalk_1.default.green('  ✓')
+            : chalk_1.default.red('  ✗');
+        const typeLabel = chalk_1.default.dim(`[${step.type}]`);
+        console.log(`    ${status} ${typeLabel} ${chalk_1.default.dim(step.stdout.split('\n')[0].slice(0, 80))}`);
+        if (step.stderr) {
+            console.log(chalk_1.default.red(`      ${step.stderr.split('\n')[0]}`));
+        }
+    }
+}
 async function cmdInit() {
     const target = (0, path_1.resolve)(process.cwd(), '.md-pipe.yml');
     if ((0, fs_1.existsSync)(target)) {
@@ -147,9 +188,13 @@ async function cmdWatch(opts) {
         console.error(chalk_1.default.red(`Watch directory not found: ${config.watch}`));
         process.exit(1);
     }
+    const totalTriggers = config.triggers.length + config.pipelines.length;
     watcher.on('ready', () => {
         console.log(chalk_1.default.green('✓') + ` Watching ${config.watch}`);
-        console.log(chalk_1.default.dim(`  ${config.triggers.length} trigger(s) loaded`));
+        if (config.triggers.length > 0)
+            console.log(chalk_1.default.dim(`  ${config.triggers.length} trigger(s)`));
+        if (config.pipelines.length > 0)
+            console.log(chalk_1.default.dim(`  ${config.pipelines.length} pipeline(s)`));
         if (config.debounce)
             console.log(chalk_1.default.dim(`  debounce: ${config.debounce}ms`));
         if (opts.dryRun)
@@ -182,10 +227,16 @@ async function cmdWatch(opts) {
             console.log(chalk_1.default.red('    ' + result.stderr.replace(/\n/g, '\n    ')));
         }
     });
+    watcher.on('pipeline', (result) => {
+        if (opts.json) {
+            console.log(JSON.stringify(result));
+            return;
+        }
+        formatPipelineResult(result, opts.debug);
+    });
     watcher.on('error', (err, filePath) => {
         console.error(chalk_1.default.red(`Error${filePath ? ` (${filePath})` : ''}: ${err.message}`));
     });
-    // Graceful shutdown
     process.on('SIGINT', async () => {
         console.log(chalk_1.default.dim('\nStopping watcher...'));
         await watcher.stop();
@@ -231,6 +282,29 @@ function cmdOnce(opts) {
     if (result.errors.length > 0)
         process.exit(1);
 }
+async function cmdRun(opts) {
+    if (!opts.pipelineName) {
+        console.error(chalk_1.default.red('Usage: md-pipe run <pipeline-name> [file]'));
+        console.error(chalk_1.default.dim('  Run a specific pipeline manually.'));
+        process.exit(1);
+    }
+    const config = getConfig(opts.config);
+    const result = await (0, run_command_js_1.runPipelineCommand)(config, opts.pipelineName, opts.file, opts.dryRun);
+    if (opts.json) {
+        console.log(JSON.stringify(result, null, 2));
+        return;
+    }
+    console.log(chalk_1.default.bold(`md-pipe run ${opts.pipelineName}`) + (opts.file ? ` ${opts.file}` : ''));
+    console.log();
+    for (const pr of result.results) {
+        formatPipelineResult(pr, opts.debug);
+    }
+    for (const err of result.errors) {
+        console.error(chalk_1.default.red(err));
+    }
+    if (!result.success)
+        process.exit(1);
+}
 function cmdTest(opts) {
     if (!opts.file) {
         console.error(chalk_1.default.red('Usage: md-pipe test <file>'));
@@ -246,11 +320,12 @@ function cmdTest(opts) {
     console.log(chalk_1.default.dim(`  Frontmatter: ${JSON.stringify(result.frontmatter)}`));
     console.log(chalk_1.default.dim(`  Tags: [${result.tags.join(', ')}]\n`));
     if (result.matches.length === 0) {
-        console.log(chalk_1.default.yellow('  No triggers matched this file.'));
+        console.log(chalk_1.default.yellow('  No triggers or pipelines matched this file.'));
         return;
     }
     for (const m of result.matches) {
-        console.log(chalk_1.default.green('  ✓') + ` ${chalk_1.default.cyan(m.triggerName)}: ${m.reason}`);
+        const label = m.type === 'pipeline' ? chalk_1.default.magenta('[pipeline]') : chalk_1.default.blue('[trigger]');
+        console.log(chalk_1.default.green('  ✓') + ` ${label} ${chalk_1.default.cyan(m.triggerName)}: ${m.reason}`);
     }
 }
 async function main() {
@@ -267,6 +342,9 @@ async function main() {
             case 'once':
                 cmdOnce(opts);
                 break;
+            case 'run':
+                await cmdRun(opts);
+                break;
             case 'test':
                 cmdTest(opts);
                 break;

package/dist/core/config.d.ts

@@ -1,3 +1,4 @@
+import type { PipelineDef } from './pipeline.js';
 export interface TriggerMatch {
     path?: string;
     frontmatter?: Record<string, unknown>;
@@ -16,6 +17,7 @@ export interface MdPipeConfig {
     watch: string;
     configDir: string;
     triggers: TriggerDef[];
+    pipelines: PipelineDef[];
     debounce?: number;
 }
 export declare function findConfigFile(dir: string): string | null;