@cloud-cli/on 0.1.9 → 1.2.3

package/README.md

@@ -1,180 +1,245 @@
-# Workflow Design
+# 🏃 `@cloud-cli/on`
 
-This is a task runner using webhooks to process incoming events.
+General-purpose workflows
 
-Every event is sent to a daemon as an HTTP request, with a JSON body, and can trigger one or more workflows.
-Every workflow is a set of steps, which can run on containers or in a shell on the host.
+A self-hosted, lightweight, high-performance CI/CD runner engine built for Node.js.
 
-## Running on Docker
+Designed with a strict **security-first boundary**, native **JavaScript AST evaluation**, zero-DSL template literals, and a **built-in terminal log web dashboard**.
 
-- Steps run inside a docker container.
-- All steps run in the same workspace folder.
-- The current folder is mounted as a volume at /workspace by default. This can be changed by specifying a volume with `.` as the host path.
+---
 
-## Running on the host machine
+## 🌟 Key Highlights
 
-- Steps are sent as stdin to a shell subprocess
+- **Strict Code/Data Separation:** `run:` steps are executed verbatim as raw process scripts. Expressions and dynamic data bindings are isolated strictly to `env:`, eliminating shell-injection vectors entirely.
+- **Standard ES Template Syntax (`${...}`):** No custom DSL wrappers like `${{ }}` or `{{ }}`. If a field contains `${...}`, it evaluates standard JavaScript template string logic via AST.
+- **Deterministic Field Evaluation:** No silent fallbacks or ambiguous type conversions. Plain strings remain literal strings; conditions in `if:` fields run as strict JS boolean expressions.
+- **Built-in Dark Mode Web UI (`/runs`):** Monitor job statuses live, inspect workspace inputs, and view ANSI-colored terminal log streams rendered in real-time.
+- **System & Container Execution Drivers:** Run steps directly as detached host process groups or inside isolated Docker/Systemd transient units.
+- **Automatic Secret Redaction:** Secrets loaded from `.env` are automatically masked (`***`) across all terminal log outputs and report snapshots.
 
-## General configuration syntax
+---
 
-Running with Docker:
+## 📦 Project Structure
+
+```text
+my-project/
+├── .on/                     # Workflow definitions directory
+│   ├── release.yml
+│   └── test.yml
+├── .env                     # Local secrets (git-ignored)
+├── runner.config.mjs        # (Optional) Engine configuration
+└── package.json
 
-```sh
-curl -X POST http://localhost:11235/ -d '{ "event-name": {...} }'
 ```
 
-```yaml
-description: Run tests and build
+---
 
-vars: &vars
-  image: node:latest
+## 🚀 Quick Start
+
+### 1. Install & Run
+
+Run the engine directly via `npx` or `pnpm dlx`:
+
+```bash
+# Start full engine (Ingress HTTP Gateway + 5 Worker Loops)
+npx @cloud-cli/on start
 
-on:
-  event-name:
-    runner: docker
-    if:
-      - ${inputs.action} == 'published'
-    secrets:
-      - /path/to/secrets
-      - /path/to/.env
-    mappings:
-      <field>: <path.to.value.in.inputs>
-    env:
-      A_SECRET: "${secrets.A_SECRET}"
-      A_VALUE: "${inputs.some.value}"
-    defaults:
-      <<: *vars
-      volumes:
-        .: /home
-        /dev/shm: /dev/shm
-      args:
-        net: host
-        dns: 1.2.3.4
-    steps:
-      - pnpm i
-      - pnpm run build
-      - pnpm run test
-    triggers:
-      - path/to/output.json
 ```
 
-Running with a shell on the same machine as the server:
+### 2. Configure Secrets (`.env`)
 
-```sh
-curl -X POST http://localhost:11235/ -d '{ "package": {...} }'
+Secrets are automatically loaded from `.env` at the root of your project. Prefix secrets with `SECRET_`:
+
+```env
+SECRET_NPM_TOKEN="npm_1234567890abcdef"
+SECRET_GITHUB_TOKEN="ghp_1234567890abcdef"
+SECRET_GITHUB_WEBHOOK_SECRET="my-webhook-secret"
 ```
 
+`SECRET_GITHUB_WEBHOOK_SECRET` is required to validate incoming webhooks from GitHub
+
+### 3. Define a Workflow (`.on/release.yml`)
+
 ```yaml
-description: Auto-release library
+name: Build and Publish Release
+
 on:
-  event-name:
-    runner: shell
-    secrets:
-      - /path/to/secrets
-      - /path/to/.env
-    mappings:
-      <field>: <path.to.value.in.json.payload>
+  github:
+    if: inputs.event === 'push' && inputs.branch === 'main'
+
+concurrency:
+  group: release-${inputs.repo}
+  cancel-in-progress: true
+
+steps:
+  - id: checkout
+    name: Checkout Code Repository
+    env:
+      CLONE_URL: ${inputs.clone_url}
+      COMMIT_SHA: ${inputs.commit_sha}
+    run: |
+      git clone --depth 1 "$CLONE_URL" .
+      git checkout "$COMMIT_SHA"
+
+  - id: install-and-build
+    name: Install Dependencies & Build
+    run: |
+      pnpm install
+      pnpm run build
+
+  - id: publish
+    name: Publish to NPM
     env:
-      A_SECRET: "${secrets.A_SECRET}"
-      A_VALUE: "${inputs.some.value}"
-    steps:
-      - pnpm i
-      - pnpm run build
-      - pnpm run release
-    triggers:
-      - path/to/output.json
+      NPM_TOKEN: ${secrets.NPM_TOKEN}
+    run: |
+      echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" > ~/.npmrc
+      npx --yes semantic-release@24 -b main --no-ci
 ```
 
-### Event Payload
+---
 
-The webhooks can have any shape. To match events to workflows, we look for the top-level keys in the request body, and matching workflows that expect them to be present.
+## 💻 CLI Usage & Commands
 
-For example, given the request:
+```bash
+npx @cloud-cli/on [command] [options]
 
-```sh
-curl -X POST http://localhost:11235/ -d '{ "published": { "value" : 123 } }'
 ```
 
-Then the workflow should map the `published` key:
+### Commands
 
-```yaml
-on:
-  published:
-    steps:
-      - echo ${inputs.value}
-```
+| Command                 | Description                                                                              |
+| ----------------------- | ---------------------------------------------------------------------------------------- |
+| **`start`** _(default)_ | Runs both Webhook Ingress Gateway and Worker execution loops together.                   |
+| **`start-server`**      | Runs Webhook Ingress Gateway only (API / Gateway mode).                                  |
+| **`start-workers`**     | Runs Worker Polling loops only (Scalable Worker mode).                                   |
+| **`validate`**          | Parses and validates all YAML workflows in your workflows folder without executing jobs. |
 
-Here, the expression `inputs` in the workflow context is defined as the value set in the `published` key from the parsed JSON.
-The expression `${inputs.value}` contains `123`.
+### CLI and Environment Options
 
-To differentiate between events coming from the same place, multiple webhooks can be created.
-The incoming webhook URL can have a path, and that is considered as the event source.
+| Flag | Option        | Default               | Env                   | Description                               |
+| ---- | ------------- | --------------------- | --------------------- | ----------------------------------------- |
+| `-h` | `--help`      | —                     | -                     | Prints CLI help message and exits.        |
+| `-c` | `--config`    | `./runner.config.mjs` | `RUNNER_CONFIG_FILE`  | Path to JavaScript configuration file.    |
+| `-d` | `--database`  | -                     | `RUNNER_DATABASE_URL` | SQLite database file path or HTTP URL.    |
+| `-w` | `--workflows` | `.on/`                | `RUNNER_WORKFLOWS`    | Directory where workflow YAML files live. |
+| `-p` | `--port`      | `11235`               | `PORT`                | Port for the Ingress HTTP server.         |
+| `-k` | `--workers`   | `5`                   | `RUNNER_WORKERS`      | Number of worker loop threads to spawn.   |
+|      |               |                       | `RUNNER_ADMIN_SECRET` | Admin token to refresh secrets via API    |
 
-Consider this request:
+### Secrets
 
-```sh
-curl -X POST http://localhost:11235/source -d '{ "event": { "value" : 123 } }'
+---
+
+## ⚙️ Configuration Reference
+
+You can customize engine behavior using `runner.config.mjs` in your project root:
+
+```javascript
+// runner.config.mjs
+import { HtmlReporter, SlackReporter, JsonFileReporter } from '@cloud-cli/on/reporters';
+
+export default {
+  port: 3000,
+  workers: 5,
+  workflows: '/home/workflows/',
+  storagePath: '/tmp/workspaces',
+  database: 'https://remote.db.com/',
+
+  // Global environment variables passed to all steps
+  env: {
+    NODE_ENV: 'production',
+  },
+
+  // Custom execution reporters
+  reporters: [
+    new JsonFileReporter({ outputDir: './reports/json' }),
+    new HtmlReporter({ outputDir: './reports/html' }),
+    new SlackReporter({
+      webhookUrl: process.env.SLACK_WEBHOOK_URL,
+      channel: '#ci-deployments',
+    }),
+  ],
+};
 ```
 
-The key `source` is added to the event payload. The workflow should now be defined as:
-
-```yaml
-on:
-  source:
-    published:
-      steps:
-        - echo ${inputs.value}
-```
+---
 
-This difference in payload processing allows multiple webhooks from the same source, to separate different events with a similar JSON body.
+## 📐 Deterministic Evaluation Rules
 
-### Secrets
+To prevent syntax ambiguity and injection risks, fields in workflow definitions operate under **three strict modes**:
 
-The daemon can fetch secrets from its host environment, or from a file.
+```
+┌──────────────────────────────────────────────────────────────────────────┐
+│ 1. RAW PASSTHROUGH MODE (`run:`)                                         │
+│ • Executed verbatim as a shell process command.                          │
+│ • No string replacements or engine parsing performed.                    │
+│ • Access environment variables strictly via shell syntax: $MY_VAR.       │
+└──────────────────────────────────────────────────────────────────────────┘
+
+┌──────────────────────────────────────────────────────────────────────────┐
+│ 2. EXPRESSION MODE (`if:`, `eval:`)                                      │
+│ • Evaluated strictly as pure JavaScript expressions via Acorn AST.       │
+│ • Must be valid JS syntax (e.g. `inputs.branch === 'main'`).             │
+│ • Automatically coerced to boolean in `if:` conditions.                  │
+└──────────────────────────────────────────────────────────────────────────┘
+
+┌──────────────────────────────────────────────────────────────────────────┐
+│ 3. DETERMINISTIC VALUE MODE (`env:`, `name:`, `image:`, `group:`)        │
+│ • Plain strings WITHOUT `${}` remain 100% raw literal strings.            │
+│ • Strings WITH `${...}` evaluate as standard ES Template Literals.       │
+│ • Example: `node:${inputs.node_version}-alpine`                          │
+└──────────────────────────────────────────────────────────────────────────┘
 
-### Inputs
+```
 
-Inputs are defined from the incoming JSON payload. The payload is parsed and made available as the `inputs` variable
+### Context Scope Available in Expressions
 
-### Mappings
+Within `${...}`, `if:`, and `eval:` contexts, the following object scopes are exposed:
 
-These are shortcuts to make scripting easier.
+- **`inputs`**: Payload key-values received from incoming webhooks.
+- **`env`**: Merged environment variables from global config and workflow definitions.
+- **`secrets`**: Unmasked secret values loaded from `.env` or environment variables (`SECRET_` prefix stripped).
+- **`steps`**: Execution statuses and outputs from previous steps in the workflow (`steps.<id>.status`, `steps.<id>.outputs`).
+- **`BUILTIN_HELPERS`**: JS utilities including `String`, `Number`, `Boolean`, and `JSON.parse` / `JSON.stringify`.
 
-After the JSON payload is parsed, these mappings are evaluated, and added to `inputs` as shortcuts for long/deep properties in the payload.
+---
 
-For example: from a GitHub webhook event that contains a lot of fields, we can define `image` from `package.package_version.package_url`
+## 🌐 Webhook Ingress Gateway & Dashboard
 
-```yaml
-mappings:
-  image: ${inputs.package.package_version.package_url}
-```
+The Ingress Gateway listens for incoming HTTP requests and serves the live web UI.
 
-### Environment variables
+### Endpoint Matrix
 
-After resolving secrets and mappings, we proceed to resolve env variables from template strings or literal strings.
+| Method     | Endpoint           | Description                                                                            |
+| ---------- | ------------------ | -------------------------------------------------------------------------------------- |
+| **`POST`** | `/webhooks/github` | Webhook endpoint for GitHub events. Evaluates `on.github.if` triggers.                 |
+| **`GET`**  | `/runs`            | **Dashboard:** Live dark-mode monitoring page listing recent jobs and worker health.   |
+| **`GET`**  | `/runs/:jobId`     | **Job Report:** Interactive HTML trace view with step timings and terminal log output. |
 
-An env variable is interpreted a JS template string, with `${value}` syntax used to interpolate values from `inputs` or `secrets`.
+### Dashboard Features
 
-## Sequence of operation
+- **Real-time Auto-Refresh:** `/runs` automatically refreshes job queue statuses (`PENDING`, `RUNNING`, `SUCCESS`, `FAILED`, `CANCELLED`).
+- **ANSI Terminal Rendering:** Uses `ansi_up` to render bash colors, bold highlights, and console outputs accurately in step log boxes.
+- **Payload Inspection:** View JSON inputs received from webhooks for easy debugging.
 
-For every incoming event, these steps are followed:
+---
 
-- Parse and validate payload
-- Load secrets
-- Map inputs
-- Populate env with secrets
-- Populate env with additional workflow definitions (section `env`)
-- Create a temporary working directory
-- Add a volume to defaults at `/workspace`, or a custom path, if a volume with a host path `.` is defined in the workflow
-- Run steps:
-  - For every step, either a string, or a step definition is accepted.
-  - If string, run with the defaults defined in the workflow
-  - If a definition, merge defaults into it, and run the step
-  - The step is a shell command, executed inside a short-lived container
-- Trigger new events
+## 🔒 Security & Hardening
 
-## Triggers
+1. **Environment Variable Boundary:**
+   By forcing shell steps to consume data via process environment variables (`$CLONE_URL`), malicious webhook payloads containing shell delimiters (e.g. `; rm -rf /`) cannot mutate shell script execution trees.
+2. **Prototype Pollution Protection:**
+   AST evaluation explicitly blocks access to dangerous JS properties (`constructor`, `__proto__`, `prototype`).
+3. **Payload Size Guard:**
+   The Ingress server enforces a strict 5MB payload limit to prevent Out-Of-Memory (OOM) denial-of-service attacks.
+4. **Signal Traps & Resource Cleanup:**
+   Graceful process traps (`SIGINT`, `SIGTERM`) ensure active job handles are safely terminated, file descriptors are closed, and temp `.env`/`.out` files are removed via `try ... finally` blocks.
 
-After steps are executed, a list of one or more JSON files can be defined to trigger new workflows.
+## Development
 
-These files are read one by one and sent back to the daemon as new events.
+```bash
+pnpm i
+pnpm run lint
+pnpm run test
+pnpm run build
+```

package/dist/db-client.d.ts

@@ -0,0 +1,14 @@
+declare function query(method: 'get' | 'run' | 'all', statement: string, data?: Array<string | number | null>, pragma?: string[]): Promise<any>;
+export declare const get: (statement: string, data?: (string | number | null)[] | undefined, pragma?: string[] | undefined) => Promise<any>;
+export declare const run: (statement: string, data?: (string | number | null)[] | undefined, pragma?: string[] | undefined) => Promise<any>;
+export declare const all: (statement: string, data?: (string | number | null)[] | undefined, pragma?: string[] | undefined) => Promise<any>;
+export declare function pragma(p: any): void;
+declare const _default: {
+    query: typeof query;
+    get: (statement: string, data?: (string | number | null)[] | undefined, pragma?: string[] | undefined) => Promise<any>;
+    run: (statement: string, data?: (string | number | null)[] | undefined, pragma?: string[] | undefined) => Promise<any>;
+    all: (statement: string, data?: (string | number | null)[] | undefined, pragma?: string[] | undefined) => Promise<any>;
+    pragma: typeof pragma;
+};
+export default _default;
+export declare function setUrl(u: any): void;

package/dist/drivers/index.d.ts

@@ -0,0 +1,2 @@
+import { ExecutionDriver } from '../types.js';
+export declare function resolveDriver(): Promise<ExecutionDriver>;

package/dist/drivers/standard-process.driver.d.ts

@@ -0,0 +1,10 @@
+import { ExecutionDriver, StepContext, StepExecutionHandle } from '../types.js';
+export declare class StandardProcessDriver implements ExecutionDriver {
+    name: string;
+    isSupported(): Promise<boolean>;
+    execute(ctx: StepContext): Promise<StepExecutionHandle>;
+    /**
+     * Kills the entire process group tree (-PID) with unref escalation
+     */
+    private killProcessGroup;
+}

package/dist/drivers/systemd.driver.d.ts

@@ -0,0 +1,6 @@
+import { ExecutionDriver, StepContext, StepExecutionHandle } from '../types.js';
+export declare class SystemdDriver implements ExecutionDriver {
+    name: string;
+    isSupported(): Promise<boolean>;
+    execute(ctx: StepContext): Promise<StepExecutionHandle>;
+}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+export { HtmlReporter } from './reporters/html.reporter.js';
+export { JsonFileReporter } from './reporters/json-file.reporter.js';
+export { SlackReporter } from './reporters/slack.reporter.js';

package/dist/log-redactor.d.ts

@@ -0,0 +1,6 @@
+import { Transform, TransformCallback } from 'node:stream';
+export declare class SecretRedactorStream extends Transform {
+    private secretValues;
+    constructor(secretValues: string[]);
+    _transform(chunk: any, _encoding: BufferEncoding, callback: TransformCallback): void;
+}