@azizikri/hookpipe 1.0.0

package/src/cli/serve.js

@@ -0,0 +1,75 @@
+import { loadConfig } from '../utils/config.js';
+import { createLogger } from '../utils/logger.js';
+import { initDatabase, closeDatabase } from '../db/index.js';
+import { SqliteQueue } from '../queue/sqlite-queue.js';
+import { PipelineLoader } from '../pipeline-loader.js';
+import { PluginLoader } from '../plugin-loader.js';
+import { registry as destinationRegistry } from '../destinations/index.js';
+import { DeliveryWorker } from '../delivery/worker.js';
+import { createServer } from '../server.js';
+
+export function serveCommand(program) {
+  program
+    .command('serve')
+    .description('Start the hookpipe webhook server')
+    .option('-p, --port <number>', 'Port to listen on')
+    .option('-H, --host <string>', 'Host to bind to')
+    .option('-c, --config <path>', 'Config file path')
+    .option('--pipelines <dir>', 'Pipelines directory')
+    .option('--plugins <dir>', 'Plugins directory')
+    .action(async (opts) => {
+      const overrides = {};
+      if (opts.port) overrides.port = Number(opts.port);
+      if (opts.host) overrides.host = opts.host;
+      if (opts.pipelines) overrides.pipelines = { dir: opts.pipelines };
+      if (opts.plugins) overrides.plugins = { dir: opts.plugins };
+      if (opts.config) overrides.configPath = opts.config;
+
+      const config = loadConfig(overrides);
+      const logger = createLogger({ level: config.log.level });
+      const db = initDatabase(config.db.path);
+      const queue = new SqliteQueue(db, config);
+      const pipelineLoader = new PipelineLoader(config.pipelines.dir);
+      const pluginLoader = new PluginLoader(config.plugins.dir);
+      const worker = new DeliveryWorker({
+        queue,
+        db,
+        destinationRegistry,
+        logger,
+        getPipeline: (id) => pipelineLoader.get(id),
+        config: { pollIntervalMs: config.queue?.poll_interval_ms ?? 1000, concurrency: config.queue?.concurrency ?? 5 },
+      });
+
+      const server = createServer({
+        pipelineLoader,
+        pluginLoader,
+        queue,
+        db,
+        logger,
+        config,
+      });
+
+      await pipelineLoader.loadAll();
+      await pipelineLoader.startWatching();
+      worker.start();
+
+      const host = config.host || '0.0.0.0';
+      const port = config.port || 3000;
+
+      await server.listen({ port, host });
+      logger.info(`Server listening on http://${host}:${port}`);
+
+      const shutdown = async () => {
+        logger.info('Shutting down...');
+        worker.stop();
+        pipelineLoader.stopWatching();
+        server.close();
+        closeDatabase(db);
+        logger.info('Shutdown complete');
+        process.exit(0);
+      };
+
+      process.on('SIGTERM', shutdown);
+      process.on('SIGINT', shutdown);
+    });
+}

package/src/cli/test.js

@@ -0,0 +1,135 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { loadConfig } from '../utils/config.js';
+import { PipelineLoader } from '../pipeline-loader.js';
+import { PluginLoader } from '../plugin-loader.js';
+import { verifyWebhookAuth } from '../auth/hmac.js';
+import { renderTemplate } from '../templates/handlebars.js';
+
+function collectHeader(value, previous) {
+  return previous.concat([value]);
+}
+
+export function testCommand(program) {
+  program
+    .command('test')
+    .description('Dry-run a webhook through a pipeline')
+    .argument('<pipeline-id>', 'Pipeline ID to test')
+    .option('-f, --file <path>', 'JSON file to use as payload')
+    .option('-d, --data <json>', 'Inline JSON payload')
+    .option('--header <header>', 'Custom header (repeatable, format: "Key: Value")', collectHeader, [])
+    .option('--skip-auth', 'Skip HMAC authentication check')
+    .option('-c, --config <path>', 'Config file path')
+    .action(async (pipelineId, opts) => {
+      try {
+        await runTest(pipelineId, opts);
+      } catch (err) {
+        console.error(`Error: ${err.message}`);
+        process.exit(1);
+      }
+    });
+}
+
+async function runTest(pipelineId, opts) {
+  const overrides = {};
+  if (opts.config) overrides.configPath = opts.config;
+
+  const config = loadConfig(overrides);
+  const pipelineLoader = new PipelineLoader(config.pipelines.dir);
+  const pluginLoader = new PluginLoader(config.plugins.dir);
+
+  await pipelineLoader.loadAll();
+
+  const pipeline = pipelineLoader.get(pipelineId);
+  if (!pipeline) {
+    throw new Error(`Pipeline "${pipelineId}" not found`);
+  }
+
+  // Read payload
+  let payload = {};
+  if (opts.file) {
+    const filePath = resolve(opts.file);
+    const raw = readFileSync(filePath, 'utf-8');
+    payload = JSON.parse(raw);
+  } else if (opts.data) {
+    payload = JSON.parse(opts.data);
+  }
+
+  // Parse headers
+  const headers = {};
+  for (const h of opts.header) {
+    const colonIdx = h.indexOf(':');
+    if (colonIdx > 0) {
+      const key = h.slice(0, colonIdx).trim().toLowerCase();
+      const value = h.slice(colonIdx + 1).trim();
+      headers[key] = value;
+    }
+  }
+
+  const result = {
+    pipeline: pipelineId,
+    payload,
+    headers,
+    steps: {},
+  };
+
+  // Auth check
+  if (pipeline.auth && !opts.skipAuth) {
+    const rawBody = JSON.stringify(payload);
+    const authResult = verifyWebhookAuth(rawBody, headers, pipeline.auth);
+    result.steps.auth = {
+      status: authResult.valid ? 'passed' : 'failed',
+      error: authResult.error || null,
+    };
+  } else if (opts.skipAuth) {
+    result.steps.auth = { status: 'skipped' };
+  }
+
+  // Filter check
+  if (pipeline.filter) {
+    try {
+      const filterFn = pluginLoader.loadFilter(pipeline.filter);
+      const filterResult = filterFn(payload, headers, pipeline.filter_config || {});
+      result.steps.filter = {
+        status: filterResult.pass ? 'pass' : 'drop',
+        reason: filterResult.reason || null,
+      };
+    } catch (err) {
+      result.steps.filter = { status: 'error', error: err.message };
+    }
+  }
+
+  // Transform check
+  if (pipeline.transform) {
+    try {
+      const transformFn = pluginLoader.loadTransform(pipeline.transform);
+      const transformed = transformFn(payload, headers, pipeline.filter_config || {});
+      result.steps.transform = {
+        before: payload,
+        after: transformed,
+      };
+      if (transformed !== null) {
+        payload = transformed;
+      }
+    } catch (err) {
+      result.steps.transform = { status: 'error', error: err.message };
+    }
+  }
+
+  // Destinations (dry-run preview)
+  if (pipeline.destinations) {
+    result.steps.destinations = pipeline.destinations.map((dest) => {
+      const body = (result.steps.transform && result.steps.transform.after !== undefined)
+        ? result.steps.transform.after
+        : payload;
+      return {
+        url: dest.url,
+        method: (dest.method || 'POST').toUpperCase(),
+        headers: dest.headers || {},
+        bodyPreview: body != null ? JSON.stringify(body).slice(0, 500) : '(empty)',
+      };
+    });
+  }
+
+  console.log(JSON.stringify(result, null, 2));
+}

package/src/db/index.js

@@ -0,0 +1,60 @@
+import Database from 'better-sqlite3';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const MIGRATIONS_DIR = path.join(__dirname, 'migrations');
+
+export function initDatabase(dbPath) {
+  const dir = path.dirname(dbPath);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+
+  const db = new Database(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.pragma('foreign_keys = ON');
+
+  runMigrations(db);
+
+  return db;
+}
+
+export function runMigrations(db) {
+  // Ensure migrations table exists
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS migrations (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      name TEXT NOT NULL UNIQUE,
+      applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+    )
+  `);
+
+  // Read migration files sorted alphabetically
+  const files = fs.readdirSync(MIGRATIONS_DIR)
+    .filter(f => f.endsWith('.sql'))
+    .sort();
+
+  const applied = new Set(
+    db.prepare('SELECT name FROM migrations').all().map(r => r.name)
+  );
+
+  for (const file of files) {
+    if (applied.has(file)) continue;
+
+    const sql = fs.readFileSync(path.join(MIGRATIONS_DIR, file), 'utf-8');
+    db.exec(sql);
+
+    // Record migration (only if not already recorded by the SQL itself)
+    const exists = db.prepare('SELECT 1 FROM migrations WHERE name = ?').get(file);
+    if (!exists) {
+      db.prepare('INSERT INTO migrations (name) VALUES (?)').run(file);
+    }
+  }
+}
+
+export function closeDatabase(db) {
+  db.close();
+}

package/src/db/migrations/001_initial.sql

@@ -0,0 +1,107 @@
+-- Migration: 001_initial
+-- Description: Create initial hookpipe schema
+-- Date: 2024-01-01
+
+BEGIN;
+
+-- Migration tracking
+CREATE TABLE IF NOT EXISTS migrations (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name TEXT NOT NULL UNIQUE,
+  applied_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Webhook delivery records
+CREATE TABLE IF NOT EXISTS deliveries (
+  id TEXT PRIMARY KEY,
+  pipeline_id TEXT NOT NULL,
+  received_at TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending',
+  payload TEXT NOT NULL,
+  headers TEXT NOT NULL,
+  source_ip TEXT,
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Individual delivery attempt records
+CREATE TABLE IF NOT EXISTS delivery_attempts (
+  id TEXT PRIMARY KEY,
+  delivery_id TEXT NOT NULL REFERENCES deliveries(id),
+  destination_id TEXT NOT NULL,
+  attempt_number INTEGER NOT NULL,
+  status TEXT NOT NULL,
+  status_code INTEGER,
+  response_body TEXT,
+  error_message TEXT,
+  duration_ms INTEGER,
+  attempted_at TEXT NOT NULL DEFAULT (datetime('now')),
+  UNIQUE(delivery_id, destination_id, attempt_number)
+);
+
+-- Failed deliveries moved to DLQ
+CREATE TABLE IF NOT EXISTS dead_letters (
+  id TEXT PRIMARY KEY,
+  delivery_id TEXT NOT NULL REFERENCES deliveries(id),
+  pipeline_id TEXT NOT NULL,
+  destination_id TEXT NOT NULL,
+  payload TEXT NOT NULL,
+  headers TEXT NOT NULL,
+  error_message TEXT,
+  attempts INTEGER NOT NULL,
+  created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- In-process queue backing table
+CREATE TABLE IF NOT EXISTS queue_jobs (
+  id TEXT PRIMARY KEY,
+  delivery_id TEXT NOT NULL REFERENCES deliveries(id),
+  destination_id TEXT NOT NULL,
+  pipeline_id TEXT NOT NULL,
+  payload TEXT NOT NULL,
+  headers TEXT NOT NULL,
+  status TEXT NOT NULL DEFAULT 'pending',
+  attempts INTEGER NOT NULL DEFAULT 0,
+  max_attempts INTEGER NOT NULL DEFAULT 3,
+  next_attempt_at TEXT NOT NULL DEFAULT (datetime('now')),
+  locked_at TEXT,
+  locked_by TEXT,
+  error_message TEXT,
+  created_at TEXT NOT NULL DEFAULT (datetime('now')),
+  updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Per-pipeline statistics
+CREATE TABLE IF NOT EXISTS pipeline_stats (
+  pipeline_id TEXT PRIMARY KEY,
+  total_received INTEGER NOT NULL DEFAULT 0,
+  total_delivered INTEGER NOT NULL DEFAULT 0,
+  total_failed INTEGER NOT NULL DEFAULT 0,
+  total_filtered INTEGER NOT NULL DEFAULT 0,
+  last_received_at TEXT,
+  updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+);
+
+-- Indexes: deliveries
+CREATE INDEX IF NOT EXISTS idx_deliveries_pipeline_created
+  ON deliveries(pipeline_id, created_at);
+CREATE INDEX IF NOT EXISTS idx_deliveries_status
+  ON deliveries(status);
+
+-- Indexes: delivery_attempts
+CREATE INDEX IF NOT EXISTS idx_delivery_attempts_delivery_id
+  ON delivery_attempts(delivery_id);
+CREATE INDEX IF NOT EXISTS idx_delivery_attempts_attempted_at
+  ON delivery_attempts(attempted_at);
+
+-- Indexes: dead_letters
+CREATE INDEX IF NOT EXISTS idx_dead_letters_pipeline_created
+  ON dead_letters(pipeline_id, created_at);
+
+-- Indexes: queue_jobs
+CREATE INDEX IF NOT EXISTS idx_queue_jobs_status_next_attempt
+  ON queue_jobs(status, next_attempt_at);
+CREATE INDEX IF NOT EXISTS idx_queue_jobs_delivery_id
+  ON queue_jobs(delivery_id);
+
+COMMIT;

package/src/db/queries.js

@@ -0,0 +1,196 @@
+// --- Deliveries ---
+
+export function insertDelivery(db, { id, pipelineId, payload, headers, sourceIp, status }) {
+  const stmt = db.prepare(`
+    INSERT INTO deliveries (id, pipeline_id, payload, headers, source_ip, status, received_at)
+    VALUES (?, ?, ?, ?, ?, ?, datetime('now'))
+  `);
+  stmt.run(id, pipelineId, JSON.stringify(payload), JSON.stringify(headers), sourceIp, status);
+}
+
+export function getDelivery(db, id) {
+  const row = db.prepare('SELECT * FROM deliveries WHERE id = ?').get(id);
+  if (!row) return null;
+  row.payload = JSON.parse(row.payload);
+  row.headers = JSON.parse(row.headers);
+  return row;
+}
+
+export function listDeliveries(db, { pipelineId, status, limit = 50, offset = 0, since } = {}) {
+  const conditions = [];
+  const params = [];
+
+  if (pipelineId) {
+    conditions.push('pipeline_id = ?');
+    params.push(pipelineId);
+  }
+  if (status) {
+    conditions.push('status = ?');
+    params.push(status);
+  }
+  if (since) {
+    conditions.push('created_at >= ?');
+    params.push(since);
+  }
+
+  const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+  const sql = `SELECT * FROM deliveries ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`;
+  params.push(limit, offset);
+
+  const rows = db.prepare(sql).all(...params);
+  return rows.map(row => {
+    row.payload = JSON.parse(row.payload);
+    row.headers = JSON.parse(row.headers);
+    return row;
+  });
+}
+
+export function updateDeliveryStatus(db, id, status) {
+  db.prepare(`UPDATE deliveries SET status = ?, updated_at = datetime('now') WHERE id = ?`).run(status, id);
+}
+
+// --- Delivery Attempts ---
+
+export function insertAttempt(db, { id, deliveryId, destinationId, attemptNumber, status, statusCode, responseBody, errorMessage, durationMs }) {
+  db.prepare(`
+    INSERT INTO delivery_attempts (id, delivery_id, destination_id, attempt_number, status, status_code, response_body, error_message, duration_ms)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(id, deliveryId, destinationId, attemptNumber, status, statusCode, responseBody, errorMessage, durationMs);
+}
+
+export function getAttemptsByDelivery(db, deliveryId) {
+  return db.prepare('SELECT * FROM delivery_attempts WHERE delivery_id = ? ORDER BY attempt_number ASC').all(deliveryId);
+}
+
+// --- Dead Letters ---
+
+export function insertDeadLetter(db, { id, deliveryId, pipelineId, destinationId, payload, headers, errorMessage, attempts }) {
+  db.prepare(`
+    INSERT INTO dead_letters (id, delivery_id, pipeline_id, destination_id, payload, headers, error_message, attempts)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+  `).run(id, deliveryId, pipelineId, destinationId, JSON.stringify(payload), JSON.stringify(headers), errorMessage, attempts);
+}
+
+export function getDeadLetter(db, id) {
+  const row = db.prepare('SELECT * FROM dead_letters WHERE id = ?').get(id);
+  if (!row) return null;
+  row.payload = JSON.parse(row.payload);
+  row.headers = JSON.parse(row.headers);
+  return row;
+}
+
+export function listDeadLetters(db, { pipelineId, limit = 50, offset = 0 } = {}) {
+  const conditions = [];
+  const params = [];
+
+  if (pipelineId) {
+    conditions.push('pipeline_id = ?');
+    params.push(pipelineId);
+  }
+
+  const where = conditions.length ? `WHERE ${conditions.join(' AND ')}` : '';
+  const sql = `SELECT * FROM dead_letters ${where} ORDER BY created_at DESC LIMIT ? OFFSET ?`;
+  params.push(limit, offset);
+
+  const rows = db.prepare(sql).all(...params);
+  return rows.map(row => {
+    row.payload = JSON.parse(row.payload);
+    row.headers = JSON.parse(row.headers);
+    return row;
+  });
+}
+
+// --- Queue Jobs ---
+
+export function insertQueueJob(db, { id, deliveryId, destinationId, pipelineId, payload, headers, maxAttempts }) {
+  db.prepare(`
+    INSERT INTO queue_jobs (id, delivery_id, destination_id, pipeline_id, payload, headers, max_attempts)
+    VALUES (?, ?, ?, ?, ?, ?, ?)
+  `).run(id, deliveryId, destinationId, pipelineId, JSON.stringify(payload), JSON.stringify(headers), maxAttempts);
+}
+
+export function dequeueJobs(db, limit, lockId) {
+  const now = new Date().toISOString();
+  const dequeue = db.transaction(() => {
+    const jobs = db.prepare(`
+      SELECT * FROM queue_jobs
+      WHERE status = 'pending' AND next_attempt_at <= datetime('now')
+      LIMIT ?
+    `).all(limit);
+
+    if (jobs.length === 0) return [];
+
+    const ids = jobs.map(j => j.id);
+    const placeholders = ids.map(() => '?').join(',');
+    db.prepare(`
+      UPDATE queue_jobs
+      SET status = 'processing', locked_at = datetime('now'), locked_by = ?, updated_at = datetime('now')
+      WHERE id IN (${placeholders})
+    `).run(lockId, ...ids);
+
+    return db.prepare(`
+      SELECT * FROM queue_jobs WHERE id IN (${placeholders})
+    `).all(...ids).map(row => {
+      row.payload = JSON.parse(row.payload);
+      row.headers = JSON.parse(row.headers);
+      return row;
+    });
+  });
+  return dequeue();
+}
+
+export function ackJob(db, jobId) {
+  db.prepare(`UPDATE queue_jobs SET status = 'completed', updated_at = datetime('now') WHERE id = ?`).run(jobId);
+}
+
+export function nackJob(db, jobId, errorMessage, nextAttemptAt) {
+  db.prepare(`
+    UPDATE queue_jobs
+    SET status = 'pending',
+        attempts = attempts + 1,
+        error_message = ?,
+        next_attempt_at = ?,
+        locked_at = NULL,
+        locked_by = NULL,
+        updated_at = datetime('now')
+    WHERE id = ?
+  `).run(errorMessage, nextAttemptAt, jobId);
+}
+
+export function moveJobToDeadLetter(db, jobId) {
+  db.prepare(`UPDATE queue_jobs SET status = 'dead_letter', updated_at = datetime('now') WHERE id = ?`).run(jobId);
+}
+
+export function getJobsByDelivery(db, deliveryId) {
+  return db.prepare('SELECT * FROM queue_jobs WHERE delivery_id = ? ORDER BY created_at ASC').all(deliveryId).map(row => {
+    row.payload = JSON.parse(row.payload);
+    row.headers = JSON.parse(row.headers);
+    return row;
+  });
+}
+
+// --- Pipeline Stats ---
+
+export function incrementStat(db, pipelineId, field) {
+  const validFields = ['total_received', 'total_delivered', 'total_failed', 'total_filtered'];
+  if (!validFields.includes(field)) {
+    throw new Error(`Invalid stat field: ${field}`);
+  }
+
+  const lastReceivedClause = field === 'total_received'
+    ? ", last_received_at = datetime('now')"
+    : '';
+
+  db.prepare(`
+    INSERT INTO pipeline_stats (pipeline_id, ${field}, updated_at${field === 'total_received' ? ', last_received_at' : ''})
+    VALUES (?, 1, datetime('now')${field === 'total_received' ? ", datetime('now')" : ''})
+    ON CONFLICT(pipeline_id) DO UPDATE SET
+      ${field} = ${field} + 1,
+      updated_at = datetime('now')
+      ${lastReceivedClause}
+  `).run(pipelineId);
+}
+
+export function getStats(db, pipelineId) {
+  return db.prepare('SELECT * FROM pipeline_stats WHERE pipeline_id = ?').get(pipelineId) || null;
+}

package/src/delivery/retry.js

@@ -0,0 +1,63 @@
+const DEFAULTS = {
+  backoff: 'exponential',
+  initialDelayMs: 1000,
+  maxDelayMs: 300000,
+};
+
+/**
+ * Calculate the delay before the next retry attempt.
+ * @param {number} attempt - Current attempt number (1-based)
+ * @param {object} config - Retry configuration
+ * @returns {number} Delay in milliseconds (integer)
+ */
+export function calculateNextDelay(attempt, config = {}) {
+  const backoff = config.backoff || DEFAULTS.backoff;
+  const initialDelayMs = config.initialDelayMs ?? DEFAULTS.initialDelayMs;
+  const maxDelayMs = config.maxDelayMs ?? DEFAULTS.maxDelayMs;
+
+  let baseDelay;
+
+  switch (backoff) {
+    case 'linear':
+      baseDelay = Math.min(initialDelayMs * attempt, maxDelayMs);
+      return baseDelay + jitter(baseDelay);
+
+    case 'fixed':
+      return initialDelayMs;
+
+    case 'exponential':
+    default:
+      baseDelay = Math.min(initialDelayMs * 2 ** (attempt - 1), maxDelayMs);
+      return baseDelay + jitter(baseDelay);
+  }
+}
+
+/**
+ * Determine if another retry should be attempted.
+ * @param {number} attempt - Current attempt number
+ * @param {number} maxAttempts - Maximum allowed attempts
+ * @returns {boolean}
+ */
+export function shouldRetry(attempt, maxAttempts) {
+  return attempt < maxAttempts;
+}
+
+/**
+ * Get the ISO 8601 timestamp for when the next attempt should occur.
+ * @param {number} attempt - Current attempt number
+ * @param {object} config - Retry configuration
+ * @returns {string} ISO 8601 date string
+ */
+export function getNextAttemptTime(attempt, config) {
+  const delay = calculateNextDelay(attempt, config);
+  return new Date(Date.now() + delay).toISOString();
+}
+
+/**
+ * Add random jitter of 0-25% of the base delay.
+ * @param {number} baseDelay
+ * @returns {number} Integer jitter value
+ */
+function jitter(baseDelay) {
+  return Math.floor(Math.random() * baseDelay * 0.25);
+}