@azizikri/hookpipe 1.0.0

package/src/delivery/worker.js

@@ -0,0 +1,192 @@
+import { ulid } from 'ulid';
+import { calculateNextDelay, shouldRetry, getNextAttemptTime } from './retry.js';
+import { insertAttempt, updateDeliveryStatus, incrementStat } from '../db/queries.js';
+
+const DEFAULT_CONFIG = {
+  pollIntervalMs: 1000,
+  concurrency: 5,
+};
+
+const DEFAULT_RETRY_CONFIG = {
+  maxAttempts: 3,
+  backoff: 'exponential',
+  initialDelayMs: 1000,
+  maxDelayMs: 300000,
+};
+
+export class DeliveryWorker {
+  #queue;
+  #db;
+  #destinationRegistry;
+  #logger;
+  #config;
+  #getPipeline;
+  #interval = null;
+  #isPolling = false;
+
+  constructor({ queue, db, destinationRegistry, logger, config = {}, getPipeline }) {
+    if (typeof getPipeline !== 'function') {
+      throw new Error('DeliveryWorker requires getPipeline(pipelineId)');
+    }
+
+    this.#queue = queue;
+    this.#db = db;
+    this.#destinationRegistry = destinationRegistry;
+    this.#logger = logger;
+    this.#config = { ...DEFAULT_CONFIG, ...config };
+    this.#getPipeline = getPipeline;
+  }
+
+  start() {
+    if (this.#interval) {
+      return;
+    }
+
+    this.#interval = setInterval(() => {
+      this.#poll().catch((error) => {
+        this.#logger?.error?.({ error }, 'delivery worker poll failed');
+      });
+    }, this.#config.pollIntervalMs);
+  }
+
+  stop() {
+    if (!this.#interval) {
+      return;
+    }
+
+    clearInterval(this.#interval);
+    this.#interval = null;
+  }
+
+  async #poll() {
+    if (this.#isPolling) {
+      return;
+    }
+
+    this.#isPolling = true;
+    try {
+      const jobs = await this.#queue.dequeue(this.#config.concurrency);
+      await Promise.all(jobs.map((job) => this.#processJob(job)));
+    } finally {
+      this.#isPolling = false;
+    }
+  }
+
+  async #processJob(job) {
+    const startedAt = Date.now();
+    const pipeline = await this.#getPipeline(job.pipeline_id);
+    const destConfig = getDestinationConfig(pipeline, job.destination_id);
+    const retryConfig = getRetryConfig(pipeline);
+    const adapter = this.#destinationRegistry.getAdapter(destConfig.type);
+    const payload = typeof job.payload === 'string' ? JSON.parse(job.payload) : job.payload;
+    const headers = typeof job.headers === 'string' ? JSON.parse(job.headers) : job.headers;
+    const context = {
+      deliveryId: job.delivery_id,
+      pipelineId: job.pipeline_id,
+      attempt: job.attempts,
+      headers,
+    };
+
+    try {
+      const response = await adapter.send(payload, destConfig, context);
+      const durationMs = Date.now() - startedAt;
+
+      if (response?.success === false) {
+        const error = new Error(response.error || `HTTP ${response.statusCode}`);
+        error.statusCode = response.statusCode;
+        error.responseBody = response.responseBody;
+        insertAttempt(this.#db, {
+          id: ulid(),
+          deliveryId: job.delivery_id,
+          destinationId: destConfig.id ?? destConfig.type,
+          attemptNumber: job.attempts,
+          status: 'failure',
+          statusCode: response.statusCode ?? null,
+          responseBody: response.responseBody ?? null,
+          errorMessage: error.message,
+          durationMs,
+        });
+        await this.#handleRetryOrDlq(job, destConfig, retryConfig, error);
+        return;
+      }
+
+      await this.#queue.ack(job.id);
+      insertAttempt(this.#db, {
+        id: ulid(),
+        deliveryId: job.delivery_id,
+        destinationId: destConfig.id ?? destConfig.type,
+        attemptNumber: job.attempts,
+        status: 'success',
+        statusCode: response?.statusCode ?? null,
+        responseBody: response?.responseBody ?? null,
+        errorMessage: null,
+        durationMs,
+      });
+      updateDeliveryStatus(this.#db, job.delivery_id, 'delivered');
+      incrementStat(this.#db, job.pipeline_id, 'total_delivered');
+    } catch (error) {
+      const durationMs = Date.now() - startedAt;
+      insertAttempt(this.#db, {
+        id: ulid(),
+        deliveryId: job.delivery_id,
+        destinationId: destConfig.id ?? destConfig.type,
+        attemptNumber: job.attempts,
+        status: 'failure',
+        statusCode: null,
+        responseBody: null,
+        errorMessage: error instanceof Error ? error.message : String(error),
+        durationMs,
+      });
+      await this.#handleRetryOrDlq(job, destConfig, retryConfig, error);
+    }
+  }
+
+  async #handleRetryOrDlq(job, destConfig, retryConfig, error) {
+    // If on_failure is 'log', just log and mark completed (no retry, no DLQ)
+    if (destConfig.on_failure === 'log') {
+      await this.#queue.ack(job.id);
+      this.#logger?.warn?.({ deliveryId: job.delivery_id, error: error.message }, 'delivery failed, on_failure=log, not retrying');
+      incrementStat(this.#db, job.pipeline_id, 'total_failed');
+      return;
+    }
+
+    if (shouldRetry(job.attempts, retryConfig.maxAttempts)) {
+      const nextAttemptAt = getNextAttemptTime(job.attempts, retryConfig);
+      await this.#queue.nack(job.id, error, nextAttemptAt);
+      return;
+    }
+
+    await this.#queue.moveToDeadLetter(job.id, error);
+    updateDeliveryStatus(this.#db, job.delivery_id, 'dead_letter');
+    incrementStat(this.#db, job.pipeline_id, 'total_failed');
+  }
+}
+
+function getDestinationConfig(pipeline, destinationId) {
+  // Support both array-based destinations and single destination config
+  if (Array.isArray(pipeline?.destinations)) {
+    const dest = destinationId
+      ? pipeline.destinations.find((d) => d.id === destinationId)
+      : pipeline.destinations[0];
+    if (!dest?.type) {
+      throw new Error(`Pipeline ${pipeline?.id ?? 'unknown'} destination '${destinationId}' not found or missing type`);
+    }
+    return dest;
+  }
+  const destConfig = pipeline?.destination ?? pipeline?.destConfig ?? pipeline?.destinationConfig;
+  if (!destConfig?.type) {
+    throw new Error(`Pipeline ${pipeline?.id ?? 'unknown'} is missing destination config`);
+  }
+  return destConfig;
+}
+
+function getRetryConfig(pipeline) {
+  const raw = pipeline?.retry ?? pipeline?.retryConfig ?? {};
+  return {
+    ...DEFAULT_RETRY_CONFIG,
+    maxAttempts: raw.maxAttempts ?? raw.max_attempts ?? DEFAULT_RETRY_CONFIG.maxAttempts,
+    backoff: raw.backoff ?? DEFAULT_RETRY_CONFIG.backoff,
+    initialDelayMs: raw.initialDelayMs ?? raw.initial_delay_ms ?? DEFAULT_RETRY_CONFIG.initialDelayMs,
+    maxDelayMs: raw.maxDelayMs ?? raw.max_delay_ms ?? DEFAULT_RETRY_CONFIG.maxDelayMs,
+  };
+}

package/src/destinations/http.js

@@ -0,0 +1,60 @@
+import { DestinationAdapter } from './interface.js';
+import { renderTemplate } from '../templates/handlebars.js';
+
+export class HttpDestinationAdapter extends DestinationAdapter {
+  get type() {
+    return 'http';
+  }
+
+  async send(payload, destConfig, context) {
+    const {
+      url,
+      method = 'POST',
+      headers: configHeaders = {},
+      body_template,
+      timeout_ms = 10000,
+    } = destConfig;
+
+    const body = body_template
+      ? renderTemplate(body_template, { payload, headers: context.headers, context })
+      : JSON.stringify(payload);
+
+    const headers = { 'Content-Type': 'application/json' };
+
+    for (const [key, value] of Object.entries(configHeaders)) {
+      headers[key] = value.replace(/\$\{(\w+)\}/g, (_, envVar) => process.env[envVar] ?? '');
+    }
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout_ms);
+    const start = Date.now();
+
+    try {
+      const response = await fetch(url, {
+        method,
+        headers,
+        body,
+        signal: controller.signal,
+      });
+
+      const responseBody = await response.text();
+      const statusCode = response.status;
+
+      return {
+        success: statusCode >= 200 && statusCode < 300,
+        statusCode,
+        responseBody,
+        durationMs: Date.now() - start,
+      };
+    } catch (err) {
+      return {
+        success: false,
+        statusCode: 0,
+        error: err.name === 'AbortError' ? 'Request timeout' : err.message,
+        durationMs: Date.now() - start,
+      };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+}

package/src/destinations/index.js

@@ -0,0 +1,30 @@
+import { HttpDestinationAdapter } from './http.js';
+
+export class DestinationRegistry {
+  constructor() {
+    this._adapters = new Map();
+    this.register(new HttpDestinationAdapter());
+  }
+
+  register(adapter) {
+    this._adapters.set(adapter.type, adapter);
+  }
+
+  getAdapter(type) {
+    const adapter = this._adapters.get(type);
+    if (!adapter) {
+      throw new Error(`No destination adapter registered for type: "${type}"`);
+    }
+    return adapter;
+  }
+
+  getTypes() {
+    return [...this._adapters.keys()];
+  }
+
+  has(type) {
+    return this._adapters.has(type);
+  }
+}
+
+export const registry = new DestinationRegistry();

package/src/destinations/interface.js

@@ -0,0 +1,47 @@
+/**
+ * Base destination adapter class.
+ *
+ * All destination adapters (HTTP, SQS, etc.) must extend this class
+ * and implement the `type` getter and `send()` method.
+ *
+ * @abstract
+ */
+export class DestinationAdapter {
+  /**
+   * The adapter type identifier (e.g., 'http', 'sqs').
+   * Subclasses MUST override this getter.
+   *
+   * @returns {string}
+   */
+  get type() {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Send a payload to the destination.
+   *
+   * @param {object} payload - The transformed webhook payload.
+   * @param {object} destConfig - Destination configuration from pipeline YAML
+   *   (url, method, headers, body_template, timeout_ms, etc.).
+   * @param {object} context - Delivery context.
+   * @param {string} context.deliveryId - Unique delivery ID.
+   * @param {string} context.pipelineId - Pipeline identifier.
+   * @param {number} context.attempt - Current attempt number.
+   * @param {object} context.headers - Original webhook headers.
+   * @returns {Promise<{success: boolean, statusCode?: number, responseBody?: string, error?: string, durationMs: number}>}
+   */
+  async send(payload, destConfig, context) {
+    throw new Error('Not implemented');
+  }
+
+  /**
+   * Optional health check for the destination.
+   * Subclasses may override for active health probing.
+   *
+   * @param {object} destConfig - Destination configuration.
+   * @returns {Promise<{healthy: boolean, error?: string}>}
+   */
+  async healthCheck(destConfig) {
+    return { healthy: true };
+  }
+}

package/src/pipeline-loader.js

@@ -0,0 +1,151 @@
+import { readdir, readFile } from 'fs/promises';
+import path from 'path';
+import { EventEmitter } from 'events';
+import yaml from 'js-yaml';
+import chokidar from 'chokidar';
+
+const VALID_ID_RE = /^[a-z0-9-]+$/;
+const ENV_VAR_RE = /\$\{([^}]+)\}/g;
+const KNOWN_TYPES = new Set(['http']);
+
+function interpolateEnv(value) {
+  if (typeof value === 'string') {
+    return value.replace(ENV_VAR_RE, (_, varName) => process.env[varName] || '');
+  }
+  if (Array.isArray(value)) {
+    return value.map(interpolateEnv);
+  }
+  if (value && typeof value === 'object') {
+    const result = {};
+    for (const [k, v] of Object.entries(value)) {
+      result[k] = interpolateEnv(v);
+    }
+    return result;
+  }
+  return value;
+}
+
+function validate(config, filePath) {
+  if (!config || !config.id) {
+    throw new Error(`Pipeline id is required (file: ${filePath})`);
+  }
+  if (!VALID_ID_RE.test(config.id)) {
+    throw new Error(`Invalid id "${config.id}" - must be URL-safe (lowercase alphanumeric + hyphens only) (file: ${filePath})`);
+  }
+  if (!config.destinations || !Array.isArray(config.destinations) || config.destinations.length === 0) {
+    throw new Error(`Pipeline destinations is required and must be a non-empty array (file: ${filePath})`);
+  }
+  for (const dest of config.destinations) {
+    if (!dest.id) {
+      throw new Error(`Destination must have an id (file: ${filePath})`);
+    }
+    if (!dest.type) {
+      throw new Error(`Destination must have a type (file: ${filePath})`);
+    }
+    if (!KNOWN_TYPES.has(dest.type)) {
+      throw new Error(`Unknown destination type "${dest.type}" (file: ${filePath})`);
+    }
+    if (!dest.url) {
+      throw new Error(`Destination must have a url (file: ${filePath})`);
+    }
+  }
+}
+
+export class PipelineLoader extends EventEmitter {
+  #pipelines = new Map();
+  #pipelinesDir;
+  #watcher = null;
+  #fileToPipelineId = new Map();
+
+  constructor(pipelinesDir, opts = {}) {
+    super();
+    this.#pipelinesDir = pipelinesDir;
+  }
+
+  async loadAll() {
+    const files = await this.#scanYamlFiles(this.#pipelinesDir);
+    for (const file of files) {
+      await this.#loadFile(file);
+    }
+  }
+
+  get(pipelineId) {
+    return this.#pipelines.get(pipelineId);
+  }
+
+  getAll() {
+    return this.#pipelines;
+  }
+
+  startWatching() {
+    this.#watcher = chokidar.watch(this.#pipelinesDir, {
+      ignoreInitial: true,
+      awaitWriteFinish: { stabilityThreshold: 200, pollInterval: 50 },
+    });
+
+    this.#watcher.on('add', (filePath) => this.#handleFileChange(filePath));
+    this.#watcher.on('change', (filePath) => this.#handleFileChange(filePath));
+    this.#watcher.on('unlink', (filePath) => this.#handleFileRemove(filePath));
+
+    return new Promise((resolve) => {
+      this.#watcher.on('ready', resolve);
+    });
+  }
+
+  stopWatching() {
+    if (this.#watcher) {
+      this.#watcher.close();
+      this.#watcher = null;
+    }
+  }
+
+  async #scanYamlFiles(dir) {
+    const entries = await readdir(dir, { recursive: true, withFileTypes: true });
+    return entries
+      .filter((e) => e.isFile() && (e.name.endsWith('.yaml') || e.name.endsWith('.yml')))
+      .map((e) => path.join(e.parentPath || e.path, e.name));
+  }
+
+  async #loadFile(filePath) {
+    try {
+      const content = await readFile(filePath, 'utf-8');
+      const raw = yaml.load(content);
+      validate(raw, filePath);
+      const config = interpolateEnv(raw);
+      this.#pipelines.set(config.id, config);
+      this.#fileToPipelineId.set(filePath, config.id);
+      this.emit('loaded', config);
+    } catch (err) {
+      this.emit('error', err);
+    }
+  }
+
+  async #handleFileChange(filePath) {
+    if (!filePath.endsWith('.yaml') && !filePath.endsWith('.yml')) return;
+    try {
+      const content = await readFile(filePath, 'utf-8');
+      const raw = yaml.load(content);
+      validate(raw, filePath);
+      const config = interpolateEnv(raw);
+      // Remove old pipeline ID if the file previously had a different ID
+      const oldId = this.#fileToPipelineId.get(filePath);
+      if (oldId && oldId !== config.id) {
+        this.#pipelines.delete(oldId);
+      }
+      this.#pipelines.set(config.id, config);
+      this.#fileToPipelineId.set(filePath, config.id);
+      this.emit('reloaded', config);
+    } catch (err) {
+      this.emit('error', err);
+    }
+  }
+
+  #handleFileRemove(filePath) {
+    const pipelineId = this.#fileToPipelineId.get(filePath);
+    if (pipelineId) {
+      this.#pipelines.delete(pipelineId);
+      this.#fileToPipelineId.delete(filePath);
+      this.emit('removed', pipelineId);
+    }
+  }
+}

package/src/plugin-loader.js

@@ -0,0 +1,88 @@
+import { createRequire } from 'module';
+import path from 'path';
+import fs from 'fs';
+
+/**
+ * Loads CJS transform/filter plugins from a specified directory.
+ *
+ * Plugin API contracts:
+ * - Transform: `module.exports.transform = function(payload, headers, config) -> object|null`
+ * - Filter: `module.exports.filter = function(payload, headers, config) -> { pass: boolean, reason?: string }`
+ */
+export class PluginLoader {
+  #pluginsDir;
+  #cache = new Map();
+  #require;
+
+  /**
+   * @param {string} pluginsDir - Absolute path to the plugins directory
+   */
+  constructor(pluginsDir) {
+    this.#pluginsDir = pluginsDir;
+    this.#require = createRequire(import.meta.url);
+  }
+
+  /**
+   * Load a transform plugin. Returns the module's `transform` function.
+   * @param {string} relativePath - Path relative to pluginsDir
+   * @returns {function(object, object, object): object|null}
+   */
+  loadTransform(relativePath) {
+    const mod = this.#loadModule(relativePath);
+    if (typeof mod.transform !== 'function') {
+      throw new Error(
+        `Plugin "${relativePath}" does not export a transform function`
+      );
+    }
+    return mod.transform;
+  }
+
+  /**
+   * Load a filter plugin. Returns the module's `filter` function.
+   * @param {string} relativePath - Path relative to pluginsDir
+   * @returns {function(object, object, object): { pass: boolean, reason?: string }}
+   */
+  loadFilter(relativePath) {
+    const mod = this.#loadModule(relativePath);
+    if (typeof mod.filter !== 'function') {
+      throw new Error(
+        `Plugin "${relativePath}" does not export a filter function`
+      );
+    }
+    return mod.filter;
+  }
+
+  /** Clear the plugin cache (for hot-reload support). */
+  clearCache() {
+    for (const resolvedPath of this.#cache.keys()) {
+      delete this.#require.cache[resolvedPath];
+    }
+    this.#cache.clear();
+  }
+
+  /**
+   * Check if a plugin is cached.
+   * @param {string} relativePath - Path relative to pluginsDir
+   * @returns {boolean}
+   */
+  isLoaded(relativePath) {
+    const resolved = path.resolve(this.#pluginsDir, relativePath);
+    return this.#cache.has(resolved);
+  }
+
+  #loadModule(relativePath) {
+    const resolved = path.resolve(this.#pluginsDir, relativePath);
+
+    if (this.#cache.has(resolved)) {
+      return this.#cache.get(resolved);
+    }
+
+    if (!fs.existsSync(resolved)) {
+      throw new Error(`Plugin not found: ${resolved}`);
+    }
+
+    const mod = this.#require(resolved);
+    this.#cache.set(resolved, mod);
+    return mod;
+  }
+}

package/src/queue/interface.js

@@ -0,0 +1,68 @@
+/**
+ * Abstract base class defining the queue contract for hookpipe.
+ * All queue implementations (SQLite, PostgreSQL, etc.) must extend this class
+ * and implement every method.
+ */
+export class QueueInterface {
+  /**
+   * Add a job to the queue.
+   * @param {Object} job - The job to enqueue
+   * @param {string} job.deliveryId - Unique delivery identifier
+   * @param {string} job.destinationId - Target destination ID
+   * @param {string} job.pipelineId - Pipeline this job belongs to
+   * @param {Object} job.payload - The webhook payload body
+   * @param {Object} job.headers - Original request headers
+   * @param {number} job.maxAttempts - Maximum retry attempts allowed
+   * @returns {Promise<Object>} The created job record
+   */
+  async enqueue(job) {
+    throw new Error('QueueInterface.enqueue() not implemented');
+  }
+
+  /**
+   * Fetch up to `limit` ready jobs (status=pending, next_attempt_at <= now).
+   * @param {number} [limit=1] - Maximum number of jobs to dequeue
+   * @returns {Promise<Array<Object>>} Array of job objects ready for processing
+   */
+  async dequeue(limit = 1) {
+    throw new Error('QueueInterface.dequeue() not implemented');
+  }
+
+  /**
+   * Mark a job as completed successfully.
+   * @param {string} jobId - The job ID to acknowledge
+   * @returns {Promise<void>}
+   */
+  async ack(jobId) {
+    throw new Error('QueueInterface.ack() not implemented');
+  }
+
+  /**
+   * Mark a job as failed and schedule a retry.
+   * @param {string} jobId - The job ID that failed
+   * @param {Error} error - The error that occurred
+   * @param {Date} nextAttemptAt - When to retry the job
+   * @returns {Promise<void>}
+   */
+  async nack(jobId, error, nextAttemptAt) {
+    throw new Error('QueueInterface.nack() not implemented');
+  }
+
+  /**
+   * Move an exhausted job to the dead letter queue.
+   * @param {string} jobId - The job ID to move
+   * @param {Error} error - The final error
+   * @returns {Promise<void>}
+   */
+  async moveToDeadLetter(jobId, error) {
+    throw new Error('QueueInterface.moveToDeadLetter() not implemented');
+  }
+
+  /**
+   * Return queue statistics.
+   * @returns {Promise<{pending: number, processing: number, completed: number, failed: number, deadLetter: number}>}
+   */
+  async getStats() {
+    throw new Error('QueueInterface.getStats() not implemented');
+  }
+}