@azizikri/hookpipe 1.0.0

package/src/queue/sqlite-queue.js

@@ -0,0 +1,83 @@
+import { QueueInterface } from './interface.js';
+import { insertQueueJob, dequeueJobs, ackJob, nackJob, moveJobToDeadLetter, insertDeadLetter } from '../db/queries.js';
+import { ulid } from 'ulid';
+
+export class SqliteQueue extends QueueInterface {
+  #db;
+  #config;
+
+  constructor(db, config = {}) {
+    super();
+    this.#db = db;
+    this.#config = {
+      pollIntervalMs: config.pollIntervalMs ?? 1000,
+      concurrency: config.concurrency ?? 5,
+    };
+  }
+
+  async enqueue(job) {
+    const id = ulid();
+    insertQueueJob(this.#db, {
+      id,
+      deliveryId: job.deliveryId,
+      destinationId: job.destinationId,
+      pipelineId: job.pipelineId,
+      payload: job.payload,
+      headers: job.headers,
+      maxAttempts: job.maxAttempts,
+    });
+    return { id, ...job };
+  }
+
+  async dequeue(limit = 1) {
+    const lockId = ulid();
+    return dequeueJobs(this.#db, limit, lockId);
+  }
+
+  async ack(jobId) {
+    ackJob(this.#db, jobId);
+  }
+
+  async nack(jobId, error, nextAttemptAt) {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+    nackJob(this.#db, jobId, errorMessage, nextAttemptAt);
+  }
+
+  async moveToDeadLetter(jobId, error) {
+    const job = this.#db.prepare('SELECT * FROM queue_jobs WHERE id = ?').get(jobId);
+    if (!job) throw new Error(`Job not found: ${jobId}`);
+
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    insertDeadLetter(this.#db, {
+      id: ulid(),
+      deliveryId: job.delivery_id,
+      pipelineId: job.pipeline_id,
+      destinationId: job.destination_id,
+      payload: JSON.parse(job.payload),
+      headers: JSON.parse(job.headers),
+      errorMessage,
+      attempts: job.attempts,
+    });
+
+    moveJobToDeadLetter(this.#db, jobId);
+  }
+
+  async getStats() {
+    const rows = this.#db.prepare(`
+      SELECT status, COUNT(*) as count FROM queue_jobs GROUP BY status
+    `).all();
+
+    const stats = { pending: 0, processing: 0, completed: 0, failed: 0, deadLetter: 0 };
+    for (const row of rows) {
+      switch (row.status) {
+        case 'pending': stats.pending = row.count; break;
+        case 'processing': stats.processing = row.count; break;
+        case 'completed': stats.completed = row.count; break;
+        case 'failed': stats.failed = row.count; break;
+        case 'dead_letter': stats.deadLetter = row.count; break;
+      }
+    }
+    return stats;
+  }
+}

package/src/server.js

@@ -0,0 +1,124 @@
+import Fastify from 'fastify';
+import { ulid } from 'ulid';
+import { verifyWebhookAuth } from './auth/hmac.js';
+import { insertDelivery, incrementStat } from './db/queries.js';
+
+function getPluginPath(pluginConfig) {
+  return typeof pluginConfig === 'string' ? pluginConfig : pluginConfig.path;
+}
+
+function getPluginConfig(pluginConfig) {
+  return typeof pluginConfig === 'string' ? undefined : pluginConfig.config;
+}
+
+function getPipelineDestinations(pipeline) {
+  if (Array.isArray(pipeline.destinations)) {
+    return pipeline.destinations;
+  }
+  return pipeline.destination ? [pipeline.destination] : [];
+}
+
+function getMaxAttempts(pipeline) {
+  return pipeline.retry?.maxAttempts ?? pipeline.retry?.max_attempts ?? 3;
+}
+
+export function createServer(deps) {
+  const { pipelineLoader, pluginLoader, queue, db, logger } = deps;
+  const server = Fastify({ logger: false });
+
+  server.addContentTypeParser('application/json', { parseAs: 'buffer' }, (req, body, done) => {
+    req.rawBody = body;
+    try {
+      done(null, JSON.parse(body));
+    } catch (error) {
+      done(error);
+    }
+  });
+
+  server.get('/health', async () => ({
+    status: 'ok',
+    uptime: process.uptime(),
+    timestamp: new Date().toISOString(),
+  }));
+
+  server.post('/hook/:pipelineId', async (request, reply) => {
+    try {
+      const { pipelineId } = request.params;
+      const pipeline = await pipelineLoader.get(pipelineId);
+
+      if (!pipeline) {
+        return reply.code(404).send({ error: 'Pipeline not found' });
+      }
+
+      if (pipeline.auth) {
+        const authResult = verifyWebhookAuth(request.rawBody ?? request.raw.rawBody, request.headers, pipeline.auth);
+        if (!authResult.valid) {
+          return reply.code(401).send({
+            error: 'Authentication failed',
+            details: authResult.error,
+          });
+        }
+      }
+
+      let payload = request.body;
+
+      if (pipeline.filter) {
+        const filter = await pluginLoader.loadFilter(getPluginPath(pipeline.filter));
+        const filterResult = await filter(payload, request.headers, pipeline.filter_config || getPluginConfig(pipeline.filter) || {});
+        if (filterResult?.pass === false) {
+          incrementStat(db, pipeline.id ?? pipelineId, 'total_filtered');
+          return reply.code(200).send({
+            status: 'filtered',
+            reason: filterResult.reason,
+          });
+        }
+      }
+
+      if (pipeline.transform) {
+        const transform = await pluginLoader.loadTransform(getPluginPath(pipeline.transform));
+        const transformed = await transform(payload, request.headers, pipeline.filter_config || getPluginConfig(pipeline.transform) || {});
+        if (transformed === null) {
+          incrementStat(db, pipeline.id ?? pipelineId, 'total_filtered');
+          return reply.code(200).send({
+            status: 'filtered',
+            reason: 'Transform returned null',
+          });
+        }
+        payload = transformed;
+      }
+
+      const deliveryId = ulid();
+      const resolvedPipelineId = pipeline.id ?? pipelineId;
+      const headers = request.headers;
+
+      insertDelivery(db, {
+        id: deliveryId,
+        pipelineId: resolvedPipelineId,
+        payload,
+        headers,
+        sourceIp: request.ip,
+        status: 'queued',
+      });
+
+      for (const destination of getPipelineDestinations(pipeline)) {
+        await queue.enqueue({
+          deliveryId,
+          destinationId: destination.id,
+          pipelineId: resolvedPipelineId,
+          payload,
+          headers,
+          maxAttempts: getMaxAttempts(pipeline),
+        });
+      }
+
+      incrementStat(db, resolvedPipelineId, 'total_received');
+
+      return reply.code(200).send({ status: 'accepted', deliveryId });
+    } catch (error) {
+      logger?.error?.(error, 'Webhook ingestion failed');
+      return reply.code(500).send({ error: 'Internal server error' });
+    }
+  });
+
+  return server;
+}

package/src/templates/handlebars.js

@@ -0,0 +1,21 @@
+import Handlebars from 'handlebars';
+
+const templateCache = new Map();
+
+Handlebars.registerHelper('json', (context) => new Handlebars.SafeString(JSON.stringify(context)));
+Handlebars.registerHelper('upper', (str) => str.toUpperCase());
+Handlebars.registerHelper('lower', (str) => str.toLowerCase());
+Handlebars.registerHelper('default', (value, fallback) => value || fallback);
+
+export function renderTemplate(templateString, data) {
+  let compiledTemplate = templateCache.get(templateString);
+  if (!compiledTemplate) {
+    compiledTemplate = Handlebars.compile(templateString);
+    templateCache.set(templateString, compiledTemplate);
+  }
+  return compiledTemplate(data);
+}
+
+export function registerHelper(name, fn) {
+  Handlebars.registerHelper(name, fn);
+}

package/src/utils/config.js

@@ -0,0 +1,115 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import yaml from 'js-yaml';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const DEFAULTS = {
+  host: '0.0.0.0',
+  port: 3000,
+  db: { path: './data/hookpipe.db' },
+  log: { level: 'info' },
+  pipelines: { dir: './pipelines' },
+  plugins: { dir: './plugins' },
+  retry: { maxAttempts: 3, backoff: 'exponential', initialDelayMs: 1000, maxDelayMs: 300000 },
+  queue: { pollIntervalMs: 1000, concurrency: 5 },
+};
+
+const ENV_MAP = {
+  HOOKPIPE_PORT: 'port',
+  HOOKPIPE_HOST: 'host',
+  HOOKPIPE_DB_PATH: 'db.path',
+  HOOKPIPE_LOG_LEVEL: 'log.level',
+  HOOKPIPE_PIPELINES_DIR: 'pipelines.dir',
+  HOOKPIPE_PLUGINS_DIR: 'plugins.dir',
+};
+
+function deepMerge(target, source) {
+  const result = { ...target };
+  for (const key of Object.keys(source)) {
+    if (
+      source[key] !== null &&
+      typeof source[key] === 'object' &&
+      !Array.isArray(source[key]) &&
+      typeof target[key] === 'object' &&
+      target[key] !== null
+    ) {
+      result[key] = deepMerge(target[key], source[key]);
+    } else {
+      result[key] = source[key];
+    }
+  }
+  return result;
+}
+
+function interpolateEnv(obj) {
+  if (typeof obj === 'string') {
+    return obj.replace(/\$\{([^}]+)\}/g, (_, varName) => process.env[varName] ?? '');
+  }
+  if (obj !== null && typeof obj === 'object' && !Array.isArray(obj)) {
+    const result = {};
+    for (const [key, value] of Object.entries(obj)) {
+      result[key] = interpolateEnv(value);
+    }
+    return result;
+  }
+  if (Array.isArray(obj)) {
+    return obj.map(interpolateEnv);
+  }
+  return obj;
+}
+
+function setNested(obj, dotPath, value) {
+  const keys = dotPath.split('.');
+  const last = keys.pop();
+  let current = obj;
+  for (const key of keys) {
+    if (!(key in current) || typeof current[key] !== 'object') {
+      current[key] = {};
+    }
+    current = current[key];
+  }
+  current[last] = value;
+}
+
+function loadYaml(filePath) {
+  try {
+    const content = fs.readFileSync(filePath, 'utf8');
+    return yaml.load(content) || {};
+  } catch {
+    return {};
+  }
+}
+
+function resolveConfigPath(overridePath) {
+  if (overridePath) return overridePath;
+  if (process.env.HOOKPIPE_CONFIG) return process.env.HOOKPIPE_CONFIG;
+  return path.join(process.cwd(), 'hookpipe.config.yaml');
+}
+
+function applyEnv() {
+  const envConfig = {};
+  for (const [envVar, configPath] of Object.entries(ENV_MAP)) {
+    if (process.env[envVar] !== undefined) {
+      let value = process.env[envVar];
+      if (configPath === 'port') value = Number(value);
+      setNested(envConfig, configPath, value);
+    }
+  }
+  return envConfig;
+}
+
+export function loadConfig(overrides = {}) {
+  const { _configPath, configPath, ...cliOverrides } = overrides;
+
+  const configFilePath = resolveConfigPath(_configPath || configPath);
+  const fileConfig = interpolateEnv(loadYaml(configFilePath));
+  const envConfig = applyEnv();
+
+  let config = deepMerge(DEFAULTS, fileConfig);
+  config = deepMerge(config, envConfig);
+  config = deepMerge(config, cliOverrides);
+
+  return config;
+}

package/src/utils/crypto.js

@@ -0,0 +1,44 @@
+import { createHmac, timingSafeEqual } from 'node:crypto';
+
+/**
+ * Constant-time string comparison to prevent timing attacks.
+ */
+export function timingSafeCompare(a, b) {
+  const bufA = Buffer.from(a);
+  const bufB = Buffer.from(b);
+
+  if (bufA.length !== bufB.length) {
+    // Still do a comparison to avoid leaking length info via timing
+    timingSafeEqual(bufA, bufA);
+    return false;
+  }
+
+  return timingSafeEqual(bufA, bufB);
+}
+
+/**
+ * Compute HMAC hex digest for a given algorithm, secret, and payload.
+ */
+export function computeHmac(algorithm, secret, payload) {
+  return createHmac(algorithm, secret).update(payload).digest('hex');
+}
+
+/**
+ * Verify a webhook signature against a computed HMAC.
+ *
+ * @param {object} opts
+ * @param {'sha256'|'sha1'} opts.algorithm
+ * @param {string} opts.secret
+ * @param {string|Buffer} opts.payload
+ * @param {string} opts.signature - The signature header value
+ * @param {string} [opts.prefix] - Prefix to strip (e.g., 'sha256=')
+ * @returns {boolean}
+ */
+export function verifySignature({ algorithm, secret, payload, signature, prefix }) {
+  const expected = computeHmac(algorithm, secret, payload);
+  const actual = prefix && signature.startsWith(prefix)
+    ? signature.slice(prefix.length)
+    : signature;
+
+  return timingSafeCompare(expected, actual);
+}

package/src/utils/logger.js

@@ -0,0 +1,35 @@
+import pino from 'pino';
+
+const DEFAULT_REDACT_PATHS = ['*.secret', '*.password', '*.token'];
+
+/**
+ * Create a configured pino logger instance.
+ * @param {object} [opts] - Options
+ * @param {string} [opts.level] - Log level (defaults to HOOKPIPE_LOG_LEVEL env or 'info')
+ * @param {string[]} [opts.redact] - Paths to redact
+ * @param {object} [opts.stream] - Custom writable stream (for testing)
+ * @returns {import('pino').Logger}
+ */
+export function createLogger(opts = {}) {
+  const {
+    level = process.env.HOOKPIPE_LOG_LEVEL || 'info',
+    redact = DEFAULT_REDACT_PATHS,
+    stream,
+    ...rest
+  } = opts;
+
+  const config = {
+    level,
+    redact: {
+      paths: redact,
+      censor: '[Redacted]',
+    },
+    ...rest,
+  };
+
+  return stream ? pino(config, stream) : pino(config);
+}
+
+const logger = createLogger();
+
+export default logger;