@vaultgradient/pipequery-cli 0.6.1 → 0.7.0

package/__tests__/rest-source.test.ts

@@ -0,0 +1,191 @@
+/**
+ * REST source — env-var interpolation in url / headers / params,
+ * plus auth: bearer helper. Issue #37 (slice).
+ *
+ * Spins up a real http server in-process so we exercise the actual
+ * outbound request shape rather than mocking globalThis.fetch.
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { RestSourceAdapter } from '../src/server/sources/rest.js';
+
+interface Captured {
+  url: string;
+  headers: Record<string, string | string[] | undefined>;
+}
+
+interface Harness {
+  url: string;
+  captured: Captured[];
+  /** Body the server returns for the next request. Default: empty array. */
+  setResponse(body: unknown, status?: number): void;
+  close: () => Promise<void>;
+}
+
+async function makeServer(): Promise<Harness> {
+  const captured: Captured[] = [];
+  let nextBody: unknown = [];
+  let nextStatus = 200;
+
+  const server: Server = createServer((req: IncomingMessage, res: ServerResponse) => {
+    captured.push({ url: req.url ?? '', headers: req.headers });
+    res.statusCode = nextStatus;
+    res.setHeader('content-type', 'application/json');
+    res.end(JSON.stringify(nextBody));
+  });
+  await new Promise<void>((r) => server.listen(0, '127.0.0.1', r));
+  const port = (server.address() as AddressInfo).port;
+  return {
+    url: `http://127.0.0.1:${port}`,
+    captured,
+    setResponse(body, status = 200) {
+      nextBody = body;
+      nextStatus = status;
+    },
+    close: () => new Promise<void>((r) => server.close(() => r())),
+  };
+}
+
+describe('REST source — env interpolation + auth: bearer (#37 slice)', () => {
+  let server: Harness;
+  let envBackup: Record<string, string | undefined>;
+
+  beforeEach(async () => {
+    server = await makeServer();
+    envBackup = {
+      MY_TOKEN: process.env.MY_TOKEN,
+      MY_KEY: process.env.MY_KEY,
+      USER_PATH: process.env.USER_PATH,
+      MY_REGION: process.env.MY_REGION,
+    };
+  });
+  afterEach(async () => {
+    await server.close();
+    // Restore env
+    for (const [k, v] of Object.entries(envBackup)) {
+      if (v === undefined) delete process.env[k];
+      else process.env[k] = v;
+    }
+  });
+
+  it('expands ${ENV_VAR} in URL', async () => {
+    process.env.USER_PATH = 'users';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: `${server.url}/api/\${USER_PATH}`,
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].url).toBe('/api/users');
+  });
+
+  it('expands ${ENV_VAR} in headers', async () => {
+    process.env.MY_KEY = 'sk-secret-abc';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { 'X-API-Key': '${MY_KEY}', 'X-Static': 'literal' },
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].headers['x-api-key']).toBe('sk-secret-abc');
+    expect(server.captured[0].headers['x-static']).toBe('literal');
+  });
+
+  it('expands ${ENV_VAR} in query params and merges with existing query string', async () => {
+    process.env.MY_REGION = 'eu-west-1';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: `${server.url}/api?fixed=1`,
+      params: { region: '${MY_REGION}', kind: 'live' },
+    });
+    await adapter.start();
+    adapter.stop();
+    // Query string contains both the inline ?fixed=1 and the appended params,
+    // joined with `&`. URLSearchParams encodes dashes as-is.
+    const seen = server.captured[0].url;
+    expect(seen).toContain('fixed=1');
+    expect(seen).toContain('region=eu-west-1');
+    expect(seen).toContain('kind=live');
+    expect(seen.startsWith('/api?')).toBe(true);
+  });
+
+  it('emits empty string and a warning for missing env vars (does not embed literal "${FOO}")', async () => {
+    // No MY_TOKEN set
+    delete process.env.MY_TOKEN;
+    server.setResponse([]);
+    const stderrCaptured: string[] = [];
+    const origWrite = process.stderr.write.bind(process.stderr);
+    process.stderr.write = ((chunk: string | Uint8Array) => {
+      stderrCaptured.push(typeof chunk === 'string' ? chunk : chunk.toString());
+      return true;
+    }) as typeof process.stderr.write;
+
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { Authorization: 'Bearer ${MY_TOKEN}' },
+    });
+    try {
+      await adapter.start();
+      adapter.stop();
+    } finally {
+      process.stderr.write = origWrite;
+    }
+
+    // The literal "${...}" must NOT appear in the outbound header. Node
+    // trims trailing whitespace from header values on the wire, so the
+    // received value is "Bearer" not "Bearer ".
+    const auth = server.captured[0].headers['authorization'] as string;
+    expect(auth).not.toContain('${');
+    expect(auth.startsWith('Bearer')).toBe(true);
+    expect(stderrCaptured.some((s) => s.includes('MY_TOKEN'))).toBe(true);
+  });
+
+  it('auth: bearer adds Authorization: Bearer <token> with env interpolation', async () => {
+    process.env.MY_TOKEN = 'jwt.abc.123';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      auth: { kind: 'bearer', token: '${MY_TOKEN}' },
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].headers['authorization']).toBe('Bearer jwt.abc.123');
+  });
+
+  it('auth: bearer wins over a hand-rolled headers.Authorization on collision', async () => {
+    // If a user sets both, they almost certainly meant the auth helper —
+    // bearer is the one that actually plumbs the token through env vars.
+    process.env.MY_TOKEN = 'jwt.xyz';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { Authorization: 'Bearer should-be-overridden' },
+      auth: { kind: 'bearer', token: '${MY_TOKEN}' },
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].headers['authorization']).toBe('Bearer jwt.xyz');
+  });
+
+  it('back-compat: no env vars + no auth → behaves exactly like before', async () => {
+    server.setResponse([{ id: 1 }, { id: 2 }]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { 'X-Custom': 'value' },
+    });
+    await adapter.start();
+    expect(adapter.getData()).toEqual([{ id: 1 }, { id: 2 }]);
+    expect(server.captured[0].headers['x-custom']).toBe('value');
+    expect(server.captured[0].headers['authorization']).toBeUndefined();
+    adapter.stop();
+  });
+});

package/__tests__/websocket-source.test.ts

@@ -0,0 +1,187 @@
+/**
+ * WebSocket source tests — covers the new subscribe-on-open + heartbeat
+ * behaviour from issue #35. Spins up a real `ws` server in-process so
+ * we exercise the actual JSON round-trip rather than mocking the socket.
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { WebSocketServer, type WebSocket as WSocket } from 'ws';
+import type { AddressInfo } from 'node:net';
+import { WebSocketSourceAdapter } from '../src/server/sources/websocket.js';
+
+interface Harness {
+  url: string;
+  /** Messages received by the server, parsed as JSON when possible. */
+  received: unknown[];
+  /** First connected client — used to push messages back at the adapter. */
+  clientPromise: Promise<WSocket>;
+  /** Force-close the active client connection (simulates a server restart). */
+  forceCloseClient: () => void;
+  close: () => Promise<void>;
+}
+
+async function makeServer(): Promise<Harness> {
+  const wss = new WebSocketServer({ port: 0 });
+  const received: unknown[] = [];
+  let resolveClient!: (ws: WSocket) => void;
+  const clientPromise = new Promise<WSocket>((res) => {
+    resolveClient = res;
+  });
+  let activeClient: WSocket | null = null;
+
+  wss.on('connection', (ws) => {
+    activeClient = ws;
+    ws.on('message', (raw) => {
+      try {
+        received.push(JSON.parse(raw.toString()));
+      } catch {
+        received.push(raw.toString());
+      }
+    });
+    resolveClient(ws);
+  });
+
+  await new Promise<void>((res) => wss.once('listening', res));
+  const port = (wss.address() as AddressInfo).port;
+  return {
+    url: `ws://127.0.0.1:${port}`,
+    received,
+    clientPromise,
+    forceCloseClient: () => activeClient?.close(),
+    close: () =>
+      new Promise<void>((res) => {
+        for (const c of wss.clients) c.terminate();
+        wss.close(() => res());
+      }),
+  };
+}
+
+describe('WebSocketSourceAdapter — subscribe + heartbeat', () => {
+  let server: Harness;
+  beforeEach(async () => {
+    server = await makeServer();
+  });
+  afterEach(async () => {
+    await server.close();
+  });
+
+  it('sends a single subscribe payload immediately after open', async () => {
+    const adapter = new WebSocketSourceAdapter({
+      type: 'websocket',
+      url: server.url,
+      subscribe: { method: 'SUBSCRIBE', params: ['btcusdt@ticker'], id: 1 },
+    });
+    await adapter.start();
+    await server.clientPromise;
+    // Give the server a tick to flush the message into `received`.
+    await new Promise((r) => setTimeout(r, 50));
+    expect(server.received).toEqual([
+      { method: 'SUBSCRIBE', params: ['btcusdt@ticker'], id: 1 },
+    ]);
+    adapter.stop();
+  });
+
+  it('sends an array of subscribe payloads in order', async () => {
+    const adapter = new WebSocketSourceAdapter({
+      type: 'websocket',
+      url: server.url,
+      subscribe: [
+        { method: 'SUBSCRIBE', params: ['btc@ticker'], id: 1 },
+        { method: 'SUBSCRIBE', params: ['eth@ticker'], id: 2 },
+        { method: 'SUBSCRIBE', params: ['sol@ticker'], id: 3 },
+      ],
+    });
+    await adapter.start();
+    await server.clientPromise;
+    await new Promise((r) => setTimeout(r, 50));
+    expect(server.received).toHaveLength(3);
+    expect((server.received[0] as { id: number }).id).toBe(1);
+    expect((server.received[1] as { id: number }).id).toBe(2);
+    expect((server.received[2] as { id: number }).id).toBe(3);
+    adapter.stop();
+  });
+
+  it('sends nothing when subscribe is omitted (back-compat)', async () => {
+    const adapter = new WebSocketSourceAdapter({
+      type: 'websocket',
+      url: server.url,
+    });
+    await adapter.start();
+    await server.clientPromise;
+    await new Promise((r) => setTimeout(r, 50));
+    expect(server.received).toEqual([]);
+    adapter.stop();
+  });
+
+  it('emits heartbeat payloads on the configured second-resolution interval', async () => {
+    const adapter = new WebSocketSourceAdapter({
+      type: 'websocket',
+      url: server.url,
+      heartbeat: { payload: { method: 'PING' }, interval: '1s' },
+    });
+    await adapter.start();
+    await server.clientPromise;
+    // Wait long enough for at least 2 ticks plus the open-time slack.
+    await new Promise((r) => setTimeout(r, 2300));
+    const pings = server.received.filter(
+      (m) => typeof m === 'object' && m !== null && (m as { method?: string }).method === 'PING',
+    );
+    expect(pings.length).toBeGreaterThanOrEqual(2);
+    adapter.stop();
+  }, 5000);
+
+  it('re-sends subscribe payloads on reconnect', async () => {
+    const adapter = new WebSocketSourceAdapter({
+      type: 'websocket',
+      url: server.url,
+      subscribe: { method: 'SUBSCRIBE', params: ['btc@ticker'], id: 1 },
+    });
+    await adapter.start();
+    await server.clientPromise;
+    await new Promise((r) => setTimeout(r, 50));
+    expect(server.received).toHaveLength(1);
+
+    // Server force-closes the client; adapter's internal reconnect (5s
+    // delay) should fire and re-run start(), which re-sends subscribe.
+    server.forceCloseClient();
+    await new Promise((r) => setTimeout(r, 5500));
+    expect(server.received.length).toBeGreaterThanOrEqual(2);
+    expect((server.received[server.received.length - 1] as { method: string }).method).toBe(
+      'SUBSCRIBE',
+    );
+    adapter.stop();
+  }, 10_000);
+
+  it('stops heartbeat timer and does not reconnect after stop()', async () => {
+    const adapter = new WebSocketSourceAdapter({
+      type: 'websocket',
+      url: server.url,
+      subscribe: { method: 'SUBSCRIBE', params: ['x'], id: 1 },
+      heartbeat: { payload: { method: 'PING' }, interval: '1s' },
+    });
+    await adapter.start();
+    await server.clientPromise;
+    await new Promise((r) => setTimeout(r, 100));
+    const beforeStop = server.received.length;
+
+    adapter.stop();
+    // Wait through one would-be reconnect window + a few heartbeat ticks.
+    await new Promise((r) => setTimeout(r, 6000));
+    // No new SUBSCRIBE re-sent (would indicate reconnect happened) and
+    // no new PING ticks (heartbeat would have kept firing).
+    expect(server.received.length).toBe(beforeStop);
+  }, 10_000);
+
+  it('still ingests messages from the server (no regression to base behaviour)', async () => {
+    const adapter = new WebSocketSourceAdapter({
+      type: 'websocket',
+      url: server.url,
+    });
+    await adapter.start();
+    const client = await server.clientPromise;
+    client.send(JSON.stringify({ price: 100 }));
+    client.send(JSON.stringify({ price: 101 }));
+    await new Promise((r) => setTimeout(r, 50));
+    expect(adapter.getData()).toEqual([{ price: 100 }, { price: 101 }]);
+    adapter.stop();
+  });
+});

package/dist/index.js

@@ -149,6 +149,21 @@ function parseDuration(input) {
   return parseFloat(match[1]) * UNITS[match[2]];
 }
 
+// src/utils/expandEnv.ts
+function expandEnv(input, context) {
+  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
+    const value = process.env[name];
+    if (value === void 0) {
+      process.stderr.write(
+        `[pipequery] env var "${name}" referenced in ${context} is not set
+`
+      );
+      return "";
+    }
+    return value;
+  });
+}
+
 // src/server/sources/rest.ts
 function extractPath(obj, path) {
   let current = obj;
@@ -199,14 +214,17 @@ var RestSourceAdapter = class {
   }
   async fetch() {
     try {
-      let url = this.config.url;
+      let url = expandEnv(this.config.url, "rest source URL");
       if (this.config.params) {
-        const params = new URLSearchParams(this.config.params);
+        const expanded = {};
+        for (const [k, v] of Object.entries(this.config.params)) {
+          expanded[k] = expandEnv(v, `rest source param "${k}"`);
+        }
+        const params = new URLSearchParams(expanded);
         url += (url.includes("?") ? "&" : "?") + params.toString();
       }
-      const res = await globalThis.fetch(url, {
-        headers: this.config.headers
-      });
+      const headers = this.buildHeaders();
+      const res = await globalThis.fetch(url, { headers });
       if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
       const json = await res.json();
       this.data = this.config.dataPath ? extractPath(json, this.config.dataPath) : Array.isArray(json) ? json : [json];
@@ -220,6 +238,28 @@ var RestSourceAdapter = class {
   notify() {
     for (const cb of this.listeners) cb(this.data);
   }
+  /**
+   * Build the final HTTP headers for one fetch.
+   *
+   * Two sources contribute, in order: any user-declared `headers` (with
+   * env-var interpolation applied per value), then the `auth` helper
+   * (currently just `bearer`). Auth wins on collision — if a user sets
+   * both `headers.Authorization` and `auth.kind === 'bearer'`, the bearer
+   * helper is what they actually meant.
+   */
+  buildHeaders() {
+    const headers = {};
+    if (this.config.headers) {
+      for (const [k, v] of Object.entries(this.config.headers)) {
+        headers[k] = expandEnv(v, `rest source header "${k}"`);
+      }
+    }
+    if (this.config.auth?.kind === "bearer") {
+      const token = expandEnv(this.config.auth.token, "rest source bearer token");
+      headers["Authorization"] = `Bearer ${token}`;
+    }
+    return Object.keys(headers).length > 0 ? headers : void 0;
+  }
 };
 
 // src/server/sources/websocket.ts
@@ -236,14 +276,35 @@ var WebSocketSourceAdapter = class {
   error;
   listeners = /* @__PURE__ */ new Set();
   maxBuffer;
+  heartbeatTimer = null;
+  /** Tracks whether stop() was called so reconnect respects intent. */
+  stopped = false;
   async start() {
+    this.stopped = false;
     return new Promise((resolve10, reject) => {
-      this.ws = new WebSocket(this.config.url);
-      this.ws.on("open", () => {
+      const ws = new WebSocket(this.config.url);
+      this.ws = ws;
+      let settled = false;
+      ws.on("open", () => {
         this.error = void 0;
-        resolve10();
+        try {
+          this.sendSubscribePayloads(ws);
+          this.startHeartbeat(ws);
+        } catch (err) {
+          this.error = err instanceof Error ? err.message : String(err);
+          ws.close();
+          if (!settled) {
+            settled = true;
+            reject(err instanceof Error ? err : new Error(this.error));
+          }
+          return;
+        }
+        if (!settled) {
+          settled = true;
+          resolve10();
+        }
       });
-      this.ws.on("message", (raw) => {
+      ws.on("message", (raw) => {
         try {
           const parsed = JSON.parse(raw.toString());
           if (Array.isArray(parsed)) {
@@ -260,19 +321,26 @@ var WebSocketSourceAdapter = class {
         } catch {
         }
       });
-      this.ws.on("error", (err) => {
+      ws.on("error", (err) => {
         this.error = err.message;
-        reject(err);
+        if (!settled) {
+          settled = true;
+          reject(err);
+        }
       });
-      this.ws.on("close", () => {
+      ws.on("close", () => {
+        this.stopHeartbeat();
+        if (this.stopped) return;
         setTimeout(() => {
-          if (this.ws) this.start().catch(() => {
+          if (!this.stopped) this.start().catch(() => {
           });
         }, 5e3);
       });
     });
   }
   stop() {
+    this.stopped = true;
+    this.stopHeartbeat();
     const ws = this.ws;
     this.ws = null;
     ws?.close();
@@ -295,6 +363,33 @@ var WebSocketSourceAdapter = class {
   notify() {
     for (const cb of this.listeners) cb(this.data);
   }
+  sendSubscribePayloads(ws) {
+    const subscribe = this.config.subscribe;
+    if (subscribe === void 0 || subscribe === null) return;
+    const list = Array.isArray(subscribe) ? subscribe : [subscribe];
+    for (const payload of list) {
+      ws.send(JSON.stringify(payload));
+    }
+  }
+  startHeartbeat(ws) {
+    this.stopHeartbeat();
+    const hb = this.config.heartbeat;
+    if (!hb) return;
+    const intervalMs = parseDuration(hb.interval);
+    this.heartbeatTimer = setInterval(() => {
+      if (ws.readyState !== WebSocket.OPEN) return;
+      try {
+        ws.send(JSON.stringify(hb.payload));
+      } catch {
+      }
+    }, intervalMs);
+  }
+  stopHeartbeat() {
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+  }
 };
 
 // src/server/sources/file.ts
@@ -399,19 +494,6 @@ var StaticSourceAdapter = class {
 import pg from "pg";
 var DEFAULT_INTERVAL_MS = 3e4;
 var DEFAULT_MAX_ROWS = 1e4;
-function expandEnv(input) {
-  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
-    const value = process.env[name];
-    if (value === void 0) {
-      process.stderr.write(
-        `[pipequery] env var "${name}" referenced in postgres source URL is not set
-`
-      );
-      return "";
-    }
-    return value;
-  });
-}
 function resolveSsl(ssl) {
   if (ssl === false) return false;
   if (ssl === "no-verify") return { rejectUnauthorized: false };
@@ -433,7 +515,7 @@ var PostgresSourceAdapter = class {
   intervalMs;
   maxRows;
   async start() {
-    const connectionString = expandEnv(this.config.url);
+    const connectionString = expandEnv(this.config.url, "postgres source URL");
     this.pool = new pg.Pool({
       connectionString,
       ssl: resolveSsl(this.config.ssl),
@@ -511,19 +593,6 @@ function stripTrailingSemicolon(sql) {
 import mysql from "mysql2/promise";
 var DEFAULT_INTERVAL_MS2 = 3e4;
 var DEFAULT_MAX_ROWS2 = 1e4;
-function expandEnv2(input) {
-  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
-    const value = process.env[name];
-    if (value === void 0) {
-      process.stderr.write(
-        `[pipequery] env var "${name}" referenced in mysql source URL is not set
-`
-      );
-      return "";
-    }
-    return value;
-  });
-}
 function resolveSsl2(ssl) {
   if (ssl === false) return void 0;
   if (ssl === "no-verify") return { rejectUnauthorized: false };
@@ -545,7 +614,7 @@ var MysqlSourceAdapter = class {
   intervalMs;
   maxRows;
   async start() {
-    const uri = expandEnv2(this.config.url);
+    const uri = expandEnv(this.config.url, "mysql source URL");
     this.pool = mysql.createPool({
       uri,
       ssl: resolveSsl2(this.config.ssl),
@@ -716,22 +785,9 @@ function stripTrailingSemicolon3(sql) {
 import { randomBytes } from "crypto";
 import { Kafka, logLevel } from "kafkajs";
 var DEFAULT_MAX_BUFFER = 1e3;
-function expandEnv3(input) {
-  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => {
-    const value = process.env[name];
-    if (value === void 0) {
-      process.stderr.write(
-        `[pipequery] env var "${name}" referenced in kafka source is not set
-`
-      );
-      return "";
-    }
-    return value;
-  });
-}
 function normalizeBrokers(brokers) {
   const list = Array.isArray(brokers) ? brokers : brokers.split(",");
-  return list.map((b) => expandEnv3(b.trim())).filter(Boolean);
+  return list.map((b) => expandEnv(b.trim(), "kafka source brokers")).filter(Boolean);
 }
 var KafkaSourceAdapter = class {
   constructor(config) {
@@ -760,8 +816,8 @@ var KafkaSourceAdapter = class {
       ssl: this.config.ssl ?? false,
       sasl: this.config.sasl ? {
         mechanism: this.config.sasl.mechanism,
-        username: expandEnv3(this.config.sasl.username),
-        password: expandEnv3(this.config.sasl.password)
+        username: expandEnv(this.config.sasl.username, "kafka source SASL username"),
+        password: expandEnv(this.config.sasl.password, "kafka source SASL password")
       } : void 0,
       // Silence kafkajs's own logger — we surface errors via SourceStatus.
       logLevel: logLevel.ERROR
@@ -2900,7 +2956,7 @@ function escapeHtml(s) {
 
 // src/telegram/notifier.ts
 var botCache = /* @__PURE__ */ new Map();
-function expandEnv4(input) {
+function expandEnv2(input) {
   return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name) => process.env[name] ?? "");
 }
 function getBot(token) {
@@ -2922,7 +2978,7 @@ var TelegramNotifier = class {
         "TelegramNotifier: no bot token. Set notify.telegram.botToken or PIPEQUERY_TG_BOT_TOKEN."
       );
     }
-    this.token = expandEnv4(tokenSource);
+    this.token = expandEnv2(tokenSource);
     this.chatId = cfg.chatId;
     this.template = cfg.message;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vaultgradient/pipequery-cli",
-  "version": "0.6.1",
+  "version": "0.7.0",
   "description": "CLI for PipeQuery — data sources, aggregations, APIs, and terminal dashboards",
   "type": "module",
   "bin": {

package/src/config/schema.ts

@@ -35,12 +35,34 @@ export interface RestSourceConfig {
   headers?: Record<string, string>;
   dataPath?: string;
   interval?: string;
+  /** Optional auth helper. Currently supports `bearer`; more kinds (basic,
+   *  apiKeyHeader, hmac-*) tracked in issue #37. The token is interpolated
+   *  through ${ENV_VAR} like every other string field. */
+  auth?: BearerAuth;
+}
+
+export interface BearerAuth {
+  kind: 'bearer';
+  token: string;
 }
 
 export interface WebSocketSourceConfig {
   type: 'websocket';
   url: string;
   maxBuffer?: number;
+  /** Payload(s) to send to the server immediately after the WebSocket
+   *  opens. Most exchange feeds (Binance, Coinbase, Kraken, OKX, Bybit, …)
+   *  require this — without it, no data is delivered. Each payload is
+   *  JSON-stringified and sent in order. Re-sent on every reconnect. */
+  subscribe?: unknown | unknown[];
+  /** Optional keepalive: sends `payload` to the server every `interval`
+   *  while the socket is open. Most exchanges close idle sockets within
+   *  a few minutes; a periodic ping keeps the stream alive. */
+  heartbeat?: {
+    payload: unknown;
+    /** Duration like "30s", "1m". */
+    interval: string;
+  };
 }
 
 export interface FileSourceConfig {

package/src/server/sources/kafka.ts

@@ -1,26 +1,14 @@
 import { randomBytes } from 'node:crypto';
 import { Kafka, logLevel, type Consumer, type SASLOptions } from 'kafkajs';
 import type { KafkaSourceConfig } from '../../config/schema.js';
+import { expandEnv } from '../../utils/expandEnv.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
 
 const DEFAULT_MAX_BUFFER = 1000;
 
-function expandEnv(input: string): string {
-  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
-    const value = process.env[name];
-    if (value === undefined) {
-      process.stderr.write(
-        `[pipequery] env var "${name}" referenced in kafka source is not set\n`,
-      );
-      return '';
-    }
-    return value;
-  });
-}
-
 function normalizeBrokers(brokers: string | string[]): string[] {
   const list = Array.isArray(brokers) ? brokers : brokers.split(',');
-  return list.map((b) => expandEnv(b.trim())).filter(Boolean);
+  return list.map((b) => expandEnv(b.trim(), 'kafka source brokers')).filter(Boolean);
 }
 
 export class KafkaSourceAdapter implements SourceAdapter {
@@ -55,8 +43,8 @@ export class KafkaSourceAdapter implements SourceAdapter {
       sasl: this.config.sasl
         ? ({
             mechanism: this.config.sasl.mechanism,
-            username: expandEnv(this.config.sasl.username),
-            password: expandEnv(this.config.sasl.password),
+            username: expandEnv(this.config.sasl.username, 'kafka source SASL username'),
+            password: expandEnv(this.config.sasl.password, 'kafka source SASL password'),
           } as SASLOptions)
         : undefined,
       // Silence kafkajs's own logger — we surface errors via SourceStatus.

package/src/server/sources/mysql.ts

@@ -1,24 +1,12 @@
 import mysql from 'mysql2/promise';
 import type { MysqlSourceConfig } from '../../config/schema.js';
 import { parseDuration } from '../../utils/parseDuration.js';
+import { expandEnv } from '../../utils/expandEnv.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
 
 const DEFAULT_INTERVAL_MS = 30_000;
 const DEFAULT_MAX_ROWS = 10_000;
 
-function expandEnv(input: string): string {
-  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
-    const value = process.env[name];
-    if (value === undefined) {
-      process.stderr.write(
-        `[pipequery] env var "${name}" referenced in mysql source URL is not set\n`,
-      );
-      return '';
-    }
-    return value;
-  });
-}
-
 function resolveSsl(ssl: MysqlSourceConfig['ssl']): mysql.PoolOptions['ssl'] {
   if (ssl === false) return undefined;
   if (ssl === 'no-verify') return { rejectUnauthorized: false };
@@ -41,7 +29,7 @@ export class MysqlSourceAdapter implements SourceAdapter {
   }
 
   async start(): Promise<void> {
-    const uri = expandEnv(this.config.url);
+    const uri = expandEnv(this.config.url, 'mysql source URL');
     this.pool = mysql.createPool({
       uri,
       ssl: resolveSsl(this.config.ssl),

package/src/server/sources/postgres.ts

@@ -1,30 +1,12 @@
 import pg from 'pg';
 import type { PostgresSourceConfig } from '../../config/schema.js';
 import { parseDuration } from '../../utils/parseDuration.js';
+import { expandEnv } from '../../utils/expandEnv.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
 
 const DEFAULT_INTERVAL_MS = 30_000;
 const DEFAULT_MAX_ROWS = 10_000;
 
-/**
- * Resolve ${ENV_VAR} placeholders in a string from process.env.
- * Missing vars expand to empty string (with a stderr warning) so a
- * mis-typed env doesn't silently embed literal "${FOO}" into a
- * Postgres connection string.
- */
-function expandEnv(input: string): string {
-  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
-    const value = process.env[name];
-    if (value === undefined) {
-      process.stderr.write(
-        `[pipequery] env var "${name}" referenced in postgres source URL is not set\n`,
-      );
-      return '';
-    }
-    return value;
-  });
-}
-
 function resolveSsl(ssl: PostgresSourceConfig['ssl']): pg.ClientConfig['ssl'] {
   if (ssl === false) return false;
   if (ssl === 'no-verify') return { rejectUnauthorized: false };
@@ -48,7 +30,7 @@ export class PostgresSourceAdapter implements SourceAdapter {
   }
 
   async start(): Promise<void> {
-    const connectionString = expandEnv(this.config.url);
+    const connectionString = expandEnv(this.config.url, 'postgres source URL');
     this.pool = new pg.Pool({
       connectionString,
       ssl: resolveSsl(this.config.ssl),

package/src/server/sources/rest.ts

@@ -1,5 +1,6 @@
 import type { RestSourceConfig } from '../../config/schema.js';
 import { parseDuration } from '../../utils/parseDuration.js';
+import { expandEnv } from '../../utils/expandEnv.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
 
 function extractPath(obj: unknown, path: string): unknown[] {
@@ -57,15 +58,19 @@ export class RestSourceAdapter implements SourceAdapter {
 
   private async fetch(): Promise<void> {
     try {
-      let url = this.config.url;
+      let url = expandEnv(this.config.url, 'rest source URL');
       if (this.config.params) {
-        const params = new URLSearchParams(this.config.params);
+        const expanded: Record<string, string> = {};
+        for (const [k, v] of Object.entries(this.config.params)) {
+          expanded[k] = expandEnv(v, `rest source param "${k}"`);
+        }
+        const params = new URLSearchParams(expanded);
         url += (url.includes('?') ? '&' : '?') + params.toString();
       }
 
-      const res = await globalThis.fetch(url, {
-        headers: this.config.headers,
-      });
+      const headers = this.buildHeaders();
+
+      const res = await globalThis.fetch(url, { headers });
 
       if (!res.ok) throw new Error(`HTTP ${res.status}: ${res.statusText}`);
 
@@ -82,4 +87,27 @@ export class RestSourceAdapter implements SourceAdapter {
   private notify(): void {
     for (const cb of this.listeners) cb(this.data);
   }
+
+  /**
+   * Build the final HTTP headers for one fetch.
+   *
+   * Two sources contribute, in order: any user-declared `headers` (with
+   * env-var interpolation applied per value), then the `auth` helper
+   * (currently just `bearer`). Auth wins on collision — if a user sets
+   * both `headers.Authorization` and `auth.kind === 'bearer'`, the bearer
+   * helper is what they actually meant.
+   */
+  private buildHeaders(): Record<string, string> | undefined {
+    const headers: Record<string, string> = {};
+    if (this.config.headers) {
+      for (const [k, v] of Object.entries(this.config.headers)) {
+        headers[k] = expandEnv(v, `rest source header "${k}"`);
+      }
+    }
+    if (this.config.auth?.kind === 'bearer') {
+      const token = expandEnv(this.config.auth.token, 'rest source bearer token');
+      headers['Authorization'] = `Bearer ${token}`;
+    }
+    return Object.keys(headers).length > 0 ? headers : undefined;
+  }
 }

package/src/server/sources/websocket.ts

@@ -1,6 +1,7 @@
 import WebSocket from 'ws';
 import type { WebSocketSourceConfig } from '../../config/schema.js';
 import type { SourceAdapter, SourceStatus } from './types.js';
+import { parseDuration } from '../../utils/parseDuration.js';
 
 export class WebSocketSourceAdapter implements SourceAdapter {
   private data: unknown[] = [];
@@ -9,21 +10,44 @@ export class WebSocketSourceAdapter implements SourceAdapter {
   private error?: string;
   private listeners = new Set<(data: unknown[]) => void>();
   private maxBuffer: number;
+  private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
+  /** Tracks whether stop() was called so reconnect respects intent. */
+  private stopped = false;
 
   constructor(private config: WebSocketSourceConfig) {
     this.maxBuffer = config.maxBuffer ?? 1000;
   }
 
   async start(): Promise<void> {
+    this.stopped = false;
     return new Promise((resolve, reject) => {
-      this.ws = new WebSocket(this.config.url);
+      const ws = new WebSocket(this.config.url);
+      this.ws = ws;
+      let settled = false;
 
-      this.ws.on('open', () => {
+      ws.on('open', () => {
         this.error = undefined;
-        resolve();
+        try {
+          this.sendSubscribePayloads(ws);
+          this.startHeartbeat(ws);
+        } catch (err) {
+          // A failure to send the subscribe frame leaves the socket open
+          // but useless; record it and reject the start() promise.
+          this.error = err instanceof Error ? err.message : String(err);
+          ws.close();
+          if (!settled) {
+            settled = true;
+            reject(err instanceof Error ? err : new Error(this.error));
+          }
+          return;
+        }
+        if (!settled) {
+          settled = true;
+          resolve();
+        }
       });
 
-      this.ws.on('message', (raw) => {
+      ws.on('message', (raw) => {
         try {
           const parsed = JSON.parse(raw.toString());
           if (Array.isArray(parsed)) {
@@ -42,21 +66,30 @@ export class WebSocketSourceAdapter implements SourceAdapter {
         }
       });
 
-      this.ws.on('error', (err) => {
+      ws.on('error', (err) => {
         this.error = err.message;
-        reject(err);
+        if (!settled) {
+          settled = true;
+          reject(err);
+        }
       });
 
-      this.ws.on('close', () => {
-        // Auto-reconnect after 5s
+      ws.on('close', () => {
+        this.stopHeartbeat();
+        // Auto-reconnect after 5s unless stop() was explicitly called.
+        // Re-running start() also re-sends the subscribe payloads, which
+        // is exactly what every exchange expects after a connection drop.
+        if (this.stopped) return;
         setTimeout(() => {
-          if (this.ws) this.start().catch(() => {});
+          if (!this.stopped) this.start().catch(() => {});
         }, 5000);
       });
     });
   }
 
   stop(): void {
+    this.stopped = true;
+    this.stopHeartbeat();
     const ws = this.ws;
     this.ws = null;
     ws?.close();
@@ -83,4 +116,35 @@ export class WebSocketSourceAdapter implements SourceAdapter {
   private notify(): void {
     for (const cb of this.listeners) cb(this.data);
   }
+
+  private sendSubscribePayloads(ws: WebSocket): void {
+    const subscribe = this.config.subscribe;
+    if (subscribe === undefined || subscribe === null) return;
+    const list = Array.isArray(subscribe) ? subscribe : [subscribe];
+    for (const payload of list) {
+      ws.send(JSON.stringify(payload));
+    }
+  }
+
+  private startHeartbeat(ws: WebSocket): void {
+    this.stopHeartbeat();
+    const hb = this.config.heartbeat;
+    if (!hb) return;
+    const intervalMs = parseDuration(hb.interval);
+    this.heartbeatTimer = setInterval(() => {
+      if (ws.readyState !== WebSocket.OPEN) return;
+      try {
+        ws.send(JSON.stringify(hb.payload));
+      } catch {
+        // Ignore — next 'close' / 'error' will surface anything fatal.
+      }
+    }, intervalMs);
+  }
+
+  private stopHeartbeat(): void {
+    if (this.heartbeatTimer) {
+      clearInterval(this.heartbeatTimer);
+      this.heartbeatTimer = null;
+    }
+  }
 }

package/src/utils/expandEnv.ts

@@ -0,0 +1,23 @@
+/**
+ * Resolve `${ENV_VAR}` placeholders in a string from process.env.
+ *
+ * Missing vars expand to empty string and emit a stderr warning so a
+ * mistyped env doesn't silently embed literal "${FOO}" into a connection
+ * string, header, URL, etc.
+ *
+ * `context` is included in the warning to help users locate the source
+ * of the misconfiguration (e.g. "postgres source URL", "rest source
+ * Authorization header"). Pass a short, human-readable string.
+ */
+export function expandEnv(input: string, context: string): string {
+  return input.replace(/\$\{([A-Z0-9_]+)\}/gi, (_, name: string) => {
+    const value = process.env[name];
+    if (value === undefined) {
+      process.stderr.write(
+        `[pipequery] env var "${name}" referenced in ${context} is not set\n`,
+      );
+      return '';
+    }
+    return value;
+  });
+}