@vaultgradient/pipequery-cli 0.3.2

package/src/docker/templates.ts

@@ -0,0 +1,43 @@
+export function generateDockerfile(port: number): string {
+  return `FROM node:20-alpine
+WORKDIR /app
+
+# Copy config and data files
+COPY pipequery.yaml .
+COPY data/ data/ 2>/dev/null || true
+
+# Install PipeQuery CLI
+RUN npm install -g @andreadito/pq
+
+EXPOSE ${port}
+
+HEALTHCHECK --interval=30s --timeout=5s --start-period=10s \\
+  CMD wget -qO- http://localhost:${port}/health || exit 1
+
+CMD ["pq", "serve", "--host", "0.0.0.0"]
+`;
+}
+
+export function generateDockerCompose(port: number): string {
+  return `services:
+  pipequery:
+    build: .
+    ports:
+      - "${port}:${port}"
+    volumes:
+      - ./pipequery.yaml:/app/pipequery.yaml:ro
+      - ./data:/app/data:ro
+    restart: unless-stopped
+    environment:
+      - NODE_ENV=production
+`;
+}
+
+export function generateDockerignore(): string {
+  return `node_modules
+.pipequery
+.git
+*.log
+dist
+`;
+}

package/src/engine.ts

@@ -0,0 +1,5 @@
+// Re-export the PipeQuery engine from the parent package.
+// This module centralizes the import so we can switch between
+// workspace reference and published package easily.
+export { query, compile, parseQuery, clearCache, LiveQuery, liveQuery } from '../../src/engine/index.js';
+export type { DataContext, CompiledQuery, LiveQueryOptions, LiveQueryStats } from '../../src/engine/index.js';

package/src/index.ts

@@ -0,0 +1,164 @@
+import { Command } from 'commander';
+import chalk from 'chalk';
+import { initCommand } from './commands/init.js';
+import { serveCommand } from './commands/serve.js';
+import { queryCommand } from './commands/query.js';
+import { sourceListCommand, sourceAddCommand, sourceRemoveCommand, sourceTestCommand } from './commands/source.js';
+import { endpointListCommand, endpointAddCommand, endpointRemoveCommand } from './commands/endpoint.js';
+import { dashboardCommand } from './commands/dashboard.js';
+import { monitorCommand } from './commands/monitor.js';
+import { remoteDeployCommand, remoteConnectCommand, remoteStatusCommand } from './commands/remote.js';
+import { completionCommand } from './commands/completion.js';
+import { stopCommand } from './commands/stop.js';
+import { startRepl } from './commands/repl.js';
+import { printBanner } from './utils/banner.js';
+
+const program = new Command();
+
+program
+  .name('pq')
+  .description('PipeQuery CLI — data sources, aggregations, APIs, and terminal dashboards')
+  .version('0.1.0')
+  .addHelpText('beforeAll', () => {
+    printBanner();
+    return '';
+  })
+  .configureHelp({
+    subcommandTerm: (cmd) => chalk.hex('#06b6d4')(cmd.name()) + (cmd.usage() ? ' ' + chalk.dim(cmd.usage()) : ''),
+  });
+
+// ─── pq init ─────────────────────────────────────────────────────────────────
+
+program
+  .command('init')
+  .description('Create a pipequery.yaml config file')
+  .action(initCommand);
+
+// ─── pq serve ────────────────────────────────────────────────────────────────
+
+program
+  .command('serve')
+  .description('Start the PipeQuery server')
+  .option('-p, --port <port>', 'Server port', parseInt)
+  .option('-H, --host <host>', 'Server host')
+  .option('-d, --daemon', 'Run as a background daemon')
+  .option('--stop', 'Stop a running daemon')
+  .action(serveCommand);
+
+// ─── pq query ────────────────────────────────────────────────────────────────
+
+program
+  .command('query <expression>')
+  .description('Run an ad-hoc PipeQuery expression')
+  .option('-f, --format <format>', 'Output format: table, json', 'table')
+  .action(queryCommand);
+
+// ─── pq source ───────────────────────────────────────────────────────────────
+
+const source = program.command('source').description('Manage data sources');
+
+source
+  .command('list')
+  .description('List configured data sources')
+  .action(sourceListCommand);
+
+source
+  .command('add <name>')
+  .description('Add a data source')
+  .requiredOption('-t, --type <type>', 'Source type: rest, websocket, file')
+  .option('-u, --url <url>', 'URL for REST or WebSocket sources')
+  .option('-i, --interval <interval>', 'Poll interval for REST (e.g., 30s, 5m)', '30s')
+  .option('-d, --data-path <path>', 'Dot-path to extract data from response')
+  .option('-p, --path <path>', 'File path for file sources')
+  .option('-w, --watch', 'Watch file for changes')
+  .action(sourceAddCommand);
+
+source
+  .command('remove <name>')
+  .description('Remove a data source')
+  .action(sourceRemoveCommand);
+
+source
+  .command('test <name>')
+  .description('Test a data source and show sample data')
+  .action(sourceTestCommand);
+
+// ─── pq endpoint ─────────────────────────────────────────────────────────────
+
+const endpoint = program.command('endpoint').description('Manage API endpoints');
+
+endpoint
+  .command('list')
+  .description('List API endpoints')
+  .action(endpointListCommand);
+
+endpoint
+  .command('add <path>')
+  .description('Register an API endpoint')
+  .requiredOption('-q, --query <query>', 'PipeQuery expression')
+  .option('-c, --cache <duration>', 'Cache duration (e.g., 30s)')
+  .action(endpointAddCommand);
+
+endpoint
+  .command('remove <path>')
+  .description('Remove an API endpoint')
+  .action(endpointRemoveCommand);
+
+// ─── pq dashboard ────────────────────────────────────────────────────────────
+
+program
+  .command('dashboard')
+  .description('Launch the interactive terminal dashboard')
+  .option('-n, --name <name>', 'Dashboard name from config', 'main')
+  .action(dashboardCommand);
+
+// ─── pq monitor ─────────────────────────────────────────────────────────────
+
+program
+  .command('monitor')
+  .description('Live server monitor — sources, endpoints, logs')
+  .action(monitorCommand);
+
+// ─── pq remote ───────────────────────────────────────────────────────────────
+
+const remote = program.command('remote').description('Manage remote deployment');
+
+remote
+  .command('deploy')
+  .description('Generate Dockerfile and docker-compose.yaml')
+  .action(remoteDeployCommand);
+
+remote
+  .command('connect <url>')
+  .description('Connect CLI to a remote PipeQuery server')
+  .action(remoteConnectCommand);
+
+remote
+  .command('status')
+  .description('Check remote server health')
+  .action(remoteStatusCommand);
+
+// ─── pq stop ────────────────────────────────────────────────────────────────
+
+program
+  .command('stop')
+  .description('Stop the running PipeQuery server (daemon or foreground)')
+  .option('-f, --force', 'Force kill (SIGKILL instead of SIGTERM)')
+  .action(stopCommand);
+
+// ─── pq completion ───────────────────────────────────────────────────────────
+
+program
+  .command('completion')
+  .description('Generate shell completion script')
+  .option('-s, --shell <shell>', 'Shell type: bash, zsh, fish')
+  .action(completionCommand);
+
+// ─── Run ─────────────────────────────────────────────────────────────────────
+
+// If no args, start interactive REPL
+if (process.argv.length <= 2) {
+  startRepl();
+} else {
+  program.parse();
+}

package/src/server/activity-log.ts

@@ -0,0 +1,28 @@
+export interface ActivityEntry {
+  time: string;
+  level: 'info' | 'warn' | 'error';
+  message: string;
+}
+
+const MAX_ENTRIES = 100;
+const entries: ActivityEntry[] = [];
+
+export function logActivity(level: ActivityEntry['level'], message: string) {
+  entries.push({
+    time: new Date().toISOString(),
+    level,
+    message,
+  });
+  if (entries.length > MAX_ENTRIES) {
+    entries.splice(0, entries.length - MAX_ENTRIES);
+  }
+}
+
+export function getActivityLog(since?: string): ActivityEntry[] {
+  if (!since) return [...entries];
+  return entries.filter((e) => e.time > since);
+}
+
+export function info(msg: string) { logActivity('info', msg); }
+export function warn(msg: string) { logActivity('warn', msg); }
+export function error(msg: string) { logActivity('error', msg); }

package/src/server/app.ts

@@ -0,0 +1,101 @@
+import Fastify from 'fastify';
+import cors from '@fastify/cors';
+import type { PipeQueryConfig, EndpointConfig, SourceConfig } from '../config/schema.js';
+import { SourceManager } from './sources/manager.js';
+import { registerHealthRoutes } from './routes/health.js';
+import { registerControlRoutes } from './routes/control.js';
+import { registerDynamicRoutes } from './routes/dynamic.js';
+import { registerSSERoutes } from './routes/sse.js';
+import { saveConfig } from '../config/loader.js';
+import * as activity from './activity-log.js';
+import { getActivityLog } from './activity-log.js';
+
+export interface ServerContext {
+  app: ReturnType<typeof Fastify>;
+  sourceManager: SourceManager;
+  endpoints: Map<string, EndpointConfig>;
+}
+
+export async function createServer(
+  config: PipeQueryConfig,
+  configPath: string,
+  cwd: string,
+): Promise<ServerContext> {
+  const app = Fastify({ logger: false });
+  await app.register(cors);
+
+  const sourceManager = new SourceManager(cwd);
+  const startedAt = new Date();
+
+  activity.info('Server starting...');
+
+  // Initialize sources from config
+  for (const [name, sourceConfig] of Object.entries(config.sources)) {
+    try {
+      await sourceManager.addSource(name, sourceConfig);
+      activity.info(`Source "${name}" loaded (${sourceConfig.type})`);
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      activity.error(`Source "${name}" failed: ${msg}`);
+      console.error(`Failed to start source "${name}":`, msg);
+    }
+  }
+
+  // Build endpoint registry
+  const endpoints = new Map<string, EndpointConfig>();
+  for (const [path, endpointConfig] of Object.entries(config.endpoints)) {
+    endpoints.set(path, endpointConfig);
+    activity.info(`Endpoint registered: ${path}`);
+  }
+
+  // Config persistence callbacks (save changes back to YAML)
+  const configCallbacks = {
+    onSourceChange: async (name: string, sourceConfig: SourceConfig | null) => {
+      if (sourceConfig) {
+        config.sources[name] = sourceConfig;
+        activity.info(`Source "${name}" added (${sourceConfig.type})`);
+      } else {
+        delete config.sources[name];
+        activity.warn(`Source "${name}" removed`);
+      }
+      await saveConfig(config, configPath);
+    },
+    onEndpointChange: async (path: string, endpointConfig: EndpointConfig | null) => {
+      if (endpointConfig) {
+        config.endpoints[path] = endpointConfig;
+        activity.info(`Endpoint "${path}" added`);
+      } else {
+        delete config.endpoints[path];
+        activity.warn(`Endpoint "${path}" removed`);
+      }
+      await saveConfig(config, configPath);
+    },
+  };
+
+  // Log requests (skip health/status/activity to reduce noise)
+  app.addHook('onResponse', async (request, reply) => {
+    const url = request.url;
+    if (url === '/health' || url === '/status' || url === '/api/_control/activity') return;
+    if (url === '/api/_control/sources' && request.method === 'GET') return;
+    if (url === '/api/_control/endpoints' && request.method === 'GET') return;
+
+    const status = reply.statusCode;
+    const level = status >= 400 ? 'error' as const : 'info' as const;
+    activity.logActivity(level, `${request.method} ${url} → ${status}`);
+  });
+
+  // Activity log endpoint (for monitor)
+  app.get<{ Querystring: { since?: string } }>('/api/_control/activity', async (req) => {
+    return getActivityLog(req.query.since);
+  });
+
+  // Register routes
+  registerHealthRoutes(app, sourceManager, startedAt);
+  registerControlRoutes(app, sourceManager, { endpoints }, configCallbacks);
+  registerSSERoutes(app, sourceManager);
+  registerDynamicRoutes(app, sourceManager, endpoints);
+
+  activity.info('Server ready');
+
+  return { app, sourceManager, endpoints };
+}

package/src/server/cache.ts

@@ -0,0 +1,30 @@
+interface CacheEntry {
+  data: unknown;
+  expiresAt: number;
+}
+
+export class ResponseCache {
+  private entries = new Map<string, CacheEntry>();
+
+  get(key: string): unknown | undefined {
+    const entry = this.entries.get(key);
+    if (!entry) return undefined;
+    if (Date.now() > entry.expiresAt) {
+      this.entries.delete(key);
+      return undefined;
+    }
+    return entry.data;
+  }
+
+  set(key: string, data: unknown, ttlMs: number): void {
+    this.entries.set(key, { data, expiresAt: Date.now() + ttlMs });
+  }
+
+  invalidate(key: string): void {
+    this.entries.delete(key);
+  }
+
+  clear(): void {
+    this.entries.clear();
+  }
+}

package/src/server/routes/control.ts

@@ -0,0 +1,100 @@
+import type { FastifyInstance } from 'fastify';
+import { query } from '../../engine.js';
+import type { SourceConfig, EndpointConfig } from '../../config/schema.js';
+import type { SourceManager } from '../sources/manager.js';
+
+interface EndpointRegistry {
+  endpoints: Map<string, EndpointConfig>;
+  onAdd?: (path: string, config: EndpointConfig) => void;
+  onRemove?: (path: string) => void;
+}
+
+export function registerControlRoutes(
+  app: FastifyInstance,
+  sourceManager: SourceManager,
+  endpointRegistry: EndpointRegistry,
+  configCallbacks: {
+    onSourceChange: (name: string, config: SourceConfig | null) => Promise<void>;
+    onEndpointChange: (path: string, config: EndpointConfig | null) => Promise<void>;
+  },
+) {
+  // ─── Sources ─────────────────────────────────────────────────────────────
+
+  app.get('/api/_control/sources', async () => {
+    return sourceManager.getAllStatuses();
+  });
+
+  app.post<{ Body: { name: string; config: SourceConfig } }>('/api/_control/sources', async (req) => {
+    const { name, config } = req.body;
+    await sourceManager.addSource(name, config);
+    await configCallbacks.onSourceChange(name, config);
+    return { ok: true, name, status: sourceManager.getSourceStatus(name) };
+  });
+
+  app.delete<{ Params: { name: string } }>('/api/_control/sources/:name', async (req) => {
+    const { name } = req.params;
+    const removed = sourceManager.removeSource(name);
+    if (!removed) return { ok: false, error: `Source "${name}" not found` };
+    await configCallbacks.onSourceChange(name, null);
+    return { ok: true, name };
+  });
+
+  app.post<{ Params: { name: string } }>('/api/_control/sources/:name/test', async (req) => {
+    const { name } = req.params;
+    const data = sourceManager.getSourceData(name);
+    if (!data) return { ok: false, error: `Source "${name}" not found` };
+    return {
+      ok: true,
+      rowCount: data.length,
+      sample: data.slice(0, 3),
+      status: sourceManager.getSourceStatus(name),
+    };
+  });
+
+  // ─── Endpoints ───────────────────────────────────────────────────────────
+
+  app.get('/api/_control/endpoints', async () => {
+    const result: Record<string, EndpointConfig> = {};
+    for (const [path, config] of endpointRegistry.endpoints) {
+      result[path] = config;
+    }
+    return result;
+  });
+
+  app.post<{ Body: { path: string; config: EndpointConfig } }>('/api/_control/endpoints', async (req) => {
+    const { path, config } = req.body;
+    endpointRegistry.endpoints.set(path, config);
+    endpointRegistry.onAdd?.(path, config);
+    await configCallbacks.onEndpointChange(path, config);
+    return { ok: true, path };
+  });
+
+  app.delete<{ Body: { path: string } }>('/api/_control/endpoints', async (req) => {
+    const path = req.body.path;
+    const removed = endpointRegistry.endpoints.delete(path);
+    if (!removed) return { ok: false, error: `Endpoint "${path}" not found` };
+    endpointRegistry.onRemove?.(path);
+    await configCallbacks.onEndpointChange(path, null);
+    return { ok: true, path };
+  });
+
+  // ─── Refresh all sources ─────────────────────────────────────────────────
+
+  app.post('/api/_control/refresh', async () => {
+    await sourceManager.refreshAll();
+    return { ok: true };
+  });
+
+  // ─── Ad-hoc query ────────────────────────────────────────────────────────
+
+  app.post<{ Body: { query: string } }>('/api/_control/query', async (req) => {
+    const { query: expr } = req.body;
+    const context = sourceManager.getContext();
+    try {
+      const result = query(context, expr);
+      return { ok: true, result };
+    } catch (err) {
+      return { ok: false, error: err instanceof Error ? err.message : String(err) };
+    }
+  });
+}

package/src/server/routes/dynamic.ts

@@ -0,0 +1,51 @@
+import type { FastifyInstance } from 'fastify';
+import { query } from '../../engine.js';
+import type { EndpointConfig } from '../../config/schema.js';
+import type { SourceManager } from '../sources/manager.js';
+import { ResponseCache } from '../cache.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+
+export function registerDynamicRoutes(
+  app: FastifyInstance,
+  sourceManager: SourceManager,
+  endpoints: Map<string, EndpointConfig>,
+) {
+  const cache = new ResponseCache();
+
+  // Use onRequest hook to intercept user-defined endpoints
+  // This avoids Fastify wildcard route conflicts with control routes
+  app.addHook('onRequest', async (req, reply) => {
+    const path = req.url.split('?')[0];
+
+    // Only handle /api/ paths that aren't control routes
+    if (!path.startsWith('/api/') || path.startsWith('/api/_control')) return;
+
+    const config = endpoints.get(path);
+    if (!config) return; // Let it fall through to 404
+
+    // Check cache
+    if (config.cache) {
+      const cached = cache.get(path);
+      if (cached !== undefined) {
+        reply.send(cached);
+        return;
+      }
+    }
+
+    // Execute query
+    const context = sourceManager.getContext();
+    try {
+      const result = query(context, config.query);
+
+      if (config.cache) {
+        cache.set(path, result, parseDuration(config.cache));
+      }
+
+      reply.send(result);
+    } catch (err) {
+      reply.status(500).send({
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  });
+}

package/src/server/routes/health.ts

@@ -0,0 +1,16 @@
+import type { FastifyInstance } from 'fastify';
+import type { SourceManager } from '../sources/manager.js';
+
+export function registerHealthRoutes(app: FastifyInstance, sourceManager: SourceManager, startedAt: Date) {
+  app.get('/health', async () => {
+    return { status: 'ok', uptime: Date.now() - startedAt.getTime() };
+  });
+
+  app.get('/status', async () => {
+    return {
+      status: 'ok',
+      uptime: Date.now() - startedAt.getTime(),
+      sources: sourceManager.getAllStatuses(),
+    };
+  });
+}

package/src/server/routes/sse.ts

@@ -0,0 +1,96 @@
+import type { FastifyInstance } from 'fastify';
+import { query } from '../../engine.js';
+import type { SourceManager } from '../sources/manager.js';
+
+/**
+ * SSE endpoint for real-time dashboard updates.
+ *
+ * Clients connect to GET /api/_control/sse?queries=<json-encoded-array>
+ * and receive server-sent events whenever source data changes.
+ * Updates are throttled to max once per second to avoid overwhelming clients.
+ *
+ * Each event is a JSON object: { panelIndex: number, result: unknown }
+ */
+export function registerSSERoutes(app: FastifyInstance, sourceManager: SourceManager) {
+  app.get('/api/_control/sse', async (req, reply) => {
+    const queriesParam = (req.query as Record<string, string>).queries;
+    if (!queriesParam) {
+      return reply.status(400).send({ error: 'queries parameter required (JSON array of strings)' });
+    }
+
+    let queries: string[];
+    try {
+      queries = JSON.parse(queriesParam);
+      if (!Array.isArray(queries)) throw new Error();
+    } catch {
+      return reply.status(400).send({ error: 'queries must be a JSON array of strings' });
+    }
+
+    // Set SSE headers
+    reply.raw.writeHead(200, {
+      'Content-Type': 'text/event-stream',
+      'Cache-Control': 'no-cache',
+      'Connection': 'keep-alive',
+      'Access-Control-Allow-Origin': '*',
+    });
+
+    // Send initial data for all panels
+    sendAllPanels(reply.raw, queries, sourceManager);
+
+    // Throttled update: max once per second
+    let pending = false;
+    let timer: ReturnType<typeof setTimeout> | null = null;
+
+    const throttledSend = () => {
+      if (timer) {
+        pending = true;
+        return;
+      }
+      sendAllPanels(reply.raw, queries, sourceManager);
+      timer = setTimeout(() => {
+        timer = null;
+        if (pending) {
+          pending = false;
+          throttledSend();
+        }
+      }, 1000);
+    };
+
+    // Subscribe to source updates
+    const unsubscribe = sourceManager.onSourceUpdate(throttledSend);
+
+    // Periodic heartbeat to keep connection alive
+    const heartbeat = setInterval(() => {
+      reply.raw.write(':heartbeat\n\n');
+    }, 15_000);
+
+    // Cleanup on close
+    req.raw.on('close', () => {
+      unsubscribe();
+      clearInterval(heartbeat);
+      if (timer) clearTimeout(timer);
+    });
+
+    // Don't let Fastify auto-close the response
+    await reply.hijack();
+  });
+}
+
+function sendAllPanels(
+  res: import('node:http').ServerResponse,
+  queries: string[],
+  sourceManager: SourceManager,
+) {
+  const context = sourceManager.getContext();
+
+  for (let i = 0; i < queries.length; i++) {
+    try {
+      const result = query(context, queries[i]);
+      const event = JSON.stringify({ panelIndex: i, result });
+      res.write(`data: ${event}\n\n`);
+    } catch (err) {
+      const event = JSON.stringify({ panelIndex: i, error: err instanceof Error ? err.message : String(err) });
+      res.write(`data: ${event}\n\n`);
+    }
+  }
+}