@vaultgradient/pipequery-cli 0.3.2 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vaultgradient/pipequery-cli",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "CLI for PipeQuery — data sources, aggregations, APIs, and terminal dashboards",
   "type": "module",
   "bin": {
@@ -14,19 +14,27 @@
   },
   "dependencies": {
     "@fastify/cors": "^11.2.0",
+    "@modelcontextprotocol/sdk": "^1.29.0",
+    "better-sqlite3": "^12.9.0",
     "chalk": "^5.6.2",
     "chokidar": "^5.0.0",
     "commander": "^14.0.3",
     "csv-parse": "^6.2.1",
     "fastify": "^5.8.5",
     "ink": "^7.0.1",
+    "kafkajs": "^2.2.4",
+    "mysql2": "^3.22.2",
     "ora": "^9.3.0",
+    "pg": "^8.20.0",
     "react": "^19.2.5",
     "ws": "^8.20.0",
-    "yaml": "^2.8.3"
+    "yaml": "^2.8.3",
+    "zod": "^4.3.6"
   },
   "devDependencies": {
+    "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^25.6.0",
+    "@types/pg": "^8.20.0",
     "@types/react": "^19.2.14",
     "@types/ws": "^8.18.1",
     "tsup": "^8.5.1",

package/src/commands/mcp.ts

@@ -0,0 +1,87 @@
+import { loadConfig } from '../config/loader.js';
+import { log } from '../utils/logger.js';
+import { AttachedProvider, LocalProvider, type Provider } from '../mcp/provider.js';
+import { runStdio } from '../mcp/stdio.js';
+import { runHttp } from '../mcp/http.js';
+import { createMcpServer } from '../mcp/server.js';
+import { Client } from '@modelcontextprotocol/sdk/client/index.js';
+import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js';
+
+interface McpServeOptions {
+  http?: boolean;
+  port?: number;
+  host?: string;
+  attach?: string;
+  authToken?: string;
+}
+
+export async function mcpServeCommand(opts: McpServeOptions): Promise<void> {
+  const provider = await buildProvider(opts.attach);
+  const isStdio = !opts.http;
+
+  if (isStdio) {
+    // In stdio mode, stdout is reserved for JSON-RPC. All logs go to stderr.
+    await runStdio(provider);
+    return;
+  }
+
+  const port = opts.port ?? 3001;
+  const host = opts.host ?? '127.0.0.1';
+  // CLI flag takes precedence over env var. No token => unauthenticated
+  // (runHttp warns loudly at startup).
+  const authToken = opts.authToken ?? process.env.PIPEQUERY_MCP_TOKEN;
+  const handle = await runHttp(provider, { port, host, authToken });
+
+  log.success(`PipeQuery MCP server listening on ${handle.url}`);
+  log.info('Connect any MCP client to the URL above (Claude, Cursor, custom agents).');
+  log.info('Press Ctrl+C to stop.');
+
+  const shutdown = async () => {
+    log.info('Shutting down MCP server...');
+    await handle.close();
+    process.exit(0);
+  };
+
+  process.on('SIGINT', () => void shutdown());
+  process.on('SIGTERM', () => void shutdown());
+}
+
+export async function mcpInspectCommand(opts: { attach?: string }): Promise<void> {
+  const provider = await buildProvider(opts.attach);
+  const mcp = createMcpServer(provider);
+
+  const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+  await mcp.connect(serverTransport);
+
+  const client = new Client({ name: 'pq-mcp-inspect', version: '0.1.0' });
+  await client.connect(clientTransport);
+
+  try {
+    const { tools } = await client.listTools();
+    const summary = {
+      server: 'pipequery',
+      toolCount: tools.length,
+      tools: tools.map((t) => ({
+        name: t.name,
+        title: t.title,
+        description: t.description,
+        inputSchema: t.inputSchema,
+      })),
+    };
+    process.stdout.write(JSON.stringify(summary, null, 2) + '\n');
+  } finally {
+    await client.close();
+    await mcp.close();
+    await provider.dispose();
+  }
+}
+
+async function buildProvider(attachUrl?: string): Promise<Provider> {
+  if (attachUrl) {
+    return new AttachedProvider(attachUrl);
+  }
+
+  const cwd = process.cwd();
+  const { config } = await loadConfig(cwd);
+  return new LocalProvider(config, cwd);
+}

package/src/commands/source.ts

@@ -51,6 +51,16 @@ export async function sourceAddCommand(name: string, opts: {
   dataPath?: string;
   path?: string;
   watch?: boolean;
+  query?: string;
+  ssl?: string;
+  maxRows?: string;
+  readonly?: boolean;
+  brokers?: string;
+  topic?: string;
+  groupId?: string;
+  fromBeginning?: boolean;
+  valueFormat?: string;
+  maxBuffer?: string;
 }) {
   const cwd = process.cwd();
   const { config, path: configPath } = await loadConfig(cwd);
@@ -79,8 +89,60 @@ export async function sourceAddCommand(name: string, opts: {
         ...(opts.watch ? { watch: true } : {}),
       };
       break;
+    case 'postgres':
+      if (!opts.url) throw new Error('--url is required for Postgres sources');
+      if (!opts.query) throw new Error('--query is required for Postgres sources');
+      sourceConfig = {
+        type: 'postgres',
+        url: opts.url,
+        query: opts.query,
+        interval: opts.interval ?? '30s',
+        ...(opts.ssl === 'no-verify' ? { ssl: 'no-verify' as const } : {}),
+        ...(opts.ssl === 'false' ? { ssl: false as const } : {}),
+        ...(opts.maxRows ? { maxRows: Number.parseInt(opts.maxRows, 10) } : {}),
+      };
+      break;
+    case 'mysql':
+      if (!opts.url) throw new Error('--url is required for MySQL sources');
+      if (!opts.query) throw new Error('--query is required for MySQL sources');
+      sourceConfig = {
+        type: 'mysql',
+        url: opts.url,
+        query: opts.query,
+        interval: opts.interval ?? '30s',
+        ...(opts.ssl === 'no-verify' ? { ssl: 'no-verify' as const } : {}),
+        ...(opts.ssl === 'false' ? { ssl: false as const } : {}),
+        ...(opts.maxRows ? { maxRows: Number.parseInt(opts.maxRows, 10) } : {}),
+      };
+      break;
+    case 'sqlite':
+      if (!opts.path) throw new Error('--path is required for SQLite sources');
+      if (!opts.query) throw new Error('--query is required for SQLite sources');
+      sourceConfig = {
+        type: 'sqlite',
+        path: opts.path,
+        query: opts.query,
+        interval: opts.interval ?? '30s',
+        ...(opts.maxRows ? { maxRows: Number.parseInt(opts.maxRows, 10) } : {}),
+        ...(opts.readonly === false ? { readonly: false } : {}),
+      };
+      break;
+    case 'kafka':
+      if (!opts.brokers) throw new Error('--brokers is required for Kafka sources (comma-separated)');
+      if (!opts.topic) throw new Error('--topic is required for Kafka sources');
+      sourceConfig = {
+        type: 'kafka',
+        brokers: opts.brokers,
+        topic: opts.topic,
+        ...(opts.groupId ? { groupId: opts.groupId } : {}),
+        ...(opts.fromBeginning ? { fromBeginning: true } : {}),
+        ...(opts.maxBuffer ? { maxBuffer: Number.parseInt(opts.maxBuffer, 10) } : {}),
+        ...(opts.valueFormat ? { valueFormat: opts.valueFormat as 'json' | 'string' | 'raw' } : {}),
+        ...(opts.ssl === 'true' ? { ssl: true } : {}),
+      };
+      break;
     default:
-      throw new Error(`Unknown source type: ${opts.type}. Use: rest, websocket, file`);
+      throw new Error(`Unknown source type: ${opts.type}. Use: rest, websocket, file, postgres, mysql, sqlite, kafka`);
   }
 
   // Save to config

package/src/config/schema.ts

@@ -21,7 +21,11 @@ export type SourceConfig =
   | RestSourceConfig
   | WebSocketSourceConfig
   | FileSourceConfig
-  | StaticSourceConfig;
+  | StaticSourceConfig
+  | PostgresSourceConfig
+  | MysqlSourceConfig
+  | SqliteSourceConfig
+  | KafkaSourceConfig;
 
 export interface RestSourceConfig {
   type: 'rest';
@@ -50,6 +54,109 @@ export interface StaticSourceConfig {
   data: unknown[];
 }
 
+export interface PostgresSourceConfig {
+  type: 'postgres';
+  /**
+   * Postgres connection URL. Supports `${ENV_VAR}` interpolation so
+   * credentials can live in the environment instead of the yaml.
+   * Example: `postgres://${DB_USER}:${DB_PASS}@db.internal:5432/app`.
+   */
+  url: string;
+  /**
+   * The SELECT query to run on each poll. Must be a single statement.
+   * It's the caller's responsibility to add LIMIT/WHERE clauses that
+   * bound the result set; `maxRows` below is a hard safety cap.
+   */
+  query: string;
+  /** Poll interval, e.g. "30s", "5m". Defaults to "30s". */
+  interval?: string;
+  /**
+   * SSL policy. `"require"` verifies the cert (production default for
+   * most hosted Postgres). `"no-verify"` skips verification (useful for
+   * self-signed dev setups). `false` disables TLS entirely.
+   */
+  ssl?: 'require' | 'no-verify' | false;
+  /** Safety cap. If the query returns more rows, the fetch errors. Default 10000. */
+  maxRows?: number;
+}
+
+export interface MysqlSourceConfig {
+  type: 'mysql';
+  /**
+   * MySQL connection URL. Supports `${ENV_VAR}` interpolation so
+   * credentials can live in the environment instead of the yaml.
+   * Example: `mysql://${DB_USER}:${DB_PASS}@db.internal:3306/app`.
+   */
+  url: string;
+  /** The SELECT query to run on each poll. */
+  query: string;
+  /** Poll interval, e.g. "30s", "5m". Defaults to "30s". */
+  interval?: string;
+  /**
+   * SSL policy. `"require"` verifies the cert (production default).
+   * `"no-verify"` skips verification. `false` disables TLS.
+   */
+  ssl?: 'require' | 'no-verify' | false;
+  /** Safety cap. Default 10000. */
+  maxRows?: number;
+}
+
+export interface KafkaSourceConfig {
+  type: 'kafka';
+  /**
+   * Comma-separated list of Kafka bootstrap brokers, or an array. Supports
+   * `${ENV_VAR}` interpolation. Example: `"broker1:9092,broker2:9092"`.
+   */
+  brokers: string | string[];
+  /** Topic to subscribe to. Only one topic per source; use multiple sources for multiple topics. */
+  topic: string;
+  /**
+   * Consumer group ID. If omitted, an ephemeral per-process group is generated
+   * so this pipequery instance sees the full firehose. Set explicitly to share
+   * a partitioned subscription across pipequery replicas.
+   */
+  groupId?: string;
+  /** Start reading from the topic's beginning on first connect. Default false (latest). */
+  fromBeginning?: boolean;
+  /** Ring buffer size. Oldest messages are evicted. Default 1000. */
+  maxBuffer?: number;
+  /**
+   * How to decode the message value.
+   *   - `"json"` (default): `JSON.parse(buffer.toString())`; non-JSON messages are skipped.
+   *   - `"string"`: UTF-8 string, stored as `{ value }`.
+   *   - `"raw"`: base64-encoded bytes, stored as `{ value }`.
+   */
+  valueFormat?: 'json' | 'string' | 'raw';
+  /** SSL / TLS toggle. Default false (plaintext). */
+  ssl?: boolean;
+  /**
+   * SASL authentication. Supports `${ENV_VAR}` interpolation in username/password.
+   */
+  sasl?: {
+    mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
+    username: string;
+    password: string;
+  };
+}
+
+export interface SqliteSourceConfig {
+  type: 'sqlite';
+  /**
+   * Path to the SQLite database file, relative to the directory
+   * containing pipequery.yaml. Use `:memory:` for an in-memory database
+   * (useful for tests; data will not persist across restarts).
+   */
+  path: string;
+  /** The SELECT query to run on each poll. */
+  query: string;
+  /** Poll interval, e.g. "30s", "5m". Defaults to "30s". */
+  interval?: string;
+  /** Open the database read-only. Recommended for query-only sources. Default true. */
+  readonly?: boolean;
+  /** Safety cap. Default 10000. */
+  maxRows?: number;
+}
+
 // ─── Endpoint config ─────────────────────────────────────────────────────────
 
 export interface EndpointConfig {

package/src/docker/templates.ts

@@ -7,7 +7,7 @@ COPY pipequery.yaml .
 COPY data/ data/ 2>/dev/null || true
 
 # Install PipeQuery CLI
-RUN npm install -g @andreadito/pq
+RUN npm install -g @vaultgradient/pipequery-cli
 
 EXPOSE ${port}
 

package/src/index.ts

@@ -8,6 +8,7 @@ import { endpointListCommand, endpointAddCommand, endpointRemoveCommand } from '
 import { dashboardCommand } from './commands/dashboard.js';
 import { monitorCommand } from './commands/monitor.js';
 import { remoteDeployCommand, remoteConnectCommand, remoteStatusCommand } from './commands/remote.js';
+import { mcpServeCommand, mcpInspectCommand } from './commands/mcp.js';
 import { completionCommand } from './commands/completion.js';
 import { stopCommand } from './commands/stop.js';
 import { startRepl } from './commands/repl.js';
@@ -65,12 +66,22 @@ source
 source
   .command('add <name>')
   .description('Add a data source')
-  .requiredOption('-t, --type <type>', 'Source type: rest, websocket, file')
-  .option('-u, --url <url>', 'URL for REST or WebSocket sources')
-  .option('-i, --interval <interval>', 'Poll interval for REST (e.g., 30s, 5m)', '30s')
-  .option('-d, --data-path <path>', 'Dot-path to extract data from response')
-  .option('-p, --path <path>', 'File path for file sources')
-  .option('-w, --watch', 'Watch file for changes')
+  .requiredOption('-t, --type <type>', 'Source type: rest, websocket, file, postgres, mysql, sqlite, kafka')
+  .option('-u, --url <url>', 'Connection URL for REST / WebSocket / Postgres / MySQL sources')
+  .option('-i, --interval <interval>', 'Poll interval (e.g., 30s, 5m)', '30s')
+  .option('-d, --data-path <path>', 'Dot-path to extract data from REST response')
+  .option('-p, --path <path>', 'File path for file or sqlite sources')
+  .option('-w, --watch', 'Watch file for changes (file sources)')
+  .option('-q, --query <sql>', 'SELECT query for Postgres / MySQL / SQLite sources')
+  .option('--ssl <mode>', 'Postgres / MySQL SSL: require (default), no-verify, false; Kafka: true / false')
+  .option('--max-rows <n>', 'DB sources: safety cap on rows per fetch (default 10000)')
+  .option('--no-readonly', 'SQLite: open the database read-write instead of read-only')
+  .option('-b, --brokers <list>', 'Kafka: comma-separated bootstrap brokers (supports ${ENV_VAR})')
+  .option('--topic <topic>', 'Kafka topic to subscribe to')
+  .option('--group-id <id>', 'Kafka consumer group (default: ephemeral per-process)')
+  .option('--from-beginning', 'Kafka: read topic from the start instead of latest')
+  .option('--value-format <fmt>', 'Kafka value decoding: json (default), string, raw')
+  .option('--max-buffer <n>', 'Kafka / WebSocket: ring buffer size (default 1000)')
   .action(sourceAddCommand);
 
 source
@@ -138,6 +149,26 @@ remote
   .description('Check remote server health')
   .action(remoteStatusCommand);
 
+// ─── pq mcp ──────────────────────────────────────────────────────────────────
+
+const mcp = program.command('mcp').description('Model Context Protocol server for AI clients');
+
+mcp
+  .command('serve')
+  .description('Start an MCP server exposing pipequery as tools (for Claude, Cursor, Copilot, etc.)')
+  .option('--http', 'Serve over HTTP/SSE instead of stdio')
+  .option('-p, --port <port>', 'HTTP port (when --http is set)', parseInt)
+  .option('-H, --host <host>', 'HTTP host (when --http is set)')
+  .option('--attach <url>', 'Attach to a running `pq serve` instance at the given URL instead of loading pipequery.yaml locally')
+  .option('--auth-token <token>', 'Require Authorization: Bearer <token> on HTTP requests. Falls back to $PIPEQUERY_MCP_TOKEN. Without either, the endpoint is unauthenticated.')
+  .action(mcpServeCommand);
+
+mcp
+  .command('inspect')
+  .description('Print the MCP tool schemas as JSON (useful for directory submission / debugging)')
+  .option('--attach <url>', 'Attach to a running `pq serve` instance at the given URL instead of loading pipequery.yaml locally')
+  .action(mcpInspectCommand);
+
 // ─── pq stop ────────────────────────────────────────────────────────────────
 
 program

package/src/mcp/http.ts

@@ -0,0 +1,191 @@
+import { createServer as createHttpServer } from 'node:http';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { randomUUID } from 'node:crypto';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import type { Provider } from './provider.js';
+import { createMcpServer, type McpServerInfo } from './server.js';
+
+export interface HttpServerOptions {
+  host?: string;
+  port: number;
+  path?: string;
+  info?: McpServerInfo;
+  /**
+   * Require clients to send `Authorization: Bearer <token>` matching this
+   * value. When undefined, the endpoint is UNAUTHENTICATED and the server
+   * prints a loud stderr warning at startup. Intended for localhost /
+   * trusted-network deployments only when auth is unset.
+   *
+   * For hosted deployments, always set this (via `--auth-token` flag or
+   * the `PIPEQUERY_MCP_TOKEN` environment variable).
+   */
+  authToken?: string;
+}
+
+export interface HttpServerHandle {
+  close(): Promise<void>;
+  url: string;
+}
+
+/**
+ * Stand up a standalone HTTP server hosting the MCP Streamable HTTP transport.
+ *
+ * Stateful mode: each client gets a session ID on its first (initialize)
+ * request and reuses it for subsequent tool calls. Sessions are in-memory;
+ * this is single-instance by design. Multi-tenant / horizontally-scaled
+ * hosting is out of scope for Phase 0.
+ */
+export async function runHttp(provider: Provider, opts: HttpServerOptions): Promise<HttpServerHandle> {
+  const host = opts.host ?? '127.0.0.1';
+  const port = opts.port;
+  const mountPath = opts.path ?? '/mcp';
+
+  const transports = new Map<string, StreamableHTTPServerTransport>();
+  const authToken = opts.authToken;
+
+  if (!authToken) {
+    process.stderr.write(
+      '[pipequery-mcp] WARNING: HTTP endpoint is UNAUTHENTICATED. ' +
+        'Set --auth-token or PIPEQUERY_MCP_TOKEN for anything beyond localhost.\n',
+    );
+  }
+
+  const attachTransport = async (): Promise<StreamableHTTPServerTransport> => {
+    const transport = new StreamableHTTPServerTransport({
+      sessionIdGenerator: () => randomUUID(),
+      onsessioninitialized: (sessionId) => {
+        transports.set(sessionId, transport);
+      },
+      onsessionclosed: (sessionId) => {
+        transports.delete(sessionId);
+      },
+    });
+    const mcp = createMcpServer(provider, opts.info);
+    await mcp.connect(transport);
+    return transport;
+  };
+
+  const server = createHttpServer(async (req, res) => {
+    if (!req.url) {
+      res.statusCode = 400;
+      res.end('Bad Request');
+      return;
+    }
+
+    const requestPath = req.url.split('?')[0];
+    if (requestPath !== mountPath) {
+      res.statusCode = 404;
+      res.end('Not Found');
+      return;
+    }
+
+    // Bearer-token auth check (only when a token is configured).
+    if (authToken && !isAuthorized(req, authToken)) {
+      res.statusCode = 401;
+      res.setHeader('content-type', 'application/json');
+      res.setHeader('www-authenticate', 'Bearer realm="pipequery-mcp"');
+      res.end(
+        JSON.stringify({
+          jsonrpc: '2.0',
+          error: { code: -32001, message: 'Unauthorized: missing or invalid bearer token' },
+          id: null,
+        }),
+      );
+      return;
+    }
+
+    try {
+      const body = await readJsonBody(req);
+      const sessionId = getSessionId(req);
+      let transport = sessionId ? transports.get(sessionId) : undefined;
+      if (!transport) {
+        // No session yet — this should be an initialize request. Mint a new transport.
+        transport = await attachTransport();
+      }
+      await transport.handleRequest(req, res, body);
+    } catch (err) {
+      process.stderr.write(
+        `[pipequery-mcp] transport error: ${err instanceof Error ? err.stack ?? err.message : String(err)}\n`,
+      );
+      if (!res.headersSent) {
+        res.statusCode = 500;
+        res.setHeader('content-type', 'application/json');
+        res.end(JSON.stringify({
+          jsonrpc: '2.0',
+          error: {
+            code: -32603,
+            message: err instanceof Error ? err.message : String(err),
+          },
+          id: null,
+        }));
+      } else {
+        res.end();
+      }
+    }
+  });
+
+  await new Promise<void>((resolve) => server.listen(port, host, resolve));
+
+  return {
+    url: `http://${host}:${port}${mountPath}`,
+    async close() {
+      await new Promise<void>((resolve, reject) => {
+        server.close((err) => (err ? reject(err) : resolve()));
+      });
+      for (const t of transports.values()) {
+        await t.close();
+      }
+      transports.clear();
+      await provider.dispose();
+    },
+  };
+}
+
+function isAuthorized(req: IncomingMessage, expected: string): boolean {
+  const header = req.headers['authorization'];
+  const raw = typeof header === 'string' ? header : Array.isArray(header) ? header[0] : undefined;
+  if (!raw) return false;
+  // Format: "Bearer <token>", case-insensitive scheme.
+  const match = raw.match(/^Bearer\s+(.+)$/i);
+  if (!match) return false;
+  const token = match[1].trim();
+  // Constant-time comparison — avoids timing oracles on short tokens.
+  return timingSafeEqual(token, expected);
+}
+
+function timingSafeEqual(a: string, b: string): boolean {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return diff === 0;
+}
+
+function getSessionId(req: IncomingMessage): string | undefined {
+  const header = req.headers['mcp-session-id'];
+  if (typeof header === 'string') return header;
+  if (Array.isArray(header)) return header[0];
+  return undefined;
+}
+
+async function readJsonBody(req: IncomingMessage): Promise<unknown> {
+  const method = req.method?.toUpperCase();
+  if (method === 'GET' || method === 'DELETE') return undefined;
+
+  const chunks: Buffer[] = [];
+  for await (const chunk of req) {
+    chunks.push(chunk as Buffer);
+  }
+  if (chunks.length === 0) return undefined;
+
+  const text = Buffer.concat(chunks).toString('utf8');
+  if (!text.trim()) return undefined;
+
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error('Invalid JSON body');
+  }
+}
+
+// Suppress unused-import warning in some configurations; ServerResponse is part of the signature readers care about.
+export type { ServerResponse };