@vaultgradient/pipequery-cli 0.3.3 → 0.6.0

package/src/mcp/http.ts

@@ -0,0 +1,191 @@
+import { createServer as createHttpServer } from 'node:http';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { randomUUID } from 'node:crypto';
+import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js';
+import type { Provider } from './provider.js';
+import { createMcpServer, type McpServerInfo } from './server.js';
+
+export interface HttpServerOptions {
+  host?: string;
+  port: number;
+  path?: string;
+  info?: McpServerInfo;
+  /**
+   * Require clients to send `Authorization: Bearer <token>` matching this
+   * value. When undefined, the endpoint is UNAUTHENTICATED and the server
+   * prints a loud stderr warning at startup. Intended for localhost /
+   * trusted-network deployments only when auth is unset.
+   *
+   * For hosted deployments, always set this (via `--auth-token` flag or
+   * the `PIPEQUERY_MCP_TOKEN` environment variable).
+   */
+  authToken?: string;
+}
+
+export interface HttpServerHandle {
+  close(): Promise<void>;
+  url: string;
+}
+
+/**
+ * Stand up a standalone HTTP server hosting the MCP Streamable HTTP transport.
+ *
+ * Stateful mode: each client gets a session ID on its first (initialize)
+ * request and reuses it for subsequent tool calls. Sessions are in-memory;
+ * this is single-instance by design. Multi-tenant / horizontally-scaled
+ * hosting is out of scope for Phase 0.
+ */
+export async function runHttp(provider: Provider, opts: HttpServerOptions): Promise<HttpServerHandle> {
+  const host = opts.host ?? '127.0.0.1';
+  const port = opts.port;
+  const mountPath = opts.path ?? '/mcp';
+
+  const transports = new Map<string, StreamableHTTPServerTransport>();
+  const authToken = opts.authToken;
+
+  if (!authToken) {
+    process.stderr.write(
+      '[pipequery-mcp] WARNING: HTTP endpoint is UNAUTHENTICATED. ' +
+        'Set --auth-token or PIPEQUERY_MCP_TOKEN for anything beyond localhost.\n',
+    );
+  }
+
+  const attachTransport = async (): Promise<StreamableHTTPServerTransport> => {
+    const transport = new StreamableHTTPServerTransport({
+      sessionIdGenerator: () => randomUUID(),
+      onsessioninitialized: (sessionId) => {
+        transports.set(sessionId, transport);
+      },
+      onsessionclosed: (sessionId) => {
+        transports.delete(sessionId);
+      },
+    });
+    const mcp = createMcpServer(provider, opts.info);
+    await mcp.connect(transport);
+    return transport;
+  };
+
+  const server = createHttpServer(async (req, res) => {
+    if (!req.url) {
+      res.statusCode = 400;
+      res.end('Bad Request');
+      return;
+    }
+
+    const requestPath = req.url.split('?')[0];
+    if (requestPath !== mountPath) {
+      res.statusCode = 404;
+      res.end('Not Found');
+      return;
+    }
+
+    // Bearer-token auth check (only when a token is configured).
+    if (authToken && !isAuthorized(req, authToken)) {
+      res.statusCode = 401;
+      res.setHeader('content-type', 'application/json');
+      res.setHeader('www-authenticate', 'Bearer realm="pipequery-mcp"');
+      res.end(
+        JSON.stringify({
+          jsonrpc: '2.0',
+          error: { code: -32001, message: 'Unauthorized: missing or invalid bearer token' },
+          id: null,
+        }),
+      );
+      return;
+    }
+
+    try {
+      const body = await readJsonBody(req);
+      const sessionId = getSessionId(req);
+      let transport = sessionId ? transports.get(sessionId) : undefined;
+      if (!transport) {
+        // No session yet — this should be an initialize request. Mint a new transport.
+        transport = await attachTransport();
+      }
+      await transport.handleRequest(req, res, body);
+    } catch (err) {
+      process.stderr.write(
+        `[pipequery-mcp] transport error: ${err instanceof Error ? err.stack ?? err.message : String(err)}\n`,
+      );
+      if (!res.headersSent) {
+        res.statusCode = 500;
+        res.setHeader('content-type', 'application/json');
+        res.end(JSON.stringify({
+          jsonrpc: '2.0',
+          error: {
+            code: -32603,
+            message: err instanceof Error ? err.message : String(err),
+          },
+          id: null,
+        }));
+      } else {
+        res.end();
+      }
+    }
+  });
+
+  await new Promise<void>((resolve) => server.listen(port, host, resolve));
+
+  return {
+    url: `http://${host}:${port}${mountPath}`,
+    async close() {
+      await new Promise<void>((resolve, reject) => {
+        server.close((err) => (err ? reject(err) : resolve()));
+      });
+      for (const t of transports.values()) {
+        await t.close();
+      }
+      transports.clear();
+      await provider.dispose();
+    },
+  };
+}
+
+function isAuthorized(req: IncomingMessage, expected: string): boolean {
+  const header = req.headers['authorization'];
+  const raw = typeof header === 'string' ? header : Array.isArray(header) ? header[0] : undefined;
+  if (!raw) return false;
+  // Format: "Bearer <token>", case-insensitive scheme.
+  const match = raw.match(/^Bearer\s+(.+)$/i);
+  if (!match) return false;
+  const token = match[1].trim();
+  // Constant-time comparison — avoids timing oracles on short tokens.
+  return timingSafeEqual(token, expected);
+}
+
+function timingSafeEqual(a: string, b: string): boolean {
+  if (a.length !== b.length) return false;
+  let diff = 0;
+  for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
+  return diff === 0;
+}
+
+function getSessionId(req: IncomingMessage): string | undefined {
+  const header = req.headers['mcp-session-id'];
+  if (typeof header === 'string') return header;
+  if (Array.isArray(header)) return header[0];
+  return undefined;
+}
+
+async function readJsonBody(req: IncomingMessage): Promise<unknown> {
+  const method = req.method?.toUpperCase();
+  if (method === 'GET' || method === 'DELETE') return undefined;
+
+  const chunks: Buffer[] = [];
+  for await (const chunk of req) {
+    chunks.push(chunk as Buffer);
+  }
+  if (chunks.length === 0) return undefined;
+
+  const text = Buffer.concat(chunks).toString('utf8');
+  if (!text.trim()) return undefined;
+
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new Error('Invalid JSON body');
+  }
+}
+
+// Suppress unused-import warning in some configurations; ServerResponse is part of the signature readers care about.
+export type { ServerResponse };

package/src/mcp/provider.ts

@@ -0,0 +1,223 @@
+import type { DataContext } from '../engine.js';
+import type { EndpointConfig, PipeQueryConfig } from '../config/schema.js';
+import type { SourceStatus } from '../server/sources/types.js';
+import { SourceManager } from '../server/sources/manager.js';
+import { query } from '../engine.js';
+
+export interface ProviderSourceInfo {
+  name: string;
+  status: SourceStatus;
+}
+
+export interface ProviderSourceDescription {
+  name: string;
+  status: SourceStatus;
+  sample: unknown[];
+  fields: string[];
+}
+
+export interface Provider {
+  listSources(): Promise<ProviderSourceInfo[]>;
+  describeSource(name: string, sampleSize?: number): Promise<ProviderSourceDescription | undefined>;
+  listEndpoints(): Promise<Array<{ path: string; config: EndpointConfig }>>;
+  callEndpoint(path: string): Promise<unknown>;
+  runQuery(expression: string): Promise<unknown>;
+  dispose(): Promise<void>;
+}
+
+// ─── LocalProvider ──────────────────────────────────────────────────────────
+// Boots its own SourceManager from a loaded config. Used when `pq mcp serve`
+// runs standalone (no attached `pq serve` daemon).
+
+export class LocalProvider implements Provider {
+  private sourceManager: SourceManager;
+  private endpoints: Map<string, EndpointConfig>;
+  private ready: Promise<void>;
+
+  constructor(private config: PipeQueryConfig, cwd: string) {
+    this.sourceManager = new SourceManager(cwd);
+    this.endpoints = new Map(Object.entries(config.endpoints));
+    this.ready = this.bootSources();
+  }
+
+  private async bootSources(): Promise<void> {
+    for (const [name, src] of Object.entries(this.config.sources)) {
+      try {
+        await this.sourceManager.addSource(name, src);
+      } catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        // Print to stderr; stdout is reserved for the MCP JSON-RPC stream in stdio mode.
+        process.stderr.write(`[pipequery-mcp] Source "${name}" failed to start: ${msg}\n`);
+      }
+    }
+  }
+
+  async listSources(): Promise<ProviderSourceInfo[]> {
+    await this.ready;
+    const statuses = this.sourceManager.getAllStatuses();
+    return Object.entries(statuses).map(([name, status]) => ({ name, status }));
+  }
+
+  async describeSource(name: string, sampleSize = 5): Promise<ProviderSourceDescription | undefined> {
+    await this.ready;
+    const status = this.sourceManager.getSourceStatus(name);
+    const data = this.sourceManager.getSourceData(name);
+    if (!status || !data) return undefined;
+
+    const sample = data.slice(0, sampleSize);
+    const fields = inferFields(data);
+    return { name, status, sample, fields };
+  }
+
+  async listEndpoints(): Promise<Array<{ path: string; config: EndpointConfig }>> {
+    await this.ready;
+    return [...this.endpoints].map(([path, config]) => ({ path, config }));
+  }
+
+  async callEndpoint(path: string): Promise<unknown> {
+    await this.ready;
+    const endpoint = this.endpoints.get(path);
+    if (!endpoint) {
+      throw new Error(`Endpoint "${path}" not found`);
+    }
+    const context = this.sourceManager.getContext();
+    return query(context, endpoint.query);
+  }
+
+  async runQuery(expression: string): Promise<unknown> {
+    await this.ready;
+    const context = this.sourceManager.getContext();
+    return query(context, expression);
+  }
+
+  async dispose(): Promise<void> {
+    await this.sourceManager.dispose();
+  }
+
+  // Exposed for the HTTP plugin that wants to mount the MCP server into an
+  // existing `createServer` instance; the caller passes in their own manager.
+  static withSourceManager(sourceManager: SourceManager, endpoints: Map<string, EndpointConfig>): Provider {
+    return new AttachedLocalProvider(sourceManager, endpoints);
+  }
+}
+
+class AttachedLocalProvider implements Provider {
+  constructor(private sourceManager: SourceManager, private endpoints: Map<string, EndpointConfig>) {}
+
+  async listSources(): Promise<ProviderSourceInfo[]> {
+    const statuses = this.sourceManager.getAllStatuses();
+    return Object.entries(statuses).map(([name, status]) => ({ name, status }));
+  }
+
+  async describeSource(name: string, sampleSize = 5): Promise<ProviderSourceDescription | undefined> {
+    const status = this.sourceManager.getSourceStatus(name);
+    const data = this.sourceManager.getSourceData(name);
+    if (!status || !data) return undefined;
+    return { name, status, sample: data.slice(0, sampleSize), fields: inferFields(data) };
+  }
+
+  async listEndpoints() {
+    return [...this.endpoints].map(([path, config]) => ({ path, config }));
+  }
+
+  async callEndpoint(path: string): Promise<unknown> {
+    const endpoint = this.endpoints.get(path);
+    if (!endpoint) throw new Error(`Endpoint "${path}" not found`);
+    return query(this.sourceManager.getContext(), endpoint.query);
+  }
+
+  async runQuery(expression: string): Promise<unknown> {
+    return query(this.sourceManager.getContext(), expression);
+  }
+
+  async dispose(): Promise<void> {
+    // Intentionally noop: the owning server disposes the SourceManager.
+  }
+}
+
+// ─── AttachedProvider ───────────────────────────────────────────────────────
+// Talks to a running `pq serve` via its existing /api/_control/* endpoints.
+
+export class AttachedProvider implements Provider {
+  constructor(private baseUrl: string) {
+    this.baseUrl = baseUrl.replace(/\/$/, '');
+  }
+
+  private async get<T>(path: string): Promise<T> {
+    const res = await fetch(`${this.baseUrl}${path}`);
+    if (!res.ok) throw new Error(`GET ${path} failed: HTTP ${res.status}`);
+    return res.json() as Promise<T>;
+  }
+
+  private async post<T>(path: string, body: unknown): Promise<T> {
+    const res = await fetch(`${this.baseUrl}${path}`, {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify(body),
+    });
+    if (!res.ok) throw new Error(`POST ${path} failed: HTTP ${res.status}`);
+    return res.json() as Promise<T>;
+  }
+
+  async listSources(): Promise<ProviderSourceInfo[]> {
+    const statuses = await this.get<Record<string, SourceStatus>>('/api/_control/sources');
+    return Object.entries(statuses).map(([name, status]) => ({ name, status }));
+  }
+
+  async describeSource(name: string, sampleSize = 5): Promise<ProviderSourceDescription | undefined> {
+    const res = await this.post<{
+      ok: boolean;
+      error?: string;
+      rowCount?: number;
+      sample?: unknown[];
+      status?: SourceStatus;
+    }>(`/api/_control/sources/${encodeURIComponent(name)}/test`, {});
+    if (!res.ok || !res.status || !res.sample) return undefined;
+    return {
+      name,
+      status: res.status,
+      sample: res.sample.slice(0, sampleSize),
+      fields: inferFields(res.sample),
+    };
+  }
+
+  async listEndpoints() {
+    const endpoints = await this.get<Record<string, EndpointConfig>>('/api/_control/endpoints');
+    return Object.entries(endpoints).map(([path, config]) => ({ path, config }));
+  }
+
+  async callEndpoint(path: string): Promise<unknown> {
+    // Just hit the actual public endpoint on the running server.
+    const normalized = path.startsWith('/') ? path : `/${path}`;
+    const res = await fetch(`${this.baseUrl}${normalized}`);
+    if (!res.ok) throw new Error(`GET ${normalized} failed: HTTP ${res.status}`);
+    return res.json();
+  }
+
+  async runQuery(expression: string): Promise<unknown> {
+    const res = await this.post<{ ok: boolean; result?: unknown; error?: string }>(
+      '/api/_control/query',
+      { query: expression },
+    );
+    if (!res.ok) throw new Error(res.error ?? 'Query failed');
+    return res.result;
+  }
+
+  async dispose(): Promise<void> {
+    // Stateless — nothing to clean up.
+  }
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function inferFields(rows: unknown[]): string[] {
+  const fields = new Set<string>();
+  for (const row of rows.slice(0, 20)) {
+    if (row && typeof row === 'object' && !Array.isArray(row)) {
+      for (const k of Object.keys(row)) fields.add(k);
+    }
+  }
+  return [...fields];
+}
+
+export type { DataContext };

package/src/mcp/server.ts

@@ -0,0 +1,138 @@
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { z } from 'zod';
+import type { Provider } from './provider.js';
+
+export interface McpServerInfo {
+  name: string;
+  version: string;
+}
+
+const DEFAULT_INFO: McpServerInfo = {
+  name: 'pipequery',
+  version: '0.1.0',
+};
+
+/**
+ * Build an MCP server wired to the supplied provider.
+ *
+ * Tools exposed:
+ *   - query              run a PipeQuery expression against live sources
+ *   - list_sources       list configured sources and their health
+ *   - describe_source    return a sample + inferred field names for a source
+ *   - list_endpoints     list pre-configured endpoints and their stored queries
+ *   - call_endpoint      execute a pre-configured endpoint by path
+ */
+export function createMcpServer(provider: Provider, info: McpServerInfo = DEFAULT_INFO): McpServer {
+  const mcp = new McpServer(info);
+
+  mcp.registerTool(
+    'query',
+    {
+      title: 'Run a PipeQuery expression',
+      description:
+        'Execute a PipeQuery pipe-based expression (e.g. `crypto | where(price > 100) | sort(price desc) | first(5)`) against all currently configured sources. Returns the computed result as JSON.',
+      inputSchema: {
+        expression: z
+          .string()
+          .min(1)
+          .describe('The PipeQuery expression. Source names used in the expression must be configured; call `list_sources` to see them.'),
+      },
+    },
+    async ({ expression }) => {
+      const result = await provider.runQuery(expression);
+      return toolResult(result);
+    },
+  );
+
+  mcp.registerTool(
+    'list_sources',
+    {
+      title: 'List configured data sources',
+      description:
+        'List every data source configured in pipequery.yaml along with its current health, row count, and last-fetch timestamp.',
+    },
+    async () => {
+      const sources = await provider.listSources();
+      return toolResult({ count: sources.length, sources });
+    },
+  );
+
+  mcp.registerTool(
+    'describe_source',
+    {
+      title: 'Describe a data source',
+      description:
+        'Return a sample of rows and the inferred field names for a single source. Useful before constructing a query.',
+      inputSchema: {
+        name: z.string().min(1).describe('Source name as returned by list_sources.'),
+        sample_size: z
+          .number()
+          .int()
+          .positive()
+          .max(100)
+          .optional()
+          .describe('Number of rows to include in the sample. Defaults to 5, max 100.'),
+      },
+    },
+    async ({ name, sample_size }) => {
+      const description = await provider.describeSource(name, sample_size);
+      if (!description) {
+        return toolError(`Source "${name}" not found`);
+      }
+      return toolResult(description);
+    },
+  );
+
+  mcp.registerTool(
+    'list_endpoints',
+    {
+      title: 'List pre-configured query endpoints',
+      description:
+        'Return every endpoint declared in pipequery.yaml along with its stored query expression. These endpoints can be invoked via `call_endpoint`.',
+    },
+    async () => {
+      const endpoints = await provider.listEndpoints();
+      return toolResult({ count: endpoints.length, endpoints });
+    },
+  );
+
+  mcp.registerTool(
+    'call_endpoint',
+    {
+      title: 'Invoke a pre-configured endpoint',
+      description:
+        'Execute the stored query for an endpoint path (e.g. `/api/top-coins`) and return its JSON result. Equivalent to the HTTP endpoint on `pq serve`, without needing a live HTTP call.',
+      inputSchema: {
+        path: z.string().min(1).describe('Endpoint path, exactly as declared in pipequery.yaml (typically begins with `/`).'),
+      },
+    },
+    async ({ path }) => {
+      try {
+        const result = await provider.callEndpoint(path);
+        return toolResult(result);
+      } catch (err) {
+        return toolError(err instanceof Error ? err.message : String(err));
+      }
+    },
+  );
+
+  return mcp;
+}
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function toolResult(data: unknown): { content: Array<{ type: 'text'; text: string }> } {
+  return {
+    content: [{ type: 'text', text: JSON.stringify(data, null, 2) }],
+  };
+}
+
+function toolError(message: string): {
+  content: Array<{ type: 'text'; text: string }>;
+  isError: true;
+} {
+  return {
+    content: [{ type: 'text', text: message }],
+    isError: true,
+  };
+}

package/src/mcp/stdio.ts

@@ -0,0 +1,26 @@
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import type { Provider } from './provider.js';
+import { createMcpServer, type McpServerInfo } from './server.js';
+
+export async function runStdio(provider: Provider, info?: McpServerInfo): Promise<void> {
+  const mcp = createMcpServer(provider, info);
+  const transport = new StdioServerTransport();
+  await mcp.connect(transport);
+
+  // Clean up on stdio close — Claude Desktop etc. close the child process
+  // by closing its stdin.
+  const cleanup = async () => {
+    try {
+      await mcp.close();
+    } finally {
+      await provider.dispose();
+    }
+  };
+
+  process.stdin.on('close', () => {
+    void cleanup().then(() => process.exit(0));
+  });
+
+  process.on('SIGINT', () => void cleanup().then(() => process.exit(0)));
+  process.on('SIGTERM', () => void cleanup().then(() => process.exit(0)));
+}

package/src/server/app.ts

@@ -2,6 +2,7 @@ import Fastify from 'fastify';
 import cors from '@fastify/cors';
 import type { PipeQueryConfig, EndpointConfig, SourceConfig } from '../config/schema.js';
 import { SourceManager } from './sources/manager.js';
+import { WatchManager } from './watches/manager.js';
 import { registerHealthRoutes } from './routes/health.js';
 import { registerControlRoutes } from './routes/control.js';
 import { registerDynamicRoutes } from './routes/dynamic.js';
@@ -14,6 +15,7 @@ export interface ServerContext {
   app: ReturnType<typeof Fastify>;
   sourceManager: SourceManager;
   endpoints: Map<string, EndpointConfig>;
+  watchManager: WatchManager;
 }
 
 export async function createServer(
@@ -95,7 +97,14 @@ export async function createServer(
   registerSSERoutes(app, sourceManager);
   registerDynamicRoutes(app, sourceManager, endpoints);
 
+  // Watches: register and start polling. Failures in any single watch are
+  // isolated — see WatchManager for the per-watch error handling.
+  const watchManager = new WatchManager(sourceManager);
+  if (config.watches && Object.keys(config.watches).length > 0) {
+    watchManager.start(config.watches);
+  }
+
   activity.info('Server ready');
 
-  return { app, sourceManager, endpoints };
+  return { app, sourceManager, endpoints, watchManager };
 }