@vaultgradient/pipequery-cli 0.7.0 → 0.10.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vaultgradient/pipequery-cli",
-  "version": "0.7.0",
+  "version": "0.10.0",
   "description": "CLI for PipeQuery — data sources, aggregations, APIs, and terminal dashboards",
   "type": "module",
   "bin": {
@@ -14,6 +14,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.91.1",
+    "@clickhouse/client": "^1.18.3",
     "@fastify/cors": "^11.2.0",
     "@modelcontextprotocol/sdk": "^1.29.0",
     "better-sqlite3": "^12.9.0",
@@ -25,10 +26,12 @@
     "grammy": "^1.42.0",
     "ink": "^7.0.1",
     "kafkajs": "^2.2.4",
+    "mongodb": "^7.2.0",
     "mysql2": "^3.22.2",
     "ora": "^9.3.0",
     "pg": "^8.20.0",
     "react": "^19.2.5",
+    "snowflake-sdk": "^2.4.0",
     "ws": "^8.20.0",
     "yaml": "^2.8.3",
     "zod": "^4.3.6"
@@ -43,11 +46,17 @@
     "typescript": "~6.0.0",
     "vitest": "^4.1.4"
   },
+  "overrides": {
+    "gcp-metadata": "^8.0.0"
+  },
   "repository": {
     "type": "git",
     "url": "https://github.com/andreadito/pipequery.git",
     "directory": "cli"
   },
   "author": "andreadito",
-  "license": "MIT"
+  "license": "MIT",
+  "engines": {
+    "node": ">=22"
+  }
 }

package/src/commands/telegram.ts

@@ -1,14 +1,17 @@
+import { resolve } from 'node:path';
 import { loadConfig } from '../config/loader.js';
 import { log } from '../utils/logger.js';
 import { LocalProvider, AttachedProvider, type Provider } from '../mcp/provider.js';
 import { buildBot } from '../telegram/bot.js';
 import { createNLTranslator } from '../telegram/nl.js';
+import { createEventLogger } from '../telegram/event-log.js';
 
 interface TelegramServeOptions {
   botToken?: string;
   attach?: string;
   allowUser?: string[];
   anthropicKey?: string;
+  logFile?: string;
 }
 
 export async function telegramServeCommand(opts: TelegramServeOptions): Promise<void> {
@@ -26,9 +29,13 @@ export async function telegramServeCommand(opts: TelegramServeOptions): Promise<
     ? createNLTranslator(provider, { apiKey: anthropicKey })
     : undefined;
 
+  const logFilePath = opts.logFile ? resolve(process.cwd(), opts.logFile) : undefined;
+  const logger = createEventLogger({ filePath: logFilePath });
+
   const bot = buildBot(token, provider, {
     allowUsers: opts.allowUser ?? [],
     nl,
+    logger,
   });
 
   const me = await bot.api.getMe();
@@ -38,6 +45,9 @@ export async function telegramServeCommand(opts: TelegramServeOptions): Promise<
   } else {
     log.dim('Natural-language translation disabled. Pass --anthropic-key or set ANTHROPIC_API_KEY to enable.');
   }
+  if (logFilePath) {
+    log.info(`Event log → ${logFilePath} (JSONL)`);
+  }
   if (!opts.allowUser || opts.allowUser.length === 0) {
     log.warn('No --allow-user set: anyone with the bot username can query. Add allowlist for production.');
   }

package/src/config/schema.ts

@@ -26,7 +26,10 @@ export type SourceConfig =
   | PostgresSourceConfig
   | MysqlSourceConfig
   | SqliteSourceConfig
-  | KafkaSourceConfig;
+  | KafkaSourceConfig
+  | SnowflakeSourceConfig
+  | ClickhouseSourceConfig
+  | MongoSourceConfig;
 
 export interface RestSourceConfig {
   type: 'rest';
@@ -245,3 +248,67 @@ export interface TelegramNotifyConfig {
    */
   message?: string;
 }
+
+// ─── Snowflake / ClickHouse / MongoDB ───────────────────────────────────────
+
+export interface SnowflakeSourceConfig {
+  type: 'snowflake';
+  /** Snowflake account locator, e.g. `myorg-myaccount`. Supports `${ENV_VAR}`. */
+  account: string;
+  /** Username. Supports `${ENV_VAR}`. */
+  username: string;
+  /** Password. Supports `${ENV_VAR}`. (Key-pair / OAuth out of scope for v1.) */
+  password: string;
+  /** Database to USE. Supports `${ENV_VAR}`. */
+  database?: string;
+  /** Schema to USE. Supports `${ENV_VAR}`. */
+  schema?: string;
+  /** Compute warehouse to USE. Supports `${ENV_VAR}`. */
+  warehouse?: string;
+  /** Optional role. Supports `${ENV_VAR}`. */
+  role?: string;
+  /** SELECT query polled on every interval. */
+  query: string;
+  /** Poll interval, e.g. "60s". Default: 60s. */
+  interval?: string;
+  /** Hard cap on rows per fetch (default 10000). */
+  maxRows?: number;
+}
+
+export interface ClickhouseSourceConfig {
+  type: 'clickhouse';
+  /** ClickHouse HTTP/HTTPS endpoint, e.g. `https://host:8443`. Supports `${ENV_VAR}`. */
+  url: string;
+  /** Username. Supports `${ENV_VAR}`. */
+  username?: string;
+  /** Password. Supports `${ENV_VAR}`. */
+  password?: string;
+  /** Database to query. Defaults to `default`. */
+  database?: string;
+  /** SELECT query polled on every interval. */
+  query: string;
+  /** Poll interval, e.g. "30s". Default: 30s. */
+  interval?: string;
+  /** Hard cap on rows per fetch (default 10000). */
+  maxRows?: number;
+}
+
+export interface MongoSourceConfig {
+  type: 'mongodb';
+  /** MongoDB connection URI. Supports `${ENV_VAR}`. */
+  url: string;
+  /** Database name. Supports `${ENV_VAR}`. */
+  database: string;
+  /** Collection to read from. */
+  collection: string;
+  /**
+   * Optional default filter applied on every poll (Mongo `$match` stage).
+   * Plain JSON object — `{ active: true }` etc. The filter is `AND`-merged
+   * with any push-down `where()` predicates the engine compiles in.
+   */
+  filter?: Record<string, unknown>;
+  /** Poll interval, e.g. "30s". Default: 30s. */
+  interval?: string;
+  /** Hard cap on documents per fetch (default 10000). */
+  maxRows?: number;
+}

package/src/index.ts

@@ -71,7 +71,7 @@ source
 source
   .command('add <name>')
   .description('Add a data source')
-  .requiredOption('-t, --type <type>', 'Source type: rest, websocket, file, postgres, mysql, sqlite, kafka')
+  .requiredOption('-t, --type <type>', 'Source type: rest, websocket, file, postgres, mysql, sqlite, kafka, snowflake, clickhouse, mongodb')
   .option('-u, --url <url>', 'Connection URL for REST / WebSocket / Postgres / MySQL sources')
   .option('-i, --interval <interval>', 'Poll interval (e.g., 30s, 5m)', '30s')
   .option('-d, --data-path <path>', 'Dot-path to extract data from REST response')
@@ -185,6 +185,7 @@ telegram
   .option('--attach <url>', 'Attach to a running `pq serve` instance instead of loading pipequery.yaml locally')
   .option('--allow-user <handle>', 'Allowlist a user (numeric id or @username); repeatable. With no allowlist, anyone with the bot username can query.', collectRepeated, [])
   .option('--anthropic-key <key>', 'Anthropic API key (or set $ANTHROPIC_API_KEY) — enables natural-language → pipequery translation on plain-text messages via claude-haiku-4-5.')
+  .option('--log-file <path>', 'Append every bot event (commands, NL queries, unauthorized attempts) as JSONL to <path>. Stderr always carries a colored, human-readable copy.')
   .action(telegramServeCommand);
 
 // ─── pq watch ────────────────────────────────────────────────────────────────

package/src/mcp/provider.ts

@@ -2,7 +2,6 @@ import type { DataContext } from '../engine.js';
 import type { EndpointConfig, PipeQueryConfig } from '../config/schema.js';
 import type { SourceStatus } from '../server/sources/types.js';
 import { SourceManager } from '../server/sources/manager.js';
-import { query } from '../engine.js';
 
 export interface ProviderSourceInfo {
   name: string;
@@ -80,14 +79,12 @@ export class LocalProvider implements Provider {
     if (!endpoint) {
       throw new Error(`Endpoint "${path}" not found`);
     }
-    const context = this.sourceManager.getContext();
-    return query(context, endpoint.query);
+    return this.sourceManager.runQuery(endpoint.query);
   }
 
   async runQuery(expression: string): Promise<unknown> {
     await this.ready;
-    const context = this.sourceManager.getContext();
-    return query(context, expression);
+    return this.sourceManager.runQuery(expression);
   }
 
   async dispose(): Promise<void> {
@@ -123,11 +120,11 @@ class AttachedLocalProvider implements Provider {
   async callEndpoint(path: string): Promise<unknown> {
     const endpoint = this.endpoints.get(path);
     if (!endpoint) throw new Error(`Endpoint "${path}" not found`);
-    return query(this.sourceManager.getContext(), endpoint.query);
+    return this.sourceManager.runQuery(endpoint.query);
   }
 
   async runQuery(expression: string): Promise<unknown> {
-    return query(this.sourceManager.getContext(), expression);
+    return this.sourceManager.runQuery(expression);
   }
 
   async dispose(): Promise<void> {

package/src/server/routes/control.ts

@@ -1,5 +1,4 @@
 import type { FastifyInstance } from 'fastify';
-import { query } from '../../engine.js';
 import type { SourceConfig, EndpointConfig } from '../../config/schema.js';
 import type { SourceManager } from '../sources/manager.js';
 
@@ -89,9 +88,8 @@ export function registerControlRoutes(
 
   app.post<{ Body: { query: string } }>('/api/_control/query', async (req) => {
     const { query: expr } = req.body;
-    const context = sourceManager.getContext();
     try {
-      const result = query(context, expr);
+      const result = await sourceManager.runQuery(expr);
       return { ok: true, result };
     } catch (err) {
       return { ok: false, error: err instanceof Error ? err.message : String(err) };

package/src/server/routes/dynamic.ts

@@ -1,5 +1,4 @@
 import type { FastifyInstance } from 'fastify';
-import { query } from '../../engine.js';
 import type { EndpointConfig } from '../../config/schema.js';
 import type { SourceManager } from '../sources/manager.js';
 import { ResponseCache } from '../cache.js';
@@ -32,10 +31,9 @@ export function registerDynamicRoutes(
       }
     }
 
-    // Execute query
-    const context = sourceManager.getContext();
+    // Execute query (push-down preferred when the source supports it)
     try {
-      const result = query(context, config.query);
+      const result = await sourceManager.runQuery(config.query);
 
       if (config.cache) {
         cache.set(path, result, parseDuration(config.cache));

package/src/server/sources/clickhouse.ts

@@ -0,0 +1,153 @@
+import { createClient, type ClickHouseClient } from '@clickhouse/client';
+import type { ClickhouseSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import { expandEnv } from '../../utils/expandEnv.js';
+import { parseQuery } from '../../engine.js';
+import { compileClickhousePushdown } from './pushdown/clickhouse.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+
+export class ClickhouseSourceAdapter implements SourceAdapter {
+  private client: ClickHouseClient | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+
+  constructor(private config: ClickhouseSourceConfig) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+  }
+
+  async start(): Promise<void> {
+    this.client = createClient({
+      url: expandEnv(this.config.url, 'clickhouse source URL'),
+      ...(this.config.username
+        ? { username: expandEnv(this.config.username, 'clickhouse source username') }
+        : {}),
+      ...(this.config.password
+        ? { password: expandEnv(this.config.password, 'clickhouse source password') }
+        : {}),
+      ...(this.config.database ? { database: this.config.database } : {}),
+    });
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.client) {
+      void this.client.close();
+      this.client = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  /**
+   * Adapter-native push-down. ClickHouse dialect inlines literals (vs.
+   * Postgres / MySQL / Snowflake which bind via placeholders) so the
+   * `params` array returned in the success result is always empty;
+   * the SQL is fully self-contained.
+   */
+  async runPushdown(expression: string): Promise<
+    | { ok: true; rows: unknown[]; sql: string; params: unknown[] }
+    | { ok: false; reason: string }
+  > {
+    if (!this.client) {
+      return { ok: false, reason: 'ClickHouse client not initialized — call start() first' };
+    }
+    let pipeline;
+    try {
+      pipeline = parseQuery(expression);
+    } catch (err) {
+      return { ok: false, reason: `parse error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const compiled = compileClickhousePushdown(pipeline, this.config.query);
+    if (!compiled.ok) {
+      return { ok: false, reason: compiled.reason };
+    }
+    try {
+      const rows = await this.executeSql(compiled.compiled.sql);
+      return {
+        ok: true,
+        rows,
+        sql: compiled.compiled.sql,
+        params: compiled.compiled.params,
+      };
+    } catch (err) {
+      return { ok: false, reason: `clickhouse error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.client) {
+      this.error = 'ClickHouse client not initialized';
+      return;
+    }
+    try {
+      const capQuery = `SELECT * FROM (${stripTrailingSemicolon(this.config.query)}) AS pq_src LIMIT ${this.maxRows + 1}`;
+      const rows = await this.executeSql(capQuery);
+      if (rows.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Add LIMIT/WHERE to bound the result set, or raise maxRows in the source config.`,
+        );
+      }
+      this.data = rows;
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private async executeSql(sql: string): Promise<unknown[]> {
+    const result = await this.client!.query({
+      query: sql,
+      format: 'JSONEachRow',
+    });
+    const rows = await result.json();
+    return Array.isArray(rows) ? rows : [];
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}
+
+function stripTrailingSemicolon(sql: string): string {
+  return sql.trim().replace(/;+\s*$/, '');
+}

package/src/server/sources/manager.ts

@@ -1,5 +1,5 @@
 import type { SourceConfig } from '../../config/schema.js';
-import type { SourceAdapter, SourceStatus } from './types.js';
+import type { PushdownResult, SourceAdapter, SourceStatus } from './types.js';
 import { RestSourceAdapter } from './rest.js';
 import { WebSocketSourceAdapter } from './websocket.js';
 import { FileSourceAdapter } from './file.js';
@@ -8,6 +8,10 @@ import { PostgresSourceAdapter } from './postgres.js';
 import { MysqlSourceAdapter } from './mysql.js';
 import { SqliteSourceAdapter } from './sqlite.js';
 import { KafkaSourceAdapter } from './kafka.js';
+import { SnowflakeSourceAdapter } from './snowflake.js';
+import { ClickhouseSourceAdapter } from './clickhouse.js';
+import { MongoSourceAdapter } from './mongodb.js';
+import { parseQuery, query } from '../../engine.js';
 
 export type DataContext = Record<string, unknown[]>;
 
@@ -98,6 +102,66 @@ export class SourceManager {
     );
   }
 
+  /**
+   * Run a pipe expression, preferring adapter-native push-down when the
+   * source supports it.
+   *
+   * Decision tree:
+   *   1. Parse the expression. If it can't parse, fall straight through to
+   *      the in-process engine, which will throw the same parse error a
+   *      caller would have seen before push-down existed (no behaviour
+   *      change for malformed input).
+   *   2. Look up the source named in the pipeline. If it exists and its
+   *      adapter exposes `runPushdown`, try it. On `{ ok: true }` we're
+   *      done — return the rows. On `{ ok: false }` (decline) or any
+   *      thrown error, fall back to in-process.
+   *   3. In-process: query(getContext(), expression) — the existing path.
+   *
+   * The push-down attempt is best-effort and *invisible to callers*: same
+   * return type as `query()`, no new failure modes. Telemetry (which
+   * branch ran, latency, source-engine SQL) belongs in the audit log when
+   * Phase 5 lands; not threaded through here.
+   */
+  async runQuery(expression: string): Promise<unknown> {
+    const pushed = await this.tryPushdown(expression);
+    if (pushed && pushed.ok) return pushed.rows;
+    return query(this.getContext(), expression);
+  }
+
+  /**
+   * Inspect the pipeline AST and dispatch to an adapter's runPushdown when
+   * eligible. Returns `null` when push-down isn't even attempted (no parse,
+   * no source, no capability). Returns a PushdownResult when it was tried —
+   * `{ ok: true }` for success, `{ ok: false, reason }` for decline.
+   *
+   * Exposed on the manager (rather than tucked inside runQuery) so the
+   * future `event.audit` channel can record `pushed_down: bool` and the
+   * `source_engine_query` SQL string for governance. Today only Postgres
+   * implements runPushdown; MySQL / SQLite / ClickHouse will follow the
+   * same interface.
+   */
+  async tryPushdown(expression: string): Promise<PushdownResult | null> {
+    let pipeline;
+    try {
+      pipeline = parseQuery(expression);
+    } catch {
+      // Bad expression — let the in-process engine raise the canonical error.
+      return null;
+    }
+    const entry = this.sources.get(pipeline.source);
+    if (!entry) return null;
+    if (!entry.adapter.runPushdown) return null;
+    try {
+      return await entry.adapter.runPushdown(expression);
+    } catch (err) {
+      // A throw from runPushdown is treated as a decline: same fallback
+      // path as `{ ok: false }`. We don't surface the thrown error because
+      // the in-process retry will produce a more useful one (or succeed).
+      const reason = err instanceof Error ? err.message : String(err);
+      return { ok: false, reason: `pushdown threw: ${reason}` };
+    }
+  }
+
   async dispose(): Promise<void> {
     this.globalListeners.clear();
     for (const entry of this.sources.values()) {
@@ -125,6 +189,12 @@ export class SourceManager {
         return new SqliteSourceAdapter(config, this.cwd);
       case 'kafka':
         return new KafkaSourceAdapter(config);
+      case 'snowflake':
+        return new SnowflakeSourceAdapter(config);
+      case 'clickhouse':
+        return new ClickhouseSourceAdapter(config);
+      case 'mongodb':
+        return new MongoSourceAdapter(config);
       default:
         throw new Error(`Unknown source type: ${(config as { type: string }).type}`);
     }

package/src/server/sources/mongodb.ts

@@ -0,0 +1,165 @@
+import { MongoClient, type Db, type Collection } from 'mongodb';
+import type { MongoSourceConfig } from '../../config/schema.js';
+import { parseDuration } from '../../utils/parseDuration.js';
+import { expandEnv } from '../../utils/expandEnv.js';
+import { parseQuery } from '../../engine.js';
+import { compileMongoPushdown, type MongoPlan } from './pushdown/mongodb.js';
+import type { SourceAdapter, SourceStatus } from './types.js';
+
+const DEFAULT_INTERVAL_MS = 30_000;
+const DEFAULT_MAX_ROWS = 10_000;
+
+export class MongoSourceAdapter implements SourceAdapter {
+  private client: MongoClient | null = null;
+  private db: Db | null = null;
+  private collection: Collection | null = null;
+  private data: unknown[] = [];
+  private timer: ReturnType<typeof setInterval> | null = null;
+  private lastFetch: Date | null = null;
+  private error?: string;
+  private listeners = new Set<(data: unknown[]) => void>();
+  private readonly intervalMs: number;
+  private readonly maxRows: number;
+
+  constructor(private config: MongoSourceConfig) {
+    this.intervalMs = config.interval ? parseDuration(config.interval) : DEFAULT_INTERVAL_MS;
+    this.maxRows = config.maxRows ?? DEFAULT_MAX_ROWS;
+  }
+
+  async start(): Promise<void> {
+    const uri = expandEnv(this.config.url, 'mongodb source URL');
+    this.client = new MongoClient(uri);
+    await this.client.connect();
+    this.db = this.client.db(expandEnv(this.config.database, 'mongodb source database'));
+    this.collection = this.db.collection(this.config.collection);
+
+    await this.fetch();
+    this.timer = setInterval(() => {
+      void this.fetch();
+    }, this.intervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = null;
+    }
+    if (this.client) {
+      void this.client.close();
+      this.client = null;
+      this.db = null;
+      this.collection = null;
+    }
+  }
+
+  getData(): unknown[] {
+    return this.data;
+  }
+
+  getStatus(): SourceStatus {
+    return {
+      healthy: !this.error,
+      rowCount: this.data.length,
+      lastFetch: this.lastFetch,
+      error: this.error,
+    };
+  }
+
+  onUpdate(callback: (data: unknown[]) => void): () => void {
+    this.listeners.add(callback);
+    return () => this.listeners.delete(callback);
+  }
+
+  async refresh(): Promise<void> {
+    await this.fetch();
+  }
+
+  /**
+   * Adapter-native push-down to MongoDB. The compiler emits either a
+   * `find()` plan (when the pipeline only uses where/sort/first/select)
+   * or an `aggregate()` pipeline (when grouping is involved).
+   *
+   * The success result's `sql` field carries a JSON serialization of the
+   * compiled plan (Mongo doesn't have SQL); `params` is empty since
+   * Mongo plans bind values inline as part of the BSON document.
+   */
+  async runPushdown(expression: string): Promise<
+    | { ok: true; rows: unknown[]; sql: string; params: unknown[] }
+    | { ok: false; reason: string }
+  > {
+    if (!this.collection) {
+      return { ok: false, reason: 'Mongo collection not initialized — call start() first' };
+    }
+    let pipeline;
+    try {
+      pipeline = parseQuery(expression);
+    } catch (err) {
+      return { ok: false, reason: `parse error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const compiled = compileMongoPushdown(pipeline, this.config.filter);
+    if (!compiled.ok) {
+      return { ok: false, reason: compiled.reason };
+    }
+    try {
+      const rows = await this.runPlan(compiled.compiled);
+      return {
+        ok: true,
+        rows,
+        sql: JSON.stringify(compiled.compiled),
+        params: [],
+      };
+    } catch (err) {
+      return { ok: false, reason: `mongo error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+
+  private async runPlan(plan: MongoPlan): Promise<unknown[]> {
+    const coll = this.collection!;
+    if (plan.kind === 'find') {
+      let cursor = coll.find(plan.filter);
+      if (plan.projection) cursor = cursor.project(plan.projection);
+      if (plan.sort) cursor = cursor.sort(plan.sort);
+      if (plan.limit !== undefined) cursor = cursor.limit(plan.limit);
+      return cursor.toArray();
+    }
+    // aggregate
+    return coll.aggregate(plan.pipeline).toArray();
+  }
+
+  private async fetch(): Promise<void> {
+    if (!this.collection) {
+      this.error = 'Mongo collection not initialized';
+      return;
+    }
+    try {
+      // Apply the optional default filter (yaml-level) plus the maxRows cap.
+      const filter = this.config.filter ?? {};
+      const docs = await this.collection
+        .find(filter)
+        .limit(this.maxRows + 1)
+        .toArray();
+      if (docs.length > this.maxRows) {
+        throw new Error(
+          `Query returned more than maxRows=${this.maxRows}. ` +
+            `Tighten the filter, or raise maxRows in the source config.`,
+        );
+      }
+      // Strip Mongo's BSON _id by default — most pipequery users don't want it
+      // surfacing in result rows. The original is still queryable in source
+      // queries that explicitly project it.
+      this.data = docs.map((d) => {
+        const { _id, ...rest } = d as Record<string, unknown>;
+        return rest;
+      });
+      this.lastFetch = new Date();
+      this.error = undefined;
+      this.notify();
+    } catch (err) {
+      this.error = err instanceof Error ? err.message : String(err);
+    }
+  }
+
+  private notify(): void {
+    for (const cb of this.listeners) cb(this.data);
+  }
+}