@vaultgradient/pipequery-cli 0.7.0 → 0.10.1

package/__tests__/postgres-pushdown.smoke.test.ts

@@ -0,0 +1,158 @@
+/**
+ * Live smoke test for the Postgres push-down prototype.
+ *
+ * Requires a local Postgres on 127.0.0.1:5432 with a `pq_pd` database, a
+ * `pq_user` role (password `pq_pw`), and an `orders` table seeded with the
+ * fixtures the test below expects. The seeding lives outside the test on
+ * purpose — push-down is a "does the round trip work end-to-end" thing,
+ * not a unit-test concern.
+ *
+ * Skipped by default so CI (and contributors without a local Postgres)
+ * stay green. To run:
+ *
+ *     POSTGRES_PUSHDOWN_SMOKE=1 npm test
+ *
+ * Or point at a different instance / credentials via POSTGRES_PUSHDOWN_URL.
+ */
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { PostgresSourceAdapter } from '../src/server/sources/postgres.js';
+
+const URL = process.env.POSTGRES_PUSHDOWN_URL ?? 'postgres://pq_user:pq_pw@127.0.0.1:5432/pq_pd';
+const ENABLED = process.env.POSTGRES_PUSHDOWN_SMOKE === '1';
+
+describe.skipIf(!ENABLED)('postgres pushdown — live', () => {
+  let adapter: PostgresSourceAdapter;
+
+  beforeAll(async () => {
+    adapter = new PostgresSourceAdapter({
+      type: 'postgres',
+      url: URL,
+      query: 'SELECT * FROM orders',
+      ssl: false,
+    });
+    await adapter.start();
+  });
+
+  afterAll(() => {
+    adapter.stop();
+  });
+
+  it('pushes down where + sort + first into one Postgres query', async () => {
+    const r = await adapter.runPushdown(
+      "orders | where(status == 'paid' && amount > 100) | sort(amount desc) | first(2)",
+    );
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+
+    // Verify the SQL shape — these are the assertions that prove "push-down,
+    // not in-memory filter": the WHERE / ORDER BY / LIMIT clauses must be
+    // present in the SQL we sent to Postgres.
+    expect(r.sql).toMatch(/SELECT \* FROM \(SELECT \* FROM orders\) AS pq_src/);
+    expect(r.sql).toMatch(/WHERE.*"status".*=.*\$1.*AND.*"amount".*>.*\$2/i);
+    expect(r.sql).toMatch(/ORDER BY "amount" DESC/);
+    expect(r.sql).toMatch(/LIMIT 2$/);
+    expect(r.params).toEqual(['paid', 100]);
+
+    // Result: two highest-amount paid orders. Frank ($1500) and Carol ($899.99).
+    expect(r.rows).toHaveLength(2);
+    expect((r.rows[0] as { customer: string }).customer).toBe('Frank');
+    expect((r.rows[1] as { customer: string }).customer).toBe('Carol');
+  });
+
+  it('declines push-down when the pipeline contains an unsupported op', async () => {
+    const r = await adapter.runPushdown('orders | groupBy(status) | rollup(sum(amount) as total)');
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.reason).toMatch(/GroupByOp|not yet pushable/);
+  });
+
+  it('handles NULL semantics correctly', async () => {
+    const r = await adapter.runPushdown("orders | where(customer != null)");
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toMatch(/"customer" IS NOT NULL/);
+  });
+
+  it('parameterizes string literals (no SQL injection surface)', async () => {
+    const r = await adapter.runPushdown(
+      "orders | where(customer == \"Robert'); DROP TABLE orders; --\")",
+    );
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    // The malicious string is bound as $1, not concatenated. SQL stays clean.
+    expect(r.sql).not.toContain('DROP TABLE');
+    expect(r.params).toEqual(["Robert'); DROP TABLE orders; --"]);
+    expect(r.rows).toHaveLength(0);
+
+    // Confirm the table is still intact afterwards.
+    const sanity = await adapter.runPushdown('orders | first(1)');
+    expect(sanity.ok).toBe(true);
+    if (sanity.ok) expect(sanity.rows.length).toBe(1);
+  });
+
+  it('pushes down rollup with GROUP BY + aggregates', async () => {
+    const r = await adapter.runPushdown(
+      "orders | rollup(status, sum(amount) as total, count() as n) | sort(total desc)",
+    );
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toContain('GROUP BY "status"');
+    expect(r.sql).toContain('SUM("amount") AS "total"');
+    expect(r.sql).toContain('COUNT(*) AS "n"');
+    expect(r.sql).toContain('ORDER BY "total" DESC');
+    // Each row carries the aggregate columns we asked for.
+    expect(r.rows.length).toBeGreaterThan(0);
+    expect(r.rows[0]).toHaveProperty('status');
+    expect(r.rows[0]).toHaveProperty('total');
+    expect(r.rows[0]).toHaveProperty('n');
+  });
+
+  it('pushes down a single-row aggregate (no GROUP BY)', async () => {
+    const r = await adapter.runPushdown('orders | rollup(sum(amount) as total)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).not.toContain('GROUP BY');
+    expect(r.rows).toHaveLength(1);
+    expect((r.rows[0] as { total: unknown }).total).not.toBeNull();
+  });
+
+  it('pushes down pipeline-terminal count() to COUNT(*)', async () => {
+    const r = await adapter.runPushdown('orders | count()');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toMatch(/^SELECT COUNT\(\*\) AS "count" FROM/);
+    expect(r.rows).toHaveLength(1);
+  });
+
+  it('pushes down select with alias', async () => {
+    const r = await adapter.runPushdown('orders | select(customer as name, amount) | first(3)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toContain('"customer" AS "name"');
+    expect(r.rows.length).toBeLessThanOrEqual(3);
+    // Result rows have the alias, not the original column name.
+    if (r.rows.length > 0) {
+      expect(r.rows[0]).toHaveProperty('name');
+      expect(r.rows[0]).not.toHaveProperty('customer');
+    }
+  });
+
+  it('pushes down full-row distinct', async () => {
+    const r = await adapter.runPushdown('orders | select(status) | distinct()');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toMatch(/^SELECT DISTINCT "status" FROM/);
+  });
+
+  it('declines distinct(field) — DISTINCT ON not portable', async () => {
+    const r = await adapter.runPushdown('orders | distinct(status)');
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.reason).toMatch(/distinct/i);
+  });
+
+  it('still declines unsupported aggregates (e.g. percentile)', async () => {
+    const r = await adapter.runPushdown('orders | rollup(percentile(amount, 0.95) as p95)');
+    expect(r.ok).toBe(false);
+  });
+});

package/__tests__/pushdown-routing.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Auto-routing tests for SourceManager.runQuery — the "try push-down,
+ * fall back to in-process" decision tree introduced for issue #29 follow-up.
+ *
+ * No real Postgres needed: we register sources whose adapters expose a
+ * fake `runPushdown` so the routing logic is testable in isolation.
+ *
+ * `addSource()` only constructs adapters from `SourceConfig.type`, so we
+ * use a `static` source as the carrier and reach in to attach a
+ * `runPushdown` after the fact. That lets us exercise the manager's
+ * dispatch surface without inventing a new public extensibility hook.
+ */
+import { describe, expect, it } from 'vitest';
+import { SourceManager } from '../src/server/sources/manager.js';
+import type { PushdownResult, SourceAdapter } from '../src/server/sources/types.js';
+
+interface SourceWithAdapter {
+  adapter: SourceAdapter;
+}
+
+/**
+ * Attach a fake `runPushdown` to an already-registered source's adapter.
+ * Returns a tracker so tests can assert call shapes.
+ */
+function attachFakePushdown(
+  sm: SourceManager,
+  name: string,
+  impl: (expression: string) => Promise<PushdownResult>,
+): { calls: string[] } {
+  const calls: string[] = [];
+  // Reach into the private map. This is test-only; production callers go
+  // through the typed methods. Keeps the test from inventing a public
+  // extensibility hook that production code wouldn't use.
+  const sources = (sm as unknown as { sources: Map<string, SourceWithAdapter> }).sources;
+  const entry = sources.get(name);
+  if (!entry) throw new Error(`source "${name}" not registered`);
+  entry.adapter.runPushdown = async (expression: string) => {
+    calls.push(expression);
+    return impl(expression);
+  };
+  return { calls };
+}
+
+describe('SourceManager.runQuery — push-down auto-routing', () => {
+  it('uses push-down when the adapter accepts the expression', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ id: 1, total: 10 }],
+    });
+    const tracker = attachFakePushdown(sm, 'orders', async () => ({
+      ok: true,
+      rows: [{ id: 99, pushed: true }],
+      sql: 'SELECT * FROM orders WHERE total > 0',
+    }));
+
+    const rows = await sm.runQuery('orders | where(total > 0)');
+
+    expect(rows).toEqual([{ id: 99, pushed: true }]);
+    expect(tracker.calls).toEqual(['orders | where(total > 0)']);
+    await sm.dispose();
+  });
+
+  it('falls back to in-process when the adapter declines', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ total: 1 }, { total: 2 }, { total: 3 }],
+    });
+    const tracker = attachFakePushdown(sm, 'orders', async () => ({
+      ok: false,
+      reason: 'simulated decline',
+    }));
+
+    const rows = (await sm.runQuery('orders | rollup(sum(total) as t)')) as Array<{ t: number }>;
+
+    expect(tracker.calls).toHaveLength(1); // adapter was asked
+    expect(rows[0].t).toBe(6); // but in-process actually computed the result
+    await sm.dispose();
+  });
+
+  it('falls back to in-process when the adapter throws', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ total: 5 }, { total: 7 }],
+    });
+    const tracker = attachFakePushdown(sm, 'orders', async () => {
+      throw new Error('connection lost');
+    });
+
+    // Push-down threw, so we fall back to in-process; the in-process result
+    // proves the fallback fired and computed correctly.
+    const rows = (await sm.runQuery('orders | rollup(sum(total) as t)')) as Array<{ t: number }>;
+    expect(tracker.calls).toHaveLength(1);
+    expect(rows[0].t).toBe(12);
+    await sm.dispose();
+  });
+
+  it('skips push-down entirely when the adapter has no runPushdown', async () => {
+    const sm = new SourceManager(process.cwd());
+    // No fake attached → entry.adapter.runPushdown is undefined.
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ total: 1 }, { total: 2 }],
+    });
+
+    const rows = (await sm.runQuery('orders | rollup(sum(total) as t)')) as Array<{ t: number }>;
+    expect(rows[0].t).toBe(3);
+    await sm.dispose();
+  });
+
+  it('skips push-down when the named source does not exist (in-process raises)', async () => {
+    const sm = new SourceManager(process.cwd());
+    // No source registered. tryPushdown returns null; runQuery falls
+    // through to in-process, which throws the canonical "source not found"
+    // engine error. We verify the fall-through happened by observing that
+    // the error is the engine's, not a push-down-specific one.
+    await expect(sm.runQuery('ghost | first(1)')).rejects.toThrow();
+    await sm.dispose();
+  });
+
+  it('non-pipe expressions still parse-fail through the in-process path', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', { type: 'static', data: [] });
+    // tryPushdown swallows the parse error and returns null; in-process
+    // re-parses and surfaces the engine's error message — same as before
+    // push-down existed.
+    await expect(sm.runQuery('this is not a pipe expression')).rejects.toThrow();
+    await sm.dispose();
+  });
+
+  it('tryPushdown returns null when push-down is not even attempted', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', { type: 'static', data: [{ id: 1 }] });
+    // No runPushdown attached → null (signals "not attempted").
+    expect(await sm.tryPushdown('orders | first(1)')).toBeNull();
+    // Source not found → null.
+    expect(await sm.tryPushdown('ghost | first(1)')).toBeNull();
+    // Bad parse → null.
+    expect(await sm.tryPushdown('totally not pipequery')).toBeNull();
+    await sm.dispose();
+  });
+});

package/__tests__/snowflake-pushdown-compile.test.ts

@@ -0,0 +1,74 @@
+/**
+ * Unit tests for compileSnowflakePushdown — no Snowflake account needed.
+ *
+ * Snowflake reuses the dialect-parametric compiler with `"x"` quoting (like
+ * Postgres) and `?` placeholders (like MySQL). These tests focus on the
+ * Snowflake-specific surface plus a couple of canary cases proving the
+ * operator semantics carry across dialects unchanged.
+ */
+import { describe, expect, it } from 'vitest';
+import { parseQuery } from '../src/engine.js';
+import { compileSnowflakePushdown } from '../src/server/sources/pushdown/snowflake.js';
+
+const BASE = 'SELECT * FROM ORDERS';
+
+function compile(expression: string) {
+  const pipeline = parseQuery(expression);
+  return compileSnowflakePushdown(pipeline, BASE);
+}
+
+describe('compileSnowflakePushdown — dialect surface', () => {
+  it('quotes identifiers with double quotes', () => {
+    const r = compile('orders | sort(amount desc) | first(5)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toContain('"amount"');
+    expect(r.compiled.sql).not.toContain('`amount`');
+  });
+
+  it('uses ? positional placeholders', () => {
+    const r = compile("orders | where(status == 'paid' && amount > 100)");
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toMatch(/"status".*=.*\?.*AND.*"amount".*>.*\?/);
+    expect(r.compiled.sql).not.toContain('$1');
+    expect(r.compiled.params).toEqual(['paid', 100]);
+  });
+});
+
+describe('compileSnowflakePushdown — operator parity with other dialects', () => {
+  it('rollup with GROUP BY + aggregates', () => {
+    const r = compile('orders | rollup(country, sum(total) as revenue) | sort(revenue desc) | first(5)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toContain('SUM("total") AS "revenue"');
+    expect(r.compiled.sql).toContain('GROUP BY "country"');
+    expect(r.compiled.sql).toContain('ORDER BY "revenue" DESC');
+    expect(r.compiled.sql).toMatch(/LIMIT 5$/);
+  });
+
+  it('translates count() and distinct_count', () => {
+    const r1 = compile('orders | count()');
+    expect(r1.ok).toBe(true);
+    if (r1.ok) expect(r1.compiled.sql).toContain('COUNT(*) AS "count"');
+
+    const r2 = compile('orders | rollup(distinct_count(email) as users)');
+    expect(r2.ok).toBe(true);
+    if (r2.ok) expect(r2.compiled.sql).toContain('COUNT(DISTINCT "email") AS "users"');
+  });
+
+  it('NULL semantics → IS NULL / IS NOT NULL', () => {
+    const r = compile('orders | where(customer != null)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toContain('"customer" IS NOT NULL');
+  });
+
+  it('parameterizes hostile string literals', () => {
+    const r = compile('orders | where(customer == "Robert\'); DROP TABLE orders; --")');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).not.toContain('DROP TABLE');
+    expect(r.compiled.params).toContain("Robert'); DROP TABLE orders; --");
+  });
+});