@vaultgradient/pipequery-cli 0.6.1 → 0.10.0

package/__tests__/postgres-pushdown.smoke.test.ts

@@ -0,0 +1,158 @@
+/**
+ * Live smoke test for the Postgres push-down prototype.
+ *
+ * Requires a local Postgres on 127.0.0.1:5432 with a `pq_pd` database, a
+ * `pq_user` role (password `pq_pw`), and an `orders` table seeded with the
+ * fixtures the test below expects. The seeding lives outside the test on
+ * purpose — push-down is a "does the round trip work end-to-end" thing,
+ * not a unit-test concern.
+ *
+ * Skipped by default so CI (and contributors without a local Postgres)
+ * stay green. To run:
+ *
+ *     POSTGRES_PUSHDOWN_SMOKE=1 npm test
+ *
+ * Or point at a different instance / credentials via POSTGRES_PUSHDOWN_URL.
+ */
+import { afterAll, beforeAll, describe, expect, it } from 'vitest';
+import { PostgresSourceAdapter } from '../src/server/sources/postgres.js';
+
+const URL = process.env.POSTGRES_PUSHDOWN_URL ?? 'postgres://pq_user:pq_pw@127.0.0.1:5432/pq_pd';
+const ENABLED = process.env.POSTGRES_PUSHDOWN_SMOKE === '1';
+
+describe.skipIf(!ENABLED)('postgres pushdown — live', () => {
+  let adapter: PostgresSourceAdapter;
+
+  beforeAll(async () => {
+    adapter = new PostgresSourceAdapter({
+      type: 'postgres',
+      url: URL,
+      query: 'SELECT * FROM orders',
+      ssl: false,
+    });
+    await adapter.start();
+  });
+
+  afterAll(() => {
+    adapter.stop();
+  });
+
+  it('pushes down where + sort + first into one Postgres query', async () => {
+    const r = await adapter.runPushdown(
+      "orders | where(status == 'paid' && amount > 100) | sort(amount desc) | first(2)",
+    );
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+
+    // Verify the SQL shape — these are the assertions that prove "push-down,
+    // not in-memory filter": the WHERE / ORDER BY / LIMIT clauses must be
+    // present in the SQL we sent to Postgres.
+    expect(r.sql).toMatch(/SELECT \* FROM \(SELECT \* FROM orders\) AS pq_src/);
+    expect(r.sql).toMatch(/WHERE.*"status".*=.*\$1.*AND.*"amount".*>.*\$2/i);
+    expect(r.sql).toMatch(/ORDER BY "amount" DESC/);
+    expect(r.sql).toMatch(/LIMIT 2$/);
+    expect(r.params).toEqual(['paid', 100]);
+
+    // Result: two highest-amount paid orders. Frank ($1500) and Carol ($899.99).
+    expect(r.rows).toHaveLength(2);
+    expect((r.rows[0] as { customer: string }).customer).toBe('Frank');
+    expect((r.rows[1] as { customer: string }).customer).toBe('Carol');
+  });
+
+  it('declines push-down when the pipeline contains an unsupported op', async () => {
+    const r = await adapter.runPushdown('orders | groupBy(status) | rollup(sum(amount) as total)');
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.reason).toMatch(/GroupByOp|not yet pushable/);
+  });
+
+  it('handles NULL semantics correctly', async () => {
+    const r = await adapter.runPushdown("orders | where(customer != null)");
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toMatch(/"customer" IS NOT NULL/);
+  });
+
+  it('parameterizes string literals (no SQL injection surface)', async () => {
+    const r = await adapter.runPushdown(
+      "orders | where(customer == \"Robert'); DROP TABLE orders; --\")",
+    );
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    // The malicious string is bound as $1, not concatenated. SQL stays clean.
+    expect(r.sql).not.toContain('DROP TABLE');
+    expect(r.params).toEqual(["Robert'); DROP TABLE orders; --"]);
+    expect(r.rows).toHaveLength(0);
+
+    // Confirm the table is still intact afterwards.
+    const sanity = await adapter.runPushdown('orders | first(1)');
+    expect(sanity.ok).toBe(true);
+    if (sanity.ok) expect(sanity.rows.length).toBe(1);
+  });
+
+  it('pushes down rollup with GROUP BY + aggregates', async () => {
+    const r = await adapter.runPushdown(
+      "orders | rollup(status, sum(amount) as total, count() as n) | sort(total desc)",
+    );
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toContain('GROUP BY "status"');
+    expect(r.sql).toContain('SUM("amount") AS "total"');
+    expect(r.sql).toContain('COUNT(*) AS "n"');
+    expect(r.sql).toContain('ORDER BY "total" DESC');
+    // Each row carries the aggregate columns we asked for.
+    expect(r.rows.length).toBeGreaterThan(0);
+    expect(r.rows[0]).toHaveProperty('status');
+    expect(r.rows[0]).toHaveProperty('total');
+    expect(r.rows[0]).toHaveProperty('n');
+  });
+
+  it('pushes down a single-row aggregate (no GROUP BY)', async () => {
+    const r = await adapter.runPushdown('orders | rollup(sum(amount) as total)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).not.toContain('GROUP BY');
+    expect(r.rows).toHaveLength(1);
+    expect((r.rows[0] as { total: unknown }).total).not.toBeNull();
+  });
+
+  it('pushes down pipeline-terminal count() to COUNT(*)', async () => {
+    const r = await adapter.runPushdown('orders | count()');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toMatch(/^SELECT COUNT\(\*\) AS "count" FROM/);
+    expect(r.rows).toHaveLength(1);
+  });
+
+  it('pushes down select with alias', async () => {
+    const r = await adapter.runPushdown('orders | select(customer as name, amount) | first(3)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toContain('"customer" AS "name"');
+    expect(r.rows.length).toBeLessThanOrEqual(3);
+    // Result rows have the alias, not the original column name.
+    if (r.rows.length > 0) {
+      expect(r.rows[0]).toHaveProperty('name');
+      expect(r.rows[0]).not.toHaveProperty('customer');
+    }
+  });
+
+  it('pushes down full-row distinct', async () => {
+    const r = await adapter.runPushdown('orders | select(status) | distinct()');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.sql).toMatch(/^SELECT DISTINCT "status" FROM/);
+  });
+
+  it('declines distinct(field) — DISTINCT ON not portable', async () => {
+    const r = await adapter.runPushdown('orders | distinct(status)');
+    expect(r.ok).toBe(false);
+    if (r.ok) return;
+    expect(r.reason).toMatch(/distinct/i);
+  });
+
+  it('still declines unsupported aggregates (e.g. percentile)', async () => {
+    const r = await adapter.runPushdown('orders | rollup(percentile(amount, 0.95) as p95)');
+    expect(r.ok).toBe(false);
+  });
+});

package/__tests__/pushdown-routing.test.ts

@@ -0,0 +1,144 @@
+/**
+ * Auto-routing tests for SourceManager.runQuery — the "try push-down,
+ * fall back to in-process" decision tree introduced for issue #29 follow-up.
+ *
+ * No real Postgres needed: we register sources whose adapters expose a
+ * fake `runPushdown` so the routing logic is testable in isolation.
+ *
+ * `addSource()` only constructs adapters from `SourceConfig.type`, so we
+ * use a `static` source as the carrier and reach in to attach a
+ * `runPushdown` after the fact. That lets us exercise the manager's
+ * dispatch surface without inventing a new public extensibility hook.
+ */
+import { describe, expect, it } from 'vitest';
+import { SourceManager } from '../src/server/sources/manager.js';
+import type { PushdownResult, SourceAdapter } from '../src/server/sources/types.js';
+
+interface SourceWithAdapter {
+  adapter: SourceAdapter;
+}
+
+/**
+ * Attach a fake `runPushdown` to an already-registered source's adapter.
+ * Returns a tracker so tests can assert call shapes.
+ */
+function attachFakePushdown(
+  sm: SourceManager,
+  name: string,
+  impl: (expression: string) => Promise<PushdownResult>,
+): { calls: string[] } {
+  const calls: string[] = [];
+  // Reach into the private map. This is test-only; production callers go
+  // through the typed methods. Keeps the test from inventing a public
+  // extensibility hook that production code wouldn't use.
+  const sources = (sm as unknown as { sources: Map<string, SourceWithAdapter> }).sources;
+  const entry = sources.get(name);
+  if (!entry) throw new Error(`source "${name}" not registered`);
+  entry.adapter.runPushdown = async (expression: string) => {
+    calls.push(expression);
+    return impl(expression);
+  };
+  return { calls };
+}
+
+describe('SourceManager.runQuery — push-down auto-routing', () => {
+  it('uses push-down when the adapter accepts the expression', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ id: 1, total: 10 }],
+    });
+    const tracker = attachFakePushdown(sm, 'orders', async () => ({
+      ok: true,
+      rows: [{ id: 99, pushed: true }],
+      sql: 'SELECT * FROM orders WHERE total > 0',
+    }));
+
+    const rows = await sm.runQuery('orders | where(total > 0)');
+
+    expect(rows).toEqual([{ id: 99, pushed: true }]);
+    expect(tracker.calls).toEqual(['orders | where(total > 0)']);
+    await sm.dispose();
+  });
+
+  it('falls back to in-process when the adapter declines', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ total: 1 }, { total: 2 }, { total: 3 }],
+    });
+    const tracker = attachFakePushdown(sm, 'orders', async () => ({
+      ok: false,
+      reason: 'simulated decline',
+    }));
+
+    const rows = (await sm.runQuery('orders | rollup(sum(total) as t)')) as Array<{ t: number }>;
+
+    expect(tracker.calls).toHaveLength(1); // adapter was asked
+    expect(rows[0].t).toBe(6); // but in-process actually computed the result
+    await sm.dispose();
+  });
+
+  it('falls back to in-process when the adapter throws', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ total: 5 }, { total: 7 }],
+    });
+    const tracker = attachFakePushdown(sm, 'orders', async () => {
+      throw new Error('connection lost');
+    });
+
+    // Push-down threw, so we fall back to in-process; the in-process result
+    // proves the fallback fired and computed correctly.
+    const rows = (await sm.runQuery('orders | rollup(sum(total) as t)')) as Array<{ t: number }>;
+    expect(tracker.calls).toHaveLength(1);
+    expect(rows[0].t).toBe(12);
+    await sm.dispose();
+  });
+
+  it('skips push-down entirely when the adapter has no runPushdown', async () => {
+    const sm = new SourceManager(process.cwd());
+    // No fake attached → entry.adapter.runPushdown is undefined.
+    await sm.addSource('orders', {
+      type: 'static',
+      data: [{ total: 1 }, { total: 2 }],
+    });
+
+    const rows = (await sm.runQuery('orders | rollup(sum(total) as t)')) as Array<{ t: number }>;
+    expect(rows[0].t).toBe(3);
+    await sm.dispose();
+  });
+
+  it('skips push-down when the named source does not exist (in-process raises)', async () => {
+    const sm = new SourceManager(process.cwd());
+    // No source registered. tryPushdown returns null; runQuery falls
+    // through to in-process, which throws the canonical "source not found"
+    // engine error. We verify the fall-through happened by observing that
+    // the error is the engine's, not a push-down-specific one.
+    await expect(sm.runQuery('ghost | first(1)')).rejects.toThrow();
+    await sm.dispose();
+  });
+
+  it('non-pipe expressions still parse-fail through the in-process path', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', { type: 'static', data: [] });
+    // tryPushdown swallows the parse error and returns null; in-process
+    // re-parses and surfaces the engine's error message — same as before
+    // push-down existed.
+    await expect(sm.runQuery('this is not a pipe expression')).rejects.toThrow();
+    await sm.dispose();
+  });
+
+  it('tryPushdown returns null when push-down is not even attempted', async () => {
+    const sm = new SourceManager(process.cwd());
+    await sm.addSource('orders', { type: 'static', data: [{ id: 1 }] });
+    // No runPushdown attached → null (signals "not attempted").
+    expect(await sm.tryPushdown('orders | first(1)')).toBeNull();
+    // Source not found → null.
+    expect(await sm.tryPushdown('ghost | first(1)')).toBeNull();
+    // Bad parse → null.
+    expect(await sm.tryPushdown('totally not pipequery')).toBeNull();
+    await sm.dispose();
+  });
+});

package/__tests__/rest-source.test.ts

@@ -0,0 +1,191 @@
+/**
+ * REST source — env-var interpolation in url / headers / params,
+ * plus auth: bearer helper. Issue #37 (slice).
+ *
+ * Spins up a real http server in-process so we exercise the actual
+ * outbound request shape rather than mocking globalThis.fetch.
+ */
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { createServer, type Server, type IncomingMessage, type ServerResponse } from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { RestSourceAdapter } from '../src/server/sources/rest.js';
+
+interface Captured {
+  url: string;
+  headers: Record<string, string | string[] | undefined>;
+}
+
+interface Harness {
+  url: string;
+  captured: Captured[];
+  /** Body the server returns for the next request. Default: empty array. */
+  setResponse(body: unknown, status?: number): void;
+  close: () => Promise<void>;
+}
+
+async function makeServer(): Promise<Harness> {
+  const captured: Captured[] = [];
+  let nextBody: unknown = [];
+  let nextStatus = 200;
+
+  const server: Server = createServer((req: IncomingMessage, res: ServerResponse) => {
+    captured.push({ url: req.url ?? '', headers: req.headers });
+    res.statusCode = nextStatus;
+    res.setHeader('content-type', 'application/json');
+    res.end(JSON.stringify(nextBody));
+  });
+  await new Promise<void>((r) => server.listen(0, '127.0.0.1', r));
+  const port = (server.address() as AddressInfo).port;
+  return {
+    url: `http://127.0.0.1:${port}`,
+    captured,
+    setResponse(body, status = 200) {
+      nextBody = body;
+      nextStatus = status;
+    },
+    close: () => new Promise<void>((r) => server.close(() => r())),
+  };
+}
+
+describe('REST source — env interpolation + auth: bearer (#37 slice)', () => {
+  let server: Harness;
+  let envBackup: Record<string, string | undefined>;
+
+  beforeEach(async () => {
+    server = await makeServer();
+    envBackup = {
+      MY_TOKEN: process.env.MY_TOKEN,
+      MY_KEY: process.env.MY_KEY,
+      USER_PATH: process.env.USER_PATH,
+      MY_REGION: process.env.MY_REGION,
+    };
+  });
+  afterEach(async () => {
+    await server.close();
+    // Restore env
+    for (const [k, v] of Object.entries(envBackup)) {
+      if (v === undefined) delete process.env[k];
+      else process.env[k] = v;
+    }
+  });
+
+  it('expands ${ENV_VAR} in URL', async () => {
+    process.env.USER_PATH = 'users';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: `${server.url}/api/\${USER_PATH}`,
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].url).toBe('/api/users');
+  });
+
+  it('expands ${ENV_VAR} in headers', async () => {
+    process.env.MY_KEY = 'sk-secret-abc';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { 'X-API-Key': '${MY_KEY}', 'X-Static': 'literal' },
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].headers['x-api-key']).toBe('sk-secret-abc');
+    expect(server.captured[0].headers['x-static']).toBe('literal');
+  });
+
+  it('expands ${ENV_VAR} in query params and merges with existing query string', async () => {
+    process.env.MY_REGION = 'eu-west-1';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: `${server.url}/api?fixed=1`,
+      params: { region: '${MY_REGION}', kind: 'live' },
+    });
+    await adapter.start();
+    adapter.stop();
+    // Query string contains both the inline ?fixed=1 and the appended params,
+    // joined with `&`. URLSearchParams encodes dashes as-is.
+    const seen = server.captured[0].url;
+    expect(seen).toContain('fixed=1');
+    expect(seen).toContain('region=eu-west-1');
+    expect(seen).toContain('kind=live');
+    expect(seen.startsWith('/api?')).toBe(true);
+  });
+
+  it('emits empty string and a warning for missing env vars (does not embed literal "${FOO}")', async () => {
+    // No MY_TOKEN set
+    delete process.env.MY_TOKEN;
+    server.setResponse([]);
+    const stderrCaptured: string[] = [];
+    const origWrite = process.stderr.write.bind(process.stderr);
+    process.stderr.write = ((chunk: string | Uint8Array) => {
+      stderrCaptured.push(typeof chunk === 'string' ? chunk : chunk.toString());
+      return true;
+    }) as typeof process.stderr.write;
+
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { Authorization: 'Bearer ${MY_TOKEN}' },
+    });
+    try {
+      await adapter.start();
+      adapter.stop();
+    } finally {
+      process.stderr.write = origWrite;
+    }
+
+    // The literal "${...}" must NOT appear in the outbound header. Node
+    // trims trailing whitespace from header values on the wire, so the
+    // received value is "Bearer" not "Bearer ".
+    const auth = server.captured[0].headers['authorization'] as string;
+    expect(auth).not.toContain('${');
+    expect(auth.startsWith('Bearer')).toBe(true);
+    expect(stderrCaptured.some((s) => s.includes('MY_TOKEN'))).toBe(true);
+  });
+
+  it('auth: bearer adds Authorization: Bearer <token> with env interpolation', async () => {
+    process.env.MY_TOKEN = 'jwt.abc.123';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      auth: { kind: 'bearer', token: '${MY_TOKEN}' },
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].headers['authorization']).toBe('Bearer jwt.abc.123');
+  });
+
+  it('auth: bearer wins over a hand-rolled headers.Authorization on collision', async () => {
+    // If a user sets both, they almost certainly meant the auth helper —
+    // bearer is the one that actually plumbs the token through env vars.
+    process.env.MY_TOKEN = 'jwt.xyz';
+    server.setResponse([]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { Authorization: 'Bearer should-be-overridden' },
+      auth: { kind: 'bearer', token: '${MY_TOKEN}' },
+    });
+    await adapter.start();
+    adapter.stop();
+    expect(server.captured[0].headers['authorization']).toBe('Bearer jwt.xyz');
+  });
+
+  it('back-compat: no env vars + no auth → behaves exactly like before', async () => {
+    server.setResponse([{ id: 1 }, { id: 2 }]);
+    const adapter = new RestSourceAdapter({
+      type: 'rest',
+      url: server.url,
+      headers: { 'X-Custom': 'value' },
+    });
+    await adapter.start();
+    expect(adapter.getData()).toEqual([{ id: 1 }, { id: 2 }]);
+    expect(server.captured[0].headers['x-custom']).toBe('value');
+    expect(server.captured[0].headers['authorization']).toBeUndefined();
+    adapter.stop();
+  });
+});

package/__tests__/snowflake-pushdown-compile.test.ts

@@ -0,0 +1,74 @@
+/**
+ * Unit tests for compileSnowflakePushdown — no Snowflake account needed.
+ *
+ * Snowflake reuses the dialect-parametric compiler with `"x"` quoting (like
+ * Postgres) and `?` placeholders (like MySQL). These tests focus on the
+ * Snowflake-specific surface plus a couple of canary cases proving the
+ * operator semantics carry across dialects unchanged.
+ */
+import { describe, expect, it } from 'vitest';
+import { parseQuery } from '../src/engine.js';
+import { compileSnowflakePushdown } from '../src/server/sources/pushdown/snowflake.js';
+
+const BASE = 'SELECT * FROM ORDERS';
+
+function compile(expression: string) {
+  const pipeline = parseQuery(expression);
+  return compileSnowflakePushdown(pipeline, BASE);
+}
+
+describe('compileSnowflakePushdown — dialect surface', () => {
+  it('quotes identifiers with double quotes', () => {
+    const r = compile('orders | sort(amount desc) | first(5)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toContain('"amount"');
+    expect(r.compiled.sql).not.toContain('`amount`');
+  });
+
+  it('uses ? positional placeholders', () => {
+    const r = compile("orders | where(status == 'paid' && amount > 100)");
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toMatch(/"status".*=.*\?.*AND.*"amount".*>.*\?/);
+    expect(r.compiled.sql).not.toContain('$1');
+    expect(r.compiled.params).toEqual(['paid', 100]);
+  });
+});
+
+describe('compileSnowflakePushdown — operator parity with other dialects', () => {
+  it('rollup with GROUP BY + aggregates', () => {
+    const r = compile('orders | rollup(country, sum(total) as revenue) | sort(revenue desc) | first(5)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toContain('SUM("total") AS "revenue"');
+    expect(r.compiled.sql).toContain('GROUP BY "country"');
+    expect(r.compiled.sql).toContain('ORDER BY "revenue" DESC');
+    expect(r.compiled.sql).toMatch(/LIMIT 5$/);
+  });
+
+  it('translates count() and distinct_count', () => {
+    const r1 = compile('orders | count()');
+    expect(r1.ok).toBe(true);
+    if (r1.ok) expect(r1.compiled.sql).toContain('COUNT(*) AS "count"');
+
+    const r2 = compile('orders | rollup(distinct_count(email) as users)');
+    expect(r2.ok).toBe(true);
+    if (r2.ok) expect(r2.compiled.sql).toContain('COUNT(DISTINCT "email") AS "users"');
+  });
+
+  it('NULL semantics → IS NULL / IS NOT NULL', () => {
+    const r = compile('orders | where(customer != null)');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).toContain('"customer" IS NOT NULL');
+  });
+
+  it('parameterizes hostile string literals', () => {
+    const r = compile('orders | where(customer == "Robert\'); DROP TABLE orders; --")');
+    expect(r.ok).toBe(true);
+    if (!r.ok) return;
+    expect(r.compiled.sql).not.toContain('DROP TABLE');
+    expect(r.compiled.params).toContain("Robert'); DROP TABLE orders; --");
+  });
+});