@pdpp/cli 0.1.0-beta.7 → 0.1.0-beta.8

package/src/read/commands.js

@@ -0,0 +1,250 @@
+import { ConnectError, normalizeProviderUrl, readStoredCredential } from '../connect/flow.js';
+import { parseArgs, requirePositional } from '../ref/args.js';
+import { PdppHttpError, PdppUsageError } from '../ref/errors.js';
+import { resolveFormat, writeData, writeEnvelopeWarnings } from '../ref/output.js';
+
+const COMMANDS = new Set(['schema', 'streams', 'query-records', 'fetch', 'search', 'aggregate']);
+
+export function readHelp(binName = 'pdpp') {
+  return `Grant-scoped reads (uses pdpp connect/token cache, never owner credentials):
+  ${binName} read schema <provider-url> [--view compact] [--stream <name>] [--connection-id <cin>] [--cache-root <dir>] [--format json|table]
+  ${binName} read streams <provider-url> [--connection-id <cin>] [--cache-root <dir>] [--format json|table]
+  ${binName} read query-records <provider-url> <stream> [--connection-id <cin>] [--limit <n>] [--cursor <cursor>] [--fields a,b] [--sort <spec>] [--count none|estimated|exact] [--filter-json <json>] [--format json|jsonl|table]
+  ${binName} read fetch <provider-url> <stream> <record-id> [--connection-id <cin>] [--fields a,b] [--format json|table]
+  ${binName} read search <provider-url> <query> [--connection-id <cin>] [--streams a,b] [--mode lexical|semantic|hybrid] [--limit <n>] [--format json|jsonl|table]
+  ${binName} read aggregate <provider-url> <stream> --metric <metric> [--field <field>] [--connection-id <cin>] [--group-by <field> | --group-by-time <field> --granularity <unit>] [--limit <n>] [--format json|table]`;
+}
+
+export async function runRead(argv, io = {}, fetchImpl = globalThis.fetch) {
+  const out = io.stdout || process.stdout;
+  const err = io.stderr || process.stderr;
+  const [command, ...rest] = argv;
+
+  if (!command || command === '--help' || command === '-h' || command === 'help') {
+    out.write(`${readHelp()}\n`);
+    return 0;
+  }
+
+  if (!COMMANDS.has(command)) {
+    throw new PdppUsageError(`Unknown read command: ${command}`);
+  }
+
+  const { flags, positionals } = parseArgs(rest);
+  const providerUrl = requirePositional(positionals, 0, 'provider-url');
+  let credential;
+  let normalizedProviderUrl;
+  try {
+    const stored = await readStoredCredential(providerUrl, { cacheRoot: flags['cache-root'] });
+    credential = stored.credential;
+    normalizedProviderUrl = stored.providerUrl;
+  } catch (error) {
+    if (error instanceof ConnectError) {
+      throw new PdppUsageError(error.message);
+    }
+    throw error;
+  }
+
+  const request = buildReadRequest(command, positionals.slice(1), flags, normalizedProviderUrl);
+  const body = await fetchReadJson(request, credential.access_token, fetchImpl);
+  writeData(projectOutput(body, flags), resolveFormat(flags, 'json', 'json'), out);
+  writeEnvelopeWarnings(body, err);
+  return 0;
+}
+
+export function buildReadRequest(command, positionals, flags, providerUrl) {
+  const origin = normalizeProviderUrl(providerUrl);
+  if (!origin) throw new PdppUsageError(`Invalid provider URL: ${providerUrl}`);
+
+  if (command === 'schema') {
+    return {
+      method: 'GET',
+      url: buildUrl(origin, '/v1/schema', pickQuery(flags, ['connector-id', 'connection-id', 'stream', 'view'])),
+    };
+  }
+
+  if (command === 'streams') {
+    return {
+      method: 'GET',
+      url: buildUrl(origin, '/v1/streams', pickQuery(flags, ['connection-id', 'connector-instance-id'])),
+    };
+  }
+
+  if (command === 'query-records') {
+    const stream = requirePositional(positionals, 0, 'stream');
+    const query = {
+      ...pickQuery(flags, [
+        'connection-id',
+        'connector-instance-id',
+        'cursor',
+        'limit',
+        'order',
+        'sort',
+        'count',
+        'changes-since',
+      ]),
+      ...csvQuery(flags, 'fields'),
+      ...jsonFilterQuery(flags),
+    };
+    return { method: 'GET', url: buildUrl(origin, `/v1/streams/${encodeURIComponent(stream)}/records`, query) };
+  }
+
+  if (command === 'fetch') {
+    const stream = requirePositional(positionals, 0, 'stream');
+    const recordId = requirePositional(positionals, 1, 'record-id');
+    const query = {
+      ...pickQuery(flags, ['connection-id', 'connector-instance-id']),
+      ...csvQuery(flags, 'fields'),
+    };
+    return {
+      method: 'GET',
+      url: buildUrl(
+        origin,
+        `/v1/streams/${encodeURIComponent(stream)}/records/${encodeURIComponent(recordId)}`,
+        query,
+      ),
+    };
+  }
+
+  if (command === 'search') {
+    const queryText = requirePositional(positionals, 0, 'query');
+    const mode = flags.mode ? String(flags.mode) : undefined;
+    const path = mode === 'semantic' ? '/v1/search/semantic' : mode === 'hybrid' ? '/v1/search/hybrid' : '/v1/search';
+    const query = {
+      q: queryText,
+      ...pickQuery(flags, ['connection-id', 'connector-instance-id', 'cursor', 'limit']),
+      ...csvQuery(flags, 'streams'),
+    };
+    return { method: 'GET', url: buildUrl(origin, path, query) };
+  }
+
+  if (command === 'aggregate') {
+    const stream = requirePositional(positionals, 0, 'stream');
+    if (!flags.metric) throw new PdppUsageError('Missing required flag: --metric');
+    if (flags['group-by'] && flags['group-by-time']) {
+      throw new PdppUsageError('Use only one of --group-by or --group-by-time.');
+    }
+    const query = pickQuery(flags, [
+      'connection-id',
+      'connector-instance-id',
+      'field',
+      'granularity',
+      'limit',
+      'metric',
+      'time-zone',
+    ]);
+    if (flags['group-by']) query.group_by = flags['group-by'];
+    if (flags['group-by-time']) query.group_by_time = flags['group-by-time'];
+    return { method: 'GET', url: buildUrl(origin, `/v1/streams/${encodeURIComponent(stream)}/aggregate`, query) };
+  }
+
+  throw new PdppUsageError(`Unsupported read command: ${command}`);
+}
+
+async function fetchReadJson(request, token, fetchImpl) {
+  let resp;
+  try {
+    resp = await fetchImpl(request.url, {
+      method: request.method,
+      headers: {
+        Accept: 'application/json',
+        Authorization: `Bearer ${token}`,
+      },
+    });
+  } catch (error) {
+    throw new PdppUsageError(`Network request failed: ${error.message}`);
+  }
+
+  const text = typeof resp.text === 'function' ? await resp.text() : '';
+  let parsed = null;
+  if (text) {
+    try {
+      parsed = JSON.parse(text);
+    } catch {
+      parsed = text;
+    }
+  }
+
+  if (resp.status >= 400) {
+    const message =
+      parsed?.error_description ||
+      parsed?.error?.message ||
+      parsed?.message ||
+      `HTTP ${resp.status} ${resp.statusText || ''}`.trim();
+    throw new PdppHttpError(String(message), resp.status, parsed, {
+      request_id: resp.headers?.get?.('x-request-id') ?? null,
+    });
+  }
+
+  return parsed;
+}
+
+function buildUrl(origin, path, query = {}) {
+  const url = new URL(path, `${origin}/`);
+  for (const [key, value] of Object.entries(query)) {
+    appendQuery(url, key, value);
+  }
+  return url.toString();
+}
+
+function appendQuery(url, key, value) {
+  if (value === undefined || value === null || value === '') return;
+  if (Array.isArray(value)) {
+    for (const entry of value) appendQuery(url, key, entry);
+    return;
+  }
+  url.searchParams.append(key, String(value));
+}
+
+function pickQuery(flags, names) {
+  const query = {};
+  for (const name of names) {
+    const value = flags[name];
+    if (value === undefined || value === true) continue;
+    query[name.replaceAll('-', '_')] = value;
+  }
+  return query;
+}
+
+function csvQuery(flags, name) {
+  const raw = flags[name];
+  if (typeof raw !== 'string' || raw.trim() === '') return {};
+  return { [name]: raw.split(',').map((entry) => entry.trim()).filter(Boolean) };
+}
+
+function jsonFilterQuery(flags) {
+  if (flags.filter !== undefined && flags['filter-json'] !== undefined) {
+    throw new PdppUsageError('Use only one of --filter or --filter-json.');
+  }
+  if (typeof flags.filter === 'string') {
+    return { filter: flags.filter };
+  }
+  if (flags['filter-json'] === undefined) return {};
+
+  let parsed;
+  try {
+    parsed = JSON.parse(flags['filter-json']);
+  } catch (error) {
+    throw new PdppUsageError(`--filter-json must be valid JSON: ${error.message}`);
+  }
+  if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+    throw new PdppUsageError('--filter-json must be a JSON object.');
+  }
+  const query = {};
+  for (const [field, value] of Object.entries(parsed)) {
+    if (value && typeof value === 'object' && !Array.isArray(value)) {
+      for (const [op, opValue] of Object.entries(value)) {
+        query[`filter[${field}][${op}]`] = opValue;
+      }
+    } else {
+      query[`filter[${field}]`] = value;
+    }
+  }
+  return query;
+}
+
+function projectOutput(body, flags) {
+  if (!flags.data) return body;
+  if (body && typeof body === 'object' && Array.isArray(body.data)) return body.data;
+  if (body && typeof body === 'object' && Array.isArray(body.records)) return body.records;
+  return body;
+}

package/src/ref/auth.js

@@ -0,0 +1,179 @@
+// Owner-action auth resolution for `pdpp ref call`.
+//
+// The reference server exposes two owner surfaces with two different auth
+// modes — the single fact that owners keep rediscovering:
+//
+//   /_ref/*      operator/diagnostics control plane  → owner SESSION COOKIE
+//   /v1/owner/*  owner control machine API           → owner BEARER token
+//
+// Every mutating /_ref/* route is guarded by `requireOwnerSession` alone; none
+// require a CSRF token, because the server exempts any POST whose Content-Type
+// is exactly `application/json` (a JSON body can't be forged into a cross-origin
+// browser POST without a CORS preflight). So a JSON owner-cookie POST to /_ref/*
+// needs no `_csrf` handling at all. This module encodes that model and refuses
+// the mismatched pairing (cookie→/v1/owner or bearer→/_ref), which would
+// otherwise surface as a confusing 401/404 — the "401 = wrong auth, 404 = wrong
+// path" trap from the live route map.
+
+import { PdppUsageError } from './errors.js';
+import { ownerSessionHeaders } from './fetch.js';
+
+export const AUTH_COOKIE = 'cookie';
+export const AUTH_BEARER = 'bearer';
+
+// Infer the auth mode a path expects from its prefix. Returns 'cookie',
+// 'bearer', or null when the prefix is not an owner surface we recognize.
+export function inferAuthMode(path) {
+  const p = normalizePath(path);
+  if (p.startsWith('/v1/owner/') || p === '/v1/owner') {
+    return AUTH_BEARER;
+  }
+  if (p.startsWith('/_ref/') || p === '/_ref') {
+    return AUTH_COOKIE;
+  }
+  return null;
+}
+
+function normalizePath(path) {
+  if (typeof path !== 'string' || !path) {
+    return '';
+  }
+  // Strip a query/hash and a leading origin if the caller passed a full URL.
+  let p = path;
+  try {
+    if (/^https?:\/\//i.test(p)) {
+      p = new URL(p).pathname;
+    }
+  } catch {
+    // fall through; treat as a path
+  }
+  const q = p.indexOf('?');
+  if (q !== -1) p = p.slice(0, q);
+  const h = p.indexOf('#');
+  if (h !== -1) p = p.slice(0, h);
+  if (!p.startsWith('/')) p = `/${p}`;
+  return p;
+}
+
+// Resolve the effective auth mode for a call: an explicit `--auth` override if
+// present and valid, otherwise the path-inferred mode. Throws a usage error
+// when an override conflicts with the path's surface, or when neither an
+// override nor a recognized prefix is available.
+export function resolveAuthMode(path, override) {
+  const inferred = inferAuthMode(path);
+
+  if (override !== undefined && override !== null && override !== '') {
+    const chosen = String(override).toLowerCase();
+    if (chosen !== AUTH_COOKIE && chosen !== AUTH_BEARER) {
+      throw new PdppUsageError(`Invalid --auth value: ${override}. Use "cookie" or "bearer".`);
+    }
+    if (inferred && inferred !== chosen) {
+      throw new PdppUsageError(mismatchMessage(normalizePath(path), inferred, chosen));
+    }
+    return chosen;
+  }
+
+  if (!inferred) {
+    throw new PdppUsageError(
+      `Cannot infer owner auth mode for path "${path}". ` +
+        'Owner routes are /_ref/* (cookie) or /v1/owner/* (bearer). ' +
+        'Pass --auth cookie|bearer to call a non-standard path explicitly.'
+    );
+  }
+  return inferred;
+}
+
+function mismatchMessage(path, inferred, chosen) {
+  const surface = inferred === AUTH_COOKIE ? '/_ref/*' : '/v1/owner/*';
+  const correct = inferred === AUTH_COOKIE ? 'cookie' : 'bearer';
+  return (
+    `Auth mismatch: ${path} is a ${surface} route and uses ${correct} auth, ` +
+    `but --auth ${chosen} was given. ` +
+    '/_ref/* uses the owner session cookie; /v1/owner/* uses the owner bearer. ' +
+    'Pointing the wrong auth at a route returns a confusing 401/404. ' +
+    `Drop --auth (it is inferred) or use --auth ${correct}.`
+  );
+}
+
+// Build the request headers for the chosen auth mode. For cookie auth, resolves
+// the owner session (flag > env > 0600 cache) and never echoes it. For bearer
+// auth, reads the owner token from --owner-token-stdin or PDPP_OWNER_TOKEN and
+// never accepts it on argv. Throws a usage error when the required secret is
+// absent. The returned object includes only the auth header; content-type is
+// added by the caller for bodies.
+export async function buildAuthHeaders({ mode, referenceUrl, flags, io, env = process.env }) {
+  if (mode === AUTH_COOKIE) {
+    const headers = ownerSessionHeaders({
+      ownerSession: flags['owner-session'] || '',
+      referenceUrl,
+      cacheRoot: flags['cache-root'],
+    });
+    if (!headers.Cookie) {
+      throw new PdppUsageError(
+        'No owner session available. Run `pdpp ref login <reference-url>` first, ' +
+          'pass --owner-session <cookie>, or set PDPP_OWNER_SESSION_COOKIE.'
+      );
+    }
+    return headers;
+  }
+
+  if (mode === AUTH_BEARER) {
+    const token = await resolveOwnerToken(flags, io, env);
+    if (!token) {
+      throw new PdppUsageError(
+        'No owner bearer available for a /v1/owner/* call. ' +
+          'Pipe it via `--owner-token-stdin` or set PDPP_OWNER_TOKEN. ' +
+          'The token is never accepted on the command line.'
+      );
+    }
+    return { Authorization: `Bearer ${token}` };
+  }
+
+  throw new PdppUsageError(`Unknown auth mode: ${mode}`);
+}
+
+async function resolveOwnerToken(flags, io, env) {
+  if (flags['owner-token-stdin']) {
+    return readFirstLine((io && io.stdin) || process.stdin);
+  }
+  const fromEnv = env.PDPP_OWNER_TOKEN;
+  if (typeof fromEnv === 'string' && fromEnv.length > 0) {
+    return fromEnv.trim();
+  }
+  return null;
+}
+
+function readFirstLine(stream) {
+  return new Promise((resolve, reject) => {
+    if (!stream || typeof stream.on !== 'function') {
+      resolve('');
+      return;
+    }
+    let buf = '';
+    stream.setEncoding?.('utf8');
+    const onData = (chunk) => {
+      buf += chunk;
+      const nl = buf.indexOf('\n');
+      if (nl !== -1) {
+        cleanup();
+        resolve(buf.slice(0, nl).replace(/\r$/, ''));
+      }
+    };
+    const onEnd = () => {
+      cleanup();
+      resolve(buf.replace(/\r?\n$/, ''));
+    };
+    const onError = (e) => {
+      cleanup();
+      reject(e);
+    };
+    function cleanup() {
+      stream.off?.('data', onData);
+      stream.off?.('end', onEnd);
+      stream.off?.('error', onError);
+    }
+    stream.on('data', onData);
+    stream.on('end', onEnd);
+    stream.on('error', onError);
+  });
+}

package/src/ref/commands/call.js

@@ -0,0 +1,168 @@
+// `pdpp ref call` — a generic owner-authenticated HTTP caller for the long tail
+// of owner-only reference routes (dataset reconcile, run cancel, schedule
+// pause/resume, and any other /_ref/* or /v1/owner/* route).
+//
+// It exists so owner lanes stop rediscovering the auth model on every new
+// action. The auth mode is inferred from the path prefix and can be overridden:
+//
+//   /_ref/*      → owner session cookie   (cached, --owner-session, or env)
+//   /v1/owner/*  → owner bearer           (--owner-token-stdin or PDPP_OWNER_TOKEN)
+//
+// Bodies are always sent as application/json, which the reference server treats
+// as CSRF-exempt — so there is no `_csrf` handling anywhere. Secrets are never
+// printed: only the response body (stdout) and a `METHOD path → status` line
+// (stderr) are emitted.
+//
+// Usage:
+//   pdpp ref call <method> <path> [--as-url <url>]
+//     [--data <json> | --data-stdin]
+//     [--auth cookie|bearer]
+//     [--owner-session <cookie>] [--owner-token-stdin]
+//     [--cache-root <dir>] [--format json|table]
+//     [--status-only]
+
+import { parseArgs, requirePositional } from '../args.js';
+import { resolveAuthMode, buildAuthHeaders } from '../auth.js';
+import { PdppUsageError, PdppHttpError } from '../errors.js';
+import { resolveReferenceUrl } from '../fetch.js';
+import { resolveFormat, writeData, writeEnvelopeWarnings } from '../output.js';
+
+const METHODS_WITH_BODY = new Set(['POST', 'PUT', 'PATCH', 'DELETE']);
+const KNOWN_METHODS = new Set(['GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD']);
+
+export async function runRefCall(argv, io = {}, fetchImpl = globalThis.fetch) {
+  const out = io.stdout || process.stdout;
+  const err = io.stderr || process.stderr;
+
+  const { flags, positionals } = parseArgs(argv);
+  const method = requirePositional(positionals, 0, 'method').toUpperCase();
+  if (!KNOWN_METHODS.has(method)) {
+    throw new PdppUsageError(
+      `Unsupported method: ${method}. Use one of ${[...KNOWN_METHODS].join(', ')}.`
+    );
+  }
+  const path = requirePositional(positionals, 1, 'path');
+
+  const referenceUrl = resolveReferenceUrl(flags);
+  const mode = resolveAuthMode(path, flags.auth);
+  const authHeaders = await buildAuthHeaders({ mode, referenceUrl, flags, io });
+
+  const body = await resolveBody(flags, io, method);
+
+  const headers = { Accept: 'application/json', ...authHeaders };
+  const init = { method, headers, redirect: 'manual' };
+  if (body !== undefined) {
+    headers['Content-Type'] = 'application/json';
+    init.body = body;
+  }
+
+  const url = `${referenceUrl}${path.startsWith('/') ? path : `/${path}`}`;
+
+  let resp;
+  try {
+    resp = await fetchImpl(url, init);
+  } catch (e) {
+    throw new PdppUsageError(`Network request to ${path} failed: ${e.message}`);
+  }
+
+  // Always surface the status line on stderr so machine-readable stdout stays
+  // clean. The URL is intentionally the path only — never the cookie/bearer.
+  err.write(`${method} ${path} → ${resp.status} ${resp.statusText || ''}`.trimEnd() + '\n');
+
+  if (flags['status-only']) {
+    return statusExitCode(resp.status);
+  }
+
+  const text = await readBody(resp);
+  let parsed = null;
+  if (text) {
+    try {
+      parsed = JSON.parse(text);
+    } catch {
+      parsed = text;
+    }
+  }
+
+  if (resp.status >= 400) {
+    const message =
+      (parsed && typeof parsed === 'object' && (parsed.error_description || parsed.error?.message || parsed.message)) ||
+      `HTTP ${resp.status} ${resp.statusText || ''}`.trim();
+    throw new PdppHttpError(String(message), resp.status, parsed);
+  }
+
+  if (parsed !== null && parsed !== '') {
+    const format = resolveFormat(flags, 'json', 'json');
+    writeData(parsed, format, out);
+    if (parsed && typeof parsed === 'object') {
+      writeEnvelopeWarnings(parsed, err);
+    }
+  }
+  return 0;
+}
+
+async function resolveBody(flags, io, method) {
+  const hasInline = typeof flags.data === 'string';
+  const hasStdin = Boolean(flags['data-stdin']);
+  if (hasInline && hasStdin) {
+    throw new PdppUsageError('Use only one of --data or --data-stdin, not both.');
+  }
+
+  let raw;
+  if (hasInline) {
+    raw = flags.data;
+  } else if (hasStdin) {
+    raw = await readAll((io && io.stdin) || process.stdin);
+  } else {
+    return undefined;
+  }
+
+  const trimmed = String(raw).trim();
+  if (!trimmed) {
+    return undefined;
+  }
+
+  // Validate that the body is JSON before sending; the server only parses
+  // application/json, and sending malformed JSON would surface as an opaque
+  // 400. Re-serialize so we send canonical JSON with the right content-type.
+  let value;
+  try {
+    value = JSON.parse(trimmed);
+  } catch (e) {
+    throw new PdppUsageError(`--data must be valid JSON: ${e.message}`);
+  }
+  if (!METHODS_WITH_BODY.has(method)) {
+    throw new PdppUsageError(`A request body is not valid for ${method}.`);
+  }
+  return JSON.stringify(value);
+}
+
+function statusExitCode(status) {
+  if (status >= 200 && status < 400) return 0;
+  if (status === 401) return 3;
+  if (status === 403) return 4;
+  if (status === 404) return 5;
+  return 1;
+}
+
+async function readBody(resp) {
+  if (typeof resp.text === 'function') {
+    return await resp.text();
+  }
+  return '';
+}
+
+function readAll(stream) {
+  return new Promise((resolve, reject) => {
+    if (!stream || typeof stream.on !== 'function') {
+      resolve('');
+      return;
+    }
+    let buf = '';
+    stream.setEncoding?.('utf8');
+    stream.on('data', (chunk) => {
+      buf += chunk;
+    });
+    stream.on('end', () => resolve(buf));
+    stream.on('error', reject);
+  });
+}