@vaultgradient/pipequery-cli 0.10.3 → 0.10.5

package/dist/sources/manager.d.ts

@@ -0,0 +1,296 @@
+type SourceConfig = RestSourceConfig | WebSocketSourceConfig | FileSourceConfig | StaticSourceConfig | PostgresSourceConfig | MysqlSourceConfig | SqliteSourceConfig | KafkaSourceConfig | SnowflakeSourceConfig | ClickhouseSourceConfig | MongoSourceConfig;
+interface RestSourceConfig {
+    type: 'rest';
+    url: string;
+    params?: Record<string, string>;
+    headers?: Record<string, string>;
+    dataPath?: string;
+    interval?: string;
+    /** Optional auth helper. Currently supports `bearer`; more kinds (basic,
+     *  apiKeyHeader, hmac-*) tracked in issue #37. The token is interpolated
+     *  through ${ENV_VAR} like every other string field. */
+    auth?: BearerAuth;
+}
+interface BearerAuth {
+    kind: 'bearer';
+    token: string;
+}
+interface WebSocketSourceConfig {
+    type: 'websocket';
+    url: string;
+    maxBuffer?: number;
+    /** Payload(s) to send to the server immediately after the WebSocket
+     *  opens. Most exchange feeds (Binance, Coinbase, Kraken, OKX, Bybit, …)
+     *  require this — without it, no data is delivered. Each payload is
+     *  JSON-stringified and sent in order. Re-sent on every reconnect. */
+    subscribe?: unknown | unknown[];
+    /** Optional keepalive: sends `payload` to the server every `interval`
+     *  while the socket is open. Most exchanges close idle sockets within
+     *  a few minutes; a periodic ping keeps the stream alive. */
+    heartbeat?: {
+        payload: unknown;
+        /** Duration like "30s", "1m". */
+        interval: string;
+    };
+}
+interface FileSourceConfig {
+    type: 'file';
+    path: string;
+    format?: 'json' | 'csv';
+    watch?: boolean;
+}
+interface StaticSourceConfig {
+    type: 'static';
+    data: unknown[];
+}
+interface PostgresSourceConfig {
+    type: 'postgres';
+    /**
+     * Postgres connection URL. Supports `${ENV_VAR}` interpolation so
+     * credentials can live in the environment instead of the yaml.
+     * Example: `postgres://${DB_USER}:${DB_PASS}@db.internal:5432/app`.
+     */
+    url: string;
+    /**
+     * The SELECT query to run on each poll. Must be a single statement.
+     * It's the caller's responsibility to add LIMIT/WHERE clauses that
+     * bound the result set; `maxRows` below is a hard safety cap.
+     */
+    query: string;
+    /** Poll interval, e.g. "30s", "5m". Defaults to "30s". */
+    interval?: string;
+    /**
+     * SSL policy. `"require"` verifies the cert (production default for
+     * most hosted Postgres). `"no-verify"` skips verification (useful for
+     * self-signed dev setups). `false` disables TLS entirely.
+     */
+    ssl?: 'require' | 'no-verify' | false;
+    /** Safety cap. If the query returns more rows, the fetch errors. Default 10000. */
+    maxRows?: number;
+}
+interface MysqlSourceConfig {
+    type: 'mysql';
+    /**
+     * MySQL connection URL. Supports `${ENV_VAR}` interpolation so
+     * credentials can live in the environment instead of the yaml.
+     * Example: `mysql://${DB_USER}:${DB_PASS}@db.internal:3306/app`.
+     */
+    url: string;
+    /** The SELECT query to run on each poll. */
+    query: string;
+    /** Poll interval, e.g. "30s", "5m". Defaults to "30s". */
+    interval?: string;
+    /**
+     * SSL policy. `"require"` verifies the cert (production default).
+     * `"no-verify"` skips verification. `false` disables TLS.
+     */
+    ssl?: 'require' | 'no-verify' | false;
+    /** Safety cap. Default 10000. */
+    maxRows?: number;
+}
+interface KafkaSourceConfig {
+    type: 'kafka';
+    /**
+     * Comma-separated list of Kafka bootstrap brokers, or an array. Supports
+     * `${ENV_VAR}` interpolation. Example: `"broker1:9092,broker2:9092"`.
+     */
+    brokers: string | string[];
+    /** Topic to subscribe to. Only one topic per source; use multiple sources for multiple topics. */
+    topic: string;
+    /**
+     * Consumer group ID. If omitted, an ephemeral per-process group is generated
+     * so this pipequery instance sees the full firehose. Set explicitly to share
+     * a partitioned subscription across pipequery replicas.
+     */
+    groupId?: string;
+    /** Start reading from the topic's beginning on first connect. Default false (latest). */
+    fromBeginning?: boolean;
+    /** Ring buffer size. Oldest messages are evicted. Default 1000. */
+    maxBuffer?: number;
+    /**
+     * How to decode the message value.
+     *   - `"json"` (default): `JSON.parse(buffer.toString())`; non-JSON messages are skipped.
+     *   - `"string"`: UTF-8 string, stored as `{ value }`.
+     *   - `"raw"`: base64-encoded bytes, stored as `{ value }`.
+     */
+    valueFormat?: 'json' | 'string' | 'raw';
+    /** SSL / TLS toggle. Default false (plaintext). */
+    ssl?: boolean;
+    /**
+     * SASL authentication. Supports `${ENV_VAR}` interpolation in username/password.
+     */
+    sasl?: {
+        mechanism: 'plain' | 'scram-sha-256' | 'scram-sha-512';
+        username: string;
+        password: string;
+    };
+}
+interface SqliteSourceConfig {
+    type: 'sqlite';
+    /**
+     * Path to the SQLite database file, relative to the directory
+     * containing pipequery.yaml. Use `:memory:` for an in-memory database
+     * (useful for tests; data will not persist across restarts).
+     */
+    path: string;
+    /** The SELECT query to run on each poll. */
+    query: string;
+    /** Poll interval, e.g. "30s", "5m". Defaults to "30s". */
+    interval?: string;
+    /** Open the database read-only. Recommended for query-only sources. Default true. */
+    readonly?: boolean;
+    /** Safety cap. Default 10000. */
+    maxRows?: number;
+}
+interface SnowflakeSourceConfig {
+    type: 'snowflake';
+    /** Snowflake account locator, e.g. `myorg-myaccount`. Supports `${ENV_VAR}`. */
+    account: string;
+    /** Username. Supports `${ENV_VAR}`. */
+    username: string;
+    /** Password. Supports `${ENV_VAR}`. (Key-pair / OAuth out of scope for v1.) */
+    password: string;
+    /** Database to USE. Supports `${ENV_VAR}`. */
+    database?: string;
+    /** Schema to USE. Supports `${ENV_VAR}`. */
+    schema?: string;
+    /** Compute warehouse to USE. Supports `${ENV_VAR}`. */
+    warehouse?: string;
+    /** Optional role. Supports `${ENV_VAR}`. */
+    role?: string;
+    /** SELECT query polled on every interval. */
+    query: string;
+    /** Poll interval, e.g. "60s". Default: 60s. */
+    interval?: string;
+    /** Hard cap on rows per fetch (default 10000). */
+    maxRows?: number;
+}
+interface ClickhouseSourceConfig {
+    type: 'clickhouse';
+    /** ClickHouse HTTP/HTTPS endpoint, e.g. `https://host:8443`. Supports `${ENV_VAR}`. */
+    url: string;
+    /** Username. Supports `${ENV_VAR}`. */
+    username?: string;
+    /** Password. Supports `${ENV_VAR}`. */
+    password?: string;
+    /** Database to query. Defaults to `default`. */
+    database?: string;
+    /** SELECT query polled on every interval. */
+    query: string;
+    /** Poll interval, e.g. "30s". Default: 30s. */
+    interval?: string;
+    /** Hard cap on rows per fetch (default 10000). */
+    maxRows?: number;
+}
+interface MongoSourceConfig {
+    type: 'mongodb';
+    /** MongoDB connection URI. Supports `${ENV_VAR}`. */
+    url: string;
+    /** Database name. Supports `${ENV_VAR}`. */
+    database: string;
+    /** Collection to read from. */
+    collection: string;
+    /**
+     * Optional default filter applied on every poll (Mongo `$match` stage).
+     * Plain JSON object — `{ active: true }` etc. The filter is `AND`-merged
+     * with any push-down `where()` predicates the engine compiles in.
+     */
+    filter?: Record<string, unknown>;
+    /** Poll interval, e.g. "30s". Default: 30s. */
+    interval?: string;
+    /** Hard cap on documents per fetch (default 10000). */
+    maxRows?: number;
+}
+
+interface SourceStatus {
+    healthy: boolean;
+    rowCount: number;
+    lastFetch: Date | null;
+    error?: string;
+}
+/**
+ * Result of an adapter-native push-down attempt.
+ *
+ * `ok: true` means the adapter executed the pipe expression natively (e.g.
+ * Postgres compiled it to SQL and ran it on the database) and returned rows
+ * without materializing the source query in memory. `sql` / `params` are
+ * optional telemetry — useful for audit logs and debugging, not required
+ * for callers to act on.
+ *
+ * `ok: false` is the explicit decline channel: the adapter recognised the
+ * shape but declined (unsupported op, multi-source pipeline, etc.). The
+ * caller should fall back to in-process execution against `getContext()`.
+ *
+ * Any thrown error means *neither* path succeeded — let it propagate; the
+ * caller has no useful fallback.
+ */
+type PushdownResult = {
+    ok: true;
+    rows: unknown[];
+    sql?: string;
+    params?: unknown[];
+} | {
+    ok: false;
+    reason: string;
+};
+
+type DataContext = Record<string, unknown[]>;
+type UpdateListener = (sourceName: string, data: unknown[]) => void;
+declare class SourceManager {
+    private cwd;
+    private sources;
+    private globalListeners;
+    constructor(cwd: string);
+    addSource(name: string, config: SourceConfig): Promise<void>;
+    removeSource(name: string): boolean;
+    getContext(): DataContext;
+    getSourceData(name: string): unknown[] | undefined;
+    getSourceStatus(name: string): SourceStatus | undefined;
+    getAllStatuses(): Record<string, SourceStatus>;
+    getSourceNames(): string[];
+    /**
+     * Subscribe to data updates from any source.
+     * Used by SSE endpoint for real-time push to TUI clients.
+     * Returns unsubscribe function.
+     */
+    onSourceUpdate(callback: UpdateListener): () => void;
+    refreshAll(): Promise<void>;
+    /**
+     * Run a pipe expression, preferring adapter-native push-down when the
+     * source supports it.
+     *
+     * Decision tree:
+     *   1. Parse the expression. If it can't parse, fall straight through to
+     *      the in-process engine, which will throw the same parse error a
+     *      caller would have seen before push-down existed (no behaviour
+     *      change for malformed input).
+     *   2. Look up the source named in the pipeline. If it exists and its
+     *      adapter exposes `runPushdown`, try it. On `{ ok: true }` we're
+     *      done — return the rows. On `{ ok: false }` (decline) or any
+     *      thrown error, fall back to in-process.
+     *   3. In-process: query(getContext(), expression) — the existing path.
+     *
+     * The push-down attempt is best-effort and *invisible to callers*: same
+     * return type as `query()`, no new failure modes. Telemetry (which
+     * branch ran, latency, source-engine SQL) belongs in the audit log when
+     * Phase 5 lands; not threaded through here.
+     */
+    runQuery(expression: string): Promise<unknown>;
+    /**
+     * Inspect the pipeline AST and dispatch to an adapter's runPushdown when
+     * eligible. Returns `null` when push-down isn't even attempted (no parse,
+     * no source, no capability). Returns a PushdownResult when it was tried —
+     * `{ ok: true }` for success, `{ ok: false, reason }` for decline.
+     *
+     * Exposed on the manager (rather than tucked inside runQuery) so the
+     * future `event.audit` channel can record `pushed_down: bool` and the
+     * `source_engine_query` SQL string for governance. Today only Postgres
+     * implements runPushdown; MySQL / SQLite / ClickHouse will follow the
+     * same interface.
+     */
+    tryPushdown(expression: string): Promise<PushdownResult | null>;
+    dispose(): Promise<void>;
+    private createAdapter;
+}
+
+export { type DataContext, SourceManager };