@sun-asterisk/sungen 3.2.10 → 3.2.11

package/src/orchestrator/templates/specs-db.ts

@@ -21,9 +21,13 @@ const ident = (s: string): string => {
   return s;
 };
 
-// Single source of truth for the "not supported" error message below and for the mysql/sqlite
-// placeholder rewrite condition (`rewritePlaceholders`) — keep engine names in sync with `DataSourceConfig.engine`.
+// SQL engines: verified via the direct guarded-SELECT path and needing the `$n`→`?` placeholder
+// rewrite. This literal is the single source of truth for that rewrite condition and is quoted, by
+// name, in the sungen-gherkin-syntax skill's "direct SQL verification" note — keep them in sync.
 const SUPPORTED_ENGINES = ['postgres', 'sqlite', 'mysql'] as const;
+// NoSQL engines: implement the Engine contract natively (no SQL rewrite). Tracked separately so the
+// SQL-rewrite guard and its skill-parity test stay pinned to the three SQL engines above.
+const NOSQL_ENGINES = ['cosmos', 'mongodb', 'dynamodb'] as const;
 
 interface SshConfig {
   host: string;              // jump host reachable from the runner
@@ -36,7 +40,7 @@ interface SshConfig {
 }
 
 interface DataSourceConfig {
-  engine: 'postgres' | 'mysql' | 'sqlite';
+  engine: 'postgres' | 'mysql' | 'sqlite' | 'cosmos' | 'mongodb' | 'dynamodb';
   url: string;
   readonly?: boolean;
   statement_timeout_ms?: number;
@@ -44,6 +48,17 @@ interface DataSourceConfig {
   // Fallback path: tunnel the DB SOCKET through an SSH bastion. DB-only — the browser/E2E
   // still run on the runner; only DB traffic crosses. See docs/spec/sungen_data_driver_ssh_tunnel_spec.md.
   ssh?: SshConfig;
+  // Cosmos DB (NoSQL API): connect via endpoint+key (or a connection-string `url`); `database` is the
+  // Cosmos database, `container` the default container when a step/catalog names no `[table]`.
+  endpoint?: string;
+  key?: string;
+  database?: string;
+  container?: string;
+  // DynamoDB: no connection URL — `region` + credentials from the AWS env chain (AWS_* / IAM role).
+  // `keySchema` maps each table to its key attributes so declarative steps can route GetItem/Query
+  // instead of an unaffordable full-table Scan; a non-key filter is refused (use a raw @query SELECT).
+  region?: string;
+  keySchema?: Record<string, { partitionKey: string; sortKey?: string | null }>;
 }
 
 /**
@@ -125,37 +140,365 @@ function loadConfig(): Record<string, DataSourceConfig> {
 }
 
 /**
- * `build()` always emits PG-style `$1..$n` placeholders; mysql2/better-sqlite3 both bind with `?`.
- * Only real placeholders are rewritten — a `$1` sitting inside a single-quoted string literal (e.g.
- * a catalog query `WHERE label = '$1 off'`) is left untouched. SQL escapes an inner quote by doubling
- * it (`''`), which the simple in-string toggle handles (an even number of quotes returns to the same state).
+ * `build()` always emits PG-style `$1..$n` placeholders; mysql2/better-sqlite3 both bind with
+ * positional `?` (left-to-right). Rewriting `$n`→`?` must therefore ALSO reorder the params to the
+ * textual placeholder order: an author-written catalog query `WHERE a = $2 AND b = $1` becomes
+ * `a = ? AND b = ?`, so the params must be bound `[params[1], params[0]]` — otherwise the values
+ * silently swap and the query returns wrong rows. A repeated `$k` duplicates its value at each `?`.
+ * Postgres binds `$n` natively, so it is returned untouched.
+ *
+ * Only real placeholders are rewritten — a `$1` inside a single-quoted string literal (e.g.
+ * `WHERE label = '$1 off'`) is left untouched. SQL escapes an inner quote by doubling it (`''`),
+ * which the simple in-string toggle handles (an even number of quotes returns to the same state).
+ *
+ * DynamoDB PartiQL binds the same positional `?` as mysql2/sqlite, so it shares this path — the
+ * reorder is essential there too (an author-written PartiQL `WHERE sk = $2 AND pk = $1`).
  */
-export function rewritePlaceholders(engine: string, sql: string): string {
-  if (engine !== 'sqlite' && engine !== 'mysql') return sql;
+export function rewritePlaceholders(engine: string, sql: string, params: any[]): { sql: string; params: any[] } {
+  if (engine !== 'sqlite' && engine !== 'mysql' && engine !== 'dynamodb') return { sql, params };
   let out = '';
   let inStr = false;
+  const newParams: any[] = [];
   for (let i = 0; i < sql.length; i++) {
     const c = sql[i];
     if (c === "'") { inStr = !inStr; out += c; continue; }
     if (!inStr && c === '$' && sql[i + 1] >= '0' && sql[i + 1] <= '9') {
       let j = i + 1;
       while (j < sql.length && sql[j] >= '0' && sql[j] <= '9') j++;
+      const k = Number(sql.slice(i + 1, j)); // 1-based placeholder index
       out += '?';
+      newParams.push(params[k - 1]);
       i = j - 1;
       continue;
     }
     out += c;
   }
-  return out;
+  return { sql: out, params: newParams };
+}
+
+/**
+ * Engine contract — the assertion surface is engine-agnostic; each engine owns its own dialect so
+ * a NoSQL store (Cosmos / MongoDB / DynamoDB) can back the same declarative steps as SQL.
+ *   findRows(table, filter, limit) — rows matching an equality filter, capped at `limit`
+ *   countRows(table, filter)       — count of rows matching an equality filter
+ *   runQuery(label, sql, params)   — a raw catalog query (read-only). SQL dialects (incl. Cosmos SQL
+ *                                    and DynamoDB PartiQL) take a SQL string; MongoDB's non-SQL
+ *                                    find/pipeline shapes are dispatched separately via fetchQueryShape.
+ */
+interface Engine {
+  findRows(table: string, filter: Record<string, any>, limit: number): Promise<any[]>;
+  countRows(table: string, filter: Record<string, any>): Promise<number>;
+  runQuery(label: string, sql: string, params: any[]): Promise<any[]>;
+  // Non-SQL raw query (MongoDB find/pipeline). Only engines with a non-SQL catalog shape implement it;
+  // fetchQueryShape refuses the call on an engine that doesn't.
+  runShape?(label: string, shape: ShapeQuery, params: Record<string, any>): Promise<any[]>;
+}
+
+/** A MongoDB catalog entry's runtime shape: exactly one of find/pipeline, with `:param` placeholders bound at runtime. */
+type ShapeQuery = {
+  find?: { collection: string; filter?: Record<string, any>; projection?: Record<string, any>; sort?: Record<string, any>; limit?: number };
+  pipeline?: { collection: string; stages: any[] };
+};
+
+/** Read-only guard (second layer): a named SQL query must be a single SELECT/WITH statement. */
+function assertSelectOnly(label: string, sql: string): void {
+  const s = sql.trim().replace(/;\s*$/, '');
+  if (!/^(SELECT|WITH)\b/i.test(s)) throw new Error(`Data Driver: ${label} is not a read-only SELECT — refused.`);
+  if (s.includes(';')) throw new Error(`Data Driver: ${label} contains multiple statements — refused.`);
+  if (/\b(INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|TRUNCATE|GRANT|REVOKE|MERGE|REPLACE|CALL|EXEC|EXECUTE|ATTACH|PRAGMA|VACUUM)\b/i.test(s)) {
+    throw new Error(`Data Driver: ${label} contains a write/DDL keyword — refused.`);
+  }
+}
+
+/**
+ * SQL engine (postgres / mysql / sqlite). Builds the exact guarded, parameterized SELECTs the Data
+ * Driver has always emitted — `SELECT * … WHERE col = $n LIMIT`, `SELECT count(*)`, the `$n`→`?`
+ * reorder, and the read-only catalog guard — so the query strings are byte-identical to the
+ * pre-refactor path. `exec` is the lazy-loaded driver call (pg/mysql2/better-sqlite3).
+ */
+class SqlEngine implements Engine {
+  constructor(private engineName: string, private exec: (sql: string, params: any[]) => Promise<any[]>) {}
+
+  private rewrite(sql: string, params: any[]): { sql: string; params: any[] } {
+    return rewritePlaceholders(this.engineName, sql, params);
+  }
+
+  async findRows(table: string, filter: Record<string, any>, limit: number): Promise<any[]> {
+    const cols = Object.keys(filter);
+    const where = cols.map((c, i) => `${ident(c)} = $${i + 1}`).join(' AND ');
+    const sql = `SELECT * FROM ${ident(table)}${where ? ' WHERE ' + where : ''} LIMIT ${limit}`;
+    const q = this.rewrite(sql, cols.map((c) => filter[c]));
+    return this.exec(q.sql, q.params);
+  }
+
+  async countRows(table: string, filter: Record<string, any>): Promise<number> {
+    const cols = Object.keys(filter);
+    const where = cols.map((c, i) => `${ident(c)} = $${i + 1}`).join(' AND ');
+    const sql = `SELECT count(*) AS n FROM ${ident(table)}${where ? ' WHERE ' + where : ''}`;
+    const q = this.rewrite(sql, cols.map((c) => filter[c]));
+    const rows = await this.exec(q.sql, q.params);
+    return Number(rows[0]?.n ?? rows[0]?.['count(*)'] ?? 0);
+  }
+
+  async runQuery(label: string, sql: string, params: any[]): Promise<any[]> {
+    assertSelectOnly(label, sql);
+    const q = this.rewrite(sql, params);
+    return this.exec(q.sql, q.params);
+  }
 }
 
-type Engine = { query(sql: string, params: any[]): Promise<any[]>; };
+/** Minimal shape of an @azure/cosmos container the engine uses — kept local so the SDK stays a lazy dep. */
+type CosmosContainer = { items: { query(spec: { query: string; parameters: Array<{ name: string; value: any }> }, opts?: any): { fetchAll(): Promise<{ resources: any[] }> } } };
+
+/**
+ * Azure Cosmos DB (NoSQL API) engine. The Cosmos query API is read-only by construction — `.query()`
+ * only ever reads; mutations require item-level APIs this engine never calls — so the SELECT guard is
+ * defense-in-depth. Cosmos binds NAMED params (`@p`), so it is immune to the positional-`?` reorder
+ * hazard. A step/catalog `[table]` selects the container; `SELECT * FROM c` uses the mandatory `c` alias.
+ */
+export class CosmosEngine implements Engine {
+  constructor(
+    private getContainer: (name?: string) => CosmosContainer,
+    private opts: { timeoutMs: number; maxRows: number },
+  ) {}
+
+  private async query(container: CosmosContainer, spec: { query: string; parameters: Array<{ name: string; value: any }> }): Promise<any[]> {
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(), this.opts.timeoutMs);
+    try {
+      const res = await container.items.query(spec, { maxItemCount: this.opts.maxRows, abortSignal: ac.signal }).fetchAll();
+      return res.resources.slice(0, this.opts.maxRows);
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  /** Build `c.<col> = @p<i>` equality predicates over the mandatory `c` alias. Columns are identifier-checked. */
+  private whereFor(filter: Record<string, any>): { where: string; parameters: Array<{ name: string; value: any }> } {
+    const cols = Object.keys(filter);
+    const where = cols.map((c, i) => `c.${ident(c)} = @p${i}`).join(' AND ');
+    const parameters = cols.map((c, i) => ({ name: `@p${i}`, value: filter[c] }));
+    return { where: where ? ` WHERE ${where}` : '', parameters };
+  }
+
+  async findRows(table: string, filter: Record<string, any>, limit: number): Promise<any[]> {
+    const { where, parameters } = this.whereFor(filter);
+    // TOP bounds the fetch server-side (like the SQL engine's LIMIT) — without it, fetchAll() would
+    // drain every matching row into memory before the client-side slice, wasting RUs + latency.
+    const cap = Math.min(limit, this.opts.maxRows);
+    const rows = await this.query(this.getContainer(table), { query: `SELECT TOP ${cap} * FROM c${where}`, parameters });
+    return rows.slice(0, limit);
+  }
+
+  async countRows(table: string, filter: Record<string, any>): Promise<number> {
+    const { where, parameters } = this.whereFor(filter);
+    // SELECT VALUE COUNT(1) returns the scalar count as the single resource.
+    const rows = await this.query(this.getContainer(table), { query: `SELECT VALUE COUNT(1) FROM c${where}`, parameters });
+    return Number(rows[0] ?? 0);
+  }
+
+  async runQuery(label: string, sql: string, params: any[]): Promise<any[]> {
+    assertSelectOnly(label, sql); // defense-in-depth; the query API is read-only anyway
+    const { query, parameters } = cosmosBindParams(sql, params);
+    return this.query(this.getContainer(), { query, parameters });
+  }
+}
+
+/**
+ * Rewrite compiled positional `$n` placeholders to Cosmos NAMED params (`@pN`) and collect the
+ * matching parameter list. A `$n` inside a single-quoted string literal is left untouched (same
+ * in-string toggle as the SQL rewrite); a repeated `$n` reuses its one `@pN` entry.
+ */
+export function cosmosBindParams(sql: string, params: any[]): { query: string; parameters: Array<{ name: string; value: any }> } {
+  const parameters: Array<{ name: string; value: any }> = [];
+  const seen = new Set<number>();
+  let out = '';
+  let inStr = false;
+  for (let i = 0; i < sql.length; i++) {
+    const c = sql[i];
+    if (c === "'") { inStr = !inStr; out += c; continue; }
+    if (!inStr && c === '$' && sql[i + 1] >= '0' && sql[i + 1] <= '9') {
+      let j = i + 1;
+      while (j < sql.length && sql[j] >= '0' && sql[j] <= '9') j++;
+      const k = Number(sql.slice(i + 1, j));
+      out += `@p${k}`;
+      if (!seen.has(k)) { seen.add(k); parameters.push({ name: `@p${k}`, value: params[k - 1] }); }
+      i = j - 1;
+      continue;
+    }
+    out += c;
+  }
+  return { query: out, parameters };
+}
+
+/** Minimal shapes of the mongodb driver objects the engine uses — kept local so the SDK stays a lazy dep. */
+type MongoCursor = { project(p: any): MongoCursor; sort(s: any): MongoCursor; limit(n: number): MongoCursor; maxTimeMS(t: number): MongoCursor; toArray(): Promise<any[]> };
+type MongoCollection = { find(filter: any): MongoCursor; countDocuments(filter: any, opts?: any): Promise<number>; aggregate(pipeline: any[], opts?: any): { toArray(): Promise<any[]> } };
+
+/** Forbidden MongoDB operators (write-out / server-side JS) — refused at runtime as defense-in-depth. */
+const MONGO_FORBIDDEN = /\$out|\$merge|\$where|\$function|\$accumulator|mapReduce/i;
+
+/** Replace every whole-string `:param` in a find/pipeline shape with its bound native value (type-preserving). */
+function substituteShapeParams(obj: any, params: Record<string, any>): any {
+  if (typeof obj === 'string') {
+    const m = obj.match(/^:([A-Za-z_][A-Za-z0-9_]*)$/);
+    return m ? params[m[1]] : obj;
+  }
+  if (Array.isArray(obj)) return obj.map((v) => substituteShapeParams(v, params));
+  if (obj && typeof obj === 'object') {
+    const out: Record<string, any> = {};
+    for (const [k, v] of Object.entries(obj)) out[k] = substituteShapeParams(v, params);
+    return out;
+  }
+  return obj;
+}
+
+/** Refuse a shape whose serialized form contains a write/JS-exec operator (nested or aliased). */
+function assertMongoReadOnly(label: string, shape: any): void {
+  const m = JSON.stringify(shape ?? {}).match(MONGO_FORBIDDEN);
+  if (m) throw new Error(`Data Driver: ${label} uses forbidden MongoDB operator "${m[0]}" — refused (read-only find/aggregate only).`);
+}
+
+/**
+ * MongoDB engine. There is no SQL — declarative steps map to native find()/countDocuments(), and the
+ * raw `@query` path takes a find/pipeline shape (via fetchQueryShape → runShape) rather than a SQL
+ * string. Type-strict matching (`{age:30}` ≠ `{age:"30"}`) is why DB params bind via raw() (native
+ * type), not the stringifying get(). Read-only is enforced first by the server-side read role, with
+ * the operator blocklist as in-driver defense-in-depth. Per-query `maxTimeMS` + row cap bound cost.
+ */
+export class MongoEngine implements Engine {
+  constructor(
+    private getCollection: (name: string) => MongoCollection,
+    private opts: { timeoutMs: number; maxRows: number },
+  ) {}
+
+  async findRows(table: string, filter: Record<string, any>, limit: number): Promise<any[]> {
+    return this.getCollection(table).find(filter).limit(limit).maxTimeMS(this.opts.timeoutMs).toArray();
+  }
+
+  async countRows(table: string, filter: Record<string, any>): Promise<number> {
+    return this.getCollection(table).countDocuments(filter, { maxTimeMS: this.opts.timeoutMs });
+  }
+
+  async runQuery(label: string): Promise<any[]> {
+    throw new Error(`Data Driver: ${label} — MongoDB has no SQL; author a find:/pipeline: catalog entry instead of sql:.`);
+  }
+
+  async runShape(label: string, shape: ShapeQuery, params: Record<string, any>): Promise<any[]> {
+    if (shape.find) {
+      const f = substituteShapeParams(shape.find, params) as NonNullable<ShapeQuery['find']>;
+      assertMongoReadOnly(label, f);
+      let cursor = this.getCollection(f.collection).find(f.filter ?? {});
+      if (f.projection) cursor = cursor.project(f.projection);
+      if (f.sort) cursor = cursor.sort(f.sort);
+      cursor = cursor.limit(f.limit ?? this.opts.maxRows).maxTimeMS(this.opts.timeoutMs);
+      return cursor.toArray();
+    }
+    if (shape.pipeline) {
+      const p = substituteShapeParams(shape.pipeline, params) as NonNullable<ShapeQuery['pipeline']>;
+      assertMongoReadOnly(label, p);
+      const rows = await this.getCollection(p.collection).aggregate(p.stages, { maxTimeMS: this.opts.timeoutMs }).toArray();
+      return rows.slice(0, this.opts.maxRows);
+    }
+    throw new Error(`Data Driver: ${label} — catalog shape has neither find: nor pipeline:.`);
+  }
+}
+
+/** The three read-only DynamoDB operations the engine needs, over the auto-marshalling Document client. */
+type DynamoOps = {
+  executeStatement(input: { Statement: string; Parameters?: any[]; Limit?: number; NextToken?: string }, signal?: AbortSignal): Promise<{ Items?: any[]; NextToken?: string }>;
+  getItem(input: { TableName: string; Key: Record<string, any> }, signal?: AbortSignal): Promise<{ Item?: any }>;
+  query(input: { TableName: string; KeyConditionExpression: string; ExpressionAttributeNames: Record<string, string>; ExpressionAttributeValues: Record<string, any>; Select?: string; Limit?: number; ExclusiveStartKey?: Record<string, any> }, signal?: AbortSignal): Promise<{ Items?: any[]; Count?: number; LastEvaluatedKey?: Record<string, any> }>;
+};
+
+type KeySchema = Record<string, { partitionKey: string; sortKey?: string | null }>;
+type DynamoRoute = { kind: 'get'; key: Record<string, any> } | { kind: 'query'; pk: string; value: any };
+
+/**
+ * DynamoDB engine. The clean primary path is raw `@query` PartiQL SELECT (positional `?`, reordered
+ * via the shared placeholder rewrite). Declarative assertRow/count is structurally constrained by
+ * DynamoDB's key model: it routes GetItem/Query only for pk / pk+sk filters (via the `keySchema`
+ * registry) and REFUSES a non-key filter rather than falling back to an unaffordable full-table Scan.
+ * Read-only is primarily IAM (`dynamodb:PartiQLSelect` only); the SELECT guard is defense-in-depth.
+ */
+export class DynamoEngine implements Engine {
+  constructor(
+    private ops: DynamoOps,
+    private keySchema: KeySchema,
+    private opts: { timeoutMs: number; maxRows: number },
+  ) {}
+
+  private async withTimeout<T>(fn: (signal: AbortSignal) => Promise<T>): Promise<T> {
+    const ac = new AbortController();
+    const timer = setTimeout(() => ac.abort(), this.opts.timeoutMs);
+    try { return await fn(ac.signal); } finally { clearTimeout(timer); }
+  }
+
+  /** Decide how a flat equality filter maps to a key-based access pattern, or refuse it. */
+  private route(table: string, filter: Record<string, any>): DynamoRoute {
+    const ks = this.keySchema[table];
+    if (!ks) throw new Error(`Data Driver: DynamoDB table "${table}" has no keySchema entry — add it under the datasource's keySchema:, or use a raw @query PartiQL SELECT.`);
+    const pk = ks.partitionKey;
+    const sk = ks.sortKey || undefined;
+    const keyAttrs = sk ? [pk, sk] : [pk];
+    const keys = Object.keys(filter);
+    const nonKey = keys.filter((k) => !keyAttrs.includes(k));
+    if (nonKey.length) throw new Error(`Data Driver: DynamoDB "${table}" filter has non-key attribute(s) [${nonKey.join(', ')}] — routing would require a full-table Scan (refused). Use a raw @query PartiQL SELECT instead.`);
+    if (!keys.includes(pk)) throw new Error(`Data Driver: DynamoDB "${table}" filter must include the partition key "${pk}".`);
+    if (sk && keys.includes(sk)) return { kind: 'get', key: { [pk]: filter[pk], [sk]: filter[sk] } };
+    if (!sk) return { kind: 'get', key: { [pk]: filter[pk] } };
+    return { kind: 'query', pk, value: filter[pk] }; // pk only on a table that has a sort key → Query
+  }
+
+  async findRows(table: string, filter: Record<string, any>, limit: number): Promise<any[]> {
+    const route = this.route(table, filter);
+    const cap = Math.min(limit, this.opts.maxRows);
+    if (route.kind === 'get') {
+      const res = await this.withTimeout((s) => this.ops.getItem({ TableName: table, Key: route.key }, s));
+      return res.Item ? [res.Item] : [];
+    }
+    const res = await this.withTimeout((s) => this.ops.query({ TableName: table, KeyConditionExpression: '#pk = :pk', ExpressionAttributeNames: { '#pk': route.pk }, ExpressionAttributeValues: { ':pk': route.value }, Limit: cap }, s));
+    return (res.Items ?? []).slice(0, cap);
+  }
+
+  async countRows(table: string, filter: Record<string, any>): Promise<number> {
+    const route = this.route(table, filter);
+    if (route.kind === 'get') {
+      const res = await this.withTimeout((s) => this.ops.getItem({ TableName: table, Key: route.key }, s));
+      return res.Item ? 1 : 0;
+    }
+    // A Query with Select:COUNT evaluates at most ~1MB per request and returns a PARTIAL Count with a
+    // LastEvaluatedKey for the rest — accumulate across pages so the exact-equality assert is correct.
+    let count = 0;
+    let startKey: Record<string, any> | undefined;
+    do {
+      const res = await this.withTimeout((s) => this.ops.query({ TableName: table, KeyConditionExpression: '#pk = :pk', ExpressionAttributeNames: { '#pk': route.pk }, ExpressionAttributeValues: { ':pk': route.value }, Select: 'COUNT', ExclusiveStartKey: startKey }, s));
+      count += Number(res.Count ?? 0);
+      startKey = res.LastEvaluatedKey;
+    } while (startKey);
+    return count;
+  }
+
+  async runQuery(label: string, sql: string, params: any[]): Promise<any[]> {
+    assertSelectOnly(label, sql); // defense-in-depth; primary read-only guarantee is IAM PartiQLSelect-only
+    const bound = rewritePlaceholders('dynamodb', sql, params); // $n → ? positional, reordered to textual order
+    const items: any[] = [];
+    let nextToken: string | undefined;
+    do {
+      const page = await this.withTimeout((s) => this.ops.executeStatement({ Statement: bound.sql, Parameters: bound.params.length ? bound.params : undefined, Limit: this.opts.maxRows, NextToken: nextToken }, s));
+      items.push(...(page.Items ?? []));
+      nextToken = page.NextToken;
+    } while (nextToken && items.length < this.opts.maxRows); // manual pagination, bounded by the row cap
+    return items.slice(0, this.opts.maxRows);
+  }
+}
 
 class DataSource {
   private configs: Record<string, DataSourceConfig> | null = null;
   private engines = new Map<string, Engine>();
   private tunnels: Array<{ close: () => void }> = [];
   private mysqlPools: Array<{ end: () => Promise<void> }> = [];
+  private mongoClients: Array<{ close: () => Promise<void> }> = [];
 
   private cfg(name?: string): { key: string; conf: DataSourceConfig } {
     if (!this.configs) this.configs = loadConfig();
@@ -168,8 +511,27 @@ class DataSource {
   private async engine(name?: string): Promise<{ engine: Engine; conf: DataSourceConfig }> {
     const { key, conf } = this.cfg(name);
     if (this.engines.has(key)) return { engine: this.engines.get(key)!, conf };
-    if (!conf.url) throw new Error(`Data Driver: datasource "${key}" has no url (set it in .env.qa).`);
     let engine: Engine;
+    if (conf.engine === 'postgres' || conf.engine === 'mysql' || conf.engine === 'sqlite') {
+      // url is required for the SQL engines (host-based, or a sqlite file path). NoSQL engines that
+      // have no connection URL (e.g. DynamoDB region+creds) supply their own config and skip this.
+      if (!conf.url) throw new Error(`Data Driver: datasource "${key}" has no url (set it in .env.qa).`);
+      engine = new SqlEngine(conf.engine, await this.sqlExec(key, conf));
+    } else if (conf.engine === 'cosmos') {
+      engine = this.cosmosEngine(key, conf);
+    } else if (conf.engine === 'mongodb') {
+      engine = this.mongoEngine(key, conf);
+    } else if (conf.engine === 'dynamodb') {
+      engine = this.dynamoEngine(key, conf);
+    } else {
+      throw new Error(`Data Driver: engine "${conf.engine}" not supported (supported: ${[...SUPPORTED_ENGINES, ...NOSQL_ENGINES].join(' | ')}). For an unlisted store, verify via @api or @manual.`);
+    }
+    this.engines.set(key, engine);
+    return { engine, conf };
+  }
+
+  /** Build the lazy driver-call closure for a SQL engine (pg/mysql/sqlite), incl. the optional SSH-tunnel fallback. */
+  private async sqlExec(key: string, conf: DataSourceConfig): Promise<(sql: string, params: any[]) => Promise<any[]>> {
     if (conf.engine === 'postgres') {
       let connectionString = conf.url;
       if (conf.ssh) {                                   // Fallback path: tunnel the DB socket through a bastion
@@ -181,8 +543,9 @@ class DataSource {
       }
       const { Pool } = require('pg');
       const pool = new Pool({ connectionString, max: 2, statement_timeout: conf.statement_timeout_ms ?? 4000 });
-      engine = { query: async (sql, params) => (await pool.query(sql, params)).rows };
-    } else if (conf.engine === 'mysql') {
+      return async (sql, params) => (await pool.query(sql, params)).rows;
+    }
+    if (conf.engine === 'mysql') {
       let connectionString = conf.url;
       if (conf.ssh) {                                   // Fallback path: tunnel the DB socket through a bastion
         const u = new URL(conf.url);
@@ -199,58 +562,89 @@ class DataSource {
       // against a timer. On overrun we reject and destroy the connection (removes it from the pool
       // rather than releasing a still-busy socket); on success we release it back.
       const timeoutMs = conf.statement_timeout_ms ?? 4000;
-      engine = {
-        query: async (sql, params) => {
-          const conn = await pool.getConnection();
-          const qp = conn.query(sql, params);
-          qp.catch(() => { /* swallow the post-timeout rejection from the abandoned query */ });
-          let timer: NodeJS.Timeout | undefined;
-          let timedOut = false;
-          const guard = new Promise<never>((_, reject) => {
-            timer = setTimeout(() => { timedOut = true; reject(new Error(`Data Driver: mysql query exceeded ${timeoutMs}ms — aborted.`)); }, timeoutMs);
-          });
-          try {
-            const [rows] = await Promise.race([qp, guard]);
-            return rows as any[];
-          } finally {
-            clearTimeout(timer);
-            if (timedOut) { try { conn.destroy(); } catch { /* best-effort */ } }
-            else { try { conn.release(); } catch { /* already gone */ } }
-          }
-        },
+      return async (sql, params) => {
+        const conn = await pool.getConnection();
+        const qp = conn.query(sql, params);
+        qp.catch(() => { /* swallow the post-timeout rejection from the abandoned query */ });
+        let timer: NodeJS.Timeout | undefined;
+        let timedOut = false;
+        const guard = new Promise<never>((_, reject) => {
+          timer = setTimeout(() => { timedOut = true; reject(new Error(`Data Driver: mysql query exceeded ${timeoutMs}ms — aborted.`)); }, timeoutMs);
+        });
+        try {
+          const [rows] = await Promise.race([qp, guard]);
+          return rows as any[];
+        } finally {
+          clearTimeout(timer);
+          if (timedOut) { try { conn.destroy(); } catch { /* best-effort */ } }
+          else { try { conn.release(); } catch { /* already gone */ } }
+        }
       };
-    } else if (conf.engine === 'sqlite') {
-      if (conf.ssh) console.warn(`Data Driver: datasource "${key}" sets ssh: but engine is sqlite (file-based) — ssh ignored.`);
-      const Database = require('better-sqlite3');
-      const db = new Database(conf.url.replace(/^sqlite:/, ''), { readonly: conf.readonly !== false });
-      engine = { query: async (sql, params) => db.prepare(sql).all(...params) };
-    } else {
-      throw new Error(`Data Driver: engine "${conf.engine}" not supported (supported: ${SUPPORTED_ENGINES.join(' | ')}). For non-SQL stores (Cosmos NoSQL, Mongo…) verify via @api or @manual.`);
     }
-    this.engines.set(key, engine);
-    return { engine, conf };
+    // sqlite
+    if (conf.ssh) console.warn(`Data Driver: datasource "${key}" sets ssh: but engine is sqlite (file-based) — ssh ignored.`);
+    const Database = require('better-sqlite3');
+    const db = new Database(conf.url.replace(/^sqlite:/, ''), { readonly: conf.readonly !== false });
+    return async (sql, params) => db.prepare(sql).all(...params);
+  }
+
+  /** Build the Cosmos engine — connect via endpoint+key or a connection-string url; resolve containers by name. */
+  private cosmosEngine(key: string, conf: DataSourceConfig): Engine {
+    const { CosmosClient } = require('@azure/cosmos');
+    let client: any;
+    if (conf.endpoint && conf.key) client = new CosmosClient({ endpoint: conf.endpoint, key: conf.key });
+    else if (conf.url) client = new CosmosClient(conf.url); // AccountEndpoint=…;AccountKey=…; connection string
+    else throw new Error(`Data Driver: cosmos datasource "${key}" needs endpoint+key (or a connection-string url).`);
+    if (!conf.database) throw new Error(`Data Driver: cosmos datasource "${key}" needs a database name.`);
+    const getContainer = (name?: string): CosmosContainer => {
+      const cn = name || conf.container;
+      if (!cn) throw new Error(`Data Driver: cosmos datasource "${key}" — no container (set container: in config, or name a [table]).`);
+      return client.database(conf.database!).container(cn);
+    };
+    return new CosmosEngine(getContainer, { timeoutMs: conf.statement_timeout_ms ?? 4000, maxRows: conf.max_rows ?? 100 });
+  }
+
+  /** Build the MongoDB engine — one reused client per datasource; the db comes from `database` or the URI default. */
+  private mongoEngine(key: string, conf: DataSourceConfig): Engine {
+    if (!conf.url) throw new Error(`Data Driver: mongodb datasource "${key}" needs a url (mongodb:// URI).`);
+    const { MongoClient } = require('mongodb');
+    const client = new MongoClient(conf.url);       // operations auto-connect; one client is reused
+    this.mongoClients.push(client);
+    const dbHandle = client.db(conf.database);       // undefined → the URI's default database
+    return new MongoEngine((name: string) => dbHandle.collection(name), { timeoutMs: conf.statement_timeout_ms ?? 5000, maxRows: conf.max_rows ?? 50 });
   }
 
-  /** Close open MySQL pools + SSH tunnels (optional explicit teardown; tunnels are unref'd so the process exits regardless). */
+  /** Build the DynamoDB engine — region + credentials from the AWS env chain; wraps the Document client's three read ops. */
+  private dynamoEngine(key: string, conf: DataSourceConfig): Engine {
+    if (!conf.region) throw new Error(`Data Driver: dynamodb datasource "${key}" needs a region (DynamoDB has no url).`);
+    const { DynamoDBClient } = require('@aws-sdk/client-dynamodb');
+    const { DynamoDBDocumentClient, ExecuteStatementCommand, GetCommand, QueryCommand } = require('@aws-sdk/lib-dynamodb');
+    const base = new DynamoDBClient({ region: conf.region }); // creds resolved from env / IAM role
+    const doc = DynamoDBDocumentClient.from(base, { marshallOptions: { removeUndefinedValues: true } });
+    const ops: DynamoOps = {
+      executeStatement: (input, signal) => doc.send(new ExecuteStatementCommand(input), { abortSignal: signal }),
+      getItem: (input, signal) => doc.send(new GetCommand(input), { abortSignal: signal }),
+      query: (input, signal) => doc.send(new QueryCommand(input), { abortSignal: signal }),
+    };
+    return new DynamoEngine(ops, conf.keySchema ?? {}, { timeoutMs: conf.statement_timeout_ms ?? 4000, maxRows: conf.max_rows ?? 1000 });
+  }
+
+  /** Close open MySQL pools + Mongo clients + SSH tunnels (optional explicit teardown; tunnels are unref'd so the process exits regardless). */
   close(): void {
     for (const p of this.mysqlPools) { try { void p.end(); } catch { /* best-effort */ } }
     this.mysqlPools = [];
+    for (const c of this.mongoClients) { try { void c.close(); } catch { /* best-effort */ } }
+    this.mongoClients = [];
     for (const t of this.tunnels) t.close();
     this.tunnels = [];
   }
 
-  private build(table: string, filter: Record<string, any>): { sql: string; params: any[] } {
-    const cols = Object.keys(filter);
-    const where = cols.map((c, i) => `${ident(c)} = $${i + 1}`).join(' AND ');
-    const sql = `SELECT * FROM ${ident(table)}${where ? ' WHERE ' + where : ''} LIMIT 50`;
-    return { sql, params: cols.map((c) => filter[c]) };
-  }
+  // --- Engine-agnostic assertions: build nothing, just delegate to the engine + compare here ----
 
   /** A row matching `filter` must exist; if `expected` given, assert those columns on the first match. */
   async assertRow(table: string, filter: Record<string, any>, expected?: Record<string, any>, datasource?: string): Promise<void> {
-    const { engine, conf } = await this.engine(datasource);
-    const { sql, params } = this.build(table, filter);
-    const rows = await engine.query(this.sqlFor(conf, sql), params);
+    const { engine } = await this.engine(datasource);
+    const rows = await engine.findRows(table, filter, 50);
     expect(rows.length, `Expected a row in "${table}" where ${desc(filter)} — found ${rows.length}`).toBeGreaterThanOrEqual(1);
     if (expected) {
       const row = rows[0];
@@ -263,48 +657,38 @@ class DataSource {
 
   /** No row matching `filter` may exist. */
   async assertNoRow(table: string, filter: Record<string, any>, datasource?: string): Promise<void> {
-    const { engine, conf } = await this.engine(datasource);
-    const { sql, params } = this.build(table, filter);
-    const rows = await engine.query(this.sqlFor(conf, sql), params);
+    const { engine } = await this.engine(datasource);
+    const rows = await engine.findRows(table, filter, 50);
     expect(rows.length, `Expected NO row in "${table}" where ${desc(filter)} — found ${rows.length}`).toBe(0);
   }
 
   /** Exactly `count` rows must match `filter`. */
   async assertCount(table: string, filter: Record<string, any>, count: number, datasource?: string): Promise<void> {
-    const { engine, conf } = await this.engine(datasource);
+    const { engine } = await this.engine(datasource);
+    const n = await engine.countRows(table, filter);
     const cols = Object.keys(filter);
-    const where = cols.map((c, i) => `${ident(c)} = $${i + 1}`).join(' AND ');
-    const sql = `SELECT count(*) AS n FROM ${ident(table)}${where ? ' WHERE ' + where : ''}`;
-    const rows = await engine.query(this.sqlFor(conf, sql), cols.map((c) => filter[c]));
-    const n = Number(rows[0]?.n ?? rows[0]?.['count(*)'] ?? 0);
     expect(n, `Expected ${count} row(s) in "${table}"${cols.length ? ' where ' + desc(filter) : ''} — found ${n}`).toBe(Number(count));
   }
 
-  /** Rewrites $1/$2 placeholders to `?` for engines that don't support PG-style positional params. */
-  private sqlFor(conf: DataSourceConfig, sql: string): string {
-    return rewritePlaceholders(conf.engine, sql);
-  }
-
-  // --- Named queries (catalog-backed; SQL is resolved + embedded at compile time) -----------
-  /** Read-only guard (second layer): a named query must be a single SELECT/WITH statement. */
-  private assertSelectOnly(label: string, sql: string): void {
-    const s = sql.trim().replace(/;\s*$/, '');
-    if (!/^(SELECT|WITH)\b/i.test(s)) throw new Error(`Data Driver: ${label} is not a read-only SELECT — refused.`);
-    if (s.includes(';')) throw new Error(`Data Driver: ${label} contains multiple statements — refused.`);
-    if (/\b(INSERT|UPDATE|DELETE|DROP|ALTER|CREATE|TRUNCATE|GRANT|REVOKE|MERGE|REPLACE|CALL|EXEC|EXECUTE|ATTACH|PRAGMA|VACUUM)\b/i.test(s)) {
-      throw new Error(`Data Driver: ${label} contains a write/DDL keyword — refused.`);
-    }
-  }
-
   /**
    * Run a catalog query (read-only) and return its rows. The result is bound to a `{{name}}`
    * variable via `testData.bind(...)`, so the scenario asserts on it with `expect …` steps and
    * path access (`{{name.count}}`, `{{name.first.col}}`, `{{name[2].col}}`).
    */
   async fetchQuery(label: string, sql: string, params: any[], datasource?: string): Promise<any[]> {
-    this.assertSelectOnly(label, sql);
-    const { engine, conf } = await this.engine(datasource);
-    return engine.query(this.sqlFor(conf, sql), params);
+    const { engine } = await this.engine(datasource);
+    return engine.runQuery(label, sql, params);
+  }
+
+  /**
+   * Run a MongoDB find/pipeline catalog query (read-only) and return its documents. Same binding
+   * contract as fetchQuery, but the query is a structured shape (not SQL) with `:param` placeholders
+   * bound from `params` at runtime. Refused on an engine that has no non-SQL shape support.
+   */
+  async fetchQueryShape(label: string, shape: ShapeQuery, params: Record<string, any>, datasource?: string): Promise<any[]> {
+    const { engine } = await this.engine(datasource);
+    if (!engine.runShape) throw new Error(`Data Driver: ${label} is a find/pipeline query, but this datasource's engine has no such support (find:/pipeline: is MongoDB-only).`);
+    return engine.runShape(label, shape, params);
   }
 }