@jordanalec/dtk 1.1.0 → 1.2.0

package/README.md

@@ -369,6 +369,76 @@ await suite()
 
 ---
 
+### sql
+
+Runs raw SQL queries, executes statements, calls stored procedures, and runs transactions against PostgreSQL, MySQL, or MSSQL via knex.
+
+```bash
+dtk add sql
+```
+
+Env vars appended to `.env.template`:
+
+```
+SQL_CONNECTION_STRING=
+```
+
+Because the sql service holds an open connection pool, it must always be disconnected after use. Create the service outside the suite and call `disconnect()` in a `finally` block so it is guaranteed to run even when a step fails:
+
+```ts
+import "../load-env.js";
+import { suite } from "../suite.js";
+import { createSqlService } from "../services/sql.js";
+
+const sql = createSqlService({
+  client: 'pg', // or 'mysql2' or 'mssql'
+  connection: process.env.SQL_CONNECTION_STRING!,
+});
+
+try {
+  await suite()
+    .step("query", async () => {
+      const rows = await sql.query<{ id: number; name: string }>(
+        "SELECT id, name FROM users WHERE active = ?",
+        [true]
+      );
+      console.log(rows);
+      return rows;
+    })
+    .step("insert", async () => {
+      const affected = await sql.execute(
+        "INSERT INTO users (name, email, active) VALUES (?, ?, ?)",
+        ["Alice", "alice@example.com", true]
+      );
+      console.log("rows inserted:", affected);
+      return affected;
+    })
+    .step("transaction", async () => {
+      await sql.transaction(async (ops) => {
+        await ops.execute("UPDATE accounts SET balance = balance - ? WHERE id = ?", [100, 1]);
+        await ops.execute("UPDATE accounts SET balance = balance + ? WHERE id = ?", [100, 2]);
+      });
+    })
+    .run("stopOnError");
+} finally {
+  await sql.disconnect();
+}
+```
+
+Available methods on the sql service:
+
+| Method | Description |
+|---|---|
+| `query<T>(sql, params?)` | Runs a SELECT and returns the result rows as `T[]` |
+| `execute(sql, params?)` | Runs an INSERT / UPDATE / DELETE and returns the affected row count |
+| `callProc<T>(name, params?)` | Calls a stored procedure and returns result rows as `T[]` |
+| `transaction(fn)` | Runs `fn` inside a transaction; rolls back automatically on error |
+| `disconnect()` | Destroys the connection pool -- always call this in a `finally` block |
+
+Supported clients: `pg` (PostgreSQL), `mysql2` (MySQL / MariaDB), `mssql` (SQL Server).
+
+---
+
 ## Writing runbooks
 
 A runbook is a TypeScript file that uses the `suite()` builder to chain steps and run them in sequence.
@@ -780,4 +850,5 @@ templates/
     aws-s3/
     open-ai/
     redis/
+    sql/
 ```

package/dist/add.js

@@ -13,6 +13,7 @@ const PLUGIN_MAP = {
     'aws-s3': 'aws-s3',
     'open-ai': 'open-ai',
     'redis': 'redis',
+    'sql': 'sql',
 };
 export const addCommand = new Command('add')
     .description('Add a service plugin to the project')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jordanalec/dtk",
-  "version": "1.1.0",
+  "version": "1.2.0",
   "description": "CLI scaffolding tool for generating self-contained TypeScript runbook projects",
   "keywords": [
     "cli",
@@ -45,6 +45,7 @@
     "axios": "^1.6.0",
     "dotenv": "^16.4.0",
     "jest": "^30.3.0",
+    "knex": "^3.1.0",
     "redis": "^4.7.1",
     "ts-jest": "^29.4.9",
     "tsx": "^4.0.0",

package/templates/init/GUIDE.md

@@ -461,6 +461,76 @@ await suite()
 
 ---
 
+### sql
+
+Runs raw SQL queries, executes statements, calls stored procedures, and runs transactions against PostgreSQL, MySQL, or MSSQL via knex.
+
+```bash
+dtk add sql
+```
+
+Required env vars:
+
+```
+SQL_CONNECTION_STRING=
+```
+
+Because the sql service holds an open connection pool, it must always be disconnected after use. Create the service outside the suite and call `disconnect()` in a `finally` block so it is guaranteed to run even when a step fails:
+
+```ts
+import "../load-env.js";
+import { suite } from "../suite.js";
+import { createSqlService } from "../services/sql.js";
+
+const sql = createSqlService({
+  client: 'pg', // or 'mysql2' or 'mssql'
+  connection: process.env.SQL_CONNECTION_STRING!,
+});
+
+try {
+  await suite()
+    .step("query", async () => {
+      const rows = await sql.query<{ id: number; name: string }>(
+        "SELECT id, name FROM users WHERE active = ?",
+        [true]
+      );
+      console.log(rows);
+      return rows;
+    })
+    .step("insert", async () => {
+      const affected = await sql.execute(
+        "INSERT INTO users (name, email, active) VALUES (?, ?, ?)",
+        ["Alice", "alice@example.com", true]
+      );
+      console.log("rows inserted:", affected);
+      return affected;
+    })
+    .step("transaction", async () => {
+      await sql.transaction(async (ops) => {
+        await ops.execute("UPDATE accounts SET balance = balance - ? WHERE id = ?", [100, 1]);
+        await ops.execute("UPDATE accounts SET balance = balance + ? WHERE id = ?", [100, 2]);
+      });
+    })
+    .run("stopOnError");
+} finally {
+  await sql.disconnect();
+}
+```
+
+Available methods on the sql service:
+
+| Method | Description |
+|---|---|
+| `query<T>(sql, params?)` | Runs a SELECT and returns the result rows as `T[]` |
+| `execute(sql, params?)` | Runs an INSERT / UPDATE / DELETE and returns the affected row count |
+| `callProc<T>(name, params?)` | Calls a stored procedure and returns result rows as `T[]` |
+| `transaction(fn)` | Runs `fn` inside a transaction; rolls back automatically on error |
+| `disconnect()` | Destroys the connection pool -- always call this in a `finally` block |
+
+Supported clients: `pg` (PostgreSQL), `mysql2` (MySQL / MariaDB), `mssql` (SQL Server).
+
+---
+
 ## Writing a custom service
 
 If there is no plugin for the service you need, wire one in manually. Four files are involved.

package/templates/init/README.md

@@ -37,7 +37,7 @@ Install the dtk CLI if you haven't already, then run from this directory:
 dtk add <plugin>
 ```
 
-Available plugins: `aws-sqs`, `aws-sns`, `aws-dynamo`, `aws-s3`, `open-ai`
+Available plugins: `aws-sqs`, `aws-sns`, `aws-dynamo`, `aws-s3`, `open-ai`, `redis`, `sql`
 
 Each plugin adds a service, wires it into the suite, appends env vars to `.env.template`, creates an example runbook, and installs dependencies automatically.
 

package/templates/plugins/sql/env.txt

@@ -0,0 +1 @@
+SQL_CONNECTION_STRING=

package/templates/plugins/sql/example.ts

@@ -0,0 +1,49 @@
+import "../load-env.js";
+import { suite } from "../suite.js";
+import { createSqlService } from "../services/sql.js";
+
+type User = { id: number; name: string; email: string; };
+type TransferResult = { status: string; };
+
+const sql = createSqlService({
+  client: 'pg', // change to 'mysql2' or 'mssql' as needed
+  connection: process.env.SQL_CONNECTION_STRING!,
+});
+
+try {
+  await suite()
+    .step("query-users", async () => {
+      const users = await sql.query<User>(
+        "SELECT id, name, email FROM users WHERE active = ?",
+        [true]
+      );
+      console.log("active users:", users);
+      return users;
+    })
+    .step("insert-user", async () => {
+      const affected = await sql.execute(
+        "INSERT INTO users (name, email, active) VALUES (?, ?, ?)",
+        ["Alice", "alice@example.com", true]
+      );
+      console.log("rows inserted:", affected);
+      return affected;
+    })
+    .step("call-stored-proc", async () => {
+      const result = await sql.callProc<TransferResult>(
+        "usp_activate_user",
+        [42]
+      );
+      console.log("proc result:", result);
+      return result;
+    })
+    .step("transfer-in-transaction", async () => {
+      await sql.transaction(async (ops) => {
+        await ops.execute("UPDATE accounts SET balance = balance - ? WHERE id = ?", [100, 1]);
+        await ops.execute("UPDATE accounts SET balance = balance + ? WHERE id = ?", [100, 2]);
+      });
+      console.log("transfer complete");
+    })
+    .run("stopOnError");
+} finally {
+  await sql.disconnect();
+}

package/templates/plugins/sql/plugin.json

@@ -0,0 +1,40 @@
+{
+  "name": "sql",
+  "description": "SQL -- query, execute, callProc, transaction across pg / mysql2 / mssql",
+  "dependencies": {
+    "knex": "^3.1.0",
+    "pg": "^8.13.0",
+    "mysql2": "^3.11.0",
+    "mssql": "^11.0.1"
+  },
+  "files": [
+    { "src": "service.ts", "dest": "src/services/sql.ts" },
+    { "src": "types.ts", "dest": "src/types/sql.ts" },
+    { "src": "service.test.ts", "dest": "src/services/sql.test.ts" }
+  ],
+  "env": "env.txt",
+  "example": "example.ts",
+  "transforms": {
+    "service.ts": [
+      { "from": "./types.js", "to": "../types/sql.js" }
+    ],
+    "service.test.ts": [
+      { "from": "./service.js", "to": "./sql.js" }
+    ]
+  },
+  "patches": {
+    "src/suite.ts": {
+      "imports": [
+        "import { createSqlService } from \"./services/sql.js\";",
+        "import type { SqlConfig } from \"./types/sql.js\";"
+      ],
+      "configs": "  private sqlConfig?: SqlConfig;",
+      "methods": "  sql(config: SqlConfig): this { this.sqlConfig = config; return this; }",
+      "services": "        sql: createSqlService(this.sqlConfig),"
+    },
+    "src/types/suite.ts": {
+      "type-imports": "import type { SqlOps } from \"./sql.js\";",
+      "service-types": "    sql: { query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>; execute(sql: string, params?: unknown[]): Promise<number>; callProc<T = Record<string, unknown>>(name: string, params?: unknown[]): Promise<T[]>; transaction<T>(fn: (ops: SqlOps) => Promise<T>): Promise<T>; disconnect(): Promise<void>; };"
+    }
+  }
+}

package/templates/plugins/sql/service.test.ts

@@ -0,0 +1,224 @@
+import { createSqlService } from './service.js';
+
+jest.mock('knex');
+import knexLib from 'knex';
+
+const mockTrxRaw = jest.fn();
+const mockTrx = { raw: mockTrxRaw };
+
+const mockRaw = jest.fn();
+const mockDestroy = jest.fn().mockResolvedValue(undefined);
+const mockTransaction = jest.fn();
+
+const mockDb = {
+  raw: mockRaw,
+  destroy: mockDestroy,
+  transaction: mockTransaction,
+};
+
+beforeEach(() => {
+  jest.clearAllMocks();
+  (knexLib as unknown as jest.Mock).mockReturnValue(mockDb);
+  mockDestroy.mockResolvedValue(undefined);
+});
+
+describe('createSqlService', () => {
+  describe('configuration', () => {
+    it('throws when any method is called without config', async () => {
+      const sql = createSqlService();
+      await expect(sql.query('SELECT 1')).rejects.toThrow('sql service is not configured');
+    });
+
+    it('creates a knex instance with the provided client and connection', async () => {
+      mockRaw.mockResolvedValue({ rows: [] });
+      const sql = createSqlService({ client: 'pg', connection: 'postgresql://localhost/test' });
+      await sql.query('SELECT 1');
+      expect(knexLib).toHaveBeenCalledWith({ client: 'pg', connection: 'postgresql://localhost/test' });
+    });
+
+    it('reuses the same knex instance across multiple calls', async () => {
+      mockRaw.mockResolvedValue({ rows: [] });
+      const sql = createSqlService({ client: 'pg', connection: 'postgresql://localhost/test' });
+      await sql.query('SELECT 1');
+      await sql.query('SELECT 2');
+      expect(knexLib).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe('query', () => {
+    it('returns result.rows for pg', async () => {
+      mockRaw.mockResolvedValue({ rows: [{ id: 1 }] });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      const result = await sql.query('SELECT * FROM users');
+      expect(result).toEqual([{ id: 1 }]);
+    });
+
+    it('returns result[0] for mysql2', async () => {
+      mockRaw.mockResolvedValue([[{ id: 1 }], []]);
+      const sql = createSqlService({ client: 'mysql2', connection: '' });
+      const result = await sql.query('SELECT * FROM users');
+      expect(result).toEqual([{ id: 1 }]);
+    });
+
+    it('returns result[0] for mssql', async () => {
+      mockRaw.mockResolvedValue([[{ id: 1 }]]);
+      const sql = createSqlService({ client: 'mssql', connection: '' });
+      const result = await sql.query('SELECT * FROM users');
+      expect(result).toEqual([{ id: 1 }]);
+    });
+
+    it('passes sql and params to raw', async () => {
+      mockRaw.mockResolvedValue({ rows: [] });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      await sql.query('SELECT * FROM users WHERE id = ?', [42]);
+      expect(mockRaw).toHaveBeenCalledWith('SELECT * FROM users WHERE id = ?', [42]);
+    });
+
+    it('defaults params to empty array when omitted', async () => {
+      mockRaw.mockResolvedValue({ rows: [] });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      await sql.query('SELECT 1');
+      expect(mockRaw).toHaveBeenCalledWith('SELECT 1', []);
+    });
+  });
+
+  describe('execute', () => {
+    it('returns rowCount for pg', async () => {
+      mockRaw.mockResolvedValue({ rowCount: 3 });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      const result = await sql.execute('DELETE FROM users WHERE active = ?', [false]);
+      expect(result).toBe(3);
+    });
+
+    it('returns affectedRows for mysql2', async () => {
+      mockRaw.mockResolvedValue([{ affectedRows: 2 }]);
+      const sql = createSqlService({ client: 'mysql2', connection: '' });
+      const result = await sql.execute('UPDATE users SET active = ?', [true]);
+      expect(result).toBe(2);
+    });
+
+    it('returns rowsAffected[0] for mssql', async () => {
+      mockRaw.mockResolvedValue({ rowsAffected: [5] });
+      const sql = createSqlService({ client: 'mssql', connection: '' });
+      const result = await sql.execute('INSERT INTO logs (msg) VALUES (?)', ['test']);
+      expect(result).toBe(5);
+    });
+
+    it('returns 0 when pg rowCount is null', async () => {
+      mockRaw.mockResolvedValue({ rowCount: null });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      const result = await sql.execute('SELECT 1');
+      expect(result).toBe(0);
+    });
+  });
+
+  describe('callProc', () => {
+    it('uses CALL syntax for pg', async () => {
+      mockRaw.mockResolvedValue({ rows: [] });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      await sql.callProc('my_proc', [1, 'test']);
+      expect(mockRaw).toHaveBeenCalledWith('CALL my_proc(?, ?)', [1, 'test']);
+    });
+
+    it('uses CALL syntax for mysql2', async () => {
+      mockRaw.mockResolvedValue([[[]], {}]);
+      const sql = createSqlService({ client: 'mysql2', connection: '' });
+      await sql.callProc('my_proc', [1]);
+      expect(mockRaw).toHaveBeenCalledWith('CALL my_proc(?)', [1]);
+    });
+
+    it('uses EXEC syntax for mssql', async () => {
+      mockRaw.mockResolvedValue([[{ result: 1 }]]);
+      const sql = createSqlService({ client: 'mssql', connection: '' });
+      await sql.callProc('usp_my_proc', [1, 2]);
+      expect(mockRaw).toHaveBeenCalledWith('EXEC usp_my_proc ?, ?', [1, 2]);
+    });
+
+    it('uses EXEC without params for mssql when no params given', async () => {
+      mockRaw.mockResolvedValue([[{ result: 1 }]]);
+      const sql = createSqlService({ client: 'mssql', connection: '' });
+      await sql.callProc('usp_my_proc');
+      expect(mockRaw).toHaveBeenCalledWith('EXEC usp_my_proc', []);
+    });
+
+    it('returns result.rows for pg', async () => {
+      mockRaw.mockResolvedValue({ rows: [{ status: 'ok' }] });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      const result = await sql.callProc('my_proc', []);
+      expect(result).toEqual([{ status: 'ok' }]);
+    });
+
+    it('returns result[0] for mssql', async () => {
+      mockRaw.mockResolvedValue([[{ status: 'ok' }]]);
+      const sql = createSqlService({ client: 'mssql', connection: '' });
+      const result = await sql.callProc('usp_my_proc', []);
+      expect(result).toEqual([{ status: 'ok' }]);
+    });
+
+    it('returns the first result set for mysql2', async () => {
+      mockRaw.mockResolvedValue([[[{ status: 'ok' }], {}], []]);
+      const sql = createSqlService({ client: 'mysql2', connection: '' });
+      const result = await sql.callProc('my_proc', []);
+      expect(result).toEqual([{ status: 'ok' }]);
+    });
+  });
+
+  describe('transaction', () => {
+    it('calls knex transaction and passes SqlOps to fn', async () => {
+      mockTrxRaw.mockResolvedValue({ rows: [{ id: 1 }] });
+      mockTransaction.mockImplementation((fn: (trx: typeof mockTrx) => Promise<unknown>) => fn(mockTrx));
+
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      const result = await sql.transaction(async (ops) => ops.query('SELECT 1'));
+
+      expect(mockTransaction).toHaveBeenCalledTimes(1);
+      expect(result).toEqual([{ id: 1 }]);
+    });
+
+    it('ops.execute within transaction uses the transaction executor', async () => {
+      mockTrxRaw.mockResolvedValue({ rowCount: 2 });
+      mockTransaction.mockImplementation((fn: (trx: typeof mockTrx) => Promise<unknown>) => fn(mockTrx));
+
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      const affected = await sql.transaction(async (ops) => ops.execute('DELETE FROM tmp'));
+
+      expect(mockTrxRaw).toHaveBeenCalledWith('DELETE FROM tmp', []);
+      expect(affected).toBe(2);
+    });
+
+    it('ops.callProc within transaction uses the transaction executor', async () => {
+      mockTrxRaw.mockResolvedValue({ rows: [] });
+      mockTransaction.mockImplementation((fn: (trx: typeof mockTrx) => Promise<unknown>) => fn(mockTrx));
+
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      await sql.transaction(async (ops) => ops.callProc('my_proc', [1]));
+
+      expect(mockTrxRaw).toHaveBeenCalledWith('CALL my_proc(?)', [1]);
+    });
+  });
+
+  describe('disconnect', () => {
+    it('destroys the knex instance and clears the reference', async () => {
+      mockRaw.mockResolvedValue({ rows: [] });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      await sql.query('SELECT 1');
+      await sql.disconnect();
+      expect(mockDestroy).toHaveBeenCalledTimes(1);
+    });
+
+    it('is a no-op when never connected', async () => {
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      await expect(sql.disconnect()).resolves.toBeUndefined();
+      expect(mockDestroy).not.toHaveBeenCalled();
+    });
+
+    it('creates a new knex instance after reconnecting following disconnect', async () => {
+      mockRaw.mockResolvedValue({ rows: [] });
+      const sql = createSqlService({ client: 'pg', connection: '' });
+      await sql.query('SELECT 1');
+      await sql.disconnect();
+      await sql.query('SELECT 1');
+      expect(knexLib).toHaveBeenCalledTimes(2);
+    });
+  });
+});

package/templates/plugins/sql/service.ts

@@ -0,0 +1,87 @@
+import knex, { Knex } from 'knex';
+import type { SqlConfig, SqlOps } from './types.js';
+
+export function createSqlService(config?: SqlConfig) {
+  const ensureConfig = () => {
+    if (!config) throw new Error("sql service is not configured -- call .sql(config) on the suite");
+  };
+
+  let db: Knex | null = null;
+
+  const getDb = (): Knex => {
+    ensureConfig();
+    if (!db) {
+      db = knex({ client: config!.client, connection: config!.connection });
+    }
+    return db;
+  };
+
+  const extractRows = <T>(result: unknown): T[] => {
+    if (config!.client === 'pg') return (result as { rows: T[] }).rows;
+    return ((result as unknown[])[0] as T[]) ?? [];
+  };
+
+  const extractAffected = (result: unknown): number => {
+    if (config!.client === 'pg') return (result as { rowCount: number }).rowCount ?? 0;
+    if (config!.client === 'mssql') return (result as { rowsAffected: number[] }).rowsAffected?.[0] ?? 0;
+    return ((result as [{ affectedRows: number }])[0]?.affectedRows) ?? 0;
+  };
+
+  const buildCallSql = (name: string, params: unknown[]): string => {
+    const placeholders = params.map(() => '?').join(', ');
+    return config!.client === 'mssql'
+      ? `EXEC ${name}${params.length ? ` ${placeholders}` : ''}`
+      : `CALL ${name}(${placeholders})`;
+  };
+
+  const extractProcRows = <T>(result: unknown): T[] => {
+    if (config!.client === 'pg') return (result as { rows: T[] }).rows;
+    if (config!.client === 'mssql') return ((result as unknown[])[0] as T[]) ?? [];
+    // mysql2 procs return [[resultSet, OkPacket], fields] -- first element of resultSet is the rows
+    const first = ((result as unknown[][])[0])?.[0];
+    return (Array.isArray(first) ? first : (result as unknown[])[0]) as T[];
+  };
+
+  const bind = (params?: unknown[]) => (params ?? []) as any[];
+
+  const makeOps = (executor: Knex | Knex.Transaction): SqlOps => ({
+    async query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]> {
+      const result = await executor.raw(sql, bind(params));
+      return extractRows<T>(result);
+    },
+    async execute(sql: string, params?: unknown[]): Promise<number> {
+      const result = await executor.raw(sql, bind(params));
+      return extractAffected(result);
+    },
+    async callProc<T = Record<string, unknown>>(name: string, params?: unknown[]): Promise<T[]> {
+      const bound = bind(params);
+      const result = await executor.raw(buildCallSql(name, bound), bound);
+      return extractProcRows<T>(result);
+    },
+  });
+
+  return {
+    async query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]> {
+      const result = await getDb().raw(sql, bind(params));
+      return extractRows<T>(result);
+    },
+    async execute(sql: string, params?: unknown[]): Promise<number> {
+      const result = await getDb().raw(sql, bind(params));
+      return extractAffected(result);
+    },
+    async callProc<T = Record<string, unknown>>(name: string, params?: unknown[]): Promise<T[]> {
+      const bound = bind(params);
+      const result = await getDb().raw(buildCallSql(name, bound), bound);
+      return extractProcRows<T>(result);
+    },
+    async transaction<T>(fn: (ops: SqlOps) => Promise<T>): Promise<T> {
+      return getDb().transaction(trx => fn(makeOps(trx)));
+    },
+    async disconnect(): Promise<void> {
+      if (db) {
+        await db.destroy();
+        db = null;
+      }
+    },
+  };
+}

package/templates/plugins/sql/types.ts

@@ -0,0 +1,10 @@
+export interface SqlConfig {
+  client: 'pg' | 'mysql2' | 'mssql';
+  connection: string | Record<string, unknown>;
+}
+
+export interface SqlOps {
+  query<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>;
+  execute(sql: string, params?: unknown[]): Promise<number>;
+  callProc<T = Record<string, unknown>>(name: string, params?: unknown[]): Promise<T[]>;
+}