dbt-js 0.1.1 → 0.1.2

package/src/render.js

@@ -1,62 +1,65 @@
-// Minimal template renderer. Supported constructs:
-//   {{ ref('model') }}  {{ this }}  {{ source('src', 'table') }}
-//   {{ var('name') }}  {{ var('name', default) }}
-//   {{ batch_start }}  {{ batch_end }}          (microbatch models only)
-//   {{ timezone }}                              (the model's config timezone)
-//   {% if is_incremental() %} ... {% endif %}   (no nesting)
-const CONFIG_RE = /\/\*\s*config:\s*[\s\S]*?\*\//;
-const IF_INCREMENTAL_RE = /\{%\s*if\s+is_incremental\(\)\s*%\}([\s\S]*?)\{%\s*endif\s*%\}/g;
-const REF_RE = /\{\{\s*ref\(\s*['"](\w+)['"]\s*\)\s*\}\}/g;
-const THIS_RE = /\{\{\s*this\s*\}\}/g;
-const SOURCE_RE = /\{\{\s*source\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)\s*\}\}/g;
-const VAR_RE = /\{\{\s*var\(\s*['"](\w+)['"]\s*(?:,\s*('[^']*'|"[^"]*"|[^)\s]+))?\s*\)\s*\}\}/g;
-const BATCH_RE = /\{\{\s*(batch_start|batch_end)\s*\}\}/g;
-const TIMEZONE_RE = /\{\{\s*timezone\s*\}\}/g;
-const LEFTOVER_RE = /\{\{[\s\S]*?\}\}|\{%[\s\S]*?%\}|\{\{|\{%/;
-
-const quoteIdent = (s) => `"${s.replace(/"/g, '""')}"`;
-const stripQuotes = (s) => (/^(['"]).*\1$/s.test(s) ? s.slice(1, -1) : s);
-
-// Cheap dependency extraction for DAG building — scans ref() calls without
-// rendering, so missing vars or incremental branches can't hide a dependency.
-export function extractRefs(rawSql) {
-  return [...rawSql.matchAll(REF_RE)].map((m) => m[1]);
-}
-
-// ctx: { name, schema, vars, isIncremental, sources, batchStart?, batchEnd?, timezone? }
-export function render(rawSql, ctx) {
-  const refs = [];
-  let sql = rawSql.replace(CONFIG_RE, '');
-  sql = sql.replace(IF_INCREMENTAL_RE, (_, body) => (ctx.isIncremental ? body : ''));
-  sql = sql.replace(REF_RE, (_, name) => {
-    refs.push(name);
-    return `${quoteIdent(ctx.schema)}.${quoteIdent(name)}`;
-  });
-  sql = sql.replace(THIS_RE, () => `${quoteIdent(ctx.schema)}.${quoteIdent(ctx.name)}`);
-  sql = sql.replace(SOURCE_RE, (_, src, table) => {
-    const decl = ctx.sources?.[src];
-    if (!decl?.schema) {
-      throw new Error(
-        `'${ctx.name}' uses undeclared source '${src}' — add it under "sources" in dbtjs.config.json`
-      );
-    }
-    return `${quoteIdent(decl.schema)}.${quoteIdent(table)}`;
-  });
-  if (ctx.batchStart != null) {
-    // only microbatch runs supply these; elsewhere the token falls through to the leftover guard
-    sql = sql.replace(BATCH_RE, (_, which) => (which === 'batch_start' ? ctx.batchStart : ctx.batchEnd));
-  }
-  // raw substitution (like batch_start) — author quotes it in SQL if needed
-  sql = sql.replace(TIMEZONE_RE, ctx.timezone ?? 'UTC');
-  sql = sql.replace(VAR_RE, (_, name, def) => {
-    const value = ctx.vars?.[name];
-    if (value !== undefined && value !== null) return String(value);
-    if (def !== undefined) return stripQuotes(def);
-    throw new Error(`Missing var '${name}' in '${ctx.name}' (no default given) — pass --vars '{"${name}": ...}'`);
-  });
-  const leftover = sql.match(LEFTOVER_RE);
-  if (leftover) {
-    throw new Error(`Unrecognized template expression in '${ctx.name}': ${leftover[0].slice(0, 80)}`);
-  }
-  return { sql: sql.trim(), refs };
-}
+// Minimal template renderer. Supported constructs:
+//   {{ ref('model') }}  {{ this }}  {{ source('src', 'table') }}
+//   {{ var('name') }}  {{ var('name', default) }}
+//   {{ batch_start }}  {{ batch_end }}          (microbatch models only)
+//   {{ timezone }}                              (the model's config timezone)
+//   {% if is_incremental() %} ... {% endif %}   (no nesting)
+const CONFIG_RE = /\/\*\s*config:\s*[\s\S]*?\*\//;
+const IF_INCREMENTAL_RE = /\{%\s*if\s+is_incremental\(\)\s*%\}([\s\S]*?)\{%\s*endif\s*%\}/g;
+const REF_RE = /\{\{\s*ref\(\s*['"](\w+)['"]\s*\)\s*\}\}/g;
+const THIS_RE = /\{\{\s*this\s*\}\}/g;
+const SOURCE_RE = /\{\{\s*source\(\s*['"](\w+)['"]\s*,\s*['"](\w+)['"]\s*\)\s*\}\}/g;
+const VAR_RE = /\{\{\s*var\(\s*['"](\w+)['"]\s*(?:,\s*('[^']*'|"[^"]*"|[^)\s]+))?\s*\)\s*\}\}/g;
+const BATCH_RE = /\{\{\s*(batch_start|batch_end)\s*\}\}/g;
+const TIMEZONE_RE = /\{\{\s*timezone\s*\}\}/g;
+const LEFTOVER_RE = /\{\{[\s\S]*?\}\}|\{%[\s\S]*?%\}|\{\{|\{%/;
+
+const quoteIdent = (s) => `"${s.replace(/"/g, '""')}"`;
+const stripQuotes = (s) => (/^(['"]).*\1$/s.test(s) ? s.slice(1, -1) : s);
+
+// Cheap dependency extraction for DAG building — scans ref() calls without
+// rendering, so missing vars or incremental branches can't hide a dependency.
+export function extractRefs(rawSql) {
+  return [...rawSql.matchAll(REF_RE)].map((m) => m[1]);
+}
+
+// ctx: { name, schema, vars, isIncremental, sources, batchStart?, batchEnd?, timezone? }
+export function render(rawSql, ctx) {
+  const refs = [];
+  let sql = rawSql.replace(CONFIG_RE, '');
+  sql = sql.replace(IF_INCREMENTAL_RE, (_, body) => (ctx.isIncremental ? body : ''));
+  sql = sql.replace(REF_RE, (_, name) => {
+    refs.push(name);
+    return `${quoteIdent(ctx.schema)}.${quoteIdent(name)}`;
+  });
+  sql = sql.replace(THIS_RE, () => `${quoteIdent(ctx.schema)}.${quoteIdent(ctx.name)}`);
+  sql = sql.replace(SOURCE_RE, (_, src, table) => {
+    const decl = ctx.sources?.[src];
+    if (!decl?.schema) {
+      throw new Error(
+        `'${ctx.name}' uses undeclared source '${src}' — add it under "sources" in dbtjs.config.json`
+      );
+    }
+    // an attached database (DuckDB ATTACH) adds a catalog qualifier:
+    // "database"."schema"."table"; without it the name stays two-part
+    const prefix = decl.database ? `${quoteIdent(decl.database)}.` : '';
+    return `${prefix}${quoteIdent(decl.schema)}.${quoteIdent(table)}`;
+  });
+  if (ctx.batchStart != null) {
+    // only microbatch runs supply these; elsewhere the token falls through to the leftover guard
+    sql = sql.replace(BATCH_RE, (_, which) => (which === 'batch_start' ? ctx.batchStart : ctx.batchEnd));
+  }
+  // raw substitution (like batch_start) — author quotes it in SQL if needed
+  sql = sql.replace(TIMEZONE_RE, ctx.timezone ?? 'UTC');
+  sql = sql.replace(VAR_RE, (_, name, def) => {
+    const value = ctx.vars?.[name];
+    if (value !== undefined && value !== null) return String(value);
+    if (def !== undefined) return stripQuotes(def);
+    throw new Error(`Missing var '${name}' in '${ctx.name}' (no default given) — pass --vars '{"${name}": ...}'`);
+  });
+  const leftover = sql.match(LEFTOVER_RE);
+  if (leftover) {
+    throw new Error(`Unrecognized template expression in '${ctx.name}': ${leftover[0].slice(0, 80)}`);
+  }
+  return { sql: sql.trim(), refs };
+}

package/src/seed.js

@@ -1,68 +1,68 @@
-import { readFileSync } from 'node:fs';
-import { parse } from 'csv-parse/sync';
-import { quoteIdent, rel, withTransaction } from './db.js';
-
-const BATCH_SIZE = 500;
-
-export async function loadSeed(client, seed, projectCfg) {
-  const rows = parse(readFileSync(seed.path, 'utf8'), {
-    columns: true,
-    skip_empty_lines: true,
-    trim: true,
-  });
-  if (!rows.length) throw new Error(`Seed '${seed.name}' has no data rows`);
-
-  const columns = Object.keys(rows[0]);
-  const overrides = projectCfg.seeds?.columnTypes?.[seed.name] ?? {};
-  const mysql = client.dialect === 'mysql';
-  const sqlite = client.dialect === 'sqlite';
-  const types = columns.map((c) => {
-    const t = overrides[c] ?? inferType(rows.map((r) => r[c]));
-    // bare NUMERIC is DECIMAL(10,0) on MySQL — would silently round decimals
-    return mysql && t === 'numeric' ? 'decimal(38,10)' : t;
-  });
-  const target = rel(projectCfg.schema, seed.name);
-  // stay under SQLite's 32766-bind-variable cap (and Postgres's 65535) for wide CSVs
-  const batchSize = Math.max(1, Math.min(BATCH_SIZE, Math.floor(32000 / columns.length)));
-
-  await withTransaction(client, async () => {
-    await client.query(`DROP TABLE IF EXISTS ${target}${sqlite ? '' : ' CASCADE'}`);
-    const defs = columns.map((c, i) => `${quoteIdent(c)} ${types[i]}`).join(', ');
-    await client.query(`CREATE TABLE ${target} (${defs})`);
-    for (let i = 0; i < rows.length; i += batchSize) {
-      const batch = rows.slice(i, i + batchSize);
-      const params = [];
-      const tuples = batch.map(
-        (row) =>
-          `(${columns
-            .map((c, j) => {
-              let v = row[c] === '' ? null : row[c];
-              // MySQL booleans are TINYINT(1); the string 'true' errors under
-              // strict mode. SQLite would store the TEXT 'true', which is falsy
-              // in CASE WHEN (and better-sqlite3 can't bind true/false anyway).
-              if ((mysql || sqlite) && v !== null && types[j] === 'boolean')
-                v = /^(true|t)$/i.test(v) ? 1 : 0;
-              params.push(v);
-              return `$${params.length}`;
-            })
-            .join(', ')})`
-      );
-      await client.query(`INSERT INTO ${target} VALUES ${tuples.join(', ')}`, params);
-    }
-  });
-  return { rowCount: rows.length };
-}
-
-// Minimal inference: integer/bigint, numeric, boolean, else text.
-// Empty strings load as NULL and are excluded from inference.
-// Anything fancier (dates, etc.) → seeds.columnTypes override in dbtjs.config.json.
-export function inferType(values) {
-  const present = values.filter((v) => v !== '');
-  if (!present.length) return 'text';
-  if (present.every((v) => /^-?\d+$/.test(v))) {
-    return present.some((v) => Math.abs(Number(v)) > 2147483647) ? 'bigint' : 'integer';
-  }
-  if (present.every((v) => /^-?\d*\.?\d+$/.test(v))) return 'numeric';
-  if (present.every((v) => /^(true|false|t|f)$/i.test(v))) return 'boolean';
-  return 'text';
-}
+import { readFileSync } from 'node:fs';
+import { parse } from 'csv-parse/sync';
+import { quoteIdent, rel, withTransaction } from './db.js';
+
+const BATCH_SIZE = 500;
+
+export async function loadSeed(client, seed, projectCfg) {
+  const rows = parse(readFileSync(seed.path, 'utf8'), {
+    columns: true,
+    skip_empty_lines: true,
+    trim: true,
+  });
+  if (!rows.length) throw new Error(`Seed '${seed.name}' has no data rows`);
+
+  const columns = Object.keys(rows[0]);
+  const overrides = projectCfg.seeds?.columnTypes?.[seed.name] ?? {};
+  const mysql = client.dialect === 'mysql';
+  const sqlite = client.dialect === 'sqlite';
+  const types = columns.map((c) => {
+    const t = overrides[c] ?? inferType(rows.map((r) => r[c]));
+    // bare NUMERIC is DECIMAL(10,0) on MySQL — would silently round decimals
+    return mysql && t === 'numeric' ? 'decimal(38,10)' : t;
+  });
+  const target = rel(projectCfg.schema, seed.name);
+  // stay under SQLite's 32766-bind-variable cap (and Postgres's 65535) for wide CSVs
+  const batchSize = Math.max(1, Math.min(BATCH_SIZE, Math.floor(32000 / columns.length)));
+
+  await withTransaction(client, async () => {
+    await client.query(`DROP TABLE IF EXISTS ${target}${sqlite ? '' : ' CASCADE'}`);
+    const defs = columns.map((c, i) => `${quoteIdent(c)} ${types[i]}`).join(', ');
+    await client.query(`CREATE TABLE ${target} (${defs})`);
+    for (let i = 0; i < rows.length; i += batchSize) {
+      const batch = rows.slice(i, i + batchSize);
+      const params = [];
+      const tuples = batch.map(
+        (row) =>
+          `(${columns
+            .map((c, j) => {
+              let v = row[c] === '' ? null : row[c];
+              // MySQL booleans are TINYINT(1); the string 'true' errors under
+              // strict mode. SQLite would store the TEXT 'true', which is falsy
+              // in CASE WHEN (and better-sqlite3 can't bind true/false anyway).
+              if ((mysql || sqlite) && v !== null && types[j] === 'boolean')
+                v = /^(true|t)$/i.test(v) ? 1 : 0;
+              params.push(v);
+              return `$${params.length}`;
+            })
+            .join(', ')})`
+      );
+      await client.query(`INSERT INTO ${target} VALUES ${tuples.join(', ')}`, params);
+    }
+  });
+  return { rowCount: rows.length };
+}
+
+// Minimal inference: integer/bigint, numeric, boolean, else text.
+// Empty strings load as NULL and are excluded from inference.
+// Anything fancier (dates, etc.) → seeds.columnTypes override in dbtjs.config.json.
+export function inferType(values) {
+  const present = values.filter((v) => v !== '');
+  if (!present.length) return 'text';
+  if (present.every((v) => /^-?\d+$/.test(v))) {
+    return present.some((v) => Math.abs(Number(v)) > 2147483647) ? 'bigint' : 'integer';
+  }
+  if (present.every((v) => /^-?\d*\.?\d+$/.test(v))) return 'numeric';
+  if (present.every((v) => /^(true|false|t|f)$/i.test(v))) return 'boolean';
+  return 'text';
+}

package/src/tests.js

@@ -1,49 +1,49 @@
-import { quoteIdent, rel } from './db.js';
-
-// Each test compiles to a SELECT returning violating rows; any row = FAIL.
-// NULLs only violate not_null (dbt semantics).
-export function buildTests(models, schema) {
-  const tests = [];
-  for (const model of models) {
-    for (const [column, specs] of Object.entries(model.config.tests ?? {})) {
-      const target = rel(schema, model.name);
-      const col = quoteIdent(column);
-      for (const spec of specs) {
-        if (spec === 'not_null') {
-          tests.push({
-            id: `${model.name}.${column}.not_null`,
-            model: model.name,
-            sql: `SELECT * FROM ${target} WHERE ${col} IS NULL`,
-            params: [],
-          });
-        } else if (spec === 'unique') {
-          tests.push({
-            id: `${model.name}.${column}.unique`,
-            model: model.name,
-            sql: `SELECT ${col}, count(*) AS n FROM ${target} WHERE ${col} IS NOT NULL GROUP BY ${col} HAVING count(*) > 1`,
-            params: [],
-          });
-        } else if (spec?.accepted_values?.length) {
-          const placeholders = spec.accepted_values.map((_, i) => `$${i + 1}`).join(', ');
-          tests.push({
-            id: `${model.name}.${column}.accepted_values`,
-            model: model.name,
-            sql: `SELECT ${col}, count(*) AS n FROM ${target} WHERE ${col} IS NOT NULL AND ${col} NOT IN (${placeholders}) GROUP BY ${col}`,
-            params: spec.accepted_values,
-          });
-        } else {
-          throw new Error(`Unknown test ${JSON.stringify(spec)} on ${model.name}.${column}`);
-        }
-      }
-    }
-  }
-  return tests;
-}
-
-export async function runTest(client, test) {
-  const count = await client.query(`SELECT count(*) AS n FROM (${test.sql}) q`, test.params);
-  const violations = Number(count.rows[0].n);
-  if (violations === 0) return { pass: true };
-  const sample = await client.query(`${test.sql} LIMIT 10`, test.params);
-  return { pass: false, violations, sample: sample.rows };
-}
+import { quoteIdent, rel } from './db.js';
+
+// Each test compiles to a SELECT returning violating rows; any row = FAIL.
+// NULLs only violate not_null (dbt semantics).
+export function buildTests(models, schema) {
+  const tests = [];
+  for (const model of models) {
+    for (const [column, specs] of Object.entries(model.config.tests ?? {})) {
+      const target = rel(schema, model.name);
+      const col = quoteIdent(column);
+      for (const spec of specs) {
+        if (spec === 'not_null') {
+          tests.push({
+            id: `${model.name}.${column}.not_null`,
+            model: model.name,
+            sql: `SELECT * FROM ${target} WHERE ${col} IS NULL`,
+            params: [],
+          });
+        } else if (spec === 'unique') {
+          tests.push({
+            id: `${model.name}.${column}.unique`,
+            model: model.name,
+            sql: `SELECT ${col}, count(*) AS n FROM ${target} WHERE ${col} IS NOT NULL GROUP BY ${col} HAVING count(*) > 1`,
+            params: [],
+          });
+        } else if (spec?.accepted_values?.length) {
+          const placeholders = spec.accepted_values.map((_, i) => `$${i + 1}`).join(', ');
+          tests.push({
+            id: `${model.name}.${column}.accepted_values`,
+            model: model.name,
+            sql: `SELECT ${col}, count(*) AS n FROM ${target} WHERE ${col} IS NOT NULL AND ${col} NOT IN (${placeholders}) GROUP BY ${col}`,
+            params: spec.accepted_values,
+          });
+        } else {
+          throw new Error(`Unknown test ${JSON.stringify(spec)} on ${model.name}.${column}`);
+        }
+      }
+    }
+  }
+  return tests;
+}
+
+export async function runTest(client, test) {
+  const count = await client.query(`SELECT count(*) AS n FROM (${test.sql}) q`, test.params);
+  const violations = Number(count.rows[0].n);
+  if (violations === 0) return { pass: true };
+  const sample = await client.query(`${test.sql} LIMIT 10`, test.params);
+  return { pass: false, violations, sample: sample.rows };
+}