@sqg/sqg 0.20.0 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -117,12 +117,15 @@ console.log(user?.name);
117
117
  | Annotation | Description |
118
118
  |------------|-------------|
119
119
  | `-- MIGRATE name` | Schema migration (CREATE TABLE, etc.) |
120
+ | `-- BASELINE name` | Schema created outside SQG (ETL, sibling service). Runs before migrations for type-checking; **not** emitted in `getMigrations()` |
121
+ | `-- BASELINE name :source=pg` | Schema of a `type: postgres` source. Applied natively to a managed testcontainer and attached into DuckDB for introspection (DuckDB generators; needs Docker) |
120
122
  | `-- QUERY name` | SELECT query returning rows |
121
123
  | `-- QUERY name :one` | Query returning single row or undefined |
122
124
  | `-- QUERY name :pluck` | Return single (first) column value |
123
125
  | `-- QUERY name :result=Foo` | Name the row type (Java). Add it to ONE query and every same-shape query shares it. Full-table `SELECT *` reuses the `TABLE` row type without annotation. |
124
126
  | `-- EXEC name` | INSERT/UPDATE/DELETE (no result rows) |
125
127
  | `-- TESTDATA name` | Test data, runs after migrations |
128
+ | `-- TABLE name :appender` | Generate a type-safe bulk insert appender (DuckDB, PostgreSQL) |
126
129
  | `@set var = value` | Define parameter with sample value |
127
130
  | `${var}` | Reference parameter in query |
128
131
 
package/dist/index.mjs CHANGED
@@ -4,7 +4,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
4
4
  import { LRParser } from "@lezer/lr";
5
5
  import { homedir } from "node:os";
6
6
  import { basename, dirname, extname, join, relative, resolve } from "node:path";
7
- import { camelCase, pascalCase, snakeCase } from "es-toolkit/string";
7
+ import { camelCase, escapeRegExp, pascalCase, snakeCase } from "es-toolkit/string";
8
8
  import Handlebars from "handlebars";
9
9
  import YAML from "yaml";
10
10
  import { z } from "zod";
@@ -229,6 +229,7 @@ SQL Annotation Syntax:
229
229
  -- QUERY <name> [:one] [:pluck] Select query (returns rows)
230
230
  -- EXEC <name> [:batch] Execute statement (INSERT/UPDATE/DELETE)
231
231
  -- BASELINE <name> Schema created outside SQG (runs before migrations, not tracked)
232
+ -- BASELINE <name> :source=<src> Schema of a 'type: postgres' source (applied to a managed testcontainer, attached into DuckDB)
232
233
  -- MIGRATE <name> Schema migration (runs in source order; name is any identifier, e.g. "1" or "add_email")
233
234
  -- TESTDATA <name> Test data setup (not generated)
234
235
  -- TABLE <name> :appender Table for bulk insert appender (DuckDB, PostgreSQL)
@@ -242,6 +243,7 @@ Modifiers:
242
243
  :all Return all rows (default)
243
244
  :batch Generate a JDBC batch method for an EXEC (Java only)
244
245
  :appender Generate bulk insert appender for TABLE annotation
246
+ :source=Name Mark a BASELINE block as the schema of a postgres source (Name)
245
247
  :result=Name Name (and share) the row type (Java only). Annotate ONE query
246
248
  with :result=Name — every other query in the same file with the
247
249
  same column shape automatically gets the same name; you do NOT
@@ -1496,6 +1498,19 @@ var PythonGenerator = class extends BaseGenerator {
1496
1498
  if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
1497
1499
  return `"${value}"`;
1498
1500
  });
1501
+ Handlebars.registerHelper("sqlExpr", (queryHelper) => {
1502
+ const parts = queryHelper.sqlQueryParts;
1503
+ if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) {
1504
+ const value = queryHelper.sqlQuery;
1505
+ if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
1506
+ return `"${value}"`;
1507
+ }
1508
+ let out = "f\"\"\"";
1509
+ for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/\{/g, "{{").replace(/\}/g, "}}");
1510
+ else if (part.name.startsWith("sources_")) out += ` '{${part.name}}'`;
1511
+ else out += "?";
1512
+ return `${out}"""`;
1513
+ });
1499
1514
  Handlebars.registerHelper("pyType", (column) => {
1500
1515
  return this.getPyType(column);
1501
1516
  });
@@ -1620,6 +1635,7 @@ var TsGenerator = class extends BaseGenerator {
1620
1635
  async beforeGenerate(_projectDir, gen, _queries, _tables) {
1621
1636
  if (this.typeMapper instanceof TypeScriptTypeMapper) this.typeMapper.safeIntegers = !!gen.config?.safeIntegers;
1622
1637
  Handlebars.registerHelper("quote", (value) => this.quote(value));
1638
+ Handlebars.registerHelper("sqlExpr", (queryHelper) => this.sqlExpr(queryHelper));
1623
1639
  Handlebars.registerHelper("appendMethod", (column) => {
1624
1640
  if (column.type instanceof ListType) return "List";
1625
1641
  const typeStr = column.type?.toString().toUpperCase() || "";
@@ -1772,6 +1788,22 @@ var TsGenerator = class extends BaseGenerator {
1772
1788
  quote(value) {
1773
1789
  return value.includes("\n") || value.includes("'") ? `\`${value}\`` : `'${value}'`;
1774
1790
  }
1791
+ /**
1792
+ * Build the SQL expression for a query. Without file `sources`, this is the
1793
+ * plain quoted SQL string (unchanged). When a `${sources_x}` reference is
1794
+ * present it is interpolated into a template literal as a runtime argument
1795
+ * (the value is supplied by the caller, not bound), mirroring the Java
1796
+ * generator — `ATTACH '` + arg + `'` style.
1797
+ */
1798
+ sqlExpr(queryHelper) {
1799
+ const parts = queryHelper.sqlQueryParts;
1800
+ if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) return this.quote(queryHelper.sqlQuery);
1801
+ let out = "`";
1802
+ for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
1803
+ else if (part.name.startsWith("sources_")) out += ` '\${${part.name}}'`;
1804
+ else out += "?";
1805
+ return `${out}\``;
1806
+ }
1775
1807
  };
1776
1808
  //#endregion
1777
1809
  //#region src/generators/typescript-duckdb-generator.ts
@@ -1878,6 +1910,69 @@ function getGenerator(generator) {
1878
1910
  }
1879
1911
  }
1880
1912
  //#endregion
1913
+ //#region src/sources.ts
1914
+ /**
1915
+ * For each `type: postgres` source: start a throwaway Postgres testcontainer,
1916
+ * run the source's schema (the BASELINE blocks tagged `:source=<name>`) natively
1917
+ * against it so introspection sees true Postgres types, and return the connection
1918
+ * info to ATTACH into DuckDB.
1919
+ *
1920
+ * The synthesized ATTACH (done by the DuckDB adapter) and these schema blocks are
1921
+ * not emitted into generated code — at runtime the application attaches the real
1922
+ * production database under the same alias.
1923
+ */
1924
+ async function preparePostgresSources(sources, queries, reporter) {
1925
+ const containers = [];
1926
+ const attachments = [];
1927
+ const teardown = async () => {
1928
+ for (const container of containers) try {
1929
+ await container.stop();
1930
+ } catch {}
1931
+ };
1932
+ try {
1933
+ for (const source of sources) {
1934
+ if (source.url) {
1935
+ attachments.push({
1936
+ alias: source.name,
1937
+ connectionUri: source.url
1938
+ });
1939
+ continue;
1940
+ }
1941
+ reporter?.onContainerStarting?.();
1942
+ let container;
1943
+ try {
1944
+ container = await new PostgreSqlContainer(source.image).withDatabase("sqg-db").withUsername("sqg").withPassword("secret").start();
1945
+ } catch (e) {
1946
+ throw new DatabaseError(`Could not start a Postgres container for source '${source.name}': ${e.message}`, "postgres", "Postgres sources need Docker to introspect schema. Start Docker, or set 'url' on the source to point at an existing Postgres database (e.g. url: $DATABASE_URL).");
1947
+ }
1948
+ containers.push(container);
1949
+ const connectionUri = container.getConnectionUri();
1950
+ reporter?.onContainerStarted?.(connectionUri);
1951
+ const schemaBlocks = queries.filter((q) => q.isBaseline && q.sourceTarget === source.name);
1952
+ const client = new Client({ connectionString: connectionUri });
1953
+ await client.connect();
1954
+ try {
1955
+ for (const block of schemaBlocks) await client.query(block.rawQuery);
1956
+ } catch (e) {
1957
+ throw new DatabaseError(`Failed to apply schema for postgres source '${source.name}': ${e.message}`, "postgres", `Check that the BASELINE blocks tagged ':source=${source.name}' are valid PostgreSQL.`);
1958
+ } finally {
1959
+ await client.end();
1960
+ }
1961
+ attachments.push({
1962
+ alias: source.name,
1963
+ connectionUri
1964
+ });
1965
+ }
1966
+ } catch (e) {
1967
+ await teardown();
1968
+ throw e;
1969
+ }
1970
+ return {
1971
+ attachments,
1972
+ teardown
1973
+ };
1974
+ }
1975
+ //#endregion
1881
1976
  //#region src/sqltool.ts
1882
1977
  const GENERATED_FILE_COMMENT = "This file is generated by SQG (https://sqg.dev). Do not edit manually.";
1883
1978
  const configSchema = z.object({ result: z.record(z.string(), z.string()).optional() });
@@ -2054,18 +2149,23 @@ var TableHelper = class {
2054
2149
  return this.generator.typeMapper;
2055
2150
  }
2056
2151
  };
2057
- function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config) {
2152
+ function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config, postgresSourceNames = []) {
2058
2153
  const templateSrc = readFileSync(templatePath, "utf-8");
2059
2154
  const template = Handlebars.compile(templateSrc);
2060
2155
  Handlebars.registerHelper("mapType", (column) => generator.mapType(column));
2061
2156
  Handlebars.registerHelper("plusOne", (value) => value + 1);
2062
2157
  const migrations = queries.filter((q) => q.isMigrate).map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q)));
2063
2158
  const tableHelpers = generator.supportsAppenders(engine) ? tables.filter((t) => !t.skipGenerateFunction).map((t) => new TableHelper(t, generator)) : [];
2159
+ const attachers = engine === "duckdb" ? postgresSourceNames.map((sourceName) => ({
2160
+ alias: sourceName,
2161
+ functionName: generator.getFunctionName(`attach_${sourceName}`)
2162
+ })) : [];
2064
2163
  return template({
2065
2164
  generatedComment: GENERATED_FILE_COMMENT,
2066
2165
  migrations,
2067
2166
  queries: queries.map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q))),
2068
2167
  tables: tableHelpers,
2168
+ attachers,
2069
2169
  className: generator.getClassName(name),
2070
2170
  projectName,
2071
2171
  config
@@ -2090,13 +2190,16 @@ const ProjectSchema = z.object({
2090
2190
  template: z.string().optional().describe("Custom Handlebars template path"),
2091
2191
  output: z.string().min(1, "Output path is required").describe("Output file or directory path"),
2092
2192
  config: z.any().optional().describe("Generator-specific configuration")
2093
- })).min(1, "At least one generator is required").describe("Code generators to run")
2094
- })).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
2193
+ }).strict()).min(1, "At least one generator is required").describe("Code generators to run")
2194
+ }).strict()).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
2095
2195
  sources: z.array(z.object({
2096
- path: z.string().describe("Path to source file (supports $HOME)"),
2097
- name: z.string().optional().describe("Variable name override")
2098
- })).optional().describe("External source files to include as variables")
2099
- });
2196
+ type: z.enum(["file", "postgres"]).optional().describe("Source type: 'file' (default) or 'postgres'"),
2197
+ path: z.string().optional().describe("Path to source file for file sources (supports $HOME)"),
2198
+ name: z.string().optional().describe("Variable name (file sources) or DuckDB attach alias (postgres sources)"),
2199
+ image: z.string().optional().describe("Docker image for postgres sources (default: postgres:16-alpine)"),
2200
+ url: z.string().optional().describe("Postgres source: connect to this existing database for introspection instead of starting a container. Use a literal DSN or '$ENV_VAR' to read from the environment. When set, ':source=' BASELINE blocks are not applied (the live schema is used), which avoids schema drift.")
2201
+ }).strict().refine((s) => s.type === "postgres" ? !!s.name : !!s.path, { message: "file sources require 'path'; postgres sources require 'name'" })).optional().describe("External sources: files inlined as variables, or postgres databases attached for introspection")
2202
+ }).strict();
2100
2203
  var ExtraVariable = class {
2101
2204
  constructor(name, value) {
2102
2205
  this.name = name;
@@ -2104,7 +2207,7 @@ var ExtraVariable = class {
2104
2207
  }
2105
2208
  };
2106
2209
  function createExtraVariables(sources, suppressLogging = false) {
2107
- return sources.map((source) => {
2210
+ return sources.filter((source) => source.type !== "postgres").map((source) => {
2108
2211
  const path = source.path;
2109
2212
  const resolvedPath = path.replace("$HOME", homedir());
2110
2213
  const varName = `sources_${(source.name ?? basename(path, extname(resolvedPath))).replace(/\s+/g, "_")}`;
@@ -2112,6 +2215,27 @@ function createExtraVariables(sources, suppressLogging = false) {
2112
2215
  return new ExtraVariable(varName, `'${resolvedPath}'`);
2113
2216
  });
2114
2217
  }
2218
+ /**
2219
+ * Resolve a postgres source `url`. A value of the form `$VAR` or `${VAR}` is
2220
+ * read from the environment (so credentials need not live in sqg.yaml); any
2221
+ * other value is treated as a literal DSN.
2222
+ */
2223
+ function resolveSourceUrl(url, sourceName) {
2224
+ const match = url.match(/^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/);
2225
+ if (!match) return url;
2226
+ const envName = match[1];
2227
+ const value = process.env[envName];
2228
+ if (!value) throw new SqgError(`Postgres source '${sourceName}' references environment variable '${envName}' which is not set`, "CONFIG_VALIDATION_ERROR", `Set ${envName} to the database connection string, or use a literal DSN in the 'url' field`);
2229
+ return value;
2230
+ }
2231
+ /** Postgres sources resolved from the project config (with image defaulted). */
2232
+ function getPostgresSources(sources) {
2233
+ return sources.filter((source) => source.type === "postgres").map((source) => ({
2234
+ name: source.name,
2235
+ image: source.image ?? "postgres:16-alpine",
2236
+ url: source.url ? resolveSourceUrl(source.url, source.name) : void 0
2237
+ }));
2238
+ }
2115
2239
  function buildProjectFromCliOptions(options) {
2116
2240
  if (!isValidGenerator(options.generator)) {
2117
2241
  const similar = findSimilarGenerators(options.generator);
@@ -2250,7 +2374,7 @@ function assignResultTypeNames(queries, tables, projectDir = process.cwd()) {
2250
2374
  }
2251
2375
  const fullMatch = tryFullTableMatch(group.queries[0], tablesByName);
2252
2376
  if (fullMatch) {
2253
- claim(fullMatch, shape, group.queries[0], `full-table match`);
2377
+ claim(fullMatch, shape, group.queries[0], "full-table match");
2254
2378
  for (const q of group.queries) q.resultTypeName = fullMatch;
2255
2379
  continue;
2256
2380
  }
@@ -2303,14 +2427,14 @@ function validateQueries(queries) {
2303
2427
  };
2304
2428
  }
2305
2429
  }
2306
- async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false) {
2430
+ async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false, postgresSourceNames = []) {
2307
2431
  await generator.beforeGenerate(projectDir, gen, queries, tables);
2308
2432
  const templatePath = join(dirname(new URL(import.meta.url).pathname), gen.template ?? generator.template);
2309
2433
  const name = gen.name ?? basename(file, extname(file));
2310
2434
  const sourceFile = generateSourceFile(name, queries, tables, templatePath, generator, engine, gen.projectName ?? name, {
2311
2435
  migrations: true,
2312
2436
  ...gen.config
2313
- });
2437
+ }, postgresSourceNames);
2314
2438
  if (writeToStdout) {
2315
2439
  process.stdout.write(sourceFile);
2316
2440
  if (!sourceFile.endsWith("\n")) process.stdout.write("\n");
@@ -2411,6 +2535,11 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
2411
2535
  const results = [];
2412
2536
  try {
2413
2537
  const extraVariables = createExtraVariables(project.sources ?? [], writeToStdout);
2538
+ const pgSources = getPostgresSources(project.sources ?? []);
2539
+ const pgSourceNames = new Set(pgSources.map((s) => s.name));
2540
+ if (pgSources.length > 0) {
2541
+ if (!project.sql.some((sql) => sql.gen.some((gen) => getGeneratorEngine(gen.generator) === "duckdb"))) throw new SqgError("Postgres sources require a DuckDB generator (they are attached into DuckDB for introspection)", "CONFIG_VALIDATION_ERROR", "Add a 'typescript/duckdb' or 'java/duckdb' generator, or remove the postgres sources");
2542
+ }
2414
2543
  const files = [];
2415
2544
  for (const sql of project.sql) {
2416
2545
  const gensByEngine = /* @__PURE__ */ new Map();
@@ -2428,18 +2557,31 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
2428
2557
  const parseResult = parseSQLQueries(fullPath, extraVariables);
2429
2558
  queries = parseResult.queries;
2430
2559
  tables = parseResult.tables;
2560
+ for (const q of queries) if (q.sourceTarget && !pgSourceNames.has(q.sourceTarget)) throw SqgError.inQuery(`BASELINE block targets unknown postgres source '${q.sourceTarget}'`, "CONFIG_VALIDATION_ERROR", q.id, sqlFile, { suggestion: `Declare a source with 'type: postgres, name: ${q.sourceTarget}' in sqg.yaml` });
2431
2561
  } catch (e) {
2432
2562
  if (e instanceof SqgError) throw e;
2433
2563
  throw SqgError.inFile(`Failed to parse SQL file: ${e.message}`, "SQL_PARSE_ERROR", sqlFile, { suggestion: "Check SQL syntax and annotation format" });
2434
2564
  }
2565
+ const referencedPgSources = pgSources.filter((source) => {
2566
+ const catalogRef = new RegExp(`\\b${escapeRegExp(source.name)}\\.`);
2567
+ return queries.some((q) => q.sourceTarget === source.name || catalogRef.test(q.rawQuery));
2568
+ });
2569
+ const referencedPgSourceNames = referencedPgSources.map((s) => s.name);
2435
2570
  for (const [engine, gens] of gensByEngine) {
2436
2571
  const executableQueries = queries.filter((q) => !q.skipGenerateFunction);
2437
2572
  const reporterAny = reporter;
2438
2573
  if (reporterAny?.setQueryTotal) reporterAny.setQueryTotal(executableQueries.length);
2574
+ let preparedSources;
2439
2575
  try {
2440
2576
  const dbEngine = getDatabaseEngine(engine);
2577
+ let attachments;
2578
+ if (engine === "duckdb" && referencedPgSources.length > 0) {
2579
+ ui?.startPhase("Starting postgres source containers...");
2580
+ preparedSources = await preparePostgresSources(referencedPgSources, queries, reporter);
2581
+ attachments = preparedSources.attachments;
2582
+ }
2441
2583
  ui?.startPhase(`Initializing ${engine} database...`);
2442
- await dbEngine.initializeDatabase(queries, reporter);
2584
+ await dbEngine.initializeDatabase(queries, reporter, { attachments });
2443
2585
  ui?.startPhase(`Introspecting ${executableQueries.length} queries...`);
2444
2586
  await dbEngine.executeQueries(queries, reporter);
2445
2587
  if (tables.length > 0) await dbEngine.introspectTables(tables, reporter);
@@ -2456,6 +2598,8 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
2456
2598
  file: sqlFile,
2457
2599
  engine
2458
2600
  });
2601
+ } finally {
2602
+ if (preparedSources) await preparedSources.teardown();
2459
2603
  }
2460
2604
  ui?.startPhase("Generating code...");
2461
2605
  const genStart = performance.now();
@@ -2464,7 +2608,7 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
2464
2608
  const outputPath = await writeGeneratedFile(projectDir, {
2465
2609
  ...gen,
2466
2610
  projectName: project.name
2467
- }, generator, sqlFile, queries, tables, engine, writeToStdout);
2611
+ }, generator, sqlFile, queries, tables, engine, writeToStdout, referencedPgSourceNames);
2468
2612
  if (outputPath !== null) {
2469
2613
  files.push(outputPath);
2470
2614
  results.push({
@@ -2537,6 +2681,13 @@ var SQLQuery = class {
2537
2681
  parameterTypes;
2538
2682
  /** Explicit row-type name from `:result=Name` modifier, when present. */
2539
2683
  resultTypeOverride;
2684
+ /**
2685
+ * Name of the postgres `source` this block targets, from a `:source=name`
2686
+ * modifier on a BASELINE block. Such blocks define the schema of an external
2687
+ * postgres source and run natively against that source's container during
2688
+ * generation (not against the DuckDB introspection connection).
2689
+ */
2690
+ sourceTarget;
2540
2691
  /** Final row-type identifier assigned by assignResultTypeNames (after introspection). */
2541
2692
  resultTypeName;
2542
2693
  constructor(filename, id, rawQuery, queryAnonymous, queryNamed, queryPositional, type, isOne, isPluck, isBatch, variables, config, line) {
@@ -2653,6 +2804,8 @@ function parseSQLQueries(filePath, extraVariables) {
2653
2804
  const isPluck = modifiers.includes(":pluck");
2654
2805
  const isBatch = modifiers.includes(":batch");
2655
2806
  const resultTypeOverride = modifiers.find((m) => m.startsWith(":result="))?.slice(8);
2807
+ const sourceTarget = modifiers.find((m) => m.startsWith(":source="))?.slice(8);
2808
+ if (sourceTarget && queryType !== "BASELINE") throw SqgError.inQuery(`The ':source=' modifier is only valid on BASELINE blocks, not ${queryType}`, "SQL_PARSE_ERROR", name, filePath, { suggestion: "Move the schema for a postgres source into a '-- BASELINE <name> :source=<source>' block" });
2656
2809
  let configStr = getStr("Config", true);
2657
2810
  if (configStr?.endsWith("*/")) configStr = configStr.slice(0, -2);
2658
2811
  let config = null;
@@ -2692,8 +2845,12 @@ function parseSQLQueries(filePath, extraVariables) {
2692
2845
  });
2693
2846
  }
2694
2847
  trim() {
2695
- const lastPart = this.sqlParts.length > 0 ? this.sqlParts[this.sqlParts.length - 1] : null;
2696
- if (lastPart && typeof lastPart === "string") this.sqlParts[this.sqlParts.length - 1] = lastPart.trimEnd();
2848
+ const isBlankString = (p) => typeof p === "string" && p.trim() === "";
2849
+ while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[0])) this.sqlParts.shift();
2850
+ if (this.sqlParts.length > 0 && typeof this.sqlParts[0] === "string") this.sqlParts[0] = this.sqlParts[0].trimStart();
2851
+ while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[this.sqlParts.length - 1])) this.sqlParts.pop();
2852
+ const lastIdx = this.sqlParts.length - 1;
2853
+ if (lastIdx >= 0 && typeof this.sqlParts[lastIdx] === "string") this.sqlParts[lastIdx] = this.sqlParts[lastIdx].trimEnd();
2697
2854
  }
2698
2855
  parameters() {
2699
2856
  return this.sqlParts.filter((part) => typeof part !== "string" && !part.name.startsWith("sources_"));
@@ -2808,6 +2965,7 @@ function parseSQLQueries(filePath, extraVariables) {
2808
2965
  });
2809
2966
  const query = new SQLQuery(filePath, name, sqlContentStr, sql.toSqlWithAnonymousPlaceholders(), sql.toSqlWithNamedPlaceholders(), sql.toSqlWithPositionalPlaceholders(), queryType, isOne, isPluck, isBatch, variables, config, annotationLine);
2810
2967
  query.resultTypeOverride = resultTypeOverride;
2968
+ query.sourceTarget = sourceTarget;
2811
2969
  checkDuplicate(queryType, name);
2812
2970
  queries.push(query);
2813
2971
  consola.debug(`Added query: ${name} (${queryType})`);
@@ -2824,7 +2982,7 @@ function parseSQLQueries(filePath, extraVariables) {
2824
2982
  //#endregion
2825
2983
  //#region src/db/types.ts
2826
2984
  async function initializeDatabase(queries, execQueries, reporter) {
2827
- const baselineQueries = queries.filter((q) => q.isBaseline);
2985
+ const baselineQueries = queries.filter((q) => q.isBaseline && !q.sourceTarget);
2828
2986
  for (const query of baselineQueries) try {
2829
2987
  await execQueries(query);
2830
2988
  } catch (error) {
@@ -2880,9 +3038,17 @@ function convertType(type) {
2880
3038
  const duckdb = new class {
2881
3039
  db;
2882
3040
  connection;
2883
- async initializeDatabase(queries, reporter) {
3041
+ async initializeDatabase(queries, reporter, options) {
2884
3042
  this.db = await DuckDBInstance.create(":memory:");
2885
3043
  this.connection = await this.db.connect();
3044
+ for (const attachment of options?.attachments ?? []) {
3045
+ const sql = `ATTACH '${attachment.connectionUri}' AS ${attachment.alias} (TYPE postgres)`;
3046
+ try {
3047
+ await this.connection.run(sql);
3048
+ } catch (e) {
3049
+ throw new DatabaseError(`Failed to attach source '${attachment.alias}': ${e.message}`, "duckdb", "Check that the postgres source container started and is reachable.");
3050
+ }
3051
+ }
2886
3052
  await initializeDatabase(queries, async (query) => {
2887
3053
  try {
2888
3054
  await this.connection.run(query.rawQuery);
package/dist/sqg.mjs CHANGED
@@ -13,7 +13,7 @@ import { Server } from "@modelcontextprotocol/sdk/server/index.js";
13
13
  import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
14
14
  import { CallToolRequestSchema, ListResourcesRequestSchema, ListToolsRequestSchema, ReadResourceRequestSchema } from "@modelcontextprotocol/sdk/types.js";
15
15
  import YAML from "yaml";
16
- import { camelCase, pascalCase, snakeCase } from "es-toolkit/string";
16
+ import { camelCase, escapeRegExp, pascalCase, snakeCase } from "es-toolkit/string";
17
17
  import Handlebars from "handlebars";
18
18
  import { z } from "zod";
19
19
  import { DuckDBEnumType, DuckDBInstance, DuckDBListType, DuckDBMapType, DuckDBStructType } from "@duckdb/node-api";
@@ -221,6 +221,7 @@ SQL Annotation Syntax:
221
221
  -- QUERY <name> [:one] [:pluck] Select query (returns rows)
222
222
  -- EXEC <name> [:batch] Execute statement (INSERT/UPDATE/DELETE)
223
223
  -- BASELINE <name> Schema created outside SQG (runs before migrations, not tracked)
224
+ -- BASELINE <name> :source=<src> Schema of a 'type: postgres' source (applied to a managed testcontainer, attached into DuckDB)
224
225
  -- MIGRATE <name> Schema migration (runs in source order; name is any identifier, e.g. "1" or "add_email")
225
226
  -- TESTDATA <name> Test data setup (not generated)
226
227
  -- TABLE <name> :appender Table for bulk insert appender (DuckDB, PostgreSQL)
@@ -234,6 +235,7 @@ Modifiers:
234
235
  :all Return all rows (default)
235
236
  :batch Generate a JDBC batch method for an EXEC (Java only)
236
237
  :appender Generate bulk insert appender for TABLE annotation
238
+ :source=Name Mark a BASELINE block as the schema of a postgres source (Name)
237
239
  :result=Name Name (and share) the row type (Java only). Annotate ONE query
238
240
  with :result=Name — every other query in the same file with the
239
241
  same column shape automatically gets the same name; you do NOT
@@ -783,6 +785,13 @@ var SQLQuery = class {
783
785
  parameterTypes;
784
786
  /** Explicit row-type name from `:result=Name` modifier, when present. */
785
787
  resultTypeOverride;
788
+ /**
789
+ * Name of the postgres `source` this block targets, from a `:source=name`
790
+ * modifier on a BASELINE block. Such blocks define the schema of an external
791
+ * postgres source and run natively against that source's container during
792
+ * generation (not against the DuckDB introspection connection).
793
+ */
794
+ sourceTarget;
786
795
  /** Final row-type identifier assigned by assignResultTypeNames (after introspection). */
787
796
  resultTypeName;
788
797
  constructor(filename, id, rawQuery, queryAnonymous, queryNamed, queryPositional, type, isOne, isPluck, isBatch, variables, config, line) {
@@ -899,6 +908,8 @@ function parseSQLQueries(filePath, extraVariables) {
899
908
  const isPluck = modifiers.includes(":pluck");
900
909
  const isBatch = modifiers.includes(":batch");
901
910
  const resultTypeOverride = modifiers.find((m) => m.startsWith(":result="))?.slice(8);
911
+ const sourceTarget = modifiers.find((m) => m.startsWith(":source="))?.slice(8);
912
+ if (sourceTarget && queryType !== "BASELINE") throw SqgError.inQuery(`The ':source=' modifier is only valid on BASELINE blocks, not ${queryType}`, "SQL_PARSE_ERROR", name, filePath, { suggestion: "Move the schema for a postgres source into a '-- BASELINE <name> :source=<source>' block" });
902
913
  let configStr = getStr("Config", true);
903
914
  if (configStr?.endsWith("*/")) configStr = configStr.slice(0, -2);
904
915
  let config = null;
@@ -938,8 +949,12 @@ function parseSQLQueries(filePath, extraVariables) {
938
949
  });
939
950
  }
940
951
  trim() {
941
- const lastPart = this.sqlParts.length > 0 ? this.sqlParts[this.sqlParts.length - 1] : null;
942
- if (lastPart && typeof lastPart === "string") this.sqlParts[this.sqlParts.length - 1] = lastPart.trimEnd();
952
+ const isBlankString = (p) => typeof p === "string" && p.trim() === "";
953
+ while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[0])) this.sqlParts.shift();
954
+ if (this.sqlParts.length > 0 && typeof this.sqlParts[0] === "string") this.sqlParts[0] = this.sqlParts[0].trimStart();
955
+ while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[this.sqlParts.length - 1])) this.sqlParts.pop();
956
+ const lastIdx = this.sqlParts.length - 1;
957
+ if (lastIdx >= 0 && typeof this.sqlParts[lastIdx] === "string") this.sqlParts[lastIdx] = this.sqlParts[lastIdx].trimEnd();
943
958
  }
944
959
  parameters() {
945
960
  return this.sqlParts.filter((part) => typeof part !== "string" && !part.name.startsWith("sources_"));
@@ -1054,6 +1069,7 @@ function parseSQLQueries(filePath, extraVariables) {
1054
1069
  });
1055
1070
  const query = new SQLQuery(filePath, name, sqlContentStr, sql.toSqlWithAnonymousPlaceholders(), sql.toSqlWithNamedPlaceholders(), sql.toSqlWithPositionalPlaceholders(), queryType, isOne, isPluck, isBatch, variables, config, annotationLine);
1056
1071
  query.resultTypeOverride = resultTypeOverride;
1072
+ query.sourceTarget = sourceTarget;
1057
1073
  checkDuplicate(queryType, name);
1058
1074
  queries.push(query);
1059
1075
  consola.debug(`Added query: ${name} (${queryType})`);
@@ -1070,7 +1086,7 @@ function parseSQLQueries(filePath, extraVariables) {
1070
1086
  //#endregion
1071
1087
  //#region src/db/types.ts
1072
1088
  async function initializeDatabase(queries, execQueries, reporter) {
1073
- const baselineQueries = queries.filter((q) => q.isBaseline);
1089
+ const baselineQueries = queries.filter((q) => q.isBaseline && !q.sourceTarget);
1074
1090
  for (const query of baselineQueries) try {
1075
1091
  await execQueries(query);
1076
1092
  } catch (error) {
@@ -1126,9 +1142,17 @@ function convertType(type) {
1126
1142
  const duckdb = new class {
1127
1143
  db;
1128
1144
  connection;
1129
- async initializeDatabase(queries, reporter) {
1145
+ async initializeDatabase(queries, reporter, options) {
1130
1146
  this.db = await DuckDBInstance.create(":memory:");
1131
1147
  this.connection = await this.db.connect();
1148
+ for (const attachment of options?.attachments ?? []) {
1149
+ const sql = `ATTACH '${attachment.connectionUri}' AS ${attachment.alias} (TYPE postgres)`;
1150
+ try {
1151
+ await this.connection.run(sql);
1152
+ } catch (e) {
1153
+ throw new DatabaseError(`Failed to attach source '${attachment.alias}': ${e.message}`, "duckdb", "Check that the postgres source container started and is reachable.");
1154
+ }
1155
+ }
1132
1156
  await initializeDatabase(queries, async (query) => {
1133
1157
  try {
1134
1158
  await this.connection.run(query.rawQuery);
@@ -2726,6 +2750,19 @@ var PythonGenerator = class extends BaseGenerator {
2726
2750
  if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
2727
2751
  return `"${value}"`;
2728
2752
  });
2753
+ Handlebars.registerHelper("sqlExpr", (queryHelper) => {
2754
+ const parts = queryHelper.sqlQueryParts;
2755
+ if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) {
2756
+ const value = queryHelper.sqlQuery;
2757
+ if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
2758
+ return `"${value}"`;
2759
+ }
2760
+ let out = "f\"\"\"";
2761
+ for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/\{/g, "{{").replace(/\}/g, "}}");
2762
+ else if (part.name.startsWith("sources_")) out += ` '{${part.name}}'`;
2763
+ else out += "?";
2764
+ return `${out}"""`;
2765
+ });
2729
2766
  Handlebars.registerHelper("pyType", (column) => {
2730
2767
  return this.getPyType(column);
2731
2768
  });
@@ -2850,6 +2887,7 @@ var TsGenerator = class extends BaseGenerator {
2850
2887
  async beforeGenerate(_projectDir, gen, _queries, _tables) {
2851
2888
  if (this.typeMapper instanceof TypeScriptTypeMapper) this.typeMapper.safeIntegers = !!gen.config?.safeIntegers;
2852
2889
  Handlebars.registerHelper("quote", (value) => this.quote(value));
2890
+ Handlebars.registerHelper("sqlExpr", (queryHelper) => this.sqlExpr(queryHelper));
2853
2891
  Handlebars.registerHelper("appendMethod", (column) => {
2854
2892
  if (column.type instanceof ListType) return "List";
2855
2893
  const typeStr = column.type?.toString().toUpperCase() || "";
@@ -3002,6 +3040,22 @@ var TsGenerator = class extends BaseGenerator {
3002
3040
  quote(value) {
3003
3041
  return value.includes("\n") || value.includes("'") ? `\`${value}\`` : `'${value}'`;
3004
3042
  }
3043
+ /**
3044
+ * Build the SQL expression for a query. Without file `sources`, this is the
3045
+ * plain quoted SQL string (unchanged). When a `${sources_x}` reference is
3046
+ * present it is interpolated into a template literal as a runtime argument
3047
+ * (the value is supplied by the caller, not bound), mirroring the Java
3048
+ * generator — `ATTACH '` + arg + `'` style.
3049
+ */
3050
+ sqlExpr(queryHelper) {
3051
+ const parts = queryHelper.sqlQueryParts;
3052
+ if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) return this.quote(queryHelper.sqlQuery);
3053
+ let out = "`";
3054
+ for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
3055
+ else if (part.name.startsWith("sources_")) out += ` '\${${part.name}}'`;
3056
+ else out += "?";
3057
+ return `${out}\``;
3058
+ }
3005
3059
  };
3006
3060
  //#endregion
3007
3061
  //#region src/generators/typescript-duckdb-generator.ts
@@ -3108,6 +3162,69 @@ function getGenerator(generator) {
3108
3162
  }
3109
3163
  }
3110
3164
  //#endregion
3165
+ //#region src/sources.ts
3166
+ /**
3167
+ * For each `type: postgres` source: start a throwaway Postgres testcontainer,
3168
+ * run the source's schema (the BASELINE blocks tagged `:source=<name>`) natively
3169
+ * against it so introspection sees true Postgres types, and return the connection
3170
+ * info to ATTACH into DuckDB.
3171
+ *
3172
+ * The synthesized ATTACH (done by the DuckDB adapter) and these schema blocks are
3173
+ * not emitted into generated code — at runtime the application attaches the real
3174
+ * production database under the same alias.
3175
+ */
3176
+ async function preparePostgresSources(sources, queries, reporter) {
3177
+ const containers = [];
3178
+ const attachments = [];
3179
+ const teardown = async () => {
3180
+ for (const container of containers) try {
3181
+ await container.stop();
3182
+ } catch {}
3183
+ };
3184
+ try {
3185
+ for (const source of sources) {
3186
+ if (source.url) {
3187
+ attachments.push({
3188
+ alias: source.name,
3189
+ connectionUri: source.url
3190
+ });
3191
+ continue;
3192
+ }
3193
+ reporter?.onContainerStarting?.();
3194
+ let container;
3195
+ try {
3196
+ container = await new PostgreSqlContainer(source.image).withDatabase("sqg-db").withUsername("sqg").withPassword("secret").start();
3197
+ } catch (e) {
3198
+ throw new DatabaseError(`Could not start a Postgres container for source '${source.name}': ${e.message}`, "postgres", "Postgres sources need Docker to introspect schema. Start Docker, or set 'url' on the source to point at an existing Postgres database (e.g. url: $DATABASE_URL).");
3199
+ }
3200
+ containers.push(container);
3201
+ const connectionUri = container.getConnectionUri();
3202
+ reporter?.onContainerStarted?.(connectionUri);
3203
+ const schemaBlocks = queries.filter((q) => q.isBaseline && q.sourceTarget === source.name);
3204
+ const client = new Client({ connectionString: connectionUri });
3205
+ await client.connect();
3206
+ try {
3207
+ for (const block of schemaBlocks) await client.query(block.rawQuery);
3208
+ } catch (e) {
3209
+ throw new DatabaseError(`Failed to apply schema for postgres source '${source.name}': ${e.message}`, "postgres", `Check that the BASELINE blocks tagged ':source=${source.name}' are valid PostgreSQL.`);
3210
+ } finally {
3211
+ await client.end();
3212
+ }
3213
+ attachments.push({
3214
+ alias: source.name,
3215
+ connectionUri
3216
+ });
3217
+ }
3218
+ } catch (e) {
3219
+ await teardown();
3220
+ throw e;
3221
+ }
3222
+ return {
3223
+ attachments,
3224
+ teardown
3225
+ };
3226
+ }
3227
+ //#endregion
3111
3228
  //#region src/sqltool.ts
3112
3229
  const GENERATED_FILE_COMMENT = "This file is generated by SQG (https://sqg.dev). Do not edit manually.";
3113
3230
  const configSchema = z.object({ result: z.record(z.string(), z.string()).optional() });
@@ -3284,18 +3401,23 @@ var TableHelper = class {
3284
3401
  return this.generator.typeMapper;
3285
3402
  }
3286
3403
  };
3287
- function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config) {
3404
+ function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config, postgresSourceNames = []) {
3288
3405
  const templateSrc = readFileSync(templatePath, "utf-8");
3289
3406
  const template = Handlebars.compile(templateSrc);
3290
3407
  Handlebars.registerHelper("mapType", (column) => generator.mapType(column));
3291
3408
  Handlebars.registerHelper("plusOne", (value) => value + 1);
3292
3409
  const migrations = queries.filter((q) => q.isMigrate).map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q)));
3293
3410
  const tableHelpers = generator.supportsAppenders(engine) ? tables.filter((t) => !t.skipGenerateFunction).map((t) => new TableHelper(t, generator)) : [];
3411
+ const attachers = engine === "duckdb" ? postgresSourceNames.map((sourceName) => ({
3412
+ alias: sourceName,
3413
+ functionName: generator.getFunctionName(`attach_${sourceName}`)
3414
+ })) : [];
3294
3415
  return template({
3295
3416
  generatedComment: GENERATED_FILE_COMMENT,
3296
3417
  migrations,
3297
3418
  queries: queries.map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q))),
3298
3419
  tables: tableHelpers,
3420
+ attachers,
3299
3421
  className: generator.getClassName(name),
3300
3422
  projectName,
3301
3423
  config
@@ -3320,13 +3442,16 @@ const ProjectSchema = z.object({
3320
3442
  template: z.string().optional().describe("Custom Handlebars template path"),
3321
3443
  output: z.string().min(1, "Output path is required").describe("Output file or directory path"),
3322
3444
  config: z.any().optional().describe("Generator-specific configuration")
3323
- })).min(1, "At least one generator is required").describe("Code generators to run")
3324
- })).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
3445
+ }).strict()).min(1, "At least one generator is required").describe("Code generators to run")
3446
+ }).strict()).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
3325
3447
  sources: z.array(z.object({
3326
- path: z.string().describe("Path to source file (supports $HOME)"),
3327
- name: z.string().optional().describe("Variable name override")
3328
- })).optional().describe("External source files to include as variables")
3329
- });
3448
+ type: z.enum(["file", "postgres"]).optional().describe("Source type: 'file' (default) or 'postgres'"),
3449
+ path: z.string().optional().describe("Path to source file for file sources (supports $HOME)"),
3450
+ name: z.string().optional().describe("Variable name (file sources) or DuckDB attach alias (postgres sources)"),
3451
+ image: z.string().optional().describe("Docker image for postgres sources (default: postgres:16-alpine)"),
3452
+ url: z.string().optional().describe("Postgres source: connect to this existing database for introspection instead of starting a container. Use a literal DSN or '$ENV_VAR' to read from the environment. When set, ':source=' BASELINE blocks are not applied (the live schema is used), which avoids schema drift.")
3453
+ }).strict().refine((s) => s.type === "postgres" ? !!s.name : !!s.path, { message: "file sources require 'path'; postgres sources require 'name'" })).optional().describe("External sources: files inlined as variables, or postgres databases attached for introspection")
3454
+ }).strict();
3330
3455
  var ExtraVariable = class {
3331
3456
  constructor(name, value) {
3332
3457
  this.name = name;
@@ -3334,7 +3459,7 @@ var ExtraVariable = class {
3334
3459
  }
3335
3460
  };
3336
3461
  function createExtraVariables(sources, suppressLogging = false) {
3337
- return sources.map((source) => {
3462
+ return sources.filter((source) => source.type !== "postgres").map((source) => {
3338
3463
  const path = source.path;
3339
3464
  const resolvedPath = path.replace("$HOME", homedir());
3340
3465
  const varName = `sources_${(source.name ?? basename(path, extname(resolvedPath))).replace(/\s+/g, "_")}`;
@@ -3342,6 +3467,27 @@ function createExtraVariables(sources, suppressLogging = false) {
3342
3467
  return new ExtraVariable(varName, `'${resolvedPath}'`);
3343
3468
  });
3344
3469
  }
3470
+ /**
3471
+ * Resolve a postgres source `url`. A value of the form `$VAR` or `${VAR}` is
3472
+ * read from the environment (so credentials need not live in sqg.yaml); any
3473
+ * other value is treated as a literal DSN.
3474
+ */
3475
+ function resolveSourceUrl(url, sourceName) {
3476
+ const match = url.match(/^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/);
3477
+ if (!match) return url;
3478
+ const envName = match[1];
3479
+ const value = process.env[envName];
3480
+ if (!value) throw new SqgError(`Postgres source '${sourceName}' references environment variable '${envName}' which is not set`, "CONFIG_VALIDATION_ERROR", `Set ${envName} to the database connection string, or use a literal DSN in the 'url' field`);
3481
+ return value;
3482
+ }
3483
+ /** Postgres sources resolved from the project config (with image defaulted). */
3484
+ function getPostgresSources(sources) {
3485
+ return sources.filter((source) => source.type === "postgres").map((source) => ({
3486
+ name: source.name,
3487
+ image: source.image ?? "postgres:16-alpine",
3488
+ url: source.url ? resolveSourceUrl(source.url, source.name) : void 0
3489
+ }));
3490
+ }
3345
3491
  function buildProjectFromCliOptions(options) {
3346
3492
  if (!isValidGenerator(options.generator)) {
3347
3493
  const similar = findSimilarGenerators(options.generator);
@@ -3480,7 +3626,7 @@ function assignResultTypeNames(queries, tables, projectDir = process.cwd()) {
3480
3626
  }
3481
3627
  const fullMatch = tryFullTableMatch(group.queries[0], tablesByName);
3482
3628
  if (fullMatch) {
3483
- claim(fullMatch, shape, group.queries[0], `full-table match`);
3629
+ claim(fullMatch, shape, group.queries[0], "full-table match");
3484
3630
  for (const q of group.queries) q.resultTypeName = fullMatch;
3485
3631
  continue;
3486
3632
  }
@@ -3533,14 +3679,14 @@ function validateQueries(queries) {
3533
3679
  };
3534
3680
  }
3535
3681
  }
3536
- async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false) {
3682
+ async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false, postgresSourceNames = []) {
3537
3683
  await generator.beforeGenerate(projectDir, gen, queries, tables);
3538
3684
  const templatePath = join(dirname(new URL(import.meta.url).pathname), gen.template ?? generator.template);
3539
3685
  const name = gen.name ?? basename(file, extname(file));
3540
3686
  const sourceFile = generateSourceFile(name, queries, tables, templatePath, generator, engine, gen.projectName ?? name, {
3541
3687
  migrations: true,
3542
3688
  ...gen.config
3543
- });
3689
+ }, postgresSourceNames);
3544
3690
  if (writeToStdout) {
3545
3691
  process.stdout.write(sourceFile);
3546
3692
  if (!sourceFile.endsWith("\n")) process.stdout.write("\n");
@@ -3641,6 +3787,11 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
3641
3787
  const results = [];
3642
3788
  try {
3643
3789
  const extraVariables = createExtraVariables(project.sources ?? [], writeToStdout);
3790
+ const pgSources = getPostgresSources(project.sources ?? []);
3791
+ const pgSourceNames = new Set(pgSources.map((s) => s.name));
3792
+ if (pgSources.length > 0) {
3793
+ if (!project.sql.some((sql) => sql.gen.some((gen) => getGeneratorEngine(gen.generator) === "duckdb"))) throw new SqgError("Postgres sources require a DuckDB generator (they are attached into DuckDB for introspection)", "CONFIG_VALIDATION_ERROR", "Add a 'typescript/duckdb' or 'java/duckdb' generator, or remove the postgres sources");
3794
+ }
3644
3795
  const files = [];
3645
3796
  for (const sql of project.sql) {
3646
3797
  const gensByEngine = /* @__PURE__ */ new Map();
@@ -3658,18 +3809,31 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
3658
3809
  const parseResult = parseSQLQueries(fullPath, extraVariables);
3659
3810
  queries = parseResult.queries;
3660
3811
  tables = parseResult.tables;
3812
+ for (const q of queries) if (q.sourceTarget && !pgSourceNames.has(q.sourceTarget)) throw SqgError.inQuery(`BASELINE block targets unknown postgres source '${q.sourceTarget}'`, "CONFIG_VALIDATION_ERROR", q.id, sqlFile, { suggestion: `Declare a source with 'type: postgres, name: ${q.sourceTarget}' in sqg.yaml` });
3661
3813
  } catch (e) {
3662
3814
  if (e instanceof SqgError) throw e;
3663
3815
  throw SqgError.inFile(`Failed to parse SQL file: ${e.message}`, "SQL_PARSE_ERROR", sqlFile, { suggestion: "Check SQL syntax and annotation format" });
3664
3816
  }
3817
+ const referencedPgSources = pgSources.filter((source) => {
3818
+ const catalogRef = new RegExp(`\\b${escapeRegExp(source.name)}\\.`);
3819
+ return queries.some((q) => q.sourceTarget === source.name || catalogRef.test(q.rawQuery));
3820
+ });
3821
+ const referencedPgSourceNames = referencedPgSources.map((s) => s.name);
3665
3822
  for (const [engine, gens] of gensByEngine) {
3666
3823
  const executableQueries = queries.filter((q) => !q.skipGenerateFunction);
3667
3824
  const reporterAny = reporter;
3668
3825
  if (reporterAny?.setQueryTotal) reporterAny.setQueryTotal(executableQueries.length);
3826
+ let preparedSources;
3669
3827
  try {
3670
3828
  const dbEngine = getDatabaseEngine(engine);
3829
+ let attachments;
3830
+ if (engine === "duckdb" && referencedPgSources.length > 0) {
3831
+ ui?.startPhase("Starting postgres source containers...");
3832
+ preparedSources = await preparePostgresSources(referencedPgSources, queries, reporter);
3833
+ attachments = preparedSources.attachments;
3834
+ }
3671
3835
  ui?.startPhase(`Initializing ${engine} database...`);
3672
- await dbEngine.initializeDatabase(queries, reporter);
3836
+ await dbEngine.initializeDatabase(queries, reporter, { attachments });
3673
3837
  ui?.startPhase(`Introspecting ${executableQueries.length} queries...`);
3674
3838
  await dbEngine.executeQueries(queries, reporter);
3675
3839
  if (tables.length > 0) await dbEngine.introspectTables(tables, reporter);
@@ -3686,6 +3850,8 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
3686
3850
  file: sqlFile,
3687
3851
  engine
3688
3852
  });
3853
+ } finally {
3854
+ if (preparedSources) await preparedSources.teardown();
3689
3855
  }
3690
3856
  ui?.startPhase("Generating code...");
3691
3857
  const genStart = performance.now();
@@ -3694,7 +3860,7 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
3694
3860
  const outputPath = await writeGeneratedFile(projectDir, {
3695
3861
  ...gen,
3696
3862
  projectName: project.name
3697
- }, generator, sqlFile, queries, tables, engine, writeToStdout);
3863
+ }, generator, sqlFile, queries, tables, engine, writeToStdout, referencedPgSourceNames);
3698
3864
  if (outputPath !== null) {
3699
3865
  files.push(outputPath);
3700
3866
  results.push({
@@ -3728,7 +3894,7 @@ async function processProject(projectPath, ui) {
3728
3894
  //#region src/mcp-server.ts
3729
3895
  const server = new Server({
3730
3896
  name: "sqg-mcp",
3731
- version: process.env.npm_package_version ?? "0.20.0"
3897
+ version: process.env.npm_package_version ?? "0.21.1"
3732
3898
  }, { capabilities: {
3733
3899
  tools: {},
3734
3900
  resources: {}
@@ -4199,7 +4365,7 @@ function formatMs(ms) {
4199
4365
  }
4200
4366
  //#endregion
4201
4367
  //#region src/sqg.ts
4202
- const version = process.env.npm_package_version ?? "0.20.0";
4368
+ const version = process.env.npm_package_version ?? "0.21.1";
4203
4369
  updateNotifier({ pkg: {
4204
4370
  name: "@sqg/sqg",
4205
4371
  version
@@ -69,7 +69,7 @@ export class {{className}} {
69
69
  {{#unless skipGenerateFunction}}
70
70
  {{functionName}}({{#each variables}}{{name}}: {{type}}{{#unless @last}}, {{/unless}}{{/each}}): {{> returnType }} {
71
71
  const stmt = this.prepare<[{{> paramTypes}}], {{> resultType}}>('{{id}}',
72
- {{{quote sqlQuery}}}{{#if isPluck}}, true{{/if}});
72
+ {{{sqlExpr this}}}{{#if isPluck}}, true{{/if}});
73
73
  {{> execute}}
74
74
  }
75
75
  {{/unless}}
@@ -223,6 +223,17 @@ public class {{className}} {
223
223
 
224
224
  {{{declareEnums queries}}}
225
225
 
226
+ {{#each attachers}}
227
+ /** Attach the "{{alias}}" postgres source under the same alias used during generation. Loads the postgres extension (idempotent) so callers need not rely on DuckDB autoloading. */
228
+ public void {{functionName}}(String connectionString) throws SQLException {
229
+ try (var stmt = connection.createStatement()) {
230
+ stmt.execute("INSTALL postgres");
231
+ stmt.execute("LOAD postgres");
232
+ stmt.execute("ATTACH '" + connectionString + "' AS {{alias}} (TYPE postgres)");
233
+ }
234
+ }
235
+
236
+ {{/each}}
226
237
  {{#each queries}}
227
238
  {{#unless skipGenerateFunction}}
228
239
  {{>columnTypesRecord}}
@@ -135,6 +135,18 @@ class {{className}}:
135
135
  {{/if}}
136
136
  {{/if}}
137
137
 
138
+ {{#each attachers}}
139
+ def {{functionName}}(self, connection_string: str) -> None:
140
+ """Attach the "{{alias}}" postgres source under the same alias used during generation.
141
+
142
+ Loads the postgres extension (idempotent) so callers need not rely on
143
+ DuckDB autoloading.
144
+ """
145
+ self._conn.execute("INSTALL postgres")
146
+ self._conn.execute("LOAD postgres")
147
+ self._conn.execute(f"ATTACH '{connection_string}' AS {{alias}} (TYPE postgres)")
148
+
149
+ {{/each}}
138
150
  {{#each queries}}
139
151
  {{#unless skipGenerateFunction}}
140
152
  {{#if isQuery}}
@@ -142,7 +154,7 @@ class {{className}}:
142
154
  {{#if isOne}}
143
155
  def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> {{{pyTypeOrNone (lookup columns 0)}}}:
144
156
  row = self._conn.execute(
145
- {{{quote sqlQuery}}}, {{> params}}
157
+ {{{sqlExpr this}}}, {{> params}}
146
158
  ).fetchone()
147
159
  if row is None:
148
160
  return None
@@ -150,19 +162,19 @@ class {{className}}:
150
162
 
151
163
  def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> tuple[Any, ...] | None:
152
164
  return self._conn.execute(
153
- {{{quote sqlQuery}}}, {{> params}}
165
+ {{{sqlExpr this}}}, {{> params}}
154
166
  ).fetchone()
155
167
 
156
168
  {{else}}
157
169
  def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[{{{pyType (lookup columns 0)}}}]:
158
170
  rows = self._conn.execute(
159
- {{{quote sqlQuery}}}, {{> params}}
171
+ {{{sqlExpr this}}}, {{> params}}
160
172
  ).fetchall()
161
173
  return [r[0] for r in rows]
162
174
 
163
175
  def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[tuple[Any, ...]]:
164
176
  return self._conn.execute(
165
- {{{quote sqlQuery}}}, {{> params}}
177
+ {{{sqlExpr this}}}, {{> params}}
166
178
  ).fetchall()
167
179
 
168
180
  {{/if}}
@@ -170,7 +182,7 @@ class {{className}}:
170
182
  {{#if isOne}}
171
183
  def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> {{{rowType}}} | None:
172
184
  row = self._conn.execute(
173
- {{{quote sqlQuery}}}, {{> params}}
185
+ {{{sqlExpr this}}}, {{> params}}
174
186
  ).fetchone()
175
187
  if row is None:
176
188
  return None
@@ -178,19 +190,19 @@ class {{className}}:
178
190
 
179
191
  def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> tuple[Any, ...] | None:
180
192
  return self._conn.execute(
181
- {{{quote sqlQuery}}}, {{> params}}
193
+ {{{sqlExpr this}}}, {{> params}}
182
194
  ).fetchone()
183
195
 
184
196
  {{else}}
185
197
  def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[{{{rowType}}}]:
186
198
  rows = self._conn.execute(
187
- {{{quote sqlQuery}}}, {{> params}}
199
+ {{{sqlExpr this}}}, {{> params}}
188
200
  ).fetchall()
189
201
  return [{{{constructRow this}}} for row in rows]
190
202
 
191
203
  def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[tuple[Any, ...]]:
192
204
  return self._conn.execute(
193
- {{{quote sqlQuery}}}, {{> params}}
205
+ {{{sqlExpr this}}}, {{> params}}
194
206
  ).fetchall()
195
207
 
196
208
  {{/if}}
@@ -198,7 +210,7 @@ class {{className}}:
198
210
  {{else}}
199
211
  def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> None:
200
212
  self._conn.execute(
201
- {{{quote sqlQuery}}}, {{> params}}
213
+ {{{sqlExpr this}}}, {{> params}}
202
214
  )
203
215
 
204
216
  {{/if}}
@@ -58,10 +58,19 @@ export class {{className}} {
58
58
  );
59
59
  }
60
60
 
61
+ {{#each attachers}}
62
+ /** Attach the "{{alias}}" postgres source under the same alias used during generation. Loads the postgres extension (idempotent) so callers need not rely on DuckDB autoloading. */
63
+ async {{functionName}}(connectionString: string): Promise<void> {
64
+ await this.conn.run("INSTALL postgres");
65
+ await this.conn.run("LOAD postgres");
66
+ await this.conn.run(`ATTACH '${connectionString}' AS {{alias}} (TYPE postgres)`);
67
+ }
68
+
69
+ {{/each}}
61
70
  {{#each queries}}
62
71
  {{#unless skipGenerateFunction}}
63
72
  async {{functionName}}({{#each variables}}{{name}}: {{type}}{{#unless @last}}, {{/unless}}{{/each}}): Promise<{{> returnType }}> {
64
- const sql = {{{quote sqlQuery}}};
73
+ const sql = {{{sqlExpr this}}};
65
74
  {{#if (hasListParams this)}}
66
75
  const stmt = await this.conn.prepare(sql);
67
76
  {{{bindStatements this}}}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@sqg/sqg",
3
- "version": "0.20.0",
3
+ "version": "0.21.1",
4
4
  "description": "SQG - SQL Query Generator - Type-safe code generation from SQL (https://sqg.dev)",
5
5
  "type": "module",
6
6
  "main": "dist/index.mjs",