@sqg/sqg 0.20.0 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -0
- package/dist/index.mjs +184 -18
- package/dist/sqg.mjs +186 -20
- package/dist/templates/better-sqlite3.hbs +1 -1
- package/dist/templates/java-jdbc.hbs +11 -0
- package/dist/templates/python.hbs +21 -9
- package/dist/templates/typescript-duckdb.hbs +10 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -117,12 +117,15 @@ console.log(user?.name);
|
|
|
117
117
|
| Annotation | Description |
|
|
118
118
|
|------------|-------------|
|
|
119
119
|
| `-- MIGRATE name` | Schema migration (CREATE TABLE, etc.) |
|
|
120
|
+
| `-- BASELINE name` | Schema created outside SQG (ETL, sibling service). Runs before migrations for type-checking; **not** emitted in `getMigrations()` |
|
|
121
|
+
| `-- BASELINE name :source=pg` | Schema of a `type: postgres` source. Applied natively to a managed testcontainer and attached into DuckDB for introspection (DuckDB generators; needs Docker) |
|
|
120
122
|
| `-- QUERY name` | SELECT query returning rows |
|
|
121
123
|
| `-- QUERY name :one` | Query returning single row or undefined |
|
|
122
124
|
| `-- QUERY name :pluck` | Return single (first) column value |
|
|
123
125
|
| `-- QUERY name :result=Foo` | Name the row type (Java). Add it to ONE query and every same-shape query shares it. Full-table `SELECT *` reuses the `TABLE` row type without annotation. |
|
|
124
126
|
| `-- EXEC name` | INSERT/UPDATE/DELETE (no result rows) |
|
|
125
127
|
| `-- TESTDATA name` | Test data, runs after migrations |
|
|
128
|
+
| `-- TABLE name :appender` | Generate a type-safe bulk insert appender (DuckDB, PostgreSQL) |
|
|
126
129
|
| `@set var = value` | Define parameter with sample value |
|
|
127
130
|
| `${var}` | Reference parameter in query |
|
|
128
131
|
|
package/dist/index.mjs
CHANGED
|
@@ -4,7 +4,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
|
4
4
|
import { LRParser } from "@lezer/lr";
|
|
5
5
|
import { homedir } from "node:os";
|
|
6
6
|
import { basename, dirname, extname, join, relative, resolve } from "node:path";
|
|
7
|
-
import { camelCase, pascalCase, snakeCase } from "es-toolkit/string";
|
|
7
|
+
import { camelCase, escapeRegExp, pascalCase, snakeCase } from "es-toolkit/string";
|
|
8
8
|
import Handlebars from "handlebars";
|
|
9
9
|
import YAML from "yaml";
|
|
10
10
|
import { z } from "zod";
|
|
@@ -229,6 +229,7 @@ SQL Annotation Syntax:
|
|
|
229
229
|
-- QUERY <name> [:one] [:pluck] Select query (returns rows)
|
|
230
230
|
-- EXEC <name> [:batch] Execute statement (INSERT/UPDATE/DELETE)
|
|
231
231
|
-- BASELINE <name> Schema created outside SQG (runs before migrations, not tracked)
|
|
232
|
+
-- BASELINE <name> :source=<src> Schema of a 'type: postgres' source (applied to a managed testcontainer, attached into DuckDB)
|
|
232
233
|
-- MIGRATE <name> Schema migration (runs in source order; name is any identifier, e.g. "1" or "add_email")
|
|
233
234
|
-- TESTDATA <name> Test data setup (not generated)
|
|
234
235
|
-- TABLE <name> :appender Table for bulk insert appender (DuckDB, PostgreSQL)
|
|
@@ -242,6 +243,7 @@ Modifiers:
|
|
|
242
243
|
:all Return all rows (default)
|
|
243
244
|
:batch Generate a JDBC batch method for an EXEC (Java only)
|
|
244
245
|
:appender Generate bulk insert appender for TABLE annotation
|
|
246
|
+
:source=Name Mark a BASELINE block as the schema of a postgres source (Name)
|
|
245
247
|
:result=Name Name (and share) the row type (Java only). Annotate ONE query
|
|
246
248
|
with :result=Name — every other query in the same file with the
|
|
247
249
|
same column shape automatically gets the same name; you do NOT
|
|
@@ -1496,6 +1498,19 @@ var PythonGenerator = class extends BaseGenerator {
|
|
|
1496
1498
|
if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
|
|
1497
1499
|
return `"${value}"`;
|
|
1498
1500
|
});
|
|
1501
|
+
Handlebars.registerHelper("sqlExpr", (queryHelper) => {
|
|
1502
|
+
const parts = queryHelper.sqlQueryParts;
|
|
1503
|
+
if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) {
|
|
1504
|
+
const value = queryHelper.sqlQuery;
|
|
1505
|
+
if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
|
|
1506
|
+
return `"${value}"`;
|
|
1507
|
+
}
|
|
1508
|
+
let out = "f\"\"\"";
|
|
1509
|
+
for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/\{/g, "{{").replace(/\}/g, "}}");
|
|
1510
|
+
else if (part.name.startsWith("sources_")) out += ` '{${part.name}}'`;
|
|
1511
|
+
else out += "?";
|
|
1512
|
+
return `${out}"""`;
|
|
1513
|
+
});
|
|
1499
1514
|
Handlebars.registerHelper("pyType", (column) => {
|
|
1500
1515
|
return this.getPyType(column);
|
|
1501
1516
|
});
|
|
@@ -1620,6 +1635,7 @@ var TsGenerator = class extends BaseGenerator {
|
|
|
1620
1635
|
async beforeGenerate(_projectDir, gen, _queries, _tables) {
|
|
1621
1636
|
if (this.typeMapper instanceof TypeScriptTypeMapper) this.typeMapper.safeIntegers = !!gen.config?.safeIntegers;
|
|
1622
1637
|
Handlebars.registerHelper("quote", (value) => this.quote(value));
|
|
1638
|
+
Handlebars.registerHelper("sqlExpr", (queryHelper) => this.sqlExpr(queryHelper));
|
|
1623
1639
|
Handlebars.registerHelper("appendMethod", (column) => {
|
|
1624
1640
|
if (column.type instanceof ListType) return "List";
|
|
1625
1641
|
const typeStr = column.type?.toString().toUpperCase() || "";
|
|
@@ -1772,6 +1788,22 @@ var TsGenerator = class extends BaseGenerator {
|
|
|
1772
1788
|
quote(value) {
|
|
1773
1789
|
return value.includes("\n") || value.includes("'") ? `\`${value}\`` : `'${value}'`;
|
|
1774
1790
|
}
|
|
1791
|
+
/**
|
|
1792
|
+
* Build the SQL expression for a query. Without file `sources`, this is the
|
|
1793
|
+
* plain quoted SQL string (unchanged). When a `${sources_x}` reference is
|
|
1794
|
+
* present it is interpolated into a template literal as a runtime argument
|
|
1795
|
+
* (the value is supplied by the caller, not bound), mirroring the Java
|
|
1796
|
+
* generator — `ATTACH '` + arg + `'` style.
|
|
1797
|
+
*/
|
|
1798
|
+
sqlExpr(queryHelper) {
|
|
1799
|
+
const parts = queryHelper.sqlQueryParts;
|
|
1800
|
+
if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) return this.quote(queryHelper.sqlQuery);
|
|
1801
|
+
let out = "`";
|
|
1802
|
+
for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
|
|
1803
|
+
else if (part.name.startsWith("sources_")) out += ` '\${${part.name}}'`;
|
|
1804
|
+
else out += "?";
|
|
1805
|
+
return `${out}\``;
|
|
1806
|
+
}
|
|
1775
1807
|
};
|
|
1776
1808
|
//#endregion
|
|
1777
1809
|
//#region src/generators/typescript-duckdb-generator.ts
|
|
@@ -1878,6 +1910,69 @@ function getGenerator(generator) {
|
|
|
1878
1910
|
}
|
|
1879
1911
|
}
|
|
1880
1912
|
//#endregion
|
|
1913
|
+
//#region src/sources.ts
|
|
1914
|
+
/**
|
|
1915
|
+
* For each `type: postgres` source: start a throwaway Postgres testcontainer,
|
|
1916
|
+
* run the source's schema (the BASELINE blocks tagged `:source=<name>`) natively
|
|
1917
|
+
* against it so introspection sees true Postgres types, and return the connection
|
|
1918
|
+
* info to ATTACH into DuckDB.
|
|
1919
|
+
*
|
|
1920
|
+
* The synthesized ATTACH (done by the DuckDB adapter) and these schema blocks are
|
|
1921
|
+
* not emitted into generated code — at runtime the application attaches the real
|
|
1922
|
+
* production database under the same alias.
|
|
1923
|
+
*/
|
|
1924
|
+
async function preparePostgresSources(sources, queries, reporter) {
|
|
1925
|
+
const containers = [];
|
|
1926
|
+
const attachments = [];
|
|
1927
|
+
const teardown = async () => {
|
|
1928
|
+
for (const container of containers) try {
|
|
1929
|
+
await container.stop();
|
|
1930
|
+
} catch {}
|
|
1931
|
+
};
|
|
1932
|
+
try {
|
|
1933
|
+
for (const source of sources) {
|
|
1934
|
+
if (source.url) {
|
|
1935
|
+
attachments.push({
|
|
1936
|
+
alias: source.name,
|
|
1937
|
+
connectionUri: source.url
|
|
1938
|
+
});
|
|
1939
|
+
continue;
|
|
1940
|
+
}
|
|
1941
|
+
reporter?.onContainerStarting?.();
|
|
1942
|
+
let container;
|
|
1943
|
+
try {
|
|
1944
|
+
container = await new PostgreSqlContainer(source.image).withDatabase("sqg-db").withUsername("sqg").withPassword("secret").start();
|
|
1945
|
+
} catch (e) {
|
|
1946
|
+
throw new DatabaseError(`Could not start a Postgres container for source '${source.name}': ${e.message}`, "postgres", "Postgres sources need Docker to introspect schema. Start Docker, or set 'url' on the source to point at an existing Postgres database (e.g. url: $DATABASE_URL).");
|
|
1947
|
+
}
|
|
1948
|
+
containers.push(container);
|
|
1949
|
+
const connectionUri = container.getConnectionUri();
|
|
1950
|
+
reporter?.onContainerStarted?.(connectionUri);
|
|
1951
|
+
const schemaBlocks = queries.filter((q) => q.isBaseline && q.sourceTarget === source.name);
|
|
1952
|
+
const client = new Client({ connectionString: connectionUri });
|
|
1953
|
+
await client.connect();
|
|
1954
|
+
try {
|
|
1955
|
+
for (const block of schemaBlocks) await client.query(block.rawQuery);
|
|
1956
|
+
} catch (e) {
|
|
1957
|
+
throw new DatabaseError(`Failed to apply schema for postgres source '${source.name}': ${e.message}`, "postgres", `Check that the BASELINE blocks tagged ':source=${source.name}' are valid PostgreSQL.`);
|
|
1958
|
+
} finally {
|
|
1959
|
+
await client.end();
|
|
1960
|
+
}
|
|
1961
|
+
attachments.push({
|
|
1962
|
+
alias: source.name,
|
|
1963
|
+
connectionUri
|
|
1964
|
+
});
|
|
1965
|
+
}
|
|
1966
|
+
} catch (e) {
|
|
1967
|
+
await teardown();
|
|
1968
|
+
throw e;
|
|
1969
|
+
}
|
|
1970
|
+
return {
|
|
1971
|
+
attachments,
|
|
1972
|
+
teardown
|
|
1973
|
+
};
|
|
1974
|
+
}
|
|
1975
|
+
//#endregion
|
|
1881
1976
|
//#region src/sqltool.ts
|
|
1882
1977
|
const GENERATED_FILE_COMMENT = "This file is generated by SQG (https://sqg.dev). Do not edit manually.";
|
|
1883
1978
|
const configSchema = z.object({ result: z.record(z.string(), z.string()).optional() });
|
|
@@ -2054,18 +2149,23 @@ var TableHelper = class {
|
|
|
2054
2149
|
return this.generator.typeMapper;
|
|
2055
2150
|
}
|
|
2056
2151
|
};
|
|
2057
|
-
function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config) {
|
|
2152
|
+
function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config, postgresSourceNames = []) {
|
|
2058
2153
|
const templateSrc = readFileSync(templatePath, "utf-8");
|
|
2059
2154
|
const template = Handlebars.compile(templateSrc);
|
|
2060
2155
|
Handlebars.registerHelper("mapType", (column) => generator.mapType(column));
|
|
2061
2156
|
Handlebars.registerHelper("plusOne", (value) => value + 1);
|
|
2062
2157
|
const migrations = queries.filter((q) => q.isMigrate).map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q)));
|
|
2063
2158
|
const tableHelpers = generator.supportsAppenders(engine) ? tables.filter((t) => !t.skipGenerateFunction).map((t) => new TableHelper(t, generator)) : [];
|
|
2159
|
+
const attachers = engine === "duckdb" ? postgresSourceNames.map((sourceName) => ({
|
|
2160
|
+
alias: sourceName,
|
|
2161
|
+
functionName: generator.getFunctionName(`attach_${sourceName}`)
|
|
2162
|
+
})) : [];
|
|
2064
2163
|
return template({
|
|
2065
2164
|
generatedComment: GENERATED_FILE_COMMENT,
|
|
2066
2165
|
migrations,
|
|
2067
2166
|
queries: queries.map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q))),
|
|
2068
2167
|
tables: tableHelpers,
|
|
2168
|
+
attachers,
|
|
2069
2169
|
className: generator.getClassName(name),
|
|
2070
2170
|
projectName,
|
|
2071
2171
|
config
|
|
@@ -2090,13 +2190,16 @@ const ProjectSchema = z.object({
|
|
|
2090
2190
|
template: z.string().optional().describe("Custom Handlebars template path"),
|
|
2091
2191
|
output: z.string().min(1, "Output path is required").describe("Output file or directory path"),
|
|
2092
2192
|
config: z.any().optional().describe("Generator-specific configuration")
|
|
2093
|
-
})).min(1, "At least one generator is required").describe("Code generators to run")
|
|
2094
|
-
})).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
|
|
2193
|
+
}).strict()).min(1, "At least one generator is required").describe("Code generators to run")
|
|
2194
|
+
}).strict()).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
|
|
2095
2195
|
sources: z.array(z.object({
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2196
|
+
type: z.enum(["file", "postgres"]).optional().describe("Source type: 'file' (default) or 'postgres'"),
|
|
2197
|
+
path: z.string().optional().describe("Path to source file for file sources (supports $HOME)"),
|
|
2198
|
+
name: z.string().optional().describe("Variable name (file sources) or DuckDB attach alias (postgres sources)"),
|
|
2199
|
+
image: z.string().optional().describe("Docker image for postgres sources (default: postgres:16-alpine)"),
|
|
2200
|
+
url: z.string().optional().describe("Postgres source: connect to this existing database for introspection instead of starting a container. Use a literal DSN or '$ENV_VAR' to read from the environment. When set, ':source=' BASELINE blocks are not applied (the live schema is used), which avoids schema drift.")
|
|
2201
|
+
}).strict().refine((s) => s.type === "postgres" ? !!s.name : !!s.path, { message: "file sources require 'path'; postgres sources require 'name'" })).optional().describe("External sources: files inlined as variables, or postgres databases attached for introspection")
|
|
2202
|
+
}).strict();
|
|
2100
2203
|
var ExtraVariable = class {
|
|
2101
2204
|
constructor(name, value) {
|
|
2102
2205
|
this.name = name;
|
|
@@ -2104,7 +2207,7 @@ var ExtraVariable = class {
|
|
|
2104
2207
|
}
|
|
2105
2208
|
};
|
|
2106
2209
|
function createExtraVariables(sources, suppressLogging = false) {
|
|
2107
|
-
return sources.map((source) => {
|
|
2210
|
+
return sources.filter((source) => source.type !== "postgres").map((source) => {
|
|
2108
2211
|
const path = source.path;
|
|
2109
2212
|
const resolvedPath = path.replace("$HOME", homedir());
|
|
2110
2213
|
const varName = `sources_${(source.name ?? basename(path, extname(resolvedPath))).replace(/\s+/g, "_")}`;
|
|
@@ -2112,6 +2215,27 @@ function createExtraVariables(sources, suppressLogging = false) {
|
|
|
2112
2215
|
return new ExtraVariable(varName, `'${resolvedPath}'`);
|
|
2113
2216
|
});
|
|
2114
2217
|
}
|
|
2218
|
+
/**
|
|
2219
|
+
* Resolve a postgres source `url`. A value of the form `$VAR` or `${VAR}` is
|
|
2220
|
+
* read from the environment (so credentials need not live in sqg.yaml); any
|
|
2221
|
+
* other value is treated as a literal DSN.
|
|
2222
|
+
*/
|
|
2223
|
+
function resolveSourceUrl(url, sourceName) {
|
|
2224
|
+
const match = url.match(/^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/);
|
|
2225
|
+
if (!match) return url;
|
|
2226
|
+
const envName = match[1];
|
|
2227
|
+
const value = process.env[envName];
|
|
2228
|
+
if (!value) throw new SqgError(`Postgres source '${sourceName}' references environment variable '${envName}' which is not set`, "CONFIG_VALIDATION_ERROR", `Set ${envName} to the database connection string, or use a literal DSN in the 'url' field`);
|
|
2229
|
+
return value;
|
|
2230
|
+
}
|
|
2231
|
+
/** Postgres sources resolved from the project config (with image defaulted). */
|
|
2232
|
+
function getPostgresSources(sources) {
|
|
2233
|
+
return sources.filter((source) => source.type === "postgres").map((source) => ({
|
|
2234
|
+
name: source.name,
|
|
2235
|
+
image: source.image ?? "postgres:16-alpine",
|
|
2236
|
+
url: source.url ? resolveSourceUrl(source.url, source.name) : void 0
|
|
2237
|
+
}));
|
|
2238
|
+
}
|
|
2115
2239
|
function buildProjectFromCliOptions(options) {
|
|
2116
2240
|
if (!isValidGenerator(options.generator)) {
|
|
2117
2241
|
const similar = findSimilarGenerators(options.generator);
|
|
@@ -2250,7 +2374,7 @@ function assignResultTypeNames(queries, tables, projectDir = process.cwd()) {
|
|
|
2250
2374
|
}
|
|
2251
2375
|
const fullMatch = tryFullTableMatch(group.queries[0], tablesByName);
|
|
2252
2376
|
if (fullMatch) {
|
|
2253
|
-
claim(fullMatch, shape, group.queries[0],
|
|
2377
|
+
claim(fullMatch, shape, group.queries[0], "full-table match");
|
|
2254
2378
|
for (const q of group.queries) q.resultTypeName = fullMatch;
|
|
2255
2379
|
continue;
|
|
2256
2380
|
}
|
|
@@ -2303,14 +2427,14 @@ function validateQueries(queries) {
|
|
|
2303
2427
|
};
|
|
2304
2428
|
}
|
|
2305
2429
|
}
|
|
2306
|
-
async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false) {
|
|
2430
|
+
async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false, postgresSourceNames = []) {
|
|
2307
2431
|
await generator.beforeGenerate(projectDir, gen, queries, tables);
|
|
2308
2432
|
const templatePath = join(dirname(new URL(import.meta.url).pathname), gen.template ?? generator.template);
|
|
2309
2433
|
const name = gen.name ?? basename(file, extname(file));
|
|
2310
2434
|
const sourceFile = generateSourceFile(name, queries, tables, templatePath, generator, engine, gen.projectName ?? name, {
|
|
2311
2435
|
migrations: true,
|
|
2312
2436
|
...gen.config
|
|
2313
|
-
});
|
|
2437
|
+
}, postgresSourceNames);
|
|
2314
2438
|
if (writeToStdout) {
|
|
2315
2439
|
process.stdout.write(sourceFile);
|
|
2316
2440
|
if (!sourceFile.endsWith("\n")) process.stdout.write("\n");
|
|
@@ -2411,6 +2535,11 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
2411
2535
|
const results = [];
|
|
2412
2536
|
try {
|
|
2413
2537
|
const extraVariables = createExtraVariables(project.sources ?? [], writeToStdout);
|
|
2538
|
+
const pgSources = getPostgresSources(project.sources ?? []);
|
|
2539
|
+
const pgSourceNames = new Set(pgSources.map((s) => s.name));
|
|
2540
|
+
if (pgSources.length > 0) {
|
|
2541
|
+
if (!project.sql.some((sql) => sql.gen.some((gen) => getGeneratorEngine(gen.generator) === "duckdb"))) throw new SqgError("Postgres sources require a DuckDB generator (they are attached into DuckDB for introspection)", "CONFIG_VALIDATION_ERROR", "Add a 'typescript/duckdb' or 'java/duckdb' generator, or remove the postgres sources");
|
|
2542
|
+
}
|
|
2414
2543
|
const files = [];
|
|
2415
2544
|
for (const sql of project.sql) {
|
|
2416
2545
|
const gensByEngine = /* @__PURE__ */ new Map();
|
|
@@ -2428,18 +2557,31 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
2428
2557
|
const parseResult = parseSQLQueries(fullPath, extraVariables);
|
|
2429
2558
|
queries = parseResult.queries;
|
|
2430
2559
|
tables = parseResult.tables;
|
|
2560
|
+
for (const q of queries) if (q.sourceTarget && !pgSourceNames.has(q.sourceTarget)) throw SqgError.inQuery(`BASELINE block targets unknown postgres source '${q.sourceTarget}'`, "CONFIG_VALIDATION_ERROR", q.id, sqlFile, { suggestion: `Declare a source with 'type: postgres, name: ${q.sourceTarget}' in sqg.yaml` });
|
|
2431
2561
|
} catch (e) {
|
|
2432
2562
|
if (e instanceof SqgError) throw e;
|
|
2433
2563
|
throw SqgError.inFile(`Failed to parse SQL file: ${e.message}`, "SQL_PARSE_ERROR", sqlFile, { suggestion: "Check SQL syntax and annotation format" });
|
|
2434
2564
|
}
|
|
2565
|
+
const referencedPgSources = pgSources.filter((source) => {
|
|
2566
|
+
const catalogRef = new RegExp(`\\b${escapeRegExp(source.name)}\\.`);
|
|
2567
|
+
return queries.some((q) => q.sourceTarget === source.name || catalogRef.test(q.rawQuery));
|
|
2568
|
+
});
|
|
2569
|
+
const referencedPgSourceNames = referencedPgSources.map((s) => s.name);
|
|
2435
2570
|
for (const [engine, gens] of gensByEngine) {
|
|
2436
2571
|
const executableQueries = queries.filter((q) => !q.skipGenerateFunction);
|
|
2437
2572
|
const reporterAny = reporter;
|
|
2438
2573
|
if (reporterAny?.setQueryTotal) reporterAny.setQueryTotal(executableQueries.length);
|
|
2574
|
+
let preparedSources;
|
|
2439
2575
|
try {
|
|
2440
2576
|
const dbEngine = getDatabaseEngine(engine);
|
|
2577
|
+
let attachments;
|
|
2578
|
+
if (engine === "duckdb" && referencedPgSources.length > 0) {
|
|
2579
|
+
ui?.startPhase("Starting postgres source containers...");
|
|
2580
|
+
preparedSources = await preparePostgresSources(referencedPgSources, queries, reporter);
|
|
2581
|
+
attachments = preparedSources.attachments;
|
|
2582
|
+
}
|
|
2441
2583
|
ui?.startPhase(`Initializing ${engine} database...`);
|
|
2442
|
-
await dbEngine.initializeDatabase(queries, reporter);
|
|
2584
|
+
await dbEngine.initializeDatabase(queries, reporter, { attachments });
|
|
2443
2585
|
ui?.startPhase(`Introspecting ${executableQueries.length} queries...`);
|
|
2444
2586
|
await dbEngine.executeQueries(queries, reporter);
|
|
2445
2587
|
if (tables.length > 0) await dbEngine.introspectTables(tables, reporter);
|
|
@@ -2456,6 +2598,8 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
2456
2598
|
file: sqlFile,
|
|
2457
2599
|
engine
|
|
2458
2600
|
});
|
|
2601
|
+
} finally {
|
|
2602
|
+
if (preparedSources) await preparedSources.teardown();
|
|
2459
2603
|
}
|
|
2460
2604
|
ui?.startPhase("Generating code...");
|
|
2461
2605
|
const genStart = performance.now();
|
|
@@ -2464,7 +2608,7 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
2464
2608
|
const outputPath = await writeGeneratedFile(projectDir, {
|
|
2465
2609
|
...gen,
|
|
2466
2610
|
projectName: project.name
|
|
2467
|
-
}, generator, sqlFile, queries, tables, engine, writeToStdout);
|
|
2611
|
+
}, generator, sqlFile, queries, tables, engine, writeToStdout, referencedPgSourceNames);
|
|
2468
2612
|
if (outputPath !== null) {
|
|
2469
2613
|
files.push(outputPath);
|
|
2470
2614
|
results.push({
|
|
@@ -2537,6 +2681,13 @@ var SQLQuery = class {
|
|
|
2537
2681
|
parameterTypes;
|
|
2538
2682
|
/** Explicit row-type name from `:result=Name` modifier, when present. */
|
|
2539
2683
|
resultTypeOverride;
|
|
2684
|
+
/**
|
|
2685
|
+
* Name of the postgres `source` this block targets, from a `:source=name`
|
|
2686
|
+
* modifier on a BASELINE block. Such blocks define the schema of an external
|
|
2687
|
+
* postgres source and run natively against that source's container during
|
|
2688
|
+
* generation (not against the DuckDB introspection connection).
|
|
2689
|
+
*/
|
|
2690
|
+
sourceTarget;
|
|
2540
2691
|
/** Final row-type identifier assigned by assignResultTypeNames (after introspection). */
|
|
2541
2692
|
resultTypeName;
|
|
2542
2693
|
constructor(filename, id, rawQuery, queryAnonymous, queryNamed, queryPositional, type, isOne, isPluck, isBatch, variables, config, line) {
|
|
@@ -2653,6 +2804,8 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
2653
2804
|
const isPluck = modifiers.includes(":pluck");
|
|
2654
2805
|
const isBatch = modifiers.includes(":batch");
|
|
2655
2806
|
const resultTypeOverride = modifiers.find((m) => m.startsWith(":result="))?.slice(8);
|
|
2807
|
+
const sourceTarget = modifiers.find((m) => m.startsWith(":source="))?.slice(8);
|
|
2808
|
+
if (sourceTarget && queryType !== "BASELINE") throw SqgError.inQuery(`The ':source=' modifier is only valid on BASELINE blocks, not ${queryType}`, "SQL_PARSE_ERROR", name, filePath, { suggestion: "Move the schema for a postgres source into a '-- BASELINE <name> :source=<source>' block" });
|
|
2656
2809
|
let configStr = getStr("Config", true);
|
|
2657
2810
|
if (configStr?.endsWith("*/")) configStr = configStr.slice(0, -2);
|
|
2658
2811
|
let config = null;
|
|
@@ -2692,8 +2845,12 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
2692
2845
|
});
|
|
2693
2846
|
}
|
|
2694
2847
|
trim() {
|
|
2695
|
-
const
|
|
2696
|
-
|
|
2848
|
+
const isBlankString = (p) => typeof p === "string" && p.trim() === "";
|
|
2849
|
+
while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[0])) this.sqlParts.shift();
|
|
2850
|
+
if (this.sqlParts.length > 0 && typeof this.sqlParts[0] === "string") this.sqlParts[0] = this.sqlParts[0].trimStart();
|
|
2851
|
+
while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[this.sqlParts.length - 1])) this.sqlParts.pop();
|
|
2852
|
+
const lastIdx = this.sqlParts.length - 1;
|
|
2853
|
+
if (lastIdx >= 0 && typeof this.sqlParts[lastIdx] === "string") this.sqlParts[lastIdx] = this.sqlParts[lastIdx].trimEnd();
|
|
2697
2854
|
}
|
|
2698
2855
|
parameters() {
|
|
2699
2856
|
return this.sqlParts.filter((part) => typeof part !== "string" && !part.name.startsWith("sources_"));
|
|
@@ -2808,6 +2965,7 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
2808
2965
|
});
|
|
2809
2966
|
const query = new SQLQuery(filePath, name, sqlContentStr, sql.toSqlWithAnonymousPlaceholders(), sql.toSqlWithNamedPlaceholders(), sql.toSqlWithPositionalPlaceholders(), queryType, isOne, isPluck, isBatch, variables, config, annotationLine);
|
|
2810
2967
|
query.resultTypeOverride = resultTypeOverride;
|
|
2968
|
+
query.sourceTarget = sourceTarget;
|
|
2811
2969
|
checkDuplicate(queryType, name);
|
|
2812
2970
|
queries.push(query);
|
|
2813
2971
|
consola.debug(`Added query: ${name} (${queryType})`);
|
|
@@ -2824,7 +2982,7 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
2824
2982
|
//#endregion
|
|
2825
2983
|
//#region src/db/types.ts
|
|
2826
2984
|
async function initializeDatabase(queries, execQueries, reporter) {
|
|
2827
|
-
const baselineQueries = queries.filter((q) => q.isBaseline);
|
|
2985
|
+
const baselineQueries = queries.filter((q) => q.isBaseline && !q.sourceTarget);
|
|
2828
2986
|
for (const query of baselineQueries) try {
|
|
2829
2987
|
await execQueries(query);
|
|
2830
2988
|
} catch (error) {
|
|
@@ -2880,9 +3038,17 @@ function convertType(type) {
|
|
|
2880
3038
|
const duckdb = new class {
|
|
2881
3039
|
db;
|
|
2882
3040
|
connection;
|
|
2883
|
-
async initializeDatabase(queries, reporter) {
|
|
3041
|
+
async initializeDatabase(queries, reporter, options) {
|
|
2884
3042
|
this.db = await DuckDBInstance.create(":memory:");
|
|
2885
3043
|
this.connection = await this.db.connect();
|
|
3044
|
+
for (const attachment of options?.attachments ?? []) {
|
|
3045
|
+
const sql = `ATTACH '${attachment.connectionUri}' AS ${attachment.alias} (TYPE postgres)`;
|
|
3046
|
+
try {
|
|
3047
|
+
await this.connection.run(sql);
|
|
3048
|
+
} catch (e) {
|
|
3049
|
+
throw new DatabaseError(`Failed to attach source '${attachment.alias}': ${e.message}`, "duckdb", "Check that the postgres source container started and is reachable.");
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
2886
3052
|
await initializeDatabase(queries, async (query) => {
|
|
2887
3053
|
try {
|
|
2888
3054
|
await this.connection.run(query.rawQuery);
|
package/dist/sqg.mjs
CHANGED
|
@@ -13,7 +13,7 @@ import { Server } from "@modelcontextprotocol/sdk/server/index.js";
|
|
|
13
13
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
14
14
|
import { CallToolRequestSchema, ListResourcesRequestSchema, ListToolsRequestSchema, ReadResourceRequestSchema } from "@modelcontextprotocol/sdk/types.js";
|
|
15
15
|
import YAML from "yaml";
|
|
16
|
-
import { camelCase, pascalCase, snakeCase } from "es-toolkit/string";
|
|
16
|
+
import { camelCase, escapeRegExp, pascalCase, snakeCase } from "es-toolkit/string";
|
|
17
17
|
import Handlebars from "handlebars";
|
|
18
18
|
import { z } from "zod";
|
|
19
19
|
import { DuckDBEnumType, DuckDBInstance, DuckDBListType, DuckDBMapType, DuckDBStructType } from "@duckdb/node-api";
|
|
@@ -221,6 +221,7 @@ SQL Annotation Syntax:
|
|
|
221
221
|
-- QUERY <name> [:one] [:pluck] Select query (returns rows)
|
|
222
222
|
-- EXEC <name> [:batch] Execute statement (INSERT/UPDATE/DELETE)
|
|
223
223
|
-- BASELINE <name> Schema created outside SQG (runs before migrations, not tracked)
|
|
224
|
+
-- BASELINE <name> :source=<src> Schema of a 'type: postgres' source (applied to a managed testcontainer, attached into DuckDB)
|
|
224
225
|
-- MIGRATE <name> Schema migration (runs in source order; name is any identifier, e.g. "1" or "add_email")
|
|
225
226
|
-- TESTDATA <name> Test data setup (not generated)
|
|
226
227
|
-- TABLE <name> :appender Table for bulk insert appender (DuckDB, PostgreSQL)
|
|
@@ -234,6 +235,7 @@ Modifiers:
|
|
|
234
235
|
:all Return all rows (default)
|
|
235
236
|
:batch Generate a JDBC batch method for an EXEC (Java only)
|
|
236
237
|
:appender Generate bulk insert appender for TABLE annotation
|
|
238
|
+
:source=Name Mark a BASELINE block as the schema of a postgres source (Name)
|
|
237
239
|
:result=Name Name (and share) the row type (Java only). Annotate ONE query
|
|
238
240
|
with :result=Name — every other query in the same file with the
|
|
239
241
|
same column shape automatically gets the same name; you do NOT
|
|
@@ -783,6 +785,13 @@ var SQLQuery = class {
|
|
|
783
785
|
parameterTypes;
|
|
784
786
|
/** Explicit row-type name from `:result=Name` modifier, when present. */
|
|
785
787
|
resultTypeOverride;
|
|
788
|
+
/**
|
|
789
|
+
* Name of the postgres `source` this block targets, from a `:source=name`
|
|
790
|
+
* modifier on a BASELINE block. Such blocks define the schema of an external
|
|
791
|
+
* postgres source and run natively against that source's container during
|
|
792
|
+
* generation (not against the DuckDB introspection connection).
|
|
793
|
+
*/
|
|
794
|
+
sourceTarget;
|
|
786
795
|
/** Final row-type identifier assigned by assignResultTypeNames (after introspection). */
|
|
787
796
|
resultTypeName;
|
|
788
797
|
constructor(filename, id, rawQuery, queryAnonymous, queryNamed, queryPositional, type, isOne, isPluck, isBatch, variables, config, line) {
|
|
@@ -899,6 +908,8 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
899
908
|
const isPluck = modifiers.includes(":pluck");
|
|
900
909
|
const isBatch = modifiers.includes(":batch");
|
|
901
910
|
const resultTypeOverride = modifiers.find((m) => m.startsWith(":result="))?.slice(8);
|
|
911
|
+
const sourceTarget = modifiers.find((m) => m.startsWith(":source="))?.slice(8);
|
|
912
|
+
if (sourceTarget && queryType !== "BASELINE") throw SqgError.inQuery(`The ':source=' modifier is only valid on BASELINE blocks, not ${queryType}`, "SQL_PARSE_ERROR", name, filePath, { suggestion: "Move the schema for a postgres source into a '-- BASELINE <name> :source=<source>' block" });
|
|
902
913
|
let configStr = getStr("Config", true);
|
|
903
914
|
if (configStr?.endsWith("*/")) configStr = configStr.slice(0, -2);
|
|
904
915
|
let config = null;
|
|
@@ -938,8 +949,12 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
938
949
|
});
|
|
939
950
|
}
|
|
940
951
|
trim() {
|
|
941
|
-
const
|
|
942
|
-
|
|
952
|
+
const isBlankString = (p) => typeof p === "string" && p.trim() === "";
|
|
953
|
+
while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[0])) this.sqlParts.shift();
|
|
954
|
+
if (this.sqlParts.length > 0 && typeof this.sqlParts[0] === "string") this.sqlParts[0] = this.sqlParts[0].trimStart();
|
|
955
|
+
while (this.sqlParts.length > 0 && isBlankString(this.sqlParts[this.sqlParts.length - 1])) this.sqlParts.pop();
|
|
956
|
+
const lastIdx = this.sqlParts.length - 1;
|
|
957
|
+
if (lastIdx >= 0 && typeof this.sqlParts[lastIdx] === "string") this.sqlParts[lastIdx] = this.sqlParts[lastIdx].trimEnd();
|
|
943
958
|
}
|
|
944
959
|
parameters() {
|
|
945
960
|
return this.sqlParts.filter((part) => typeof part !== "string" && !part.name.startsWith("sources_"));
|
|
@@ -1054,6 +1069,7 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
1054
1069
|
});
|
|
1055
1070
|
const query = new SQLQuery(filePath, name, sqlContentStr, sql.toSqlWithAnonymousPlaceholders(), sql.toSqlWithNamedPlaceholders(), sql.toSqlWithPositionalPlaceholders(), queryType, isOne, isPluck, isBatch, variables, config, annotationLine);
|
|
1056
1071
|
query.resultTypeOverride = resultTypeOverride;
|
|
1072
|
+
query.sourceTarget = sourceTarget;
|
|
1057
1073
|
checkDuplicate(queryType, name);
|
|
1058
1074
|
queries.push(query);
|
|
1059
1075
|
consola.debug(`Added query: ${name} (${queryType})`);
|
|
@@ -1070,7 +1086,7 @@ function parseSQLQueries(filePath, extraVariables) {
|
|
|
1070
1086
|
//#endregion
|
|
1071
1087
|
//#region src/db/types.ts
|
|
1072
1088
|
async function initializeDatabase(queries, execQueries, reporter) {
|
|
1073
|
-
const baselineQueries = queries.filter((q) => q.isBaseline);
|
|
1089
|
+
const baselineQueries = queries.filter((q) => q.isBaseline && !q.sourceTarget);
|
|
1074
1090
|
for (const query of baselineQueries) try {
|
|
1075
1091
|
await execQueries(query);
|
|
1076
1092
|
} catch (error) {
|
|
@@ -1126,9 +1142,17 @@ function convertType(type) {
|
|
|
1126
1142
|
const duckdb = new class {
|
|
1127
1143
|
db;
|
|
1128
1144
|
connection;
|
|
1129
|
-
async initializeDatabase(queries, reporter) {
|
|
1145
|
+
async initializeDatabase(queries, reporter, options) {
|
|
1130
1146
|
this.db = await DuckDBInstance.create(":memory:");
|
|
1131
1147
|
this.connection = await this.db.connect();
|
|
1148
|
+
for (const attachment of options?.attachments ?? []) {
|
|
1149
|
+
const sql = `ATTACH '${attachment.connectionUri}' AS ${attachment.alias} (TYPE postgres)`;
|
|
1150
|
+
try {
|
|
1151
|
+
await this.connection.run(sql);
|
|
1152
|
+
} catch (e) {
|
|
1153
|
+
throw new DatabaseError(`Failed to attach source '${attachment.alias}': ${e.message}`, "duckdb", "Check that the postgres source container started and is reachable.");
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1132
1156
|
await initializeDatabase(queries, async (query) => {
|
|
1133
1157
|
try {
|
|
1134
1158
|
await this.connection.run(query.rawQuery);
|
|
@@ -2726,6 +2750,19 @@ var PythonGenerator = class extends BaseGenerator {
|
|
|
2726
2750
|
if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
|
|
2727
2751
|
return `"${value}"`;
|
|
2728
2752
|
});
|
|
2753
|
+
Handlebars.registerHelper("sqlExpr", (queryHelper) => {
|
|
2754
|
+
const parts = queryHelper.sqlQueryParts;
|
|
2755
|
+
if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) {
|
|
2756
|
+
const value = queryHelper.sqlQuery;
|
|
2757
|
+
if (value.includes("\n") || value.includes("'") || value.includes("\"")) return `"""\\\n${value}"""`;
|
|
2758
|
+
return `"${value}"`;
|
|
2759
|
+
}
|
|
2760
|
+
let out = "f\"\"\"";
|
|
2761
|
+
for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/\{/g, "{{").replace(/\}/g, "}}");
|
|
2762
|
+
else if (part.name.startsWith("sources_")) out += ` '{${part.name}}'`;
|
|
2763
|
+
else out += "?";
|
|
2764
|
+
return `${out}"""`;
|
|
2765
|
+
});
|
|
2729
2766
|
Handlebars.registerHelper("pyType", (column) => {
|
|
2730
2767
|
return this.getPyType(column);
|
|
2731
2768
|
});
|
|
@@ -2850,6 +2887,7 @@ var TsGenerator = class extends BaseGenerator {
|
|
|
2850
2887
|
async beforeGenerate(_projectDir, gen, _queries, _tables) {
|
|
2851
2888
|
if (this.typeMapper instanceof TypeScriptTypeMapper) this.typeMapper.safeIntegers = !!gen.config?.safeIntegers;
|
|
2852
2889
|
Handlebars.registerHelper("quote", (value) => this.quote(value));
|
|
2890
|
+
Handlebars.registerHelper("sqlExpr", (queryHelper) => this.sqlExpr(queryHelper));
|
|
2853
2891
|
Handlebars.registerHelper("appendMethod", (column) => {
|
|
2854
2892
|
if (column.type instanceof ListType) return "List";
|
|
2855
2893
|
const typeStr = column.type?.toString().toUpperCase() || "";
|
|
@@ -3002,6 +3040,22 @@ var TsGenerator = class extends BaseGenerator {
|
|
|
3002
3040
|
quote(value) {
|
|
3003
3041
|
return value.includes("\n") || value.includes("'") ? `\`${value}\`` : `'${value}'`;
|
|
3004
3042
|
}
|
|
3043
|
+
/**
|
|
3044
|
+
* Build the SQL expression for a query. Without file `sources`, this is the
|
|
3045
|
+
* plain quoted SQL string (unchanged). When a `${sources_x}` reference is
|
|
3046
|
+
* present it is interpolated into a template literal as a runtime argument
|
|
3047
|
+
* (the value is supplied by the caller, not bound), mirroring the Java
|
|
3048
|
+
* generator — `ATTACH '` + arg + `'` style.
|
|
3049
|
+
*/
|
|
3050
|
+
sqlExpr(queryHelper) {
|
|
3051
|
+
const parts = queryHelper.sqlQueryParts;
|
|
3052
|
+
if (!parts.some((p) => typeof p !== "string" && p.name.startsWith("sources_"))) return this.quote(queryHelper.sqlQuery);
|
|
3053
|
+
let out = "`";
|
|
3054
|
+
for (const part of parts) if (typeof part === "string") out += part.replace(/\\/g, "\\\\").replace(/`/g, "\\`").replace(/\$\{/g, "\\${");
|
|
3055
|
+
else if (part.name.startsWith("sources_")) out += ` '\${${part.name}}'`;
|
|
3056
|
+
else out += "?";
|
|
3057
|
+
return `${out}\``;
|
|
3058
|
+
}
|
|
3005
3059
|
};
|
|
3006
3060
|
//#endregion
|
|
3007
3061
|
//#region src/generators/typescript-duckdb-generator.ts
|
|
@@ -3108,6 +3162,69 @@ function getGenerator(generator) {
|
|
|
3108
3162
|
}
|
|
3109
3163
|
}
|
|
3110
3164
|
//#endregion
|
|
3165
|
+
//#region src/sources.ts
|
|
3166
|
+
/**
|
|
3167
|
+
* For each `type: postgres` source: start a throwaway Postgres testcontainer,
|
|
3168
|
+
* run the source's schema (the BASELINE blocks tagged `:source=<name>`) natively
|
|
3169
|
+
* against it so introspection sees true Postgres types, and return the connection
|
|
3170
|
+
* info to ATTACH into DuckDB.
|
|
3171
|
+
*
|
|
3172
|
+
* The synthesized ATTACH (done by the DuckDB adapter) and these schema blocks are
|
|
3173
|
+
* not emitted into generated code — at runtime the application attaches the real
|
|
3174
|
+
* production database under the same alias.
|
|
3175
|
+
*/
|
|
3176
|
+
async function preparePostgresSources(sources, queries, reporter) {
|
|
3177
|
+
const containers = [];
|
|
3178
|
+
const attachments = [];
|
|
3179
|
+
const teardown = async () => {
|
|
3180
|
+
for (const container of containers) try {
|
|
3181
|
+
await container.stop();
|
|
3182
|
+
} catch {}
|
|
3183
|
+
};
|
|
3184
|
+
try {
|
|
3185
|
+
for (const source of sources) {
|
|
3186
|
+
if (source.url) {
|
|
3187
|
+
attachments.push({
|
|
3188
|
+
alias: source.name,
|
|
3189
|
+
connectionUri: source.url
|
|
3190
|
+
});
|
|
3191
|
+
continue;
|
|
3192
|
+
}
|
|
3193
|
+
reporter?.onContainerStarting?.();
|
|
3194
|
+
let container;
|
|
3195
|
+
try {
|
|
3196
|
+
container = await new PostgreSqlContainer(source.image).withDatabase("sqg-db").withUsername("sqg").withPassword("secret").start();
|
|
3197
|
+
} catch (e) {
|
|
3198
|
+
throw new DatabaseError(`Could not start a Postgres container for source '${source.name}': ${e.message}`, "postgres", "Postgres sources need Docker to introspect schema. Start Docker, or set 'url' on the source to point at an existing Postgres database (e.g. url: $DATABASE_URL).");
|
|
3199
|
+
}
|
|
3200
|
+
containers.push(container);
|
|
3201
|
+
const connectionUri = container.getConnectionUri();
|
|
3202
|
+
reporter?.onContainerStarted?.(connectionUri);
|
|
3203
|
+
const schemaBlocks = queries.filter((q) => q.isBaseline && q.sourceTarget === source.name);
|
|
3204
|
+
const client = new Client({ connectionString: connectionUri });
|
|
3205
|
+
await client.connect();
|
|
3206
|
+
try {
|
|
3207
|
+
for (const block of schemaBlocks) await client.query(block.rawQuery);
|
|
3208
|
+
} catch (e) {
|
|
3209
|
+
throw new DatabaseError(`Failed to apply schema for postgres source '${source.name}': ${e.message}`, "postgres", `Check that the BASELINE blocks tagged ':source=${source.name}' are valid PostgreSQL.`);
|
|
3210
|
+
} finally {
|
|
3211
|
+
await client.end();
|
|
3212
|
+
}
|
|
3213
|
+
attachments.push({
|
|
3214
|
+
alias: source.name,
|
|
3215
|
+
connectionUri
|
|
3216
|
+
});
|
|
3217
|
+
}
|
|
3218
|
+
} catch (e) {
|
|
3219
|
+
await teardown();
|
|
3220
|
+
throw e;
|
|
3221
|
+
}
|
|
3222
|
+
return {
|
|
3223
|
+
attachments,
|
|
3224
|
+
teardown
|
|
3225
|
+
};
|
|
3226
|
+
}
|
|
3227
|
+
//#endregion
|
|
3111
3228
|
//#region src/sqltool.ts
|
|
3112
3229
|
const GENERATED_FILE_COMMENT = "This file is generated by SQG (https://sqg.dev). Do not edit manually.";
|
|
3113
3230
|
const configSchema = z.object({ result: z.record(z.string(), z.string()).optional() });
|
|
@@ -3284,18 +3401,23 @@ var TableHelper = class {
|
|
|
3284
3401
|
return this.generator.typeMapper;
|
|
3285
3402
|
}
|
|
3286
3403
|
};
|
|
3287
|
-
function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config) {
|
|
3404
|
+
function generateSourceFile(name, queries, tables, templatePath, generator, engine, projectName, config, postgresSourceNames = []) {
|
|
3288
3405
|
const templateSrc = readFileSync(templatePath, "utf-8");
|
|
3289
3406
|
const template = Handlebars.compile(templateSrc);
|
|
3290
3407
|
Handlebars.registerHelper("mapType", (column) => generator.mapType(column));
|
|
3291
3408
|
Handlebars.registerHelper("plusOne", (value) => value + 1);
|
|
3292
3409
|
const migrations = queries.filter((q) => q.isMigrate).map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q)));
|
|
3293
3410
|
const tableHelpers = generator.supportsAppenders(engine) ? tables.filter((t) => !t.skipGenerateFunction).map((t) => new TableHelper(t, generator)) : [];
|
|
3411
|
+
const attachers = engine === "duckdb" ? postgresSourceNames.map((sourceName) => ({
|
|
3412
|
+
alias: sourceName,
|
|
3413
|
+
functionName: generator.getFunctionName(`attach_${sourceName}`)
|
|
3414
|
+
})) : [];
|
|
3294
3415
|
return template({
|
|
3295
3416
|
generatedComment: GENERATED_FILE_COMMENT,
|
|
3296
3417
|
migrations,
|
|
3297
3418
|
queries: queries.map((q) => new SqlQueryHelper(q, generator, generator.getStatement(q))),
|
|
3298
3419
|
tables: tableHelpers,
|
|
3420
|
+
attachers,
|
|
3299
3421
|
className: generator.getClassName(name),
|
|
3300
3422
|
projectName,
|
|
3301
3423
|
config
|
|
@@ -3320,13 +3442,16 @@ const ProjectSchema = z.object({
|
|
|
3320
3442
|
template: z.string().optional().describe("Custom Handlebars template path"),
|
|
3321
3443
|
output: z.string().min(1, "Output path is required").describe("Output file or directory path"),
|
|
3322
3444
|
config: z.any().optional().describe("Generator-specific configuration")
|
|
3323
|
-
})).min(1, "At least one generator is required").describe("Code generators to run")
|
|
3324
|
-
})).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
|
|
3445
|
+
}).strict()).min(1, "At least one generator is required").describe("Code generators to run")
|
|
3446
|
+
}).strict()).min(1, "At least one SQL configuration is required").describe("SQL file configurations"),
|
|
3325
3447
|
sources: z.array(z.object({
|
|
3326
|
-
|
|
3327
|
-
|
|
3328
|
-
|
|
3329
|
-
|
|
3448
|
+
type: z.enum(["file", "postgres"]).optional().describe("Source type: 'file' (default) or 'postgres'"),
|
|
3449
|
+
path: z.string().optional().describe("Path to source file for file sources (supports $HOME)"),
|
|
3450
|
+
name: z.string().optional().describe("Variable name (file sources) or DuckDB attach alias (postgres sources)"),
|
|
3451
|
+
image: z.string().optional().describe("Docker image for postgres sources (default: postgres:16-alpine)"),
|
|
3452
|
+
url: z.string().optional().describe("Postgres source: connect to this existing database for introspection instead of starting a container. Use a literal DSN or '$ENV_VAR' to read from the environment. When set, ':source=' BASELINE blocks are not applied (the live schema is used), which avoids schema drift.")
|
|
3453
|
+
}).strict().refine((s) => s.type === "postgres" ? !!s.name : !!s.path, { message: "file sources require 'path'; postgres sources require 'name'" })).optional().describe("External sources: files inlined as variables, or postgres databases attached for introspection")
|
|
3454
|
+
}).strict();
|
|
3330
3455
|
var ExtraVariable = class {
|
|
3331
3456
|
constructor(name, value) {
|
|
3332
3457
|
this.name = name;
|
|
@@ -3334,7 +3459,7 @@ var ExtraVariable = class {
|
|
|
3334
3459
|
}
|
|
3335
3460
|
};
|
|
3336
3461
|
function createExtraVariables(sources, suppressLogging = false) {
|
|
3337
|
-
return sources.map((source) => {
|
|
3462
|
+
return sources.filter((source) => source.type !== "postgres").map((source) => {
|
|
3338
3463
|
const path = source.path;
|
|
3339
3464
|
const resolvedPath = path.replace("$HOME", homedir());
|
|
3340
3465
|
const varName = `sources_${(source.name ?? basename(path, extname(resolvedPath))).replace(/\s+/g, "_")}`;
|
|
@@ -3342,6 +3467,27 @@ function createExtraVariables(sources, suppressLogging = false) {
|
|
|
3342
3467
|
return new ExtraVariable(varName, `'${resolvedPath}'`);
|
|
3343
3468
|
});
|
|
3344
3469
|
}
|
|
3470
|
+
/**
|
|
3471
|
+
* Resolve a postgres source `url`. A value of the form `$VAR` or `${VAR}` is
|
|
3472
|
+
* read from the environment (so credentials need not live in sqg.yaml); any
|
|
3473
|
+
* other value is treated as a literal DSN.
|
|
3474
|
+
*/
|
|
3475
|
+
function resolveSourceUrl(url, sourceName) {
|
|
3476
|
+
const match = url.match(/^\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?$/);
|
|
3477
|
+
if (!match) return url;
|
|
3478
|
+
const envName = match[1];
|
|
3479
|
+
const value = process.env[envName];
|
|
3480
|
+
if (!value) throw new SqgError(`Postgres source '${sourceName}' references environment variable '${envName}' which is not set`, "CONFIG_VALIDATION_ERROR", `Set ${envName} to the database connection string, or use a literal DSN in the 'url' field`);
|
|
3481
|
+
return value;
|
|
3482
|
+
}
|
|
3483
|
+
/** Postgres sources resolved from the project config (with image defaulted). */
|
|
3484
|
+
function getPostgresSources(sources) {
|
|
3485
|
+
return sources.filter((source) => source.type === "postgres").map((source) => ({
|
|
3486
|
+
name: source.name,
|
|
3487
|
+
image: source.image ?? "postgres:16-alpine",
|
|
3488
|
+
url: source.url ? resolveSourceUrl(source.url, source.name) : void 0
|
|
3489
|
+
}));
|
|
3490
|
+
}
|
|
3345
3491
|
function buildProjectFromCliOptions(options) {
|
|
3346
3492
|
if (!isValidGenerator(options.generator)) {
|
|
3347
3493
|
const similar = findSimilarGenerators(options.generator);
|
|
@@ -3480,7 +3626,7 @@ function assignResultTypeNames(queries, tables, projectDir = process.cwd()) {
|
|
|
3480
3626
|
}
|
|
3481
3627
|
const fullMatch = tryFullTableMatch(group.queries[0], tablesByName);
|
|
3482
3628
|
if (fullMatch) {
|
|
3483
|
-
claim(fullMatch, shape, group.queries[0],
|
|
3629
|
+
claim(fullMatch, shape, group.queries[0], "full-table match");
|
|
3484
3630
|
for (const q of group.queries) q.resultTypeName = fullMatch;
|
|
3485
3631
|
continue;
|
|
3486
3632
|
}
|
|
@@ -3533,14 +3679,14 @@ function validateQueries(queries) {
|
|
|
3533
3679
|
};
|
|
3534
3680
|
}
|
|
3535
3681
|
}
|
|
3536
|
-
async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false) {
|
|
3682
|
+
async function writeGeneratedFile(projectDir, gen, generator, file, queries, tables, engine, writeToStdout = false, postgresSourceNames = []) {
|
|
3537
3683
|
await generator.beforeGenerate(projectDir, gen, queries, tables);
|
|
3538
3684
|
const templatePath = join(dirname(new URL(import.meta.url).pathname), gen.template ?? generator.template);
|
|
3539
3685
|
const name = gen.name ?? basename(file, extname(file));
|
|
3540
3686
|
const sourceFile = generateSourceFile(name, queries, tables, templatePath, generator, engine, gen.projectName ?? name, {
|
|
3541
3687
|
migrations: true,
|
|
3542
3688
|
...gen.config
|
|
3543
|
-
});
|
|
3689
|
+
}, postgresSourceNames);
|
|
3544
3690
|
if (writeToStdout) {
|
|
3545
3691
|
process.stdout.write(sourceFile);
|
|
3546
3692
|
if (!sourceFile.endsWith("\n")) process.stdout.write("\n");
|
|
@@ -3641,6 +3787,11 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
3641
3787
|
const results = [];
|
|
3642
3788
|
try {
|
|
3643
3789
|
const extraVariables = createExtraVariables(project.sources ?? [], writeToStdout);
|
|
3790
|
+
const pgSources = getPostgresSources(project.sources ?? []);
|
|
3791
|
+
const pgSourceNames = new Set(pgSources.map((s) => s.name));
|
|
3792
|
+
if (pgSources.length > 0) {
|
|
3793
|
+
if (!project.sql.some((sql) => sql.gen.some((gen) => getGeneratorEngine(gen.generator) === "duckdb"))) throw new SqgError("Postgres sources require a DuckDB generator (they are attached into DuckDB for introspection)", "CONFIG_VALIDATION_ERROR", "Add a 'typescript/duckdb' or 'java/duckdb' generator, or remove the postgres sources");
|
|
3794
|
+
}
|
|
3644
3795
|
const files = [];
|
|
3645
3796
|
for (const sql of project.sql) {
|
|
3646
3797
|
const gensByEngine = /* @__PURE__ */ new Map();
|
|
@@ -3658,18 +3809,31 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
3658
3809
|
const parseResult = parseSQLQueries(fullPath, extraVariables);
|
|
3659
3810
|
queries = parseResult.queries;
|
|
3660
3811
|
tables = parseResult.tables;
|
|
3812
|
+
for (const q of queries) if (q.sourceTarget && !pgSourceNames.has(q.sourceTarget)) throw SqgError.inQuery(`BASELINE block targets unknown postgres source '${q.sourceTarget}'`, "CONFIG_VALIDATION_ERROR", q.id, sqlFile, { suggestion: `Declare a source with 'type: postgres, name: ${q.sourceTarget}' in sqg.yaml` });
|
|
3661
3813
|
} catch (e) {
|
|
3662
3814
|
if (e instanceof SqgError) throw e;
|
|
3663
3815
|
throw SqgError.inFile(`Failed to parse SQL file: ${e.message}`, "SQL_PARSE_ERROR", sqlFile, { suggestion: "Check SQL syntax and annotation format" });
|
|
3664
3816
|
}
|
|
3817
|
+
const referencedPgSources = pgSources.filter((source) => {
|
|
3818
|
+
const catalogRef = new RegExp(`\\b${escapeRegExp(source.name)}\\.`);
|
|
3819
|
+
return queries.some((q) => q.sourceTarget === source.name || catalogRef.test(q.rawQuery));
|
|
3820
|
+
});
|
|
3821
|
+
const referencedPgSourceNames = referencedPgSources.map((s) => s.name);
|
|
3665
3822
|
for (const [engine, gens] of gensByEngine) {
|
|
3666
3823
|
const executableQueries = queries.filter((q) => !q.skipGenerateFunction);
|
|
3667
3824
|
const reporterAny = reporter;
|
|
3668
3825
|
if (reporterAny?.setQueryTotal) reporterAny.setQueryTotal(executableQueries.length);
|
|
3826
|
+
let preparedSources;
|
|
3669
3827
|
try {
|
|
3670
3828
|
const dbEngine = getDatabaseEngine(engine);
|
|
3829
|
+
let attachments;
|
|
3830
|
+
if (engine === "duckdb" && referencedPgSources.length > 0) {
|
|
3831
|
+
ui?.startPhase("Starting postgres source containers...");
|
|
3832
|
+
preparedSources = await preparePostgresSources(referencedPgSources, queries, reporter);
|
|
3833
|
+
attachments = preparedSources.attachments;
|
|
3834
|
+
}
|
|
3671
3835
|
ui?.startPhase(`Initializing ${engine} database...`);
|
|
3672
|
-
await dbEngine.initializeDatabase(queries, reporter);
|
|
3836
|
+
await dbEngine.initializeDatabase(queries, reporter, { attachments });
|
|
3673
3837
|
ui?.startPhase(`Introspecting ${executableQueries.length} queries...`);
|
|
3674
3838
|
await dbEngine.executeQueries(queries, reporter);
|
|
3675
3839
|
if (tables.length > 0) await dbEngine.introspectTables(tables, reporter);
|
|
@@ -3686,6 +3850,8 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
3686
3850
|
file: sqlFile,
|
|
3687
3851
|
engine
|
|
3688
3852
|
});
|
|
3853
|
+
} finally {
|
|
3854
|
+
if (preparedSources) await preparedSources.teardown();
|
|
3689
3855
|
}
|
|
3690
3856
|
ui?.startPhase("Generating code...");
|
|
3691
3857
|
const genStart = performance.now();
|
|
@@ -3694,7 +3860,7 @@ async function processProjectFromConfig(project, projectDir, writeToStdout = fal
|
|
|
3694
3860
|
const outputPath = await writeGeneratedFile(projectDir, {
|
|
3695
3861
|
...gen,
|
|
3696
3862
|
projectName: project.name
|
|
3697
|
-
}, generator, sqlFile, queries, tables, engine, writeToStdout);
|
|
3863
|
+
}, generator, sqlFile, queries, tables, engine, writeToStdout, referencedPgSourceNames);
|
|
3698
3864
|
if (outputPath !== null) {
|
|
3699
3865
|
files.push(outputPath);
|
|
3700
3866
|
results.push({
|
|
@@ -3728,7 +3894,7 @@ async function processProject(projectPath, ui) {
|
|
|
3728
3894
|
//#region src/mcp-server.ts
|
|
3729
3895
|
const server = new Server({
|
|
3730
3896
|
name: "sqg-mcp",
|
|
3731
|
-
version: process.env.npm_package_version ?? "0.
|
|
3897
|
+
version: process.env.npm_package_version ?? "0.21.1"
|
|
3732
3898
|
}, { capabilities: {
|
|
3733
3899
|
tools: {},
|
|
3734
3900
|
resources: {}
|
|
@@ -4199,7 +4365,7 @@ function formatMs(ms) {
|
|
|
4199
4365
|
}
|
|
4200
4366
|
//#endregion
|
|
4201
4367
|
//#region src/sqg.ts
|
|
4202
|
-
const version = process.env.npm_package_version ?? "0.
|
|
4368
|
+
const version = process.env.npm_package_version ?? "0.21.1";
|
|
4203
4369
|
updateNotifier({ pkg: {
|
|
4204
4370
|
name: "@sqg/sqg",
|
|
4205
4371
|
version
|
|
@@ -69,7 +69,7 @@ export class {{className}} {
|
|
|
69
69
|
{{#unless skipGenerateFunction}}
|
|
70
70
|
{{functionName}}({{#each variables}}{{name}}: {{type}}{{#unless @last}}, {{/unless}}{{/each}}): {{> returnType }} {
|
|
71
71
|
const stmt = this.prepare<[{{> paramTypes}}], {{> resultType}}>('{{id}}',
|
|
72
|
-
{{{
|
|
72
|
+
{{{sqlExpr this}}}{{#if isPluck}}, true{{/if}});
|
|
73
73
|
{{> execute}}
|
|
74
74
|
}
|
|
75
75
|
{{/unless}}
|
|
@@ -223,6 +223,17 @@ public class {{className}} {
|
|
|
223
223
|
|
|
224
224
|
{{{declareEnums queries}}}
|
|
225
225
|
|
|
226
|
+
{{#each attachers}}
|
|
227
|
+
/** Attach the "{{alias}}" postgres source under the same alias used during generation. Loads the postgres extension (idempotent) so callers need not rely on DuckDB autoloading. */
|
|
228
|
+
public void {{functionName}}(String connectionString) throws SQLException {
|
|
229
|
+
try (var stmt = connection.createStatement()) {
|
|
230
|
+
stmt.execute("INSTALL postgres");
|
|
231
|
+
stmt.execute("LOAD postgres");
|
|
232
|
+
stmt.execute("ATTACH '" + connectionString + "' AS {{alias}} (TYPE postgres)");
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
{{/each}}
|
|
226
237
|
{{#each queries}}
|
|
227
238
|
{{#unless skipGenerateFunction}}
|
|
228
239
|
{{>columnTypesRecord}}
|
|
@@ -135,6 +135,18 @@ class {{className}}:
|
|
|
135
135
|
{{/if}}
|
|
136
136
|
{{/if}}
|
|
137
137
|
|
|
138
|
+
{{#each attachers}}
|
|
139
|
+
def {{functionName}}(self, connection_string: str) -> None:
|
|
140
|
+
"""Attach the "{{alias}}" postgres source under the same alias used during generation.
|
|
141
|
+
|
|
142
|
+
Loads the postgres extension (idempotent) so callers need not rely on
|
|
143
|
+
DuckDB autoloading.
|
|
144
|
+
"""
|
|
145
|
+
self._conn.execute("INSTALL postgres")
|
|
146
|
+
self._conn.execute("LOAD postgres")
|
|
147
|
+
self._conn.execute(f"ATTACH '{connection_string}' AS {{alias}} (TYPE postgres)")
|
|
148
|
+
|
|
149
|
+
{{/each}}
|
|
138
150
|
{{#each queries}}
|
|
139
151
|
{{#unless skipGenerateFunction}}
|
|
140
152
|
{{#if isQuery}}
|
|
@@ -142,7 +154,7 @@ class {{className}}:
|
|
|
142
154
|
{{#if isOne}}
|
|
143
155
|
def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> {{{pyTypeOrNone (lookup columns 0)}}}:
|
|
144
156
|
row = self._conn.execute(
|
|
145
|
-
{{{
|
|
157
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
146
158
|
).fetchone()
|
|
147
159
|
if row is None:
|
|
148
160
|
return None
|
|
@@ -150,19 +162,19 @@ class {{className}}:
|
|
|
150
162
|
|
|
151
163
|
def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> tuple[Any, ...] | None:
|
|
152
164
|
return self._conn.execute(
|
|
153
|
-
{{{
|
|
165
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
154
166
|
).fetchone()
|
|
155
167
|
|
|
156
168
|
{{else}}
|
|
157
169
|
def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[{{{pyType (lookup columns 0)}}}]:
|
|
158
170
|
rows = self._conn.execute(
|
|
159
|
-
{{{
|
|
171
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
160
172
|
).fetchall()
|
|
161
173
|
return [r[0] for r in rows]
|
|
162
174
|
|
|
163
175
|
def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[tuple[Any, ...]]:
|
|
164
176
|
return self._conn.execute(
|
|
165
|
-
{{{
|
|
177
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
166
178
|
).fetchall()
|
|
167
179
|
|
|
168
180
|
{{/if}}
|
|
@@ -170,7 +182,7 @@ class {{className}}:
|
|
|
170
182
|
{{#if isOne}}
|
|
171
183
|
def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> {{{rowType}}} | None:
|
|
172
184
|
row = self._conn.execute(
|
|
173
|
-
{{{
|
|
185
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
174
186
|
).fetchone()
|
|
175
187
|
if row is None:
|
|
176
188
|
return None
|
|
@@ -178,19 +190,19 @@ class {{className}}:
|
|
|
178
190
|
|
|
179
191
|
def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> tuple[Any, ...] | None:
|
|
180
192
|
return self._conn.execute(
|
|
181
|
-
{{{
|
|
193
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
182
194
|
).fetchone()
|
|
183
195
|
|
|
184
196
|
{{else}}
|
|
185
197
|
def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[{{{rowType}}}]:
|
|
186
198
|
rows = self._conn.execute(
|
|
187
|
-
{{{
|
|
199
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
188
200
|
).fetchall()
|
|
189
201
|
return [{{{constructRow this}}} for row in rows]
|
|
190
202
|
|
|
191
203
|
def {{functionName}}_raw(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> list[tuple[Any, ...]]:
|
|
192
204
|
return self._conn.execute(
|
|
193
|
-
{{{
|
|
205
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
194
206
|
).fetchall()
|
|
195
207
|
|
|
196
208
|
{{/if}}
|
|
@@ -198,7 +210,7 @@ class {{className}}:
|
|
|
198
210
|
{{else}}
|
|
199
211
|
def {{functionName}}(self{{#each variables}}, {{name}}: {{{type}}}{{/each}}) -> None:
|
|
200
212
|
self._conn.execute(
|
|
201
|
-
{{{
|
|
213
|
+
{{{sqlExpr this}}}, {{> params}}
|
|
202
214
|
)
|
|
203
215
|
|
|
204
216
|
{{/if}}
|
|
@@ -58,10 +58,19 @@ export class {{className}} {
|
|
|
58
58
|
);
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
{{#each attachers}}
|
|
62
|
+
/** Attach the "{{alias}}" postgres source under the same alias used during generation. Loads the postgres extension (idempotent) so callers need not rely on DuckDB autoloading. */
|
|
63
|
+
async {{functionName}}(connectionString: string): Promise<void> {
|
|
64
|
+
await this.conn.run("INSTALL postgres");
|
|
65
|
+
await this.conn.run("LOAD postgres");
|
|
66
|
+
await this.conn.run(`ATTACH '${connectionString}' AS {{alias}} (TYPE postgres)`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
{{/each}}
|
|
61
70
|
{{#each queries}}
|
|
62
71
|
{{#unless skipGenerateFunction}}
|
|
63
72
|
async {{functionName}}({{#each variables}}{{name}}: {{type}}{{#unless @last}}, {{/unless}}{{/each}}): Promise<{{> returnType }}> {
|
|
64
|
-
const sql = {{{
|
|
73
|
+
const sql = {{{sqlExpr this}}};
|
|
65
74
|
{{#if (hasListParams this)}}
|
|
66
75
|
const stmt = await this.conn.prepare(sql);
|
|
67
76
|
{{{bindStatements this}}}
|