@sqlanvil/cli 1.23.0 → 1.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +277 -53
- package/package.json +1 -1
package/bundle.js
CHANGED
|
@@ -49315,10 +49315,25 @@ function unknownToValueShallow(raw) {
|
|
|
49315
49315
|
})();
|
|
49316
49316
|
|
|
49317
49317
|
const nativeRequire = typeof __webpack_require__ === "function" ? __non_webpack_require__ : require;
|
|
49318
|
+
function globToRegExp(pattern) {
|
|
49319
|
+
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\*/g, ".*");
|
|
49320
|
+
return new RegExp(`^${escaped}$`);
|
|
49321
|
+
}
|
|
49318
49322
|
function matchPatterns(patterns, values) {
|
|
49319
49323
|
const fullyQualifiedActions = [];
|
|
49320
49324
|
patterns.forEach(pattern => {
|
|
49321
|
-
if (pattern.includes("
|
|
49325
|
+
if (pattern.includes("*")) {
|
|
49326
|
+
const regExp = globToRegExp(pattern);
|
|
49327
|
+
const scope = pattern.includes(".")
|
|
49328
|
+
? values
|
|
49329
|
+
: values.map(value => value.split(".").slice(-1)[0]);
|
|
49330
|
+
values.forEach((value, i) => {
|
|
49331
|
+
if (regExp.test(scope[i])) {
|
|
49332
|
+
fullyQualifiedActions.push(value);
|
|
49333
|
+
}
|
|
49334
|
+
});
|
|
49335
|
+
}
|
|
49336
|
+
else if (pattern.includes(".")) {
|
|
49322
49337
|
if (values.includes(pattern)) {
|
|
49323
49338
|
fullyQualifiedActions.push(pattern);
|
|
49324
49339
|
}
|
|
@@ -50606,7 +50621,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
50606
50621
|
.filter(validationQuery => !!validationQuery.query);
|
|
50607
50622
|
}
|
|
50608
50623
|
|
|
50609
|
-
const version = "1.
|
|
50624
|
+
const version = "1.24.0";
|
|
50610
50625
|
const dataformVersion = "3.0.61";
|
|
50611
50626
|
|
|
50612
50627
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -50987,6 +51002,123 @@ async function test$1(dbadapter, timeoutMs = 10000) {
|
|
|
50987
51002
|
}
|
|
50988
51003
|
}
|
|
50989
51004
|
|
|
51005
|
+
function claudeMdBridgeContents() {
|
|
51006
|
+
return "@AGENTS.md\n";
|
|
51007
|
+
}
|
|
51008
|
+
function agentsMdContents(options) {
|
|
51009
|
+
const { warehouse, defaultDataset, version, converted } = options;
|
|
51010
|
+
const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
|
|
51011
|
+
const isSupabase = warehouse === "supabase";
|
|
51012
|
+
const isMysql = warehouse === "mysql";
|
|
51013
|
+
const sections = [];
|
|
51014
|
+
sections.push(`<!-- generated by sqlanvil ${version} — this file is yours; edit freely -->
|
|
51015
|
+
|
|
51016
|
+
# SQLAnvil project — guide for coding agents
|
|
51017
|
+
|
|
51018
|
+
This is a **SQLAnvil** data project (warehouse: **${warehouse}**, default dataset/schema:
|
|
51019
|
+
\`${defaultDataset}\`). SQLAnvil is a fork of Dataform, so Dataform instincts are mostly
|
|
51020
|
+
right: \`.sqlx\` files, \`config {}\` blocks, \`\${ref()}\`, \`\${self()}\`, declarations,
|
|
51021
|
+
assertions, tags, incremental tables, and \`pre_operations\`/\`post_operations\` all work the
|
|
51022
|
+
same. The rules below are where the differences bite.
|
|
51023
|
+
|
|
51024
|
+
## The essentials
|
|
51025
|
+
|
|
51026
|
+
- The CLI is \`sqlanvil\` (\`npm i -g @sqlanvil/cli\`) — there is no \`dataform\` CLI and no
|
|
51027
|
+
npm scripts in this project.
|
|
51028
|
+
- \`workflow_settings.yaml\` pins \`sqlanvilCoreVersion:\` — sqlanvil's OWN SemVer line, NOT
|
|
51029
|
+
\`dataformCoreVersion:\`.
|
|
51030
|
+
- Credentials live in the gitignored \`.df-credentials.json\`. Never commit it; never put
|
|
51031
|
+
secrets in \`workflow_settings.yaml\`.
|
|
51032
|
+
- The statement separator in \`operations\` / \`pre_operations\` / \`post_operations\` is
|
|
51033
|
+
\`---\` on its own line — NEVER \`;\`. Dollar-quoted bodies (\`$$ … ; … $$\`) survive
|
|
51034
|
+
intact. A \`table\`/\`view\` model body is exactly ONE \`SELECT\` — no \`---\`.
|
|
51035
|
+
- Procedures, functions, triggers, grants, extensions → \`type: "operations"\` (there is no
|
|
51036
|
+
dedicated action type).
|
|
51037
|
+
- Two different \`uniqueKey\`s: top-level \`uniqueKey\` on an incremental is the merge/upsert
|
|
51038
|
+
key; \`assertions: { uniqueKey }\` generates a data-quality assertion. They are independent.
|
|
51039
|
+
- Always reference tables with \`\${ref("name")}\` — never hardcoded paths. Declare external
|
|
51040
|
+
tables (\`type: "declaration"\`) before ref'ing them.
|
|
51041
|
+
- Document every table: \`description:\` + \`columns: {}\` compile to real comments.
|
|
51042
|
+
|
|
51043
|
+
## Commands
|
|
51044
|
+
|
|
51045
|
+
\`\`\`bash
|
|
51046
|
+
sqlanvil compile .
|
|
51047
|
+
sqlanvil validate . --credentials .df-credentials.json # planner-checks the whole DAG without executing — run this before every run
|
|
51048
|
+
sqlanvil run . --credentials .df-credentials.json
|
|
51049
|
+
sqlanvil run . --credentials .df-credentials.json --actions <name> --include-deps
|
|
51050
|
+
sqlanvil test . --credentials .df-credentials.json
|
|
51051
|
+
\`\`\`
|
|
51052
|
+
|
|
51053
|
+
\`target/\` holds compile/run artifacts (queryable via \`sqlanvil query\`/\`inspect\`) — keep
|
|
51054
|
+
it gitignored.`);
|
|
51055
|
+
if (isPostgresLike) {
|
|
51056
|
+
sections.push(`## ${isSupabase ? "Supabase/PostgreSQL" : "PostgreSQL"} specifics
|
|
51057
|
+
|
|
51058
|
+
- \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
|
|
51059
|
+
sslMode defaultSchema\` — not nested under a key, not \`username\`/\`databaseName\`/\`ssl\`.${isSupabase
|
|
51060
|
+
? `
|
|
51061
|
+
Use the Session pooler host (\`aws-…pooler.supabase.com\`, user \`postgres.<project-ref>\`,
|
|
51062
|
+
port 5432): the direct \`db.<ref>.supabase.co\` host is IPv6-only and fails on most networks.`
|
|
51063
|
+
: ""}
|
|
51064
|
+
- Storage, indexes, and partitioning are first-class in the \`postgres: {}\` config block —
|
|
51065
|
+
never hand-roll \`CREATE INDEX\` or \`SET (fillfactor …)\` in \`post_operations\`.
|
|
51066
|
+
- Index \`method\` is a NUMERIC enum (\`BTREE=0, HASH=1, GIN=2, GIST=3, BRIN=4\`) — omit it
|
|
51067
|
+
for btree; the string \`method: "btree"\` fails the config check. \`opclass\` is a single
|
|
51068
|
+
string, not an array.
|
|
51069
|
+
- No BigQuery-isms: \`bigquery: {}\`, \`partitionBy\`, \`clusterBy\`, \`OPTIONS(...)\` — use
|
|
51070
|
+
\`postgres: { partition: {...}, indexes: [...] }\` (partition \`kind\`: RANGE=0, LIST=1,
|
|
51071
|
+
HASH=2).
|
|
51072
|
+
- Materialized views: \`type: "view", materialized: true\` (+ optional \`postgres: {
|
|
51073
|
+
refreshPolicy: "on_dependency_change", noData, indexes }\`). Default is drop+recreate each
|
|
51074
|
+
run; in-place refresh does not pick up definition changes.
|
|
51075
|
+
- One-time DDL on incrementals (e.g. \`ADD PRIMARY KEY\`) must be wrapped in
|
|
51076
|
+
\`\${when(!incremental(), \\\`…\\\`)}\` — an unwrapped \`post_operations\` block re-runs on
|
|
51077
|
+
EVERY append and errors the second time.
|
|
51078
|
+
- Read tables in OTHER warehouses via named \`connections:\` in \`workflow_settings.yaml\`
|
|
51079
|
+
plus a \`connection:\`-tagged declaration with \`columnTypes\` in POSTGRES types — scaffold
|
|
51080
|
+
it with \`sqlanvil introspect <connection> <schema.table>\`. Never hand-write FDW/foreign
|
|
51081
|
+
tables.${isSupabase
|
|
51082
|
+
? `
|
|
51083
|
+
- Supabase extras: the \`supabase: {}\` block (\`enableRls\`, \`vectors\`) and dedicated
|
|
51084
|
+
action types \`rlsPolicy\`, \`realtimePublication\`, \`vectorIndex\`. \`enableRls\` only
|
|
51085
|
+
flips RLS on — declare policies via \`rlsPolicy\` actions.`
|
|
51086
|
+
: ""}`);
|
|
51087
|
+
}
|
|
51088
|
+
if (isMysql) {
|
|
51089
|
+
sections.push(`## MySQL/MariaDB specifics (several Postgres rules INVERT here)
|
|
51090
|
+
|
|
51091
|
+
- \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
|
|
51092
|
+
sslMode\` — no \`defaultSchema\` (MySQL's "schema" IS the database).
|
|
51093
|
+
- Use the \`mysql: {}\` config block for indexes, table options (\`engine\`, \`charset\`,
|
|
51094
|
+
\`rowFormat\`), and partitioning — never \`postgres: {}\` (wrong dialect, silently ignored).
|
|
51095
|
+
- An incremental's \`uniqueKey\` compiles to \`INSERT … ON DUPLICATE KEY UPDATE\` and the
|
|
51096
|
+
adapter AUTO-CREATES the matching unique index — do not add your own PK/unique index for
|
|
51097
|
+
the merge.
|
|
51098
|
+
- \`materialized: true\` is an emulated snapshot (drop + CTAS each run), not a native
|
|
51099
|
+
matview; no \`refreshPolicy\`/\`noData\`.
|
|
51100
|
+
- Backtick-quote identifiers in raw DDL; never \`DELIMITER\` (client-only directive — the
|
|
51101
|
+
\`---\` separator already keeps \`;\` bodies intact).
|
|
51102
|
+
- A mysql warehouse cannot read cross-warehouse \`connections:\` (single write warehouse;
|
|
51103
|
+
MySQL works as a SOURCE for a Postgres/Supabase project, not the other way).`);
|
|
51104
|
+
}
|
|
51105
|
+
if (converted) {
|
|
51106
|
+
sections.push(`## This project was converted from Dataform
|
|
51107
|
+
|
|
51108
|
+
- Work the to-do list in \`migration-report.md\` (machine-readable twin:
|
|
51109
|
+
\`migration-report.json\`). Inline \`SQLANVIL-MIGRATE:\` markers flag every rewritten or
|
|
51110
|
+
questionable spot.
|
|
51111
|
+
- \`sqlanvil validate\` is the completion loop — converge on all-PASS.
|
|
51112
|
+
- The original Dataform repo was read-only during conversion and remains the rollback path.`);
|
|
51113
|
+
}
|
|
51114
|
+
sections.push(`---
|
|
51115
|
+
|
|
51116
|
+
Docs: https://sqlanvil.com/docs (agent bundle: https://sqlanvil.com/llms.txt) · Full
|
|
51117
|
+
authoring skill: \`npx skills add SQLAnvil/agent-skills\` · Hosted CI/scheduling:
|
|
51118
|
+
https://app.sqlanvil.com (remote MCP server at \`/api/mcp\`).`);
|
|
51119
|
+
return sections.join("\n\n") + "\n";
|
|
51120
|
+
}
|
|
51121
|
+
|
|
50990
51122
|
const gitIgnoreContents = `
|
|
50991
51123
|
${CREDENTIALS_FILENAME}
|
|
50992
51124
|
.df-credentials*.json
|
|
@@ -51084,6 +51216,20 @@ async function init(projectDir, projectConfig, options = {}) {
|
|
|
51084
51216
|
filesWritten.push(workflowSettingsYamlPath);
|
|
51085
51217
|
fs__namespace$1.writeFileSync(gitignorePath, gitIgnoreContents);
|
|
51086
51218
|
filesWritten.push(gitignorePath);
|
|
51219
|
+
const agentsMdPath = path__namespace.join(projectDir, "AGENTS.md");
|
|
51220
|
+
if (!fs__namespace$1.existsSync(agentsMdPath)) {
|
|
51221
|
+
fs__namespace$1.writeFileSync(agentsMdPath, agentsMdContents({
|
|
51222
|
+
warehouse,
|
|
51223
|
+
defaultDataset: workflowSettings.defaultDataset || "public",
|
|
51224
|
+
version
|
|
51225
|
+
}));
|
|
51226
|
+
filesWritten.push(agentsMdPath);
|
|
51227
|
+
}
|
|
51228
|
+
const claudeMdPath = path__namespace.join(projectDir, "CLAUDE.md");
|
|
51229
|
+
if (!fs__namespace$1.existsSync(claudeMdPath)) {
|
|
51230
|
+
fs__namespace$1.writeFileSync(claudeMdPath, claudeMdBridgeContents());
|
|
51231
|
+
filesWritten.push(claudeMdPath);
|
|
51232
|
+
}
|
|
51087
51233
|
if (!isBigQuery) {
|
|
51088
51234
|
fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), (_a = options.credentialsJson) !== null && _a !== void 0 ? _a : (warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse)));
|
|
51089
51235
|
filesWritten.push(path__namespace.join(projectDir, CREDENTIALS_FILENAME));
|
|
@@ -54409,7 +54555,8 @@ ${description} columnTypes: {
|
|
|
54409
54555
|
],
|
|
54410
54556
|
};
|
|
54411
54557
|
}
|
|
54412
|
-
function convertTarget(source, jsMode) {
|
|
54558
|
+
function convertTarget(source, jsMode, dialect = "postgres") {
|
|
54559
|
+
const dialectPass = dialect === "postgres";
|
|
54413
54560
|
const span = jsMode ? null : findConfigBlock(source);
|
|
54414
54561
|
const findings = [];
|
|
54415
54562
|
const lines = source.split("\n");
|
|
@@ -54447,7 +54594,7 @@ function convertTarget(source, jsMode) {
|
|
|
54447
54594
|
line = line.replace(/,/, " ");
|
|
54448
54595
|
}
|
|
54449
54596
|
}
|
|
54450
|
-
if (inConfig && commentingConfigBlockDepth === 0) {
|
|
54597
|
+
if (dialectPass && inConfig && commentingConfigBlockDepth === 0) {
|
|
54451
54598
|
const m = CONFIG_BQ_KEYS.exec(line);
|
|
54452
54599
|
if (m) {
|
|
54453
54600
|
const note = `BigQuery-only setting — not applicable on Postgres (partitioning/clustering → postgres: {}; project qualifiers drop)`;
|
|
@@ -54471,7 +54618,7 @@ function convertTarget(source, jsMode) {
|
|
|
54471
54618
|
out.push(`// ${line}`);
|
|
54472
54619
|
continue;
|
|
54473
54620
|
}
|
|
54474
|
-
if (!inConfig || jsMode) {
|
|
54621
|
+
if (dialectPass && (!inConfig || jsMode)) {
|
|
54475
54622
|
for (const rule of SQL_RULES) {
|
|
54476
54623
|
rule.pattern.lastIndex = 0;
|
|
54477
54624
|
if (!rule.pattern.test(line))
|
|
@@ -54523,6 +54670,7 @@ function readSourceConfig(srcDir) {
|
|
|
54523
54670
|
const s = (k) => (typeof raw[k] === "string" ? raw[k] : null);
|
|
54524
54671
|
return {
|
|
54525
54672
|
defaultProject: (_a = s("defaultProject")) !== null && _a !== void 0 ? _a : s("defaultDatabase"),
|
|
54673
|
+
defaultLocation: s("defaultLocation"),
|
|
54526
54674
|
defaultDataset: (_c = (_b = s("defaultDataset")) !== null && _b !== void 0 ? _b : s("defaultSchema")) !== null && _c !== void 0 ? _c : "dataform",
|
|
54527
54675
|
defaultAssertionDataset: (_d = s("defaultAssertionDataset")) !== null && _d !== void 0 ? _d : s("assertionSchema"),
|
|
54528
54676
|
vars: (_e = raw.vars) !== null && _e !== void 0 ? _e : null,
|
|
@@ -54530,7 +54678,7 @@ function readSourceConfig(srcDir) {
|
|
|
54530
54678
|
};
|
|
54531
54679
|
}
|
|
54532
54680
|
async function migrateDataform(opts) {
|
|
54533
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
54681
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
54534
54682
|
const srcDir = path__namespace.resolve(opts.srcDir);
|
|
54535
54683
|
const outDir = path__namespace.resolve(opts.outDir);
|
|
54536
54684
|
if (containsPath(srcDir, outDir)) {
|
|
@@ -54542,11 +54690,14 @@ async function migrateDataform(opts) {
|
|
|
54542
54690
|
if (fs__namespace$1.existsSync(outDir) && fs__namespace$1.readdirSync(outDir).length > 0) {
|
|
54543
54691
|
throw new Error(`Output directory ${outDir} is not empty.`);
|
|
54544
54692
|
}
|
|
54693
|
+
const targetWarehouse = (_a = opts.targetWarehouse) !== null && _a !== void 0 ? _a : "supabase";
|
|
54694
|
+
const isBigQueryTarget = targetWarehouse === "bigquery";
|
|
54545
54695
|
const sourceConfig = readSourceConfig(srcDir);
|
|
54546
54696
|
const writer = new OutWriter(outDir);
|
|
54547
54697
|
skippedForSafety = new Set();
|
|
54548
54698
|
const files = walkFiles(srcDir);
|
|
54549
54699
|
const report = {
|
|
54700
|
+
targetWarehouse,
|
|
54550
54701
|
generator: `sqlanvil migrate-dataform ${version}`,
|
|
54551
54702
|
sourceDir: srcDir,
|
|
54552
54703
|
sourceConfig: sourceConfig.raw,
|
|
@@ -54568,16 +54719,16 @@ async function migrateDataform(opts) {
|
|
|
54568
54719
|
: { type: null, schema: null, database: null, name: null, description: null };
|
|
54569
54720
|
sqlxFiles.push({ rel, source, parsed });
|
|
54570
54721
|
report.inventory.sqlxFiles++;
|
|
54571
|
-
const t = (
|
|
54572
|
-
report.inventory.byType[t] = ((
|
|
54722
|
+
const t = (_b = parsed.type) !== null && _b !== void 0 ? _b : "(none)";
|
|
54723
|
+
report.inventory.byType[t] = ((_c = report.inventory.byType[t]) !== null && _c !== void 0 ? _c : 0) + 1;
|
|
54573
54724
|
}
|
|
54574
54725
|
const resolveDatabase = (db) => { var _a; return !db || db === "<defaultDatabase>" ? (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "UNKNOWN_PROJECT" : db; };
|
|
54575
54726
|
const declarationSchemas = new Set();
|
|
54576
54727
|
const targetSchemas = new Set();
|
|
54577
54728
|
const connectionMap = new Map();
|
|
54578
|
-
for (const f of sqlxFiles) {
|
|
54729
|
+
for (const f of isBigQueryTarget ? [] : sqlxFiles) {
|
|
54579
54730
|
if (f.parsed.type === "declaration") {
|
|
54580
|
-
const dataset = (
|
|
54731
|
+
const dataset = (_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset;
|
|
54581
54732
|
const project = resolveDatabase(f.parsed.database);
|
|
54582
54733
|
declarationSchemas.add(dataset);
|
|
54583
54734
|
const name = connectionNameFor(project);
|
|
@@ -54598,7 +54749,7 @@ async function migrateDataform(opts) {
|
|
|
54598
54749
|
}
|
|
54599
54750
|
}
|
|
54600
54751
|
else if (f.parsed.type) {
|
|
54601
|
-
targetSchemas.add((
|
|
54752
|
+
targetSchemas.add((_e = f.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset);
|
|
54602
54753
|
}
|
|
54603
54754
|
}
|
|
54604
54755
|
report.connections = [...connectionMap.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
@@ -54610,8 +54761,20 @@ async function migrateDataform(opts) {
|
|
|
54610
54761
|
}
|
|
54611
54762
|
if (rel.endsWith(".sqlx")) {
|
|
54612
54763
|
const info = sqlxFiles.find(f => f.rel === rel);
|
|
54764
|
+
if (isBigQueryTarget) {
|
|
54765
|
+
const { content, findings } = convertTarget(info.source, false, "none");
|
|
54766
|
+
writer.write(rel, content);
|
|
54767
|
+
report.files.push({
|
|
54768
|
+
file: rel,
|
|
54769
|
+
action: info.parsed.type === "declaration" ? "declaration" : "target",
|
|
54770
|
+
type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
|
|
54771
|
+
status: findings.length === 0 ? "clean" : "rewritten",
|
|
54772
|
+
findings,
|
|
54773
|
+
});
|
|
54774
|
+
continue;
|
|
54775
|
+
}
|
|
54613
54776
|
if (info.parsed.type === "declaration") {
|
|
54614
|
-
const dataset = (
|
|
54777
|
+
const dataset = (_g = info.parsed.schema) !== null && _g !== void 0 ? _g : sourceConfig.defaultDataset;
|
|
54615
54778
|
const project = resolveDatabase(info.parsed.database);
|
|
54616
54779
|
const connection = connectionNameFor(project);
|
|
54617
54780
|
const { content, findings } = convertDeclaration(info.source, info.parsed, connection, dataset, rel);
|
|
@@ -54631,7 +54794,7 @@ async function migrateDataform(opts) {
|
|
|
54631
54794
|
report.files.push({
|
|
54632
54795
|
file: rel,
|
|
54633
54796
|
action: "target",
|
|
54634
|
-
type: (
|
|
54797
|
+
type: (_h = info.parsed.type) !== null && _h !== void 0 ? _h : "(none)",
|
|
54635
54798
|
status: findings.length === 0
|
|
54636
54799
|
? "clean"
|
|
54637
54800
|
: findings.some(f => f.kind === "flag")
|
|
@@ -54645,10 +54808,10 @@ async function migrateDataform(opts) {
|
|
|
54645
54808
|
if (rel.endsWith(".js") && (rel.startsWith("definitions/") || rel.startsWith("includes/"))) {
|
|
54646
54809
|
report.inventory.jsFiles++;
|
|
54647
54810
|
const source = fs__namespace$1.readFileSync(abs, "utf8");
|
|
54648
|
-
const { content, findings } = convertTarget(source, true);
|
|
54811
|
+
const { content, findings } = convertTarget(source, true, isBigQueryTarget ? "none" : "postgres");
|
|
54649
54812
|
writer.write(rel, content);
|
|
54650
54813
|
const isIncludes = rel.startsWith("includes/");
|
|
54651
|
-
if (isIncludes && findings.length > 0) {
|
|
54814
|
+
if (isIncludes && findings.length > 0 && !isBigQueryTarget) {
|
|
54652
54815
|
report.warnings.push(`includes helper ${rel} emits BigQuery-flavored SQL — review its ${findings.length} finding(s)`);
|
|
54653
54816
|
}
|
|
54654
54817
|
report.files.push({
|
|
@@ -54667,11 +54830,12 @@ async function migrateDataform(opts) {
|
|
|
54667
54830
|
writer.write(rel, fs__namespace$1.readFileSync(abs));
|
|
54668
54831
|
report.files.push({ file: rel, action: "copied", status: "clean", findings: [] });
|
|
54669
54832
|
}
|
|
54670
|
-
const settings =
|
|
54671
|
-
|
|
54833
|
+
const settings = isBigQueryTarget
|
|
54834
|
+
? Object.assign(Object.assign(Object.assign({}, (sourceConfig.defaultProject ? { defaultProject: sourceConfig.defaultProject } : {})), (sourceConfig.defaultLocation ? { defaultLocation: sourceConfig.defaultLocation } : {})), { defaultDataset: sourceConfig.defaultDataset, defaultAssertionDataset: (_j = sourceConfig.defaultAssertionDataset) !== null && _j !== void 0 ? _j : "sqlanvil_assertions", sqlanvilCoreVersion: (_k = opts.coreVersion) !== null && _k !== void 0 ? _k : version }) : {
|
|
54835
|
+
warehouse: targetWarehouse,
|
|
54672
54836
|
defaultDataset: sourceConfig.defaultDataset.toLowerCase(),
|
|
54673
|
-
defaultAssertionDataset: ((
|
|
54674
|
-
sqlanvilCoreVersion: (
|
|
54837
|
+
defaultAssertionDataset: ((_l = sourceConfig.defaultAssertionDataset) !== null && _l !== void 0 ? _l : "sqlanvil_assertions").toLowerCase(),
|
|
54838
|
+
sqlanvilCoreVersion: (_m = opts.coreVersion) !== null && _m !== void 0 ? _m : version,
|
|
54675
54839
|
};
|
|
54676
54840
|
if (sourceConfig.vars)
|
|
54677
54841
|
settings.vars = sourceConfig.vars;
|
|
@@ -54690,6 +54854,22 @@ async function migrateDataform(opts) {
|
|
|
54690
54854
|
}));
|
|
54691
54855
|
}
|
|
54692
54856
|
writer.write("workflow_settings.yaml", jsYaml.dump(settings));
|
|
54857
|
+
if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "AGENTS.md"))) {
|
|
54858
|
+
writer.write("AGENTS.md", agentsMdContents({
|
|
54859
|
+
warehouse: targetWarehouse,
|
|
54860
|
+
defaultDataset: settings.defaultDataset,
|
|
54861
|
+
version: (_o = opts.coreVersion) !== null && _o !== void 0 ? _o : version,
|
|
54862
|
+
converted: true
|
|
54863
|
+
}));
|
|
54864
|
+
report.files.push({ file: "AGENTS.md", action: "generated", status: "clean", findings: [] });
|
|
54865
|
+
}
|
|
54866
|
+
else {
|
|
54867
|
+
report.warnings.push("Source AGENTS.md copied as-is — its guidance targets Dataform; consider replacing it with the sqlanvil version (`sqlanvil init --bare` in an empty dir generates one).");
|
|
54868
|
+
}
|
|
54869
|
+
if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "CLAUDE.md"))) {
|
|
54870
|
+
writer.write("CLAUDE.md", claudeMdBridgeContents());
|
|
54871
|
+
report.files.push({ file: "CLAUDE.md", action: "generated", status: "clean", findings: [] });
|
|
54872
|
+
}
|
|
54693
54873
|
report.skippedForSafety = [...skippedForSafety].sort();
|
|
54694
54874
|
if (skippedForSafety.size > 0) {
|
|
54695
54875
|
report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
|
|
@@ -54716,28 +54896,38 @@ function renderReportMd(r) {
|
|
|
54716
54896
|
lines.push(` - ${t}: ${n}`);
|
|
54717
54897
|
}
|
|
54718
54898
|
lines.push("");
|
|
54719
|
-
|
|
54720
|
-
|
|
54721
|
-
|
|
54722
|
-
`
|
|
54723
|
-
|
|
54724
|
-
|
|
54725
|
-
|
|
54726
|
-
lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
|
|
54727
|
-
`${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
|
|
54728
|
-
(c.datasets.length > 8 ? ", …" : ""));
|
|
54729
|
-
lines.push(` - \`${c.introspectExample}\``);
|
|
54899
|
+
if (r.targetWarehouse === "bigquery") {
|
|
54900
|
+
lines.push(`## Same-warehouse conversion (BigQuery → BigQuery)`);
|
|
54901
|
+
lines.push("");
|
|
54902
|
+
lines.push(`This is a tooling swap: the project keeps running on the SAME BigQuery warehouse. SQL ` +
|
|
54903
|
+
`bodies, \`bigquery: {}\` config blocks, and declarations passed through untouched; ` +
|
|
54904
|
+
`only the compile global was renamed (\`dataform.projectConfig\` → ` +
|
|
54905
|
+
`\`sqlanvil.projectConfig\`) and the settings file converted.`);
|
|
54730
54906
|
}
|
|
54731
|
-
|
|
54907
|
+
else {
|
|
54908
|
+
lines.push(`## Source connections (${r.connections.length})`);
|
|
54732
54909
|
lines.push("");
|
|
54733
|
-
lines.push(
|
|
54910
|
+
lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
|
|
54911
|
+
`(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
|
|
54912
|
+
`before it can extract; scaffold them with \`sqlanvil introspect\`:`);
|
|
54734
54913
|
lines.push("");
|
|
54735
|
-
|
|
54736
|
-
.
|
|
54737
|
-
|
|
54738
|
-
|
|
54739
|
-
`
|
|
54740
|
-
|
|
54914
|
+
for (const c of r.connections) {
|
|
54915
|
+
lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
|
|
54916
|
+
`${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
|
|
54917
|
+
(c.datasets.length > 8 ? ", …" : ""));
|
|
54918
|
+
lines.push(` - \`${c.introspectExample}\``);
|
|
54919
|
+
}
|
|
54920
|
+
if (r.overlappingSchemas.length > 0) {
|
|
54921
|
+
lines.push("");
|
|
54922
|
+
lines.push(`## ⚠ Schemas that are BOTH source and target`);
|
|
54923
|
+
lines.push("");
|
|
54924
|
+
lines.push(`These schemas have declarations AND materializing actions: ${r.overlappingSchemas
|
|
54925
|
+
.map(s => `\`${s}\``)
|
|
54926
|
+
.join(", ")}. ` +
|
|
54927
|
+
`Their declared tables are probably produced by this pipeline (or a sibling) — once those ` +
|
|
54928
|
+
`producers move to Postgres, convert the declarations to plain local declarations instead ` +
|
|
54929
|
+
`of BigQuery connections.`);
|
|
54930
|
+
}
|
|
54741
54931
|
}
|
|
54742
54932
|
lines.push("");
|
|
54743
54933
|
lines.push(`## Target files (${targets.length})`);
|
|
@@ -54759,7 +54949,9 @@ function renderReportMd(r) {
|
|
|
54759
54949
|
lines.push("");
|
|
54760
54950
|
lines.push(`## Declarations (${declarations.length})`);
|
|
54761
54951
|
lines.push("");
|
|
54762
|
-
lines.push(
|
|
54952
|
+
lines.push(r.targetWarehouse === "bigquery"
|
|
54953
|
+
? `Declarations are unchanged — the sources are native to the warehouse.`
|
|
54954
|
+
: `Every declaration was rewritten to ride its connection. The BigQuery data does not move.`);
|
|
54763
54955
|
if (r.warnings.length > 0) {
|
|
54764
54956
|
lines.push("");
|
|
54765
54957
|
lines.push(`## Warnings`);
|
|
@@ -54770,12 +54962,23 @@ function renderReportMd(r) {
|
|
|
54770
54962
|
lines.push("");
|
|
54771
54963
|
lines.push(`## Finish the migration`);
|
|
54772
54964
|
lines.push("");
|
|
54773
|
-
|
|
54774
|
-
|
|
54775
|
-
|
|
54776
|
-
|
|
54777
|
-
|
|
54778
|
-
|
|
54965
|
+
if (r.targetWarehouse === "bigquery") {
|
|
54966
|
+
lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
|
|
54967
|
+
lines.push(`2. Credentials: \`gcloud auth application-default login\` (ADC), or a service-account ` +
|
|
54968
|
+
`key in a gitignored \`.df-credentials.json\`.`);
|
|
54969
|
+
lines.push(`3. \`sqlanvil validate\` — dry-runs every model against BigQuery without executing; ` +
|
|
54970
|
+
`all-PASS means the swap is complete.`);
|
|
54971
|
+
lines.push(`4. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
|
|
54972
|
+
`sample of actions — they should match apart from the compile-global rename.`);
|
|
54973
|
+
}
|
|
54974
|
+
else {
|
|
54975
|
+
lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
|
|
54976
|
+
lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
|
|
54977
|
+
lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
|
|
54978
|
+
`model is the migration to-do list. Work the flagged files until validate is green.`);
|
|
54979
|
+
lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
|
|
54980
|
+
`the \`SQLANVIL-MIGRATE:\` markers is the fastest path through the dialect work.`);
|
|
54981
|
+
}
|
|
54779
54982
|
lines.push("");
|
|
54780
54983
|
return lines.join("\n");
|
|
54781
54984
|
}
|
|
@@ -55316,6 +55519,7 @@ const CREDS_PASSWORD_QUESTION = "Password? (leave empty to fill in later)";
|
|
|
55316
55519
|
const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
|
|
55317
55520
|
const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
|
|
55318
55521
|
const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
|
|
55522
|
+
const CONVERT_TARGET_QUESTION = "Target warehouse? (bigquery = keep running on BigQuery, tooling swap only; supabase/postgres = move off BigQuery)";
|
|
55319
55523
|
const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
|
|
55320
55524
|
"aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
|
|
55321
55525
|
"db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
|
|
@@ -55434,13 +55638,22 @@ async function runConvertFlow() {
|
|
|
55434
55638
|
throw new Error("No usable source directory provided.");
|
|
55435
55639
|
}
|
|
55436
55640
|
const outDir = actuallyResolve(askRequired(CONVERT_OUT_QUESTION));
|
|
55641
|
+
const targetWarehouse = askChoice(CONVERT_TARGET_QUESTION, ["bigquery", "supabase", "postgres"], "bigquery");
|
|
55437
55642
|
print("\nConverting...\n");
|
|
55438
|
-
const report = await migrateDataform({ srcDir, outDir });
|
|
55643
|
+
const report = await migrateDataform({ srcDir, outDir, targetWarehouse });
|
|
55439
55644
|
printMigrationSummary(report, outDir);
|
|
55645
|
+
if (targetWarehouse === "bigquery") {
|
|
55646
|
+
print("\nNext steps:");
|
|
55647
|
+
print(` 1. sqlanvil compile ${outDir}`);
|
|
55648
|
+
print(" 2. Credentials: gcloud auth application-default login (ADC), or a service-account " +
|
|
55649
|
+
`key in a gitignored ${CREDENTIALS_FILENAME}.`);
|
|
55650
|
+
print(` 3. sqlanvil validate ${outDir} (dry-runs every model against BigQuery; all-PASS = swap complete)`);
|
|
55651
|
+
return 0;
|
|
55652
|
+
}
|
|
55440
55653
|
if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
|
|
55441
55654
|
const credentialsPath = path__namespace.join(outDir, CREDENTIALS_FILENAME);
|
|
55442
55655
|
ensureGitignoreCoversCredentials(outDir);
|
|
55443
|
-
fs__namespace$1.writeFileSync(credentialsPath, collectCredentials(
|
|
55656
|
+
fs__namespace$1.writeFileSync(credentialsPath, collectCredentials(targetWarehouse));
|
|
55444
55657
|
printSuccess(`Credentials written to ${credentialsPath}`);
|
|
55445
55658
|
}
|
|
55446
55659
|
print("\nNext steps:");
|
|
@@ -56829,6 +57042,14 @@ const interactiveOption = option("interactive", {
|
|
|
56829
57042
|
type: "boolean",
|
|
56830
57043
|
default: false
|
|
56831
57044
|
});
|
|
57045
|
+
const targetWarehouseOption = option("target-warehouse", {
|
|
57046
|
+
describe: "Where the CONVERTED project runs: supabase/postgres = move off BigQuery (sources become " +
|
|
57047
|
+
"named connections, SQL gets the dialect pass); bigquery = keep the SAME warehouse — a " +
|
|
57048
|
+
"tooling swap with SQL and bigquery:{} blocks untouched.",
|
|
57049
|
+
type: "string",
|
|
57050
|
+
choices: ["supabase", "postgres", "bigquery"],
|
|
57051
|
+
default: "supabase"
|
|
57052
|
+
});
|
|
56832
57053
|
const warehouseOption = option("warehouse", {
|
|
56833
57054
|
describe: "Target warehouse for the new project.",
|
|
56834
57055
|
type: "string",
|
|
@@ -57695,10 +57916,12 @@ function runCli() {
|
|
|
57695
57916
|
},
|
|
57696
57917
|
{
|
|
57697
57918
|
format: "migrate-dataform <source-dir> <out-dir>",
|
|
57698
|
-
description: "Convert a Dataform/BigQuery project
|
|
57699
|
-
"
|
|
57700
|
-
"
|
|
57701
|
-
"gets safe rewrites + inline SQLANVIL-MIGRATE markers
|
|
57919
|
+
description: "Convert a Dataform/BigQuery project to sqlanvil. The source directory is READ-ONLY; " +
|
|
57920
|
+
"the converted project + migration-report.{md,json} land in <out-dir> (must be empty). " +
|
|
57921
|
+
"Default target (supabase/postgres) MOVES the warehouse: sources stay in BigQuery as " +
|
|
57922
|
+
"named connections and target SQL gets safe rewrites + inline SQLANVIL-MIGRATE markers. " +
|
|
57923
|
+
"--target-warehouse bigquery is a TOOLING SWAP: the project keeps running on the same " +
|
|
57924
|
+
"BigQuery warehouse, SQL and bigquery:{} blocks untouched.",
|
|
57702
57925
|
positionalOptions: [
|
|
57703
57926
|
positionalOption("source-dir", {
|
|
57704
57927
|
describe: "The Dataform project to convert (never modified)."
|
|
@@ -57707,11 +57930,12 @@ function runCli() {
|
|
|
57707
57930
|
describe: "Where the converted sqlanvil project is written (created; must be empty)."
|
|
57708
57931
|
})
|
|
57709
57932
|
],
|
|
57710
|
-
options: [],
|
|
57933
|
+
options: [targetWarehouseOption],
|
|
57711
57934
|
processFn: async (argv) => {
|
|
57712
57935
|
const report = await migrateDataform({
|
|
57713
57936
|
srcDir: argv["source-dir"],
|
|
57714
|
-
outDir: argv["out-dir"]
|
|
57937
|
+
outDir: argv["out-dir"],
|
|
57938
|
+
targetWarehouse: argv[targetWarehouseOption.name]
|
|
57715
57939
|
});
|
|
57716
57940
|
printMigrationSummary(report, argv["out-dir"]);
|
|
57717
57941
|
print(`Next: sqlanvil compile ${argv["out-dir"]}`);
|