@sqlanvil/cli 1.23.0 → 1.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -4
- package/bundle.js +296 -53
- package/package.json +9 -2
package/README.md
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
SQLAnvil is an open-source fork of [Dataform OSS](https://github.com/dataform-co/dataform) (Apache 2.0), extended with first-class PostgreSQL, Supabase, and MySQL/MariaDB support. Define your data transformations in SQLX, have SQLAnvil compile them to idiomatic SQL for your warehouse, and run, validate, and document the whole DAG from one CLI.
|
|
6
6
|
|
|
7
|
-
> **SQLAnvil is not affiliated with or endorsed by Google.** The Dataform name and related marks are trademarks of Google LLC. See [NOTICE](NOTICE) for attribution.
|
|
7
|
+
> **SQLAnvil is not affiliated with or endorsed by Google.** The Dataform name and related marks are trademarks of Google LLC. See [NOTICE](https://github.com/SQLAnvil/sqlanvil/blob/main/NOTICE) for attribution.
|
|
8
8
|
|
|
9
9
|
---
|
|
10
10
|
|
|
@@ -86,7 +86,7 @@ Full documentation at **[sqlanvil.com/docs](https://sqlanvil.com/docs/)**.
|
|
|
86
86
|
- [Validate](https://sqlanvil.com/docs/guides/validate/) · [Named Environments](https://sqlanvil.com/docs/guides/environments/) · [Artifacts & Catalog](https://sqlanvil.com/docs/guides/artifacts/)
|
|
87
87
|
- [Configs Reference](https://sqlanvil.com/docs/reference/configs/) · [What's New](https://sqlanvil.com/docs/whats-new/)
|
|
88
88
|
|
|
89
|
-
Writing sqlanvil projects with an AI agent? Point it at [`AGENTS.md`](AGENTS.md) — it corrects
|
|
89
|
+
Writing sqlanvil projects with an AI agent? Point it at [`AGENTS.md`](https://github.com/SQLAnvil/sqlanvil/blob/main/AGENTS.md) — it corrects
|
|
90
90
|
the Dataform/BigQuery priors that otherwise produce broken sqlanvil code. Contributor/design
|
|
91
91
|
docs live in the separate [`sqlanvil/docs`](https://github.com/sqlanvil/docs) repo.
|
|
92
92
|
|
|
@@ -123,12 +123,12 @@ bazel test //core/... //cli/...
|
|
|
123
123
|
```
|
|
124
124
|
|
|
125
125
|
A Docker-based build container (`./scripts/docker-bazel`) remains available as an optional
|
|
126
|
-
hermetic environment. See [contributing.md](contributing.md) for full instructions.
|
|
126
|
+
hermetic environment. See [contributing.md](https://github.com/SQLAnvil/sqlanvil/blob/main/contributing.md) for full instructions.
|
|
127
127
|
|
|
128
128
|
---
|
|
129
129
|
|
|
130
130
|
## License
|
|
131
131
|
|
|
132
|
-
Apache License 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE).
|
|
132
|
+
Apache License 2.0. See [LICENSE](https://github.com/SQLAnvil/sqlanvil/blob/main/LICENSE) and [NOTICE](https://github.com/SQLAnvil/sqlanvil/blob/main/NOTICE).
|
|
133
133
|
|
|
134
134
|
SQLAnvil is a derivative of Dataform OSS (originally developed by Dataform Co and contributed to by Google LLC). It has been fully renamed and extended with first-class PostgreSQL/Supabase/MySQL support. Licensed under Apache License 2.0.
|
package/bundle.js
CHANGED
|
@@ -49315,10 +49315,25 @@ function unknownToValueShallow(raw) {
|
|
|
49315
49315
|
})();
|
|
49316
49316
|
|
|
49317
49317
|
const nativeRequire = typeof __webpack_require__ === "function" ? __non_webpack_require__ : require;
|
|
49318
|
+
function globToRegExp(pattern) {
|
|
49319
|
+
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\*/g, ".*");
|
|
49320
|
+
return new RegExp(`^${escaped}$`);
|
|
49321
|
+
}
|
|
49318
49322
|
function matchPatterns(patterns, values) {
|
|
49319
49323
|
const fullyQualifiedActions = [];
|
|
49320
49324
|
patterns.forEach(pattern => {
|
|
49321
|
-
if (pattern.includes("
|
|
49325
|
+
if (pattern.includes("*")) {
|
|
49326
|
+
const regExp = globToRegExp(pattern);
|
|
49327
|
+
const scope = pattern.includes(".")
|
|
49328
|
+
? values
|
|
49329
|
+
: values.map(value => value.split(".").slice(-1)[0]);
|
|
49330
|
+
values.forEach((value, i) => {
|
|
49331
|
+
if (regExp.test(scope[i])) {
|
|
49332
|
+
fullyQualifiedActions.push(value);
|
|
49333
|
+
}
|
|
49334
|
+
});
|
|
49335
|
+
}
|
|
49336
|
+
else if (pattern.includes(".")) {
|
|
49322
49337
|
if (values.includes(pattern)) {
|
|
49323
49338
|
fullyQualifiedActions.push(pattern);
|
|
49324
49339
|
}
|
|
@@ -50606,7 +50621,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
50606
50621
|
.filter(validationQuery => !!validationQuery.query);
|
|
50607
50622
|
}
|
|
50608
50623
|
|
|
50609
|
-
const version = "1.
|
|
50624
|
+
const version = "1.25.0";
|
|
50610
50625
|
const dataformVersion = "3.0.61";
|
|
50611
50626
|
|
|
50612
50627
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -50987,6 +51002,142 @@ async function test$1(dbadapter, timeoutMs = 10000) {
|
|
|
50987
51002
|
}
|
|
50988
51003
|
}
|
|
50989
51004
|
|
|
51005
|
+
function claudeMdBridgeContents() {
|
|
51006
|
+
return "@AGENTS.md\n";
|
|
51007
|
+
}
|
|
51008
|
+
function agentsMdContents(options) {
|
|
51009
|
+
const { warehouse, defaultDataset, version, converted } = options;
|
|
51010
|
+
const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
|
|
51011
|
+
const isSupabase = warehouse === "supabase";
|
|
51012
|
+
const isMysql = warehouse === "mysql";
|
|
51013
|
+
const sections = [];
|
|
51014
|
+
sections.push(`<!-- generated by sqlanvil ${version} — this file is yours; edit freely -->
|
|
51015
|
+
|
|
51016
|
+
# SQLAnvil project — guide for coding agents
|
|
51017
|
+
|
|
51018
|
+
This is a **SQLAnvil** data project (warehouse: **${warehouse}**, default dataset/schema:
|
|
51019
|
+
\`${defaultDataset}\`). SQLAnvil is a fork of Dataform, so Dataform instincts are mostly
|
|
51020
|
+
right: \`.sqlx\` files, \`config {}\` blocks, \`\${ref()}\`, \`\${self()}\`, declarations,
|
|
51021
|
+
assertions, tags, incremental tables, and \`pre_operations\`/\`post_operations\` all work the
|
|
51022
|
+
same. The rules below are where the differences bite.
|
|
51023
|
+
|
|
51024
|
+
## The essentials
|
|
51025
|
+
|
|
51026
|
+
- The CLI is \`sqlanvil\` (\`npm i -g @sqlanvil/cli\`) — there is no \`dataform\` CLI and no
|
|
51027
|
+
npm scripts in this project.
|
|
51028
|
+
- \`workflow_settings.yaml\` pins \`sqlanvilCoreVersion:\` — sqlanvil's OWN SemVer line, NOT
|
|
51029
|
+
\`dataformCoreVersion:\`.
|
|
51030
|
+
- Credentials live in the gitignored \`.df-credentials.json\`. Never commit it; never put
|
|
51031
|
+
secrets in \`workflow_settings.yaml\`.
|
|
51032
|
+
- The statement separator in \`operations\` / \`pre_operations\` / \`post_operations\` is
|
|
51033
|
+
\`---\` on its own line — NEVER \`;\`. Dollar-quoted bodies (\`$$ … ; … $$\`) survive
|
|
51034
|
+
intact. A \`table\`/\`view\` model body is exactly ONE \`SELECT\` — no \`---\`.
|
|
51035
|
+
- Procedures, functions, triggers, grants, extensions → \`type: "operations"\` (there is no
|
|
51036
|
+
dedicated action type).
|
|
51037
|
+
- Two different \`uniqueKey\`s: top-level \`uniqueKey\` on an incremental is the merge/upsert
|
|
51038
|
+
key; \`assertions: { uniqueKey }\` generates a data-quality assertion. They are independent.
|
|
51039
|
+
- Always reference tables with \`\${ref("name")}\` — never hardcoded paths. Declare external
|
|
51040
|
+
tables (\`type: "declaration"\`) before ref'ing them.
|
|
51041
|
+
- Document every table: \`description:\` + \`columns: {}\` compile to real comments.
|
|
51042
|
+
|
|
51043
|
+
## Commands
|
|
51044
|
+
|
|
51045
|
+
\`\`\`bash
|
|
51046
|
+
sqlanvil compile .
|
|
51047
|
+
sqlanvil validate . --credentials .df-credentials.json # planner-checks the whole DAG without executing — run this before every run
|
|
51048
|
+
sqlanvil run . --credentials .df-credentials.json
|
|
51049
|
+
sqlanvil run . --credentials .df-credentials.json --actions <name> --include-deps
|
|
51050
|
+
sqlanvil test . --credentials .df-credentials.json
|
|
51051
|
+
\`\`\`
|
|
51052
|
+
|
|
51053
|
+
\`target/\` holds compile/run artifacts (queryable via \`sqlanvil query\`/\`inspect\`) — keep
|
|
51054
|
+
it gitignored.`);
|
|
51055
|
+
if (isPostgresLike) {
|
|
51056
|
+
sections.push(`## ${isSupabase ? "Supabase/PostgreSQL" : "PostgreSQL"} specifics
|
|
51057
|
+
|
|
51058
|
+
- \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
|
|
51059
|
+
sslMode defaultSchema\` — not nested under a key, not \`username\`/\`databaseName\`/\`ssl\`.${isSupabase
|
|
51060
|
+
? `
|
|
51061
|
+
Use the Session pooler host (\`aws-…pooler.supabase.com\`, user \`postgres.<project-ref>\`,
|
|
51062
|
+
port 5432): the direct \`db.<ref>.supabase.co\` host is IPv6-only and fails on most networks.`
|
|
51063
|
+
: ""}
|
|
51064
|
+
- Storage, indexes, and partitioning are first-class in the \`postgres: {}\` config block —
|
|
51065
|
+
never hand-roll \`CREATE INDEX\` or \`SET (fillfactor …)\` in \`post_operations\`.
|
|
51066
|
+
- Index \`method\` is a NUMERIC enum (\`BTREE=0, HASH=1, GIN=2, GIST=3, BRIN=4\`) — omit it
|
|
51067
|
+
for btree; the string \`method: "btree"\` fails the config check. \`opclass\` is a single
|
|
51068
|
+
string, not an array.
|
|
51069
|
+
- No BigQuery-isms: \`bigquery: {}\`, \`partitionBy\`, \`clusterBy\`, \`OPTIONS(...)\` — use
|
|
51070
|
+
\`postgres: { partition: {...}, indexes: [...] }\` (partition \`kind\`: RANGE=0, LIST=1,
|
|
51071
|
+
HASH=2).
|
|
51072
|
+
- Materialized views: \`type: "view", materialized: true\` (+ optional \`postgres: {
|
|
51073
|
+
refreshPolicy: "on_dependency_change", noData, indexes }\`). Default is drop+recreate each
|
|
51074
|
+
run; in-place refresh does not pick up definition changes.
|
|
51075
|
+
- One-time DDL on incrementals (e.g. \`ADD PRIMARY KEY\`) must be wrapped in
|
|
51076
|
+
\`\${when(!incremental(), \\\`…\\\`)}\` — an unwrapped \`post_operations\` block re-runs on
|
|
51077
|
+
EVERY append and errors the second time.
|
|
51078
|
+
- Read tables in OTHER warehouses via named \`connections:\` in \`workflow_settings.yaml\`
|
|
51079
|
+
plus a \`connection:\`-tagged declaration with \`columnTypes\` in POSTGRES types — scaffold
|
|
51080
|
+
it with \`sqlanvil introspect <connection> <schema.table>\`. Never hand-write FDW/foreign
|
|
51081
|
+
tables.${isSupabase
|
|
51082
|
+
? `
|
|
51083
|
+
- Supabase extras: the \`supabase: {}\` block (\`enableRls\`, \`vectors\`) and dedicated
|
|
51084
|
+
action types \`rlsPolicy\`, \`realtimePublication\`, \`vectorIndex\`. \`enableRls\` only
|
|
51085
|
+
flips RLS on — declare policies via \`rlsPolicy\` actions.`
|
|
51086
|
+
: ""}`);
|
|
51087
|
+
}
|
|
51088
|
+
if (isMysql) {
|
|
51089
|
+
sections.push(`## MySQL/MariaDB specifics (several Postgres rules INVERT here)
|
|
51090
|
+
|
|
51091
|
+
- \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
|
|
51092
|
+
sslMode\` — no \`defaultSchema\` (MySQL's "schema" IS the database).
|
|
51093
|
+
- Use the \`mysql: {}\` config block for indexes, table options (\`engine\`, \`charset\`,
|
|
51094
|
+
\`rowFormat\`), and partitioning — never \`postgres: {}\` (wrong dialect, silently ignored).
|
|
51095
|
+
- An incremental's \`uniqueKey\` compiles to \`INSERT … ON DUPLICATE KEY UPDATE\` and the
|
|
51096
|
+
adapter AUTO-CREATES the matching unique index — do not add your own PK/unique index for
|
|
51097
|
+
the merge.
|
|
51098
|
+
- \`materialized: true\` is an emulated snapshot (drop + CTAS each run), not a native
|
|
51099
|
+
matview; no \`refreshPolicy\`/\`noData\`.
|
|
51100
|
+
- Backtick-quote identifiers in raw DDL; never \`DELIMITER\` (client-only directive — the
|
|
51101
|
+
\`---\` separator already keeps \`;\` bodies intact).
|
|
51102
|
+
- A mysql warehouse cannot read cross-warehouse \`connections:\` (single write warehouse;
|
|
51103
|
+
MySQL works as a SOURCE for a Postgres/Supabase project, not the other way).`);
|
|
51104
|
+
}
|
|
51105
|
+
const commentMechanics = isPostgresLike
|
|
51106
|
+
? "`COMMENT ON TABLE|VIEW|MATERIALIZED VIEW|COLUMN`"
|
|
51107
|
+
: isMysql
|
|
51108
|
+
? "table/column comments (tables + incrementals only — MySQL views can't carry comments)"
|
|
51109
|
+
: "table + column descriptions (nested fields included)";
|
|
51110
|
+
const constraintNote = warehouse === "bigquery"
|
|
51111
|
+
? "append `NOT ENFORCED` (BigQuery stores them as catalog metadata for query engines)"
|
|
51112
|
+
: "real enforced constraints";
|
|
51113
|
+
sections.push(`## Metadata for AI/BI engines
|
|
51114
|
+
|
|
51115
|
+
- \`description:\` + \`columns: {}\` persist into the warehouse catalog on every run as
|
|
51116
|
+
${commentMechanics} — BI copilots and text-to-SQL agents read them back from there. Name
|
|
51117
|
+
join targets in column descriptions (\`customer_id: "Foreign key to
|
|
51118
|
+
public.customers.customer_id."\`).
|
|
51119
|
+
- PK/FK constraints go in \`post_operations\` (no config field for them) — ${constraintNote};
|
|
51120
|
+
on an incremental, wrap the \`ALTER\` in \`\${when(!incremental(), \\\`…\\\`)}\`.
|
|
51121
|
+
- \`assertions: { uniqueKey | uniqueKeys: [[…],[…]], nonNull, rowConditions }\` make key
|
|
51122
|
+
claims TESTED on every run (\`uniqueKey\` and \`uniqueKeys\` are mutually exclusive).
|
|
51123
|
+
- Guide: https://sqlanvil.com/docs/guides/metadata/`);
|
|
51124
|
+
if (converted) {
|
|
51125
|
+
sections.push(`## This project was converted from Dataform
|
|
51126
|
+
|
|
51127
|
+
- Work the to-do list in \`migration-report.md\` (machine-readable twin:
|
|
51128
|
+
\`migration-report.json\`). Inline \`SQLANVIL-MIGRATE:\` markers flag every rewritten or
|
|
51129
|
+
questionable spot.
|
|
51130
|
+
- \`sqlanvil validate\` is the completion loop — converge on all-PASS.
|
|
51131
|
+
- The original Dataform repo was read-only during conversion and remains the rollback path.`);
|
|
51132
|
+
}
|
|
51133
|
+
sections.push(`---
|
|
51134
|
+
|
|
51135
|
+
Docs: https://sqlanvil.com/docs (agent bundle: https://sqlanvil.com/llms.txt) · Full
|
|
51136
|
+
authoring skill: \`npx skills add SQLAnvil/agent-skills\` · Hosted CI/scheduling:
|
|
51137
|
+
https://app.sqlanvil.com (remote MCP server at \`/api/mcp\`).`);
|
|
51138
|
+
return sections.join("\n\n") + "\n";
|
|
51139
|
+
}
|
|
51140
|
+
|
|
50990
51141
|
const gitIgnoreContents = `
|
|
50991
51142
|
${CREDENTIALS_FILENAME}
|
|
50992
51143
|
.df-credentials*.json
|
|
@@ -51084,6 +51235,20 @@ async function init(projectDir, projectConfig, options = {}) {
|
|
|
51084
51235
|
filesWritten.push(workflowSettingsYamlPath);
|
|
51085
51236
|
fs__namespace$1.writeFileSync(gitignorePath, gitIgnoreContents);
|
|
51086
51237
|
filesWritten.push(gitignorePath);
|
|
51238
|
+
const agentsMdPath = path__namespace.join(projectDir, "AGENTS.md");
|
|
51239
|
+
if (!fs__namespace$1.existsSync(agentsMdPath)) {
|
|
51240
|
+
fs__namespace$1.writeFileSync(agentsMdPath, agentsMdContents({
|
|
51241
|
+
warehouse,
|
|
51242
|
+
defaultDataset: workflowSettings.defaultDataset || "public",
|
|
51243
|
+
version
|
|
51244
|
+
}));
|
|
51245
|
+
filesWritten.push(agentsMdPath);
|
|
51246
|
+
}
|
|
51247
|
+
const claudeMdPath = path__namespace.join(projectDir, "CLAUDE.md");
|
|
51248
|
+
if (!fs__namespace$1.existsSync(claudeMdPath)) {
|
|
51249
|
+
fs__namespace$1.writeFileSync(claudeMdPath, claudeMdBridgeContents());
|
|
51250
|
+
filesWritten.push(claudeMdPath);
|
|
51251
|
+
}
|
|
51087
51252
|
if (!isBigQuery) {
|
|
51088
51253
|
fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), (_a = options.credentialsJson) !== null && _a !== void 0 ? _a : (warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse)));
|
|
51089
51254
|
filesWritten.push(path__namespace.join(projectDir, CREDENTIALS_FILENAME));
|
|
@@ -54409,7 +54574,8 @@ ${description} columnTypes: {
|
|
|
54409
54574
|
],
|
|
54410
54575
|
};
|
|
54411
54576
|
}
|
|
54412
|
-
function convertTarget(source, jsMode) {
|
|
54577
|
+
function convertTarget(source, jsMode, dialect = "postgres") {
|
|
54578
|
+
const dialectPass = dialect === "postgres";
|
|
54413
54579
|
const span = jsMode ? null : findConfigBlock(source);
|
|
54414
54580
|
const findings = [];
|
|
54415
54581
|
const lines = source.split("\n");
|
|
@@ -54447,7 +54613,7 @@ function convertTarget(source, jsMode) {
|
|
|
54447
54613
|
line = line.replace(/,/, " ");
|
|
54448
54614
|
}
|
|
54449
54615
|
}
|
|
54450
|
-
if (inConfig && commentingConfigBlockDepth === 0) {
|
|
54616
|
+
if (dialectPass && inConfig && commentingConfigBlockDepth === 0) {
|
|
54451
54617
|
const m = CONFIG_BQ_KEYS.exec(line);
|
|
54452
54618
|
if (m) {
|
|
54453
54619
|
const note = `BigQuery-only setting — not applicable on Postgres (partitioning/clustering → postgres: {}; project qualifiers drop)`;
|
|
@@ -54471,7 +54637,7 @@ function convertTarget(source, jsMode) {
|
|
|
54471
54637
|
out.push(`// ${line}`);
|
|
54472
54638
|
continue;
|
|
54473
54639
|
}
|
|
54474
|
-
if (!inConfig || jsMode) {
|
|
54640
|
+
if (dialectPass && (!inConfig || jsMode)) {
|
|
54475
54641
|
for (const rule of SQL_RULES) {
|
|
54476
54642
|
rule.pattern.lastIndex = 0;
|
|
54477
54643
|
if (!rule.pattern.test(line))
|
|
@@ -54523,6 +54689,7 @@ function readSourceConfig(srcDir) {
|
|
|
54523
54689
|
const s = (k) => (typeof raw[k] === "string" ? raw[k] : null);
|
|
54524
54690
|
return {
|
|
54525
54691
|
defaultProject: (_a = s("defaultProject")) !== null && _a !== void 0 ? _a : s("defaultDatabase"),
|
|
54692
|
+
defaultLocation: s("defaultLocation"),
|
|
54526
54693
|
defaultDataset: (_c = (_b = s("defaultDataset")) !== null && _b !== void 0 ? _b : s("defaultSchema")) !== null && _c !== void 0 ? _c : "dataform",
|
|
54527
54694
|
defaultAssertionDataset: (_d = s("defaultAssertionDataset")) !== null && _d !== void 0 ? _d : s("assertionSchema"),
|
|
54528
54695
|
vars: (_e = raw.vars) !== null && _e !== void 0 ? _e : null,
|
|
@@ -54530,7 +54697,7 @@ function readSourceConfig(srcDir) {
|
|
|
54530
54697
|
};
|
|
54531
54698
|
}
|
|
54532
54699
|
async function migrateDataform(opts) {
|
|
54533
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
54700
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
54534
54701
|
const srcDir = path__namespace.resolve(opts.srcDir);
|
|
54535
54702
|
const outDir = path__namespace.resolve(opts.outDir);
|
|
54536
54703
|
if (containsPath(srcDir, outDir)) {
|
|
@@ -54542,11 +54709,14 @@ async function migrateDataform(opts) {
|
|
|
54542
54709
|
if (fs__namespace$1.existsSync(outDir) && fs__namespace$1.readdirSync(outDir).length > 0) {
|
|
54543
54710
|
throw new Error(`Output directory ${outDir} is not empty.`);
|
|
54544
54711
|
}
|
|
54712
|
+
const targetWarehouse = (_a = opts.targetWarehouse) !== null && _a !== void 0 ? _a : "supabase";
|
|
54713
|
+
const isBigQueryTarget = targetWarehouse === "bigquery";
|
|
54545
54714
|
const sourceConfig = readSourceConfig(srcDir);
|
|
54546
54715
|
const writer = new OutWriter(outDir);
|
|
54547
54716
|
skippedForSafety = new Set();
|
|
54548
54717
|
const files = walkFiles(srcDir);
|
|
54549
54718
|
const report = {
|
|
54719
|
+
targetWarehouse,
|
|
54550
54720
|
generator: `sqlanvil migrate-dataform ${version}`,
|
|
54551
54721
|
sourceDir: srcDir,
|
|
54552
54722
|
sourceConfig: sourceConfig.raw,
|
|
@@ -54568,16 +54738,16 @@ async function migrateDataform(opts) {
|
|
|
54568
54738
|
: { type: null, schema: null, database: null, name: null, description: null };
|
|
54569
54739
|
sqlxFiles.push({ rel, source, parsed });
|
|
54570
54740
|
report.inventory.sqlxFiles++;
|
|
54571
|
-
const t = (
|
|
54572
|
-
report.inventory.byType[t] = ((
|
|
54741
|
+
const t = (_b = parsed.type) !== null && _b !== void 0 ? _b : "(none)";
|
|
54742
|
+
report.inventory.byType[t] = ((_c = report.inventory.byType[t]) !== null && _c !== void 0 ? _c : 0) + 1;
|
|
54573
54743
|
}
|
|
54574
54744
|
const resolveDatabase = (db) => { var _a; return !db || db === "<defaultDatabase>" ? (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "UNKNOWN_PROJECT" : db; };
|
|
54575
54745
|
const declarationSchemas = new Set();
|
|
54576
54746
|
const targetSchemas = new Set();
|
|
54577
54747
|
const connectionMap = new Map();
|
|
54578
|
-
for (const f of sqlxFiles) {
|
|
54748
|
+
for (const f of isBigQueryTarget ? [] : sqlxFiles) {
|
|
54579
54749
|
if (f.parsed.type === "declaration") {
|
|
54580
|
-
const dataset = (
|
|
54750
|
+
const dataset = (_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset;
|
|
54581
54751
|
const project = resolveDatabase(f.parsed.database);
|
|
54582
54752
|
declarationSchemas.add(dataset);
|
|
54583
54753
|
const name = connectionNameFor(project);
|
|
@@ -54598,7 +54768,7 @@ async function migrateDataform(opts) {
|
|
|
54598
54768
|
}
|
|
54599
54769
|
}
|
|
54600
54770
|
else if (f.parsed.type) {
|
|
54601
|
-
targetSchemas.add((
|
|
54771
|
+
targetSchemas.add((_e = f.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset);
|
|
54602
54772
|
}
|
|
54603
54773
|
}
|
|
54604
54774
|
report.connections = [...connectionMap.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
@@ -54610,8 +54780,20 @@ async function migrateDataform(opts) {
|
|
|
54610
54780
|
}
|
|
54611
54781
|
if (rel.endsWith(".sqlx")) {
|
|
54612
54782
|
const info = sqlxFiles.find(f => f.rel === rel);
|
|
54783
|
+
if (isBigQueryTarget) {
|
|
54784
|
+
const { content, findings } = convertTarget(info.source, false, "none");
|
|
54785
|
+
writer.write(rel, content);
|
|
54786
|
+
report.files.push({
|
|
54787
|
+
file: rel,
|
|
54788
|
+
action: info.parsed.type === "declaration" ? "declaration" : "target",
|
|
54789
|
+
type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
|
|
54790
|
+
status: findings.length === 0 ? "clean" : "rewritten",
|
|
54791
|
+
findings,
|
|
54792
|
+
});
|
|
54793
|
+
continue;
|
|
54794
|
+
}
|
|
54613
54795
|
if (info.parsed.type === "declaration") {
|
|
54614
|
-
const dataset = (
|
|
54796
|
+
const dataset = (_g = info.parsed.schema) !== null && _g !== void 0 ? _g : sourceConfig.defaultDataset;
|
|
54615
54797
|
const project = resolveDatabase(info.parsed.database);
|
|
54616
54798
|
const connection = connectionNameFor(project);
|
|
54617
54799
|
const { content, findings } = convertDeclaration(info.source, info.parsed, connection, dataset, rel);
|
|
@@ -54631,7 +54813,7 @@ async function migrateDataform(opts) {
|
|
|
54631
54813
|
report.files.push({
|
|
54632
54814
|
file: rel,
|
|
54633
54815
|
action: "target",
|
|
54634
|
-
type: (
|
|
54816
|
+
type: (_h = info.parsed.type) !== null && _h !== void 0 ? _h : "(none)",
|
|
54635
54817
|
status: findings.length === 0
|
|
54636
54818
|
? "clean"
|
|
54637
54819
|
: findings.some(f => f.kind === "flag")
|
|
@@ -54645,10 +54827,10 @@ async function migrateDataform(opts) {
|
|
|
54645
54827
|
if (rel.endsWith(".js") && (rel.startsWith("definitions/") || rel.startsWith("includes/"))) {
|
|
54646
54828
|
report.inventory.jsFiles++;
|
|
54647
54829
|
const source = fs__namespace$1.readFileSync(abs, "utf8");
|
|
54648
|
-
const { content, findings } = convertTarget(source, true);
|
|
54830
|
+
const { content, findings } = convertTarget(source, true, isBigQueryTarget ? "none" : "postgres");
|
|
54649
54831
|
writer.write(rel, content);
|
|
54650
54832
|
const isIncludes = rel.startsWith("includes/");
|
|
54651
|
-
if (isIncludes && findings.length > 0) {
|
|
54833
|
+
if (isIncludes && findings.length > 0 && !isBigQueryTarget) {
|
|
54652
54834
|
report.warnings.push(`includes helper ${rel} emits BigQuery-flavored SQL — review its ${findings.length} finding(s)`);
|
|
54653
54835
|
}
|
|
54654
54836
|
report.files.push({
|
|
@@ -54667,11 +54849,12 @@ async function migrateDataform(opts) {
|
|
|
54667
54849
|
writer.write(rel, fs__namespace$1.readFileSync(abs));
|
|
54668
54850
|
report.files.push({ file: rel, action: "copied", status: "clean", findings: [] });
|
|
54669
54851
|
}
|
|
54670
|
-
const settings =
|
|
54671
|
-
|
|
54852
|
+
const settings = isBigQueryTarget
|
|
54853
|
+
? Object.assign(Object.assign(Object.assign({}, (sourceConfig.defaultProject ? { defaultProject: sourceConfig.defaultProject } : {})), (sourceConfig.defaultLocation ? { defaultLocation: sourceConfig.defaultLocation } : {})), { defaultDataset: sourceConfig.defaultDataset, defaultAssertionDataset: (_j = sourceConfig.defaultAssertionDataset) !== null && _j !== void 0 ? _j : "sqlanvil_assertions", sqlanvilCoreVersion: (_k = opts.coreVersion) !== null && _k !== void 0 ? _k : version }) : {
|
|
54854
|
+
warehouse: targetWarehouse,
|
|
54672
54855
|
defaultDataset: sourceConfig.defaultDataset.toLowerCase(),
|
|
54673
|
-
defaultAssertionDataset: ((
|
|
54674
|
-
sqlanvilCoreVersion: (
|
|
54856
|
+
defaultAssertionDataset: ((_l = sourceConfig.defaultAssertionDataset) !== null && _l !== void 0 ? _l : "sqlanvil_assertions").toLowerCase(),
|
|
54857
|
+
sqlanvilCoreVersion: (_m = opts.coreVersion) !== null && _m !== void 0 ? _m : version,
|
|
54675
54858
|
};
|
|
54676
54859
|
if (sourceConfig.vars)
|
|
54677
54860
|
settings.vars = sourceConfig.vars;
|
|
@@ -54690,6 +54873,22 @@ async function migrateDataform(opts) {
|
|
|
54690
54873
|
}));
|
|
54691
54874
|
}
|
|
54692
54875
|
writer.write("workflow_settings.yaml", jsYaml.dump(settings));
|
|
54876
|
+
if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "AGENTS.md"))) {
|
|
54877
|
+
writer.write("AGENTS.md", agentsMdContents({
|
|
54878
|
+
warehouse: targetWarehouse,
|
|
54879
|
+
defaultDataset: settings.defaultDataset,
|
|
54880
|
+
version: (_o = opts.coreVersion) !== null && _o !== void 0 ? _o : version,
|
|
54881
|
+
converted: true
|
|
54882
|
+
}));
|
|
54883
|
+
report.files.push({ file: "AGENTS.md", action: "generated", status: "clean", findings: [] });
|
|
54884
|
+
}
|
|
54885
|
+
else {
|
|
54886
|
+
report.warnings.push("Source AGENTS.md copied as-is — its guidance targets Dataform; consider replacing it with the sqlanvil version (`sqlanvil init --bare` in an empty dir generates one).");
|
|
54887
|
+
}
|
|
54888
|
+
if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "CLAUDE.md"))) {
|
|
54889
|
+
writer.write("CLAUDE.md", claudeMdBridgeContents());
|
|
54890
|
+
report.files.push({ file: "CLAUDE.md", action: "generated", status: "clean", findings: [] });
|
|
54891
|
+
}
|
|
54693
54892
|
report.skippedForSafety = [...skippedForSafety].sort();
|
|
54694
54893
|
if (skippedForSafety.size > 0) {
|
|
54695
54894
|
report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
|
|
@@ -54716,28 +54915,38 @@ function renderReportMd(r) {
|
|
|
54716
54915
|
lines.push(` - ${t}: ${n}`);
|
|
54717
54916
|
}
|
|
54718
54917
|
lines.push("");
|
|
54719
|
-
|
|
54720
|
-
|
|
54721
|
-
|
|
54722
|
-
`
|
|
54723
|
-
|
|
54724
|
-
|
|
54725
|
-
|
|
54726
|
-
lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
|
|
54727
|
-
`${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
|
|
54728
|
-
(c.datasets.length > 8 ? ", …" : ""));
|
|
54729
|
-
lines.push(` - \`${c.introspectExample}\``);
|
|
54918
|
+
if (r.targetWarehouse === "bigquery") {
|
|
54919
|
+
lines.push(`## Same-warehouse conversion (BigQuery → BigQuery)`);
|
|
54920
|
+
lines.push("");
|
|
54921
|
+
lines.push(`This is a tooling swap: the project keeps running on the SAME BigQuery warehouse. SQL ` +
|
|
54922
|
+
`bodies, \`bigquery: {}\` config blocks, and declarations passed through untouched; ` +
|
|
54923
|
+
`only the compile global was renamed (\`dataform.projectConfig\` → ` +
|
|
54924
|
+
`\`sqlanvil.projectConfig\`) and the settings file converted.`);
|
|
54730
54925
|
}
|
|
54731
|
-
|
|
54926
|
+
else {
|
|
54927
|
+
lines.push(`## Source connections (${r.connections.length})`);
|
|
54732
54928
|
lines.push("");
|
|
54733
|
-
lines.push(
|
|
54929
|
+
lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
|
|
54930
|
+
`(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
|
|
54931
|
+
`before it can extract; scaffold them with \`sqlanvil introspect\`:`);
|
|
54734
54932
|
lines.push("");
|
|
54735
|
-
|
|
54736
|
-
.
|
|
54737
|
-
|
|
54738
|
-
|
|
54739
|
-
`
|
|
54740
|
-
|
|
54933
|
+
for (const c of r.connections) {
|
|
54934
|
+
lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
|
|
54935
|
+
`${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
|
|
54936
|
+
(c.datasets.length > 8 ? ", …" : ""));
|
|
54937
|
+
lines.push(` - \`${c.introspectExample}\``);
|
|
54938
|
+
}
|
|
54939
|
+
if (r.overlappingSchemas.length > 0) {
|
|
54940
|
+
lines.push("");
|
|
54941
|
+
lines.push(`## ⚠ Schemas that are BOTH source and target`);
|
|
54942
|
+
lines.push("");
|
|
54943
|
+
lines.push(`These schemas have declarations AND materializing actions: ${r.overlappingSchemas
|
|
54944
|
+
.map(s => `\`${s}\``)
|
|
54945
|
+
.join(", ")}. ` +
|
|
54946
|
+
`Their declared tables are probably produced by this pipeline (or a sibling) — once those ` +
|
|
54947
|
+
`producers move to Postgres, convert the declarations to plain local declarations instead ` +
|
|
54948
|
+
`of BigQuery connections.`);
|
|
54949
|
+
}
|
|
54741
54950
|
}
|
|
54742
54951
|
lines.push("");
|
|
54743
54952
|
lines.push(`## Target files (${targets.length})`);
|
|
@@ -54759,7 +54968,9 @@ function renderReportMd(r) {
|
|
|
54759
54968
|
lines.push("");
|
|
54760
54969
|
lines.push(`## Declarations (${declarations.length})`);
|
|
54761
54970
|
lines.push("");
|
|
54762
|
-
lines.push(
|
|
54971
|
+
lines.push(r.targetWarehouse === "bigquery"
|
|
54972
|
+
? `Declarations are unchanged — the sources are native to the warehouse.`
|
|
54973
|
+
: `Every declaration was rewritten to ride its connection. The BigQuery data does not move.`);
|
|
54763
54974
|
if (r.warnings.length > 0) {
|
|
54764
54975
|
lines.push("");
|
|
54765
54976
|
lines.push(`## Warnings`);
|
|
@@ -54770,12 +54981,23 @@ function renderReportMd(r) {
|
|
|
54770
54981
|
lines.push("");
|
|
54771
54982
|
lines.push(`## Finish the migration`);
|
|
54772
54983
|
lines.push("");
|
|
54773
|
-
|
|
54774
|
-
|
|
54775
|
-
|
|
54776
|
-
|
|
54777
|
-
|
|
54778
|
-
|
|
54984
|
+
if (r.targetWarehouse === "bigquery") {
|
|
54985
|
+
lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
|
|
54986
|
+
lines.push(`2. Credentials: \`gcloud auth application-default login\` (ADC), or a service-account ` +
|
|
54987
|
+
`key in a gitignored \`.df-credentials.json\`.`);
|
|
54988
|
+
lines.push(`3. \`sqlanvil validate\` — dry-runs every model against BigQuery without executing; ` +
|
|
54989
|
+
`all-PASS means the swap is complete.`);
|
|
54990
|
+
lines.push(`4. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
|
|
54991
|
+
`sample of actions — they should match apart from the compile-global rename.`);
|
|
54992
|
+
}
|
|
54993
|
+
else {
|
|
54994
|
+
lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
|
|
54995
|
+
lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
|
|
54996
|
+
lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
|
|
54997
|
+
`model is the migration to-do list. Work the flagged files until validate is green.`);
|
|
54998
|
+
lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
|
|
54999
|
+
`the \`SQLANVIL-MIGRATE:\` markers is the fastest path through the dialect work.`);
|
|
55000
|
+
}
|
|
54779
55001
|
lines.push("");
|
|
54780
55002
|
return lines.join("\n");
|
|
54781
55003
|
}
|
|
@@ -55316,6 +55538,7 @@ const CREDS_PASSWORD_QUESTION = "Password? (leave empty to fill in later)";
|
|
|
55316
55538
|
const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
|
|
55317
55539
|
const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
|
|
55318
55540
|
const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
|
|
55541
|
+
const CONVERT_TARGET_QUESTION = "Target warehouse? (bigquery = keep running on BigQuery, tooling swap only; supabase/postgres = move off BigQuery)";
|
|
55319
55542
|
const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
|
|
55320
55543
|
"aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
|
|
55321
55544
|
"db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
|
|
@@ -55434,13 +55657,22 @@ async function runConvertFlow() {
|
|
|
55434
55657
|
throw new Error("No usable source directory provided.");
|
|
55435
55658
|
}
|
|
55436
55659
|
const outDir = actuallyResolve(askRequired(CONVERT_OUT_QUESTION));
|
|
55660
|
+
const targetWarehouse = askChoice(CONVERT_TARGET_QUESTION, ["bigquery", "supabase", "postgres"], "bigquery");
|
|
55437
55661
|
print("\nConverting...\n");
|
|
55438
|
-
const report = await migrateDataform({ srcDir, outDir });
|
|
55662
|
+
const report = await migrateDataform({ srcDir, outDir, targetWarehouse });
|
|
55439
55663
|
printMigrationSummary(report, outDir);
|
|
55664
|
+
if (targetWarehouse === "bigquery") {
|
|
55665
|
+
print("\nNext steps:");
|
|
55666
|
+
print(` 1. sqlanvil compile ${outDir}`);
|
|
55667
|
+
print(" 2. Credentials: gcloud auth application-default login (ADC), or a service-account " +
|
|
55668
|
+
`key in a gitignored ${CREDENTIALS_FILENAME}.`);
|
|
55669
|
+
print(` 3. sqlanvil validate ${outDir} (dry-runs every model against BigQuery; all-PASS = swap complete)`);
|
|
55670
|
+
return 0;
|
|
55671
|
+
}
|
|
55440
55672
|
if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
|
|
55441
55673
|
const credentialsPath = path__namespace.join(outDir, CREDENTIALS_FILENAME);
|
|
55442
55674
|
ensureGitignoreCoversCredentials(outDir);
|
|
55443
|
-
fs__namespace$1.writeFileSync(credentialsPath, collectCredentials(
|
|
55675
|
+
fs__namespace$1.writeFileSync(credentialsPath, collectCredentials(targetWarehouse));
|
|
55444
55676
|
printSuccess(`Credentials written to ${credentialsPath}`);
|
|
55445
55677
|
}
|
|
55446
55678
|
print("\nNext steps:");
|
|
@@ -56829,6 +57061,14 @@ const interactiveOption = option("interactive", {
|
|
|
56829
57061
|
type: "boolean",
|
|
56830
57062
|
default: false
|
|
56831
57063
|
});
|
|
57064
|
+
const targetWarehouseOption = option("target-warehouse", {
|
|
57065
|
+
describe: "Where the CONVERTED project runs: supabase/postgres = move off BigQuery (sources become " +
|
|
57066
|
+
"named connections, SQL gets the dialect pass); bigquery = keep the SAME warehouse — a " +
|
|
57067
|
+
"tooling swap with SQL and bigquery:{} blocks untouched.",
|
|
57068
|
+
type: "string",
|
|
57069
|
+
choices: ["supabase", "postgres", "bigquery"],
|
|
57070
|
+
default: "supabase"
|
|
57071
|
+
});
|
|
56832
57072
|
const warehouseOption = option("warehouse", {
|
|
56833
57073
|
describe: "Target warehouse for the new project.",
|
|
56834
57074
|
type: "string",
|
|
@@ -57695,10 +57935,12 @@ function runCli() {
|
|
|
57695
57935
|
},
|
|
57696
57936
|
{
|
|
57697
57937
|
format: "migrate-dataform <source-dir> <out-dir>",
|
|
57698
|
-
description: "Convert a Dataform/BigQuery project
|
|
57699
|
-
"
|
|
57700
|
-
"
|
|
57701
|
-
"gets safe rewrites + inline SQLANVIL-MIGRATE markers
|
|
57938
|
+
description: "Convert a Dataform/BigQuery project to sqlanvil. The source directory is READ-ONLY; " +
|
|
57939
|
+
"the converted project + migration-report.{md,json} land in <out-dir> (must be empty). " +
|
|
57940
|
+
"Default target (supabase/postgres) MOVES the warehouse: sources stay in BigQuery as " +
|
|
57941
|
+
"named connections and target SQL gets safe rewrites + inline SQLANVIL-MIGRATE markers. " +
|
|
57942
|
+
"--target-warehouse bigquery is a TOOLING SWAP: the project keeps running on the same " +
|
|
57943
|
+
"BigQuery warehouse, SQL and bigquery:{} blocks untouched.",
|
|
57702
57944
|
positionalOptions: [
|
|
57703
57945
|
positionalOption("source-dir", {
|
|
57704
57946
|
describe: "The Dataform project to convert (never modified)."
|
|
@@ -57707,11 +57949,12 @@ function runCli() {
|
|
|
57707
57949
|
describe: "Where the converted sqlanvil project is written (created; must be empty)."
|
|
57708
57950
|
})
|
|
57709
57951
|
],
|
|
57710
|
-
options: [],
|
|
57952
|
+
options: [targetWarehouseOption],
|
|
57711
57953
|
processFn: async (argv) => {
|
|
57712
57954
|
const report = await migrateDataform({
|
|
57713
57955
|
srcDir: argv["source-dir"],
|
|
57714
|
-
outDir: argv["out-dir"]
|
|
57956
|
+
outDir: argv["out-dir"],
|
|
57957
|
+
targetWarehouse: argv[targetWarehouseOption.name]
|
|
57715
57958
|
});
|
|
57716
57959
|
printMigrationSummary(report, argv["out-dir"]);
|
|
57717
57960
|
print(`Next: sqlanvil compile ${argv["out-dir"]}`);
|
package/package.json
CHANGED
|
@@ -46,7 +46,14 @@
|
|
|
46
46
|
"**/wrap-ansi": "7.0.0",
|
|
47
47
|
"**/string-width": "4.1.0"
|
|
48
48
|
},
|
|
49
|
-
"homepage": "https://
|
|
49
|
+
"homepage": "https://sqlanvil.com",
|
|
50
|
+
"repository": {
|
|
51
|
+
"type": "git",
|
|
52
|
+
"url": "git+https://github.com/sqlanvil/sqlanvil.git"
|
|
53
|
+
},
|
|
54
|
+
"bugs": {
|
|
55
|
+
"url": "https://github.com/sqlanvil/sqlanvil/issues"
|
|
56
|
+
},
|
|
50
57
|
"license": "Apache-2.0",
|
|
51
58
|
"keywords": [
|
|
52
59
|
"sqlanvil",
|
|
@@ -64,7 +71,7 @@
|
|
|
64
71
|
"bin": {
|
|
65
72
|
"sqlanvil": "bundle.js"
|
|
66
73
|
},
|
|
67
|
-
"version": "1.
|
|
74
|
+
"version": "1.25.0",
|
|
68
75
|
"name": "@sqlanvil/cli",
|
|
69
76
|
"description": "sqlanvil command line interface.",
|
|
70
77
|
"main": "bundle.js"
|