@sqlanvil/cli 1.22.0 → 1.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bundle.js +1358 -888
- package/package.json +1 -1
package/bundle.js
CHANGED
|
@@ -49315,10 +49315,25 @@ function unknownToValueShallow(raw) {
|
|
|
49315
49315
|
})();
|
|
49316
49316
|
|
|
49317
49317
|
const nativeRequire = typeof __webpack_require__ === "function" ? __non_webpack_require__ : require;
|
|
49318
|
+
function globToRegExp(pattern) {
|
|
49319
|
+
const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&").replace(/\\\*/g, ".*");
|
|
49320
|
+
return new RegExp(`^${escaped}$`);
|
|
49321
|
+
}
|
|
49318
49322
|
function matchPatterns(patterns, values) {
|
|
49319
49323
|
const fullyQualifiedActions = [];
|
|
49320
49324
|
patterns.forEach(pattern => {
|
|
49321
|
-
if (pattern.includes("
|
|
49325
|
+
if (pattern.includes("*")) {
|
|
49326
|
+
const regExp = globToRegExp(pattern);
|
|
49327
|
+
const scope = pattern.includes(".")
|
|
49328
|
+
? values
|
|
49329
|
+
: values.map(value => value.split(".").slice(-1)[0]);
|
|
49330
|
+
values.forEach((value, i) => {
|
|
49331
|
+
if (regExp.test(scope[i])) {
|
|
49332
|
+
fullyQualifiedActions.push(value);
|
|
49333
|
+
}
|
|
49334
|
+
});
|
|
49335
|
+
}
|
|
49336
|
+
else if (pattern.includes(".")) {
|
|
49322
49337
|
if (values.includes(pattern)) {
|
|
49323
49338
|
fullyQualifiedActions.push(pattern);
|
|
49324
49339
|
}
|
|
@@ -50606,7 +50621,7 @@ function collectEvaluationQueries(queryOrAction, concatenate, queryModifier = (q
|
|
|
50606
50621
|
.filter(validationQuery => !!validationQuery.query);
|
|
50607
50622
|
}
|
|
50608
50623
|
|
|
50609
|
-
const version = "1.
|
|
50624
|
+
const version = "1.24.0";
|
|
50610
50625
|
const dataformVersion = "3.0.61";
|
|
50611
50626
|
|
|
50612
50627
|
async function build(compiledGraph, runConfig, dbadapter) {
|
|
@@ -50987,6 +51002,123 @@ async function test$1(dbadapter, timeoutMs = 10000) {
|
|
|
50987
51002
|
}
|
|
50988
51003
|
}
|
|
50989
51004
|
|
|
51005
|
+
function claudeMdBridgeContents() {
|
|
51006
|
+
return "@AGENTS.md\n";
|
|
51007
|
+
}
|
|
51008
|
+
function agentsMdContents(options) {
|
|
51009
|
+
const { warehouse, defaultDataset, version, converted } = options;
|
|
51010
|
+
const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
|
|
51011
|
+
const isSupabase = warehouse === "supabase";
|
|
51012
|
+
const isMysql = warehouse === "mysql";
|
|
51013
|
+
const sections = [];
|
|
51014
|
+
sections.push(`<!-- generated by sqlanvil ${version} — this file is yours; edit freely -->
|
|
51015
|
+
|
|
51016
|
+
# SQLAnvil project — guide for coding agents
|
|
51017
|
+
|
|
51018
|
+
This is a **SQLAnvil** data project (warehouse: **${warehouse}**, default dataset/schema:
|
|
51019
|
+
\`${defaultDataset}\`). SQLAnvil is a fork of Dataform, so Dataform instincts are mostly
|
|
51020
|
+
right: \`.sqlx\` files, \`config {}\` blocks, \`\${ref()}\`, \`\${self()}\`, declarations,
|
|
51021
|
+
assertions, tags, incremental tables, and \`pre_operations\`/\`post_operations\` all work the
|
|
51022
|
+
same. The rules below are where the differences bite.
|
|
51023
|
+
|
|
51024
|
+
## The essentials
|
|
51025
|
+
|
|
51026
|
+
- The CLI is \`sqlanvil\` (\`npm i -g @sqlanvil/cli\`) — there is no \`dataform\` CLI and no
|
|
51027
|
+
npm scripts in this project.
|
|
51028
|
+
- \`workflow_settings.yaml\` pins \`sqlanvilCoreVersion:\` — sqlanvil's OWN SemVer line, NOT
|
|
51029
|
+
\`dataformCoreVersion:\`.
|
|
51030
|
+
- Credentials live in the gitignored \`.df-credentials.json\`. Never commit it; never put
|
|
51031
|
+
secrets in \`workflow_settings.yaml\`.
|
|
51032
|
+
- The statement separator in \`operations\` / \`pre_operations\` / \`post_operations\` is
|
|
51033
|
+
\`---\` on its own line — NEVER \`;\`. Dollar-quoted bodies (\`$$ … ; … $$\`) survive
|
|
51034
|
+
intact. A \`table\`/\`view\` model body is exactly ONE \`SELECT\` — no \`---\`.
|
|
51035
|
+
- Procedures, functions, triggers, grants, extensions → \`type: "operations"\` (there is no
|
|
51036
|
+
dedicated action type).
|
|
51037
|
+
- Two different \`uniqueKey\`s: top-level \`uniqueKey\` on an incremental is the merge/upsert
|
|
51038
|
+
key; \`assertions: { uniqueKey }\` generates a data-quality assertion. They are independent.
|
|
51039
|
+
- Always reference tables with \`\${ref("name")}\` — never hardcoded paths. Declare external
|
|
51040
|
+
tables (\`type: "declaration"\`) before ref'ing them.
|
|
51041
|
+
- Document every table: \`description:\` + \`columns: {}\` compile to real comments.
|
|
51042
|
+
|
|
51043
|
+
## Commands
|
|
51044
|
+
|
|
51045
|
+
\`\`\`bash
|
|
51046
|
+
sqlanvil compile .
|
|
51047
|
+
sqlanvil validate . --credentials .df-credentials.json # planner-checks the whole DAG without executing — run this before every run
|
|
51048
|
+
sqlanvil run . --credentials .df-credentials.json
|
|
51049
|
+
sqlanvil run . --credentials .df-credentials.json --actions <name> --include-deps
|
|
51050
|
+
sqlanvil test . --credentials .df-credentials.json
|
|
51051
|
+
\`\`\`
|
|
51052
|
+
|
|
51053
|
+
\`target/\` holds compile/run artifacts (queryable via \`sqlanvil query\`/\`inspect\`) — keep
|
|
51054
|
+
it gitignored.`);
|
|
51055
|
+
if (isPostgresLike) {
|
|
51056
|
+
sections.push(`## ${isSupabase ? "Supabase/PostgreSQL" : "PostgreSQL"} specifics
|
|
51057
|
+
|
|
51058
|
+
- \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
|
|
51059
|
+
sslMode defaultSchema\` — not nested under a key, not \`username\`/\`databaseName\`/\`ssl\`.${isSupabase
|
|
51060
|
+
? `
|
|
51061
|
+
Use the Session pooler host (\`aws-…pooler.supabase.com\`, user \`postgres.<project-ref>\`,
|
|
51062
|
+
port 5432): the direct \`db.<ref>.supabase.co\` host is IPv6-only and fails on most networks.`
|
|
51063
|
+
: ""}
|
|
51064
|
+
- Storage, indexes, and partitioning are first-class in the \`postgres: {}\` config block —
|
|
51065
|
+
never hand-roll \`CREATE INDEX\` or \`SET (fillfactor …)\` in \`post_operations\`.
|
|
51066
|
+
- Index \`method\` is a NUMERIC enum (\`BTREE=0, HASH=1, GIN=2, GIST=3, BRIN=4\`) — omit it
|
|
51067
|
+
for btree; the string \`method: "btree"\` fails the config check. \`opclass\` is a single
|
|
51068
|
+
string, not an array.
|
|
51069
|
+
- No BigQuery-isms: \`bigquery: {}\`, \`partitionBy\`, \`clusterBy\`, \`OPTIONS(...)\` — use
|
|
51070
|
+
\`postgres: { partition: {...}, indexes: [...] }\` (partition \`kind\`: RANGE=0, LIST=1,
|
|
51071
|
+
HASH=2).
|
|
51072
|
+
- Materialized views: \`type: "view", materialized: true\` (+ optional \`postgres: {
|
|
51073
|
+
refreshPolicy: "on_dependency_change", noData, indexes }\`). Default is drop+recreate each
|
|
51074
|
+
run; in-place refresh does not pick up definition changes.
|
|
51075
|
+
- One-time DDL on incrementals (e.g. \`ADD PRIMARY KEY\`) must be wrapped in
|
|
51076
|
+
\`\${when(!incremental(), \\\`…\\\`)}\` — an unwrapped \`post_operations\` block re-runs on
|
|
51077
|
+
EVERY append and errors the second time.
|
|
51078
|
+
- Read tables in OTHER warehouses via named \`connections:\` in \`workflow_settings.yaml\`
|
|
51079
|
+
plus a \`connection:\`-tagged declaration with \`columnTypes\` in POSTGRES types — scaffold
|
|
51080
|
+
it with \`sqlanvil introspect <connection> <schema.table>\`. Never hand-write FDW/foreign
|
|
51081
|
+
tables.${isSupabase
|
|
51082
|
+
? `
|
|
51083
|
+
- Supabase extras: the \`supabase: {}\` block (\`enableRls\`, \`vectors\`) and dedicated
|
|
51084
|
+
action types \`rlsPolicy\`, \`realtimePublication\`, \`vectorIndex\`. \`enableRls\` only
|
|
51085
|
+
flips RLS on — declare policies via \`rlsPolicy\` actions.`
|
|
51086
|
+
: ""}`);
|
|
51087
|
+
}
|
|
51088
|
+
if (isMysql) {
|
|
51089
|
+
sections.push(`## MySQL/MariaDB specifics (several Postgres rules INVERT here)
|
|
51090
|
+
|
|
51091
|
+
- \`.df-credentials.json\` is a FLAT object with exactly \`host port database user password
|
|
51092
|
+
sslMode\` — no \`defaultSchema\` (MySQL's "schema" IS the database).
|
|
51093
|
+
- Use the \`mysql: {}\` config block for indexes, table options (\`engine\`, \`charset\`,
|
|
51094
|
+
\`rowFormat\`), and partitioning — never \`postgres: {}\` (wrong dialect, silently ignored).
|
|
51095
|
+
- An incremental's \`uniqueKey\` compiles to \`INSERT … ON DUPLICATE KEY UPDATE\` and the
|
|
51096
|
+
adapter AUTO-CREATES the matching unique index — do not add your own PK/unique index for
|
|
51097
|
+
the merge.
|
|
51098
|
+
- \`materialized: true\` is an emulated snapshot (drop + CTAS each run), not a native
|
|
51099
|
+
matview; no \`refreshPolicy\`/\`noData\`.
|
|
51100
|
+
- Backtick-quote identifiers in raw DDL; never \`DELIMITER\` (client-only directive — the
|
|
51101
|
+
\`---\` separator already keeps \`;\` bodies intact).
|
|
51102
|
+
- A mysql warehouse cannot read cross-warehouse \`connections:\` (single write warehouse;
|
|
51103
|
+
MySQL works as a SOURCE for a Postgres/Supabase project, not the other way).`);
|
|
51104
|
+
}
|
|
51105
|
+
if (converted) {
|
|
51106
|
+
sections.push(`## This project was converted from Dataform
|
|
51107
|
+
|
|
51108
|
+
- Work the to-do list in \`migration-report.md\` (machine-readable twin:
|
|
51109
|
+
\`migration-report.json\`). Inline \`SQLANVIL-MIGRATE:\` markers flag every rewritten or
|
|
51110
|
+
questionable spot.
|
|
51111
|
+
- \`sqlanvil validate\` is the completion loop — converge on all-PASS.
|
|
51112
|
+
- The original Dataform repo was read-only during conversion and remains the rollback path.`);
|
|
51113
|
+
}
|
|
51114
|
+
sections.push(`---
|
|
51115
|
+
|
|
51116
|
+
Docs: https://sqlanvil.com/docs (agent bundle: https://sqlanvil.com/llms.txt) · Full
|
|
51117
|
+
authoring skill: \`npx skills add SQLAnvil/agent-skills\` · Hosted CI/scheduling:
|
|
51118
|
+
https://app.sqlanvil.com (remote MCP server at \`/api/mcp\`).`);
|
|
51119
|
+
return sections.join("\n\n") + "\n";
|
|
51120
|
+
}
|
|
51121
|
+
|
|
50990
51122
|
const gitIgnoreContents = `
|
|
50991
51123
|
${CREDENTIALS_FILENAME}
|
|
50992
51124
|
.df-credentials*.json
|
|
@@ -51016,7 +51148,8 @@ function mysqlCredentialsTemplate() {
|
|
|
51016
51148
|
};
|
|
51017
51149
|
return `${JSON.stringify(template, null, 2)}\n`;
|
|
51018
51150
|
}
|
|
51019
|
-
async function init(projectDir, projectConfig) {
|
|
51151
|
+
async function init(projectDir, projectConfig, options = {}) {
|
|
51152
|
+
var _a;
|
|
51020
51153
|
const workflowSettingsYamlPath = path__namespace.join(projectDir, "workflow_settings.yaml");
|
|
51021
51154
|
const packageJsonPath = path__namespace.join(projectDir, "package.json");
|
|
51022
51155
|
const gitignorePath = path__namespace.join(projectDir, ".gitignore");
|
|
@@ -51064,7 +51197,11 @@ async function init(projectDir, projectConfig) {
|
|
|
51064
51197
|
if (projectConfig.builtinAssertionNamePrefix) {
|
|
51065
51198
|
workflowSettings.builtinAssertionNamePrefix = projectConfig.builtinAssertionNamePrefix;
|
|
51066
51199
|
}
|
|
51067
|
-
|
|
51200
|
+
const includeSample = options.includeSample !== false;
|
|
51201
|
+
const withBigQuerySource = includeSample &&
|
|
51202
|
+
warehouse !== "mysql" &&
|
|
51203
|
+
(!isPostgresLike || options.includeBigQuerySource !== false);
|
|
51204
|
+
if (isPostgresLike && withBigQuerySource) {
|
|
51068
51205
|
workflowSettings.connections = {
|
|
51069
51206
|
bigquery_public: sqlanvil.ConnectionConfig.create({
|
|
51070
51207
|
platform: "bigquery",
|
|
@@ -51079,8 +51216,22 @@ async function init(projectDir, projectConfig) {
|
|
|
51079
51216
|
filesWritten.push(workflowSettingsYamlPath);
|
|
51080
51217
|
fs__namespace$1.writeFileSync(gitignorePath, gitIgnoreContents);
|
|
51081
51218
|
filesWritten.push(gitignorePath);
|
|
51219
|
+
const agentsMdPath = path__namespace.join(projectDir, "AGENTS.md");
|
|
51220
|
+
if (!fs__namespace$1.existsSync(agentsMdPath)) {
|
|
51221
|
+
fs__namespace$1.writeFileSync(agentsMdPath, agentsMdContents({
|
|
51222
|
+
warehouse,
|
|
51223
|
+
defaultDataset: workflowSettings.defaultDataset || "public",
|
|
51224
|
+
version
|
|
51225
|
+
}));
|
|
51226
|
+
filesWritten.push(agentsMdPath);
|
|
51227
|
+
}
|
|
51228
|
+
const claudeMdPath = path__namespace.join(projectDir, "CLAUDE.md");
|
|
51229
|
+
if (!fs__namespace$1.existsSync(claudeMdPath)) {
|
|
51230
|
+
fs__namespace$1.writeFileSync(claudeMdPath, claudeMdBridgeContents());
|
|
51231
|
+
filesWritten.push(claudeMdPath);
|
|
51232
|
+
}
|
|
51082
51233
|
if (!isBigQuery) {
|
|
51083
|
-
fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse));
|
|
51234
|
+
fs__namespace$1.writeFileSync(path__namespace.join(projectDir, CREDENTIALS_FILENAME), (_a = options.credentialsJson) !== null && _a !== void 0 ? _a : (warehouse === "mysql" ? mysqlCredentialsTemplate() : postgresCredentialsTemplate(warehouse)));
|
|
51084
51235
|
filesWritten.push(path__namespace.join(projectDir, CREDENTIALS_FILENAME));
|
|
51085
51236
|
}
|
|
51086
51237
|
const mkdir = (...segments) => {
|
|
@@ -51101,7 +51252,10 @@ async function init(projectDir, projectConfig) {
|
|
|
51101
51252
|
const reportingDir = mkdir("definitions", "outputs", "reporting");
|
|
51102
51253
|
const testDir = mkdir("definitions", "test");
|
|
51103
51254
|
keep(mkdir("includes"));
|
|
51104
|
-
|
|
51255
|
+
if (!includeSample) {
|
|
51256
|
+
[sourcesDir, intermediateDir, salesDir, reportingDir, testDir].forEach(keep);
|
|
51257
|
+
return { filesWritten, dirsCreated };
|
|
51258
|
+
}
|
|
51105
51259
|
write(path__namespace.join(sourcesDir, "app_orders.sqlx"), `-- A declaration: a table that already exists in your warehouse (created by your
|
|
51106
51260
|
-- application, an import, or another pipeline). Point name/dataset at a real table —
|
|
51107
51261
|
-- downstream models read it with \${ref("app_orders")}.
|
|
@@ -54401,7 +54555,8 @@ ${description} columnTypes: {
|
|
|
54401
54555
|
],
|
|
54402
54556
|
};
|
|
54403
54557
|
}
|
|
54404
|
-
function convertTarget(source, jsMode) {
|
|
54558
|
+
function convertTarget(source, jsMode, dialect = "postgres") {
|
|
54559
|
+
const dialectPass = dialect === "postgres";
|
|
54405
54560
|
const span = jsMode ? null : findConfigBlock(source);
|
|
54406
54561
|
const findings = [];
|
|
54407
54562
|
const lines = source.split("\n");
|
|
@@ -54439,7 +54594,7 @@ function convertTarget(source, jsMode) {
|
|
|
54439
54594
|
line = line.replace(/,/, " ");
|
|
54440
54595
|
}
|
|
54441
54596
|
}
|
|
54442
|
-
if (inConfig && commentingConfigBlockDepth === 0) {
|
|
54597
|
+
if (dialectPass && inConfig && commentingConfigBlockDepth === 0) {
|
|
54443
54598
|
const m = CONFIG_BQ_KEYS.exec(line);
|
|
54444
54599
|
if (m) {
|
|
54445
54600
|
const note = `BigQuery-only setting — not applicable on Postgres (partitioning/clustering → postgres: {}; project qualifiers drop)`;
|
|
@@ -54463,7 +54618,7 @@ function convertTarget(source, jsMode) {
|
|
|
54463
54618
|
out.push(`// ${line}`);
|
|
54464
54619
|
continue;
|
|
54465
54620
|
}
|
|
54466
|
-
if (!inConfig || jsMode) {
|
|
54621
|
+
if (dialectPass && (!inConfig || jsMode)) {
|
|
54467
54622
|
for (const rule of SQL_RULES) {
|
|
54468
54623
|
rule.pattern.lastIndex = 0;
|
|
54469
54624
|
if (!rule.pattern.test(line))
|
|
@@ -54515,6 +54670,7 @@ function readSourceConfig(srcDir) {
|
|
|
54515
54670
|
const s = (k) => (typeof raw[k] === "string" ? raw[k] : null);
|
|
54516
54671
|
return {
|
|
54517
54672
|
defaultProject: (_a = s("defaultProject")) !== null && _a !== void 0 ? _a : s("defaultDatabase"),
|
|
54673
|
+
defaultLocation: s("defaultLocation"),
|
|
54518
54674
|
defaultDataset: (_c = (_b = s("defaultDataset")) !== null && _b !== void 0 ? _b : s("defaultSchema")) !== null && _c !== void 0 ? _c : "dataform",
|
|
54519
54675
|
defaultAssertionDataset: (_d = s("defaultAssertionDataset")) !== null && _d !== void 0 ? _d : s("assertionSchema"),
|
|
54520
54676
|
vars: (_e = raw.vars) !== null && _e !== void 0 ? _e : null,
|
|
@@ -54522,7 +54678,7 @@ function readSourceConfig(srcDir) {
|
|
|
54522
54678
|
};
|
|
54523
54679
|
}
|
|
54524
54680
|
async function migrateDataform(opts) {
|
|
54525
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
54681
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
54526
54682
|
const srcDir = path__namespace.resolve(opts.srcDir);
|
|
54527
54683
|
const outDir = path__namespace.resolve(opts.outDir);
|
|
54528
54684
|
if (containsPath(srcDir, outDir)) {
|
|
@@ -54534,11 +54690,14 @@ async function migrateDataform(opts) {
|
|
|
54534
54690
|
if (fs__namespace$1.existsSync(outDir) && fs__namespace$1.readdirSync(outDir).length > 0) {
|
|
54535
54691
|
throw new Error(`Output directory ${outDir} is not empty.`);
|
|
54536
54692
|
}
|
|
54693
|
+
const targetWarehouse = (_a = opts.targetWarehouse) !== null && _a !== void 0 ? _a : "supabase";
|
|
54694
|
+
const isBigQueryTarget = targetWarehouse === "bigquery";
|
|
54537
54695
|
const sourceConfig = readSourceConfig(srcDir);
|
|
54538
54696
|
const writer = new OutWriter(outDir);
|
|
54539
54697
|
skippedForSafety = new Set();
|
|
54540
54698
|
const files = walkFiles(srcDir);
|
|
54541
54699
|
const report = {
|
|
54700
|
+
targetWarehouse,
|
|
54542
54701
|
generator: `sqlanvil migrate-dataform ${version}`,
|
|
54543
54702
|
sourceDir: srcDir,
|
|
54544
54703
|
sourceConfig: sourceConfig.raw,
|
|
@@ -54560,16 +54719,16 @@ async function migrateDataform(opts) {
|
|
|
54560
54719
|
: { type: null, schema: null, database: null, name: null, description: null };
|
|
54561
54720
|
sqlxFiles.push({ rel, source, parsed });
|
|
54562
54721
|
report.inventory.sqlxFiles++;
|
|
54563
|
-
const t = (
|
|
54564
|
-
report.inventory.byType[t] = ((
|
|
54722
|
+
const t = (_b = parsed.type) !== null && _b !== void 0 ? _b : "(none)";
|
|
54723
|
+
report.inventory.byType[t] = ((_c = report.inventory.byType[t]) !== null && _c !== void 0 ? _c : 0) + 1;
|
|
54565
54724
|
}
|
|
54566
54725
|
const resolveDatabase = (db) => { var _a; return !db || db === "<defaultDatabase>" ? (_a = sourceConfig.defaultProject) !== null && _a !== void 0 ? _a : "UNKNOWN_PROJECT" : db; };
|
|
54567
54726
|
const declarationSchemas = new Set();
|
|
54568
54727
|
const targetSchemas = new Set();
|
|
54569
54728
|
const connectionMap = new Map();
|
|
54570
|
-
for (const f of sqlxFiles) {
|
|
54729
|
+
for (const f of isBigQueryTarget ? [] : sqlxFiles) {
|
|
54571
54730
|
if (f.parsed.type === "declaration") {
|
|
54572
|
-
const dataset = (
|
|
54731
|
+
const dataset = (_d = f.parsed.schema) !== null && _d !== void 0 ? _d : sourceConfig.defaultDataset;
|
|
54573
54732
|
const project = resolveDatabase(f.parsed.database);
|
|
54574
54733
|
declarationSchemas.add(dataset);
|
|
54575
54734
|
const name = connectionNameFor(project);
|
|
@@ -54590,7 +54749,7 @@ async function migrateDataform(opts) {
|
|
|
54590
54749
|
}
|
|
54591
54750
|
}
|
|
54592
54751
|
else if (f.parsed.type) {
|
|
54593
|
-
targetSchemas.add((
|
|
54752
|
+
targetSchemas.add((_e = f.parsed.schema) !== null && _e !== void 0 ? _e : sourceConfig.defaultDataset);
|
|
54594
54753
|
}
|
|
54595
54754
|
}
|
|
54596
54755
|
report.connections = [...connectionMap.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
@@ -54602,8 +54761,20 @@ async function migrateDataform(opts) {
|
|
|
54602
54761
|
}
|
|
54603
54762
|
if (rel.endsWith(".sqlx")) {
|
|
54604
54763
|
const info = sqlxFiles.find(f => f.rel === rel);
|
|
54764
|
+
if (isBigQueryTarget) {
|
|
54765
|
+
const { content, findings } = convertTarget(info.source, false, "none");
|
|
54766
|
+
writer.write(rel, content);
|
|
54767
|
+
report.files.push({
|
|
54768
|
+
file: rel,
|
|
54769
|
+
action: info.parsed.type === "declaration" ? "declaration" : "target",
|
|
54770
|
+
type: (_f = info.parsed.type) !== null && _f !== void 0 ? _f : "(none)",
|
|
54771
|
+
status: findings.length === 0 ? "clean" : "rewritten",
|
|
54772
|
+
findings,
|
|
54773
|
+
});
|
|
54774
|
+
continue;
|
|
54775
|
+
}
|
|
54605
54776
|
if (info.parsed.type === "declaration") {
|
|
54606
|
-
const dataset = (
|
|
54777
|
+
const dataset = (_g = info.parsed.schema) !== null && _g !== void 0 ? _g : sourceConfig.defaultDataset;
|
|
54607
54778
|
const project = resolveDatabase(info.parsed.database);
|
|
54608
54779
|
const connection = connectionNameFor(project);
|
|
54609
54780
|
const { content, findings } = convertDeclaration(info.source, info.parsed, connection, dataset, rel);
|
|
@@ -54623,7 +54794,7 @@ async function migrateDataform(opts) {
|
|
|
54623
54794
|
report.files.push({
|
|
54624
54795
|
file: rel,
|
|
54625
54796
|
action: "target",
|
|
54626
|
-
type: (
|
|
54797
|
+
type: (_h = info.parsed.type) !== null && _h !== void 0 ? _h : "(none)",
|
|
54627
54798
|
status: findings.length === 0
|
|
54628
54799
|
? "clean"
|
|
54629
54800
|
: findings.some(f => f.kind === "flag")
|
|
@@ -54637,10 +54808,10 @@ async function migrateDataform(opts) {
|
|
|
54637
54808
|
if (rel.endsWith(".js") && (rel.startsWith("definitions/") || rel.startsWith("includes/"))) {
|
|
54638
54809
|
report.inventory.jsFiles++;
|
|
54639
54810
|
const source = fs__namespace$1.readFileSync(abs, "utf8");
|
|
54640
|
-
const { content, findings } = convertTarget(source, true);
|
|
54811
|
+
const { content, findings } = convertTarget(source, true, isBigQueryTarget ? "none" : "postgres");
|
|
54641
54812
|
writer.write(rel, content);
|
|
54642
54813
|
const isIncludes = rel.startsWith("includes/");
|
|
54643
|
-
if (isIncludes && findings.length > 0) {
|
|
54814
|
+
if (isIncludes && findings.length > 0 && !isBigQueryTarget) {
|
|
54644
54815
|
report.warnings.push(`includes helper ${rel} emits BigQuery-flavored SQL — review its ${findings.length} finding(s)`);
|
|
54645
54816
|
}
|
|
54646
54817
|
report.files.push({
|
|
@@ -54659,11 +54830,12 @@ async function migrateDataform(opts) {
|
|
|
54659
54830
|
writer.write(rel, fs__namespace$1.readFileSync(abs));
|
|
54660
54831
|
report.files.push({ file: rel, action: "copied", status: "clean", findings: [] });
|
|
54661
54832
|
}
|
|
54662
|
-
const settings =
|
|
54663
|
-
|
|
54833
|
+
const settings = isBigQueryTarget
|
|
54834
|
+
? Object.assign(Object.assign(Object.assign({}, (sourceConfig.defaultProject ? { defaultProject: sourceConfig.defaultProject } : {})), (sourceConfig.defaultLocation ? { defaultLocation: sourceConfig.defaultLocation } : {})), { defaultDataset: sourceConfig.defaultDataset, defaultAssertionDataset: (_j = sourceConfig.defaultAssertionDataset) !== null && _j !== void 0 ? _j : "sqlanvil_assertions", sqlanvilCoreVersion: (_k = opts.coreVersion) !== null && _k !== void 0 ? _k : version }) : {
|
|
54835
|
+
warehouse: targetWarehouse,
|
|
54664
54836
|
defaultDataset: sourceConfig.defaultDataset.toLowerCase(),
|
|
54665
|
-
defaultAssertionDataset: ((
|
|
54666
|
-
sqlanvilCoreVersion: (
|
|
54837
|
+
defaultAssertionDataset: ((_l = sourceConfig.defaultAssertionDataset) !== null && _l !== void 0 ? _l : "sqlanvil_assertions").toLowerCase(),
|
|
54838
|
+
sqlanvilCoreVersion: (_m = opts.coreVersion) !== null && _m !== void 0 ? _m : version,
|
|
54667
54839
|
};
|
|
54668
54840
|
if (sourceConfig.vars)
|
|
54669
54841
|
settings.vars = sourceConfig.vars;
|
|
@@ -54682,6 +54854,22 @@ async function migrateDataform(opts) {
|
|
|
54682
54854
|
}));
|
|
54683
54855
|
}
|
|
54684
54856
|
writer.write("workflow_settings.yaml", jsYaml.dump(settings));
|
|
54857
|
+
if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "AGENTS.md"))) {
|
|
54858
|
+
writer.write("AGENTS.md", agentsMdContents({
|
|
54859
|
+
warehouse: targetWarehouse,
|
|
54860
|
+
defaultDataset: settings.defaultDataset,
|
|
54861
|
+
version: (_o = opts.coreVersion) !== null && _o !== void 0 ? _o : version,
|
|
54862
|
+
converted: true
|
|
54863
|
+
}));
|
|
54864
|
+
report.files.push({ file: "AGENTS.md", action: "generated", status: "clean", findings: [] });
|
|
54865
|
+
}
|
|
54866
|
+
else {
|
|
54867
|
+
report.warnings.push("Source AGENTS.md copied as-is — its guidance targets Dataform; consider replacing it with the sqlanvil version (`sqlanvil init --bare` in an empty dir generates one).");
|
|
54868
|
+
}
|
|
54869
|
+
if (!fs__namespace$1.existsSync(path__namespace.join(outDir, "CLAUDE.md"))) {
|
|
54870
|
+
writer.write("CLAUDE.md", claudeMdBridgeContents());
|
|
54871
|
+
report.files.push({ file: "CLAUDE.md", action: "generated", status: "clean", findings: [] });
|
|
54872
|
+
}
|
|
54685
54873
|
report.skippedForSafety = [...skippedForSafety].sort();
|
|
54686
54874
|
if (skippedForSafety.size > 0) {
|
|
54687
54875
|
report.warnings.push(`Skipped ${skippedForSafety.size} file(s)/dir(s) for safety (credentials, build artifacts, VCS) — see skippedForSafety`);
|
|
@@ -54708,28 +54896,38 @@ function renderReportMd(r) {
|
|
|
54708
54896
|
lines.push(` - ${t}: ${n}`);
|
|
54709
54897
|
}
|
|
54710
54898
|
lines.push("");
|
|
54711
|
-
|
|
54712
|
-
|
|
54713
|
-
|
|
54714
|
-
`
|
|
54715
|
-
|
|
54716
|
-
|
|
54717
|
-
|
|
54718
|
-
lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
|
|
54719
|
-
`${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
|
|
54720
|
-
(c.datasets.length > 8 ? ", …" : ""));
|
|
54721
|
-
lines.push(` - \`${c.introspectExample}\``);
|
|
54899
|
+
if (r.targetWarehouse === "bigquery") {
|
|
54900
|
+
lines.push(`## Same-warehouse conversion (BigQuery → BigQuery)`);
|
|
54901
|
+
lines.push("");
|
|
54902
|
+
lines.push(`This is a tooling swap: the project keeps running on the SAME BigQuery warehouse. SQL ` +
|
|
54903
|
+
`bodies, \`bigquery: {}\` config blocks, and declarations passed through untouched; ` +
|
|
54904
|
+
`only the compile global was renamed (\`dataform.projectConfig\` → ` +
|
|
54905
|
+
`\`sqlanvil.projectConfig\`) and the settings file converted.`);
|
|
54722
54906
|
}
|
|
54723
|
-
|
|
54907
|
+
else {
|
|
54908
|
+
lines.push(`## Source connections (${r.connections.length})`);
|
|
54724
54909
|
lines.push("");
|
|
54725
|
-
lines.push(
|
|
54910
|
+
lines.push(`Declarations keep reading BigQuery through named \`connections:\` in workflow_settings.yaml ` +
|
|
54911
|
+
`(mode: runner-extract — no Vault, no wrappers). Each declaration needs \`columnTypes\` ` +
|
|
54912
|
+
`before it can extract; scaffold them with \`sqlanvil introspect\`:`);
|
|
54726
54913
|
lines.push("");
|
|
54727
|
-
|
|
54728
|
-
.
|
|
54729
|
-
|
|
54730
|
-
|
|
54731
|
-
`
|
|
54732
|
-
|
|
54914
|
+
for (const c of r.connections) {
|
|
54915
|
+
lines.push(`- **${c.name}** — project \`${c.project}\`, ${c.declarationCount} declaration(s) across ` +
|
|
54916
|
+
`${c.datasets.length} dataset(s): ${c.datasets.slice(0, 8).map(d => `\`${d}\``).join(", ")}` +
|
|
54917
|
+
(c.datasets.length > 8 ? ", …" : ""));
|
|
54918
|
+
lines.push(` - \`${c.introspectExample}\``);
|
|
54919
|
+
}
|
|
54920
|
+
if (r.overlappingSchemas.length > 0) {
|
|
54921
|
+
lines.push("");
|
|
54922
|
+
lines.push(`## ⚠ Schemas that are BOTH source and target`);
|
|
54923
|
+
lines.push("");
|
|
54924
|
+
lines.push(`These schemas have declarations AND materializing actions: ${r.overlappingSchemas
|
|
54925
|
+
.map(s => `\`${s}\``)
|
|
54926
|
+
.join(", ")}. ` +
|
|
54927
|
+
`Their declared tables are probably produced by this pipeline (or a sibling) — once those ` +
|
|
54928
|
+
`producers move to Postgres, convert the declarations to plain local declarations instead ` +
|
|
54929
|
+
`of BigQuery connections.`);
|
|
54930
|
+
}
|
|
54733
54931
|
}
|
|
54734
54932
|
lines.push("");
|
|
54735
54933
|
lines.push(`## Target files (${targets.length})`);
|
|
@@ -54751,7 +54949,9 @@ function renderReportMd(r) {
|
|
|
54751
54949
|
lines.push("");
|
|
54752
54950
|
lines.push(`## Declarations (${declarations.length})`);
|
|
54753
54951
|
lines.push("");
|
|
54754
|
-
lines.push(
|
|
54952
|
+
lines.push(r.targetWarehouse === "bigquery"
|
|
54953
|
+
? `Declarations are unchanged — the sources are native to the warehouse.`
|
|
54954
|
+
: `Every declaration was rewritten to ride its connection. The BigQuery data does not move.`);
|
|
54755
54955
|
if (r.warnings.length > 0) {
|
|
54756
54956
|
lines.push("");
|
|
54757
54957
|
lines.push(`## Warnings`);
|
|
@@ -54762,958 +54962,1201 @@ function renderReportMd(r) {
|
|
|
54762
54962
|
lines.push("");
|
|
54763
54963
|
lines.push(`## Finish the migration`);
|
|
54764
54964
|
lines.push("");
|
|
54765
|
-
|
|
54766
|
-
|
|
54767
|
-
|
|
54768
|
-
|
|
54769
|
-
|
|
54770
|
-
|
|
54965
|
+
if (r.targetWarehouse === "bigquery") {
|
|
54966
|
+
lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
|
|
54967
|
+
lines.push(`2. Credentials: \`gcloud auth application-default login\` (ADC), or a service-account ` +
|
|
54968
|
+
`key in a gitignored \`.df-credentials.json\`.`);
|
|
54969
|
+
lines.push(`3. \`sqlanvil validate\` — dry-runs every model against BigQuery without executing; ` +
|
|
54970
|
+
`all-PASS means the swap is complete.`);
|
|
54971
|
+
lines.push(`4. Compare \`sqlanvil compile\` output with \`dataform compile\` on the source for a ` +
|
|
54972
|
+
`sample of actions — they should match apart from the compile-global rename.`);
|
|
54973
|
+
}
|
|
54974
|
+
else {
|
|
54975
|
+
lines.push(`1. \`sqlanvil compile\` — should already succeed.`);
|
|
54976
|
+
lines.push(`2. Scaffold \`columnTypes\` for the declarations you actually read (see connections above).`);
|
|
54977
|
+
lines.push(`3. \`sqlanvil validate\` against your Postgres/Supabase warehouse — PASS/FAILURE/BLOCKED per ` +
|
|
54978
|
+
`model is the migration to-do list. Work the flagged files until validate is green.`);
|
|
54979
|
+
lines.push(`4. This report is machine-readable (migration-report.json) — pointing an AI agent at it plus ` +
|
|
54980
|
+
`the \`SQLANVIL-MIGRATE:\` markers is the fastest path through the dialect work.`);
|
|
54981
|
+
}
|
|
54771
54982
|
lines.push("");
|
|
54772
54983
|
return lines.join("\n");
|
|
54773
54984
|
}
|
|
54774
54985
|
|
|
54775
|
-
|
|
54776
|
-
|
|
54777
|
-
|
|
54778
|
-
|
|
54779
|
-
|
|
54780
|
-
|
|
54781
|
-
import platform
|
|
54782
|
-
py_version = platform.python_version()
|
|
54783
|
-
|
|
54784
|
-
def load_packaging():
|
|
54785
|
-
try:
|
|
54786
|
-
from packaging.specifiers import SpecifierSet
|
|
54787
|
-
from packaging.requirements import Requirement
|
|
54788
|
-
return SpecifierSet, Requirement
|
|
54789
|
-
except ImportError:
|
|
54790
|
-
pass
|
|
54791
|
-
try:
|
|
54792
|
-
from pip._vendor.packaging.specifiers import SpecifierSet
|
|
54793
|
-
from pip._vendor.packaging.requirements import Requirement
|
|
54794
|
-
return SpecifierSet, Requirement
|
|
54795
|
-
except ImportError:
|
|
54796
|
-
return None, None
|
|
54797
|
-
|
|
54798
|
-
SpecifierSet, Requirement = load_packaging()
|
|
54799
|
-
|
|
54800
|
-
def mini_parse(v):
|
|
54801
|
-
import re
|
|
54802
|
-
return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
|
|
54803
|
-
|
|
54804
|
-
def mini_check(version, spec):
|
|
54805
|
-
import re
|
|
54806
|
-
ok = True
|
|
54807
|
-
for clause in spec.split(","):
|
|
54808
|
-
clause = clause.strip()
|
|
54809
|
-
m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
|
|
54810
|
-
if not m:
|
|
54811
|
-
return None
|
|
54812
|
-
op, want = m.group(1), m.group(2).strip()
|
|
54813
|
-
if want.endswith(".*"):
|
|
54814
|
-
prefix = mini_parse(want[:-2])
|
|
54815
|
-
have = mini_parse(version)[:len(prefix)]
|
|
54816
|
-
eq = have == prefix
|
|
54817
|
-
if (op in ("==", "===") and not eq) or (op == "!=" and eq):
|
|
54818
|
-
ok = False
|
|
54819
|
-
continue
|
|
54820
|
-
a, b = mini_parse(version), mini_parse(want)
|
|
54821
|
-
n = max(len(a), len(b))
|
|
54822
|
-
a = a + [0] * (n - len(a))
|
|
54823
|
-
b = b + [0] * (n - len(b))
|
|
54824
|
-
if op in ("==", "==="):
|
|
54825
|
-
res = a == b
|
|
54826
|
-
elif op == "!=":
|
|
54827
|
-
res = a != b
|
|
54828
|
-
elif op == "<=":
|
|
54829
|
-
res = a <= b
|
|
54830
|
-
elif op == ">=":
|
|
54831
|
-
res = a >= b
|
|
54832
|
-
elif op == "<":
|
|
54833
|
-
res = a < b
|
|
54834
|
-
elif op == ">":
|
|
54835
|
-
res = a > b
|
|
54836
|
-
else: # ~= : at least b, within b's series
|
|
54837
|
-
wb = mini_parse(want)
|
|
54838
|
-
res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
|
|
54839
|
-
if not res:
|
|
54840
|
-
ok = False
|
|
54841
|
-
return ok
|
|
54842
|
-
|
|
54843
|
-
def check_version(version, spec, what):
|
|
54844
|
-
if SpecifierSet is not None:
|
|
54845
|
-
try:
|
|
54846
|
-
if version not in SpecifierSet(spec, prereleases=True):
|
|
54847
|
-
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
54848
|
-
return
|
|
54849
|
-
except Exception as e:
|
|
54850
|
-
errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
|
|
54851
|
-
return
|
|
54852
|
-
if mini_check(version, spec) is False:
|
|
54853
|
-
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
54854
|
-
|
|
54855
|
-
if version_spec:
|
|
54856
|
-
check_version(py_version, version_spec, "python interpreter")
|
|
54857
|
-
|
|
54858
|
-
if requirements_path:
|
|
54859
|
-
from importlib import metadata
|
|
54860
|
-
try:
|
|
54861
|
-
lines = open(requirements_path).read().splitlines()
|
|
54862
|
-
except OSError as e:
|
|
54863
|
-
errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
|
|
54864
|
-
lines = []
|
|
54865
|
-
for raw in lines:
|
|
54866
|
-
line = raw.split("#")[0].strip()
|
|
54867
|
-
if not line or line.startswith("-"):
|
|
54868
|
-
continue
|
|
54869
|
-
if Requirement is not None:
|
|
54870
|
-
try:
|
|
54871
|
-
req = Requirement(line)
|
|
54872
|
-
except Exception:
|
|
54873
|
-
errors.append('requirements: cannot parse "%s"' % line)
|
|
54874
|
-
continue
|
|
54875
|
-
if req.marker is not None and not req.marker.evaluate():
|
|
54876
|
-
continue
|
|
54877
|
-
name, spec = req.name, str(req.specifier)
|
|
54878
|
-
else:
|
|
54879
|
-
import re
|
|
54880
|
-
if ";" in line:
|
|
54881
|
-
line = line.split(";")[0].strip()
|
|
54882
|
-
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
|
|
54883
|
-
if not m:
|
|
54884
|
-
errors.append('requirements: cannot parse "%s"' % line)
|
|
54885
|
-
continue
|
|
54886
|
-
name, spec = m.group(1), (m.group(3) or "").strip()
|
|
54887
|
-
try:
|
|
54888
|
-
installed = metadata.version(name)
|
|
54889
|
-
except metadata.PackageNotFoundError:
|
|
54890
|
-
errors.append(
|
|
54891
|
-
"requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
|
|
54892
|
-
)
|
|
54893
|
-
continue
|
|
54894
|
-
if spec:
|
|
54895
|
-
check_version(installed, spec, "requirements: %s" % name)
|
|
54896
|
-
|
|
54897
|
-
try:
|
|
54898
|
-
with open(script_path, "rb") as f:
|
|
54899
|
-
compile(f.read(), script_path, "exec")
|
|
54900
|
-
except SyntaxError as e:
|
|
54901
|
-
errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
|
|
54902
|
-
except (OSError, ValueError) as e:
|
|
54903
|
-
errors.append("syntax: cannot compile %s: %s" % (script_path, e))
|
|
54904
|
-
|
|
54905
|
-
print(json.dumps({"interpreter_version": py_version, "errors": errors}))
|
|
54906
|
-
`;
|
|
54907
|
-
const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
|
|
54908
|
-
childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
|
|
54909
|
-
if (err) {
|
|
54910
|
-
reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
|
|
54911
|
-
}
|
|
54912
|
-
else {
|
|
54913
|
-
resolve(stdout);
|
|
54914
|
-
}
|
|
54915
|
-
});
|
|
54916
|
-
});
|
|
54917
|
-
const failure = (message) => ({
|
|
54918
|
-
status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
|
|
54919
|
-
error: { message }
|
|
54920
|
-
});
|
|
54921
|
-
async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
|
|
54922
|
-
if (script.language !== "python") {
|
|
54923
|
-
return [failure(`no validator for script language "${script.language}"`)];
|
|
54924
|
-
}
|
|
54925
|
-
let interpreter;
|
|
54926
|
-
try {
|
|
54927
|
-
interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
|
|
54928
|
-
}
|
|
54929
|
-
catch (e) {
|
|
54930
|
-
return [failure(e.message)];
|
|
54931
|
-
}
|
|
54932
|
-
let stdout;
|
|
54933
|
-
try {
|
|
54934
|
-
stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
|
|
54935
|
-
}
|
|
54936
|
-
catch (e) {
|
|
54937
|
-
return [failure(e.message)];
|
|
54938
|
-
}
|
|
54939
|
-
let parsed;
|
|
54940
|
-
try {
|
|
54941
|
-
parsed = JSON.parse(stdout);
|
|
54986
|
+
function actuallyResolve(...filePaths) {
|
|
54987
|
+
return path__namespace.resolve(...filePaths.map(filePath => untildify__default["default"](filePath)));
|
|
54988
|
+
}
|
|
54989
|
+
function assertPathExists(checkPath) {
|
|
54990
|
+
if (!fs__namespace$1.existsSync(checkPath)) {
|
|
54991
|
+
throw new Error(`${checkPath} does not exist!`);
|
|
54942
54992
|
}
|
|
54943
|
-
|
|
54944
|
-
|
|
54993
|
+
}
|
|
54994
|
+
function compiledGraphHasErrors(graph) {
|
|
54995
|
+
var _a, _b;
|
|
54996
|
+
return ((_b = (_a = graph.graphErrors) === null || _a === void 0 ? void 0 : _a.compilationErrors) === null || _b === void 0 ? void 0 : _b.length) > 0;
|
|
54997
|
+
}
|
|
54998
|
+
function formatExecutionSuffix(jobIds, bytesBilled) {
|
|
54999
|
+
const jobMetadataParts = [];
|
|
55000
|
+
if (jobIds.length > 0) {
|
|
55001
|
+
jobMetadataParts.push(`\n \t jobId: ${jobIds.join(", ")}`);
|
|
54945
55002
|
}
|
|
54946
|
-
if (
|
|
54947
|
-
|
|
55003
|
+
if (bytesBilled.length > 0) {
|
|
55004
|
+
jobMetadataParts.push(`\n \t Bytes billed: ${bytesBilled.join(", ")}`);
|
|
54948
55005
|
}
|
|
54949
|
-
return
|
|
54950
|
-
}
|
|
54951
|
-
|
|
54952
|
-
function targetKey(target) {
|
|
54953
|
-
return [target.database, target.schema, target.name].filter(Boolean).join(".");
|
|
55006
|
+
return jobMetadataParts.length > 0 ? ` ${jobMetadataParts}` : "";
|
|
54954
55007
|
}
|
|
54955
|
-
function
|
|
54956
|
-
|
|
54957
|
-
|
|
54958
|
-
const dependents = new Map();
|
|
54959
|
-
for (const n of nodes) {
|
|
54960
|
-
const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
|
|
54961
|
-
indegree.set(n.key, inGraphDeps.length);
|
|
54962
|
-
for (const dep of inGraphDeps) {
|
|
54963
|
-
dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
|
|
54964
|
-
}
|
|
55008
|
+
function formatBytesInHumanReadableFormat(bytes) {
|
|
55009
|
+
if (bytes <= 0) {
|
|
55010
|
+
return '0 B';
|
|
54965
55011
|
}
|
|
54966
|
-
const
|
|
54967
|
-
|
|
54968
|
-
|
|
54969
|
-
|
|
54970
|
-
|
|
54971
|
-
|
|
54972
|
-
|
|
54973
|
-
|
|
54974
|
-
|
|
54975
|
-
|
|
55012
|
+
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
|
|
55013
|
+
const k = 1024;
|
|
55014
|
+
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
|
55015
|
+
const value = (bytes / Math.pow(k, i)).toFixed(2);
|
|
55016
|
+
return `${value} ${units[i]}`;
|
|
55017
|
+
}
|
|
55018
|
+
function promptForIcebergConfig() {
|
|
55019
|
+
print(ICEBERG_CONFIG_PROMPT_TEXT);
|
|
55020
|
+
print(ICEBERG_CONFIG_PROMPT_HINT);
|
|
55021
|
+
const tempIcebergConfig = {};
|
|
55022
|
+
let bucketName;
|
|
55023
|
+
while (true) {
|
|
55024
|
+
print(ICEBERG_BUCKET_NAME_HINT);
|
|
55025
|
+
bucketName = interactiveQuestion(ICEBERG_BUCKET_NAME_PROMPT_QUESTION);
|
|
55026
|
+
try {
|
|
55027
|
+
if (bucketName) {
|
|
55028
|
+
validateIcebergConfigBucketName(bucketName);
|
|
55029
|
+
tempIcebergConfig.bucketName = bucketName;
|
|
55030
|
+
}
|
|
55031
|
+
break;
|
|
54976
55032
|
}
|
|
54977
|
-
|
|
54978
|
-
|
|
54979
|
-
|
|
54980
|
-
|
|
54981
|
-
|
|
54982
|
-
|
|
54983
|
-
|
|
55033
|
+
catch (e) {
|
|
55034
|
+
printError(`Validation Error: ${e.message}`);
|
|
55035
|
+
print("Please try again.");
|
|
55036
|
+
}
|
|
55037
|
+
}
|
|
55038
|
+
let tableFolderRoot;
|
|
55039
|
+
while (true) {
|
|
55040
|
+
print(ICEBERG_TABLE_FOLDER_ROOT_HINT);
|
|
55041
|
+
tableFolderRoot = interactiveQuestion(ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION);
|
|
55042
|
+
try {
|
|
55043
|
+
if (tableFolderRoot) {
|
|
55044
|
+
validateIcebergConfigTableFolderRoot(tableFolderRoot);
|
|
55045
|
+
tempIcebergConfig.tableFolderRoot = tableFolderRoot;
|
|
54984
55046
|
}
|
|
55047
|
+
break;
|
|
55048
|
+
}
|
|
55049
|
+
catch (e) {
|
|
55050
|
+
printError(`Validation Error: ${e.message}`);
|
|
55051
|
+
print("Please try again.");
|
|
54985
55052
|
}
|
|
54986
55053
|
}
|
|
54987
|
-
|
|
54988
|
-
|
|
54989
|
-
|
|
54990
|
-
|
|
54991
|
-
|
|
55054
|
+
let tableFolderSubpath;
|
|
55055
|
+
while (true) {
|
|
55056
|
+
print(ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT);
|
|
55057
|
+
tableFolderSubpath = interactiveQuestion(ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION);
|
|
55058
|
+
try {
|
|
55059
|
+
if (tableFolderSubpath) {
|
|
55060
|
+
validateIcebergConfigTableFolderSubpath(tableFolderSubpath);
|
|
55061
|
+
tempIcebergConfig.tableFolderSubpath = tableFolderSubpath;
|
|
55062
|
+
}
|
|
55063
|
+
break;
|
|
55064
|
+
}
|
|
55065
|
+
catch (e) {
|
|
55066
|
+
printError(`Validation Error: ${e.message}`);
|
|
55067
|
+
print("Please try again.");
|
|
55068
|
+
}
|
|
55069
|
+
}
|
|
55070
|
+
let connection;
|
|
55071
|
+
while (true) {
|
|
55072
|
+
print(ICEBERG_CONNECTION_HINT);
|
|
55073
|
+
connection = interactiveQuestion(ICEBERG_CONNECTION_QUESTION);
|
|
55074
|
+
try {
|
|
55075
|
+
if (connection) {
|
|
55076
|
+
validateConnectionFormat(connection);
|
|
55077
|
+
tempIcebergConfig.connection = connection;
|
|
54992
55078
|
}
|
|
55079
|
+
break;
|
|
55080
|
+
}
|
|
55081
|
+
catch (e) {
|
|
55082
|
+
printError(`Validation Error: ${e.message}`);
|
|
55083
|
+
print("Please try again.");
|
|
54993
55084
|
}
|
|
54994
55085
|
}
|
|
54995
|
-
|
|
54996
|
-
|
|
54997
|
-
|
|
54998
|
-
|
|
54999
|
-
|
|
55000
|
-
return status !== undefined && status !== "PASS";
|
|
55001
|
-
});
|
|
55086
|
+
printSuccess(ICEBERG_CONFIG_COLLECTED_TEXT);
|
|
55087
|
+
if (Object.keys(tempIcebergConfig).length > 0) {
|
|
55088
|
+
return tempIcebergConfig;
|
|
55089
|
+
}
|
|
55090
|
+
return undefined;
|
|
55002
55091
|
}
|
|
55003
|
-
|
|
55004
|
-
|
|
55005
|
-
|
|
55092
|
+
function validateIcebergConfigBucketName(bucketName) {
|
|
55093
|
+
if (bucketName.length < 3 || bucketName.length > 63) {
|
|
55094
|
+
throw new Error("Bucket name must be between 3 and 63 characters long.");
|
|
55095
|
+
}
|
|
55096
|
+
if (!/^[a-z0-9][a-z0-9-._]{1,220}[a-z0-9]$/.test(bucketName) || bucketName.includes("..")) {
|
|
55097
|
+
throw new Error("Invalid bucket name. Must start and end with a letter or number, contain only lowercase letters, numbers, hyphens (-), underscores (_), and periods (.).");
|
|
55098
|
+
}
|
|
55099
|
+
if (bucketName.startsWith("goog") || bucketName.includes("--")) {
|
|
55100
|
+
throw new Error("Bucket name cannot start with 'goog' or contain '--'.");
|
|
55101
|
+
}
|
|
55102
|
+
if (bucketName.includes("google") || bucketName.includes("g00gle")) {
|
|
55103
|
+
throw new Error("Bucket name cannot contain 'google' or close misspellings such as 'g00gle'.");
|
|
55104
|
+
}
|
|
55006
55105
|
}
|
|
55007
|
-
function
|
|
55008
|
-
|
|
55009
|
-
|
|
55106
|
+
function validateIcebergConfigTableFolderRoot(tableFolderRoot) {
|
|
55107
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9-._]{1,220}[a-zA-Z0-9]$/.test(tableFolderRoot) || tableFolderRoot.includes("..")) {
|
|
55108
|
+
throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), and periods (.).");
|
|
55109
|
+
}
|
|
55010
55110
|
}
|
|
55011
|
-
function
|
|
55012
|
-
|
|
55013
|
-
|
|
55014
|
-
|
|
55015
|
-
|
|
55111
|
+
function validateIcebergConfigTableFolderSubpath(tableFolderSubpath) {
|
|
55112
|
+
if (!/^[a-zA-Z0-9][a-zA-Z0-9-._/]{1,220}[a-zA-Z0-9]$/.test(tableFolderSubpath) || tableFolderSubpath.includes("..")) {
|
|
55113
|
+
throw new Error("Invalid input. Must start and end with a letter or number, contain only letters (a-z, A-Z), numbers, hyphens (-), underscores (_), periods (.), and forward slashes (/). The sequence '..' is also disallowed.");
|
|
55114
|
+
}
|
|
55115
|
+
if (tableFolderSubpath.includes("./") || tableFolderSubpath.includes("../")) {
|
|
55116
|
+
throw new Error("Input cannot contain './' or '../'.");
|
|
55117
|
+
}
|
|
55016
55118
|
}
|
|
55119
|
+
const ICEBERG_BUCKET_NAME_HINT = "The bucket name must comply with https://cloud.google.com/storage/docs/buckets#naming. If you do not want to provide a workflow-level default bucket name, leave this input empty.\n";
|
|
55120
|
+
const ICEBERG_TABLE_FOLDER_ROOT_HINT = "Table folder root must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores and periods. If you do not want to provide a workflow_level default table folder root, leave this input empty.\n";
|
|
55121
|
+
const ICEBERG_TABLE_FOLDER_ROOT_SUBPATH_HINT = "Table folder subpath must start and end with a letter or a number. It can only contain letters, numbers, hyphens, underscores, periods and forward slashes. If you do not want to provide a workflow_level default table folder subpath, leave this input empty.\n";
|
|
55122
|
+
const ICEBERG_CONFIG_PROMPT_HINT = "Set repository-level configuration for Iceberg bucket name, table folder root, table folder subpath and connection. If you do not want to set a field, enter an empty string in response to the prompt.\n";
|
|
55123
|
+
const ICEBERG_CONFIG_PROMPT_TEXT = "\n--- Iceberg Configuration ---\n";
|
|
55124
|
+
const ICEBERG_CONFIG_COLLECTED_TEXT = "Iceberg configuration collected.\n";
|
|
55125
|
+
const ICEBERG_BUCKET_NAME_PROMPT_QUESTION = "Enter the default Iceberg bucket name:";
|
|
55126
|
+
const ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION = "Enter the default Iceberg table folder root:";
|
|
55127
|
+
const ICEBERG_TABLE_FOLDER_SUBPATH_PROMPT_QUESTION = "Enter the default Iceberg table folder subpath:";
|
|
55128
|
+
const ICEBERG_CONNECTION_HINT = "The connection can have the form `{project}.{location}.{connection_id}` or `projects/{project}/locations/{location}/connections/{connection_id} or be set to DEFAULT. If you do not want to provide a workflow-level default connection, leave this input empty.\n";
|
|
55129
|
+
const ICEBERG_CONNECTION_QUESTION = "Enter the default connection:";
|
|
55017
55130
|
|
|
55018
|
-
const
|
|
55019
|
-
|
|
55131
|
+
const noColor = process.env.NO_COLOR && ["1", "true", "yes"].includes(process.env.NO_COLOR.toLowerCase());
|
|
55132
|
+
const ansiColorCodes = Object.freeze({
|
|
55133
|
+
red: 91,
|
|
55134
|
+
green: 32,
|
|
55135
|
+
yellow: 93,
|
|
55136
|
+
cyan: 36
|
|
55137
|
+
});
|
|
55138
|
+
function output(text, ansiColorCode) {
|
|
55139
|
+
if (noColor) {
|
|
55140
|
+
return text;
|
|
55141
|
+
}
|
|
55142
|
+
return `\x1b[${ansiColorCode}m${text}\x1b[0m`;
|
|
55143
|
+
}
|
|
55144
|
+
const successOutput = (text) => output(text, ansiColorCodes.green);
|
|
55145
|
+
const warningOutput = (text) => output(text, ansiColorCodes.yellow);
|
|
55146
|
+
const errorOutput = (text) => output(text, ansiColorCodes.red);
|
|
55147
|
+
const calloutOutput = (text) => output(text, ansiColorCodes.cyan);
|
|
55148
|
+
const write = (stream, text, indentCount) => stream.write(`${" ".repeat(indentCount)}${text}\n`);
|
|
55149
|
+
const writeStdOut = (text, indentCount = 0) => write(process.stdout, text, indentCount);
|
|
55150
|
+
const writeStdErr = (text, indentCount = 0) => write(process.stderr, text, indentCount);
|
|
55151
|
+
const DEFAULT_PROMPT = "> ";
|
|
55152
|
+
function question(questionText) {
|
|
55153
|
+
return prompt(questionText);
|
|
55154
|
+
}
|
|
55155
|
+
function interactiveQuestion(questionText) {
|
|
55156
|
+
if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
|
|
55157
|
+
const testInput = getTestInput(questionText);
|
|
55158
|
+
print(`${questionText} ${testInput}\n`);
|
|
55159
|
+
return testInput;
|
|
55160
|
+
}
|
|
55161
|
+
print(`${questionText} `);
|
|
55162
|
+
return readlineSync__namespace.question("");
|
|
55163
|
+
}
|
|
55164
|
+
function getTestInput(questionText) {
|
|
55165
|
+
const envVar = process.env.DATAFORM_CLI_TEST_INPUTS;
|
|
55166
|
+
if (!envVar) {
|
|
55167
|
+
throw new Error("Environment variable DATAFORM_CLI_TEST_INPUTS not set.");
|
|
55168
|
+
}
|
|
55020
55169
|
try {
|
|
55021
|
-
const
|
|
55022
|
-
|
|
55023
|
-
|
|
55024
|
-
|
|
55170
|
+
const parsed = JSON.parse(envVar);
|
|
55171
|
+
if (typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed)) {
|
|
55172
|
+
const inputs = new Map(Object.entries(parsed).map(([key, value]) => [key, String(value)]));
|
|
55173
|
+
const trimmedQuestion = questionText.trim();
|
|
55174
|
+
const answer = inputs.get(trimmedQuestion);
|
|
55175
|
+
if (answer !== undefined) {
|
|
55176
|
+
print(`[TEST_INPUT for "${trimmedQuestion}"]: "${answer}"`);
|
|
55177
|
+
return answer;
|
|
55025
55178
|
}
|
|
55026
|
-
|
|
55179
|
+
else {
|
|
55180
|
+
throw new Error(`[MISSING TEST_INPUT for "${trimmedQuestion}"]`);
|
|
55027
55181
|
}
|
|
55028
55182
|
}
|
|
55183
|
+
else {
|
|
55184
|
+
throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: Expected a JSON object, but got ${typeof parsed}.`);
|
|
55185
|
+
}
|
|
55029
55186
|
}
|
|
55030
55187
|
catch (e) {
|
|
55188
|
+
throw new Error(`Failed to parse DATAFORM_CLI_TEST_INPUTS: ${e.message || e}`);
|
|
55031
55189
|
}
|
|
55032
55190
|
}
|
|
55033
|
-
function
|
|
55034
|
-
|
|
55035
|
-
|
|
55036
|
-
|
|
55037
|
-
|
|
55038
|
-
|
|
55039
|
-
|
|
55040
|
-
|
|
55191
|
+
function passwordQuestion(questionText) {
|
|
55192
|
+
return prompt(questionText, {
|
|
55193
|
+
hideEchoBack: true,
|
|
55194
|
+
mask: ""
|
|
55195
|
+
});
|
|
55196
|
+
}
|
|
55197
|
+
function interactivePasswordQuestion(questionText) {
|
|
55198
|
+
if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS !== undefined) {
|
|
55199
|
+
const testInput = getTestInput(questionText);
|
|
55200
|
+
print(`${questionText} ********\n`);
|
|
55201
|
+
return testInput;
|
|
55041
55202
|
}
|
|
55203
|
+
return passwordQuestion(questionText);
|
|
55042
55204
|
}
|
|
55043
|
-
function
|
|
55044
|
-
return (
|
|
55205
|
+
function selectionQuestion(questionText, options) {
|
|
55206
|
+
return readlineSync__namespace.keyInSelect(options, `${questionText}\n`, {
|
|
55207
|
+
cancel: false
|
|
55208
|
+
});
|
|
55045
55209
|
}
|
|
55046
|
-
|
|
55047
|
-
|
|
55048
|
-
|
|
55049
|
-
|
|
55050
|
-
|
|
55051
|
-
|
|
55052
|
-
|
|
55053
|
-
|
|
55054
|
-
|
|
55055
|
-
|
|
55056
|
-
|
|
55057
|
-
|
|
55058
|
-
|
|
55059
|
-
|
|
55060
|
-
|
|
55061
|
-
|
|
55062
|
-
|
|
55063
|
-
kind: "assertion",
|
|
55064
|
-
type: "assertion",
|
|
55065
|
-
target: assertion.target,
|
|
55066
|
-
action: assertion
|
|
55067
|
-
});
|
|
55210
|
+
function prompt(questionText, options) {
|
|
55211
|
+
writeStdOut(questionText);
|
|
55212
|
+
return readlineSync__namespace.prompt(Object.assign(Object.assign({}, options), { prompt: (options && options.prompt && options.prompt + DEFAULT_PROMPT) || DEFAULT_PROMPT }));
|
|
55213
|
+
}
|
|
55214
|
+
function print(text) {
|
|
55215
|
+
writeStdOut(text);
|
|
55216
|
+
}
|
|
55217
|
+
function printSuccess(text) {
|
|
55218
|
+
writeStdOut(successOutput(text));
|
|
55219
|
+
}
|
|
55220
|
+
function printError(errorText, indentCount = 0) {
|
|
55221
|
+
writeStdErr(errorOutput(errorText), indentCount);
|
|
55222
|
+
}
|
|
55223
|
+
function printInitResult(result) {
|
|
55224
|
+
if (result.dirsCreated && result.dirsCreated.length) {
|
|
55225
|
+
writeStdOut(successOutput("Directories successfully created:"));
|
|
55226
|
+
result.dirsCreated.forEach(dir => writeStdOut(dir, 1));
|
|
55068
55227
|
}
|
|
55069
|
-
|
|
55070
|
-
|
|
55071
|
-
|
|
55072
|
-
}
|
|
55073
|
-
nodes.push({
|
|
55074
|
-
key: targetKey(operation.target),
|
|
55075
|
-
dependencyKeys: depKeys(operation.dependencyTargets),
|
|
55076
|
-
kind: "operation",
|
|
55077
|
-
type: "operation",
|
|
55078
|
-
target: operation.target,
|
|
55079
|
-
action: undefined
|
|
55080
|
-
});
|
|
55228
|
+
if (result.filesWritten && result.filesWritten.length) {
|
|
55229
|
+
writeStdOut(successOutput("Files successfully written:"));
|
|
55230
|
+
result.filesWritten.forEach(file => writeStdOut(file, 1));
|
|
55081
55231
|
}
|
|
55082
|
-
|
|
55083
|
-
|
|
55084
|
-
|
|
55085
|
-
|
|
55086
|
-
|
|
55087
|
-
|
|
55088
|
-
|
|
55089
|
-
|
|
55090
|
-
|
|
55091
|
-
|
|
55092
|
-
|
|
55093
|
-
|
|
55232
|
+
}
|
|
55233
|
+
function printInitCredsResult(writtenFilePath) {
|
|
55234
|
+
writeStdOut(successOutput("Credentials file successfully written:"));
|
|
55235
|
+
writeStdOut(writtenFilePath, 1);
|
|
55236
|
+
writeStdOut("To change connection settings, edit this file directly.");
|
|
55237
|
+
}
|
|
55238
|
+
function isInteractive({ stream = process.stdout } = {}) {
|
|
55239
|
+
return Boolean(stream && stream.isTTY &&
|
|
55240
|
+
process.env.TERM !== 'dumb' &&
|
|
55241
|
+
!('CI' in process.env));
|
|
55242
|
+
}
|
|
55243
|
+
var compiledGraphOutputType;
|
|
55244
|
+
(function (compiledGraphOutputType) {
|
|
55245
|
+
compiledGraphOutputType["Json"] = "json";
|
|
55246
|
+
compiledGraphOutputType["Dot"] = "dot";
|
|
55247
|
+
compiledGraphOutputType["Summary"] = "summary";
|
|
55248
|
+
})(compiledGraphOutputType || (compiledGraphOutputType = {}));
|
|
55249
|
+
function printCompiledGraph(graph, outputType, quietCompilation) {
|
|
55250
|
+
const interactive = isInteractive();
|
|
55251
|
+
if (outputType === compiledGraphOutputType.Json) {
|
|
55252
|
+
writeStdOut(prettyJsonStringify(graph));
|
|
55094
55253
|
}
|
|
55095
|
-
|
|
55096
|
-
|
|
55097
|
-
continue;
|
|
55098
|
-
}
|
|
55099
|
-
nodes.push({
|
|
55100
|
-
key: targetKey(script.target),
|
|
55101
|
-
dependencyKeys: depKeys(script.dependencyTargets),
|
|
55102
|
-
kind: "script",
|
|
55103
|
-
type: "script",
|
|
55104
|
-
target: script.target,
|
|
55105
|
-
script,
|
|
55106
|
-
action: undefined
|
|
55107
|
-
});
|
|
55254
|
+
else if (outputType === compiledGraphOutputType.Dot) {
|
|
55255
|
+
writeStdOut(dotRepresentation(graph, interactive));
|
|
55108
55256
|
}
|
|
55109
|
-
|
|
55110
|
-
|
|
55111
|
-
|
|
55112
|
-
|
|
55113
|
-
|
|
55114
|
-
|
|
55115
|
-
|
|
55116
|
-
|
|
55117
|
-
|
|
55118
|
-
if (
|
|
55119
|
-
|
|
55120
|
-
|
|
55121
|
-
|
|
55122
|
-
results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
|
|
55123
|
-
continue;
|
|
55124
|
-
}
|
|
55125
|
-
if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
|
|
55126
|
-
statusByKey.set(node.key, "BLOCKED");
|
|
55127
|
-
results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
|
|
55128
|
-
continue;
|
|
55257
|
+
else {
|
|
55258
|
+
const actionCount = 0 +
|
|
55259
|
+
(graph.tables ? graph.tables.length : 0) +
|
|
55260
|
+
(graph.assertions ? graph.assertions.length : 0) +
|
|
55261
|
+
(graph.operations ? graph.operations.length : 0);
|
|
55262
|
+
writeStdOut(successOutput(`Compiled ${actionCount} action(s).`));
|
|
55263
|
+
if (graph.tables && graph.tables.length) {
|
|
55264
|
+
graph.tables.forEach(setOrValidateTableEnumType);
|
|
55265
|
+
writeStdOut(`${graph.tables.length} dataset(s)${quietCompilation ? "." : ":"}`);
|
|
55266
|
+
if (!quietCompilation) {
|
|
55267
|
+
graph.tables.forEach(compiledTable => {
|
|
55268
|
+
writeStdOut(`${datasetString(compiledTable.target, tableTypeEnumToString(compiledTable.enumType), compiledTable.disabled)}`, 1);
|
|
55269
|
+
});
|
|
55129
55270
|
}
|
|
55130
|
-
|
|
55131
|
-
|
|
55132
|
-
|
|
55133
|
-
|
|
55134
|
-
|
|
55135
|
-
|
|
55136
|
-
|
|
55137
|
-
if (status === "PASS" && node.kind === "table") {
|
|
55138
|
-
try {
|
|
55139
|
-
await deps.execute(deps.validationStubSql(node.table));
|
|
55140
|
-
}
|
|
55141
|
-
catch (e) {
|
|
55142
|
-
}
|
|
55271
|
+
}
|
|
55272
|
+
if (graph.assertions && graph.assertions.length) {
|
|
55273
|
+
writeStdOut(`${graph.assertions.length} assertion(s)${quietCompilation ? "." : ":"}`);
|
|
55274
|
+
if (!quietCompilation) {
|
|
55275
|
+
graph.assertions.forEach(assertion => {
|
|
55276
|
+
writeStdOut(assertionString(assertion.target, assertion.disabled), 1);
|
|
55277
|
+
});
|
|
55143
55278
|
}
|
|
55144
55279
|
}
|
|
55145
|
-
|
|
55146
|
-
|
|
55147
|
-
|
|
55148
|
-
|
|
55149
|
-
|
|
55150
|
-
|
|
55151
|
-
}
|
|
55152
|
-
catch (e) {
|
|
55153
|
-
}
|
|
55280
|
+
if (graph.operations && graph.operations.length) {
|
|
55281
|
+
writeStdOut(`${graph.operations.length} operation(s)${quietCompilation ? "." : ":"}`);
|
|
55282
|
+
if (!quietCompilation) {
|
|
55283
|
+
graph.operations.forEach(operation => {
|
|
55284
|
+
writeStdOut(operationString(operation.target, operation.disabled), 1);
|
|
55285
|
+
});
|
|
55154
55286
|
}
|
|
55155
55287
|
}
|
|
55156
55288
|
}
|
|
55157
|
-
return results;
|
|
55158
55289
|
}
|
|
55159
|
-
|
|
55160
|
-
|
|
55161
|
-
|
|
55162
|
-
|
|
55163
|
-
|
|
55164
|
-
const clientConfig = {
|
|
55165
|
-
host: credentials.host,
|
|
55166
|
-
port: credentials.port,
|
|
55167
|
-
database: credentials.database,
|
|
55168
|
-
user: credentials.user,
|
|
55169
|
-
password: credentials.password,
|
|
55170
|
-
ssl: sslEnabled
|
|
55171
|
-
? {
|
|
55172
|
-
rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
|
|
55173
|
-
}
|
|
55174
|
-
: false
|
|
55175
|
-
};
|
|
55176
|
-
const queryExecutor = new PgPoolExecutor(clientConfig, options);
|
|
55177
|
-
try {
|
|
55178
|
-
await queryExecutor.verifyConnection();
|
|
55179
|
-
}
|
|
55180
|
-
catch (e) {
|
|
55181
|
-
await queryExecutor.close().catch(() => undefined);
|
|
55182
|
-
throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
|
|
55183
|
-
`as "${credentials.user}": ${e.message}`, e);
|
|
55184
|
-
}
|
|
55185
|
-
return new SupabaseDbAdapter(queryExecutor);
|
|
55186
|
-
}
|
|
55187
|
-
constructor(queryExecutor) {
|
|
55188
|
-
super(queryExecutor);
|
|
55189
|
-
}
|
|
55190
|
-
async verifyServiceRolePermissions() {
|
|
55191
|
-
var _a;
|
|
55192
|
-
try {
|
|
55193
|
-
const result = await this.execute("select current_user");
|
|
55194
|
-
const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
|
|
55195
|
-
return !!currentUser;
|
|
55196
|
-
}
|
|
55197
|
-
catch (_b) {
|
|
55198
|
-
return false;
|
|
55199
|
-
}
|
|
55200
|
-
}
|
|
55201
|
-
async validatePublicationState(pubName = "supabase_realtime") {
|
|
55202
|
-
try {
|
|
55203
|
-
const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
|
|
55204
|
-
params: [pubName]
|
|
55205
|
-
});
|
|
55206
|
-
return result.rows.length > 0;
|
|
55207
|
-
}
|
|
55208
|
-
catch (_a) {
|
|
55209
|
-
return false;
|
|
55290
|
+
function formatStackTraceForQuietCompilation(compileError) {
|
|
55291
|
+
var _a, _b;
|
|
55292
|
+
if ((_a = compileError === null || compileError === void 0 ? void 0 : compileError.message) === null || _a === void 0 ? void 0 : _a.includes("Unexpected identifier")) {
|
|
55293
|
+
if (!compileError.stack) {
|
|
55294
|
+
return "";
|
|
55210
55295
|
}
|
|
55296
|
+
const stackLines = ((_b = compileError.stack) === null || _b === void 0 ? void 0 : _b.split("\n")) || [];
|
|
55297
|
+
return stackLines.slice(0, Math.min(3, stackLines.length)).join("\n");
|
|
55211
55298
|
}
|
|
55299
|
+
return "";
|
|
55212
55300
|
}
|
|
55213
|
-
|
|
55214
|
-
|
|
55215
|
-
|
|
55216
|
-
|
|
55217
|
-
|
|
55218
|
-
|
|
55219
|
-
throw new Error(`${checkPath} does not exist!`);
|
|
55301
|
+
function printCompiledGraphErrors(graphErrors, quietCompilation) {
|
|
55302
|
+
if (graphErrors.compilationErrors && graphErrors.compilationErrors.length > 0) {
|
|
55303
|
+
printError("Compilation errors:", 1);
|
|
55304
|
+
graphErrors.compilationErrors.forEach(compileError => {
|
|
55305
|
+
writeStdErr(`${calloutOutput(compileError.fileName)}: ${errorOutput(quietCompilation ? (compileError.message + " " + formatStackTraceForQuietCompilation(compileError) || compileError.stack) : (compileError.stack || compileError.message))}`, 1);
|
|
55306
|
+
});
|
|
55220
55307
|
}
|
|
55221
55308
|
}
|
|
55222
|
-
function
|
|
55223
|
-
|
|
55224
|
-
|
|
55225
|
-
|
|
55226
|
-
function formatExecutionSuffix(jobIds, bytesBilled) {
|
|
55227
|
-
const jobMetadataParts = [];
|
|
55228
|
-
if (jobIds.length > 0) {
|
|
55229
|
-
jobMetadataParts.push(`\n \t jobId: ${jobIds.join(", ")}`);
|
|
55230
|
-
}
|
|
55231
|
-
if (bytesBilled.length > 0) {
|
|
55232
|
-
jobMetadataParts.push(`\n \t Bytes billed: ${bytesBilled.join(", ")}`);
|
|
55309
|
+
function printTestResult(testResult) {
|
|
55310
|
+
writeStdOut(`${testResult.name}: ${testResult.successful ? successOutput("passed") : errorOutput("failed")}`);
|
|
55311
|
+
if (!testResult.successful) {
|
|
55312
|
+
testResult.messages.forEach(message => writeStdErr(message, 1));
|
|
55233
55313
|
}
|
|
55234
|
-
return jobMetadataParts.length > 0 ? ` ${jobMetadataParts}` : "";
|
|
55235
55314
|
}
|
|
55236
|
-
function
|
|
55237
|
-
if (
|
|
55238
|
-
|
|
55315
|
+
function printExecutionGraph(executionGraph, asJson) {
|
|
55316
|
+
if (asJson) {
|
|
55317
|
+
writeStdOut(prettyJsonStringify(executionGraph.toJSON()));
|
|
55239
55318
|
}
|
|
55240
|
-
|
|
55241
|
-
|
|
55242
|
-
|
|
55243
|
-
|
|
55244
|
-
|
|
55245
|
-
}
|
|
55246
|
-
|
|
55247
|
-
|
|
55248
|
-
|
|
55249
|
-
const tempIcebergConfig = {};
|
|
55250
|
-
let bucketName;
|
|
55251
|
-
while (true) {
|
|
55252
|
-
print(ICEBERG_BUCKET_NAME_HINT);
|
|
55253
|
-
bucketName = interactiveQuestion(ICEBERG_BUCKET_NAME_PROMPT_QUESTION);
|
|
55254
|
-
try {
|
|
55255
|
-
if (bucketName) {
|
|
55256
|
-
validateIcebergConfigBucketName(bucketName);
|
|
55257
|
-
tempIcebergConfig.bucketName = bucketName;
|
|
55319
|
+
else {
|
|
55320
|
+
const actionsByType = {
|
|
55321
|
+
table: [],
|
|
55322
|
+
assertion: [],
|
|
55323
|
+
operation: []
|
|
55324
|
+
};
|
|
55325
|
+
executionGraph.actions.forEach(action => {
|
|
55326
|
+
if (!(action.type === "table" || action.type === "assertion" || action.type === "operation")) {
|
|
55327
|
+
throw new Error(`Unrecognized action type: ${action.type}`);
|
|
55258
55328
|
}
|
|
55259
|
-
|
|
55260
|
-
}
|
|
55261
|
-
|
|
55262
|
-
|
|
55263
|
-
|
|
55329
|
+
actionsByType[action.type].push(action);
|
|
55330
|
+
});
|
|
55331
|
+
const datasetActions = actionsByType.table;
|
|
55332
|
+
if (datasetActions && datasetActions.length) {
|
|
55333
|
+
writeStdOut(`${datasetActions.length} dataset(s):`);
|
|
55334
|
+
datasetActions.forEach(datasetAction => writeStdOut(datasetString(datasetAction.target, datasetAction.type, datasetAction.tasks.length === 0), 1));
|
|
55264
55335
|
}
|
|
55265
|
-
|
|
55266
|
-
|
|
55267
|
-
|
|
55268
|
-
|
|
55269
|
-
tableFolderRoot = interactiveQuestion(ICEBERG_TABLE_FOLDER_ROOT_PROMPT_QUESTION);
|
|
55270
|
-
try {
|
|
55271
|
-
if (tableFolderRoot) {
|
|
55272
|
-
validateIcebergConfigTableFolderRoot(tableFolderRoot);
|
|
55273
|
-
tempIcebergConfig.tableFolderRoot = tableFolderRoot;
|
|
55274
|
-
}
|
|
55275
|
-
break;
|
|
55336
|
+
const assertionActions = actionsByType.assertion;
|
|
55337
|
+
if (assertionActions && assertionActions.length) {
|
|
55338
|
+
writeStdOut(`${assertionActions.length} assertion(s):`);
|
|
55339
|
+
assertionActions.forEach(assertionAction => writeStdOut(assertionString(assertionAction.target, assertionAction.tasks.length === 0), 1));
|
|
55276
55340
|
}
|
|
55277
|
-
|
|
55278
|
-
|
|
55279
|
-
|
|
55341
|
+
const operationActions = actionsByType.operation;
|
|
55342
|
+
if (operationActions && operationActions.length) {
|
|
55343
|
+
writeStdOut(`${operationActions.length} operation(s):`);
|
|
55344
|
+
operationActions.forEach(operationAction => writeStdOut(operationString(operationAction.target, operationAction.tasks.length === 0), 1));
|
|
55280
55345
|
}
|
|
55281
55346
|
}
|
|
55282
|
-
|
|
55283
|
-
|
|
55284
|
-
|
|
55285
|
-
|
|
55286
|
-
|
|
55287
|
-
|
|
55288
|
-
|
|
55289
|
-
|
|
55347
|
+
}
|
|
55348
|
+
function printExecutedAction(executedAction, executionAction, dryRun) {
|
|
55349
|
+
const jobIds = executedAction.tasks
|
|
55350
|
+
.filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
|
|
55351
|
+
.map(task => task.metadata.bigquery.jobId);
|
|
55352
|
+
const bytesBilled = executedAction.tasks
|
|
55353
|
+
.filter(task => { var _a, _b; return (_b = (_a = task.metadata) === null || _a === void 0 ? void 0 : _a.bigquery) === null || _b === void 0 ? void 0 : _b.jobId; })
|
|
55354
|
+
.map(task => {
|
|
55355
|
+
var _a, _b, _c, _d;
|
|
55356
|
+
const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
|
|
55357
|
+
return formatBytesInHumanReadableFormat(bytes);
|
|
55358
|
+
});
|
|
55359
|
+
const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
|
|
55360
|
+
switch (executedAction.status) {
|
|
55361
|
+
case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
|
|
55362
|
+
switch (executionAction.type) {
|
|
55363
|
+
case "table": {
|
|
55364
|
+
writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55365
|
+
return;
|
|
55366
|
+
}
|
|
55367
|
+
case "assertion": {
|
|
55368
|
+
writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55369
|
+
return;
|
|
55370
|
+
}
|
|
55371
|
+
case "operation": {
|
|
55372
|
+
writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55373
|
+
return;
|
|
55374
|
+
}
|
|
55290
55375
|
}
|
|
55291
|
-
break;
|
|
55292
55376
|
}
|
|
55293
|
-
|
|
55294
|
-
|
|
55295
|
-
|
|
55377
|
+
case sqlanvil.ActionResult.ExecutionStatus.FAILED: {
|
|
55378
|
+
switch (executionAction.type) {
|
|
55379
|
+
case "table": {
|
|
55380
|
+
writeStdErr(`${errorOutput("Dataset creation failed: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55381
|
+
break;
|
|
55382
|
+
}
|
|
55383
|
+
case "assertion": {
|
|
55384
|
+
writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55385
|
+
break;
|
|
55386
|
+
}
|
|
55387
|
+
case "operation": {
|
|
55388
|
+
writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55389
|
+
break;
|
|
55390
|
+
}
|
|
55391
|
+
}
|
|
55392
|
+
printExecutedActionErrors(executedAction, executionAction);
|
|
55393
|
+
return;
|
|
55296
55394
|
}
|
|
55297
|
-
|
|
55298
|
-
|
|
55299
|
-
|
|
55300
|
-
|
|
55301
|
-
|
|
55302
|
-
|
|
55303
|
-
|
|
55304
|
-
|
|
55305
|
-
|
|
55395
|
+
case sqlanvil.ActionResult.ExecutionStatus.SKIPPED: {
|
|
55396
|
+
switch (executionAction.type) {
|
|
55397
|
+
case "table": {
|
|
55398
|
+
writeStdOut(`${warningOutput("Skipping dataset creation: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
|
|
55399
|
+
return;
|
|
55400
|
+
}
|
|
55401
|
+
case "assertion": {
|
|
55402
|
+
writeStdOut(`${warningOutput("Skipping assertion execution: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55403
|
+
return;
|
|
55404
|
+
}
|
|
55405
|
+
case "operation": {
|
|
55406
|
+
writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55407
|
+
return;
|
|
55408
|
+
}
|
|
55306
55409
|
}
|
|
55307
|
-
|
|
55410
|
+
return;
|
|
55308
55411
|
}
|
|
55309
|
-
|
|
55310
|
-
|
|
55311
|
-
|
|
55412
|
+
case sqlanvil.ActionResult.ExecutionStatus.DISABLED: {
|
|
55413
|
+
switch (executionAction.type) {
|
|
55414
|
+
case "table": {
|
|
55415
|
+
writeStdOut(`${warningOutput("Dataset creation disabled: ")} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}`);
|
|
55416
|
+
return;
|
|
55417
|
+
}
|
|
55418
|
+
case "assertion": {
|
|
55419
|
+
writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55420
|
+
return;
|
|
55421
|
+
}
|
|
55422
|
+
case "operation": {
|
|
55423
|
+
writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55424
|
+
return;
|
|
55425
|
+
}
|
|
55426
|
+
}
|
|
55427
|
+
return;
|
|
55312
55428
|
}
|
|
55313
55429
|
}
|
|
55314
|
-
printSuccess(ICEBERG_CONFIG_COLLECTED_TEXT);
|
|
55315
|
-
if (Object.keys(tempIcebergConfig).length > 0) {
|
|
55316
|
-
return tempIcebergConfig;
|
|
55317
|
-
}
|
|
55318
|
-
return undefined;
|
|
55319
55430
|
}
|
|
55320
|
-
function
|
|
55321
|
-
|
|
55322
|
-
|
|
55431
|
+
function printFormatFilesResult(formatResults) {
|
|
55432
|
+
const sorted = formatResults.sort((a, b) => a.filename.localeCompare(b.filename));
|
|
55433
|
+
const successfulFormatResults = sorted.filter(result => !result.err && !result.needsFormatting);
|
|
55434
|
+
const needsFormattingResults = sorted.filter(result => !result.err && result.needsFormatting);
|
|
55435
|
+
const failedFormatResults = sorted.filter(result => !!result.err);
|
|
55436
|
+
if (successfulFormatResults.length > 0) {
|
|
55437
|
+
printSuccess("Successfully formatted:");
|
|
55438
|
+
successfulFormatResults.forEach(result => writeStdOut(result.filename, 1));
|
|
55323
55439
|
}
|
|
55324
|
-
if (
|
|
55325
|
-
|
|
55440
|
+
if (needsFormattingResults.length > 0) {
|
|
55441
|
+
printError("Files that need formatting:");
|
|
55442
|
+
needsFormattingResults.forEach(result => writeStdErr(result.filename, 1));
|
|
55326
55443
|
}
|
|
55327
|
-
if (
|
|
55328
|
-
|
|
55444
|
+
if (failedFormatResults.length > 0) {
|
|
55445
|
+
printError("Errors encountered during formatting:");
|
|
55446
|
+
failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
|
|
55329
55447
|
}
|
|
55330
|
-
|
|
55331
|
-
|
|
55448
|
+
}
|
|
55449
|
+
function datasetString(target, datasetType, disabled) {
|
|
55450
|
+
return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
|
|
55451
|
+
}
|
|
55452
|
+
function assertionString(target, disabled) {
|
|
55453
|
+
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55454
|
+
}
|
|
55455
|
+
function operationString(target, disabled) {
|
|
55456
|
+
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55457
|
+
}
|
|
55458
|
+
function plainTargetString(target) {
|
|
55459
|
+
return `${target.schema}.${target.name}`;
|
|
55460
|
+
}
|
|
55461
|
+
function targetString(target) {
|
|
55462
|
+
return calloutOutput(`${target.schema}.${target.name}`);
|
|
55463
|
+
}
|
|
55464
|
+
function dotRepresentation(graph, interactive) {
|
|
55465
|
+
var _a, _b, _c;
|
|
55466
|
+
const nodes = [];
|
|
55467
|
+
const edges = [];
|
|
55468
|
+
const formatTarget = interactive ? targetString : plainTargetString;
|
|
55469
|
+
(_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
|
|
55470
|
+
var _a;
|
|
55471
|
+
const nodeName = `${formatTarget(table.target)}`;
|
|
55472
|
+
nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
|
|
55473
|
+
(_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55474
|
+
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55475
|
+
});
|
|
55476
|
+
});
|
|
55477
|
+
(_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
|
|
55478
|
+
var _a;
|
|
55479
|
+
const nodeName = `${formatTarget(assertion.target)}`;
|
|
55480
|
+
nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
|
|
55481
|
+
(_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55482
|
+
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55483
|
+
});
|
|
55484
|
+
});
|
|
55485
|
+
(_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
|
|
55486
|
+
var _a;
|
|
55487
|
+
const nodeName = `${formatTarget(operation.target)}`;
|
|
55488
|
+
nodes.push(`"${nodeName}" [label="${formatTarget(operation.target)}"`);
|
|
55489
|
+
(_a = operation.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55490
|
+
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55491
|
+
});
|
|
55492
|
+
});
|
|
55493
|
+
return `digraph {\n${nodes.join(";\n")};\n${edges.join(";\n")};\n}`;
|
|
55494
|
+
}
|
|
55495
|
+
function printExecutedActionErrors(executedAction, executionAction) {
|
|
55496
|
+
const failingTasks = executedAction.tasks.filter(task => task.status === sqlanvil.TaskResult.ExecutionStatus.FAILED);
|
|
55497
|
+
failingTasks.forEach((task, i) => {
|
|
55498
|
+
executionAction.tasks[i].statement.split("\n").forEach(line => {
|
|
55499
|
+
writeStdErr(`${DEFAULT_PROMPT}${line}`, 1);
|
|
55500
|
+
});
|
|
55501
|
+
printError(task.errorMessage, 1);
|
|
55502
|
+
});
|
|
55503
|
+
}
|
|
55504
|
+
|
|
55505
|
+
const INIT_MODE_QUESTION = "Start a fresh project, or convert an existing Dataform project? (fresh/convert)";
|
|
55506
|
+
const INIT_WAREHOUSE_QUESTION = "Which warehouse? (supabase/postgres/bigquery/mysql)";
|
|
55507
|
+
const INIT_PROJECT_DIR_QUESTION = "Project directory?";
|
|
55508
|
+
const INIT_BQ_PROJECT_QUESTION = "Google Cloud project ID (the default project models build in)?";
|
|
55509
|
+
const INIT_BQ_LOCATION_QUESTION = "Default BigQuery location (e.g. US, us-central1)?";
|
|
55510
|
+
const INIT_DEFAULT_SCHEMA_QUESTION = "Default schema for models?";
|
|
55511
|
+
const INIT_INCLUDE_SAMPLE_QUESTION = "Include the sample project (sources -> staging views -> outputs -> assertion)?";
|
|
55512
|
+
const INIT_INCLUDE_BQ_SOURCE_QUESTION = "Include the cross-warehouse BigQuery sample source (Google public ZIP data)?";
|
|
55513
|
+
const INIT_CONFIGURE_CREDS_QUESTION = `Configure warehouse credentials now (written to the gitignored ${CREDENTIALS_FILENAME})?`;
|
|
55514
|
+
const CREDS_HOST_QUESTION = "Host?";
|
|
55515
|
+
const CREDS_PORT_QUESTION = "Port?";
|
|
55516
|
+
const CREDS_DATABASE_QUESTION = "Database?";
|
|
55517
|
+
const CREDS_USER_QUESTION = "User?";
|
|
55518
|
+
const CREDS_PASSWORD_QUESTION = "Password? (leave empty to fill in later)";
|
|
55519
|
+
const CREDS_SSLMODE_QUESTION = "SSL mode? (disable/require)";
|
|
55520
|
+
const CONVERT_SOURCE_QUESTION = "Path to the Dataform project to convert (read-only, never modified)?";
|
|
55521
|
+
const CONVERT_OUT_QUESTION = "Directory for the converted sqlanvil project (created; must be empty)?";
|
|
55522
|
+
const CONVERT_TARGET_QUESTION = "Target warehouse? (bigquery = keep running on BigQuery, tooling swap only; supabase/postgres = move off BigQuery)";
|
|
55523
|
+
const SUPABASE_POOLER_HINT = "Use the SESSION POOLER connection (Supabase Dashboard -> Connect -> Session pooler): host " +
|
|
55524
|
+
"aws-<n>-<region>.pooler.supabase.com, user postgres.<project-ref>. The direct " +
|
|
55525
|
+
"db.<ref>.supabase.co host is IPv6-only and unreachable from most networks.";
|
|
55526
|
+
const MAX_ATTEMPTS = 20;
|
|
55527
|
+
function withDefault(question, defaultValue) {
|
|
55528
|
+
return defaultValue ? `${question} [${defaultValue}]` : question;
|
|
55529
|
+
}
|
|
55530
|
+
function ask(question, defaultValue) {
|
|
55531
|
+
const answer = interactiveQuestion(withDefault(question, defaultValue)).trim();
|
|
55532
|
+
return answer === "" ? defaultValue !== null && defaultValue !== void 0 ? defaultValue : "" : answer;
|
|
55533
|
+
}
|
|
55534
|
+
function askRequired(question) {
|
|
55535
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55536
|
+
const answer = ask(question);
|
|
55537
|
+
if (answer !== "") {
|
|
55538
|
+
return answer;
|
|
55539
|
+
}
|
|
55540
|
+
printError("A value is required.");
|
|
55332
55541
|
}
|
|
55542
|
+
throw new Error(`No answer provided for: ${question}`);
|
|
55333
55543
|
}
|
|
55334
|
-
function
|
|
55335
|
-
|
|
55336
|
-
|
|
55544
|
+
function askChoice(question, choices, defaultValue) {
|
|
55545
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55546
|
+
const answer = ask(question, defaultValue).toLowerCase();
|
|
55547
|
+
if (choices.includes(answer)) {
|
|
55548
|
+
return answer;
|
|
55549
|
+
}
|
|
55550
|
+
printError(`Please answer one of: ${choices.join(", ")}.`);
|
|
55337
55551
|
}
|
|
55552
|
+
throw new Error(`No valid answer provided for: ${question}`);
|
|
55338
55553
|
}
|
|
55339
|
-
function
|
|
55340
|
-
|
|
55341
|
-
|
|
55554
|
+
function askYesNo(question, defaultValue) {
|
|
55555
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55556
|
+
const answer = ask(question, defaultValue ? "y" : "n").toLowerCase();
|
|
55557
|
+
if (answer === "y" || answer === "yes") {
|
|
55558
|
+
return true;
|
|
55559
|
+
}
|
|
55560
|
+
if (answer === "n" || answer === "no") {
|
|
55561
|
+
return false;
|
|
55562
|
+
}
|
|
55563
|
+
printError("Please answer y or n.");
|
|
55342
55564
|
}
|
|
55343
|
-
|
|
55344
|
-
|
|
55565
|
+
throw new Error(`No valid answer provided for: ${question}`);
|
|
55566
|
+
}
|
|
55567
|
+
function askInt(question, defaultValue) {
|
|
55568
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55569
|
+
const value = parseInt(ask(question, String(defaultValue)), 10);
|
|
55570
|
+
if (!isNaN(value)) {
|
|
55571
|
+
return value;
|
|
55572
|
+
}
|
|
55573
|
+
printError("Entered value must be an integer.");
|
|
55345
55574
|
}
|
|
55575
|
+
throw new Error(`No valid answer provided for: ${question}`);
|
|
55346
55576
|
}
|
|
55347
|
-
|
|
55348
|
-
|
|
55349
|
-
const
|
|
55350
|
-
|
|
55351
|
-
|
|
55352
|
-
|
|
55353
|
-
|
|
55354
|
-
|
|
55355
|
-
|
|
55356
|
-
|
|
55357
|
-
|
|
55358
|
-
|
|
55359
|
-
const noColor = process.env.NO_COLOR && ["1", "true", "yes"].includes(process.env.NO_COLOR.toLowerCase());
|
|
55360
|
-
const ansiColorCodes = Object.freeze({
|
|
55361
|
-
red: 91,
|
|
55362
|
-
green: 32,
|
|
55363
|
-
yellow: 93,
|
|
55364
|
-
cyan: 36
|
|
55365
|
-
});
|
|
55366
|
-
function output(text, ansiColorCode) {
|
|
55367
|
-
if (noColor) {
|
|
55368
|
-
return text;
|
|
55577
|
+
function collectCredentials(warehouse) {
|
|
55578
|
+
if (warehouse === "mysql") {
|
|
55579
|
+
const creds = {
|
|
55580
|
+
host: ask(CREDS_HOST_QUESTION, "localhost"),
|
|
55581
|
+
port: askInt(CREDS_PORT_QUESTION, 3306),
|
|
55582
|
+
database: ask(CREDS_DATABASE_QUESTION, "sqlanvil"),
|
|
55583
|
+
user: ask(CREDS_USER_QUESTION, "root"),
|
|
55584
|
+
password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
|
|
55585
|
+
sslMode: ask(CREDS_SSLMODE_QUESTION, "disable")
|
|
55586
|
+
};
|
|
55587
|
+
return `${JSON.stringify(creds, null, 2)}\n`;
|
|
55369
55588
|
}
|
|
55370
|
-
|
|
55589
|
+
const isSupabase = warehouse === "supabase";
|
|
55590
|
+
if (isSupabase) {
|
|
55591
|
+
print(SUPABASE_POOLER_HINT);
|
|
55592
|
+
}
|
|
55593
|
+
const host = ask(CREDS_HOST_QUESTION, isSupabase ? "aws-1-<region>.pooler.supabase.com" : "localhost");
|
|
55594
|
+
if (/^db\..+\.supabase\.co$/.test(host)) {
|
|
55595
|
+
printError("Warning: that is the DIRECT Supabase host, which is IPv6-only — connections fail with " +
|
|
55596
|
+
"ENOTFOUND on most networks. Prefer the Session pooler host (Dashboard -> Connect).");
|
|
55597
|
+
}
|
|
55598
|
+
const creds = {
|
|
55599
|
+
host,
|
|
55600
|
+
port: askInt(CREDS_PORT_QUESTION, 5432),
|
|
55601
|
+
database: ask(CREDS_DATABASE_QUESTION, "postgres"),
|
|
55602
|
+
user: ask(CREDS_USER_QUESTION, isSupabase ? "postgres.<your-project-ref>" : "postgres"),
|
|
55603
|
+
password: interactivePasswordQuestion(CREDS_PASSWORD_QUESTION),
|
|
55604
|
+
sslMode: ask(CREDS_SSLMODE_QUESTION, isSupabase ? "require" : "disable"),
|
|
55605
|
+
defaultSchema: "public"
|
|
55606
|
+
};
|
|
55607
|
+
return `${JSON.stringify(creds, null, 2)}\n`;
|
|
55371
55608
|
}
|
|
55372
|
-
|
|
55373
|
-
const
|
|
55374
|
-
const
|
|
55375
|
-
|
|
55376
|
-
|
|
55377
|
-
|
|
55378
|
-
|
|
55379
|
-
|
|
55380
|
-
function question(questionText) {
|
|
55381
|
-
return prompt(questionText);
|
|
55609
|
+
function printMigrationSummary(report, outDir) {
|
|
55610
|
+
const targets = report.files.filter(f => f.action === "target");
|
|
55611
|
+
const flagged = targets.filter(f => f.status === "flagged").length;
|
|
55612
|
+
printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
|
|
55613
|
+
`${report.connections.length} source connection(s) over ` +
|
|
55614
|
+
`${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
|
|
55615
|
+
`${targets.length} target file(s), ${flagged} flagged for dialect review.`);
|
|
55616
|
+
print(`Report: ${path__namespace.join(outDir, "migration-report.md")}`);
|
|
55382
55617
|
}
|
|
55383
|
-
function
|
|
55384
|
-
|
|
55385
|
-
|
|
55386
|
-
|
|
55387
|
-
return
|
|
55618
|
+
function ensureGitignoreCoversCredentials(dir) {
|
|
55619
|
+
const gitignorePath = path__namespace.join(dir, ".gitignore");
|
|
55620
|
+
const existing = fs__namespace$1.existsSync(gitignorePath) ? fs__namespace$1.readFileSync(gitignorePath, "utf8") : "";
|
|
55621
|
+
if (existing.includes(".df-credentials")) {
|
|
55622
|
+
return;
|
|
55388
55623
|
}
|
|
55389
|
-
|
|
55390
|
-
|
|
55624
|
+
const separator = existing === "" || existing.endsWith("\n") ? "" : "\n";
|
|
55625
|
+
fs__namespace$1.writeFileSync(gitignorePath, `${existing}${separator}.df-credentials*.json\n`);
|
|
55391
55626
|
}
|
|
55392
|
-
function
|
|
55393
|
-
|
|
55394
|
-
|
|
55395
|
-
|
|
55627
|
+
async function runConvertFlow() {
|
|
55628
|
+
let srcDir = "";
|
|
55629
|
+
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
|
|
55630
|
+
const candidate = actuallyResolve(askRequired(CONVERT_SOURCE_QUESTION));
|
|
55631
|
+
if (fs__namespace$1.existsSync(candidate)) {
|
|
55632
|
+
srcDir = candidate;
|
|
55633
|
+
break;
|
|
55634
|
+
}
|
|
55635
|
+
printError(`${candidate} does not exist.`);
|
|
55636
|
+
}
|
|
55637
|
+
if (!srcDir) {
|
|
55638
|
+
throw new Error("No usable source directory provided.");
|
|
55639
|
+
}
|
|
55640
|
+
const outDir = actuallyResolve(askRequired(CONVERT_OUT_QUESTION));
|
|
55641
|
+
const targetWarehouse = askChoice(CONVERT_TARGET_QUESTION, ["bigquery", "supabase", "postgres"], "bigquery");
|
|
55642
|
+
print("\nConverting...\n");
|
|
55643
|
+
const report = await migrateDataform({ srcDir, outDir, targetWarehouse });
|
|
55644
|
+
printMigrationSummary(report, outDir);
|
|
55645
|
+
if (targetWarehouse === "bigquery") {
|
|
55646
|
+
print("\nNext steps:");
|
|
55647
|
+
print(` 1. sqlanvil compile ${outDir}`);
|
|
55648
|
+
print(" 2. Credentials: gcloud auth application-default login (ADC), or a service-account " +
|
|
55649
|
+
`key in a gitignored ${CREDENTIALS_FILENAME}.`);
|
|
55650
|
+
print(` 3. sqlanvil validate ${outDir} (dry-runs every model against BigQuery; all-PASS = swap complete)`);
|
|
55651
|
+
return 0;
|
|
55396
55652
|
}
|
|
55397
|
-
|
|
55398
|
-
const
|
|
55399
|
-
|
|
55400
|
-
|
|
55401
|
-
|
|
55402
|
-
|
|
55403
|
-
|
|
55404
|
-
|
|
55405
|
-
|
|
55406
|
-
|
|
55407
|
-
|
|
55408
|
-
|
|
55409
|
-
|
|
55653
|
+
if (askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
|
|
55654
|
+
const credentialsPath = path__namespace.join(outDir, CREDENTIALS_FILENAME);
|
|
55655
|
+
ensureGitignoreCoversCredentials(outDir);
|
|
55656
|
+
fs__namespace$1.writeFileSync(credentialsPath, collectCredentials(targetWarehouse));
|
|
55657
|
+
printSuccess(`Credentials written to ${credentialsPath}`);
|
|
55658
|
+
}
|
|
55659
|
+
print("\nNext steps:");
|
|
55660
|
+
print(` 1. sqlanvil compile ${outDir}`);
|
|
55661
|
+
print(` 2. Review ${path__namespace.join(outDir, "migration-report.md")} — BigQuery source connections ` +
|
|
55662
|
+
`need columnTypes (the introspect commands are in the report).`);
|
|
55663
|
+
print(` 3. sqlanvil validate ${outDir} (PASS/FAILURE/BLOCKED per model = the migration to-do list)`);
|
|
55664
|
+
return 0;
|
|
55665
|
+
}
|
|
55666
|
+
async function runFreshFlow(defaultProjectDir) {
|
|
55667
|
+
const warehouse = askChoice(INIT_WAREHOUSE_QUESTION, ["supabase", "postgres", "bigquery", "mysql"], "supabase");
|
|
55668
|
+
const isPostgresLike = warehouse === "postgres" || warehouse === "supabase";
|
|
55669
|
+
const projectDir = actuallyResolve(ask(INIT_PROJECT_DIR_QUESTION, defaultProjectDir));
|
|
55670
|
+
const projectConfig = { warehouse };
|
|
55671
|
+
if (warehouse === "bigquery") {
|
|
55672
|
+
projectConfig.defaultDatabase = askRequired(INIT_BQ_PROJECT_QUESTION);
|
|
55673
|
+
projectConfig.defaultLocation = askRequired(INIT_BQ_LOCATION_QUESTION);
|
|
55674
|
+
}
|
|
55675
|
+
projectConfig.defaultSchema = ask(INIT_DEFAULT_SCHEMA_QUESTION, isPostgresLike ? "public" : "sqlanvil");
|
|
55676
|
+
const includeSample = askYesNo(INIT_INCLUDE_SAMPLE_QUESTION, true);
|
|
55677
|
+
const includeBigQuerySource = includeSample && isPostgresLike ? askYesNo(INIT_INCLUDE_BQ_SOURCE_QUESTION, true) : undefined;
|
|
55678
|
+
let credentialsJson;
|
|
55679
|
+
if (warehouse !== "bigquery" && askYesNo(INIT_CONFIGURE_CREDS_QUESTION, true)) {
|
|
55680
|
+
credentialsJson = collectCredentials(warehouse);
|
|
55681
|
+
}
|
|
55682
|
+
print("\nWriting project files...\n");
|
|
55683
|
+
const result = await init(projectDir, projectConfig, {
|
|
55684
|
+
includeSample,
|
|
55685
|
+
includeBigQuerySource,
|
|
55686
|
+
credentialsJson
|
|
55687
|
+
});
|
|
55688
|
+
printInitResult(result);
|
|
55689
|
+
const steps = [];
|
|
55690
|
+
if (warehouse === "bigquery") {
|
|
55691
|
+
steps.push(`sqlanvil init-creds ${projectDir} (BigQuery credentials)`);
|
|
55692
|
+
}
|
|
55693
|
+
else if (!credentialsJson) {
|
|
55694
|
+
steps.push(`Edit ${CREDENTIALS_FILENAME} (gitignored) with your warehouse credentials.`);
|
|
55695
|
+
}
|
|
55696
|
+
if (includeSample) {
|
|
55697
|
+
steps.push("Point definitions/sources/app_orders.sqlx at a real table in your warehouse.");
|
|
55698
|
+
}
|
|
55699
|
+
if (isPostgresLike && includeBigQuerySource) {
|
|
55700
|
+
steps.push("Replace REPLACE_WITH_YOUR_GCP_PROJECT in workflow_settings.yaml (billingProject for " +
|
|
55701
|
+
"the BigQuery sample source).");
|
|
55702
|
+
}
|
|
55703
|
+
steps.push(`sqlanvil compile ${projectDir}`);
|
|
55704
|
+
steps.push(`sqlanvil validate ${projectDir} (checks every model against the warehouse without executing)`);
|
|
55705
|
+
print("\nNext steps:");
|
|
55706
|
+
steps.forEach((step, index) => print(` ${index + 1}. ${step}`));
|
|
55707
|
+
return 0;
|
|
55708
|
+
}
|
|
55709
|
+
async function runInteractiveInit(defaultProjectDir) {
|
|
55710
|
+
if (!process.stdin.isTTY && process.env.DATAFORM_CLI_TEST_INPUTS === undefined) {
|
|
55711
|
+
printError("init --interactive needs an interactive terminal (stdin is not a TTY). " +
|
|
55712
|
+
"Use the non-interactive form instead: sqlanvil init [project-dir] --warehouse=... " +
|
|
55713
|
+
'(see "sqlanvil help init").');
|
|
55714
|
+
return 1;
|
|
55715
|
+
}
|
|
55716
|
+
const mode = askChoice(INIT_MODE_QUESTION, ["fresh", "convert"], "fresh");
|
|
55717
|
+
if (mode === "convert") {
|
|
55718
|
+
return runConvertFlow();
|
|
55719
|
+
}
|
|
55720
|
+
return runFreshFlow(defaultProjectDir);
|
|
55721
|
+
}
|
|
55722
|
+
|
|
55723
|
+
const CHECKER_SOURCE = `
|
|
55724
|
+
import json, os, sys
|
|
55725
|
+
|
|
55726
|
+
script_path, requirements_path, version_spec = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
55727
|
+
errors = []
|
|
55728
|
+
|
|
55729
|
+
import platform
|
|
55730
|
+
py_version = platform.python_version()
|
|
55731
|
+
|
|
55732
|
+
def load_packaging():
|
|
55733
|
+
try:
|
|
55734
|
+
from packaging.specifiers import SpecifierSet
|
|
55735
|
+
from packaging.requirements import Requirement
|
|
55736
|
+
return SpecifierSet, Requirement
|
|
55737
|
+
except ImportError:
|
|
55738
|
+
pass
|
|
55739
|
+
try:
|
|
55740
|
+
from pip._vendor.packaging.specifiers import SpecifierSet
|
|
55741
|
+
from pip._vendor.packaging.requirements import Requirement
|
|
55742
|
+
return SpecifierSet, Requirement
|
|
55743
|
+
except ImportError:
|
|
55744
|
+
return None, None
|
|
55745
|
+
|
|
55746
|
+
SpecifierSet, Requirement = load_packaging()
|
|
55747
|
+
|
|
55748
|
+
def mini_parse(v):
|
|
55749
|
+
import re
|
|
55750
|
+
return [int(x) for x in re.findall(r"[0-9]+", v.split("+")[0])][:4]
|
|
55751
|
+
|
|
55752
|
+
def mini_check(version, spec):
|
|
55753
|
+
import re
|
|
55754
|
+
ok = True
|
|
55755
|
+
for clause in spec.split(","):
|
|
55756
|
+
clause = clause.strip()
|
|
55757
|
+
m = re.match(r"(===|==|!=|<=|>=|~=|<|>)\\s*(.+)", clause)
|
|
55758
|
+
if not m:
|
|
55759
|
+
return None
|
|
55760
|
+
op, want = m.group(1), m.group(2).strip()
|
|
55761
|
+
if want.endswith(".*"):
|
|
55762
|
+
prefix = mini_parse(want[:-2])
|
|
55763
|
+
have = mini_parse(version)[:len(prefix)]
|
|
55764
|
+
eq = have == prefix
|
|
55765
|
+
if (op in ("==", "===") and not eq) or (op == "!=" and eq):
|
|
55766
|
+
ok = False
|
|
55767
|
+
continue
|
|
55768
|
+
a, b = mini_parse(version), mini_parse(want)
|
|
55769
|
+
n = max(len(a), len(b))
|
|
55770
|
+
a = a + [0] * (n - len(a))
|
|
55771
|
+
b = b + [0] * (n - len(b))
|
|
55772
|
+
if op in ("==", "==="):
|
|
55773
|
+
res = a == b
|
|
55774
|
+
elif op == "!=":
|
|
55775
|
+
res = a != b
|
|
55776
|
+
elif op == "<=":
|
|
55777
|
+
res = a <= b
|
|
55778
|
+
elif op == ">=":
|
|
55779
|
+
res = a >= b
|
|
55780
|
+
elif op == "<":
|
|
55781
|
+
res = a < b
|
|
55782
|
+
elif op == ">":
|
|
55783
|
+
res = a > b
|
|
55784
|
+
else: # ~= : at least b, within b's series
|
|
55785
|
+
wb = mini_parse(want)
|
|
55786
|
+
res = a >= b and a[:max(len(wb) - 1, 1)] == b[:max(len(wb) - 1, 1)]
|
|
55787
|
+
if not res:
|
|
55788
|
+
ok = False
|
|
55789
|
+
return ok
|
|
55790
|
+
|
|
55791
|
+
def check_version(version, spec, what):
|
|
55792
|
+
if SpecifierSet is not None:
|
|
55793
|
+
try:
|
|
55794
|
+
if version not in SpecifierSet(spec, prereleases=True):
|
|
55795
|
+
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
55796
|
+
return
|
|
55797
|
+
except Exception as e:
|
|
55798
|
+
errors.append('%s: could not evaluate specifier "%s": %s' % (what, spec, e))
|
|
55799
|
+
return
|
|
55800
|
+
if mini_check(version, spec) is False:
|
|
55801
|
+
errors.append('%s: installed %s does not satisfy "%s"' % (what, version, spec))
|
|
55802
|
+
|
|
55803
|
+
if version_spec:
|
|
55804
|
+
check_version(py_version, version_spec, "python interpreter")
|
|
55805
|
+
|
|
55806
|
+
if requirements_path:
|
|
55807
|
+
from importlib import metadata
|
|
55808
|
+
try:
|
|
55809
|
+
lines = open(requirements_path).read().splitlines()
|
|
55810
|
+
except OSError as e:
|
|
55811
|
+
errors.append("requirements: cannot read %s: %s" % (requirements_path, e))
|
|
55812
|
+
lines = []
|
|
55813
|
+
for raw in lines:
|
|
55814
|
+
line = raw.split("#")[0].strip()
|
|
55815
|
+
if not line or line.startswith("-"):
|
|
55816
|
+
continue
|
|
55817
|
+
if Requirement is not None:
|
|
55818
|
+
try:
|
|
55819
|
+
req = Requirement(line)
|
|
55820
|
+
except Exception:
|
|
55821
|
+
errors.append('requirements: cannot parse "%s"' % line)
|
|
55822
|
+
continue
|
|
55823
|
+
if req.marker is not None and not req.marker.evaluate():
|
|
55824
|
+
continue
|
|
55825
|
+
name, spec = req.name, str(req.specifier)
|
|
55826
|
+
else:
|
|
55827
|
+
import re
|
|
55828
|
+
if ";" in line:
|
|
55829
|
+
line = line.split(";")[0].strip()
|
|
55830
|
+
m = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)\\s*(\\[[^\\]]*\\])?\\s*(.*)$", line)
|
|
55831
|
+
if not m:
|
|
55832
|
+
errors.append('requirements: cannot parse "%s"' % line)
|
|
55833
|
+
continue
|
|
55834
|
+
name, spec = m.group(1), (m.group(3) or "").strip()
|
|
55835
|
+
try:
|
|
55836
|
+
installed = metadata.version(name)
|
|
55837
|
+
except metadata.PackageNotFoundError:
|
|
55838
|
+
errors.append(
|
|
55839
|
+
"requirements: %s is not installed%s" % (name, " (need %s)" % spec if spec else "")
|
|
55840
|
+
)
|
|
55841
|
+
continue
|
|
55842
|
+
if spec:
|
|
55843
|
+
check_version(installed, spec, "requirements: %s" % name)
|
|
55844
|
+
|
|
55845
|
+
try:
|
|
55846
|
+
with open(script_path, "rb") as f:
|
|
55847
|
+
compile(f.read(), script_path, "exec")
|
|
55848
|
+
except SyntaxError as e:
|
|
55849
|
+
errors.append("syntax: %s (line %s)" % (e.msg, e.lineno))
|
|
55850
|
+
except (OSError, ValueError) as e:
|
|
55851
|
+
errors.append("syntax: cannot compile %s: %s" % (script_path, e))
|
|
55852
|
+
|
|
55853
|
+
print(json.dumps({"interpreter_version": py_version, "errors": errors}))
|
|
55854
|
+
`;
|
|
55855
|
+
const defaultCheckerExec = (interpreter, args, cwd) => new Promise((resolve, reject) => {
|
|
55856
|
+
childProcess.execFile(interpreter, args, { cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
|
|
55857
|
+
if (err) {
|
|
55858
|
+
reject(new Error(`checker failed under ${interpreter}: ${err.message}\n${stderr}`.trim()));
|
|
55410
55859
|
}
|
|
55411
55860
|
else {
|
|
55412
|
-
|
|
55861
|
+
resolve(stdout);
|
|
55413
55862
|
}
|
|
55863
|
+
});
|
|
55864
|
+
});
|
|
55865
|
+
const failure = (message) => ({
|
|
55866
|
+
status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE,
|
|
55867
|
+
error: { message }
|
|
55868
|
+
});
|
|
55869
|
+
async function checkScriptAction(script, projectDir, checkerExec = defaultCheckerExec) {
|
|
55870
|
+
if (script.language !== "python") {
|
|
55871
|
+
return [failure(`no validator for script language "${script.language}"`)];
|
|
55872
|
+
}
|
|
55873
|
+
let interpreter;
|
|
55874
|
+
try {
|
|
55875
|
+
interpreter = resolveInterpreter({ language: script.language, envRoot: script.envRoot }, projectDir);
|
|
55876
|
+
}
|
|
55877
|
+
catch (e) {
|
|
55878
|
+
return [failure(e.message)];
|
|
55879
|
+
}
|
|
55880
|
+
let stdout;
|
|
55881
|
+
try {
|
|
55882
|
+
stdout = await checkerExec(interpreter, ["-c", CHECKER_SOURCE, script.scriptFilename, script.depsFile || "", script.runtimeVersion || ""], projectDir);
|
|
55414
55883
|
}
|
|
55415
55884
|
catch (e) {
|
|
55416
|
-
|
|
55885
|
+
return [failure(e.message)];
|
|
55417
55886
|
}
|
|
55418
|
-
|
|
55419
|
-
|
|
55420
|
-
|
|
55421
|
-
cancel: false
|
|
55422
|
-
});
|
|
55423
|
-
}
|
|
55424
|
-
function prompt(questionText, options) {
|
|
55425
|
-
writeStdOut(questionText);
|
|
55426
|
-
return readlineSync__namespace.prompt(Object.assign(Object.assign({}, options), { prompt: (options && options.prompt && options.prompt + DEFAULT_PROMPT) || DEFAULT_PROMPT }));
|
|
55427
|
-
}
|
|
55428
|
-
function print(text) {
|
|
55429
|
-
writeStdOut(text);
|
|
55430
|
-
}
|
|
55431
|
-
function printSuccess(text) {
|
|
55432
|
-
writeStdOut(successOutput(text));
|
|
55433
|
-
}
|
|
55434
|
-
function printError(errorText, indentCount = 0) {
|
|
55435
|
-
writeStdErr(errorOutput(errorText), indentCount);
|
|
55436
|
-
}
|
|
55437
|
-
function printInitResult(result) {
|
|
55438
|
-
if (result.dirsCreated && result.dirsCreated.length) {
|
|
55439
|
-
writeStdOut(successOutput("Directories successfully created:"));
|
|
55440
|
-
result.dirsCreated.forEach(dir => writeStdOut(dir, 1));
|
|
55887
|
+
let parsed;
|
|
55888
|
+
try {
|
|
55889
|
+
parsed = JSON.parse(stdout);
|
|
55441
55890
|
}
|
|
55442
|
-
|
|
55443
|
-
|
|
55444
|
-
result.filesWritten.forEach(file => writeStdOut(file, 1));
|
|
55891
|
+
catch (e) {
|
|
55892
|
+
return [failure(`checker returned unparseable output: ${stdout.slice(0, 500)}`)];
|
|
55445
55893
|
}
|
|
55894
|
+
if (parsed.errors && parsed.errors.length > 0) {
|
|
55895
|
+
return parsed.errors.map(failure);
|
|
55896
|
+
}
|
|
55897
|
+
return [{ status: sqlanvil.QueryEvaluation.QueryEvaluationStatus.SUCCESS }];
|
|
55446
55898
|
}
|
|
55447
|
-
|
|
55448
|
-
|
|
55449
|
-
|
|
55450
|
-
writeStdOut("To change connection settings, edit this file directly.");
|
|
55451
|
-
}
|
|
55452
|
-
function isInteractive({ stream = process.stdout } = {}) {
|
|
55453
|
-
return Boolean(stream && stream.isTTY &&
|
|
55454
|
-
process.env.TERM !== 'dumb' &&
|
|
55455
|
-
!('CI' in process.env));
|
|
55899
|
+
|
|
55900
|
+
function targetKey(target) {
|
|
55901
|
+
return [target.database, target.schema, target.name].filter(Boolean).join(".");
|
|
55456
55902
|
}
|
|
55457
|
-
|
|
55458
|
-
(
|
|
55459
|
-
|
|
55460
|
-
|
|
55461
|
-
|
|
55462
|
-
|
|
55463
|
-
|
|
55464
|
-
|
|
55465
|
-
|
|
55466
|
-
|
|
55467
|
-
}
|
|
55468
|
-
else if (outputType === compiledGraphOutputType.Dot) {
|
|
55469
|
-
writeStdOut(dotRepresentation(graph, interactive));
|
|
55903
|
+
function topoOrder(nodes) {
|
|
55904
|
+
const byKey = new Map(nodes.map(n => [n.key, n]));
|
|
55905
|
+
const indegree = new Map();
|
|
55906
|
+
const dependents = new Map();
|
|
55907
|
+
for (const n of nodes) {
|
|
55908
|
+
const inGraphDeps = Array.from(new Set(n.dependencyKeys.filter(d => byKey.has(d) && d !== n.key)));
|
|
55909
|
+
indegree.set(n.key, inGraphDeps.length);
|
|
55910
|
+
for (const dep of inGraphDeps) {
|
|
55911
|
+
dependents.set(dep, (dependents.get(dep) || []).concat(n.key));
|
|
55912
|
+
}
|
|
55470
55913
|
}
|
|
55471
|
-
|
|
55472
|
-
|
|
55473
|
-
|
|
55474
|
-
|
|
55475
|
-
|
|
55476
|
-
|
|
55477
|
-
|
|
55478
|
-
|
|
55479
|
-
|
|
55480
|
-
|
|
55481
|
-
graph.tables.forEach(compiledTable => {
|
|
55482
|
-
writeStdOut(`${datasetString(compiledTable.target, tableTypeEnumToString(compiledTable.enumType), compiledTable.disabled)}`, 1);
|
|
55483
|
-
});
|
|
55484
|
-
}
|
|
55914
|
+
const ready = nodes
|
|
55915
|
+
.filter(n => (indegree.get(n.key) || 0) === 0)
|
|
55916
|
+
.map(n => n.key)
|
|
55917
|
+
.sort();
|
|
55918
|
+
const ordered = [];
|
|
55919
|
+
const emitted = new Set();
|
|
55920
|
+
while (ready.length) {
|
|
55921
|
+
const key = ready.shift();
|
|
55922
|
+
if (emitted.has(key)) {
|
|
55923
|
+
continue;
|
|
55485
55924
|
}
|
|
55486
|
-
|
|
55487
|
-
|
|
55488
|
-
|
|
55489
|
-
|
|
55490
|
-
|
|
55491
|
-
|
|
55925
|
+
emitted.add(key);
|
|
55926
|
+
ordered.push(byKey.get(key));
|
|
55927
|
+
for (const dep of (dependents.get(key) || []).slice().sort()) {
|
|
55928
|
+
indegree.set(dep, (indegree.get(dep) || 0) - 1);
|
|
55929
|
+
if ((indegree.get(dep) || 0) <= 0 && !emitted.has(dep)) {
|
|
55930
|
+
ready.push(dep);
|
|
55931
|
+
ready.sort();
|
|
55492
55932
|
}
|
|
55493
55933
|
}
|
|
55494
|
-
|
|
55495
|
-
|
|
55496
|
-
|
|
55497
|
-
|
|
55498
|
-
|
|
55499
|
-
|
|
55934
|
+
}
|
|
55935
|
+
if (ordered.length < nodes.length) {
|
|
55936
|
+
for (const n of nodes.slice().sort((a, b) => a.key.localeCompare(b.key))) {
|
|
55937
|
+
if (!emitted.has(n.key)) {
|
|
55938
|
+
emitted.add(n.key);
|
|
55939
|
+
ordered.push(n);
|
|
55500
55940
|
}
|
|
55501
55941
|
}
|
|
55502
55942
|
}
|
|
55943
|
+
return ordered;
|
|
55503
55944
|
}
|
|
55504
|
-
function
|
|
55505
|
-
|
|
55506
|
-
|
|
55507
|
-
|
|
55508
|
-
|
|
55945
|
+
function dependencyBlocked(dependencyKeys, statusByKey) {
|
|
55946
|
+
return dependencyKeys.some(dep => {
|
|
55947
|
+
const status = statusByKey.get(dep);
|
|
55948
|
+
return status !== undefined && status !== "PASS";
|
|
55949
|
+
});
|
|
55950
|
+
}
|
|
55951
|
+
const VALIDATE_SHADOW_PREFIX = "sqlanvil_validate_";
|
|
55952
|
+
function validateShadowSuffix(nowMs) {
|
|
55953
|
+
return `${VALIDATE_SHADOW_PREFIX}${nowMs}`;
|
|
55954
|
+
}
|
|
55955
|
+
function parseShadowTimestamp(schemaName) {
|
|
55956
|
+
const match = schemaName.match(/sqlanvil_validate_(\d+)/);
|
|
55957
|
+
return match ? Number(match[1]) : null;
|
|
55958
|
+
}
|
|
55959
|
+
function shadowSchemasToSweep(schemaNames, nowMs, maxAgeMs) {
|
|
55960
|
+
return schemaNames.filter(name => {
|
|
55961
|
+
const ts = parseShadowTimestamp(name);
|
|
55962
|
+
return ts !== null && nowMs - ts > maxAgeMs;
|
|
55963
|
+
});
|
|
55964
|
+
}
|
|
55965
|
+
|
|
55966
|
+
const SHADOW_MAX_AGE_MS = 60 * 60 * 1000;
|
|
55967
|
+
async function sweepOrphanShadows(deps, nowMs, maxAgeMs = SHADOW_MAX_AGE_MS) {
|
|
55968
|
+
try {
|
|
55969
|
+
const schemas = await deps.listSchemas();
|
|
55970
|
+
for (const schema of shadowSchemasToSweep(schemas, nowMs, maxAgeMs)) {
|
|
55971
|
+
try {
|
|
55972
|
+
await deps.execute(deps.dropSchemaCascadeSql(schema));
|
|
55973
|
+
}
|
|
55974
|
+
catch (e) {
|
|
55975
|
+
}
|
|
55509
55976
|
}
|
|
55510
|
-
const stackLines = ((_b = compileError.stack) === null || _b === void 0 ? void 0 : _b.split("\n")) || [];
|
|
55511
|
-
return stackLines.slice(0, Math.min(3, stackLines.length)).join("\n");
|
|
55512
55977
|
}
|
|
55513
|
-
|
|
55514
|
-
}
|
|
55515
|
-
function printCompiledGraphErrors(graphErrors, quietCompilation) {
|
|
55516
|
-
if (graphErrors.compilationErrors && graphErrors.compilationErrors.length > 0) {
|
|
55517
|
-
printError("Compilation errors:", 1);
|
|
55518
|
-
graphErrors.compilationErrors.forEach(compileError => {
|
|
55519
|
-
writeStdErr(`${calloutOutput(compileError.fileName)}: ${errorOutput(quietCompilation ? (compileError.message + " " + formatStackTraceForQuietCompilation(compileError) || compileError.stack) : (compileError.stack || compileError.message))}`, 1);
|
|
55520
|
-
});
|
|
55978
|
+
catch (e) {
|
|
55521
55979
|
}
|
|
55522
55980
|
}
|
|
55523
|
-
function
|
|
55524
|
-
|
|
55525
|
-
|
|
55526
|
-
|
|
55981
|
+
function tableType(enumType) {
|
|
55982
|
+
switch (enumType) {
|
|
55983
|
+
case sqlanvil.TableType.VIEW:
|
|
55984
|
+
return "view";
|
|
55985
|
+
case sqlanvil.TableType.INCREMENTAL:
|
|
55986
|
+
return "incremental";
|
|
55987
|
+
default:
|
|
55988
|
+
return "table";
|
|
55527
55989
|
}
|
|
55528
55990
|
}
|
|
55529
|
-
function
|
|
55530
|
-
|
|
55531
|
-
|
|
55991
|
+
function depKeys(deps) {
|
|
55992
|
+
return (deps || []).map(targetKey);
|
|
55993
|
+
}
|
|
55994
|
+
async function validate(compiledGraph, deps, options = {}) {
|
|
55995
|
+
const nodes = [];
|
|
55996
|
+
for (const table of compiledGraph.tables || []) {
|
|
55997
|
+
nodes.push({
|
|
55998
|
+
key: targetKey(table.target),
|
|
55999
|
+
dependencyKeys: depKeys(table.dependencyTargets),
|
|
56000
|
+
kind: "table",
|
|
56001
|
+
type: tableType(table.enumType),
|
|
56002
|
+
target: table.target,
|
|
56003
|
+
table,
|
|
56004
|
+
action: table
|
|
56005
|
+
});
|
|
55532
56006
|
}
|
|
55533
|
-
|
|
55534
|
-
|
|
55535
|
-
|
|
55536
|
-
|
|
55537
|
-
|
|
55538
|
-
|
|
55539
|
-
|
|
55540
|
-
|
|
55541
|
-
throw new Error(`Unrecognized action type: ${action.type}`);
|
|
55542
|
-
}
|
|
55543
|
-
actionsByType[action.type].push(action);
|
|
56007
|
+
for (const assertion of compiledGraph.assertions || []) {
|
|
56008
|
+
nodes.push({
|
|
56009
|
+
key: targetKey(assertion.target),
|
|
56010
|
+
dependencyKeys: depKeys(assertion.dependencyTargets),
|
|
56011
|
+
kind: "assertion",
|
|
56012
|
+
type: "assertion",
|
|
56013
|
+
target: assertion.target,
|
|
56014
|
+
action: assertion
|
|
55544
56015
|
});
|
|
55545
|
-
|
|
55546
|
-
|
|
55547
|
-
|
|
55548
|
-
|
|
56016
|
+
}
|
|
56017
|
+
for (const operation of compiledGraph.operations || []) {
|
|
56018
|
+
if (!operation.target) {
|
|
56019
|
+
continue;
|
|
55549
56020
|
}
|
|
55550
|
-
|
|
55551
|
-
|
|
55552
|
-
|
|
55553
|
-
|
|
56021
|
+
nodes.push({
|
|
56022
|
+
key: targetKey(operation.target),
|
|
56023
|
+
dependencyKeys: depKeys(operation.dependencyTargets),
|
|
56024
|
+
kind: "operation",
|
|
56025
|
+
type: "operation",
|
|
56026
|
+
target: operation.target,
|
|
56027
|
+
action: undefined
|
|
56028
|
+
});
|
|
56029
|
+
}
|
|
56030
|
+
for (const imp of compiledGraph.imports || []) {
|
|
56031
|
+
if (!imp.target) {
|
|
56032
|
+
continue;
|
|
55554
56033
|
}
|
|
55555
|
-
|
|
55556
|
-
|
|
55557
|
-
|
|
55558
|
-
|
|
56034
|
+
nodes.push({
|
|
56035
|
+
key: targetKey(imp.target),
|
|
56036
|
+
dependencyKeys: depKeys(imp.dependencyTargets),
|
|
56037
|
+
kind: "import",
|
|
56038
|
+
type: "import",
|
|
56039
|
+
target: imp.target,
|
|
56040
|
+
action: undefined
|
|
56041
|
+
});
|
|
56042
|
+
}
|
|
56043
|
+
for (const script of compiledGraph.scripts || []) {
|
|
56044
|
+
if (!script.target) {
|
|
56045
|
+
continue;
|
|
55559
56046
|
}
|
|
56047
|
+
nodes.push({
|
|
56048
|
+
key: targetKey(script.target),
|
|
56049
|
+
dependencyKeys: depKeys(script.dependencyTargets),
|
|
56050
|
+
kind: "script",
|
|
56051
|
+
type: "script",
|
|
56052
|
+
target: script.target,
|
|
56053
|
+
script,
|
|
56054
|
+
action: undefined
|
|
56055
|
+
});
|
|
55560
56056
|
}
|
|
55561
|
-
|
|
55562
|
-
|
|
55563
|
-
const
|
|
55564
|
-
|
|
55565
|
-
|
|
55566
|
-
|
|
55567
|
-
|
|
55568
|
-
.map(task => {
|
|
55569
|
-
var _a, _b, _c, _d;
|
|
55570
|
-
const bytes = (_d = (_c = (_b = (_a = task.metadata.bigquery) === null || _a === void 0 ? void 0 : _a.totalBytesBilled) === null || _b === void 0 ? void 0 : _b.toNumber) === null || _c === void 0 ? void 0 : _c.call(_b)) !== null && _d !== void 0 ? _d : 0;
|
|
55571
|
-
return formatBytesInHumanReadableFormat(bytes);
|
|
55572
|
-
});
|
|
55573
|
-
const executionSuffix = formatExecutionSuffix(jobIds, bytesBilled);
|
|
55574
|
-
switch (executedAction.status) {
|
|
55575
|
-
case sqlanvil.ActionResult.ExecutionStatus.SUCCESSFUL: {
|
|
55576
|
-
switch (executionAction.type) {
|
|
55577
|
-
case "table": {
|
|
55578
|
-
writeStdOut(`${successOutput(`Table ${dryRun ? "dry run success" : "created"}: `)} ${datasetString(executionAction.target, executionAction.tableType, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55579
|
-
return;
|
|
55580
|
-
}
|
|
55581
|
-
case "assertion": {
|
|
55582
|
-
writeStdOut(`${successOutput(`Assertion ${dryRun ? "dry run success" : "passed"}: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55583
|
-
return;
|
|
55584
|
-
}
|
|
55585
|
-
case "operation": {
|
|
55586
|
-
writeStdOut(`${successOutput(`Operation ${dryRun ? "dry run success" : "completed successfully"}: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55587
|
-
return;
|
|
55588
|
-
}
|
|
55589
|
-
}
|
|
56057
|
+
const ordered = topoOrder(nodes);
|
|
56058
|
+
const shadowSchemas = Array.from(new Set(nodes.filter(n => n.kind === "table").map(n => n.target.schema)));
|
|
56059
|
+
const statusByKey = new Map();
|
|
56060
|
+
const results = [];
|
|
56061
|
+
try {
|
|
56062
|
+
for (const schema of shadowSchemas) {
|
|
56063
|
+
await deps.execute(deps.createSchemaSql(schema));
|
|
55590
56064
|
}
|
|
55591
|
-
|
|
55592
|
-
|
|
55593
|
-
|
|
55594
|
-
|
|
55595
|
-
|
|
55596
|
-
}
|
|
55597
|
-
|
|
55598
|
-
writeStdErr(`${errorOutput("Assertion failed: ")} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55599
|
-
break;
|
|
55600
|
-
}
|
|
55601
|
-
case "operation": {
|
|
55602
|
-
writeStdErr(`${errorOutput("Operation failed: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}${executionSuffix}`);
|
|
55603
|
-
break;
|
|
55604
|
-
}
|
|
56065
|
+
for (const node of ordered) {
|
|
56066
|
+
if (node.kind === "operation" ||
|
|
56067
|
+
node.kind === "import" ||
|
|
56068
|
+
(node.kind === "script" && !deps.checkScript)) {
|
|
56069
|
+
statusByKey.set(node.key, "SKIPPED");
|
|
56070
|
+
results.push({ target: node.target, type: node.type, status: "SKIPPED", errors: [] });
|
|
56071
|
+
continue;
|
|
55605
56072
|
}
|
|
55606
|
-
|
|
55607
|
-
|
|
55608
|
-
|
|
55609
|
-
|
|
55610
|
-
|
|
55611
|
-
|
|
55612
|
-
|
|
55613
|
-
|
|
55614
|
-
|
|
55615
|
-
|
|
55616
|
-
|
|
55617
|
-
|
|
56073
|
+
if (dependencyBlocked(node.dependencyKeys, statusByKey)) {
|
|
56074
|
+
statusByKey.set(node.key, "BLOCKED");
|
|
56075
|
+
results.push({ target: node.target, type: node.type, status: "BLOCKED", errors: [] });
|
|
56076
|
+
continue;
|
|
56077
|
+
}
|
|
56078
|
+
const evaluations = node.kind === "script"
|
|
56079
|
+
? await deps.checkScript(node.script)
|
|
56080
|
+
: await deps.evaluate(node.action);
|
|
56081
|
+
const failed = evaluations.some(e => e.status === sqlanvil.QueryEvaluation.QueryEvaluationStatus.FAILURE);
|
|
56082
|
+
const status = failed ? "FAILURE" : "PASS";
|
|
56083
|
+
statusByKey.set(node.key, status);
|
|
56084
|
+
results.push({ target: node.target, type: node.type, status, errors: evaluations });
|
|
56085
|
+
if (status === "PASS" && node.kind === "table") {
|
|
56086
|
+
try {
|
|
56087
|
+
await deps.execute(deps.validationStubSql(node.table));
|
|
55618
56088
|
}
|
|
55619
|
-
|
|
55620
|
-
writeStdOut(`${warningOutput("Skipping operation execution: ")} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55621
|
-
return;
|
|
56089
|
+
catch (e) {
|
|
55622
56090
|
}
|
|
55623
56091
|
}
|
|
55624
|
-
return;
|
|
55625
56092
|
}
|
|
55626
|
-
|
|
55627
|
-
|
|
55628
|
-
|
|
55629
|
-
|
|
55630
|
-
|
|
55631
|
-
|
|
55632
|
-
case "assertion": {
|
|
55633
|
-
writeStdOut(`${warningOutput(`Assertion execution disabled: `)} ${assertionString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55634
|
-
return;
|
|
56093
|
+
}
|
|
56094
|
+
finally {
|
|
56095
|
+
if (!options.keepShadow) {
|
|
56096
|
+
for (const schema of shadowSchemas.slice().reverse()) {
|
|
56097
|
+
try {
|
|
56098
|
+
await deps.execute(deps.dropSchemaCascadeSql(schema));
|
|
55635
56099
|
}
|
|
55636
|
-
|
|
55637
|
-
writeStdOut(`${warningOutput(`Operation execution disabled: `)} ${operationString(executionAction.target, executionAction.tasks.length === 0)}`);
|
|
55638
|
-
return;
|
|
56100
|
+
catch (e) {
|
|
55639
56101
|
}
|
|
55640
56102
|
}
|
|
55641
|
-
return;
|
|
55642
56103
|
}
|
|
55643
56104
|
}
|
|
56105
|
+
return results;
|
|
55644
56106
|
}
|
|
55645
|
-
|
|
55646
|
-
|
|
55647
|
-
|
|
55648
|
-
|
|
55649
|
-
|
|
55650
|
-
|
|
55651
|
-
|
|
55652
|
-
|
|
55653
|
-
|
|
55654
|
-
|
|
55655
|
-
|
|
55656
|
-
|
|
56107
|
+
|
|
56108
|
+
class SupabaseDbAdapter extends PostgresDbAdapter {
|
|
56109
|
+
static async create(credentials, options) {
|
|
56110
|
+
const sslMode = (credentials.sslMode || "").toLowerCase();
|
|
56111
|
+
const sslEnabled = !(options === null || options === void 0 ? void 0 : options.disableSslForTestsOnly) && sslMode !== "disable";
|
|
56112
|
+
const clientConfig = {
|
|
56113
|
+
host: credentials.host,
|
|
56114
|
+
port: credentials.port,
|
|
56115
|
+
database: credentials.database,
|
|
56116
|
+
user: credentials.user,
|
|
56117
|
+
password: credentials.password,
|
|
56118
|
+
ssl: sslEnabled
|
|
56119
|
+
? {
|
|
56120
|
+
rejectUnauthorized: sslMode === "verify-ca" || sslMode === "verify-full"
|
|
56121
|
+
}
|
|
56122
|
+
: false
|
|
56123
|
+
};
|
|
56124
|
+
const queryExecutor = new PgPoolExecutor(clientConfig, options);
|
|
56125
|
+
try {
|
|
56126
|
+
await queryExecutor.verifyConnection();
|
|
56127
|
+
}
|
|
56128
|
+
catch (e) {
|
|
56129
|
+
await queryExecutor.close().catch(() => undefined);
|
|
56130
|
+
throw new ErrorWithCause(`Could not connect to Supabase Postgres at ${credentials.host}:${credentials.port} ` +
|
|
56131
|
+
`as "${credentials.user}": ${e.message}`, e);
|
|
56132
|
+
}
|
|
56133
|
+
return new SupabaseDbAdapter(queryExecutor);
|
|
55657
56134
|
}
|
|
55658
|
-
|
|
55659
|
-
|
|
55660
|
-
failedFormatResults.forEach(result => writeStdOut(`${result.filename}: ${result.err.message}`, 1));
|
|
56135
|
+
constructor(queryExecutor) {
|
|
56136
|
+
super(queryExecutor);
|
|
55661
56137
|
}
|
|
55662
|
-
|
|
55663
|
-
function datasetString(target, datasetType, disabled) {
|
|
55664
|
-
return `${targetString(target)} [${datasetType}]${disabled ? " [disabled]" : ""}`;
|
|
55665
|
-
}
|
|
55666
|
-
function assertionString(target, disabled) {
|
|
55667
|
-
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55668
|
-
}
|
|
55669
|
-
function operationString(target, disabled) {
|
|
55670
|
-
return `${targetString(target)}${disabled ? " [disabled]" : ""}`;
|
|
55671
|
-
}
|
|
55672
|
-
function plainTargetString(target) {
|
|
55673
|
-
return `${target.schema}.${target.name}`;
|
|
55674
|
-
}
|
|
55675
|
-
function targetString(target) {
|
|
55676
|
-
return calloutOutput(`${target.schema}.${target.name}`);
|
|
55677
|
-
}
|
|
55678
|
-
function dotRepresentation(graph, interactive) {
|
|
55679
|
-
var _a, _b, _c;
|
|
55680
|
-
const nodes = [];
|
|
55681
|
-
const edges = [];
|
|
55682
|
-
const formatTarget = interactive ? targetString : plainTargetString;
|
|
55683
|
-
(_a = graph.tables) === null || _a === void 0 ? void 0 : _a.forEach(table => {
|
|
55684
|
-
var _a;
|
|
55685
|
-
const nodeName = `${formatTarget(table.target)}`;
|
|
55686
|
-
nodes.push(`"${nodeName}" [label="${formatTarget(table.target)} [${tableTypeEnumToString(table.enumType)}]"]`);
|
|
55687
|
-
(_a = table.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55688
|
-
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55689
|
-
});
|
|
55690
|
-
});
|
|
55691
|
-
(_b = graph.assertions) === null || _b === void 0 ? void 0 : _b.forEach(assertion => {
|
|
55692
|
-
var _a;
|
|
55693
|
-
const nodeName = `${formatTarget(assertion.target)}`;
|
|
55694
|
-
nodes.push(`"${nodeName}" [label="${formatTarget(assertion.target)}"]`);
|
|
55695
|
-
(_a = assertion.dependencyTargets) === null || _a === void 0 ? void 0 : _a.forEach(dependencyTarget => {
|
|
55696
|
-
edges.push(`"${formatTarget(dependencyTarget)}" -> "${nodeName}"`);
|
|
55697
|
-
});
|
|
55698
|
-
});
|
|
55699
|
-
(_c = graph.operations) === null || _c === void 0 ? void 0 : _c.forEach(operation => {
|
|
56138
|
+
async verifyServiceRolePermissions() {
|
|
55700
56139
|
var _a;
|
|
55701
|
-
|
|
55702
|
-
|
|
55703
|
-
|
|
55704
|
-
|
|
55705
|
-
}
|
|
55706
|
-
|
|
55707
|
-
|
|
55708
|
-
}
|
|
55709
|
-
|
|
55710
|
-
|
|
55711
|
-
|
|
55712
|
-
|
|
55713
|
-
|
|
55714
|
-
|
|
55715
|
-
|
|
55716
|
-
|
|
56140
|
+
try {
|
|
56141
|
+
const result = await this.execute("select current_user");
|
|
56142
|
+
const currentUser = (_a = result.rows[0]) === null || _a === void 0 ? void 0 : _a.current_user;
|
|
56143
|
+
return !!currentUser;
|
|
56144
|
+
}
|
|
56145
|
+
catch (_b) {
|
|
56146
|
+
return false;
|
|
56147
|
+
}
|
|
56148
|
+
}
|
|
56149
|
+
async validatePublicationState(pubName = "supabase_realtime") {
|
|
56150
|
+
try {
|
|
56151
|
+
const result = await this.execute(`select 1 from pg_publication where pubname = $1`, {
|
|
56152
|
+
params: [pubName]
|
|
56153
|
+
});
|
|
56154
|
+
return result.rows.length > 0;
|
|
56155
|
+
}
|
|
56156
|
+
catch (_a) {
|
|
56157
|
+
return false;
|
|
56158
|
+
}
|
|
56159
|
+
}
|
|
55717
56160
|
}
|
|
55718
56161
|
|
|
55719
56162
|
function getBigQueryCredentials() {
|
|
@@ -56587,6 +57030,26 @@ const icebergOption = option("iceberg", {
|
|
|
56587
57030
|
type: "boolean",
|
|
56588
57031
|
default: false
|
|
56589
57032
|
});
|
|
57033
|
+
const bareOption = option("bare", {
|
|
57034
|
+
describe: "Skip the sample project files — scaffold bare (gitkept) directories only.",
|
|
57035
|
+
type: "boolean",
|
|
57036
|
+
default: false
|
|
57037
|
+
});
|
|
57038
|
+
const interactiveOption = option("interactive", {
|
|
57039
|
+
describe: "Guided Q&A setup: start a fresh project (warehouse, sample project, credentials) or " +
|
|
57040
|
+
"convert an existing Dataform project. Other init arguments are ignored except " +
|
|
57041
|
+
"[project-dir], which seeds the directory prompt.",
|
|
57042
|
+
type: "boolean",
|
|
57043
|
+
default: false
|
|
57044
|
+
});
|
|
57045
|
+
const targetWarehouseOption = option("target-warehouse", {
|
|
57046
|
+
describe: "Where the CONVERTED project runs: supabase/postgres = move off BigQuery (sources become " +
|
|
57047
|
+
"named connections, SQL gets the dialect pass); bigquery = keep the SAME warehouse — a " +
|
|
57048
|
+
"tooling swap with SQL and bigquery:{} blocks untouched.",
|
|
57049
|
+
type: "string",
|
|
57050
|
+
choices: ["supabase", "postgres", "bigquery"],
|
|
57051
|
+
default: "supabase"
|
|
57052
|
+
});
|
|
56590
57053
|
const warehouseOption = option("warehouse", {
|
|
56591
57054
|
describe: "Target warehouse for the new project.",
|
|
56592
57055
|
type: "string",
|
|
@@ -56836,14 +57299,17 @@ function runCli() {
|
|
|
56836
57299
|
{
|
|
56837
57300
|
format: `init [${projectDirOption.name}] [${ProjectConfigOptions.defaultDatabase.name}]` +
|
|
56838
57301
|
` [${ProjectConfigOptions.defaultLocation.name}]`,
|
|
56839
|
-
description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB)."
|
|
57302
|
+
description: "Create a new sqlanvil project (BigQuery, Postgres, Supabase, or MySQL/MariaDB). " +
|
|
57303
|
+
"Use --interactive for a guided setup, including converting a Dataform project.",
|
|
56840
57304
|
positionalOptions: [
|
|
56841
57305
|
projectDirOption,
|
|
56842
57306
|
positionalOption(ProjectConfigOptions.defaultDatabase.name, {
|
|
56843
57307
|
describe: "The default database to use, equivalent to Google Cloud Project ID."
|
|
56844
57308
|
}, (argv) => {
|
|
56845
57309
|
const warehouse = argv[warehouseOption.name] || "bigquery";
|
|
56846
|
-
if (warehouse === "bigquery" &&
|
|
57310
|
+
if (warehouse === "bigquery" &&
|
|
57311
|
+
!argv[interactiveOption.name] &&
|
|
57312
|
+
!argv[ProjectConfigOptions.defaultDatabase.name]) {
|
|
56847
57313
|
throw new Error(`The ${ProjectConfigOptions.defaultDatabase.name} positional argument is ` +
|
|
56848
57314
|
`required for BigQuery projects. Use "sqlanvil help init" for more info.`);
|
|
56849
57315
|
}
|
|
@@ -56853,15 +57319,20 @@ function runCli() {
|
|
|
56853
57319
|
"https://cloud.google.com/bigquery/docs/locations for supported values."
|
|
56854
57320
|
}, (argv) => {
|
|
56855
57321
|
const warehouse = argv[warehouseOption.name] || "bigquery";
|
|
56856
|
-
if (warehouse === "bigquery" &&
|
|
57322
|
+
if (warehouse === "bigquery" &&
|
|
57323
|
+
!argv[interactiveOption.name] &&
|
|
57324
|
+
!argv[ProjectConfigOptions.defaultLocation.name]) {
|
|
56857
57325
|
throw new Error(`The ${ProjectConfigOptions.defaultLocation.name} positional argument is ` +
|
|
56858
57326
|
`required for BigQuery projects. Use "sqlanvil help init" for more info.`);
|
|
56859
57327
|
}
|
|
56860
57328
|
})
|
|
56861
57329
|
],
|
|
56862
|
-
options: [warehouseOption, icebergOption],
|
|
57330
|
+
options: [warehouseOption, icebergOption, bareOption, interactiveOption],
|
|
56863
57331
|
processFn: async (argv) => {
|
|
56864
57332
|
const projectDir = argv[projectDirOption.name];
|
|
57333
|
+
if (argv[interactiveOption.name]) {
|
|
57334
|
+
return runInteractiveInit(projectDir);
|
|
57335
|
+
}
|
|
56865
57336
|
const warehouse = argv[warehouseOption.name] || "bigquery";
|
|
56866
57337
|
const projectConfig = { warehouse };
|
|
56867
57338
|
if (warehouse === "bigquery") {
|
|
@@ -56875,7 +57346,9 @@ function runCli() {
|
|
|
56875
57346
|
}
|
|
56876
57347
|
}
|
|
56877
57348
|
print("Writing project files...\n");
|
|
56878
|
-
const initResult = await init(projectDir, projectConfig
|
|
57349
|
+
const initResult = await init(projectDir, projectConfig, {
|
|
57350
|
+
includeSample: !argv[bareOption.name]
|
|
57351
|
+
});
|
|
56879
57352
|
printInitResult(initResult);
|
|
56880
57353
|
return 0;
|
|
56881
57354
|
}
|
|
@@ -57443,10 +57916,12 @@ function runCli() {
|
|
|
57443
57916
|
},
|
|
57444
57917
|
{
|
|
57445
57918
|
format: "migrate-dataform <source-dir> <out-dir>",
|
|
57446
|
-
description: "Convert a Dataform/BigQuery project
|
|
57447
|
-
"
|
|
57448
|
-
"
|
|
57449
|
-
"gets safe rewrites + inline SQLANVIL-MIGRATE markers
|
|
57919
|
+
description: "Convert a Dataform/BigQuery project to sqlanvil. The source directory is READ-ONLY; " +
|
|
57920
|
+
"the converted project + migration-report.{md,json} land in <out-dir> (must be empty). " +
|
|
57921
|
+
"Default target (supabase/postgres) MOVES the warehouse: sources stay in BigQuery as " +
|
|
57922
|
+
"named connections and target SQL gets safe rewrites + inline SQLANVIL-MIGRATE markers. " +
|
|
57923
|
+
"--target-warehouse bigquery is a TOOLING SWAP: the project keeps running on the same " +
|
|
57924
|
+
"BigQuery warehouse, SQL and bigquery:{} blocks untouched.",
|
|
57450
57925
|
positionalOptions: [
|
|
57451
57926
|
positionalOption("source-dir", {
|
|
57452
57927
|
describe: "The Dataform project to convert (never modified)."
|
|
@@ -57455,19 +57930,14 @@ function runCli() {
|
|
|
57455
57930
|
describe: "Where the converted sqlanvil project is written (created; must be empty)."
|
|
57456
57931
|
})
|
|
57457
57932
|
],
|
|
57458
|
-
options: [],
|
|
57933
|
+
options: [targetWarehouseOption],
|
|
57459
57934
|
processFn: async (argv) => {
|
|
57460
57935
|
const report = await migrateDataform({
|
|
57461
57936
|
srcDir: argv["source-dir"],
|
|
57462
|
-
outDir: argv["out-dir"]
|
|
57937
|
+
outDir: argv["out-dir"],
|
|
57938
|
+
targetWarehouse: argv[targetWarehouseOption.name]
|
|
57463
57939
|
});
|
|
57464
|
-
|
|
57465
|
-
const flagged = targets.filter(f => f.status === "flagged").length;
|
|
57466
|
-
printSuccess(`Converted ${report.inventory.sqlxFiles} .sqlx file(s): ` +
|
|
57467
|
-
`${report.connections.length} source connection(s) over ` +
|
|
57468
|
-
`${report.files.filter(f => f.action === "declaration").length} declaration(s); ` +
|
|
57469
|
-
`${targets.length} target file(s), ${flagged} flagged for dialect review.`);
|
|
57470
|
-
print(`Report: ${path__namespace.join(argv["out-dir"], "migration-report.md")}`);
|
|
57940
|
+
printMigrationSummary(report, argv["out-dir"]);
|
|
57471
57941
|
print(`Next: sqlanvil compile ${argv["out-dir"]}`);
|
|
57472
57942
|
return 0;
|
|
57473
57943
|
}
|