@danielarndt0/cnpj-db-loader 2.4.0-beta.2 → 2.4.0-beta.3

package/dist/index.js

@@ -8411,6 +8411,18 @@ var STAGING_TABLE_BY_DATASET3 = {
   partners: "staging_partners",
   simples_options: "staging_simples_options"
 };
+var STEP_ORDER = [
+  "setup",
+  "load-domains",
+  "load-companies",
+  "load-establishments",
+  "load-partners",
+  "load-simples",
+  "materialize",
+  "materialize-secondary-cnaes",
+  "indexes",
+  "analyze"
+];
 function quoteSqlLiteral(value) {
   return `'${value.replace(/'/g, "''")}'`;
 }
@@ -8428,6 +8440,9 @@ function receitaCopyCommand(tableName, columns, filePath) {
   const normalizedFilePath = normalizePathForPsql(filePath);
   return `\\copy ${tableName} (${columns.join(", ")}) from ${quoteSqlLiteral(normalizedFilePath)} with (format csv, header false, delimiter ';', quote '"', escape '"')`;
 }
+function echo(message) {
+  return `\\echo ${quoteSqlLiteral(message)}`;
+}
 function datasetColumns(dataset) {
   return DATASET_LAYOUTS[dataset].fields.map((field) => field.columnName);
 }
@@ -8454,7 +8469,7 @@ function partnerDedupeExpression(alias) {
 function materializeCompaniesSql() {
   const columns = companiesLayout.fields.map((field) => field.columnName);
   return [
-    "\\echo 'Materializing companies...'",
+    echo("[materialize] Materializing companies..."),
     "with source as (",
     "  select",
     `    ${columns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8468,7 +8483,8 @@ function materializeCompaniesSql() {
     `select ${columns.join(", ")}`,
     "from deduped",
     "on conflict (cnpj_root) do update set",
-    `  ${updateAssignments(columns, ["cnpj_root"])};`
+    `  ${updateAssignments(columns, ["cnpj_root"])};`,
+    echo("[materialize] Companies materialization completed.")
   ].join("\n");
 }
 function materializeEstablishmentsSql() {
@@ -8477,7 +8493,7 @@ function materializeEstablishmentsSql() {
   );
   const insertColumns = [...baseColumns, "cnpj_full"];
   return [
-    "\\echo 'Materializing establishments and secondary CNAEs...'",
+    echo("[materialize] Materializing establishments..."),
     "with source as (",
     "  select",
     `    ${baseColumns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8487,14 +8503,29 @@ function materializeEstablishmentsSql() {
     "),",
     "deduped as (",
     "  select * from source where dedupe_rank = 1",
+    ")",
+    `insert into establishments (${insertColumns.join(", ")})`,
+    `select ${insertColumns.join(", ")}`,
+    "from deduped",
+    "on conflict (cnpj_full) do update set",
+    `  ${updateAssignments(insertColumns, ["cnpj_root", "cnpj_order", "cnpj_check_digits", "cnpj_full"])};`,
+    echo("[materialize] Establishments materialization completed.")
+  ].join("\n");
+}
+function materializeSecondaryCnaesSql() {
+  return [
+    echo(
+      "[materialize-secondary-cnaes] Materializing establishment secondary CNAEs..."
+    ),
+    "with source as (",
+    "  select",
+    "    staging.cnpj_root || staging.cnpj_order || staging.cnpj_check_digits as cnpj_full,",
+    "    staging.secondary_cnaes_raw,",
+    "    row_number() over (partition by staging.cnpj_root || staging.cnpj_order || staging.cnpj_check_digits order by staging.staging_id desc) as dedupe_rank",
+    "  from staging_establishments staging",
     "),",
-    "upserted as (",
-    `  insert into establishments (${insertColumns.join(", ")})`,
-    `  select ${insertColumns.join(", ")}`,
-    "  from deduped",
-    "  on conflict (cnpj_full) do update set",
-    `    ${updateAssignments(insertColumns, ["cnpj_root", "cnpj_order", "cnpj_check_digits", "cnpj_full"])}`,
-    "  returning cnpj_full",
+    "deduped as (",
+    "  select * from source where dedupe_rank = 1",
     "),",
     "deleted_secondary_cnaes as (",
     "  delete from establishment_secondary_cnaes target",
@@ -8515,14 +8546,17 @@ function materializeEstablishmentsSql() {
     "insert into establishment_secondary_cnaes (cnpj_full, cnae_code)",
     "select cnpj_full, cnae_code",
     "from secondary_cnaes_source",
-    "on conflict (cnpj_full, cnae_code) do nothing;"
+    "on conflict (cnpj_full, cnae_code) do nothing;",
+    echo(
+      "[materialize-secondary-cnaes] Secondary CNAEs materialization completed."
+    )
   ].join("\n");
 }
 function materializePartnersSql() {
   const baseColumns = partnersLayout.fields.map((field) => field.columnName);
   const insertColumns = [...baseColumns, "partner_dedupe_key"];
   return [
-    "\\echo 'Materializing partners...'",
+    echo("[materialize] Materializing partners..."),
     "with source as (",
     "  select",
     `    ${baseColumns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8542,13 +8576,14 @@ function materializePartnersSql() {
     `select ${insertColumns.join(", ")}`,
     "from deduped",
     "on conflict (partner_dedupe_key) do update set",
-    `  ${updateAssignments(insertColumns, ["partner_dedupe_key"])};`
+    `  ${updateAssignments(insertColumns, ["partner_dedupe_key"])};`,
+    echo("[materialize] Partners materialization completed.")
   ].join("\n");
 }
 function materializeSimplesSql() {
   const columns = simplesLayout.fields.map((field) => field.columnName);
   return [
-    "\\echo 'Materializing simples options...'",
+    echo("[materialize] Materializing simples options..."),
     "with source as (",
     "  select",
     `    ${columns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8562,7 +8597,8 @@ function materializeSimplesSql() {
     `select ${columns.join(", ")}`,
     "from deduped",
     "on conflict (cnpj_root) do update set",
-    `  ${updateAssignments(columns, ["cnpj_root"])};`
+    `  ${updateAssignments(columns, ["cnpj_root"])};`,
+    echo("[materialize] Simples options materialization completed.")
   ].join("\n");
 }
 function copyDomainSql(dataset, files) {
@@ -8572,12 +8608,20 @@ function copyDomainSql(dataset, files) {
   const columns = datasetColumns(dataset);
   const tempTable = `tmp_hybrid_${dataset}`;
   const lines = [
-    `\\echo 'Loading ${dataset} lookup data...'`,
+    echo(`[load-domains] Loading ${dataset} lookup data...`),
     `drop table if exists ${tempTable};`,
     `create temporary table ${tempTable} (code text, description text);`
   ];
-  for (const file of files) {
-    lines.push(csvCopyCommand(tempTable, columns, file.absolutePath));
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[load-domains] Loading ${dataset} file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      csvCopyCommand(tempTable, columns, file.absolutePath),
+      echo(
+        `[load-domains] Loaded ${dataset} file ${index + 1} of ${files.length}.`
+      )
+    );
   }
   lines.push(
     `insert into ${dataset} (${columns.join(", ")})`,
@@ -8598,12 +8642,17 @@ function copyStagingSql(dataset, files) {
     return [];
   }
   const columns = datasetColumns(dataset);
-  return [
-    `\\echo 'Loading ${dataset} staging data...'`,
-    ...files.map(
-      (file) => csvCopyCommand(tableName, columns, file.absolutePath)
-    )
-  ];
+  const lines = [echo(`[load-${dataset}] Loading ${dataset} staging data...`)];
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[load-${dataset}] Loading file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      csvCopyCommand(tableName, columns, file.absolutePath),
+      echo(`[load-${dataset}] Loaded file ${index + 1} of ${files.length}.`)
+    );
+  }
+  return lines;
 }
 function csvFilesByDataset(files) {
   const grouped = {};
@@ -8629,7 +8678,9 @@ function rawTableName(dataset) {
 function createRawTempTableSql(dataset) {
   const columns = DATASET_LAYOUTS[dataset].fields.map((field) => `  ${quoteIdentifier(field.columnName)} text`).join(",\n");
   return [
+    "set client_min_messages to warning;",
     `drop table if exists ${rawTableName(dataset)};`,
+    "reset client_min_messages;",
     `create temporary table ${rawTableName(dataset)} (`,
     columns,
     ");"
@@ -8711,11 +8762,21 @@ function rawDomainSql(dataset, files) {
   const columns = layout.fields.map((field) => field.columnName);
   const tableName = rawTableName(dataset);
   const lines = [
-    `\\echo 'Loading ${dataset} lookup data directly from sanitized Receita files...'`,
+    echo(
+      `[load-domains] Loading ${dataset} lookup data directly from sanitized Receita files...`
+    ),
     createRawTempTableSql(dataset)
   ];
-  for (const file of files) {
-    lines.push(receitaCopyCommand(tableName, columns, file.absolutePath));
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[load-domains] Loading ${dataset} file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      receitaCopyCommand(tableName, columns, file.absolutePath),
+      echo(
+        `[load-domains] Loaded ${dataset} file ${index + 1} of ${files.length}.`
+      )
+    );
   }
   lines.push(
     `insert into ${dataset} (${columns.join(", ")})`,
@@ -8725,7 +8786,8 @@ function rawDomainSql(dataset, files) {
     `from ${tableName}`,
     "where nullif(btrim(code), '') is not null",
     "order by code",
-    "on conflict (code) do update set description = excluded.description;"
+    "on conflict (code) do update set description = excluded.description;",
+    echo(`[load-domains] ${dataset} lookup data completed.`)
   );
   return lines;
 }
@@ -8744,70 +8806,363 @@ function rawStagingSql(dataset, files) {
   const expressions = layout.fields.map(
     (field) => `  ${fieldExpression(dataset, field, alias)} as ${field.columnName}`
   );
+  const stepName = loadStepName(dataset);
   const lines = [
-    `\\echo 'Loading ${dataset} staging data directly from sanitized Receita files...'`,
+    echo(
+      `[${stepName}] Loading ${dataset} staging data directly from sanitized Receita files...`
+    ),
+    `truncate table ${targetTable} restart identity;`,
     createRawTempTableSql(dataset)
   ];
-  for (const file of files) {
-    lines.push(receitaCopyCommand(tableName, columns, file.absolutePath));
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[${stepName}] Loading file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      receitaCopyCommand(tableName, columns, file.absolutePath),
+      echo(`[${stepName}] Loaded file ${index + 1} of ${files.length}.`)
+    );
   }
   lines.push(
+    echo(
+      `[${stepName}] Transforming ${dataset} raw rows into ${targetTable}...`
+    ),
     `insert into ${targetTable} (${columns.join(", ")})`,
     "select",
     expressions.join(",\n"),
-    `from ${tableName} ${alias};`
+    `from ${tableName} ${alias};`,
+    echo(`[${stepName}] ${dataset} staging load completed.`)
   );
   return lines;
 }
-function generatePostgresDirectImportScript(input) {
-  const grouped = csvFilesByDataset(input.files);
-  const lines = [
-    "-- CNPJ DB Loader hybrid PostgreSQL import script",
-    "-- Generated from PostgreSQL-ready CSV files exported by cnpj-db-loader postgres export-csv.",
-    "-- Execute with psql, for example:",
-    '--   psql "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
-    "",
+function loadStepName(dataset) {
+  switch (dataset) {
+    case "companies":
+      return "load-companies";
+    case "establishments":
+      return "load-establishments";
+    case "partners":
+      return "load-partners";
+    case "simples_options":
+      return "load-simples";
+    default:
+      return `load-${dataset}`;
+  }
+}
+function scriptHeader(title, sourceEncoding) {
+  return [
+    `-- ${title}`,
+    "-- Generated by cnpj-db-loader postgres generate-script.",
     "\\set ON_ERROR_STOP on",
-    "\\echo 'Starting CNPJ DB Loader hybrid PostgreSQL import...'",
-    "",
-    "begin;",
-    "",
-    "-- Keep the final schema and seed data managed by sql/schema.sql.",
-    "-- This script only resets staging tables and then upserts final data.",
-    "truncate table staging_companies restart identity;",
-    "truncate table staging_establishments restart identity;",
-    "truncate table staging_partners restart identity;",
-    "truncate table staging_simples_options restart identity;",
+    ...sourceEncoding ? [
+      echo(
+        `Using source file encoding ${sourceEncoding} for psql copy operations...`
+      ),
+      `set client_encoding to ${quoteSqlLiteral(sourceEncoding)};`
+    ] : [],
     ""
   ];
-  for (const dataset of DOMAIN_DATASETS) {
-    lines.push(...copyDomainSql(dataset, grouped[dataset] ?? []), "");
+}
+function wrapTransaction(lines, mode, shouldWrap) {
+  if (!shouldWrap || mode !== "phase") {
+    return [...lines];
   }
-  for (const dataset of STAGING_DATASETS) {
-    lines.push(...copyStagingSql(dataset, grouped[dataset] ?? []), "");
+  return ["begin;", "", ...lines, "", "commit;"];
+}
+function buildStepScript(title, body, input, wrapInPhaseTransaction) {
+  return [
+    ...scriptHeader(title, input.sourceEncoding),
+    ...wrapTransaction(body, input.transactionMode, wrapInPhaseTransaction),
+    ""
+  ].join("\n");
+}
+function includeSet(input) {
+  const selected = new Set(input.include);
+  if (input.skipIndexes) {
+    selected.delete("indexes");
   }
-  lines.push(...materializationAndAnalyzeSql());
-  return lines.join("\n");
+  if (input.skipAnalyze) {
+    selected.delete("analyze");
+  }
+  return selected;
+}
+function hasAnyFinalMaterialization(selected) {
+  return selected.has("companies") || selected.has("establishments") || selected.has("partners") || selected.has("simples");
+}
+function materializeSql(selected) {
+  const lines = [echo("[materialize] Starting final table materialization...")];
+  if (selected.has("companies")) {
+    lines.push(materializeCompaniesSql(), "");
+  }
+  if (selected.has("establishments")) {
+    lines.push(materializeEstablishmentsSql(), "");
+  }
+  if (selected.has("partners")) {
+    lines.push(materializePartnersSql(), "");
+  }
+  if (selected.has("simples")) {
+    lines.push(materializeSimplesSql(), "");
+  }
+  lines.push(echo("[materialize] Final table materialization completed."));
+  return lines;
+}
+function indexesSql() {
+  return [
+    echo(
+      "[indexes] No additional index operations are generated in this beta."
+    ),
+    "-- Indexes are expected to be managed by the schema generated by cnpj-db-loader schema generate.",
+    "-- A future fast-rebuild mode may generate DROP/CREATE INDEX operations here."
+  ];
+}
+function analyzeSql(selected) {
+  const tables = /* @__PURE__ */ new Set();
+  if (selected.has("companies")) {
+    tables.add("companies");
+  }
+  if (selected.has("establishments")) {
+    tables.add("establishments");
+  }
+  if (selected.has("secondary-cnaes")) {
+    tables.add("establishment_secondary_cnaes");
+  }
+  if (selected.has("partners")) {
+    tables.add("partners");
+  }
+  if (selected.has("simples")) {
+    tables.add("simples_options");
+  }
+  if (selected.has("domains")) {
+    for (const dataset of DOMAIN_DATASETS) {
+      tables.add(dataset);
+    }
+  }
+  return [
+    echo("[analyze] Refreshing planner statistics..."),
+    ...[...tables].map((table) => `analyze ${table};`),
+    echo("[analyze] Planner statistics refreshed.")
+  ];
 }
-function generatePostgresSanitizedDirectImportScript(input) {
+function step(name, file, dependsOn, included) {
+  return { name, file, dependsOn, included };
+}
+function generatePostgresDirectScriptFiles(input) {
   const grouped = directFilesByDataset(input.files);
-  const lines = [
-    "-- CNPJ DB Loader direct PostgreSQL import script",
+  const selected = includeSet(input);
+  if (!DOMAIN_DATASETS.some((dataset) => (grouped[dataset] ?? []).length > 0)) {
+    selected.delete("domains");
+  }
+  if ((grouped.companies ?? []).length === 0) {
+    selected.delete("companies");
+  }
+  if ((grouped.establishments ?? []).length === 0) {
+    selected.delete("establishments");
+    selected.delete("secondary-cnaes");
+  }
+  if ((grouped.partners ?? []).length === 0) {
+    selected.delete("partners");
+  }
+  if ((grouped.simples_options ?? []).length === 0) {
+    selected.delete("simples");
+  }
+  const scripts = {};
+  const steps = [];
+  const setupIncluded = true;
+  steps.push(step("setup", "setup.sql", [], setupIncluded));
+  scripts["setup.sql"] = [
+    ...scriptHeader(
+      "CNPJ DB Loader PostgreSQL direct import setup",
+      input.sourceEncoding
+    ),
+    echo("[setup] Preparing PostgreSQL direct import session..."),
+    "-- The database schema must be applied before running these scripts.",
+    "-- This setup script configures the psql session used by the generated orchestrator.",
+    echo("[setup] Setup completed."),
+    ""
+  ].join("\n");
+  const domainsIncluded = selected.has("domains") && DOMAIN_DATASETS.some((dataset) => (grouped[dataset] ?? []).length > 0);
+  steps.push(
+    step("load-domains", "load-domains.sql", ["setup"], domainsIncluded)
+  );
+  if (domainsIncluded) {
+    const lines = [echo("[load-domains] Starting domain tables load...")];
+    for (const dataset of DOMAIN_DATASETS) {
+      lines.push(...rawDomainSql(dataset, grouped[dataset] ?? []), "");
+    }
+    lines.push(echo("[load-domains] Domain tables load completed."));
+    scripts["load-domains.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import domains step",
+      lines,
+      input,
+      true
+    );
+  }
+  const datasetSteps = [
+    {
+      dataset: "companies",
+      name: "load-companies",
+      file: "load-companies.sql",
+      include: "companies"
+    },
+    {
+      dataset: "establishments",
+      name: "load-establishments",
+      file: "load-establishments.sql",
+      include: "establishments"
+    },
+    {
+      dataset: "partners",
+      name: "load-partners",
+      file: "load-partners.sql",
+      include: "partners"
+    },
+    {
+      dataset: "simples_options",
+      name: "load-simples",
+      file: "load-simples.sql",
+      include: "simples"
+    }
+  ];
+  for (const item of datasetSteps) {
+    const files = grouped[item.dataset] ?? [];
+    const included = selected.has(item.include) && files.length > 0;
+    steps.push(step(item.name, item.file, ["setup"], included));
+    if (included) {
+      scripts[item.file] = buildStepScript(
+        `CNPJ DB Loader PostgreSQL direct import ${item.name} step`,
+        rawStagingSql(item.dataset, files),
+        input,
+        true
+      );
+    }
+  }
+  const materializeIncluded = hasAnyFinalMaterialization(selected);
+  steps.push(
+    step(
+      "materialize",
+      "materialize.sql",
+      datasetSteps.filter((item) => selected.has(item.include)).map((item) => item.name),
+      materializeIncluded
+    )
+  );
+  if (materializeIncluded) {
+    scripts["materialize.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import materialization step",
+      materializeSql(selected),
+      input,
+      true
+    );
+  }
+  const secondaryIncluded = selected.has("secondary-cnaes") && selected.has("establishments");
+  steps.push(
+    step(
+      "materialize-secondary-cnaes",
+      "materialize-secondary-cnaes.sql",
+      ["load-establishments"],
+      secondaryIncluded
+    )
+  );
+  if (secondaryIncluded) {
+    scripts["materialize-secondary-cnaes.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import secondary CNAEs step",
+      [materializeSecondaryCnaesSql()],
+      input,
+      true
+    );
+  }
+  const indexesIncluded = selected.has("indexes");
+  steps.push(
+    step(
+      "indexes",
+      "indexes.sql",
+      materializeIncluded ? ["materialize"] : ["setup"],
+      indexesIncluded
+    )
+  );
+  if (indexesIncluded) {
+    scripts["indexes.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import indexes step",
+      indexesSql(),
+      input,
+      true
+    );
+  }
+  const analyzeIncluded = selected.has("analyze");
+  const analyzeDependencies = [
+    ...domainsIncluded ? ["load-domains"] : [],
+    ...materializeIncluded ? ["materialize"] : [],
+    ...secondaryIncluded ? ["materialize-secondary-cnaes"] : []
+  ];
+  steps.push(
+    step(
+      "analyze",
+      "analyze.sql",
+      analyzeDependencies.length > 0 ? analyzeDependencies : ["setup"],
+      analyzeIncluded
+    )
+  );
+  if (analyzeIncluded) {
+    scripts["analyze.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import analyze step",
+      analyzeSql(selected),
+      input,
+      true
+    );
+  }
+  const orchestratorLines = [
+    "-- CNPJ DB Loader direct PostgreSQL import orchestrator",
     "-- Generated from sanitized Receita files by cnpj-db-loader postgres generate-script.",
-    "-- This path avoids rewriting the dataset into a second CSV tree.",
     "-- Execute with psql, for example:",
-    '--   psql "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
+    '--   psql -d "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
     "",
     "\\set ON_ERROR_STOP on",
-    `\\echo 'Using source file encoding ${input.sourceEncoding} for psql copy operations...'`,
+    echo(
+      `Using source file encoding ${input.sourceEncoding} for psql copy operations...`
+    ),
     `set client_encoding to ${quoteSqlLiteral(input.sourceEncoding)};`,
-    "\\echo 'Starting CNPJ DB Loader direct PostgreSQL import from sanitized files...'",
+    echo(
+      `Starting CNPJ DB Loader direct PostgreSQL import using transaction mode ${input.transactionMode}...`
+    ),
+    "",
+    ...input.transactionMode === "single" ? ["begin;", ""] : []
+  ];
+  for (const name of STEP_ORDER) {
+    const currentStep = steps.find((item) => item.name === name);
+    if (!currentStep?.included) {
+      continue;
+    }
+    orchestratorLines.push(
+      echo(
+        `[orchestrator] Running ${currentStep.name} (${currentStep.file})...`
+      ),
+      `\\ir ${currentStep.file}`,
+      echo(`[orchestrator] Completed ${currentStep.name}.`),
+      ""
+    );
+  }
+  orchestratorLines.push(
+    ...input.transactionMode === "single" ? ["commit;", ""] : [],
+    echo("CNPJ DB Loader hybrid PostgreSQL import completed."),
+    ""
+  );
+  scripts["import-postgres-direct.sql"] = orchestratorLines.join("\n");
+  return { scripts, steps };
+}
+function generatePostgresDirectImportScript(input) {
+  const grouped = csvFilesByDataset(input.files);
+  const lines = [
+    "-- CNPJ DB Loader hybrid PostgreSQL import script",
+    "-- Generated from PostgreSQL-ready CSV files exported by cnpj-db-loader postgres export-csv.",
+    "-- Execute with psql, for example:",
+    '--   psql -d "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
+    "",
+    "\\set ON_ERROR_STOP on",
+    echo("Starting CNPJ DB Loader hybrid PostgreSQL import..."),
     "",
     "begin;",
     "",
     "-- Keep the final schema and seed data managed by sql/schema.sql.",
-    "-- This script copies sanitized Receita files into temporary raw tables,",
-    "-- transforms values inside PostgreSQL, resets staging tables and upserts final data.",
+    "-- This script only resets staging tables and then upserts final data.",
     "truncate table staging_companies restart identity;",
     "truncate table staging_establishments restart identity;",
     "truncate table staging_partners restart identity;",
@@ -8815,10 +9170,10 @@ function generatePostgresSanitizedDirectImportScript(input) {
     ""
   ];
   for (const dataset of DOMAIN_DATASETS) {
-    lines.push(...rawDomainSql(dataset, grouped[dataset] ?? []), "");
+    lines.push(...copyDomainSql(dataset, grouped[dataset] ?? []), "");
   }
   for (const dataset of STAGING_DATASETS) {
-    lines.push(...rawStagingSql(dataset, grouped[dataset] ?? []), "");
+    lines.push(...copyStagingSql(dataset, grouped[dataset] ?? []), "");
   }
   lines.push(...materializationAndAnalyzeSql());
   return lines.join("\n");
@@ -8829,11 +9184,13 @@ function materializationAndAnalyzeSql() {
     "",
     materializeEstablishmentsSql(),
     "",
+    materializeSecondaryCnaesSql(),
+    "",
     materializePartnersSql(),
     "",
     materializeSimplesSql(),
     "",
-    "\\echo 'Refreshing planner statistics...'",
+    echo("Refreshing planner statistics..."),
     "analyze companies;",
     "analyze establishments;",
     "analyze establishment_secondary_cnaes;",
@@ -8848,7 +9205,7 @@ function materializationAndAnalyzeSql() {
     "",
     "commit;",
     "",
-    "\\echo 'CNPJ DB Loader hybrid PostgreSQL import completed.'",
+    echo("CNPJ DB Loader hybrid PostgreSQL import completed."),
     ""
   ];
 }
@@ -9055,6 +9412,29 @@ async function exportPostgresCsvDataset(inputPath, options = {}) {
 import { mkdir as mkdir9, stat as stat7, writeFile as writeFile6 } from "fs/promises";
 import path17 from "path";
 var DEFAULT_SOURCE_ENCODING = "UTF8";
+var DEFAULT_TRANSACTION_MODE = "single";
+var ALL_INCLUDE_TARGETS = [
+  "domains",
+  "companies",
+  "establishments",
+  "partners",
+  "simples",
+  "secondary-cnaes",
+  "indexes",
+  "analyze"
+];
+var INCLUDE_TARGETS_BY_DATASET = {
+  companies: "companies",
+  establishments: "establishments",
+  partners: "partners",
+  simples_options: "simples",
+  countries: "domains",
+  cities: "domains",
+  partner_qualifications: "domains",
+  legal_natures: "domains",
+  reasons: "domains",
+  cnaes: "domains"
+};
 function defaultPostgresDirectOutputPath(inputPath) {
   const baseName = path17.basename(inputPath);
   if (baseName.toLowerCase() === "sanitized") {
@@ -9063,7 +9443,7 @@ function defaultPostgresDirectOutputPath(inputPath) {
   return path17.join(path17.dirname(inputPath), `${baseName}-postgres-direct`);
 }
 function inferNextStep5(scriptPath) {
-  return `psql "postgres://postgres:postgres@localhost:5432/cnpj" -f ${scriptPath.replace(/\\/g, "/")}`;
+  return `psql -d "postgres://postgres:postgres@localhost:5432/cnpj" -f ${scriptPath.replace(/\\/g, "/")}`;
 }
 function normalizeSourceEncoding(value) {
   const encoding = (value ?? DEFAULT_SOURCE_ENCODING).trim();
@@ -9074,6 +9454,41 @@ function normalizeSourceEncoding(value) {
   }
   return encoding.toUpperCase();
 }
+function normalizeTransactionMode(value) {
+  const mode = value ?? DEFAULT_TRANSACTION_MODE;
+  if (!["single", "phase", "none"].includes(mode)) {
+    throw new ValidationError(
+      `Invalid transaction mode: ${String(value)}. Use single, phase or none.`
+    );
+  }
+  return mode;
+}
+function isIncludeTarget(value) {
+  return ALL_INCLUDE_TARGETS.includes(value);
+}
+function normalizeIncludeTargets(include, dataset) {
+  if (include && include.length > 0) {
+    const unique = [...new Set(include)];
+    const invalid = unique.filter((item) => !isIncludeTarget(item));
+    if (invalid.length > 0) {
+      throw new ValidationError(
+        `Invalid include target(s): ${invalid.join(", ")}. Use ${ALL_INCLUDE_TARGETS.join(", ")}.`
+      );
+    }
+    return unique;
+  }
+  if (dataset) {
+    const target = INCLUDE_TARGETS_BY_DATASET[dataset];
+    if (!target) {
+      return [];
+    }
+    if (target === "establishments") {
+      return ["establishments", "secondary-cnaes", "analyze"];
+    }
+    return [target, "analyze"];
+  }
+  return [...ALL_INCLUDE_TARGETS];
+}
 async function generatePostgresDirectScript(inputPath, options = {}) {
   if (options.dataset && !isImportDatasetType(options.dataset)) {
     throw new ValidationError(`Unsupported dataset type: ${options.dataset}.`);
@@ -9089,6 +9504,10 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
     options.outputPath ?? defaultPostgresDirectOutputPath(validatedPath)
   );
   const sourceEncoding = normalizeSourceEncoding(options.sourceEncoding);
+  const transactionMode = normalizeTransactionMode(options.transactionMode);
+  const include = normalizeIncludeTargets(options.include, options.dataset);
+  const skipIndexes = options.skipIndexes ?? false;
+  const skipAnalyze = options.skipAnalyze ?? false;
   const inspected = await inspectFiles(validatedPath);
   const recognizedFiles = inspected.entries.filter((entry) => entry.entryKind === "file").flatMap((entry) => {
     if (!isImportDatasetType(entry.inferredType)) {
@@ -9116,7 +9535,11 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
     outputPath,
     totalFiles: recognizedFiles.length,
     datasets,
-    sourceEncoding
+    sourceEncoding,
+    transactionMode,
+    include,
+    skipIndexes,
+    skipAnalyze
   });
   await mkdir9(outputPath, { recursive: true });
   const sourceFiles = [];
@@ -9152,11 +9575,21 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
   }
   const scriptName = options.scriptName ?? "import-postgres-direct.sql";
   const scriptPath = path17.join(outputPath, scriptName);
-  const script = generatePostgresSanitizedDirectImportScript({
+  const generated = generatePostgresDirectScriptFiles({
     files: sourceFiles,
-    sourceEncoding
+    sourceEncoding,
+    transactionMode,
+    include,
+    skipIndexes,
+    skipAnalyze
   });
-  await writeFile6(scriptPath, script, "utf8");
+  const scriptFiles = [];
+  for (const [fileName, script] of Object.entries(generated.scripts)) {
+    const outputFileName = fileName === "import-postgres-direct.sql" ? scriptName : fileName;
+    const outputFilePath = path17.join(outputPath, outputFileName);
+    await writeFile6(outputFilePath, script, "utf8");
+    scriptFiles.push(outputFilePath);
+  }
   const manifestPath = path17.join(outputPath, "manifest.json");
   const summaryDatasets = [...summariesByDataset.values()].sort(
     (left, right) => IMPORT_ORDER.indexOf(left.dataset) - IMPORT_ORDER.indexOf(right.dataset)
@@ -9168,13 +9601,19 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
   const manifest = {
     generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
     mode: "direct-sanitized-script",
+    transactionMode,
+    include,
+    skipIndexes,
+    skipAnalyze,
     inputPath: path17.resolve(inputPath),
     validatedPath,
     outputPath,
     scriptPath,
+    scriptFiles,
     sourceEncoding,
     totalFiles: sourceFiles.length,
     totalBytes,
+    steps: generated.steps,
     datasets: summaryDatasets
   };
   await writeFile6(
@@ -9197,15 +9636,19 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
     scriptPath,
     manifestPath,
     sourceEncoding,
+    transactionMode,
     totalFiles: sourceFiles.length,
     totalBytes,
     datasets: summaryDatasets,
+    scriptFiles,
+    steps: generated.steps,
     warnings: [
       ...validation.ok ? [] : validation.errors,
       "This script imports sanitized Receita files directly with psql \\copy. It avoids rewriting the full dataset into a second CSV tree.",
-      "The generated script expects the database schema generated by cnpj-db-loader to be applied before execution.",
+      "The generated scripts expect the database schema generated by cnpj-db-loader to be applied before execution.",
       "The direct PostgreSQL script now defaults to UTF8 because the sanitize command writes clean UTF-8 files.",
-      "Use --source-encoding WIN1252 or LATIN1 only when generating scripts for legacy sanitized files produced by older loader versions."
+      "Use --source-encoding WIN1252 or LATIN1 only when generating scripts for legacy sanitized files produced by older loader versions.",
+      "The generated import is now modular. Use import-postgres-direct.sql as the orchestrator or run individual phase scripts manually."
     ],
     nextStep: inferNextStep5(scriptPath)
   };