@danielarndt0/cnpj-db-loader 2.4.0-beta.1 → 2.4.0-beta.3

package/dist/index.js

@@ -7853,81 +7853,264 @@ function isRecognizedSanitizeEntry(entry) {
   return entry.entryKind === "file" && entry.inferredType !== "zip-archive" && entry.inferredType !== "unknown";
 }
 
+// src/services/sanitize/encoding.ts
+import { StringDecoder } from "string_decoder";
+var WINDOWS_1252_C1_MAP = {
+  128: "\u20AC",
+  130: "\u201A",
+  131: "\u0192",
+  132: "\u201E",
+  133: "\u2026",
+  134: "\u2020",
+  135: "\u2021",
+  136: "\u02C6",
+  137: "\u2030",
+  138: "\u0160",
+  139: "\u2039",
+  140: "\u0152",
+  142: "\u017D",
+  145: "\u2018",
+  146: "\u2019",
+  147: "\u201C",
+  148: "\u201D",
+  149: "\u2022",
+  150: "\u2013",
+  151: "\u2014",
+  152: "\u02DC",
+  153: "\u2122",
+  154: "\u0161",
+  155: "\u203A",
+  156: "\u0153",
+  158: "\u017E",
+  159: "\u0178"
+};
+function normalizeSanitizeSourceEncoding(value) {
+  const normalized = (value ?? "WIN1252").trim().toUpperCase().replace(/_/g, "-");
+  switch (normalized) {
+    case "WIN1252":
+    case "WINDOWS-1252":
+    case "CP1252":
+      return "WIN1252";
+    case "LATIN1":
+    case "LATIN-1":
+    case "ISO-8859-1":
+    case "ISO8859-1":
+      return "LATIN1";
+    case "UTF8":
+    case "UTF-8":
+      return "UTF8";
+    default:
+      throw new ValidationError(
+        `Unsupported sanitize source encoding: ${value}. Supported values: WIN1252, LATIN1, UTF8.`
+      );
+  }
+}
+function isAllowedControlCodePoint(codePoint) {
+  return codePoint === 9 || codePoint === 10 || codePoint === 13;
+}
+function isProblematicControlCodePoint(codePoint) {
+  if (isAllowedControlCodePoint(codePoint)) {
+    return false;
+  }
+  return codePoint >= 0 && codePoint <= 31 || codePoint === 127 || codePoint >= 128 && codePoint <= 159 || codePoint === 65279;
+}
+function sanitizeDecodedText(text) {
+  const output = [];
+  let invalidBytesRemoved = 0;
+  let controlCharsRemoved = 0;
+  for (const char of text) {
+    const codePoint = char.codePointAt(0);
+    if (codePoint === 65533) {
+      invalidBytesRemoved += 1;
+      continue;
+    }
+    if (isProblematicControlCodePoint(codePoint)) {
+      controlCharsRemoved += 1;
+      continue;
+    }
+    output.push(char);
+  }
+  return {
+    text: output.join(""),
+    invalidBytesRemoved,
+    controlCharsRemoved
+  };
+}
+var SanitizeEncodingNormalizer = class {
+  constructor(sourceEncoding) {
+    this.sourceEncoding = sourceEncoding;
+    this.utf8Decoder = sourceEncoding === "UTF8" ? new StringDecoder("utf8") : void 0;
+  }
+  sourceEncoding;
+  utf8Decoder;
+  normalizeChunk(chunk) {
+    if (this.sourceEncoding === "UTF8") {
+      const decoded = this.utf8Decoder.write(chunk);
+      const sanitized = sanitizeDecodedText(decoded);
+      const nulBytesRemoved = [...decoded].filter(
+        (char) => char === "\0"
+      ).length;
+      return {
+        ...sanitized,
+        nulBytesRemoved
+      };
+    }
+    return this.normalizeSingleByteChunk(chunk);
+  }
+  flush() {
+    if (!this.utf8Decoder) {
+      return {
+        text: "",
+        nulBytesRemoved: 0,
+        invalidBytesRemoved: 0,
+        controlCharsRemoved: 0
+      };
+    }
+    const decoded = this.utf8Decoder.end();
+    const sanitized = sanitizeDecodedText(decoded);
+    const nulBytesRemoved = [...decoded].filter((char) => char === "\0").length;
+    return {
+      ...sanitized,
+      nulBytesRemoved
+    };
+  }
+  normalizeSingleByteChunk(chunk) {
+    const output = [];
+    let nulBytesRemoved = 0;
+    let invalidBytesRemoved = 0;
+    let controlCharsRemoved = 0;
+    for (const byte of chunk) {
+      if (byte === 0) {
+        nulBytesRemoved += 1;
+        continue;
+      }
+      if (byte < 32 || byte === 127) {
+        if (isAllowedControlCodePoint(byte)) {
+          output.push(String.fromCharCode(byte));
+        } else {
+          controlCharsRemoved += 1;
+        }
+        continue;
+      }
+      if (byte >= 128 && byte <= 159) {
+        if (this.sourceEncoding === "WIN1252") {
+          const mapped = WINDOWS_1252_C1_MAP[byte];
+          if (mapped === void 0) {
+            invalidBytesRemoved += 1;
+          } else {
+            output.push(mapped);
+          }
+        } else {
+          controlCharsRemoved += 1;
+        }
+        continue;
+      }
+      output.push(String.fromCharCode(byte));
+    }
+    return {
+      text: output.join(""),
+      nulBytesRemoved,
+      invalidBytesRemoved,
+      controlCharsRemoved
+    };
+  }
+};
+
 // src/services/sanitize/runner.ts
 import { createReadStream as createReadStream2, createWriteStream as createWriteStream2 } from "fs";
 import { mkdir as mkdir7 } from "fs/promises";
 import path13 from "path";
-function stripNulBytes(chunk) {
-  let removed = 0;
-  for (let index = 0; index < chunk.length; index += 1) {
-    if (chunk[index] === 0) {
-      removed += 1;
-    }
+async function writeUtf8(output, value) {
+  if (value.length === 0) {
+    return;
   }
-  if (removed === 0) {
-    return { buffer: chunk, removed: 0 };
+  if (!output.write(value, "utf8")) {
+    await new Promise((resolve, reject) => {
+      output.once("drain", resolve);
+      output.once("error", reject);
+    });
   }
-  const sanitized = Buffer.allocUnsafe(chunk.length - removed);
-  let outputIndex = 0;
-  for (let index = 0; index < chunk.length; index += 1) {
-    const value = chunk[index];
-    if (value !== 0) {
-      sanitized[outputIndex] = value;
-      outputIndex += 1;
+}
+function countNewlines(value) {
+  let count = 0;
+  for (let index = 0; index < value.length; index += 1) {
+    if (value[index] === "\n") {
+      count += 1;
     }
   }
-  return { buffer: sanitized, removed };
+  return count;
 }
-async function sanitizeDatasetFile(plan, onChunk) {
+async function sanitizeDatasetFile(plan, onChunk, options = {}) {
   await mkdir7(path13.dirname(plan.outputPath), { recursive: true });
+  const sourceEncoding = normalizeSanitizeSourceEncoding(
+    options.sourceEncoding
+  );
+  const normalizer = new SanitizeEncodingNormalizer(sourceEncoding);
   const input = createReadStream2(plan.absolutePath);
-  const output = createWriteStream2(plan.outputPath);
+  const output = createWriteStream2(plan.outputPath, { encoding: "utf8" });
   let totalBytesRead = 0;
   let totalBytesWritten = 0;
   let nulBytesRemoved = 0;
+  let invalidBytesRemoved = 0;
+  let controlCharsRemoved = 0;
   let lineCount = 0;
-  let sawAnyByte = false;
-  let lastByteWasNewline = false;
+  let sawAnyCharacter = false;
+  let lastCharacterWasNewline = false;
+  const processText = async (text) => {
+    if (text.length === 0) {
+      return;
+    }
+    sawAnyCharacter = true;
+    lineCount += countNewlines(text);
+    lastCharacterWasNewline = text.endsWith("\n");
+    totalBytesWritten += Buffer.byteLength(text, "utf8");
+    await writeUtf8(output, text);
+  };
   try {
     for await (const chunk of input) {
       const chunkBuffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
       totalBytesRead += chunkBuffer.length;
-      const { buffer, removed } = stripNulBytes(chunkBuffer);
-      nulBytesRemoved += removed;
-      sawAnyByte = sawAnyByte || buffer.length > 0;
-      for (let index = 0; index < buffer.length; index += 1) {
-        if (buffer[index] === 10) {
-          lineCount += 1;
-        }
-      }
-      if (buffer.length > 0) {
-        lastByteWasNewline = buffer[buffer.length - 1] === 10;
-      }
-      totalBytesWritten += buffer.length;
-      output.write(buffer);
+      const normalized = normalizer.normalizeChunk(chunkBuffer);
+      nulBytesRemoved += normalized.nulBytesRemoved;
+      invalidBytesRemoved += normalized.invalidBytesRemoved;
+      controlCharsRemoved += normalized.controlCharsRemoved;
+      await processText(normalized.text);
       onChunk?.({
         bytesProcessed: chunkBuffer.length,
         fileBytesProcessed: totalBytesRead,
         currentFileSize: plan.fileSize,
         processedRows: lineCount,
-        nulBytesRemoved
+        nulBytesRemoved,
+        invalidBytesRemoved,
+        controlCharsRemoved
       });
     }
-    if (sawAnyByte && !lastByteWasNewline) {
+    const flushed = normalizer.flush();
+    nulBytesRemoved += flushed.nulBytesRemoved;
+    invalidBytesRemoved += flushed.invalidBytesRemoved;
+    controlCharsRemoved += flushed.controlCharsRemoved;
+    await processText(flushed.text);
+    if (sawAnyCharacter && !lastCharacterWasNewline) {
       lineCount += 1;
     }
   } finally {
     input.close();
     output.end();
-    await new Promise((resolve) => output.on("finish", () => resolve()));
+    await new Promise((resolve, reject) => {
+      output.on("finish", () => resolve());
+      output.on("error", (error) => reject(error));
+    });
   }
   return {
     plan,
     totalBytesRead,
     totalBytesWritten,
+    sourceEncoding,
     nulBytesRemoved,
+    invalidBytesRemoved,
+    controlCharsRemoved,
     lineCount,
-    changed: nulBytesRemoved > 0 || totalBytesRead !== totalBytesWritten
+    changed: nulBytesRemoved > 0 || invalidBytesRemoved > 0 || controlCharsRemoved > 0 || totalBytesRead !== totalBytesWritten
   };
 }
 
@@ -7990,40 +8173,54 @@ async function sanitizeInputDirectory(inputPath, options = {}) {
       "No recognized validated dataset files were found for sanitization."
     );
   }
+  const sourceEncoding = normalizeSanitizeSourceEncoding(
+    options.sourceEncoding
+  );
   options.onProgress?.({
     kind: "start",
     validatedPath,
     outputPath,
     totalFiles: plan.totalFiles,
     totalBytes: plan.totalBytes,
-    datasets: plan.datasets
+    datasets: plan.datasets,
+    sourceEncoding
   });
   let processedFiles = 0;
   let processedRows = 0;
   let processedBytes = 0;
   let nulBytesRemoved = 0;
+  let invalidBytesRemoved = 0;
+  let controlCharsRemoved = 0;
   let changedFiles = 0;
   const fileSummaries = [];
   for (const [index, filePlan] of plan.files.entries()) {
-    const fileResult = await sanitizeDatasetFile(filePlan, (chunk) => {
-      options.onProgress?.({
-        kind: "progress",
-        currentFileDisplayPath: filePlan.displayPath,
-        fileIndex: index + 1,
-        totalFiles: plan.totalFiles,
-        bytesProcessed: processedBytes + chunk.fileBytesProcessed,
-        totalBytes: plan.totalBytes,
-        fileBytesProcessed: chunk.fileBytesProcessed,
-        currentFileSize: chunk.currentFileSize,
-        processedRows: processedRows + chunk.processedRows,
-        nulBytesRemoved: nulBytesRemoved + chunk.nulBytesRemoved,
-        changedFiles
-      });
-    });
+    const fileResult = await sanitizeDatasetFile(
+      filePlan,
+      (chunk) => {
+        options.onProgress?.({
+          kind: "progress",
+          currentFileDisplayPath: filePlan.displayPath,
+          fileIndex: index + 1,
+          totalFiles: plan.totalFiles,
+          bytesProcessed: processedBytes + chunk.fileBytesProcessed,
+          totalBytes: plan.totalBytes,
+          fileBytesProcessed: chunk.fileBytesProcessed,
+          currentFileSize: chunk.currentFileSize,
+          processedRows: processedRows + chunk.processedRows,
+          nulBytesRemoved: nulBytesRemoved + chunk.nulBytesRemoved,
+          invalidBytesRemoved: invalidBytesRemoved + chunk.invalidBytesRemoved,
+          controlCharsRemoved: controlCharsRemoved + chunk.controlCharsRemoved,
+          changedFiles
+        });
+      },
+      { sourceEncoding }
+    );
     processedFiles += 1;
     processedRows += fileResult.lineCount;
     processedBytes += fileResult.totalBytesRead;
     nulBytesRemoved += fileResult.nulBytesRemoved;
+    invalidBytesRemoved += fileResult.invalidBytesRemoved;
+    controlCharsRemoved += fileResult.controlCharsRemoved;
     changedFiles += fileResult.changed ? 1 : 0;
     fileSummaries.push({
       dataset: filePlan.dataset,
@@ -8031,7 +8228,9 @@ async function sanitizeInputDirectory(inputPath, options = {}) {
       outputPath: filePlan.outputPath,
       lineCount: fileResult.lineCount,
       changed: fileResult.changed,
-      nulBytesRemoved: fileResult.nulBytesRemoved
+      nulBytesRemoved: fileResult.nulBytesRemoved,
+      invalidBytesRemoved: fileResult.invalidBytesRemoved,
+      controlCharsRemoved: fileResult.controlCharsRemoved
     });
   }
   options.onProgress?.({
@@ -8039,6 +8238,8 @@ async function sanitizeInputDirectory(inputPath, options = {}) {
     totalFiles: plan.totalFiles,
     processedRows,
     nulBytesRemoved,
+    invalidBytesRemoved,
+    controlCharsRemoved,
     changedFiles,
     totalBytes: plan.totalBytes
   });
@@ -8050,13 +8251,17 @@ async function sanitizeInputDirectory(inputPath, options = {}) {
     totalBytes: plan.totalBytes,
     processedFiles,
     processedRows,
+    sourceEncoding,
     nulBytesRemoved,
+    invalidBytesRemoved,
+    controlCharsRemoved,
     changedFiles,
     unchangedFiles: plan.totalFiles - changedFiles,
     datasets: plan.datasets,
     files: fileSummaries,
     warnings: [
-      "Sanitization prepares a clean dataset tree for import by removing known low-level byte issues such as NUL bytes before PostgreSQL loading begins.",
+      "Sanitization now writes UTF-8 output and removes invalid bytes plus problematic control characters before PostgreSQL loading begins.",
+      "The PostgreSQL direct import path can use --source-encoding UTF8 when reading files generated by this sanitization command.",
       "The import command still keeps quarantine and row-level recovery for unexpected issues, but sanitizing first reduces the amount of slow fallback work during import."
     ],
     nextStep: inferNextStep3(outputPath)
@@ -8206,6 +8411,18 @@ var STAGING_TABLE_BY_DATASET3 = {
   partners: "staging_partners",
   simples_options: "staging_simples_options"
 };
+var STEP_ORDER = [
+  "setup",
+  "load-domains",
+  "load-companies",
+  "load-establishments",
+  "load-partners",
+  "load-simples",
+  "materialize",
+  "materialize-secondary-cnaes",
+  "indexes",
+  "analyze"
+];
 function quoteSqlLiteral(value) {
   return `'${value.replace(/'/g, "''")}'`;
 }
@@ -8223,6 +8440,9 @@ function receitaCopyCommand(tableName, columns, filePath) {
   const normalizedFilePath = normalizePathForPsql(filePath);
   return `\\copy ${tableName} (${columns.join(", ")}) from ${quoteSqlLiteral(normalizedFilePath)} with (format csv, header false, delimiter ';', quote '"', escape '"')`;
 }
+function echo(message) {
+  return `\\echo ${quoteSqlLiteral(message)}`;
+}
 function datasetColumns(dataset) {
   return DATASET_LAYOUTS[dataset].fields.map((field) => field.columnName);
 }
@@ -8249,7 +8469,7 @@ function partnerDedupeExpression(alias) {
 function materializeCompaniesSql() {
   const columns = companiesLayout.fields.map((field) => field.columnName);
   return [
-    "\\echo 'Materializing companies...'",
+    echo("[materialize] Materializing companies..."),
     "with source as (",
     "  select",
     `    ${columns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8263,7 +8483,8 @@ function materializeCompaniesSql() {
     `select ${columns.join(", ")}`,
     "from deduped",
     "on conflict (cnpj_root) do update set",
-    `  ${updateAssignments(columns, ["cnpj_root"])};`
+    `  ${updateAssignments(columns, ["cnpj_root"])};`,
+    echo("[materialize] Companies materialization completed.")
   ].join("\n");
 }
 function materializeEstablishmentsSql() {
@@ -8272,7 +8493,7 @@ function materializeEstablishmentsSql() {
   );
   const insertColumns = [...baseColumns, "cnpj_full"];
   return [
-    "\\echo 'Materializing establishments and secondary CNAEs...'",
+    echo("[materialize] Materializing establishments..."),
     "with source as (",
     "  select",
     `    ${baseColumns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8282,14 +8503,29 @@ function materializeEstablishmentsSql() {
     "),",
     "deduped as (",
     "  select * from source where dedupe_rank = 1",
+    ")",
+    `insert into establishments (${insertColumns.join(", ")})`,
+    `select ${insertColumns.join(", ")}`,
+    "from deduped",
+    "on conflict (cnpj_full) do update set",
+    `  ${updateAssignments(insertColumns, ["cnpj_root", "cnpj_order", "cnpj_check_digits", "cnpj_full"])};`,
+    echo("[materialize] Establishments materialization completed.")
+  ].join("\n");
+}
+function materializeSecondaryCnaesSql() {
+  return [
+    echo(
+      "[materialize-secondary-cnaes] Materializing establishment secondary CNAEs..."
+    ),
+    "with source as (",
+    "  select",
+    "    staging.cnpj_root || staging.cnpj_order || staging.cnpj_check_digits as cnpj_full,",
+    "    staging.secondary_cnaes_raw,",
+    "    row_number() over (partition by staging.cnpj_root || staging.cnpj_order || staging.cnpj_check_digits order by staging.staging_id desc) as dedupe_rank",
+    "  from staging_establishments staging",
     "),",
-    "upserted as (",
-    `  insert into establishments (${insertColumns.join(", ")})`,
-    `  select ${insertColumns.join(", ")}`,
-    "  from deduped",
-    "  on conflict (cnpj_full) do update set",
-    `    ${updateAssignments(insertColumns, ["cnpj_root", "cnpj_order", "cnpj_check_digits", "cnpj_full"])}`,
-    "  returning cnpj_full",
+    "deduped as (",
+    "  select * from source where dedupe_rank = 1",
     "),",
     "deleted_secondary_cnaes as (",
     "  delete from establishment_secondary_cnaes target",
@@ -8310,14 +8546,17 @@ function materializeEstablishmentsSql() {
     "insert into establishment_secondary_cnaes (cnpj_full, cnae_code)",
     "select cnpj_full, cnae_code",
     "from secondary_cnaes_source",
-    "on conflict (cnpj_full, cnae_code) do nothing;"
+    "on conflict (cnpj_full, cnae_code) do nothing;",
+    echo(
+      "[materialize-secondary-cnaes] Secondary CNAEs materialization completed."
+    )
   ].join("\n");
 }
 function materializePartnersSql() {
   const baseColumns = partnersLayout.fields.map((field) => field.columnName);
   const insertColumns = [...baseColumns, "partner_dedupe_key"];
   return [
-    "\\echo 'Materializing partners...'",
+    echo("[materialize] Materializing partners..."),
     "with source as (",
     "  select",
     `    ${baseColumns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8337,13 +8576,14 @@ function materializePartnersSql() {
     `select ${insertColumns.join(", ")}`,
     "from deduped",
     "on conflict (partner_dedupe_key) do update set",
-    `  ${updateAssignments(insertColumns, ["partner_dedupe_key"])};`
+    `  ${updateAssignments(insertColumns, ["partner_dedupe_key"])};`,
+    echo("[materialize] Partners materialization completed.")
   ].join("\n");
 }
 function materializeSimplesSql() {
   const columns = simplesLayout.fields.map((field) => field.columnName);
   return [
-    "\\echo 'Materializing simples options...'",
+    echo("[materialize] Materializing simples options..."),
     "with source as (",
     "  select",
     `    ${columns.map((column) => `source.${column}`).join(",\n    ")},`,
@@ -8357,7 +8597,8 @@ function materializeSimplesSql() {
     `select ${columns.join(", ")}`,
     "from deduped",
     "on conflict (cnpj_root) do update set",
-    `  ${updateAssignments(columns, ["cnpj_root"])};`
+    `  ${updateAssignments(columns, ["cnpj_root"])};`,
+    echo("[materialize] Simples options materialization completed.")
   ].join("\n");
 }
 function copyDomainSql(dataset, files) {
@@ -8367,12 +8608,20 @@ function copyDomainSql(dataset, files) {
   const columns = datasetColumns(dataset);
   const tempTable = `tmp_hybrid_${dataset}`;
   const lines = [
-    `\\echo 'Loading ${dataset} lookup data...'`,
+    echo(`[load-domains] Loading ${dataset} lookup data...`),
     `drop table if exists ${tempTable};`,
     `create temporary table ${tempTable} (code text, description text);`
   ];
-  for (const file of files) {
-    lines.push(csvCopyCommand(tempTable, columns, file.absolutePath));
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[load-domains] Loading ${dataset} file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      csvCopyCommand(tempTable, columns, file.absolutePath),
+      echo(
+        `[load-domains] Loaded ${dataset} file ${index + 1} of ${files.length}.`
+      )
+    );
   }
   lines.push(
     `insert into ${dataset} (${columns.join(", ")})`,
@@ -8393,12 +8642,17 @@ function copyStagingSql(dataset, files) {
     return [];
   }
   const columns = datasetColumns(dataset);
-  return [
-    `\\echo 'Loading ${dataset} staging data...'`,
-    ...files.map(
-      (file) => csvCopyCommand(tableName, columns, file.absolutePath)
-    )
-  ];
+  const lines = [echo(`[load-${dataset}] Loading ${dataset} staging data...`)];
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[load-${dataset}] Loading file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      csvCopyCommand(tableName, columns, file.absolutePath),
+      echo(`[load-${dataset}] Loaded file ${index + 1} of ${files.length}.`)
+    );
+  }
+  return lines;
 }
 function csvFilesByDataset(files) {
   const grouped = {};
@@ -8424,7 +8678,9 @@ function rawTableName(dataset) {
 function createRawTempTableSql(dataset) {
   const columns = DATASET_LAYOUTS[dataset].fields.map((field) => `  ${quoteIdentifier(field.columnName)} text`).join(",\n");
   return [
+    "set client_min_messages to warning;",
     `drop table if exists ${rawTableName(dataset)};`,
+    "reset client_min_messages;",
     `create temporary table ${rawTableName(dataset)} (`,
     columns,
     ");"
@@ -8506,11 +8762,21 @@ function rawDomainSql(dataset, files) {
   const columns = layout.fields.map((field) => field.columnName);
   const tableName = rawTableName(dataset);
   const lines = [
-    `\\echo 'Loading ${dataset} lookup data directly from sanitized Receita files...'`,
+    echo(
+      `[load-domains] Loading ${dataset} lookup data directly from sanitized Receita files...`
+    ),
     createRawTempTableSql(dataset)
   ];
-  for (const file of files) {
-    lines.push(receitaCopyCommand(tableName, columns, file.absolutePath));
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[load-domains] Loading ${dataset} file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      receitaCopyCommand(tableName, columns, file.absolutePath),
+      echo(
+        `[load-domains] Loaded ${dataset} file ${index + 1} of ${files.length}.`
+      )
+    );
   }
   lines.push(
     `insert into ${dataset} (${columns.join(", ")})`,
@@ -8520,7 +8786,8 @@ function rawDomainSql(dataset, files) {
     `from ${tableName}`,
     "where nullif(btrim(code), '') is not null",
     "order by code",
-    "on conflict (code) do update set description = excluded.description;"
+    "on conflict (code) do update set description = excluded.description;",
+    echo(`[load-domains] ${dataset} lookup data completed.`)
   );
   return lines;
 }
@@ -8539,70 +8806,363 @@ function rawStagingSql(dataset, files) {
   const expressions = layout.fields.map(
     (field) => `  ${fieldExpression(dataset, field, alias)} as ${field.columnName}`
   );
+  const stepName = loadStepName(dataset);
   const lines = [
-    `\\echo 'Loading ${dataset} staging data directly from sanitized Receita files...'`,
+    echo(
+      `[${stepName}] Loading ${dataset} staging data directly from sanitized Receita files...`
+    ),
+    `truncate table ${targetTable} restart identity;`,
     createRawTempTableSql(dataset)
   ];
-  for (const file of files) {
-    lines.push(receitaCopyCommand(tableName, columns, file.absolutePath));
+  for (const [index, file] of files.entries()) {
+    lines.push(
+      echo(
+        `[${stepName}] Loading file ${index + 1} of ${files.length}: ${file.relativePath}`
+      ),
+      receitaCopyCommand(tableName, columns, file.absolutePath),
+      echo(`[${stepName}] Loaded file ${index + 1} of ${files.length}.`)
+    );
   }
   lines.push(
+    echo(
+      `[${stepName}] Transforming ${dataset} raw rows into ${targetTable}...`
+    ),
     `insert into ${targetTable} (${columns.join(", ")})`,
     "select",
     expressions.join(",\n"),
-    `from ${tableName} ${alias};`
+    `from ${tableName} ${alias};`,
+    echo(`[${stepName}] ${dataset} staging load completed.`)
   );
   return lines;
 }
-function generatePostgresDirectImportScript(input) {
-  const grouped = csvFilesByDataset(input.files);
-  const lines = [
-    "-- CNPJ DB Loader hybrid PostgreSQL import script",
-    "-- Generated from PostgreSQL-ready CSV files exported by cnpj-db-loader postgres export-csv.",
-    "-- Execute with psql, for example:",
-    '--   psql "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
-    "",
+function loadStepName(dataset) {
+  switch (dataset) {
+    case "companies":
+      return "load-companies";
+    case "establishments":
+      return "load-establishments";
+    case "partners":
+      return "load-partners";
+    case "simples_options":
+      return "load-simples";
+    default:
+      return `load-${dataset}`;
+  }
+}
+function scriptHeader(title, sourceEncoding) {
+  return [
+    `-- ${title}`,
+    "-- Generated by cnpj-db-loader postgres generate-script.",
     "\\set ON_ERROR_STOP on",
-    "\\echo 'Starting CNPJ DB Loader hybrid PostgreSQL import...'",
-    "",
-    "begin;",
-    "",
-    "-- Keep the final schema and seed data managed by sql/schema.sql.",
-    "-- This script only resets staging tables and then upserts final data.",
-    "truncate table staging_companies restart identity;",
-    "truncate table staging_establishments restart identity;",
-    "truncate table staging_partners restart identity;",
-    "truncate table staging_simples_options restart identity;",
+    ...sourceEncoding ? [
+      echo(
+        `Using source file encoding ${sourceEncoding} for psql copy operations...`
+      ),
+      `set client_encoding to ${quoteSqlLiteral(sourceEncoding)};`
+    ] : [],
     ""
   ];
-  for (const dataset of DOMAIN_DATASETS) {
-    lines.push(...copyDomainSql(dataset, grouped[dataset] ?? []), "");
+}
+function wrapTransaction(lines, mode, shouldWrap) {
+  if (!shouldWrap || mode !== "phase") {
+    return [...lines];
   }
-  for (const dataset of STAGING_DATASETS) {
-    lines.push(...copyStagingSql(dataset, grouped[dataset] ?? []), "");
+  return ["begin;", "", ...lines, "", "commit;"];
+}
+function buildStepScript(title, body, input, wrapInPhaseTransaction) {
+  return [
+    ...scriptHeader(title, input.sourceEncoding),
+    ...wrapTransaction(body, input.transactionMode, wrapInPhaseTransaction),
+    ""
+  ].join("\n");
+}
+function includeSet(input) {
+  const selected = new Set(input.include);
+  if (input.skipIndexes) {
+    selected.delete("indexes");
   }
-  lines.push(...materializationAndAnalyzeSql());
-  return lines.join("\n");
+  if (input.skipAnalyze) {
+    selected.delete("analyze");
+  }
+  return selected;
+}
+function hasAnyFinalMaterialization(selected) {
+  return selected.has("companies") || selected.has("establishments") || selected.has("partners") || selected.has("simples");
+}
+function materializeSql(selected) {
+  const lines = [echo("[materialize] Starting final table materialization...")];
+  if (selected.has("companies")) {
+    lines.push(materializeCompaniesSql(), "");
+  }
+  if (selected.has("establishments")) {
+    lines.push(materializeEstablishmentsSql(), "");
+  }
+  if (selected.has("partners")) {
+    lines.push(materializePartnersSql(), "");
+  }
+  if (selected.has("simples")) {
+    lines.push(materializeSimplesSql(), "");
+  }
+  lines.push(echo("[materialize] Final table materialization completed."));
+  return lines;
+}
+function indexesSql() {
+  return [
+    echo(
+      "[indexes] No additional index operations are generated in this beta."
+    ),
+    "-- Indexes are expected to be managed by the schema generated by cnpj-db-loader schema generate.",
+    "-- A future fast-rebuild mode may generate DROP/CREATE INDEX operations here."
+  ];
+}
+function analyzeSql(selected) {
+  const tables = /* @__PURE__ */ new Set();
+  if (selected.has("companies")) {
+    tables.add("companies");
+  }
+  if (selected.has("establishments")) {
+    tables.add("establishments");
+  }
+  if (selected.has("secondary-cnaes")) {
+    tables.add("establishment_secondary_cnaes");
+  }
+  if (selected.has("partners")) {
+    tables.add("partners");
+  }
+  if (selected.has("simples")) {
+    tables.add("simples_options");
+  }
+  if (selected.has("domains")) {
+    for (const dataset of DOMAIN_DATASETS) {
+      tables.add(dataset);
+    }
+  }
+  return [
+    echo("[analyze] Refreshing planner statistics..."),
+    ...[...tables].map((table) => `analyze ${table};`),
+    echo("[analyze] Planner statistics refreshed.")
+  ];
+}
+function step(name, file, dependsOn, included) {
+  return { name, file, dependsOn, included };
 }
-function generatePostgresSanitizedDirectImportScript(input) {
+function generatePostgresDirectScriptFiles(input) {
   const grouped = directFilesByDataset(input.files);
-  const lines = [
-    "-- CNPJ DB Loader direct PostgreSQL import script",
+  const selected = includeSet(input);
+  if (!DOMAIN_DATASETS.some((dataset) => (grouped[dataset] ?? []).length > 0)) {
+    selected.delete("domains");
+  }
+  if ((grouped.companies ?? []).length === 0) {
+    selected.delete("companies");
+  }
+  if ((grouped.establishments ?? []).length === 0) {
+    selected.delete("establishments");
+    selected.delete("secondary-cnaes");
+  }
+  if ((grouped.partners ?? []).length === 0) {
+    selected.delete("partners");
+  }
+  if ((grouped.simples_options ?? []).length === 0) {
+    selected.delete("simples");
+  }
+  const scripts = {};
+  const steps = [];
+  const setupIncluded = true;
+  steps.push(step("setup", "setup.sql", [], setupIncluded));
+  scripts["setup.sql"] = [
+    ...scriptHeader(
+      "CNPJ DB Loader PostgreSQL direct import setup",
+      input.sourceEncoding
+    ),
+    echo("[setup] Preparing PostgreSQL direct import session..."),
+    "-- The database schema must be applied before running these scripts.",
+    "-- This setup script configures the psql session used by the generated orchestrator.",
+    echo("[setup] Setup completed."),
+    ""
+  ].join("\n");
+  const domainsIncluded = selected.has("domains") && DOMAIN_DATASETS.some((dataset) => (grouped[dataset] ?? []).length > 0);
+  steps.push(
+    step("load-domains", "load-domains.sql", ["setup"], domainsIncluded)
+  );
+  if (domainsIncluded) {
+    const lines = [echo("[load-domains] Starting domain tables load...")];
+    for (const dataset of DOMAIN_DATASETS) {
+      lines.push(...rawDomainSql(dataset, grouped[dataset] ?? []), "");
+    }
+    lines.push(echo("[load-domains] Domain tables load completed."));
+    scripts["load-domains.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import domains step",
+      lines,
+      input,
+      true
+    );
+  }
+  const datasetSteps = [
+    {
+      dataset: "companies",
+      name: "load-companies",
+      file: "load-companies.sql",
+      include: "companies"
+    },
+    {
+      dataset: "establishments",
+      name: "load-establishments",
+      file: "load-establishments.sql",
+      include: "establishments"
+    },
+    {
+      dataset: "partners",
+      name: "load-partners",
+      file: "load-partners.sql",
+      include: "partners"
+    },
+    {
+      dataset: "simples_options",
+      name: "load-simples",
+      file: "load-simples.sql",
+      include: "simples"
+    }
+  ];
+  for (const item of datasetSteps) {
+    const files = grouped[item.dataset] ?? [];
+    const included = selected.has(item.include) && files.length > 0;
+    steps.push(step(item.name, item.file, ["setup"], included));
+    if (included) {
+      scripts[item.file] = buildStepScript(
+        `CNPJ DB Loader PostgreSQL direct import ${item.name} step`,
+        rawStagingSql(item.dataset, files),
+        input,
+        true
+      );
+    }
+  }
+  const materializeIncluded = hasAnyFinalMaterialization(selected);
+  steps.push(
+    step(
+      "materialize",
+      "materialize.sql",
+      datasetSteps.filter((item) => selected.has(item.include)).map((item) => item.name),
+      materializeIncluded
+    )
+  );
+  if (materializeIncluded) {
+    scripts["materialize.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import materialization step",
+      materializeSql(selected),
+      input,
+      true
+    );
+  }
+  const secondaryIncluded = selected.has("secondary-cnaes") && selected.has("establishments");
+  steps.push(
+    step(
+      "materialize-secondary-cnaes",
+      "materialize-secondary-cnaes.sql",
+      ["load-establishments"],
+      secondaryIncluded
+    )
+  );
+  if (secondaryIncluded) {
+    scripts["materialize-secondary-cnaes.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import secondary CNAEs step",
+      [materializeSecondaryCnaesSql()],
+      input,
+      true
+    );
+  }
+  const indexesIncluded = selected.has("indexes");
+  steps.push(
+    step(
+      "indexes",
+      "indexes.sql",
+      materializeIncluded ? ["materialize"] : ["setup"],
+      indexesIncluded
+    )
+  );
+  if (indexesIncluded) {
+    scripts["indexes.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import indexes step",
+      indexesSql(),
+      input,
+      true
+    );
+  }
+  const analyzeIncluded = selected.has("analyze");
+  const analyzeDependencies = [
+    ...domainsIncluded ? ["load-domains"] : [],
+    ...materializeIncluded ? ["materialize"] : [],
+    ...secondaryIncluded ? ["materialize-secondary-cnaes"] : []
+  ];
+  steps.push(
+    step(
+      "analyze",
+      "analyze.sql",
+      analyzeDependencies.length > 0 ? analyzeDependencies : ["setup"],
+      analyzeIncluded
+    )
+  );
+  if (analyzeIncluded) {
+    scripts["analyze.sql"] = buildStepScript(
+      "CNPJ DB Loader PostgreSQL direct import analyze step",
+      analyzeSql(selected),
+      input,
+      true
+    );
+  }
+  const orchestratorLines = [
+    "-- CNPJ DB Loader direct PostgreSQL import orchestrator",
     "-- Generated from sanitized Receita files by cnpj-db-loader postgres generate-script.",
-    "-- This path avoids rewriting the dataset into a second CSV tree.",
     "-- Execute with psql, for example:",
-    '--   psql "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
+    '--   psql -d "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
     "",
     "\\set ON_ERROR_STOP on",
-    `\\echo 'Using source file encoding ${input.sourceEncoding} for psql copy operations...'`,
+    echo(
+      `Using source file encoding ${input.sourceEncoding} for psql copy operations...`
+    ),
     `set client_encoding to ${quoteSqlLiteral(input.sourceEncoding)};`,
-    "\\echo 'Starting CNPJ DB Loader direct PostgreSQL import from sanitized files...'",
+    echo(
+      `Starting CNPJ DB Loader direct PostgreSQL import using transaction mode ${input.transactionMode}...`
+    ),
+    "",
+    ...input.transactionMode === "single" ? ["begin;", ""] : []
+  ];
+  for (const name of STEP_ORDER) {
+    const currentStep = steps.find((item) => item.name === name);
+    if (!currentStep?.included) {
+      continue;
+    }
+    orchestratorLines.push(
+      echo(
+        `[orchestrator] Running ${currentStep.name} (${currentStep.file})...`
+      ),
+      `\\ir ${currentStep.file}`,
+      echo(`[orchestrator] Completed ${currentStep.name}.`),
+      ""
+    );
+  }
+  orchestratorLines.push(
+    ...input.transactionMode === "single" ? ["commit;", ""] : [],
+    echo("CNPJ DB Loader hybrid PostgreSQL import completed."),
+    ""
+  );
+  scripts["import-postgres-direct.sql"] = orchestratorLines.join("\n");
+  return { scripts, steps };
+}
+function generatePostgresDirectImportScript(input) {
+  const grouped = csvFilesByDataset(input.files);
+  const lines = [
+    "-- CNPJ DB Loader hybrid PostgreSQL import script",
+    "-- Generated from PostgreSQL-ready CSV files exported by cnpj-db-loader postgres export-csv.",
+    "-- Execute with psql, for example:",
+    '--   psql -d "postgres://postgres:postgres@localhost:5432/cnpj" -f import-postgres-direct.sql',
+    "",
+    "\\set ON_ERROR_STOP on",
+    echo("Starting CNPJ DB Loader hybrid PostgreSQL import..."),
     "",
     "begin;",
     "",
     "-- Keep the final schema and seed data managed by sql/schema.sql.",
-    "-- This script copies sanitized Receita files into temporary raw tables,",
-    "-- transforms values inside PostgreSQL, resets staging tables and upserts final data.",
+    "-- This script only resets staging tables and then upserts final data.",
     "truncate table staging_companies restart identity;",
     "truncate table staging_establishments restart identity;",
     "truncate table staging_partners restart identity;",
@@ -8610,10 +9170,10 @@ function generatePostgresSanitizedDirectImportScript(input) {
     ""
   ];
   for (const dataset of DOMAIN_DATASETS) {
-    lines.push(...rawDomainSql(dataset, grouped[dataset] ?? []), "");
+    lines.push(...copyDomainSql(dataset, grouped[dataset] ?? []), "");
   }
   for (const dataset of STAGING_DATASETS) {
-    lines.push(...rawStagingSql(dataset, grouped[dataset] ?? []), "");
+    lines.push(...copyStagingSql(dataset, grouped[dataset] ?? []), "");
   }
   lines.push(...materializationAndAnalyzeSql());
   return lines.join("\n");
@@ -8624,11 +9184,13 @@ function materializationAndAnalyzeSql() {
     "",
     materializeEstablishmentsSql(),
     "",
+    materializeSecondaryCnaesSql(),
+    "",
     materializePartnersSql(),
     "",
     materializeSimplesSql(),
     "",
-    "\\echo 'Refreshing planner statistics...'",
+    echo("Refreshing planner statistics..."),
     "analyze companies;",
     "analyze establishments;",
     "analyze establishment_secondary_cnaes;",
@@ -8643,7 +9205,7 @@ function materializationAndAnalyzeSql() {
     "",
     "commit;",
     "",
-    "\\echo 'CNPJ DB Loader hybrid PostgreSQL import completed.'",
+    echo("CNPJ DB Loader hybrid PostgreSQL import completed."),
     ""
   ];
 }
@@ -8849,7 +9411,30 @@ async function exportPostgresCsvDataset(inputPath, options = {}) {
 // src/services/postgres-direct/generator.ts
 import { mkdir as mkdir9, stat as stat7, writeFile as writeFile6 } from "fs/promises";
 import path17 from "path";
-var DEFAULT_SOURCE_ENCODING = "WIN1252";
+var DEFAULT_SOURCE_ENCODING = "UTF8";
+var DEFAULT_TRANSACTION_MODE = "single";
+var ALL_INCLUDE_TARGETS = [
+  "domains",
+  "companies",
+  "establishments",
+  "partners",
+  "simples",
+  "secondary-cnaes",
+  "indexes",
+  "analyze"
+];
+var INCLUDE_TARGETS_BY_DATASET = {
+  companies: "companies",
+  establishments: "establishments",
+  partners: "partners",
+  simples_options: "simples",
+  countries: "domains",
+  cities: "domains",
+  partner_qualifications: "domains",
+  legal_natures: "domains",
+  reasons: "domains",
+  cnaes: "domains"
+};
 function defaultPostgresDirectOutputPath(inputPath) {
   const baseName = path17.basename(inputPath);
   if (baseName.toLowerCase() === "sanitized") {
@@ -8858,17 +9443,52 @@ function defaultPostgresDirectOutputPath(inputPath) {
   return path17.join(path17.dirname(inputPath), `${baseName}-postgres-direct`);
 }
 function inferNextStep5(scriptPath) {
-  return `psql "postgres://postgres:postgres@localhost:5432/cnpj" -f ${scriptPath.replace(/\\/g, "/")}`;
+  return `psql -d "postgres://postgres:postgres@localhost:5432/cnpj" -f ${scriptPath.replace(/\\/g, "/")}`;
 }
 function normalizeSourceEncoding(value) {
   const encoding = (value ?? DEFAULT_SOURCE_ENCODING).trim();
   if (!/^[A-Za-z0-9_-]+$/.test(encoding)) {
     throw new ValidationError(
-      `Invalid source encoding: ${value}. Use a PostgreSQL client encoding name such as WIN1252 or UTF8.`
+      `Invalid source encoding: ${value}. Use a PostgreSQL client encoding name such as UTF8, WIN1252 or LATIN1.`
     );
   }
   return encoding.toUpperCase();
 }
+function normalizeTransactionMode(value) {
+  const mode = value ?? DEFAULT_TRANSACTION_MODE;
+  if (!["single", "phase", "none"].includes(mode)) {
+    throw new ValidationError(
+      `Invalid transaction mode: ${String(value)}. Use single, phase or none.`
+    );
+  }
+  return mode;
+}
+function isIncludeTarget(value) {
+  return ALL_INCLUDE_TARGETS.includes(value);
+}
+function normalizeIncludeTargets(include, dataset) {
+  if (include && include.length > 0) {
+    const unique = [...new Set(include)];
+    const invalid = unique.filter((item) => !isIncludeTarget(item));
+    if (invalid.length > 0) {
+      throw new ValidationError(
+        `Invalid include target(s): ${invalid.join(", ")}. Use ${ALL_INCLUDE_TARGETS.join(", ")}.`
+      );
+    }
+    return unique;
+  }
+  if (dataset) {
+    const target = INCLUDE_TARGETS_BY_DATASET[dataset];
+    if (!target) {
+      return [];
+    }
+    if (target === "establishments") {
+      return ["establishments", "secondary-cnaes", "analyze"];
+    }
+    return [target, "analyze"];
+  }
+  return [...ALL_INCLUDE_TARGETS];
+}
 async function generatePostgresDirectScript(inputPath, options = {}) {
   if (options.dataset && !isImportDatasetType(options.dataset)) {
     throw new ValidationError(`Unsupported dataset type: ${options.dataset}.`);
@@ -8884,6 +9504,10 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
     options.outputPath ?? defaultPostgresDirectOutputPath(validatedPath)
   );
   const sourceEncoding = normalizeSourceEncoding(options.sourceEncoding);
+  const transactionMode = normalizeTransactionMode(options.transactionMode);
+  const include = normalizeIncludeTargets(options.include, options.dataset);
+  const skipIndexes = options.skipIndexes ?? false;
+  const skipAnalyze = options.skipAnalyze ?? false;
   const inspected = await inspectFiles(validatedPath);
   const recognizedFiles = inspected.entries.filter((entry) => entry.entryKind === "file").flatMap((entry) => {
     if (!isImportDatasetType(entry.inferredType)) {
@@ -8911,7 +9535,11 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
     outputPath,
     totalFiles: recognizedFiles.length,
     datasets,
-    sourceEncoding
+    sourceEncoding,
+    transactionMode,
+    include,
+    skipIndexes,
+    skipAnalyze
   });
   await mkdir9(outputPath, { recursive: true });
   const sourceFiles = [];
@@ -8947,11 +9575,21 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
   }
   const scriptName = options.scriptName ?? "import-postgres-direct.sql";
   const scriptPath = path17.join(outputPath, scriptName);
-  const script = generatePostgresSanitizedDirectImportScript({
+  const generated = generatePostgresDirectScriptFiles({
     files: sourceFiles,
-    sourceEncoding
+    sourceEncoding,
+    transactionMode,
+    include,
+    skipIndexes,
+    skipAnalyze
   });
-  await writeFile6(scriptPath, script, "utf8");
+  const scriptFiles = [];
+  for (const [fileName, script] of Object.entries(generated.scripts)) {
+    const outputFileName = fileName === "import-postgres-direct.sql" ? scriptName : fileName;
+    const outputFilePath = path17.join(outputPath, outputFileName);
+    await writeFile6(outputFilePath, script, "utf8");
+    scriptFiles.push(outputFilePath);
+  }
   const manifestPath = path17.join(outputPath, "manifest.json");
   const summaryDatasets = [...summariesByDataset.values()].sort(
     (left, right) => IMPORT_ORDER.indexOf(left.dataset) - IMPORT_ORDER.indexOf(right.dataset)
@@ -8963,13 +9601,19 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
   const manifest = {
     generatedAt: (/* @__PURE__ */ new Date()).toISOString(),
     mode: "direct-sanitized-script",
+    transactionMode,
+    include,
+    skipIndexes,
+    skipAnalyze,
     inputPath: path17.resolve(inputPath),
     validatedPath,
     outputPath,
     scriptPath,
+    scriptFiles,
     sourceEncoding,
     totalFiles: sourceFiles.length,
     totalBytes,
+    steps: generated.steps,
     datasets: summaryDatasets
   };
   await writeFile6(
@@ -8992,14 +9636,19 @@ async function generatePostgresDirectScript(inputPath, options = {}) {
     scriptPath,
     manifestPath,
     sourceEncoding,
+    transactionMode,
     totalFiles: sourceFiles.length,
     totalBytes,
     datasets: summaryDatasets,
+    scriptFiles,
+    steps: generated.steps,
     warnings: [
       ...validation.ok ? [] : validation.errors,
       "This script imports sanitized Receita files directly with psql \\copy. It avoids rewriting the full dataset into a second CSV tree.",
-      "The generated script expects the database schema generated by cnpj-db-loader to be applied before execution.",
-      "Use --source-encoding UTF8 only if your sanitized files are already UTF-8. The default WIN1252 matches the usual Receita file encoding."
+      "The generated scripts expect the database schema generated by cnpj-db-loader to be applied before execution.",
+      "The direct PostgreSQL script now defaults to UTF8 because the sanitize command writes clean UTF-8 files.",
+      "Use --source-encoding WIN1252 or LATIN1 only when generating scripts for legacy sanitized files produced by older loader versions.",
+      "The generated import is now modular. Use import-postgres-direct.sql as the orchestrator or run individual phase scripts manually."
     ],
     nextStep: inferNextStep5(scriptPath)
   };