@danielarndt0/cnpj-db-loader 2.2.0 → 2.3.0

package/README.md

@@ -17,6 +17,7 @@ This version focuses on the real loading workflow:
   - exact preparatory scanning for total rows and total batches before import starts
   - persisted import plans reused on resume for the same validated input and batch size
   - staged bulk loads for the large datasets through PostgreSQL COPY
+  - automatic materialization of `establishment_secondary_cnaes` from establishment secondary CNAE data
   - direct final-schema upserts for the smaller domain datasets
   - checkpoint-based resume by file and byte offset
   - row quarantine for invalid or constraint-breaking records without stopping the import
@@ -95,7 +96,7 @@ The import internals are now split into dedicated modules such as planner, sourc
 
 The CLI now exposes a split workflow as well: `import` runs the full pipeline, `import load` stops after staging/direct writes, `import materialize` resumes from the saved plan and pushes staged rows into the final tables, and `database cleanup ...` exposes safe maintenance commands for staging tables, simplified final materialized tables, checkpoints, and saved plans.
 
-Materialization progress is now checkpointed separately from file-load checkpoints, and the materializer works in resumable chunks controlled by `--materialize-batch-size`. During long final materialization steps, the CLI keeps the live progress output on a dedicated MATERIALIZING stage while reducing per-chunk checkpoint and JSONL write overhead so resumable chunks stay fast. The simplified final schema keeps raw secondary CNAE text in establishments and derives helper fields such as partner dedupe keys during materialization only when they are still stored physically in the target schema.
+Materialization progress is now checkpointed separately from file-load checkpoints, and the materializer works in resumable chunks controlled by `--materialize-batch-size`. During long final materialization steps, the CLI keeps the live progress output on a dedicated MATERIALIZING stage while reducing per-chunk checkpoint and JSONL write overhead so resumable chunks stay fast. The simplified final schema keeps raw secondary CNAE text in establishments and also materializes `establishment_secondary_cnaes` so APIs can query one row per secondary CNAE without running a separate backfill script.
 
 The Federal Revenue commands write the same structured command logs and keep the remote-download phase outside the import internals. Existing completed ZIP files are skipped by default, temporary `.part` files are used while downloads are still in progress, and each reference keeps a local manifest for `status`, `retry`, `clean`, and future runner automation.
 

package/dist/cli.js

@@ -1512,8 +1512,14 @@ function maskDatabaseLabel(url) {
 
 // src/services/database/cleanup.ts
 var MATERIALIZED_DATASET_TABLES = {
-  companies: ["simples_options", "partners", "establishments", "companies"],
-  establishments: ["establishments"],
+  companies: [
+    "simples_options",
+    "partners",
+    "establishment_secondary_cnaes",
+    "establishments",
+    "companies"
+  ],
+  establishments: ["establishment_secondary_cnaes", "establishments"],
   partners: ["partners"],
   simples_options: ["simples_options"]
 };
@@ -1557,6 +1563,22 @@ function collectMaterializedTables(dataset) {
   }
   return [...orderedTables];
 }
+async function tableExists3(client, tableName) {
+  const result = await client.query(
+    "select to_regclass(current_schema() || '.' || $1) as exists",
+    [tableName]
+  );
+  return Boolean(result.rows[0]?.exists);
+}
+async function filterExistingTables(client, tableNames) {
+  const existingTables = [];
+  for (const tableName of tableNames) {
+    if (await tableExists3(client, tableName)) {
+      existingTables.push(tableName);
+    }
+  }
+  return existingTables;
+}
 async function deleteLoadCheckpoints(client, dataset) {
   await ensureCheckpointTable(client);
   const result = dataset ? await client.query(`delete from import_checkpoints where dataset = $1`, [
@@ -1662,7 +1684,10 @@ async function cleanupDatabaseMaterializedTables(client, input2) {
     targetDatabase,
     dataset: input2.dataset
   });
-  const tableNames = collectMaterializedTables(input2.dataset);
+  const tableNames = await filterExistingTables(
+    client,
+    collectMaterializedTables(input2.dataset)
+  );
   if (tableNames.length > 0) {
     await client.query(`truncate ${tableNames.join(", ")}`);
   }
@@ -2261,6 +2286,7 @@ function createIndexesSql() {
   return [
     "-- Operational indexes",
     "create index if not exists idx_establishments_cnpj_root on establishments (cnpj_root);",
+    "create index if not exists idx_establishment_secondary_cnaes_cnae_code on establishment_secondary_cnaes (cnae_code);",
     "create index if not exists idx_partners_cnpj_root on partners (cnpj_root);",
     "create index if not exists idx_import_plans_status on import_plans (status);",
     "create index if not exists idx_import_plans_load_status on import_plans (load_status);",
@@ -2314,6 +2340,15 @@ function createPartnersSql() {
     ");"
   ].join("\n");
 }
+function createEstablishmentSecondaryCnaesSql() {
+  return [
+    "create table if not exists establishment_secondary_cnaes (",
+    "  cnpj_full text not null,",
+    "  cnae_code text not null,",
+    "  primary key (cnpj_full, cnae_code)",
+    ");"
+  ].join("\n");
+}
 function createSimplesSql() {
   return [
     "create table if not exists simples_options (",
@@ -2331,6 +2366,7 @@ function createOperationalSchemaParts() {
     "-- Final operational tables (simplified for fast first-load materialization)",
     createCompaniesSql(),
     createEstablishmentsSql(),
+    createEstablishmentSecondaryCnaesSql(),
     createPartnersSql(),
     createSimplesSql()
   ];
@@ -4279,6 +4315,68 @@ function buildChunkInsertSql(input2) {
     values: [input2.lastStagingId, input2.chunkSize]
   };
 }
+function buildEstablishmentsChunkInsertSql(input2) {
+  const chunkSelectList = [
+    "source.staging_id",
+    ...input2.selectColumns.map((column) => `source.${column}`),
+    `${buildEstablishmentCnpjFullExpression("source")} as cnpj_full`
+  ].join(",\n    ");
+  const insertSelectList = input2.insertColumns.join(", ");
+  const secondaryCnaesCtes = input2.includeSecondaryCnaesTable ? [
+    ",",
+    "deleted_secondary_cnaes as (",
+    "  delete from establishment_secondary_cnaes target",
+    "  using (select distinct cnpj_full from inserted_establishments) source_keys",
+    "  where target.cnpj_full = source_keys.cnpj_full",
+    "  returning 1",
+    "),",
+    "secondary_cnaes_source as (",
+    "  select distinct",
+    "    chunked.cnpj_full,",
+    "    btrim(cnae_code) as cnae_code",
+    "  from chunked",
+    "  inner join inserted_establishments inserted",
+    "    on inserted.cnpj_full = chunked.cnpj_full",
+    "  cross join lateral unnest(string_to_array(chunked.secondary_cnaes_raw, ',')) as cnae_code",
+    "  where chunked.secondary_cnaes_raw is not null",
+    "    and chunked.secondary_cnaes_raw <> ''",
+    "    and btrim(cnae_code) <> ''",
+    "),",
+    "inserted_secondary_cnaes as (",
+    "  insert into establishment_secondary_cnaes (cnpj_full, cnae_code)",
+    "  select cnpj_full, cnae_code",
+    "  from secondary_cnaes_source",
+    "  on conflict (cnpj_full, cnae_code) do nothing",
+    "  returning 1",
+    ")"
+  ] : [];
+  return {
+    text: [
+      "with chunked as (",
+      `  select
+    ${chunkSelectList}`,
+      "  from staging_establishments source",
+      "  where source.staging_id > $1",
+      "  order by source.staging_id asc",
+      "  limit $2",
+      "),",
+      "inserted_establishments as (",
+      `  insert into establishments (${input2.insertColumns.join(", ")})`,
+      `  select ${insertSelectList}`,
+      "  from chunked",
+      ...input2.conflictClause ? [input2.conflictClause] : [],
+      "  returning cnpj_full",
+      ")",
+      ...secondaryCnaesCtes,
+      "select",
+      "  coalesce(max(staging_id), $1::bigint)::bigint as max_staging_id,",
+      "  count(*)::bigint as source_rows,",
+      "  count(*)::bigint as affected_rows",
+      "from chunked;"
+    ].join("\n"),
+    values: [input2.lastStagingId, input2.chunkSize]
+  };
+}
 function buildPartnersChunkInsertSql(input2) {
   const baseColumns = MATERIALIZATION_COLUMNS.partners;
   const chunkSelectList = [
@@ -4356,9 +4454,7 @@ function buildMaterializationChunkQuery(input2) {
       });
     case "establishments": {
       const insertColumns = input2.schemaCapabilities.includeEstablishmentCnpjFullInInsert ? [...baseColumns, "cnpj_full"] : [...baseColumns];
-      return buildChunkInsertSql({
-        stagingTable: "staging_establishments",
-        targetTable: "establishments",
+      return buildEstablishmentsChunkInsertSql({
         insertColumns,
         selectColumns: baseColumns,
         conflictClause: useConflictClause ? getConflictClause(
@@ -4368,7 +4464,7 @@ function buildMaterializationChunkQuery(input2) {
         ) : "",
         lastStagingId: input2.lastStagingId,
         chunkSize: input2.chunkSize,
-        extraSelects: input2.schemaCapabilities.includeEstablishmentCnpjFullInInsert ? [`${buildEstablishmentCnpjFullExpression("source")} as cnpj_full`] : []
+        includeSecondaryCnaesTable: input2.schemaCapabilities.includeEstablishmentSecondaryCnaesTable
       });
     }
     case "simples_options":
@@ -4819,6 +4915,57 @@ async function validateDatasetCheckpoint(input2) {
     targetRows
   };
 }
+async function readEstablishmentSecondaryCnaesCount(client) {
+  const result = await client.query(
+    `select count(*)::bigint as total_count from establishment_secondary_cnaes`
+  );
+  return Number.parseInt(result.rows[0]?.total_count ?? "0", 10);
+}
+async function hasEstablishmentsWithSecondaryCnaes(client) {
+  const result = await client.query(
+    `select exists (
+       select 1
+         from establishments
+        where secondary_cnaes_raw is not null
+          and secondary_cnaes_raw <> ''
+        limit 1
+     ) as exists`
+  );
+  return result.rows[0]?.exists ?? false;
+}
+async function backfillEstablishmentSecondaryCnaesFromFinal(input2) {
+  const existingRows = await readEstablishmentSecondaryCnaesCount(input2.client);
+  if (existingRows > 0) {
+    return 0;
+  }
+  if (!await hasEstablishmentsWithSecondaryCnaes(input2.client)) {
+    return 0;
+  }
+  const startedAt = performance4.now();
+  const result = await input2.client.query(
+    `insert into establishment_secondary_cnaes (cnpj_full, cnae_code)
+     select distinct
+       e.cnpj_full,
+       btrim(cnae_code) as cnae_code
+     from establishments e
+     cross join lateral unnest(
+       string_to_array(e.secondary_cnaes_raw, ',')
+     ) as cnae_code
+     where e.secondary_cnaes_raw is not null
+       and e.secondary_cnaes_raw <> ''
+       and btrim(cnae_code) <> ''
+     on conflict (cnpj_full, cnae_code) do nothing`
+  );
+  const insertedRows = result.rowCount ?? 0;
+  await appendJsonLinesLog(input2.progressLogPath, {
+    kind: "establishment_secondary_cnaes_backfilled",
+    targetTable: "establishment_secondary_cnaes",
+    insertedRows,
+    durationMs: performance4.now() - startedAt,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString()
+  });
+  return insertedRows;
+}
 async function materializeDatasetByChunks(input2) {
   const startedAt = performance4.now();
   const validationReason = "Validating the live staging row count and staging cursor before resuming materialization.";
@@ -5265,6 +5412,12 @@ async function materializeStagedDatasets(input2) {
         totalBatches: input2.totalBatches,
         completedDatasets: summary.datasets.length
       });
+      if (dataset === "establishments" && input2.schemaCapabilities.includeEstablishmentSecondaryCnaesTable) {
+        await backfillEstablishmentSecondaryCnaesFromFinal({
+          client: input2.client,
+          progressLogPath: input2.progressLogPath
+        });
+      }
       const tracker = input2.datasetPerformanceTrackers.get(dataset);
       if (tracker) {
         tracker.materializationDurationMs += result.durationMs;
@@ -5488,6 +5641,12 @@ function canInsertIntoColumn(rows, tableName, columnName) {
   }
   return row.is_generated.toUpperCase() !== "ALWAYS";
 }
+function hasRequiredColumns(rows, tableName, columnNames) {
+  const availableColumns = new Set(
+    rows.filter((item) => item.table_name === tableName).map((item) => item.column_name)
+  );
+  return columnNames.every((columnName) => availableColumns.has(columnName));
+}
 async function detectImportSchemaCapabilities(client) {
   const [columnResult, lookupConstraintResult] = await Promise.all([
     client.query(
@@ -5496,6 +5655,7 @@ async function detectImportSchemaCapabilities(client) {
         where table_schema = current_schema()
           and (
             (table_name = 'establishments' and column_name = 'cnpj_full') or
+            (table_name = 'establishment_secondary_cnaes' and column_name in ('cnpj_full', 'cnae_code')) or
             (table_name = 'partners' and column_name = 'partner_dedupe_key')
           )`
     ),
@@ -5508,7 +5668,7 @@ async function detectImportSchemaCapabilities(client) {
            inner join pg_class target_table on target_table.oid = constraint_item.confrelid
           where constraint_item.contype = 'f'
             and source_namespace.nspname = current_schema()
-            and source_table.relname in ('companies', 'establishments', 'partners')
+            and source_table.relname in ('companies', 'establishments', 'partners', 'establishment_secondary_cnaes')
             and target_table.relname in (
               'countries',
               'cities',
@@ -5531,6 +5691,11 @@ async function detectImportSchemaCapabilities(client) {
       "establishments",
       "cnpj_full"
     ),
+    includeEstablishmentSecondaryCnaesTable: hasRequiredColumns(
+      columnResult.rows,
+      "establishment_secondary_cnaes",
+      ["cnpj_full", "cnae_code"]
+    ),
     includePartnerDedupeKeyInInsert: canInsertIntoColumn(
       columnResult.rows,
       "partners",