@danielarndt0/cnpj-db-loader 2.2.1 → 2.3.0

package/dist/index.d.ts

@@ -144,6 +144,7 @@ type ImportPlanRecord = {
 };
 type ImportSchemaCapabilities = {
     includeEstablishmentCnpjFullInInsert: boolean;
+    includeEstablishmentSecondaryCnaesTable: boolean;
     includePartnerDedupeKeyInInsert: boolean;
     requiresLookupReconciliation: boolean;
 };

package/dist/index.js

@@ -1499,8 +1499,14 @@ function maskDatabaseLabel(url) {
 
 // src/services/database/cleanup.ts
 var MATERIALIZED_DATASET_TABLES = {
-  companies: ["simples_options", "partners", "establishments", "companies"],
-  establishments: ["establishments"],
+  companies: [
+    "simples_options",
+    "partners",
+    "establishment_secondary_cnaes",
+    "establishments",
+    "companies"
+  ],
+  establishments: ["establishment_secondary_cnaes", "establishments"],
   partners: ["partners"],
   simples_options: ["simples_options"]
 };
@@ -1544,6 +1550,22 @@ function collectMaterializedTables(dataset) {
   }
   return [...orderedTables];
 }
+async function tableExists3(client, tableName) {
+  const result = await client.query(
+    "select to_regclass(current_schema() || '.' || $1) as exists",
+    [tableName]
+  );
+  return Boolean(result.rows[0]?.exists);
+}
+async function filterExistingTables(client, tableNames) {
+  const existingTables = [];
+  for (const tableName of tableNames) {
+    if (await tableExists3(client, tableName)) {
+      existingTables.push(tableName);
+    }
+  }
+  return existingTables;
+}
 async function deleteLoadCheckpoints(client, dataset) {
   await ensureCheckpointTable(client);
   const result = dataset ? await client.query(`delete from import_checkpoints where dataset = $1`, [
@@ -1649,7 +1671,10 @@ async function cleanupDatabaseMaterializedTables(client, input) {
     targetDatabase,
     dataset: input.dataset
   });
-  const tableNames = collectMaterializedTables(input.dataset);
+  const tableNames = await filterExistingTables(
+    client,
+    collectMaterializedTables(input.dataset)
+  );
   if (tableNames.length > 0) {
     await client.query(`truncate ${tableNames.join(", ")}`);
   }
@@ -2280,6 +2305,7 @@ function createIndexesSql() {
   return [
     "-- Operational indexes",
     "create index if not exists idx_establishments_cnpj_root on establishments (cnpj_root);",
+    "create index if not exists idx_establishment_secondary_cnaes_cnae_code on establishment_secondary_cnaes (cnae_code);",
     "create index if not exists idx_partners_cnpj_root on partners (cnpj_root);",
     "create index if not exists idx_import_plans_status on import_plans (status);",
     "create index if not exists idx_import_plans_load_status on import_plans (load_status);",
@@ -2333,6 +2359,15 @@ function createPartnersSql() {
     ");"
   ].join("\n");
 }
+function createEstablishmentSecondaryCnaesSql() {
+  return [
+    "create table if not exists establishment_secondary_cnaes (",
+    "  cnpj_full text not null,",
+    "  cnae_code text not null,",
+    "  primary key (cnpj_full, cnae_code)",
+    ");"
+  ].join("\n");
+}
 function createSimplesSql() {
   return [
     "create table if not exists simples_options (",
@@ -2350,6 +2385,7 @@ function createOperationalSchemaParts() {
     "-- Final operational tables (simplified for fast first-load materialization)",
     createCompaniesSql(),
     createEstablishmentsSql(),
+    createEstablishmentSecondaryCnaesSql(),
     createPartnersSql(),
     createSimplesSql()
   ];
@@ -4311,6 +4347,68 @@ function buildChunkInsertSql(input) {
     values: [input.lastStagingId, input.chunkSize]
   };
 }
+function buildEstablishmentsChunkInsertSql(input) {
+  const chunkSelectList = [
+    "source.staging_id",
+    ...input.selectColumns.map((column) => `source.${column}`),
+    `${buildEstablishmentCnpjFullExpression("source")} as cnpj_full`
+  ].join(",\n    ");
+  const insertSelectList = input.insertColumns.join(", ");
+  const secondaryCnaesCtes = input.includeSecondaryCnaesTable ? [
+    ",",
+    "deleted_secondary_cnaes as (",
+    "  delete from establishment_secondary_cnaes target",
+    "  using (select distinct cnpj_full from inserted_establishments) source_keys",
+    "  where target.cnpj_full = source_keys.cnpj_full",
+    "  returning 1",
+    "),",
+    "secondary_cnaes_source as (",
+    "  select distinct",
+    "    chunked.cnpj_full,",
+    "    btrim(cnae_code) as cnae_code",
+    "  from chunked",
+    "  inner join inserted_establishments inserted",
+    "    on inserted.cnpj_full = chunked.cnpj_full",
+    "  cross join lateral unnest(string_to_array(chunked.secondary_cnaes_raw, ',')) as cnae_code",
+    "  where chunked.secondary_cnaes_raw is not null",
+    "    and chunked.secondary_cnaes_raw <> ''",
+    "    and btrim(cnae_code) <> ''",
+    "),",
+    "inserted_secondary_cnaes as (",
+    "  insert into establishment_secondary_cnaes (cnpj_full, cnae_code)",
+    "  select cnpj_full, cnae_code",
+    "  from secondary_cnaes_source",
+    "  on conflict (cnpj_full, cnae_code) do nothing",
+    "  returning 1",
+    ")"
+  ] : [];
+  return {
+    text: [
+      "with chunked as (",
+      `  select
+    ${chunkSelectList}`,
+      "  from staging_establishments source",
+      "  where source.staging_id > $1",
+      "  order by source.staging_id asc",
+      "  limit $2",
+      "),",
+      "inserted_establishments as (",
+      `  insert into establishments (${input.insertColumns.join(", ")})`,
+      `  select ${insertSelectList}`,
+      "  from chunked",
+      ...input.conflictClause ? [input.conflictClause] : [],
+      "  returning cnpj_full",
+      ")",
+      ...secondaryCnaesCtes,
+      "select",
+      "  coalesce(max(staging_id), $1::bigint)::bigint as max_staging_id,",
+      "  count(*)::bigint as source_rows,",
+      "  count(*)::bigint as affected_rows",
+      "from chunked;"
+    ].join("\n"),
+    values: [input.lastStagingId, input.chunkSize]
+  };
+}
 function buildPartnersChunkInsertSql(input) {
   const baseColumns = MATERIALIZATION_COLUMNS.partners;
   const chunkSelectList = [
@@ -4388,9 +4486,7 @@ function buildMaterializationChunkQuery(input) {
       });
     case "establishments": {
       const insertColumns = input.schemaCapabilities.includeEstablishmentCnpjFullInInsert ? [...baseColumns, "cnpj_full"] : [...baseColumns];
-      return buildChunkInsertSql({
-        stagingTable: "staging_establishments",
-        targetTable: "establishments",
+      return buildEstablishmentsChunkInsertSql({
         insertColumns,
         selectColumns: baseColumns,
         conflictClause: useConflictClause ? getConflictClause(
@@ -4400,7 +4496,7 @@ function buildMaterializationChunkQuery(input) {
         ) : "",
         lastStagingId: input.lastStagingId,
         chunkSize: input.chunkSize,
-        extraSelects: input.schemaCapabilities.includeEstablishmentCnpjFullInInsert ? [`${buildEstablishmentCnpjFullExpression("source")} as cnpj_full`] : []
+        includeSecondaryCnaesTable: input.schemaCapabilities.includeEstablishmentSecondaryCnaesTable
       });
     }
     case "simples_options":
@@ -4851,6 +4947,57 @@ async function validateDatasetCheckpoint(input) {
     targetRows
   };
 }
+async function readEstablishmentSecondaryCnaesCount(client) {
+  const result = await client.query(
+    `select count(*)::bigint as total_count from establishment_secondary_cnaes`
+  );
+  return Number.parseInt(result.rows[0]?.total_count ?? "0", 10);
+}
+async function hasEstablishmentsWithSecondaryCnaes(client) {
+  const result = await client.query(
+    `select exists (
+       select 1
+         from establishments
+        where secondary_cnaes_raw is not null
+          and secondary_cnaes_raw <> ''
+        limit 1
+     ) as exists`
+  );
+  return result.rows[0]?.exists ?? false;
+}
+async function backfillEstablishmentSecondaryCnaesFromFinal(input) {
+  const existingRows = await readEstablishmentSecondaryCnaesCount(input.client);
+  if (existingRows > 0) {
+    return 0;
+  }
+  if (!await hasEstablishmentsWithSecondaryCnaes(input.client)) {
+    return 0;
+  }
+  const startedAt = performance4.now();
+  const result = await input.client.query(
+    `insert into establishment_secondary_cnaes (cnpj_full, cnae_code)
+     select distinct
+       e.cnpj_full,
+       btrim(cnae_code) as cnae_code
+     from establishments e
+     cross join lateral unnest(
+       string_to_array(e.secondary_cnaes_raw, ',')
+     ) as cnae_code
+     where e.secondary_cnaes_raw is not null
+       and e.secondary_cnaes_raw <> ''
+       and btrim(cnae_code) <> ''
+     on conflict (cnpj_full, cnae_code) do nothing`
+  );
+  const insertedRows = result.rowCount ?? 0;
+  await appendJsonLinesLog(input.progressLogPath, {
+    kind: "establishment_secondary_cnaes_backfilled",
+    targetTable: "establishment_secondary_cnaes",
+    insertedRows,
+    durationMs: performance4.now() - startedAt,
+    timestamp: (/* @__PURE__ */ new Date()).toISOString()
+  });
+  return insertedRows;
+}
 async function materializeDatasetByChunks(input) {
   const startedAt = performance4.now();
   const validationReason = "Validating the live staging row count and staging cursor before resuming materialization.";
@@ -5297,6 +5444,12 @@ async function materializeStagedDatasets(input) {
         totalBatches: input.totalBatches,
         completedDatasets: summary.datasets.length
       });
+      if (dataset === "establishments" && input.schemaCapabilities.includeEstablishmentSecondaryCnaesTable) {
+        await backfillEstablishmentSecondaryCnaesFromFinal({
+          client: input.client,
+          progressLogPath: input.progressLogPath
+        });
+      }
       const tracker = input.datasetPerformanceTrackers.get(dataset);
       if (tracker) {
         tracker.materializationDurationMs += result.durationMs;
@@ -5520,6 +5673,12 @@ function canInsertIntoColumn(rows, tableName, columnName) {
   }
   return row.is_generated.toUpperCase() !== "ALWAYS";
 }
+function hasRequiredColumns(rows, tableName, columnNames) {
+  const availableColumns = new Set(
+    rows.filter((item) => item.table_name === tableName).map((item) => item.column_name)
+  );
+  return columnNames.every((columnName) => availableColumns.has(columnName));
+}
 async function detectImportSchemaCapabilities(client) {
   const [columnResult, lookupConstraintResult] = await Promise.all([
     client.query(
@@ -5528,6 +5687,7 @@ async function detectImportSchemaCapabilities(client) {
         where table_schema = current_schema()
           and (
             (table_name = 'establishments' and column_name = 'cnpj_full') or
+            (table_name = 'establishment_secondary_cnaes' and column_name in ('cnpj_full', 'cnae_code')) or
             (table_name = 'partners' and column_name = 'partner_dedupe_key')
           )`
     ),
@@ -5540,7 +5700,7 @@ async function detectImportSchemaCapabilities(client) {
            inner join pg_class target_table on target_table.oid = constraint_item.confrelid
           where constraint_item.contype = 'f'
             and source_namespace.nspname = current_schema()
-            and source_table.relname in ('companies', 'establishments', 'partners')
+            and source_table.relname in ('companies', 'establishments', 'partners', 'establishment_secondary_cnaes')
             and target_table.relname in (
               'countries',
               'cities',
@@ -5563,6 +5723,11 @@ async function detectImportSchemaCapabilities(client) {
       "establishments",
       "cnpj_full"
     ),
+    includeEstablishmentSecondaryCnaesTable: hasRequiredColumns(
+      columnResult.rows,
+      "establishment_secondary_cnaes",
+      ["cnpj_full", "cnae_code"]
+    ),
     includePartnerDedupeKeyInInsert: canInsertIntoColumn(
       columnResult.rows,
       "partners",