@dataif/cli 0.1.0

package/templates/dataif/pipelines/dataif_pipelines/services/pnp_raw_ingestion_service.py

@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from dataif_pipelines.connectors.base.types import NormalizedRecord, RawRecord, RunContext
+from dataif_pipelines.connectors.nilo_pecanha.transform import normalize_domain_record, normalize_record
+from dataif_pipelines.repositories import pnp_raw_repository
+
+
+@dataclass(frozen=True)
+class NormalizationResult:
+    normalized_records: list[NormalizedRecord]
+    quarantine_rows: list[dict[str, Any]]
+
+
+def normalize_raw_records(raw_records: list[RawRecord], run_context: RunContext) -> NormalizationResult:
+    normalized: list[NormalizedRecord] = []
+    quarantine_rows: list[dict[str, Any]] = []
+
+    for raw_record in raw_records:
+        payload = raw_record.get("payload")
+        if not isinstance(payload, dict):
+            continue
+
+        source_url = str(raw_record.get("source_url") or run_context.source_url)
+        legacy_record = normalize_record(
+            payload=payload,
+            source_url=source_url,
+            run_id=run_context.run_id,
+            endpoint_id=int(raw_record.get("endpoint_id")),
+            endpoint_key=str(raw_record.get("endpoint_key") or "default"),
+            source_kind=str(raw_record.get("source_kind") or "powerbi_microdados"),
+        )
+
+        try:
+            domain_record = normalize_domain_record(
+                payload=payload,
+                run_id=run_context.run_id,
+                instance_key=str(payload.get("instance_key") or "").strip() or None,
+                source_url=source_url,
+            )
+        except Exception as exc:
+            quarantine_rows.append(
+                {
+                    "run_id": run_context.run_id,
+                    "instance_key": str(payload.get("instance_key") or "").strip() or None,
+                    "source_url": source_url,
+                    "source_row_number": payload.get("source_row_number"),
+                    "error_type": "unsupported_domain",
+                    "error_message": str(exc),
+                    "raw_line_text": json.dumps(payload, ensure_ascii=True, sort_keys=True),
+                    "details_json": {"tipo_microdados": payload.get("tipo_microdados")},
+                }
+            )
+            continue
+
+        normalized.append({**legacy_record, **domain_record})
+
+    return NormalizationResult(normalized_records=normalized, quarantine_rows=quarantine_rows)
+
+
+def load_raw_batch(
+    dsn: str,
+    *,
+    normalized_records: list[dict[str, Any]],
+    pending_assets: list[dict[str, Any]],
+    pending_catalog_entries: list[dict[str, Any]],
+    pending_run_selection: list[dict[str, Any]],
+    pending_downloads: list[dict[str, Any]],
+    pending_quarantine: list[dict[str, Any]],
+    write_legacy: bool = False,
+) -> dict[str, int]:
+    return pnp_raw_repository.load_raw_batch(
+        dsn,
+        normalized_records=normalized_records,
+        pending_assets=pending_assets,
+        pending_catalog_entries=pending_catalog_entries,
+        pending_run_selection=pending_run_selection,
+        pending_downloads=pending_downloads,
+        pending_quarantine=pending_quarantine,
+        write_legacy=write_legacy,
+    )
+
+
+def upsert_raw_metadata(
+    dsn: str,
+    *,
+    pending_assets: list[dict[str, Any]],
+    pending_catalog_entries: list[dict[str, Any]],
+    pending_run_selection: list[dict[str, Any]],
+    pending_downloads: list[dict[str, Any]],
+    write_legacy: bool = False,
+    include_download_columns: bool = True,
+) -> dict[str, Any]:
+    return pnp_raw_repository.upsert_raw_metadata(
+        dsn,
+        pending_assets=pending_assets,
+        pending_catalog_entries=pending_catalog_entries,
+        pending_run_selection=pending_run_selection,
+        pending_downloads=pending_downloads,
+        write_legacy=write_legacy,
+        include_download_columns=include_download_columns,
+    )
+
+
+def load_raw_record_chunk(
+    dsn: str,
+    *,
+    normalized_records: list[dict[str, Any]],
+    pending_quarantine: list[dict[str, Any]],
+    download_id_by_url: dict[str, int],
+    pending_assets: list[dict[str, Any]] | None = None,
+    write_legacy: bool = False,
+) -> dict[str, int]:
+    return pnp_raw_repository.load_raw_record_chunk(
+        dsn,
+        normalized_records=normalized_records,
+        pending_quarantine=pending_quarantine,
+        download_id_by_url=download_id_by_url,
+        pending_assets=pending_assets,
+        write_legacy=write_legacy,
+    )

package/templates/dataif/pipelines/dataif_pipelines/services/pnp_staging_service.py

@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import psycopg2
+from psycopg2.extras import Json, RealDictCursor
+
+from dataif_pipelines.repositories import pnp_raw_repository
+
+_ENDPOINT_SQL = {
+    "Matrículas": {
+        "domain": "020_pnp_matriculas.sql",
+    },
+    "Eficiência Acadêmica": {
+        "domain": "030_pnp_eficiencia_academica.sql",
+    },
+    "Servidores": {
+        "domain": "040_pnp_servidores.sql",
+    },
+    "Financeiro": {
+        "domain": "050_pnp_financeiro.sql",
+    },
+}
+
+
+def _resolve_sql_dir() -> Path:
+    candidates = (
+        Path(__file__).resolve().parents[2] / "sql" / "staging",
+        Path(__file__).resolve().parents[3] / "sql" / "staging",
+    )
+    for candidate in candidates:
+        if candidate.exists():
+            return candidate
+    raise FileNotFoundError("staging SQL directory not found in expected locations")
+
+
+def _read_sql_file(filename: str) -> str:
+    return (_resolve_sql_dir() / filename).read_text(encoding="utf-8")
+
+
+def _list_run_download_batches(dsn: str, *, run_id: str, instance_key: str | None) -> list[dict[str, Any]]:
+    with psycopg2.connect(dsn, cursor_factory=RealDictCursor) as conn, conn.cursor() as cur:
+        cur.execute(
+            """
+            SELECT
+              download_id,
+              tipo_microdados
+            FROM raw.pnp_downloads
+            WHERE run_id = %s
+              AND instance_key IS NOT DISTINCT FROM %s
+              AND status = 'success'
+            ORDER BY
+              CASE tipo_microdados
+                WHEN 'Matrículas' THEN 1
+                WHEN 'Eficiência Acadêmica' THEN 2
+                WHEN 'Servidores' THEN 3
+                WHEN 'Financeiro' THEN 4
+                ELSE 99
+              END,
+              download_id
+            """,
+            (run_id, instance_key),
+        )
+        return [dict(row) for row in cur.fetchall()]
+
+
+def _collect_materialized_counts(dsn: str, *, run_id: str) -> dict[str, int]:
+    with psycopg2.connect(dsn, cursor_factory=RealDictCursor) as conn, conn.cursor() as cur:
+        counts: dict[str, int] = {}
+        for table_name, result_key in (
+            ("staging.pnp_matriculas", "matriculas_count"),
+            ("staging.pnp_eficiencia_academica", "eficiencia_academica_count"),
+            ("staging.pnp_servidores", "servidores_count"),
+            ("staging.pnp_financeiro", "financeiro_count"),
+        ):
+            cur.execute(f"SELECT COUNT(*) AS count_value FROM {table_name} WHERE run_id = %s", (run_id,))
+            counts[result_key] = int((cur.fetchone() or {}).get("count_value") or 0)
+        counts["deduplicated_count"] = (
+            counts["matriculas_count"]
+            + counts["eficiencia_academica_count"]
+            + counts["servidores_count"]
+            + counts["financeiro_count"]
+        )
+        return counts
+
+
+def materialize_instance_staging(dsn: str, *, run_id: str, instance_key: str | None) -> dict[str, Any]:
+    raw_checks = pnp_raw_repository.collect_run_checks(dsn, run_id)
+    started_at = datetime.now(tz=UTC)
+    batch_rows = _list_run_download_batches(dsn, run_id=run_id, instance_key=instance_key)
+    counts = {
+        "matriculas_count": 0,
+        "eficiencia_academica_count": 0,
+        "servidores_count": 0,
+        "financeiro_count": 0,
+        "deduplicated_count": 0,
+    }
+    try:
+        with psycopg2.connect(dsn, cursor_factory=RealDictCursor) as conn, conn.cursor() as cur:
+            cur.execute(
+                """
+                INSERT INTO staging.pnp_ingestion_runs (
+                  run_id,
+                  instance_key,
+                  status,
+                  selected_download_count,
+                  downloaded_file_count,
+                  raw_record_count,
+                  deduplicated_record_count,
+                  quality_status,
+                  quality_summary_json,
+                  started_at,
+                  finished_at,
+                  updated_at
+                )
+                VALUES (%s, %s, 'running', %s, %s, %s, 0, NULL, '{}'::jsonb, %s, NULL, NOW())
+                ON CONFLICT (run_id) DO UPDATE
+                SET
+                  instance_key = EXCLUDED.instance_key,
+                  status = EXCLUDED.status,
+                  selected_download_count = EXCLUDED.selected_download_count,
+                  downloaded_file_count = EXCLUDED.downloaded_file_count,
+                  raw_record_count = EXCLUDED.raw_record_count,
+                  deduplicated_record_count = 0,
+                  quality_status = NULL,
+                  quality_summary_json = '{}'::jsonb,
+                  started_at = EXCLUDED.started_at,
+                  finished_at = NULL,
+                  updated_at = NOW()
+                """,
+                (
+                    run_id,
+                    instance_key,
+                    int(raw_checks.get("run_selection_count") or 0),
+                    int(raw_checks.get("download_count") or 0),
+                    int(raw_checks.get("raw_count") or 0),
+                    started_at,
+                ),
+            )
+            conn.commit()
+
+        for batch in batch_rows:
+            tipo_microdados = str(batch.get("tipo_microdados") or "")
+            sql_files = _ENDPOINT_SQL.get(tipo_microdados)
+            if not sql_files:
+                continue
+            params = {
+                "instance_key": instance_key,
+                "run_id": run_id,
+                "download_id": int(batch["download_id"]),
+            }
+            with psycopg2.connect(dsn, cursor_factory=RealDictCursor) as conn, conn.cursor() as cur:
+                cur.execute("SET LOCAL synchronous_commit = OFF")
+                cur.execute(_read_sql_file(sql_files["domain"]), params)
+                conn.commit()
+
+        counts = _collect_materialized_counts(dsn, run_id=run_id)
+
+        with psycopg2.connect(dsn, cursor_factory=RealDictCursor) as conn, conn.cursor() as cur:
+            cur.execute(
+                """
+                UPDATE staging.pnp_ingestion_runs
+                SET
+                  status = 'success',
+                  deduplicated_record_count = %s,
+                  quality_status = 'passed',
+                  quality_summary_json = %s,
+                  finished_at = NOW(),
+                  updated_at = NOW()
+                WHERE run_id = %s
+                """,
+                (
+                    counts["deduplicated_count"],
+                    Json(
+                        {
+                            "raw_checks": raw_checks,
+                            "staging_counts": {
+                                "matriculas_count": counts["matriculas_count"],
+                                "eficiencia_academica_count": counts["eficiencia_academica_count"],
+                                "servidores_count": counts["servidores_count"],
+                                "financeiro_count": counts["financeiro_count"],
+                                "deduplicated_count": counts["deduplicated_count"],
+                            },
+                            "batch_count": len(batch_rows),
+                        }
+                    ),
+                    run_id,
+                ),
+            )
+    except Exception as exc:
+        with psycopg2.connect(dsn, cursor_factory=RealDictCursor) as error_conn:
+            with error_conn.cursor() as cur:
+                cur.execute(
+                    """
+                    INSERT INTO staging.pnp_ingestion_runs (
+                      run_id,
+                      instance_key,
+                      status,
+                      selected_download_count,
+                      downloaded_file_count,
+                      raw_record_count,
+                      deduplicated_record_count,
+                      quality_status,
+                      quality_summary_json,
+                      started_at,
+                      finished_at,
+                      updated_at
+                    )
+                    VALUES (%s, %s, 'running', %s, %s, %s, 0, NULL, '{}'::jsonb, %s, NULL, NOW())
+                    ON CONFLICT (run_id) DO UPDATE
+                    SET
+                      instance_key = EXCLUDED.instance_key,
+                      status = EXCLUDED.status,
+                      selected_download_count = EXCLUDED.selected_download_count,
+                      downloaded_file_count = EXCLUDED.downloaded_file_count,
+                      raw_record_count = EXCLUDED.raw_record_count,
+                      deduplicated_record_count = 0,
+                      quality_status = NULL,
+                      quality_summary_json = '{}'::jsonb,
+                      started_at = EXCLUDED.started_at,
+                      finished_at = NULL,
+                      updated_at = NOW()
+                    """,
+                    (
+                        run_id,
+                        instance_key,
+                        int(raw_checks.get("run_selection_count") or 0),
+                        int(raw_checks.get("download_count") or 0),
+                        int(raw_checks.get("raw_count") or 0),
+                        started_at,
+                    ),
+                )
+        raise
+
+    return {
+        "run_id": run_id,
+        "instance_key": instance_key,
+        "matriculas_count": counts["matriculas_count"],
+        "eficiencia_academica_count": counts["eficiencia_academica_count"],
+        "servidores_count": counts["servidores_count"],
+        "financeiro_count": counts["financeiro_count"],
+        "deduplicated_record_count": counts["deduplicated_count"],
+        "batch_count": len(batch_rows),
+    }
+
+
+def collect_staging_checks(dsn: str, run_id: str) -> dict[str, Any]:
+    with psycopg2.connect(dsn, cursor_factory=RealDictCursor) as conn, conn.cursor() as cur:
+        cur.execute(
+            """
+            SELECT
+              run_id,
+              instance_key,
+              status,
+              selected_download_count,
+              downloaded_file_count,
+              raw_record_count,
+              deduplicated_record_count,
+              quality_status,
+              quality_summary_json,
+              started_at,
+              finished_at
+            FROM staging.pnp_ingestion_runs
+            WHERE run_id = %s
+            """,
+            (run_id,),
+        )
+        row = cur.fetchone()
+        return dict(row) if row else {"run_id": run_id}

package/templates/dataif/pipelines/dataif_pipelines/services/powerbi_catalog_service.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from typing import Any
+
+from dataif_pipelines.connectors.nilo_pecanha.powerbi_microdados import (
+    MicrodadosCatalogEntry,
+    PowerBIMicrodadosClient,
+    PowerBIMicrodadosContext,
+)
+
+
+@dataclass(frozen=True)
+class CatalogSelectionResult:
+    instance_key: str | None
+    selected_years: tuple[str, ...]
+    selected_microdados_types: tuple[str, ...]
+    selection_source: str
+    context: PowerBIMicrodadosContext
+    catalog_entries: tuple[MicrodadosCatalogEntry, ...]
+    selected_entries: tuple[MicrodadosCatalogEntry, ...]
+
+
+def create_powerbi_client(*, page_url: str, timeout_seconds: int) -> PowerBIMicrodadosClient:
+    return PowerBIMicrodadosClient(page_url=page_url, timeout_seconds=timeout_seconds)
+
+
+def resolve_catalog_selection(
+    *,
+    client: PowerBIMicrodadosClient,
+    request_params: dict[str, Any],
+) -> CatalogSelectionResult:
+    instance_key = str(request_params.get("instance_key") or "").strip() or None
+    selected_years = tuple(
+        str(item).strip()
+        for item in (request_params.get("selected_years") or [])
+        if isinstance(item, str) and item.strip()
+    )
+    selected_microdados_types = tuple(
+        str(item).strip()
+        for item in (request_params.get("selected_microdados_types") or [])
+        if isinstance(item, str) and item.strip()
+    )
+    if not selected_years:
+        raise RuntimeError("powerbi_microdados mode requires selected_years")
+    if not selected_microdados_types:
+        raise RuntimeError("powerbi_microdados mode requires selected_microdados_types")
+
+    context, catalog_entries = client.fetch_catalog()
+    selected_downloads = [
+        item
+        for item in (request_params.get("selected_downloads") or [])
+        if isinstance(item, dict)
+    ]
+
+    if selected_downloads:
+        selected_keys = {
+            (
+                str(item.get("ano_base") or "").strip(),
+                str(item.get("tipo_microdados") or "").strip(),
+                str(item.get("microdados_url") or "").strip(),
+            )
+            for item in selected_downloads
+            if str(item.get("ano_base") or "").strip()
+            and str(item.get("tipo_microdados") or "").strip()
+            and str(item.get("microdados_url") or "").strip()
+        }
+        filtered_entries = [
+            entry
+            for entry in catalog_entries
+            if (entry.ano_base, entry.tipo_microdados, entry.microdados_url) in selected_keys
+        ]
+        missing_keys = selected_keys - {
+            (entry.ano_base, entry.tipo_microdados, entry.microdados_url)
+            for entry in filtered_entries
+        }
+        if missing_keys:
+            missing_text = ", ".join(
+                f"{ano_base}/{tipo_microdados}"
+                for ano_base, tipo_microdados, _microdados_url in sorted(missing_keys)
+            )
+            raise RuntimeError(f"powerbi_microdados selected_downloads missing from public catalog: {missing_text}")
+        selection_source = "selected_downloads"
+    else:
+        filtered_entries = [
+            entry
+            for entry in catalog_entries
+            if entry.ano_base in selected_years and entry.tipo_microdados in selected_microdados_types
+        ]
+        selection_source = "catalog_filter"
+
+    if not filtered_entries:
+        raise RuntimeError("powerbi_microdados selection matched no download links in the public catalog")
+
+    return CatalogSelectionResult(
+        instance_key=instance_key,
+        selected_years=selected_years,
+        selected_microdados_types=selected_microdados_types,
+        selection_source=selection_source,
+        context=context,
+        catalog_entries=tuple(catalog_entries),
+        selected_entries=tuple(filtered_entries),
+    )
+
+
+def build_catalog_entry_rows(
+    *,
+    run_id: str,
+    selection: CatalogSelectionResult,
+) -> list[dict[str, Any]]:
+    selected_keys = {
+        (entry.ano_base, entry.tipo_microdados, entry.microdados_url)
+        for entry in selection.selected_entries
+    }
+    rows: list[dict[str, Any]] = []
+    for entry in selection.catalog_entries:
+        rows.append(
+            {
+                "run_id": run_id,
+                "instance_key": selection.instance_key,
+                "ano_base": entry.ano_base,
+                "tipo_microdados": entry.tipo_microdados,
+                "microdados_url": entry.microdados_url,
+                "resource_key": selection.context.resource_key,
+                "visual_id": selection.context.visual_id,
+                "api_base_url": selection.context.api_base_url,
+                "catalog_hash": hashlib.sha256(
+                    f"{entry.ano_base}|{entry.tipo_microdados}|{entry.microdados_url}".encode("utf-8")
+                ).hexdigest(),
+                "is_selected": (entry.ano_base, entry.tipo_microdados, entry.microdados_url) in selected_keys,
+            }
+        )
+    return rows
+
+
+def build_run_selection_rows(
+    *,
+    run_id: str,
+    selection: CatalogSelectionResult,
+) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for selection_rank, entry in enumerate(selection.selected_entries, start=1):
+        rows.append(
+            {
+                "run_id": run_id,
+                "instance_key": selection.instance_key,
+                "ano_base": entry.ano_base,
+                "tipo_microdados": entry.tipo_microdados,
+                "microdados_url": entry.microdados_url,
+                "selection_source": selection.selection_source,
+                "selection_rank": selection_rank,
+                "details_json": {
+                    "page_url": selection.context.page_url,
+                    "resource_key": selection.context.resource_key,
+                },
+            }
+        )
+    return rows

package/templates/dataif/pipelines/sql/staging/020_pnp_matriculas.sql

@@ -0,0 +1,112 @@
+WITH selected_rows AS (
+    SELECT
+        src.*,
+        ROW_NUMBER() OVER (
+            PARTITION BY src.instance_key, src.record_hash
+            ORDER BY src.raw_record_id DESC
+        ) AS dedup_rank
+    FROM raw.pnp_matriculas_src src
+    LEFT JOIN raw.pnp_downloads downloads ON downloads.download_id = src.download_id
+    JOIN raw.pnp_instance_selection selection
+      ON selection.instance_key = src.instance_key
+     AND selection.is_active = TRUE
+     AND selection.ano_base = src.ano_base
+     AND selection.tipo_microdados = src.tipo_microdados
+     AND (
+        selection.configured_microdados_url IS NULL
+        OR selection.configured_microdados_url = downloads.microdados_url
+     )
+    WHERE src.run_id = %(run_id)s
+      AND src.instance_key IS NOT DISTINCT FROM %(instance_key)s
+      AND src.download_id = %(download_id)s
+),
+deduplicated_rows AS (
+    SELECT *
+    FROM selected_rows
+    WHERE dedup_rank = 1
+)
+INSERT INTO staging.pnp_matriculas (
+    raw_record_id,
+    run_id,
+    instance_key,
+    ano,
+    instituicao,
+    regiao,
+    uf,
+    municipio,
+    sexo,
+    cor_raca,
+    renda_familiar,
+    faixa_etaria,
+    situacao_matricula,
+    modalidade_ensino,
+    tipo_curso,
+    tipo_oferta,
+    turno,
+    eixo_tecnologico,
+    subeixo_tecnologico,
+    nome_curso,
+    total_inscritos,
+    vagas_ofertadas,
+    processed_at
+)
+SELECT
+    deduplicated_rows.raw_record_id,
+    deduplicated_rows.run_id,
+    deduplicated_rows.instance_key,
+    CASE
+        WHEN NULLIF(deduplicated_rows.ano, '') ~ '^[0-9]{1,4}$' THEN deduplicated_rows.ano::INTEGER
+        ELSE NULL
+    END AS ano,
+    NULLIF(deduplicated_rows.instituicao, '') AS instituicao,
+    NULLIF(deduplicated_rows.regiao, '') AS regiao,
+    NULLIF(deduplicated_rows.uf, '') AS uf,
+    NULLIF(deduplicated_rows.municipio, '') AS municipio,
+    NULLIF(deduplicated_rows.sexo, '') AS sexo,
+    NULLIF(deduplicated_rows.cor_raca, '') AS cor_raca,
+    NULLIF(deduplicated_rows.renda_familiar, '') AS renda_familiar,
+    NULLIF(deduplicated_rows.faixa_etaria, '') AS faixa_etaria,
+    NULLIF(deduplicated_rows.situacao_de_matricula, '') AS situacao_matricula,
+    NULLIF(deduplicated_rows.modalidade_de_ensino, '') AS modalidade_ensino,
+    NULLIF(deduplicated_rows.tipo_de_curso, '') AS tipo_curso,
+    NULLIF(deduplicated_rows.tipo_de_oferta, '') AS tipo_oferta,
+    NULLIF(deduplicated_rows.turno, '') AS turno,
+    NULLIF(deduplicated_rows.eixo_tecnologico, '') AS eixo_tecnologico,
+    NULLIF(deduplicated_rows.subeixo_tecnologico, '') AS subeixo_tecnologico,
+    NULLIF(deduplicated_rows.nome_de_curso, '') AS nome_curso,
+    CASE
+        WHEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.total_de_inscritos), ''), '.', ''), ',', '.') ~ '^-?[0-9]+(\.[0-9]+)?$'
+            THEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.total_de_inscritos), ''), '.', ''), ',', '.')::NUMERIC
+        ELSE NULL
+    END AS total_inscritos,
+    CASE
+        WHEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.vagas_ofertadas), ''), '.', ''), ',', '.') ~ '^-?[0-9]+(\.[0-9]+)?$'
+            THEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.vagas_ofertadas), ''), '.', ''), ',', '.')::NUMERIC
+        ELSE NULL
+    END AS vagas_ofertadas,
+    NOW()
+FROM deduplicated_rows
+ON CONFLICT (raw_record_id) DO UPDATE
+SET
+    run_id = EXCLUDED.run_id,
+    instance_key = EXCLUDED.instance_key,
+    ano = EXCLUDED.ano,
+    instituicao = EXCLUDED.instituicao,
+    regiao = EXCLUDED.regiao,
+    uf = EXCLUDED.uf,
+    municipio = EXCLUDED.municipio,
+    sexo = EXCLUDED.sexo,
+    cor_raca = EXCLUDED.cor_raca,
+    renda_familiar = EXCLUDED.renda_familiar,
+    faixa_etaria = EXCLUDED.faixa_etaria,
+    situacao_matricula = EXCLUDED.situacao_matricula,
+    modalidade_ensino = EXCLUDED.modalidade_ensino,
+    tipo_curso = EXCLUDED.tipo_curso,
+    tipo_oferta = EXCLUDED.tipo_oferta,
+    turno = EXCLUDED.turno,
+    eixo_tecnologico = EXCLUDED.eixo_tecnologico,
+    subeixo_tecnologico = EXCLUDED.subeixo_tecnologico,
+    nome_curso = EXCLUDED.nome_curso,
+    total_inscritos = EXCLUDED.total_inscritos,
+    vagas_ofertadas = EXCLUDED.vagas_ofertadas,
+    processed_at = NOW();