@dataif/cli 0.1.0

package/templates/dataif/sql/migrations/002_pnp_phase2_admin_config_backfill.sql

@@ -0,0 +1,184 @@
+WITH explicit_connections AS (
+  SELECT DISTINCT ON (connection_key)
+    COALESCE(NULLIF(request_params->>'connection_key', ''), REPLACE(endpoint_key, '__connection', '')) AS connection_key,
+    COALESCE(
+      NULLIF(request_params->>'connection_name', ''),
+      NULLIF(description, ''),
+      COALESCE(NULLIF(request_params->>'connection_key', ''), REPLACE(endpoint_key, '__connection', ''))
+    ) AS connection_name,
+    COALESCE(NULLIF(page_url, ''), NULLIF(api_endpoint_url, ''), NULLIF(csv_url, '')) AS page_url,
+    is_active,
+    jsonb_build_object(
+      'legacy_source', 'config.connector_endpoints',
+      'legacy_endpoint_key', endpoint_key,
+      'legacy_description', description,
+      'legacy_request_params', request_params
+    ) AS metadata,
+    created_at,
+    updated_at,
+    CASE
+      WHEN COALESCE(request_params->>'deleted', 'false') = 'true'
+        THEN COALESCE(NULLIF(request_params->>'deleted_at', '')::timestamptz, updated_at)
+      ELSE NULL
+    END AS deleted_at
+  FROM config.connector_endpoints
+  WHERE connector_id = 'nilo_pecanha'
+    AND request_params->>'mode' = 'powerbi_microdados'
+    AND COALESCE(request_params->>'entity_type', 'pipeline') = 'connection'
+    AND COALESCE(NULLIF(request_params->>'connection_key', ''), REPLACE(endpoint_key, '__connection', '')) IS NOT NULL
+  ORDER BY connection_key, updated_at DESC, id DESC
+)
+INSERT INTO raw.pnp_connections (
+  connection_key,
+  connection_name,
+  page_url,
+  is_active,
+  metadata,
+  created_at,
+  updated_at,
+  deleted_at
+)
+SELECT
+  connection_key,
+  connection_name,
+  page_url,
+  is_active,
+  metadata,
+  created_at,
+  updated_at,
+  deleted_at
+FROM explicit_connections
+WHERE connection_key IS NOT NULL
+  AND connection_name IS NOT NULL
+  AND page_url IS NOT NULL
+ON CONFLICT (connection_key) DO UPDATE
+SET
+  connection_name = EXCLUDED.connection_name,
+  page_url = EXCLUDED.page_url,
+  is_active = EXCLUDED.is_active,
+  metadata = EXCLUDED.metadata,
+  updated_at = EXCLUDED.updated_at,
+  deleted_at = EXCLUDED.deleted_at;
+
+WITH instances_needing_connection AS (
+  SELECT DISTINCT
+    page_url
+  FROM raw.pnp_instances
+  WHERE page_url IS NOT NULL
+    AND (
+      COALESCE(connection_key, '') = ''
+      OR NOT EXISTS (
+        SELECT 1
+        FROM raw.pnp_connections c
+        WHERE c.connection_key = raw.pnp_instances.connection_key
+      )
+    )
+),
+fallback_connections AS (
+  SELECT
+    CASE
+      WHEN ROW_NUMBER() OVER (ORDER BY page_url) = 1 THEN 'pnp_conn_principal'
+      ELSE format('pnp_conn_principal_%s', lpad(ROW_NUMBER() OVER (ORDER BY page_url)::text, 2, '0'))
+    END AS connection_key,
+    CASE
+      WHEN ROW_NUMBER() OVER (ORDER BY page_url) = 1 THEN 'PNP Principal'
+      ELSE format('PNP Principal %s', ROW_NUMBER() OVER (ORDER BY page_url))
+    END AS connection_name,
+    page_url
+  FROM instances_needing_connection
+)
+INSERT INTO raw.pnp_connections (
+  connection_key,
+  connection_name,
+  page_url,
+  is_active,
+  metadata
+)
+SELECT
+  connection_key,
+  connection_name,
+  page_url,
+  TRUE,
+  jsonb_build_object(
+    'backfill_source', 'raw.pnp_instances.page_url',
+    'synthetic', TRUE
+  )
+FROM fallback_connections
+ON CONFLICT (connection_key) DO UPDATE
+SET
+  connection_name = EXCLUDED.connection_name,
+  page_url = EXCLUDED.page_url,
+  metadata = raw.pnp_connections.metadata || EXCLUDED.metadata;
+
+WITH resolved_connections AS (
+  SELECT
+    i.instance_key,
+    COALESCE(i.connection_key, fallback.connection_key) AS connection_key,
+    COALESCE(NULLIF(i.connection_name, ''), explicit.connection_name, fallback.connection_name) AS connection_name
+  FROM raw.pnp_instances i
+  LEFT JOIN raw.pnp_connections explicit
+    ON explicit.connection_key = i.connection_key
+  LEFT JOIN raw.pnp_connections fallback
+    ON fallback.page_url = i.page_url
+   AND COALESCE(i.connection_key, '') = ''
+)
+UPDATE raw.pnp_instances i
+SET
+  connection_key = resolved.connection_key,
+  connection_name = resolved.connection_name,
+  updated_at = NOW()
+FROM resolved_connections resolved
+WHERE i.instance_key = resolved.instance_key
+  AND resolved.connection_key IS NOT NULL
+  AND (
+    COALESCE(i.connection_key, '') <> resolved.connection_key
+    OR COALESCE(i.connection_name, '') <> COALESCE(resolved.connection_name, '')
+  );
+
+WITH connection_status AS (
+  SELECT
+    c.connection_key,
+    BOOL_OR(i.deleted_at IS NULL AND i.is_active = TRUE) AS has_active_instances,
+    MAX(i.deleted_at) AS latest_instance_deleted_at
+  FROM raw.pnp_connections c
+  LEFT JOIN raw.pnp_instances i
+    ON i.connection_key = c.connection_key
+  GROUP BY c.connection_key
+)
+UPDATE raw.pnp_connections c
+SET
+  is_active = COALESCE(status.has_active_instances, c.is_active),
+  deleted_at = CASE
+    WHEN COALESCE(status.has_active_instances, FALSE) THEN NULL
+    ELSE COALESCE(c.deleted_at, status.latest_instance_deleted_at)
+  END,
+  updated_at = NOW()
+FROM connection_status status
+WHERE c.connection_key = status.connection_key;
+
+WITH instance_connection AS (
+  SELECT
+    instance_key,
+    connection_key,
+    connection_name
+  FROM raw.pnp_instances
+  WHERE connection_key IS NOT NULL
+)
+UPDATE config.connector_endpoints e
+SET
+  request_params = jsonb_set(
+    jsonb_set(COALESCE(e.request_params, '{}'::jsonb), '{connection_key}', to_jsonb(ic.connection_key), TRUE),
+    '{connection_name}',
+    to_jsonb(ic.connection_name),
+    TRUE
+  ),
+  updated_at = NOW()
+FROM instance_connection ic
+WHERE e.connector_id = 'nilo_pecanha'
+  AND e.request_params->>'mode' = 'powerbi_microdados'
+  AND COALESCE(e.request_params->>'entity_type', 'pipeline') <> 'connection'
+  AND COALESCE(NULLIF(e.request_params->>'instance_key', ''), NULLIF(e.request_params->>'pipeline_key', '')) = ic.instance_key
+  AND (
+    COALESCE(e.request_params->>'connection_key', '') <> ic.connection_key
+    OR COALESCE(e.request_params->>'connection_name', '') <> ic.connection_name
+  );

package/templates/dataif/sql/migrations/003_pnp_phase3_raw_tabular_backfill.sql

@@ -0,0 +1,3 @@
+-- Archived legacy migration.
+-- The final PNP contract materializes directly from raw domain tables into staging
+-- domain tables and does not retain the legacy raw compatibility backfill flow.

package/templates/dataif/sql/migrations/004_pnp_phase3_raw_backfill_support_index.sql

@@ -0,0 +1,3 @@
+-- Archived legacy migration.
+-- Support indexes for the removed compatibility backfill are no longer part of the
+-- operational PNP schema.

package/templates/dataif/sql/migrations/005_pnp_phase7_staging_support_indexes.sql

@@ -0,0 +1,2 @@
+-- Archived legacy migration.
+-- Staging support indexes for the removed intermediate base table are no longer used.

package/templates/dataif/sql/migrations/006_pnp_phase7_staging_autovacuum_tuning.sql

@@ -0,0 +1,2 @@
+-- Archived legacy migration.
+-- Autovacuum tuning for the removed intermediate base table is no longer used.

package/templates/dataif/sql/migrations/007_pnp_phase7b_run_packages.sql

@@ -0,0 +1,20 @@
+CREATE TABLE IF NOT EXISTS raw.pnp_run_packages (
+  package_id BIGSERIAL PRIMARY KEY,
+  run_id TEXT NOT NULL REFERENCES raw.pnp_runs(run_id) ON DELETE CASCADE,
+  instance_key TEXT,
+  airflow_dag_id TEXT,
+  airflow_dag_run_id TEXT,
+  airflow_task_id TEXT NOT NULL,
+  package_type TEXT NOT NULL,
+  package_name TEXT NOT NULL,
+  package_status TEXT NOT NULL,
+  records_affected BIGINT,
+  payload_json JSONB NOT NULL DEFAULT '{}'::jsonb,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_raw_pnp_run_packages_run
+  ON raw.pnp_run_packages (run_id, created_at DESC);
+
+CREATE INDEX IF NOT EXISTS idx_raw_pnp_run_packages_task
+  ON raw.pnp_run_packages (airflow_task_id, created_at DESC);

package/templates/dataif/sql/migrations/008_pnp_phase7a_pipeline_endpoints.sql

@@ -0,0 +1,169 @@
+CREATE TABLE IF NOT EXISTS raw.pnp_endpoint_tables (
+  endpoint_key TEXT PRIMARY KEY,
+  endpoint_name TEXT NOT NULL,
+  tipo_microdados TEXT NOT NULL UNIQUE,
+  raw_table_schema TEXT NOT NULL DEFAULT 'raw',
+  raw_table_name TEXT NOT NULL,
+  staging_table_schema TEXT NOT NULL DEFAULT 'staging',
+  staging_table_name TEXT,
+  curated_relation_schema TEXT NOT NULL DEFAULT 'curated',
+  curated_relation_name TEXT,
+  is_active BOOLEAN NOT NULL DEFAULT TRUE,
+  metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  CONSTRAINT uq_raw_pnp_endpoint_tables_raw_table UNIQUE (raw_table_schema, raw_table_name)
+);
+
+DROP TRIGGER IF EXISTS trg_pnp_endpoint_tables_updated_at ON raw.pnp_endpoint_tables;
+CREATE TRIGGER trg_pnp_endpoint_tables_updated_at
+BEFORE UPDATE ON raw.pnp_endpoint_tables
+FOR EACH ROW
+EXECUTE FUNCTION raw.touch_updated_at();
+
+CREATE INDEX IF NOT EXISTS idx_raw_pnp_endpoint_tables_active
+  ON raw.pnp_endpoint_tables (is_active, endpoint_name);
+
+CREATE TABLE IF NOT EXISTS raw.pnp_pipeline_endpoints (
+  pipeline_endpoint_id BIGSERIAL PRIMARY KEY,
+  instance_key TEXT NOT NULL REFERENCES raw.pnp_instances(instance_key) ON DELETE CASCADE,
+  connection_key TEXT,
+  endpoint_key TEXT NOT NULL REFERENCES raw.pnp_endpoint_tables(endpoint_key),
+  selection_source TEXT,
+  is_active BOOLEAN NOT NULL DEFAULT TRUE,
+  metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  CONSTRAINT uq_raw_pnp_pipeline_endpoints UNIQUE (instance_key, endpoint_key)
+);
+
+DROP TRIGGER IF EXISTS trg_pnp_pipeline_endpoints_updated_at ON raw.pnp_pipeline_endpoints;
+CREATE TRIGGER trg_pnp_pipeline_endpoints_updated_at
+BEFORE UPDATE ON raw.pnp_pipeline_endpoints
+FOR EACH ROW
+EXECUTE FUNCTION raw.touch_updated_at();
+
+CREATE INDEX IF NOT EXISTS idx_raw_pnp_pipeline_endpoints_instance
+  ON raw.pnp_pipeline_endpoints (instance_key, is_active, endpoint_key);
+
+CREATE INDEX IF NOT EXISTS idx_raw_pnp_pipeline_endpoints_connection
+  ON raw.pnp_pipeline_endpoints (connection_key, is_active, instance_key);
+
+INSERT INTO raw.pnp_endpoint_tables (
+  endpoint_key,
+  endpoint_name,
+  tipo_microdados,
+  raw_table_schema,
+  raw_table_name,
+  staging_table_schema,
+  staging_table_name,
+  curated_relation_schema,
+  curated_relation_name,
+  metadata
+)
+VALUES
+  (
+    'matriculas',
+    'Matrículas',
+    'Matrículas',
+    'raw',
+    'pnp_matriculas_src',
+    'staging',
+    'pnp_matriculas',
+    'curated',
+    NULL,
+    jsonb_build_object('domain_key', 'matriculas')
+  ),
+  (
+    'eficiencia_academica',
+    'Eficiência Acadêmica',
+    'Eficiência Acadêmica',
+    'raw',
+    'pnp_eficiencia_academica_src',
+    'staging',
+    'pnp_eficiencia_academica',
+    'curated',
+    NULL,
+    jsonb_build_object('domain_key', 'eficiencia_academica')
+  ),
+  (
+    'servidores',
+    'Servidores',
+    'Servidores',
+    'raw',
+    'pnp_servidores_src',
+    'staging',
+    'pnp_servidores',
+    'curated',
+    NULL,
+    jsonb_build_object('domain_key', 'servidores')
+  ),
+  (
+    'financeiro',
+    'Financeiro',
+    'Financeiro',
+    'raw',
+    'pnp_financeiro_src',
+    'staging',
+    'pnp_financeiro',
+    'curated',
+    NULL,
+    jsonb_build_object('domain_key', 'financeiro')
+  )
+ON CONFLICT (endpoint_key) DO UPDATE
+SET
+  endpoint_name = EXCLUDED.endpoint_name,
+  tipo_microdados = EXCLUDED.tipo_microdados,
+  raw_table_schema = EXCLUDED.raw_table_schema,
+  raw_table_name = EXCLUDED.raw_table_name,
+  staging_table_schema = EXCLUDED.staging_table_schema,
+  staging_table_name = EXCLUDED.staging_table_name,
+  curated_relation_schema = EXCLUDED.curated_relation_schema,
+  curated_relation_name = EXCLUDED.curated_relation_name,
+  is_active = TRUE,
+  metadata = EXCLUDED.metadata,
+  updated_at = NOW();
+
+INSERT INTO raw.pnp_pipeline_endpoints (
+  instance_key,
+  connection_key,
+  endpoint_key,
+  selection_source,
+  is_active,
+  metadata
+)
+SELECT
+  i.instance_key,
+  i.connection_key,
+  et.endpoint_key,
+  'phase7a_backfill',
+  COALESCE(i.is_active, TRUE) AND BOOL_OR(COALESCE(s.is_active, TRUE)),
+  jsonb_build_object(
+    'tipo_microdados', s.tipo_microdados,
+    'raw_table', format('%s.%s', et.raw_table_schema, et.raw_table_name),
+    'staging_table', CASE
+      WHEN et.staging_table_name IS NULL THEN NULL
+      ELSE format('%s.%s', et.staging_table_schema, et.staging_table_name)
+    END
+  )
+FROM raw.pnp_instances i
+JOIN raw.pnp_instance_selection s
+  ON s.instance_key = i.instance_key
+JOIN raw.pnp_endpoint_tables et
+  ON et.tipo_microdados = s.tipo_microdados
+GROUP BY
+  i.instance_key,
+  i.connection_key,
+  et.endpoint_key,
+  s.tipo_microdados,
+  et.raw_table_schema,
+  et.raw_table_name,
+  et.staging_table_schema,
+  et.staging_table_name
+ON CONFLICT (instance_key, endpoint_key) DO UPDATE
+SET
+  connection_key = EXCLUDED.connection_key,
+  selection_source = EXCLUDED.selection_source,
+  is_active = EXCLUDED.is_active,
+  metadata = EXCLUDED.metadata,
+  updated_at = NOW();

package/templates/dataif/sql/migrations/009_pnp_phase8_curated.sql

@@ -0,0 +1,35 @@
+\ir ../views_curated/010_vw_pnp_admin_ingestao.sql
+\ir ../views_curated/020_vw_pnp_qualidade_dados.sql
+\ir ../views_curated/030_vw_pnp_matriculas.sql
+\ir ../views_curated/040_vw_pnp_eficiencia.sql
+\ir ../views_curated/050_vw_pnp_servidores.sql
+\ir ../views_curated/060_vw_pnp_financeiro.sql
+\ir ../views_curated/070_vw_pnp_vanna.sql
+\ir ../views_curated/004_mv_pnp_dashboard_fast.sql
+
+UPDATE raw.pnp_endpoint_tables
+SET
+  curated_relation_schema = 'curated',
+  curated_relation_name = CASE endpoint_key
+    WHEN 'matriculas' THEN 'vw_pnp_matriculas_perfil'
+    WHEN 'eficiencia_academica' THEN 'vw_pnp_eficiencia_situacao'
+    WHEN 'servidores' THEN 'vw_pnp_servidores_quadro'
+    WHEN 'financeiro' THEN 'vw_pnp_financeiro_execucao'
+    ELSE curated_relation_name
+  END,
+  metadata = jsonb_set(
+    COALESCE(metadata, '{}'::jsonb),
+    '{curated_relation}',
+    to_jsonb(
+      CASE endpoint_key
+        WHEN 'matriculas' THEN 'curated.vw_pnp_matriculas_perfil'
+        WHEN 'eficiencia_academica' THEN 'curated.vw_pnp_eficiencia_situacao'
+        WHEN 'servidores' THEN 'curated.vw_pnp_servidores_quadro'
+        WHEN 'financeiro' THEN 'curated.vw_pnp_financeiro_execucao'
+        ELSE NULL
+      END
+    ),
+    true
+  ),
+  updated_at = NOW()
+WHERE endpoint_key IN ('matriculas', 'eficiencia_academica', 'servidores', 'financeiro');

package/templates/dataif/sql/migrations/010_pnp_phase10_staging_incremental_upsert.sql

@@ -0,0 +1,3 @@
+-- Archived legacy migration.
+-- Incremental upsert maintenance for the removed intermediate staging base table is
+-- no longer part of the final PNP contract.

package/templates/dataif/sql/migrations/010_pnp_pipeline_uuid.sql

@@ -0,0 +1,51 @@
+CREATE EXTENSION IF NOT EXISTS pgcrypto;
+
+ALTER TABLE raw.pnp_instances
+  ADD COLUMN IF NOT EXISTS pipeline_id UUID;
+
+ALTER TABLE raw.pnp_instances
+  ALTER COLUMN pipeline_id SET DEFAULT gen_random_uuid();
+
+UPDATE raw.pnp_instances
+SET pipeline_id = gen_random_uuid()
+WHERE pipeline_id IS NULL;
+
+ALTER TABLE raw.pnp_instances
+  ALTER COLUMN pipeline_id SET NOT NULL;
+
+DO $$
+BEGIN
+  IF NOT EXISTS (
+    SELECT 1
+    FROM pg_constraint
+    WHERE conname = 'uq_raw_pnp_instances_pipeline_id'
+      AND conrelid = 'raw.pnp_instances'::regclass
+  ) THEN
+    ALTER TABLE raw.pnp_instances
+      ADD CONSTRAINT uq_raw_pnp_instances_pipeline_id UNIQUE (pipeline_id);
+  END IF;
+END $$;
+
+ALTER TABLE raw.pnp_instance_selection
+  ADD COLUMN IF NOT EXISTS pipeline_id UUID;
+
+UPDATE raw.pnp_instance_selection selection
+SET pipeline_id = instances.pipeline_id
+FROM raw.pnp_instances instances
+WHERE instances.instance_key = selection.instance_key
+  AND (selection.pipeline_id IS NULL OR selection.pipeline_id <> instances.pipeline_id);
+
+ALTER TABLE raw.pnp_pipeline_endpoints
+  ADD COLUMN IF NOT EXISTS pipeline_id UUID;
+
+UPDATE raw.pnp_pipeline_endpoints pipeline_endpoints
+SET pipeline_id = instances.pipeline_id
+FROM raw.pnp_instances instances
+WHERE instances.instance_key = pipeline_endpoints.instance_key
+  AND (pipeline_endpoints.pipeline_id IS NULL OR pipeline_endpoints.pipeline_id <> instances.pipeline_id);
+
+CREATE INDEX IF NOT EXISTS idx_raw_pnp_instance_selection_pipeline
+  ON raw.pnp_instance_selection (pipeline_id, is_active, ano_base, tipo_microdados);
+
+CREATE INDEX IF NOT EXISTS idx_raw_pnp_pipeline_endpoints_pipeline
+  ON raw.pnp_pipeline_endpoints (pipeline_id, is_active, endpoint_key);

package/templates/dataif/sql/migrations/011_app_settings.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS config.app_settings (
+  setting_key TEXT PRIMARY KEY,
+  setting_value JSONB NOT NULL,
+  metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

package/templates/dataif/sql/staging/020_pnp_matriculas.sql

@@ -0,0 +1,112 @@
+WITH selected_rows AS (
+    SELECT
+        src.*,
+        ROW_NUMBER() OVER (
+            PARTITION BY src.instance_key, src.record_hash
+            ORDER BY src.raw_record_id DESC
+        ) AS dedup_rank
+    FROM raw.pnp_matriculas_src src
+    LEFT JOIN raw.pnp_downloads downloads ON downloads.download_id = src.download_id
+    JOIN raw.pnp_instance_selection selection
+      ON selection.instance_key = src.instance_key
+     AND selection.is_active = TRUE
+     AND selection.ano_base = src.ano_base
+     AND selection.tipo_microdados = src.tipo_microdados
+     AND (
+        selection.configured_microdados_url IS NULL
+        OR selection.configured_microdados_url = downloads.microdados_url
+     )
+    WHERE src.run_id = %(run_id)s
+      AND src.instance_key IS NOT DISTINCT FROM %(instance_key)s
+      AND src.download_id = %(download_id)s
+),
+deduplicated_rows AS (
+    SELECT *
+    FROM selected_rows
+    WHERE dedup_rank = 1
+)
+INSERT INTO staging.pnp_matriculas (
+    raw_record_id,
+    run_id,
+    instance_key,
+    ano,
+    instituicao,
+    regiao,
+    uf,
+    municipio,
+    sexo,
+    cor_raca,
+    renda_familiar,
+    faixa_etaria,
+    situacao_matricula,
+    modalidade_ensino,
+    tipo_curso,
+    tipo_oferta,
+    turno,
+    eixo_tecnologico,
+    subeixo_tecnologico,
+    nome_curso,
+    total_inscritos,
+    vagas_ofertadas,
+    processed_at
+)
+SELECT
+    deduplicated_rows.raw_record_id,
+    deduplicated_rows.run_id,
+    deduplicated_rows.instance_key,
+    CASE
+        WHEN NULLIF(deduplicated_rows.ano, '') ~ '^[0-9]{1,4}$' THEN deduplicated_rows.ano::INTEGER
+        ELSE NULL
+    END AS ano,
+    NULLIF(deduplicated_rows.instituicao, '') AS instituicao,
+    NULLIF(deduplicated_rows.regiao, '') AS regiao,
+    NULLIF(deduplicated_rows.uf, '') AS uf,
+    NULLIF(deduplicated_rows.municipio, '') AS municipio,
+    NULLIF(deduplicated_rows.sexo, '') AS sexo,
+    NULLIF(deduplicated_rows.cor_raca, '') AS cor_raca,
+    NULLIF(deduplicated_rows.renda_familiar, '') AS renda_familiar,
+    NULLIF(deduplicated_rows.faixa_etaria, '') AS faixa_etaria,
+    NULLIF(deduplicated_rows.situacao_de_matricula, '') AS situacao_matricula,
+    NULLIF(deduplicated_rows.modalidade_de_ensino, '') AS modalidade_ensino,
+    NULLIF(deduplicated_rows.tipo_de_curso, '') AS tipo_curso,
+    NULLIF(deduplicated_rows.tipo_de_oferta, '') AS tipo_oferta,
+    NULLIF(deduplicated_rows.turno, '') AS turno,
+    NULLIF(deduplicated_rows.eixo_tecnologico, '') AS eixo_tecnologico,
+    NULLIF(deduplicated_rows.subeixo_tecnologico, '') AS subeixo_tecnologico,
+    NULLIF(deduplicated_rows.nome_de_curso, '') AS nome_curso,
+    CASE
+        WHEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.total_de_inscritos), ''), '.', ''), ',', '.') ~ '^-?[0-9]+(\.[0-9]+)?$'
+            THEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.total_de_inscritos), ''), '.', ''), ',', '.')::NUMERIC
+        ELSE NULL
+    END AS total_inscritos,
+    CASE
+        WHEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.vagas_ofertadas), ''), '.', ''), ',', '.') ~ '^-?[0-9]+(\.[0-9]+)?$'
+            THEN REPLACE(REPLACE(NULLIF(BTRIM(deduplicated_rows.vagas_ofertadas), ''), '.', ''), ',', '.')::NUMERIC
+        ELSE NULL
+    END AS vagas_ofertadas,
+    NOW()
+FROM deduplicated_rows
+ON CONFLICT (raw_record_id) DO UPDATE
+SET
+    run_id = EXCLUDED.run_id,
+    instance_key = EXCLUDED.instance_key,
+    ano = EXCLUDED.ano,
+    instituicao = EXCLUDED.instituicao,
+    regiao = EXCLUDED.regiao,
+    uf = EXCLUDED.uf,
+    municipio = EXCLUDED.municipio,
+    sexo = EXCLUDED.sexo,
+    cor_raca = EXCLUDED.cor_raca,
+    renda_familiar = EXCLUDED.renda_familiar,
+    faixa_etaria = EXCLUDED.faixa_etaria,
+    situacao_matricula = EXCLUDED.situacao_matricula,
+    modalidade_ensino = EXCLUDED.modalidade_ensino,
+    tipo_curso = EXCLUDED.tipo_curso,
+    tipo_oferta = EXCLUDED.tipo_oferta,
+    turno = EXCLUDED.turno,
+    eixo_tecnologico = EXCLUDED.eixo_tecnologico,
+    subeixo_tecnologico = EXCLUDED.subeixo_tecnologico,
+    nome_curso = EXCLUDED.nome_curso,
+    total_inscritos = EXCLUDED.total_inscritos,
+    vagas_ofertadas = EXCLUDED.vagas_ofertadas,
+    processed_at = NOW();

package/templates/dataif/sql/staging/030_pnp_eficiencia_academica.sql

@@ -0,0 +1,83 @@
+WITH selected_rows AS (
+    SELECT
+        src.*,
+        ROW_NUMBER() OVER (
+            PARTITION BY src.instance_key, src.record_hash
+            ORDER BY src.raw_record_id DESC
+        ) AS dedup_rank
+    FROM raw.pnp_eficiencia_academica_src src
+    LEFT JOIN raw.pnp_downloads downloads ON downloads.download_id = src.download_id
+    JOIN raw.pnp_instance_selection selection
+      ON selection.instance_key = src.instance_key
+     AND selection.is_active = TRUE
+     AND selection.ano_base = src.ano_base
+     AND selection.tipo_microdados = src.tipo_microdados
+     AND (
+        selection.configured_microdados_url IS NULL
+        OR selection.configured_microdados_url = downloads.microdados_url
+     )
+    WHERE src.run_id = %(run_id)s
+      AND src.instance_key IS NOT DISTINCT FROM %(instance_key)s
+      AND src.download_id = %(download_id)s
+),
+deduplicated_rows AS (
+    SELECT *
+    FROM selected_rows
+    WHERE dedup_rank = 1
+)
+INSERT INTO staging.pnp_eficiencia_academica (
+    raw_record_id,
+    run_id,
+    instance_key,
+    ano,
+    instituicao,
+    regiao,
+    uf,
+    municipio,
+    sexo,
+    cor_raca,
+    renda_familiar,
+    faixa_etaria,
+    categoria_situacao,
+    situacao_matricula,
+    matricula_atendida,
+    processed_at
+)
+SELECT
+    deduplicated_rows.raw_record_id,
+    deduplicated_rows.run_id,
+    deduplicated_rows.instance_key,
+    CASE
+        WHEN NULLIF(deduplicated_rows.ano, '') ~ '^[0-9]{1,4}$' THEN deduplicated_rows.ano::INTEGER
+        ELSE NULL
+    END AS ano,
+    NULLIF(deduplicated_rows.instituicao, '') AS instituicao,
+    NULLIF(deduplicated_rows.regiao, '') AS regiao,
+    NULLIF(deduplicated_rows.uf, '') AS uf,
+    NULLIF(deduplicated_rows.municipio, '') AS municipio,
+    NULLIF(deduplicated_rows.sexo, '') AS sexo,
+    NULLIF(deduplicated_rows.cor_raca, '') AS cor_raca,
+    NULLIF(deduplicated_rows.renda_familiar, '') AS renda_familiar,
+    NULLIF(deduplicated_rows.faixa_etaria, '') AS faixa_etaria,
+    NULLIF(deduplicated_rows.categoria_da_situacao, '') AS categoria_situacao,
+    NULLIF(deduplicated_rows.situacao_de_matricula, '') AS situacao_matricula,
+    NULLIF(deduplicated_rows.matricula_atendida, '') AS matricula_atendida,
+    NOW()
+FROM deduplicated_rows
+ON CONFLICT (raw_record_id) DO UPDATE
+SET
+    run_id = EXCLUDED.run_id,
+    instance_key = EXCLUDED.instance_key,
+    ano = EXCLUDED.ano,
+    instituicao = EXCLUDED.instituicao,
+    regiao = EXCLUDED.regiao,
+    uf = EXCLUDED.uf,
+    municipio = EXCLUDED.municipio,
+    sexo = EXCLUDED.sexo,
+    cor_raca = EXCLUDED.cor_raca,
+    renda_familiar = EXCLUDED.renda_familiar,
+    faixa_etaria = EXCLUDED.faixa_etaria,
+    categoria_situacao = EXCLUDED.categoria_situacao,
+    situacao_matricula = EXCLUDED.situacao_matricula,
+    matricula_atendida = EXCLUDED.matricula_atendida,
+    processed_at = NOW();