@dataif/cli 0.1.0

package/templates/dataif/infra/keycloak/realm-dataif.json

@@ -0,0 +1,73 @@
+{
+  "realm": "dataif",
+  "enabled": true,
+  "displayName": "dataif",
+  "roles": {
+    "realm": [
+      {
+        "name": "admin"
+      },
+      {
+        "name": "viewer"
+      }
+    ]
+  },
+  "users": [
+    {
+      "username": "dataif-admin",
+      "enabled": true,
+      "emailVerified": true,
+      "firstName": "Dataif",
+      "lastName": "Admin",
+      "email": "admin@dataif.local",
+      "credentials": [
+        {
+          "type": "password",
+          "value": "admin",
+          "temporary": false
+        }
+      ],
+      "realmRoles": [
+        "admin"
+      ]
+    }
+  ],
+  "clients": [
+    {
+      "clientId": "dataif-web",
+      "enabled": true,
+      "publicClient": true,
+      "standardFlowEnabled": true,
+      "directAccessGrantsEnabled": true,
+      "redirectUris": [
+        "http://localhost:5173/*",
+        "http://localhost:5174/*"
+      ],
+      "webOrigins": [
+        "http://localhost:5173",
+        "http://localhost:5174"
+      ],
+      "protocolMappers": [
+        {
+          "name": "aud-dataif-api",
+          "protocol": "openid-connect",
+          "protocolMapper": "oidc-audience-mapper",
+          "consentRequired": false,
+          "config": {
+            "included.client.audience": "dataif-api",
+            "id.token.claim": "false",
+            "access.token.claim": "true"
+          }
+        }
+      ]
+    },
+    {
+      "clientId": "dataif-api",
+      "enabled": true,
+      "publicClient": false,
+      "serviceAccountsEnabled": true,
+      "secret": "dataif-api-secret",
+      "protocol": "openid-connect"
+    }
+  ]
+}

package/templates/dataif/infra/ollama/Dockerfile

@@ -0,0 +1,9 @@
+FROM curlimages/curl:8.11.1
+
+USER root
+WORKDIR /bootstrap
+
+COPY bootstrap-model.sh sabia-7b.Modelfile ./
+RUN chmod +x /bootstrap/bootstrap-model.sh
+
+ENTRYPOINT ["/bootstrap/bootstrap-model.sh"]

package/templates/dataif/infra/ollama/bootstrap-model.sh

@@ -0,0 +1,100 @@
+#!/bin/sh
+set -eu
+
+enabled="${OLLAMA_MODEL_BOOTSTRAP_ENABLED:-true}"
+if [ "$enabled" = "false" ] || [ "$enabled" = "0" ] || [ "$enabled" = "no" ]; then
+  echo "Ollama model bootstrap disabled."
+  exit 0
+fi
+
+ollama_base_url="${OLLAMA_BASE_URL:-http://ollama:11434}"
+model_name="${OLLAMA_MODEL_NAME:-${VANNA_OLLAMA_MODEL:-sabia-7b}}"
+gguf_url="${OLLAMA_MODEL_GGUF_URL:-}"
+gguf_file="${OLLAMA_MODEL_GGUF_FILE:-sabia-7b.Q4_K_M.gguf}"
+model_dir="/models/${model_name}"
+model_path="${model_dir}/${gguf_file}"
+modelfile_template="${OLLAMA_MODELFILE_TEMPLATE:-/bootstrap/sabia-7b.Modelfile}"
+
+if [ -z "$model_name" ]; then
+  echo "OLLAMA_MODEL_NAME or VANNA_OLLAMA_MODEL must be set." >&2
+  exit 2
+fi
+
+wait_for_ollama() {
+  tries="${OLLAMA_BOOTSTRAP_WAIT_RETRIES:-60}"
+  delay="${OLLAMA_BOOTSTRAP_WAIT_SECONDS:-2}"
+  i=1
+  while [ "$i" -le "$tries" ]; do
+    if curl -fsS "${ollama_base_url%/}/api/tags" >/tmp/ollama-tags.json; then
+      return 0
+    fi
+    echo "Waiting for Ollama at ${ollama_base_url} (${i}/${tries})..."
+    sleep "$delay"
+    i=$((i + 1))
+  done
+  echo "Ollama did not become reachable at ${ollama_base_url}." >&2
+  exit 1
+}
+
+model_exists() {
+  curl -fsS "${ollama_base_url%/}/api/tags" >/tmp/ollama-tags.json
+  grep -Eq "\"(name|model)\"[[:space:]]*:[[:space:]]*\"${model_name}(:latest)?\"" /tmp/ollama-tags.json
+}
+
+download_gguf() {
+  if [ -f "$model_path" ]; then
+    echo "GGUF already present at ${model_path}."
+    return 0
+  fi
+  if [ -z "$gguf_url" ]; then
+    echo "OLLAMA_MODEL_GGUF_URL is required because ${model_path} is missing." >&2
+    exit 2
+  fi
+
+  mkdir -p "$model_dir"
+  tmp_path="${model_path}.part"
+  echo "Downloading ${model_name} GGUF to ${model_path}..."
+  if [ -n "${HF_TOKEN:-}" ]; then
+    curl -fL --retry 5 --retry-delay 10 -H "Authorization: Bearer ${HF_TOKEN}" -o "$tmp_path" "$gguf_url"
+  else
+    curl -fL --retry 5 --retry-delay 10 -o "$tmp_path" "$gguf_url"
+  fi
+
+  mv "$tmp_path" "$model_path"
+}
+
+json_escape_file() {
+  sed "s#__MODEL_GGUF_PATH__#${model_path}#g" "$modelfile_template" \
+    | sed 's/\\/\\\\/g; s/"/\\"/g' \
+    | awk '{printf "%s\\n", $0}'
+}
+
+create_model() {
+  escaped_modelfile="$(json_escape_file)"
+  payload="/tmp/ollama-create-model.json"
+  printf '{"name":"%s","modelfile":"%s","stream":false}\n' "$model_name" "$escaped_modelfile" >"$payload"
+
+  echo "Creating Ollama model ${model_name} from ${model_path}..."
+  curl -fsS \
+    -H "Content-Type: application/json" \
+    --data-binary "@${payload}" \
+    "${ollama_base_url%/}/api/create"
+  echo
+}
+
+wait_for_ollama
+if model_exists; then
+  echo "Ollama model ${model_name} already exists."
+  exit 0
+fi
+
+download_gguf
+create_model
+
+if model_exists; then
+  echo "Ollama model ${model_name} is ready."
+  exit 0
+fi
+
+echo "Ollama model ${model_name} was not visible after creation." >&2
+exit 1

package/templates/dataif/infra/ollama/sabia-7b.Modelfile

@@ -0,0 +1,14 @@
+FROM __MODEL_GGUF_PATH__
+
+PARAMETER temperature 0.1
+PARAMETER top_p 0.9
+PARAMETER top_k 40
+PARAMETER num_ctx 2048
+PARAMETER repeat_penalty 1.1
+
+SYSTEM """
+Voce gera SQL PostgreSQL para consultas analiticas sobre a camada curated.
+Responda apenas com uma consulta SELECT.
+Use somente tabelas e views explicitamente apresentadas no contexto.
+Nao use DDL, DML, schemas operacionais ou multiplas statements.
+"""

package/templates/dataif/infra/postgres/Dockerfile

@@ -0,0 +1,4 @@
+FROM postgres:16-alpine
+
+COPY infra/init-db /docker-entrypoint-initdb.d
+

package/templates/dataif/pipelines/airflow/dags/generated/.gitkeep

@@ -0,0 +1 @@
+

package/templates/dataif/pipelines/airflow/dags/generated/2020_financeiro_fcc6f1f3_sync.py

@@ -0,0 +1,9 @@
+from dataif_pipelines.airflow.pnp_pipeline_factory import build_pipeline_dag
+
+
+dag = build_pipeline_dag(
+    dag_id='2020_financeiro_fcc6f1f3_sync',
+    pipeline_id='fcc6f1f3-ca8f-480e-bc7a-d48dcc8516ee',
+    instance_key='pnp_pipe_pnp_2020_financeiro',
+    schedule=None,
+)

package/templates/dataif/pipelines/dataif_pipelines/__init__.py

@@ -0,0 +1 @@
+"""Pipelines package for dataif."""

package/templates/dataif/pipelines/dataif_pipelines/airflow/__init__.py

@@ -0,0 +1 @@
+

package/templates/dataif/pipelines/dataif_pipelines/airflow/pnp_pipeline_factory.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from airflow.decorators import dag, task
+from airflow.operators.python import get_current_context
+from airflow.utils.trigger_rule import TriggerRule
+
+from dataif_pipelines.orchestration import pnp_workflow
+
+
+def _context_metadata() -> dict[str, object]:
+    context = get_current_context()
+    dag_run = context["dag_run"]
+    task_instance = context["ti"]
+    logical_date = context.get("logical_date")
+    return {
+        "dag_id": dag_run.dag_id,
+        "dag_run_id": dag_run.run_id,
+        "logical_date": logical_date.isoformat() if logical_date else None,
+        "task_id": task_instance.task_id,
+        "map_index": getattr(task_instance, "map_index", -1),
+    }
+
+
+def _task_states() -> tuple[str, dict[str, str]]:
+    context = get_current_context()
+    dag_run = context["dag_run"]
+    current_task_id = context["ti"].task_id
+    task_states: dict[str, str] = {}
+    for task_instance in dag_run.get_task_instances():
+        if task_instance.task_id == current_task_id:
+            continue
+        task_states[task_instance.task_id] = str(task_instance.state)
+    failed = any(state in {"failed", "upstream_failed"} for state in task_states.values())
+    return ("failed" if failed else "success", task_states)
+
+
+def build_pipeline_dag(*, dag_id: str, pipeline_id: str | None = None, instance_key: str, schedule: str | None):
+    @dag(
+        dag_id=dag_id,
+        start_date=datetime(2025, 1, 1),
+        schedule=schedule,
+        catchup=False,
+        max_active_runs=1,
+        default_args={
+            "owner": "dataif",
+            "depends_on_past": False,
+            "retries": 1,
+            "retry_delay": timedelta(minutes=5),
+        },
+        tags=["dataif", "governo", "nilo_pecanha", "pnp", "pipeline", instance_key, *( [pipeline_id] if pipeline_id else [] )],
+    )
+    def _build():
+        @task
+        def register_run() -> dict[str, object]:
+            context = get_current_context()
+            dag_run = context["dag_run"]
+            conf = getattr(dag_run, "conf", None) or {}
+            metadata = _context_metadata()
+            operation = str(conf.get("operation") or "sync").strip().lower() or "sync"
+            trigger_origin = "manual" if conf else "scheduled"
+            return pnp_workflow.register_pipeline_run(
+                {
+                    "instance_key": instance_key,
+                    "pipeline_id": pipeline_id,
+                    "operation": operation,
+                    "trigger_mode": f"airflow_{trigger_origin}_{operation}",
+                    "requested_by": str(conf.get("requested_by") or f"airflow.{metadata['dag_id']}"),
+                    "dag_id": str(metadata["dag_id"]),
+                    "dag_run_id": str(metadata["dag_run_id"]),
+                    "logical_date": metadata["logical_date"],
+                },
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        @task
+        def load_instance_config(run_ref: dict[str, object]) -> dict[str, object]:
+            metadata = _context_metadata()
+            return pnp_workflow.load_instance_config(
+                run_ref,
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        @task
+        def resolve_powerbi_catalog(run_ref: dict[str, object], instance_config: dict[str, object]) -> dict[str, object]:
+            metadata = _context_metadata()
+            return pnp_workflow.resolve_catalog(
+                run_ref,
+                instance_config,
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        @task.branch
+        def select_execution_path(run_ref: dict[str, object]) -> str:
+            operation = str(run_ref.get("operation") or "sync").strip().lower()
+            if operation == "validate":
+                return "finalize_run"
+            return "extract_raw"
+
+        @task
+        def extract_raw(run_ref: dict[str, object], instance_config: dict[str, object]) -> dict[str, object]:
+            metadata = _context_metadata()
+            return pnp_workflow.sync_raw(
+                run_ref,
+                instance_config,
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        @task
+        def materialize_staging(run_ref: dict[str, object]) -> dict[str, object]:
+            metadata = _context_metadata()
+            return pnp_workflow.materialize_staging(
+                run_ref,
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        @task
+        def build_curated_views(run_ref: dict[str, object]) -> dict[str, object]:
+            metadata = _context_metadata()
+            return pnp_workflow.materialize_curated(
+                run_ref,
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        @task
+        def run_quality_checks(run_ref: dict[str, object]) -> dict[str, object]:
+            metadata = _context_metadata()
+            return pnp_workflow.run_quality_checks(
+                run_ref,
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        @task(trigger_rule=TriggerRule.ALL_DONE)
+        def finalize_run(run_ref: dict[str, object]) -> dict[str, object]:
+            metadata = _context_metadata()
+            dag_status, task_states = _task_states()
+            return pnp_workflow.finalize_run(
+                run_ref,
+                dag_status=dag_status,
+                task_states=task_states,
+                task_id=str(metadata["task_id"]),
+                map_index=int(metadata["map_index"]),
+            )
+
+        run_ref = register_run()
+        instance_config = load_instance_config(run_ref)
+        catalog = resolve_powerbi_catalog(run_ref, instance_config)
+        execution_path = select_execution_path(run_ref)
+        raw = extract_raw(run_ref, instance_config)
+        staging = materialize_staging(run_ref)
+        curated = build_curated_views(run_ref)
+        quality = run_quality_checks(run_ref)
+        final = finalize_run(run_ref)
+
+        run_ref >> instance_config >> catalog >> execution_path
+        execution_path >> raw >> staging >> curated >> quality >> final
+        execution_path >> final
+
+    return _build()

package/templates/dataif/pipelines/dataif_pipelines/connectors/__init__.py

@@ -0,0 +1 @@
+"""Connectors for dataif pipelines."""

package/templates/dataif/pipelines/dataif_pipelines/connectors/base/__init__.py

@@ -0,0 +1 @@
+"""Base connector abstractions."""

package/templates/dataif/pipelines/dataif_pipelines/connectors/base/connector.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterable
+
+from .types import NormalizedRecord, RawRecord, RunContext
+
+
+class BaseConnector(ABC):
+    @abstractmethod
+    def connector_id(self) -> str:
+        raise NotImplementedError
+
+    @abstractmethod
+    def fetch(self, run_context: RunContext) -> list[RawRecord]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def normalize(self, raw_records: list[RawRecord], run_context: RunContext) -> list[NormalizedRecord]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def load_raw(self, normalized_records: list[NormalizedRecord], run_context: RunContext) -> int:
+        raise NotImplementedError
+
+    @abstractmethod
+    def post_load_checks(self, run_id: str) -> dict[str, object]:
+        raise NotImplementedError

package/templates/dataif/pipelines/dataif_pipelines/connectors/base/types.py

@@ -0,0 +1,14 @@
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+
+@dataclass(frozen=True)
+class RunContext:
+    run_id: str
+    started_at: datetime
+    source_url: str
+
+
+RawRecord = dict[str, Any]
+NormalizedRecord = dict[str, Any]

package/templates/dataif/pipelines/dataif_pipelines/connectors/nilo_pecanha/__init__.py

@@ -0,0 +1 @@
+"""Nilo Pecanha connector."""

package/templates/dataif/pipelines/dataif_pipelines/connectors/nilo_pecanha/config.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+from dataif_pipelines.connectors.nilo_pecanha.powerbi_microdados import DEFAULT_POWERBI_MICRODADOS_URL
+
+
+@dataclass(frozen=True)
+class NiloConfig:
+    endpoint: str
+    timeout_seconds: int
+
+
+def load_config() -> NiloConfig:
+    return NiloConfig(
+        endpoint=os.getenv("NILO_PECANHA_ENDPOINT", DEFAULT_POWERBI_MICRODADOS_URL),
+        timeout_seconds=int(os.getenv("NILO_TIMEOUT_SECONDS", "60")),
+    )