javi-forge 0.1.0

package/ai-config/skills/data-ai/duckdb-analytics/SKILL.md

@@ -0,0 +1,440 @@
+---
+name: duckdb-analytics
+description: >
+  DuckDB OLAP analytics with Parquet, S3, and columnar queries for BI/ML.
+  Trigger: duckdb, olap, analytics, parquet, columnar, data warehouse, BI
+
+tools:
+  - Read
+  - Write
+  - Bash
+  - Grep
+
+metadata:
+  author: plataforma-industrial
+  version: "2.0"
+  tags: [duckdb, olap, analytics, parquet, bi]
+  updated: "2026-02"
+---
+
+# DuckDB OLAP Analytics
+
+> Columnar analytics engine for Parquet files, S3 data lakes, and BI workloads.
+
+## Stack
+
+```yaml
+DuckDB: 0.10+
+Go Driver: github.com/marcboeker/go-duckdb
+Python: duckdb 0.10+
+Extensions: httpfs, parquet, postgres
+```
+
+## When to Use
+
+- Analytical queries over historical data
+- Reading Parquet files from S3/MinIO
+- BI reports and aggregations
+- ML feature engineering
+- Time-series analysis on archived data
+
+### Data Architecture
+
+```
+TimescaleDB (OLTP - Hot, <90 days)
+    |
+    | ETL / Export
+    v
+Parquet Files (S3/MinIO - Warm)
+    |
+    | Direct read
+    v
+DuckDB (OLAP - Analytical queries)
+    |
+    v
+BI Tools / Reports / ML
+```
+
+## Python Setup
+
+```python
+import duckdb
+
+class DuckDBAnalytics:
+    def __init__(self, db_path: str = ":memory:"):
+        self.conn = duckdb.connect(db_path)
+        self._setup()
+
+    def _setup(self):
+        self.conn.execute("INSTALL httpfs; LOAD httpfs;")
+        self.conn.execute("INSTALL parquet; LOAD parquet;")
+
+    def configure_s3(self, endpoint: str, access_key: str, secret_key: str):
+        self.conn.execute(f"SET s3_endpoint = '{endpoint}';")
+        self.conn.execute(f"SET s3_access_key_id = '{access_key}';")
+        self.conn.execute(f"SET s3_secret_access_key = '{secret_key}';")
+        self.conn.execute("SET s3_use_ssl = false;")
+
+    def query_df(self, sql: str):
+        return self.conn.execute(sql).fetchdf()
+
+    def query(self, sql: str) -> list[dict]:
+        result = self.conn.execute(sql)
+        columns = [desc[0] for desc in result.description]
+        return [dict(zip(columns, row)) for row in result.fetchall()]
+```
+
+## Go Setup
+
+```go
+package analytics
+
+import (
+    "database/sql"
+    _ "github.com/marcboeker/go-duckdb"
+)
+
+type DuckDB struct {
+    db *sql.DB
+}
+
+func NewDuckDB(path string) (*DuckDB, error) {
+    db, err := sql.Open("duckdb", path)
+    if err != nil {
+        return nil, err
+    }
+    db.Exec("INSTALL parquet; LOAD parquet;")
+    return &DuckDB{db: db}, nil
+}
+```
+
+## Schema for Analytics
+
+### Fact Tables
+
+```sql
+CREATE SCHEMA IF NOT EXISTS analytics;
+
+CREATE TABLE IF NOT EXISTS analytics.sensor_readings (
+    time TIMESTAMP NOT NULL,
+    sensor_id UUID NOT NULL,
+    sensor_name VARCHAR,
+    tenant_id UUID,
+    value DOUBLE,
+    unit VARCHAR,
+    -- Pre-computed time dimensions
+    year INTEGER,
+    month INTEGER,
+    week INTEGER,
+    day_of_week INTEGER,
+    hour INTEGER,
+    partition_date DATE
+);
+
+CREATE TABLE IF NOT EXISTS analytics.events (
+    id UUID,
+    event_type VARCHAR,
+    entity_id UUID,
+    tenant_id UUID,
+    payload JSON,
+    occurred_at TIMESTAMP NOT NULL,
+    year INTEGER,
+    month INTEGER,
+    partition_date DATE
+);
+```
+
+### Dimension Tables
+
+```sql
+-- Date dimension
+CREATE TABLE IF NOT EXISTS analytics.dim_date (
+    date_key DATE PRIMARY KEY,
+    year INTEGER,
+    quarter INTEGER,
+    month INTEGER,
+    month_name VARCHAR,
+    week INTEGER,
+    day_of_week INTEGER,
+    day_name VARCHAR,
+    is_weekend BOOLEAN
+);
+
+-- Populate 10 years
+INSERT INTO analytics.dim_date
+SELECT
+    date_key,
+    YEAR(date_key), QUARTER(date_key), MONTH(date_key),
+    MONTHNAME(date_key), WEEKOFYEAR(date_key),
+    DAYOFWEEK(date_key), DAYNAME(date_key),
+    DAYOFWEEK(date_key) IN (0, 6)
+FROM (
+    SELECT UNNEST(generate_series(
+        DATE '2020-01-01', DATE '2030-12-31', INTERVAL 1 DAY
+    ))::DATE as date_key
+);
+```
+
+## Loading from Parquet
+
+### Direct Query (No Load)
+
+```sql
+-- Query directly from Parquet files
+SELECT
+    sensor_id,
+    AVG(value) as avg_value,
+    COUNT(*) as readings
+FROM read_parquet('s3://data/readings/**/*.parquet')
+WHERE partition_date >= '2024-01-01'
+GROUP BY sensor_id;
+```
+
+### Load into Table
+
+```sql
+-- Load with transformation
+INSERT INTO analytics.sensor_readings
+SELECT
+    time::TIMESTAMP,
+    sensor_id::UUID,
+    sensor_name,
+    tenant_id::UUID,
+    value,
+    unit,
+    YEAR(time), MONTH(time), WEEKOFYEAR(time),
+    DAYOFWEEK(time), HOUR(time),
+    time::DATE as partition_date
+FROM read_parquet('s3://data/readings/date=2024-01-15/*.parquet')
+WHERE partition_date NOT IN (SELECT DISTINCT partition_date FROM analytics.sensor_readings);
+```
+
+### Python ETL
+
+```python
+from datetime import datetime, timedelta
+
+conn = duckdb.connect("analytics.duckdb")
+conn.execute("SET s3_endpoint = 'minio:9000';")
+conn.execute("SET s3_access_key_id = 'admin';")
+conn.execute("SET s3_secret_access_key = 'password';")
+
+# Load last 30 days
+for i in range(30):
+    date = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
+    conn.execute(f"""
+        INSERT INTO analytics.sensor_readings
+        SELECT * FROM read_parquet('s3://data/readings/date={date}/*.parquet')
+    """)
+```
+
+## Analytical Views
+
+### Daily Summary
+
+```sql
+CREATE OR REPLACE VIEW reports.daily_summary AS
+SELECT
+    partition_date,
+    sensor_id,
+    sensor_name,
+    COUNT(*) as reading_count,
+    AVG(value) as avg_value,
+    MIN(value) as min_value,
+    MAX(value) as max_value,
+    STDDEV(value) as stddev_value
+FROM analytics.sensor_readings
+GROUP BY partition_date, sensor_id, sensor_name;
+```
+
+### Monthly KPIs
+
+```sql
+CREATE OR REPLACE VIEW reports.monthly_kpis AS
+SELECT
+    tenant_id,
+    year,
+    month,
+    COUNT(DISTINCT sensor_id) as active_sensors,
+    SUM(reading_count) as total_readings,
+    AVG(avg_value) as overall_avg
+FROM reports.daily_summary
+GROUP BY tenant_id, year, month;
+```
+
+## Advanced Analytics
+
+### Z-Score Anomaly Detection
+
+```sql
+WITH stats AS (
+    SELECT
+        sensor_id,
+        AVG(value) as avg_val,
+        STDDEV(value) as std_val
+    FROM analytics.sensor_readings
+    WHERE partition_date >= CURRENT_DATE - INTERVAL 90 DAY
+    GROUP BY sensor_id
+)
+SELECT
+    r.time,
+    r.sensor_id,
+    r.value,
+    (r.value - s.avg_val) / NULLIF(s.std_val, 0) as z_score,
+    CASE
+        WHEN ABS(r.value - s.avg_val) > 3 * s.std_val THEN 'ANOMALY'
+        WHEN ABS(r.value - s.avg_val) > 2 * s.std_val THEN 'WARNING'
+        ELSE 'NORMAL'
+    END as status
+FROM analytics.sensor_readings r
+JOIN stats s ON s.sensor_id = r.sensor_id
+WHERE r.partition_date >= CURRENT_DATE - INTERVAL 7 DAY
+  AND ABS(r.value - s.avg_val) > 2 * s.std_val;
+```
+
+### Correlation Analysis
+
+```sql
+SELECT
+    sensor_name,
+    CORR(temperature, gas_flow) as temp_gas_correlation,
+    COUNT(*) as samples
+FROM analytics.readings
+WHERE partition_date >= CURRENT_DATE - INTERVAL 30 DAY
+GROUP BY sensor_name
+HAVING COUNT(*) > 100;
+```
+
+### Window Functions
+
+```sql
+SELECT
+    partition_date,
+    sensor_id,
+    value,
+    -- Month cumulative
+    SUM(value) OVER (
+        PARTITION BY sensor_id, year, month
+        ORDER BY partition_date
+    ) as month_cumulative,
+    -- 7-day moving average
+    AVG(value) OVER (
+        PARTITION BY sensor_id
+        ORDER BY partition_date
+        ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
+    ) as ma7,
+    -- Week-over-week change
+    value - LAG(value, 7) OVER (
+        PARTITION BY sensor_id ORDER BY partition_date
+    ) as wow_change
+FROM analytics.daily_agg;
+```
+
+### Percentiles
+
+```sql
+SELECT
+    sensor_name,
+    MIN(value) as min,
+    PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY value) as p25,
+    PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY value) as median,
+    PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY value) as p75,
+    MAX(value) as max
+FROM analytics.sensor_readings
+WHERE partition_date >= CURRENT_DATE - INTERVAL 30 DAY
+GROUP BY sensor_name;
+```
+
+## Export to Parquet
+
+```sql
+-- Simple export
+COPY (SELECT * FROM reports.monthly_kpis)
+TO 'monthly_kpis.parquet' (FORMAT PARQUET);
+
+-- Partitioned export
+COPY analytics.sensor_readings
+TO 'sensor_readings' (FORMAT PARQUET, PARTITION_BY (year, month));
+
+-- Export to S3
+COPY reports.daily_summary
+TO 's3://data/reports/daily_summary.parquet' (FORMAT PARQUET);
+```
+
+## Python/Pandas Integration
+
+```python
+# Query to DataFrame
+df = conn.execute("""
+    SELECT partition_date, sensor_id, AVG(value) as avg
+    FROM analytics.sensor_readings
+    GROUP BY partition_date, sensor_id
+""").fetchdf()
+
+# DataFrame to DuckDB
+import pandas as pd
+df = pd.DataFrame({'date': [...], 'value': [...]})
+conn.register('temp_data', df)
+conn.execute("INSERT INTO analytics.processed SELECT * FROM temp_data")
+```
+
+## DuckDB vs TimescaleDB
+
+| Aspect | DuckDB | TimescaleDB |
+|--------|--------|-------------|
+| Type | OLAP (analytical) | OLTP (transactional) |
+| Storage | Columnar | Row-based + chunks |
+| Inserts | Batch (slow individual) | Streaming (fast) |
+| Queries | Massive aggregations | Point + range |
+| Latency | Seconds | Milliseconds |
+| Use case | Reports, BI, ML | Dashboard, alerts |
+
+## Performance Tips
+
+```sql
+-- Filter early (pushdown to Parquet)
+SELECT * FROM read_parquet('data/*.parquet')
+WHERE year = 2024 AND month = 3;  -- Good
+
+-- Project only needed columns
+SELECT sensor_id, AVG(value)
+FROM read_parquet('data.parquet')
+GROUP BY sensor_id;  -- Columnar = efficient
+
+-- Configure resources
+SET threads = 4;
+SET memory_limit = '4GB';
+
+-- Analyze query plan
+EXPLAIN ANALYZE SELECT ...;
+```
+
+## Quick Reference
+
+| Task | Command |
+|------|---------|
+| Load Parquet | `SELECT * FROM read_parquet('file.parquet')` |
+| Load from S3 | `SELECT * FROM read_parquet('s3://bucket/path/*.parquet')` |
+| Export Parquet | `COPY (query) TO 'file.parquet' (FORMAT PARQUET)` |
+| In-memory DB | `duckdb.connect(':memory:')` |
+| Check tables | `SELECT * FROM duckdb_tables()` |
+| Memory usage | `SELECT * FROM duckdb_memory()` |
+
+## Resources
+
+- [DuckDB Documentation](https://duckdb.org/docs/)
+- [go-duckdb](https://github.com/marcboeker/go-duckdb)
+- [DuckDB Python API](https://duckdb.org/docs/api/python/overview)
+
+---
+
+## Changelog
+
+- **2.0** - Condensed from Plataforma Industrial SKILL-DUCKDB
+
+## Related Skills
+
+- `timescaledb`: Time-series source data
+- `scikit-learn`: ML data preparation
+- `powerbi`: BI visualization
+- `sqlite-embedded`: Embedded alternative

package/ai-config/skills/data-ai/langchain/SKILL.md

@@ -0,0 +1,238 @@
+---
+name: langchain
+description: >
+  Build LLM applications with chains, agents, and RAG using LangChain and LangGraph.
+  Trigger: langchain, llm app, agent, rag, chains, langgraph
+tools:
+  - Read
+  - Write
+  - Bash
+  - Grep
+metadata:
+  author: plataforma-industrial
+  version: "2.0"
+  tags: [langchain, llm, agents, rag, ai]
+  updated: "2026-02"
+---
+
+# LangChain Skill
+
+Build LLM-powered applications with chains, agents, RAG, and streaming.
+
+## Stack
+
+```yaml
+langchain: 0.1+
+langchain-openai: 0.0.8+
+langchain-community: 0.0.20+
+langgraph: 0.0.26+
+langsmith: 0.1+
+```
+
+## Project Structure
+
+```
+src/
+├── agents/          # LangGraph agents
+├── chains/          # LCEL chains
+├── prompts/         # Prompt templates
+├── tools/           # Custom tools
+└── vectorstore/     # RAG components
+```
+
+## Basic Chain with Structured Output
+
+```python
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.output_parsers import PydanticOutputParser
+from pydantic import BaseModel, Field
+
+class Analysis(BaseModel):
+    status: str = Field(description="Status: normal, warning, critical")
+    findings: list[str] = Field(description="Key findings")
+    confidence: float = Field(description="Confidence 0-1")
+
+def create_analysis_chain():
+    llm = ChatOpenAI(model="gpt-4-turbo-preview", temperature=0)
+    parser = PydanticOutputParser(pydantic_object=Analysis)
+
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", "Analyze data.\n{format_instructions}"),
+        ("human", "{input}")
+    ])
+
+    chain = prompt | llm | parser
+    return chain
+
+# Usage
+result = await chain.ainvoke({
+    "input": data,
+    "format_instructions": parser.get_format_instructions()
+})
+```
+
+## Custom Tools
+
+```python
+from langchain_core.tools import tool
+import httpx
+
+@tool
+async def fetch_data(resource_id: str, hours: int = 24) -> str:
+    """Fetch data for a resource.
+
+    Args:
+        resource_id: The resource UUID
+        hours: Hours to look back (default 24)
+    """
+    async with httpx.AsyncClient() as client:
+        response = await client.get(
+            f"http://api:8080/resources/{resource_id}",
+            params={"hours": hours}
+        )
+        return response.text
+
+@tool
+async def update_status(resource_id: str, status: str) -> str:
+    """Update resource status."""
+    async with httpx.AsyncClient() as client:
+        response = await client.patch(
+            f"http://api:8080/resources/{resource_id}",
+            json={"status": status}
+        )
+        return "Updated" if response.status_code == 200 else f"Failed: {response.text}"
+```
+
+## LangGraph Agent
+
+```python
+from typing import TypedDict, Annotated, Sequence
+from langchain_core.messages import BaseMessage, HumanMessage
+from langgraph.graph import StateGraph, END
+from langgraph.prebuilt import ToolNode
+import operator
+
+class AgentState(TypedDict):
+    messages: Annotated[Sequence[BaseMessage], operator.add]
+    context: dict
+
+def create_agent(tools: list):
+    llm = ChatOpenAI(model="gpt-4-turbo-preview", temperature=0)
+    llm_with_tools = llm.bind_tools(tools)
+
+    def agent_node(state: AgentState):
+        system_message = f"Context: {state['context']}"
+        response = llm_with_tools.invoke([
+            {"role": "system", "content": system_message},
+            *state["messages"]
+        ])
+        return {"messages": [response]}
+
+    def should_continue(state: AgentState):
+        last_message = state["messages"][-1]
+        if hasattr(last_message, "tool_calls") and last_message.tool_calls:
+            return "tools"
+        return END
+
+    workflow = StateGraph(AgentState)
+    workflow.add_node("agent", agent_node)
+    workflow.add_node("tools", ToolNode(tools))
+    workflow.set_entry_point("agent")
+    workflow.add_conditional_edges("agent", should_continue, {"tools": "tools", END: END})
+    workflow.add_edge("tools", "agent")
+
+    return workflow.compile()
+
+# Usage
+agent = create_agent([fetch_data, update_status])
+result = await agent.ainvoke({
+    "messages": [HumanMessage(content="Check resource status")],
+    "context": {"user_id": "123"}
+})
+```
+
+## RAG Chain
+
+```python
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.vectorstores import Chroma
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_core.runnables import RunnablePassthrough
+
+def create_rag_chain(vectorstore):
+    llm = ChatOpenAI(model="gpt-4-turbo-preview", temperature=0)
+    retriever = vectorstore.as_retriever(search_type="mmr", search_kwargs={"k": 5})
+
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", "Answer using context:\n{context}"),
+        ("human", "{question}")
+    ])
+
+    def format_docs(docs):
+        return "\n\n".join(doc.page_content for doc in docs)
+
+    chain = (
+        {"context": retriever | format_docs, "question": RunnablePassthrough()}
+        | prompt
+        | llm
+    )
+    return chain
+```
+
+## Streaming Response (FastAPI)
+
+```python
+from fastapi import APIRouter
+from fastapi.responses import StreamingResponse
+
+router = APIRouter()
+
+@router.post("/chat/stream")
+async def chat_stream(request: ChatRequest):
+    llm = ChatOpenAI(model="gpt-4-turbo-preview", streaming=True)
+    prompt = ChatPromptTemplate.from_messages([
+        ("system", "You are a helpful assistant."),
+        ("human", "{message}")
+    ])
+    chain = prompt | llm
+
+    async def generate():
+        async for chunk in chain.astream({"message": request.message}):
+            if chunk.content:
+                yield f"data: {chunk.content}\n\n"
+        yield "data: [DONE]\n\n"
+
+    return StreamingResponse(generate(), media_type="text/event-stream")
+```
+
+## LangSmith Tracing
+
+```python
+import os
+from langsmith import Client
+
+def setup_langsmith():
+    os.environ["LANGCHAIN_TRACING_V2"] = "true"
+    os.environ["LANGCHAIN_PROJECT"] = "my-project"
+    os.environ["LANGCHAIN_API_KEY"] = os.getenv("LANGSMITH_API_KEY")
+
+# Add feedback
+async def log_feedback(run_id: str, score: float):
+    Client().create_feedback(run_id=run_id, key="user-rating", score=score)
+```
+
+## Best Practices
+
+1. **Use structured outputs** - PydanticOutputParser for reliable parsing
+2. **Async operations** - Always use `ainvoke`, `astream` for I/O
+3. **Error handling** - Use RunnableConfig with retries and timeouts
+4. **Cache embeddings** - CacheBackedEmbeddings for repeated queries
+5. **Tracing** - Enable LangSmith for debugging and monitoring
+
+## Related Skills
+
+- `ai-ml`: Full AI/ML patterns
+- `vector-db`: RAG with vector stores
+- `fastapi`: API integration
+- `redis-cache`: LLM response caching