javi-forge 1.1.0 → 1.3.0

package/ai-config/skills/data-ai/duckdb-analytics/SKILL.md

@@ -1,440 +0,0 @@
----
-name: duckdb-analytics
-description: >
-  DuckDB OLAP analytics with Parquet, S3, and columnar queries for BI/ML.
-  Trigger: duckdb, olap, analytics, parquet, columnar, data warehouse, BI
-
-tools:
-  - Read
-  - Write
-  - Bash
-  - Grep
-
-metadata:
-  author: plataforma-industrial
-  version: "2.0"
-  tags: [duckdb, olap, analytics, parquet, bi]
-  updated: "2026-02"
----
-
-# DuckDB OLAP Analytics
-
-> Columnar analytics engine for Parquet files, S3 data lakes, and BI workloads.
-
-## Stack
-
-```yaml
-DuckDB: 0.10+
-Go Driver: github.com/marcboeker/go-duckdb
-Python: duckdb 0.10+
-Extensions: httpfs, parquet, postgres
-```
-
-## When to Use
-
-- Analytical queries over historical data
-- Reading Parquet files from S3/MinIO
-- BI reports and aggregations
-- ML feature engineering
-- Time-series analysis on archived data
-
-### Data Architecture
-
-```
-TimescaleDB (OLTP - Hot, <90 days)
-    |
-    | ETL / Export
-    v
-Parquet Files (S3/MinIO - Warm)
-    |
-    | Direct read
-    v
-DuckDB (OLAP - Analytical queries)
-    |
-    v
-BI Tools / Reports / ML
-```
-
-## Python Setup
-
-```python
-import duckdb
-
-class DuckDBAnalytics:
-    def __init__(self, db_path: str = ":memory:"):
-        self.conn = duckdb.connect(db_path)
-        self._setup()
-
-    def _setup(self):
-        self.conn.execute("INSTALL httpfs; LOAD httpfs;")
-        self.conn.execute("INSTALL parquet; LOAD parquet;")
-
-    def configure_s3(self, endpoint: str, access_key: str, secret_key: str):
-        self.conn.execute(f"SET s3_endpoint = '{endpoint}';")
-        self.conn.execute(f"SET s3_access_key_id = '{access_key}';")
-        self.conn.execute(f"SET s3_secret_access_key = '{secret_key}';")
-        self.conn.execute("SET s3_use_ssl = false;")
-
-    def query_df(self, sql: str):
-        return self.conn.execute(sql).fetchdf()
-
-    def query(self, sql: str) -> list[dict]:
-        result = self.conn.execute(sql)
-        columns = [desc[0] for desc in result.description]
-        return [dict(zip(columns, row)) for row in result.fetchall()]
-```
-
-## Go Setup
-
-```go
-package analytics
-
-import (
-    "database/sql"
-    _ "github.com/marcboeker/go-duckdb"
-)
-
-type DuckDB struct {
-    db *sql.DB
-}
-
-func NewDuckDB(path string) (*DuckDB, error) {
-    db, err := sql.Open("duckdb", path)
-    if err != nil {
-        return nil, err
-    }
-    db.Exec("INSTALL parquet; LOAD parquet;")
-    return &DuckDB{db: db}, nil
-}
-```
-
-## Schema for Analytics
-
-### Fact Tables
-
-```sql
-CREATE SCHEMA IF NOT EXISTS analytics;
-
-CREATE TABLE IF NOT EXISTS analytics.sensor_readings (
-    time TIMESTAMP NOT NULL,
-    sensor_id UUID NOT NULL,
-    sensor_name VARCHAR,
-    tenant_id UUID,
-    value DOUBLE,
-    unit VARCHAR,
-    -- Pre-computed time dimensions
-    year INTEGER,
-    month INTEGER,
-    week INTEGER,
-    day_of_week INTEGER,
-    hour INTEGER,
-    partition_date DATE
-);
-
-CREATE TABLE IF NOT EXISTS analytics.events (
-    id UUID,
-    event_type VARCHAR,
-    entity_id UUID,
-    tenant_id UUID,
-    payload JSON,
-    occurred_at TIMESTAMP NOT NULL,
-    year INTEGER,
-    month INTEGER,
-    partition_date DATE
-);
-```
-
-### Dimension Tables
-
-```sql
--- Date dimension
-CREATE TABLE IF NOT EXISTS analytics.dim_date (
-    date_key DATE PRIMARY KEY,
-    year INTEGER,
-    quarter INTEGER,
-    month INTEGER,
-    month_name VARCHAR,
-    week INTEGER,
-    day_of_week INTEGER,
-    day_name VARCHAR,
-    is_weekend BOOLEAN
-);
-
--- Populate 10 years
-INSERT INTO analytics.dim_date
-SELECT
-    date_key,
-    YEAR(date_key), QUARTER(date_key), MONTH(date_key),
-    MONTHNAME(date_key), WEEKOFYEAR(date_key),
-    DAYOFWEEK(date_key), DAYNAME(date_key),
-    DAYOFWEEK(date_key) IN (0, 6)
-FROM (
-    SELECT UNNEST(generate_series(
-        DATE '2020-01-01', DATE '2030-12-31', INTERVAL 1 DAY
-    ))::DATE as date_key
-);
-```
-
-## Loading from Parquet
-
-### Direct Query (No Load)
-
-```sql
--- Query directly from Parquet files
-SELECT
-    sensor_id,
-    AVG(value) as avg_value,
-    COUNT(*) as readings
-FROM read_parquet('s3://data/readings/**/*.parquet')
-WHERE partition_date >= '2024-01-01'
-GROUP BY sensor_id;
-```
-
-### Load into Table
-
-```sql
--- Load with transformation
-INSERT INTO analytics.sensor_readings
-SELECT
-    time::TIMESTAMP,
-    sensor_id::UUID,
-    sensor_name,
-    tenant_id::UUID,
-    value,
-    unit,
-    YEAR(time), MONTH(time), WEEKOFYEAR(time),
-    DAYOFWEEK(time), HOUR(time),
-    time::DATE as partition_date
-FROM read_parquet('s3://data/readings/date=2024-01-15/*.parquet')
-WHERE partition_date NOT IN (SELECT DISTINCT partition_date FROM analytics.sensor_readings);
-```
-
-### Python ETL
-
-```python
-from datetime import datetime, timedelta
-
-conn = duckdb.connect("analytics.duckdb")
-conn.execute("SET s3_endpoint = 'minio:9000';")
-conn.execute("SET s3_access_key_id = 'admin';")
-conn.execute("SET s3_secret_access_key = 'password';")
-
-# Load last 30 days
-for i in range(30):
-    date = (datetime.now() - timedelta(days=i)).strftime("%Y-%m-%d")
-    conn.execute(f"""
-        INSERT INTO analytics.sensor_readings
-        SELECT * FROM read_parquet('s3://data/readings/date={date}/*.parquet')
-    """)
-```
-
-## Analytical Views
-
-### Daily Summary
-
-```sql
-CREATE OR REPLACE VIEW reports.daily_summary AS
-SELECT
-    partition_date,
-    sensor_id,
-    sensor_name,
-    COUNT(*) as reading_count,
-    AVG(value) as avg_value,
-    MIN(value) as min_value,
-    MAX(value) as max_value,
-    STDDEV(value) as stddev_value
-FROM analytics.sensor_readings
-GROUP BY partition_date, sensor_id, sensor_name;
-```
-
-### Monthly KPIs
-
-```sql
-CREATE OR REPLACE VIEW reports.monthly_kpis AS
-SELECT
-    tenant_id,
-    year,
-    month,
-    COUNT(DISTINCT sensor_id) as active_sensors,
-    SUM(reading_count) as total_readings,
-    AVG(avg_value) as overall_avg
-FROM reports.daily_summary
-GROUP BY tenant_id, year, month;
-```
-
-## Advanced Analytics
-
-### Z-Score Anomaly Detection
-
-```sql
-WITH stats AS (
-    SELECT
-        sensor_id,
-        AVG(value) as avg_val,
-        STDDEV(value) as std_val
-    FROM analytics.sensor_readings
-    WHERE partition_date >= CURRENT_DATE - INTERVAL 90 DAY
-    GROUP BY sensor_id
-)
-SELECT
-    r.time,
-    r.sensor_id,
-    r.value,
-    (r.value - s.avg_val) / NULLIF(s.std_val, 0) as z_score,
-    CASE
-        WHEN ABS(r.value - s.avg_val) > 3 * s.std_val THEN 'ANOMALY'
-        WHEN ABS(r.value - s.avg_val) > 2 * s.std_val THEN 'WARNING'
-        ELSE 'NORMAL'
-    END as status
-FROM analytics.sensor_readings r
-JOIN stats s ON s.sensor_id = r.sensor_id
-WHERE r.partition_date >= CURRENT_DATE - INTERVAL 7 DAY
-  AND ABS(r.value - s.avg_val) > 2 * s.std_val;
-```
-
-### Correlation Analysis
-
-```sql
-SELECT
-    sensor_name,
-    CORR(temperature, gas_flow) as temp_gas_correlation,
-    COUNT(*) as samples
-FROM analytics.readings
-WHERE partition_date >= CURRENT_DATE - INTERVAL 30 DAY
-GROUP BY sensor_name
-HAVING COUNT(*) > 100;
-```
-
-### Window Functions
-
-```sql
-SELECT
-    partition_date,
-    sensor_id,
-    value,
-    -- Month cumulative
-    SUM(value) OVER (
-        PARTITION BY sensor_id, year, month
-        ORDER BY partition_date
-    ) as month_cumulative,
-    -- 7-day moving average
-    AVG(value) OVER (
-        PARTITION BY sensor_id
-        ORDER BY partition_date
-        ROWS BETWEEN 6 PRECEDING AND CURRENT ROW
-    ) as ma7,
-    -- Week-over-week change
-    value - LAG(value, 7) OVER (
-        PARTITION BY sensor_id ORDER BY partition_date
-    ) as wow_change
-FROM analytics.daily_agg;
-```
-
-### Percentiles
-
-```sql
-SELECT
-    sensor_name,
-    MIN(value) as min,
-    PERCENTILE_CONT(0.25) WITHIN GROUP (ORDER BY value) as p25,
-    PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY value) as median,
-    PERCENTILE_CONT(0.75) WITHIN GROUP (ORDER BY value) as p75,
-    MAX(value) as max
-FROM analytics.sensor_readings
-WHERE partition_date >= CURRENT_DATE - INTERVAL 30 DAY
-GROUP BY sensor_name;
-```
-
-## Export to Parquet
-
-```sql
--- Simple export
-COPY (SELECT * FROM reports.monthly_kpis)
-TO 'monthly_kpis.parquet' (FORMAT PARQUET);
-
--- Partitioned export
-COPY analytics.sensor_readings
-TO 'sensor_readings' (FORMAT PARQUET, PARTITION_BY (year, month));
-
--- Export to S3
-COPY reports.daily_summary
-TO 's3://data/reports/daily_summary.parquet' (FORMAT PARQUET);
-```
-
-## Python/Pandas Integration
-
-```python
-# Query to DataFrame
-df = conn.execute("""
-    SELECT partition_date, sensor_id, AVG(value) as avg
-    FROM analytics.sensor_readings
-    GROUP BY partition_date, sensor_id
-""").fetchdf()
-
-# DataFrame to DuckDB
-import pandas as pd
-df = pd.DataFrame({'date': [...], 'value': [...]})
-conn.register('temp_data', df)
-conn.execute("INSERT INTO analytics.processed SELECT * FROM temp_data")
-```
-
-## DuckDB vs TimescaleDB
-
-| Aspect | DuckDB | TimescaleDB |
-|--------|--------|-------------|
-| Type | OLAP (analytical) | OLTP (transactional) |
-| Storage | Columnar | Row-based + chunks |
-| Inserts | Batch (slow individual) | Streaming (fast) |
-| Queries | Massive aggregations | Point + range |
-| Latency | Seconds | Milliseconds |
-| Use case | Reports, BI, ML | Dashboard, alerts |
-
-## Performance Tips
-
-```sql
--- Filter early (pushdown to Parquet)
-SELECT * FROM read_parquet('data/*.parquet')
-WHERE year = 2024 AND month = 3;  -- Good
-
--- Project only needed columns
-SELECT sensor_id, AVG(value)
-FROM read_parquet('data.parquet')
-GROUP BY sensor_id;  -- Columnar = efficient
-
--- Configure resources
-SET threads = 4;
-SET memory_limit = '4GB';
-
--- Analyze query plan
-EXPLAIN ANALYZE SELECT ...;
-```
-
-## Quick Reference
-
-| Task | Command |
-|------|---------|
-| Load Parquet | `SELECT * FROM read_parquet('file.parquet')` |
-| Load from S3 | `SELECT * FROM read_parquet('s3://bucket/path/*.parquet')` |
-| Export Parquet | `COPY (query) TO 'file.parquet' (FORMAT PARQUET)` |
-| In-memory DB | `duckdb.connect(':memory:')` |
-| Check tables | `SELECT * FROM duckdb_tables()` |
-| Memory usage | `SELECT * FROM duckdb_memory()` |
-
-## Resources
-
-- [DuckDB Documentation](https://duckdb.org/docs/)
-- [go-duckdb](https://github.com/marcboeker/go-duckdb)
-- [DuckDB Python API](https://duckdb.org/docs/api/python/overview)
-
----
-
-## Changelog
-
-- **2.0** - Condensed from Plataforma Industrial SKILL-DUCKDB
-
-## Related Skills
-
-- `timescaledb`: Time-series source data
-- `scikit-learn`: ML data preparation
-- `powerbi`: BI visualization
-- `sqlite-embedded`: Embedded alternative

package/ai-config/skills/data-ai/langchain/SKILL.md

@@ -1,238 +0,0 @@
----
-name: langchain
-description: >
-  Build LLM applications with chains, agents, and RAG using LangChain and LangGraph.
-  Trigger: langchain, llm app, agent, rag, chains, langgraph
-tools:
-  - Read
-  - Write
-  - Bash
-  - Grep
-metadata:
-  author: plataforma-industrial
-  version: "2.0"
-  tags: [langchain, llm, agents, rag, ai]
-  updated: "2026-02"
----
-
-# LangChain Skill
-
-Build LLM-powered applications with chains, agents, RAG, and streaming.
-
-## Stack
-
-```yaml
-langchain: 0.1+
-langchain-openai: 0.0.8+
-langchain-community: 0.0.20+
-langgraph: 0.0.26+
-langsmith: 0.1+
-```
-
-## Project Structure
-
-```
-src/
-├── agents/          # LangGraph agents
-├── chains/          # LCEL chains
-├── prompts/         # Prompt templates
-├── tools/           # Custom tools
-└── vectorstore/     # RAG components
-```
-
-## Basic Chain with Structured Output
-
-```python
-from langchain_openai import ChatOpenAI
-from langchain_core.prompts import ChatPromptTemplate
-from langchain_core.output_parsers import PydanticOutputParser
-from pydantic import BaseModel, Field
-
-class Analysis(BaseModel):
-    status: str = Field(description="Status: normal, warning, critical")
-    findings: list[str] = Field(description="Key findings")
-    confidence: float = Field(description="Confidence 0-1")
-
-def create_analysis_chain():
-    llm = ChatOpenAI(model="gpt-4-turbo-preview", temperature=0)
-    parser = PydanticOutputParser(pydantic_object=Analysis)
-
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", "Analyze data.\n{format_instructions}"),
-        ("human", "{input}")
-    ])
-
-    chain = prompt | llm | parser
-    return chain
-
-# Usage
-result = await chain.ainvoke({
-    "input": data,
-    "format_instructions": parser.get_format_instructions()
-})
-```
-
-## Custom Tools
-
-```python
-from langchain_core.tools import tool
-import httpx
-
-@tool
-async def fetch_data(resource_id: str, hours: int = 24) -> str:
-    """Fetch data for a resource.
-
-    Args:
-        resource_id: The resource UUID
-        hours: Hours to look back (default 24)
-    """
-    async with httpx.AsyncClient() as client:
-        response = await client.get(
-            f"http://api:8080/resources/{resource_id}",
-            params={"hours": hours}
-        )
-        return response.text
-
-@tool
-async def update_status(resource_id: str, status: str) -> str:
-    """Update resource status."""
-    async with httpx.AsyncClient() as client:
-        response = await client.patch(
-            f"http://api:8080/resources/{resource_id}",
-            json={"status": status}
-        )
-        return "Updated" if response.status_code == 200 else f"Failed: {response.text}"
-```
-
-## LangGraph Agent
-
-```python
-from typing import TypedDict, Annotated, Sequence
-from langchain_core.messages import BaseMessage, HumanMessage
-from langgraph.graph import StateGraph, END
-from langgraph.prebuilt import ToolNode
-import operator
-
-class AgentState(TypedDict):
-    messages: Annotated[Sequence[BaseMessage], operator.add]
-    context: dict
-
-def create_agent(tools: list):
-    llm = ChatOpenAI(model="gpt-4-turbo-preview", temperature=0)
-    llm_with_tools = llm.bind_tools(tools)
-
-    def agent_node(state: AgentState):
-        system_message = f"Context: {state['context']}"
-        response = llm_with_tools.invoke([
-            {"role": "system", "content": system_message},
-            *state["messages"]
-        ])
-        return {"messages": [response]}
-
-    def should_continue(state: AgentState):
-        last_message = state["messages"][-1]
-        if hasattr(last_message, "tool_calls") and last_message.tool_calls:
-            return "tools"
-        return END
-
-    workflow = StateGraph(AgentState)
-    workflow.add_node("agent", agent_node)
-    workflow.add_node("tools", ToolNode(tools))
-    workflow.set_entry_point("agent")
-    workflow.add_conditional_edges("agent", should_continue, {"tools": "tools", END: END})
-    workflow.add_edge("tools", "agent")
-
-    return workflow.compile()
-
-# Usage
-agent = create_agent([fetch_data, update_status])
-result = await agent.ainvoke({
-    "messages": [HumanMessage(content="Check resource status")],
-    "context": {"user_id": "123"}
-})
-```
-
-## RAG Chain
-
-```python
-from langchain_openai import OpenAIEmbeddings
-from langchain_community.vectorstores import Chroma
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_core.runnables import RunnablePassthrough
-
-def create_rag_chain(vectorstore):
-    llm = ChatOpenAI(model="gpt-4-turbo-preview", temperature=0)
-    retriever = vectorstore.as_retriever(search_type="mmr", search_kwargs={"k": 5})
-
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", "Answer using context:\n{context}"),
-        ("human", "{question}")
-    ])
-
-    def format_docs(docs):
-        return "\n\n".join(doc.page_content for doc in docs)
-
-    chain = (
-        {"context": retriever | format_docs, "question": RunnablePassthrough()}
-        | prompt
-        | llm
-    )
-    return chain
-```
-
-## Streaming Response (FastAPI)
-
-```python
-from fastapi import APIRouter
-from fastapi.responses import StreamingResponse
-
-router = APIRouter()
-
-@router.post("/chat/stream")
-async def chat_stream(request: ChatRequest):
-    llm = ChatOpenAI(model="gpt-4-turbo-preview", streaming=True)
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", "You are a helpful assistant."),
-        ("human", "{message}")
-    ])
-    chain = prompt | llm
-
-    async def generate():
-        async for chunk in chain.astream({"message": request.message}):
-            if chunk.content:
-                yield f"data: {chunk.content}\n\n"
-        yield "data: [DONE]\n\n"
-
-    return StreamingResponse(generate(), media_type="text/event-stream")
-```
-
-## LangSmith Tracing
-
-```python
-import os
-from langsmith import Client
-
-def setup_langsmith():
-    os.environ["LANGCHAIN_TRACING_V2"] = "true"
-    os.environ["LANGCHAIN_PROJECT"] = "my-project"
-    os.environ["LANGCHAIN_API_KEY"] = os.getenv("LANGSMITH_API_KEY")
-
-# Add feedback
-async def log_feedback(run_id: str, score: float):
-    Client().create_feedback(run_id=run_id, key="user-rating", score=score)
-```
-
-## Best Practices
-
-1. **Use structured outputs** - PydanticOutputParser for reliable parsing
-2. **Async operations** - Always use `ainvoke`, `astream` for I/O
-3. **Error handling** - Use RunnableConfig with retries and timeouts
-4. **Cache embeddings** - CacheBackedEmbeddings for repeated queries
-5. **Tracing** - Enable LangSmith for debugging and monitoring
-
-## Related Skills
-
-- `ai-ml`: Full AI/ML patterns
-- `vector-db`: RAG with vector stores
-- `fastapi`: API integration
-- `redis-cache`: LLM response caching