javi-forge 0.1.0

package/ai-config/skills/data-ai/scikit-learn/SKILL.md

@@ -0,0 +1,321 @@
+---
+name: scikit-learn
+description: >
+  Classical ML with scikit-learn for anomaly detection, classification, and clustering.
+  Trigger: sklearn, scikit-learn, classical ml, anomaly detection, classification, clustering
+tools:
+  - Read
+  - Write
+  - Bash
+  - Grep
+metadata:
+  author: plataforma-industrial
+  version: "2.0"
+  tags: [sklearn, ml, classification, anomaly-detection, clustering]
+  updated: "2026-02"
+---
+
+# Scikit-learn Skill
+
+Classical ML for anomaly detection, classification, regression, and clustering.
+
+## Stack
+
+```yaml
+scikit-learn: 1.4+
+pandas: 2.2+
+numpy: 1.26+
+joblib: 1.3+
+imbalanced-learn: 0.12+
+```
+
+## Anomaly Detection
+
+### Isolation Forest
+
+```python
+from sklearn.ensemble import IsolationForest
+from sklearn.preprocessing import StandardScaler
+from sklearn.pipeline import Pipeline
+import numpy as np
+import joblib
+
+class AnomalyDetector:
+    def __init__(self, contamination: float = 0.05, n_estimators: int = 100):
+        self.pipeline = Pipeline([
+            ('scaler', StandardScaler()),
+            ('detector', IsolationForest(
+                n_estimators=n_estimators,
+                contamination=contamination,
+                random_state=42,
+                n_jobs=-1
+            ))
+        ])
+
+    def fit(self, X: np.ndarray):
+        self.pipeline.fit(X)
+        return self
+
+    def predict(self, X: np.ndarray):
+        labels = self.pipeline.predict(X)  # -1 anomaly, 1 normal
+        scores = self.pipeline.decision_function(X)
+        return labels, scores
+
+    def predict_proba(self, X: np.ndarray):
+        scores = self.pipeline.decision_function(X)
+        return 1 / (1 + np.exp(scores))  # Convert to probability
+
+    def save(self, path: str):
+        joblib.dump(self.pipeline, path)
+
+    @classmethod
+    def load(cls, path: str):
+        detector = cls()
+        detector.pipeline = joblib.load(path)
+        return detector
+```
+
+### One-Class SVM & LOF
+
+```python
+from sklearn.svm import OneClassSVM
+from sklearn.neighbors import LocalOutlierFactor
+from sklearn.preprocessing import RobustScaler
+
+# One-Class SVM (robust to outliers)
+class RobustDetector:
+    def __init__(self, nu: float = 0.05):
+        self.pipeline = Pipeline([
+            ('scaler', RobustScaler()),
+            ('svm', OneClassSVM(nu=nu, kernel='rbf', gamma='scale'))
+        ])
+
+    def fit(self, X):
+        self.pipeline.fit(X)
+        return self
+
+    def predict(self, X):
+        return self.pipeline.predict(X)
+
+# Local Outlier Factor (streaming)
+class StreamingLOF:
+    def __init__(self, n_neighbors: int = 20, contamination: float = 0.05):
+        self.lof = LocalOutlierFactor(
+            n_neighbors=n_neighbors,
+            contamination=contamination,
+            novelty=True  # Enable predict on new data
+        )
+
+    def fit(self, X):
+        self.lof.fit(X)
+        return self
+
+    def predict(self, X):
+        return self.lof.predict(X)
+```
+
+## Classification
+
+```python
+from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from sklearn.pipeline import Pipeline
+from sklearn.metrics import classification_report
+import pandas as pd
+
+class EquipmentClassifier:
+    STATES = ['normal', 'degraded', 'maintenance_required', 'critical']
+
+    def __init__(self, model_type: str = 'random_forest'):
+        self.label_encoder = LabelEncoder()
+        self.label_encoder.fit(self.STATES)
+
+        classifier = RandomForestClassifier(
+            n_estimators=200, max_depth=10, min_samples_split=5,
+            class_weight='balanced', n_jobs=-1, random_state=42
+        ) if model_type == 'random_forest' else GradientBoostingClassifier(
+            n_estimators=200, max_depth=5, learning_rate=0.1, random_state=42
+        )
+
+        self.pipeline = Pipeline([
+            ('scaler', StandardScaler()),
+            ('classifier', classifier)
+        ])
+
+    def fit(self, X: np.ndarray, y: np.ndarray):
+        if isinstance(y[0], str):
+            y = self.label_encoder.transform(y)
+        self.pipeline.fit(X, y)
+        return self
+
+    def predict(self, X: np.ndarray):
+        y_pred = self.pipeline.predict(X)
+        return self.label_encoder.inverse_transform(y_pred)
+
+    def predict_proba(self, X: np.ndarray) -> pd.DataFrame:
+        proba = self.pipeline.predict_proba(X)
+        return pd.DataFrame(proba, columns=self.STATES)
+
+    def feature_importance(self, feature_names: list) -> pd.DataFrame:
+        importances = self.pipeline.named_steps['classifier'].feature_importances_
+        return pd.DataFrame({'feature': feature_names, 'importance': importances}).sort_values('importance', ascending=False)
+```
+
+## Feature Engineering
+
+```python
+def extract_features(df: pd.DataFrame, window_size: int = 60) -> pd.DataFrame:
+    features = pd.DataFrame()
+
+    for col in ['temperature', 'pressure', 'vibration', 'flow_rate']:
+        features[f'{col}_mean'] = df[col].rolling(window_size).mean()
+        features[f'{col}_std'] = df[col].rolling(window_size).std()
+        features[f'{col}_min'] = df[col].rolling(window_size).min()
+        features[f'{col}_max'] = df[col].rolling(window_size).max()
+        features[f'{col}_range'] = features[f'{col}_max'] - features[f'{col}_min']
+        features[f'{col}_diff'] = df[col].diff()
+        features[f'{col}_diff_mean'] = features[f'{col}_diff'].rolling(window_size).mean()
+
+    return features.dropna()
+```
+
+## Forecasting (Multi-step)
+
+```python
+from sklearn.multioutput import MultiOutputRegressor
+from sklearn.ensemble import GradientBoostingRegressor
+
+class Forecaster:
+    def __init__(self, horizon: int = 12, lookback: int = 24):
+        self.horizon = horizon
+        self.lookback = lookback
+        self.pipeline = Pipeline([
+            ('scaler', StandardScaler()),
+            ('regressor', MultiOutputRegressor(
+                GradientBoostingRegressor(n_estimators=100, max_depth=5, random_state=42),
+                n_jobs=-1
+            ))
+        ])
+
+    def create_sequences(self, data: np.ndarray):
+        X, y = [], []
+        for i in range(len(data) - self.lookback - self.horizon + 1):
+            X.append(data[i:i + self.lookback].flatten())
+            y.append(data[i + self.lookback:i + self.lookback + self.horizon].flatten())
+        return np.array(X), np.array(y)
+
+    def fit(self, data: np.ndarray):
+        X, y = self.create_sequences(data)
+        self.pipeline.fit(X, y)
+        return self
+
+    def predict(self, recent_data: np.ndarray) -> np.ndarray:
+        X = recent_data.flatten().reshape(1, -1)
+        y_pred = self.pipeline.predict(X)
+        return y_pred.reshape(self.horizon, recent_data.shape[1])
+```
+
+## Clustering
+
+```python
+from sklearn.cluster import KMeans, DBSCAN
+from sklearn.decomposition import PCA
+from sklearn.metrics import silhouette_score
+
+class ModeDetector:
+    def __init__(self, n_modes: int = None, method: str = 'kmeans'):
+        self.n_modes = n_modes
+        self.scaler = StandardScaler()
+        self.pca = PCA(n_components=0.95)
+        self.clusterer = None
+
+    def fit(self, X: np.ndarray):
+        X_scaled = self.scaler.fit_transform(X)
+        X_pca = self.pca.fit_transform(X_scaled)
+
+        if self.n_modes is None:
+            self.n_modes = self._find_optimal_k(X_pca)
+
+        self.clusterer = KMeans(n_clusters=self.n_modes, n_init=10, random_state=42)
+        self.clusterer.fit(X_pca)
+        return self
+
+    def predict(self, X: np.ndarray):
+        X_pca = self.pca.transform(self.scaler.transform(X))
+        return self.clusterer.predict(X_pca)
+
+    def _find_optimal_k(self, X: np.ndarray, max_k: int = 10) -> int:
+        scores = [silhouette_score(X, KMeans(n_clusters=k, n_init=10).fit_predict(X))
+                  for k in range(2, max_k + 1)]
+        return np.argmax(scores) + 2
+```
+
+## Hyperparameter Tuning
+
+```python
+from sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit
+
+def tune_classifier(X: np.ndarray, y: np.ndarray):
+    tscv = TimeSeriesSplit(n_splits=5)
+
+    param_grid = {
+        'n_estimators': [100, 200, 300],
+        'max_depth': [5, 10, 15, None],
+        'min_samples_split': [2, 5, 10],
+        'min_samples_leaf': [1, 2, 4],
+    }
+
+    search = RandomizedSearchCV(
+        RandomForestClassifier(random_state=42, n_jobs=-1),
+        param_grid,
+        n_iter=50,
+        cv=tscv,
+        scoring='f1_weighted',
+        n_jobs=-1
+    )
+    search.fit(X, y)
+    return search.best_estimator_, search.best_params_
+```
+
+## Model Persistence
+
+```python
+import json
+from pathlib import Path
+from datetime import datetime
+
+def save_model(model, path: str, metadata: dict = None, feature_names: list = None):
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    joblib.dump(model, path)
+
+    meta = {
+        'saved_at': datetime.utcnow().isoformat(),
+        'sklearn_version': __import__('sklearn').__version__,
+        'feature_names': feature_names,
+        **(metadata or {})
+    }
+    path.with_suffix('.json').write_text(json.dumps(meta, indent=2))
+
+def load_model(path: str):
+    path = Path(path)
+    model = joblib.load(path)
+    meta_path = path.with_suffix('.json')
+    metadata = json.loads(meta_path.read_text()) if meta_path.exists() else {}
+    return model, metadata
+```
+
+## Best Practices
+
+1. **Use pipelines** - Combine preprocessing and model for reproducibility
+2. **Handle imbalance** - SMOTE or `class_weight='balanced'`
+3. **Time series CV** - Use `TimeSeriesSplit` instead of random splits
+4. **Feature importance** - Analyze with `.feature_importances_`
+5. **Version models** - Save with metadata (version, features, metrics)
+
+## Related Skills
+
+- `mlflow`: Experiment tracking
+- `pytorch`: Deep learning alternative
+- `duckdb-analytics`: Data preprocessing
+- `onnx-inference`: Model deployment

package/ai-config/skills/data-ai/vector-db/SKILL.md

@@ -0,0 +1,301 @@
+---
+name: vector-db
+description: >
+  Vector databases for RAG and semantic search with ChromaDB and pgvector.
+  Trigger: vector database, embeddings, rag, semantic search, chromadb, pgvector
+tools:
+  - Read
+  - Write
+  - Bash
+  - Grep
+metadata:
+  author: plataforma-industrial
+  version: "2.0"
+  tags: [vector-db, rag, embeddings, semantic-search, chromadb, pgvector]
+  updated: "2026-02"
+---
+
+# Vector Database Skill
+
+Vector databases for RAG and semantic search.
+
+## Stack
+
+```yaml
+# Primary
+chromadb: 0.4+
+pgvector: 0.6+  # PostgreSQL extension
+
+# Alternatives
+qdrant-client: 1.7+
+pinecone-client: 3.0+
+
+# Embeddings
+openai: 1.12+
+sentence-transformers: 2.5+
+```
+
+## ChromaDB
+
+```python
+import chromadb
+from chromadb.config import Settings
+from chromadb.utils import embedding_functions
+import os
+
+class ChromaVectorStore:
+    def __init__(self, collection_name: str = "documents", persist_dir: str = "./chroma_db"):
+        self.client = chromadb.PersistentClient(
+            path=persist_dir,
+            settings=Settings(anonymized_telemetry=False)
+        )
+
+        self.embedding_fn = embedding_functions.OpenAIEmbeddingFunction(
+            api_key=os.getenv("OPENAI_API_KEY"),
+            model_name="text-embedding-3-small"
+        )
+
+        self.collection = self.client.get_or_create_collection(
+            name=collection_name,
+            embedding_function=self.embedding_fn,
+            metadata={"hnsw:space": "cosine"}
+        )
+
+    def add_documents(self, documents: list[str], metadatas: list[dict] = None, ids: list[str] = None):
+        if ids is None:
+            ids = [f"doc_{i}" for i in range(len(documents))]
+        self.collection.add(documents=documents, metadatas=metadatas, ids=ids)
+
+    def query(self, query_text: str, n_results: int = 5, where: dict = None):
+        return self.collection.query(
+            query_texts=[query_text],
+            n_results=n_results,
+            where=where,
+            include=["documents", "metadatas", "distances"]
+        )
+
+    def delete(self, ids: list[str] = None, where: dict = None):
+        if ids:
+            self.collection.delete(ids=ids)
+        elif where:
+            self.collection.delete(where=where)
+```
+
+## pgvector (PostgreSQL)
+
+### Schema
+
+```sql
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE documents (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    content TEXT NOT NULL,
+    embedding vector(1536),  -- OpenAI dimension
+    metadata JSONB DEFAULT '{}',
+    tenant_id UUID NOT NULL,
+    created_at TIMESTAMPTZ DEFAULT NOW()
+);
+
+-- HNSW index for fast search
+CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64);
+CREATE INDEX idx_documents_tenant ON documents (tenant_id);
+CREATE INDEX idx_documents_metadata ON documents USING gin (metadata);
+```
+
+### Python Client
+
+```python
+import asyncpg
+from pgvector.asyncpg import register_vector
+import numpy as np
+from openai import AsyncOpenAI
+
+class PgVectorStore:
+    def __init__(self, pool: asyncpg.Pool, embedding_model: str = "text-embedding-3-small"):
+        self.pool = pool
+        self.openai = AsyncOpenAI()
+        self.embedding_model = embedding_model
+
+    async def setup(self):
+        async with self.pool.acquire() as conn:
+            await register_vector(conn)
+
+    async def embed(self, texts: list[str]) -> list[np.ndarray]:
+        response = await self.openai.embeddings.create(model=self.embedding_model, input=texts)
+        return [np.array(e.embedding) for e in response.data]
+
+    async def add_documents(self, contents: list[str], metadatas: list[dict], tenant_id: str) -> list[str]:
+        embeddings = await self.embed(contents)
+        ids = []
+        async with self.pool.acquire() as conn:
+            for content, embedding, metadata in zip(contents, embeddings, metadatas):
+                row = await conn.fetchrow(
+                    "INSERT INTO documents (content, embedding, metadata, tenant_id) VALUES ($1, $2, $3, $4) RETURNING id",
+                    content, embedding, metadata, tenant_id
+                )
+                ids.append(str(row['id']))
+        return ids
+
+    async def similarity_search(self, query: str, tenant_id: str, k: int = 5, metadata_filter: dict = None):
+        query_embedding = (await self.embed([query]))[0]
+
+        sql = """
+            SELECT id, content, metadata, 1 - (embedding <=> $1) AS similarity
+            FROM documents WHERE tenant_id = $2
+        """
+        params = [query_embedding, tenant_id]
+
+        if metadata_filter:
+            sql += " AND metadata @> $3"
+            params.append(metadata_filter)
+
+        sql += f" ORDER BY embedding <=> $1 LIMIT ${len(params) + 1}"
+        params.append(k)
+
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(sql, *params)
+
+        return [{"id": str(r['id']), "content": r['content'], "metadata": dict(r['metadata']), "similarity": float(r['similarity'])} for r in rows]
+
+    async def hybrid_search(self, query: str, tenant_id: str, k: int = 5, keyword_weight: float = 0.3):
+        query_embedding = (await self.embed([query]))[0]
+
+        sql = """
+            WITH vector_search AS (
+                SELECT id, content, metadata, 1 - (embedding <=> $1) AS vector_score
+                FROM documents WHERE tenant_id = $2
+                ORDER BY embedding <=> $1 LIMIT $3 * 2
+            ),
+            keyword_search AS (
+                SELECT id, ts_rank(to_tsvector('english', content), plainto_tsquery('english', $4)) AS keyword_score
+                FROM documents WHERE tenant_id = $2 AND to_tsvector('english', content) @@ plainto_tsquery('english', $4)
+            )
+            SELECT v.id, v.content, v.metadata,
+                   (1 - $5) * v.vector_score + $5 * COALESCE(k.keyword_score, 0) AS combined_score
+            FROM vector_search v LEFT JOIN keyword_search k ON v.id = k.id
+            ORDER BY combined_score DESC LIMIT $3
+        """
+
+        async with self.pool.acquire() as conn:
+            rows = await conn.fetch(sql, query_embedding, tenant_id, k, query, keyword_weight)
+
+        return [{"id": str(r['id']), "content": r['content'], "metadata": dict(r['metadata']), "score": float(r['combined_score'])} for r in rows]
+```
+
+## Document Chunking
+
+```python
+from langchain_text_splitters import RecursiveCharacterTextSplitter, MarkdownHeaderTextSplitter
+
+def chunk_documents(documents: list[dict], chunk_size: int = 1000, chunk_overlap: int = 200) -> list[dict]:
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separators=["\n\n", "\n", ". ", " ", ""]
+    )
+
+    chunks = []
+    for doc in documents:
+        for i, chunk in enumerate(splitter.split_text(doc["content"])):
+            chunks.append({
+                "content": chunk,
+                "metadata": {**doc.get("metadata", {}), "source_id": doc.get("id"), "chunk_index": i}
+            })
+    return chunks
+
+def chunk_markdown(content: str, source_metadata: dict = None) -> list[dict]:
+    header_splitter = MarkdownHeaderTextSplitter(
+        headers_to_split_on=[("#", "h1"), ("##", "h2"), ("###", "h3")]
+    )
+    size_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+
+    chunks = []
+    for split in header_splitter.split_text(content):
+        text, metadata = split.page_content, {**(source_metadata or {}), **split.metadata}
+        if len(text) > 1000:
+            for i, sub_chunk in enumerate(size_splitter.split_text(text)):
+                chunks.append({"content": sub_chunk, "metadata": {**metadata, "sub_chunk": i}})
+        else:
+            chunks.append({"content": text, "metadata": metadata})
+    return chunks
+```
+
+## RAG Chain
+
+```python
+from openai import AsyncOpenAI
+
+class RAGChain:
+    def __init__(self, vector_store: PgVectorStore, model: str = "gpt-4-turbo-preview", k: int = 5):
+        self.vector_store = vector_store
+        self.openai = AsyncOpenAI()
+        self.model = model
+        self.k = k
+
+    async def query(self, question: str, tenant_id: str, system_prompt: str = None, metadata_filter: dict = None):
+        docs = await self.vector_store.similarity_search(
+            query=question, tenant_id=tenant_id, k=self.k, metadata_filter=metadata_filter
+        )
+
+        context = "\n\n---\n\n".join([f"[Source: {d['metadata'].get('source', 'Unknown')}]\n{d['content']}" for d in docs])
+
+        if system_prompt is None:
+            system_prompt = "Answer based on the provided context. If not in context, say so. Cite sources."
+
+        response = await self.openai.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": f"Context:\n{context}\n\nQuestion: {question}"}
+            ],
+            temperature=0
+        )
+
+        return {
+            "answer": response.choices[0].message.content,
+            "sources": [d['metadata'].get('source') for d in docs],
+            "docs": docs
+        }
+```
+
+## Indexing Pipeline
+
+```python
+class IndexingPipeline:
+    def __init__(self, vector_store: PgVectorStore, chunk_size: int = 1000, batch_size: int = 100):
+        self.vector_store = vector_store
+        self.chunk_size = chunk_size
+        self.batch_size = batch_size
+
+    async def index_documents(self, documents: list[dict], tenant_id: str) -> int:
+        chunks = chunk_documents(documents, chunk_size=self.chunk_size)
+        total = 0
+
+        for i in range(0, len(chunks), self.batch_size):
+            batch = chunks[i:i + self.batch_size]
+            await self.vector_store.add_documents(
+                contents=[c['content'] for c in batch],
+                metadatas=[c['metadata'] for c in batch],
+                tenant_id=tenant_id
+            )
+            total += len(batch)
+            print(f"Indexed {total}/{len(chunks)}")
+
+        return total
+```
+
+## Best Practices
+
+1. **Chunk size** - Technical docs: 1000-1500, FAQs: 300-500
+2. **Include overlap** - 10-20% overlap for context continuity
+3. **Use metadata** - Filter by type, source, date
+4. **Hybrid search** - Combine vector + keyword for better recall
+5. **Re-rank results** - Cross-encoder for precision: `cross-encoder/ms-marco-MiniLM-L-6-v2`
+
+## Related Skills
+
+- `langchain`: RAG implementation
+- `ai-ml`: Embedding generation
+- `fastapi`: Vector search API
+- `redis-cache`: Hybrid search caching