javi-forge 1.2.0 → 1.4.0

package/ai-config/skills/data-ai/scikit-learn/SKILL.md

@@ -1,321 +0,0 @@
----
-name: scikit-learn
-description: >
-  Classical ML with scikit-learn for anomaly detection, classification, and clustering.
-  Trigger: sklearn, scikit-learn, classical ml, anomaly detection, classification, clustering
-tools:
-  - Read
-  - Write
-  - Bash
-  - Grep
-metadata:
-  author: plataforma-industrial
-  version: "2.0"
-  tags: [sklearn, ml, classification, anomaly-detection, clustering]
-  updated: "2026-02"
----
-
-# Scikit-learn Skill
-
-Classical ML for anomaly detection, classification, regression, and clustering.
-
-## Stack
-
-```yaml
-scikit-learn: 1.4+
-pandas: 2.2+
-numpy: 1.26+
-joblib: 1.3+
-imbalanced-learn: 0.12+
-```
-
-## Anomaly Detection
-
-### Isolation Forest
-
-```python
-from sklearn.ensemble import IsolationForest
-from sklearn.preprocessing import StandardScaler
-from sklearn.pipeline import Pipeline
-import numpy as np
-import joblib
-
-class AnomalyDetector:
-    def __init__(self, contamination: float = 0.05, n_estimators: int = 100):
-        self.pipeline = Pipeline([
-            ('scaler', StandardScaler()),
-            ('detector', IsolationForest(
-                n_estimators=n_estimators,
-                contamination=contamination,
-                random_state=42,
-                n_jobs=-1
-            ))
-        ])
-
-    def fit(self, X: np.ndarray):
-        self.pipeline.fit(X)
-        return self
-
-    def predict(self, X: np.ndarray):
-        labels = self.pipeline.predict(X)  # -1 anomaly, 1 normal
-        scores = self.pipeline.decision_function(X)
-        return labels, scores
-
-    def predict_proba(self, X: np.ndarray):
-        scores = self.pipeline.decision_function(X)
-        return 1 / (1 + np.exp(scores))  # Convert to probability
-
-    def save(self, path: str):
-        joblib.dump(self.pipeline, path)
-
-    @classmethod
-    def load(cls, path: str):
-        detector = cls()
-        detector.pipeline = joblib.load(path)
-        return detector
-```
-
-### One-Class SVM & LOF
-
-```python
-from sklearn.svm import OneClassSVM
-from sklearn.neighbors import LocalOutlierFactor
-from sklearn.preprocessing import RobustScaler
-
-# One-Class SVM (robust to outliers)
-class RobustDetector:
-    def __init__(self, nu: float = 0.05):
-        self.pipeline = Pipeline([
-            ('scaler', RobustScaler()),
-            ('svm', OneClassSVM(nu=nu, kernel='rbf', gamma='scale'))
-        ])
-
-    def fit(self, X):
-        self.pipeline.fit(X)
-        return self
-
-    def predict(self, X):
-        return self.pipeline.predict(X)
-
-# Local Outlier Factor (streaming)
-class StreamingLOF:
-    def __init__(self, n_neighbors: int = 20, contamination: float = 0.05):
-        self.lof = LocalOutlierFactor(
-            n_neighbors=n_neighbors,
-            contamination=contamination,
-            novelty=True  # Enable predict on new data
-        )
-
-    def fit(self, X):
-        self.lof.fit(X)
-        return self
-
-    def predict(self, X):
-        return self.lof.predict(X)
-```
-
-## Classification
-
-```python
-from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
-from sklearn.preprocessing import StandardScaler, LabelEncoder
-from sklearn.pipeline import Pipeline
-from sklearn.metrics import classification_report
-import pandas as pd
-
-class EquipmentClassifier:
-    STATES = ['normal', 'degraded', 'maintenance_required', 'critical']
-
-    def __init__(self, model_type: str = 'random_forest'):
-        self.label_encoder = LabelEncoder()
-        self.label_encoder.fit(self.STATES)
-
-        classifier = RandomForestClassifier(
-            n_estimators=200, max_depth=10, min_samples_split=5,
-            class_weight='balanced', n_jobs=-1, random_state=42
-        ) if model_type == 'random_forest' else GradientBoostingClassifier(
-            n_estimators=200, max_depth=5, learning_rate=0.1, random_state=42
-        )
-
-        self.pipeline = Pipeline([
-            ('scaler', StandardScaler()),
-            ('classifier', classifier)
-        ])
-
-    def fit(self, X: np.ndarray, y: np.ndarray):
-        if isinstance(y[0], str):
-            y = self.label_encoder.transform(y)
-        self.pipeline.fit(X, y)
-        return self
-
-    def predict(self, X: np.ndarray):
-        y_pred = self.pipeline.predict(X)
-        return self.label_encoder.inverse_transform(y_pred)
-
-    def predict_proba(self, X: np.ndarray) -> pd.DataFrame:
-        proba = self.pipeline.predict_proba(X)
-        return pd.DataFrame(proba, columns=self.STATES)
-
-    def feature_importance(self, feature_names: list) -> pd.DataFrame:
-        importances = self.pipeline.named_steps['classifier'].feature_importances_
-        return pd.DataFrame({'feature': feature_names, 'importance': importances}).sort_values('importance', ascending=False)
-```
-
-## Feature Engineering
-
-```python
-def extract_features(df: pd.DataFrame, window_size: int = 60) -> pd.DataFrame:
-    features = pd.DataFrame()
-
-    for col in ['temperature', 'pressure', 'vibration', 'flow_rate']:
-        features[f'{col}_mean'] = df[col].rolling(window_size).mean()
-        features[f'{col}_std'] = df[col].rolling(window_size).std()
-        features[f'{col}_min'] = df[col].rolling(window_size).min()
-        features[f'{col}_max'] = df[col].rolling(window_size).max()
-        features[f'{col}_range'] = features[f'{col}_max'] - features[f'{col}_min']
-        features[f'{col}_diff'] = df[col].diff()
-        features[f'{col}_diff_mean'] = features[f'{col}_diff'].rolling(window_size).mean()
-
-    return features.dropna()
-```
-
-## Forecasting (Multi-step)
-
-```python
-from sklearn.multioutput import MultiOutputRegressor
-from sklearn.ensemble import GradientBoostingRegressor
-
-class Forecaster:
-    def __init__(self, horizon: int = 12, lookback: int = 24):
-        self.horizon = horizon
-        self.lookback = lookback
-        self.pipeline = Pipeline([
-            ('scaler', StandardScaler()),
-            ('regressor', MultiOutputRegressor(
-                GradientBoostingRegressor(n_estimators=100, max_depth=5, random_state=42),
-                n_jobs=-1
-            ))
-        ])
-
-    def create_sequences(self, data: np.ndarray):
-        X, y = [], []
-        for i in range(len(data) - self.lookback - self.horizon + 1):
-            X.append(data[i:i + self.lookback].flatten())
-            y.append(data[i + self.lookback:i + self.lookback + self.horizon].flatten())
-        return np.array(X), np.array(y)
-
-    def fit(self, data: np.ndarray):
-        X, y = self.create_sequences(data)
-        self.pipeline.fit(X, y)
-        return self
-
-    def predict(self, recent_data: np.ndarray) -> np.ndarray:
-        X = recent_data.flatten().reshape(1, -1)
-        y_pred = self.pipeline.predict(X)
-        return y_pred.reshape(self.horizon, recent_data.shape[1])
-```
-
-## Clustering
-
-```python
-from sklearn.cluster import KMeans, DBSCAN
-from sklearn.decomposition import PCA
-from sklearn.metrics import silhouette_score
-
-class ModeDetector:
-    def __init__(self, n_modes: int = None, method: str = 'kmeans'):
-        self.n_modes = n_modes
-        self.scaler = StandardScaler()
-        self.pca = PCA(n_components=0.95)
-        self.clusterer = None
-
-    def fit(self, X: np.ndarray):
-        X_scaled = self.scaler.fit_transform(X)
-        X_pca = self.pca.fit_transform(X_scaled)
-
-        if self.n_modes is None:
-            self.n_modes = self._find_optimal_k(X_pca)
-
-        self.clusterer = KMeans(n_clusters=self.n_modes, n_init=10, random_state=42)
-        self.clusterer.fit(X_pca)
-        return self
-
-    def predict(self, X: np.ndarray):
-        X_pca = self.pca.transform(self.scaler.transform(X))
-        return self.clusterer.predict(X_pca)
-
-    def _find_optimal_k(self, X: np.ndarray, max_k: int = 10) -> int:
-        scores = [silhouette_score(X, KMeans(n_clusters=k, n_init=10).fit_predict(X))
-                  for k in range(2, max_k + 1)]
-        return np.argmax(scores) + 2
-```
-
-## Hyperparameter Tuning
-
-```python
-from sklearn.model_selection import RandomizedSearchCV, TimeSeriesSplit
-
-def tune_classifier(X: np.ndarray, y: np.ndarray):
-    tscv = TimeSeriesSplit(n_splits=5)
-
-    param_grid = {
-        'n_estimators': [100, 200, 300],
-        'max_depth': [5, 10, 15, None],
-        'min_samples_split': [2, 5, 10],
-        'min_samples_leaf': [1, 2, 4],
-    }
-
-    search = RandomizedSearchCV(
-        RandomForestClassifier(random_state=42, n_jobs=-1),
-        param_grid,
-        n_iter=50,
-        cv=tscv,
-        scoring='f1_weighted',
-        n_jobs=-1
-    )
-    search.fit(X, y)
-    return search.best_estimator_, search.best_params_
-```
-
-## Model Persistence
-
-```python
-import json
-from pathlib import Path
-from datetime import datetime
-
-def save_model(model, path: str, metadata: dict = None, feature_names: list = None):
-    path = Path(path)
-    path.parent.mkdir(parents=True, exist_ok=True)
-    joblib.dump(model, path)
-
-    meta = {
-        'saved_at': datetime.utcnow().isoformat(),
-        'sklearn_version': __import__('sklearn').__version__,
-        'feature_names': feature_names,
-        **(metadata or {})
-    }
-    path.with_suffix('.json').write_text(json.dumps(meta, indent=2))
-
-def load_model(path: str):
-    path = Path(path)
-    model = joblib.load(path)
-    meta_path = path.with_suffix('.json')
-    metadata = json.loads(meta_path.read_text()) if meta_path.exists() else {}
-    return model, metadata
-```
-
-## Best Practices
-
-1. **Use pipelines** - Combine preprocessing and model for reproducibility
-2. **Handle imbalance** - SMOTE or `class_weight='balanced'`
-3. **Time series CV** - Use `TimeSeriesSplit` instead of random splits
-4. **Feature importance** - Analyze with `.feature_importances_`
-5. **Version models** - Save with metadata (version, features, metrics)
-
-## Related Skills
-
-- `mlflow`: Experiment tracking
-- `pytorch`: Deep learning alternative
-- `duckdb-analytics`: Data preprocessing
-- `onnx-inference`: Model deployment

package/ai-config/skills/data-ai/vector-db/SKILL.md

@@ -1,301 +0,0 @@
----
-name: vector-db
-description: >
-  Vector databases for RAG and semantic search with ChromaDB and pgvector.
-  Trigger: vector database, embeddings, rag, semantic search, chromadb, pgvector
-tools:
-  - Read
-  - Write
-  - Bash
-  - Grep
-metadata:
-  author: plataforma-industrial
-  version: "2.0"
-  tags: [vector-db, rag, embeddings, semantic-search, chromadb, pgvector]
-  updated: "2026-02"
----
-
-# Vector Database Skill
-
-Vector databases for RAG and semantic search.
-
-## Stack
-
-```yaml
-# Primary
-chromadb: 0.4+
-pgvector: 0.6+  # PostgreSQL extension
-
-# Alternatives
-qdrant-client: 1.7+
-pinecone-client: 3.0+
-
-# Embeddings
-openai: 1.12+
-sentence-transformers: 2.5+
-```
-
-## ChromaDB
-
-```python
-import chromadb
-from chromadb.config import Settings
-from chromadb.utils import embedding_functions
-import os
-
-class ChromaVectorStore:
-    def __init__(self, collection_name: str = "documents", persist_dir: str = "./chroma_db"):
-        self.client = chromadb.PersistentClient(
-            path=persist_dir,
-            settings=Settings(anonymized_telemetry=False)
-        )
-
-        self.embedding_fn = embedding_functions.OpenAIEmbeddingFunction(
-            api_key=os.getenv("OPENAI_API_KEY"),
-            model_name="text-embedding-3-small"
-        )
-
-        self.collection = self.client.get_or_create_collection(
-            name=collection_name,
-            embedding_function=self.embedding_fn,
-            metadata={"hnsw:space": "cosine"}
-        )
-
-    def add_documents(self, documents: list[str], metadatas: list[dict] = None, ids: list[str] = None):
-        if ids is None:
-            ids = [f"doc_{i}" for i in range(len(documents))]
-        self.collection.add(documents=documents, metadatas=metadatas, ids=ids)
-
-    def query(self, query_text: str, n_results: int = 5, where: dict = None):
-        return self.collection.query(
-            query_texts=[query_text],
-            n_results=n_results,
-            where=where,
-            include=["documents", "metadatas", "distances"]
-        )
-
-    def delete(self, ids: list[str] = None, where: dict = None):
-        if ids:
-            self.collection.delete(ids=ids)
-        elif where:
-            self.collection.delete(where=where)
-```
-
-## pgvector (PostgreSQL)
-
-### Schema
-
-```sql
-CREATE EXTENSION IF NOT EXISTS vector;
-
-CREATE TABLE documents (
-    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-    content TEXT NOT NULL,
-    embedding vector(1536),  -- OpenAI dimension
-    metadata JSONB DEFAULT '{}',
-    tenant_id UUID NOT NULL,
-    created_at TIMESTAMPTZ DEFAULT NOW()
-);
-
--- HNSW index for fast search
-CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64);
-CREATE INDEX idx_documents_tenant ON documents (tenant_id);
-CREATE INDEX idx_documents_metadata ON documents USING gin (metadata);
-```
-
-### Python Client
-
-```python
-import asyncpg
-from pgvector.asyncpg import register_vector
-import numpy as np
-from openai import AsyncOpenAI
-
-class PgVectorStore:
-    def __init__(self, pool: asyncpg.Pool, embedding_model: str = "text-embedding-3-small"):
-        self.pool = pool
-        self.openai = AsyncOpenAI()
-        self.embedding_model = embedding_model
-
-    async def setup(self):
-        async with self.pool.acquire() as conn:
-            await register_vector(conn)
-
-    async def embed(self, texts: list[str]) -> list[np.ndarray]:
-        response = await self.openai.embeddings.create(model=self.embedding_model, input=texts)
-        return [np.array(e.embedding) for e in response.data]
-
-    async def add_documents(self, contents: list[str], metadatas: list[dict], tenant_id: str) -> list[str]:
-        embeddings = await self.embed(contents)
-        ids = []
-        async with self.pool.acquire() as conn:
-            for content, embedding, metadata in zip(contents, embeddings, metadatas):
-                row = await conn.fetchrow(
-                    "INSERT INTO documents (content, embedding, metadata, tenant_id) VALUES ($1, $2, $3, $4) RETURNING id",
-                    content, embedding, metadata, tenant_id
-                )
-                ids.append(str(row['id']))
-        return ids
-
-    async def similarity_search(self, query: str, tenant_id: str, k: int = 5, metadata_filter: dict = None):
-        query_embedding = (await self.embed([query]))[0]
-
-        sql = """
-            SELECT id, content, metadata, 1 - (embedding <=> $1) AS similarity
-            FROM documents WHERE tenant_id = $2
-        """
-        params = [query_embedding, tenant_id]
-
-        if metadata_filter:
-            sql += " AND metadata @> $3"
-            params.append(metadata_filter)
-
-        sql += f" ORDER BY embedding <=> $1 LIMIT ${len(params) + 1}"
-        params.append(k)
-
-        async with self.pool.acquire() as conn:
-            rows = await conn.fetch(sql, *params)
-
-        return [{"id": str(r['id']), "content": r['content'], "metadata": dict(r['metadata']), "similarity": float(r['similarity'])} for r in rows]
-
-    async def hybrid_search(self, query: str, tenant_id: str, k: int = 5, keyword_weight: float = 0.3):
-        query_embedding = (await self.embed([query]))[0]
-
-        sql = """
-            WITH vector_search AS (
-                SELECT id, content, metadata, 1 - (embedding <=> $1) AS vector_score
-                FROM documents WHERE tenant_id = $2
-                ORDER BY embedding <=> $1 LIMIT $3 * 2
-            ),
-            keyword_search AS (
-                SELECT id, ts_rank(to_tsvector('english', content), plainto_tsquery('english', $4)) AS keyword_score
-                FROM documents WHERE tenant_id = $2 AND to_tsvector('english', content) @@ plainto_tsquery('english', $4)
-            )
-            SELECT v.id, v.content, v.metadata,
-                   (1 - $5) * v.vector_score + $5 * COALESCE(k.keyword_score, 0) AS combined_score
-            FROM vector_search v LEFT JOIN keyword_search k ON v.id = k.id
-            ORDER BY combined_score DESC LIMIT $3
-        """
-
-        async with self.pool.acquire() as conn:
-            rows = await conn.fetch(sql, query_embedding, tenant_id, k, query, keyword_weight)
-
-        return [{"id": str(r['id']), "content": r['content'], "metadata": dict(r['metadata']), "score": float(r['combined_score'])} for r in rows]
-```
-
-## Document Chunking
-
-```python
-from langchain_text_splitters import RecursiveCharacterTextSplitter, MarkdownHeaderTextSplitter
-
-def chunk_documents(documents: list[dict], chunk_size: int = 1000, chunk_overlap: int = 200) -> list[dict]:
-    splitter = RecursiveCharacterTextSplitter(
-        chunk_size=chunk_size,
-        chunk_overlap=chunk_overlap,
-        separators=["\n\n", "\n", ". ", " ", ""]
-    )
-
-    chunks = []
-    for doc in documents:
-        for i, chunk in enumerate(splitter.split_text(doc["content"])):
-            chunks.append({
-                "content": chunk,
-                "metadata": {**doc.get("metadata", {}), "source_id": doc.get("id"), "chunk_index": i}
-            })
-    return chunks
-
-def chunk_markdown(content: str, source_metadata: dict = None) -> list[dict]:
-    header_splitter = MarkdownHeaderTextSplitter(
-        headers_to_split_on=[("#", "h1"), ("##", "h2"), ("###", "h3")]
-    )
-    size_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
-
-    chunks = []
-    for split in header_splitter.split_text(content):
-        text, metadata = split.page_content, {**(source_metadata or {}), **split.metadata}
-        if len(text) > 1000:
-            for i, sub_chunk in enumerate(size_splitter.split_text(text)):
-                chunks.append({"content": sub_chunk, "metadata": {**metadata, "sub_chunk": i}})
-        else:
-            chunks.append({"content": text, "metadata": metadata})
-    return chunks
-```
-
-## RAG Chain
-
-```python
-from openai import AsyncOpenAI
-
-class RAGChain:
-    def __init__(self, vector_store: PgVectorStore, model: str = "gpt-4-turbo-preview", k: int = 5):
-        self.vector_store = vector_store
-        self.openai = AsyncOpenAI()
-        self.model = model
-        self.k = k
-
-    async def query(self, question: str, tenant_id: str, system_prompt: str = None, metadata_filter: dict = None):
-        docs = await self.vector_store.similarity_search(
-            query=question, tenant_id=tenant_id, k=self.k, metadata_filter=metadata_filter
-        )
-
-        context = "\n\n---\n\n".join([f"[Source: {d['metadata'].get('source', 'Unknown')}]\n{d['content']}" for d in docs])
-
-        if system_prompt is None:
-            system_prompt = "Answer based on the provided context. If not in context, say so. Cite sources."
-
-        response = await self.openai.chat.completions.create(
-            model=self.model,
-            messages=[
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": f"Context:\n{context}\n\nQuestion: {question}"}
-            ],
-            temperature=0
-        )
-
-        return {
-            "answer": response.choices[0].message.content,
-            "sources": [d['metadata'].get('source') for d in docs],
-            "docs": docs
-        }
-```
-
-## Indexing Pipeline
-
-```python
-class IndexingPipeline:
-    def __init__(self, vector_store: PgVectorStore, chunk_size: int = 1000, batch_size: int = 100):
-        self.vector_store = vector_store
-        self.chunk_size = chunk_size
-        self.batch_size = batch_size
-
-    async def index_documents(self, documents: list[dict], tenant_id: str) -> int:
-        chunks = chunk_documents(documents, chunk_size=self.chunk_size)
-        total = 0
-
-        for i in range(0, len(chunks), self.batch_size):
-            batch = chunks[i:i + self.batch_size]
-            await self.vector_store.add_documents(
-                contents=[c['content'] for c in batch],
-                metadatas=[c['metadata'] for c in batch],
-                tenant_id=tenant_id
-            )
-            total += len(batch)
-            print(f"Indexed {total}/{len(chunks)}")
-
-        return total
-```
-
-## Best Practices
-
-1. **Chunk size** - Technical docs: 1000-1500, FAQs: 300-500
-2. **Include overlap** - 10-20% overlap for context continuity
-3. **Use metadata** - Filter by type, source, date
-4. **Hybrid search** - Combine vector + keyword for better recall
-5. **Re-rank results** - Cross-encoder for precision: `cross-encoder/ms-marco-MiniLM-L-6-v2`
-
-## Related Skills
-
-- `langchain`: RAG implementation
-- `ai-ml`: Embedding generation
-- `fastapi`: Vector search API
-- `redis-cache`: Hybrid search caching