@nockdev/awf 6.2.0 → 6.2.2

package/.agent/skills/ai-ml/rag-patterns/data/retrieval-patterns.yaml

@@ -0,0 +1,101 @@
+# =============================================================================
+# retrieval-patterns.yaml — Advanced Retrieval Patterns
+# =============================================================================
+
+version: "1.0.0"
+
+hybrid_search:
+  description: "Combine dense (vector) + sparse (keyword) search"
+  method: |
+    Score = α × dense_score + (1-α) × sparse_score
+    α = 0.7 (typical, tune per use case)
+  implementations:
+    - name: "Qdrant"
+      method: "Sparse + dense vectors natively"
+    - name: "Weaviate"
+      method: "BM25 + vector built-in"
+    - name: "pgvector + pg_trgm"
+      method: "Vector similarity + trigram text search"
+  example: |
+    # Qdrant hybrid search
+    from qdrant_client.models import SparseVector
+    results = client.query_points(
+        collection_name="docs",
+        prefetch=[
+            Prefetch(query=dense_vector, using="dense", limit=20),
+            Prefetch(query=sparse_vector, using="sparse", limit=20),
+        ],
+        query=FusionQuery(fusion=Fusion.RRF),  # Reciprocal Rank Fusion
+        limit=5,
+    )
+
+reranking:
+  description: "Re-score top-K results with a cross-encoder"
+  models:
+    - "Cohere Rerank v3"
+    - "BAAI/bge-reranker-v2-m3"
+    - "cross-encoder/ms-marco-MiniLM-L-6-v2"
+  pipeline: |
+    1. Retrieve top-20 from vector search
+    2. Re-rank with cross-encoder model
+    3. Return top-5 after reranking
+  example_python: |
+    from cohere import Client
+    co = Client("API_KEY")
+
+    results = co.rerank(
+        model="rerank-v3.5",
+        query="What is RAG?",
+        documents=retrieved_chunks,
+        top_n=5
+    )
+
+hyde:
+  name: "Hypothetical Document Embeddings"
+  description: |
+    Generate a hypothetical answer to the query, embed it, then
+    search for similar real documents. Improves retrieval for
+    questions that don't lexically match the source.
+  pipeline: |
+    Query → LLM generates hypothetical answer → Embed answer → Search
+  example_python: |
+    # 1. Generate hypothetical answer
+    hypothetical = llm.generate(f"Answer this question: {query}")
+
+    # 2. Embed the hypothetical answer
+    embedding = embed_model.encode(hypothetical)
+
+    # 3. Search with hypothetical embedding
+    results = vector_store.similarity_search(embedding, k=5)
+
+self_query:
+  description: "LLM extracts structured filters from natural language query"
+  example: |
+    Query: "What papers about transformers were published after 2023?"
+    → filter: {"year": {">": 2023}, "topic": "transformers"}
+    → vector search with metadata filter
+
+multi_query:
+  description: "Generate multiple query variants for broader retrieval"
+  pipeline: |
+    Original query → LLM generates 3-5 variants → Search each → Deduplicate → Rank
+  example: |
+    Query: "How to handle errors in Go?"
+    Variants:
+      - "Go error handling patterns"
+      - "golang error wrapping best practices"
+      - "Go error types and sentinel errors"
+
+contextual_compression:
+  description: "Extract only relevant parts from retrieved chunks"
+  pipeline: |
+    Retrieved chunk (500 tokens) → LLM extracts relevant sentences → Compressed (100 tokens)
+  benefit: "Fits more relevant context in LLM prompt"
+
+parent_document:
+  description: "Store small chunks for retrieval, return parent doc for context"
+  pipeline: |
+    Document → Split into small chunks (200 tokens) → Index
+    Query → Match small chunk → Return parent section (2000 tokens)
+
+# _DOMYH Awesome Code • Retrieval Patterns_

package/.agent/skills/ai-ml/rag-patterns/data/vector-databases.yaml

@@ -0,0 +1,103 @@
+# =============================================================================
+# vector-databases.yaml — Vector Database Comparison
+# =============================================================================
+
+version: "1.0.0"
+
+databases:
+  pgvector:
+    type: "PostgreSQL extension"
+    hosting: "Self-hosted or managed (Supabase, Neon)"
+    best_for: "Already using PostgreSQL, small-medium scale"
+    setup: |
+      CREATE EXTENSION vector;
+      CREATE TABLE documents (
+          id SERIAL PRIMARY KEY,
+          content TEXT,
+          metadata JSONB,
+          embedding vector(1536)
+      );
+      CREATE INDEX ON documents USING ivfflat (embedding vector_cosine_ops) WITH (lists = 100);
+    query: |
+      SELECT content, 1 - (embedding <=> $1) AS similarity
+      FROM documents
+      ORDER BY embedding <=> $1
+      LIMIT 5;
+
+  pinecone:
+    type: "Managed SaaS"
+    hosting: "Cloud (AWS, GCP, Azure)"
+    best_for: "Production, serverless, managed"
+    example_python: |
+      from pinecone import Pinecone
+      pc = Pinecone(api_key="YOUR_KEY")
+      index = pc.Index("my-index")
+
+      # Upsert
+      index.upsert(vectors=[
+          {"id": "doc1", "values": embedding, "metadata": {"source": "doc.pdf"}}
+      ])
+
+      # Query
+      results = index.query(vector=query_embedding, top_k=5, include_metadata=True)
+
+  qdrant:
+    type: "Self-hosted / Cloud"
+    hosting: "Docker, Kubernetes, Qdrant Cloud"
+    best_for: "Full control, advanced filtering"
+    features: ["Payload filtering", "Sparse vectors", "Hybrid search"]
+    example_python: |
+      from qdrant_client import QdrantClient
+      client = QdrantClient("localhost", port=6333)
+
+      client.upsert(collection_name="docs", points=[
+          {"id": 1, "vector": embedding, "payload": {"source": "doc.pdf"}}
+      ])
+
+      results = client.search(
+          collection_name="docs",
+          query_vector=query_embedding,
+          query_filter={"must": [{"key": "source", "match": {"value": "doc.pdf"}}]},
+          limit=5
+      )
+
+  chroma:
+    type: "Embedded / Server"
+    hosting: "In-process or Docker"
+    best_for: "Prototyping, development, small datasets"
+    example_python: |
+      import chromadb
+      client = chromadb.Client()
+      collection = client.create_collection("docs")
+
+      collection.add(
+          documents=["text1", "text2"],
+          metadatas=[{"source": "a"}, {"source": "b"}],
+          ids=["1", "2"]
+      )
+
+      results = collection.query(query_texts=["search query"], n_results=5)
+
+  weaviate:
+    type: "Self-hosted / Cloud"
+    best_for: "Multi-modal (text + images), hybrid search built-in"
+
+  milvus:
+    type: "Self-hosted"
+    best_for: "Large scale (billion vectors), GPU acceleration"
+
+selection_guide:
+  - scenario: "Already using PostgreSQL"
+    choice: "pgvector"
+  - scenario: "Need managed, production-ready"
+    choice: "Pinecone"
+  - scenario: "Need advanced filtering, self-hosted"
+    choice: "Qdrant"
+  - scenario: "Quick prototype, local dev"
+    choice: "Chroma"
+  - scenario: "Multi-modal (text + images)"
+    choice: "Weaviate"
+  - scenario: "Billion+ vectors, GPU"
+    choice: "Milvus"
+
+# _DOMYH Awesome Code • Vector Databases_

package/.agent/skills/ai-ml/vector-search/META.yaml

@@ -0,0 +1,63 @@
+name: vector-search
+version: "1.0.0"
+display: Vector Search & Embeddings
+category: ai-ml
+tier: 1
+priority: 2
+
+desc: "Vector databases & semantic search. Embeddings, similarity search, hybrid search (vector + BM25), indexing strategies, and production patterns."
+
+  ### Data Files (3 domains, ~21 patterns)
+  - vector-dbs.yaml: comparison table, features, pricing (7)
+  - embedding-models.yaml: model comparison, dimensions, perf (7)
+  - search-patterns.yaml: hybrid search, filtering, reranking (7)
+
+triggers:
+  file_patterns:
+    - "**/embeddings/**"
+    - "**/vector/**"
+    - "chromadb/**"
+  keywords:
+    - vector database
+    - vector search
+    - vector store
+    - embeddings
+    - semantic search
+    - similarity search
+    - cosine similarity
+    - HNSW
+    - ANN
+    - approximate nearest neighbor
+    - pinecone
+    - weaviate
+    - qdrant
+    - chromadb
+    - chroma
+    - milvus
+    - pgvector
+    - text embedding
+    - embedding model
+    - hybrid search
+    - dense retrieval
+
+caps:
+  - Embedding generation (OpenAI, Cohere, Voyage, local models)
+  - Vector DB selection (Pinecone, Weaviate, Qdrant, ChromaDB, Milvus)
+  - pgvector (PostgreSQL native vector search)
+  - Indexing strategies (HNSW, IVF, PQ, SCANN)
+  - Hybrid Search (vector + keyword BM25)
+  - Metadata filtering & namespace isolation
+  - Batch upsert & real-time indexing
+  - Distance metrics (cosine, L2 euclidean, dot product)
+  - Reranking (Cohere Rerank, cross-encoder)
+  - Chunking strategies for documents
+
+related_skills:
+  - ai-agents
+  - rag-patterns
+  - database
+
+hsa_adapter:
+  enabled: true
+  data_dir: "skills/ai-ml/vector-search/data"
+  preload_data: true

package/.agent/skills/ai-ml/vector-search/SKILL.md

@@ -0,0 +1,110 @@
+# Vector Search & Embeddings
+
+> Vector databases • Semantic search • Hybrid retrieval  
+> Pinecone • Weaviate • Qdrant • ChromaDB • pgvector
+
+---
+
+## Khi Nào Dùng
+
+- Implement semantic search (tìm kiếm theo ý nghĩa)
+- Build RAG (Retrieval-Augmented Generation) pipeline
+- Chọn vector database phù hợp
+- Optimize embedding & search performance
+
+## Architecture
+
+```
+Documents → Chunking → Embedding Model → Vector DB
+                                              │
+Query → Embedding Model ──────────────────────┤
+                                              ▼
+                                     Similarity Search
+                                              │
+                                     Reranking (optional)
+                                              │
+                                     Top-K Results
+```
+
+## Vector DB Selection
+
+| DB           | Type                 | Scale     | Best For                    |
+| ------------ | -------------------- | --------- | --------------------------- |
+| **Pinecone** | Managed cloud        | Billions  | Production SaaS, zero-ops   |
+| **Weaviate** | Self-hosted/cloud    | Millions  | Hybrid search, multi-modal  |
+| **Qdrant**   | Self-hosted/cloud    | Millions  | Performance, filtering      |
+| **ChromaDB** | Embedded/self-hosted | Thousands | Dev/prototype, simple       |
+| **Milvus**   | Self-hosted          | Billions  | Enterprise, GPU-accelerated |
+| **pgvector** | PostgreSQL extension | Millions  | Already using PostgreSQL    |
+
+## Core Patterns
+
+### Embedding Generation
+
+```typescript
+// OpenAI embeddings
+const response = await openai.embeddings.create({
+  model: "text-embedding-3-small",
+  input: "Search query or document text",
+});
+const embedding = response.data[0].embedding; // 1536 dimensions
+```
+
+### Chunking Strategy
+
+| Strategy   | Chunk Size      | Overlap    | Best For        |
+| ---------- | --------------- | ---------- | --------------- |
+| Fixed-size | 500-1000 tokens | 50-100     | General purpose |
+| Sentence   | 3-5 sentences   | 1 sentence | Q&A             |
+| Semantic   | Variable        | Paragraph  | Long documents  |
+| Code       | Function/class  | None       | Codebase search |
+
+### Hybrid Search (Vector + BM25)
+
+```typescript
+// Weaviate hybrid search
+const result = await client.graphql
+  .get()
+  .withClassName("Document")
+  .withHybrid({ query: "machine learning", alpha: 0.5 }) // 0=BM25, 1=vector
+  .withLimit(10)
+  .do();
+```
+
+### pgvector Quick Start
+
+```sql
+CREATE EXTENSION vector;
+CREATE TABLE documents (
+  id SERIAL PRIMARY KEY,
+  content TEXT,
+  embedding vector(1536)
+);
+CREATE INDEX ON documents USING hnsw (embedding vector_cosine_ops);
+
+-- Search
+SELECT content, 1 - (embedding <=> query_embedding) AS similarity
+FROM documents ORDER BY embedding <=> query_embedding LIMIT 10;
+```
+
+## Distance Metrics
+
+| Metric             | Formula    | When to Use                         |
+| ------------------ | ---------- | ----------------------------------- |
+| **Cosine**         | 1 - cos(θ) | Normalized embeddings (most common) |
+| **L2 (Euclidean)** | √Σ(a-b)²   | When magnitude matters              |
+| **Dot Product**    | Σ(a×b)     | Already normalized, fastest         |
+
+## Common Traps
+
+| Trap                | Fix                                        |
+| ------------------- | ------------------------------------------ |
+| Wrong chunk size    | Test 256/512/1024 tokens, measure recall   |
+| Slow search         | Use HNSW index, reduce dimensions          |
+| Poor recall         | Hybrid search (vector + BM25)              |
+| High embedding cost | Batch embeddings, cache, use smaller model |
+| Stale data          | Implement incremental indexing pipeline    |
+
+---
+
+_DOMYH Awesome Code • Vector Search Skill v1.0.0_

package/.agent/skills/ai-ml/vector-search/data/embedding-models.yaml

@@ -0,0 +1,117 @@
+# Embedding Models Comparison — HSA Tier 3 Data
+version: "1.0.0"
+
+patterns:
+  - id: EMB-001
+    name: OpenAI text-embedding-3
+    category: commercial
+    description: |
+      OpenAI embedding models.
+    text: |
+      Models:
+      - text-embedding-3-small: 1536 dims, $0.02/1M tokens, best value
+      - text-embedding-3-large: 3072 dims, $0.13/1M tokens, best quality
+      - text-embedding-ada-002: 1536 dims (legacy, deprecated)
+
+      Features: Dimension reduction via dimensions parameter
+      API: openai.embeddings.create()
+    example: |
+      const response = await openai.embeddings.create({
+        model: "text-embedding-3-small",
+        input: ["search query", "document text"],
+        dimensions: 512  // Optional: reduce for cost savings
+      });
+      const embeddings = response.data.map(d => d.embedding);
+
+  - id: EMB-002
+    name: Cohere Embed v3
+    category: commercial
+    description: |
+      Cohere embedding with separate query/document modes.
+    text: |
+      Model: embed-multilingual-v3.0
+      Dims: 1024
+      Special: input_type parameter (search_query vs search_document)
+      Multilingual: 100+ languages
+    example: |
+      const response = await cohere.embed({
+        texts: ["search query"],
+        model: "embed-multilingual-v3.0",
+        inputType: "search_query",  // or "search_document" for indexing
+        embeddingTypes: ["float"]
+      });
+
+  - id: EMB-003
+    name: Voyage AI
+    category: commercial
+    description: |
+      Specialized embeddings for code and text.
+    text: |
+      Models:
+      - voyage-3: General purpose, 1024 dims
+      - voyage-code-3: Code-optimized, 1024 dims
+      - voyage-3-lite: Lightweight, 512 dims
+
+      Best for: Code search, technical documentation
+
+  - id: EMB-004
+    name: Local Models (Sentence-Transformers)
+    category: open-source
+    description: |
+      Free, self-hosted embedding models.
+    text: |
+      Top models:
+      - all-MiniLM-L6-v2: 384 dims, fastest, good quality
+      - all-mpnet-base-v2: 768 dims, best quality/speed
+      - multilingual-e5-large: 1024 dims, multilingual
+      - nomic-embed-text-v1.5: 768 dims, Matryoshka
+
+      Frameworks: sentence-transformers, ollama, fastembed
+    example: |
+      from sentence_transformers import SentenceTransformer
+      model = SentenceTransformer("all-MiniLM-L6-v2")
+      embeddings = model.encode(["text1", "text2"])
+
+  - id: EMB-005
+    name: Chunking Strategies
+    category: preprocessing
+    description: |
+      How to split documents before embedding.
+    text: |
+      | Strategy | Chunk Size | Overlap | Use Case |
+      |----------|-----------|---------|----------|
+      | Fixed tokens | 500-1000 | 50-100 | General purpose |
+      | Sentences | 3-5 sentences | 1 | Q&A systems |
+      | Paragraphs | Variable | 0 | Articles, blogs |
+      | Semantic | Variable | 0 | Mixed content |
+      | Code | Function/class | 0 | Codebase search |
+      | Markdown | Headers | 0 | Documentation |
+
+      Tools: LangChain TextSplitters, LlamaIndex NodeParsers
+
+  - id: EMB-006
+    name: Embedding Best Practices
+    category: optimization
+    description: |
+      Production embedding optimization.
+    text: |
+      1. Batch embeddings (reduce API calls)
+      2. Cache embeddings (avoid re-computing)
+      3. Use smaller dimensions when possible
+      4. Separate query vs document embeddings if model supports it
+      5. Normalize embeddings for cosine similarity
+      6. Use Matryoshka models for dimension flexibility
+
+  - id: EMB-007
+    name: Model Selection Guide
+    category: comparison
+    description: |
+      How to choose embedding model.
+    text: |
+      Decision flow:
+      Budget zero → Local: all-MiniLM-L6-v2 or nomic-embed
+      Need multilingual → Cohere embed-multilingual-v3
+      Code search → Voyage code-3
+      General best quality → OpenAI text-embedding-3-large
+      Best cost/quality → OpenAI text-embedding-3-small
+      Privacy/latency → Local: all-mpnet-base-v2

package/.agent/skills/ai-ml/vector-search/data/search-patterns.yaml

@@ -0,0 +1,118 @@
+# Search Patterns — HSA Tier 3 Data
+version: "1.0.0"
+
+patterns:
+  - id: SRCH-001
+    name: Hybrid Search
+    category: retrieval
+    description: |
+      Combine vector similarity with keyword BM25 for better recall.
+    text: |
+      Alpha parameter: 0.0 = pure BM25, 1.0 = pure vector
+      Best: alpha 0.4-0.7 (balanced)
+      Benefit: Catches exact matches that vectors miss
+    example: |
+      // Weaviate hybrid search
+      const results = await collection.query.hybrid("machine learning", {
+        alpha: 0.5,
+        limit: 10,
+        returnMetadata: ["score"]
+      });
+
+      // pgvector + ts_rank hybrid
+      SELECT content,
+        (0.5 * ts_rank(to_tsvector(content), query) + 
+         0.5 * (1 - (embedding <=> query_embedding))) AS score
+      FROM documents, plainto_tsquery('machine learning') query
+      ORDER BY score DESC LIMIT 10;
+
+  - id: SRCH-002
+    name: Metadata Filtering
+    category: retrieval
+    description: |
+      Pre-filter vectors by metadata before similarity search.
+    text: |
+      Strategies:
+      - Pre-filter: Filter first, then vector search (faster)
+      - Post-filter: Vector search first, then filter (better recall)
+      Common filters: date range, category, language, access level
+    example: |
+      // Pinecone with filter
+      await index.query({
+        vector: queryEmbedding,
+        topK: 10,
+        filter: {
+          category: { $eq: "documentation" },
+          date: { $gte: "2025-01-01" }
+        }
+      });
+
+  - id: SRCH-003
+    name: Reranking
+    category: retrieval
+    description: |
+      Rerank initial results for higher precision.
+    text: |
+      Pipeline: Query → Retrieve top-100 → Rerank → Return top-10
+      Models: Cohere Rerank, cross-encoder/ms-marco
+      Benefit: 10-30% precision improvement
+    example: |
+      // Cohere Rerank
+      const reranked = await cohere.rerank({
+        model: "rerank-multilingual-v3.0",
+        query: "What is machine learning?",
+        documents: initialResults.map(r => r.text),
+        topN: 10
+      });
+
+  - id: SRCH-004
+    name: Multi-vector Search
+    category: retrieval
+    description: |
+      Use multiple vectors per document for richer representation.
+    text: |
+      Approaches:
+      - ColBERT: Token-level vectors (late interaction)
+      - Multi-chunk: One vector per chunk, link to parent doc
+      - Multi-modal: Text vector + image vector per document
+
+  - id: SRCH-005
+    name: Namespace Isolation
+    category: architecture
+    description: |
+      Separate vectors by tenant/project for multi-tenancy.
+    text: |
+      Strategies:
+      - Pinecone: Use namespaces
+      - Weaviate: Use multi-tenancy
+      - Qdrant: Use collection aliases or payload filtering
+      - pgvector: Use separate schemas or tenant_id column
+
+  - id: SRCH-006
+    name: Incremental Indexing
+    category: pipeline
+    description: |
+      Keep vector index in sync with source data.
+    text: |
+      Pipeline:
+      1. Detect new/modified/deleted documents
+      2. Chunk modified documents
+      3. Generate embeddings (batch)
+      4. Upsert vectors with document IDs
+      5. Delete vectors for removed documents
+
+      Tools: LlamaIndex IngestionPipeline, custom CDC pipeline
+
+  - id: SRCH-007
+    name: Evaluation Metrics
+    category: quality
+    description: |
+      Measure search quality.
+    text: |
+      Metrics:
+      - Recall@K: % of relevant docs in top-K results
+      - MRR: Mean Reciprocal Rank
+      - NDCG: Normalized Discounted Cumulative Gain
+      - Latency: p50/p95/p99 query time
+
+      Tools: RAGAS, BEIR benchmark, custom eval sets