@rune-kit/rune 2.8.0 → 2.11.0

package/extensions/ai-ml/skills/embedding-search.md

@@ -1,66 +1,66 @@
----
-name: "embedding-search"
-pack: "@rune/ai-ml"
-description: "Embedding-based search — semantic search, hybrid search (BM25 + vector), similarity thresholds, index optimization."
-model: sonnet
-tools: [Read, Edit, Write, Grep, Glob, Bash]
----
-
-# embedding-search
-
-Embedding-based search — semantic search, hybrid search (BM25 + vector), similarity thresholds, index optimization.
-
-#### Workflow
-
-**Step 1 — Detect search implementation**
-Use Grep to find search code: `similarity_search`, `vector_search`, `fts`, `tsvector`, `BM25`. Read search handlers to understand: query flow, ranking strategy, and result formatting.
-
-**Step 2 — Audit search quality**
-Check for: pure vector search without keyword fallback (misses exact matches), no similarity threshold (returns irrelevant results at low scores), missing query embedding cache (repeated queries re-embed), no hybrid scoring (BM25 for exact + vector for semantic), and unoptimized vector index (HNSW parameters not tuned).
-
-**Step 3 — Emit hybrid search**
-Emit: combined BM25 + vector search with reciprocal rank fusion, similarity threshold filtering, query embedding cache, and HNSW index tuning.
-
-#### Example
-
-```typescript
-// Hybrid search — BM25 + vector with reciprocal rank fusion
-async function hybridSearch(query: string, limit = 10) {
-  // Parallel: keyword (BM25) + semantic (vector)
-  const [keywordResults, vectorResults] = await Promise.all([
-    db.execute(sql`
-      SELECT id, content, ts_rank(search_vector, plainto_tsquery(${query})) AS bm25_score
-      FROM documents
-      WHERE search_vector @@ plainto_tsquery(${query})
-      ORDER BY bm25_score DESC LIMIT ${limit * 2}
-    `),
-    db.execute(sql`
-      SELECT id, content, 1 - (embedding <=> ${await getEmbedding(query)}) AS vector_score
-      FROM documents
-      ORDER BY embedding <=> ${await getEmbedding(query)}
-      LIMIT ${limit * 2}
-    `),
-  ]);
-
-  // Reciprocal rank fusion (k=60)
-  const scores = new Map<string, number>();
-  const K = 60;
-  keywordResults.forEach((r, i) => scores.set(r.id, (scores.get(r.id) || 0) + 1 / (K + i + 1)));
-  vectorResults.forEach((r, i) => scores.set(r.id, (scores.get(r.id) || 0) + 1 / (K + i + 1)));
-
-  return [...scores.entries()]
-    .sort((a, b) => b[1] - a[1])
-    .slice(0, limit)
-    .filter(([_, score]) => score > 0.01); // threshold
-}
-
-// Embedding cache (avoid re-embedding repeated queries)
-const embeddingCache = new Map<string, number[]>();
-async function getEmbedding(text: string): Promise<number[]> {
-  const cached = embeddingCache.get(text);
-  if (cached) return cached;
-  const { data } = await openai.embeddings.create({ model: 'text-embedding-3-small', input: text });
-  embeddingCache.set(text, data[0].embedding);
-  return data[0].embedding;
-}
-```
+---
+name: "embedding-search"
+pack: "@rune/ai-ml"
+description: "Embedding-based search — semantic search, hybrid search (BM25 + vector), similarity thresholds, index optimization."
+model: sonnet
+tools: [Read, Edit, Write, Grep, Glob, Bash]
+---
+
+# embedding-search
+
+Embedding-based search — semantic search, hybrid search (BM25 + vector), similarity thresholds, index optimization.
+
+#### Workflow
+
+**Step 1 — Detect search implementation**
+Use Grep to find search code: `similarity_search`, `vector_search`, `fts`, `tsvector`, `BM25`. Read search handlers to understand: query flow, ranking strategy, and result formatting.
+
+**Step 2 — Audit search quality**
+Check for: pure vector search without keyword fallback (misses exact matches), no similarity threshold (returns irrelevant results at low scores), missing query embedding cache (repeated queries re-embed), no hybrid scoring (BM25 for exact + vector for semantic), and unoptimized vector index (HNSW parameters not tuned).
+
+**Step 3 — Emit hybrid search**
+Emit: combined BM25 + vector search with reciprocal rank fusion, similarity threshold filtering, query embedding cache, and HNSW index tuning.
+
+#### Example
+
+```typescript
+// Hybrid search — BM25 + vector with reciprocal rank fusion
+async function hybridSearch(query: string, limit = 10) {
+  // Parallel: keyword (BM25) + semantic (vector)
+  const [keywordResults, vectorResults] = await Promise.all([
+    db.execute(sql`
+      SELECT id, content, ts_rank(search_vector, plainto_tsquery(${query})) AS bm25_score
+      FROM documents
+      WHERE search_vector @@ plainto_tsquery(${query})
+      ORDER BY bm25_score DESC LIMIT ${limit * 2}
+    `),
+    db.execute(sql`
+      SELECT id, content, 1 - (embedding <=> ${await getEmbedding(query)}) AS vector_score
+      FROM documents
+      ORDER BY embedding <=> ${await getEmbedding(query)}
+      LIMIT ${limit * 2}
+    `),
+  ]);
+
+  // Reciprocal rank fusion (k=60)
+  const scores = new Map<string, number>();
+  const K = 60;
+  keywordResults.forEach((r, i) => scores.set(r.id, (scores.get(r.id) || 0) + 1 / (K + i + 1)));
+  vectorResults.forEach((r, i) => scores.set(r.id, (scores.get(r.id) || 0) + 1 / (K + i + 1)));
+
+  return [...scores.entries()]
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, limit)
+    .filter(([_, score]) => score > 0.01); // threshold
+}
+
+// Embedding cache (avoid re-embedding repeated queries)
+const embeddingCache = new Map<string, number[]>();
+async function getEmbedding(text: string): Promise<number[]> {
+  const cached = embeddingCache.get(text);
+  if (cached) return cached;
+  const { data } = await openai.embeddings.create({ model: 'text-embedding-3-small', input: text });
+  embeddingCache.set(text, data[0].embedding);
+  return data[0].embedding;
+}
+```

package/extensions/ai-ml/skills/fine-tuning-guide.md

@@ -1,74 +1,74 @@
----
-name: "fine-tuning-guide"
-pack: "@rune/ai-ml"
-description: "Fine-tuning workflows — dataset preparation, training configuration, evaluation metrics, deployment, A/B testing."
-model: sonnet
-tools: [Read, Edit, Write, Grep, Glob, Bash]
----
-
-# fine-tuning-guide
-
-Fine-tuning workflows — dataset preparation, training configuration, evaluation metrics, deployment, A/B testing.
-
-#### Workflow
-
-**Step 1 — Audit training data**
-Use Read to examine the dataset files. Check for: data format (JSONL with `messages` array), train/eval split (eval must not overlap with train), sufficient examples (minimum 50, recommended 200+), balanced class distribution, and PII in training data.
-
-**Step 2 — Prepare and validate dataset**
-Emit: JSONL formatter that validates each example, train/eval splitter with stratification, token count estimator (cost preview), and data quality checks (duplicate detection, format validation).
-
-**Step 3 — Execute fine-tuning and evaluate**
-Emit: fine-tune API call with hyperparameters, evaluation script that compares base vs fine-tuned on held-out set, and A/B deployment configuration.
-
-#### Example
-
-```python
-# Fine-tuning workflow — prepare, train, evaluate
-import json
-import openai
-from sklearn.model_selection import train_test_split
-
-# Step 1: Prepare JSONL dataset
-def prepare_dataset(examples: list[dict], output_prefix: str):
-    train, eval_set = train_test_split(examples, test_size=0.2, random_state=42)
-
-    for split_name, split_data in [("train", train), ("eval", eval_set)]:
-        path = f"{output_prefix}_{split_name}.jsonl"
-        with open(path, "w") as f:
-            for ex in split_data:
-                f.write(json.dumps({"messages": [
-                    {"role": "system", "content": ex["system"]},
-                    {"role": "user", "content": ex["input"]},
-                    {"role": "assistant", "content": ex["output"]},
-                ]}) + "\n")
-        print(f"Wrote {len(split_data)} examples to {path}")
-
-# Step 2: Launch fine-tuning
-def start_fine_tune(train_file: str, eval_file: str):
-    train_id = openai.files.create(file=open(train_file, "rb"), purpose="fine-tune").id
-    eval_id = openai.files.create(file=open(eval_file, "rb"), purpose="fine-tune").id
-
-    job = openai.fine_tuning.jobs.create(
-        training_file=train_id,
-        validation_file=eval_id,
-        model="gpt-4o-mini-2024-07-18",
-        hyperparameters={"n_epochs": 3, "batch_size": "auto", "learning_rate_multiplier": "auto"},
-    )
-    print(f"Fine-tuning job: {job.id} — status: {job.status}")
-    return job
-
-# Step 3: Evaluate base vs fine-tuned
-def evaluate(base_model: str, ft_model: str, eval_set: list[dict]) -> dict:
-    results = {"base": {"correct": 0}, "finetuned": {"correct": 0}}
-    for ex in eval_set:
-        for label, model in [("base", base_model), ("finetuned", ft_model)]:
-            response = openai.chat.completions.create(
-                model=model, messages=ex["messages"][:2], max_tokens=500,
-            )
-            if response.choices[0].message.content.strip() == ex["messages"][2]["content"].strip():
-                results[label]["correct"] += 1
-    for label in results:
-        results[label]["accuracy"] = results[label]["correct"] / len(eval_set)
-    return results
-```
+---
+name: "fine-tuning-guide"
+pack: "@rune/ai-ml"
+description: "Fine-tuning workflows — dataset preparation, training configuration, evaluation metrics, deployment, A/B testing."
+model: sonnet
+tools: [Read, Edit, Write, Grep, Glob, Bash]
+---
+
+# fine-tuning-guide
+
+Fine-tuning workflows — dataset preparation, training configuration, evaluation metrics, deployment, A/B testing.
+
+#### Workflow
+
+**Step 1 — Audit training data**
+Use Read to examine the dataset files. Check for: data format (JSONL with `messages` array), train/eval split (eval must not overlap with train), sufficient examples (minimum 50, recommended 200+), balanced class distribution, and PII in training data.
+
+**Step 2 — Prepare and validate dataset**
+Emit: JSONL formatter that validates each example, train/eval splitter with stratification, token count estimator (cost preview), and data quality checks (duplicate detection, format validation).
+
+**Step 3 — Execute fine-tuning and evaluate**
+Emit: fine-tune API call with hyperparameters, evaluation script that compares base vs fine-tuned on held-out set, and A/B deployment configuration.
+
+#### Example
+
+```python
+# Fine-tuning workflow — prepare, train, evaluate
+import json
+import openai
+from sklearn.model_selection import train_test_split
+
+# Step 1: Prepare JSONL dataset
+def prepare_dataset(examples: list[dict], output_prefix: str):
+    train, eval_set = train_test_split(examples, test_size=0.2, random_state=42)
+
+    for split_name, split_data in [("train", train), ("eval", eval_set)]:
+        path = f"{output_prefix}_{split_name}.jsonl"
+        with open(path, "w") as f:
+            for ex in split_data:
+                f.write(json.dumps({"messages": [
+                    {"role": "system", "content": ex["system"]},
+                    {"role": "user", "content": ex["input"]},
+                    {"role": "assistant", "content": ex["output"]},
+                ]}) + "\n")
+        print(f"Wrote {len(split_data)} examples to {path}")
+
+# Step 2: Launch fine-tuning
+def start_fine_tune(train_file: str, eval_file: str):
+    train_id = openai.files.create(file=open(train_file, "rb"), purpose="fine-tune").id
+    eval_id = openai.files.create(file=open(eval_file, "rb"), purpose="fine-tune").id
+
+    job = openai.fine_tuning.jobs.create(
+        training_file=train_id,
+        validation_file=eval_id,
+        model="gpt-4o-mini-2024-07-18",
+        hyperparameters={"n_epochs": 3, "batch_size": "auto", "learning_rate_multiplier": "auto"},
+    )
+    print(f"Fine-tuning job: {job.id} — status: {job.status}")
+    return job
+
+# Step 3: Evaluate base vs fine-tuned
+def evaluate(base_model: str, ft_model: str, eval_set: list[dict]) -> dict:
+    results = {"base": {"correct": 0}, "finetuned": {"correct": 0}}
+    for ex in eval_set:
+        for label, model in [("base", base_model), ("finetuned", ft_model)]:
+            response = openai.chat.completions.create(
+                model=model, messages=ex["messages"][:2], max_tokens=500,
+            )
+            if response.choices[0].message.content.strip() == ex["messages"][2]["content"].strip():
+                results[label]["correct"] += 1
+    for label in results:
+        results[label]["accuracy"] = results[label]["correct"] / len(eval_set)
+    return results
+```

package/extensions/ai-ml/skills/llm-architect.md

@@ -1,125 +1,125 @@
----
-name: "llm-architect"
-pack: "@rune/ai-ml"
-description: "LLM system architecture — model selection, prompt engineering patterns, evaluation frameworks, cost optimization, multi-model routing, and guardrail design."
-model: opus
-tools: [Read, Edit, Write, Grep, Glob, Bash]
----
-
-# llm-architect
-
-LLM system architecture — model selection, prompt engineering patterns, evaluation frameworks, cost optimization, multi-model routing, and guardrail design.
-
-#### Workflow
-
-**Step 1 — Assess LLM requirements**
-Understand the use case: what does the LLM need to do? Classify into:
-- **Generation**: open-ended text (blog, email, creative writing)
-- **Extraction**: structured data from unstructured input (JSON from text, entities, classification)
-- **Reasoning**: multi-step logic (math, code generation, planning)
-- **Conversation**: multi-turn dialogue with memory
-- **Agentic**: tool use, function calling, autonomous task execution
-
-For each class, identify: latency requirements (real-time < 2s, async < 30s, batch), accuracy requirements (critical = needs eval suite, casual = spot check), cost sensitivity (per-call budget), and data sensitivity (PII, HIPAA, can data leave the network?).
-
-**Step 2 — Model selection matrix**
-Based on requirements, recommend model tier:
-
-| Requirement | Recommended | Fallback |
-|------------|-------------|----------|
-| Fast + cheap (classification, routing) | Haiku / GPT-4o-mini | Local (Llama 3) |
-| Balanced (code, summaries, RAG) | Sonnet / GPT-4o | Haiku with retry |
-| Deep reasoning (architecture, math) | Opus / o1 | Sonnet with chain-of-thought |
-| On-premise required | Llama 3 / Mistral | Ollama local deployment |
-| Multimodal (vision + text) | Sonnet / GPT-4o | Local LLaVA |
-
-Emit: primary model, fallback model, estimated cost per 1K calls, and latency p50/p99.
-
-**Step 3 — Prompt architecture**
-Design the prompt structure:
-- **System prompt**: Role definition, constraints, output format. Keep under 500 tokens for cost efficiency.
-- **Few-shot examples**: 2-3 examples for extraction/classification tasks. Format matches expected output exactly.
-- **Chain-of-thought**: For reasoning tasks, explicitly request step-by-step thinking before final answer.
-- **Structured output**: JSON mode or tool use for extraction. Define schema with Zod/Pydantic for validation.
-
-**Step 4 — Guardrails and evaluation**
-Design safety and quality layers:
-- **Input guardrails**: PII detection, prompt injection detection, topic filtering
-- **Output guardrails**: Schema validation, hallucination checks, toxicity filtering
-- **Evaluation framework**: Define eval dataset (50+ examples), metrics (accuracy, latency, cost), and regression threshold (new prompt must not drop > 2% on any metric)
-
-Save architecture doc to `.rune/ai/llm-architecture.md`.
-
-#### Example
-
-```typescript
-// Multi-model router with fallback
-interface ModelConfig {
-  id: string;
-  provider: 'anthropic' | 'openai' | 'local';
-  costPer1kTokens: number;
-  maxTokens: number;
-  latencyP50Ms: number;
-}
-
-const MODELS: Record<string, ModelConfig> = {
-  fast: {
-    id: 'claude-haiku-4-5-20251001',
-    provider: 'anthropic',
-    costPer1kTokens: 0.001,
-    maxTokens: 4096,
-    latencyP50Ms: 200,
-  },
-  balanced: {
-    id: 'claude-sonnet-4-6',
-    provider: 'anthropic',
-    costPer1kTokens: 0.01,
-    maxTokens: 8192,
-    latencyP50Ms: 800,
-  },
-  deep: {
-    id: 'claude-opus-4-6',
-    provider: 'anthropic',
-    costPer1kTokens: 0.05,
-    maxTokens: 16384,
-    latencyP50Ms: 2000,
-  },
-};
-
-type TaskComplexity = 'trivial' | 'standard' | 'complex';
-
-function selectModel(complexity: TaskComplexity): ModelConfig {
-  const map: Record<TaskComplexity, string> = {
-    trivial: 'fast',
-    standard: 'balanced',
-    complex: 'deep',
-  };
-  return MODELS[map[complexity]];
-}
-
-// Prompt architecture template
-const systemPrompt = `You are a ${role} assistant.
-
-CONSTRAINTS:
-- ${constraints.join('\n- ')}
-
-OUTPUT FORMAT:
-Return valid JSON matching this schema:
-${JSON.stringify(outputSchema, null, 2)}
-
-Do not include explanations outside the JSON.`;
-
-// Guardrail: validate structured output
-import { z } from 'zod';
-
-const OutputSchema = z.object({
-  classification: z.enum(['positive', 'negative', 'neutral']),
-  confidence: z.number().min(0).max(1),
-  reasoning: z.string().max(200),
-});
-
-function validateOutput(raw: string): z.infer<typeof OutputSchema> {
-  const parsed = JSON.parse(raw);
-  return OutputSchema.parse(parsed); // throws if invalid
-}
-```
+---
+name: "llm-architect"
+pack: "@rune/ai-ml"
+description: "LLM system architecture — model selection, prompt engineering patterns, evaluation frameworks, cost optimization, multi-model routing, and guardrail design."
+model: opus
+tools: [Read, Edit, Write, Grep, Glob, Bash]
+---
+
+# llm-architect
+
+LLM system architecture — model selection, prompt engineering patterns, evaluation frameworks, cost optimization, multi-model routing, and guardrail design.
+
+#### Workflow
+
+**Step 1 — Assess LLM requirements**
+Understand the use case: what does the LLM need to do? Classify into:
+- **Generation**: open-ended text (blog, email, creative writing)
+- **Extraction**: structured data from unstructured input (JSON from text, entities, classification)
+- **Reasoning**: multi-step logic (math, code generation, planning)
+- **Conversation**: multi-turn dialogue with memory
+- **Agentic**: tool use, function calling, autonomous task execution
+
+For each class, identify: latency requirements (real-time < 2s, async < 30s, batch), accuracy requirements (critical = needs eval suite, casual = spot check), cost sensitivity (per-call budget), and data sensitivity (PII, HIPAA, can data leave the network?).
+
+**Step 2 — Model selection matrix**
+Based on requirements, recommend model tier:
+
+| Requirement | Recommended | Fallback |
+|------------|-------------|----------|
+| Fast + cheap (classification, routing) | Haiku / GPT-4o-mini | Local (Llama 3) |
+| Balanced (code, summaries, RAG) | Sonnet / GPT-4o | Haiku with retry |
+| Deep reasoning (architecture, math) | Opus / o1 | Sonnet with chain-of-thought |
+| On-premise required | Llama 3 / Mistral | Ollama local deployment |
+| Multimodal (vision + text) | Sonnet / GPT-4o | Local LLaVA |
+
+Emit: primary model, fallback model, estimated cost per 1K calls, and latency p50/p99.
+
+**Step 3 — Prompt architecture**
+Design the prompt structure:
+- **System prompt**: Role definition, constraints, output format. Keep under 500 tokens for cost efficiency.
+- **Few-shot examples**: 2-3 examples for extraction/classification tasks. Format matches expected output exactly.
+- **Chain-of-thought**: For reasoning tasks, explicitly request step-by-step thinking before final answer.
+- **Structured output**: JSON mode or tool use for extraction. Define schema with Zod/Pydantic for validation.
+
+**Step 4 — Guardrails and evaluation**
+Design safety and quality layers:
+- **Input guardrails**: PII detection, prompt injection detection, topic filtering
+- **Output guardrails**: Schema validation, hallucination checks, toxicity filtering
+- **Evaluation framework**: Define eval dataset (50+ examples), metrics (accuracy, latency, cost), and regression threshold (new prompt must not drop > 2% on any metric)
+
+Save architecture doc to `.rune/ai/llm-architecture.md`.
+
+#### Example
+
+```typescript
+// Multi-model router with fallback
+interface ModelConfig {
+  id: string;
+  provider: 'anthropic' | 'openai' | 'local';
+  costPer1kTokens: number;
+  maxTokens: number;
+  latencyP50Ms: number;
+}
+
+const MODELS: Record<string, ModelConfig> = {
+  fast: {
+    id: 'claude-haiku-4-5-20251001',
+    provider: 'anthropic',
+    costPer1kTokens: 0.001,
+    maxTokens: 4096,
+    latencyP50Ms: 200,
+  },
+  balanced: {
+    id: 'claude-sonnet-4-6',
+    provider: 'anthropic',
+    costPer1kTokens: 0.01,
+    maxTokens: 8192,
+    latencyP50Ms: 800,
+  },
+  deep: {
+    id: 'claude-opus-4-6',
+    provider: 'anthropic',
+    costPer1kTokens: 0.05,
+    maxTokens: 16384,
+    latencyP50Ms: 2000,
+  },
+};
+
+type TaskComplexity = 'trivial' | 'standard' | 'complex';
+
+function selectModel(complexity: TaskComplexity): ModelConfig {
+  const map: Record<TaskComplexity, string> = {
+    trivial: 'fast',
+    standard: 'balanced',
+    complex: 'deep',
+  };
+  return MODELS[map[complexity]];
+}
+
+// Prompt architecture template
+const systemPrompt = `You are a ${role} assistant.
+
+CONSTRAINTS:
+- ${constraints.join('\n- ')}
+
+OUTPUT FORMAT:
+Return valid JSON matching this schema:
+${JSON.stringify(outputSchema, null, 2)}
+
+Do not include explanations outside the JSON.`;
+
+// Guardrail: validate structured output
+import { z } from 'zod';
+
+const OutputSchema = z.object({
+  classification: z.enum(['positive', 'negative', 'neutral']),
+  confidence: z.number().min(0).max(1),
+  reasoning: z.string().max(200),
+});
+
+function validateOutput(raw: string): z.infer<typeof OutputSchema> {
+  const parsed = JSON.parse(raw);
+  return OutputSchema.parse(parsed); // throws if invalid
+}
+```