@rfxlamia/skillkit 1.1.0 → 1.2.0

package/skills/skills/framework-critical-thinking/references/fallback_handler.md

@@ -0,0 +1,239 @@
+# Fallback Handler
+
+## Purpose
+
+The Fallback Handler manages graceful degradation when the agent encounters situations it cannot handle, ensuring appropriate escalation to human oversight or alternative approaches.
+
+## Fallback Levels
+
+### Level 1: Alternative Approach
+
+Try a different method before escalating.
+
+```python
+class AlternativeApproachFallback:
+    def __init__(self, available_approaches):
+        self.approaches = available_approaches
+        self.attempted = []
+
+    def try_fallback(self, task, current_approach, failure_reason):
+        """Try alternative approach."""
+        for approach in self.approaches:
+            if approach not in self.attempted:
+                self.attempted.append(approach)
+                return {
+                    'action': 'try_alternative',
+                    'approach': approach,
+                    'reason': f"Switching from {current_approach} due to: {failure_reason}"
+                }
+
+        return {'action': 'escalate', 'reason': 'All approaches exhausted'}
+```
+
+### Level 2: Simplified Task
+
+Decompose or simplify the task.
+
+```python
+class SimplificationFallback:
+    def simplify(self, task):
+        """Create simplified version of task."""
+        simplifications = []
+
+        # Remove non-essential constraints
+        if 'constraints' in task:
+            essential_only = {
+                k: v for k, v in task['constraints'].items()
+                if v.get('essential', False)
+            }
+            simplifications.append({
+                'type': 'reduced_constraints',
+                'task': {**task, 'constraints': essential_only}
+            })
+
+        # Reduce scope
+        if 'scope' in task:
+            reduced_scope = self.reduce_scope(task['scope'])
+            simplifications.append({
+                'type': 'reduced_scope',
+                'task': {**task, 'scope': reduced_scope}
+            })
+
+        return simplifications
+
+    def try_simplified(self, task, agent):
+        """Try simplified versions."""
+        for simplified in self.simplify(task):
+            result = agent.attempt(simplified['task'])
+            if result['success']:
+                return {
+                    'action': 'use_simplified',
+                    'simplification_type': simplified['type'],
+                    'result': result
+                }
+
+        return {'action': 'escalate', 'reason': 'All simplifications failed'}
+```
+
+### Level 3: Partial Solution
+
+Provide partial results with clear caveats.
+
+```python
+class PartialSolutionFallback:
+    def create_partial_solution(self, progress, failed_components):
+        """Generate partial solution from progress."""
+        return {
+            'action': 'provide_partial',
+            'solution': {
+                'completed_components': progress['completed'],
+                'partial_results': progress['results'],
+                'failed_components': failed_components,
+                'confidence': progress['confidence']
+            },
+            'caveats': [
+                f"Solution incomplete: {len(failed_components)} components failed",
+                "Results may not be fully valid",
+                "Human review strongly recommended"
+            ]
+        }
+```
+
+### Level 4: Human Handoff
+
+Escalate to human operator.
+
+```python
+class HumanHandoffFallback:
+    def prepare_handoff(self, task, failure_context, attempted_solutions):
+        """Prepare comprehensive handoff package."""
+        return {
+            'action': 'human_handoff',
+            'handoff_package': {
+                'original_task': task,
+                'failure_context': {
+                    'reason': failure_context['reason'],
+                    'step': failure_context['step'],
+                    'error': failure_context.get('error')
+                },
+                'attempted_solutions': attempted_solutions,
+                'current_state': failure_context.get('state'),
+                'partial_results': failure_context.get('partial_results'),
+                'recommended_action': failure_context.get('recommendation'),
+                'urgency': self.assess_urgency(failure_context)
+            }
+        }
+
+    def assess_urgency(self, context):
+        """Assess urgency of handoff."""
+        if context.get('critical_path', False):
+            return 'critical'
+        if context.get('time_sensitive', False):
+            return 'high'
+        return 'normal'
+```
+
+## Fallback Decision Tree
+
+```
+Agent encounters failure
+│
+├─ Can try alternative approach?
+│  └─ YES → Try alternative method
+│     └─ Success? → Return result
+│     └─ Fail? → Continue to next fallback
+│
+├─ Can simplify task?
+│  └─ YES → Try simplified version
+│     └─ Success? → Return partial with caveats
+│     └─ Fail? → Continue to next fallback
+│
+├─ Has partial progress?
+│  └─ YES → Return partial solution with warnings
+│
+└─ Handoff to human
+   ├─ Prepare context package
+   ├─ Escalate with priority
+   └─ Log for analysis
+```
+
+## Implementation
+
+```python
+class FallbackHandler:
+    def __init__(self, config):
+        self.fallbacks = [
+            AlternativeApproachFallback(config['approaches']),
+            SimplificationFallback(),
+            PartialSolutionFallback(),
+            HumanHandoffFallback()
+        ]
+        self.max_fallback_depth = config.get('max_depth', 3)
+
+    def handle_failure(self, task, context, depth=0):
+        """Execute fallback chain."""
+        if depth >= self.max_fallback_depth:
+            return self.fallbacks[-1].prepare_handoff(
+                task, context, context.get('attempted', [])
+            )
+
+        for fallback in self.fallbacks:
+            result = fallback.try_fallback(task, context)
+
+            if result['action'] != 'escalate':
+                return result
+
+        # All fallbacks exhausted
+        return self.fallbacks[-1].prepare_handoff(
+            task, context, context.get('attempted', [])
+        )
+```
+
+## Handoff Context Package
+
+What to include when handing off to humans:
+
+```python
+HANDOFF_CONTEXT = {
+    'task_description': 'Original task and requirements',
+    'failure_point': 'Where and why the agent failed',
+    'attempted_approaches': 'What was tried',
+    'intermediate_results': 'What was successfully completed',
+    'confidence_assessment': 'Agent confidence in partial results',
+    'recommended_resolution': 'Agent suggestion for human',
+    'time_constraints': 'Any deadlines or urgency',
+    'domain_expertise_needed': 'What expertise is required'
+}
+```
+
+## Usage Example
+
+```python
+handler = FallbackHandler(config={
+    'approaches': ['cot', 'tot', 'got'],
+    'max_depth': 3
+})
+
+try:
+    result = agent.solve(task)
+except AgentFailure as e:
+    fallback_result = handler.handle_failure(
+        task,
+        context={
+            'reason': str(e),
+            'step': e.step,
+            'state': agent.current_state,
+            'attempted': agent.attempted_approaches
+        }
+    )
+
+    if fallback_result['action'] == 'human_handoff':
+        escalate_to_human(fallback_result['handoff_package'])
+    else:
+        return fallback_result
+```
+
+---
+
+**Sources:**
+- [Graceful Degradation in AI Systems](https://www.microsoft.com/en-us/research/publication/graceful-degradation/)

package/skills/skills/framework-critical-thinking/references/memory_curator.md

@@ -0,0 +1,161 @@
+# Memory Curator
+
+## Table of Contents
+
+- [Purpose](#purpose)
+- [Memory Types (CoALA Model)](#memory-types-coala-model)
+- [Episodic Memory Structure](#episodic-memory-structure)
+- [Quality Weighting System](#quality-weighting-system)
+- [Memory Operations](#memory-operations)
+- [Memory Consolidation](#memory-consolidation)
+- [Experience Replay](#experience-replay)
+- [Usage Example](#usage-example)
+- [Best Practices](#best-practices)
+
+## Purpose
+
+The Memory Curator manages episodic memory with quality weighting to enable agents to learn from past experiences while preventing memory pollution from unsuccessful episodes.
+
+## Memory Types (CoALA Model)
+
+Following the CoALA (Cognitive Architectures for Language Agents) framework:
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                     Memory Systems                          │
+├──────────────┬──────────────┬──────────────┬────────────────┤
+│   Working    │   Episodic   │   Semantic   │   Procedural   │
+│   Memory     │   Memory     │   Memory     │   Memory       │
+├──────────────┼──────────────┼──────────────┼────────────────┤
+│ - Current    │ - Past       │ - Facts      │ - Skills       │
+│   context    │   experiences│ - Concepts   │ - Procedures   │
+│ - Active     │ - Specific   │ - Relations  │ - Tool usage   │
+│   task       │   episodes   │              │                │
+├──────────────┼──────────────┼──────────────┼────────────────┤
+│ Limited      │ Large,       │ Large,       │ Learned        │
+│ capacity     │ selectable   │ queryable    │ patterns       │
+└──────────────┴──────────────┴──────────────┴────────────────┘
+```
+
+This module focuses primarily on **Episodic Memory** management.
+
+## Episodic Memory Structure
+
+```python
+class EpisodicMemory:
+    def __init__(self):
+        self.episodes = []
+
+class Episode:
+    def __init__(self):
+        self.id = generate_id()
+        self.timestamp = datetime.now()
+        self.task = None
+        self.reasoning_trace = []
+        self.outcome = None
+        self.quality_score = 0.0
+        self.retrieval_count = 0
+        self.last_accessed = None
+        self.tags = []
+        self.embedding = None
+```
+
+## Quality Weighting System
+
+### Scoring Dimensions
+
+```python
+class QualityScorer:
+    def score_episode(self, episode):
+        scores = {
+            'outcome_success': self.outcome_score(episode),
+            'reasoning_quality': self.reasoning_score(episode),
+            'efficiency': self.efficiency_score(episode),
+            'reusability': self.reusability_score(episode)
+        }
+
+        # Weighted combination
+        weights = {
+            'outcome_success': 0.4,
+            'reasoning_quality': 0.3,
+            'efficiency': 0.2,
+            'reusability': 0.1
+        }
+
+        episode.quality_score = sum(
+            scores[k] * weights[k] for k in scores
+        )
+
+        return episode.quality_score
+```
+
+### Quality Tiers
+
+```python
+QUALITY_TIERS = {
+    'exemplar': (0.9, 1.0),      # Always retain, priority retrieval
+    'good': (0.7, 0.9),          # Standard retention
+    'acceptable': (0.5, 0.7),    # Conditional retention
+    'poor': (0.0, 0.5)           # Flag for review or discard
+}
+```
+
+## Memory Operations
+
+### Storage
+
+```python
+def store_episode(self, task, reasoning_trace, outcome):
+    episode = Episode()
+    episode.task = task
+    episode.reasoning_trace = reasoning_trace
+    episode.outcome = outcome
+    episode.embedding = self.compute_embedding(task + " " + str(reasoning_trace))
+    episode.quality_score = self.quality_scorer.score_episode(episode)
+    episode.tags = self.auto_tag(task, reasoning_trace)
+
+    if self.should_retain(episode):
+        self.episodes.append(episode)
+        self.maintain_capacity()
+
+    return episode
+```
+
+### Retrieval
+
+```python
+def retrieve_similar(self, query, k=5, quality_threshold=0.6):
+    query_embedding = self.compute_embedding(query)
+    scored_episodes = []
+
+    for episode in self.episodes:
+        if episode.quality_score < quality_threshold:
+            continue
+        similarity = cosine_similarity(query_embedding, episode.embedding)
+        adjusted_score = similarity * (0.5 + 0.5 * episode.quality_score)
+        scored_episodes.append((episode, adjusted_score))
+
+    scored_episodes.sort(key=lambda x: x[1], reverse=True)
+    return [ep for ep, _ in scored_episodes[:k]]
+```
+
+## Usage Example
+
+```python
+curator = MemoryCurator(max_episodes=1000, quality_threshold=0.5)
+
+# Store experience
+episode = curator.store_episode(
+    task="Solve differential equation",
+    reasoning_trace=[...],
+    outcome={'success': True, 'quality': 0.9}
+)
+
+# Retrieve similar
+similar = curator.retrieve_similar("Solve differential equation", k=3)
+```
+
+---
+
+**Sources:**
+- [Cognitive Architectures for Language Agents (CoALA)](https://arxiv.org/abs/2309.02427)

package/skills/skills/framework-critical-thinking/references/metacognitive_monitor.md

@@ -0,0 +1,297 @@
+# Metacognitive Monitor
+
+## Purpose
+
+The Metacognitive Monitor provides self-assessment and error detection capabilities for AI agents. It implements the Producer-Critic pattern to continuously evaluate the quality of reasoning and trigger corrective actions when needed.
+
+## Architecture
+
+### Producer-Critic Pattern
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│                    Master Agent                             │
+└──────────────┬──────────────────────────────┬───────────────┘
+               │                              │
+               ▼                              ▼
+┌──────────────────────┐          ┌──────────────────────┐
+│   Producer Agent     │          │    Critic Agent      │
+│   (Generate)         │          │    (Evaluate)        │
+└──────────┬───────────┘          └──────────┬───────────┘
+           │                                 │
+           │         ┌──────────────┐        │
+           └────────►│   Output     │◄───────┘
+                     │   Quality    │
+                     └──────────────┘
+                            │
+                            ▼
+                     ┌──────────────┐
+                     │  Decision    │
+                     │  Continue /  │
+                     │  Refine /    │
+                     │  Handoff     │
+                     └──────────────┘
+```
+
+### Components
+
+#### 1. Confidence Scorer
+
+Evaluates confidence at multiple levels:
+
+```python
+class ConfidenceScorer:
+    def assess_step(self, reasoning_step):
+        return {
+            'certainty_score': self.compute_certainty(reasoning_step),
+            'consistency_check': self.check_internal_consistency(reasoning_step),
+            'grounding_score': self.verify_grounding(reasoning_step),
+            'overall_confidence': weighted_average([
+                certainty * 0.4,
+                consistency * 0.3,
+                grounding * 0.3
+            ])
+        }
+```
+
+**Scoring dimensions:**
+- **Certainty:** How definitive is the statement? (0.0 - 1.0)
+- **Consistency:** Does it align with previous steps?
+- **Grounding:** Is it supported by evidence/context?
+
+#### 2. Anomaly Detector
+
+Identifies unusual patterns in reasoning:
+
+```python
+class AnomalyDetector:
+    def detect(self, thought_history):
+        anomalies = []
+
+        # Pattern 1: Sudden topic shift
+        if self.topic_coherence_score(thought_history) < 0.5:
+            anomalies.append(Anomaly.TOPIC_DRIFT)
+
+        # Pattern 2: Repetitive reasoning
+        if self.repetition_detected(thought_history):
+            anomalies.append(Anomaly.LOOPING)
+
+        # Pattern 3: Confidence degradation
+        if self.confidence_trend(thought_history) == 'decreasing':
+            anomalies.append(Anomaly.DEGRADING_QUALITY)
+
+        # Pattern 4: Contradiction emergence
+        if self.find_contradictions(thought_history):
+            anomalies.append(Anomaly.CONTRADICTION)
+
+        return anomalies
+```
+
+#### 3. Reflection Controller
+
+Manages when and how to trigger reflection:
+
+```python
+class ReflectionController:
+    def __init__(self):
+        self.reflection_budget = 3  # Max reflections per task
+        self.min_confidence_threshold = 0.7
+        self.criticality_boost = 1.2  # Multiply threshold for critical tasks
+
+    def should_reflect(self, current_state):
+        if self.reflection_budget <= 0:
+            return False, "Budget exhausted"
+
+        adjusted_threshold = self.min_confidence_threshold
+        if current_state.task_criticality == 'HIGH':
+            adjusted_threshold *= self.criticality_boost
+
+        if current_state.confidence < adjusted_threshold:
+            return True, f"Confidence {current_state.confidence} below threshold {adjusted_threshold}"
+
+        if current_state.anomalies_detected:
+            return True, f"Anomalies detected: {current_state.anomalies}"
+
+        return False, "No reflection needed"
+```
+
+## Implementation
+
+### Basic Monitor Setup
+
+```python
+class MetacognitiveMonitor:
+    def __init__(self, config):
+        self.confidence_scorer = ConfidenceScorer()
+        self.anomaly_detector = AnomalyDetector()
+        self.reflection_controller = ReflectionController(config)
+        self.thought_history = []
+
+    def monitor(self, current_step, context):
+        # Score the current step
+        confidence = self.confidence_scorer.assess_step(current_step)
+
+        # Update history
+        self.thought_history.append({
+            'step': current_step,
+            'confidence': confidence,
+            'timestamp': now()
+        })
+
+        # Detect anomalies
+        anomalies = self.anomaly_detector.detect(self.thought_history)
+
+        # Create state snapshot
+        state = MonitorState(
+            confidence=confidence['overall_confidence'],
+            anomalies=anomalies,
+            thought_count=len(self.thought_history),
+            task_criticality=context.criticality
+        )
+
+        # Decide on action
+        should_reflect, reason = self.reflection_controller.should_reflect(state)
+
+        return MonitorResult(
+            confidence=confidence,
+            anomalies=anomalies,
+            should_reflect=should_reflect,
+            reflection_reason=reason,
+            recommended_action=self.determine_action(state, should_reflect)
+        )
+
+    def determine_action(self, state, should_reflect):
+        if state.confidence < 0.3:
+            return Action.HUMAN_HANDOFF
+        elif should_reflect:
+            return Action.REFLECT_AND_REFINE
+        elif state.anomalies:
+            return Action.REVIEW_ANOMALIES
+        else:
+            return Action.CONTINUE
+```
+
+### Confidence Calibration
+
+Calibrate confidence scores to match actual accuracy:
+
+```python
+def calibrate_confidence(raw_confidence, task_type):
+    # Use historical performance data
+    calibration_map = {
+        'arithmetic': lambda x: x * 0.95,      # Usually overconfident
+        'reasoning': lambda x: x * 0.85,       # Often overconfident
+        'creative': lambda x: x * 0.70,        # Highly overconfident
+        'factual': lambda x: min(x * 1.1, 1.0) # Often underconfident
+    }
+
+    calibrator = calibration_map.get(task_type, lambda x: x)
+    return calibrator(raw_confidence)
+```
+
+## Monitoring Heuristics
+
+### 1. Reasoning Quality Indicators
+
+**Positive indicators:**
+- Progressive narrowing of solution space
+- Increasing specificity in claims
+- Consistent terminology usage
+- Appropriate uncertainty acknowledgment
+
+**Negative indicators:**
+- Circular reasoning
+- Unjustified leaps
+- Confusing correlation with causation
+- Overgeneralization
+
+### 2. Confidence-Accuracy Correlation
+
+Track the relationship between stated confidence and actual correctness:
+
+```python
+def compute_calibration_error(confidence_history, accuracy_history):
+    # Perfect calibration: confidence == accuracy
+    # Overconfidence: confidence > accuracy
+    # Underconfidence: confidence < accuracy
+
+    expected_accuracy = sum(confidence_history) / len(confidence_history)
+    actual_accuracy = sum(accuracy_history) / len(accuracy_history)
+
+    return abs(expected_accuracy - actual_accuracy)
+```
+
+## Human Handoff Protocols
+
+### When to Handoff
+
+| Condition | Threshold | Action |
+|-----------|-----------|--------|
+| Very low confidence | < 0.3 | Immediate handoff |
+| Repeated reflection failures | > 3 attempts | Handoff with context |
+| Critical task + uncertainty | Confidence < 0.8 | Request verification |
+| Anomaly cluster | > 2 anomalies | Handoff for review |
+
+### Handoff Context
+
+Include in handoff:
+- Current reasoning chain
+- Confidence trajectory
+- Detected anomalies
+- Attempted corrections
+- Specific uncertainty points
+
+## Usage Example
+
+```python
+# Initialize monitor
+monitor = MetacognitiveMonitor(config={
+    'reflection_budget': 3,
+    'confidence_threshold': 0.7,
+    'enable_human_handoff': True
+})
+
+# Use in agent loop
+for step in reasoning_process:
+    result = monitor.monitor(step, context={'criticality': 'HIGH'})
+
+    if result.recommended_action == Action.HUMAN_HANDOFF:
+        await handoff_to_human(result)
+        break
+
+    elif result.recommended_action == Action.REFLECT_AND_REFINE:
+        reflection = generate_reflection(result.anomalies)
+        refined_step = refine_with_reflection(step, reflection)
+        step = refined_step
+
+    elif result.recommended_action == Action.REVIEW_ANOMALIES:
+        for anomaly in result.anomalies:
+            log_warning(f"Anomaly detected: {anomaly}")
+
+    # Continue with (possibly refined) step
+    execute(step)
+```
+
+## Performance Metrics
+
+Track these metrics to evaluate monitor effectiveness:
+
+1. **False Positive Rate:** Reflection triggered when not needed
+2. **False Negative Rate:** Missed errors that should have been caught
+3. **Calibration Error:** Difference between confidence and accuracy
+4. **Reflection Success Rate:** % of reflections that improve output
+5. **Average Reflections per Task:** Efficiency measure
+
+## Best Practices
+
+1. **Start with high thresholds** and gradually lower based on performance
+2. **Log all monitoring decisions** for post-hoc analysis
+3. **Calibrate confidence** per task type using historical data
+4. **Respect the reflection budget** to avoid infinite loops
+5. **Provide clear handoff reasons** for human reviewers
+
+---
+
+**Sources:**
+- [Self-Corrective Agent Architecture](https://www.emergentmind.com/topics/self-corrective-agent-architecture)
+- [Building Metacognitive AI Agents](https://rewire.it/blog/building-metacognitive-ai-agents-complete-guide/)