enya-agent 0.1.0

package/crates/enact-context/src/token_counter.rs

@@ -0,0 +1,148 @@
+//! Token Counter
+//!
+//! Token counting using tiktoken for accurate context management.
+
+use thiserror::Error;
+use tiktoken_rs::{cl100k_base, CoreBPE};
+
+/// Token counter errors
+#[derive(Debug, Error)]
+pub enum TokenCounterError {
+    #[error("Failed to initialize tokenizer: {0}")]
+    InitError(String),
+}
+
+/// Token counter using tiktoken
+///
+/// Uses cl100k_base encoding (GPT-4, GPT-3.5-turbo, text-embedding-ada-002)
+pub struct TokenCounter {
+    bpe: CoreBPE,
+}
+
+impl TokenCounter {
+    /// Create a new token counter with cl100k_base encoding
+    pub fn new() -> Result<Self, TokenCounterError> {
+        let bpe = cl100k_base().map_err(|e| TokenCounterError::InitError(e.to_string()))?;
+        Ok(Self { bpe })
+    }
+
+    /// Count tokens in a string
+    pub fn count(&self, text: &str) -> usize {
+        self.bpe.encode_with_special_tokens(text).len()
+    }
+
+    /// Count tokens for a chat message (includes role overhead)
+    ///
+    /// Approximates the token overhead for chat formatting:
+    /// - Each message has ~4 tokens overhead (role, separators)
+    /// - System messages have additional overhead
+    pub fn count_message(&self, role: &str, content: &str) -> usize {
+        let content_tokens = self.count(content);
+        let role_overhead = match role {
+            "system" => 4,
+            "user" => 4,
+            "assistant" => 4,
+            "function" | "tool" => 5,
+            _ => 4,
+        };
+        content_tokens + role_overhead
+    }
+
+    /// Count tokens for multiple messages
+    pub fn count_messages(&self, messages: &[(String, String)]) -> usize {
+        messages
+            .iter()
+            .map(|(role, content)| self.count_message(role, content))
+            .sum::<usize>()
+            + 3 // priming tokens
+    }
+
+    /// Truncate text to fit within a token limit
+    ///
+    /// Returns the truncated text and the actual token count.
+    pub fn truncate(&self, text: &str, max_tokens: usize) -> (String, usize) {
+        let tokens = self.bpe.encode_with_special_tokens(text);
+        if tokens.len() <= max_tokens {
+            return (text.to_string(), tokens.len());
+        }
+
+        let truncated_tokens = &tokens[..max_tokens];
+        let truncated_text = self.bpe.decode(truncated_tokens.to_vec())
+            .unwrap_or_else(|_| text[..text.len() / 2].to_string());
+
+        (truncated_text, max_tokens)
+    }
+
+    /// Split text into chunks of approximately equal token size
+    pub fn chunk(&self, text: &str, chunk_size: usize) -> Vec<String> {
+        let tokens = self.bpe.encode_with_special_tokens(text);
+        let mut chunks = Vec::new();
+
+        for chunk_tokens in tokens.chunks(chunk_size) {
+            if let Ok(chunk_text) = self.bpe.decode(chunk_tokens.to_vec()) {
+                chunks.push(chunk_text);
+            }
+        }
+
+        chunks
+    }
+
+    /// Estimate if text will fit within budget
+    pub fn will_fit(&self, text: &str, budget: usize) -> bool {
+        self.count(text) <= budget
+    }
+}
+
+impl Default for TokenCounter {
+    fn default() -> Self {
+        Self::new().expect("Failed to initialize default token counter")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_count_tokens() {
+        let counter = TokenCounter::new().unwrap();
+
+        // Empty string
+        assert_eq!(counter.count(""), 0);
+
+        // Simple text
+        let count = counter.count("Hello, world!");
+        assert!(count > 0);
+        assert!(count < 10);
+    }
+
+    #[test]
+    fn test_count_message() {
+        let counter = TokenCounter::new().unwrap();
+
+        let content_tokens = counter.count("Hello");
+        let message_tokens = counter.count_message("user", "Hello");
+
+        // Message should have overhead
+        assert!(message_tokens > content_tokens);
+    }
+
+    #[test]
+    fn test_truncate() {
+        let counter = TokenCounter::new().unwrap();
+
+        let long_text = "This is a long text that we want to truncate to a smaller size.";
+        let (truncated, count) = counter.truncate(long_text, 5);
+
+        assert!(count <= 5);
+        assert!(truncated.len() < long_text.len());
+    }
+
+    #[test]
+    fn test_will_fit() {
+        let counter = TokenCounter::new().unwrap();
+
+        assert!(counter.will_fit("Hello", 100));
+        assert!(!counter.will_fit("Hello ".repeat(1000).as_str(), 10));
+    }
+}

package/crates/enact-context/src/window.rs

@@ -0,0 +1,372 @@
+//! Context Window Management
+//!
+//! The ContextWindow manages all segments within token budget limits.
+//!
+//! @see packages/enact-schemas/src/context.schemas.ts
+
+use crate::budget::{BudgetHealth, ContextBudget};
+use crate::compactor::{CompactionResult, CompactionStrategyType, Compactor};
+use crate::segment::{ContextSegment, ContextSegmentType};
+use crate::token_counter::TokenCounter;
+use chrono::{DateTime, Utc};
+use enact_core::kernel::ExecutionId;
+use serde::{Deserialize, Serialize};
+use std::time::Instant;
+use thiserror::Error;
+
+/// Context window errors
+#[derive(Debug, Error)]
+pub enum ContextWindowError {
+    #[error("Token counter error: {0}")]
+    TokenCounter(String),
+
+    #[error("Budget exceeded: need {needed} tokens, only {available} available")]
+    BudgetExceeded { needed: usize, available: usize },
+
+    #[error("Segment budget exceeded for {segment_type:?}: need {needed}, max {max}")]
+    SegmentBudgetExceeded {
+        segment_type: ContextSegmentType,
+        needed: usize,
+        max: usize,
+    },
+
+    #[error("Compaction failed: {0}")]
+    CompactionFailed(String),
+}
+
+/// Context window state
+///
+/// Matches `contextWindowStateSchema` in @enact/schemas
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct ContextWindowState {
+    /// Execution ID
+    pub execution_id: ExecutionId,
+
+    /// All segments currently in context
+    pub segments: Vec<ContextSegment>,
+
+    /// Current budget state
+    pub budget: ContextBudget,
+
+    /// Compaction history
+    pub compaction_history: Vec<CompactionResult>,
+
+    /// Number of compactions performed
+    pub compaction_count: u32,
+
+    /// Total tokens saved by compaction
+    pub total_tokens_saved: usize,
+
+    /// Current health status
+    pub health: BudgetHealth,
+
+    /// Last updated timestamp
+    pub updated_at: DateTime<Utc>,
+}
+
+/// Context Window - manages segments within budget
+pub struct ContextWindow {
+    /// Execution ID
+    execution_id: ExecutionId,
+
+    /// All segments in the context
+    segments: Vec<ContextSegment>,
+
+    /// Token budget
+    budget: ContextBudget,
+
+    /// Token counter
+    token_counter: TokenCounter,
+
+    /// Compaction history
+    compaction_history: Vec<CompactionResult>,
+
+    /// Next sequence number
+    next_sequence: u64,
+}
+
+impl ContextWindow {
+    /// Create a new context window
+    pub fn new(budget: ContextBudget) -> Result<Self, ContextWindowError> {
+        let token_counter = TokenCounter::new()
+            .map_err(|e| ContextWindowError::TokenCounter(e.to_string()))?;
+
+        Ok(Self {
+            execution_id: budget.execution_id.clone(),
+            segments: Vec::new(),
+            budget,
+            token_counter,
+            compaction_history: Vec::new(),
+            next_sequence: 0,
+        })
+    }
+
+    /// Create with a preset budget
+    pub fn with_preset_gpt4_128k(execution_id: ExecutionId) -> Result<Self, ContextWindowError> {
+        Self::new(ContextBudget::preset_gpt4_128k(execution_id))
+    }
+
+    /// Create with Claude 200K preset
+    pub fn with_preset_claude_200k(execution_id: ExecutionId) -> Result<Self, ContextWindowError> {
+        Self::new(ContextBudget::preset_claude_200k(execution_id))
+    }
+
+    /// Create with default (8K) preset
+    pub fn with_preset_default(execution_id: ExecutionId) -> Result<Self, ContextWindowError> {
+        Self::new(ContextBudget::preset_default(execution_id))
+    }
+
+    /// Get the execution ID
+    pub fn execution_id(&self) -> &ExecutionId {
+        &self.execution_id
+    }
+
+    /// Get all segments
+    pub fn segments(&self) -> &[ContextSegment] {
+        &self.segments
+    }
+
+    /// Get the budget
+    pub fn budget(&self) -> &ContextBudget {
+        &self.budget
+    }
+
+    /// Get mutable budget
+    pub fn budget_mut(&mut self) -> &mut ContextBudget {
+        &mut self.budget
+    }
+
+    /// Count tokens for text
+    pub fn count_tokens(&self, text: &str) -> usize {
+        self.token_counter.count(text)
+    }
+
+    /// Add a segment with automatic token counting
+    pub fn add_segment_auto(&mut self, mut segment: ContextSegment) -> Result<(), ContextWindowError> {
+        // Count tokens if not already set
+        if segment.token_count == 0 {
+            segment.token_count = self.token_counter.count(&segment.content);
+        }
+
+        self.add_segment(segment)
+    }
+
+    /// Add a segment to the context window
+    pub fn add_segment(&mut self, mut segment: ContextSegment) -> Result<(), ContextWindowError> {
+        // Check segment budget
+        if let Some(seg_budget) = self.budget.get_segment(segment.segment_type) {
+            let new_usage = seg_budget.current_tokens + segment.token_count;
+            if new_usage > seg_budget.max_tokens {
+                return Err(ContextWindowError::SegmentBudgetExceeded {
+                    segment_type: segment.segment_type,
+                    needed: segment.token_count,
+                    max: seg_budget.max_tokens - seg_budget.current_tokens,
+                });
+            }
+        }
+
+        // Check total budget
+        let new_total = self.budget.used_tokens + segment.token_count;
+        if new_total > self.budget.available_tokens {
+            return Err(ContextWindowError::BudgetExceeded {
+                needed: segment.token_count,
+                available: self.budget.remaining(),
+            });
+        }
+
+        // Assign sequence number
+        segment.sequence = self.next_sequence;
+        self.next_sequence += 1;
+
+        // Update budget
+        self.budget.add_tokens(segment.segment_type, segment.token_count);
+
+        // Add segment
+        self.segments.push(segment);
+
+        Ok(())
+    }
+
+    /// Remove a segment by ID
+    pub fn remove_segment(&mut self, segment_id: &str) -> bool {
+        if let Some(pos) = self.segments.iter().position(|s| s.id == segment_id) {
+            let segment = self.segments.remove(pos);
+            self.budget.remove_tokens(segment.segment_type, segment.token_count);
+            true
+        } else {
+            false
+        }
+    }
+
+    /// Get segments of a specific type
+    pub fn segments_of_type(&self, segment_type: ContextSegmentType) -> Vec<&ContextSegment> {
+        self.segments
+            .iter()
+            .filter(|s| s.segment_type == segment_type)
+            .collect()
+    }
+
+    /// Total tokens currently used
+    pub fn used_tokens(&self) -> usize {
+        self.budget.used_tokens
+    }
+
+    /// Remaining tokens available
+    pub fn remaining_tokens(&self) -> usize {
+        self.budget.remaining()
+    }
+
+    /// Check if the window needs compaction
+    pub fn needs_compaction(&self) -> bool {
+        self.budget.is_warning()
+    }
+
+    /// Check if the window is in critical state
+    pub fn is_critical(&self) -> bool {
+        self.budget.is_critical()
+    }
+
+    /// Get health status
+    pub fn health(&self) -> BudgetHealth {
+        self.budget.health()
+    }
+
+    /// Compact the context using the given compactor
+    pub fn compact(&mut self, compactor: &Compactor) -> Result<CompactionResult, ContextWindowError> {
+        let start = Instant::now();
+        let tokens_before = self.budget.used_tokens;
+
+        let result = match compactor.strategy().strategy_type {
+            CompactionStrategyType::Truncate => {
+                compactor.compact_truncate(&mut self.segments, tokens_before)
+            }
+            CompactionStrategyType::SlidingWindow => {
+                compactor.compact_sliding_window(&mut self.segments)
+            }
+            _ => {
+                // Other strategies not implemented yet
+                return Err(ContextWindowError::CompactionFailed(
+                    format!("Strategy {:?} not implemented", compactor.strategy().strategy_type)
+                ));
+            }
+        };
+
+        let duration_ms = start.elapsed().as_millis() as u64;
+
+        match result {
+            Ok(tokens_removed) => {
+                // Recalculate budget
+                self.recalculate_budget();
+
+                let tokens_after = self.budget.used_tokens;
+                let segments_compacted = (tokens_removed > 0) as usize;
+
+                let compaction_result = CompactionResult::success(
+                    self.execution_id.clone(),
+                    compactor.strategy().strategy_type,
+                    tokens_before,
+                    tokens_after,
+                    segments_compacted,
+                    duration_ms,
+                );
+
+                self.compaction_history.push(compaction_result.clone());
+                Ok(compaction_result)
+            }
+            Err(e) => {
+                let compaction_result = CompactionResult::failure(
+                    self.execution_id.clone(),
+                    compactor.strategy().strategy_type,
+                    tokens_before,
+                    e.to_string(),
+                    duration_ms,
+                );
+
+                self.compaction_history.push(compaction_result.clone());
+                Err(ContextWindowError::CompactionFailed(e.to_string()))
+            }
+        }
+    }
+
+    /// Recalculate budget from current segments
+    fn recalculate_budget(&mut self) {
+        // Reset all segment budgets
+        for seg_budget in &mut self.budget.segments {
+            seg_budget.current_tokens = 0;
+        }
+
+        // Sum up tokens by segment type
+        for segment in &self.segments {
+            self.budget.add_tokens(segment.segment_type, segment.token_count);
+        }
+    }
+
+    /// Build the context as a single string (for LLM calls)
+    pub fn build_context(&self) -> String {
+        let mut parts: Vec<&str> = Vec::new();
+
+        // Sort segments by sequence
+        let mut sorted: Vec<&ContextSegment> = self.segments.iter().collect();
+        sorted.sort_by_key(|s| s.sequence);
+
+        for segment in sorted {
+            parts.push(&segment.content);
+        }
+
+        parts.join("\n\n")
+    }
+
+    /// Get the current state (for serialization)
+    pub fn state(&self) -> ContextWindowState {
+        ContextWindowState {
+            execution_id: self.execution_id.clone(),
+            segments: self.segments.clone(),
+            budget: self.budget.clone(),
+            compaction_history: self.compaction_history.clone(),
+            compaction_count: self.compaction_history.len() as u32,
+            total_tokens_saved: self.compaction_history.iter().map(|r| r.tokens_saved).sum(),
+            health: self.budget.health(),
+            updated_at: Utc::now(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn test_execution_id() -> ExecutionId {
+        ExecutionId::new()
+    }
+
+    #[test]
+    fn test_create_window() {
+        let budget = ContextBudget::preset_default(test_execution_id());
+        let window = ContextWindow::new(budget).unwrap();
+
+        assert_eq!(window.used_tokens(), 0);
+        assert!(window.remaining_tokens() > 0);
+    }
+
+    #[test]
+    fn test_add_segment() {
+        let budget = ContextBudget::preset_default(test_execution_id());
+        let mut window = ContextWindow::new(budget).unwrap();
+
+        let segment = ContextSegment::system("You are a helpful assistant.", 10);
+        window.add_segment(segment).unwrap();
+
+        assert_eq!(window.segments().len(), 1);
+        assert_eq!(window.used_tokens(), 10);
+    }
+
+    #[test]
+    fn test_health_tracking() {
+        let budget = ContextBudget::preset_default(test_execution_id());
+        let window = ContextWindow::new(budget).unwrap();
+
+        assert_eq!(window.health(), BudgetHealth::Healthy);
+        assert!(!window.needs_compaction());
+    }
+}

package/crates/enact-core/Cargo.toml

@@ -0,0 +1,42 @@
+[package]
+name = "enact-core"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Core agent runtime for Enact - Graph-Native AI agents"
+repository.workspace = true
+homepage.workspace = true
+keywords = ["agent", "runtime", "ai", "llm", "graph"]
+categories.workspace = true
+
+[dependencies]
+tokio.workspace = true
+async-trait.workspace = true
+async-stream.workspace = true
+futures.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+tracing.workspace = true
+uuid.workspace = true
+svix-ksuid.workspace = true
+chrono.workspace = true
+thiserror.workspace = true
+anyhow.workspace = true
+reqwest.workspace = true
+tokio-util.workspace = true
+
+# Compression & Hashing (feat-04: Artifact Lifecycle)
+miniz_oxide.workspace = true
+sha2.workspace = true
+hex.workspace = true
+aes-gcm.workspace = true
+rand.workspace = true
+
+serde_yaml = "0.9.34"
+urlencoding = { workspace = true }
+html-escape = { workspace = true }
+regex = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["test-util"] }
+tempfile.workspace = true

package/crates/enact-core/README.md

@@ -0,0 +1,98 @@
+# enact-core
+
+`enact-core` is the core graph-first agent runtime crate for the Enact
+platform. It provides the execution kernel, graph engine, and streaming
+infrastructure for deterministic, auditable agent execution.
+
+## Key Features
+
+### Execution Kernel
+- **Reducer-based state machine** - Pure function state transitions for determinism
+- **Tenant isolation** - `TenantContext` required for all executions
+- **Agentic DAG** - Dynamic step discovery with policy-bounded execution
+- **Parallel step execution** - Independent steps execute concurrently via `tokio::join!`
+- **Circular graph detection** - Topological sort validation at compile time
+
+### Mid-Execution Control
+- **Inbox integration** - Inject guidance, evidence, and control signals mid-execution
+- **Async cancellation** - Cooperative cancellation via `CancellationToken`
+- **Checkpointing** - Save and restore execution state
+
+### Content Protection
+- **ProtectedEventEmitter** - Optional content protection pipeline
+- **PII detection** - Via `enact-guardrails` integration
+- **Context-aware masking** - Different protection based on destination
+
+### Tool Execution
+- **ToolExecutor** - Policy-aware tool execution
+- **Policy decision events** - All tool policy decisions emitted for audit trail
+- **Trust levels** - Untrusted, Low, Medium, High, System
+
+### Observability
+- **StreamEvent** - AI SDK compatible wire format with `data-*` prefix
+- **EventStore** - Append-only event persistence
+- **Artifact lifecycle** - First-class artifacts with deterministic IDs
+
+## Module Structure
+
+```
+src/
+├── kernel/           # Execution kernel (state machine, reducer, events)
+│   ├── kernel.rs     # ExecutionKernel entry point
+│   ├── reducer.rs    # Pure state transitions
+│   ├── event.rs      # ExecutionEvent types
+│   ├── error.rs      # Error taxonomy with retry policies
+│   ├── execution_state.rs  # ExecutionState, WaitReason
+│   ├── interrupt.rs  # Interrupt handling for HITL
+│   └── artifact/     # Artifact lifecycle (feat-04)
+├── context/          # TenantContext, RuntimeContext
+├── flow/             # Execution semantics (sequential, parallel, conditional)
+├── callable/         # Callable trait, LlmCallable
+├── policy/           # ExecutionPolicy, ToolPolicy, TenantPolicy
+├── streaming/        # SSE streaming, EventStore, ProtectedEventEmitter
+├── graph/            # StateGraph, CompiledGraph, Node
+├── tool/             # Tool trait, ToolExecutor, policy enforcement
+├── inbox/            # Mid-execution guidance (INV-INBOX-*)
+├── providers/        # ModelProvider trait
+├── telemetry/        # OpenTelemetry integration
+└── runner/           # Thin wiring shell
+```
+
+## Usage
+
+```rust
+use enact_core::prelude::*;
+use enact_core::context::TenantContext;
+
+// Create kernel with required tenant context
+let tenant = TenantContext::new(TenantId::from("tenant_123"));
+let kernel = ExecutionKernel::new(tenant)
+    .with_inbox(Arc::new(InMemoryInboxStore::new()))
+    .with_artifact_store(Arc::new(InMemoryArtifactStore::new()));
+
+// Compile and execute graph
+let graph = StateGraph::new("my_agent")
+    .add_node("step1", my_callable)
+    .add_edge("start", "step1");
+
+let compiled = graph.compile()?;
+let result = kernel.execute_graph(&compiled, input).await?;
+```
+
+## Invariants
+
+See [04-KERNEL_INVARIANTS.md](../../docs/TECHNICAL/04-KERNEL_INVARIANTS.md) for:
+- Execution invariants (determinism, replay, tenant isolation)
+- Graph execution invariants (parallel execution, cycle detection)
+- Inbox invariants (INV-INBOX-001 through INV-INBOX-004)
+- Cancellation invariants (cooperative, propagating, observable)
+- Artifact lifecycle invariants (first-class, deterministic IDs, immutable)
+- Content protection invariants (protected emission, bypass for control)
+- Tool policy invariants (auditable decisions, ToolExecutor enforcement)
+
+## Related Documentation
+
+- [02-ENACT-CORE-ARCHITECTURE.md](../../docs/TECHNICAL/02-ENACT-CORE-ARCHITECTURE.md) - Full architecture
+- [07-EXECUTION_LIFECYCLE.md](../../docs/TECHNICAL/07-EXECUTION_LIFECYCLE.md) - Execution lifecycle
+- [31-MID-EXECUTION-GUIDANCE.md](../../docs/TECHNICAL/31-MID-EXECUTION-GUIDANCE.md) - Inbox system
+- [17-GUARDRAILS-PROTECTION.md](../../docs/TECHNICAL/17-GUARDRAILS-PROTECTION.md) - PII protection