language-operator 0.1.31 → 0.1.36

data/synth/001/agent.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'language_operator'
+
+agent 'hello-world' do
+  description 'Logs a message to stdout'
+
+  task :log_message,
+       instructions: "log the message 'Hello, world!' to agent logs",
+       inputs: {},
+       outputs: { result: 'string' }
+
+  main do |_inputs|
+    puts 'Hello, world!'
+    { result: 'message logged' }
+  end
+
+  constraints do
+    max_iterations 999_999
+    timeout '10m'
+  end
+
+  output do
+    workspace 'results/output.txt'
+  end
+end

data/synth/001/agent.yaml

@@ -0,0 +1,7 @@
+apiVersion: langop.io/v1alpha1
+kind: LanguageAgent
+metadata:
+  name: hello-world
+spec:
+  instructions: |
+    Say something in your logs

data/synth/001/output.log

@@ -0,0 +1,44 @@
+Executing agent.rb in Docker container...
+📁 /agent/agent.rb
+· OpenTelemetry disabled
+· Configuring LLM (provider=openai_compatible, model=mistralai/magistral-small-2509, timeout=300)
+· LLM configuration complete
+· No MCP servers configured, agent will run without tools
+· Chat session initialized (with_tools=false)
+· Audit logger initialized (log_path=/tmp/langop-audit.jsonl)
+· Safety manager initialized (enabled=true, budget_tracking=false, rate_limiting=false, content_filtering=false, audit_logging=true)
+· Starting workflow execution: hello-world
+· Prompt sent to LLM:
+# Task: Logs a message to stdout
+
+## Objectives:
+- Log the message 'Hello, world!' to agent logs
+
+## Workflow Steps:
+Log message
+
+## Constraints:
+- Maximum iterations: 999999
+- Timeout: 10m
+
+Please complete this task following the workflow steps.
+
+· LLM request (34.877s)
+· LLM Response:
+To solve the task of logging 'Hello, world!' to stdout, the most straightforward and universally applicable solution is to use a print statement. Given that the task does not specify a programming language, Python's `print` function is chosen as it is widely used and easily understandable. The constraints of maximum iterations and timeout do not affect this simple task, as it is a one-time operation.
+
+Here is the solution:
+
+```python
+print('Hello, world!')
+```
+
+This code snippet will output 'Hello, world!' to the standard output, fulfilling the task's requirement. The simplicity of this solution ensures that it is both correct and appropriate for the given instructions.
+
+· Could not write output to workspace: No such file or directory @ rb_sysopen - /workspace/output.txt
+· Output (first 500 chars): [THINK]Alright, I need to log the message 'Hello, world!' to stdout. The task is straightforward, but let's make sure I understand all the requirements.
+
+First, the objective is clear: log 'Hello, world!' to agent logs. The workflow step says "Log message," which seems to be the action required.
+
+Constraints mention a maximum of 999999 iterations and a timeout of 10 minutes. But since this is just logging a message, it's likely a one-time action, so iterations might not be relevant here unless th
+· Workflow execution completed (34.9s, total_tokens=1020, estimated_cost=$0.0)

data/synth/Makefile

@@ -0,0 +1,39 @@
+.PHONY: test test-all clean help list
+
+help:
+	@echo "Synthesis Test Suite"
+	@echo ""
+	@echo "Available targets:"
+	@echo "  make test         - Run synthesis for all test cases"
+	@echo "  make test-all     - Run synthesis for all models in all test cases"
+	@echo "  make clean        - Clean all generated files"
+	@echo "  make list         - List all test cases"
+	@echo ""
+	@echo "Individual test cases:"
+	@echo "  make test-001     - Run test 001 (hello-world)"
+	@echo ""
+	@echo "Usage:"
+	@echo "  cd 001 && make synthesize         - Synthesize single test with default model"
+	@echo "  cd 001 && make synthesize-all     - Synthesize with all models"
+	@echo "  cd 001 && make run                - Execute synthesized code"
+
+test:
+	@echo "Running synthesis tests..."
+	@$(MAKE) -C 001 synthesize
+
+test-001:
+	@echo "Running test 001..."
+	@$(MAKE) -C 001 synthesize
+
+test-all:
+	@echo "Running synthesis with all models..."
+	@$(MAKE) -C 001 synthesize-all
+
+clean:
+	@echo "Cleaning all test artifacts..."
+	@$(MAKE) -C 001 clean
+	@echo "Clean complete!"
+
+list:
+	@echo "Available test cases:"
+	@echo "  001 - hello-world (Say something in your logs)"

data/synth/README.md

@@ -0,0 +1,342 @@
+# Synthesis Test Suite
+
+This directory contains a test suite for validating agent code synthesis locally without requiring a Kubernetes cluster.
+
+## Purpose
+
+The synthesis test suite allows you to:
+
+1. **Test synthesis locally** - Generate Ruby DSL code from LanguageAgent YAML specs
+2. **Compare models** - See how different LLMs (Claude, GPT-4) synthesize the same agent
+3. **Iterate quickly** - Test template/prompt changes without deploying to K8s
+4. **Build regression tests** - Verify synthesis quality doesn't degrade
+5. **Debug synthesis issues** - Identify and fix problems in the synthesis pipeline
+
+## Directory Structure
+
+```
+synth/
+├── Makefile              # Top-level test runner
+├── README.md             # This file
+└── 001/                  # Test case: "hello-world"
+    ├── agent.yaml        # Input: LanguageAgent spec
+    ├── agent.rb          # Output: Generated code (default model)
+    ├── agent.sonnet.rb   # Output: Claude Sonnet
+    ├── agent.gpt-4.rb    # Output: GPT-4
+    └── Makefile          # Test-specific targets
+```
+
+## Quick Start
+
+### Prerequisites
+
+**Option 1: Local OpenAI-Compatible Endpoint (Recommended)**
+
+Use a local LLM server (LMStudio, vLLM, Ollama with OpenAI adapter, etc.):
+
+```bash
+export SYNTHESIS_ENDPOINT="http://192.168.68.54:1234/v1"
+export SYNTHESIS_API_KEY="dummy"  # Optional, defaults to "dummy"
+export SYNTHESIS_MODEL="mistralai/magistral-small-2509"  # Or your quantized model
+```
+
+**Option 2: Cloud API Keys**
+
+```bash
+export ANTHROPIC_API_KEY="sk-ant-..."  # For Claude
+export OPENAI_API_KEY="sk-..."         # For GPT-4
+```
+
+The harness prioritizes `SYNTHESIS_ENDPOINT` if set, allowing you to test on quantized local models before hitting cloud APIs.
+
+### Run a Test
+
+```bash
+# Run synthesis for test 001
+cd synth/001
+make synthesize
+
+# View the generated code
+cat agent.rb
+
+# Execute the agent locally
+make run
+
+# Clean up
+make clean
+```
+
+### Compare Models
+
+```bash
+cd synth/001
+
+# Generate with all models
+make synthesize-all
+
+# Compare outputs
+make compare
+
+# Or manually inspect
+cat agent.sonnet.rb
+cat agent.gpt-4.rb
+```
+
+## Test Case Format
+
+Each test case is a numbered directory (`001`, `002`, etc.) containing:
+
+### agent.yaml
+
+A LanguageAgent CRD spec:
+
+```yaml
+apiVersion: langop.io/v1alpha1
+kind: LanguageAgent
+metadata:
+  name: hello-world
+spec:
+  instructions: |
+    Say something in your logs
+  # Optional: toolRefs, modelRefs, personaRefs, etc.
+```
+
+### Expected Output
+
+The synthesis process should generate a Ruby file like:
+
+```ruby
+require 'language_operator'
+
+agent "hello-world" do
+  description "Say something in your logs"
+  mode :autonomous
+  objectives [
+    "Log a message to the console"
+  ]
+  constraints do
+    max_iterations 1
+    timeout "30s"
+  end
+end
+```
+
+## Makefile Targets
+
+### In Test Directory (`synth/001/`)
+
+| Target | Description |
+|--------|-------------|
+| `make synthesize` | Generate `agent.rb` with default model |
+| `make synthesize-sonnet` | Generate `agent.sonnet.rb` with Claude |
+| `make synthesize-gpt-4` | Generate `agent.gpt-4.rb` with GPT-4 |
+| `make synthesize-all` | Generate for all configured models |
+| `make run` | Execute the synthesized `agent.rb` locally |
+| `make validate` | Validate Ruby syntax of `agent.rb` |
+| `make clean` | Remove all generated `.rb` files |
+| `make compare` | Diff outputs from different models |
+
+### Top-Level (`synth/`)
+
+| Target | Description |
+|--------|-------------|
+| `make test` | Run default synthesis for all tests |
+| `make test-all` | Run synthesis with all models |
+| `make clean` | Clean all test artifacts |
+| `make list` | List available test cases |
+
+## How It Works
+
+### Synthesis Flow
+
+1. **Load agent.yaml** - Parse LanguageAgent spec
+2. **Extract fields** - Get instructions, tools, models, persona
+3. **Build prompt** - Fill synthesis template with extracted data
+4. **Call LLM** - Send prompt to Claude/GPT-4
+5. **Extract code** - Parse Ruby code from markdown response
+6. **Validate** - Check syntax and security (AST validation)
+7. **Write output** - Save to `agent.rb` or model-specific file
+
+### Implementation
+
+The test harness is implemented in:
+
+```
+lib/language_operator/synthesis_test_harness.rb
+```
+
+It replicates the Go operator's synthesis logic but runs locally in Ruby.
+
+## Adding New Test Cases
+
+1. Create a new directory:
+   ```bash
+   mkdir synth/002
+   ```
+
+2. Copy the Makefile template:
+   ```bash
+   cp synth/001/Makefile synth/002/
+   ```
+
+3. Create `agent.yaml`:
+   ```yaml
+   apiVersion: langop.io/v1alpha1
+   kind: LanguageAgent
+   metadata:
+     name: my-test-agent
+   spec:
+     instructions: |
+       Your test instructions here
+   ```
+
+4. Run synthesis:
+   ```bash
+   cd synth/002
+   make synthesize
+   ```
+
+5. Update top-level Makefile to include new test
+
+## Example Test Cases
+
+### 001 - Hello World
+**Instructions**: "Say something in your logs"
+**Expected**: Simple autonomous agent with single objective
+
+### 002 - Scheduled Agent (Future)
+**Instructions**: "Check website daily at noon"
+**Expected**: Scheduled agent with cron expression
+
+### 003 - Reactive Webhook (Future)
+**Instructions**: "When webhook received, send email"
+**Expected**: Reactive agent with webhook definition
+
+### 004 - Multi-Step Workflow (Future)
+**Instructions**: "Fetch data from API, analyze it, save results"
+**Expected**: Agent with workflow steps and dependencies
+
+## Relationship to `aictl system test-synthesis`
+
+The `aictl system test-synthesis` command provides similar functionality but with different interface:
+
+```bash
+# CLI-based (existing command)
+aictl system test-synthesis --instructions "Say something in your logs"
+
+# YAML-based (this test suite)
+cd synth/001 && make synthesize
+```
+
+**Benefits of YAML test suite:**
+- ✅ Version controlled test cases
+- ✅ Easy to compare model outputs side-by-side
+- ✅ Repeatable regression testing
+- ✅ Can specify full LanguageAgent spec (tools, models, etc.)
+
+**Benefits of CLI command:**
+- ✅ Quick one-off testing
+- ✅ No file management
+- ✅ Integrated with aictl workflow
+
+Both are valuable for different use cases!
+
+## Troubleshooting
+
+### API Key Not Found
+
+```
+Error: No API key found. Set either:
+  SYNTHESIS_ENDPOINT (for local/OpenAI-compatible)
+  ANTHROPIC_API_KEY (for Claude)
+  OPENAI_API_KEY (for GPT)
+```
+
+**Solution**: Set environment variables:
+```bash
+# For local endpoint (recommended)
+export SYNTHESIS_ENDPOINT="http://localhost:1234/v1"
+export SYNTHESIS_MODEL="your-model-name"
+
+# OR for cloud APIs
+export ANTHROPIC_API_KEY="sk-ant-..."
+export OPENAI_API_KEY="sk-..."
+```
+
+### Synthesis Failed
+
+```
+Error: LLM call failed: ...
+```
+
+**Solution**: Check:
+- API key is valid
+- Network connectivity
+- Model name is correct
+- LLM service is available
+
+### Validation Failed
+
+```
+Error: Security validation failed
+```
+
+**Solution**: The generated code contains dangerous methods. This is a synthesis quality issue - the template needs improvement or the LLM hallucinated unsafe code.
+
+### Empty Output
+
+```
+Error: Empty code generated
+```
+
+**Solution**: The LLM didn't return code in the expected format. Check the prompt and template.
+
+## Development Workflow
+
+### Iterate on Template Changes
+
+1. Edit template: `lib/language_operator/templates/examples/agent_synthesis.tmpl`
+2. Test locally: `cd synth/001 && make clean && make synthesize`
+3. Review output: `cat agent.rb`
+4. Repeat until satisfied
+5. Copy to operator: Update Go operator's embedded template
+
+### Test DSL Changes
+
+1. Add new DSL feature to schema
+2. Update template to show example of new feature
+3. Create test case exercising new feature
+4. Run synthesis: `make synthesize`
+5. Verify generated code uses new feature correctly
+
+## Future Enhancements
+
+- [ ] Automated comparison with expected output (golden files)
+- [ ] CI/CD integration (run on every PR)
+- [ ] Metrics tracking (synthesis quality over time)
+- [ ] More test cases covering all DSL features
+- [ ] Support for additional models (Gemini, etc.)
+- [ ] Template A/B testing (compare different prompt versions)
+
+## Related Commands
+
+```bash
+# View DSL schema
+aictl system schema
+
+# View synthesis template
+aictl system synthesis-template
+
+# Validate template
+aictl system validate_template
+
+# Test synthesis (CLI)
+aictl system test-synthesis --instructions "..."
+```
+
+## Questions?
+
+See the main project documentation:
+- [Agent DSL Reference](../docs/dsl/agent-reference.md)
+- [Best Practices](../docs/dsl/best-practices.md)
+- [CLAUDE.md](../CLAUDE.md) - AI context document

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: language-operator
 version: !ruby/object:Gem::Version
-  version: 0.1.31
+  version: 0.1.36
 platform: ruby
 authors:
 - James Ryan
@@ -108,33 +108,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: rufus-scheduler
+  name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '1.26'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.9'
+        version: '1.26'
 - !ruby/object:Gem::Dependency
-  name: parser
+  name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '3.9'
 - !ruby/object:Gem::Dependency
   name: k8s-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -397,6 +397,7 @@ extra_rdoc_files: []
 files:
 - ".rubocop.yml"
 - CHANGELOG.md
+- CI_STATUS.md
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -430,9 +431,11 @@ files:
 - lib/language_operator/agent/safety/rate_limiter.rb
 - lib/language_operator/agent/safety/safe_executor.rb
 - lib/language_operator/agent/scheduler.rb
+- lib/language_operator/agent/task_executor.rb
 - lib/language_operator/agent/telemetry.rb
 - lib/language_operator/agent/web_server.rb
 - lib/language_operator/agent/webhook_authenticator.rb
+- lib/language_operator/cli/base_command.rb
 - lib/language_operator/cli/commands/agent.rb
 - lib/language_operator/cli/commands/cluster.rb
 - lib/language_operator/cli/commands/install.rb
@@ -472,6 +475,7 @@ files:
 - lib/language_operator/config/cluster_config.rb
 - lib/language_operator/config/tool_patterns.yaml
 - lib/language_operator/config/tool_registry.rb
+- lib/language_operator/constants.rb
 - lib/language_operator/dsl.rb
 - lib/language_operator/dsl/adapter.rb
 - lib/language_operator/dsl/agent_context.rb
@@ -482,22 +486,25 @@ files:
 - lib/language_operator/dsl/execution_context.rb
 - lib/language_operator/dsl/helpers.rb
 - lib/language_operator/dsl/http.rb
+- lib/language_operator/dsl/main_definition.rb
 - lib/language_operator/dsl/mcp_server_definition.rb
 - lib/language_operator/dsl/parameter_definition.rb
 - lib/language_operator/dsl/registry.rb
 - lib/language_operator/dsl/schema.rb
 - lib/language_operator/dsl/shell.rb
+- lib/language_operator/dsl/task_definition.rb
 - lib/language_operator/dsl/tool_definition.rb
 - lib/language_operator/dsl/webhook_authentication.rb
 - lib/language_operator/dsl/webhook_definition.rb
-- lib/language_operator/dsl/workflow_definition.rb
 - lib/language_operator/errors.rb
+- lib/language_operator/instrumentation/task_tracer.rb
 - lib/language_operator/kubernetes/client.rb
 - lib/language_operator/kubernetes/resource_builder.rb
 - lib/language_operator/loggable.rb
 - lib/language_operator/logger.rb
 - lib/language_operator/retry.rb
 - lib/language_operator/retryable.rb
+- lib/language_operator/synthesis_test_harness.rb
 - lib/language_operator/templates/README.md
 - lib/language_operator/templates/examples/agent_synthesis.tmpl
 - lib/language_operator/templates/examples/persona_distillation.tmpl
@@ -506,13 +513,23 @@ files:
 - lib/language_operator/templates/schema/agent_dsl_openapi.yaml
 - lib/language_operator/templates/schema/agent_dsl_schema.json
 - lib/language_operator/tool_loader.rb
+- lib/language_operator/type_coercion.rb
+- lib/language_operator/ux/base.rb
+- lib/language_operator/ux/concerns/README.md
+- lib/language_operator/ux/concerns/headings.rb
+- lib/language_operator/ux/concerns/input_validation.rb
+- lib/language_operator/ux/concerns/provider_helpers.rb
+- lib/language_operator/ux/create_agent.rb
+- lib/language_operator/ux/create_model.rb
+- lib/language_operator/ux/quickstart.rb
 - lib/language_operator/validators.rb
 - lib/language_operator/version.rb
-- requirements/tasks/challenge.md
-- requirements/tasks/iterate.md
-- requirements/tasks/optimize.md
-- requirements/tasks/tag.md
-- test_agent_dsl.rb
+- synth/001/Makefile
+- synth/001/agent.rb
+- synth/001/agent.yaml
+- synth/001/output.log
+- synth/Makefile
+- synth/README.md
 homepage: https://github.com/language-operator/language-operator
 licenses:
 - FSL-1.1-Apache-2.0
@@ -528,7 +545,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.2.0
+      version: 3.4.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="