language-operator 0.1.63 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.plan.md +127 -0
- data/.rspec +3 -0
- data/Gemfile +2 -0
- data/Gemfile.lock +4 -1
- data/Makefile +34 -80
- data/README.md +20 -1
- data/components/agent/Gemfile +1 -1
- data/docs/cheat-sheet.md +173 -0
- data/docs/observability.md +208 -0
- data/lib/language_operator/agent/base.rb +10 -1
- data/lib/language_operator/agent/event_config.rb +172 -0
- data/lib/language_operator/agent/safety/ast_validator.rb +1 -1
- data/lib/language_operator/agent/safety/safe_executor.rb +5 -1
- data/lib/language_operator/agent/task_executor.rb +97 -7
- data/lib/language_operator/agent/telemetry.rb +25 -3
- data/lib/language_operator/agent/web_server.rb +6 -9
- data/lib/language_operator/agent.rb +24 -14
- data/lib/language_operator/cli/commands/agent/base.rb +155 -64
- data/lib/language_operator/cli/commands/agent/code_operations.rb +157 -16
- data/lib/language_operator/cli/commands/cluster.rb +2 -2
- data/lib/language_operator/cli/commands/status.rb +2 -2
- data/lib/language_operator/cli/commands/system/synthesize.rb +1 -1
- data/lib/language_operator/cli/errors/suggestions.rb +1 -1
- data/lib/language_operator/cli/formatters/value_formatter.rb +1 -1
- data/lib/language_operator/cli/helpers/ux_helper.rb +3 -4
- data/lib/language_operator/config.rb +3 -3
- data/lib/language_operator/constants/kubernetes_labels.rb +2 -2
- data/lib/language_operator/constants.rb +1 -0
- data/lib/language_operator/dsl/task_definition.rb +18 -7
- data/lib/language_operator/instrumentation/task_tracer.rb +44 -3
- data/lib/language_operator/kubernetes/client.rb +112 -1
- data/lib/language_operator/templates/schema/CHANGELOG.md +28 -0
- data/lib/language_operator/templates/schema/agent_dsl_openapi.yaml +1 -1
- data/lib/language_operator/templates/schema/agent_dsl_schema.json +1 -1
- data/lib/language_operator/type_coercion.rb +22 -8
- data/lib/language_operator/version.rb +1 -1
- data/synth/002/agent.rb +23 -12
- data/synth/002/output.log +88 -15
- data/synth/003/Makefile +17 -4
- data/synth/003/agent.txt +1 -1
- data/synth/004/Makefile +54 -0
- data/synth/004/README.md +281 -0
- data/synth/004/instructions.txt +1 -0
- metadata +11 -6
- data/lib/language_operator/cli/commands/agent/learning.rb +0 -289
- data/synth/003/agent.optimized.rb +0 -66
- data/synth/003/agent.synthesized.rb +0 -41
data/synth/002/output.log
CHANGED
|
@@ -1,21 +1,94 @@
|
|
|
1
|
+
[36m⚬[0m [2mStreaming logs for agent 's002'...[0m
|
|
2
|
+
|
|
3
|
+
[1;36m⚬[0m Language Operator v0.1.61
|
|
1
4
|
[1;36m⚬[0m OpenTelemetry disabled
|
|
2
|
-
[1;36m⚬[0m Configuring LLM (provider=openai_compatible, model=
|
|
5
|
+
[1;36m⚬[0m Configuring LLM (provider=openai_compatible, model=qwen3-coder:30b, timeout=300)
|
|
3
6
|
[1;36m⚬[0m LLM configuration complete
|
|
4
7
|
[1;36m⚬[0m No MCP servers configured, agent will run without tools
|
|
5
|
-
[1;36m⚬[0m Chat session initialized (with_tools=false)
|
|
6
|
-
[1;36m⚬[0m
|
|
8
|
+
[1;36m⚬[0m Chat session initialized (with_tools=false, total_tools=0)
|
|
9
|
+
[1;36m⚬[0m Agent running in scheduled mode - executing once (agent_name=s002, dsl_version=v1)
|
|
10
|
+
[1;36m⚬[0m Executing main block (agent=s002, task_count=2)
|
|
7
11
|
[1;36m⚬[0m Executing main block (inputs_keys=[])
|
|
8
|
-
[1;36m⚬[0m
|
|
9
|
-
[1;36m⚬[0m
|
|
10
|
-
[1;36m⚬[0m
|
|
11
|
-
[1;36m⚬[0m
|
|
12
|
-
[1;36m⚬[0m
|
|
13
|
-
|
|
14
|
-
[1;36m⚬[0m
|
|
15
|
-
[1;36m⚬[0m
|
|
12
|
+
[1;36m⚬[0m Sending prompt to LLM (task=generate_fortune, prompt_length=632, available_tools=[])
|
|
13
|
+
[1;36m⚬[0m LLM response received, extracting content (task=generate_fortune, response_class=RubyLLM::Message, has_tool_calls=, tool_call_count=0)
|
|
14
|
+
[1;36m⚬[0m Neural task response received (task=generate_fortune, response_length=563)
|
|
15
|
+
[1;36m⚬[0m Parsing neural task response (task=generate_fortune)
|
|
16
|
+
[1;36m⚬[0m LLM thinking captured (event=llm_thinking, task=generate_fortune, thinking_steps=1, thinking=["\nI need to generate a short, positive fortune message that is inspiring and uplifting, under 100 words. The message should be meaningful and encourage the reader. I'll create a brief, motivational message about perseverance and positive outcomes.\n\n"], thinking_preview=
|
|
17
|
+
I need to generate a short, positive fortune message that is inspiring and uplifting, under 100 w...)
|
|
18
|
+
[1;36m⚬[0m Response parsed successfully (task=generate_fortune, output_keys=[:fortune])
|
|
19
|
+
[1;36m⚬[0m Validating task outputs (task=generate_fortune)
|
|
20
|
+
[1;36m⚬[0m Sending prompt to LLM (task=format_output, prompt_length=918, available_tools=[])
|
|
21
|
+
[1;36m⚬[0m LLM response received, extracting content (task=format_output, response_class=RubyLLM::Message, has_tool_calls=, tool_call_count=0)
|
|
22
|
+
[1;36m⚬[0m Neural task response received (task=format_output, response_length=583)
|
|
23
|
+
[1;36m⚬[0m Parsing neural task response (task=format_output)
|
|
24
|
+
[1;36m⚬[0m LLM thinking captured (event=llm_thinking, task=format_output, thinking_steps=1, thinking=["\nI need to format the fortune message into a readable output string with the title 'Your Fortune:' as specified. The input fortune message is already provided and needs to be wrapped with the title. I'll create a JSON object with the formatted message.\n\n"], thinking_preview=
|
|
25
|
+
I need to format the fortune message into a readable output string with the title 'Your Fortune:'...)
|
|
26
|
+
[1;36m⚬[0m Response parsed successfully (task=format_output, output_keys=[:message])
|
|
27
|
+
[1;36m⚬[0m Validating task outputs (task=format_output)
|
|
28
|
+
[1;36m⚬[0m Main execution (13.01s)
|
|
16
29
|
[1;36m⚬[0m Main block completed
|
|
17
|
-
[1;36m⚬[0m Main block execution completed (result={
|
|
18
|
-
Your
|
|
19
|
-
[
|
|
20
|
-
|
|
30
|
+
[1;36m⚬[0m Main block execution completed (result={message: "Your Fortune: Every challenge you face today is building strength you didn't know you had. Trust the process, believe in yourself, and remember that brighter days are always ahead. Your hard work and dedication will soon bring rewarding results. Keep moving forward with hope and determination."})
|
|
31
|
+
Your Fortune: Every challenge you face today is building strength you didn't know you had. Trust the process, believe in yourself, and remember that brighter days are always ahead. Your hard work and dedication will soon bring rewarding results. Keep moving forward with hope and determination.
|
|
32
|
+
[1;36m⚬[0m Scheduled execution completed - exiting (agent_name=s002)
|
|
33
|
+
[1;36m⚬[0m Language Operator v0.1.61
|
|
34
|
+
[1;36m⚬[0m OpenTelemetry disabled
|
|
35
|
+
[1;36m⚬[0m Configuring LLM (provider=openai_compatible, model=qwen3-coder:30b, timeout=300)
|
|
36
|
+
[1;36m⚬[0m LLM configuration complete
|
|
37
|
+
[1;36m⚬[0m No MCP servers configured, agent will run without tools
|
|
38
|
+
[1;36m⚬[0m Chat session initialized (with_tools=false, total_tools=0)
|
|
39
|
+
[1;36m⚬[0m Agent running in scheduled mode - executing once (agent_name=s002, dsl_version=v1)
|
|
40
|
+
[1;36m⚬[0m Executing main block (agent=s002, task_count=2)
|
|
41
|
+
[1;36m⚬[0m Executing main block (inputs_keys=[])
|
|
42
|
+
[1;36m⚬[0m Sending prompt to LLM (task=generate_fortune, prompt_length=632, available_tools=[])
|
|
43
|
+
[1;36m⚬[0m LLM response received, extracting content (task=generate_fortune, response_class=RubyLLM::Message, has_tool_calls=, tool_call_count=0)
|
|
44
|
+
[1;36m⚬[0m Neural task response received (task=generate_fortune, response_length=707)
|
|
45
|
+
[1;36m⚬[0m Parsing neural task response (task=generate_fortune)
|
|
46
|
+
[1;36m⚬[0m LLM thinking captured (event=llm_thinking, task=generate_fortune, thinking_steps=1, thinking=["\nI need to create a short, positive fortune message that is inspiring and uplifting, under 100 words. The message should be generally encouraging and positive in nature. Let me think of a message that conveys hope, perseverance, or personal growth in a concise way.\n\nI'll create a fortune that emphasizes the power of persistence and believing in oneself - a common theme that's universally uplifting and inspiring.\n"], thinking_preview=
|
|
47
|
+
I need to create a short, positive fortune message that is inspiring and uplifting, under 100 wor...)
|
|
48
|
+
[1;36m⚬[0m Response parsed successfully (task=generate_fortune, output_keys=[:fortune])
|
|
49
|
+
[1;36m⚬[0m Validating task outputs (task=generate_fortune)
|
|
50
|
+
[1;36m⚬[0m Sending prompt to LLM (task=format_output, prompt_length=893, available_tools=[])
|
|
51
|
+
[1;36m⚬[0m LLM response received, extracting content (task=format_output, response_class=RubyLLM::Message, has_tool_calls=, tool_call_count=0)
|
|
52
|
+
[1;36m⚬[0m Neural task response received (task=format_output, response_length=745)
|
|
53
|
+
[1;36m⚬[0m Parsing neural task response (task=format_output)
|
|
54
|
+
[1;36m⚬[0m LLM thinking captured (event=llm_thinking, task=format_output, thinking_steps=1, thinking=["\nI need to format the fortune message into a readable output string with the title 'Your Fortune:' as requested. The fortune message is quite long, so I'll format it nicely with proper spacing and structure. I'll create a formatted string that includes the title and the fortune content.\n\nI'll create a clean, readable format that presents the fortune in an appealing way while keeping it concise and maintaining the inspirational tone.\n"], thinking_preview=
|
|
55
|
+
I need to format the fortune message into a readable output string with the title 'Your Fortune:'...)
|
|
56
|
+
[1;36m⚬[0m Response parsed successfully (task=format_output, output_keys=[:message])
|
|
57
|
+
[1;36m⚬[0m Validating task outputs (task=format_output)
|
|
58
|
+
[1;36m⚬[0m Main execution (12.698s)
|
|
59
|
+
[1;36m⚬[0m Main block completed
|
|
60
|
+
[1;36m⚬[0m Main block execution completed (result={message: "Your Fortune:\n\nEvery challenge you face today is building the strength you'll need for tomorrow. Believe in your ability to grow through difficulties and trust that better days are ahead. You have within you everything you need to succeed and create the life you desire."})
|
|
61
|
+
Your Fortune:
|
|
21
62
|
|
|
63
|
+
Every challenge you face today is building the strength you'll need for tomorrow. Believe in your ability to grow through difficulties and trust that better days are ahead. You have within you everything you need to succeed and create the life you desire.
|
|
64
|
+
[1;36m⚬[0m Scheduled execution completed - exiting (agent_name=s002)
|
|
65
|
+
[1;36m⚬[0m Language Operator v0.1.61
|
|
66
|
+
[1;36m⚬[0m OpenTelemetry disabled
|
|
67
|
+
[1;36m⚬[0m Configuring LLM (provider=openai_compatible, model=qwen3-coder:30b, timeout=300)
|
|
68
|
+
[1;36m⚬[0m LLM configuration complete
|
|
69
|
+
[1;36m⚬[0m No MCP servers configured, agent will run without tools
|
|
70
|
+
[1;36m⚬[0m Chat session initialized (with_tools=false, total_tools=0)
|
|
71
|
+
[1;36m⚬[0m Agent running in scheduled mode - executing once (agent_name=s002, dsl_version=v1)
|
|
72
|
+
[1;36m⚬[0m Executing main block (agent=s002, task_count=2)
|
|
73
|
+
[1;36m⚬[0m Executing main block (inputs_keys=[])
|
|
74
|
+
[1;36m⚬[0m Sending prompt to LLM (task=generate_fortune, prompt_length=632, available_tools=[])
|
|
75
|
+
[1;36m⚬[0m LLM response received, extracting content (task=generate_fortune, response_class=RubyLLM::Message, has_tool_calls=, tool_call_count=0)
|
|
76
|
+
[1;36m⚬[0m Neural task response received (task=generate_fortune, response_length=591)
|
|
77
|
+
[1;36m⚬[0m Parsing neural task response (task=generate_fortune)
|
|
78
|
+
[1;36m⚬[0m LLM thinking captured (event=llm_thinking, task=generate_fortune, thinking_steps=1, thinking=["\nI need to create a positive, inspiring fortune message under 100 words. The message should be uplifting and motivational. Let me craft something that feels genuine and encouraging.\n\nThe fortune should be short and impactful, focusing on positivity and inspiration. I'll make sure it's under 100 words as requested.\n"], thinking_preview=
|
|
79
|
+
I need to create a positive, inspiring fortune message under 100 words. The message should be upl...)
|
|
80
|
+
[1;36m⚬[0m Response parsed successfully (task=generate_fortune, output_keys=[:fortune])
|
|
81
|
+
[1;36m⚬[0m Validating task outputs (task=generate_fortune)
|
|
82
|
+
[1;36m⚬[0m Sending prompt to LLM (task=format_output, prompt_length=877, available_tools=[])
|
|
83
|
+
[1;36m⚬[0m LLM response received, extracting content (task=format_output, response_class=RubyLLM::Message, has_tool_calls=, tool_call_count=0)
|
|
84
|
+
[1;36m⚬[0m Neural task response received (task=format_output, response_length=491)
|
|
85
|
+
[1;36m⚬[0m Parsing neural task response (task=format_output)
|
|
86
|
+
[1;36m⚬[0m LLM thinking captured (event=llm_thinking, task=format_output, thinking_steps=1, thinking=["\nI need to format the fortune message with the title 'Your Fortune:' as requested. The message should be properly formatted with this title prefix. I'll create a JSON object with the formatted message.\n"], thinking_preview=
|
|
87
|
+
I need to format the fortune message with the title 'Your Fortune:' as requested. The message sho...)
|
|
88
|
+
[1;36m⚬[0m Response parsed successfully (task=format_output, output_keys=[:message])
|
|
89
|
+
[1;36m⚬[0m Validating task outputs (task=format_output)
|
|
90
|
+
[1;36m⚬[0m Main execution (12.256s)
|
|
91
|
+
[1;36m⚬[0m Main block completed
|
|
92
|
+
[1;36m⚬[0m Main block execution completed (result={message: "Your Fortune: Your kindness creates ripples of joy that extend far beyond what you can see. Every small act of compassion matters, and your positive energy lights up the world around you. Trust in your ability to make a difference, one moment at a time."})
|
|
93
|
+
Your Fortune: Your kindness creates ripples of joy that extend far beyond what you can see. Every small act of compassion matters, and your positive energy lights up the world around you. Trust in your ability to make a difference, one moment at a time.
|
|
94
|
+
[1;36m⚬[0m Scheduled execution completed - exiting (agent_name=s002)
|
data/synth/003/Makefile
CHANGED
|
@@ -10,21 +10,34 @@ create:
|
|
|
10
10
|
run:
|
|
11
11
|
@JOB_NAME=$(AGENT)-$(shell date +%s); \
|
|
12
12
|
kubectl create job --from=cronjob/$(AGENT) $$JOB_NAME && \
|
|
13
|
-
trap "kubectl delete job $$JOB_NAME" EXIT; \
|
|
14
13
|
kubectl wait --for=condition=ready pod -l job-name=$$JOB_NAME --timeout=60s && \
|
|
15
|
-
kubectl logs -f job/$$JOB_NAME
|
|
14
|
+
kubectl logs -f job/$$JOB_NAME && \
|
|
15
|
+
sleep 15 && \
|
|
16
|
+
kubectl delete job $$JOB_NAME
|
|
16
17
|
|
|
17
18
|
code:
|
|
18
19
|
$(AICTL) code $(AGENT)
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
|
|
21
|
+
versions:
|
|
22
|
+
$(AICTL) versions $(AGENT)
|
|
23
|
+
|
|
24
|
+
inspect:
|
|
25
|
+
$(AICTL) inspect $(AGENT)
|
|
26
|
+
|
|
27
|
+
learning-status:
|
|
28
|
+
$(AICTL) learning status $(AGENT)
|
|
29
|
+
|
|
30
|
+
learn:
|
|
31
|
+
kubectl patch languageagent $(AGENT) --type='merge' -p='{"status":{"runsPendingLearning":10}}'
|
|
32
|
+
|
|
22
33
|
|
|
23
34
|
logs:
|
|
24
35
|
$(AICTL) logs $(AGENT)
|
|
25
36
|
|
|
26
37
|
clean:
|
|
27
38
|
$(AICTL) delete $(AGENT) --force
|
|
39
|
+
kubectl delete configmaps -l app.kubernetes.io/name=$(AGENT) --ignore-not-found=true
|
|
40
|
+
kubectl delete configmaps --field-selector metadata.name~=$(AGENT)-v --ignore-not-found=true
|
|
28
41
|
|
|
29
42
|
save:
|
|
30
43
|
$(AICTL) code $(AGENT) --raw > agent.rb
|
data/synth/003/agent.txt
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
Write a story one sentence at a time, with one new sentence every
|
|
1
|
+
Write a story one sentence at a time, with one new sentence every 10 minutes.
|
data/synth/004/Makefile
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
.PHONY: create code logs clean events validate run
|
|
2
|
+
|
|
3
|
+
AGENT := s004
|
|
4
|
+
AICTL := bundle exec ../../bin/aictl agent
|
|
5
|
+
TOOLS := workspace
|
|
6
|
+
|
|
7
|
+
create:
|
|
8
|
+
cat instructions.txt | $(AICTL) create --name $(AGENT) --tools "$(TOOLS)"
|
|
9
|
+
|
|
10
|
+
run:
|
|
11
|
+
@JOB_NAME=$(AGENT)-$(shell date +%s); \
|
|
12
|
+
kubectl create job --from=cronjob/$(AGENT) $$JOB_NAME && \
|
|
13
|
+
trap "kubectl delete job $$JOB_NAME" EXIT; \
|
|
14
|
+
kubectl wait --for=condition=ready pod -l job-name=$$JOB_NAME --timeout=60s && \
|
|
15
|
+
kubectl logs -f job/$$JOB_NAME
|
|
16
|
+
|
|
17
|
+
code:
|
|
18
|
+
$(AICTL) code $(AGENT)
|
|
19
|
+
|
|
20
|
+
inspect:
|
|
21
|
+
$(AICTL) inspect $(AGENT)
|
|
22
|
+
|
|
23
|
+
logs:
|
|
24
|
+
$(AICTL) logs $(AGENT)
|
|
25
|
+
|
|
26
|
+
# Health monitoring specific commands
|
|
27
|
+
events:
|
|
28
|
+
@echo "=== Checking for Task Completion events ===" && \
|
|
29
|
+
kubectl get events -n language-operator --field-selector involvedObject.name=$(AGENT) --sort-by='.lastTimestamp' || \
|
|
30
|
+
echo "No events found - this may indicate observability issues"
|
|
31
|
+
|
|
32
|
+
validate:
|
|
33
|
+
@echo "=== Validating observability infrastructure ===" && \
|
|
34
|
+
echo "1. Checking if agent exists..." && \
|
|
35
|
+
$(AICTL) inspect $(AGENT) && \
|
|
36
|
+
echo "2. Checking recent logs..." && \
|
|
37
|
+
$(AICTL) logs $(AGENT) --tail=20 && \
|
|
38
|
+
echo "3. Checking for events..." && \
|
|
39
|
+
make events && \
|
|
40
|
+
echo "4. Checking CronJob status..." && \
|
|
41
|
+
kubectl get cronjob $(AGENT) -n language-operator
|
|
42
|
+
|
|
43
|
+
health:
|
|
44
|
+
@echo "=== System Health Check Status ===" && \
|
|
45
|
+
echo "Last execution logs:" && \
|
|
46
|
+
$(AICTL) logs $(AGENT) --tail=50 | grep -E "(healthy|error|fail|success|reachable)" || \
|
|
47
|
+
echo "No health status found in recent logs"
|
|
48
|
+
|
|
49
|
+
clean:
|
|
50
|
+
$(AICTL) delete $(AGENT) --force
|
|
51
|
+
|
|
52
|
+
save:
|
|
53
|
+
$(AICTL) code $(AGENT) --raw > agent.rb
|
|
54
|
+
$(AICTL) logs $(AGENT) > output.log
|
data/synth/004/README.md
ADDED
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
# 004 - System Health Monitoring & Observability Validation
|
|
2
|
+
|
|
3
|
+
## Instructions
|
|
4
|
+
|
|
5
|
+
"Check if the system is healthy every 5 minutes and report any issues"
|
|
6
|
+
|
|
7
|
+
## Significance
|
|
8
|
+
|
|
9
|
+
This validates observability infrastructure and system self-monitoring capabilities in DSL v1.
|
|
10
|
+
|
|
11
|
+
While tests 001-003 validated synthesis, neural execution, and learning, this test validates that the system can properly observe and report on its own health - including the critical infrastructure required for observability itself.
|
|
12
|
+
|
|
13
|
+
This is the first synthesis test focused on infrastructure resilience and event emission validation.
|
|
14
|
+
|
|
15
|
+
## What This Demonstrates
|
|
16
|
+
|
|
17
|
+
### 1. Self-Observability Architecture
|
|
18
|
+
|
|
19
|
+
The synthesized agent should naturally generate tasks that validate:
|
|
20
|
+
- **Kubernetes API connectivity** - Can reach cluster API for event emission
|
|
21
|
+
- **LLM endpoint availability** - Can execute neural tasks
|
|
22
|
+
- **Monitoring system integration** - Can send telemetry data
|
|
23
|
+
- **Event emission functionality** - Task Completion events are generated
|
|
24
|
+
- **Network policy compliance** - Agent can reach required services
|
|
25
|
+
|
|
26
|
+
### 2. Infrastructure Resilience Testing
|
|
27
|
+
|
|
28
|
+
```ruby
|
|
29
|
+
# Expected synthesis pattern
|
|
30
|
+
task :check_kubernetes_api,
|
|
31
|
+
instructions: "Test connectivity to Kubernetes API server",
|
|
32
|
+
outputs: { reachable: 'boolean', response_time_ms: 'integer' }
|
|
33
|
+
|
|
34
|
+
task :check_llm_endpoint,
|
|
35
|
+
instructions: "Verify LLM model is responding",
|
|
36
|
+
outputs: { available: 'boolean', model_name: 'string' }
|
|
37
|
+
|
|
38
|
+
task :check_monitoring_system,
|
|
39
|
+
instructions: "Test OpenTelemetry/monitoring endpoint connectivity",
|
|
40
|
+
outputs: { telemetry_working: 'boolean', endpoint: 'string' }
|
|
41
|
+
|
|
42
|
+
task :validate_event_emission,
|
|
43
|
+
instructions: "Ensure task execution events are being emitted properly",
|
|
44
|
+
outputs: { events_working: 'boolean', last_event_time: 'string' }
|
|
45
|
+
|
|
46
|
+
main do |inputs|
|
|
47
|
+
# Systematic health validation
|
|
48
|
+
api_status = execute_task(:check_kubernetes_api)
|
|
49
|
+
llm_status = execute_task(:check_llm_endpoint)
|
|
50
|
+
monitoring_status = execute_task(:check_monitoring_system)
|
|
51
|
+
events_status = execute_task(:validate_event_emission)
|
|
52
|
+
|
|
53
|
+
# Overall health determination and reporting
|
|
54
|
+
overall_healthy = api_status[:reachable] &&
|
|
55
|
+
llm_status[:available] &&
|
|
56
|
+
monitoring_status[:telemetry_working] &&
|
|
57
|
+
events_status[:events_working]
|
|
58
|
+
|
|
59
|
+
{
|
|
60
|
+
healthy: overall_healthy,
|
|
61
|
+
components: {
|
|
62
|
+
kubernetes: api_status,
|
|
63
|
+
llm: llm_status,
|
|
64
|
+
monitoring: monitoring_status,
|
|
65
|
+
events: events_status
|
|
66
|
+
},
|
|
67
|
+
timestamp: Time.now.iso8601
|
|
68
|
+
}
|
|
69
|
+
end
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
### 3. Scheduled Health Monitoring
|
|
73
|
+
|
|
74
|
+
```ruby
|
|
75
|
+
mode :scheduled
|
|
76
|
+
schedule "*/5 * * * *" # Every 5 minutes
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Each execution:
|
|
80
|
+
1. Tests all critical infrastructure components
|
|
81
|
+
2. Generates multiple Task Completion events (validation target)
|
|
82
|
+
3. Reports comprehensive health status
|
|
83
|
+
4. Creates telemetry spans for observability
|
|
84
|
+
5. Exits and waits for next scheduled run
|
|
85
|
+
|
|
86
|
+
### 4. Event Emission Validation (Critical)
|
|
87
|
+
|
|
88
|
+
This test specifically validates the Task Completion event infrastructure that was failing due to NetworkPolicy issues. Each health check execution should generate:
|
|
89
|
+
|
|
90
|
+
- **Task Start events** for each health check task
|
|
91
|
+
- **Task Completion events** (success/failure) for each task
|
|
92
|
+
- **OpenTelemetry traces** with health check spans
|
|
93
|
+
- **Agent execution events** for overall run status
|
|
94
|
+
|
|
95
|
+
**Key Validation**: If Task Completion events are missing, the health check itself reports the observability system as unhealthy.
|
|
96
|
+
|
|
97
|
+
### 5. Meta-Validation Property
|
|
98
|
+
|
|
99
|
+
The test has a beautiful self-validating property:
|
|
100
|
+
|
|
101
|
+
```
|
|
102
|
+
IF observability infrastructure is broken
|
|
103
|
+
THEN health check cannot report properly
|
|
104
|
+
THEN system correctly reports as unhealthy
|
|
105
|
+
THEREFORE test reveals infrastructure issues
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
This makes it impossible for observability problems to hide - they become part of the health status.
|
|
109
|
+
|
|
110
|
+
## Why This Matters
|
|
111
|
+
|
|
112
|
+
### Fills Critical Testing Gap
|
|
113
|
+
|
|
114
|
+
Current synthesis test coverage:
|
|
115
|
+
- ✅ **001**: Basic synthesis functionality
|
|
116
|
+
- ✅ **002**: Neural task execution + scheduling
|
|
117
|
+
- ✅ **003**: Progressive learning + optimization
|
|
118
|
+
- ✅ **004**: Infrastructure resilience + observability
|
|
119
|
+
|
|
120
|
+
### Validates Production Readiness
|
|
121
|
+
|
|
122
|
+
Real-world deployments require:
|
|
123
|
+
- **Health monitoring** - System can detect its own problems
|
|
124
|
+
- **Event emission** - Observability data is generated correctly
|
|
125
|
+
- **Network policies** - Security restrictions don't break functionality
|
|
126
|
+
- **Infrastructure dependencies** - All required services are reachable
|
|
127
|
+
|
|
128
|
+
### Catches Infrastructure Configuration Errors
|
|
129
|
+
|
|
130
|
+
This test would immediately detect:
|
|
131
|
+
- NetworkPolicy blocking Kubernetes API access
|
|
132
|
+
- Broken OpenTelemetry configuration
|
|
133
|
+
- LLM endpoint connectivity issues
|
|
134
|
+
- Missing RBAC permissions for event creation
|
|
135
|
+
- Firewall rules blocking required traffic
|
|
136
|
+
|
|
137
|
+
### Self-Healing Validation
|
|
138
|
+
|
|
139
|
+
The test validates that agents can:
|
|
140
|
+
- Detect infrastructure problems
|
|
141
|
+
- Report issues clearly
|
|
142
|
+
- Continue functioning despite partial failures
|
|
143
|
+
- Provide actionable diagnostic information
|
|
144
|
+
|
|
145
|
+
## Real-World Use Cases This Enables
|
|
146
|
+
|
|
147
|
+
### Cluster Health Monitoring
|
|
148
|
+
```ruby
|
|
149
|
+
# Production monitoring agent
|
|
150
|
+
"Monitor cluster resources and alert if anything is degraded"
|
|
151
|
+
# → Generates comprehensive cluster health dashboard
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### Service Dependency Validation
|
|
155
|
+
```ruby
|
|
156
|
+
# Service mesh health checking
|
|
157
|
+
"Test all service connections and report any broken integrations"
|
|
158
|
+
# → Validates microservice communication patterns
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### Infrastructure Compliance Auditing
|
|
162
|
+
```ruby
|
|
163
|
+
# Security and compliance monitoring
|
|
164
|
+
"Check if all security policies are working correctly"
|
|
165
|
+
# → Validates network policies, RBAC, pod security standards
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
### Self-Healing System Validation
|
|
169
|
+
```ruby
|
|
170
|
+
# Resilience testing
|
|
171
|
+
"Verify the system can recover from common failure scenarios"
|
|
172
|
+
# → Tests automatic recovery mechanisms
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
## Expected Synthesis Outcomes
|
|
176
|
+
|
|
177
|
+
### Neural Phase (Initial Synthesis)
|
|
178
|
+
- **All tasks neural** - Uses LLM to determine health check methods
|
|
179
|
+
- **Flexible validation** - Adapts to different infrastructure configurations
|
|
180
|
+
- **Comprehensive checking** - Tests all critical system components
|
|
181
|
+
|
|
182
|
+
### Learning Phase (After Pattern Detection)
|
|
183
|
+
- **Deterministic checks become symbolic** - API connectivity, endpoint tests
|
|
184
|
+
- **Complex analysis stays neural** - Overall health determination, anomaly detection
|
|
185
|
+
- **Optimized execution** - Faster, cheaper health checks over time
|
|
186
|
+
|
|
187
|
+
### Progressive Optimization
|
|
188
|
+
```ruby
|
|
189
|
+
# Run 1-10: All neural health checks
|
|
190
|
+
# Run 11+: Learned symbolic implementations for standard checks
|
|
191
|
+
# Always: Neural analysis of overall system health patterns
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
## Validation Criteria
|
|
195
|
+
|
|
196
|
+
### Infrastructure Health
|
|
197
|
+
- [ ] Kubernetes API connectivity verified
|
|
198
|
+
- [ ] LLM endpoint responsiveness confirmed
|
|
199
|
+
- [ ] Monitoring system integration working
|
|
200
|
+
- [ ] Network policies allow required access
|
|
201
|
+
|
|
202
|
+
### Event Emission
|
|
203
|
+
- [ ] Task Completion events generated for each task
|
|
204
|
+
- [ ] Events contain proper metadata and timing
|
|
205
|
+
- [ ] OpenTelemetry traces collected successfully
|
|
206
|
+
- [ ] No event emission timeout errors
|
|
207
|
+
|
|
208
|
+
### Error Handling
|
|
209
|
+
- [ ] Graceful handling of connectivity failures
|
|
210
|
+
- [ ] Clear reporting of specific issues
|
|
211
|
+
- [ ] Continued operation despite partial failures
|
|
212
|
+
- [ ] Actionable diagnostic information provided
|
|
213
|
+
|
|
214
|
+
### Schedule Reliability
|
|
215
|
+
- [ ] Executions occur every 5 minutes as scheduled
|
|
216
|
+
- [ ] Consistent performance across runs
|
|
217
|
+
- [ ] Proper cleanup after each execution
|
|
218
|
+
- [ ] No resource leaks or accumulation
|
|
219
|
+
|
|
220
|
+
## Comparison to Traditional Monitoring
|
|
221
|
+
|
|
222
|
+
### Traditional Approach (Prometheus/Nagios)
|
|
223
|
+
```yaml
|
|
224
|
+
# Static configuration, manual setup
|
|
225
|
+
- name: check_api
|
|
226
|
+
command: curl -f https://k8s-api/healthz
|
|
227
|
+
interval: 5m
|
|
228
|
+
|
|
229
|
+
- name: check_llm
|
|
230
|
+
command: curl -f https://llm-endpoint/health
|
|
231
|
+
interval: 5m
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
### Language Operator Approach (Organic Functions)
|
|
235
|
+
```ruby
|
|
236
|
+
# Natural language → comprehensive monitoring
|
|
237
|
+
"Check if the system is healthy every 5 minutes and report any issues"
|
|
238
|
+
|
|
239
|
+
# Synthesizes intelligent monitoring that:
|
|
240
|
+
# - Adapts to infrastructure changes
|
|
241
|
+
# - Learns optimal check patterns
|
|
242
|
+
# - Provides contextual analysis
|
|
243
|
+
# - Integrates with existing observability
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
## Files Generated
|
|
247
|
+
|
|
248
|
+
| File | Purpose |
|
|
249
|
+
|------|---------|
|
|
250
|
+
| `instructions.txt` | Single-sentence natural language instruction |
|
|
251
|
+
| `agent.synthesized.rb` | Initial neural health monitoring agent |
|
|
252
|
+
| `Makefile` | Synthesis and execution commands |
|
|
253
|
+
| `output.log` | Health check execution logs |
|
|
254
|
+
|
|
255
|
+
## Synthesis Commands
|
|
256
|
+
|
|
257
|
+
```bash
|
|
258
|
+
# Create and deploy health monitoring agent
|
|
259
|
+
make create
|
|
260
|
+
|
|
261
|
+
# Monitor health check execution
|
|
262
|
+
make logs
|
|
263
|
+
|
|
264
|
+
# Check for Task Completion events
|
|
265
|
+
make events
|
|
266
|
+
|
|
267
|
+
# Validate observability infrastructure
|
|
268
|
+
make validate
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
## Expected Impact
|
|
272
|
+
|
|
273
|
+
This test validates that Language Operator can synthesize production-ready monitoring infrastructure from a simple natural language instruction, while ensuring the observability systems required for operational visibility are functioning correctly.
|
|
274
|
+
|
|
275
|
+
The zen aspect: A simple request for health monitoring becomes a comprehensive validation of the entire system's ability to observe itself.
|
|
276
|
+
|
|
277
|
+
## Related Tests
|
|
278
|
+
|
|
279
|
+
- [001 - Minimal Synthesis](../001/README.md) - Basic synthesis validation
|
|
280
|
+
- [002 - Neural Execution](../002/README.md) - Neural task execution + scheduling
|
|
281
|
+
- [003 - Progressive Learning](../003/README.md) - Learning and optimization
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
Check if the system is healthy every 5 minutes and report any issues
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: language-operator
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.66
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- James Ryan
|
|
8
8
|
bindir: bin
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date:
|
|
10
|
+
date: 2025-12-09 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: mcp
|
|
@@ -453,6 +453,8 @@ extra_rdoc_files: []
|
|
|
453
453
|
files:
|
|
454
454
|
- ".claude/commands/persona.md"
|
|
455
455
|
- ".claude/commands/task.md"
|
|
456
|
+
- ".plan.md"
|
|
457
|
+
- ".rspec"
|
|
456
458
|
- ".rubocop.yml"
|
|
457
459
|
- ".rubocop_custom/use_ux_helper.rb"
|
|
458
460
|
- CHANGELOG.md
|
|
@@ -495,10 +497,12 @@ files:
|
|
|
495
497
|
- docs/agent-internals.md
|
|
496
498
|
- docs/best-practices.md
|
|
497
499
|
- docs/chat-endpoints.md
|
|
500
|
+
- docs/cheat-sheet.md
|
|
498
501
|
- docs/cli-reference.md
|
|
499
502
|
- docs/constraints.md
|
|
500
503
|
- docs/how-agents-work.md
|
|
501
504
|
- docs/installation.md
|
|
505
|
+
- docs/observability.md
|
|
502
506
|
- docs/quickstart.md
|
|
503
507
|
- docs/schema-versioning.md
|
|
504
508
|
- docs/understanding-generated-code.md
|
|
@@ -508,6 +512,7 @@ files:
|
|
|
508
512
|
- lib/language_operator.rb
|
|
509
513
|
- lib/language_operator/agent.rb
|
|
510
514
|
- lib/language_operator/agent/base.rb
|
|
515
|
+
- lib/language_operator/agent/event_config.rb
|
|
511
516
|
- lib/language_operator/agent/executor.rb
|
|
512
517
|
- lib/language_operator/agent/instrumentation.rb
|
|
513
518
|
- lib/language_operator/agent/metrics_tracker.rb
|
|
@@ -529,7 +534,6 @@ files:
|
|
|
529
534
|
- lib/language_operator/cli/commands/agent/helpers/cluster_llm_client.rb
|
|
530
535
|
- lib/language_operator/cli/commands/agent/helpers/code_parser.rb
|
|
531
536
|
- lib/language_operator/cli/commands/agent/helpers/synthesis_watcher.rb
|
|
532
|
-
- lib/language_operator/cli/commands/agent/learning.rb
|
|
533
537
|
- lib/language_operator/cli/commands/agent/lifecycle.rb
|
|
534
538
|
- lib/language_operator/cli/commands/agent/logs.rb
|
|
535
539
|
- lib/language_operator/cli/commands/agent/workspace.rb
|
|
@@ -645,9 +649,10 @@ files:
|
|
|
645
649
|
- synth/002/output.log
|
|
646
650
|
- synth/003/Makefile
|
|
647
651
|
- synth/003/README.md
|
|
648
|
-
- synth/003/agent.optimized.rb
|
|
649
|
-
- synth/003/agent.synthesized.rb
|
|
650
652
|
- synth/003/agent.txt
|
|
653
|
+
- synth/004/Makefile
|
|
654
|
+
- synth/004/README.md
|
|
655
|
+
- synth/004/instructions.txt
|
|
651
656
|
- synth/README.md
|
|
652
657
|
homepage: https://github.com/language-operator/language-operator
|
|
653
658
|
licenses:
|
|
@@ -671,7 +676,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
671
676
|
- !ruby/object:Gem::Version
|
|
672
677
|
version: '0'
|
|
673
678
|
requirements: []
|
|
674
|
-
rubygems_version: 3.6.
|
|
679
|
+
rubygems_version: 3.6.6
|
|
675
680
|
specification_version: 4
|
|
676
681
|
summary: Ruby SDK for Language Operator
|
|
677
682
|
test_files: []
|