agent99 0.0.3 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. checksums.yaml +4 -4
  2. data/A2A_SPEC-dev.md +1829 -0
  3. data/CHANGELOG.md +38 -0
  4. data/COMMITS.md +196 -0
  5. data/DOCS.md +96 -0
  6. data/README.md +212 -84
  7. data/Rakefile +62 -0
  8. data/docs/AI/htm.md +215 -0
  9. data/docs/AI/htm.rb +141 -0
  10. data/docs/AI/htm_demo.db +0 -0
  11. data/docs/AI/notes_on_htm_implementation.md +1319 -0
  12. data/docs/AI/some_code.rb +692 -0
  13. data/docs/advanced-topics/a2a-protocol.md +13 -0
  14. data/docs/{advanced_features.md → advanced-topics/advanced-features.md} +9 -4
  15. data/docs/{control_actions.md → advanced-topics/control-actions.md} +2 -0
  16. data/docs/advanced-topics/model-context-protocol.md +4 -0
  17. data/docs/advanced-topics/multi-agent-processing.md +674 -0
  18. data/docs/agent-development/request-response-handling.md +512 -0
  19. data/docs/agent99_framework/central_registry.md +94 -0
  20. data/docs/agent99_framework/message_client.md +120 -0
  21. data/docs/agent99_framework/registry_client.md +119 -0
  22. data/docs/api-reference/agent99-base.md +463 -0
  23. data/docs/api-reference/message-clients.md +495 -0
  24. data/docs/{api_reference.md → api-reference/overview.md} +14 -4
  25. data/docs/api-reference/registry-client.md +470 -0
  26. data/docs/api-reference/schemas.md +518 -0
  27. data/docs/assets/css/custom.css +27 -0
  28. data/docs/assets/images/agent-lifecycle.svg +73 -0
  29. data/docs/assets/images/agent-registry-process.svg +86 -0
  30. data/docs/assets/images/agent-registry-processes.svg +114 -0
  31. data/docs/assets/images/agent-types-overview.svg +51 -0
  32. data/docs/assets/images/agent99-architecture.svg +85 -0
  33. data/docs/assets/images/agent99_logo.png +0 -0
  34. data/docs/assets/images/control-actions-state.svg +83 -0
  35. data/docs/assets/images/knowledge-graph.svg +77 -0
  36. data/docs/assets/images/message-processing-flow.svg +148 -0
  37. data/docs/assets/images/multi-agent-system.svg +66 -0
  38. data/docs/assets/images/proxy-pattern-sequence.svg +48 -0
  39. data/docs/assets/images/request-flow.svg +97 -0
  40. data/docs/assets/images/request-processing-lifecycle.svg +50 -0
  41. data/docs/assets/images/request-response-sequence.svg +39 -0
  42. data/docs/{agent_lifecycle.md → core-concepts/agent-lifecycle.md} +2 -0
  43. data/docs/core-concepts/agent-types.md +255 -0
  44. data/docs/{architecture.md → core-concepts/architecture.md} +5 -5
  45. data/docs/core-concepts/what-is-an-agent.md +293 -0
  46. data/docs/diagrams/message-flow-sequence.svg +198 -0
  47. data/docs/diagrams/p2p-network-topology.svg +181 -0
  48. data/docs/diagrams/smart-transport-routing.svg +165 -0
  49. data/docs/diagrams/three-layer-architecture.svg +77 -0
  50. data/docs/diagrams/transport-extension-api.svg +309 -0
  51. data/docs/diagrams/transport-extension-architecture.svg +234 -0
  52. data/docs/diagrams/transport-selection-flowchart.svg +264 -0
  53. data/docs/examples/advanced-examples.md +951 -0
  54. data/docs/examples/basic-examples.md +268 -0
  55. data/docs/{agent_discovery.md → framework-components/agent-discovery.md} +9 -5
  56. data/docs/{agent_registry_processes.md → framework-components/agent-registry.md} +9 -3
  57. data/docs/{message_processing.md → framework-components/message-processing.md} +3 -1
  58. data/docs/getting-started/basic-example.md +306 -0
  59. data/docs/getting-started/installation.md +160 -0
  60. data/docs/getting-started/overview.md +64 -0
  61. data/docs/getting-started/quick-start.md +179 -0
  62. data/docs/index.md +97 -0
  63. data/docs/operations/breaking-changes.md +26 -0
  64. data/examples/DEMO.md +148 -0
  65. data/examples/README.md +50 -0
  66. data/examples/agent_watcher.rb +5 -1
  67. data/examples/bad_agent.rb +32 -0
  68. data/examples/chief_agent.rb +17 -6
  69. data/examples/control.rb +16 -7
  70. data/examples/example_agent.rb +16 -3
  71. data/examples/maxwell_agent86.rb +15 -26
  72. data/examples/registry.rb +10 -9
  73. data/examples/run_demo.rb +433 -0
  74. data/lib/agent99/agent_discovery.rb +4 -0
  75. data/lib/agent99/agent_lifecycle.rb +34 -10
  76. data/lib/agent99/amqp_message_client.rb +2 -2
  77. data/lib/agent99/base.rb +6 -2
  78. data/lib/agent99/message_processing.rb +6 -10
  79. data/lib/agent99/registry_client.rb +15 -11
  80. data/lib/agent99/tcp_message_client.rb +183 -0
  81. data/lib/agent99/version.rb +1 -1
  82. data/lib/agent99.rb +1 -1
  83. data/mkdocs.yml +195 -0
  84. data/p2p_plan.md +533 -0
  85. data/p2p_roadmap.md +299 -0
  86. data/registry_plan.md +1818 -0
  87. metadata +93 -30
  88. data/docs/README.md +0 -57
  89. data/docs/diagrams/agent_registry_processes.dot +0 -42
  90. data/docs/diagrams/agent_registry_processes.png +0 -0
  91. data/docs/diagrams/high_level_architecture.dot +0 -26
  92. data/docs/diagrams/high_level_architecture.png +0 -0
  93. data/docs/diagrams/request_flow.dot +0 -42
  94. data/docs/diagrams/request_flow.png +0 -0
  95. /data/docs/{extending_the_framework.md → advanced-topics/extending-the-framework.md} +0 -0
  96. /data/docs/{custom_agent_implementation.md → agent-development/custom-agent-implementation.md} +0 -0
  97. /data/docs/{error_handling_and_logging.md → agent-development/error-handling-and-logging.md} +0 -0
  98. /data/docs/{schema_definition.md → agent-development/schema-definition.md} +0 -0
  99. /data/docs/{messaging_system.md → framework-components/messaging-system.md} +0 -0
  100. /data/docs/{configuration.md → operations/configuration.md} +0 -0
  101. /data/docs/{preformance_considerations.md → operations/performance-considerations.md} +0 -0
  102. /data/docs/{security.md → operations/security.md} +0 -0
  103. /data/docs/{troubleshooting.md → operations/troubleshooting.md} +0 -0
data/README.md CHANGED
@@ -1,62 +1,102 @@
1
- # Agent99 Framework (AFW)
2
-
3
- **Under Development** Initial release has no AI components - its just a generic client-server / request-response micro-services system using a peer-to-peer messaging broker and a centralized agent registry. To keep up with the version changes review [The Changelog](./CHANGELOG.md) file.
4
-
5
- Agent99 is a Ruby-based framework for building and managing AI agents in a distributed system. It provides a robust foundation for creating intelligent agents that can communicate, discover each other, and perform various tasks.
6
-
7
- ## Hype!
8
-
9
- 🌟 **Introducing Agent99**: Your Friendly Ruby Sidekick for Building Intelligent Software Agents! 🌟
10
-
11
- Do you ever miss the charm and wit of Barbara Feldon's iconic character from *Get Smart*? Just as Agent 99 was always ready to tackle mischief and save the day, **Agent99** is here to help you create your very own software agents that can communicate, collaborate, and conquer tasks in a smart and efficient way!
12
-
13
- ### Why Choose Agent99?
14
-
15
- 🔍 **Independent Agents, Unified Communication**: Say goodbye to complex service orchestration! Agent99 empowers you to build micro-services that communicate seamlessly through a robust peer-to-peer messaging network. Each agent operates independently while working together like a well-oiled machine!
16
-
17
- 🤖 **Smart Automation for the Modern Age**: In the era of AI, why settle for less? With Agent99, you can develop software agents that respond intelligently to incoming requests—just like a secret agent reacting to their next mission! Deploy your own digital spies to gather data, complete tasks, and more.
18
-
19
- 📦 **Quick Setup & Easy Integration**: Just like Agent 99’s quick thinking, our library is designed for rapid development. Get up and running in no time, whether you’re building a small project or an enterprise-level solution!
20
-
21
-
22
- ### Features that Make Agent99 a Must-Have:
23
-
24
- - 🌐 **Peer-to-Peer Messaging**: Ensure that your agents can communicate effortlessly, just like Agent 99 and Max.
25
- - 🚀 **Agile Development**: Design and deploy agents at lightning speed, giving you the freedom to innovate.
26
- - 🔒 **Secure Communication**: Built with security in mind, because even our agents deserve to keep their secrets safe.
27
- - 📊 **Scalability**: Expand your network of agents as your needs grow—no mission is too big!
28
-
29
- ### Get Smart! Join the Revolution!
30
-
31
- Whether you’re a seasoned Ruby developer or just getting started, Agent99 provides the tools you need to build your very own quirky, intelligent agents. Just like Agent 99, your agents will be clever, adaptable, and ready to tackle any challenge!
32
-
33
- ### How to Get Started:
1
+ # Agent99
2
+ <br/>
3
+
4
+ > [!CAUTION]
5
+ > ## ⚠️ Under Development ⚠️
6
+ > Initial release has no AI components - it's a generic client-server / request-response micro-services system using peer-to-peer messaging and centralized agent registry. Review [The Changelog](./CHANGELOG.md) for version changes.
7
+ <br /><br />
8
+
9
+ <div align="center">
10
+ <table>
11
+ <tr>
12
+ <td width="40%" align="center" valign="top">
13
+ <img src="docs/assets/images/agent99_logo.png" alt="Agent99 - Ruby Framework for Intelligent Software Agents" width="1200">
14
+ <br /><br />
15
+ [Comprehensive Documentation Website](https://madbomber.github.io/agent99/)
16
+ </td>
17
+ <td width="60%" align="left" valign="top">
18
+ Agent99 is a Ruby-based framework for building and managing intelligent software agents in a distributed system. Like the clever Agent 99 from Get Smart, your agents will be smart, adaptable, and ready to tackle any challenge through seamless peer-to-peer communication and centralized discovery.
19
+ <br/><br/>
20
+ <h3>Key Features</h3>
21
+ <ul>
22
+ <li><strong>🌐 <a href="#agent-discovery">Agent Discovery</a></strong> - Find agents by capabilities</li>
23
+ <li><strong>📡 <a href="#flexible-communication">Peer-to-Peer Messaging</a></strong> - AMQP & NATS support</li>
24
+ <li><strong>🔄 <a href="#message-processing">Message Processing</a></strong> - Requests, responses & control</li>
25
+ <li><strong>📋 <a href="#registry-integration">Registry Integration</a></strong> - Centralized agent registry</li>
26
+ <li><strong>⚡ <a href="#control-actions">Control Actions</a></strong> - Pause, resume, update agents</li>
27
+ <li><strong>🔧 <a href="#agent-lifecycle-management">Lifecycle Management</a></strong> - Easy setup & teardown</li>
28
+ <li><strong>🚀 <a href="#multi-agent-processing">Multi-Agent Processing</a></strong> - Thread isolation support</li>
29
+ <li><strong>📊 <a href="#error-handling-and-logging">Error Handling & Logging</a></strong> - Built-in management</li>
30
+ </ul>
31
+ </td>
32
+ </tr>
33
+ </table>
34
+ </div>
35
+
36
+ ## Table of Contents
37
+
38
+ * [Quick Start](#quick-start)
39
+ * [Prerequisites](#prerequisites)
40
+ * [Architecture Overview](#architecture-overview)
41
+ * [Installation](#installation)
42
+ * [Usage](#usage)
43
+ * [Configuration](#configuration)
44
+ * [Agent Lifecycle](#agent-lifecycle)
45
+ * [Message Processing](#message-processing)
46
+ * [Registry Integration](#registry-integration)
47
+ * [Examples](#examples)
48
+ * [Evolving Standards](#evolving-standards)
49
+ * [Contributing](#contributing)
50
+ * [Roadmap](#roadmap)
51
+ * [License](#license)
52
+
53
+ ## Quick Start
54
+
55
+ Jump right in with our step-by-step guide:
56
+
57
+ 1. **Prerequisites**: [Install message broker](#prerequisites) (NATS or RabbitMQ)
58
+ 2. **Install Agent99**: Add to Gemfile and bundle install
59
+ 3. **Start Registry**: Launch the central agent registry
60
+ 4. **Create Your First Agent**: Build a simple greeter agent
61
+ 5. **Test Communication**: Verify agents can discover and communicate
62
+
63
+ 📖 **Detailed walkthrough**: [Getting Started Guide](docs/getting-started/quick-start.md)
64
+
65
+ ## Prerequisites
66
+
67
+ Agent99 requires a message broker for inter-agent communication. Choose one:
68
+
69
+ ### NATS (Recommended)
70
+ ```bash
71
+ # macOS
72
+ brew install nats-server
73
+
74
+ # Start NATS
75
+ nats-server
76
+ ```
34
77
 
35
- 1. **Install**: Simply add Agent99 to your Gemfile and run `bundle install`.
36
- 2. **Create an Agent**: Use our simple API to define and deploy your first agent.
37
- 3. **Watch Them Work**: Sit back and relax as your agents spring into action, communicating with one another to accomplish tasks that would impress even the chief of CONTROL!
78
+ ### RabbitMQ
79
+ ```bash
80
+ # macOS
81
+ brew install rabbitmq
38
82
 
39
- ### Spread the Word!
83
+ # Start RabbitMQ
84
+ brew services start rabbitmq
85
+ ```
40
86
 
41
- **Join the Agent99 community** on GitHub and share your experiences, tips, and feedback. Let’s build a world of smart agents together! And remember, just like Agent 99 always had Max’s back, we’ve got yours during your development journey.
87
+ 📋 **More installation options**: [Installation Guide](docs/getting-started/installation.md)
42
88
 
43
- 🕵️‍♂️ **Become an Agent!** Dive into the world of Agent99 today: [GitHub Repository](#) 📖
89
+ ## Architecture Overview
44
90
 
45
- **Agent99** – Because when it comes to building software agents, it's all about being smart!
91
+ ![Agent99 Architecture](docs/assets/images/agent99-architecture.svg)
46
92
 
47
- ... end of the Hype; now back to normal.
93
+ Agent99 follows a distributed architecture with three core components:
48
94
 
49
- ## Features
95
+ - **🏛️ Central Registry**: Service discovery and agent registration
96
+ - **📡 Message Broker**: Peer-to-peer communication backbone (NATS/AMQP)
97
+ - **🤖 Agents**: Independent services with specific capabilities
50
98
 
51
- - Agent Lifecycle Management: Easy setup and teardown of agents
52
- - Message Processing: Handle requests, responses, and control messages
53
- - Agent Discovery: Find other agents based on capabilities
54
- - Flexible Communication: Support for both AMQP and NATS messaging systems
55
- - Registry Integration: Register and discover agents through a central registry
56
- - Error Handling and Logging: Built-in error management and logging capabilities
57
- - Control Actions: Pause, resume, update configuration, and request status of agents
58
- - Dynamic Agent Loading: Support for runtime loading and deployment of new agents
59
- - Multi-Agent Processing: Run multiple agents within the same process using thread isolation
99
+ 📚 **Deep dive**: [Architecture Documentation](docs/core-concepts/architecture.md)
60
100
 
61
101
  ## Installation
62
102
 
@@ -80,63 +120,151 @@ $ gem install agent99
80
120
 
81
121
  ## Usage
82
122
 
83
- Here's a basic example of how to create an AI agent:
123
+ Create your first agent in three simple steps:
84
124
 
125
+ ### 1. Define Request Schema
85
126
  ```ruby
86
- require 'agent99'
87
-
88
- class MyAgentRequest < SimpleJsonSchemaBuilder::Base
127
+ class GreeterRequest < SimpleJsonSchemaBuilder::Base
89
128
  object do
90
129
  object :header, schema: Agent99::HeaderSchema
91
-
92
- # Define your agents parameters ....
93
- string :greeting, required: false, examples: ["Hello"]
94
- string :name, required: true, examples: ["World"]
130
+ string :name, required: true, examples: ["World"]
95
131
  end
96
132
  end
133
+ ```
97
134
 
98
- class MyAgent < Agent99::Base
99
- REQUEST_SCHEMA = MyAgentRequest.schema
100
-
101
- def capabilities
102
- ['text_processing', 'sentiment_analysis']
103
- # TODO: make up mind on keyword or unstructured text
104
- end
105
-
106
- def receive_request
107
- # Handle the validated incoming requests
108
- response = { result: "Processed request" }
109
-
110
- # Not every request needs a response
111
- send_response(response)
135
+ ### 2. Implement Agent Class
136
+ ```ruby
137
+ class GreeterAgent < Agent99::Base
138
+ def info
139
+ {
140
+ name: self.class.to_s,
141
+ type: :server,
142
+ capabilities: ['greeter', 'hello_world'],
143
+ request_schema: GreeterRequest.schema
144
+ }
112
145
  end
113
146
 
114
- def receive_response
115
- # You sent a request to another agent
116
- # now handle the response.
147
+ def process_request(payload)
148
+ name = payload.dig(:name)
149
+ send_response(result: "Hello, #{name}!")
117
150
  end
118
151
  end
152
+ ```
119
153
 
120
- agent = MyAgent.new
154
+ ### 3. Run Your Agent
155
+ ```ruby
156
+ require 'agent99'
157
+ agent = GreeterAgent.new
121
158
  agent.run
122
159
  ```
123
160
 
161
+ 🎯 **More examples**: [Usage Examples](docs/examples/)
162
+
124
163
  ## Configuration
125
164
 
126
- The framework can be configured through environment variables:
165
+ Configure Agent99 through environment variables:
166
+
167
+ ### Core Settings
168
+ - `AGENT99_REGISTRY_URL`: Registry service URL (default: `http://localhost:4567`)
169
+ - `AGENT99_LOG_LEVEL`: Logging verbosity (default: `INFO`)
170
+
171
+ ### NATS Configuration
172
+ - `NATS_URL`: NATS server URL (default: `nats://localhost:4222`)
173
+ - `NATS_USER`: Username for authentication
174
+ - `NATS_PASS`: Password for authentication
175
+
176
+ ### AMQP/RabbitMQ Configuration
177
+ - `RABBITMQ_URL`: RabbitMQ server URL (default: `amqp://localhost:5672`)
178
+ - `RABBITMQ_USER`: Username (default: `guest`)
179
+ - `RABBITMQ_PASS`: Password (default: `guest`)
180
+
181
+ 📚 **Complete configuration guide**: [Configuration Documentation](docs/configuration/)
182
+
183
+ ## Agent Lifecycle
184
+
185
+ ![Agent Lifecycle](docs/assets/images/agent-lifecycle.svg)
186
+
187
+ Agent99 manages the complete lifecycle of your agents:
188
+
189
+ - **🎬 Initialization**: Agent registers with the central registry
190
+ - **⚡ Running**: Agent processes requests and sends responses
191
+ - **⏸️ Paused**: Agent temporarily stops processing (control action)
192
+ - **🔄 Updated**: Agent receives new configuration or capabilities
193
+ - **🛑 Shutdown**: Agent gracefully disconnects and cleans up
194
+
195
+ 🔄 **Detailed lifecycle management**: [Agent Lifecycle Guide](docs/core-concepts/agent-lifecycle.md)
196
+
197
+ ## Message Processing
198
+
199
+ ![Message Processing Flow](docs/assets/images/message-processing-flow.svg)
200
+
201
+ Agent99 handles three types of messages:
202
+
203
+ - **📨 Requests**: Incoming work for agents to process
204
+ - **📤 Responses**: Results sent back to requesters
205
+ - **🎛️ Control**: Management commands (pause, resume, update)
206
+
207
+ Each message is validated, routed, and processed with full error handling and logging.
208
+
209
+ ⚙️ **Message processing details**: [Message Processing Guide](docs/framework-components/message-processing.md)
210
+
211
+ ## Registry Integration
212
+
213
+ ![Agent Registry Process](docs/assets/images/agent-registry-processes.svg)
214
+
215
+ The central registry enables dynamic service discovery:
216
+
217
+ 1. **📝 Registration**: Agents announce their capabilities
218
+ 2. **🔍 Discovery**: Find agents by capability or name
219
+ 3. **📋 Management**: Monitor and control registered agents
220
+ 4. **🚪 Withdrawal**: Clean removal from registry
221
+
222
+ The registry supports both HTTP REST API and direct client integration.
223
+
224
+ 🏛️ **Registry implementation**: [Registry Documentation](docs/framework-components/agent-registry.md)
225
+
226
+ ## Examples
227
+
228
+ Explore real-world implementations:
229
+
230
+ - **🚀 [Demo Runner](examples/run_demo.rb)**: Complete working system
231
+ - **👋 [Greeter Agent](examples/greeter_agent.rb)**: Simple request-response
232
+ - **🔄 [Multi-Agent System](examples/multi_agent/)**: Coordinated agents
233
+ - **📊 [Monitoring Dashboard](examples/dashboard/)**: Agent status monitoring
234
+
235
+ 💡 **All examples**: [Examples Directory](docs/examples/)
236
+
237
+ ## Evolving Standards
238
+
239
+ - [agntcy.org](https://agntcy.org) - Agency protocol initiative
240
+ - [Agent2Agent](https://a2a-protocol.org/latest/) - Linux Foundation protocol
127
241
 
128
- - `REGISTRY_BASE_URL`: URL of the agent registry service (default: 'http://localhost:4567') See the default registry service in the examples folder.
129
242
 
130
243
  ## Contributing
131
244
 
132
245
  Bug reports and pull requests are welcome on GitHub at https://github.com/MadBomber/agent99.
133
246
 
134
- ## Short-term Roadmap
247
+ ## Roadmap
248
+
249
+ ### Short-term (v0.1.x)
250
+ - **🔍 Enhanced Discovery**: Semantic search with vector database (SQLite + embeddings)
251
+ - **📋 Schema Validation**: Complete request/response schema definitions
252
+ - **🤖 AI Integration**: RAG-enabled agents using prompts as tools
253
+ - **📊 Monitoring**: Built-in metrics and health checks
254
+
255
+ ### Medium-term (v0.2.x)
256
+ - **🔐 Security**: Authentication and authorization framework
257
+ - **⚡ Performance**: Connection pooling and message batching
258
+ - **🌍 Multi-broker**: Support for multiple message brokers simultaneously
259
+ - **📈 Scaling**: Load balancing and auto-scaling capabilities
260
+
261
+ ### Long-term (v1.0+)
262
+ - **🧠 Intelligence**: Built-in ML/AI capabilities
263
+ - **🔗 Interoperability**: Full Agent2Agent protocol compliance
264
+ - **☁️ Cloud-native**: Kubernetes operators and cloud integrations
265
+
266
+ ⚠️ **Breaking changes**: [Migration Guide v0.0.4](docs/breaking_change_v0.0.4.md)
135
267
 
136
- - In the example registry, replace the Array(Hash) datastore with sqlite3 with a vector table to support discovery using semantic search.
137
- - Treat the agent like a Tool w/r/t RAG for prompts.
138
- - Add AgentRequest schema to agent's info in the registry.
139
- - Add AgentResponse schema to define the `result` element in the response JSON payload
140
268
 
141
269
  ## License
142
270
 
data/Rakefile CHANGED
@@ -6,3 +6,65 @@ require "minitest/test_task"
6
6
  Minitest::TestTask.create
7
7
 
8
8
  task default: :test
9
+
10
+ # Additional test tasks for Agent99
11
+ require "rake/testtask"
12
+
13
+ # Unit tests only
14
+ Rake::TestTask.new(:test_unit) do |t|
15
+ t.libs << "lib"
16
+ t.libs << "test"
17
+ t.test_files = FileList["test/agent99/*test*.rb"]
18
+ t.verbose = true
19
+ end
20
+
21
+ # Integration tests only
22
+ Rake::TestTask.new(:test_integration) do |t|
23
+ t.libs << "lib"
24
+ t.libs << "test"
25
+ t.test_files = FileList["test/integration/*test*.rb"]
26
+ t.verbose = true
27
+ end
28
+
29
+ # System tests only
30
+ Rake::TestTask.new(:test_system) do |t|
31
+ t.libs << "lib"
32
+ t.libs << "test"
33
+ t.test_files = FileList["test/system/*test*.rb"]
34
+ t.verbose = true
35
+ end
36
+
37
+ desc "Run tests with coverage reporting (requires simplecov gem)"
38
+ task :test_coverage do
39
+ ENV["COVERAGE"] = "true"
40
+ Rake::Task[:test].invoke
41
+ end
42
+
43
+ desc "Run tests in verbose mode"
44
+ task :test_verbose do
45
+ ENV["VERBOSE"] = "true"
46
+ Rake::Task[:test].invoke
47
+ end
48
+
49
+ desc "Show test statistics"
50
+ task :test_stats do
51
+ unit_tests = Dir.glob("test/agent99/*test*.rb").size
52
+ integration_tests = Dir.glob("test/integration/*test*.rb").size
53
+ system_tests = Dir.glob("test/system/*test*.rb").size
54
+ total_tests = unit_tests + integration_tests + system_tests
55
+
56
+ puts "Test Statistics:"
57
+ puts " Unit tests: #{unit_tests}"
58
+ puts " Integration tests: #{integration_tests}"
59
+ puts " System tests: #{system_tests}"
60
+ puts " Total test files: #{total_tests}"
61
+
62
+ # Count test methods
63
+ test_methods = 0
64
+ Dir.glob("test/**/*test*.rb").each do |file|
65
+ content = File.read(file)
66
+ test_methods += content.scan(/def test_/).size
67
+ end
68
+
69
+ puts " Total test methods: #{test_methods}"
70
+ end
data/docs/AI/htm.md ADDED
@@ -0,0 +1,215 @@
1
+ # Hierarchical Temporal Memory in AI Systems
2
+
3
+ ## Table of Contents
4
+ - [Introduction](#introduction)
5
+ - [Understanding Hierarchical Temporal Memory](#understanding-hierarchical-temporal-memory)
6
+ - [Layers of Memory](#layers-of-memory)
7
+ - [Topic Columns](#topic-columns)
8
+ - [Memory Cell Promotion and Forgetting](#memory-cell-promotion-and-forgetting)
9
+ - [Implications for AI Systems](#implications-for-ai-systems)
10
+ - [Conclusion](#conclusion)
11
+ - [Some Code Ideas](#some-code-ideas)
12
+
13
+ ## Introduction
14
+
15
+ Hierarchical Temporal Memory (HTM) is a computational model that mimics some aspects of the neocortex, developed by the Numenta research company. It focuses on how the brain processes information over time and is designed to learn patterns and sequences in data. HTM operates on principles of hierarchy, meaning it organizes information in layers, and temporal memory, which enables it to remember sequences and make predictions based on temporal patterns. This model is particularly effective for tasks involving time series data and has applications in anomaly detection, robotics, and various cognitive computing tasks.
16
+
17
+ ## Understanding Hierarchical Temporal Memory
18
+
19
+ Hierarchical Temporal Memory is inspired by the theory of how the brain works, particularly focusing on how it learns patterns and sequences. The fundamental ideas were put forward by Jeff Hawkins, co-founder of Numenta, who theorized that real intelligence is derived from understanding and manipulating temporal sequences.
20
+
21
+ HTM leverages a structure that is hierarchical and spatially organized. Each level of the hierarchy processes information at varying degrees of abstraction. The bottom layers deal with raw sensory input, while the upper layers interpret this data into meaningful contexts.
22
+
23
+ This model is particularly valuable in applications involving complex time series, where the demand for understanding sequences over time is crucial. The capability of HTM to generalize from sparse data makes it unique compared to traditional machine learning techniques.
24
+
25
+ ## Layers of Memory
26
+
27
+ HTM's architecture consists of several layers, each designed for specific functions. The bottom layer, known as the input layer, is responsible for receiving raw data inputs. Subsequent layers process this information, identifying patterns and sequences that inform decisions.
28
+
29
+ 1. **Input Layer**: Receives sensory data and converts it into a usable format for the next layer.
30
+ 2. **Temporal Pooling Layer**: Captures the temporal dependencies in the data.
31
+ 3. **Spatial Pooling Layer**: Encodes spatial relationships and builds a representation of the input data.
32
+
33
+ This multi-layered approach enables HTM to build increasingly sophisticated representations of the data at each layer, akin to how the neocortex might operate.
34
+
35
+ ## Topic Columns
36
+
37
+ Each "column" in HTM can be viewed as a mini-neural network that processes input from the layer below it. These columns work in parallel to learn spatial and temporal patterns. Thus, they respond to specific features of the incoming data, allowing HTM to develop a hierarchy of data features. Columns also facilitate learning through inhibition, promoting the most active neurons while suppressing others.
38
+
39
+ ## Memory Cell Promotion and Forgetting
40
+
41
+ The mechanisms by which HTM promotes or forgets memory cells mirror more biological processes.
42
+
43
+ ### Promotion
44
+
45
+ Cells that consistently provide accurate predictions based on the learned sequence are promoted. For instance, if certain neurons within a column recognize patterns that lead to accurate outcomes, they're trained to engage more actively depending on their performance over time.
46
+
47
+ ### Forgetting
48
+
49
+ Forgetting is equally important; neurons that fail to respond to valid sequences are gradually demoted. This ensures that the model does not become overly reliant on outdated information, allowing new patterns and sequences to emerge and be incorporated.
50
+
51
+ ## Implications for AI Systems
52
+
53
+ By modeling the functioning of the neocortex, HTM introduces a new paradigm in machine learning, emphasizing time and sequences over static data processing. This has significant implications in various fields:
54
+
55
+ - **Anomaly Detection**: In finance, HTM can be used to identify unusual patterns in transactions that may indicate fraud.
56
+ - **Robotics**: Robots leveraging HTM can learn from their experiences in real-time, adapting to environmental changes dynamically.
57
+ - **Natural Language Processing**: Understanding the temporal relationships between phrases can enhance language models, leading to more contextually accurate interpretations.
58
+
59
+ ## Conclusion
60
+
61
+ Hierarchical Temporal Memory presents a powerful framework for understanding and replicating the cognitive processes of the human brain. With its ability to learn from temporal patterns, it opens new frontiers in artificial intelligence and machine learning.
62
+
63
+ ## Some Code Ideas
64
+
65
+ ```ruby
66
+ require 'active_record'
67
+
68
+ class CreateHtmTable < ActiveRecord::Migration[7.0]
69
+ def change
70
+ create_table :htm do |t|
71
+ t.uuid :event, null: false
72
+ t.text :proposition, null: false
73
+ t.string :state, null: false
74
+ t.datetime :last_accessed, null: false
75
+ t.integer :hit_count, null: false, default: 0
76
+ t.float :vector, null: false, array: true, limit: 2048
77
+
78
+ t.timestamps
79
+ end
80
+
81
+ add_index :htm, :event, unique: true
82
+ add_index :htm, :state
83
+
84
+ execute <<-SQL
85
+ ALTER TABLE htm
86
+ ADD CONSTRAINT check_state
87
+ CHECK (state IN ('STM', 'MTM', 'LTM', 'PM'))
88
+ SQL
89
+
90
+ execute <<-SQL
91
+ ALTER TABLE htm
92
+ ADD CONSTRAINT check_vector_length
93
+ CHECK (array_length(vector, 1) = 2048)
94
+ SQL
95
+ end
96
+ end
97
+ ```
98
+
99
+ This is a notional model for the table. I'm having second thoughts about keeping track of the hit count. I think maybe access time is all that is needed and that is provided by the AR timestamps.
100
+
101
+ ```ruby
102
+ require 'active_record'
103
+ require 'neighbors'
104
+
105
+ class Htm < ActiveRecord::Base
106
+ STATES = %w[STM MTM LTM PM].freeze
107
+
108
+ validates :event, presence: true
109
+ validates :proposition, presence: true
110
+ validates :state, presence: true, inclusion: { in: STATES }
111
+ validates :vector, presence: true, length: { is: 2048 }
112
+
113
+ def self.search_and_update(event_uuid:, query_vector:, top_k: 10)
114
+ records = where(event: event_uuid)
115
+ vectors = records.pluck(:vector)
116
+
117
+ neighbor_search = Neighbors::NearestNeighbors.new(vectors)
118
+ nearest_indices = neighbor_search.search(query_vector, k: top_k)
119
+
120
+ nearest_records = records.where(id: records.ids.values_at(*nearest_indices))
121
+
122
+ nearest_records.each do |record|
123
+ record.hit_count += 1
124
+ record.last_accessed = Time.current
125
+ record.save!
126
+ end
127
+
128
+ nearest_records
129
+ end
130
+
131
+ def update_state
132
+ event_config = StateConfig.find_by(event: event)
133
+ current_time = Time.current
134
+ time_since_creation = current_time - created_at
135
+ time_since_last_access = current_time - last_accessed
136
+
137
+ if should_promote?(event_config, time_since_creation)
138
+ promote_state
139
+ elsif should_demote?(event_config, time_since_last_access)
140
+ demote_state
141
+ end
142
+ end
143
+
144
+ private
145
+
146
+ def should_promote?(config, time_delta)
147
+ current_state_index = STATES.index(state)
148
+ return false if current_state_index == STATES.length - 1
149
+
150
+ next_state = STATES[current_state_index + 1]
151
+ time_delta > config.promotion_time(next_state) &&
152
+ hit_count > config.promotion_hits(next_state)
153
+ end
154
+
155
+ def should_demote?(config, time_delta)
156
+ current_state_index = STATES.index(state)
157
+ return false if current_state_index == 0
158
+
159
+ time_delta < config.demotion_time(state) &&
160
+ hit_count < config.demotion_hits(state)
161
+ end
162
+
163
+ def promote_state
164
+ current_index = STATES.index(state)
165
+ self.state = STATES[current_index + 1] if current_index < STATES.length - 1
166
+ save!
167
+ end
168
+
169
+ def demote_state
170
+ current_index = STATES.index(state)
171
+ self.state = STATES[current_index - 1] if current_index > 0
172
+ save!
173
+ end
174
+ end
175
+
176
+ class StateConfig < ActiveRecord::Base
177
+ def promotion_time(state)
178
+ case state
179
+ when 'MTM' then mtm_promotion_time
180
+ when 'LTM' then ltm_promotion_time
181
+ when 'PM' then pm_promotion_time
182
+ else 0
183
+ end
184
+ end
185
+
186
+ def promotion_hits(state)
187
+ case state
188
+ when 'MTM' then mtm_promotion_hits
189
+ when 'LTM' then ltm_promotion_hits
190
+ when 'PM' then pm_promotion_hits
191
+ else 0
192
+ end
193
+ end
194
+
195
+ def demotion_time(state)
196
+ case state
197
+ when 'MTM' then mtm_demotion_time
198
+ when 'LTM' then ltm_demotion_time
199
+ when 'PM' then pm_demotion_time
200
+ else Float::INFINITY
201
+ end
202
+ end
203
+
204
+ def demotion_hits(state)
205
+ case state
206
+ when 'MTM' then mtm_demotion_hits
207
+ when 'LTM' then ltm_demotion_hits
208
+ when 'PM' then pm_demotion_hits
209
+ else Float::INFINITY
210
+ end
211
+ end
212
+ end
213
+ ```
214
+
215
+