agent99 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/A2A_SPEC-dev.md +1829 -0
- data/CHANGELOG.md +38 -0
- data/COMMITS.md +196 -0
- data/DOCS.md +96 -0
- data/README.md +212 -84
- data/Rakefile +62 -0
- data/docs/AI/htm.md +215 -0
- data/docs/AI/htm.rb +141 -0
- data/docs/AI/htm_demo.db +0 -0
- data/docs/AI/notes_on_htm_implementation.md +1319 -0
- data/docs/AI/some_code.rb +692 -0
- data/docs/advanced-topics/a2a-protocol.md +13 -0
- data/docs/{advanced_features.md → advanced-topics/advanced-features.md} +9 -4
- data/docs/{control_actions.md → advanced-topics/control-actions.md} +2 -0
- data/docs/advanced-topics/model-context-protocol.md +4 -0
- data/docs/advanced-topics/multi-agent-processing.md +674 -0
- data/docs/agent-development/request-response-handling.md +512 -0
- data/docs/agent99_framework/central_registry.md +94 -0
- data/docs/agent99_framework/message_client.md +120 -0
- data/docs/agent99_framework/registry_client.md +119 -0
- data/docs/api-reference/agent99-base.md +463 -0
- data/docs/api-reference/message-clients.md +495 -0
- data/docs/{api_reference.md → api-reference/overview.md} +14 -4
- data/docs/api-reference/registry-client.md +470 -0
- data/docs/api-reference/schemas.md +518 -0
- data/docs/assets/css/custom.css +27 -0
- data/docs/assets/images/agent-lifecycle.svg +73 -0
- data/docs/assets/images/agent-registry-process.svg +86 -0
- data/docs/assets/images/agent-registry-processes.svg +114 -0
- data/docs/assets/images/agent-types-overview.svg +51 -0
- data/docs/assets/images/agent99-architecture.svg +85 -0
- data/docs/assets/images/agent99_logo.png +0 -0
- data/docs/assets/images/control-actions-state.svg +83 -0
- data/docs/assets/images/knowledge-graph.svg +77 -0
- data/docs/assets/images/message-processing-flow.svg +148 -0
- data/docs/assets/images/multi-agent-system.svg +66 -0
- data/docs/assets/images/proxy-pattern-sequence.svg +48 -0
- data/docs/assets/images/request-flow.svg +97 -0
- data/docs/assets/images/request-processing-lifecycle.svg +50 -0
- data/docs/assets/images/request-response-sequence.svg +39 -0
- data/docs/{agent_lifecycle.md → core-concepts/agent-lifecycle.md} +2 -0
- data/docs/core-concepts/agent-types.md +255 -0
- data/docs/{architecture.md → core-concepts/architecture.md} +5 -5
- data/docs/core-concepts/what-is-an-agent.md +293 -0
- data/docs/diagrams/message-flow-sequence.svg +198 -0
- data/docs/diagrams/p2p-network-topology.svg +181 -0
- data/docs/diagrams/smart-transport-routing.svg +165 -0
- data/docs/diagrams/three-layer-architecture.svg +77 -0
- data/docs/diagrams/transport-extension-api.svg +309 -0
- data/docs/diagrams/transport-extension-architecture.svg +234 -0
- data/docs/diagrams/transport-selection-flowchart.svg +264 -0
- data/docs/examples/advanced-examples.md +951 -0
- data/docs/examples/basic-examples.md +268 -0
- data/docs/{agent_discovery.md → framework-components/agent-discovery.md} +9 -5
- data/docs/{agent_registry_processes.md → framework-components/agent-registry.md} +9 -3
- data/docs/{message_processing.md → framework-components/message-processing.md} +3 -1
- data/docs/getting-started/basic-example.md +306 -0
- data/docs/getting-started/installation.md +160 -0
- data/docs/getting-started/overview.md +64 -0
- data/docs/getting-started/quick-start.md +179 -0
- data/docs/index.md +97 -0
- data/docs/operations/breaking-changes.md +26 -0
- data/examples/DEMO.md +148 -0
- data/examples/README.md +50 -0
- data/examples/agent_watcher.rb +5 -1
- data/examples/bad_agent.rb +32 -0
- data/examples/chief_agent.rb +17 -6
- data/examples/control.rb +16 -7
- data/examples/example_agent.rb +16 -3
- data/examples/maxwell_agent86.rb +15 -26
- data/examples/registry.rb +10 -9
- data/examples/run_demo.rb +433 -0
- data/lib/agent99/agent_discovery.rb +4 -0
- data/lib/agent99/agent_lifecycle.rb +34 -10
- data/lib/agent99/amqp_message_client.rb +2 -2
- data/lib/agent99/base.rb +6 -2
- data/lib/agent99/message_processing.rb +6 -10
- data/lib/agent99/registry_client.rb +15 -11
- data/lib/agent99/tcp_message_client.rb +183 -0
- data/lib/agent99/version.rb +1 -1
- data/lib/agent99.rb +1 -1
- data/mkdocs.yml +195 -0
- data/p2p_plan.md +533 -0
- data/p2p_roadmap.md +299 -0
- data/registry_plan.md +1818 -0
- metadata +93 -30
- data/docs/README.md +0 -57
- data/docs/diagrams/agent_registry_processes.dot +0 -42
- data/docs/diagrams/agent_registry_processes.png +0 -0
- data/docs/diagrams/high_level_architecture.dot +0 -26
- data/docs/diagrams/high_level_architecture.png +0 -0
- data/docs/diagrams/request_flow.dot +0 -42
- data/docs/diagrams/request_flow.png +0 -0
- /data/docs/{extending_the_framework.md → advanced-topics/extending-the-framework.md} +0 -0
- /data/docs/{custom_agent_implementation.md → agent-development/custom-agent-implementation.md} +0 -0
- /data/docs/{error_handling_and_logging.md → agent-development/error-handling-and-logging.md} +0 -0
- /data/docs/{schema_definition.md → agent-development/schema-definition.md} +0 -0
- /data/docs/{messaging_system.md → framework-components/messaging-system.md} +0 -0
- /data/docs/{configuration.md → operations/configuration.md} +0 -0
- /data/docs/{preformance_considerations.md → operations/performance-considerations.md} +0 -0
- /data/docs/{security.md → operations/security.md} +0 -0
- /data/docs/{troubleshooting.md → operations/troubleshooting.md} +0 -0
data/README.md
CHANGED
@@ -1,62 +1,102 @@
|
|
1
|
-
# Agent99
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
1
|
+
# Agent99
|
2
|
+
<br/>
|
3
|
+
|
4
|
+
> [!CAUTION]
|
5
|
+
> ## ⚠️ Under Development ⚠️
|
6
|
+
> Initial release has no AI components - it's a generic client-server / request-response micro-services system using peer-to-peer messaging and centralized agent registry. Review [The Changelog](./CHANGELOG.md) for version changes.
|
7
|
+
<br /><br />
|
8
|
+
|
9
|
+
<div align="center">
|
10
|
+
<table>
|
11
|
+
<tr>
|
12
|
+
<td width="40%" align="center" valign="top">
|
13
|
+
<img src="docs/assets/images/agent99_logo.png" alt="Agent99 - Ruby Framework for Intelligent Software Agents" width="1200">
|
14
|
+
<br /><br />
|
15
|
+
[Comprehensive Documentation Website](https://madbomber.github.io/agent99/)
|
16
|
+
</td>
|
17
|
+
<td width="60%" align="left" valign="top">
|
18
|
+
Agent99 is a Ruby-based framework for building and managing intelligent software agents in a distributed system. Like the clever Agent 99 from Get Smart, your agents will be smart, adaptable, and ready to tackle any challenge through seamless peer-to-peer communication and centralized discovery.
|
19
|
+
<br/><br/>
|
20
|
+
<h3>Key Features</h3>
|
21
|
+
<ul>
|
22
|
+
<li><strong>🌐 <a href="#agent-discovery">Agent Discovery</a></strong> - Find agents by capabilities</li>
|
23
|
+
<li><strong>📡 <a href="#flexible-communication">Peer-to-Peer Messaging</a></strong> - AMQP & NATS support</li>
|
24
|
+
<li><strong>🔄 <a href="#message-processing">Message Processing</a></strong> - Requests, responses & control</li>
|
25
|
+
<li><strong>📋 <a href="#registry-integration">Registry Integration</a></strong> - Centralized agent registry</li>
|
26
|
+
<li><strong>⚡ <a href="#control-actions">Control Actions</a></strong> - Pause, resume, update agents</li>
|
27
|
+
<li><strong>🔧 <a href="#agent-lifecycle-management">Lifecycle Management</a></strong> - Easy setup & teardown</li>
|
28
|
+
<li><strong>🚀 <a href="#multi-agent-processing">Multi-Agent Processing</a></strong> - Thread isolation support</li>
|
29
|
+
<li><strong>📊 <a href="#error-handling-and-logging">Error Handling & Logging</a></strong> - Built-in management</li>
|
30
|
+
</ul>
|
31
|
+
</td>
|
32
|
+
</tr>
|
33
|
+
</table>
|
34
|
+
</div>
|
35
|
+
|
36
|
+
## Table of Contents
|
37
|
+
|
38
|
+
* [Quick Start](#quick-start)
|
39
|
+
* [Prerequisites](#prerequisites)
|
40
|
+
* [Architecture Overview](#architecture-overview)
|
41
|
+
* [Installation](#installation)
|
42
|
+
* [Usage](#usage)
|
43
|
+
* [Configuration](#configuration)
|
44
|
+
* [Agent Lifecycle](#agent-lifecycle)
|
45
|
+
* [Message Processing](#message-processing)
|
46
|
+
* [Registry Integration](#registry-integration)
|
47
|
+
* [Examples](#examples)
|
48
|
+
* [Evolving Standards](#evolving-standards)
|
49
|
+
* [Contributing](#contributing)
|
50
|
+
* [Roadmap](#roadmap)
|
51
|
+
* [License](#license)
|
52
|
+
|
53
|
+
## Quick Start
|
54
|
+
|
55
|
+
Jump right in with our step-by-step guide:
|
56
|
+
|
57
|
+
1. **Prerequisites**: [Install message broker](#prerequisites) (NATS or RabbitMQ)
|
58
|
+
2. **Install Agent99**: Add to Gemfile and bundle install
|
59
|
+
3. **Start Registry**: Launch the central agent registry
|
60
|
+
4. **Create Your First Agent**: Build a simple greeter agent
|
61
|
+
5. **Test Communication**: Verify agents can discover and communicate
|
62
|
+
|
63
|
+
📖 **Detailed walkthrough**: [Getting Started Guide](docs/getting-started/quick-start.md)
|
64
|
+
|
65
|
+
## Prerequisites
|
66
|
+
|
67
|
+
Agent99 requires a message broker for inter-agent communication. Choose one:
|
68
|
+
|
69
|
+
### NATS (Recommended)
|
70
|
+
```bash
|
71
|
+
# macOS
|
72
|
+
brew install nats-server
|
73
|
+
|
74
|
+
# Start NATS
|
75
|
+
nats-server
|
76
|
+
```
|
34
77
|
|
35
|
-
|
36
|
-
|
37
|
-
|
78
|
+
### RabbitMQ
|
79
|
+
```bash
|
80
|
+
# macOS
|
81
|
+
brew install rabbitmq
|
38
82
|
|
39
|
-
|
83
|
+
# Start RabbitMQ
|
84
|
+
brew services start rabbitmq
|
85
|
+
```
|
40
86
|
|
41
|
-
|
87
|
+
📋 **More installation options**: [Installation Guide](docs/getting-started/installation.md)
|
42
88
|
|
43
|
-
|
89
|
+
## Architecture Overview
|
44
90
|
|
45
|
-
|
91
|
+

|
46
92
|
|
47
|
-
|
93
|
+
Agent99 follows a distributed architecture with three core components:
|
48
94
|
|
49
|
-
|
95
|
+
- **🏛️ Central Registry**: Service discovery and agent registration
|
96
|
+
- **📡 Message Broker**: Peer-to-peer communication backbone (NATS/AMQP)
|
97
|
+
- **🤖 Agents**: Independent services with specific capabilities
|
50
98
|
|
51
|
-
|
52
|
-
- Message Processing: Handle requests, responses, and control messages
|
53
|
-
- Agent Discovery: Find other agents based on capabilities
|
54
|
-
- Flexible Communication: Support for both AMQP and NATS messaging systems
|
55
|
-
- Registry Integration: Register and discover agents through a central registry
|
56
|
-
- Error Handling and Logging: Built-in error management and logging capabilities
|
57
|
-
- Control Actions: Pause, resume, update configuration, and request status of agents
|
58
|
-
- Dynamic Agent Loading: Support for runtime loading and deployment of new agents
|
59
|
-
- Multi-Agent Processing: Run multiple agents within the same process using thread isolation
|
99
|
+
📚 **Deep dive**: [Architecture Documentation](docs/core-concepts/architecture.md)
|
60
100
|
|
61
101
|
## Installation
|
62
102
|
|
@@ -80,63 +120,151 @@ $ gem install agent99
|
|
80
120
|
|
81
121
|
## Usage
|
82
122
|
|
83
|
-
|
123
|
+
Create your first agent in three simple steps:
|
84
124
|
|
125
|
+
### 1. Define Request Schema
|
85
126
|
```ruby
|
86
|
-
|
87
|
-
|
88
|
-
class MyAgentRequest < SimpleJsonSchemaBuilder::Base
|
127
|
+
class GreeterRequest < SimpleJsonSchemaBuilder::Base
|
89
128
|
object do
|
90
129
|
object :header, schema: Agent99::HeaderSchema
|
91
|
-
|
92
|
-
# Define your agents parameters ....
|
93
|
-
string :greeting, required: false, examples: ["Hello"]
|
94
|
-
string :name, required: true, examples: ["World"]
|
130
|
+
string :name, required: true, examples: ["World"]
|
95
131
|
end
|
96
132
|
end
|
133
|
+
```
|
97
134
|
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
def
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
response = { result: "Processed request" }
|
109
|
-
|
110
|
-
# Not every request needs a response
|
111
|
-
send_response(response)
|
135
|
+
### 2. Implement Agent Class
|
136
|
+
```ruby
|
137
|
+
class GreeterAgent < Agent99::Base
|
138
|
+
def info
|
139
|
+
{
|
140
|
+
name: self.class.to_s,
|
141
|
+
type: :server,
|
142
|
+
capabilities: ['greeter', 'hello_world'],
|
143
|
+
request_schema: GreeterRequest.schema
|
144
|
+
}
|
112
145
|
end
|
113
146
|
|
114
|
-
def
|
115
|
-
|
116
|
-
|
147
|
+
def process_request(payload)
|
148
|
+
name = payload.dig(:name)
|
149
|
+
send_response(result: "Hello, #{name}!")
|
117
150
|
end
|
118
151
|
end
|
152
|
+
```
|
119
153
|
|
120
|
-
|
154
|
+
### 3. Run Your Agent
|
155
|
+
```ruby
|
156
|
+
require 'agent99'
|
157
|
+
agent = GreeterAgent.new
|
121
158
|
agent.run
|
122
159
|
```
|
123
160
|
|
161
|
+
🎯 **More examples**: [Usage Examples](docs/examples/)
|
162
|
+
|
124
163
|
## Configuration
|
125
164
|
|
126
|
-
|
165
|
+
Configure Agent99 through environment variables:
|
166
|
+
|
167
|
+
### Core Settings
|
168
|
+
- `AGENT99_REGISTRY_URL`: Registry service URL (default: `http://localhost:4567`)
|
169
|
+
- `AGENT99_LOG_LEVEL`: Logging verbosity (default: `INFO`)
|
170
|
+
|
171
|
+
### NATS Configuration
|
172
|
+
- `NATS_URL`: NATS server URL (default: `nats://localhost:4222`)
|
173
|
+
- `NATS_USER`: Username for authentication
|
174
|
+
- `NATS_PASS`: Password for authentication
|
175
|
+
|
176
|
+
### AMQP/RabbitMQ Configuration
|
177
|
+
- `RABBITMQ_URL`: RabbitMQ server URL (default: `amqp://localhost:5672`)
|
178
|
+
- `RABBITMQ_USER`: Username (default: `guest`)
|
179
|
+
- `RABBITMQ_PASS`: Password (default: `guest`)
|
180
|
+
|
181
|
+
📚 **Complete configuration guide**: [Configuration Documentation](docs/configuration/)
|
182
|
+
|
183
|
+
## Agent Lifecycle
|
184
|
+
|
185
|
+

|
186
|
+
|
187
|
+
Agent99 manages the complete lifecycle of your agents:
|
188
|
+
|
189
|
+
- **🎬 Initialization**: Agent registers with the central registry
|
190
|
+
- **⚡ Running**: Agent processes requests and sends responses
|
191
|
+
- **⏸️ Paused**: Agent temporarily stops processing (control action)
|
192
|
+
- **🔄 Updated**: Agent receives new configuration or capabilities
|
193
|
+
- **🛑 Shutdown**: Agent gracefully disconnects and cleans up
|
194
|
+
|
195
|
+
🔄 **Detailed lifecycle management**: [Agent Lifecycle Guide](docs/core-concepts/agent-lifecycle.md)
|
196
|
+
|
197
|
+
## Message Processing
|
198
|
+
|
199
|
+

|
200
|
+
|
201
|
+
Agent99 handles three types of messages:
|
202
|
+
|
203
|
+
- **📨 Requests**: Incoming work for agents to process
|
204
|
+
- **📤 Responses**: Results sent back to requesters
|
205
|
+
- **🎛️ Control**: Management commands (pause, resume, update)
|
206
|
+
|
207
|
+
Each message is validated, routed, and processed with full error handling and logging.
|
208
|
+
|
209
|
+
⚙️ **Message processing details**: [Message Processing Guide](docs/framework-components/message-processing.md)
|
210
|
+
|
211
|
+
## Registry Integration
|
212
|
+
|
213
|
+

|
214
|
+
|
215
|
+
The central registry enables dynamic service discovery:
|
216
|
+
|
217
|
+
1. **📝 Registration**: Agents announce their capabilities
|
218
|
+
2. **🔍 Discovery**: Find agents by capability or name
|
219
|
+
3. **📋 Management**: Monitor and control registered agents
|
220
|
+
4. **🚪 Withdrawal**: Clean removal from registry
|
221
|
+
|
222
|
+
The registry supports both HTTP REST API and direct client integration.
|
223
|
+
|
224
|
+
🏛️ **Registry implementation**: [Registry Documentation](docs/framework-components/agent-registry.md)
|
225
|
+
|
226
|
+
## Examples
|
227
|
+
|
228
|
+
Explore real-world implementations:
|
229
|
+
|
230
|
+
- **🚀 [Demo Runner](examples/run_demo.rb)**: Complete working system
|
231
|
+
- **👋 [Greeter Agent](examples/greeter_agent.rb)**: Simple request-response
|
232
|
+
- **🔄 [Multi-Agent System](examples/multi_agent/)**: Coordinated agents
|
233
|
+
- **📊 [Monitoring Dashboard](examples/dashboard/)**: Agent status monitoring
|
234
|
+
|
235
|
+
💡 **All examples**: [Examples Directory](docs/examples/)
|
236
|
+
|
237
|
+
## Evolving Standards
|
238
|
+
|
239
|
+
- [agntcy.org](https://agntcy.org) - Agency protocol initiative
|
240
|
+
- [Agent2Agent](https://a2a-protocol.org/latest/) - Linux Foundation protocol
|
127
241
|
|
128
|
-
- `REGISTRY_BASE_URL`: URL of the agent registry service (default: 'http://localhost:4567') See the default registry service in the examples folder.
|
129
242
|
|
130
243
|
## Contributing
|
131
244
|
|
132
245
|
Bug reports and pull requests are welcome on GitHub at https://github.com/MadBomber/agent99.
|
133
246
|
|
134
|
-
##
|
247
|
+
## Roadmap
|
248
|
+
|
249
|
+
### Short-term (v0.1.x)
|
250
|
+
- **🔍 Enhanced Discovery**: Semantic search with vector database (SQLite + embeddings)
|
251
|
+
- **📋 Schema Validation**: Complete request/response schema definitions
|
252
|
+
- **🤖 AI Integration**: RAG-enabled agents using prompts as tools
|
253
|
+
- **📊 Monitoring**: Built-in metrics and health checks
|
254
|
+
|
255
|
+
### Medium-term (v0.2.x)
|
256
|
+
- **🔐 Security**: Authentication and authorization framework
|
257
|
+
- **⚡ Performance**: Connection pooling and message batching
|
258
|
+
- **🌍 Multi-broker**: Support for multiple message brokers simultaneously
|
259
|
+
- **📈 Scaling**: Load balancing and auto-scaling capabilities
|
260
|
+
|
261
|
+
### Long-term (v1.0+)
|
262
|
+
- **🧠 Intelligence**: Built-in ML/AI capabilities
|
263
|
+
- **🔗 Interoperability**: Full Agent2Agent protocol compliance
|
264
|
+
- **☁️ Cloud-native**: Kubernetes operators and cloud integrations
|
265
|
+
|
266
|
+
⚠️ **Breaking changes**: [Migration Guide v0.0.4](docs/breaking_change_v0.0.4.md)
|
135
267
|
|
136
|
-
- In the example registry, replace the Array(Hash) datastore with sqlite3 with a vector table to support discovery using semantic search.
|
137
|
-
- Treat the agent like a Tool w/r/t RAG for prompts.
|
138
|
-
- Add AgentRequest schema to agent's info in the registry.
|
139
|
-
- Add AgentResponse schema to define the `result` element in the response JSON payload
|
140
268
|
|
141
269
|
## License
|
142
270
|
|
data/Rakefile
CHANGED
@@ -6,3 +6,65 @@ require "minitest/test_task"
|
|
6
6
|
Minitest::TestTask.create
|
7
7
|
|
8
8
|
task default: :test
|
9
|
+
|
10
|
+
# Additional test tasks for Agent99
|
11
|
+
require "rake/testtask"
|
12
|
+
|
13
|
+
# Unit tests only
|
14
|
+
Rake::TestTask.new(:test_unit) do |t|
|
15
|
+
t.libs << "lib"
|
16
|
+
t.libs << "test"
|
17
|
+
t.test_files = FileList["test/agent99/*test*.rb"]
|
18
|
+
t.verbose = true
|
19
|
+
end
|
20
|
+
|
21
|
+
# Integration tests only
|
22
|
+
Rake::TestTask.new(:test_integration) do |t|
|
23
|
+
t.libs << "lib"
|
24
|
+
t.libs << "test"
|
25
|
+
t.test_files = FileList["test/integration/*test*.rb"]
|
26
|
+
t.verbose = true
|
27
|
+
end
|
28
|
+
|
29
|
+
# System tests only
|
30
|
+
Rake::TestTask.new(:test_system) do |t|
|
31
|
+
t.libs << "lib"
|
32
|
+
t.libs << "test"
|
33
|
+
t.test_files = FileList["test/system/*test*.rb"]
|
34
|
+
t.verbose = true
|
35
|
+
end
|
36
|
+
|
37
|
+
desc "Run tests with coverage reporting (requires simplecov gem)"
|
38
|
+
task :test_coverage do
|
39
|
+
ENV["COVERAGE"] = "true"
|
40
|
+
Rake::Task[:test].invoke
|
41
|
+
end
|
42
|
+
|
43
|
+
desc "Run tests in verbose mode"
|
44
|
+
task :test_verbose do
|
45
|
+
ENV["VERBOSE"] = "true"
|
46
|
+
Rake::Task[:test].invoke
|
47
|
+
end
|
48
|
+
|
49
|
+
desc "Show test statistics"
|
50
|
+
task :test_stats do
|
51
|
+
unit_tests = Dir.glob("test/agent99/*test*.rb").size
|
52
|
+
integration_tests = Dir.glob("test/integration/*test*.rb").size
|
53
|
+
system_tests = Dir.glob("test/system/*test*.rb").size
|
54
|
+
total_tests = unit_tests + integration_tests + system_tests
|
55
|
+
|
56
|
+
puts "Test Statistics:"
|
57
|
+
puts " Unit tests: #{unit_tests}"
|
58
|
+
puts " Integration tests: #{integration_tests}"
|
59
|
+
puts " System tests: #{system_tests}"
|
60
|
+
puts " Total test files: #{total_tests}"
|
61
|
+
|
62
|
+
# Count test methods
|
63
|
+
test_methods = 0
|
64
|
+
Dir.glob("test/**/*test*.rb").each do |file|
|
65
|
+
content = File.read(file)
|
66
|
+
test_methods += content.scan(/def test_/).size
|
67
|
+
end
|
68
|
+
|
69
|
+
puts " Total test methods: #{test_methods}"
|
70
|
+
end
|
data/docs/AI/htm.md
ADDED
@@ -0,0 +1,215 @@
|
|
1
|
+
# Hierarchical Temporal Memory in AI Systems
|
2
|
+
|
3
|
+
## Table of Contents
|
4
|
+
- [Introduction](#introduction)
|
5
|
+
- [Understanding Hierarchical Temporal Memory](#understanding-hierarchical-temporal-memory)
|
6
|
+
- [Layers of Memory](#layers-of-memory)
|
7
|
+
- [Topic Columns](#topic-columns)
|
8
|
+
- [Memory Cell Promotion and Forgetting](#memory-cell-promotion-and-forgetting)
|
9
|
+
- [Implications for AI Systems](#implications-for-ai-systems)
|
10
|
+
- [Conclusion](#conclusion)
|
11
|
+
- [Some Code Ideas](#some-code-ideas)
|
12
|
+
|
13
|
+
## Introduction
|
14
|
+
|
15
|
+
Hierarchical Temporal Memory (HTM) is a computational model that mimics some aspects of the neocortex, developed by the Numenta research company. It focuses on how the brain processes information over time and is designed to learn patterns and sequences in data. HTM operates on principles of hierarchy, meaning it organizes information in layers, and temporal memory, which enables it to remember sequences and make predictions based on temporal patterns. This model is particularly effective for tasks involving time series data and has applications in anomaly detection, robotics, and various cognitive computing tasks.
|
16
|
+
|
17
|
+
## Understanding Hierarchical Temporal Memory
|
18
|
+
|
19
|
+
Hierarchical Temporal Memory is inspired by the theory of how the brain works, particularly focusing on how it learns patterns and sequences. The fundamental ideas were put forward by Jeff Hawkins, co-founder of Numenta, who theorized that real intelligence is derived from understanding and manipulating temporal sequences.
|
20
|
+
|
21
|
+
HTM leverages a structure that is hierarchical and spatially organized. Each level of the hierarchy processes information at varying degrees of abstraction. The bottom layers deal with raw sensory input, while the upper layers interpret this data into meaningful contexts.
|
22
|
+
|
23
|
+
This model is particularly valuable in applications involving complex time series, where the demand for understanding sequences over time is crucial. The capability of HTM to generalize from sparse data makes it unique compared to traditional machine learning techniques.
|
24
|
+
|
25
|
+
## Layers of Memory
|
26
|
+
|
27
|
+
HTM's architecture consists of several layers, each designed for specific functions. The bottom layer, known as the input layer, is responsible for receiving raw data inputs. Subsequent layers process this information, identifying patterns and sequences that inform decisions.
|
28
|
+
|
29
|
+
1. **Input Layer**: Receives sensory data and converts it into a usable format for the next layer.
|
30
|
+
2. **Temporal Pooling Layer**: Captures the temporal dependencies in the data.
|
31
|
+
3. **Spatial Pooling Layer**: Encodes spatial relationships and builds a representation of the input data.
|
32
|
+
|
33
|
+
This multi-layered approach enables HTM to build increasingly sophisticated representations of the data at each layer, akin to how the neocortex might operate.
|
34
|
+
|
35
|
+
## Topic Columns
|
36
|
+
|
37
|
+
Each "column" in HTM can be viewed as a mini-neural network that processes input from the layer below it. These columns work in parallel to learn spatial and temporal patterns. Thus, they respond to specific features of the incoming data, allowing HTM to develop a hierarchy of data features. Columns also facilitate learning through inhibition, promoting the most active neurons while suppressing others.
|
38
|
+
|
39
|
+
## Memory Cell Promotion and Forgetting
|
40
|
+
|
41
|
+
The mechanisms by which HTM promotes or forgets memory cells mirror more biological processes.
|
42
|
+
|
43
|
+
### Promotion
|
44
|
+
|
45
|
+
Cells that consistently provide accurate predictions based on the learned sequence are promoted. For instance, if certain neurons within a column recognize patterns that lead to accurate outcomes, they're trained to engage more actively depending on their performance over time.
|
46
|
+
|
47
|
+
### Forgetting
|
48
|
+
|
49
|
+
Forgetting is equally important; neurons that fail to respond to valid sequences are gradually demoted. This ensures that the model does not become overly reliant on outdated information, allowing new patterns and sequences to emerge and be incorporated.
|
50
|
+
|
51
|
+
## Implications for AI Systems
|
52
|
+
|
53
|
+
By modeling the functioning of the neocortex, HTM introduces a new paradigm in machine learning, emphasizing time and sequences over static data processing. This has significant implications in various fields:
|
54
|
+
|
55
|
+
- **Anomaly Detection**: In finance, HTM can be used to identify unusual patterns in transactions that may indicate fraud.
|
56
|
+
- **Robotics**: Robots leveraging HTM can learn from their experiences in real-time, adapting to environmental changes dynamically.
|
57
|
+
- **Natural Language Processing**: Understanding the temporal relationships between phrases can enhance language models, leading to more contextually accurate interpretations.
|
58
|
+
|
59
|
+
## Conclusion
|
60
|
+
|
61
|
+
Hierarchical Temporal Memory presents a powerful framework for understanding and replicating the cognitive processes of the human brain. With its ability to learn from temporal patterns, it opens new frontiers in artificial intelligence and machine learning.
|
62
|
+
|
63
|
+
## Some Code Ideas
|
64
|
+
|
65
|
+
```ruby
|
66
|
+
require 'active_record'
|
67
|
+
|
68
|
+
class CreateHtmTable < ActiveRecord::Migration[7.0]
|
69
|
+
def change
|
70
|
+
create_table :htm do |t|
|
71
|
+
t.uuid :event, null: false
|
72
|
+
t.text :proposition, null: false
|
73
|
+
t.string :state, null: false
|
74
|
+
t.datetime :last_accessed, null: false
|
75
|
+
t.integer :hit_count, null: false, default: 0
|
76
|
+
t.float :vector, null: false, array: true, limit: 2048
|
77
|
+
|
78
|
+
t.timestamps
|
79
|
+
end
|
80
|
+
|
81
|
+
add_index :htm, :event, unique: true
|
82
|
+
add_index :htm, :state
|
83
|
+
|
84
|
+
execute <<-SQL
|
85
|
+
ALTER TABLE htm
|
86
|
+
ADD CONSTRAINT check_state
|
87
|
+
CHECK (state IN ('STM', 'MTM', 'LTM', 'PM'))
|
88
|
+
SQL
|
89
|
+
|
90
|
+
execute <<-SQL
|
91
|
+
ALTER TABLE htm
|
92
|
+
ADD CONSTRAINT check_vector_length
|
93
|
+
CHECK (array_length(vector, 1) = 2048)
|
94
|
+
SQL
|
95
|
+
end
|
96
|
+
end
|
97
|
+
```
|
98
|
+
|
99
|
+
This is a notional model for the table. I'm having second thoughts about keeping track of the hit count. I think maybe access time is all that is needed and that is provided by the AR timestamps.
|
100
|
+
|
101
|
+
```ruby
|
102
|
+
require 'active_record'
|
103
|
+
require 'neighbors'
|
104
|
+
|
105
|
+
class Htm < ActiveRecord::Base
|
106
|
+
STATES = %w[STM MTM LTM PM].freeze
|
107
|
+
|
108
|
+
validates :event, presence: true
|
109
|
+
validates :proposition, presence: true
|
110
|
+
validates :state, presence: true, inclusion: { in: STATES }
|
111
|
+
validates :vector, presence: true, length: { is: 2048 }
|
112
|
+
|
113
|
+
def self.search_and_update(event_uuid:, query_vector:, top_k: 10)
|
114
|
+
records = where(event: event_uuid)
|
115
|
+
vectors = records.pluck(:vector)
|
116
|
+
|
117
|
+
neighbor_search = Neighbors::NearestNeighbors.new(vectors)
|
118
|
+
nearest_indices = neighbor_search.search(query_vector, k: top_k)
|
119
|
+
|
120
|
+
nearest_records = records.where(id: records.ids.values_at(*nearest_indices))
|
121
|
+
|
122
|
+
nearest_records.each do |record|
|
123
|
+
record.hit_count += 1
|
124
|
+
record.last_accessed = Time.current
|
125
|
+
record.save!
|
126
|
+
end
|
127
|
+
|
128
|
+
nearest_records
|
129
|
+
end
|
130
|
+
|
131
|
+
def update_state
|
132
|
+
event_config = StateConfig.find_by(event: event)
|
133
|
+
current_time = Time.current
|
134
|
+
time_since_creation = current_time - created_at
|
135
|
+
time_since_last_access = current_time - last_accessed
|
136
|
+
|
137
|
+
if should_promote?(event_config, time_since_creation)
|
138
|
+
promote_state
|
139
|
+
elsif should_demote?(event_config, time_since_last_access)
|
140
|
+
demote_state
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
private
|
145
|
+
|
146
|
+
def should_promote?(config, time_delta)
|
147
|
+
current_state_index = STATES.index(state)
|
148
|
+
return false if current_state_index == STATES.length - 1
|
149
|
+
|
150
|
+
next_state = STATES[current_state_index + 1]
|
151
|
+
time_delta > config.promotion_time(next_state) &&
|
152
|
+
hit_count > config.promotion_hits(next_state)
|
153
|
+
end
|
154
|
+
|
155
|
+
def should_demote?(config, time_delta)
|
156
|
+
current_state_index = STATES.index(state)
|
157
|
+
return false if current_state_index == 0
|
158
|
+
|
159
|
+
time_delta < config.demotion_time(state) &&
|
160
|
+
hit_count < config.demotion_hits(state)
|
161
|
+
end
|
162
|
+
|
163
|
+
def promote_state
|
164
|
+
current_index = STATES.index(state)
|
165
|
+
self.state = STATES[current_index + 1] if current_index < STATES.length - 1
|
166
|
+
save!
|
167
|
+
end
|
168
|
+
|
169
|
+
def demote_state
|
170
|
+
current_index = STATES.index(state)
|
171
|
+
self.state = STATES[current_index - 1] if current_index > 0
|
172
|
+
save!
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
class StateConfig < ActiveRecord::Base
|
177
|
+
def promotion_time(state)
|
178
|
+
case state
|
179
|
+
when 'MTM' then mtm_promotion_time
|
180
|
+
when 'LTM' then ltm_promotion_time
|
181
|
+
when 'PM' then pm_promotion_time
|
182
|
+
else 0
|
183
|
+
end
|
184
|
+
end
|
185
|
+
|
186
|
+
def promotion_hits(state)
|
187
|
+
case state
|
188
|
+
when 'MTM' then mtm_promotion_hits
|
189
|
+
when 'LTM' then ltm_promotion_hits
|
190
|
+
when 'PM' then pm_promotion_hits
|
191
|
+
else 0
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
def demotion_time(state)
|
196
|
+
case state
|
197
|
+
when 'MTM' then mtm_demotion_time
|
198
|
+
when 'LTM' then ltm_demotion_time
|
199
|
+
when 'PM' then pm_demotion_time
|
200
|
+
else Float::INFINITY
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def demotion_hits(state)
|
205
|
+
case state
|
206
|
+
when 'MTM' then mtm_demotion_hits
|
207
|
+
when 'LTM' then ltm_demotion_hits
|
208
|
+
when 'PM' then pm_demotion_hits
|
209
|
+
else Float::INFINITY
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
213
|
+
```
|
214
|
+
|
215
|
+
|