igniter 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +217 -0
- data/docs/APPLICATION_V1.md +253 -0
- data/docs/CAPABILITIES_V1.md +207 -0
- data/docs/CONSENSUS_V1.md +477 -0
- data/docs/CONTENT_ADDRESSING_V1.md +221 -0
- data/docs/DATAFLOW_V1.md +274 -0
- data/docs/MESH_V1.md +732 -0
- data/docs/NODE_CACHE_V1.md +324 -0
- data/docs/PROACTIVE_AGENTS_V1.md +293 -0
- data/docs/SERVER_V1.md +200 -1
- data/docs/SKILLS_V1.md +213 -0
- data/docs/STORE_ADAPTERS.md +41 -13
- data/docs/TEMPORAL_V1.md +174 -0
- data/docs/TOOLS_V1.md +347 -0
- data/docs/TRANSCRIPTION_V1.md +403 -0
- data/examples/README.md +37 -0
- data/examples/consensus.rb +239 -0
- data/examples/dataflow.rb +308 -0
- data/examples/elocal_webhook.rb +1 -0
- data/examples/incremental.rb +142 -0
- data/examples/llm_tools.rb +237 -0
- data/examples/mesh.rb +239 -0
- data/examples/mesh_discovery.rb +267 -0
- data/examples/mesh_gossip.rb +162 -0
- data/examples/ringcentral_routing.rb +1 -1
- data/lib/igniter/agents/ai/alert_agent.rb +111 -0
- data/lib/igniter/agents/ai/chain_agent.rb +127 -0
- data/lib/igniter/agents/ai/critic_agent.rb +163 -0
- data/lib/igniter/agents/ai/evaluator_agent.rb +193 -0
- data/lib/igniter/agents/ai/evolution_agent.rb +286 -0
- data/lib/igniter/agents/ai/health_check_agent.rb +122 -0
- data/lib/igniter/agents/ai/observer_agent.rb +184 -0
- data/lib/igniter/agents/ai/planner_agent.rb +210 -0
- data/lib/igniter/agents/ai/router_agent.rb +131 -0
- data/lib/igniter/agents/ai/self_reflection_agent.rb +175 -0
- data/lib/igniter/agents/observability/metrics_agent.rb +130 -0
- data/lib/igniter/agents/pipeline/batch_processor_agent.rb +131 -0
- data/lib/igniter/agents/proactive_agent.rb +208 -0
- data/lib/igniter/agents/reliability/retry_agent.rb +99 -0
- data/lib/igniter/agents/scheduling/cron_agent.rb +110 -0
- data/lib/igniter/agents.rb +56 -0
- data/lib/igniter/application/app_config.rb +32 -0
- data/lib/igniter/application/autoloader.rb +18 -0
- data/lib/igniter/application/generator.rb +157 -0
- data/lib/igniter/application/scheduler.rb +109 -0
- data/lib/igniter/application/yml_loader.rb +39 -0
- data/lib/igniter/application.rb +174 -0
- data/lib/igniter/capabilities.rb +68 -0
- data/lib/igniter/compiler/validators/dependencies_validator.rb +50 -2
- data/lib/igniter/compiler/validators/remote_validator.rb +2 -0
- data/lib/igniter/consensus/cluster.rb +183 -0
- data/lib/igniter/consensus/errors.rb +14 -0
- data/lib/igniter/consensus/executors.rb +43 -0
- data/lib/igniter/consensus/node.rb +320 -0
- data/lib/igniter/consensus/read_query.rb +30 -0
- data/lib/igniter/consensus/state_machine.rb +58 -0
- data/lib/igniter/consensus.rb +58 -0
- data/lib/igniter/content_addressing.rb +133 -0
- data/lib/igniter/contract.rb +12 -0
- data/lib/igniter/dataflow/aggregate_operators.rb +147 -0
- data/lib/igniter/dataflow/aggregate_state.rb +77 -0
- data/lib/igniter/dataflow/diff.rb +37 -0
- data/lib/igniter/dataflow/diff_state.rb +81 -0
- data/lib/igniter/dataflow/incremental_collection_result.rb +39 -0
- data/lib/igniter/dataflow/window_filter.rb +48 -0
- data/lib/igniter/dataflow.rb +65 -0
- data/lib/igniter/dsl/contract_builder.rb +71 -7
- data/lib/igniter/executor.rb +60 -0
- data/lib/igniter/extensions/capabilities.rb +39 -0
- data/lib/igniter/extensions/content_addressing.rb +5 -0
- data/lib/igniter/extensions/dataflow.rb +117 -0
- data/lib/igniter/extensions/incremental.rb +50 -0
- data/lib/igniter/extensions/mesh.rb +31 -0
- data/lib/igniter/fingerprint.rb +43 -0
- data/lib/igniter/incremental/formatter.rb +81 -0
- data/lib/igniter/incremental/result.rb +69 -0
- data/lib/igniter/incremental/tracker.rb +108 -0
- data/lib/igniter/incremental.rb +50 -0
- data/lib/igniter/integrations/llm/config.rb +48 -4
- data/lib/igniter/integrations/llm/executor.rb +221 -28
- data/lib/igniter/integrations/llm/providers/anthropic.rb +37 -4
- data/lib/igniter/integrations/llm/providers/openai.rb +34 -5
- data/lib/igniter/integrations/llm/transcription/providers/assemblyai.rb +200 -0
- data/lib/igniter/integrations/llm/transcription/providers/base.rb +122 -0
- data/lib/igniter/integrations/llm/transcription/providers/deepgram.rb +162 -0
- data/lib/igniter/integrations/llm/transcription/providers/openai.rb +102 -0
- data/lib/igniter/integrations/llm/transcription/transcriber.rb +145 -0
- data/lib/igniter/integrations/llm/transcription/transcript_result.rb +29 -0
- data/lib/igniter/integrations/llm.rb +37 -1
- data/lib/igniter/memory/agent_memory.rb +104 -0
- data/lib/igniter/memory/episode.rb +29 -0
- data/lib/igniter/memory/fact.rb +27 -0
- data/lib/igniter/memory/memorable.rb +90 -0
- data/lib/igniter/memory/reflection_cycle.rb +96 -0
- data/lib/igniter/memory/reflection_record.rb +28 -0
- data/lib/igniter/memory/store.rb +115 -0
- data/lib/igniter/memory/stores/in_memory.rb +136 -0
- data/lib/igniter/memory/stores/sqlite.rb +284 -0
- data/lib/igniter/memory.rb +80 -0
- data/lib/igniter/mesh/announcer.rb +55 -0
- data/lib/igniter/mesh/config.rb +45 -0
- data/lib/igniter/mesh/discovery.rb +39 -0
- data/lib/igniter/mesh/errors.rb +31 -0
- data/lib/igniter/mesh/gossip.rb +47 -0
- data/lib/igniter/mesh/peer.rb +21 -0
- data/lib/igniter/mesh/peer_registry.rb +51 -0
- data/lib/igniter/mesh/poller.rb +77 -0
- data/lib/igniter/mesh/router.rb +109 -0
- data/lib/igniter/mesh.rb +85 -0
- data/lib/igniter/metrics/collector.rb +131 -0
- data/lib/igniter/metrics/prometheus_exporter.rb +104 -0
- data/lib/igniter/metrics/snapshot.rb +8 -0
- data/lib/igniter/metrics.rb +37 -0
- data/lib/igniter/model/aggregate_node.rb +34 -0
- data/lib/igniter/model/collection_node.rb +3 -2
- data/lib/igniter/model/compute_node.rb +13 -0
- data/lib/igniter/model/remote_node.rb +18 -2
- data/lib/igniter/node_cache.rb +231 -0
- data/lib/igniter/replication/bootstrapper.rb +61 -0
- data/lib/igniter/replication/bootstrappers/gem.rb +32 -0
- data/lib/igniter/replication/bootstrappers/git.rb +39 -0
- data/lib/igniter/replication/bootstrappers/tarball.rb +56 -0
- data/lib/igniter/replication/expansion_plan.rb +38 -0
- data/lib/igniter/replication/expansion_planner.rb +142 -0
- data/lib/igniter/replication/manifest.rb +45 -0
- data/lib/igniter/replication/network_topology.rb +123 -0
- data/lib/igniter/replication/node_role.rb +42 -0
- data/lib/igniter/replication/reflective_replication_agent.rb +238 -0
- data/lib/igniter/replication/replication_agent.rb +87 -0
- data/lib/igniter/replication/role_registry.rb +73 -0
- data/lib/igniter/replication/ssh_session.rb +77 -0
- data/lib/igniter/replication.rb +54 -0
- data/lib/igniter/runtime/cache.rb +35 -6
- data/lib/igniter/runtime/execution.rb +26 -2
- data/lib/igniter/runtime/input_validator.rb +6 -2
- data/lib/igniter/runtime/node_state.rb +7 -2
- data/lib/igniter/runtime/resolver.rb +323 -31
- data/lib/igniter/runtime/stores/redis_store.rb +41 -4
- data/lib/igniter/server/client.rb +44 -1
- data/lib/igniter/server/config.rb +13 -6
- data/lib/igniter/server/handlers/event_handler.rb +4 -0
- data/lib/igniter/server/handlers/execute_handler.rb +6 -0
- data/lib/igniter/server/handlers/liveness_handler.rb +20 -0
- data/lib/igniter/server/handlers/manifest_handler.rb +34 -0
- data/lib/igniter/server/handlers/metrics_handler.rb +51 -0
- data/lib/igniter/server/handlers/peers_handler.rb +115 -0
- data/lib/igniter/server/handlers/readiness_handler.rb +47 -0
- data/lib/igniter/server/http_server.rb +54 -17
- data/lib/igniter/server/router.rb +54 -21
- data/lib/igniter/server/server_logger.rb +52 -0
- data/lib/igniter/server.rb +6 -0
- data/lib/igniter/skill/feedback.rb +116 -0
- data/lib/igniter/skill/output_schema.rb +110 -0
- data/lib/igniter/skill.rb +218 -0
- data/lib/igniter/temporal.rb +84 -0
- data/lib/igniter/tool/discoverable.rb +151 -0
- data/lib/igniter/tool.rb +52 -0
- data/lib/igniter/tool_registry.rb +144 -0
- data/lib/igniter/version.rb +1 -1
- data/lib/igniter.rb +17 -0
- metadata +128 -1
|
@@ -29,6 +29,8 @@ module Igniter
|
|
|
29
29
|
resolve_effect(node)
|
|
30
30
|
when :await
|
|
31
31
|
resolve_await(node)
|
|
32
|
+
when :aggregate
|
|
33
|
+
resolve_aggregate(node)
|
|
32
34
|
when :remote
|
|
33
35
|
resolve_remote(node)
|
|
34
36
|
else
|
|
@@ -65,6 +67,26 @@ module Igniter
|
|
|
65
67
|
NodeState.new(node: node, status: :succeeded, value: @execution.fetch_input!(node.name))
|
|
66
68
|
end
|
|
67
69
|
|
|
70
|
+
def resolve_aggregate(node)
|
|
71
|
+
unless defined?(Igniter::Dataflow)
|
|
72
|
+
raise ResolutionError,
|
|
73
|
+
"Aggregate nodes require the dataflow extension. " \
|
|
74
|
+
"Add: require 'igniter/extensions/dataflow'"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
collection_result = resolve_dependency_value(node.source_collection)
|
|
78
|
+
unless collection_result.respond_to?(:diff)
|
|
79
|
+
raise ResolutionError,
|
|
80
|
+
"Aggregate '#{node.name}' requires an incremental collection. " \
|
|
81
|
+
"Ensure '#{node.source_collection}' uses mode: :incremental"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
agg_state = @execution.aggregate_state_for(node.name)
|
|
85
|
+
agg_state.apply_diff!(collection_result.diff, collection_result)
|
|
86
|
+
|
|
87
|
+
NodeState.new(node: node, status: :succeeded, value: agg_state.value)
|
|
88
|
+
end
|
|
89
|
+
|
|
68
90
|
def resolve_effect(node)
|
|
69
91
|
dependencies = node.dependencies.each_with_object({}) do |dep, memo|
|
|
70
92
|
memo[dep] = resolve_dependency_value(dep)
|
|
@@ -87,17 +109,19 @@ module Igniter
|
|
|
87
109
|
raise PendingDependencyError.new(deferred, "Waiting for external event '#{node.event_name}'")
|
|
88
110
|
end
|
|
89
111
|
|
|
90
|
-
def resolve_remote(node) # rubocop:disable Metrics/MethodLength
|
|
112
|
+
def resolve_remote(node) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
91
113
|
unless defined?(Igniter::Server::Client)
|
|
92
114
|
raise ResolutionError,
|
|
93
115
|
"remote: nodes require `require 'igniter/server'` (server integration not loaded)"
|
|
94
116
|
end
|
|
95
117
|
|
|
118
|
+
url = resolve_remote_url(node)
|
|
119
|
+
|
|
96
120
|
inputs = node.input_mapping.each_with_object({}) do |(child_input, dep_name), memo|
|
|
97
121
|
memo[child_input] = resolve_dependency_value(dep_name)
|
|
98
122
|
end
|
|
99
123
|
|
|
100
|
-
client = Igniter::Server::Client.new(
|
|
124
|
+
client = Igniter::Server::Client.new(url, timeout: node.timeout)
|
|
101
125
|
response = client.execute(node.contract_name, inputs: inputs)
|
|
102
126
|
|
|
103
127
|
case response[:status]
|
|
@@ -106,25 +130,148 @@ module Igniter
|
|
|
106
130
|
when :failed
|
|
107
131
|
error_message = response.dig(:error, :message) || response.dig(:error, "message")
|
|
108
132
|
raise ResolutionError,
|
|
109
|
-
"Remote #{node.contract_name}@#{
|
|
133
|
+
"Remote #{node.contract_name}@#{url}: #{error_message}"
|
|
110
134
|
else
|
|
111
135
|
raise ResolutionError,
|
|
112
|
-
"Remote #{node.contract_name}@#{
|
|
136
|
+
"Remote #{node.contract_name}@#{url}: unexpected status '#{response[:status]}'"
|
|
113
137
|
end
|
|
114
138
|
rescue Igniter::Server::Client::ConnectionError => e
|
|
115
|
-
raise ResolutionError, "Cannot reach #{
|
|
139
|
+
raise ResolutionError, "Cannot reach #{url}: #{e.message}"
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Resolve the target URL for a remote node based on its routing_mode.
|
|
143
|
+
# :static → node_url directly
|
|
144
|
+
# :capability → Mesh::Router selects an alive peer (raises DeferredCapabilityError if none)
|
|
145
|
+
# :pinned → Mesh::Router asserts the peer is alive (raises IncidentError if not)
|
|
146
|
+
def resolve_remote_url(node) # rubocop:disable Metrics/MethodLength
|
|
147
|
+
case node.routing_mode
|
|
148
|
+
when :static
|
|
149
|
+
node.node_url
|
|
150
|
+
when :capability
|
|
151
|
+
unless defined?(Igniter::Mesh)
|
|
152
|
+
raise ResolutionError,
|
|
153
|
+
"remote :#{node.name} uses capability routing — add `require 'igniter/mesh'`"
|
|
154
|
+
end
|
|
155
|
+
deferred = Runtime::DeferredResult.build(
|
|
156
|
+
payload: { capability: node.capability },
|
|
157
|
+
source_node: node.name,
|
|
158
|
+
waiting_on: node.name
|
|
159
|
+
)
|
|
160
|
+
Igniter::Mesh.router.find_peer_for(node.capability, deferred)
|
|
161
|
+
when :pinned
|
|
162
|
+
unless defined?(Igniter::Mesh)
|
|
163
|
+
raise ResolutionError,
|
|
164
|
+
"remote :#{node.name} uses pinned routing — add `require 'igniter/mesh'`"
|
|
165
|
+
end
|
|
166
|
+
Igniter::Mesh.router.resolve_pinned(node.pinned_to)
|
|
167
|
+
end
|
|
116
168
|
end
|
|
117
169
|
|
|
118
|
-
def resolve_compute(node)
|
|
170
|
+
def resolve_compute(node) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
171
|
+
# Capability policy check — raises CapabilityViolationError if denied.
|
|
172
|
+
check_capability_policy!(node)
|
|
173
|
+
|
|
174
|
+
# Running state preserves dep_snapshot + value_version from the stale state.
|
|
175
|
+
# These are used for memoization (skip recompute) and value backdating.
|
|
176
|
+
running_state = @execution.cache.fetch(node.name)
|
|
177
|
+
old_dep_snapshot = running_state&.dep_snapshot
|
|
178
|
+
old_value = running_state&.value
|
|
179
|
+
old_value_version = running_state&.value_version || 0
|
|
180
|
+
|
|
181
|
+
# Resolve all dependencies (may recursively recompute upstream nodes).
|
|
119
182
|
dependencies = node.dependencies.each_with_object({}) do |dependency_name, memo|
|
|
120
183
|
memo[dependency_name] = resolve_dependency_value(dependency_name)
|
|
121
184
|
end
|
|
122
185
|
|
|
186
|
+
# Build snapshot of current dep value_versions (only regular nodes, not outputs).
|
|
187
|
+
current_dep_snapshot = build_dep_snapshot(node)
|
|
188
|
+
|
|
189
|
+
# Memoization: if all dep value_versions are unchanged, skip the compute entirely.
|
|
190
|
+
if old_dep_snapshot && old_value && old_value_version.positive? &&
|
|
191
|
+
dep_snapshot_match?(current_dep_snapshot, old_dep_snapshot)
|
|
192
|
+
@execution.events.emit(:node_skipped, node: node, status: :succeeded,
|
|
193
|
+
payload: { reason: :deps_unchanged })
|
|
194
|
+
return NodeState.new(node: node, status: :succeeded, value: old_value,
|
|
195
|
+
value_version: old_value_version,
|
|
196
|
+
dep_snapshot: current_dep_snapshot)
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Content-addressed cache: pure executor + same dep values → reuse across executions.
|
|
200
|
+
if (content_key = build_content_key(node, dependencies))
|
|
201
|
+
cached_value = Igniter::ContentAddressing.cache.fetch(content_key)
|
|
202
|
+
if cached_value
|
|
203
|
+
@execution.events.emit(:node_content_cache_hit, node: node, status: :succeeded,
|
|
204
|
+
payload: { key: content_key.to_s })
|
|
205
|
+
return NodeState.new(node: node, status: :succeeded, value: cached_value,
|
|
206
|
+
dep_snapshot: current_dep_snapshot)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# TTL cache: any compute node + same dep fingerprint → reuse across executions.
|
|
211
|
+
ttl_key = build_ttl_cache_key(node, dependencies)
|
|
212
|
+
is_coalescing_leader = false
|
|
213
|
+
|
|
214
|
+
if ttl_key
|
|
215
|
+
if (cached_value = Igniter::NodeCache.cache.fetch(ttl_key))
|
|
216
|
+
@execution.events.emit(:node_ttl_cache_hit, node: node, status: :succeeded,
|
|
217
|
+
payload: { key: ttl_key.to_s })
|
|
218
|
+
return NodeState.new(node: node, status: :succeeded, value: cached_value,
|
|
219
|
+
dep_snapshot: current_dep_snapshot)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Coalescing: if another execution is already computing this node for the same
|
|
223
|
+
# inputs, join as a follower instead of duplicating the work.
|
|
224
|
+
if node.coalesce? && (lock = Igniter::NodeCache.coalescing_lock)
|
|
225
|
+
role, flight = lock.acquire(ttl_key.hex)
|
|
226
|
+
if role == :follower
|
|
227
|
+
coalesced_value, coalesced_error = lock.wait(flight)
|
|
228
|
+
raise coalesced_error if coalesced_error
|
|
229
|
+
|
|
230
|
+
# Follower timed out — coalesced_value is nil, fall through to compute independently
|
|
231
|
+
unless coalesced_value.nil? && coalesced_error.nil? && !flight.done
|
|
232
|
+
@execution.events.emit(:node_coalesced, node: node, status: :succeeded,
|
|
233
|
+
payload: { key: ttl_key.to_s })
|
|
234
|
+
return NodeState.new(node: node, status: :succeeded, value: coalesced_value,
|
|
235
|
+
dep_snapshot: current_dep_snapshot)
|
|
236
|
+
end
|
|
237
|
+
else
|
|
238
|
+
is_coalescing_leader = true
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
123
243
|
value = call_compute(node.callable, dependencies)
|
|
124
|
-
|
|
244
|
+
if deferred_result?(value)
|
|
245
|
+
return NodeState.new(node: node, status: :pending,
|
|
246
|
+
value: normalize_deferred_result(value, node))
|
|
247
|
+
end
|
|
248
|
+
|
|
125
249
|
value = normalize_guard_value(node, value)
|
|
126
250
|
|
|
127
|
-
|
|
251
|
+
# Value backdating: if the output is unchanged, preserve value_version so that
|
|
252
|
+
# downstream nodes whose dep_snapshots reference this node won't see it as changed.
|
|
253
|
+
if old_value && old_value_version.positive? && value == old_value
|
|
254
|
+
@execution.events.emit(:node_backdated, node: node, status: :succeeded,
|
|
255
|
+
payload: { reason: :value_unchanged })
|
|
256
|
+
store_ttl_result(ttl_key, value, node, is_coalescing_leader)
|
|
257
|
+
return NodeState.new(node: node, status: :succeeded, value: value,
|
|
258
|
+
value_version: old_value_version,
|
|
259
|
+
dep_snapshot: current_dep_snapshot)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Store in content cache for future executions.
|
|
263
|
+
Igniter::ContentAddressing.cache.store(content_key, value) if content_key
|
|
264
|
+
|
|
265
|
+
# Store in TTL cache and notify any coalescing followers.
|
|
266
|
+
store_ttl_result(ttl_key, value, node, is_coalescing_leader)
|
|
267
|
+
|
|
268
|
+
NodeState.new(node: node, status: :succeeded, value: value,
|
|
269
|
+
dep_snapshot: current_dep_snapshot)
|
|
270
|
+
rescue StandardError => e
|
|
271
|
+
# If this execution was the coalescing leader, notify followers of the failure
|
|
272
|
+
# so they are unblocked (they will re-raise the error through their own path).
|
|
273
|
+
Igniter::NodeCache.coalescing_lock&.finish!(ttl_key&.hex, error: e) if is_coalescing_leader
|
|
274
|
+
raise
|
|
128
275
|
end
|
|
129
276
|
|
|
130
277
|
def call_compute(callable, dependencies)
|
|
@@ -151,11 +298,9 @@ module Igniter
|
|
|
151
298
|
end
|
|
152
299
|
|
|
153
300
|
def call_compute_object(callable, dependencies)
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
raise ResolutionError, "Unsupported callable: #{callable.class}"
|
|
158
|
-
end
|
|
301
|
+
raise ResolutionError, "Unsupported callable: #{callable.class}" unless callable.respond_to?(:call)
|
|
302
|
+
|
|
303
|
+
callable.call(**dependencies)
|
|
159
304
|
end
|
|
160
305
|
|
|
161
306
|
def resolve_composition(node)
|
|
@@ -222,6 +367,8 @@ module Igniter
|
|
|
222
367
|
end
|
|
223
368
|
|
|
224
369
|
def resolve_collection(node)
|
|
370
|
+
return resolve_incremental_collection(node) if node.mode == :incremental
|
|
371
|
+
|
|
225
372
|
items = resolve_dependency_value(node.source_dependency)
|
|
226
373
|
context_values = node.context_dependencies.each_with_object({}) do |dependency_name, memo|
|
|
227
374
|
memo[dependency_name] = resolve_dependency_value(dependency_name)
|
|
@@ -277,17 +424,93 @@ module Igniter
|
|
|
277
424
|
)
|
|
278
425
|
end
|
|
279
426
|
|
|
427
|
+
def resolve_incremental_collection(node) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
428
|
+
unless defined?(Igniter::Dataflow)
|
|
429
|
+
raise ResolutionError.new(
|
|
430
|
+
"Collection '#{node.name}' uses mode: :incremental — " \
|
|
431
|
+
"add `require 'igniter/extensions/dataflow'` to activate it",
|
|
432
|
+
context: collection_context(node)
|
|
433
|
+
)
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
items = resolve_dependency_value(node.source_dependency)
|
|
437
|
+
context_values = node.context_dependencies.each_with_object({}) do |dep_name, memo|
|
|
438
|
+
memo[dep_name] = resolve_dependency_value(dep_name)
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
normalized_items = normalize_collection_items(node, items, context_values)
|
|
442
|
+
normalized_items = Igniter::Dataflow::WindowFilter.new(node.window).apply(normalized_items) if node.window
|
|
443
|
+
|
|
444
|
+
diff_state = @execution.diff_state_for(node.name)
|
|
445
|
+
key_fn = ->(item) { extract_collection_key(node, item) }
|
|
446
|
+
diff = diff_state.compute_diff(normalized_items, key_fn)
|
|
447
|
+
|
|
448
|
+
collection_items = {}
|
|
449
|
+
|
|
450
|
+
# Reuse cached results for unchanged items (no child contract re-run)
|
|
451
|
+
diff.unchanged.each do |key|
|
|
452
|
+
cached = diff_state.cached_item_for(key)
|
|
453
|
+
collection_items[key] = cached if cached
|
|
454
|
+
emit_collection_item_event(:collection_item_reused, node, key)
|
|
455
|
+
end
|
|
456
|
+
|
|
457
|
+
# Retract removed items from the diff state
|
|
458
|
+
diff.removed.each { |key| diff_state.retract!(key) }
|
|
459
|
+
|
|
460
|
+
# Run child contracts only for added + changed items
|
|
461
|
+
to_process = normalized_items.select { |item| diff.added.include?(key_fn.call(item)) || diff.changed.include?(key_fn.call(item)) }
|
|
462
|
+
|
|
463
|
+
to_process.each do |item_inputs|
|
|
464
|
+
item_key = key_fn.call(item_inputs)
|
|
465
|
+
emit_collection_item_event(:collection_item_started, node, item_key, item_inputs: item_inputs)
|
|
466
|
+
child_contract = node.contract_class.new(item_inputs)
|
|
467
|
+
begin
|
|
468
|
+
child_contract.resolve_all
|
|
469
|
+
rescue Igniter::Error
|
|
470
|
+
nil
|
|
471
|
+
end
|
|
472
|
+
child_error = child_contract.execution.cache.values.find(&:failed?)&.error
|
|
473
|
+
|
|
474
|
+
result_item = if child_error
|
|
475
|
+
emit_collection_item_event(:collection_item_failed, node, item_key, error: child_error.message, error_type: child_error.class.name, child_execution_id: child_contract.execution.events.execution_id)
|
|
476
|
+
Runtime::CollectionResult::Item.new(key: item_key, status: :failed, error: child_error)
|
|
477
|
+
else
|
|
478
|
+
emit_collection_item_event(:collection_item_succeeded, node, item_key, child_execution_id: child_contract.execution.events.execution_id)
|
|
479
|
+
Runtime::CollectionResult::Item.new(key: item_key, status: :succeeded, result: child_contract.result)
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
collection_items[item_key] = result_item
|
|
483
|
+
diff_state.update!(item_key, item_inputs, result_item)
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
# Preserve the input array ordering in the result
|
|
487
|
+
ordered_items = normalized_items.each_with_object({}) do |item_inputs, memo|
|
|
488
|
+
key = key_fn.call(item_inputs)
|
|
489
|
+
memo[key] = collection_items[key] if collection_items.key?(key)
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
NodeState.new(
|
|
493
|
+
node: node,
|
|
494
|
+
status: :succeeded,
|
|
495
|
+
value: Igniter::Dataflow::IncrementalCollectionResult.new(items: ordered_items, diff: diff)
|
|
496
|
+
)
|
|
497
|
+
end
|
|
498
|
+
|
|
280
499
|
def resolve_dependency_value(dependency_name)
|
|
281
500
|
if @execution.compiled_graph.node?(dependency_name)
|
|
282
501
|
dependency_state = resolve(dependency_name)
|
|
283
502
|
raise dependency_state.error if dependency_state.failed?
|
|
284
|
-
|
|
503
|
+
|
|
504
|
+
if dependency_state.pending?
|
|
505
|
+
raise PendingDependencyError.new(dependency_state.value,
|
|
506
|
+
context: pending_context(dependency_state.node))
|
|
507
|
+
end
|
|
285
508
|
|
|
286
509
|
dependency_state.value
|
|
287
510
|
elsif @execution.compiled_graph.outputs_by_name.key?(dependency_name.to_sym)
|
|
288
511
|
output = @execution.compiled_graph.fetch_output(dependency_name)
|
|
289
512
|
value = @execution.send(:resolve_exported_output, output)
|
|
290
|
-
raise PendingDependencyError
|
|
513
|
+
raise PendingDependencyError, value if deferred_result?(value)
|
|
291
514
|
|
|
292
515
|
value
|
|
293
516
|
else
|
|
@@ -357,21 +580,89 @@ module Igniter
|
|
|
357
580
|
end
|
|
358
581
|
|
|
359
582
|
def normalize_error(error, node)
|
|
360
|
-
|
|
583
|
+
# Trust any Igniter::Error that already carries node context.
|
|
584
|
+
return error if error.is_a?(Igniter::Error) && error.node_name
|
|
361
585
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
586
|
+
# Domain-specific subclasses (IncidentError, DeferredCapabilityError,
|
|
587
|
+
# InvariantError, …) carry semantics the caller depends on — preserve
|
|
588
|
+
# their type unchanged. Only bare Igniter::ResolutionError instances
|
|
589
|
+
# (raised with just a message inside an executor) get enriched.
|
|
590
|
+
return error if error.is_a?(Igniter::Error) && !error.instance_of?(Igniter::ResolutionError)
|
|
591
|
+
|
|
592
|
+
node_context = {
|
|
593
|
+
graph: @execution.compiled_graph.name,
|
|
594
|
+
node_id: node.id,
|
|
595
|
+
node_name: node.name,
|
|
596
|
+
node_path: node.path,
|
|
597
|
+
source_location: node.source_location,
|
|
598
|
+
execution_id: @execution.events.execution_id
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
ResolutionError.new(error.message, context: node_context)
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
# ─── Capabilities ──────────────────────────────────────────────────────────
|
|
605
|
+
|
|
606
|
+
def check_capability_policy!(node)
|
|
607
|
+
return unless defined?(Igniter::Capabilities) && Igniter::Capabilities.policy
|
|
608
|
+
return unless node.callable.is_a?(Class) && node.callable <= Igniter::Executor
|
|
609
|
+
|
|
610
|
+
Igniter::Capabilities.policy.check!(node.name, node.callable)
|
|
611
|
+
end
|
|
612
|
+
|
|
613
|
+
# ─── Content addressing ────────────────────────────────────────────────────
|
|
614
|
+
|
|
615
|
+
# Returns a ContentKey for pure executors when content addressing is loaded.
|
|
616
|
+
# Returns nil for non-pure executors, Procs, or when the extension is absent.
|
|
617
|
+
def build_content_key(node, dep_values)
|
|
618
|
+
return unless defined?(Igniter::ContentAddressing)
|
|
619
|
+
return unless node.callable.is_a?(Class) && node.callable <= Igniter::Executor
|
|
620
|
+
return unless node.callable.pure?
|
|
621
|
+
|
|
622
|
+
Igniter::ContentAddressing::ContentKey.compute(node.callable, dep_values)
|
|
623
|
+
end
|
|
624
|
+
|
|
625
|
+
# ─── TTL cache ─────────────────────────────────────────────────────────────
|
|
626
|
+
|
|
627
|
+
# Returns a NodeCache::CacheKey when TTL caching is active for this node.
|
|
628
|
+
# Returns nil when NodeCache is not loaded, no backend is configured,
|
|
629
|
+
# or the node has no cache_ttl declared.
|
|
630
|
+
def build_ttl_cache_key(node, dep_values)
|
|
631
|
+
return unless defined?(Igniter::NodeCache)
|
|
632
|
+
return unless Igniter::NodeCache.cache
|
|
633
|
+
return unless node.respond_to?(:cache_ttl) && node.cache_ttl
|
|
634
|
+
|
|
635
|
+
dep_hex = Igniter::NodeCache::Fingerprinter.call(dep_values)
|
|
636
|
+
Igniter::NodeCache::CacheKey.new(
|
|
637
|
+
@execution.compiled_graph.name,
|
|
638
|
+
node.name,
|
|
639
|
+
dep_hex
|
|
372
640
|
)
|
|
373
641
|
end
|
|
374
642
|
|
|
643
|
+
# Stores a computed value in the TTL cache and signals any coalescing followers.
|
|
644
|
+
def store_ttl_result(ttl_key, value, node, is_leader)
|
|
645
|
+
return unless ttl_key
|
|
646
|
+
|
|
647
|
+
Igniter::NodeCache.cache.store(ttl_key, value, ttl: node.cache_ttl)
|
|
648
|
+
Igniter::NodeCache.coalescing_lock&.finish!(ttl_key.hex, value: value) if is_leader
|
|
649
|
+
end
|
|
650
|
+
|
|
651
|
+
def build_dep_snapshot(node)
|
|
652
|
+
node.dependencies.each_with_object({}) do |dep_name, memo|
|
|
653
|
+
next unless @execution.compiled_graph.node?(dep_name)
|
|
654
|
+
|
|
655
|
+
dep_state = @execution.cache.fetch(dep_name.to_sym)
|
|
656
|
+
memo[dep_name] = dep_state&.value_version
|
|
657
|
+
end
|
|
658
|
+
end
|
|
659
|
+
|
|
660
|
+
def dep_snapshot_match?(current, old)
|
|
661
|
+
return false if current.size != old.size
|
|
662
|
+
|
|
663
|
+
current.all? { |name, vv| old[name] == vv }
|
|
664
|
+
end
|
|
665
|
+
|
|
375
666
|
def normalize_guard_value(node, value)
|
|
376
667
|
return value unless node.respond_to?(:guard?) && node.guard?
|
|
377
668
|
return true if value
|
|
@@ -389,9 +680,7 @@ module Igniter
|
|
|
389
680
|
end
|
|
390
681
|
|
|
391
682
|
def normalize_collection_items(node, items, context_values = {})
|
|
392
|
-
if node.input_mapper? && items.is_a?(Hash)
|
|
393
|
-
items = items.to_a
|
|
394
|
-
end
|
|
683
|
+
items = items.to_a if node.input_mapper? && items.is_a?(Hash)
|
|
395
684
|
|
|
396
685
|
unless items.is_a?(Array)
|
|
397
686
|
raise CollectionInputError.new(
|
|
@@ -440,14 +729,17 @@ module Igniter
|
|
|
440
729
|
|
|
441
730
|
def ensure_unique_collection_keys!(node, items)
|
|
442
731
|
keys = items.map do |item|
|
|
443
|
-
item.fetch(node.key_name)
|
|
732
|
+
item.fetch(node.key_name) do
|
|
733
|
+
raise CollectionKeyError.new("Collection '#{node.name}' item is missing key '#{node.key_name}'",
|
|
734
|
+
context: collection_context(node))
|
|
735
|
+
end
|
|
444
736
|
end
|
|
445
737
|
|
|
446
738
|
duplicates = keys.group_by(&:itself).select { |_key, entries| entries.size > 1 }.keys
|
|
447
739
|
return if duplicates.empty?
|
|
448
740
|
|
|
449
741
|
raise CollectionKeyError.new(
|
|
450
|
-
"Collection '#{node.name}' has duplicate keys: #{duplicates.join(
|
|
742
|
+
"Collection '#{node.name}' has duplicate keys: #{duplicates.join(", ")}",
|
|
451
743
|
context: collection_context(node)
|
|
452
744
|
)
|
|
453
745
|
end
|
|
@@ -11,22 +11,46 @@ module Igniter
|
|
|
11
11
|
@namespace = namespace
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
# Save a snapshot. Maintains secondary indexes:
|
|
15
|
+
# {namespace}:all — set of all execution_ids
|
|
16
|
+
# {namespace}:graph:{name} — set of execution_ids for a given graph
|
|
17
|
+
# {namespace}:corr:{graph} — hash of JSON(sorted_correlation) → execution_id
|
|
18
|
+
def save(snapshot, correlation: nil, graph: nil) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
15
19
|
execution_id = snapshot[:execution_id] || snapshot["execution_id"]
|
|
16
20
|
@redis.set(redis_key(execution_id), JSON.generate(snapshot))
|
|
21
|
+
@redis.sadd(all_key, execution_id)
|
|
22
|
+
|
|
23
|
+
if graph
|
|
24
|
+
@redis.sadd(graph_key(graph), execution_id)
|
|
25
|
+
if correlation && !correlation.empty?
|
|
26
|
+
corr_json = JSON.generate(correlation.transform_keys(&:to_s).sort.to_h)
|
|
27
|
+
@redis.hset(correlation_key(graph), corr_json, execution_id)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
17
31
|
execution_id
|
|
18
32
|
end
|
|
19
33
|
|
|
20
34
|
def find_by_correlation(graph:, correlation:)
|
|
21
|
-
|
|
35
|
+
normalized = correlation.transform_keys(&:to_s).sort.to_h
|
|
36
|
+
@redis.hget(correlation_key(graph), JSON.generate(normalized))
|
|
22
37
|
end
|
|
23
38
|
|
|
24
39
|
def list_all(graph: nil)
|
|
25
|
-
|
|
40
|
+
@redis.smembers(graph ? graph_key(graph) : all_key)
|
|
26
41
|
end
|
|
27
42
|
|
|
28
43
|
def list_pending(graph: nil)
|
|
29
|
-
|
|
44
|
+
list_all(graph: graph).select do |id|
|
|
45
|
+
payload = @redis.get(redis_key(id))
|
|
46
|
+
next false unless payload
|
|
47
|
+
|
|
48
|
+
snapshot = JSON.parse(payload)
|
|
49
|
+
states = snapshot["states"] || {}
|
|
50
|
+
states.any? { |_name, state| state["status"].to_s == "pending" }
|
|
51
|
+
rescue StandardError
|
|
52
|
+
false
|
|
53
|
+
end
|
|
30
54
|
end
|
|
31
55
|
|
|
32
56
|
def fetch(execution_id)
|
|
@@ -38,6 +62,7 @@ module Igniter
|
|
|
38
62
|
|
|
39
63
|
def delete(execution_id)
|
|
40
64
|
@redis.del(redis_key(execution_id))
|
|
65
|
+
@redis.srem(all_key, execution_id)
|
|
41
66
|
end
|
|
42
67
|
|
|
43
68
|
def exist?(execution_id)
|
|
@@ -50,6 +75,18 @@ module Igniter
|
|
|
50
75
|
def redis_key(execution_id)
|
|
51
76
|
"#{@namespace}:#{execution_id}"
|
|
52
77
|
end
|
|
78
|
+
|
|
79
|
+
def all_key
|
|
80
|
+
"#{@namespace}:all"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def graph_key(graph)
|
|
84
|
+
"#{@namespace}:graph:#{graph}"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def correlation_key(graph)
|
|
88
|
+
"#{@namespace}:corr:#{graph}"
|
|
89
|
+
end
|
|
53
90
|
end
|
|
54
91
|
end
|
|
55
92
|
end
|
|
@@ -8,7 +8,7 @@ module Igniter
|
|
|
8
8
|
module Server
|
|
9
9
|
# HTTP client for calling remote igniter-server nodes.
|
|
10
10
|
# Uses only stdlib (Net::HTTP + JSON), no external gems required.
|
|
11
|
-
class Client
|
|
11
|
+
class Client # rubocop:disable Metrics/ClassLength
|
|
12
12
|
class Error < Igniter::Server::Error; end
|
|
13
13
|
class ConnectionError < Error; end
|
|
14
14
|
class RemoteError < Error; end
|
|
@@ -51,6 +51,40 @@ module Igniter
|
|
|
51
51
|
get("/v1/health")
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
+
# Fetch peer manifest: peer_name, capabilities, contracts, url.
|
|
55
|
+
def manifest
|
|
56
|
+
response = get("/v1/manifest")
|
|
57
|
+
{
|
|
58
|
+
peer_name: response["peer_name"],
|
|
59
|
+
capabilities: (response["capabilities"] || []).map(&:to_sym),
|
|
60
|
+
contracts: response["contracts"] || [],
|
|
61
|
+
url: response["url"]
|
|
62
|
+
}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Fetch the list of all peers known to the remote node.
|
|
66
|
+
# Returns an Array of hashes with :name, :url, :capabilities (Array<Symbol>).
|
|
67
|
+
def list_peers
|
|
68
|
+
Array(get("/v1/mesh/peers")).map do |p|
|
|
69
|
+
{
|
|
70
|
+
name: p["name"],
|
|
71
|
+
url: p["url"],
|
|
72
|
+
capabilities: Array(p["capabilities"]).map(&:to_sym)
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Register this node as a peer on the remote node.
|
|
78
|
+
def register_peer(name:, url:, capabilities: [])
|
|
79
|
+
post("/v1/mesh/peers",
|
|
80
|
+
{ "name" => name, "url" => url, "capabilities" => capabilities.map(&:to_s) })
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Remove a peer registration from the remote node. Best-effort.
|
|
84
|
+
def unregister_peer(name)
|
|
85
|
+
delete_request("/v1/mesh/peers/#{uri_encode(name)}")
|
|
86
|
+
end
|
|
87
|
+
|
|
54
88
|
private
|
|
55
89
|
|
|
56
90
|
def post(path, body)
|
|
@@ -72,6 +106,15 @@ module Igniter
|
|
|
72
106
|
raise ConnectionError, "Cannot connect to #{@base_url}: #{e.message}"
|
|
73
107
|
end
|
|
74
108
|
|
|
109
|
+
def delete_request(path)
|
|
110
|
+
uri = build_uri(path)
|
|
111
|
+
http = build_http(uri)
|
|
112
|
+
req = Net::HTTP::Delete.new(uri.path, json_headers)
|
|
113
|
+
parse_response(http.request(req))
|
|
114
|
+
rescue Errno::ECONNREFUSED, Errno::EADDRNOTAVAIL, SocketError, Net::OpenTimeout => e
|
|
115
|
+
raise ConnectionError, "Cannot connect to #{@base_url}: #{e.message}"
|
|
116
|
+
end
|
|
117
|
+
|
|
75
118
|
def build_uri(path)
|
|
76
119
|
URI.parse("#{@base_url}#{path}")
|
|
77
120
|
end
|
|
@@ -3,15 +3,22 @@
|
|
|
3
3
|
module Igniter
|
|
4
4
|
module Server
|
|
5
5
|
class Config
|
|
6
|
-
attr_accessor :host, :port, :store, :logger
|
|
6
|
+
attr_accessor :host, :port, :store, :logger,
|
|
7
|
+
:metrics_collector, :log_format, :drain_timeout,
|
|
8
|
+
:peer_name, :peer_capabilities
|
|
7
9
|
attr_reader :registry
|
|
8
10
|
|
|
9
11
|
def initialize
|
|
10
|
-
@host
|
|
11
|
-
@port
|
|
12
|
-
@store
|
|
13
|
-
@registry
|
|
14
|
-
@logger
|
|
12
|
+
@host = "0.0.0.0"
|
|
13
|
+
@port = 4567
|
|
14
|
+
@store = Igniter::Runtime::Stores::MemoryStore.new
|
|
15
|
+
@registry = Registry.new
|
|
16
|
+
@logger = nil
|
|
17
|
+
@metrics_collector = nil
|
|
18
|
+
@log_format = :text
|
|
19
|
+
@drain_timeout = 30
|
|
20
|
+
@peer_name = nil
|
|
21
|
+
@peer_capabilities = []
|
|
15
22
|
end
|
|
16
23
|
|
|
17
24
|
def register(name, contract_class)
|
|
@@ -4,6 +4,10 @@ module Igniter
|
|
|
4
4
|
module Server
|
|
5
5
|
module Handlers
|
|
6
6
|
class EventHandler < Base
|
|
7
|
+
def initialize(registry, store, collector: nil) # rubocop:disable Lint/UnusedMethodArgument
|
|
8
|
+
super(registry, store)
|
|
9
|
+
end
|
|
10
|
+
|
|
7
11
|
private
|
|
8
12
|
|
|
9
13
|
def handle(params:, body:) # rubocop:disable Metrics/MethodLength
|
|
@@ -4,6 +4,11 @@ module Igniter
|
|
|
4
4
|
module Server
|
|
5
5
|
module Handlers
|
|
6
6
|
class ExecuteHandler < Base
|
|
7
|
+
def initialize(registry, store, collector: nil)
|
|
8
|
+
super(registry, store)
|
|
9
|
+
@collector = collector
|
|
10
|
+
end
|
|
11
|
+
|
|
7
12
|
private
|
|
8
13
|
|
|
9
14
|
def handle(params:, body:)
|
|
@@ -18,6 +23,7 @@ module Igniter
|
|
|
18
23
|
contract_class.start(inputs, store: @store)
|
|
19
24
|
else
|
|
20
25
|
contract = contract_class.new(inputs)
|
|
26
|
+
contract.execution.events.subscribe(@collector) if @collector
|
|
21
27
|
begin
|
|
22
28
|
contract.resolve_all
|
|
23
29
|
rescue Igniter::Error
|