igniter 0.4.5 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +217 -0
- data/docs/APPLICATION_V1.md +253 -0
- data/docs/CAPABILITIES_V1.md +207 -0
- data/docs/CONSENSUS_V1.md +477 -0
- data/docs/CONTENT_ADDRESSING_V1.md +221 -0
- data/docs/DATAFLOW_V1.md +274 -0
- data/docs/MESH_V1.md +732 -0
- data/docs/NODE_CACHE_V1.md +324 -0
- data/docs/PROACTIVE_AGENTS_V1.md +293 -0
- data/docs/SERVER_V1.md +200 -1
- data/docs/SKILLS_V1.md +213 -0
- data/docs/STORE_ADAPTERS.md +41 -13
- data/docs/TEMPORAL_V1.md +174 -0
- data/docs/TOOLS_V1.md +347 -0
- data/docs/TRANSCRIPTION_V1.md +403 -0
- data/examples/README.md +37 -0
- data/examples/consensus.rb +239 -0
- data/examples/dataflow.rb +308 -0
- data/examples/elocal_webhook.rb +1 -0
- data/examples/llm_tools.rb +237 -0
- data/examples/mesh.rb +239 -0
- data/examples/mesh_discovery.rb +267 -0
- data/examples/mesh_gossip.rb +162 -0
- data/examples/ringcentral_routing.rb +1 -1
- data/lib/igniter/agents/ai/alert_agent.rb +111 -0
- data/lib/igniter/agents/ai/chain_agent.rb +127 -0
- data/lib/igniter/agents/ai/critic_agent.rb +163 -0
- data/lib/igniter/agents/ai/evaluator_agent.rb +193 -0
- data/lib/igniter/agents/ai/evolution_agent.rb +286 -0
- data/lib/igniter/agents/ai/health_check_agent.rb +122 -0
- data/lib/igniter/agents/ai/observer_agent.rb +184 -0
- data/lib/igniter/agents/ai/planner_agent.rb +210 -0
- data/lib/igniter/agents/ai/router_agent.rb +131 -0
- data/lib/igniter/agents/ai/self_reflection_agent.rb +175 -0
- data/lib/igniter/agents/observability/metrics_agent.rb +130 -0
- data/lib/igniter/agents/pipeline/batch_processor_agent.rb +131 -0
- data/lib/igniter/agents/proactive_agent.rb +208 -0
- data/lib/igniter/agents/reliability/retry_agent.rb +99 -0
- data/lib/igniter/agents/scheduling/cron_agent.rb +110 -0
- data/lib/igniter/agents.rb +56 -0
- data/lib/igniter/application/app_config.rb +32 -0
- data/lib/igniter/application/autoloader.rb +18 -0
- data/lib/igniter/application/generator.rb +157 -0
- data/lib/igniter/application/scheduler.rb +109 -0
- data/lib/igniter/application/yml_loader.rb +39 -0
- data/lib/igniter/application.rb +174 -0
- data/lib/igniter/capabilities.rb +68 -0
- data/lib/igniter/compiler/validators/dependencies_validator.rb +50 -2
- data/lib/igniter/compiler/validators/remote_validator.rb +2 -0
- data/lib/igniter/consensus/cluster.rb +183 -0
- data/lib/igniter/consensus/errors.rb +14 -0
- data/lib/igniter/consensus/executors.rb +43 -0
- data/lib/igniter/consensus/node.rb +320 -0
- data/lib/igniter/consensus/read_query.rb +30 -0
- data/lib/igniter/consensus/state_machine.rb +58 -0
- data/lib/igniter/consensus.rb +58 -0
- data/lib/igniter/content_addressing.rb +133 -0
- data/lib/igniter/contract.rb +12 -0
- data/lib/igniter/dataflow/aggregate_operators.rb +147 -0
- data/lib/igniter/dataflow/aggregate_state.rb +77 -0
- data/lib/igniter/dataflow/diff.rb +37 -0
- data/lib/igniter/dataflow/diff_state.rb +81 -0
- data/lib/igniter/dataflow/incremental_collection_result.rb +39 -0
- data/lib/igniter/dataflow/window_filter.rb +48 -0
- data/lib/igniter/dataflow.rb +65 -0
- data/lib/igniter/dsl/contract_builder.rb +71 -7
- data/lib/igniter/executor.rb +60 -0
- data/lib/igniter/extensions/capabilities.rb +39 -0
- data/lib/igniter/extensions/content_addressing.rb +5 -0
- data/lib/igniter/extensions/dataflow.rb +117 -0
- data/lib/igniter/extensions/mesh.rb +31 -0
- data/lib/igniter/fingerprint.rb +43 -0
- data/lib/igniter/integrations/llm/config.rb +48 -4
- data/lib/igniter/integrations/llm/executor.rb +221 -28
- data/lib/igniter/integrations/llm/providers/anthropic.rb +37 -4
- data/lib/igniter/integrations/llm/providers/openai.rb +34 -5
- data/lib/igniter/integrations/llm/transcription/providers/assemblyai.rb +200 -0
- data/lib/igniter/integrations/llm/transcription/providers/base.rb +122 -0
- data/lib/igniter/integrations/llm/transcription/providers/deepgram.rb +162 -0
- data/lib/igniter/integrations/llm/transcription/providers/openai.rb +102 -0
- data/lib/igniter/integrations/llm/transcription/transcriber.rb +145 -0
- data/lib/igniter/integrations/llm/transcription/transcript_result.rb +29 -0
- data/lib/igniter/integrations/llm.rb +37 -1
- data/lib/igniter/memory/agent_memory.rb +104 -0
- data/lib/igniter/memory/episode.rb +29 -0
- data/lib/igniter/memory/fact.rb +27 -0
- data/lib/igniter/memory/memorable.rb +90 -0
- data/lib/igniter/memory/reflection_cycle.rb +96 -0
- data/lib/igniter/memory/reflection_record.rb +28 -0
- data/lib/igniter/memory/store.rb +115 -0
- data/lib/igniter/memory/stores/in_memory.rb +136 -0
- data/lib/igniter/memory/stores/sqlite.rb +284 -0
- data/lib/igniter/memory.rb +80 -0
- data/lib/igniter/mesh/announcer.rb +55 -0
- data/lib/igniter/mesh/config.rb +45 -0
- data/lib/igniter/mesh/discovery.rb +39 -0
- data/lib/igniter/mesh/errors.rb +31 -0
- data/lib/igniter/mesh/gossip.rb +47 -0
- data/lib/igniter/mesh/peer.rb +21 -0
- data/lib/igniter/mesh/peer_registry.rb +51 -0
- data/lib/igniter/mesh/poller.rb +77 -0
- data/lib/igniter/mesh/router.rb +109 -0
- data/lib/igniter/mesh.rb +85 -0
- data/lib/igniter/metrics/collector.rb +131 -0
- data/lib/igniter/metrics/prometheus_exporter.rb +104 -0
- data/lib/igniter/metrics/snapshot.rb +8 -0
- data/lib/igniter/metrics.rb +37 -0
- data/lib/igniter/model/aggregate_node.rb +34 -0
- data/lib/igniter/model/collection_node.rb +3 -2
- data/lib/igniter/model/compute_node.rb +13 -0
- data/lib/igniter/model/remote_node.rb +18 -2
- data/lib/igniter/node_cache.rb +231 -0
- data/lib/igniter/replication/bootstrapper.rb +61 -0
- data/lib/igniter/replication/bootstrappers/gem.rb +32 -0
- data/lib/igniter/replication/bootstrappers/git.rb +39 -0
- data/lib/igniter/replication/bootstrappers/tarball.rb +56 -0
- data/lib/igniter/replication/expansion_plan.rb +38 -0
- data/lib/igniter/replication/expansion_planner.rb +142 -0
- data/lib/igniter/replication/manifest.rb +45 -0
- data/lib/igniter/replication/network_topology.rb +123 -0
- data/lib/igniter/replication/node_role.rb +42 -0
- data/lib/igniter/replication/reflective_replication_agent.rb +238 -0
- data/lib/igniter/replication/replication_agent.rb +87 -0
- data/lib/igniter/replication/role_registry.rb +73 -0
- data/lib/igniter/replication/ssh_session.rb +77 -0
- data/lib/igniter/replication.rb +54 -0
- data/lib/igniter/runtime/execution.rb +18 -0
- data/lib/igniter/runtime/input_validator.rb +6 -2
- data/lib/igniter/runtime/resolver.rb +254 -16
- data/lib/igniter/runtime/stores/redis_store.rb +41 -4
- data/lib/igniter/server/client.rb +44 -1
- data/lib/igniter/server/config.rb +13 -6
- data/lib/igniter/server/handlers/event_handler.rb +4 -0
- data/lib/igniter/server/handlers/execute_handler.rb +6 -0
- data/lib/igniter/server/handlers/liveness_handler.rb +20 -0
- data/lib/igniter/server/handlers/manifest_handler.rb +34 -0
- data/lib/igniter/server/handlers/metrics_handler.rb +51 -0
- data/lib/igniter/server/handlers/peers_handler.rb +115 -0
- data/lib/igniter/server/handlers/readiness_handler.rb +47 -0
- data/lib/igniter/server/http_server.rb +54 -17
- data/lib/igniter/server/router.rb +54 -21
- data/lib/igniter/server/server_logger.rb +52 -0
- data/lib/igniter/server.rb +6 -0
- data/lib/igniter/skill/feedback.rb +116 -0
- data/lib/igniter/skill/output_schema.rb +110 -0
- data/lib/igniter/skill.rb +218 -0
- data/lib/igniter/temporal.rb +84 -0
- data/lib/igniter/tool/discoverable.rb +151 -0
- data/lib/igniter/tool.rb +52 -0
- data/lib/igniter/tool_registry.rb +144 -0
- data/lib/igniter/version.rb +1 -1
- data/lib/igniter.rb +17 -0
- metadata +122 -1
|
@@ -59,6 +59,24 @@ module Igniter
|
|
|
59
59
|
self
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
+
# Returns the DiffState for an incremental collection node (created on first access).
|
|
63
|
+
# Persists across update_inputs calls for the lifetime of this Execution.
|
|
64
|
+
def diff_state_for(node_name)
|
|
65
|
+
@diff_states ||= {}
|
|
66
|
+
@diff_states[node_name.to_sym] ||= Igniter::Dataflow::DiffState.new
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
# Returns the AggregateState for an aggregate node (created on first access).
|
|
71
|
+
# Persists across update_inputs calls for the lifetime of this Execution.
|
|
72
|
+
def aggregate_state_for(node_name)
|
|
73
|
+
@aggregate_states ||= {}
|
|
74
|
+
@aggregate_states[node_name.to_sym] ||= begin
|
|
75
|
+
node = compiled_graph.fetch_node(node_name)
|
|
76
|
+
Igniter::Dataflow::AggregateState.new(node.operator)
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
62
80
|
def resume(node_name, value:)
|
|
63
81
|
node = compiled_graph.fetch_node(node_name)
|
|
64
82
|
current = cache.fetch(node.name)
|
|
@@ -64,7 +64,8 @@ module Igniter
|
|
|
64
64
|
next if inputs.key?(node.name)
|
|
65
65
|
next unless node.default?
|
|
66
66
|
|
|
67
|
-
|
|
67
|
+
d = node.default
|
|
68
|
+
inputs[node.name] = d.respond_to?(:call) ? d.call : d
|
|
68
69
|
end
|
|
69
70
|
end
|
|
70
71
|
|
|
@@ -77,7 +78,10 @@ module Igniter
|
|
|
77
78
|
end
|
|
78
79
|
|
|
79
80
|
def missing_value!(input_node)
|
|
80
|
-
|
|
81
|
+
if input_node.default?
|
|
82
|
+
d = input_node.default
|
|
83
|
+
return d.respond_to?(:call) ? d.call : d
|
|
84
|
+
end
|
|
81
85
|
return nil unless input_node.required?
|
|
82
86
|
|
|
83
87
|
raise input_error(input_node, "Missing required input: #{input_node.name}")
|
|
@@ -29,6 +29,8 @@ module Igniter
|
|
|
29
29
|
resolve_effect(node)
|
|
30
30
|
when :await
|
|
31
31
|
resolve_await(node)
|
|
32
|
+
when :aggregate
|
|
33
|
+
resolve_aggregate(node)
|
|
32
34
|
when :remote
|
|
33
35
|
resolve_remote(node)
|
|
34
36
|
else
|
|
@@ -65,6 +67,26 @@ module Igniter
|
|
|
65
67
|
NodeState.new(node: node, status: :succeeded, value: @execution.fetch_input!(node.name))
|
|
66
68
|
end
|
|
67
69
|
|
|
70
|
+
def resolve_aggregate(node)
|
|
71
|
+
unless defined?(Igniter::Dataflow)
|
|
72
|
+
raise ResolutionError,
|
|
73
|
+
"Aggregate nodes require the dataflow extension. " \
|
|
74
|
+
"Add: require 'igniter/extensions/dataflow'"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
collection_result = resolve_dependency_value(node.source_collection)
|
|
78
|
+
unless collection_result.respond_to?(:diff)
|
|
79
|
+
raise ResolutionError,
|
|
80
|
+
"Aggregate '#{node.name}' requires an incremental collection. " \
|
|
81
|
+
"Ensure '#{node.source_collection}' uses mode: :incremental"
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
agg_state = @execution.aggregate_state_for(node.name)
|
|
85
|
+
agg_state.apply_diff!(collection_result.diff, collection_result)
|
|
86
|
+
|
|
87
|
+
NodeState.new(node: node, status: :succeeded, value: agg_state.value)
|
|
88
|
+
end
|
|
89
|
+
|
|
68
90
|
def resolve_effect(node)
|
|
69
91
|
dependencies = node.dependencies.each_with_object({}) do |dep, memo|
|
|
70
92
|
memo[dep] = resolve_dependency_value(dep)
|
|
@@ -87,17 +109,19 @@ module Igniter
|
|
|
87
109
|
raise PendingDependencyError.new(deferred, "Waiting for external event '#{node.event_name}'")
|
|
88
110
|
end
|
|
89
111
|
|
|
90
|
-
def resolve_remote(node) # rubocop:disable Metrics/MethodLength
|
|
112
|
+
def resolve_remote(node) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
91
113
|
unless defined?(Igniter::Server::Client)
|
|
92
114
|
raise ResolutionError,
|
|
93
115
|
"remote: nodes require `require 'igniter/server'` (server integration not loaded)"
|
|
94
116
|
end
|
|
95
117
|
|
|
118
|
+
url = resolve_remote_url(node)
|
|
119
|
+
|
|
96
120
|
inputs = node.input_mapping.each_with_object({}) do |(child_input, dep_name), memo|
|
|
97
121
|
memo[child_input] = resolve_dependency_value(dep_name)
|
|
98
122
|
end
|
|
99
123
|
|
|
100
|
-
client = Igniter::Server::Client.new(
|
|
124
|
+
client = Igniter::Server::Client.new(url, timeout: node.timeout)
|
|
101
125
|
response = client.execute(node.contract_name, inputs: inputs)
|
|
102
126
|
|
|
103
127
|
case response[:status]
|
|
@@ -106,16 +130,47 @@ module Igniter
|
|
|
106
130
|
when :failed
|
|
107
131
|
error_message = response.dig(:error, :message) || response.dig(:error, "message")
|
|
108
132
|
raise ResolutionError,
|
|
109
|
-
"Remote #{node.contract_name}@#{
|
|
133
|
+
"Remote #{node.contract_name}@#{url}: #{error_message}"
|
|
110
134
|
else
|
|
111
135
|
raise ResolutionError,
|
|
112
|
-
"Remote #{node.contract_name}@#{
|
|
136
|
+
"Remote #{node.contract_name}@#{url}: unexpected status '#{response[:status]}'"
|
|
113
137
|
end
|
|
114
138
|
rescue Igniter::Server::Client::ConnectionError => e
|
|
115
|
-
raise ResolutionError, "Cannot reach #{
|
|
139
|
+
raise ResolutionError, "Cannot reach #{url}: #{e.message}"
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Resolve the target URL for a remote node based on its routing_mode.
|
|
143
|
+
# :static → node_url directly
|
|
144
|
+
# :capability → Mesh::Router selects an alive peer (raises DeferredCapabilityError if none)
|
|
145
|
+
# :pinned → Mesh::Router asserts the peer is alive (raises IncidentError if not)
|
|
146
|
+
def resolve_remote_url(node) # rubocop:disable Metrics/MethodLength
|
|
147
|
+
case node.routing_mode
|
|
148
|
+
when :static
|
|
149
|
+
node.node_url
|
|
150
|
+
when :capability
|
|
151
|
+
unless defined?(Igniter::Mesh)
|
|
152
|
+
raise ResolutionError,
|
|
153
|
+
"remote :#{node.name} uses capability routing — add `require 'igniter/mesh'`"
|
|
154
|
+
end
|
|
155
|
+
deferred = Runtime::DeferredResult.build(
|
|
156
|
+
payload: { capability: node.capability },
|
|
157
|
+
source_node: node.name,
|
|
158
|
+
waiting_on: node.name
|
|
159
|
+
)
|
|
160
|
+
Igniter::Mesh.router.find_peer_for(node.capability, deferred)
|
|
161
|
+
when :pinned
|
|
162
|
+
unless defined?(Igniter::Mesh)
|
|
163
|
+
raise ResolutionError,
|
|
164
|
+
"remote :#{node.name} uses pinned routing — add `require 'igniter/mesh'`"
|
|
165
|
+
end
|
|
166
|
+
Igniter::Mesh.router.resolve_pinned(node.pinned_to)
|
|
167
|
+
end
|
|
116
168
|
end
|
|
117
169
|
|
|
118
170
|
def resolve_compute(node) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
|
|
171
|
+
# Capability policy check — raises CapabilityViolationError if denied.
|
|
172
|
+
check_capability_policy!(node)
|
|
173
|
+
|
|
119
174
|
# Running state preserves dep_snapshot + value_version from the stale state.
|
|
120
175
|
# These are used for memoization (skip recompute) and value backdating.
|
|
121
176
|
running_state = @execution.cache.fetch(node.name)
|
|
@@ -141,6 +196,50 @@ module Igniter
|
|
|
141
196
|
dep_snapshot: current_dep_snapshot)
|
|
142
197
|
end
|
|
143
198
|
|
|
199
|
+
# Content-addressed cache: pure executor + same dep values → reuse across executions.
|
|
200
|
+
if (content_key = build_content_key(node, dependencies))
|
|
201
|
+
cached_value = Igniter::ContentAddressing.cache.fetch(content_key)
|
|
202
|
+
if cached_value
|
|
203
|
+
@execution.events.emit(:node_content_cache_hit, node: node, status: :succeeded,
|
|
204
|
+
payload: { key: content_key.to_s })
|
|
205
|
+
return NodeState.new(node: node, status: :succeeded, value: cached_value,
|
|
206
|
+
dep_snapshot: current_dep_snapshot)
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# TTL cache: any compute node + same dep fingerprint → reuse across executions.
|
|
211
|
+
ttl_key = build_ttl_cache_key(node, dependencies)
|
|
212
|
+
is_coalescing_leader = false
|
|
213
|
+
|
|
214
|
+
if ttl_key
|
|
215
|
+
if (cached_value = Igniter::NodeCache.cache.fetch(ttl_key))
|
|
216
|
+
@execution.events.emit(:node_ttl_cache_hit, node: node, status: :succeeded,
|
|
217
|
+
payload: { key: ttl_key.to_s })
|
|
218
|
+
return NodeState.new(node: node, status: :succeeded, value: cached_value,
|
|
219
|
+
dep_snapshot: current_dep_snapshot)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Coalescing: if another execution is already computing this node for the same
|
|
223
|
+
# inputs, join as a follower instead of duplicating the work.
|
|
224
|
+
if node.coalesce? && (lock = Igniter::NodeCache.coalescing_lock)
|
|
225
|
+
role, flight = lock.acquire(ttl_key.hex)
|
|
226
|
+
if role == :follower
|
|
227
|
+
coalesced_value, coalesced_error = lock.wait(flight)
|
|
228
|
+
raise coalesced_error if coalesced_error
|
|
229
|
+
|
|
230
|
+
# Follower timed out — coalesced_value is nil, fall through to compute independently
|
|
231
|
+
unless coalesced_value.nil? && coalesced_error.nil? && !flight.done
|
|
232
|
+
@execution.events.emit(:node_coalesced, node: node, status: :succeeded,
|
|
233
|
+
payload: { key: ttl_key.to_s })
|
|
234
|
+
return NodeState.new(node: node, status: :succeeded, value: coalesced_value,
|
|
235
|
+
dep_snapshot: current_dep_snapshot)
|
|
236
|
+
end
|
|
237
|
+
else
|
|
238
|
+
is_coalescing_leader = true
|
|
239
|
+
end
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
|
|
144
243
|
value = call_compute(node.callable, dependencies)
|
|
145
244
|
if deferred_result?(value)
|
|
146
245
|
return NodeState.new(node: node, status: :pending,
|
|
@@ -154,13 +253,25 @@ module Igniter
|
|
|
154
253
|
if old_value && old_value_version.positive? && value == old_value
|
|
155
254
|
@execution.events.emit(:node_backdated, node: node, status: :succeeded,
|
|
156
255
|
payload: { reason: :value_unchanged })
|
|
256
|
+
store_ttl_result(ttl_key, value, node, is_coalescing_leader)
|
|
157
257
|
return NodeState.new(node: node, status: :succeeded, value: value,
|
|
158
258
|
value_version: old_value_version,
|
|
159
259
|
dep_snapshot: current_dep_snapshot)
|
|
160
260
|
end
|
|
161
261
|
|
|
262
|
+
# Store in content cache for future executions.
|
|
263
|
+
Igniter::ContentAddressing.cache.store(content_key, value) if content_key
|
|
264
|
+
|
|
265
|
+
# Store in TTL cache and notify any coalescing followers.
|
|
266
|
+
store_ttl_result(ttl_key, value, node, is_coalescing_leader)
|
|
267
|
+
|
|
162
268
|
NodeState.new(node: node, status: :succeeded, value: value,
|
|
163
269
|
dep_snapshot: current_dep_snapshot)
|
|
270
|
+
rescue StandardError => e
|
|
271
|
+
# If this execution was the coalescing leader, notify followers of the failure
|
|
272
|
+
# so they are unblocked (they will re-raise the error through their own path).
|
|
273
|
+
Igniter::NodeCache.coalescing_lock&.finish!(ttl_key&.hex, error: e) if is_coalescing_leader
|
|
274
|
+
raise
|
|
164
275
|
end
|
|
165
276
|
|
|
166
277
|
def call_compute(callable, dependencies)
|
|
@@ -256,6 +367,8 @@ module Igniter
|
|
|
256
367
|
end
|
|
257
368
|
|
|
258
369
|
def resolve_collection(node)
|
|
370
|
+
return resolve_incremental_collection(node) if node.mode == :incremental
|
|
371
|
+
|
|
259
372
|
items = resolve_dependency_value(node.source_dependency)
|
|
260
373
|
context_values = node.context_dependencies.each_with_object({}) do |dependency_name, memo|
|
|
261
374
|
memo[dependency_name] = resolve_dependency_value(dependency_name)
|
|
@@ -311,6 +424,78 @@ module Igniter
|
|
|
311
424
|
)
|
|
312
425
|
end
|
|
313
426
|
|
|
427
|
+
def resolve_incremental_collection(node) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
|
|
428
|
+
unless defined?(Igniter::Dataflow)
|
|
429
|
+
raise ResolutionError.new(
|
|
430
|
+
"Collection '#{node.name}' uses mode: :incremental — " \
|
|
431
|
+
"add `require 'igniter/extensions/dataflow'` to activate it",
|
|
432
|
+
context: collection_context(node)
|
|
433
|
+
)
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
items = resolve_dependency_value(node.source_dependency)
|
|
437
|
+
context_values = node.context_dependencies.each_with_object({}) do |dep_name, memo|
|
|
438
|
+
memo[dep_name] = resolve_dependency_value(dep_name)
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
normalized_items = normalize_collection_items(node, items, context_values)
|
|
442
|
+
normalized_items = Igniter::Dataflow::WindowFilter.new(node.window).apply(normalized_items) if node.window
|
|
443
|
+
|
|
444
|
+
diff_state = @execution.diff_state_for(node.name)
|
|
445
|
+
key_fn = ->(item) { extract_collection_key(node, item) }
|
|
446
|
+
diff = diff_state.compute_diff(normalized_items, key_fn)
|
|
447
|
+
|
|
448
|
+
collection_items = {}
|
|
449
|
+
|
|
450
|
+
# Reuse cached results for unchanged items (no child contract re-run)
|
|
451
|
+
diff.unchanged.each do |key|
|
|
452
|
+
cached = diff_state.cached_item_for(key)
|
|
453
|
+
collection_items[key] = cached if cached
|
|
454
|
+
emit_collection_item_event(:collection_item_reused, node, key)
|
|
455
|
+
end
|
|
456
|
+
|
|
457
|
+
# Retract removed items from the diff state
|
|
458
|
+
diff.removed.each { |key| diff_state.retract!(key) }
|
|
459
|
+
|
|
460
|
+
# Run child contracts only for added + changed items
|
|
461
|
+
to_process = normalized_items.select { |item| diff.added.include?(key_fn.call(item)) || diff.changed.include?(key_fn.call(item)) }
|
|
462
|
+
|
|
463
|
+
to_process.each do |item_inputs|
|
|
464
|
+
item_key = key_fn.call(item_inputs)
|
|
465
|
+
emit_collection_item_event(:collection_item_started, node, item_key, item_inputs: item_inputs)
|
|
466
|
+
child_contract = node.contract_class.new(item_inputs)
|
|
467
|
+
begin
|
|
468
|
+
child_contract.resolve_all
|
|
469
|
+
rescue Igniter::Error
|
|
470
|
+
nil
|
|
471
|
+
end
|
|
472
|
+
child_error = child_contract.execution.cache.values.find(&:failed?)&.error
|
|
473
|
+
|
|
474
|
+
result_item = if child_error
|
|
475
|
+
emit_collection_item_event(:collection_item_failed, node, item_key, error: child_error.message, error_type: child_error.class.name, child_execution_id: child_contract.execution.events.execution_id)
|
|
476
|
+
Runtime::CollectionResult::Item.new(key: item_key, status: :failed, error: child_error)
|
|
477
|
+
else
|
|
478
|
+
emit_collection_item_event(:collection_item_succeeded, node, item_key, child_execution_id: child_contract.execution.events.execution_id)
|
|
479
|
+
Runtime::CollectionResult::Item.new(key: item_key, status: :succeeded, result: child_contract.result)
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
collection_items[item_key] = result_item
|
|
483
|
+
diff_state.update!(item_key, item_inputs, result_item)
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
# Preserve the input array ordering in the result
|
|
487
|
+
ordered_items = normalized_items.each_with_object({}) do |item_inputs, memo|
|
|
488
|
+
key = key_fn.call(item_inputs)
|
|
489
|
+
memo[key] = collection_items[key] if collection_items.key?(key)
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
NodeState.new(
|
|
493
|
+
node: node,
|
|
494
|
+
status: :succeeded,
|
|
495
|
+
value: Igniter::Dataflow::IncrementalCollectionResult.new(items: ordered_items, diff: diff)
|
|
496
|
+
)
|
|
497
|
+
end
|
|
498
|
+
|
|
314
499
|
def resolve_dependency_value(dependency_name)
|
|
315
500
|
if @execution.compiled_graph.node?(dependency_name)
|
|
316
501
|
dependency_state = resolve(dependency_name)
|
|
@@ -395,21 +580,74 @@ module Igniter
|
|
|
395
580
|
end
|
|
396
581
|
|
|
397
582
|
def normalize_error(error, node)
|
|
398
|
-
|
|
583
|
+
# Trust any Igniter::Error that already carries node context.
|
|
584
|
+
return error if error.is_a?(Igniter::Error) && error.node_name
|
|
399
585
|
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
586
|
+
# Domain-specific subclasses (IncidentError, DeferredCapabilityError,
|
|
587
|
+
# InvariantError, …) carry semantics the caller depends on — preserve
|
|
588
|
+
# their type unchanged. Only bare Igniter::ResolutionError instances
|
|
589
|
+
# (raised with just a message inside an executor) get enriched.
|
|
590
|
+
return error if error.is_a?(Igniter::Error) && !error.instance_of?(Igniter::ResolutionError)
|
|
591
|
+
|
|
592
|
+
node_context = {
|
|
593
|
+
graph: @execution.compiled_graph.name,
|
|
594
|
+
node_id: node.id,
|
|
595
|
+
node_name: node.name,
|
|
596
|
+
node_path: node.path,
|
|
597
|
+
source_location: node.source_location,
|
|
598
|
+
execution_id: @execution.events.execution_id
|
|
599
|
+
}
|
|
600
|
+
|
|
601
|
+
ResolutionError.new(error.message, context: node_context)
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
# ─── Capabilities ──────────────────────────────────────────────────────────
|
|
605
|
+
|
|
606
|
+
def check_capability_policy!(node)
|
|
607
|
+
return unless defined?(Igniter::Capabilities) && Igniter::Capabilities.policy
|
|
608
|
+
return unless node.callable.is_a?(Class) && node.callable <= Igniter::Executor
|
|
609
|
+
|
|
610
|
+
Igniter::Capabilities.policy.check!(node.name, node.callable)
|
|
611
|
+
end
|
|
612
|
+
|
|
613
|
+
# ─── Content addressing ────────────────────────────────────────────────────
|
|
614
|
+
|
|
615
|
+
# Returns a ContentKey for pure executors when content addressing is loaded.
|
|
616
|
+
# Returns nil for non-pure executors, Procs, or when the extension is absent.
|
|
617
|
+
def build_content_key(node, dep_values)
|
|
618
|
+
return unless defined?(Igniter::ContentAddressing)
|
|
619
|
+
return unless node.callable.is_a?(Class) && node.callable <= Igniter::Executor
|
|
620
|
+
return unless node.callable.pure?
|
|
621
|
+
|
|
622
|
+
Igniter::ContentAddressing::ContentKey.compute(node.callable, dep_values)
|
|
623
|
+
end
|
|
624
|
+
|
|
625
|
+
# ─── TTL cache ─────────────────────────────────────────────────────────────
|
|
626
|
+
|
|
627
|
+
# Returns a NodeCache::CacheKey when TTL caching is active for this node.
|
|
628
|
+
# Returns nil when NodeCache is not loaded, no backend is configured,
|
|
629
|
+
# or the node has no cache_ttl declared.
|
|
630
|
+
def build_ttl_cache_key(node, dep_values)
|
|
631
|
+
return unless defined?(Igniter::NodeCache)
|
|
632
|
+
return unless Igniter::NodeCache.cache
|
|
633
|
+
return unless node.respond_to?(:cache_ttl) && node.cache_ttl
|
|
634
|
+
|
|
635
|
+
dep_hex = Igniter::NodeCache::Fingerprinter.call(dep_values)
|
|
636
|
+
Igniter::NodeCache::CacheKey.new(
|
|
637
|
+
@execution.compiled_graph.name,
|
|
638
|
+
node.name,
|
|
639
|
+
dep_hex
|
|
410
640
|
)
|
|
411
641
|
end
|
|
412
642
|
|
|
643
|
+
# Stores a computed value in the TTL cache and signals any coalescing followers.
|
|
644
|
+
def store_ttl_result(ttl_key, value, node, is_leader)
|
|
645
|
+
return unless ttl_key
|
|
646
|
+
|
|
647
|
+
Igniter::NodeCache.cache.store(ttl_key, value, ttl: node.cache_ttl)
|
|
648
|
+
Igniter::NodeCache.coalescing_lock&.finish!(ttl_key.hex, value: value) if is_leader
|
|
649
|
+
end
|
|
650
|
+
|
|
413
651
|
def build_dep_snapshot(node)
|
|
414
652
|
node.dependencies.each_with_object({}) do |dep_name, memo|
|
|
415
653
|
next unless @execution.compiled_graph.node?(dep_name)
|
|
@@ -11,22 +11,46 @@ module Igniter
|
|
|
11
11
|
@namespace = namespace
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
-
|
|
14
|
+
# Save a snapshot. Maintains secondary indexes:
|
|
15
|
+
# {namespace}:all — set of all execution_ids
|
|
16
|
+
# {namespace}:graph:{name} — set of execution_ids for a given graph
|
|
17
|
+
# {namespace}:corr:{graph} — hash of JSON(sorted_correlation) → execution_id
|
|
18
|
+
def save(snapshot, correlation: nil, graph: nil) # rubocop:disable Metrics/MethodLength,Metrics/AbcSize
|
|
15
19
|
execution_id = snapshot[:execution_id] || snapshot["execution_id"]
|
|
16
20
|
@redis.set(redis_key(execution_id), JSON.generate(snapshot))
|
|
21
|
+
@redis.sadd(all_key, execution_id)
|
|
22
|
+
|
|
23
|
+
if graph
|
|
24
|
+
@redis.sadd(graph_key(graph), execution_id)
|
|
25
|
+
if correlation && !correlation.empty?
|
|
26
|
+
corr_json = JSON.generate(correlation.transform_keys(&:to_s).sort.to_h)
|
|
27
|
+
@redis.hset(correlation_key(graph), corr_json, execution_id)
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
17
31
|
execution_id
|
|
18
32
|
end
|
|
19
33
|
|
|
20
34
|
def find_by_correlation(graph:, correlation:)
|
|
21
|
-
|
|
35
|
+
normalized = correlation.transform_keys(&:to_s).sort.to_h
|
|
36
|
+
@redis.hget(correlation_key(graph), JSON.generate(normalized))
|
|
22
37
|
end
|
|
23
38
|
|
|
24
39
|
def list_all(graph: nil)
|
|
25
|
-
|
|
40
|
+
@redis.smembers(graph ? graph_key(graph) : all_key)
|
|
26
41
|
end
|
|
27
42
|
|
|
28
43
|
def list_pending(graph: nil)
|
|
29
|
-
|
|
44
|
+
list_all(graph: graph).select do |id|
|
|
45
|
+
payload = @redis.get(redis_key(id))
|
|
46
|
+
next false unless payload
|
|
47
|
+
|
|
48
|
+
snapshot = JSON.parse(payload)
|
|
49
|
+
states = snapshot["states"] || {}
|
|
50
|
+
states.any? { |_name, state| state["status"].to_s == "pending" }
|
|
51
|
+
rescue StandardError
|
|
52
|
+
false
|
|
53
|
+
end
|
|
30
54
|
end
|
|
31
55
|
|
|
32
56
|
def fetch(execution_id)
|
|
@@ -38,6 +62,7 @@ module Igniter
|
|
|
38
62
|
|
|
39
63
|
def delete(execution_id)
|
|
40
64
|
@redis.del(redis_key(execution_id))
|
|
65
|
+
@redis.srem(all_key, execution_id)
|
|
41
66
|
end
|
|
42
67
|
|
|
43
68
|
def exist?(execution_id)
|
|
@@ -50,6 +75,18 @@ module Igniter
|
|
|
50
75
|
def redis_key(execution_id)
|
|
51
76
|
"#{@namespace}:#{execution_id}"
|
|
52
77
|
end
|
|
78
|
+
|
|
79
|
+
def all_key
|
|
80
|
+
"#{@namespace}:all"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def graph_key(graph)
|
|
84
|
+
"#{@namespace}:graph:#{graph}"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def correlation_key(graph)
|
|
88
|
+
"#{@namespace}:corr:#{graph}"
|
|
89
|
+
end
|
|
53
90
|
end
|
|
54
91
|
end
|
|
55
92
|
end
|
|
@@ -8,7 +8,7 @@ module Igniter
|
|
|
8
8
|
module Server
|
|
9
9
|
# HTTP client for calling remote igniter-server nodes.
|
|
10
10
|
# Uses only stdlib (Net::HTTP + JSON), no external gems required.
|
|
11
|
-
class Client
|
|
11
|
+
class Client # rubocop:disable Metrics/ClassLength
|
|
12
12
|
class Error < Igniter::Server::Error; end
|
|
13
13
|
class ConnectionError < Error; end
|
|
14
14
|
class RemoteError < Error; end
|
|
@@ -51,6 +51,40 @@ module Igniter
|
|
|
51
51
|
get("/v1/health")
|
|
52
52
|
end
|
|
53
53
|
|
|
54
|
+
# Fetch peer manifest: peer_name, capabilities, contracts, url.
|
|
55
|
+
def manifest
|
|
56
|
+
response = get("/v1/manifest")
|
|
57
|
+
{
|
|
58
|
+
peer_name: response["peer_name"],
|
|
59
|
+
capabilities: (response["capabilities"] || []).map(&:to_sym),
|
|
60
|
+
contracts: response["contracts"] || [],
|
|
61
|
+
url: response["url"]
|
|
62
|
+
}
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Fetch the list of all peers known to the remote node.
|
|
66
|
+
# Returns an Array of hashes with :name, :url, :capabilities (Array<Symbol>).
|
|
67
|
+
def list_peers
|
|
68
|
+
Array(get("/v1/mesh/peers")).map do |p|
|
|
69
|
+
{
|
|
70
|
+
name: p["name"],
|
|
71
|
+
url: p["url"],
|
|
72
|
+
capabilities: Array(p["capabilities"]).map(&:to_sym)
|
|
73
|
+
}
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Register this node as a peer on the remote node.
|
|
78
|
+
def register_peer(name:, url:, capabilities: [])
|
|
79
|
+
post("/v1/mesh/peers",
|
|
80
|
+
{ "name" => name, "url" => url, "capabilities" => capabilities.map(&:to_s) })
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Remove a peer registration from the remote node. Best-effort.
|
|
84
|
+
def unregister_peer(name)
|
|
85
|
+
delete_request("/v1/mesh/peers/#{uri_encode(name)}")
|
|
86
|
+
end
|
|
87
|
+
|
|
54
88
|
private
|
|
55
89
|
|
|
56
90
|
def post(path, body)
|
|
@@ -72,6 +106,15 @@ module Igniter
|
|
|
72
106
|
raise ConnectionError, "Cannot connect to #{@base_url}: #{e.message}"
|
|
73
107
|
end
|
|
74
108
|
|
|
109
|
+
def delete_request(path)
|
|
110
|
+
uri = build_uri(path)
|
|
111
|
+
http = build_http(uri)
|
|
112
|
+
req = Net::HTTP::Delete.new(uri.path, json_headers)
|
|
113
|
+
parse_response(http.request(req))
|
|
114
|
+
rescue Errno::ECONNREFUSED, Errno::EADDRNOTAVAIL, SocketError, Net::OpenTimeout => e
|
|
115
|
+
raise ConnectionError, "Cannot connect to #{@base_url}: #{e.message}"
|
|
116
|
+
end
|
|
117
|
+
|
|
75
118
|
def build_uri(path)
|
|
76
119
|
URI.parse("#{@base_url}#{path}")
|
|
77
120
|
end
|
|
@@ -3,15 +3,22 @@
|
|
|
3
3
|
module Igniter
|
|
4
4
|
module Server
|
|
5
5
|
class Config
|
|
6
|
-
attr_accessor :host, :port, :store, :logger
|
|
6
|
+
attr_accessor :host, :port, :store, :logger,
|
|
7
|
+
:metrics_collector, :log_format, :drain_timeout,
|
|
8
|
+
:peer_name, :peer_capabilities
|
|
7
9
|
attr_reader :registry
|
|
8
10
|
|
|
9
11
|
def initialize
|
|
10
|
-
@host
|
|
11
|
-
@port
|
|
12
|
-
@store
|
|
13
|
-
@registry
|
|
14
|
-
@logger
|
|
12
|
+
@host = "0.0.0.0"
|
|
13
|
+
@port = 4567
|
|
14
|
+
@store = Igniter::Runtime::Stores::MemoryStore.new
|
|
15
|
+
@registry = Registry.new
|
|
16
|
+
@logger = nil
|
|
17
|
+
@metrics_collector = nil
|
|
18
|
+
@log_format = :text
|
|
19
|
+
@drain_timeout = 30
|
|
20
|
+
@peer_name = nil
|
|
21
|
+
@peer_capabilities = []
|
|
15
22
|
end
|
|
16
23
|
|
|
17
24
|
def register(name, contract_class)
|
|
@@ -4,6 +4,10 @@ module Igniter
|
|
|
4
4
|
module Server
|
|
5
5
|
module Handlers
|
|
6
6
|
class EventHandler < Base
|
|
7
|
+
def initialize(registry, store, collector: nil) # rubocop:disable Lint/UnusedMethodArgument
|
|
8
|
+
super(registry, store)
|
|
9
|
+
end
|
|
10
|
+
|
|
7
11
|
private
|
|
8
12
|
|
|
9
13
|
def handle(params:, body:) # rubocop:disable Metrics/MethodLength
|
|
@@ -4,6 +4,11 @@ module Igniter
|
|
|
4
4
|
module Server
|
|
5
5
|
module Handlers
|
|
6
6
|
class ExecuteHandler < Base
|
|
7
|
+
def initialize(registry, store, collector: nil)
|
|
8
|
+
super(registry, store)
|
|
9
|
+
@collector = collector
|
|
10
|
+
end
|
|
11
|
+
|
|
7
12
|
private
|
|
8
13
|
|
|
9
14
|
def handle(params:, body:)
|
|
@@ -18,6 +23,7 @@ module Igniter
|
|
|
18
23
|
contract_class.start(inputs, store: @store)
|
|
19
24
|
else
|
|
20
25
|
contract = contract_class.new(inputs)
|
|
26
|
+
contract.execution.events.subscribe(@collector) if @collector
|
|
21
27
|
begin
|
|
22
28
|
contract.resolve_all
|
|
23
29
|
rescue Igniter::Error
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Igniter
|
|
4
|
+
module Server
|
|
5
|
+
module Handlers
|
|
6
|
+
# GET /v1/live — Kubernetes liveness probe.
|
|
7
|
+
#
|
|
8
|
+
# Returns 200 as long as the process is running. This endpoint should
|
|
9
|
+
# NEVER return a non-200 status unless the process is truly broken (e.g.
|
|
10
|
+
# deadlocked). A failing liveness probe causes K8s to restart the pod.
|
|
11
|
+
class LivenessHandler < Base
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
def handle(params:, body:) # rubocop:disable Lint/UnusedMethodArgument
|
|
15
|
+
json_ok({ status: "alive", pid: Process.pid })
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Igniter
|
|
4
|
+
module Server
|
|
5
|
+
module Handlers
|
|
6
|
+
# Returns a JSON manifest describing this peer: its name, advertised
|
|
7
|
+
# capabilities, registered contracts, and its base URL.
|
|
8
|
+
# Used by Igniter::Mesh::Router health-probing and peer discovery.
|
|
9
|
+
class ManifestHandler < Base
|
|
10
|
+
def initialize(registry, store, config: nil)
|
|
11
|
+
super(registry, store)
|
|
12
|
+
@config = config
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def handle(params:, body:) # rubocop:disable Lint/UnusedMethodArgument
|
|
18
|
+
json_ok({
|
|
19
|
+
peer_name: @config&.peer_name,
|
|
20
|
+
capabilities: (@config&.peer_capabilities || []).map(&:to_s),
|
|
21
|
+
contracts: @registry.names,
|
|
22
|
+
url: node_url
|
|
23
|
+
})
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def node_url
|
|
27
|
+
return nil unless @config
|
|
28
|
+
|
|
29
|
+
"http://#{@config.host}:#{@config.port}"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|