igniter 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +217 -0
- data/docs/APPLICATION_V1.md +253 -0
- data/docs/CAPABILITIES_V1.md +207 -0
- data/docs/CONSENSUS_V1.md +477 -0
- data/docs/CONTENT_ADDRESSING_V1.md +221 -0
- data/docs/DATAFLOW_V1.md +274 -0
- data/docs/MESH_V1.md +732 -0
- data/docs/NODE_CACHE_V1.md +324 -0
- data/docs/PROACTIVE_AGENTS_V1.md +293 -0
- data/docs/SERVER_V1.md +200 -1
- data/docs/SKILLS_V1.md +213 -0
- data/docs/STORE_ADAPTERS.md +41 -13
- data/docs/TEMPORAL_V1.md +174 -0
- data/docs/TOOLS_V1.md +347 -0
- data/docs/TRANSCRIPTION_V1.md +403 -0
- data/examples/README.md +37 -0
- data/examples/consensus.rb +239 -0
- data/examples/dataflow.rb +308 -0
- data/examples/elocal_webhook.rb +1 -0
- data/examples/incremental.rb +142 -0
- data/examples/llm_tools.rb +237 -0
- data/examples/mesh.rb +239 -0
- data/examples/mesh_discovery.rb +267 -0
- data/examples/mesh_gossip.rb +162 -0
- data/examples/ringcentral_routing.rb +1 -1
- data/lib/igniter/agents/ai/alert_agent.rb +111 -0
- data/lib/igniter/agents/ai/chain_agent.rb +127 -0
- data/lib/igniter/agents/ai/critic_agent.rb +163 -0
- data/lib/igniter/agents/ai/evaluator_agent.rb +193 -0
- data/lib/igniter/agents/ai/evolution_agent.rb +286 -0
- data/lib/igniter/agents/ai/health_check_agent.rb +122 -0
- data/lib/igniter/agents/ai/observer_agent.rb +184 -0
- data/lib/igniter/agents/ai/planner_agent.rb +210 -0
- data/lib/igniter/agents/ai/router_agent.rb +131 -0
- data/lib/igniter/agents/ai/self_reflection_agent.rb +175 -0
- data/lib/igniter/agents/observability/metrics_agent.rb +130 -0
- data/lib/igniter/agents/pipeline/batch_processor_agent.rb +131 -0
- data/lib/igniter/agents/proactive_agent.rb +208 -0
- data/lib/igniter/agents/reliability/retry_agent.rb +99 -0
- data/lib/igniter/agents/scheduling/cron_agent.rb +110 -0
- data/lib/igniter/agents.rb +56 -0
- data/lib/igniter/application/app_config.rb +32 -0
- data/lib/igniter/application/autoloader.rb +18 -0
- data/lib/igniter/application/generator.rb +157 -0
- data/lib/igniter/application/scheduler.rb +109 -0
- data/lib/igniter/application/yml_loader.rb +39 -0
- data/lib/igniter/application.rb +174 -0
- data/lib/igniter/capabilities.rb +68 -0
- data/lib/igniter/compiler/validators/dependencies_validator.rb +50 -2
- data/lib/igniter/compiler/validators/remote_validator.rb +2 -0
- data/lib/igniter/consensus/cluster.rb +183 -0
- data/lib/igniter/consensus/errors.rb +14 -0
- data/lib/igniter/consensus/executors.rb +43 -0
- data/lib/igniter/consensus/node.rb +320 -0
- data/lib/igniter/consensus/read_query.rb +30 -0
- data/lib/igniter/consensus/state_machine.rb +58 -0
- data/lib/igniter/consensus.rb +58 -0
- data/lib/igniter/content_addressing.rb +133 -0
- data/lib/igniter/contract.rb +12 -0
- data/lib/igniter/dataflow/aggregate_operators.rb +147 -0
- data/lib/igniter/dataflow/aggregate_state.rb +77 -0
- data/lib/igniter/dataflow/diff.rb +37 -0
- data/lib/igniter/dataflow/diff_state.rb +81 -0
- data/lib/igniter/dataflow/incremental_collection_result.rb +39 -0
- data/lib/igniter/dataflow/window_filter.rb +48 -0
- data/lib/igniter/dataflow.rb +65 -0
- data/lib/igniter/dsl/contract_builder.rb +71 -7
- data/lib/igniter/executor.rb +60 -0
- data/lib/igniter/extensions/capabilities.rb +39 -0
- data/lib/igniter/extensions/content_addressing.rb +5 -0
- data/lib/igniter/extensions/dataflow.rb +117 -0
- data/lib/igniter/extensions/incremental.rb +50 -0
- data/lib/igniter/extensions/mesh.rb +31 -0
- data/lib/igniter/fingerprint.rb +43 -0
- data/lib/igniter/incremental/formatter.rb +81 -0
- data/lib/igniter/incremental/result.rb +69 -0
- data/lib/igniter/incremental/tracker.rb +108 -0
- data/lib/igniter/incremental.rb +50 -0
- data/lib/igniter/integrations/llm/config.rb +48 -4
- data/lib/igniter/integrations/llm/executor.rb +221 -28
- data/lib/igniter/integrations/llm/providers/anthropic.rb +37 -4
- data/lib/igniter/integrations/llm/providers/openai.rb +34 -5
- data/lib/igniter/integrations/llm/transcription/providers/assemblyai.rb +200 -0
- data/lib/igniter/integrations/llm/transcription/providers/base.rb +122 -0
- data/lib/igniter/integrations/llm/transcription/providers/deepgram.rb +162 -0
- data/lib/igniter/integrations/llm/transcription/providers/openai.rb +102 -0
- data/lib/igniter/integrations/llm/transcription/transcriber.rb +145 -0
- data/lib/igniter/integrations/llm/transcription/transcript_result.rb +29 -0
- data/lib/igniter/integrations/llm.rb +37 -1
- data/lib/igniter/memory/agent_memory.rb +104 -0
- data/lib/igniter/memory/episode.rb +29 -0
- data/lib/igniter/memory/fact.rb +27 -0
- data/lib/igniter/memory/memorable.rb +90 -0
- data/lib/igniter/memory/reflection_cycle.rb +96 -0
- data/lib/igniter/memory/reflection_record.rb +28 -0
- data/lib/igniter/memory/store.rb +115 -0
- data/lib/igniter/memory/stores/in_memory.rb +136 -0
- data/lib/igniter/memory/stores/sqlite.rb +284 -0
- data/lib/igniter/memory.rb +80 -0
- data/lib/igniter/mesh/announcer.rb +55 -0
- data/lib/igniter/mesh/config.rb +45 -0
- data/lib/igniter/mesh/discovery.rb +39 -0
- data/lib/igniter/mesh/errors.rb +31 -0
- data/lib/igniter/mesh/gossip.rb +47 -0
- data/lib/igniter/mesh/peer.rb +21 -0
- data/lib/igniter/mesh/peer_registry.rb +51 -0
- data/lib/igniter/mesh/poller.rb +77 -0
- data/lib/igniter/mesh/router.rb +109 -0
- data/lib/igniter/mesh.rb +85 -0
- data/lib/igniter/metrics/collector.rb +131 -0
- data/lib/igniter/metrics/prometheus_exporter.rb +104 -0
- data/lib/igniter/metrics/snapshot.rb +8 -0
- data/lib/igniter/metrics.rb +37 -0
- data/lib/igniter/model/aggregate_node.rb +34 -0
- data/lib/igniter/model/collection_node.rb +3 -2
- data/lib/igniter/model/compute_node.rb +13 -0
- data/lib/igniter/model/remote_node.rb +18 -2
- data/lib/igniter/node_cache.rb +231 -0
- data/lib/igniter/replication/bootstrapper.rb +61 -0
- data/lib/igniter/replication/bootstrappers/gem.rb +32 -0
- data/lib/igniter/replication/bootstrappers/git.rb +39 -0
- data/lib/igniter/replication/bootstrappers/tarball.rb +56 -0
- data/lib/igniter/replication/expansion_plan.rb +38 -0
- data/lib/igniter/replication/expansion_planner.rb +142 -0
- data/lib/igniter/replication/manifest.rb +45 -0
- data/lib/igniter/replication/network_topology.rb +123 -0
- data/lib/igniter/replication/node_role.rb +42 -0
- data/lib/igniter/replication/reflective_replication_agent.rb +238 -0
- data/lib/igniter/replication/replication_agent.rb +87 -0
- data/lib/igniter/replication/role_registry.rb +73 -0
- data/lib/igniter/replication/ssh_session.rb +77 -0
- data/lib/igniter/replication.rb +54 -0
- data/lib/igniter/runtime/cache.rb +35 -6
- data/lib/igniter/runtime/execution.rb +26 -2
- data/lib/igniter/runtime/input_validator.rb +6 -2
- data/lib/igniter/runtime/node_state.rb +7 -2
- data/lib/igniter/runtime/resolver.rb +323 -31
- data/lib/igniter/runtime/stores/redis_store.rb +41 -4
- data/lib/igniter/server/client.rb +44 -1
- data/lib/igniter/server/config.rb +13 -6
- data/lib/igniter/server/handlers/event_handler.rb +4 -0
- data/lib/igniter/server/handlers/execute_handler.rb +6 -0
- data/lib/igniter/server/handlers/liveness_handler.rb +20 -0
- data/lib/igniter/server/handlers/manifest_handler.rb +34 -0
- data/lib/igniter/server/handlers/metrics_handler.rb +51 -0
- data/lib/igniter/server/handlers/peers_handler.rb +115 -0
- data/lib/igniter/server/handlers/readiness_handler.rb +47 -0
- data/lib/igniter/server/http_server.rb +54 -17
- data/lib/igniter/server/router.rb +54 -21
- data/lib/igniter/server/server_logger.rb +52 -0
- data/lib/igniter/server.rb +6 -0
- data/lib/igniter/skill/feedback.rb +116 -0
- data/lib/igniter/skill/output_schema.rb +110 -0
- data/lib/igniter/skill.rb +218 -0
- data/lib/igniter/temporal.rb +84 -0
- data/lib/igniter/tool/discoverable.rb +151 -0
- data/lib/igniter/tool.rb +52 -0
- data/lib/igniter/tool_registry.rb +144 -0
- data/lib/igniter/version.rb +1 -1
- data/lib/igniter.rb +17 -0
- metadata +128 -1
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Igniter
|
|
4
|
+
module Metrics
|
|
5
|
+
# Formats a Collector snapshot into Prometheus text exposition format (0.0.4).
|
|
6
|
+
#
|
|
7
|
+
# https://prometheus.io/docs/instrumenting/exposition_formats/
|
|
8
|
+
#
|
|
9
|
+
# Usage:
|
|
10
|
+
# exporter = PrometheusExporter.new(collector, store: store, registry: registry)
|
|
11
|
+
# text = exporter.export # → String in Prometheus text format
|
|
12
|
+
# content_type = exporter.content_type
|
|
13
|
+
class PrometheusExporter
|
|
14
|
+
CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8"
|
|
15
|
+
|
|
16
|
+
COUNTER_META = {
|
|
17
|
+
"igniter_executions_total" =>
|
|
18
|
+
"Total contract executions completed",
|
|
19
|
+
"igniter_http_requests_total" =>
|
|
20
|
+
"Total HTTP requests received by igniter-server"
|
|
21
|
+
}.freeze
|
|
22
|
+
|
|
23
|
+
HISTOGRAM_META = {
|
|
24
|
+
"igniter_execution_duration_seconds" =>
|
|
25
|
+
"Contract execution duration in seconds",
|
|
26
|
+
"igniter_http_request_duration_seconds" =>
|
|
27
|
+
"HTTP request processing duration in seconds"
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
def initialize(collector, store:, registry:)
|
|
31
|
+
@collector = collector
|
|
32
|
+
@store = store
|
|
33
|
+
@registry = registry
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def content_type
|
|
37
|
+
CONTENT_TYPE
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def export # rubocop:disable Metrics/MethodLength
|
|
41
|
+
snap = @collector.snapshot
|
|
42
|
+
lines = []
|
|
43
|
+
|
|
44
|
+
emit_counters(lines, snap.counters)
|
|
45
|
+
emit_histograms(lines, snap.histograms)
|
|
46
|
+
emit_pending_gauge(lines)
|
|
47
|
+
|
|
48
|
+
lines.join("\n") + "\n"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
private
|
|
52
|
+
|
|
53
|
+
def emit_counters(lines, counters) # rubocop:disable Metrics/MethodLength
|
|
54
|
+
by_metric = counters.each_with_object(Hash.new { |h, k| h[k] = [] }) do |(key, val), memo|
|
|
55
|
+
name = key.split("{").first
|
|
56
|
+
memo[name] << [key, val]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
by_metric.each do |name, entries|
|
|
60
|
+
lines << "# HELP #{name} #{COUNTER_META.fetch(name, name)}"
|
|
61
|
+
lines << "# TYPE #{name} counter"
|
|
62
|
+
entries.each { |key, val| lines << "#{key} #{val}" }
|
|
63
|
+
lines << ""
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def emit_histograms(lines, histograms)
|
|
68
|
+
histograms.each do |name, by_label|
|
|
69
|
+
lines << "# HELP #{name} #{HISTOGRAM_META.fetch(name, name)}"
|
|
70
|
+
lines << "# TYPE #{name} histogram"
|
|
71
|
+
by_label.each_value { |entry| emit_histogram_entry(lines, name, entry) }
|
|
72
|
+
lines << ""
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def emit_histogram_entry(lines, name, entry) # rubocop:disable Metrics/MethodLength
|
|
77
|
+
lstr = entry[:labels].map { |k, v| "#{k}=\"#{v}\"" }.join(",")
|
|
78
|
+
sep = lstr.empty? ? "" : ","
|
|
79
|
+
|
|
80
|
+
Collector::HISTOGRAM_BUCKETS.each do |b|
|
|
81
|
+
le = b.to_s
|
|
82
|
+
lines << "#{name}_bucket{#{lstr}#{sep}le=\"#{le}\"} #{entry[:buckets][b]}"
|
|
83
|
+
end
|
|
84
|
+
lines << "#{name}_bucket{#{lstr}#{sep}le=\"+Inf\"} #{entry[:count]}"
|
|
85
|
+
lines << "#{name}_sum{#{lstr}} #{format("%.6f", entry[:sum])}"
|
|
86
|
+
lines << "#{name}_count{#{lstr}} #{entry[:count]}"
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def emit_pending_gauge(lines) # rubocop:disable Metrics/MethodLength
|
|
90
|
+
lines << "# HELP igniter_pending_executions Currently pending executions in store"
|
|
91
|
+
lines << "# TYPE igniter_pending_executions gauge"
|
|
92
|
+
|
|
93
|
+
@registry.names.each do |name|
|
|
94
|
+
count = @store.list_pending(graph: name).size
|
|
95
|
+
lines << "igniter_pending_executions{graph=\"#{name}\"} #{count}"
|
|
96
|
+
rescue StandardError
|
|
97
|
+
lines << "igniter_pending_executions{graph=\"#{name}\"} 0"
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
lines << ""
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "igniter"
|
|
4
|
+
require "igniter/metrics/snapshot"
|
|
5
|
+
require "igniter/metrics/collector"
|
|
6
|
+
require "igniter/metrics/prometheus_exporter"
|
|
7
|
+
|
|
8
|
+
module Igniter
|
|
9
|
+
# Metrics collection for Igniter contracts and igniter-server.
|
|
10
|
+
#
|
|
11
|
+
# The Collector subscribes to an Igniter::Events::Bus and maintains
|
|
12
|
+
# in-memory counters and histograms with zero external dependencies.
|
|
13
|
+
#
|
|
14
|
+
# Prometheus text format is exported via PrometheusExporter — usable
|
|
15
|
+
# directly in the /v1/metrics endpoint of igniter-server.
|
|
16
|
+
#
|
|
17
|
+
# Usage (standalone):
|
|
18
|
+
# require "igniter/metrics"
|
|
19
|
+
#
|
|
20
|
+
# collector = Igniter::Metrics::Collector.new
|
|
21
|
+
# contract.execution.events.subscribe(collector)
|
|
22
|
+
# contract.resolve_all
|
|
23
|
+
#
|
|
24
|
+
# exporter = Igniter::Metrics::PrometheusExporter.new(
|
|
25
|
+
# collector, store: store, registry: registry
|
|
26
|
+
# )
|
|
27
|
+
# puts exporter.export
|
|
28
|
+
#
|
|
29
|
+
# Usage (igniter-server — automatic when metrics_collector is set):
|
|
30
|
+
# Igniter::Server.configure do |c|
|
|
31
|
+
# c.metrics_collector = Igniter::Metrics::Collector.new
|
|
32
|
+
# end
|
|
33
|
+
#
|
|
34
|
+
module Metrics
|
|
35
|
+
class MetricsError < Igniter::Error; end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Igniter
|
|
4
|
+
module Model
|
|
5
|
+
# Represents a maintained aggregate node in the computation graph.
|
|
6
|
+
#
|
|
7
|
+
# An AggregateNode computes an aggregate value over an incremental collection
|
|
8
|
+
# and updates it in O(change) time by processing only the diff (added/changed/removed
|
|
9
|
+
# items) rather than iterating over the entire collection on every resolve.
|
|
10
|
+
#
|
|
11
|
+
# Supported built-in operators: :count, :sum, :avg, :min, :max, :group_count.
|
|
12
|
+
# Custom aggregates can be defined with initial:, add:, and remove: lambdas.
|
|
13
|
+
#
|
|
14
|
+
# The aggregate node depends on exactly one upstream incremental collection node.
|
|
15
|
+
# The compiler validates this at definition time.
|
|
16
|
+
class AggregateNode < Node
|
|
17
|
+
attr_reader :source_collection, :operator
|
|
18
|
+
|
|
19
|
+
def initialize(id:, name:, source_collection:, operator:, # rubocop:disable Metrics/ParameterLists
|
|
20
|
+
path: nil, metadata: {})
|
|
21
|
+
super(
|
|
22
|
+
id: id,
|
|
23
|
+
kind: :aggregate,
|
|
24
|
+
name: name,
|
|
25
|
+
path: path || name.to_s,
|
|
26
|
+
dependencies: [source_collection],
|
|
27
|
+
metadata: metadata
|
|
28
|
+
)
|
|
29
|
+
@source_collection = source_collection.to_sym
|
|
30
|
+
@operator = operator
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -3,9 +3,9 @@
|
|
|
3
3
|
module Igniter
|
|
4
4
|
module Model
|
|
5
5
|
class CollectionNode < Node
|
|
6
|
-
attr_reader :source_dependency, :contract_class, :key_name, :mode, :context_dependencies, :input_mapper
|
|
6
|
+
attr_reader :source_dependency, :contract_class, :key_name, :mode, :window, :context_dependencies, :input_mapper
|
|
7
7
|
|
|
8
|
-
def initialize(id:, name:, source_dependency:, contract_class:, key_name:, mode:, context_dependencies: [], input_mapper: nil, path: nil, metadata: {})
|
|
8
|
+
def initialize(id:, name:, source_dependency:, contract_class:, key_name:, mode:, window: nil, context_dependencies: [], input_mapper: nil, path: nil, metadata: {})
|
|
9
9
|
super(
|
|
10
10
|
id: id,
|
|
11
11
|
kind: :collection,
|
|
@@ -19,6 +19,7 @@ module Igniter
|
|
|
19
19
|
@contract_class = contract_class
|
|
20
20
|
@key_name = key_name.to_sym
|
|
21
21
|
@mode = mode.to_sym
|
|
22
|
+
@window = window
|
|
22
23
|
@context_dependencies = Array(context_dependencies).map(&:to_sym)
|
|
23
24
|
@input_mapper = input_mapper
|
|
24
25
|
end
|
|
@@ -57,6 +57,19 @@ module Igniter
|
|
|
57
57
|
metadata[:type] || executor_metadata[:type]
|
|
58
58
|
end
|
|
59
59
|
|
|
60
|
+
# Seconds to cache this node's result across executions (nil = no TTL cache).
|
|
61
|
+
# Requires Igniter::NodeCache.cache to be configured.
|
|
62
|
+
def cache_ttl
|
|
63
|
+
metadata[:cache_ttl]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# When true, concurrent executions with identical dep fingerprints share one
|
|
67
|
+
# computation (the follower waits for the leader's result).
|
|
68
|
+
# Requires Igniter::NodeCache.coalescing_lock to be configured.
|
|
69
|
+
def coalesce?
|
|
70
|
+
metadata[:coalesce] == true
|
|
71
|
+
end
|
|
72
|
+
|
|
60
73
|
def const?
|
|
61
74
|
metadata[:kind] == :const
|
|
62
75
|
end
|
|
@@ -4,10 +4,16 @@ module Igniter
|
|
|
4
4
|
module Model
|
|
5
5
|
# Represents a node that executes a contract on a remote igniter-server node.
|
|
6
6
|
# The result is the outputs hash returned by the remote contract.
|
|
7
|
+
#
|
|
8
|
+
# Three routing modes:
|
|
9
|
+
# :static — node_url is a hard-coded URL (original behaviour)
|
|
10
|
+
# :capability — auto-select an alive peer advertising the given capability
|
|
11
|
+
# :pinned — must use the specific named peer; IncidentError if down
|
|
7
12
|
class RemoteNode < Node
|
|
8
|
-
attr_reader :contract_name, :node_url, :input_mapping, :timeout
|
|
13
|
+
attr_reader :contract_name, :node_url, :input_mapping, :timeout, :capability, :pinned_to
|
|
9
14
|
|
|
10
|
-
def initialize(id:, name:, contract_name:,
|
|
15
|
+
def initialize(id:, name:, contract_name:, input_mapping:, # rubocop:disable Metrics/ParameterLists, Metrics/MethodLength
|
|
16
|
+
node_url: "", timeout: 30, path: nil, metadata: {}, capability: nil, pinned_to: nil)
|
|
11
17
|
super(
|
|
12
18
|
id: id,
|
|
13
19
|
kind: :remote,
|
|
@@ -20,6 +26,16 @@ module Igniter
|
|
|
20
26
|
@node_url = node_url.to_s
|
|
21
27
|
@input_mapping = input_mapping.transform_keys(&:to_sym).transform_values(&:to_sym).freeze
|
|
22
28
|
@timeout = Integer(timeout)
|
|
29
|
+
@capability = capability&.to_sym
|
|
30
|
+
@pinned_to = pinned_to&.to_s
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Returns :static, :capability, or :pinned.
|
|
34
|
+
def routing_mode
|
|
35
|
+
return :pinned if @pinned_to
|
|
36
|
+
return :capability if @capability
|
|
37
|
+
|
|
38
|
+
:static
|
|
23
39
|
end
|
|
24
40
|
end
|
|
25
41
|
end
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
module Igniter
|
|
6
|
+
# Cross-execution TTL cache and request coalescing for compute nodes.
|
|
7
|
+
#
|
|
8
|
+
# Activated per-node via the `cache_ttl:` and `coalesce:` options on `compute`:
|
|
9
|
+
#
|
|
10
|
+
# compute :available_slots, with: [...], call: CheckAvailability,
|
|
11
|
+
# cache_ttl: 60, # seconds — reuse result across executions
|
|
12
|
+
# coalesce: true # deduplicate concurrent in-flight requests
|
|
13
|
+
#
|
|
14
|
+
# == Setup
|
|
15
|
+
#
|
|
16
|
+
# require "igniter/node_cache"
|
|
17
|
+
# Igniter::NodeCache.cache = Igniter::NodeCache::Memory.new
|
|
18
|
+
# Igniter::NodeCache.coalescing_lock = Igniter::NodeCache::CoalescingLock.new
|
|
19
|
+
#
|
|
20
|
+
# Or via the configure block:
|
|
21
|
+
#
|
|
22
|
+
# Igniter.configure do |c|
|
|
23
|
+
# c.node_cache = Igniter::NodeCache::Memory.new
|
|
24
|
+
# c.node_coalescing = true # auto-creates a CoalescingLock
|
|
25
|
+
# end
|
|
26
|
+
#
|
|
27
|
+
# == AR fingerprinting (Gap 3)
|
|
28
|
+
#
|
|
29
|
+
# Cache keys are built from a stable fingerprint of dep values.
|
|
30
|
+
# For ActiveRecord objects, include Igniter::Fingerprint (or use the Railtie):
|
|
31
|
+
#
|
|
32
|
+
# class Trade < ApplicationRecord
|
|
33
|
+
# include Igniter::Fingerprint
|
|
34
|
+
# # default: "Trade:42:1712345678"
|
|
35
|
+
# end
|
|
36
|
+
module NodeCache
|
|
37
|
+
class << self
|
|
38
|
+
# Global TTL cache backend. nil = disabled (default).
|
|
39
|
+
# Must respond to: #fetch(key) → value | nil
|
|
40
|
+
# #store(key, value, ttl:)
|
|
41
|
+
attr_accessor :cache
|
|
42
|
+
|
|
43
|
+
# Global coalescing lock. nil = disabled (default).
|
|
44
|
+
attr_accessor :coalescing_lock
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# ─── Cache key ────────────────────────────────────────────────────────────
|
|
48
|
+
|
|
49
|
+
# Immutable key uniquely identifying a node result for a given set of dep values.
|
|
50
|
+
# Format: "ttl:{contract_name}:{node_name}:{dep_fingerprint_hex}"
|
|
51
|
+
class CacheKey
|
|
52
|
+
attr_reader :hex
|
|
53
|
+
|
|
54
|
+
def initialize(contract_name, node_name, dep_hex)
|
|
55
|
+
@hex = "ttl:#{contract_name}:#{node_name}:#{dep_hex}".freeze
|
|
56
|
+
freeze
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def to_s = @hex
|
|
60
|
+
def inspect = "#<NodeCache::CacheKey #{@hex}>"
|
|
61
|
+
def ==(other) = other.is_a?(CacheKey) && other.hex == @hex
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# ─── Memory backend ───────────────────────────────────────────────────────
|
|
65
|
+
|
|
66
|
+
# Thread-safe in-process TTL store. Entries expire after their TTL.
|
|
67
|
+
# Replace with a Redis-backed implementation for multi-process sharing.
|
|
68
|
+
class Memory
|
|
69
|
+
Entry = Struct.new(:value, :expires_at) do
|
|
70
|
+
def expired? = Time.now.utc > expires_at
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def initialize
|
|
74
|
+
@store = {}
|
|
75
|
+
@mu = Mutex.new
|
|
76
|
+
@hits = 0
|
|
77
|
+
@misses = 0
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Returns the cached value, or nil on miss / expiry.
|
|
81
|
+
def fetch(key)
|
|
82
|
+
@mu.synchronize do
|
|
83
|
+
entry = @store[key.hex]
|
|
84
|
+
if entry.nil? || entry.expired?
|
|
85
|
+
@store.delete(key.hex) if entry
|
|
86
|
+
@misses += 1
|
|
87
|
+
nil
|
|
88
|
+
else
|
|
89
|
+
@hits += 1
|
|
90
|
+
entry.value
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Stores value with a TTL in seconds.
|
|
96
|
+
def store(key, value, ttl:)
|
|
97
|
+
@mu.synchronize do
|
|
98
|
+
@store[key.hex] = Entry.new(value, Time.now.utc + ttl)
|
|
99
|
+
end
|
|
100
|
+
value
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Remove all expired entries. Call periodically to reclaim memory.
|
|
104
|
+
def prune!
|
|
105
|
+
@mu.synchronize do
|
|
106
|
+
now = Time.now.utc
|
|
107
|
+
@store.delete_if { |_, e| e.expires_at < now }
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def size = @mu.synchronize { @store.size }
|
|
112
|
+
def clear = @mu.synchronize { @store.clear; @hits = 0; @misses = 0 }
|
|
113
|
+
def stats = @mu.synchronize { { size: @store.size, hits: @hits, misses: @misses } }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# ─── Coalescing lock ──────────────────────────────────────────────────────
|
|
117
|
+
|
|
118
|
+
# In-flight deduplication for concurrent requests with identical dep fingerprints.
|
|
119
|
+
#
|
|
120
|
+
# When two executions race to compute the same `coalesce: true` node:
|
|
121
|
+
# • The first caller becomes the leader — computes and stores the result.
|
|
122
|
+
# • Subsequent callers become followers — wait for the leader, then reuse its result.
|
|
123
|
+
#
|
|
124
|
+
# This eliminates duplicate work for concurrent auction-style requests
|
|
125
|
+
# (e.g. 3 vendors arriving within ~50ms for the same lead).
|
|
126
|
+
#
|
|
127
|
+
# Scope: single Ruby process. For Puma multi-worker, extend with a Redis-based
|
|
128
|
+
# implementation using SETNX advisory locks + Pub/Sub.
|
|
129
|
+
class CoalescingLock
|
|
130
|
+
WAIT_TIMEOUT = 30 # seconds before a follower gives up and computes independently
|
|
131
|
+
|
|
132
|
+
InFlight = Struct.new(:mutex, :cond, :done, :value, :error, keyword_init: true)
|
|
133
|
+
|
|
134
|
+
def initialize
|
|
135
|
+
@mu = Mutex.new
|
|
136
|
+
@flights = {}
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# Attempt to acquire a computation slot for `hex`.
|
|
140
|
+
#
|
|
141
|
+
# Returns [:leader, flight] — caller is first; must compute then call finish!
|
|
142
|
+
# Returns [:follower, flight] — another caller is already computing; call wait(flight)
|
|
143
|
+
def acquire(hex)
|
|
144
|
+
@mu.synchronize do
|
|
145
|
+
if @flights.key?(hex)
|
|
146
|
+
[:follower, @flights[hex]]
|
|
147
|
+
else
|
|
148
|
+
flight = InFlight.new(
|
|
149
|
+
mutex: Mutex.new,
|
|
150
|
+
cond: ConditionVariable.new,
|
|
151
|
+
done: false,
|
|
152
|
+
value: nil,
|
|
153
|
+
error: nil
|
|
154
|
+
)
|
|
155
|
+
@flights[hex] = flight
|
|
156
|
+
[:leader, flight]
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
# Called by the leader when computation finishes (success or failure).
|
|
162
|
+
# Unblocks all waiting followers.
|
|
163
|
+
def finish!(hex, value: nil, error: nil)
|
|
164
|
+
flight = @mu.synchronize { @flights.delete(hex) }
|
|
165
|
+
return unless flight
|
|
166
|
+
|
|
167
|
+
flight.mutex.synchronize do
|
|
168
|
+
flight.value = value
|
|
169
|
+
flight.error = error
|
|
170
|
+
flight.done = true
|
|
171
|
+
flight.cond.broadcast
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
# Called by a follower after receiving the flight object from acquire.
|
|
176
|
+
# Blocks until the leader calls finish!, then returns [value, error].
|
|
177
|
+
# Times out after WAIT_TIMEOUT seconds and returns [nil, nil] (follower recomputes).
|
|
178
|
+
def wait(flight)
|
|
179
|
+
flight.mutex.synchronize do
|
|
180
|
+
deadline = Time.now + WAIT_TIMEOUT
|
|
181
|
+
until flight.done
|
|
182
|
+
remaining = deadline - Time.now
|
|
183
|
+
break if remaining <= 0
|
|
184
|
+
|
|
185
|
+
flight.cond.wait(flight.mutex, remaining)
|
|
186
|
+
end
|
|
187
|
+
[flight.value, flight.error]
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
def in_flight_count = @mu.synchronize { @flights.size }
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# ─── Fingerprinter ────────────────────────────────────────────────────────
|
|
195
|
+
|
|
196
|
+
# Produces a stable hex fingerprint for a set of dependency values.
|
|
197
|
+
# Used as part of the NodeCache::CacheKey.
|
|
198
|
+
#
|
|
199
|
+
# Supports the Igniter::Fingerprint protocol:
|
|
200
|
+
# objects responding to #igniter_fingerprint return a stable string
|
|
201
|
+
# (e.g. "Trade:42:1712345678").
|
|
202
|
+
#
|
|
203
|
+
# Fallback for unknown objects: "#ClassName@object_id" — stable within
|
|
204
|
+
# a single process but NOT across restarts. Sufficient for the in-process
|
|
205
|
+
# Memory backend; Redis-backed deployments should ensure all dep objects
|
|
206
|
+
# implement igniter_fingerprint.
|
|
207
|
+
module Fingerprinter
|
|
208
|
+
def self.call(dep_values)
|
|
209
|
+
Digest::SHA256.hexdigest(serialize(dep_values))[0..23]
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
def self.serialize(val) # rubocop:disable Metrics/CyclomaticComplexity
|
|
213
|
+
case val
|
|
214
|
+
when Hash
|
|
215
|
+
pairs = val.sort_by { |k, _| k.to_s }.map { |k, v| "#{k}:#{serialize(v)}" }
|
|
216
|
+
"{#{pairs.join(",")}}"
|
|
217
|
+
when Array then "[#{val.map { |v| serialize(v) }.join(",")}]"
|
|
218
|
+
when String then val.inspect
|
|
219
|
+
when Symbol then ":#{val}"
|
|
220
|
+
when Numeric, NilClass, TrueClass, FalseClass then val.inspect
|
|
221
|
+
else
|
|
222
|
+
if val.respond_to?(:igniter_fingerprint)
|
|
223
|
+
"fp:#{val.igniter_fingerprint}"
|
|
224
|
+
else
|
|
225
|
+
"obj:#{val.class.name}@#{val.object_id}"
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
end
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Igniter
|
|
4
|
+
module Replication
|
|
5
|
+
# Abstract base class for deployment bootstrappers.
|
|
6
|
+
#
|
|
7
|
+
# Subclasses implement three lifecycle methods:
|
|
8
|
+
# install — copy/install the application on the remote
|
|
9
|
+
# start — launch the application process
|
|
10
|
+
# verify — confirm the application is running (default impl provided)
|
|
11
|
+
class Bootstrapper
|
|
12
|
+
BootstrapError = Class.new(Igniter::Error)
|
|
13
|
+
|
|
14
|
+
# Install the application on the remote server.
|
|
15
|
+
#
|
|
16
|
+
# @param session [SSHSession] active SSH session to the remote host
|
|
17
|
+
# @param manifest [Manifest] self-description of the running instance
|
|
18
|
+
# @param env [Hash] environment variables to write on remote
|
|
19
|
+
# @param target_path [String] base directory on the remote server
|
|
20
|
+
def install(session:, manifest:, env: {}, target_path: "/opt/igniter")
|
|
21
|
+
raise NotImplementedError, "#{self.class}#install not implemented"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# Start the application on the remote server.
|
|
25
|
+
#
|
|
26
|
+
# @param session [SSHSession] active SSH session to the remote host
|
|
27
|
+
# @param manifest [Manifest] self-description of the running instance
|
|
28
|
+
# @param target_path [String] base directory on the remote server
|
|
29
|
+
def start(session:, manifest:, target_path: "/opt/igniter")
|
|
30
|
+
raise NotImplementedError, "#{self.class}#start not implemented"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Verify the application is running on the remote server.
|
|
34
|
+
# Returns true if Igniter can be loaded, false otherwise.
|
|
35
|
+
#
|
|
36
|
+
# @param session [SSHSession] active SSH session to the remote host
|
|
37
|
+
# @param target_path [String] base directory on the remote server (unused by default impl)
|
|
38
|
+
def verify(session:, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
|
|
39
|
+
result = session.exec("ruby -e 'require \"igniter\"; puts Igniter::VERSION' 2>/dev/null")
|
|
40
|
+
result[:success] && !result[:stdout].strip.empty?
|
|
41
|
+
rescue SSHSession::SSHError
|
|
42
|
+
false
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
protected
|
|
46
|
+
|
|
47
|
+
# Write environment variables to a shell-sourceable file on the remote.
|
|
48
|
+
#
|
|
49
|
+
# @param session [SSHSession] active SSH session
|
|
50
|
+
# @param env [Hash] key-value pairs to export
|
|
51
|
+
# @param path [String] remote path for the env file
|
|
52
|
+
def write_env_file(session:, env:, path:)
|
|
53
|
+
return if env.nil? || env.empty?
|
|
54
|
+
|
|
55
|
+
require "shellwords"
|
|
56
|
+
lines = env.map { |k, v| "export #{k}=#{v.to_s.shellescape}" }.join("\n")
|
|
57
|
+
session.exec!("printf '%s\\n' #{lines.shellescape} > #{path}")
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Igniter
|
|
4
|
+
module Replication
|
|
5
|
+
module Bootstrappers
|
|
6
|
+
# Deploys by installing the igniter gem from RubyGems on the remote host.
|
|
7
|
+
#
|
|
8
|
+
# Optionally pins a specific gem version. The startup script defaults to
|
|
9
|
+
# the +igniter-server+ executable installed with the gem.
|
|
10
|
+
class Gem < Bootstrapper
|
|
11
|
+
def initialize(version: nil, startup_script: nil)
|
|
12
|
+
super()
|
|
13
|
+
@version = version
|
|
14
|
+
@startup_script = startup_script
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def install(session:, manifest:, env: {}, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
|
|
18
|
+
ver_flag = @version ? " -v #{@version}" : ""
|
|
19
|
+
session.exec!("gem install igniter#{ver_flag} --no-document")
|
|
20
|
+
session.exec!("mkdir -p #{target_path}")
|
|
21
|
+
write_env_file(session: session, env: env, path: "#{target_path}/.env")
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def start(session:, manifest:, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
|
|
25
|
+
log_path = "#{target_path}/igniter.log"
|
|
26
|
+
script = @startup_script || "igniter-server"
|
|
27
|
+
session.exec!("nohup #{script} >> #{log_path} 2>&1 &")
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Igniter
|
|
4
|
+
module Replication
|
|
5
|
+
module Bootstrappers
|
|
6
|
+
# Deploys by cloning a git repository on the remote host.
|
|
7
|
+
#
|
|
8
|
+
# Requires git to be installed on the remote. After cloning, installs
|
|
9
|
+
# bundler and runs bundle install.
|
|
10
|
+
class Git < Bootstrapper
|
|
11
|
+
DEFAULT_BRANCH = "main"
|
|
12
|
+
DEFAULT_BUNDLE_OPTIONS = "--without development test"
|
|
13
|
+
|
|
14
|
+
def initialize(repo_url:, branch: DEFAULT_BRANCH, bundle_options: DEFAULT_BUNDLE_OPTIONS)
|
|
15
|
+
super()
|
|
16
|
+
@repo_url = repo_url
|
|
17
|
+
@branch = branch
|
|
18
|
+
@bundle_options = bundle_options
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def install(session:, manifest:, env: {}, target_path: "/opt/igniter") # rubocop:disable Lint/UnusedMethodArgument
|
|
22
|
+
app_path = "#{target_path}/app"
|
|
23
|
+
session.exec!("mkdir -p #{target_path}")
|
|
24
|
+
session.exec!("git clone --branch #{@branch} --depth 1 #{@repo_url} #{app_path}")
|
|
25
|
+
session.exec!("cd #{app_path} && gem install bundler --no-document")
|
|
26
|
+
session.exec!("cd #{app_path} && bundle install #{@bundle_options}")
|
|
27
|
+
write_env_file(session: session, env: env, path: "#{target_path}/.env")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def start(session:, manifest:, target_path: "/opt/igniter")
|
|
31
|
+
app_path = "#{target_path}/app"
|
|
32
|
+
log_path = "#{target_path}/igniter.log"
|
|
33
|
+
cmd = manifest.startup_command
|
|
34
|
+
session.exec!("cd #{app_path} && nohup ruby #{cmd} >> #{log_path} 2>&1 &")
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tmpdir"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
|
|
6
|
+
module Igniter
|
|
7
|
+
module Replication
|
|
8
|
+
module Bootstrappers
|
|
9
|
+
# Deploys by creating a gzipped tarball of the local source tree,
|
|
10
|
+
# uploading it via SCP, and extracting it on the remote host.
|
|
11
|
+
#
|
|
12
|
+
# This strategy works without git or internet access on the remote,
|
|
13
|
+
# making it suitable for air-gapped or private environments.
|
|
14
|
+
class Tarball < Bootstrapper
|
|
15
|
+
def install(session:, manifest:, env: {}, target_path: "/opt/igniter")
|
|
16
|
+
tarball = create_tarball(manifest)
|
|
17
|
+
begin
|
|
18
|
+
remote_tar = "/tmp/igniter_replication_#{Process.pid}.tar.gz"
|
|
19
|
+
session.upload!(tarball, remote_tar)
|
|
20
|
+
extract_and_install(session, remote_tar, target_path)
|
|
21
|
+
write_env_file(session: session, env: env, path: "#{target_path}/.env")
|
|
22
|
+
ensure
|
|
23
|
+
FileUtils.rm_f(tarball)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def start(session:, manifest:, target_path: "/opt/igniter")
|
|
28
|
+
app_path = "#{target_path}/app"
|
|
29
|
+
log_path = "#{target_path}/igniter.log"
|
|
30
|
+
cmd = File.basename(manifest.startup_command)
|
|
31
|
+
session.exec!("cd #{app_path} && nohup ruby #{cmd} >> #{log_path} 2>&1 &")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
def extract_and_install(session, remote_tar, target_path)
|
|
37
|
+
app_path = "#{target_path}/app"
|
|
38
|
+
session.exec!("mkdir -p #{app_path}")
|
|
39
|
+
session.exec!("tar -xzf #{remote_tar} -C #{app_path} --strip-components=1")
|
|
40
|
+
session.exec!("cd #{app_path} && gem install bundler --no-document")
|
|
41
|
+
session.exec!("cd #{app_path} && bundle install --without development test")
|
|
42
|
+
session.exec!("rm -f #{remote_tar}")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def create_tarball(manifest)
|
|
46
|
+
source = manifest.source_path
|
|
47
|
+
tmpfile = File.join(Dir.tmpdir, "igniter_replication_#{Process.pid}.tar.gz")
|
|
48
|
+
parent = File.dirname(source)
|
|
49
|
+
name = File.basename(source)
|
|
50
|
+
system("tar", "-czf", tmpfile, "-C", parent, name)
|
|
51
|
+
tmpfile
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|