brute 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/brute/agent.rb +14 -0
- data/lib/brute/diff.rb +24 -0
- data/lib/brute/loop/agent_stream.rb +118 -0
- data/lib/brute/loop/agent_turn.rb +520 -0
- data/lib/brute/{compactor.rb → loop/compactor.rb} +2 -0
- data/lib/brute/{doom_loop.rb → loop/doom_loop.rb} +2 -0
- data/lib/brute/loop/step.rb +332 -0
- data/lib/brute/loop/tool_call_step.rb +90 -0
- data/lib/brute/middleware/compaction_check.rb +70 -23
- data/lib/brute/middleware/doom_loop_detection.rb +110 -7
- data/lib/brute/middleware/llm_call.rb +88 -1
- data/lib/brute/middleware/message_tracking.rb +140 -10
- data/lib/brute/middleware/otel/span.rb +32 -2
- data/lib/brute/middleware/otel/token_usage.rb +38 -0
- data/lib/brute/middleware/otel/tool_calls.rb +30 -1
- data/lib/brute/middleware/otel/tool_results.rb +29 -1
- data/lib/brute/middleware/otel.rb +5 -0
- data/lib/brute/middleware/reasoning_normalizer.rb +94 -0
- data/lib/brute/middleware/retry.rb +113 -1
- data/lib/brute/middleware/session_persistence.rb +46 -3
- data/lib/brute/middleware/token_tracking.rb +78 -0
- data/lib/brute/middleware/tool_error_tracking.rb +128 -1
- data/lib/brute/middleware/tool_use_guard.rb +64 -28
- data/lib/brute/middleware/tracing.rb +63 -2
- data/lib/brute/middleware.rb +18 -0
- data/lib/brute/orchestrator/turn.rb +105 -0
- data/lib/brute/patches/buffer_nil_guard.rb +5 -0
- data/lib/brute/pipeline.rb +86 -7
- data/lib/brute/prompts/build_switch.rb +29 -0
- data/lib/brute/prompts/environment.rb +43 -0
- data/lib/brute/prompts/identity.rb +29 -0
- data/lib/brute/prompts/instructions.rb +21 -0
- data/lib/brute/prompts/max_steps.rb +25 -0
- data/lib/brute/prompts/plan_reminder.rb +25 -0
- data/lib/brute/prompts/skills.rb +13 -0
- data/lib/brute/prompts.rb +28 -0
- data/lib/brute/providers/ollama.rb +135 -0
- data/lib/brute/providers/opencode_go.rb +5 -0
- data/lib/brute/providers/opencode_zen.rb +7 -2
- data/lib/brute/providers/shell.rb +2 -2
- data/lib/brute/providers/shell_response.rb +7 -2
- data/lib/brute/providers.rb +62 -0
- data/lib/brute/queue/base_queue.rb +222 -0
- data/lib/brute/{file_mutation_queue.rb → queue/file_mutation_queue.rb} +28 -26
- data/lib/brute/queue/parallel_queue.rb +66 -0
- data/lib/brute/queue/sequential_queue.rb +63 -0
- data/lib/brute/{message_store.rb → store/message_store.rb} +155 -62
- data/lib/brute/store/session.rb +106 -0
- data/lib/brute/{snapshot_store.rb → store/snapshot_store.rb} +2 -0
- data/lib/brute/{todo_store.rb → store/todo_store.rb} +2 -0
- data/lib/brute/system_prompt.rb +101 -0
- data/lib/brute/tools/delegate.rb +59 -0
- data/lib/brute/tools/fs_patch.rb +54 -2
- data/lib/brute/tools/fs_read.rb +5 -0
- data/lib/brute/tools/fs_remove.rb +7 -2
- data/lib/brute/tools/fs_search.rb +5 -0
- data/lib/brute/tools/fs_undo.rb +7 -2
- data/lib/brute/tools/fs_write.rb +40 -2
- data/lib/brute/tools/net_fetch.rb +5 -0
- data/lib/brute/tools/question.rb +5 -0
- data/lib/brute/tools/shell.rb +5 -0
- data/lib/brute/tools/todo_read.rb +6 -1
- data/lib/brute/tools/todo_write.rb +6 -1
- data/lib/brute/tools.rb +31 -0
- data/lib/brute/version.rb +1 -1
- data/lib/brute.rb +40 -204
- metadata +31 -20
- data/lib/brute/agent_stream.rb +0 -63
- data/lib/brute/hooks.rb +0 -84
- data/lib/brute/orchestrator.rb +0 -391
- data/lib/brute/session.rb +0 -161
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "brute"
|
|
5
|
+
|
|
6
|
+
# Ensure the Ollama provider is loaded (llm.rb lazy-loads providers).
|
|
7
|
+
unless defined?(LLM::Ollama)
|
|
8
|
+
require "llm/providers/ollama"
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
module Brute
|
|
12
|
+
module Providers
|
|
13
|
+
##
|
|
14
|
+
# Brute-level wrapper around LLM::Ollama for local model inference.
|
|
15
|
+
#
|
|
16
|
+
# Adds environment-variable-based configuration so that all Brute
|
|
17
|
+
# examples and the CLI work out of the box with a local Ollama
|
|
18
|
+
# instance:
|
|
19
|
+
#
|
|
20
|
+
# OLLAMA_HOST — base URL (default: http://localhost:11434)
|
|
21
|
+
# OLLAMA_MODEL — default model (default: llm.rb's default, currently qwen3:latest)
|
|
22
|
+
#
|
|
23
|
+
# @example Auto-detect via environment
|
|
24
|
+
# export OLLAMA_HOST=http://localhost:11434
|
|
25
|
+
# ruby examples/01_basic_agent.rb
|
|
26
|
+
#
|
|
27
|
+
# @example Remote Ollama server
|
|
28
|
+
# export OLLAMA_HOST=http://192.168.1.50:11434
|
|
29
|
+
# export OLLAMA_MODEL=llama3.1:8b
|
|
30
|
+
# ruby examples/02_fix_a_bug.rb
|
|
31
|
+
#
|
|
32
|
+
class Ollama < LLM::Ollama
|
|
33
|
+
##
|
|
34
|
+
# Parse OLLAMA_HOST into host, port, and ssl components.
|
|
35
|
+
# Accepts formats like:
|
|
36
|
+
# http://localhost:11434
|
|
37
|
+
# https://ollama.example.com
|
|
38
|
+
# 192.168.1.50:11434
|
|
39
|
+
# localhost
|
|
40
|
+
#
|
|
41
|
+
# @param url [String, nil] raw OLLAMA_HOST value
|
|
42
|
+
# @return [Hash] with :host, :port, :ssl keys
|
|
43
|
+
def self.parse_host(url)
|
|
44
|
+
return { host: LLM::Ollama::HOST, port: 11434, ssl: false } if url.nil? || url.empty?
|
|
45
|
+
|
|
46
|
+
# Prepend scheme if missing so URI.parse works
|
|
47
|
+
url = "http://#{url}" unless url.match?(%r{\A\w+://})
|
|
48
|
+
uri = URI.parse(url)
|
|
49
|
+
|
|
50
|
+
{
|
|
51
|
+
host: uri.host || LLM::Ollama::HOST,
|
|
52
|
+
port: uri.port || 11434,
|
|
53
|
+
ssl: uri.scheme == "https",
|
|
54
|
+
}
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
##
|
|
58
|
+
# @param key [String] ignored (Ollama needs no auth), kept for provider interface
|
|
59
|
+
def initialize(key: "none", **)
|
|
60
|
+
config = self.class.parse_host(ENV["OLLAMA_HOST"])
|
|
61
|
+
super(key: key, host: config[:host], port: config[:port], ssl: config[:ssl], **)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
##
|
|
65
|
+
# @return [Symbol]
|
|
66
|
+
def name
|
|
67
|
+
:ollama
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
##
|
|
71
|
+
# Returns the default model, preferring OLLAMA_MODEL env var.
|
|
72
|
+
# @return [String]
|
|
73
|
+
def default_model
|
|
74
|
+
ENV["OLLAMA_MODEL"] || super
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
test do
|
|
81
|
+
parse = proc { |url| Brute::Providers::Ollama.parse_host(url) }
|
|
82
|
+
|
|
83
|
+
describe ".parse_host" do
|
|
84
|
+
it "returns defaults for nil" do
|
|
85
|
+
parse.(nil).should == { host: "localhost", port: 11434, ssl: false }
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
it "returns defaults for empty string" do
|
|
89
|
+
parse.("").should == { host: "localhost", port: 11434, ssl: false }
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
it "parses http URL with port" do
|
|
93
|
+
parse.("http://192.168.1.50:11434").should == { host: "192.168.1.50", port: 11434, ssl: false }
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
it "parses https URL" do
|
|
97
|
+
parse.("https://ollama.example.com").should == { host: "ollama.example.com", port: 443, ssl: true }
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
it "parses host:port without scheme" do
|
|
101
|
+
parse.("192.168.1.50:11434").should == { host: "192.168.1.50", port: 11434, ssl: false }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
it "parses bare hostname" do
|
|
105
|
+
parse.("myhost").should == { host: "myhost", port: 80, ssl: false }
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
describe "#name" do
|
|
110
|
+
it "returns :ollama" do
|
|
111
|
+
provider = Brute::Providers::Ollama.new
|
|
112
|
+
provider.name.should == :ollama
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
describe "#default_model" do
|
|
117
|
+
it "falls back to llm.rb default when OLLAMA_MODEL is not set" do
|
|
118
|
+
original = ENV["OLLAMA_MODEL"]
|
|
119
|
+
ENV.delete("OLLAMA_MODEL")
|
|
120
|
+
provider = Brute::Providers::Ollama.new
|
|
121
|
+
provider.default_model.should == "qwen3:latest"
|
|
122
|
+
ensure
|
|
123
|
+
ENV["OLLAMA_MODEL"] = original if original
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
it "uses OLLAMA_MODEL env var when set" do
|
|
127
|
+
original = ENV["OLLAMA_MODEL"]
|
|
128
|
+
ENV["OLLAMA_MODEL"] = "llama3.1:8b"
|
|
129
|
+
provider = Brute::Providers::Ollama.new
|
|
130
|
+
provider.default_model.should == "llama3.1:8b"
|
|
131
|
+
ensure
|
|
132
|
+
ENV["OLLAMA_MODEL"] = original
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
if __FILE__ == $0
|
|
4
|
+
require "bundler/setup"
|
|
5
|
+
require "brute"
|
|
6
|
+
end
|
|
7
|
+
|
|
3
8
|
# Ensure the OpenAI provider is loaded (llm.rb lazy-loads providers).
|
|
4
9
|
unless defined?(LLM::OpenAI)
|
|
5
10
|
require "llm/providers/openai"
|
|
@@ -41,10 +46,10 @@ module LLM
|
|
|
41
46
|
end
|
|
42
47
|
|
|
43
48
|
##
|
|
44
|
-
# Returns the default model
|
|
49
|
+
# Returns the default model.
|
|
45
50
|
# @return [String]
|
|
46
51
|
def default_model
|
|
47
|
-
"
|
|
52
|
+
"zen-bickpickle"
|
|
48
53
|
end
|
|
49
54
|
|
|
50
55
|
##
|
|
@@ -15,7 +15,7 @@ module Brute
|
|
|
15
15
|
# nix - nix eval --expr '...'
|
|
16
16
|
#
|
|
17
17
|
# The provider's #complete method returns a synthetic response
|
|
18
|
-
# containing a single "shell" tool call. The
|
|
18
|
+
# containing a single "shell" tool call. The agent loop executes
|
|
19
19
|
# it through the normal pipeline — all middleware (message tracking,
|
|
20
20
|
# session persistence, token tracking, etc.) fires as usual.
|
|
21
21
|
#
|
|
@@ -45,7 +45,7 @@ module Brute
|
|
|
45
45
|
tools = params[:tools] || []
|
|
46
46
|
|
|
47
47
|
# nil text means we received tool results (second call) —
|
|
48
|
-
# return an empty assistant response so the
|
|
48
|
+
# return an empty assistant response so the agent loop exits.
|
|
49
49
|
return ShellResponse.new(model: model, tools: tools) if text.nil?
|
|
50
50
|
|
|
51
51
|
wrap = INTERPRETERS.fetch(model, INTERPRETERS["bash"])
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
if __FILE__ == $0
|
|
4
|
+
require "bundler/setup"
|
|
5
|
+
require "brute"
|
|
6
|
+
end
|
|
7
|
+
|
|
3
8
|
require "securerandom"
|
|
4
9
|
|
|
5
10
|
module Brute
|
|
@@ -7,12 +12,12 @@ module Brute
|
|
|
7
12
|
# Synthetic completion response returned by Brute::Providers::Shell.
|
|
8
13
|
#
|
|
9
14
|
# When +command+ is present, the response contains a single assistant
|
|
10
|
-
# message with a "shell" tool call. The
|
|
15
|
+
# message with a "shell" tool call. The agent loop picks it up and
|
|
11
16
|
# executes Brute::Tools::Shell through the normal pipeline.
|
|
12
17
|
#
|
|
13
18
|
# When +command+ is nil (tool results round-trip), the response
|
|
14
19
|
# contains an empty assistant message with no tool calls, causing
|
|
15
|
-
# the
|
|
20
|
+
# the agent loop to exit.
|
|
16
21
|
#
|
|
17
22
|
class ShellResponse
|
|
18
23
|
def initialize(command: nil, model: "bash", tools: [])
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
require_relative 'providers/shell_response'
|
|
2
|
+
require_relative 'providers/shell'
|
|
3
|
+
require_relative 'providers/models_dev'
|
|
4
|
+
require_relative 'providers/opencode_zen'
|
|
5
|
+
require_relative 'providers/opencode_go'
|
|
6
|
+
require_relative 'providers/ollama'
|
|
7
|
+
|
|
8
|
+
module Brute
|
|
9
|
+
module Providers
|
|
10
|
+
ALL = {
|
|
11
|
+
'anthropic' => ->(key) { LLM.anthropic(key: key).tap { Patches::AnthropicToolRole.apply! } },
|
|
12
|
+
'openai' => ->(key) { LLM.openai(key: key) },
|
|
13
|
+
'google' => ->(key) { LLM.google(key: key) },
|
|
14
|
+
'deepseek' => ->(key) { LLM.deepseek(key: key) },
|
|
15
|
+
'ollama' => ->(_key) { Providers::Ollama.new },
|
|
16
|
+
'xai' => ->(key) { LLM.xai(key: key) },
|
|
17
|
+
'opencode_zen' => ->(key) { LLM::OpencodeZen.new(key: key) },
|
|
18
|
+
'opencode_go' => ->(key) { LLM::OpencodeGo.new(key: key) },
|
|
19
|
+
'shell' => ->(_key) { Providers::Shell.new },
|
|
20
|
+
}.freeze
|
|
21
|
+
|
|
22
|
+
# Resolve the LLM provider from environment variables.
|
|
23
|
+
#
|
|
24
|
+
# Checks in order:
|
|
25
|
+
# 1. LLM_API_KEY + LLM_PROVIDER (explicit)
|
|
26
|
+
# 2. OPENCODE_API_KEY (implicit: provider = opencode_zen)
|
|
27
|
+
# 3. ANTHROPIC_API_KEY (implicit: provider = anthropic)
|
|
28
|
+
# 4. OPENAI_API_KEY (implicit: provider = openai)
|
|
29
|
+
# 5. GOOGLE_API_KEY (implicit: provider = google)
|
|
30
|
+
# 6. OLLAMA_HOST (implicit: provider = ollama, local inference)
|
|
31
|
+
#
|
|
32
|
+
# Returns nil if no key is found. Error is deferred to the caller.
|
|
33
|
+
def self.guess_from_env
|
|
34
|
+
if ENV['LLM_API_KEY']
|
|
35
|
+
key = ENV['LLM_API_KEY']
|
|
36
|
+
name = ENV.fetch('LLM_PROVIDER', 'opencode_zen').downcase
|
|
37
|
+
elsif ENV['OPENCODE_API_KEY']
|
|
38
|
+
key = ENV['OPENCODE_API_KEY']
|
|
39
|
+
name = 'opencode_zen'
|
|
40
|
+
elsif ENV['ANTHROPIC_API_KEY']
|
|
41
|
+
key = ENV['ANTHROPIC_API_KEY']
|
|
42
|
+
name = 'anthropic'
|
|
43
|
+
elsif ENV['OPENAI_API_KEY']
|
|
44
|
+
key = ENV['OPENAI_API_KEY']
|
|
45
|
+
name = 'openai'
|
|
46
|
+
elsif ENV['GOOGLE_API_KEY']
|
|
47
|
+
key = ENV['GOOGLE_API_KEY']
|
|
48
|
+
name = 'google'
|
|
49
|
+
elsif ENV['OLLAMA_HOST']
|
|
50
|
+
key = 'none'
|
|
51
|
+
name = 'ollama'
|
|
52
|
+
else
|
|
53
|
+
return nil
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
factory = Providers::ALL[name]
|
|
57
|
+
raise "Unknown LLM provider: #{name}. Available: #{Providers::ALL.keys.join(', ')}" unless factory
|
|
58
|
+
|
|
59
|
+
factory.call(key)
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "brute"
|
|
5
|
+
|
|
6
|
+
require "async"
|
|
7
|
+
require "async/queue"
|
|
8
|
+
require "async/barrier"
|
|
9
|
+
require "async/semaphore"
|
|
10
|
+
|
|
11
|
+
module Brute
|
|
12
|
+
module Queue
|
|
13
|
+
# A queue that dequeues Step objects and runs them, honoring cancellation.
|
|
14
|
+
#
|
|
15
|
+
# Composes four async primitives:
|
|
16
|
+
# - An inbox (Async::Queue) that holds pending steps
|
|
17
|
+
# - A barrier (Async::Barrier) that tracks every task the queue spawns
|
|
18
|
+
# - A semaphore (Async::Semaphore) parented to the barrier, limiting concurrency
|
|
19
|
+
# - Workers — long-lived tasks that dequeue from the inbox and run steps
|
|
20
|
+
#
|
|
21
|
+
# The barrier-semaphore composition via parent: means every task the
|
|
22
|
+
# semaphore spawns is also tracked by the barrier. One call site
|
|
23
|
+
# (semaphore.async), two guarantees (scoped lifetime + bounded concurrency).
|
|
24
|
+
#
|
|
25
|
+
class BaseQueue
|
|
26
|
+
attr_reader :steps
|
|
27
|
+
|
|
28
|
+
def initialize(concurrency:, worker_count:, parent: Async::Task.current)
|
|
29
|
+
@steps = []
|
|
30
|
+
@inbox = Async::Queue.new
|
|
31
|
+
@barrier = Async::Barrier.new(parent: parent)
|
|
32
|
+
@semaphore = Async::Semaphore.new(concurrency, parent: @barrier)
|
|
33
|
+
@worker_count = worker_count
|
|
34
|
+
@started = false
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def <<(step)
|
|
38
|
+
@steps << step
|
|
39
|
+
@inbox.push(step)
|
|
40
|
+
self
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def first = @steps.first
|
|
44
|
+
def last = @steps.last
|
|
45
|
+
|
|
46
|
+
def start
|
|
47
|
+
return self if @started
|
|
48
|
+
@started = true
|
|
49
|
+
|
|
50
|
+
@worker_count.times do
|
|
51
|
+
@barrier.async do
|
|
52
|
+
while (step = @inbox.dequeue)
|
|
53
|
+
@semaphore.async do |task|
|
|
54
|
+
step.call(task)
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
self
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Graceful: stop accepting, wait for running work to finish.
|
|
63
|
+
def drain
|
|
64
|
+
@inbox.close
|
|
65
|
+
@barrier.wait
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Hard: close inbox, cancel pending steps, cancel running work.
|
|
69
|
+
def cancel
|
|
70
|
+
@inbox.close
|
|
71
|
+
@steps.each do |step|
|
|
72
|
+
step.cancel if step.state == :pending
|
|
73
|
+
end
|
|
74
|
+
@barrier.cancel
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
test do
|
|
81
|
+
class CountStep < Brute::Loop::Step
|
|
82
|
+
def perform(task)
|
|
83
|
+
@attributes[:counter] << @attributes[:value]
|
|
84
|
+
@attributes[:value]
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
class SleepStep < Brute::Loop::Step
|
|
89
|
+
def perform(task)
|
|
90
|
+
sleep(@attributes[:duration])
|
|
91
|
+
"slept"
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# -- enqueue --
|
|
96
|
+
|
|
97
|
+
it "appends steps to the steps list" do
|
|
98
|
+
Sync do
|
|
99
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
100
|
+
q << CountStep.new(counter: [], value: 1)
|
|
101
|
+
q.steps.size.should == 1
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
it "returns self from <<" do
|
|
106
|
+
Sync do
|
|
107
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
108
|
+
(q << CountStep.new(counter: [], value: 1)).should.be.identical_to q
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# -- first / last --
|
|
113
|
+
|
|
114
|
+
it "returns the first step" do
|
|
115
|
+
Sync do
|
|
116
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
117
|
+
s1 = CountStep.new(counter: [], value: 1)
|
|
118
|
+
s2 = CountStep.new(counter: [], value: 2)
|
|
119
|
+
q << s1 << s2
|
|
120
|
+
q.first.should.be.identical_to s1
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
it "returns the last step" do
|
|
125
|
+
Sync do
|
|
126
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
127
|
+
s1 = CountStep.new(counter: [], value: 1)
|
|
128
|
+
s2 = CountStep.new(counter: [], value: 2)
|
|
129
|
+
q << s1 << s2
|
|
130
|
+
q.last.should.be.identical_to s2
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
# -- start + drain --
|
|
135
|
+
|
|
136
|
+
it "runs steps to completion on drain" do
|
|
137
|
+
Sync do
|
|
138
|
+
results = []
|
|
139
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
140
|
+
q << CountStep.new(counter: results, value: "a")
|
|
141
|
+
q << CountStep.new(counter: results, value: "b")
|
|
142
|
+
q.start
|
|
143
|
+
q.drain
|
|
144
|
+
results.should == ["a", "b"]
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
it "completes all steps" do
|
|
149
|
+
Sync do
|
|
150
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
151
|
+
q << CountStep.new(counter: [], value: 1)
|
|
152
|
+
q << CountStep.new(counter: [], value: 2)
|
|
153
|
+
q.start
|
|
154
|
+
q.drain
|
|
155
|
+
q.steps.all? { |s| s.state == :completed }.should.be.true
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
it "captures results on each step" do
|
|
160
|
+
Sync do
|
|
161
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
162
|
+
q << CountStep.new(counter: [], value: 42)
|
|
163
|
+
q.start
|
|
164
|
+
q.drain
|
|
165
|
+
q.first.result.should == 42
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
it "is idempotent on start" do
|
|
170
|
+
Sync do
|
|
171
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
172
|
+
q << CountStep.new(counter: [], value: 1)
|
|
173
|
+
q.start
|
|
174
|
+
q.start # should not double-spawn workers
|
|
175
|
+
q.drain
|
|
176
|
+
q.steps.size.should == 1
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# -- cancel --
|
|
181
|
+
|
|
182
|
+
it "marks pending steps as cancelled" do
|
|
183
|
+
Sync do
|
|
184
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 1, worker_count: 1)
|
|
185
|
+
s1 = SleepStep.new(duration: 10)
|
|
186
|
+
s2 = SleepStep.new(duration: 10)
|
|
187
|
+
s3 = SleepStep.new(duration: 10)
|
|
188
|
+
q << s1 << s2 << s3
|
|
189
|
+
q.start
|
|
190
|
+
sleep 0.01 # let worker pick up s1
|
|
191
|
+
q.cancel
|
|
192
|
+
# s2 and s3 should be cancelled (they were pending)
|
|
193
|
+
[s2, s3].all? { |s| s.state == :cancelled }.should.be.true
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# -- concurrent execution --
|
|
198
|
+
|
|
199
|
+
it "runs multiple steps concurrently" do
|
|
200
|
+
Sync do
|
|
201
|
+
started = []
|
|
202
|
+
done = []
|
|
203
|
+
|
|
204
|
+
steps = 3.times.map do |i|
|
|
205
|
+
Class.new(Brute::Loop::Step) do
|
|
206
|
+
define_method(:perform) do |task|
|
|
207
|
+
started << i
|
|
208
|
+
sleep 0.05
|
|
209
|
+
done << i
|
|
210
|
+
i
|
|
211
|
+
end
|
|
212
|
+
end.new
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
q = Brute::Queue::BaseQueue.new(concurrency: 3, worker_count: 3)
|
|
216
|
+
steps.each { |s| q << s }
|
|
217
|
+
q.start
|
|
218
|
+
q.drain
|
|
219
|
+
done.size.should == 3
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Brute
|
|
4
|
+
module Queue
|
|
4
5
|
# Per-file serialization queue for concurrent tool execution.
|
|
5
6
|
#
|
|
6
7
|
# When tools run in parallel (via threads or async fibers), multiple tools
|
|
@@ -62,38 +63,39 @@ module Brute
|
|
|
62
63
|
|
|
63
64
|
private
|
|
64
65
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
66
|
+
# Resolve a file path to a canonical key.
|
|
67
|
+
# Uses File.realpath to follow symlinks so that aliases to the
|
|
68
|
+
# same underlying file share one mutex. Falls back to
|
|
69
|
+
# File.expand_path for files that don't exist yet (e.g., new writes).
|
|
70
|
+
def canonical_path(path)
|
|
71
|
+
resolved = File.expand_path(path)
|
|
72
|
+
begin
|
|
73
|
+
File.realpath(resolved)
|
|
74
|
+
rescue Errno::ENOENT
|
|
75
|
+
resolved
|
|
76
|
+
end
|
|
75
77
|
end
|
|
76
|
-
end
|
|
77
78
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
79
|
+
# Get (or create) a mutex for a file path and increment the waiter count.
|
|
80
|
+
def acquire_mutex(key)
|
|
81
|
+
@guard.synchronize do
|
|
82
|
+
@mutexes[key] ||= Mutex.new
|
|
83
|
+
@waiters[key] += 1
|
|
84
|
+
@mutexes[key]
|
|
85
|
+
end
|
|
84
86
|
end
|
|
85
|
-
end
|
|
86
87
|
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
88
|
+
# Decrement the waiter count and clean up the mutex if no one else needs it.
|
|
89
|
+
def release_mutex(key)
|
|
90
|
+
@guard.synchronize do
|
|
91
|
+
@waiters[key] -= 1
|
|
92
|
+
if @waiters[key] <= 0
|
|
93
|
+
@mutexes.delete(key)
|
|
94
|
+
@waiters.delete(key)
|
|
95
|
+
end
|
|
94
96
|
end
|
|
95
97
|
end
|
|
96
|
-
end
|
|
97
98
|
end
|
|
98
99
|
end
|
|
100
|
+
end
|
|
99
101
|
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "brute"
|
|
5
|
+
|
|
6
|
+
module Brute
|
|
7
|
+
module Queue
|
|
8
|
+
# A queue that processes steps concurrently up to a limit.
|
|
9
|
+
# Workers match concurrency slots.
|
|
10
|
+
class ParallelQueue < BaseQueue
|
|
11
|
+
def initialize(concurrency: 4, parent: Async::Task.current)
|
|
12
|
+
super(concurrency: concurrency, worker_count: concurrency, parent: parent)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
test do
|
|
19
|
+
it "runs steps concurrently" do
|
|
20
|
+
Sync do
|
|
21
|
+
concurrent = 0
|
|
22
|
+
max_concurrent = 0
|
|
23
|
+
|
|
24
|
+
steps = 4.times.map do
|
|
25
|
+
Class.new(Brute::Loop::Step) do
|
|
26
|
+
define_method(:perform) do |task|
|
|
27
|
+
concurrent += 1
|
|
28
|
+
max_concurrent = [max_concurrent, concurrent].max
|
|
29
|
+
sleep 0.05
|
|
30
|
+
concurrent -= 1
|
|
31
|
+
end
|
|
32
|
+
end.new
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
q = Brute::Queue::ParallelQueue.new(concurrency: 4)
|
|
36
|
+
steps.each { |s| q << s }
|
|
37
|
+
q.start
|
|
38
|
+
q.drain
|
|
39
|
+
(max_concurrent > 1).should.be.true
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
it "limits concurrency to the specified amount" do
|
|
44
|
+
Sync do
|
|
45
|
+
concurrent = 0
|
|
46
|
+
max_concurrent = 0
|
|
47
|
+
|
|
48
|
+
steps = 8.times.map do
|
|
49
|
+
Class.new(Brute::Loop::Step) do
|
|
50
|
+
define_method(:perform) do |task|
|
|
51
|
+
concurrent += 1
|
|
52
|
+
max_concurrent = [max_concurrent, concurrent].max
|
|
53
|
+
sleep 0.05
|
|
54
|
+
concurrent -= 1
|
|
55
|
+
end
|
|
56
|
+
end.new
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
q = Brute::Queue::ParallelQueue.new(concurrency: 2)
|
|
60
|
+
steps.each { |s| q << s }
|
|
61
|
+
q.start
|
|
62
|
+
q.drain
|
|
63
|
+
(max_concurrent <= 2).should.be.true
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "brute"
|
|
5
|
+
|
|
6
|
+
module Brute
|
|
7
|
+
module Queue
|
|
8
|
+
# A queue that processes steps one at a time, in order.
|
|
9
|
+
# One worker, one concurrency slot.
|
|
10
|
+
class SequentialQueue < BaseQueue
|
|
11
|
+
def initialize(parent: Async::Task.current)
|
|
12
|
+
super(concurrency: 1, worker_count: 1, parent: parent)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
test do
|
|
19
|
+
class OrderStep < Brute::Loop::Step
|
|
20
|
+
def perform(task)
|
|
21
|
+
@attributes[:log] << @attributes[:value]
|
|
22
|
+
sleep(@attributes[:delay]) if @attributes[:delay]
|
|
23
|
+
@attributes[:value]
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
it "processes steps in order" do
|
|
28
|
+
Sync do
|
|
29
|
+
log = []
|
|
30
|
+
q = Brute::Queue::SequentialQueue.new
|
|
31
|
+
q << OrderStep.new(log: log, value: "a", delay: 0.01)
|
|
32
|
+
q << OrderStep.new(log: log, value: "b", delay: 0.01)
|
|
33
|
+
q << OrderStep.new(log: log, value: "c", delay: 0.01)
|
|
34
|
+
q.start
|
|
35
|
+
q.drain
|
|
36
|
+
log.should == ["a", "b", "c"]
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
it "runs only one step at a time" do
|
|
41
|
+
Sync do
|
|
42
|
+
concurrent = 0
|
|
43
|
+
max_concurrent = 0
|
|
44
|
+
|
|
45
|
+
steps = 3.times.map do
|
|
46
|
+
Class.new(Brute::Loop::Step) do
|
|
47
|
+
define_method(:perform) do |task|
|
|
48
|
+
concurrent += 1
|
|
49
|
+
max_concurrent = [max_concurrent, concurrent].max
|
|
50
|
+
sleep 0.02
|
|
51
|
+
concurrent -= 1
|
|
52
|
+
end
|
|
53
|
+
end.new
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
q = Brute::Queue::SequentialQueue.new
|
|
57
|
+
steps.each { |s| q << s }
|
|
58
|
+
q.start
|
|
59
|
+
q.drain
|
|
60
|
+
max_concurrent.should == 1
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|