coatepec 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +5 -0
- data/CHANGELOG.md +142 -0
- data/README.md +150 -25
- data/ROADMAP.md +163 -0
- data/lib/coatepec/introspection/bounded_options.rb +22 -0
- data/lib/coatepec/introspection/controller.rb +242 -0
- data/lib/coatepec/introspection/model.rb +33 -43
- data/lib/coatepec/introspection/model_resolver.rb +32 -0
- data/lib/coatepec/introspection/route_entries.rb +77 -0
- data/lib/coatepec/introspection/routes.rb +70 -20
- data/lib/coatepec/mcp/response.rb +16 -9
- data/lib/coatepec/mcp/tools.rb +107 -79
- data/lib/coatepec/mcp.rb +4 -3
- data/lib/coatepec/project.rb +11 -1
- data/lib/coatepec/spec/failure_collapser.rb +111 -0
- data/lib/coatepec/spec/flaky_checker.rb +3 -1
- data/lib/coatepec/spec/fork_strategy.rb +3 -12
- data/lib/coatepec/spec/guarded_fork_strategy.rb +11 -10
- data/lib/coatepec/spec/path_policy.rb +45 -12
- data/lib/coatepec/spec/process_strategy.rb +22 -17
- data/lib/coatepec/spec/result.rb +61 -19
- data/lib/coatepec/spec/rspec_adapter.rb +58 -0
- data/lib/coatepec/spec/runner.rb +22 -24
- data/lib/coatepec/spec/spawn_strategy.rb +7 -8
- data/lib/coatepec/test_unit/adapter.rb +132 -0
- data/lib/coatepec/test_unit/child_entry.rb +33 -0
- data/lib/coatepec/test_unit/json_reporter.rb +107 -0
- data/lib/coatepec/test_unit/line_filtering.rb +46 -0
- data/lib/coatepec/version.rb +1 -1
- data/lib/coatepec/worker/server.rb +7 -1
- data/lib/coatepec/worker_manager.rb +12 -6
- data/lib/coatepec.rb +6 -0
- metadata +18 -5
data/lib/coatepec/mcp/tools.rb
CHANGED
|
@@ -2,67 +2,73 @@
|
|
|
2
2
|
|
|
3
3
|
module Coatepec
|
|
4
4
|
module MCP
|
|
5
|
-
# The `rails_spec_run` MCP tool: runs targeted RSpec examples
|
|
6
|
-
# the warm test worker and
|
|
5
|
+
# The `rails_spec_run` MCP tool: runs targeted RSpec examples or Minitest
|
|
6
|
+
# tests (chosen from the selector paths) against the warm test worker and
|
|
7
|
+
# returns a structured pass/fail result.
|
|
7
8
|
class SpecRunTool < ::MCP::Tool
|
|
8
|
-
|
|
9
|
-
description "Run targeted RSpec examples against a warm, isolated Rails test worker"
|
|
10
|
-
annotations(read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: true)
|
|
11
|
-
input_schema(
|
|
9
|
+
INPUT_SCHEMA = {
|
|
12
10
|
properties: {
|
|
13
11
|
paths: { type: "array", items: { type: "string" }, minItems: 1, maxItems: 100 },
|
|
14
12
|
example: { type: %w[string null] },
|
|
15
13
|
seed: { type: %w[integer null], minimum: 0, maximum: 65_535 },
|
|
16
14
|
fail_fast: { type: "boolean" },
|
|
17
|
-
timeout_seconds: { type: "integer", minimum: 1, maximum: 900 }
|
|
15
|
+
timeout_seconds: { type: "integer", minimum: 1, maximum: 900 },
|
|
16
|
+
include_passing: { type: "boolean" },
|
|
17
|
+
include_stdout: { type: "string", enum: %w[failures always never] }
|
|
18
18
|
},
|
|
19
19
|
required: ["paths"],
|
|
20
20
|
additionalProperties: false
|
|
21
|
-
|
|
21
|
+
}.freeze
|
|
22
|
+
|
|
23
|
+
# Kept a constant so the vocabulary clause stays readable next to the rest of the description.
|
|
24
|
+
RSPEC_VOCABULARY =
|
|
25
|
+
"; results use RSpec vocabulary for both frameworks: a Minitest error is status failed and is " \
|
|
26
|
+
"counted in summary.failure_count (so it can exceed the failures number Minitest prints in " \
|
|
27
|
+
"stdout; summary.error_count says how many of those were errors) and a skip is pending" \
|
|
28
|
+
"; summary.assertion_count is Minitest's assertion total, null for RSpec, as is error_count"
|
|
29
|
+
|
|
30
|
+
tool_name "rails_spec_run"
|
|
31
|
+
description "Run targeted RSpec examples (spec/**/*_spec.rb) or Minitest tests (test/**/*_test.rb) " \
|
|
32
|
+
"against a warm, isolated Rails test worker; the framework is chosen from the selector " \
|
|
33
|
+
"paths; there is no separate Minitest tool#{RSPEC_VOCABULARY}" \
|
|
34
|
+
"; returns only failed and pending examples unless include_passing is true" \
|
|
35
|
+
"; failure blocks in stdout that repeat an earlier error are rolled up into one line" \
|
|
36
|
+
"; stdout is returned only for failing runs unless include_stdout is \"always\" or \"never\""
|
|
37
|
+
annotations(read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: true)
|
|
38
|
+
input_schema(**INPUT_SCHEMA)
|
|
22
39
|
|
|
23
40
|
class << self
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
# server_context; splitting it would fight the ::MCP::Tool#call contract.
|
|
27
|
-
def call(paths:, server_context:, example: nil, seed: nil, fail_fast: false, timeout_seconds: 120)
|
|
28
|
-
# rubocop:enable Metrics/ParameterLists
|
|
41
|
+
def call(paths:, server_context:, example: nil, seed: nil, fail_fast: false, timeout_seconds: 120,
|
|
42
|
+
include_passing: false, include_stdout: "failures")
|
|
29
43
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
30
44
|
data = server_context[:worker_manager].run_spec(
|
|
31
|
-
paths: paths, example: example, seed: seed, fail_fast: fail_fast, timeout_seconds: timeout_seconds
|
|
45
|
+
paths: paths, example: example, seed: seed, fail_fast: fail_fast, timeout_seconds: timeout_seconds,
|
|
46
|
+
include_passing: include_passing, include_stdout: include_stdout
|
|
32
47
|
)
|
|
33
|
-
Response.ok(data: data, meta:
|
|
48
|
+
Response.ok(data: data, meta: Response.meta(started_at))
|
|
34
49
|
rescue Coatepec::Error => e
|
|
35
50
|
Response.error(e)
|
|
36
51
|
end
|
|
37
|
-
|
|
38
|
-
private
|
|
39
|
-
|
|
40
|
-
def meta_for(server_context, started_at)
|
|
41
|
-
duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
|
|
42
|
-
{ project_root: server_context[:project_root], environment: "test", duration_ms: duration_ms }
|
|
43
|
-
end
|
|
44
52
|
end
|
|
45
53
|
end
|
|
46
54
|
|
|
47
|
-
# The `rails_runtime_status` MCP tool: reports the test worker's
|
|
48
|
-
# versions, PID, boot_id,
|
|
49
|
-
# the Rails runtime before dispatching
|
|
50
|
-
# this tool starts (and blocks on) a
|
|
55
|
+
# The `rails_runtime_status` MCP tool: reports the test worker's
|
|
56
|
+
# Ruby/Rails versions, PID, boot_id, lifecycle state and the project
|
|
57
|
+
# root. Worker::Server#handle boots the Rails runtime before dispatching
|
|
58
|
+
# any command, so the first call to this tool starts (and blocks on) a
|
|
59
|
+
# full Rails boot just like a spec run.
|
|
51
60
|
class RuntimeStatusTool < ::MCP::Tool
|
|
52
61
|
tool_name "rails_runtime_status"
|
|
53
62
|
description "Report the Coatepec test worker's identity and boot status " \
|
|
54
|
-
"(boots the warm worker if it is not up yet)"
|
|
63
|
+
"(boots the warm worker if it is not up yet); includes the project root as project_root"
|
|
55
64
|
annotations(read_only_hint: true, destructive_hint: false, idempotent_hint: true, open_world_hint: false)
|
|
56
65
|
input_schema(properties: {}, required: [], additionalProperties: false)
|
|
57
66
|
|
|
58
67
|
class << self
|
|
59
68
|
def call(server_context:)
|
|
60
69
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
61
|
-
data = server_context[:worker_manager].status
|
|
62
|
-
|
|
63
|
-
Response.ok(data: data,
|
|
64
|
-
meta: { project_root: server_context[:project_root], environment: "test",
|
|
65
|
-
duration_ms: duration_ms })
|
|
70
|
+
data = server_context[:worker_manager].status.merge(project_root: server_context[:project_root])
|
|
71
|
+
Response.ok(data: data, meta: Response.meta(started_at))
|
|
66
72
|
rescue Coatepec::Error => e
|
|
67
73
|
Response.error(e)
|
|
68
74
|
end
|
|
@@ -84,27 +90,20 @@ module Coatepec
|
|
|
84
90
|
def call(server_context:)
|
|
85
91
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
86
92
|
data = server_context[:worker_manager].restart!
|
|
87
|
-
|
|
88
|
-
Response.ok(data: data,
|
|
89
|
-
meta: { project_root: server_context[:project_root], environment: "test",
|
|
90
|
-
duration_ms: duration_ms })
|
|
93
|
+
Response.ok(data: data, meta: Response.meta(started_at))
|
|
91
94
|
rescue Coatepec::Error => e
|
|
92
95
|
Response.error(e)
|
|
93
96
|
end
|
|
94
97
|
end
|
|
95
98
|
end
|
|
96
99
|
|
|
97
|
-
# The `rails_spec_flaky_check` MCP tool: runs targeted RSpec examples
|
|
98
|
-
# multiple times with independently random seeds and
|
|
99
|
-
#
|
|
100
|
-
# tool from rails_spec_run for the same reason rails_spec_profile
|
|
101
|
-
# see docs/superpowers/specs/2026-08-13-flaky-spec-detection-design.md.
|
|
100
|
+
# The `rails_spec_flaky_check` MCP tool: runs targeted RSpec examples or
|
|
101
|
+
# Minitest tests multiple times with independently random seeds and
|
|
102
|
+
# reports which tests' pass/fail status was inconsistent across rounds.
|
|
103
|
+
# Separate tool from rails_spec_run for the same reason rails_spec_profile
|
|
104
|
+
# is -- see docs/superpowers/specs/2026-08-13-flaky-spec-detection-design.md.
|
|
102
105
|
class FlakyCheckTool < ::MCP::Tool
|
|
103
|
-
|
|
104
|
-
description "Run targeted RSpec examples multiple times with random seeds to detect order-dependent or " \
|
|
105
|
-
"intermittent flakiness, reporting which examples' status was inconsistent across runs"
|
|
106
|
-
annotations(read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: true)
|
|
107
|
-
input_schema(
|
|
106
|
+
INPUT_SCHEMA = {
|
|
108
107
|
properties: {
|
|
109
108
|
paths: { type: "array", items: { type: "string" }, minItems: 1, maxItems: 100 },
|
|
110
109
|
example: { type: %w[string null] },
|
|
@@ -113,7 +112,14 @@ module Coatepec
|
|
|
113
112
|
},
|
|
114
113
|
required: ["paths"],
|
|
115
114
|
additionalProperties: false
|
|
116
|
-
|
|
115
|
+
}.freeze
|
|
116
|
+
|
|
117
|
+
tool_name "rails_spec_flaky_check"
|
|
118
|
+
description "Run targeted RSpec examples or Minitest tests multiple times with random seeds to detect " \
|
|
119
|
+
"order-dependent or intermittent flakiness, reporting which tests' status was inconsistent " \
|
|
120
|
+
"across runs; the framework is chosen from the selector paths"
|
|
121
|
+
annotations(read_only_hint: false, destructive_hint: true, idempotent_hint: false, open_world_hint: true)
|
|
122
|
+
input_schema(**INPUT_SCHEMA)
|
|
117
123
|
|
|
118
124
|
class << self
|
|
119
125
|
def call(paths:, server_context:, example: nil, timeout_seconds: 120, runs: 5)
|
|
@@ -121,17 +127,10 @@ module Coatepec
|
|
|
121
127
|
data = server_context[:worker_manager].check_flaky(
|
|
122
128
|
paths: paths, example: example, timeout_seconds: timeout_seconds, runs: runs
|
|
123
129
|
)
|
|
124
|
-
Response.ok(data: data, meta:
|
|
130
|
+
Response.ok(data: data, meta: Response.meta(started_at))
|
|
125
131
|
rescue Coatepec::Error => e
|
|
126
132
|
Response.error(e)
|
|
127
133
|
end
|
|
128
|
-
|
|
129
|
-
private
|
|
130
|
-
|
|
131
|
-
def meta_for(server_context, started_at)
|
|
132
|
-
duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
|
|
133
|
-
{ project_root: server_context[:project_root], environment: "test", duration_ms: duration_ms }
|
|
134
|
-
end
|
|
135
134
|
end
|
|
136
135
|
end
|
|
137
136
|
|
|
@@ -139,65 +138,94 @@ module Coatepec
|
|
|
139
138
|
# Rails app's routes.
|
|
140
139
|
class RoutesTool < ::MCP::Tool
|
|
141
140
|
tool_name "rails_routes"
|
|
142
|
-
description "Return a bounded, filterable list of the Rails app's routes"
|
|
141
|
+
description "Return a bounded, filterable list of the Rails app's routes, application routes only by " \
|
|
142
|
+
"default: mounted-engine routes (for example admin scaffolding) are withheld and the response's " \
|
|
143
|
+
"engines_excluded says how many routes matching the query the engines filter held back " \
|
|
144
|
+
"(application routes, under \"only\"); pass " \
|
|
145
|
+
"engines: \"include\" to list both or \"only\" for engine routes alone. Engine routes are " \
|
|
146
|
+
"expanded one level deep, carry the mount point in their path, and name their engine in the " \
|
|
147
|
+
"engine field (null for an application route; query also matches that field); returns columns " \
|
|
148
|
+
"(name, verb, path, controller, action, engine) and up to limit rows (default 100) in that " \
|
|
149
|
+
"order, paths without the (.:format) suffix Rails appends, with next_offset -- the offset to " \
|
|
150
|
+
"pass back for the next page, null on the last one"
|
|
143
151
|
annotations(read_only_hint: true, destructive_hint: false, idempotent_hint: true, open_world_hint: false)
|
|
144
152
|
input_schema(
|
|
145
153
|
properties: {
|
|
146
154
|
query: { type: %w[string null] },
|
|
147
155
|
limit: { type: "integer", minimum: 1, maximum: 200 },
|
|
148
|
-
offset: { type: "integer", minimum: 0 }
|
|
156
|
+
offset: { type: "integer", minimum: 0 },
|
|
157
|
+
engines: { type: "string", enum: %w[include exclude only] }
|
|
149
158
|
},
|
|
150
159
|
required: [],
|
|
151
160
|
additionalProperties: false
|
|
152
161
|
)
|
|
153
162
|
|
|
154
163
|
class << self
|
|
155
|
-
def call(server_context:, query: nil, limit:
|
|
164
|
+
def call(server_context:, query: nil, limit: 100, offset: 0, engines: Introspection::Routes::DEFAULT_ENGINES)
|
|
156
165
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
157
|
-
data = server_context[:worker_manager].routes(query: query, limit: limit, offset: offset)
|
|
158
|
-
Response.ok(data: data, meta:
|
|
166
|
+
data = server_context[:worker_manager].routes(query: query, limit: limit, offset: offset, engines: engines)
|
|
167
|
+
Response.ok(data: data, meta: Response.meta(started_at))
|
|
159
168
|
rescue Coatepec::Error => e
|
|
160
169
|
Response.error(e)
|
|
161
170
|
end
|
|
162
|
-
|
|
163
|
-
private
|
|
164
|
-
|
|
165
|
-
def meta_for(server_context, started_at)
|
|
166
|
-
duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at) * 1000).round
|
|
167
|
-
{ project_root: server_context[:project_root], environment: "test", duration_ms: duration_ms }
|
|
168
|
-
end
|
|
169
171
|
end
|
|
170
172
|
end
|
|
171
173
|
|
|
172
|
-
# The `rails_model` MCP tool: returns an ActiveRecord model's
|
|
173
|
-
#
|
|
174
|
+
# The `rails_model` MCP tool: returns an ActiveRecord model's table
|
|
175
|
+
# metadata and list counts, plus any lists named in fields.
|
|
174
176
|
class ModelTool < ::MCP::Tool
|
|
175
177
|
tool_name "rails_model"
|
|
176
|
-
description "Return
|
|
177
|
-
"without
|
|
178
|
+
description "Return an ActiveRecord model's name, table, primary key, abstract_class and counts by default " \
|
|
179
|
+
"(the size of its columns, associations, validators and enums lists, each capped at 200), without " \
|
|
180
|
+
"row data; pass fields (any of columns, associations, validators, enums) to include those lists -- " \
|
|
181
|
+
"an omitted or empty fields returns no lists, only counts; validators are de-duplicated by class, " \
|
|
182
|
+
"attributes and options, so the list holds distinct validators and can be shorter than " \
|
|
183
|
+
"klass.validators; an array-valued validator option longer than 20 entries keeps its first 20 with " \
|
|
184
|
+
"<option>_count and <option>_truncated beside it"
|
|
178
185
|
annotations(read_only_hint: true, destructive_hint: false, idempotent_hint: true, open_world_hint: false)
|
|
179
186
|
input_schema(
|
|
180
187
|
properties: {
|
|
181
|
-
name: { type: "string", pattern: '^[A-Z]\w*(?:::[A-Z]\w*)*$' }
|
|
188
|
+
name: { type: "string", pattern: '^[A-Z]\w*(?:::[A-Z]\w*)*$' },
|
|
189
|
+
fields: { type: "array", items: { type: "string", enum: %w[columns associations validators enums] },
|
|
190
|
+
uniqueItems: true }
|
|
182
191
|
},
|
|
183
192
|
required: ["name"],
|
|
184
193
|
additionalProperties: false
|
|
185
194
|
)
|
|
186
195
|
|
|
187
196
|
class << self
|
|
188
|
-
def call(name:, server_context:)
|
|
197
|
+
def call(name:, server_context:, fields: nil)
|
|
189
198
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
190
|
-
data = server_context[:worker_manager].model(name: name)
|
|
191
|
-
Response.ok(data: data, meta:
|
|
199
|
+
data = server_context[:worker_manager].model(name: name, fields: fields)
|
|
200
|
+
Response.ok(data: data, meta: Response.meta(started_at))
|
|
192
201
|
rescue Coatepec::Error => e
|
|
193
202
|
Response.error(e)
|
|
194
203
|
end
|
|
204
|
+
end
|
|
205
|
+
end
|
|
195
206
|
|
|
196
|
-
|
|
207
|
+
# The `rails_controller` MCP tool: returns a controller's actions, action
|
|
208
|
+
# callbacks, included concerns, and the routes reaching each action.
|
|
209
|
+
class ControllerTool < ::MCP::Tool
|
|
210
|
+
tool_name "rails_controller"
|
|
211
|
+
description "Return a Rails controller's actions, action callbacks, concerns, and the routes " \
|
|
212
|
+
"reaching each action, including unroutable actions and routes with no matching action"
|
|
213
|
+
annotations(read_only_hint: true, destructive_hint: false, idempotent_hint: true, open_world_hint: false)
|
|
214
|
+
input_schema(
|
|
215
|
+
properties: {
|
|
216
|
+
name: { type: "string", pattern: '^[A-Z]\w*(?:::[A-Z]\w*)*$' }
|
|
217
|
+
},
|
|
218
|
+
required: ["name"],
|
|
219
|
+
additionalProperties: false
|
|
220
|
+
)
|
|
197
221
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
222
|
+
class << self
|
|
223
|
+
def call(name:, server_context:)
|
|
224
|
+
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
225
|
+
data = server_context[:worker_manager].controller(name: name)
|
|
226
|
+
Response.ok(data: data, meta: Response.meta(started_at))
|
|
227
|
+
rescue Coatepec::Error => e
|
|
228
|
+
Response.error(e)
|
|
201
229
|
end
|
|
202
230
|
end
|
|
203
231
|
end
|
data/lib/coatepec/mcp.rb
CHANGED
|
@@ -13,14 +13,15 @@ require_relative "mcp/tools"
|
|
|
13
13
|
|
|
14
14
|
module Coatepec
|
|
15
15
|
# Wires the `rails_spec_run`, `rails_runtime_status`, `rails_runtime_restart`,
|
|
16
|
-
# `rails_spec_flaky_check`, `rails_routes`, and `
|
|
17
|
-
# `::MCP::Server` instance backed by the given project's worker manager.
|
|
16
|
+
# `rails_spec_flaky_check`, `rails_routes`, `rails_model`, and `rails_controller`
|
|
17
|
+
# tools into an `::MCP::Server` instance backed by the given project's worker manager.
|
|
18
18
|
module MCP
|
|
19
19
|
def self.build_server(project:, worker_manager:)
|
|
20
20
|
::MCP::Server.new(
|
|
21
21
|
name: "coatepec",
|
|
22
22
|
version: Coatepec::VERSION,
|
|
23
|
-
tools: [SpecRunTool, RuntimeStatusTool, RuntimeRestartTool, FlakyCheckTool,
|
|
23
|
+
tools: [SpecRunTool, RuntimeStatusTool, RuntimeRestartTool, FlakyCheckTool,
|
|
24
|
+
RoutesTool, ModelTool, ControllerTool],
|
|
24
25
|
server_context: { worker_manager: worker_manager, project_root: project.root }
|
|
25
26
|
)
|
|
26
27
|
end
|
data/lib/coatepec/project.rb
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module Coatepec
|
|
4
4
|
# A Rails application checkout rooted at an absolute path (must contain a
|
|
5
|
-
# Gemfile); knows where its own and pack/engine/gem spec
|
|
5
|
+
# Gemfile); knows where its own and pack/engine/gem spec and test
|
|
6
|
+
# directories live.
|
|
6
7
|
class Project
|
|
7
8
|
attr_reader :root
|
|
8
9
|
|
|
@@ -23,6 +24,15 @@ module Coatepec
|
|
|
23
24
|
].select { |path| File.directory?(path) }
|
|
24
25
|
end
|
|
25
26
|
|
|
27
|
+
def test_root_candidates
|
|
28
|
+
[
|
|
29
|
+
File.join(root, "test"),
|
|
30
|
+
*Dir.glob(File.join(root, "packs/*/test")),
|
|
31
|
+
*Dir.glob(File.join(root, "engines/*/test")),
|
|
32
|
+
*Dir.glob(File.join(root, "gems/*/test"))
|
|
33
|
+
].select { |path| File.directory?(path) }
|
|
34
|
+
end
|
|
35
|
+
|
|
26
36
|
def config
|
|
27
37
|
@config ||= ProjectConfig.new(root)
|
|
28
38
|
end
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Coatepec
|
|
4
|
+
module Spec
|
|
5
|
+
# Collapses failure blocks that repeat one error verbatim (a broken layout erroring every controller
|
|
6
|
+
# test, say) into the first block plus a roll-up line naming the other tests. Unrecognised text is kept.
|
|
7
|
+
module FailureCollapser
|
|
8
|
+
MINITEST_START = /\A(?:Error|Failure):\z/
|
|
9
|
+
MINITEST_ID = /\A(\S+#\S+?)(?: \[[^\]]*\])?:\z/
|
|
10
|
+
MINITEST_END = /\A\S*\brails test \S+:\d+\z/
|
|
11
|
+
RSPEC_START = /\A \d+\) (.+)\z/
|
|
12
|
+
RSPEC_END = /\A(?: \d+\) |Finished in |Failed examples:)/
|
|
13
|
+
# Backtrace frames and RSpec's "Failure/Error: <source line>" differ per test; the error text is what must match.
|
|
14
|
+
IGNORED = %r{\A\s+(?:# )?\S+:\d+(?::in .*)?\z|\A\s+Failure/Error: }
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
def call(text)
|
|
19
|
+
# read_bounded's tail byteslice can sever a multibyte character, and matching a regexp on that raises.
|
|
20
|
+
return text unless text.valid_encoding?
|
|
21
|
+
|
|
22
|
+
lines = text.lines
|
|
23
|
+
blocks = minitest_blocks(lines)
|
|
24
|
+
blocks = rspec_blocks(lines) if blocks.empty?
|
|
25
|
+
return text if blocks.size < 2
|
|
26
|
+
|
|
27
|
+
rewrite(lines, blocks)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# A block runs from the Error:/Failure: line through Rails' "bin/rails test path:LINE" rerun line.
|
|
31
|
+
def minitest_blocks(lines)
|
|
32
|
+
blocks = []
|
|
33
|
+
i = 0
|
|
34
|
+
while i < lines.size
|
|
35
|
+
id, stop = minitest_block_end(lines, i)
|
|
36
|
+
blocks << block(i, stop, id, lines[(i + 2)...stop]) if stop
|
|
37
|
+
i = (stop || i) + 1
|
|
38
|
+
end
|
|
39
|
+
blocks
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def minitest_block_end(lines, index)
|
|
43
|
+
id = lines[index].chomp.match?(MINITEST_START) && lines[index + 1]&.chomp&.[](MINITEST_ID, 1)
|
|
44
|
+
return [nil, nil] unless id
|
|
45
|
+
|
|
46
|
+
[id, ((index + 2)...lines.size).find { |j| lines[j].chomp.match?(MINITEST_END) }]
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# A block runs from " N) description" (after the Failures: heading only) to the next such line or the summary.
|
|
50
|
+
def rspec_blocks(lines)
|
|
51
|
+
rspec_starts(lines).map do |i|
|
|
52
|
+
last = rspec_block_end(lines, i)
|
|
53
|
+
block(i, last, lines[i].chomp[RSPEC_START, 1], lines[(i + 1)..last])
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Only the numbered entries under the Failures: heading, never RSpec's Pending: section.
|
|
58
|
+
def rspec_starts(lines)
|
|
59
|
+
failures_at = lines.index { |line| line.chomp == "Failures:" }
|
|
60
|
+
return [] unless failures_at
|
|
61
|
+
|
|
62
|
+
((failures_at + 1)...lines.size).select { |i| lines[i].chomp.match?(RSPEC_START) }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Stops before the next failure or the summary, then backs over the blank lines in between.
|
|
66
|
+
def rspec_block_end(lines, start)
|
|
67
|
+
stop = ((start + 1)...lines.size).find { |j| lines[j].chomp.match?(RSPEC_END) } || lines.size
|
|
68
|
+
stop -= 1 while stop > start + 1 && lines[stop - 1].strip.empty?
|
|
69
|
+
stop - 1
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def block(first, last, id, body)
|
|
73
|
+
key = body.map(&:chomp).reject { |line| line.strip.empty? || line.match?(IGNORED) }
|
|
74
|
+
{ first: first, last: last, id: id, key: key }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def rewrite(lines, blocks)
|
|
78
|
+
skipped = {}
|
|
79
|
+
notes = {}
|
|
80
|
+
blocks.group_by { |b| b[:key] }.each_value do |kept, *dropped|
|
|
81
|
+
next if dropped.empty?
|
|
82
|
+
|
|
83
|
+
dropped.each { |b| skip_block(lines, b, skipped) }
|
|
84
|
+
notes[kept[:last]] = note_for(dropped)
|
|
85
|
+
end
|
|
86
|
+
render(lines, skipped, notes)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def render(lines, skipped, notes)
|
|
90
|
+
lines.each_with_index.reject { |_, j| skipped[j] }.map { |line, j| attach_note(line, notes[j]) }.join
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# The blank line that separates the dropped block from the next one goes with it.
|
|
94
|
+
def skip_block(lines, dropped, skipped)
|
|
95
|
+
(dropped[:first]..dropped[:last]).each { |j| skipped[j] = true }
|
|
96
|
+
skipped[dropped[:last] + 1] = true if lines[dropped[:last] + 1]&.strip&.empty?
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def note_for(dropped)
|
|
100
|
+
noun = dropped.size == 1 ? "test" : "tests"
|
|
101
|
+
"#{dropped.size} more #{noun} failed with this same error: #{dropped.map { |b| b[:id] }.join(", ")}"
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def attach_note(line, note)
|
|
105
|
+
return line unless note
|
|
106
|
+
|
|
107
|
+
"#{line.chomp}\n#{note}\n"
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
@@ -44,9 +44,11 @@ module Coatepec
|
|
|
44
44
|
|
|
45
45
|
def run_one_round(paths, example, timeout_seconds)
|
|
46
46
|
seed = SecureRandom.random_number(65_536)
|
|
47
|
+
# Every round needs the full roster: a pass in one round is what makes a later failure flaky.
|
|
48
|
+
# stdout is never read from a round, so it is not captured into the result.
|
|
47
49
|
result = @runner.run(
|
|
48
50
|
paths: paths, example: example, seed: seed, fail_fast: false,
|
|
49
|
-
timeout_seconds: timeout_seconds
|
|
51
|
+
timeout_seconds: timeout_seconds, include_passing: true, include_stdout: "never"
|
|
50
52
|
)
|
|
51
53
|
{ seed: seed, status: result[:status], examples: result[:examples] }
|
|
52
54
|
end
|
|
@@ -2,29 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
module Coatepec
|
|
4
4
|
module Spec
|
|
5
|
-
# Runs
|
|
5
|
+
# Runs the test framework in a `Process.fork`ed child (Linux only): cheap and reuses
|
|
6
6
|
# the warm worker's loaded Rails boot, but isolated from the parent's
|
|
7
7
|
# ActiveRecord connections and global state.
|
|
8
8
|
class ForkStrategy < ProcessStrategy
|
|
9
9
|
private
|
|
10
10
|
|
|
11
|
-
def start(full_args, out_w, err_w)
|
|
11
|
+
def start(full_args, out_w, err_w, json_path)
|
|
12
12
|
Process.fork do
|
|
13
13
|
Process.setpgid(0, 0)
|
|
14
14
|
redirect_output(out_w, err_w)
|
|
15
15
|
# Forked children must not share the parent's live DB sockets.
|
|
16
16
|
ActiveRecord::Base.connection_handler.clear_all_connections! if defined?(ActiveRecord::Base)
|
|
17
|
-
# RSpec freezes its own "load started at" timestamp once, at the moment
|
|
18
|
-
# rspec/core.rb is first required -- in this architecture, that's when
|
|
19
|
-
# the long-lived warm worker booted, not when THIS run started. Every
|
|
20
|
-
# forked child inherits that frozen timestamp via copy-on-write, so
|
|
21
|
-
# RSpec's own "(files took N seconds to load)" reporting would
|
|
22
|
-
# otherwise measure "time since the worker booted" and grow across
|
|
23
|
-
# every run for as long as the worker stays warm. Reset it fresh
|
|
24
|
-
# before each run.
|
|
25
|
-
RSpec.configuration.start_time = RSpec::Core::Time.now
|
|
26
17
|
|
|
27
|
-
status =
|
|
18
|
+
status = @adapter.run_in_process(full_args, json_path)
|
|
28
19
|
$stdout.flush
|
|
29
20
|
$stderr.flush
|
|
30
21
|
Kernel.exit!(status)
|
|
@@ -36,13 +36,14 @@ module Coatepec
|
|
|
36
36
|
# along with a whole second result inside MCP::Response's 1 MiB cap.
|
|
37
37
|
MAX_CRASH_STDERR_BYTES = 4 * 1024
|
|
38
38
|
|
|
39
|
-
def initialize(project_root, project: nil, rails_runtime: nil)
|
|
39
|
+
def initialize(project_root, adapter: nil, project: nil, rails_runtime: nil)
|
|
40
40
|
super
|
|
41
|
-
@spawn_strategy = SpawnStrategy.new(project_root)
|
|
41
|
+
@spawn_strategy = SpawnStrategy.new(project_root, adapter: @adapter)
|
|
42
42
|
end
|
|
43
43
|
|
|
44
|
-
def run(args, timeout_seconds)
|
|
45
|
-
|
|
44
|
+
def run(args, timeout_seconds, include_passing: false, include_stdout: "failures")
|
|
45
|
+
result_options = { include_passing: include_passing, include_stdout: include_stdout }
|
|
46
|
+
return fallback_result(args, timeout_seconds, "spawn_fallback", result_options) unless guard_passes?
|
|
46
47
|
|
|
47
48
|
started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
48
49
|
begin
|
|
@@ -53,20 +54,20 @@ module Coatepec
|
|
|
53
54
|
# caller's side that is indistinguishable from a failed guard, hence
|
|
54
55
|
# the same mode. Opting into macos_fork must never surface an error
|
|
55
56
|
# that plain SpawnStrategy wouldn't have.
|
|
56
|
-
return fallback_result(args, timeout_seconds, "spawn_fallback")
|
|
57
|
+
return fallback_result(args, timeout_seconds, "spawn_fallback", result_options)
|
|
57
58
|
end
|
|
58
59
|
return result.merge(execution_mode: "fork") unless crashed?(result)
|
|
59
60
|
|
|
60
|
-
retry_after_crash(args, timeout_seconds, started_at, result)
|
|
61
|
+
retry_after_crash(args, timeout_seconds, started_at, result, result_options)
|
|
61
62
|
end
|
|
62
63
|
|
|
63
64
|
private
|
|
64
65
|
|
|
65
|
-
def retry_after_crash(args, timeout_seconds, started_at, crashed)
|
|
66
|
+
def retry_after_crash(args, timeout_seconds, started_at, crashed, result_options)
|
|
66
67
|
elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
|
|
67
68
|
remaining = [timeout_seconds - elapsed, MIN_RETRY_TIMEOUT_SECONDS].max
|
|
68
69
|
|
|
69
|
-
fallback_result(args, remaining, "spawn_after_crash")
|
|
70
|
+
fallback_result(args, remaining, "spawn_after_crash", result_options)
|
|
70
71
|
.merge(crashed_fork_stderr: crash_diagnostics(crashed))
|
|
71
72
|
end
|
|
72
73
|
|
|
@@ -79,8 +80,8 @@ module Coatepec
|
|
|
79
80
|
text.byteslice(-MAX_CRASH_STDERR_BYTES, MAX_CRASH_STDERR_BYTES)
|
|
80
81
|
end
|
|
81
82
|
|
|
82
|
-
def fallback_result(args, timeout_seconds, mode)
|
|
83
|
-
@spawn_strategy.run(args, timeout_seconds).merge(execution_mode: mode)
|
|
83
|
+
def fallback_result(args, timeout_seconds, mode, result_options)
|
|
84
|
+
@spawn_strategy.run(args, timeout_seconds, **result_options).merge(execution_mode: mode)
|
|
84
85
|
end
|
|
85
86
|
|
|
86
87
|
def guard_passes?
|
|
@@ -3,23 +3,32 @@
|
|
|
3
3
|
module Coatepec
|
|
4
4
|
module Spec
|
|
5
5
|
# Validates rails_spec_run's `paths` selectors against the allowed spec
|
|
6
|
-
# roots before any
|
|
7
|
-
# `..` traversal, symlink escapes,
|
|
8
|
-
#
|
|
6
|
+
# and test roots before any test process is started, rejecting absolute
|
|
7
|
+
# paths, `..` traversal, symlink escapes, files that are neither
|
|
8
|
+
# `_spec.rb` under a spec root nor `_test.rb` under a test root, and
|
|
9
|
+
# oversized selector lists. Also classifies the whole list as RSpec or
|
|
10
|
+
# Minitest from the selectors' shape: a list may not mix the two.
|
|
9
11
|
class PathPolicy
|
|
10
12
|
MAX_SELECTORS = 100
|
|
11
13
|
|
|
14
|
+
KINDS = {
|
|
15
|
+
rspec: { roots: :spec_root_candidates, suffix: "_spec.rb" },
|
|
16
|
+
minitest: { roots: :test_root_candidates, suffix: "_test.rb" }
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
12
19
|
def initialize(project)
|
|
13
20
|
@project = project
|
|
14
21
|
end
|
|
15
22
|
|
|
23
|
+
# Returns { selectors: [...], framework: :rspec | :minitest }.
|
|
16
24
|
def validate!(selectors)
|
|
17
25
|
raise Coatepec::Error.new(:invalid_spec_path, "No spec paths given") if selectors.nil? || selectors.empty?
|
|
18
26
|
if selectors.size > MAX_SELECTORS
|
|
19
27
|
raise Coatepec::Error.new(:invalid_spec_path, "At most #{MAX_SELECTORS} spec paths are allowed")
|
|
20
28
|
end
|
|
21
29
|
|
|
22
|
-
selectors.map { |selector| validate_one(selector) }
|
|
30
|
+
classified = selectors.map { |selector| validate_one(selector) }
|
|
31
|
+
{ selectors: classified.map { |c| c[:selector] }, framework: single_framework!(classified) }
|
|
23
32
|
end
|
|
24
33
|
|
|
25
34
|
private
|
|
@@ -31,9 +40,24 @@ module Coatepec
|
|
|
31
40
|
|
|
32
41
|
real_path = resolve_real_path!(selector, path_part)
|
|
33
42
|
reject_escape!(selector, real_path)
|
|
34
|
-
|
|
43
|
+
framework = framework_for!(selector, real_path)
|
|
44
|
+
|
|
45
|
+
{ selector: line_part ? "#{path_part}:#{line_part}" : path_part, framework: framework }
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def single_framework!(classified)
|
|
49
|
+
frameworks = classified.map { |c| c[:framework] }.uniq
|
|
50
|
+
return frameworks.first if frameworks.size == 1
|
|
35
51
|
|
|
36
|
-
|
|
52
|
+
raise Coatepec::Error.new(:mixed_test_frameworks, mixed_frameworks_message(classified))
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Names the minority selectors: the ones an agent most likely added by
|
|
56
|
+
# mistake to an otherwise single-framework call.
|
|
57
|
+
def mixed_frameworks_message(classified)
|
|
58
|
+
majority = classified.group_by { |c| c[:framework] }.max_by { |_, list| list.size }.first
|
|
59
|
+
odd = classified.reject { |c| c[:framework] == majority }.map { |c| c[:selector] }
|
|
60
|
+
"A single call may not mix RSpec and Minitest selectors; these do not match the others: #{odd.join(", ")}"
|
|
37
61
|
end
|
|
38
62
|
|
|
39
63
|
def resolve_real_path!(selector, path_part)
|
|
@@ -74,15 +98,24 @@ module Coatepec
|
|
|
74
98
|
raise Coatepec::Error.new(:invalid_spec_path, "Spec path escapes the project root: #{selector}")
|
|
75
99
|
end
|
|
76
100
|
|
|
77
|
-
|
|
78
|
-
|
|
101
|
+
# A directory under a root is that root's kind; a file must also carry
|
|
102
|
+
# the kind's suffix. The two root sets never overlap (spec/ vs test/),
|
|
103
|
+
# so the iteration order does not matter.
|
|
104
|
+
def framework_for!(selector, real_path)
|
|
105
|
+
KINDS.each do |framework, kind|
|
|
106
|
+
next unless under_any_root?(real_path, @project.public_send(kind[:roots]))
|
|
107
|
+
return framework if File.directory?(real_path) || real_path.end_with?(kind[:suffix])
|
|
108
|
+
end
|
|
79
109
|
|
|
80
|
-
raise Coatepec::Error.new(
|
|
81
|
-
|
|
110
|
+
raise Coatepec::Error.new(
|
|
111
|
+
:invalid_spec_path,
|
|
112
|
+
"Spec path is not an allowed spec/test root, a _spec.rb file under spec/, " \
|
|
113
|
+
"or a _test.rb file under test/: #{selector}"
|
|
114
|
+
)
|
|
82
115
|
end
|
|
83
116
|
|
|
84
|
-
def
|
|
85
|
-
|
|
117
|
+
def under_any_root?(real_path, roots)
|
|
118
|
+
roots.any? do |candidate|
|
|
86
119
|
real_candidate = File.realpath(candidate)
|
|
87
120
|
real_path == real_candidate || real_path.start_with?("#{real_candidate}/")
|
|
88
121
|
end
|