ruby_raider 3.13.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/runtime.yml +38 -0
- data/.github/workflows/system_tests.yml +1 -23
- data/.reek.yml +1 -0
- data/.rspec +1 -0
- data/Gemfile +2 -0
- data/README.md +16 -49
- data/RELEASE.md +22 -0
- data/bin/raider +5 -0
- data/lib/commands/scaffolding_commands.rb +2 -25
- data/lib/commands/utility_commands.rb +23 -3
- data/lib/generators/automation/automation_generator.rb +0 -5
- data/lib/generators/automation/templates/component.tt +1 -0
- data/lib/generators/automation/templates/home.tt +2 -2
- data/lib/generators/automation/templates/login.tt +6 -26
- data/lib/generators/automation/templates/page.tt +23 -18
- data/lib/generators/automation/templates/partials/home_page_selector.tt +1 -2
- data/lib/generators/automation/templates/partials/initialize_selector.tt +0 -8
- data/lib/generators/automation/templates/partials/pdp_page_selector.tt +1 -2
- data/lib/generators/automation/templates/partials/url_methods.tt +8 -2
- data/lib/generators/automation/templates/partials/visit_method.tt +0 -5
- data/lib/generators/automation/templates/pdp.tt +1 -9
- data/lib/generators/common_generator.rb +0 -2
- data/lib/generators/generator.rb +12 -19
- data/lib/generators/helper_generator.rb +21 -19
- data/lib/generators/infrastructure/templates/azure_pipelines.tt +1 -1
- data/lib/generators/infrastructure/templates/github.tt +2 -3
- data/lib/generators/infrastructure/templates/github_appium.tt +0 -8
- data/lib/generators/invoke_generators.rb +4 -23
- data/lib/generators/menu_generator.rb +1 -9
- data/lib/generators/rspec/templates/accessibility_spec.tt +1 -9
- data/lib/generators/rspec/templates/spec.tt +8 -24
- data/lib/generators/template_renderer/partial_cache.rb +3 -1
- data/lib/generators/template_renderer.rb +1 -1
- data/lib/generators/templates/common/gemfile.tt +3 -10
- data/lib/generators/templates/common/git_ignore.tt +0 -5
- data/lib/generators/templates/common/partials/mobile_config.tt +3 -1
- data/lib/generators/templates/common/partials/web_config.tt +10 -1
- data/lib/generators/templates/common/rakefile.tt +0 -15
- data/lib/generators/templates/common/read_me.tt +357 -250
- data/lib/generators/templates/common/rubocop.tt +0 -2
- data/lib/generators/templates/helpers/driver_helper.tt +8 -0
- data/lib/generators/templates/helpers/gesture_helper.tt +78 -0
- data/lib/generators/templates/helpers/network_helper.tt +521 -0
- data/lib/generators/templates/helpers/partials/quit_driver.tt +0 -2
- data/lib/generators/templates/helpers/partials/screenshot.tt +0 -2
- data/lib/generators/templates/helpers/partials/selenium_driver.tt +5 -2
- data/lib/generators/templates/helpers/screenshot_helper.tt +490 -0
- data/lib/generators/templates/helpers/spec_helper.tt +39 -29
- data/lib/generators/templates/helpers/trace_helper.tt +240 -95
- data/lib/generators/templates/helpers/video_helper.tt +3 -8
- data/lib/ruby_raider/runtime/actionability.rb +129 -0
- data/lib/ruby_raider/runtime/actionable.rb +49 -0
- data/lib/ruby_raider/runtime/by_role.rb +55 -0
- data/lib/ruby_raider/runtime/config.rb +79 -0
- data/lib/ruby_raider/runtime/context.rb +38 -0
- data/lib/ruby_raider/runtime/errors.rb +69 -0
- data/lib/ruby_raider/runtime/finders.rb +73 -0
- data/lib/ruby_raider/runtime/js/actionability.js +134 -0
- data/lib/ruby_raider/runtime/js/role_engine.js +315 -0
- data/lib/ruby_raider/runtime/locator_converter.rb +72 -0
- data/lib/ruby_raider/runtime/matchers/base.rb +222 -0
- data/lib/ruby_raider/runtime/matchers/element.rb +183 -0
- data/lib/ruby_raider/runtime/matchers/page.rb +71 -0
- data/lib/ruby_raider/runtime/matchers/presence.rb +41 -0
- data/lib/ruby_raider/runtime/matchers/retrying_block.rb +104 -0
- data/lib/ruby_raider/runtime/observer.rb +170 -0
- data/lib/ruby_raider/runtime/role_finder.rb +83 -0
- data/lib/ruby_raider/runtime/rspec.rb +78 -0
- data/lib/ruby_raider/runtime/selectors.rb +60 -0
- data/lib/ruby_raider/runtime/strict.rb +142 -0
- data/lib/ruby_raider/runtime/wait.rb +84 -0
- data/lib/ruby_raider/runtime.rb +20 -0
- data/lib/scaffolding/name_normalizer.rb +1 -1
- data/lib/scaffolding/project_detector.rb +2 -28
- data/lib/scaffolding/scaffolding.rb +0 -20
- data/lib/scaffolding/templates/component.tt +0 -14
- data/lib/scaffolding/templates/page_object.tt +2 -14
- data/lib/scaffolding/templates/spec.tt +1 -9
- data/lib/utilities/utilities.rb +10 -8
- data/lib/version +1 -1
- data/ruby_raider.gemspec +2 -1
- data/sig/commands/scaffolding_commands.rbs +0 -2
- data/sig/commands/utility_commands.rbs +6 -0
- data/sig/generators/generator.rbs +0 -4
- data/sig/generators/helper_generator.rbs +0 -1
- data/sig/generators/menu_generator.rbs +0 -1
- data/sig/ruby_raider/runtime/actionability.rbs +25 -0
- data/sig/ruby_raider/runtime/actionable.rbs +13 -0
- data/sig/ruby_raider/runtime/by_role.rbs +15 -0
- data/sig/ruby_raider/runtime/config.rbs +26 -0
- data/sig/ruby_raider/runtime/context.rbs +12 -0
- data/sig/ruby_raider/runtime/errors.rbs +36 -0
- data/sig/ruby_raider/runtime/finders.rbs +26 -0
- data/sig/ruby_raider/runtime/locator_converter.rbs +21 -0
- data/sig/ruby_raider/runtime/matchers/base.rbs +64 -0
- data/sig/ruby_raider/runtime/matchers/element.rbs +78 -0
- data/sig/ruby_raider/runtime/matchers/page.rbs +38 -0
- data/sig/ruby_raider/runtime/matchers/presence.rbs +21 -0
- data/sig/ruby_raider/runtime/matchers/retrying_block.rbs +27 -0
- data/sig/ruby_raider/runtime/observer.rbs +46 -0
- data/sig/ruby_raider/runtime/role_finder.rbs +18 -0
- data/sig/ruby_raider/runtime/selectors.rbs +21 -0
- data/sig/ruby_raider/runtime/strict.rbs +35 -0
- data/sig/ruby_raider/runtime/wait.rbs +17 -0
- data/sig/ruby_raider/runtime.rbs +2 -0
- data/sig/scaffolding/project_detector.rbs +0 -4
- data/sig/scaffolding/scaffolding.rbs +0 -5
- data/sig/utilities/utilities.rbs +2 -1
- data/sig/vendor/selenium_webdriver.rbs +71 -0
- data/spec/commands/raider_commands_spec.rb +38 -3
- data/spec/commands/scaffolding_commands_spec.rb +27 -0
- data/spec/commands/utility_commands_spec.rb +53 -0
- data/spec/generators/gesture_helper_spec.rb +123 -0
- data/spec/generators/network_helper_spec.rb +634 -0
- data/spec/generators/screenshot_helper_spec.rb +776 -0
- data/spec/generators/template_renderer_spec.rb +0 -4
- data/spec/generators/trace_helper_spec.rb +489 -97
- data/spec/generators/video_helper_spec.rb +0 -9
- data/spec/integration/commands/browser_update_after_creation_spec.rb +0 -8
- data/spec/integration/commands/scaffolding_commands_spec.rb +12 -70
- data/spec/integration/commands/utility_commands_spec.rb +19 -41
- data/spec/integration/content/azure_content_spec.rb +2 -6
- data/spec/integration/content/ci_content_spec.rb +1 -55
- data/spec/integration/content/common_content_spec.rb +1 -118
- data/spec/integration/content/config_content_spec.rb +34 -31
- data/spec/integration/content/gemfile_content_spec.rb +7 -72
- data/spec/integration/content/helper_content_spec.rb +105 -99
- data/spec/integration/content/page_content_spec.rb +85 -87
- data/spec/integration/content/readme_content_spec.rb +134 -75
- data/spec/integration/content/rspec_reporter_content_spec.rb +2 -2
- data/spec/integration/content/rubocop_content_spec.rb +1 -1
- data/spec/integration/content/syntax_validation_spec.rb +2 -2
- data/spec/integration/content/test_content_spec.rb +23 -147
- data/spec/integration/end_to_end_features_spec.rb +1 -43
- data/spec/integration/end_to_end_spec.rb +319 -326
- data/spec/integration/generators/automation_generator_spec.rb +3 -37
- data/spec/integration/generators/axe_addon_spec.rb +2 -35
- data/spec/integration/generators/azure_generator_spec.rb +1 -1
- data/spec/integration/generators/common_generator_spec.rb +1 -40
- data/spec/integration/generators/config_features_spec.rb +1 -24
- data/spec/integration/generators/github_generator_spec.rb +0 -12
- data/spec/integration/generators/helpers_generator_spec.rb +74 -80
- data/spec/integration/generators/invoke_generators_spec.rb +54 -45
- data/spec/integration/generators/rspec_generator_spec.rb +0 -6
- data/spec/integration/scaffolding_e2e_spec.rb +17 -179
- data/spec/integration/spec_helper.rb +4 -6
- data/spec/integration/trace_format_checker.rb +11 -2
- data/spec/integration/trace_page_server.rb +72 -5
- data/spec/menus/menu_generator_spec.rb +14 -39
- data/spec/runtime/actionability_spec.rb +207 -0
- data/spec/runtime/actionable_spec.rb +125 -0
- data/spec/runtime/browser/actionability_script_browser_spec.rb +150 -0
- data/spec/runtime/browser/actionable_browser_spec.rb +98 -0
- data/spec/runtime/browser/finders_browser_spec.rb +148 -0
- data/spec/runtime/browser/matchers_browser_spec.rb +67 -0
- data/spec/runtime/browser/retrying_block_browser_spec.rb +37 -0
- data/spec/runtime/browser/role_engine_browser_spec.rb +184 -0
- data/spec/runtime/browser/strict_browser_spec.rb +38 -0
- data/spec/runtime/by_role_spec.rb +52 -0
- data/spec/runtime/config_spec.rb +79 -0
- data/spec/runtime/context_spec.rb +73 -0
- data/spec/runtime/errors_spec.rb +89 -0
- data/spec/runtime/finders_spec.rb +165 -0
- data/spec/runtime/fixtures/actionability.html +70 -0
- data/spec/runtime/fixtures/locators.html +46 -0
- data/spec/runtime/fixtures/matchers.html +33 -0
- data/spec/runtime/fixtures/roles.html +162 -0
- data/spec/runtime/matchers/base_spec.rb +165 -0
- data/spec/runtime/matchers/element_spec.rb +277 -0
- data/spec/runtime/matchers/page_spec.rb +102 -0
- data/spec/runtime/matchers/presence_spec.rb +59 -0
- data/spec/runtime/matchers/retrying_block_spec.rb +165 -0
- data/spec/runtime/observer_spec.rb +341 -0
- data/spec/runtime/role_finder_spec.rb +156 -0
- data/spec/runtime/role_query_spec.rb +75 -0
- data/spec/runtime/selectors_spec.rb +135 -0
- data/spec/runtime/strict_spec.rb +257 -0
- data/spec/runtime/support/browser.rb +48 -0
- data/spec/runtime/support/fake_bridge.rb +117 -0
- data/spec/runtime/wait_spec.rb +71 -0
- data/spec/scaffolding/name_normalizer_spec.rb +3 -2
- data/spec/scaffolding/scaffold_project_detector_spec.rb +4 -43
- data/spec/system/selenium_spec.rb +1 -5
- data/spec/system/support/system_test_helper.rb +2 -5
- data/spec/utilities/headless_config_spec.rb +0 -7
- data/spec/utilities/utilities_spec.rb +24 -0
- metadata +113 -48
- data/lib/generators/automation/templates/header.tt +0 -9
- data/lib/generators/cucumber/cucumber_generator.rb +0 -58
- data/lib/generators/cucumber/templates/accessibility_feature.tt +0 -5
- data/lib/generators/cucumber/templates/accessibility_steps.tt +0 -17
- data/lib/generators/cucumber/templates/cucumber.tt +0 -1
- data/lib/generators/cucumber/templates/env.tt +0 -7
- data/lib/generators/cucumber/templates/feature.tt +0 -21
- data/lib/generators/cucumber/templates/partials/appium_env.tt +0 -26
- data/lib/generators/cucumber/templates/partials/driver_world.tt +0 -18
- data/lib/generators/cucumber/templates/partials/mobile_steps.tt +0 -20
- data/lib/generators/cucumber/templates/partials/selenium_env.tt +0 -120
- data/lib/generators/cucumber/templates/partials/watir_env.tt +0 -48
- data/lib/generators/cucumber/templates/partials/watir_world.tt +0 -4
- data/lib/generators/cucumber/templates/partials/web_steps.tt +0 -28
- data/lib/generators/cucumber/templates/steps.tt +0 -5
- data/lib/generators/cucumber/templates/user_factory.tt +0 -15
- data/lib/generators/cucumber/templates/world.tt +0 -7
- data/lib/generators/templates/helpers/browser_helper.tt +0 -31
- data/lib/generators/templates/helpers/raider_element.tt +0 -159
- data/lib/generators/templates/helpers/wait_helper.tt +0 -611
- data/lib/scaffolding/templates/feature.tt +0 -6
- data/lib/scaffolding/templates/steps.tt +0 -19
- data/sig/generators/cucumber/cucumber_generator.rbs +0 -12
- data/spec/fixtures/watir_master/cucumber_watir/Readme.md.snapshot +0 -175
- data/spec/fixtures/watir_master/cucumber_watir/features/step_definitions/login_steps.rb.snapshot +0 -24
- data/spec/fixtures/watir_master/cucumber_watir/page_objects/abstract/page.rb.snapshot +0 -37
- data/spec/fixtures/watir_master/cucumber_watir/page_objects/pages/login.rb.snapshot +0 -33
- data/spec/fixtures/watir_master/rspec_watir/Readme.md.snapshot +0 -174
- data/spec/fixtures/watir_master/rspec_watir/page_objects/abstract/page.rb.snapshot +0 -37
- data/spec/fixtures/watir_master/rspec_watir/page_objects/pages/login.rb.snapshot +0 -33
- data/spec/fixtures/watir_master/rspec_watir/spec/login_page_spec.rb.snapshot +0 -49
- data/spec/integration/content/cucumber_reporter_content_spec.rb +0 -161
- data/spec/integration/generators/cucumber_generator_spec.rb +0 -100
- data/spec/integration/helpers/wait_helper_mobile_spec.rb +0 -281
- data/spec/integration/helpers/wait_helper_spec.rb +0 -679
- data/spec/integration/helpers/wait_helper_support.rb +0 -183
- data/spec/system/watir_spec.rb +0 -38
|
@@ -13,7 +13,7 @@ require_relative 'trace_page_server'
|
|
|
13
13
|
# These tests:
|
|
14
14
|
# 1. Generate a complete project
|
|
15
15
|
# 2. Install dependencies (bundle install)
|
|
16
|
-
# 3. Run the generated tests (rspec
|
|
16
|
+
# 3. Run the generated tests (rspec)
|
|
17
17
|
# 4. Verify tests pass
|
|
18
18
|
#
|
|
19
19
|
# This ensures the template system produces working, executable test frameworks
|
|
@@ -94,21 +94,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
94
94
|
result
|
|
95
95
|
end
|
|
96
96
|
|
|
97
|
-
# Helper to run Cucumber tests
|
|
98
|
-
def run_cucumber(project_name)
|
|
99
|
-
puts "\n🥒 Running Cucumber tests in #{project_name}..."
|
|
100
|
-
result = run_in_project(project_name, 'HEADLESS=true bundle exec cucumber features/ --format pretty', timeout: 120)
|
|
101
|
-
|
|
102
|
-
puts result[:stdout] if result[:stdout].length.positive?
|
|
103
|
-
|
|
104
|
-
unless result[:success]
|
|
105
|
-
puts '❌ Cucumber tests failed:'
|
|
106
|
-
puts result[:stderr] if result[:stderr].length.positive?
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
result
|
|
110
|
-
end
|
|
111
|
-
|
|
112
97
|
# Shared example for RSpec-based projects
|
|
113
98
|
shared_examples 'executable rspec project' do |project_name|
|
|
114
99
|
let(:project) { project_name }
|
|
@@ -117,10 +102,10 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
117
102
|
expect(bundle_install(project)).to be true
|
|
118
103
|
end
|
|
119
104
|
|
|
120
|
-
it 'resolves raider_reporter
|
|
105
|
+
it 'resolves raider_reporter 1.x from RubyGems and ships the raider-reporter executable', :aggregate_failures do
|
|
121
106
|
skip 'Bundle install failed' unless bundle_install(project)
|
|
122
107
|
|
|
123
|
-
expect(File.read(File.join(project, 'Gemfile.lock'))).to match(/^ raider_reporter \(
|
|
108
|
+
expect(File.read(File.join(project, 'Gemfile.lock'))).to match(/^ raider_reporter \(1\.\d+\.\d+\)$/)
|
|
124
109
|
result = run_in_project(project, 'bundle exec raider-reporter --help', timeout: 60)
|
|
125
110
|
expect(result[:exit_code]).to eq(0), "STDOUT: #{result[:stdout]}\nSTDERR: #{result[:stderr]}"
|
|
126
111
|
expect(result[:stdout] + result[:stderr]).to include('raider-reporter comment')
|
|
@@ -157,116 +142,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
157
142
|
end
|
|
158
143
|
end
|
|
159
144
|
|
|
160
|
-
# Shared example for Cucumber-based projects
|
|
161
|
-
shared_examples 'executable cucumber project' do |project_name|
|
|
162
|
-
let(:project) { project_name }
|
|
163
|
-
|
|
164
|
-
it 'installs dependencies successfully' do
|
|
165
|
-
expect(bundle_install(project)).to be true
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
it 'runs generated Cucumber tests successfully', :slow do
|
|
169
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
170
|
-
|
|
171
|
-
result = run_cucumber(project)
|
|
172
|
-
|
|
173
|
-
expect(result[:success]).to be(true),
|
|
174
|
-
"Cucumber tests failed with exit code #{result[:exit_code]}.\n" \
|
|
175
|
-
"STDOUT: #{result[:stdout]}\n" \
|
|
176
|
-
"STDERR: #{result[:stderr]}"
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
it 'resolves raider_reporter 0.10+ from RubyGems and requires its Cucumber plugin', :aggregate_failures do
|
|
180
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
181
|
-
|
|
182
|
-
lockfile = File.read(File.join(project, 'Gemfile.lock'))
|
|
183
|
-
expect(lockfile).to match(/^ raider_reporter \(0\.1\d\.\d+\)$/)
|
|
184
|
-
expect(File.read(File.join(project, 'features/support/env.rb'))).to include("require 'raider_reporter/cucumber'")
|
|
185
|
-
end
|
|
186
|
-
|
|
187
|
-
it 'prints pretty output and keeps the exit status of a failing Cucumber run', :aggregate_failures, :slow do
|
|
188
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
189
|
-
|
|
190
|
-
result = run_cucumber_with_failure(project)
|
|
191
|
-
|
|
192
|
-
expect(result[:stdout]).to include('Feature: Reporter failure')
|
|
193
|
-
expect(result[:stdout]).to include('Scenario: A deliberately failing scenario')
|
|
194
|
-
expect(result[:stdout]).to match(/\d+ scenarios? \(\d+ failed/)
|
|
195
|
-
expect(result[:exit_code]).to eq(1)
|
|
196
|
-
end
|
|
197
|
-
|
|
198
|
-
it 'writes rerun.txt listing only the failing scenario', :aggregate_failures, :slow do
|
|
199
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
200
|
-
|
|
201
|
-
result = run_cucumber_with_failure(project)
|
|
202
|
-
|
|
203
|
-
expect(result[:exit_code]).to eq(1)
|
|
204
|
-
rerun = File.join(project, 'rerun.txt')
|
|
205
|
-
expect(File).to exist(rerun)
|
|
206
|
-
expect(File.read(rerun).strip).to eq('features/reporter_failure.feature:3')
|
|
207
|
-
end
|
|
208
|
-
|
|
209
|
-
it 'writes raider-results/run.json listing every scenario with a failing scenario screenshot',
|
|
210
|
-
:aggregate_failures, :slow do
|
|
211
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
212
|
-
|
|
213
|
-
result = run_cucumber_with_failure(project)
|
|
214
|
-
summary = result[:stdout].match(/(\d+) scenarios? \((?:(\d+) failed)?/)
|
|
215
|
-
expect(summary).not_to be_nil, "No Cucumber summary line.\nSTDOUT: #{result[:stdout]}\nSTDERR: #{result[:stderr]}"
|
|
216
|
-
|
|
217
|
-
run = JSON.parse(File.read(File.join(project, 'raider-results', 'run.json')))
|
|
218
|
-
expect(run['schema_version']).to eq(1)
|
|
219
|
-
expect(run['results'].size).to eq(summary[1].to_i)
|
|
220
|
-
expect(run['results'].count { |r| r['status'] == 'failed' }).to eq(summary[2].to_i)
|
|
221
|
-
|
|
222
|
-
failing = run['results'].find { |r| r['full_name'] == 'Reporter failure: A deliberately failing scenario' }
|
|
223
|
-
expect(failing).not_to be_nil, "Scenario missing from run.json: #{run['results'].map { |r| r['full_name'] }}"
|
|
224
|
-
expect(failing['status']).to eq('failed')
|
|
225
|
-
expect(failing['file_path']).to eq('features/reporter_failure.feature')
|
|
226
|
-
expect(failing['failure']['message']).to start_with(
|
|
227
|
-
'Then the reporter records this failure (features/reporter_failure.feature:5)'
|
|
228
|
-
)
|
|
229
|
-
screenshot = failing['attachments'].find { |a| a['name'] == 'screenshot' }
|
|
230
|
-
expect(screenshot).not_to be_nil, "No screenshot attachment: #{failing['attachments']}"
|
|
231
|
-
expect(screenshot['content_type']).to eq('image/png')
|
|
232
|
-
end
|
|
233
|
-
end
|
|
234
|
-
|
|
235
|
-
# Runs the project's features plus a deliberately failing scenario, then removes that scenario again
|
|
236
|
-
def run_cucumber_with_failure(project)
|
|
237
|
-
add_failing_cucumber_scenario(project)
|
|
238
|
-
FileUtils.rm_rf([File.join(project, 'raider-results'), File.join(project, 'rerun.txt')])
|
|
239
|
-
run_cucumber(project)
|
|
240
|
-
ensure
|
|
241
|
-
remove_failing_cucumber_scenario(project)
|
|
242
|
-
end
|
|
243
|
-
|
|
244
|
-
# Adds a scenario that always fails at its last step, after the browser has been started
|
|
245
|
-
def add_failing_cucumber_scenario(project)
|
|
246
|
-
File.write(File.join(project, 'features/reporter_failure.feature'), <<~FEATURE)
|
|
247
|
-
Feature: Reporter failure
|
|
248
|
-
|
|
249
|
-
Scenario: A deliberately failing scenario
|
|
250
|
-
Given the reporter has a started scenario
|
|
251
|
-
Then the reporter records this failure
|
|
252
|
-
FEATURE
|
|
253
|
-
File.write(File.join(project, 'features/step_definitions/reporter_failure_steps.rb'), <<~STEPS)
|
|
254
|
-
# frozen_string_literal: true
|
|
255
|
-
|
|
256
|
-
Given('the reporter has a started scenario') {}
|
|
257
|
-
|
|
258
|
-
Then('the reporter records this failure') do
|
|
259
|
-
raise 'deliberate failure for raider_reporter'
|
|
260
|
-
end
|
|
261
|
-
STEPS
|
|
262
|
-
end
|
|
263
|
-
|
|
264
|
-
# Keeps the deliberate failure from leaking into the other examples that share the project
|
|
265
|
-
def remove_failing_cucumber_scenario(project)
|
|
266
|
-
FileUtils.rm_f(File.join(project, 'features/reporter_failure.feature'))
|
|
267
|
-
FileUtils.rm_f(File.join(project, 'features/step_definitions/reporter_failure_steps.rb'))
|
|
268
|
-
end
|
|
269
|
-
|
|
270
145
|
# Runs the project's own RuboCop with the gems its bundle resolved (newer than this gem's dev bundle).
|
|
271
146
|
# It runs from a copy outside this repo, as a user's project would: inside it RuboCop would also read
|
|
272
147
|
# this repo's .rubocop.yml, gemspec and .ruby-version.
|
|
@@ -291,6 +166,13 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
291
166
|
expect_project_rubocop_clean('rspec_selenium')
|
|
292
167
|
end
|
|
293
168
|
|
|
169
|
+
it 'prints the actionability notice once per run', :slow do
|
|
170
|
+
skip 'Bundle install failed' unless bundle_install('rspec_selenium')
|
|
171
|
+
|
|
172
|
+
result = run_rspec('rspec_selenium')
|
|
173
|
+
expect(result[:stderr].scan('[raider] actionability on').size).to eq(1), "STDERR: #{result[:stderr]}"
|
|
174
|
+
end
|
|
175
|
+
|
|
294
176
|
it 'saves the login once and reuses it', :aggregate_failures, :slow do
|
|
295
177
|
skip 'Bundle install failed' unless bundle_install('rspec_selenium')
|
|
296
178
|
|
|
@@ -314,22 +196,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
314
196
|
cookie_values.select { |value| value.length >= 8 }.each { |value| expect(output).not_to include(value) }
|
|
315
197
|
end
|
|
316
198
|
end
|
|
317
|
-
|
|
318
|
-
describe 'Watir + RSpec' do
|
|
319
|
-
include_examples 'executable rspec project', 'rspec_watir'
|
|
320
|
-
end
|
|
321
|
-
|
|
322
|
-
describe 'Selenium + Cucumber' do
|
|
323
|
-
include_examples 'executable cucumber project', 'cucumber_selenium'
|
|
324
|
-
|
|
325
|
-
it 'passes its own RuboCop with its bundled gems', :slow do
|
|
326
|
-
expect_project_rubocop_clean('cucumber_selenium')
|
|
327
|
-
end
|
|
328
|
-
end
|
|
329
|
-
|
|
330
|
-
describe 'Watir + Cucumber' do
|
|
331
|
-
include_examples 'executable cucumber project', 'cucumber_watir'
|
|
332
|
-
end
|
|
333
199
|
end
|
|
334
200
|
|
|
335
201
|
# Recorded Azure Pipelines env of a push (IndividualCI) build, see raider_reporter's CiDetector specs
|
|
@@ -382,12 +248,12 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
382
248
|
describe 'Selenium + RSpec' do
|
|
383
249
|
let(:project) { generate_azure_project('rspec') }
|
|
384
250
|
|
|
385
|
-
it 'ships azure-pipelines.yml and resolves raider_reporter
|
|
251
|
+
it 'ships azure-pipelines.yml and resolves raider_reporter 1.x', :aggregate_failures do
|
|
386
252
|
expect(File).to exist(File.join(project, 'azure-pipelines.yml'))
|
|
387
253
|
expect(File).not_to exist(File.join(project, '.github/workflows'))
|
|
388
254
|
skip 'Bundle install failed' unless bundle_install(project)
|
|
389
255
|
|
|
390
|
-
expect(File.read(File.join(project, 'Gemfile.lock'))).to match(/^ raider_reporter \(
|
|
256
|
+
expect(File.read(File.join(project, 'Gemfile.lock'))).to match(/^ raider_reporter \(1\.\d+\.\d+\)$/)
|
|
391
257
|
end
|
|
392
258
|
|
|
393
259
|
it 'records provider azure_pipelines and an azure_pipelines: build key in run.json', :aggregate_failures, :slow do
|
|
@@ -400,27 +266,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
400
266
|
expect_azure_ci_block(project)
|
|
401
267
|
end
|
|
402
268
|
end
|
|
403
|
-
|
|
404
|
-
describe 'Selenium + Cucumber' do
|
|
405
|
-
let(:project) { generate_azure_project('cucumber') }
|
|
406
|
-
|
|
407
|
-
it 'ships azure-pipelines.yml and resolves raider_reporter 0.10+', :aggregate_failures do
|
|
408
|
-
expect(File).to exist(File.join(project, 'azure-pipelines.yml'))
|
|
409
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
410
|
-
|
|
411
|
-
expect(File.read(File.join(project, 'Gemfile.lock'))).to match(/^ raider_reporter \(0\.1\d\.\d+\)$/)
|
|
412
|
-
end
|
|
413
|
-
|
|
414
|
-
it 'records provider azure_pipelines and an azure_pipelines: build key in run.json', :aggregate_failures, :slow do
|
|
415
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
416
|
-
|
|
417
|
-
FileUtils.rm_rf(File.join(project, 'raider-results'))
|
|
418
|
-
result = run_in_project(project, "#{azure_env_prefix} bundle exec cucumber features/ --format pretty",
|
|
419
|
-
timeout: 120)
|
|
420
|
-
expect(result[:success]).to be(true), "STDOUT: #{result[:stdout]}\nSTDERR: #{result[:stderr]}"
|
|
421
|
-
expect_azure_ci_block(project)
|
|
422
|
-
end
|
|
423
|
-
end
|
|
424
269
|
end
|
|
425
270
|
|
|
426
271
|
# An empty skeleton (examples: false) still passes its run, so its CI is green until tests are added
|
|
@@ -444,17 +289,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
444
289
|
expect(result[:stdout]).to include('0 examples, 0 failures')
|
|
445
290
|
end
|
|
446
291
|
end
|
|
447
|
-
|
|
448
|
-
it 'runs bundle exec cucumber on a selenium cucumber skeleton with 0 scenarios', :aggregate_failures do
|
|
449
|
-
project = generate_skeleton_project('cucumber')
|
|
450
|
-
skip 'Bundle install failed' unless bundle_install(project)
|
|
451
|
-
|
|
452
|
-
['cucumber', 'cucumber features'].each do |command|
|
|
453
|
-
result = run_in_project(project, "bundle exec #{command}", timeout: 120)
|
|
454
|
-
expect(result[:exit_code]).to eq(0), "#{command}\nSTDOUT: #{result[:stdout]}\nSTDERR: #{result[:stderr]}"
|
|
455
|
-
expect(result[:stdout]).to include('0 scenarios')
|
|
456
|
-
end
|
|
457
|
-
end
|
|
458
292
|
end
|
|
459
293
|
|
|
460
294
|
# Browser e2e helpers shared by the failure video and the trace replay contexts
|
|
@@ -470,7 +304,7 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
470
304
|
File.directory?("/Applications/#{app}.app") || system("command -v #{command} > /dev/null 2>&1")
|
|
471
305
|
end
|
|
472
306
|
|
|
473
|
-
# A separate project, so the default projects keep resolving raider_reporter
|
|
307
|
+
# A separate project, so the default projects keep resolving raider_reporter 1.x from RubyGems
|
|
474
308
|
def generate_video_project(framework, automation)
|
|
475
309
|
name = "video_#{framework}_#{automation}"
|
|
476
310
|
unless LazyProjectGenerator::GENERATED.include?(name)
|
|
@@ -534,41 +368,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
534
368
|
SPEC
|
|
535
369
|
end
|
|
536
370
|
|
|
537
|
-
def add_cucumber_video_feature(project, firefox_binary)
|
|
538
|
-
File.write(File.join(project, 'features/video_replay.feature'), <<~FEATURE)
|
|
539
|
-
Feature: Video replay
|
|
540
|
-
|
|
541
|
-
Scenario: A passing scenario
|
|
542
|
-
Given the video page is open
|
|
543
|
-
Then the video page title is "Video"
|
|
544
|
-
|
|
545
|
-
Scenario: A failing scenario
|
|
546
|
-
Given the video page is open
|
|
547
|
-
Then the video page title is "deliberate video failure"
|
|
548
|
-
FEATURE
|
|
549
|
-
File.write(File.join(project, 'features/step_definitions/video_replay_steps.rb'), <<~STEPS)
|
|
550
|
-
# frozen_string_literal: true
|
|
551
|
-
|
|
552
|
-
Given('the video page is open') do
|
|
553
|
-
browser.goto('#{page_url}')
|
|
554
|
-
browser.text_field(id: 'name').set('typed')
|
|
555
|
-
browser.button(id: 'go').click
|
|
556
|
-
end
|
|
557
|
-
|
|
558
|
-
Then('the video page title is {string}') do |title|
|
|
559
|
-
raise "expected title \#{title}" unless browser.h1(id: 'title').text == title
|
|
560
|
-
end
|
|
561
|
-
STEPS
|
|
562
|
-
config = File.join(project, 'config/config.yml')
|
|
563
|
-
File.write(config, File.read(config).sub(/^browser: .*$/, 'browser: firefox'))
|
|
564
|
-
# Test setup only: the generated templates do not read FIREFOX_BINARY.
|
|
565
|
-
File.write(File.join(project, 'features/support/zz_firefox_binary.rb'), <<~RUBY) if firefox_binary
|
|
566
|
-
# frozen_string_literal: true
|
|
567
|
-
|
|
568
|
-
Selenium::WebDriver::Firefox.path = ENV['FIREFOX_BINARY'] if ENV['FIREFOX_BINARY']
|
|
569
|
-
RUBY
|
|
570
|
-
end
|
|
571
|
-
|
|
572
371
|
def video_files(project)
|
|
573
372
|
Dir.glob(File.join(project, 'raider-results', '**', '*.{webm,mp4}'))
|
|
574
373
|
end
|
|
@@ -634,34 +433,10 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
634
433
|
expect(Dir.glob(File.join(cache_dir, '**', '*'))).to be_empty
|
|
635
434
|
end
|
|
636
435
|
end
|
|
637
|
-
|
|
638
|
-
describe 'Watir + Cucumber on headless Firefox' do
|
|
639
|
-
let(:project) { 'video_cucumber_watir' }
|
|
640
|
-
let(:firefox_binary) { ENV.fetch('FIREFOX_BINARY', nil) }
|
|
641
|
-
|
|
642
|
-
before do
|
|
643
|
-
skip_unless_video_prerequisites('firefox')
|
|
644
|
-
generate_video_project('cucumber', 'watir')
|
|
645
|
-
add_cucumber_video_feature(project, firefox_binary)
|
|
646
|
-
end
|
|
647
|
-
|
|
648
|
-
after do
|
|
649
|
-
FileUtils.rm_f(File.join(project, 'features/video_replay.feature'))
|
|
650
|
-
FileUtils.rm_f(File.join(project, 'features/step_definitions/video_replay_steps.rb'))
|
|
651
|
-
end
|
|
652
|
-
|
|
653
|
-
it 'attaches video.webm to the failing scenario only and plays it in index.html', :aggregate_failures do
|
|
654
|
-
result = run_in_project(project, 'bundle exec cucumber features/video_replay.feature',
|
|
655
|
-
env: video_env('FIREFOX_BINARY' => firefox_binary), timeout: 240)
|
|
656
|
-
expect(result[:exit_code]).to eq(1), "STDOUT: #{result[:stdout]}\nSTDERR: #{result[:stderr]}"
|
|
657
|
-
expect(result[:stderr]).not_to include('[video]')
|
|
658
|
-
expect_video_for_failure_only(project, 'A failing scenario', result)
|
|
659
|
-
end
|
|
660
|
-
end
|
|
661
436
|
end
|
|
662
437
|
|
|
663
438
|
# Trace replay against headless Chrome, and one example on headless Firefox. The page is served over HTTP from this process (never file://), so
|
|
664
|
-
# WebDriver BiDi reports real network and console entries. raider_reporter ~> 0
|
|
439
|
+
# WebDriver BiDi reports real network and console entries. raider_reporter ~> 1.0 resolves from RubyGems, or
|
|
665
440
|
# from a checkout at RAIDER_REPORTER_PATH. The example with video on also needs RAIDER_FFMPEG_PATH.
|
|
666
441
|
# Set TRACE_FIXTURE_DIR to keep the failing RSpec example's trace.zip (and an unzipped copy) there.
|
|
667
442
|
context 'Trace replay', :slow do
|
|
@@ -686,8 +461,8 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
686
461
|
def skip_unless_trace_prerequisites(project, browser = 'chrome')
|
|
687
462
|
skip "#{browser == 'chrome' ? 'Google Chrome' : 'Firefox'} is not installed" unless browser_installed?(browser)
|
|
688
463
|
version = run_in_project(project, %(bundle exec ruby -e "require 'raider_reporter/version'; print RaiderReporter::VERSION"))
|
|
689
|
-
skip "the bundle does not resolve raider_reporter ~> 0
|
|
690
|
-
version[:success] && Gem::Version.new(version[:stdout]) >= Gem::Version.new('0
|
|
464
|
+
skip "the bundle does not resolve raider_reporter ~> 1.0 (set RAIDER_REPORTER_PATH): #{version[:stderr].lines.last}" unless
|
|
465
|
+
version[:success] && Gem::Version.new(version[:stdout]) >= Gem::Version.new('1.0')
|
|
691
466
|
end
|
|
692
467
|
|
|
693
468
|
def trace_env(overrides = {})
|
|
@@ -747,7 +522,7 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
747
522
|
'passes after driving the page'
|
|
748
523
|
end
|
|
749
524
|
|
|
750
|
-
# Drives the page
|
|
525
|
+
# Drives the page with plain Selenium and the runtime's matchers, so the trace has action steps.
|
|
751
526
|
# Each example prints the driver's webSocketUrl, which is set only while tracing is on.
|
|
752
527
|
def add_rspec_trace_spec(project)
|
|
753
528
|
File.write(File.join(project, 'spec/trace_replay_spec.rb'), <<~SPEC)
|
|
@@ -757,21 +532,23 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
757
532
|
|
|
758
533
|
describe 'Trace replay' do
|
|
759
534
|
it '#{failing}' do
|
|
760
|
-
driver.navigate.to('#{server.url}')
|
|
535
|
+
driver.navigate.to('#{server.url}start')
|
|
761
536
|
puts "TRACE_E2E_WEBSOCKET=\#{driver.capabilities[:web_socket_url].inspect}"
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
find_element(id: '
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
537
|
+
driver.find_element(id: 'pw').send_keys('#{secrets[0]}')
|
|
538
|
+
driver.find_element(id: 'pw2').send_keys('#{secrets[1]}')
|
|
539
|
+
form = driver.find_element(id: 'form')
|
|
540
|
+
form.find_element(id: 'name').send_keys('#{plain_value}')
|
|
541
|
+
driver.find_element(id: 'go').click
|
|
542
|
+
expect { driver.find_element(id: 'status') }.to have_text('answered by /api', wait: 5)
|
|
543
|
+
expect { driver.find_element(id: 'status').text }.to eq('answered by /api')
|
|
544
|
+
expect { driver.find_element(id: 'title') }.to have_text('deliberate trace failure', wait: 1)
|
|
768
545
|
end
|
|
769
546
|
|
|
770
547
|
it '#{passing}' do
|
|
771
548
|
driver.navigate.to('#{server.url}')
|
|
772
549
|
puts "TRACE_E2E_WEBSOCKET=\#{driver.capabilities[:web_socket_url].inspect}"
|
|
773
|
-
|
|
774
|
-
|
|
550
|
+
driver.find_element(id: 'name').send_keys('ok')
|
|
551
|
+
expect { driver.find_element(id: 'title') }.to have_text('Trace')
|
|
775
552
|
end
|
|
776
553
|
end
|
|
777
554
|
SPEC
|
|
@@ -782,6 +559,19 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
782
559
|
trace_zip(project, trace_attachments(result).first)
|
|
783
560
|
end
|
|
784
561
|
|
|
562
|
+
# Locators as id strategies, the chained find as within, the password steps masked, and no 3.x step shapes.
|
|
563
|
+
def expect_runtime_steps(steps)
|
|
564
|
+
by_id = ->(step) { step['locator'].is_a?(Hash) && step['locator']['strategy'] == 'id' && step['locator']['selector'].is_a?(String) }
|
|
565
|
+
expect(steps.select { |step| step['action'] == 'click' }).to all(satisfy(&by_id)).and(be_any)
|
|
566
|
+
expect(steps.select { |step| step['action'] == 'click' }.first['source']['file']).to end_with('spec/trace_replay_spec.rb')
|
|
567
|
+
typed = steps.select { |step| step['action'] == 'send_keys' }
|
|
568
|
+
expect(typed).to all(satisfy(&by_id))
|
|
569
|
+
expect(typed.last['locator']['within']).to be_a(Array).and(be_any)
|
|
570
|
+
expect(typed.first(2)).to all(include('value' => nil, 'secret' => true))
|
|
571
|
+
expect(steps.map { |step| step['kind'] }).not_to include('group')
|
|
572
|
+
expect(steps.map { |step| step['action'] }).not_to include('cucumber_step')
|
|
573
|
+
end
|
|
574
|
+
|
|
785
575
|
def steps_of(path)
|
|
786
576
|
TraceFormatChecker.read(path).first['steps']
|
|
787
577
|
end
|
|
@@ -824,6 +614,16 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
824
614
|
expect(trace['console']).to include(include('level' => 'error', 'text' => a_string_including('trace e2e error')))
|
|
825
615
|
end
|
|
826
616
|
|
|
617
|
+
# /start redirects to /: two entries, one per hop, both credited to the navigate step.
|
|
618
|
+
def expect_redirect_hops(trace)
|
|
619
|
+
navigate = trace['steps'].find { |step| step['kind'] == 'navigate' }
|
|
620
|
+
start = trace['network'].find { |entry| entry['url'].to_s.end_with?('/start') }
|
|
621
|
+
page = trace['network'].find { |entry| entry['url'] == server.url }
|
|
622
|
+
expect(start).to include('status' => 302, 'step_index' => navigate['index'])
|
|
623
|
+
expect(page).to include('status' => 200, 'step_index' => navigate['index'])
|
|
624
|
+
expect(page['duration_ms']).to be >= (TracePageServer::PAGE_DELAY * 1000) - 50
|
|
625
|
+
end
|
|
626
|
+
|
|
827
627
|
# The same-origin stylesheet inlined as <style>, and the page's <script> gone.
|
|
828
628
|
def expect_sanitised_snapshots(path, names)
|
|
829
629
|
entries = zip_entries(path)
|
|
@@ -847,12 +647,14 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
847
647
|
expect(trace_attachments(result_named(project, passing))).to be_empty
|
|
848
648
|
expect(trace_files(project).size).to eq(1)
|
|
849
649
|
expect_valid_trace(path)
|
|
650
|
+
expect_runtime_steps(steps_of(path))
|
|
850
651
|
|
|
851
652
|
trace, names = TraceFormatChecker.read(path)
|
|
852
653
|
expect(trace['capabilities']).to include('network' => true, 'console' => true)
|
|
853
654
|
expect_steps(trace['steps'])
|
|
854
655
|
expect_jpeg_screenshots(path, names)
|
|
855
656
|
expect_network_and_console(trace)
|
|
657
|
+
expect_redirect_hops(trace)
|
|
856
658
|
expect_sanitised_snapshots(path, names)
|
|
857
659
|
end
|
|
858
660
|
|
|
@@ -871,7 +673,7 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
871
673
|
end
|
|
872
674
|
|
|
873
675
|
it 'writes no trace and no web_socket_url with TRACE=off', :aggregate_failures do
|
|
874
|
-
run = run_in_project(project, command, env: trace_env('TRACE' => 'off'))
|
|
676
|
+
run = run_in_project(project, command, env: trace_env('TRACE' => 'off', 'BIDI' => 'off'))
|
|
875
677
|
expect_run(run)
|
|
876
678
|
expect(run[:stdout].scan(/^TRACE_E2E_WEBSOCKET=(.*)$/).flatten).to eq(%w[nil nil])
|
|
877
679
|
expect(results(project).flat_map { |result| trace_attachments(result) }).to be_empty
|
|
@@ -954,6 +756,7 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
954
756
|
expect_run(run)
|
|
955
757
|
path = failing_trace(result_named(project, failing))
|
|
956
758
|
expect_valid_trace(path)
|
|
759
|
+
expect_runtime_steps(steps_of(path))
|
|
957
760
|
|
|
958
761
|
trace, names = TraceFormatChecker.read(path)
|
|
959
762
|
expect(trace['browser']).to include('name' => 'firefox')
|
|
@@ -965,57 +768,286 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
965
768
|
end
|
|
966
769
|
end
|
|
967
770
|
|
|
968
|
-
|
|
969
|
-
|
|
771
|
+
# mock_request / block_request through WebDriver BiDi, on a page that fetches /api/* and a cross-origin URL
|
|
772
|
+
# that exists nowhere (so it only answers when mocked). Chrome by default; Edge and Firefox run only when
|
|
773
|
+
# RAIDER_E2E_BROWSERS (comma separated) lists them.
|
|
774
|
+
describe 'Network mocking' do
|
|
775
|
+
let(:project) { 'video_rspec_selenium' }
|
|
776
|
+
let(:original_config) { File.read(config) }
|
|
970
777
|
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
skip_unless_trace_prerequisites(project)
|
|
974
|
-
add_cucumber_trace_feature(project)
|
|
778
|
+
def config
|
|
779
|
+
File.join(project, 'config/config.yml')
|
|
975
780
|
end
|
|
976
781
|
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
FileUtils.rm_f(File.join(project, 'features/step_definitions/trace_replay_steps.rb'))
|
|
782
|
+
def spec_path
|
|
783
|
+
File.join(project, 'spec/network_mocking_spec.rb')
|
|
980
784
|
end
|
|
981
785
|
|
|
982
|
-
def
|
|
983
|
-
|
|
984
|
-
|
|
786
|
+
def opted_in?(browser)
|
|
787
|
+
ENV.fetch('RAIDER_E2E_BROWSERS', '').split(',').map(&:strip).include?(browser)
|
|
788
|
+
end
|
|
789
|
+
|
|
790
|
+
def skip_unless_mocking_prerequisites(browser)
|
|
791
|
+
skip "set RAIDER_E2E_BROWSERS=#{browser} to run network mocking on #{browser}" unless browser == 'chrome' || opted_in?(browser)
|
|
792
|
+
skip 'Microsoft Edge is not installed' if browser == 'edge' && !File.directory?('/Applications/Microsoft Edge.app')
|
|
793
|
+
skip_unless_trace_prerequisites(project, browser == 'firefox' ? 'firefox' : 'chrome')
|
|
794
|
+
end
|
|
985
795
|
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
When I type a name on the trace page
|
|
989
|
-
Then the trace page title is "deliberate trace failure"
|
|
990
|
-
FEATURE
|
|
991
|
-
File.write(File.join(project, 'features/step_definitions/trace_replay_steps.rb'), <<~STEPS)
|
|
796
|
+
def write_spec(body)
|
|
797
|
+
File.write(spec_path, <<~SPEC)
|
|
992
798
|
# frozen_string_literal: true
|
|
993
799
|
|
|
994
|
-
|
|
995
|
-
|
|
800
|
+
require_relative '../helpers/spec_helper'
|
|
801
|
+
|
|
802
|
+
describe 'Network mocking' do
|
|
803
|
+
#{body.lines.map { |line| line.strip.empty? ? line : " #{line}" }.join}
|
|
996
804
|
end
|
|
805
|
+
SPEC
|
|
806
|
+
end
|
|
807
|
+
|
|
808
|
+
def mocking_run(overrides = {})
|
|
809
|
+
run_in_project(project, 'bundle exec rspec spec/network_mocking_spec.rb', env: trace_env(overrides), timeout: 240)
|
|
810
|
+
end
|
|
997
811
|
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
812
|
+
def mock_example(browser)
|
|
813
|
+
write_spec(<<~BODY)
|
|
814
|
+
it 'mocks, blocks, edits and removes' do
|
|
815
|
+
driver.navigate.to('#{server.url}mock')
|
|
816
|
+
users = mock_request('#{server.url}api/users', json: { users: ['ana'] })
|
|
817
|
+
block_request('**/api/fail')
|
|
818
|
+
mock_request('**/api/echo') { { continue: { headers: { 'x-raider' => 'yes' } } } }
|
|
819
|
+
mock_request('http://mock.raider.test/api/cors', json: { ok: true })
|
|
820
|
+
driver.find_element(id: 'users').click
|
|
821
|
+
expect { driver.find_element(id: 'users-out') }.to have_text('ana', wait: 5)
|
|
822
|
+
driver.find_element(id: 'fail').click
|
|
823
|
+
expect { driver.find_element(id: 'fail-out') }.to have_text('request failed', wait: 5)
|
|
824
|
+
driver.find_element(id: 'echo').click
|
|
825
|
+
expect { driver.find_element(id: 'echo-out') }.to have_text('echo yes', wait: 5)
|
|
826
|
+
driver.find_element(id: 'cors').click
|
|
827
|
+
expect { driver.find_element(id: 'cors-out') }.to have_text('cors true', wait: 5)
|
|
828
|
+
expect { users.calls.size }.to eq(1)
|
|
829
|
+
expect { users.called? }.to be(true)
|
|
830
|
+
remove_mock(users)
|
|
831
|
+
driver.find_element(id: 'users').click
|
|
832
|
+
expect { driver.find_element(id: 'users-out') }.to have_text('real', wait: 5)
|
|
833
|
+
expect { users.calls.size }.to eq(1)
|
|
834
|
+
puts 'NETWORK_MOCKING_#{browser.upcase}_DONE'
|
|
1001
835
|
end
|
|
836
|
+
BODY
|
|
837
|
+
end
|
|
838
|
+
|
|
839
|
+
def switch_browser(name)
|
|
840
|
+
# The generated config carries browser arguments for chrome and firefox only.
|
|
841
|
+
edge_args = " edge:\n - no-sandbox\n"
|
|
842
|
+
File.write(config, original_config.sub(/^browser: .*$/, "browser: #{name}").sub(/^ firefox:/, "#{edge_args} firefox:"))
|
|
843
|
+
end
|
|
844
|
+
|
|
845
|
+
# Skips when the browser is unavailable or not opted in; otherwise points the config at it.
|
|
846
|
+
def use_browser(name)
|
|
847
|
+
skip_unless_mocking_prerequisites(name)
|
|
848
|
+
original_config
|
|
849
|
+
switch_browser(name)
|
|
850
|
+
end
|
|
851
|
+
|
|
852
|
+
before { generate_video_project('rspec', 'selenium') }
|
|
1002
853
|
|
|
1003
|
-
|
|
1004
|
-
|
|
854
|
+
after do
|
|
855
|
+
File.write(config, original_config) if File.exist?(config)
|
|
856
|
+
FileUtils.rm_f(spec_path)
|
|
857
|
+
end
|
|
858
|
+
|
|
859
|
+
%w[chrome edge firefox].each do |name|
|
|
860
|
+
it "mocks, blocks, edits headers and removes a mock on headless #{name}", :aggregate_failures do
|
|
861
|
+
use_browser(name)
|
|
862
|
+
mock_example(name)
|
|
863
|
+
run = mocking_run('TRACE' => 'off')
|
|
864
|
+
expect_run(run, exit_code: 0)
|
|
865
|
+
expect(run[:stdout]).to include("NETWORK_MOCKING_#{name.upcase}_DONE")
|
|
866
|
+
end
|
|
867
|
+
end
|
|
868
|
+
|
|
869
|
+
def write_failing_spec
|
|
870
|
+
write_spec(<<~BODY)
|
|
871
|
+
it 'fails after mocking' do
|
|
872
|
+
driver.navigate.to('#{server.url}mock')
|
|
873
|
+
mock_request('#{server.url}api/users', json: { users: ['ana'] })
|
|
874
|
+
block_request('#{server.url}api/fail')
|
|
875
|
+
driver.find_element(id: 'users').click
|
|
876
|
+
expect { driver.find_element(id: 'users-out') }.to have_text('ana', wait: 5)
|
|
877
|
+
driver.find_element(id: 'fail').click
|
|
878
|
+
expect { driver.find_element(id: 'fail-out') }.to have_text('request failed', wait: 5)
|
|
879
|
+
expect { driver.find_element(id: 'users-out') }.to have_text('deliberate mocking failure', wait: 1)
|
|
1005
880
|
end
|
|
1006
|
-
|
|
881
|
+
BODY
|
|
1007
882
|
end
|
|
1008
883
|
|
|
1009
|
-
it '
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
path = trace_zip(project, attachments.first)
|
|
884
|
+
it 'keeps mocked entries in the failing example trace', :aggregate_failures do
|
|
885
|
+
use_browser('chrome')
|
|
886
|
+
write_failing_spec
|
|
887
|
+
expect_run(mocking_run)
|
|
888
|
+
path = failing_trace(result_named(project, 'fails after mocking'))
|
|
889
|
+
keep_fixture(path)
|
|
1016
890
|
expect_valid_trace(path)
|
|
1017
|
-
|
|
1018
|
-
|
|
891
|
+
network = TraceFormatChecker.read(path).first['network']
|
|
892
|
+
users = network.find { |entry| entry['url'].to_s.end_with?('/api/users') }
|
|
893
|
+
failed = network.find { |entry| entry['url'].to_s.end_with?('/api/fail') }
|
|
894
|
+
expect(users).to include('mocked' => 'fulfilled', 'status' => 200)
|
|
895
|
+
expect(failed).to include('mocked' => 'failed')
|
|
896
|
+
end
|
|
897
|
+
|
|
898
|
+
it 'fails the example with MockError when a block raises', :aggregate_failures do
|
|
899
|
+
use_browser('chrome')
|
|
900
|
+
write_spec(<<~BODY)
|
|
901
|
+
it 'has a raising block' do
|
|
902
|
+
driver.navigate.to('#{server.url}mock')
|
|
903
|
+
mock_request('#{server.url}api/users') { raise 'boom from the block' }
|
|
904
|
+
driver.find_element(id: 'users').click
|
|
905
|
+
expect { driver.find_element(id: 'users-out') }.to have_text('users failed', wait: 5)
|
|
906
|
+
end
|
|
907
|
+
BODY
|
|
908
|
+
run = mocking_run('TRACE' => 'off')
|
|
909
|
+
expect(run[:exit_code]).to eq(1), "STDOUT: #{run[:stdout]}\nSTDERR: #{run[:stderr]}"
|
|
910
|
+
expect(run[:stdout]).to include('NetworkHelper::MockError', 'boom from the block')
|
|
911
|
+
end
|
|
912
|
+
end
|
|
913
|
+
end
|
|
914
|
+
|
|
915
|
+
# match_screenshot against real browsers, on a fixed data: page (no server, no animation). Chrome by default; Edge and
|
|
916
|
+
# Firefox run only when RAIDER_E2E_BROWSERS (comma separated) lists them. The generated spec recolours the page
|
|
917
|
+
# when SHOT_VARIANT=changed. raider_reporter comes from RubyGems (no `comparison` method yet), so the images are
|
|
918
|
+
# attached and one warning is printed; the report's comparisons field is not asserted here.
|
|
919
|
+
context 'Screenshot comparison', :slow do
|
|
920
|
+
let(:cache_dir) { Dir.mktmpdir('raider-cache') }
|
|
921
|
+
let(:baseline) { "screenshots/baselines/#{browser_name}/home.png" }
|
|
922
|
+
let(:original_config) { File.read(config_path) }
|
|
923
|
+
|
|
924
|
+
after do
|
|
925
|
+
File.write(config_path, original_config) if File.exist?(config_path)
|
|
926
|
+
FileUtils.rm_rf(File.join(project, 'screenshots'))
|
|
927
|
+
FileUtils.rm_f(File.join(project, 'spec/screenshot_comparison_spec.rb'))
|
|
928
|
+
FileUtils.rm_rf(cache_dir)
|
|
929
|
+
end
|
|
930
|
+
|
|
931
|
+
def config_path
|
|
932
|
+
File.join(project, 'config/config.yml')
|
|
933
|
+
end
|
|
934
|
+
|
|
935
|
+
def shot_page
|
|
936
|
+
html = '<body style="margin:0;font-family:monospace"><h1 id="title">Screenshot</h1><p>Fixed content</p></body>'
|
|
937
|
+
"data:text/html,#{ERB::Util.url_encode(html)}"
|
|
938
|
+
end
|
|
939
|
+
|
|
940
|
+
def recolour
|
|
941
|
+
"document.documentElement.style.background = 'rgb(200, 0, 0)'; document.body.style.background = 'rgb(200, 0, 0)'"
|
|
942
|
+
end
|
|
943
|
+
|
|
944
|
+
def shot_env(overrides = {})
|
|
945
|
+
ci = %w[CI GITHUB_ACTIONS GITLAB_CI JENKINS_URL BUILDKITE CIRCLECI TF_BUILD TRAVIS].to_h { |key| [key, nil] }
|
|
946
|
+
ci.merge('HEADLESS' => 'true', 'VIDEO' => 'off', 'TRACE' => 'off', 'RAIDER_CACHE_DIR' => cache_dir,
|
|
947
|
+
'RAIDER_TOKEN' => nil, 'RAIDER_ENDPOINT' => nil, 'SCREENSHOTS' => nil, 'SHOT_VARIANT' => nil).merge(overrides)
|
|
948
|
+
end
|
|
949
|
+
|
|
950
|
+
def use_browser(name)
|
|
951
|
+
opted_in = ENV.fetch('RAIDER_E2E_BROWSERS', '').split(',').map(&:strip).include?(name)
|
|
952
|
+
skip "set RAIDER_E2E_BROWSERS=#{name} to run screenshot comparison on #{name}" unless name == 'chrome' || opted_in
|
|
953
|
+
skip 'Microsoft Edge is not installed' if name == 'edge' && !File.directory?('/Applications/Microsoft Edge.app')
|
|
954
|
+
skip "#{name} is not installed" unless name == 'edge' || browser_installed?(name == 'firefox' ? 'firefox' : 'chrome')
|
|
955
|
+
@browser_name = name
|
|
956
|
+
original_config
|
|
957
|
+
edge_args = " edge:\n - no-sandbox\n"
|
|
958
|
+
File.write(config_path, original_config.sub(/^browser: .*$/, "browser: #{name}").sub(/^ firefox:/, "#{edge_args} firefox:"))
|
|
959
|
+
end
|
|
960
|
+
|
|
961
|
+
attr_reader :browser_name
|
|
962
|
+
|
|
963
|
+
def attachment_names(project)
|
|
964
|
+
results(project).flat_map { |result| result['attachments'].map { |attachment| attachment['name'] } }
|
|
965
|
+
end
|
|
966
|
+
|
|
967
|
+
def comparison_names(project)
|
|
968
|
+
attachment_names(project).grep(/\Acomparison:/)
|
|
969
|
+
end
|
|
970
|
+
|
|
971
|
+
def baseline_path
|
|
972
|
+
File.join(project, baseline)
|
|
973
|
+
end
|
|
974
|
+
|
|
975
|
+
def write_rspec_spec
|
|
976
|
+
File.write(File.join(project, 'spec/screenshot_comparison_spec.rb'), <<~SPEC)
|
|
977
|
+
# frozen_string_literal: true
|
|
978
|
+
|
|
979
|
+
require_relative '../helpers/spec_helper'
|
|
980
|
+
|
|
981
|
+
describe 'Screenshot comparison' do
|
|
982
|
+
it 'matches home' do
|
|
983
|
+
driver.navigate.to('#{shot_page}')
|
|
984
|
+
driver.execute_script("#{recolour}") if ENV['SHOT_VARIANT'] == 'changed'
|
|
985
|
+
match_screenshot('home')
|
|
986
|
+
end
|
|
987
|
+
end
|
|
988
|
+
SPEC
|
|
989
|
+
end
|
|
990
|
+
|
|
991
|
+
# Every failure message in run.json, as text.
|
|
992
|
+
def failure_text(project)
|
|
993
|
+
results(project).to_json
|
|
994
|
+
end
|
|
995
|
+
|
|
996
|
+
def expect_ok(run)
|
|
997
|
+
expect(run[:exit_code]).to eq(0), "STDOUT: #{run[:stdout]}\nSTDERR: #{run[:stderr]}"
|
|
998
|
+
end
|
|
999
|
+
|
|
1000
|
+
def expect_failed(run)
|
|
1001
|
+
expect(run[:exit_code]).to eq(1), "STDOUT: #{run[:stdout]}\nSTDERR: #{run[:stderr]}"
|
|
1002
|
+
end
|
|
1003
|
+
|
|
1004
|
+
# The five runs of the plan, in order; each block is one run of the generated test.
|
|
1005
|
+
def walk_through(command)
|
|
1006
|
+
run = proc do |overrides = {}|
|
|
1007
|
+
FileUtils.rm_rf(File.join(project, 'raider-results'))
|
|
1008
|
+
run_in_project(project, command, env: shot_env(overrides), timeout: 300)
|
|
1009
|
+
end
|
|
1010
|
+
|
|
1011
|
+
expect_ok(run.call) # 1: creates the baseline
|
|
1012
|
+
expect(File).to exist(baseline_path)
|
|
1013
|
+
expect(File.binread(baseline_path)).to start_with("\x89PNG".b)
|
|
1014
|
+
first_bytes = File.binread(baseline_path)
|
|
1015
|
+
|
|
1016
|
+
expect_ok(run.call) # 2: matches
|
|
1017
|
+
expect(comparison_names(project)).to be_empty
|
|
1018
|
+
|
|
1019
|
+
different = run.call('SHOT_VARIANT' => 'changed') # 3: differs
|
|
1020
|
+
expect_failed(different)
|
|
1021
|
+
expect(comparison_names(project)).to include('comparison:home:expected', 'comparison:home:actual', 'comparison:home:diff')
|
|
1022
|
+
expect(failure_text(project)).to include(baseline)
|
|
1023
|
+
expect(File.binread(baseline_path)).to eq(first_bytes)
|
|
1024
|
+
expect(different[:stderr].scan('[screenshots] this raider_reporter has no comparison method').size).to be <= 1
|
|
1025
|
+
|
|
1026
|
+
FileUtils.rm_f(baseline_path) # 4: CI without a baseline
|
|
1027
|
+
missing = run.call('CI' => '1')
|
|
1028
|
+
expect_failed(missing)
|
|
1029
|
+
expect(File).not_to exist(baseline_path)
|
|
1030
|
+
expect(failure_text(project)).to include('no baseline')
|
|
1031
|
+
|
|
1032
|
+
expect_ok(run.call('SCREENSHOTS' => 'update')) # 5: update rewrites
|
|
1033
|
+
expect(File).to exist(baseline_path)
|
|
1034
|
+
expect_ok(run.call('SHOT_VARIANT' => 'changed', 'SCREENSHOTS' => 'update'))
|
|
1035
|
+
expect(File.binread(baseline_path)).not_to eq(first_bytes)
|
|
1036
|
+
end
|
|
1037
|
+
|
|
1038
|
+
%w[chrome edge firefox].each do |name|
|
|
1039
|
+
describe "Selenium + RSpec on headless #{name}" do
|
|
1040
|
+
let(:project) { 'video_rspec_selenium' }
|
|
1041
|
+
|
|
1042
|
+
before do
|
|
1043
|
+
generate_video_project('rspec', 'selenium')
|
|
1044
|
+
use_browser(name)
|
|
1045
|
+
write_rspec_spec
|
|
1046
|
+
end
|
|
1047
|
+
|
|
1048
|
+
it 'creates, matches, fails with images, refuses on CI and updates', :aggregate_failures do
|
|
1049
|
+
walk_through('bundle exec rspec spec/screenshot_comparison_spec.rb')
|
|
1050
|
+
end
|
|
1019
1051
|
end
|
|
1020
1052
|
end
|
|
1021
1053
|
end
|
|
@@ -1035,19 +1067,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
1035
1067
|
end
|
|
1036
1068
|
end
|
|
1037
1069
|
|
|
1038
|
-
describe 'iOS + Cucumber' do
|
|
1039
|
-
it 'generates valid project structure' do
|
|
1040
|
-
expect(File).to exist('cucumber_ios/features/support/env.rb')
|
|
1041
|
-
expect(File).to exist('cucumber_ios/Gemfile')
|
|
1042
|
-
expect(File).to exist('cucumber_ios/features')
|
|
1043
|
-
end
|
|
1044
|
-
|
|
1045
|
-
it 'has valid Ruby syntax in generated files' do
|
|
1046
|
-
result = run_in_project('cucumber_ios', 'ruby -c features/support/env.rb')
|
|
1047
|
-
expect(result[:success]).to be true
|
|
1048
|
-
end
|
|
1049
|
-
end
|
|
1050
|
-
|
|
1051
1070
|
describe 'Android + RSpec' do
|
|
1052
1071
|
it 'generates valid project structure' do
|
|
1053
1072
|
expect(File).to exist('rspec_android/helpers/spec_helper.rb')
|
|
@@ -1061,19 +1080,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
1061
1080
|
end
|
|
1062
1081
|
end
|
|
1063
1082
|
|
|
1064
|
-
describe 'Android + Cucumber' do
|
|
1065
|
-
it 'generates valid project structure' do
|
|
1066
|
-
expect(File).to exist('cucumber_android/features/support/env.rb')
|
|
1067
|
-
expect(File).to exist('cucumber_android/Gemfile')
|
|
1068
|
-
expect(File).to exist('cucumber_android/features')
|
|
1069
|
-
end
|
|
1070
|
-
|
|
1071
|
-
it 'has valid Ruby syntax in generated files' do
|
|
1072
|
-
result = run_in_project('cucumber_android', 'ruby -c features/support/env.rb')
|
|
1073
|
-
expect(result[:success]).to be true
|
|
1074
|
-
end
|
|
1075
|
-
end
|
|
1076
|
-
|
|
1077
1083
|
describe 'Cross-Platform + RSpec' do
|
|
1078
1084
|
it 'generates valid project structure' do
|
|
1079
1085
|
expect(File).to exist('rspec_cross_platform/helpers/appium_helper.rb')
|
|
@@ -1086,19 +1092,6 @@ describe 'End-to-End Project Generation and Execution' do
|
|
|
1086
1092
|
expect(result[:success]).to be true
|
|
1087
1093
|
end
|
|
1088
1094
|
end
|
|
1089
|
-
|
|
1090
|
-
describe 'Cross-Platform + Cucumber' do
|
|
1091
|
-
it 'generates valid project structure' do
|
|
1092
|
-
expect(File).to exist('cucumber_cross_platform/helpers/appium_helper.rb')
|
|
1093
|
-
expect(File).to exist('cucumber_cross_platform/Gemfile')
|
|
1094
|
-
expect(File).to exist('cucumber_cross_platform/features')
|
|
1095
|
-
end
|
|
1096
|
-
|
|
1097
|
-
it 'has valid Ruby syntax in generated files' do
|
|
1098
|
-
result = run_in_project('cucumber_cross_platform', 'ruby -c features/support/env.rb')
|
|
1099
|
-
expect(result[:success]).to be true
|
|
1100
|
-
end
|
|
1101
|
-
end
|
|
1102
1095
|
end
|
|
1103
1096
|
|
|
1104
1097
|
# Verify template system performance (caching working)
|