omnitest-skeptic 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +6 -0
- data/.rspec +2 -0
- data/.rubocop.yml +5 -0
- data/.rubocop_todo.yml +36 -0
- data/Gemfile +24 -0
- data/LICENSE.txt +22 -0
- data/README.md +31 -0
- data/Rakefile +12 -0
- data/bin/skeptic +4 -0
- data/lib/omnitest/skeptic.rb +123 -0
- data/lib/omnitest/skeptic/cli.rb +156 -0
- data/lib/omnitest/skeptic/configuration.rb +48 -0
- data/lib/omnitest/skeptic/errors.rb +19 -0
- data/lib/omnitest/skeptic/evidence.rb +81 -0
- data/lib/omnitest/skeptic/property_definition.rb +8 -0
- data/lib/omnitest/skeptic/result.rb +27 -0
- data/lib/omnitest/skeptic/scenario.rb +167 -0
- data/lib/omnitest/skeptic/scenario_definition.rb +41 -0
- data/lib/omnitest/skeptic/spies.rb +43 -0
- data/lib/omnitest/skeptic/spy.rb +23 -0
- data/lib/omnitest/skeptic/test_manifest.rb +78 -0
- data/lib/omnitest/skeptic/test_statuses.rb +63 -0
- data/lib/omnitest/skeptic/test_transitions.rb +172 -0
- data/lib/omnitest/skeptic/validation.rb +39 -0
- data/lib/omnitest/skeptic/validator.rb +34 -0
- data/lib/omnitest/skeptic/validator_registry.rb +33 -0
- data/lib/omnitest/skeptic/version.rb +5 -0
- data/omnitest-skeptic.gemspec +34 -0
- data/spec/fabricators/psychic_fabricator.rb +12 -0
- data/spec/fabricators/scenario_fabricator.rb +6 -0
- data/spec/fabricators/validator_fabricator.rb +12 -0
- data/spec/fixtures/factorial.py +18 -0
- data/spec/fixtures/skeptic.yaml +16 -0
- data/spec/omnitest/skeptic/evidence_spec.rb +58 -0
- data/spec/omnitest/skeptic/result_spec.rb +51 -0
- data/spec/omnitest/skeptic/scenario_definition_spec.rb +39 -0
- data/spec/omnitest/skeptic/scenario_spec.rb +35 -0
- data/spec/omnitest/skeptic/test_manifest_spec.rb +28 -0
- data/spec/omnitest/skeptic/validator_registry_spec.rb +40 -0
- data/spec/omnitest/skeptic/validator_spec.rb +70 -0
- data/spec/omnitest/skeptic_spec.rb +65 -0
- data/spec/spec_helper.rb +65 -0
- metadata +289 -0
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
RSpec.describe Evidence do
|
6
|
+
let(:file) { Pathname('evidence.pstore').expand_path(current_dir) }
|
7
|
+
subject { described_class.new(file) }
|
8
|
+
|
9
|
+
let(:sample_data) do
|
10
|
+
{
|
11
|
+
last_attempted_action: 'foo',
|
12
|
+
last_completed_action: 'bar',
|
13
|
+
result: {
|
14
|
+
execution_result: {
|
15
|
+
command: 'echo foo',
|
16
|
+
stdout: File.read(__FILE__),
|
17
|
+
stderr: File.read(__FILE__),
|
18
|
+
exitstatus: 0
|
19
|
+
},
|
20
|
+
source_file: 'foo/bar.rb',
|
21
|
+
data: {},
|
22
|
+
validations: nil
|
23
|
+
},
|
24
|
+
spy_data: {
|
25
|
+
complex: {
|
26
|
+
nested: %w(object that can be stored)
|
27
|
+
}
|
28
|
+
},
|
29
|
+
error: ::StandardError.new,
|
30
|
+
vars: {
|
31
|
+
a: 'b',
|
32
|
+
c: 'd',
|
33
|
+
e: 'f'
|
34
|
+
},
|
35
|
+
duration: 123.5
|
36
|
+
}
|
37
|
+
end
|
38
|
+
|
39
|
+
describe '#save' do
|
40
|
+
it 'creates a file' do
|
41
|
+
expect { subject.save }.to change { file.exist? }.from(false).to(true)
|
42
|
+
end
|
43
|
+
|
44
|
+
it 'persists data that can be reloaded' do
|
45
|
+
sample_data.each do |key, value|
|
46
|
+
subject[key] = value
|
47
|
+
end
|
48
|
+
subject.save
|
49
|
+
|
50
|
+
reloaded_evidence = described_class.load(file)
|
51
|
+
original_data = subject.to_hash
|
52
|
+
reloaded_data = reloaded_evidence.to_hash
|
53
|
+
expect(reloaded_data).to eq(original_data)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
describe Result do
|
6
|
+
describe '#status' do
|
7
|
+
context 'mixed pass/fail' do
|
8
|
+
let(:subject) do
|
9
|
+
Omnitest::Skeptic::Result.new(
|
10
|
+
validations: {
|
11
|
+
'max' => { status: 'passed' },
|
12
|
+
'omnitest' => { status: 'failed', error: 'foo!' }
|
13
|
+
}
|
14
|
+
).status
|
15
|
+
end
|
16
|
+
it 'reports the failed status' do
|
17
|
+
is_expected.to eq('failed')
|
18
|
+
end
|
19
|
+
end
|
20
|
+
context 'mix passed/pending/skipped' do
|
21
|
+
let(:subject) do
|
22
|
+
Omnitest::Skeptic::Result.new(
|
23
|
+
validations: {
|
24
|
+
'max' => { status: 'passed' },
|
25
|
+
'omnitest' => { status: 'pending' },
|
26
|
+
'john doe' => { status: 'skipped' }
|
27
|
+
}
|
28
|
+
).status
|
29
|
+
end
|
30
|
+
it 'reports the passed status' do
|
31
|
+
is_expected.to eq('passed')
|
32
|
+
end
|
33
|
+
end
|
34
|
+
context 'mix pending/skipped' do
|
35
|
+
let(:subject) do
|
36
|
+
Omnitest::Skeptic::Result.new(
|
37
|
+
validations: {
|
38
|
+
'max' => { status: 'pending' },
|
39
|
+
'omnitest' => { status: 'pending' },
|
40
|
+
'john doe' => { status: 'skipped' }
|
41
|
+
}
|
42
|
+
).status
|
43
|
+
end
|
44
|
+
it 'reports the pending status' do
|
45
|
+
is_expected.to eq('pending')
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
RSpec.describe ScenarioDefinition do
|
6
|
+
let(:psychic) { Fabricate(:psychic) }
|
7
|
+
let(:definition) do
|
8
|
+
{
|
9
|
+
name: 'My test scenario',
|
10
|
+
suite: 'My API',
|
11
|
+
properties: {
|
12
|
+
foo: {
|
13
|
+
required: true,
|
14
|
+
default: 'bar'
|
15
|
+
}
|
16
|
+
}
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
subject { described_class.new(definition) }
|
21
|
+
|
22
|
+
describe '#build' do
|
23
|
+
let(:scenario) { subject.build psychic }
|
24
|
+
|
25
|
+
it 'builds a Scenario for the Project' do
|
26
|
+
expect(scenario).to be_an_instance_of Scenario
|
27
|
+
# It actually creates a Psychic for the psychic
|
28
|
+
expect(scenario.psychic.name).to eq(psychic.name)
|
29
|
+
expect(scenario.psychic.basedir.to_s).to eq(psychic.basedir.to_s)
|
30
|
+
end
|
31
|
+
|
32
|
+
xit 'finds the source' do
|
33
|
+
expected_file = Pathname.new 'spec/fixtures/factorial.py'
|
34
|
+
expect(scenario.source_file).to eq(expected_file)
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
describe Scenario do
|
4
|
+
subject(:scenario) do
|
5
|
+
project = Omnitest::Psychic.new name: 'some_sdk', cwd: 'spec/fixtures'
|
6
|
+
Fabricate(:scenario_definition, name: 'factorial', vars: {}).build(project)
|
7
|
+
end
|
8
|
+
|
9
|
+
describe '#detect' do
|
10
|
+
pending 'finds a script for the scenario'
|
11
|
+
end
|
12
|
+
|
13
|
+
describe '#check' do
|
14
|
+
pending 'checks the script complies with static rules'
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#prepare' do
|
18
|
+
pending 'meets unmet dependencies'
|
19
|
+
end
|
20
|
+
|
21
|
+
describe '#exec' do
|
22
|
+
it 'executes the scenario and returns itself' do
|
23
|
+
expect(scenario.exec).to be_an_instance_of Scenario
|
24
|
+
expect(scenario.exec).to eq(scenario)
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'stores the result' do
|
28
|
+
evidence = scenario.exec
|
29
|
+
result = evidence.result
|
30
|
+
expect(result).to be_an_instance_of Skeptic::Result
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
describe TestManifest do
|
4
|
+
describe '#from_yaml' do
|
5
|
+
subject(:manifest) { described_class.from_yaml 'spec/fixtures/skeptic.yaml' }
|
6
|
+
|
7
|
+
it 'initializes a manifest' do
|
8
|
+
expect(manifest).to be_an_instance_of TestManifest
|
9
|
+
end
|
10
|
+
|
11
|
+
it 'processes ERB' do
|
12
|
+
expect(manifest.global_env.LOCALE).to eq(ENV['LANG'])
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'parses global_env' do
|
16
|
+
expect(manifest.global_env).to be_an_instance_of TestManifest::Environment
|
17
|
+
end
|
18
|
+
|
19
|
+
it 'parses suites' do
|
20
|
+
expect(manifest.suites).to be_an_instance_of ::Hash
|
21
|
+
manifest.suites.each_value do | suite |
|
22
|
+
expect(suite).to be_an_instance_of TestManifest::Suite
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
module Omnitest
|
2
|
+
class Skeptic
|
3
|
+
describe ValidatorRegistry do
|
4
|
+
subject(:registry) { Omnitest::Skeptic::ValidatorRegistry }
|
5
|
+
|
6
|
+
describe '#register' do
|
7
|
+
it 'registers a validator' do
|
8
|
+
callback = proc do |scenario|
|
9
|
+
expect(scenario.result).to_not be_nil
|
10
|
+
expect(scenario.result.execution_result.exitstatus).to eq(0)
|
11
|
+
end
|
12
|
+
|
13
|
+
expect(registry.validators).to be_empty
|
14
|
+
registry.register(Validator.new('dummy', suite: 'java', scenario: 'hello world', &callback))
|
15
|
+
validator = registry.validators.first
|
16
|
+
expect(validator.suite).to eql('java')
|
17
|
+
expect(validator.scenario).to eql('hello world')
|
18
|
+
expect(validator.instance_variable_get('@callback')).to eql(callback)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
describe '#validators_for' do
|
23
|
+
let(:java_hello_world_validator) { Fabricate(:validator, suite: 'java', scenario: 'hello world') }
|
24
|
+
let(:java_validator) { Fabricate(:validator, suite: 'java', scenario: //) }
|
25
|
+
let(:ruby_validator) { Fabricate(:validator, suite: 'ruby') }
|
26
|
+
|
27
|
+
it 'returns registered validators that match the scope of the scenario' do
|
28
|
+
registry.register(java_hello_world_validator)
|
29
|
+
registry.register(java_validator)
|
30
|
+
registry.register(ruby_validator)
|
31
|
+
|
32
|
+
scenario = Fabricate(:scenario_definition, suite: 'java', name: 'hello world')
|
33
|
+
validators = registry.validators_for scenario
|
34
|
+
expect(validators).to include(java_hello_world_validator, java_validator)
|
35
|
+
expect(validators).to_not include(ruby_validator)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
class Skeptic
|
5
|
+
describe 'Validator' do
|
6
|
+
describe '#initialize' do
|
7
|
+
let(:global_matcher) { Validator::UNIVERSAL_MATCHER }
|
8
|
+
|
9
|
+
it 'accepts scope options and callback' do
|
10
|
+
validator = Validator.new 'dummy', suite: 'java', scenario: 'hello world' do |_scenario|
|
11
|
+
# Validate the scenario
|
12
|
+
end
|
13
|
+
expect(validator.suite).to eq('java')
|
14
|
+
end
|
15
|
+
|
16
|
+
it 'defaults suite and scenario to the universal matcher' do
|
17
|
+
validator = Validator.new 'dummy' do |_scenario|
|
18
|
+
# Validate
|
19
|
+
end
|
20
|
+
expect(validator.suite).to eq(Validator::UNIVERSAL_MATCHER)
|
21
|
+
expect(validator.scenario).to eq(Validator::UNIVERSAL_MATCHER)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
describe '#should_validate?' do
|
26
|
+
let(:scenario) do
|
27
|
+
Fabricate(:scenario_definition, suite: 'java', name: 'hello world')
|
28
|
+
end
|
29
|
+
|
30
|
+
it 'returns true if the scope matches the scope of the scenario' do
|
31
|
+
expect(validator('java', 'hello world').should_validate? scenario).to be true
|
32
|
+
expect(validator('java').should_validate? scenario).to be true
|
33
|
+
expect(validator(/j/, /hello/).should_validate? scenario).to be true
|
34
|
+
end
|
35
|
+
|
36
|
+
it 'returns false if the scope does not match' do
|
37
|
+
expect(validator('ruby', 'hello world').should_validate? scenario).to be false
|
38
|
+
expect(validator('ruby').should_validate? scenario).to be false
|
39
|
+
expect(validator(/r/, /hello/).should_validate? scenario).to be false
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
describe '#validate' do
|
44
|
+
let(:psychic) { Fabricate(:psychic) }
|
45
|
+
let(:scenario) do
|
46
|
+
Fabricate(:scenario_definition).build(psychic).tap do | scenario |
|
47
|
+
scenario.result = Result.new
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
it 'calls the validation callback' do
|
52
|
+
called = false
|
53
|
+
validator = Validator.new 'dummy' do |_scenario|
|
54
|
+
called = true
|
55
|
+
end
|
56
|
+
expect { validator.validate scenario }.to change { called }.from(false).to(true)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def validator(*args)
|
61
|
+
scope = {}
|
62
|
+
scope[:suite] = args[0]
|
63
|
+
scope[:scenario] = args[1] if args[1]
|
64
|
+
Validator.new 'dummy', scope do |_scenario|
|
65
|
+
# Dummy validator
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
module Omnitest
|
4
|
+
RSpec.describe Skeptic do
|
5
|
+
before(:each) do | _example |
|
6
|
+
write_file 'skeptic.yaml', <<-eos
|
7
|
+
---
|
8
|
+
suites:
|
9
|
+
Execution:
|
10
|
+
samples:
|
11
|
+
- success
|
12
|
+
- failure
|
13
|
+
eos
|
14
|
+
write_file 'success.rb', 'puts "foo"; exit 0'
|
15
|
+
write_file 'failure.rb', 'puts "bar"; exit 1'
|
16
|
+
|
17
|
+
Skeptic.configure do | config |
|
18
|
+
config.manifest_file = File.expand_path('skeptic.yaml', current_dir)
|
19
|
+
end
|
20
|
+
|
21
|
+
subject.clear
|
22
|
+
end
|
23
|
+
|
24
|
+
let(:psychic) { Psychic.new(cwd: current_dir) }
|
25
|
+
subject { described_class.new(psychic) }
|
26
|
+
|
27
|
+
describe '#scenarios' do
|
28
|
+
it 'returns the list of all scenarios' do
|
29
|
+
scenarios = subject.scenarios
|
30
|
+
expect(scenarios).to_not be_empty
|
31
|
+
scenarios.each do | scenario |
|
32
|
+
expect(scenario).to be_an_instance_of Skeptic::Scenario
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe '#prepare' do
|
38
|
+
let(:satisifed_dependency) { double('satisifed_dependency', met?: true) }
|
39
|
+
let(:unsatisifed_dependency) { double('unsatisifed_dependency', met?: false) }
|
40
|
+
|
41
|
+
pending 'meets unmet dependencies' do
|
42
|
+
scenarios = subject.scenarios
|
43
|
+
expect(satisifed_dependency).to_not receive(:meet)
|
44
|
+
expect(satisifed_dependency).to receive(:meet)
|
45
|
+
subject.prepare
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
describe '#exec' do
|
50
|
+
it 'calls exec on each scenario and stores an execution result' do
|
51
|
+
scenarios = subject.scenarios
|
52
|
+
results = scenarios.map(&:result)
|
53
|
+
expect(results).to all(be nil)
|
54
|
+
subject.exec
|
55
|
+
results = scenarios.map(&:result)
|
56
|
+
expect(results).to all(be_an_instance_of Skeptic::Result)
|
57
|
+
expect(scenarios.map(&:status_description)).to all(eq 'Executed')
|
58
|
+
|
59
|
+
expect(subject.scenario('success').result).to be_successful
|
60
|
+
expect(subject.scenario('failure').result).to_not be_successful
|
61
|
+
puts subject.summary
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,65 @@
|
|
1
|
+
require 'simplecov'
|
2
|
+
SimpleCov.start
|
3
|
+
|
4
|
+
require 'omnitest/skeptic'
|
5
|
+
require 'fabrication'
|
6
|
+
|
7
|
+
# For Fabricators
|
8
|
+
LANGUAGES = %w(java ruby python nodejs c# golang php)
|
9
|
+
SCENARIO_NAMES = [
|
10
|
+
'hello world',
|
11
|
+
'quine',
|
12
|
+
'my_kata'
|
13
|
+
]
|
14
|
+
|
15
|
+
require 'rspec'
|
16
|
+
require 'omnitest/skeptic'
|
17
|
+
require 'aruba'
|
18
|
+
require 'aruba/api'
|
19
|
+
|
20
|
+
# Config required for project
|
21
|
+
RSpec.configure do | config |
|
22
|
+
config.include Aruba::Api
|
23
|
+
config.before(:example) do
|
24
|
+
@aruba_timeout_seconds = 30
|
25
|
+
clean_current_dir
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
RSpec.configure do |c|
|
30
|
+
c.before(:each) do
|
31
|
+
Omnitest::Skeptic.reset
|
32
|
+
end
|
33
|
+
c.expose_current_running_example_as :example
|
34
|
+
end
|
35
|
+
|
36
|
+
# Configs recommended by RSpec
|
37
|
+
RSpec.configure do |config|
|
38
|
+
# config.warnings = true # Unfortunately this produces too many warnings in third-party code
|
39
|
+
# config.disable_monkey_patching!
|
40
|
+
config.filter_run :focus
|
41
|
+
config.run_all_when_everything_filtered = true
|
42
|
+
|
43
|
+
config.expect_with :rspec do |expectations|
|
44
|
+
# This option will default to `true` in RSpec 4.
|
45
|
+
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
|
46
|
+
end
|
47
|
+
|
48
|
+
config.mock_with :rspec do |mocks|
|
49
|
+
# Prevents you from mocking or stubbing a method that does not exist on
|
50
|
+
# a real object. This is generally recommended, and will default to
|
51
|
+
# `true` in RSpec 4.
|
52
|
+
mocks.verify_partial_doubles = true
|
53
|
+
end
|
54
|
+
|
55
|
+
if config.files_to_run.one?
|
56
|
+
# Use the documentation formatter for detailed output,
|
57
|
+
# unless a formatter has already been configured
|
58
|
+
# (e.g. via a command-line flag).
|
59
|
+
config.default_formatter = 'doc'
|
60
|
+
end
|
61
|
+
|
62
|
+
# config.profile_examples = 10
|
63
|
+
config.order = :random
|
64
|
+
Kernel.srand config.seed
|
65
|
+
end
|