omnitest-skeptic 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +6 -0
  3. data/.rspec +2 -0
  4. data/.rubocop.yml +5 -0
  5. data/.rubocop_todo.yml +36 -0
  6. data/Gemfile +24 -0
  7. data/LICENSE.txt +22 -0
  8. data/README.md +31 -0
  9. data/Rakefile +12 -0
  10. data/bin/skeptic +4 -0
  11. data/lib/omnitest/skeptic.rb +123 -0
  12. data/lib/omnitest/skeptic/cli.rb +156 -0
  13. data/lib/omnitest/skeptic/configuration.rb +48 -0
  14. data/lib/omnitest/skeptic/errors.rb +19 -0
  15. data/lib/omnitest/skeptic/evidence.rb +81 -0
  16. data/lib/omnitest/skeptic/property_definition.rb +8 -0
  17. data/lib/omnitest/skeptic/result.rb +27 -0
  18. data/lib/omnitest/skeptic/scenario.rb +167 -0
  19. data/lib/omnitest/skeptic/scenario_definition.rb +41 -0
  20. data/lib/omnitest/skeptic/spies.rb +43 -0
  21. data/lib/omnitest/skeptic/spy.rb +23 -0
  22. data/lib/omnitest/skeptic/test_manifest.rb +78 -0
  23. data/lib/omnitest/skeptic/test_statuses.rb +63 -0
  24. data/lib/omnitest/skeptic/test_transitions.rb +172 -0
  25. data/lib/omnitest/skeptic/validation.rb +39 -0
  26. data/lib/omnitest/skeptic/validator.rb +34 -0
  27. data/lib/omnitest/skeptic/validator_registry.rb +33 -0
  28. data/lib/omnitest/skeptic/version.rb +5 -0
  29. data/omnitest-skeptic.gemspec +34 -0
  30. data/spec/fabricators/psychic_fabricator.rb +12 -0
  31. data/spec/fabricators/scenario_fabricator.rb +6 -0
  32. data/spec/fabricators/validator_fabricator.rb +12 -0
  33. data/spec/fixtures/factorial.py +18 -0
  34. data/spec/fixtures/skeptic.yaml +16 -0
  35. data/spec/omnitest/skeptic/evidence_spec.rb +58 -0
  36. data/spec/omnitest/skeptic/result_spec.rb +51 -0
  37. data/spec/omnitest/skeptic/scenario_definition_spec.rb +39 -0
  38. data/spec/omnitest/skeptic/scenario_spec.rb +35 -0
  39. data/spec/omnitest/skeptic/test_manifest_spec.rb +28 -0
  40. data/spec/omnitest/skeptic/validator_registry_spec.rb +40 -0
  41. data/spec/omnitest/skeptic/validator_spec.rb +70 -0
  42. data/spec/omnitest/skeptic_spec.rb +65 -0
  43. data/spec/spec_helper.rb +65 -0
  44. metadata +289 -0
@@ -0,0 +1,58 @@
1
+ require 'spec_helper'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ RSpec.describe Evidence do
6
+ let(:file) { Pathname('evidence.pstore').expand_path(current_dir) }
7
+ subject { described_class.new(file) }
8
+
9
+ let(:sample_data) do
10
+ {
11
+ last_attempted_action: 'foo',
12
+ last_completed_action: 'bar',
13
+ result: {
14
+ execution_result: {
15
+ command: 'echo foo',
16
+ stdout: File.read(__FILE__),
17
+ stderr: File.read(__FILE__),
18
+ exitstatus: 0
19
+ },
20
+ source_file: 'foo/bar.rb',
21
+ data: {},
22
+ validations: nil
23
+ },
24
+ spy_data: {
25
+ complex: {
26
+ nested: %w(object that can be stored)
27
+ }
28
+ },
29
+ error: ::StandardError.new,
30
+ vars: {
31
+ a: 'b',
32
+ c: 'd',
33
+ e: 'f'
34
+ },
35
+ duration: 123.5
36
+ }
37
+ end
38
+
39
+ describe '#save' do
40
+ it 'creates a file' do
41
+ expect { subject.save }.to change { file.exist? }.from(false).to(true)
42
+ end
43
+
44
+ it 'persists data that can be reloaded' do
45
+ sample_data.each do |key, value|
46
+ subject[key] = value
47
+ end
48
+ subject.save
49
+
50
+ reloaded_evidence = described_class.load(file)
51
+ original_data = subject.to_hash
52
+ reloaded_data = reloaded_evidence.to_hash
53
+ expect(reloaded_data).to eq(original_data)
54
+ end
55
+ end
56
+ end
57
+ end
58
+ end
@@ -0,0 +1,51 @@
1
+ require 'spec_helper'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ describe Result do
6
+ describe '#status' do
7
+ context 'mixed pass/fail' do
8
+ let(:subject) do
9
+ Omnitest::Skeptic::Result.new(
10
+ validations: {
11
+ 'max' => { status: 'passed' },
12
+ 'omnitest' => { status: 'failed', error: 'foo!' }
13
+ }
14
+ ).status
15
+ end
16
+ it 'reports the failed status' do
17
+ is_expected.to eq('failed')
18
+ end
19
+ end
20
+ context 'mix passed/pending/skipped' do
21
+ let(:subject) do
22
+ Omnitest::Skeptic::Result.new(
23
+ validations: {
24
+ 'max' => { status: 'passed' },
25
+ 'omnitest' => { status: 'pending' },
26
+ 'john doe' => { status: 'skipped' }
27
+ }
28
+ ).status
29
+ end
30
+ it 'reports the passed status' do
31
+ is_expected.to eq('passed')
32
+ end
33
+ end
34
+ context 'mix pending/skipped' do
35
+ let(:subject) do
36
+ Omnitest::Skeptic::Result.new(
37
+ validations: {
38
+ 'max' => { status: 'pending' },
39
+ 'omnitest' => { status: 'pending' },
40
+ 'john doe' => { status: 'skipped' }
41
+ }
42
+ ).status
43
+ end
44
+ it 'reports the pending status' do
45
+ is_expected.to eq('pending')
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,39 @@
1
+ require 'spec_helper'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ RSpec.describe ScenarioDefinition do
6
+ let(:psychic) { Fabricate(:psychic) }
7
+ let(:definition) do
8
+ {
9
+ name: 'My test scenario',
10
+ suite: 'My API',
11
+ properties: {
12
+ foo: {
13
+ required: true,
14
+ default: 'bar'
15
+ }
16
+ }
17
+ }
18
+ end
19
+
20
+ subject { described_class.new(definition) }
21
+
22
+ describe '#build' do
23
+ let(:scenario) { subject.build psychic }
24
+
25
+ it 'builds a Scenario for the Project' do
26
+ expect(scenario).to be_an_instance_of Scenario
27
+ # It actually creates a Psychic for the psychic
28
+ expect(scenario.psychic.name).to eq(psychic.name)
29
+ expect(scenario.psychic.basedir.to_s).to eq(psychic.basedir.to_s)
30
+ end
31
+
32
+ xit 'finds the source' do
33
+ expected_file = Pathname.new 'spec/fixtures/factorial.py'
34
+ expect(scenario.source_file).to eq(expected_file)
35
+ end
36
+ end
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,35 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ describe Scenario do
4
+ subject(:scenario) do
5
+ project = Omnitest::Psychic.new name: 'some_sdk', cwd: 'spec/fixtures'
6
+ Fabricate(:scenario_definition, name: 'factorial', vars: {}).build(project)
7
+ end
8
+
9
+ describe '#detect' do
10
+ pending 'finds a script for the scenario'
11
+ end
12
+
13
+ describe '#check' do
14
+ pending 'checks the script complies with static rules'
15
+ end
16
+
17
+ describe '#prepare' do
18
+ pending 'meets unmet dependencies'
19
+ end
20
+
21
+ describe '#exec' do
22
+ it 'executes the scenario and returns itself' do
23
+ expect(scenario.exec).to be_an_instance_of Scenario
24
+ expect(scenario.exec).to eq(scenario)
25
+ end
26
+
27
+ it 'stores the result' do
28
+ evidence = scenario.exec
29
+ result = evidence.result
30
+ expect(result).to be_an_instance_of Skeptic::Result
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,28 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ describe TestManifest do
4
+ describe '#from_yaml' do
5
+ subject(:manifest) { described_class.from_yaml 'spec/fixtures/skeptic.yaml' }
6
+
7
+ it 'initializes a manifest' do
8
+ expect(manifest).to be_an_instance_of TestManifest
9
+ end
10
+
11
+ it 'processes ERB' do
12
+ expect(manifest.global_env.LOCALE).to eq(ENV['LANG'])
13
+ end
14
+
15
+ it 'parses global_env' do
16
+ expect(manifest.global_env).to be_an_instance_of TestManifest::Environment
17
+ end
18
+
19
+ it 'parses suites' do
20
+ expect(manifest.suites).to be_an_instance_of ::Hash
21
+ manifest.suites.each_value do | suite |
22
+ expect(suite).to be_an_instance_of TestManifest::Suite
23
+ end
24
+ end
25
+ end
26
+ end
27
+ end
28
+ end
@@ -0,0 +1,40 @@
1
+ module Omnitest
2
+ class Skeptic
3
+ describe ValidatorRegistry do
4
+ subject(:registry) { Omnitest::Skeptic::ValidatorRegistry }
5
+
6
+ describe '#register' do
7
+ it 'registers a validator' do
8
+ callback = proc do |scenario|
9
+ expect(scenario.result).to_not be_nil
10
+ expect(scenario.result.execution_result.exitstatus).to eq(0)
11
+ end
12
+
13
+ expect(registry.validators).to be_empty
14
+ registry.register(Validator.new('dummy', suite: 'java', scenario: 'hello world', &callback))
15
+ validator = registry.validators.first
16
+ expect(validator.suite).to eql('java')
17
+ expect(validator.scenario).to eql('hello world')
18
+ expect(validator.instance_variable_get('@callback')).to eql(callback)
19
+ end
20
+ end
21
+
22
+ describe '#validators_for' do
23
+ let(:java_hello_world_validator) { Fabricate(:validator, suite: 'java', scenario: 'hello world') }
24
+ let(:java_validator) { Fabricate(:validator, suite: 'java', scenario: //) }
25
+ let(:ruby_validator) { Fabricate(:validator, suite: 'ruby') }
26
+
27
+ it 'returns registered validators that match the scope of the scenario' do
28
+ registry.register(java_hello_world_validator)
29
+ registry.register(java_validator)
30
+ registry.register(ruby_validator)
31
+
32
+ scenario = Fabricate(:scenario_definition, suite: 'java', name: 'hello world')
33
+ validators = registry.validators_for scenario
34
+ expect(validators).to include(java_hello_world_validator, java_validator)
35
+ expect(validators).to_not include(ruby_validator)
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,70 @@
1
+ require 'spec_helper'
2
+
3
+ module Omnitest
4
+ class Skeptic
5
+ describe 'Validator' do
6
+ describe '#initialize' do
7
+ let(:global_matcher) { Validator::UNIVERSAL_MATCHER }
8
+
9
+ it 'accepts scope options and callback' do
10
+ validator = Validator.new 'dummy', suite: 'java', scenario: 'hello world' do |_scenario|
11
+ # Validate the scenario
12
+ end
13
+ expect(validator.suite).to eq('java')
14
+ end
15
+
16
+ it 'defaults suite and scenario to the universal matcher' do
17
+ validator = Validator.new 'dummy' do |_scenario|
18
+ # Validate
19
+ end
20
+ expect(validator.suite).to eq(Validator::UNIVERSAL_MATCHER)
21
+ expect(validator.scenario).to eq(Validator::UNIVERSAL_MATCHER)
22
+ end
23
+ end
24
+
25
+ describe '#should_validate?' do
26
+ let(:scenario) do
27
+ Fabricate(:scenario_definition, suite: 'java', name: 'hello world')
28
+ end
29
+
30
+ it 'returns true if the scope matches the scope of the scenario' do
31
+ expect(validator('java', 'hello world').should_validate? scenario).to be true
32
+ expect(validator('java').should_validate? scenario).to be true
33
+ expect(validator(/j/, /hello/).should_validate? scenario).to be true
34
+ end
35
+
36
+ it 'returns false if the scope does not match' do
37
+ expect(validator('ruby', 'hello world').should_validate? scenario).to be false
38
+ expect(validator('ruby').should_validate? scenario).to be false
39
+ expect(validator(/r/, /hello/).should_validate? scenario).to be false
40
+ end
41
+ end
42
+
43
+ describe '#validate' do
44
+ let(:psychic) { Fabricate(:psychic) }
45
+ let(:scenario) do
46
+ Fabricate(:scenario_definition).build(psychic).tap do | scenario |
47
+ scenario.result = Result.new
48
+ end
49
+ end
50
+
51
+ it 'calls the validation callback' do
52
+ called = false
53
+ validator = Validator.new 'dummy' do |_scenario|
54
+ called = true
55
+ end
56
+ expect { validator.validate scenario }.to change { called }.from(false).to(true)
57
+ end
58
+ end
59
+
60
+ def validator(*args)
61
+ scope = {}
62
+ scope[:suite] = args[0]
63
+ scope[:scenario] = args[1] if args[1]
64
+ Validator.new 'dummy', scope do |_scenario|
65
+ # Dummy validator
66
+ end
67
+ end
68
+ end
69
+ end
70
+ end
@@ -0,0 +1,65 @@
1
+ require 'spec_helper'
2
+
3
+ module Omnitest
4
+ RSpec.describe Skeptic do
5
+ before(:each) do | _example |
6
+ write_file 'skeptic.yaml', <<-eos
7
+ ---
8
+ suites:
9
+ Execution:
10
+ samples:
11
+ - success
12
+ - failure
13
+ eos
14
+ write_file 'success.rb', 'puts "foo"; exit 0'
15
+ write_file 'failure.rb', 'puts "bar"; exit 1'
16
+
17
+ Skeptic.configure do | config |
18
+ config.manifest_file = File.expand_path('skeptic.yaml', current_dir)
19
+ end
20
+
21
+ subject.clear
22
+ end
23
+
24
+ let(:psychic) { Psychic.new(cwd: current_dir) }
25
+ subject { described_class.new(psychic) }
26
+
27
+ describe '#scenarios' do
28
+ it 'returns the list of all scenarios' do
29
+ scenarios = subject.scenarios
30
+ expect(scenarios).to_not be_empty
31
+ scenarios.each do | scenario |
32
+ expect(scenario).to be_an_instance_of Skeptic::Scenario
33
+ end
34
+ end
35
+ end
36
+
37
+ describe '#prepare' do
38
+ let(:satisifed_dependency) { double('satisifed_dependency', met?: true) }
39
+ let(:unsatisifed_dependency) { double('unsatisifed_dependency', met?: false) }
40
+
41
+ pending 'meets unmet dependencies' do
42
+ scenarios = subject.scenarios
43
+ expect(satisifed_dependency).to_not receive(:meet)
44
+ expect(satisifed_dependency).to receive(:meet)
45
+ subject.prepare
46
+ end
47
+ end
48
+
49
+ describe '#exec' do
50
+ it 'calls exec on each scenario and stores an execution result' do
51
+ scenarios = subject.scenarios
52
+ results = scenarios.map(&:result)
53
+ expect(results).to all(be nil)
54
+ subject.exec
55
+ results = scenarios.map(&:result)
56
+ expect(results).to all(be_an_instance_of Skeptic::Result)
57
+ expect(scenarios.map(&:status_description)).to all(eq 'Executed')
58
+
59
+ expect(subject.scenario('success').result).to be_successful
60
+ expect(subject.scenario('failure').result).to_not be_successful
61
+ puts subject.summary
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,65 @@
1
+ require 'simplecov'
2
+ SimpleCov.start
3
+
4
+ require 'omnitest/skeptic'
5
+ require 'fabrication'
6
+
7
+ # For Fabricators
8
+ LANGUAGES = %w(java ruby python nodejs c# golang php)
9
+ SCENARIO_NAMES = [
10
+ 'hello world',
11
+ 'quine',
12
+ 'my_kata'
13
+ ]
14
+
15
+ require 'rspec'
16
+ require 'omnitest/skeptic'
17
+ require 'aruba'
18
+ require 'aruba/api'
19
+
20
+ # Config required for project
21
+ RSpec.configure do | config |
22
+ config.include Aruba::Api
23
+ config.before(:example) do
24
+ @aruba_timeout_seconds = 30
25
+ clean_current_dir
26
+ end
27
+ end
28
+
29
+ RSpec.configure do |c|
30
+ c.before(:each) do
31
+ Omnitest::Skeptic.reset
32
+ end
33
+ c.expose_current_running_example_as :example
34
+ end
35
+
36
+ # Configs recommended by RSpec
37
+ RSpec.configure do |config|
38
+ # config.warnings = true # Unfortunately this produces too many warnings in third-party code
39
+ # config.disable_monkey_patching!
40
+ config.filter_run :focus
41
+ config.run_all_when_everything_filtered = true
42
+
43
+ config.expect_with :rspec do |expectations|
44
+ # This option will default to `true` in RSpec 4.
45
+ expectations.include_chain_clauses_in_custom_matcher_descriptions = true
46
+ end
47
+
48
+ config.mock_with :rspec do |mocks|
49
+ # Prevents you from mocking or stubbing a method that does not exist on
50
+ # a real object. This is generally recommended, and will default to
51
+ # `true` in RSpec 4.
52
+ mocks.verify_partial_doubles = true
53
+ end
54
+
55
+ if config.files_to_run.one?
56
+ # Use the documentation formatter for detailed output,
57
+ # unless a formatter has already been configured
58
+ # (e.g. via a command-line flag).
59
+ config.default_formatter = 'doc'
60
+ end
61
+
62
+ # config.profile_examples = 10
63
+ config.order = :random
64
+ Kernel.srand config.seed
65
+ end