dat-analysis 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. data/.gitignore +4 -0
  2. data/Gemfile +2 -0
  3. data/LICENSE.txt +22 -0
  4. data/README.md +423 -0
  5. data/dat-analysis.gemspec +17 -0
  6. data/lib/dat/analysis.rb +446 -0
  7. data/lib/dat/analysis/library.rb +30 -0
  8. data/lib/dat/analysis/matcher.rb +43 -0
  9. data/lib/dat/analysis/registry.rb +50 -0
  10. data/lib/dat/analysis/result.rb +78 -0
  11. data/lib/dat/analysis/tally.rb +59 -0
  12. data/script/bootstrap +9 -0
  13. data/script/release +38 -0
  14. data/script/test +9 -0
  15. data/test/dat_analysis_subclassing_test.rb +119 -0
  16. data/test/dat_analysis_test.rb +822 -0
  17. data/test/fixtures/analysis/test-suite-experiment/matcher.rb +7 -0
  18. data/test/fixtures/experiment-with-classes/matcher_a.rb +5 -0
  19. data/test/fixtures/experiment-with-classes/matcher_b.rb +11 -0
  20. data/test/fixtures/experiment-with-classes/wrapper_a.rb +5 -0
  21. data/test/fixtures/experiment-with-classes/wrapper_b.rb +11 -0
  22. data/test/fixtures/experiment-with-good-and-extraneous-classes/matcher_w.rb +5 -0
  23. data/test/fixtures/experiment-with-good-and-extraneous-classes/matcher_y.rb +11 -0
  24. data/test/fixtures/experiment-with-good-and-extraneous-classes/matcher_z.rb +11 -0
  25. data/test/fixtures/experiment-with-good-and-extraneous-classes/wrapper_w.rb +5 -0
  26. data/test/fixtures/experiment-with-good-and-extraneous-classes/wrapper_y.rb +11 -0
  27. data/test/fixtures/experiment-with-good-and-extraneous-classes/wrapper_z.rb +11 -0
  28. data/test/fixtures/initialize-classes/matcher_m.rb +5 -0
  29. data/test/fixtures/initialize-classes/matcher_n.rb +11 -0
  30. data/test/fixtures/initialize-classes/wrapper_m.rb +5 -0
  31. data/test/fixtures/initialize-classes/wrapper_n.rb +11 -0
  32. data/test/fixtures/invalid-matcher/matcher.rb +1 -0
  33. metadata +128 -0
@@ -0,0 +1,30 @@
1
+ module Dat
2
+ # Internal: Keep a registry of Dat::Analysis::Matcher and
3
+ # Dat::Analysis::Result subclasses for use by an Dat::Analysis::Analysis
4
+ # instance.
5
+ class Analysis::Library
6
+
7
+ @@known_classes = []
8
+
9
+ # Public: Collect matcher and results classes created by the
10
+ # provided block.
11
+ #
12
+ # &block - Block which instantiates matcher and results classes.
13
+ #
14
+ # Returns the newly-instantiated matcher and results classes.
15
+ def self.select_classes(&block)
16
+ @@known_classes = [] # prepare for registering new classes
17
+ yield
18
+ @@known_classes # return all the newly-registered classes
19
+ end
20
+
21
+ # Public: register a matcher or results class.
22
+ #
23
+ # klass - a Dat::Analysis::Matcher or Dat::Analysis::Result subclass.
24
+ #
25
+ # Returns the current list of registered classes.
26
+ def self.add(klass)
27
+ @@known_classes << klass
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,43 @@
1
+ module Dat
2
+ # Public: Base class for science mismatch results matchers. Subclasses
3
+ # implement the `#match?` instance method, which returns true when
4
+ # a provided science mismatch result is recognized by the matcher.
5
+ #
6
+ # Subclasses are expected to define `#match?`.
7
+ #
8
+ # Subclasses may optionally define `#readable` to return an alternative
9
+ # readable String representation of a cooked science mismatch result. The
10
+ # default implementation is defined in Dat::Analysis#readable.
11
+ class Analysis::Matcher
12
+
13
+ # Public: The science mismatch result to be matched.
14
+ attr_reader :result
15
+
16
+ # Internal: Called at subclass instantiation time to register the subclass
17
+ # with Dat::Analysis::Library.
18
+ #
19
+ # subclass - The Dat::Analysis::Matcher subclass being instantiated.
20
+ #
21
+ # Not intended to be called directly.
22
+ def self.inherited(subclass)
23
+ Dat::Analysis::Library.add subclass
24
+ end
25
+
26
+ # Internal: Add this class to a Dat::Analysis instance. Intended to be
27
+ # called from Dat::Analysis to dispatch registration.
28
+ #
29
+ # analyzer - a Dat::Analysis instance for an experiment
30
+ #
31
+ # Returns the analyzer's updated list of known matcher classes.
32
+ def self.add_to_analyzer(analyzer)
33
+ analyzer.add_matcher self
34
+ end
35
+
36
+ # Public: create a new Matcher.
37
+ #
38
+ # result - a science mismatch result, to be tested via `#match?`
39
+ def initialize(result)
40
+ @result = result
41
+ end
42
+ end
43
+ end
@@ -0,0 +1,50 @@
1
+ module Dat
2
+ # Internal: Registry of Dat::Analysis::Matcher and Dat::Analysis::Result
3
+ # classes. This is used to maintain the mapping of matchers and
4
+ # results wrappers for a particular Dat::Analysis instance.
5
+ class Analysis::Registry
6
+
7
+ # Public: Create a new Registry instance.
8
+ def initialize
9
+ @known_classes = []
10
+ end
11
+
12
+ # Public: Add a matcher or results wrapper class to the registry
13
+ #
14
+ # klass - a Dat::Analysis::Matcher subclass or a Dat::Analysis::Result
15
+ # subclass, to be added to the registry.
16
+ #
17
+ # Returns the list of currently registered classes.
18
+ def add(klass)
19
+ @known_classes << klass
20
+ end
21
+
22
+ # Public: Get the list of known Dat::Analysis::Matcher subclasses
23
+ #
24
+ # Returns the list of currently known matcher classes.
25
+ def matchers
26
+ @known_classes.select {|c| c <= ::Dat::Analysis::Matcher }
27
+ end
28
+
29
+ # Public: Get the list of known Dat::Analysis::Result subclasses
30
+ #
31
+ # Returns the list of currently known result wrapper classes.
32
+ def wrappers
33
+ @known_classes.select {|c| c <= ::Dat::Analysis::Result }
34
+ end
35
+
36
+ # Public: Get list of Dat::Analysis::Matcher subclasses for which
37
+ # `#match?` is truthy for the given result.
38
+ #
39
+ # result - a cooked science mismatch result
40
+ #
41
+ # Returns a list of matchers initialized with the provided result.
42
+ def identify(result)
43
+ matchers.inject([]) do |hits, matcher|
44
+ instance = matcher.new(result)
45
+ hits << instance if instance.match?
46
+ hits
47
+ end
48
+ end
49
+ end
50
+ end
@@ -0,0 +1,78 @@
1
+ require "time"
2
+
3
+ module Dat
4
+ # Public: Base class for wrappers around science mismatch results.
5
+ #
6
+ # Instance methods defined on subclasses will be added as instance methods
7
+ # on science mismatch results handled by Dat::Analysis instances which
8
+ # add the wrapper subclass via Dat::Analysis#add or Dat::Analysis#load_classes.
9
+ class Analysis::Result
10
+
11
+ # Public: return the current science mismatch result
12
+ attr_reader :result
13
+
14
+ # Internal: Called at subclass instantiation time to register the subclass
15
+ # with Dat::Analysis::Library.
16
+ #
17
+ # subclass - The Dat::Analysis::Result subclass being instantiated.
18
+ #
19
+ # Not intended to be called directly.
20
+ def self.inherited(subclass)
21
+ Dat::Analysis::Library.add subclass
22
+ end
23
+
24
+ # Internal: Add this class to a Dat::Analysis instance. Intended to be
25
+ # called from Dat::Analysis to dispatch registration.
26
+ #
27
+ # analyzer - a Dat::Analysis instance for an experiment
28
+ #
29
+ # Returns the analyzer's updated list of known result wrapper classes.
30
+ def self.add_to_analyzer(analyzer)
31
+ analyzer.add_wrapper self
32
+ end
33
+
34
+ # Public: create a new Result wrapper.
35
+ #
36
+ # result - a science mismatch result, to be wrapped with our instance methods.
37
+ def initialize(result)
38
+ @result = result
39
+ end
40
+ end
41
+
42
+ module Analysis::Result::DefaultMethods
43
+ # Public: Get the result data for the 'control' code path.
44
+ #
45
+ # Returns the 'control' field of the result hash.
46
+ def control
47
+ self['control']
48
+ end
49
+
50
+ # Public: Get the result data for the 'candidate' code path.
51
+ #
52
+ # Returns the 'candidate' field of the result hash.
53
+ def candidate
54
+ self['candidate']
55
+ end
56
+
57
+ # Public: Get the timestamp when the result was recorded.
58
+ #
59
+ # Returns a Time object for the timestamp for this result.
60
+ def timestamp
61
+ @timestamp ||= Time.parse(self['timestamp'])
62
+ end
63
+
64
+ # Public: Get which code path was run first.
65
+ #
66
+ # Returns the 'first' field of the result hash.
67
+ def first
68
+ self['first']
69
+ end
70
+
71
+ # Public: Get the experiment name
72
+ #
73
+ # Returns the 'experiment' field of the result hash.
74
+ def experiment_name
75
+ self['experiment']
76
+ end
77
+ end
78
+ end
@@ -0,0 +1,59 @@
1
+ module Dat
2
+ # Internal: Track and summarize counts of occurrences of mismatch objects.
3
+ #
4
+ # Examples
5
+ #
6
+ # tally = Dat::Analysis::Tally.new
7
+ # tally.count('foo')
8
+ # => 1
9
+ # tally.count('bar')
10
+ # => 1
11
+ # tally.count('foo')
12
+ # => 2
13
+ # puts tally.summary
14
+ # Summary of known mismatches found:
15
+ # foo 2
16
+ # bar 1
17
+ # TOTAL: 3
18
+ # => nil
19
+ #
20
+ class Analysis::Tally
21
+
22
+ # Public: Returns the hash of recorded mismatches.
23
+ attr_reader :tally
24
+
25
+ def initialize
26
+ @tally = {}
27
+ end
28
+
29
+ # Public: record an occurrence of a mismatch class.
30
+ def count(klass)
31
+ tally[klass] ||= 0
32
+ tally[klass] += 1
33
+ end
34
+
35
+ # Public: Return a String summary of mismatches seen so far.
36
+ #
37
+ # Returns a printable String summarizing the counts of mismatches seen,
38
+ # sorted in descending count order.
39
+ def summary
40
+ return "\nNo results identified.\n" if tally.keys.empty?
41
+ result = [ "\nSummary of identified results:\n" ]
42
+ sum = 0
43
+ tally.keys.sort_by {|k| -1*tally[k] }.each do |k|
44
+ sum += tally[k]
45
+ result << "%30s: %6d" % [k, tally[k]]
46
+ end
47
+ result << "%30s: %6d" % ['TOTAL', sum]
48
+ result.join "\n"
49
+ end
50
+
51
+ # Public: prints a summary of mismatches seen so far to STDOUT (see
52
+ # `#summary` above).
53
+ #
54
+ # Returns nil.
55
+ def summarize
56
+ puts summary
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,9 @@
1
+ #!/bin/sh
2
+ # Ensure local dependencies are available.
3
+
4
+ set -e
5
+
6
+ cd $(dirname "$0")/..
7
+ rm -rf .bundle/{binstubs,config}
8
+
9
+ bundle install --binstubs .bundle/binstubs --path .bundle --quiet "$@"
@@ -0,0 +1,38 @@
1
+ #!/bin/sh
2
+ # Tag and push a release.
3
+
4
+ set -e
5
+
6
+ # Make sure we're in the project root.
7
+
8
+ cd $(dirname "$0")/..
9
+
10
+ # Build a new gem archive.
11
+
12
+ rm -rf dat-analysis-*.gem
13
+ gem build -q dat-analysis.gemspec
14
+
15
+ # Make sure we're on the master branch.
16
+
17
+ (git branch | grep -q '* master') || {
18
+ echo "Only release from the master branch."
19
+ exit 1
20
+ }
21
+
22
+ # Figure out what version we're releasing.
23
+
24
+ tag=v`ls dat-analysis-*.gem | sed 's/^dat-analysis-\(.*\)\.gem$/\1/'`
25
+
26
+ # Make sure we haven't released this version before.
27
+
28
+ git fetch -t origin
29
+
30
+ (git tag -l | grep -q "$tag") && {
31
+ echo "Whoops, there's already a '${tag}' tag."
32
+ exit 1
33
+ }
34
+
35
+ # Tag it and bag it.
36
+
37
+ gem push dat-analysis-*.gem && git tag "$tag" &&
38
+ git push origin master && git push origin "$tag"
@@ -0,0 +1,9 @@
1
+ #!/bin/sh
2
+ # Run the unit tests.
3
+
4
+ set -e
5
+
6
+ cd $(dirname "$0")/..
7
+ script/bootstrap && ruby -I lib -r rubygems \
8
+ -e 'require "bundler/setup"' \
9
+ -e '(ARGV.empty? ? Dir["test/**/*_test.rb"] : ARGV).each { |f| load f }' -- "$@"
@@ -0,0 +1,119 @@
1
+ require "minitest/autorun"
2
+ require "mocha/setup"
3
+ require "dat/analysis"
4
+
5
+ # helper class to provide mismatch results
6
+ class TestCookedAnalyzer < Dat::Analysis
7
+ attr_accessor :mismatches
8
+
9
+ def initialize(experiment_name)
10
+ super
11
+ @mismatches = [] # use a simple array for a mismatch store
12
+ end
13
+
14
+ # load data files from our fixtures path
15
+ def path
16
+ File.expand_path('fixtures/', __FILE__)
17
+ end
18
+
19
+ def cook(raw_result)
20
+ return "cooked" unless raw_result
21
+ "cooked-#{raw_result}"
22
+ end
23
+
24
+ def count
25
+ mismatches.size
26
+ end
27
+
28
+ def read
29
+ mismatches.pop
30
+ end
31
+
32
+ # neuter formatter to take simple non-structured results
33
+ def readable
34
+ current.inspect
35
+ end
36
+
37
+ # neuter calls to `puts`, make it possible to test them.
38
+ def puts(*args)
39
+ @last_printed = args.join('')
40
+ nil
41
+ end
42
+ attr_reader :last_printed # for tests: last call to puts
43
+
44
+ # neuter calls to 'print' to eliminate test output clutter
45
+ def print(*args) end
46
+ end
47
+
48
+ class DatAnalysisSubclassingTest < MiniTest::Unit::TestCase
49
+
50
+ def setup
51
+ @experiment_name = 'test-suite-experiment'
52
+ @analyzer = ::TestCookedAnalyzer.new @experiment_name
53
+ end
54
+
55
+ def test_is_0_when_count_is_overridden_and_there_are_no_mismatches
56
+ assert_equal 0, @analyzer.count
57
+ end
58
+
59
+ def test_returns_the_count_of_mismatches_when_count_is_overridden
60
+ @analyzer.mismatches.push 'mismatch'
61
+ @analyzer.mismatches.push 'mismatch'
62
+ assert_equal 2, @analyzer.count
63
+ end
64
+
65
+ def test_fetch_returns_nil_when_read_is_overridden_and_read_returns_no_mismatches
66
+ assert_nil @analyzer.fetch
67
+ end
68
+
69
+ def test_fetch_returns_the_cooked_version_of_the_next_mismatch_from_read_when_read_is_overridden
70
+ @analyzer.mismatches.push 'mismatch'
71
+ assert_equal 'cooked-mismatch', @analyzer.fetch
72
+ end
73
+
74
+ def test_raw_returns_nil_when_no_mismatches_have_been_fetched_and_cook_is_overridden
75
+ assert_nil @analyzer.raw
76
+ end
77
+
78
+ def test_current_returns_nil_when_no_mismatches_have_been_fetch_and_cook_is_overridden
79
+ assert_nil @analyzer.current
80
+ end
81
+
82
+ def test_raw_returns_nil_when_last_fetched_returns_no_results_and_cook_is_overridden
83
+ @analyzer.fetch
84
+ assert_nil @analyzer.raw
85
+ end
86
+
87
+ def test_current_returns_nil_when_last_fetched_returns_no_results_and_cook_is_overridden
88
+ @analyzer.fetch
89
+ assert_nil @analyzer.current
90
+ end
91
+
92
+ def test_raw_returns_unprocess_mismatch_when_cook_is_overridden
93
+ @analyzer.mismatches.push 'mismatch-1'
94
+ result = @analyzer.fetch
95
+ assert_equal 'mismatch-1', @analyzer.raw
96
+ end
97
+
98
+ def test_current_returns_a_cooked_mismatch_when_cook_is_overridden
99
+ @analyzer.mismatches.push 'mismatch-1'
100
+ result = @analyzer.fetch
101
+ assert_equal 'cooked-mismatch-1', @analyzer.current
102
+ end
103
+
104
+ def test_raw_updates_with_later_fetches_when_cook_is_overridden
105
+ @analyzer.mismatches.push 'mismatch-1'
106
+ @analyzer.mismatches.push 'mismatch-2'
107
+ @analyzer.fetch # discard the first one
108
+ @analyzer.fetch
109
+ assert_equal 'mismatch-1', @analyzer.raw
110
+ end
111
+
112
+ def test_current_updates_with_later_fetches_when_cook_is_overridden
113
+ @analyzer.mismatches.push 'mismatch-1'
114
+ @analyzer.mismatches.push 'mismatch-2'
115
+ @analyzer.fetch # discard the first one
116
+ @analyzer.fetch
117
+ assert_equal 'cooked-mismatch-1', @analyzer.current
118
+ end
119
+ end
@@ -0,0 +1,822 @@
1
+ require "minitest/autorun"
2
+ require "mocha/setup"
3
+ require "dat/analysis"
4
+ require "time"
5
+
6
+ # helper class to provide mismatch results
7
+ class TestMismatchAnalysis < Dat::Analysis
8
+ attr_accessor :mismatches
9
+
10
+ def initialize(experiment_name)
11
+ super
12
+ @mismatches = [] # use a simple array for a mismatch store
13
+ end
14
+
15
+ # load data files from our fixtures path
16
+ def path
17
+ File.expand_path(File.join(File.dirname(__FILE__), 'fixtures'))
18
+ end
19
+
20
+ def count
21
+ mismatches.size
22
+ end
23
+
24
+ def read
25
+ mismatches.pop
26
+ end
27
+
28
+ # neuter formatter to take simple non-structured results
29
+ def readable
30
+ current.inspect
31
+ end
32
+
33
+ # neuter calls to `puts`, make it possible to test them.
34
+ def puts(*args)
35
+ @last_printed = args.join('')
36
+ nil
37
+ end
38
+ attr_reader :last_printed # for tests: last call to puts
39
+
40
+ # neuter calls to 'print' to eliminate test output clutter
41
+ def print(*args) end
42
+ end
43
+
44
+ # for testing that a non-registered Recognizer class can still
45
+ # supply a default `#readable` method to subclasses
46
+ class TestSubclassRecognizer < Dat::Analysis::Matcher
47
+ def readable
48
+ "experiment-formatter: #{result['extra']}"
49
+ end
50
+ end
51
+
52
+ class DatAnalysisTest < MiniTest::Unit::TestCase
53
+ def setup
54
+ Dat::Analysis::Tally.any_instance.stubs(:puts)
55
+ @experiment_name = 'test-suite-experiment'
56
+ @analyzer = TestMismatchAnalysis.new @experiment_name
57
+
58
+ @timestamp = Time.now
59
+ @result = {
60
+ 'experiment' => @experiment_name,
61
+ 'control' => {
62
+ 'duration' => 0.03,
63
+ 'exception' => nil,
64
+ 'value' => true,
65
+ },
66
+ 'candidate' => {
67
+ 'duration' => 1.03,
68
+ 'exception' => nil,
69
+ 'value' => false,
70
+ },
71
+ 'first' => 'candidate',
72
+ 'extra' => 'bacon',
73
+ 'timestamp' => @timestamp.to_s
74
+ }
75
+ end
76
+
77
+ def test_preserves_the_experiment_name
78
+ assert_equal @experiment_name, @analyzer.experiment_name
79
+ end
80
+
81
+ def test_analyze_returns_nil_if_there_is_no_current_result_and_no_additional_results
82
+ assert_nil @analyzer.analyze
83
+ end
84
+
85
+ def test_analyze_leaves_tallies_empty_if_there_is_no_current_result_and_no_additional_results
86
+ @analyzer.analyze
87
+ assert_equal({}, @analyzer.tally.tally)
88
+ end
89
+
90
+ def test_analyze_returns_nil_if_there_is_a_current_result_but_no_additional_results
91
+ @analyzer.mismatches.push @result
92
+ @analyzer.fetch
93
+ assert @analyzer.current
94
+ assert_nil @analyzer.analyze
95
+ end
96
+
97
+ def test_analyze_leaves_tallies_empty_if_there_is_a_current_result_but_no_additional_results
98
+ @analyzer.mismatches.push @result
99
+ @analyzer.fetch
100
+ assert @analyzer.current
101
+ @analyzer.analyze
102
+ assert_equal({}, @analyzer.tally.tally)
103
+ end
104
+
105
+ def test_analyze_outputs_default_result_summary_and_tally_summary_when_one_unrecognized_result_is_present
106
+ @analyzer.expects(:summarize_unknown_result)
107
+ @analyzer.mismatches.push @result
108
+ @analyzer.analyze
109
+ end
110
+
111
+ def test_analyze_returns_nil_when_one_unrecognized_result_is_present
112
+ @analyzer.mismatches.push @result
113
+ assert_nil @analyzer.analyze
114
+ end
115
+
116
+ def test_analyze_leaves_current_result_set_to_first_result_when_one_unrecognized_result_is_present
117
+ @analyzer.mismatches.push @result
118
+ @analyzer.analyze
119
+ assert_equal @result, @analyzer.current
120
+ end
121
+
122
+ def test_analyze_leaves_tallies_empty_when_one_unrecognized_result_is_present
123
+ @analyzer.mismatches.push @result
124
+ @analyzer.analyze
125
+ assert_equal({}, @analyzer.tally.tally)
126
+ end
127
+
128
+ def test_analyze_outputs_default_results_summary_for_first_unrecognized_result_and_tally_summary_when_recognized_and_unrecognized_results_are_present
129
+ matcher = Class.new(Dat::Analysis::Matcher) do
130
+ def match?
131
+ result['extra'] =~ /^known-/
132
+ end
133
+ end
134
+
135
+ @analyzer.add matcher
136
+ @analyzer.mismatches.push @result.merge('extra' => 'known-1')
137
+ @analyzer.mismatches.push @result.merge('extra' => 'unknown-1')
138
+ @analyzer.mismatches.push @result.merge('extra' => 'known-2')
139
+
140
+ @analyzer.expects(:summarize_unknown_result)
141
+ @analyzer.analyze
142
+ end
143
+
144
+ def test_analyze_returns_number_of_unanalyzed_results_when_recognized_and_unrecognized_results_are_present
145
+ matcher = Class.new(Dat::Analysis::Matcher) do
146
+ def match?
147
+ result['extra'] =~ /^known-/
148
+ end
149
+ end
150
+
151
+ @analyzer.add matcher
152
+ @analyzer.mismatches.push @result.merge('extra' => 'known-1')
153
+ @analyzer.mismatches.push @result.merge('extra' => 'unknown-1')
154
+ @analyzer.mismatches.push @result.merge('extra' => 'known-2')
155
+
156
+ assert_equal 1, @analyzer.analyze
157
+ end
158
+
159
+ def test_analyze_leaves_current_result_set_to_first_unrecognized_result_when_recognized_and_unrecognized_results_are_present
160
+ matcher = Class.new(Dat::Analysis::Matcher) do
161
+ def match?
162
+ result['extra'] =~ /^known/
163
+ end
164
+ end
165
+
166
+ @analyzer.add matcher
167
+ @analyzer.mismatches.push @result.merge('extra' => 'known-1')
168
+ @analyzer.mismatches.push @result.merge('extra' => 'unknown-1')
169
+ @analyzer.mismatches.push @result.merge('extra' => 'known-2')
170
+
171
+ @analyzer.analyze
172
+ assert_equal 'unknown-1', @analyzer.current['extra']
173
+ end
174
+
175
+ def test_analyze_leaves_recognized_result_counts_in_tally_when_recognized_and_unrecognized_results_are_present
176
+ matcher1 = Class.new(Dat::Analysis::Matcher) do
177
+ def self.name() "RecognizerOne" end
178
+ def match?
179
+ result['extra'] =~ /^known-1/
180
+ end
181
+ end
182
+
183
+ matcher2 = Class.new(Dat::Analysis::Matcher) do
184
+ def self.name() "RecognizerTwo" end
185
+ def match?
186
+ result['extra'] =~ /^known-2/
187
+ end
188
+ end
189
+
190
+ @analyzer.add matcher1
191
+ @analyzer.add matcher2
192
+
193
+ @analyzer.mismatches.push @result.merge('extra' => 'known-1-last')
194
+ @analyzer.mismatches.push @result.merge('extra' => 'unknown-1')
195
+ @analyzer.mismatches.push @result.merge('extra' => 'known-10')
196
+ @analyzer.mismatches.push @result.merge('extra' => 'known-20')
197
+ @analyzer.mismatches.push @result.merge('extra' => 'known-11')
198
+ @analyzer.mismatches.push @result.merge('extra' => 'known-21')
199
+ @analyzer.mismatches.push @result.merge('extra' => 'known-12')
200
+
201
+ @analyzer.analyze
202
+
203
+ tally = @analyzer.tally.tally
204
+ assert_equal [ 'RecognizerOne', 'RecognizerTwo' ], tally.keys.sort
205
+ assert_equal 3, tally['RecognizerOne']
206
+ assert_equal 2, tally['RecognizerTwo']
207
+ end
208
+
209
+ def test_analyze_proceeds_from_stop_point_when_analyzing_with_more_results
210
+ matcher = Class.new(Dat::Analysis::Matcher) do
211
+ def match?
212
+ result['extra'] =~ /^known-/
213
+ end
214
+ end
215
+
216
+ @analyzer.add matcher
217
+ @analyzer.mismatches.push @result.merge('extra' => 'known-1')
218
+ @analyzer.mismatches.push @result.merge('extra' => 'unknown-1')
219
+ @analyzer.mismatches.push @result.merge('extra' => 'known-2')
220
+ @analyzer.mismatches.push @result.merge('extra' => 'unknown-2')
221
+ @analyzer.mismatches.push @result.merge('extra' => 'known-3')
222
+
223
+ assert_equal 3, @analyzer.analyze
224
+ assert_equal 'unknown-2', @analyzer.current['extra']
225
+ assert_equal 1, @analyzer.analyze
226
+ assert_equal 'unknown-1', @analyzer.current['extra']
227
+ assert_equal @analyzer.readable, @analyzer.last_printed
228
+ end
229
+
230
+ def test_analyze_resets_tally_between_runs_when_analyzing_later_results_after_a_stop
231
+ matcher1 = Class.new(Dat::Analysis::Matcher) do
232
+ def self.name() "RecognizerOne" end
233
+ def match?
234
+ result['extra'] =~ /^known-1/
235
+ end
236
+ end
237
+
238
+ matcher2 = Class.new(Dat::Analysis::Matcher) do
239
+ def self.name() "RecognizerTwo" end
240
+ def match?
241
+ result['extra'] =~ /^known-2/
242
+ end
243
+ end
244
+
245
+ @analyzer.add matcher1
246
+ @analyzer.add matcher2
247
+
248
+ @analyzer.mismatches.push @result.merge('extra' => 'known-1-last')
249
+ @analyzer.mismatches.push @result.merge('extra' => 'unknown-1')
250
+ @analyzer.mismatches.push @result.merge('extra' => 'known-10')
251
+ @analyzer.mismatches.push @result.merge('extra' => 'known-20')
252
+ @analyzer.mismatches.push @result.merge('extra' => 'known-11')
253
+ @analyzer.mismatches.push @result.merge('extra' => 'known-21')
254
+ @analyzer.mismatches.push @result.merge('extra' => 'known-12')
255
+
256
+ @analyzer.analyze # proceed to first stop point
257
+ @analyzer.analyze # and continue analysis
258
+
259
+ assert_equal({'RecognizerOne' => 1}, @analyzer.tally.tally)
260
+ end
261
+
262
+ def test_skip_fails_if_no_block_is_provided
263
+ assert_raises(ArgumentError) do
264
+ @analyzer.skip
265
+ end
266
+ end
267
+
268
+ def test_skip_returns_nil_if_there_is_no_current_result
269
+ remaining = @analyzer.skip do |result|
270
+ true
271
+ end
272
+
273
+ assert_nil remaining
274
+ end
275
+
276
+ def test_skip_leaves_current_alone_if_the_current_result_satisfies_the_block
277
+ @analyzer.mismatches.push @result
278
+
279
+ @analyzer.skip do |result|
280
+ true
281
+ end
282
+ end
283
+
284
+ def test_skip_returns_0_if_the_current_result_satisfies_the_block_and_no_other_results_are_available
285
+ @analyzer.mismatches.push @result
286
+
287
+ remaining = @analyzer.skip do |result|
288
+ true
289
+ end
290
+
291
+ assert_equal 0, remaining
292
+ end
293
+
294
+ def test_skip_returns_the_number_of_additional_results_if_the_current_result_satisfies_the_block_and_other_results_are_available
295
+ @analyzer.mismatches.push @result
296
+ @analyzer.mismatches.push @result
297
+ @analyzer.mismatches.push @result
298
+
299
+ remaining = @analyzer.skip do |result|
300
+ true
301
+ end
302
+
303
+ assert_equal 2, remaining
304
+ end
305
+
306
+ def test_skip_returns_nil_if_no_results_are_satisfying
307
+ @analyzer.mismatches.push @result
308
+ @analyzer.mismatches.push @result
309
+ @analyzer.mismatches.push @result
310
+
311
+ remaining = @analyzer.skip do |result|
312
+ false
313
+ end
314
+
315
+ assert_nil remaining
316
+ end
317
+
318
+ def test_skip_skips_all_results_if_no_results_are_satisfying
319
+ @analyzer.mismatches.push @result
320
+ @analyzer.mismatches.push @result
321
+ @analyzer.mismatches.push @result
322
+
323
+ remaining = @analyzer.skip do |result|
324
+ false
325
+ end
326
+
327
+ assert !@analyzer.more?
328
+ end
329
+
330
+ def test_skip_leaves_current_as_nil_if_no_results_are_satisfying
331
+ @analyzer.mismatches.push @result
332
+ @analyzer.mismatches.push @result
333
+ @analyzer.mismatches.push @result
334
+
335
+ remaining = @analyzer.skip do |result|
336
+ false
337
+ end
338
+
339
+ assert_nil @analyzer.current
340
+ end
341
+
342
+ def test_more_is_false_when_there_are_no_mismatches
343
+ assert !@analyzer.more?
344
+ end
345
+
346
+ def test_more_is_true_when_there_are_mismatches
347
+ @analyzer.mismatches.push @result
348
+ assert @analyzer.more?
349
+ end
350
+
351
+ def test_count_fails
352
+ assert_raises(NoMethodError) do
353
+ Dat::Analysis.new(@experiment_name).count
354
+ end
355
+ end
356
+
357
+ def test_fetch_fails_unless_read_is_implemented_by_a_subclass
358
+ assert_raises(NameError) do
359
+ Dat::Analysis.new(@experiment_name).fetch
360
+ end
361
+ end
362
+
363
+ def test_current_returns_nil_when_no_mismmatches_have_been_fetched
364
+ assert_nil @analyzer.current
365
+ end
366
+
367
+ def test_current_returns_nil_when_last_fetch_returned_no_results
368
+ @analyzer.fetch
369
+ assert_nil @analyzer.current
370
+ end
371
+
372
+ def test_current_returns_the_most_recent_mismatch_when_one_has_been_fetched
373
+ @analyzer.mismatches.push @result
374
+ @analyzer.fetch
375
+ assert_equal @result, @analyzer.current
376
+ end
377
+
378
+ def test_current_updates_with_later_fetches
379
+ @analyzer.mismatches.push @result
380
+ @analyzer.mismatches.push @result
381
+ @analyzer.fetch
382
+ result = @analyzer.fetch
383
+ assert_equal result, @analyzer.current
384
+ end
385
+
386
+ def test_result_is_an_alias_for_current
387
+ @analyzer.mismatches.push @result
388
+ @analyzer.mismatches.push @result
389
+ @analyzer.fetch
390
+ result = @analyzer.fetch
391
+ assert_equal result, @analyzer.result
392
+ end
393
+
394
+ def test_raw_returns_nil_when_no_mismatches_have_been_fetched
395
+ assert_nil @analyzer.raw
396
+ end
397
+
398
+ def test_raw_returns_nil_when_last_fetched_returned_no_results
399
+ @analyzer.fetch
400
+ assert_nil @analyzer.raw
401
+ end
402
+
403
+ def test_raw_returns_an_unprocessed_version_of_the_most_recent_mismatch
404
+ @analyzer.mismatches.push @result
405
+ result = @analyzer.fetch
406
+ assert_equal @result, @analyzer.raw
407
+ end
408
+
409
+ def test_raw_updates_with_later_fetches
410
+ @analyzer.mismatches.push 'mismatch-1'
411
+ @analyzer.mismatches.push 'mismatch-2'
412
+ @analyzer.fetch # discard the first one
413
+ @analyzer.fetch
414
+ assert_equal 'mismatch-1', @analyzer.raw
415
+ end
416
+
417
+ def test_when_loading_support_classes_loads_no_matchers_if_no_matcher_files_exist_on_load_path
418
+ analyzer = TestMismatchAnalysis.new('experiment-with-no-classes')
419
+ analyzer.load_classes
420
+ assert_equal [], analyzer.matchers
421
+ assert_equal [], analyzer.wrappers
422
+ end
423
+
424
+ def test_when_loading_support_classes_loads_matchers_and_wrappers_if_they_exist_on_load_path
425
+ analyzer = TestMismatchAnalysis.new('experiment-with-classes')
426
+ analyzer.load_classes
427
+ assert_equal ["MatcherA", "MatcherB", "MatcherC"], analyzer.matchers.map(&:name)
428
+ assert_equal ["WrapperA", "WrapperB", "WrapperC"], analyzer.wrappers.map(&:name)
429
+ end
430
+
431
+ def test_when_loading_support_classes_ignores_extraneous_classes_on_load_path
432
+ analyzer = TestMismatchAnalysis.new('experiment-with-good-and-extraneous-classes')
433
+ analyzer.load_classes
434
+ assert_equal ["MatcherX", "MatcherY", "MatcherZ"], analyzer.matchers.map(&:name)
435
+ assert_equal ["WrapperX", "WrapperY", "WrapperZ"], analyzer.wrappers.map(&:name)
436
+ end
437
+
438
+ def test_when_loading_support_classes_loads_classes_at_initialization_time_if_they_are_available
439
+ analyzer = TestMismatchAnalysis.new('initialize-classes')
440
+ assert_equal ["MatcherM", "MatcherN"], analyzer.matchers.map(&:name)
441
+ assert_equal ["WrapperM", "WrapperN"], analyzer.wrappers.map(&:name)
442
+ end
443
+
444
+ def test_when_loading_support_classes_does_not_load_classes_at_initialization_time_if_they_cannot_be_loaded
445
+ analyzer = TestMismatchAnalysis.new('invalid-matcher')
446
+ assert_equal [], analyzer.matchers
447
+ end
448
+
449
+ def test_loading_classes_post_initialization_fails_if_loading_has_errors
450
+ # fails at #load_classes time since we define #path later
451
+ analyzer = Dat::Analysis.new('invalid-matcher')
452
+ analyzer.path = File.expand_path(File.join(File.dirname(__FILE__), 'fixtures'))
453
+
454
+ assert_raises(Errno::EACCES) do
455
+ analyzer.load_classes
456
+ end
457
+ end
458
+
459
+ def test_result_has_an_useful_timestamp
460
+ @analyzer.mismatches.push(@result)
461
+ result = @analyzer.fetch
462
+ assert_equal @timestamp.to_i, result.timestamp.to_i
463
+ end
464
+
465
+ def test_result_has_a_method_for_first
466
+ @analyzer.mismatches.push(@result)
467
+ result = @analyzer.fetch
468
+ assert_equal @result['first'], result.first
469
+ end
470
+
471
+ def test_result_has_a_method_for_control
472
+ @analyzer.mismatches.push(@result)
473
+ result = @analyzer.fetch
474
+ assert_equal @result['control'], result.control
475
+ end
476
+
477
+ def test_result_has_a_method_for_candidate
478
+ @analyzer.mismatches.push(@result)
479
+ result = @analyzer.fetch
480
+ assert_equal @result['candidate'], result.candidate
481
+ end
482
+
483
+ def test_result_has_a_method_for_experiment_name
484
+ @analyzer.mismatches.push(@result)
485
+ result = @analyzer.fetch
486
+ assert_equal @result['experiment'], result.experiment_name
487
+ end
488
+
489
+ def test_results_helper_methods_are_not_available_on_results_unless_loaded
490
+ @analyzer.mismatches.push @result
491
+ result = @analyzer.fetch
492
+
493
+ assert_raises(NoMethodError) do
494
+ result.repository
495
+ end
496
+ end
497
+
498
+ def test_results_helper_methods_are_made_available_on_returned_results
499
+ wrapper = Class.new(Dat::Analysis::Result) do
500
+ def repository
501
+ 'github/dat-science'
502
+ end
503
+ end
504
+
505
+ @analyzer.add wrapper
506
+ @analyzer.mismatches.push @result
507
+ result = @analyzer.fetch
508
+ assert_equal 'github/dat-science', result.repository
509
+ end
510
+
511
+ def test_results_helper_methods_can_be_loaded_from_multiple_classes
512
+ wrapper1 = Class.new(Dat::Analysis::Result) do
513
+ def repository
514
+ 'github/dat-science'
515
+ end
516
+ end
517
+
518
+ wrapper2 = Class.new(Dat::Analysis::Result) do
519
+ def user
520
+ :rick
521
+ end
522
+ end
523
+
524
+ @analyzer.add wrapper1
525
+ @analyzer.add wrapper2
526
+ @analyzer.mismatches.push @result
527
+ result = @analyzer.fetch
528
+ assert_equal 'github/dat-science', result.repository
529
+ assert_equal :rick, result.user
530
+ end
531
+
532
+ def test_results_helper_methods_are_made_available_in_the_order_loaded
533
+ wrapper1 = Class.new(Dat::Analysis::Result) do
534
+ def repository
535
+ 'github/dat-science'
536
+ end
537
+ end
538
+
539
+ wrapper2 = Class.new(Dat::Analysis::Result) do
540
+ def repository
541
+ 'github/linguist'
542
+ end
543
+
544
+ def user
545
+ :rick
546
+ end
547
+ end
548
+
549
+ @analyzer.add wrapper1
550
+ @analyzer.add wrapper2
551
+ @analyzer.mismatches.push @result
552
+ result = @analyzer.fetch
553
+ assert_equal 'github/dat-science', result.repository
554
+ assert_equal :rick, result.user
555
+ end
556
+
557
+ def test_results_helper_methods_do_not_hide_existing_result_methods
558
+ wrapper = Class.new(Dat::Analysis::Result) do
559
+ def size
560
+ 'huge'
561
+ end
562
+ end
563
+
564
+ @analyzer.add wrapper
565
+ @analyzer.mismatches.push 'mismatch-1'
566
+ result = @analyzer.fetch
567
+ assert_equal 10, result.size
568
+ end
569
+
570
+ def test_methods_can_access_the_result_using_the_result_method
571
+ wrapper = Class.new(Dat::Analysis::Result) do
572
+ def esrever
573
+ result.reverse
574
+ end
575
+ end
576
+
577
+ @analyzer.add wrapper
578
+ @analyzer.mismatches.push 'mismatch-1'
579
+ result = @analyzer.fetch
580
+ assert_equal 'mismatch-1'.reverse, result.esrever
581
+ end
582
+
583
+ def test_summarize_returns_nil_and_prints_the_empty_string_if_no_result_is_current
584
+ assert_nil @analyzer.summarize
585
+ assert_equal "", @analyzer.last_printed
586
+ end
587
+
588
+ def test_summarize_returns_nil_and_prints_the_default_readable_result_if_a_result_is_current_but_no_matchers_are_known
589
+ @analyzer.mismatches.push @result
590
+ @analyzer.fetch
591
+ assert_nil @analyzer.summarize
592
+ assert_equal @analyzer.readable, @analyzer.last_printed
593
+ end
594
+
595
+ def test_summarize_returns_nil_and_prints_the_default_readable_result_if_a_result_is_current_but_not_matched_by_any_known_matchers
596
+ matcher = Class.new(Dat::Analysis::Matcher) do
597
+ def match?
598
+ false
599
+ end
600
+
601
+ def readable
602
+ 'this should never run'
603
+ end
604
+ end
605
+
606
+ @analyzer.add matcher
607
+ @analyzer.mismatches.push @result
608
+ @analyzer.fetch
609
+ assert_nil @analyzer.summarize
610
+ assert_equal @analyzer.readable, @analyzer.last_printed
611
+ end
612
+
613
+ def test_summarize_returns_nil_and_prints_the_matchers_readable_result_when_a_result_is_current_and_matched_by_a_matcher
614
+ matcher = Class.new(Dat::Analysis::Matcher) do
615
+ def match?
616
+ true
617
+ end
618
+
619
+ def readable
620
+ "recognized: #{result['extra']}"
621
+ end
622
+ end
623
+
624
+ @analyzer.add matcher
625
+ @analyzer.mismatches.push @result.merge('extra' => 'mismatch-1')
626
+ @analyzer.fetch
627
+ assert_nil @analyzer.summarize
628
+ assert_equal "recognized: mismatch-1", @analyzer.last_printed
629
+ end
630
+
631
+ def test_summarize_returns_nil_and_prints_the_default_readable_result_when_a_result_is_matched_by_a_matcher_with_no_formatter
632
+ matcher = Class.new(Dat::Analysis::Matcher) do
633
+ def match?
634
+ true
635
+ end
636
+ end
637
+
638
+ @analyzer.add matcher
639
+ @analyzer.mismatches.push @result
640
+ @analyzer.fetch
641
+ assert_nil @analyzer.summarize
642
+ assert_equal @analyzer.readable, @analyzer.last_printed
643
+ end
644
+
645
+ def test_summarize_supports_use_of_a_matcher_base_class_for_shared_formatting
646
+ matcher = Class.new(TestSubclassRecognizer) do
647
+ def match?
648
+ true
649
+ end
650
+ end
651
+
652
+ @analyzer.add matcher
653
+ @analyzer.mismatches.push @result.merge('extra' => 'mismatch-1')
654
+ @analyzer.fetch
655
+ assert_nil @analyzer.summarize
656
+ assert_equal "experiment-formatter: mismatch-1", @analyzer.last_printed
657
+ end
658
+
659
+ def test_summary_returns_nil_if_no_result_is_current
660
+ assert_nil @analyzer.summary
661
+ end
662
+
663
+ def test_summary_returns_the_default_readable_result_if_a_result_is_current_but_no_matchers_are_known
664
+ @analyzer.mismatches.push @result
665
+ @analyzer.fetch
666
+ assert_equal @analyzer.readable, @analyzer.summary
667
+ end
668
+
669
+ def test_summary_returns_the_default_readable_result_if_a_result_is_current_but_not_matched_by_any_known_matchers
670
+ matcher = Class.new(Dat::Analysis::Matcher) do
671
+ def match?
672
+ false
673
+ end
674
+
675
+ def readable
676
+ 'this should never run'
677
+ end
678
+ end
679
+
680
+ @analyzer.add matcher
681
+ @analyzer.mismatches.push @result
682
+ @analyzer.fetch
683
+ assert_equal @analyzer.readable, @analyzer.summary
684
+ end
685
+
686
+ def test_summary_returns_the_matchers_readable_result_when_a_result_is_current_and_matched_by_a_matcher
687
+ matcher = Class.new(Dat::Analysis::Matcher) do
688
+ def match?
689
+ true
690
+ end
691
+
692
+ def readable
693
+ "recognized: #{result['extra']}"
694
+ end
695
+ end
696
+
697
+ @analyzer.add matcher
698
+ @analyzer.mismatches.push @result.merge('extra' => 'mismatch-1')
699
+ @analyzer.fetch
700
+ assert_equal "recognized: mismatch-1", @analyzer.summary
701
+ end
702
+
703
+ def test_summary_formats_with_the_default_formatter_if_a_matching_matcher_does_not_define_a_formatter
704
+ matcher = Class.new(Dat::Analysis::Matcher) do
705
+ def match?
706
+ true
707
+ end
708
+ end
709
+
710
+ @analyzer.add matcher
711
+ @analyzer.mismatches.push @result
712
+ @analyzer.fetch
713
+ assert_equal @analyzer.readable, @analyzer.summary
714
+ end
715
+
716
+ def test_summary_supports_use_of_a_matcher_base_class_for_shared_formatting
717
+ matcher = Class.new(TestSubclassRecognizer) do
718
+ def match?
719
+ true
720
+ end
721
+ end
722
+
723
+ @analyzer.add matcher
724
+ @analyzer.mismatches.push @result.merge('extra' => 'mismatch-1')
725
+ @analyzer.fetch
726
+ assert_equal "experiment-formatter: mismatch-1", @analyzer.summary
727
+ end
728
+
729
+ def test_unknown_returns_nil_if_no_result_is_current
730
+ assert_nil @analyzer.unknown?
731
+ end
732
+
733
+ def test_unknown_returns_true_if_a_result_is_current_but_no_matchers_are_known
734
+ @analyzer.mismatches.push @result
735
+ @analyzer.fetch
736
+ assert_equal true, @analyzer.unknown?
737
+ end
738
+
739
+ def test_unknown_returns_true_if_current_result_is_not_matched_by_any_known_matchers
740
+ matcher = Class.new(Dat::Analysis::Matcher) do
741
+ def match?
742
+ false
743
+ end
744
+ end
745
+
746
+ @analyzer.add matcher
747
+ @analyzer.mismatches.push @result
748
+ @analyzer.fetch
749
+ assert_equal true, @analyzer.unknown?
750
+ end
751
+
752
+ def test_unknown_returns_false_if_a_matcher_class_matches_the_current_result
753
+ matcher = Class.new(Dat::Analysis::Matcher) do
754
+ def match?
755
+ true
756
+ end
757
+ end
758
+
759
+ @analyzer.add matcher
760
+ @analyzer.mismatches.push @result
761
+ @analyzer.fetch
762
+ assert_equal false, @analyzer.unknown?
763
+ end
764
+
765
+ def test_identify_returns_nil_if_no_result_is_current
766
+ assert_nil @analyzer.identify
767
+ end
768
+
769
+ def test_identify_returns_nil_if_a_result_is_current_but_no_matchers_are_known
770
+ @analyzer.mismatches.push @result
771
+ @analyzer.fetch
772
+ assert_nil @analyzer.identify
773
+ end
774
+
775
+ def test_identify_returns_nil_if_current_result_is_not_matched_by_any_known_matchers
776
+ matcher = Class.new(Dat::Analysis::Matcher) do
777
+ def match?
778
+ false
779
+ end
780
+ end
781
+
782
+ @analyzer.add matcher
783
+ @analyzer.mismatches.push @result
784
+ @analyzer.fetch
785
+ assert_nil @analyzer.identify
786
+ end
787
+
788
+ def test_identify_returns_the_matcher_class_which_matches_the_current_result
789
+ matcher = Class.new(Dat::Analysis::Matcher) do
790
+ def match?
791
+ true
792
+ end
793
+ end
794
+
795
+ @analyzer.add matcher
796
+ @analyzer.mismatches.push @result
797
+ @analyzer.fetch
798
+ assert_equal matcher, @analyzer.identify.class
799
+ end
800
+
801
+ def test_identify_fails_if_more_than_one_matcher_class_matches_the_current_result
802
+ matcher1 = Class.new(Dat::Analysis::Matcher) do
803
+ def match?
804
+ true
805
+ end
806
+ end
807
+
808
+ matcher2 = Class.new(Dat::Analysis::Matcher) do
809
+ def match?
810
+ true
811
+ end
812
+ end
813
+
814
+ @analyzer.add matcher1
815
+ @analyzer.add matcher2
816
+ @analyzer.mismatches.push @result
817
+ @analyzer.fetch
818
+ assert_raises(RuntimeError) do
819
+ @analyzer.identify
820
+ end
821
+ end
822
+ end