lab_tech 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/MIT-LICENSE +20 -0
  3. data/README.md +323 -0
  4. data/Rakefile +30 -0
  5. data/app/models/lab_tech/application_record.rb +5 -0
  6. data/app/models/lab_tech/default_cleaner.rb +87 -0
  7. data/app/models/lab_tech/experiment.rb +190 -0
  8. data/app/models/lab_tech/observation.rb +40 -0
  9. data/app/models/lab_tech/percentile.rb +41 -0
  10. data/app/models/lab_tech/result.rb +130 -0
  11. data/app/models/lab_tech/speedup.rb +65 -0
  12. data/app/models/lab_tech/summary.rb +183 -0
  13. data/config/routes.rb +2 -0
  14. data/db/migrate/20190815192130_create_experiment_tables.rb +50 -0
  15. data/lib/lab_tech.rb +176 -0
  16. data/lib/lab_tech/engine.rb +6 -0
  17. data/lib/lab_tech/version.rb +3 -0
  18. data/lib/tasks/lab_tech_tasks.rake +4 -0
  19. data/spec/dummy/Rakefile +6 -0
  20. data/spec/dummy/app/assets/config/manifest.js +1 -0
  21. data/spec/dummy/app/assets/javascripts/application.js +14 -0
  22. data/spec/dummy/app/assets/stylesheets/application.css +15 -0
  23. data/spec/dummy/app/controllers/application_controller.rb +2 -0
  24. data/spec/dummy/app/jobs/application_job.rb +2 -0
  25. data/spec/dummy/app/models/application_record.rb +3 -0
  26. data/spec/dummy/bin/bundle +3 -0
  27. data/spec/dummy/bin/rails +4 -0
  28. data/spec/dummy/bin/rake +4 -0
  29. data/spec/dummy/bin/setup +33 -0
  30. data/spec/dummy/bin/update +28 -0
  31. data/spec/dummy/config.ru +5 -0
  32. data/spec/dummy/config/application.rb +35 -0
  33. data/spec/dummy/config/boot.rb +5 -0
  34. data/spec/dummy/config/database.yml +25 -0
  35. data/spec/dummy/config/environment.rb +5 -0
  36. data/spec/dummy/config/environments/development.rb +46 -0
  37. data/spec/dummy/config/environments/production.rb +71 -0
  38. data/spec/dummy/config/environments/test.rb +36 -0
  39. data/spec/dummy/config/initializers/application_controller_renderer.rb +8 -0
  40. data/spec/dummy/config/initializers/backtrace_silencers.rb +7 -0
  41. data/spec/dummy/config/initializers/cors.rb +16 -0
  42. data/spec/dummy/config/initializers/filter_parameter_logging.rb +4 -0
  43. data/spec/dummy/config/initializers/inflections.rb +16 -0
  44. data/spec/dummy/config/initializers/mime_types.rb +4 -0
  45. data/spec/dummy/config/initializers/wrap_parameters.rb +14 -0
  46. data/spec/dummy/config/locales/en.yml +33 -0
  47. data/spec/dummy/config/puma.rb +34 -0
  48. data/spec/dummy/config/routes.rb +3 -0
  49. data/spec/dummy/config/spring.rb +6 -0
  50. data/spec/dummy/db/schema.rb +52 -0
  51. data/spec/dummy/db/test.sqlite3 +0 -0
  52. data/spec/dummy/log/development.log +0 -0
  53. data/spec/dummy/log/test.log +1519 -0
  54. data/spec/examples.txt +79 -0
  55. data/spec/models/lab_tech/default_cleaner_spec.rb +32 -0
  56. data/spec/models/lab_tech/experiment_spec.rb +110 -0
  57. data/spec/models/lab_tech/percentile_spec.rb +85 -0
  58. data/spec/models/lab_tech/result_spec.rb +198 -0
  59. data/spec/models/lab_tech/speedup_spec.rb +133 -0
  60. data/spec/models/lab_tech/summary_spec.rb +325 -0
  61. data/spec/models/lab_tech_spec.rb +23 -0
  62. data/spec/rails_helper.rb +62 -0
  63. data/spec/spec_helper.rb +98 -0
  64. data/spec/support/misc_helpers.rb +7 -0
  65. metadata +238 -0
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 2f5e6f3f09101904d4cf6fbc209768dba3f58bdb3ca9bbe157473e7d939fe124
4
+ data.tar.gz: 88dceb210ec735257a5f2cf329fe132085062b39dd2ad207dae2d0ff16392993
5
+ SHA512:
6
+ metadata.gz: b52697112c48077ac53e85af052b0e20e5425f448fba77a8139344afad7bb39c4bf497dcf5987d794d5826e5a6dfd920975a0d509d44fb16779d92342d21ffbd
7
+ data.tar.gz: d6178294c5066e7117435237b7ba2a93a1bbf15bba7152ead2c6f4393a808c0af1d271478126e50a5948ee133c855fa28ca2589da140a305cc5698271ce0fd81
@@ -0,0 +1,20 @@
1
+ Copyright 2019 Sam Livingston-Gray
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,323 @@
1
+ # LabTech!
2
+ Rails engine for using GitHub's 'Scientist' library with ActiveRecord, for those of us not operating apps at ROFLscale
3
+
4
+ Please go read [Scientist's amazing
5
+ README](https://github.com/github/scientist/blob/master/README.md). This tool
6
+ won't make any sense until you understand what Scientist is for and how it
7
+ works.
8
+
9
+ If conference videos are your thing, Jesse Toth's ["Easy Rewrites With Ruby and
10
+ Science!"](http://www.confreaks.tv/videos/rubyconf2014-easy-rewrites-with-ruby-and-science)
11
+ from RubyConf 2014 is well worth your time as well.
12
+
13
+ **NOTE: our examples assume that you have access to the Rails production
14
+ console.** If you work at a company that locks this down, you'll need to write
15
+ an administrative back-end UI to enable and disable experiments and review them
16
+ for accuracy and performance. (Please feel free to send those back to us in a
17
+ pull request; we simply haven't needed them for ourselves, so they don't
18
+ exist yet.)
19
+
20
+ ## Usage
21
+
22
+ Once you've installed the gem and run its migrations (as described in
23
+ "Installation", below), you can start running experiments.
24
+
25
+ For the purposes of this README, let's say we have a Customer Relationship
26
+ Manager (CRM) that lets its users search for leads using some predefined set of
27
+ criteria (name, location, favorite food, whatever). (Any resemblance to an
28
+ actual SaaS product we sell here at Real Geeks is... purely coincidental.)
29
+
30
+ Let's say, too, that the code behind that search started out sort of okay, but
31
+ it got worse and worse over time until someone decided it was time for a full
32
+ rewrite. The old search code lives in a method named `Lead.search`, but we've
33
+ been working on a replacement that lives in `SpiffySearch.leads`. The tests
34
+ for SpiffySearch are in great shape, and we're confident that we won't be
35
+ causing any 500 errors -- but we'd like to use a tool like Scientist to make
36
+ sure that SpiffySearch returns the same results in the same order our users
37
+ expect.
38
+
39
+ Stand back -- we're going to try SCIENCE!
40
+
41
+ The first thing we need is a name for our experiment -- let's just go with
42
+ `"spiffy-search"`.
43
+
44
+ ### Deploying an Experiment
45
+
46
+ ```ruby
47
+ LabTech.science "spiffy-search" do |exp|
48
+ exp.use { Lead.search(params[:search]) } # control
49
+ exp.try { SpiffySearch.leads(params[:search]) } # candidate
50
+ end
51
+ ```
52
+
53
+ Within the block, `exp` is an object that includes the `Scientist::Experiment`
54
+ module, so we can use any and all of the tools in the Scientist README.
55
+
56
+ However, I want to call out a few specific ones as being extremely useful for
57
+ this sort of thing. When working with code that returns interesting objects,
58
+ it's a Very Good Idea™ to make use of the `clean` method on the experiment.
59
+ It's probably also good to override the default comparison and to provide some
60
+ context for the experiment, so let's just redo that block with that in mind:
61
+
62
+ ```ruby
63
+ LabTech.science "spiffy-search" do |exp|
64
+ exp.context params: params.to_h
65
+
66
+ exp.use { Lead.search(params[:search]) } # control
67
+ exp.try { SpiffySearch.leads(params[:search]) } # candidate
68
+
69
+ exp.compare {|control, candidate| control.map(&:id) == candidate.map(&:id) }
70
+ exp.clean { |records| records.map(&:id) }
71
+ end
72
+ ```
73
+
74
+ Now that that's done, we can safely commit and deploy this code. As soon as
75
+ that code starts getting run, we'll see a new LabTech::Experiment record in the
76
+ database. However, **the candidate block will never run,** because
77
+ LabTech::Experiment records are disabled by default.
78
+
79
+ ### Enabling an Experiment
80
+
81
+ In order to enable the experiment, we'll need to go into the Rails production
82
+ console and run:
83
+
84
+ ```ruby
85
+ LabTech.enable "spiffy-search"
86
+ ```
87
+
88
+ If we have particularly high search volume and we only want to run the
89
+ experiment on a fraction of our requests, the `.enable` method takes an
90
+ optional `:percent` keyword argument, which accepts an integer in the range
91
+ `(0..100)`. So to sample only, say, 3% of our searches (selected at random),
92
+ we could run this instead:
93
+
94
+ ```ruby
95
+ LabTech.enable "spiffy-search", percent: 3
96
+ ```
97
+
98
+ ### Summarizing Experimental Results
99
+
100
+ At this point, if you have the [table_print gem](http://tableprintgem.com/)
101
+ installed, you can get a quick overview of all of your experiments by running:
102
+
103
+ ```ruby
104
+ tp LabTech::Experiment.all
105
+ ```
106
+
107
+ Either way, if you want more details on how the experiment is running, you can run:
108
+
109
+ ```ruby
110
+ tp LabTech.summarize_results "spiffy-search"
111
+ ```
112
+
113
+ This will print a terminal-friendly summary of your experimental results. I'll
114
+ talk more about this summary later, but for now, let's say we were
115
+ overconfident, and there's a bug in SpiffySearch that's raising an exception.
116
+ The summary will have a line that looks like this:
117
+
118
+ ```
119
+ 22 of 22 (100.00%) raised errors
120
+ ```
121
+
122
+ Ruh roh!
123
+
124
+ ### Summarizing Errors
125
+
126
+ We run this to see what's up:
127
+
128
+ ```
129
+ LabTech.summarize_errors "spiffy-search"
130
+ ```
131
+
132
+ And we see something that looks like this, only longer:
133
+
134
+ ```
135
+ ====================================================================================================
136
+ Comparing results for smoke-test:
137
+
138
+
139
+ ----------------------------------------------------------------------------------------------------
140
+ Result #1
141
+ * RuntimeError: nope
142
+ ----------------------------------------------------------------------------------------------------
143
+ ----------------------------------------------------------------------------------------------------
144
+ Result #2
145
+ * RuntimeError: nope
146
+ ----------------------------------------------------------------------------------------------------
147
+
148
+ ====================================================================================================
149
+ ```
150
+
151
+ If you want to see individual backtraces, you can do so by finding and
152
+ inspecting indvididual records in the Rails console. For now, though, let's
153
+ say we know where the error is...
154
+
155
+ ### Disabling and Restarting an Experiment
156
+
157
+ There's no point continuing to collect those exceptions, so we might as well
158
+ turn the experiment back off:
159
+
160
+ ```ruby
161
+ LabTech.disable "spiffy-search"
162
+ ```
163
+
164
+ We fix the exception, deploy the new code, and now we want to start the
165
+ experiment over again. We don't want the previous exceptions cluttering up our
166
+ new results, so let's clear out all those observations:
167
+
168
+ ```ruby
169
+ exp = LabTech::Experiment.named("spiffy-search")
170
+ exp.purge_data
171
+ ```
172
+
173
+ (Yes, this is a slightly more cumbersome interface than enabling or summarizing
174
+ an experiment. While deleting data is sometimes necessary, we don't want to
175
+ make it easy to do accidentally.)
176
+
177
+ ### Summarizing Experimental Results, Take Two
178
+
179
+ This time, the output from `LabTech.summarize_results "spiffy-search"` looks
180
+ more like this:
181
+
182
+ ```
183
+ --------------------------------------------------------------------------------
184
+ Experiment: smoke-test
185
+ --------------------------------------------------------------------------------
186
+ Earliest results: 2019-08-21T11:00:41-10:00
187
+ Latest result: 2019-08-21T11:23:31-10:00 (23 minutes)
188
+
189
+ 103 of 106 (97.16%) correct
190
+ 2 of 106 (1.88%) mismatched
191
+ 1 of 106 (0.94%) timed out
192
+
193
+ Median time delta: +0.000s (90% of observations between -0.000s and +0.000s)
194
+
195
+ Speedups (by percentiles):
196
+ 0% [ █ · ] -3.1x
197
+ 5% [ █ · ] -2.8x
198
+ 10% [ █ · ] -2.6x
199
+ 15% [ █ · ] -2.5x
200
+ 20% [ █ · ] -2.4x
201
+ 25% [ █ · ] -2.3x
202
+ 30% [ █ · ] -2.2x
203
+ 35% [ █ · ] -2.1x
204
+ 40% [ █ · ] -2.0x
205
+ 45% [ · █ ] +1.2x faster
206
+ 50% [ · · · · · · · · · · · · · · · · █ · · · · · · · · ] +1.8x faster
207
+ 55% [ · █ ] +2.0x faster
208
+ 60% [ · █ ] +2.1x faster
209
+ 65% [ · █ ] +2.2x faster
210
+ 70% [ · █ ] +2.4x faster
211
+ 75% [ · █ ] +2.6x faster
212
+ 80% [ · █ ] +2.6x faster
213
+ 85% [ · █ ] +2.7x faster
214
+ 90% [ · █ ] +2.8x faster
215
+ 95% [ · █ ] +3.0x faster
216
+ 100% [ · █] +6.7x faster
217
+ --------------------------------------------------------------------------------
218
+ ```
219
+
220
+ First off, we see a summary of the time range represented in this experiment.
221
+ This is a very simple "first result to last result" view that does not take
222
+ into account when the experiment was enabled.
223
+
224
+ Next, we see some counts. An individual run of an experiment may have one of
225
+ four outcomes:
226
+ - "correct" means that both control and candidate were considered equivalent
227
+ - "mismatched" means that the candidate returned a different value than the
228
+ control
229
+ - "timed out" means that the experiment's run raised a `Timeout::Error`
230
+ - "raised error" means that the experiment's run raised anything other than
231
+ `Timeout::Error`
232
+
233
+ After the counts, we see a bunch of performance data, starting with a line that
234
+ says "Median time delta" and includes the 5th and 95th percentile time deltas
235
+ as well. "Time delta" just means the difference in execution time between the
236
+ control and the candidate: negative values are faster, and positive values are
237
+ slower. (The 5th and 95th percentiles are deliberately chosen to keep us from
238
+ worrying too much about extreme values that might be outliers.)
239
+
240
+ The rest of the output is taken up by a chart that attempts to provide a handy
241
+ visual chart showing whether the candidate is faster or slower than the
242
+ control. Because it can be hard to remember what the signs signify, this also
243
+ includes the word "faster" when the candidate was faster than the control.
244
+
245
+ ### Comparing Mismatches
246
+
247
+ At this point, we might be curious about any mismatches, and want to
248
+ investigate those. Unfortunately, the chart I showed above was edited by hand
249
+ to show what the output might look like if mismatches were present, but as of
250
+ this writing I don't actually have any mismatches to show you. (I promise
251
+ that's not a humblebrag.)
252
+
253
+ However, you can get a quick, if EXTREMELY VERBOSE, listing of the first few
254
+ mismatches by running:
255
+
256
+ ```ruby
257
+ LabTech.compare_mismatches "spiffy-search", limit: 3
258
+ ```
259
+
260
+ You have the ability to customize the output of this by passing a block that
261
+ takes a "control" parameter followed by a "candidate" parameter; the return
262
+ value of that block will be printed to the console. How you do this will
263
+ largely depend on the kind of data you're collecting to validate your
264
+ experiments. There are several examples in the `lib/lab_tech.rb` file; I
265
+ encourage you to check them out.
266
+
267
+ ## Installation
268
+
269
+ **NOTE: As this gem is a Rails engine, we assume you have a Rails application to
270
+ include it in.**
271
+
272
+ Add this line to your application's Gemfile:
273
+
274
+ ```ruby
275
+ gem 'lab_tech'
276
+ ```
277
+
278
+ And then execute:
279
+ ```bash
280
+ $ bundle
281
+ ```
282
+
283
+ Or install it yourself as:
284
+ ```bash
285
+ $ gem install lab_tech
286
+ ```
287
+
288
+ Once the gem is installed, run this from your application's root (possibly with
289
+ the `bundle exec` or `bin/` prefix, or whatever else may be dictated by your
290
+ local custom and practice):
291
+
292
+ ```ruby
293
+ rails lab_tech:install:migrations db:migrate
294
+ ```
295
+
296
+ The output from that command should look like this:
297
+
298
+ ```
299
+ Copied migration 20190822175815_create_experiment_tables.lab_tech.rb from lab_tech
300
+ == 20190822175815 CreateExperimentTables: migrating ===========================
301
+ -- create_table("lab_tech_experiments")
302
+ -> 0.0147s
303
+ -- create_table("lab_tech_results")
304
+ -> 0.0152s
305
+ -- create_table("lab_tech_observations")
306
+ -> 0.0109s
307
+ == 20190822175815 CreateExperimentTables: migrated (0.0410s) ==================
308
+ ```
309
+
310
+ Once that's done, you should be good to go! See the "Usage" section, above.
311
+
312
+ ## Contributing
313
+
314
+ This gem was extracted just before its primary author left Real Geeks, so it's
315
+ not quite clear who's going to take responsibility for the gem. It's probably
316
+ a good idea to open a GitHub issue to start a conversation before undertaking
317
+ any great amount of work -- though, of course, you're perfectly welcome to fork
318
+ the gem and use your modified version at any time.
319
+
320
+ ## License
321
+
322
+ The gem is available as open source under the terms of the [MIT
323
+ License](https://opensource.org/licenses/MIT).
@@ -0,0 +1,30 @@
1
+ begin
2
+ require 'bundler/setup'
3
+ rescue LoadError
4
+ puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
5
+ end
6
+
7
+ require 'rdoc/task'
8
+
9
+ RDoc::Task.new(:rdoc) do |rdoc|
10
+ rdoc.rdoc_dir = 'rdoc'
11
+ rdoc.title = 'LabTech'
12
+ rdoc.options << '--line-numbers'
13
+ rdoc.rdoc_files.include('README.md')
14
+ rdoc.rdoc_files.include('lib/**/*.rb')
15
+ end
16
+
17
+ APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
18
+ load 'rails/tasks/engine.rake'
19
+
20
+ load 'rails/tasks/statistics.rake'
21
+
22
+ require 'bundler/gem_tasks'
23
+
24
+ require 'rspec/core'
25
+ require 'rspec/core/rake_task'
26
+
27
+ desc "Run all specs in spec directory (excluding plugin specs)"
28
+ RSpec::Core::RakeTask.new(spec: "app:db:test:prepare")
29
+
30
+ task default: :spec
@@ -0,0 +1,5 @@
1
+ module LabTech
2
+ class ApplicationRecord < ActiveRecord::Base
3
+ self.abstract_class = true
4
+ end
5
+ end
@@ -0,0 +1,87 @@
1
+ module LabTech
2
+ class DefaultCleaner
3
+ def self.call( value )
4
+ new.call( value )
5
+ end
6
+
7
+ def call( value )
8
+ clean( value, return_placeholders: false )
9
+ end
10
+
11
+ class RecordPlaceholder
12
+ attr_reader :class_name, :id
13
+ def initialize( record )
14
+ @class_name = record.class.to_s
15
+ @id = record.id
16
+ end
17
+
18
+ def to_a
19
+ [ class_name, id ]
20
+ end
21
+
22
+ def inspect
23
+ "<#{class_name} ##{id}>"
24
+ end
25
+ end
26
+
27
+ private
28
+
29
+ # In the event of a Recursion Blunder, stop before we smash the stack, so we
30
+ # can actually get a useful stack trace that doesn't overwhelm our scrollback
31
+ # buffers
32
+ MAX_DEPTH = 10
33
+ def __push__
34
+ @depth ||= 0
35
+ @depth += 1
36
+ fail "wtf are you even doing?" if @depth > MAX_DEPTH
37
+ end
38
+ def __pop__
39
+ @depth -= 1
40
+ end
41
+
42
+ def clean( value, return_placeholders: )
43
+ __push__
44
+
45
+ case value
46
+ when RecordPlaceholder
47
+ clean_record_placeholder( value, return_placeholders: return_placeholders )
48
+ when ActiveRecord::Base
49
+ clean_record( value, return_placeholders: return_placeholders )
50
+ when Array
51
+ clean_array( value, return_placeholders: return_placeholders )
52
+ else
53
+ value
54
+ end
55
+
56
+ ensure
57
+ __pop__
58
+ end
59
+
60
+ def clean_array( value, return_placeholders: )
61
+ placeholders = value.map {|e| clean(e, return_placeholders: true) }
62
+ if placeholders.all? { |e| e.kind_of?(RecordPlaceholder) } && !return_placeholders
63
+ count_placeholders( placeholders )
64
+ else
65
+ placeholders.map {|e| clean(e, return_placeholders: return_placeholders) }
66
+ end
67
+ end
68
+
69
+ def clean_record( value, return_placeholders: )
70
+ placeholder = RecordPlaceholder.new( value )
71
+ clean_record_placeholder( placeholder, return_placeholders: return_placeholders )
72
+ end
73
+
74
+ def clean_record_placeholder( value, return_placeholders: )
75
+ return_placeholders \
76
+ ? value \
77
+ : value.to_a
78
+ end
79
+
80
+ def count_placeholders( placeholders )
81
+ counts = placeholders.group_by(&:class_name).map { |class_name, summs|
82
+ [ class_name, summs.map(&:id) ]
83
+ }
84
+ Hash[ counts ]
85
+ end
86
+ end
87
+ end