lab_coat 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d819fd05745d82590e7d0f8c2e05f415b3ec6bea1cdc5def244fcfee17b969c7
-  data.tar.gz: 27819f2af7099608845196b00ae89a0ba59e9bdb8224814585a659601a261e76
+  metadata.gz: 22451390278d5dff8546f421b305107ea5e6d6636ea288eb674ab980353dcd20
+  data.tar.gz: 0a97027504a75312e42501283898021c9417b6e283e22cb5d86a614ebf939ae8
 SHA512:
-  metadata.gz: 711d1510f5d1f4d59a57830567988d74fad50bfe7c4e548cd76e67d3f46bb2dfbbff503914e6129f7023032f7986c2153a1359e600d1f7d2e6aa443d98f27c0d
-  data.tar.gz: 66ed5879e1e803045806464c6b708b1a355c0d997965e1a6ea43e42b789d4f2904b7c39223b60101c08a1857c35dba72003d9463102962da27d9f39b181c1636
+  metadata.gz: d59adedb051274080954bd28c23b0fbbdc4b6914d90a19f45e5a4c8d3d8f46a06b334e7a2c1e5698eee9adbc9bdfb0f8d4270d37c37430978d952635f4186711
+  data.tar.gz: eb3f5aadd5a01813521b81941052a699ce5ced2b7e9f8d0964942d4a419524e001a0664cd55170069a9fff149eddc6f36885d704e035cc139309115f35c8d856

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## [0.1.2] - 2024-04-15
+- use `Benchmark` to capture the duration with more details
+- add `to_h` methods to `Result` and `Observation` for convenience
+
 ## [0.1.1] - 2024-04-08
 - add `#slug` method to `Observation`
 

data/README.md

@@ -2,7 +2,7 @@
 
 ![Gem Version](https://img.shields.io/gem/v/lab_coat) ![Gem Total Downloads](https://img.shields.io/gem/dt/lab_coat)
 
-A simple experiment library to safely test new code paths.
+A simple experiment library to safely test new code paths. `LabCoat` is designed to be highly customizable and play nice with your existing tools/services.
 
 This library is heavily inspired by [Scientist](https://github.com/github/scientist), with some key differences:
 - `Experiments` are `classes`, not `modules` which means they are stateful by default.
@@ -69,13 +69,11 @@ See the [`Experiment`](lib/lab_coat/experiment.rb) class for more details.
 > [!TIP]
 > You should create a shared base class(es) to maintain consistency across experiments within your app.
 
-You may want to give your experiment some context, or state. You can do this via an initializer or writer methods just like any other Ruby class.
+You might want to give your experiment some context, or state. You can do this via an initializer or writer methods just like any other Ruby class.
 
 ```ruby
 # application_experiment.rb
 class ApplicationExperiment < LabCoat::Experiment
-  attr_reader :user, :is_admin
-
   def initialize(user)
     @user = user
     @is_admin = user.admin?
@@ -83,25 +81,19 @@ class ApplicationExperiment < LabCoat::Experiment
 end
 ```
 
-You likely want to `publish!` all experiments in a consistent way, so that you can analyze the data and make decisions. New `Experiment` authors should not have to redo the "plumbing" between your experimentation framework (e.g. `LabCoat`) and your observability (o11y) process.
+You might want to `publish!` all experiments in a consistent way so that you can analyze the data and make decisions. New `Experiment` authors should not have to redo the "plumbing" between your experimentation framework (e.g. `LabCoat`) and your observability (o11y) process.
 
 ```ruby
 # application_experiment.rb
 class ApplicationExperiment < LabCoat::Experiment
   def publish!(result)
-    YourO11yService.track_experiment_result(
-      name: result.experiment.name,
-      matched: result.matched?,
-      observations: {
-        control: result.control.publishable_value,
-        candidate: result.candidate.publishable_value,
-      }
-    )
+    payload = result.to_h.merge(user_id: @user.id)
+    YourO11yService.track_experiment_result(payload)
   end
 end
 ```
 
-You might also have a common way to enable experiments such as a feature flag system and/or common guards you want to enforce application wide. These might come from a mix of services and the `Experiment`'s state.
+You might have a common way to enable experiments such as a feature flag system and/or common guards you want to enforce application wide. These might come from a mix of services and the `Experiment`'s state.
 
 ```ruby
 # application_experiment.rb
@@ -112,19 +104,34 @@ class ApplicationExperiment < LabCoat::Experiment
 end
 ```
 
+You might want to track any errors thrown from all your experiments and route them to some service, or log them.
+
+```ruby
+# application_experiment.rb
+class ApplicationExperiment < LabCoat::Experiment
+  def raised(observation)
+    YourErrorService.report_error(
+      observation.error,
+      tags: observation.to_h
+    )
+  end
+end
+```
+
 ### Make some `Observations` via `run!`
 
 You don't have to create an `Observation` yourself; that happens automatically when you call `Experiment#run!`. The control and candidate `Observations` are packaged into a `Result` and [passed to `Experiment#publish!`](#publish-the-result).
 
 |Attribute|Description|
 |---|---|
-|`duration`|The duration of the run in `float` seconds.|
+|`duration`|The duration of the run represented as a `Benchmark::Tms` object.|
 |`error`|If the code path raised, the thrown exception is stored here.|
 |`experiment`|The `Experiment` instance this `Result` is for.|
 |`name`|Either `"control"` or `"candidate"`.|
 |`publishable_value`|A publishable representation of the `value`, as defined by `Experiment#publishable_value`.|
 |`raised?`|Whether or not the code path raised.|
 |`slug`|A combination of the `Experiment#name` and `Observation#name`, e.g. `"experiment_name.control"`|
+|`to_h`|A hash representation of the `Observation`. Useful for publishing and/or reporting.|
 |`value`|The return value of the observed code path.|
 
 `Observation` instances are passed to many of the `Experiment` methods that you may override.
@@ -145,18 +152,14 @@ def ignore?(control, candidate)
 end
 
 def publishable_value(observation)
-  if observation.raised?
-    {
-      error_class: observation.error.class.name,
-      error_message: observation.error.message
-    }
-  else
-    {
-      type: observation.name,
-      value: observation.publishable_value,
-      duration: observation.duration
-    }
-  end
+  return nil if observation.raised?
+
+  # Let's say your control and candidate blocks return objects that don't serialize nicely.
+  {
+    some_attribute: observation.value.some_attribute,
+    some_other_attribute: observation.value.some_other_attribute,
+    some_count: observation.value.some_array.count
+  }
 end
 
 # Elsewhere...
@@ -174,10 +177,26 @@ A `Result` represents a single run of an `Experiment`.
 |`experiment`|The `Experiment` instance this `Result` is for.|
 |`ignored?`|Whether or not the result should be ignored, as defined by `Experiment#ignore?`|
 |`matched?`|Whether or not the `control` and `candidate` match, as defined by `Experiment#compare`|
+|`to_h`|A hash representation of the `Result`. Useful for publishing and/or reporting.|
+
+The `Result` is passed to your implementation of `#publish!` when an `Experiment` is finished running. The `to_h` method on a Result is a good place to start and might be sufficient for most experiments.
+
+```ruby
+# your_experiment.rb
+def publish!(result)
+  return if result.ignored?
 
-The `Result` is passed to your implementation of `#publish!` when an `Experiment` is finished running.
+  puts result.to_h
+end
+```
+
+> ![NOTE]
+> All `Results` are passed to `publish!`, **including ignored ones**. It is your responsibility to call the `ignored?` method and handle those as you wish.
+
+You can always access all of the attributes of the `Result` and its `Observations` directly to fully customize what your experiment publishing looks like.
 
 ```ruby
+# your_experiment.rb
 def publish!(result)
   if result.ignored?
     puts "🙈"
@@ -189,22 +208,27 @@ def publish!(result)
   else
     control = result.control
     candidate = result.candidate
-    puts <<~MISMATCH
+    puts <<~MSG
       😮
 
       #{control.slug}
       Value: #{control.publishable_value}
-      Duration: #{control.duration}
+      Duration Real: #{control.duration.real}
+      Duration System: #{control.duration.stime}
+      Duration User: #{control.duration.utime}
       Error: #{control.error&.message}
 
       #{candidate.slug}
       Value: #{candidate.publishable_value}
-      Duration: #{candidate.duration}
+      Duration: #{candidate.duration.real}
+      Duration System: #{candidate.duration.stime}
+      Duration User: #{candidate.duration.utime}
       Error: #{candidate.error&.message}
-    MISMATCH
+    MSG
   end
 end
 ```
+
 Running a mismatched experiment with this implementation of `publish!` would produce:
 
 ```
@@ -212,12 +236,16 @@ Running a mismatched experiment with this implementation of `publish!` would pro
 
 my_experiment.control
 Value: 420
-Duration: 12.934
+Duration Real: 12.934
+Duration System: 2.134
+Duration User: 10.800
 Error:
 
 my_experiment.candidate
 Value: 69
-Duration: 9.702
+Duration Real: 9.702
+Duration System: 1.002
+Duration User: 8.700
 Error:
 ```
 
@@ -238,8 +266,10 @@ The `Observation` class can be used as a standalone wrapper for any code that yo
   if observation.raised?
     puts "error: #{observation.error.message}"
   else
-    puts "duration: #{observation.duration}"
-    puts "succeeded: #{!observation.raised?}"
+    puts <<~MSG
+      duration: #{observation.duration.real}
+      succeeded: #{!observation.raised?}
+    MSG
   end
 end
 ```

data/lib/lab_coat/observation.rb

@@ -9,13 +9,10 @@ module LabCoat
       @name = name
       @experiment = experiment
 
-      start_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
-      begin
+      @duration = Benchmark.measure(name) do
         @value = block.call
       rescue StandardError => e
         @error = e
-      ensure
-        @duration = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - start_at
       end
     end
 
@@ -28,9 +25,22 @@ module LabCoat
       !error.nil?
     end
 
-    # @return [String] String representing this Observation.
+    # @return [String] String representing this `Observation`.
     def slug
       "#{experiment.name}.#{name}"
     end
+
+    # @return [Hash] A hash representation of this `Observation`. Useful when publishing `Results`.
+    def to_h
+      {
+        name: name,
+        experiment: experiment.name,
+        slug: slug,
+        value: publishable_value,
+        duration: duration.to_h,
+        error_class: error&.class&.name,
+        error_message: error&.message
+      }.compact
+    end
   end
 end

data/lib/lab_coat/result.rb

@@ -26,5 +26,16 @@ module LabCoat
     def ignored?
       @ignored
     end
+
+    # @return [Hash] A hash representation of this `Result`. Useful when publishing.
+    def to_h
+      {
+        experiment: experiment.name,
+        matched: matched?,
+        ignored: ignored?,
+        control: control.to_h,
+        candidate: candidate.to_h
+      }
+    end
   end
 end

data/lib/lab_coat/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module LabCoat
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/lab_coat.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "benchmark"
+
 require_relative "lab_coat/version"
 require_relative "lab_coat/observation"
 require_relative "lab_coat/result"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lab_coat
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Omkar Moghe
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-09 00:00:00.000000000 Z
+date: 2024-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest