lab_coat 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d66afab349e98ec1ce140293e4372772633d9d8565ff66543f502769274614fb
+  data.tar.gz: 683b940631dc537a41049af5c49006f56c80b833d8644957c6cbb8c6b28cff2f
+SHA512:
+  metadata.gz: 1c492775bf3f0b3c9418934764891a82331761cc507e41e7de7c9085481b2860cc6fe3b5c9d048e008b15519cad72a1226ec9476248f75365ec581d634e59317
+  data.tar.gz: ead0b6e3125f239444654e492ed5414073ba9b56c33e365226b7218548c429b08599c5df4475dbf76eb9375add204126fa7f000dcd59eb1c69dce6cf65234a5b

data/.rubocop.yml

@@ -0,0 +1,8 @@
+AllCops:
+  TargetRubyVersion: 3.0
+
+Style/StringLiterals:
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  EnforcedStyle: double_quotes

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+## [0.1.0] - 2024-04-08
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Omkar Moghe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,201 @@
+# LabCoat 🥼
+
+![Gem Version](https://img.shields.io/gem/v/lab_coat) ![Gem Total Downloads](https://img.shields.io/gem/dt/lab_coat)
+
+A simple experiment library to safely test new code paths.
+
+This library is heavily inspired by [Scientist](https://github.com/github/scientist), with some key differences:
+- `Experiments` are `classes`, not `modules` which means they are stateful by default.
+- There is no app wide default experiment that gets magically set.
+- The `Result` only supports one comparison at a time, i.e. only 1 `candidate` is allowed.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+`bundle add lab_coat`
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+`gem install lab_coat`
+
+## Usage
+
+### Create an `Experiment`
+
+To do some science, i.e. test out a new code path, start by defining an `Experiment`. An experiment is any class that inherits from `LabCoat::Experiment` and implements the required methods.
+
+#### Required methods
+
+See the [`Experiment`](lib/lab_coat/experiment.rb) class for more details.
+
+|Method|Description|
+|---|---|
+|`enabled?`|Returns a `Boolean` that controls whether or not the experiment runs.|
+|`control`|The existing or default behavior. This will always be returned from `#run!`.|
+|`candidate`|The new behavior you want to test.|
+|`publish!`|This is not _technically_ required, but `Experiments` are not useful unless you can analyze the results. Override this method to record the `Result` however you wish.|
+
+#### Additional methods
+
+|Method|Description|
+|---|---|
+|`compare`|Whether or not the result is a match. This is how you can run complex/custom comparisons.|
+|`ignore?`|Whether or not the result should be ignored. Ignored `Results` are still passed to `#publish!`|
+|`raised`|Callback method that's called when an `Observation` raises.|
+
+Consider creating a shared base class(es) to create consistency across experiments within your app.
+
+```ruby
+# application_experiment.rb
+class ApplicationExperiment < LabCoat::Experiment
+end
+```
+
+You may want to give your experiment some context, or state. You can do this via an initializer or writer methods just like any other Ruby class.
+
+```ruby
+# application_experiment.rb
+class ApplicationExperiment < LabCoat::Experiment
+  attr_reader :user, :is_admin
+
+  def initialize(user)
+    @user = user
+    @is_admin = user.admin?
+  end
+end
+```
+
+You likely want to `publish!` all experiments in a uniform way, so that you can analyze the data and make decisions.
+
+```ruby
+# application_experiment.rb
+class ApplicationExperiment < LabCoat::Experiment
+  def publish!(result)
+    YourO11yService.track_experiment_result(
+      name: result.experiment.name,
+      matched: result.matched?,
+      observations: {
+        control: result.control.publishable_value,
+        candidate: result.candidate.publishable_value,
+      }
+    )
+  end
+end
+```
+
+You might also have a common way to enable experiments such as a feature flag system and/or common guards you want to enforce application wide.
+
+```ruby
+# application_experiment.rb
+class ApplicationExperiment < LabCoat::Experiment
+  def enabled?
+    !@is_admin && YourFeatureFlagService.flag_enabled?(@user.id, name)
+  end
+end
+```
+
+### Make some `Observations` via `run!`
+
+You don't have to create an `Observation` yourself; that happens automatically when you call `Experiment#run!`.
+
+|Attribute|Description|
+|---|---|
+|`name`|Either `"control"` or `"candidate"`.|
+|`experiment`|The `Experiment` instance this `Result` is for.|
+|`duration`|The duration of the run in `float` seconds.|
+|`value`|The return value of the observed code path.|
+|`publishable_value`|A publishable representation of the `value`, as defined by `Experiment#publishable_value`.|
+|`raised?`|Whether or not the code path raised.|
+|`error`|If the code path raised, the thrown exception is stored here.|
+
+`Observation` instances are passed to many of the `Experiment` methods that you may override.
+
+```ruby
+# your_experiment.rb
+def compare(control, candidate)
+  return false if control.raised? || candidate.raised?
+
+  control.value.some_method == candidate.value.some_method
+end
+
+def ignore?(control, candidate)
+  return true if control.raised? || candidate.raised?
+  return true if candidate.value.some_guard?
+
+  false
+end
+
+def publishable_value(observation)
+  if observation.raised?
+    {
+      error_class: observation.error.class.name,
+      error_message: observation.error.message
+    }
+  else
+    {
+      type: observation.name,
+      value: observation.publishable_value,
+      duration: observation.duration
+    }
+  end
+end
+
+# Elsewhere...
+YourExperiment.new(...).run!
+```
+
+### Publish the `Result`
+
+A `Result` represents a single run of an `Experiment`.
+
+|Attribute|Description|
+|---|---|
+|`experiment`|The `Experiment` instance this `Result` is for.|
+|`control`|An `Observation` instance representing the `Experiment#control` behavior|
+|`candidate`|An `Observation` instance representing the `Experiment#candidate` behavior|
+|`matched?`|Whether or not the `control` and `candidate` match, as defined by `Experiment#compare`|
+|`ignored?`|Whether or not the result should be ignored, as defined by `Experiment#ignore?`|
+
+The `Result` is passed to your implementation of `#publish!` when an `Experiment` is finished running.
+
+```ruby
+def publish!(result)
+  if result.ignored?
+    puts "🙈"
+    return
+  end
+
+  if result.matched?
+    puts "😎"
+  else
+    puts <<~MISMATCH
+      😮
+
+      [Control]
+      Value: #{result.control.publishable_value}
+      Duration: #{result.control.duration}
+      Error: #{result.control.error&.message}
+
+      [Candidate]
+      Value: #{result.candidate.publishable_value}
+      Duration: #{result.candidate.duration}
+      Error: #{result.candidate.error&.message}
+    MISMATCH
+  end
+end
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/lab_coat.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+Minitest::TestTask.create
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

data/lib/lab_coat/experiment.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module LabCoat
+  # A base experiment class meant to be subclassed to define various experiments.
+  class Experiment
+    attr_reader :name
+
+    def initialize(name)
+      @name = name
+    end
+
+    # Override this method to control whether or not the experiment runs.
+    # @return [TrueClass, FalseClass]
+    def enabled?(...)
+      raise InvalidExperimentError, "`#enabled?` must be implemented in your Experiment class."
+    end
+
+    # Override this method to define the existing aka "control" behavior. This method is always run, even when
+    # `enabled?` is false.
+    # @return [Object] Anything.
+    def control(...)
+      raise InvalidExperimentError, "`#control` must be implemented in your Experiment class."
+    end
+
+    # Override this method to define the new aka "candidate" behavior. Only run if the experiment is enabled.
+    # @return [Object] Anything.
+    def candidate(...)
+      raise InvalidExperimentError, "`#candidate` must be implemented in your Experiment class."
+    end
+
+    # Override this method to define what is considered a match or mismatch. Must return a boolean.
+    # @param control [LabCoat::Observation] The control `Observation`.
+    # @param candidate [LabCoat::Observation] The candidate `Observation`.
+    # @return [TrueClass, FalseClass]
+    def compare(control, candidate)
+      control.value == candidate.value
+    end
+
+    # Override this method to define which results are ignored. Must return a boolean.
+    def ignore?(_control, _candidate)
+      false
+    end
+
+    # Called when the control and/or candidate observations raise an error.
+    # @param observation [LabCoat::Observation]
+    def raised(observation); end
+
+    # Override this method to transform the value for publishing. This could mean turning the value into something
+    # serializable (e.g. JSON).
+    # @param observation [LabCoat::Observation]
+    def publishable_value(observation)
+      observation.value
+    end
+
+    # Override this method to publish the `Result`. It's recommended to override this once in an application wide base
+    # class.
+    # @param result [LabCoat::Result] The result of this experiment.
+    def publish!(result); end
+
+    # Runs the control and candidate and publishes the result. Always returns the result of `control`.
+    # @param context [Hash] Any data needed at runtime.
+    def run!(...) # rubocop:disable Metrics/MethodLength
+      # Run the control and exit early if the experiment is not enabled.
+      control = Observation.new("control", self) do
+        control(...)
+      end
+      raised(control) if control.raised?
+      return control.value unless enabled?(...)
+
+      candidate = Observation.new("candidate", self) do
+        candidate(...)
+      end
+      raised(candidate) if candidate.raised?
+
+      # Compare and publish the results.
+      result = Result.new(self, control, candidate)
+      publish!(result)
+
+      # Always return the control.
+      control.value
+    end
+  end
+end

data/lib/lab_coat/observation.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module LabCoat
+  # A wrapper around some behavior that captures the resulting `value` and any exceptions thrown.
+  class Observation
+    attr_reader :name, :experiment, :duration, :value, :error
+
+    def initialize(name, experiment, &block)
+      @name = name
+      @experiment = experiment
+
+      start_at = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
+      begin
+        @value = block.call
+      rescue StandardError => e
+        @error = e
+      ensure
+        @duration = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) - start_at
+      end
+    end
+
+    def publishable_value
+      @publishable_value ||= experiment.publishable_value(self)
+    end
+
+    # @return [TrueClass, FalseClass]
+    def raised?
+      !error.nil?
+    end
+  end
+end

data/lib/lab_coat/result.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module LabCoat
+  # The result of a single `Experiment` run, that is published by the `Experiment`.
+  class Result
+    attr_reader :experiment, :control, :candidate
+
+    def initialize(experiment, control, candidate)
+      @experiment = experiment
+      @control = control
+      @candidate = candidate
+      @matched = experiment.compare(control, candidate)
+      @ignored = experiment.ignore?(control, candidate)
+
+      freeze
+    end
+
+    # Whether or not the control and candidate match, as defined by `Experiment#compare`.
+    # @return [TrueClass, FalseClass]
+    def matched?
+      @matched
+    end
+
+    # Whether or not the result should be ignored, as defined by `#Experiment#ignore?`.
+    # @return [TrueClass, FalseClass]
+    def ignored?
+      @ignored
+    end
+  end
+end

data/lib/lab_coat/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LabCoat
+  VERSION = "0.1.0"
+end

data/lib/lab_coat.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "lab_coat/version"
+require_relative "lab_coat/observation"
+require_relative "lab_coat/result"
+require_relative "lab_coat/experiment"
+
+module LabCoat
+  Error = Class.new(StandardError)
+  InvalidExperimentError = Class.new(Error)
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: lab_coat
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Omkar Moghe
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-04-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.16'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.21'
+description:
+email:
+- yo@omkr.dev
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/lab_coat.rb
+- lib/lab_coat/experiment.rb
+- lib/lab_coat/observation.rb
+- lib/lab_coat/result.rb
+- lib/lab_coat/version.rb
+homepage: https://github.com/omkarmoghe/lab_coat
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/omkarmoghe/lab_coat
+  source_code_uri: https://github.com/omkarmoghe/lab_coat
+  changelog_uri: https://github.com/omkarmoghe/lab_coat/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.1
+signing_key:
+specification_version: 4
+summary: A simple experiment library to safely test new code paths.
+test_files: []