record_loader 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 159dabfb7769aebda7cb1c5fd9f77b67e728917876c769d663191caefc386a48
+  data.tar.gz: 64e7699028b6ca62daa1dd1bab3b9513f19a61affca1ce3df0f0b074cfc60315
+SHA512:
+  metadata.gz: 3dff12471dc11ca8b76a029e229240bb76f2b86e079c9149af0119525e4ce784c8875f33e17b92f3d2ca900eb7d7d6759bb589959f568af4a56ea9a6ede85454
+  data.tar.gz: a7506298753fe9e3d99a06b574312d00c975d93763c9a9715dc17ee5657cea60ab62fcacc5116dd36b57e2a538ecc26f841e2a0dc0891f9fa135052378a2595e

data/.github/workflows/cops_and_specs.yml

@@ -0,0 +1,39 @@
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
+# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
+
+name: Cops and Specs
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Ruby
+    # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
+    # change this to (see https://github.com/ruby/setup-ruby#versioning):
+    # uses: ruby/setup-ruby@v1
+      uses: ruby/setup-ruby@ec106b438a1ff6ff109590de34ddc62c540232e0
+      with:
+        ruby-version: 2.5
+    - name: Install dependencies
+      run: bundle install
+    - name: Run Rubocop
+      run: bundle exec rubocop
+    - name: Test & publish code coverage
+      uses: paambaati/codeclimate-action@v2.6.0
+      env:
+        CC_TEST_REPORTER_ID: 5c1f50b6c3beb11b20e0e1d40800ebd81a5e1724b626060e9b366f2942acc315
+      with:
+        coverageCommand: bundle exec rspec

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,51 @@
+# The behaviour of RuboCop can be controlled via the .rubocop.yml
+# configuration file. It makes it possible to enable/disable
+# certain cops (checks) and to alter their behaviour if they accept
+# any parameters. The file can be placed either in your home
+# directory or in some project directory.
+#
+# RuboCop will start looking for the configuration file in the directory
+# where the inspected file is and continue its way up to the root directory.
+#
+# See https://github.com/rubocop-hq/rubocop/blob/master/manual/configuration.md
+
+require: rubocop-rspec
+
+Layout/LineLength:
+  Max: 120
+
+# Blocks are used to structure tests and are part of the rspec dsl.
+# The standard BlockLength limits are too stringent for this purpose.
+Metrics/BlockLength:
+  ExcludedMethods:
+    - describe
+    - context
+
+# Increase the acceptable nesting levels. While we don't want
+# 4 nested contexts, it appears that the cop is also counting
+# describe. So by the time we've described out class, and the method
+# we only have room for one context. Maybe this is the cops intent
+# but feels a little strict to me.
+RSpec/NestedGroups:
+  Max: 4
+
+# Pending cops in 0.83 can be removed once enabled in bulk
+# see https://docs.rubocop.org/en/latest/versioning/:
+Layout/SpaceAroundMethodCallOperator:
+  Enabled: true
+Lint/RaiseException:
+  Enabled: true
+Lint/StructNewOverride:
+  Enabled: true
+Style/ExponentialNotation:
+  Enabled: true
+Style/HashEachMethods:
+  Enabled: true
+Style/HashTransformKeys:
+  Enabled: true
+Style/HashTransformValues:
+  Enabled: true
+Layout/EmptyLinesAroundAttributeAccessor:
+  Enabled: true
+Style/SlicingWithRange:
+  Enabled: true

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.5.3

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.4.1
+before_install: gem install bundler -v 1.17.3

data/.yardopts

@@ -0,0 +1,4 @@
+-
+LICENSE
+CHANGELOG.md
+lib/generators/record_loader/USAGE

data/CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+Keeps track of notable changes. Please remember to add new behaviours to the
+Unreleased section to make new releases easy.
+
+## [Unreleased]
+
+## [0.1.0]
+
+Initial release
+
+- [Feature] Produce testable, reproducible data migrations across multiple environments
+- [Feature] Organize data into multiple files to provide context
+- [Feature] Add development environment specific data with .dev.yml files
+- [Feature] Keep work-in-progress isolated with .wip.yml files
+- [Feature] Rails generators to quickly create new record loaders

data/Gemfile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in record_loader.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,70 @@
+PATH
+  remote: .
+  specs:
+    record_loader (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    coderay (1.1.2)
+    diff-lcs (1.3)
+    docile (1.3.2)
+    json (2.3.0)
+    method_source (1.0.0)
+    parallel (1.19.1)
+    parser (2.7.1.2)
+      ast (~> 2.4.0)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rainbow (3.0.0)
+    rake (12.3.3)
+    rexml (3.2.4)
+    rspec (3.9.0)
+      rspec-core (~> 3.9.0)
+      rspec-expectations (~> 3.9.0)
+      rspec-mocks (~> 3.9.0)
+    rspec-core (3.9.2)
+      rspec-support (~> 3.9.3)
+    rspec-expectations (3.9.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-mocks (3.9.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.9.0)
+    rspec-support (3.9.3)
+    rubocop (0.83.0)
+      parallel (~> 1.10)
+      parser (>= 2.7.0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      rexml
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-rspec (1.39.0)
+      rubocop (>= 0.68.1)
+    ruby-progressbar (1.10.1)
+    simplecov (0.17.1)
+      docile (~> 1.1)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    unicode-display_width (1.7.0)
+    yard (0.9.25)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 2.1)
+  pry (~> 0.13)
+  rake (~> 12.3)
+  record_loader!
+  rspec (~> 3.0)
+  rubocop (~> 0.82)
+  rubocop-rspec (~> 1.39)
+  simplecov (~> 0.17.0)
+  yard (~> 0.9)
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,24 @@
+The MIT License (MIT)
+
+Authors:
+James Glover <james.glover@sanger.ac.uk>
+
+Copyright (c) 2020 Genome Research Ltd
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,182 @@
+
+# RecordLoader
+
+![Cops and Specs](https://github.com/sanger/record_loader/workflows/Cops%20and%20Specs/badge.svg)
+[![Maintainability](https://api.codeclimate.com/v1/badges/0ce827d110dfced197ab/maintainability)](https://codeclimate.com/github/sanger/record_loader/maintainability)
+[![Test Coverage](https://api.codeclimate.com/v1/badges/0ce827d110dfced197ab/test_coverage)](https://codeclimate.com/github/sanger/record_loader/test_coverage)
+[![Yard Docs](http://img.shields.io/badge/yard-docs-blue.svg)](https://rubydoc.info/github/sanger/record_loader)
+
+RecordLoader provides a simple and standardized way of populating databases from information described in a series of
+organized yaml files. It is intended to be used to generate a number of idempotent tasks, which can be run in both
+your production and development environments.
+
+While written with ActiveRecord/Rails in mind, it is possible to use RecordLoader in different environments.
+
+## Key features
+
+- Produce testable, reproducible data migrations across multiple environments
+- Organize data into multiple files to provide context
+- Add development environment specific data with .dev.yml files
+- Keep work-in-progress isolated with .wip.yml files
+- Rails generators to quickly create new record loaders
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'record_loader'
+```
+
+And then execute:
+
+```bash
+    bundle
+```
+
+Or install it yourself as:
+
+```bash
+    gem install record_loader
+```
+
+If you are using Rails, you do not need to make any further changes, and all necessary hooks will be installed when
+generating your first record loader.
+
+## Usage (Rails)
+
+RecordLoader provides a generator to automatically build a loader, specs and the yaml files necessary to use it.
+In addition, the first time you use it it will automatically install the necessary rake files and configuration.
+You can access this by running:
+
+```bash
+    bundle exec rails g record_loader
+```
+
+Which will return the documentation:
+
+{include:file:lib/generators/record_loader/USAGE}
+
+### An example loader
+
+Suppose you want to create a loader to maintain a selection of product types. You'll first use the generator:
+
+```bash
+  $ bundle exec rails g record_loader
+       exist
+      create  config/record_loader/product_types/default_records.yml
+      create  lib/record_loader/product_type_loader.rb
+      create  lib/record_loader/tasks/record_loader/product_type.rake
+      create  spec/data/record_loader/product_types/two_entry_example.yml
+      create  spec/lib/record_loader/product_type_loader_spec.rb
+        skip  lib/record_loader/application_record_loader.rb
+   identical  lib/tasks/record_loader.rake
+```
+
+This will create several files:
+
+#### `lib/tasks/record_loader.rake`
+
+Adds the record_loader:all rake task which can be used to trigger all record loaders.
+
+#### `lib/record_loader/application_record_loader.rb`
+
+Application specific base class for customization.
+
+#### `config/record_loader/product_types/default_records.yml`
+
+Example yaml file to begin populating with your record information. Record Loaders will load all yaml files from within
+this directory, so it is possible to separate your records into multiple different files for better organization.
+In addition yaml files ending in `.dev.yml` and `.wip.yml` exhibit special behaviour.
+See [dev and wip files](#dev-and-wip).
+
+#### `lib/record_loader/product_type_loader.rb`
+
+The actual loader. It will look something like this:
+
+```ruby
+# frozen_string_literal: true
+# This file was automatically generated via `rails g record_loader`
+
+# RecordLoader handles automatic population and updating of database records
+# across different environments
+# @see https://rubydoc.info/github/sanger/record_loader/
+module RecordLoader
+  # Creates the specified plate types if they are not present
+  class ProductTypeLoader < ApplicationRecordLoader
+    config_folder 'product_types'
+
+    def create_or_update!(name, options)
+      ProductType.create_with(options).find_or_create_by!(name: name)
+    end
+  end
+end
+```
+
+The `config_folder` specifies which directory under `config/record_loader` will be used to source the yaml files.
+The method `create_or_update!` will create the actual records, and should be idempotent (ie. calling it multiple times will
+have the same effect as calling it once). `create_or_update!` will be called once for each entry in the yaml files,
+with the first argument being the key, and the second argument being the value, usually a hash of options.
+
+#### `lib/record_loader/tasks/record_loader/product_type.rake`
+
+This contains the `record_loader:product_type` which will trigger the record loader, and also ensures that
+`record_loader:product_type` will get invoked on calling `record_loader:all`.
+
+#### `spec/data/record_loader/product_types/two_entry_example.yml`
+
+A basic configuration for testing the loader. Tests use a separate directory to avoid coupling your specs to the data.
+
+#### `spec/lib/record_loader/product_type_loader_spec.rb`
+
+A basic rspec spec file for testing your loader. By default this just confirms that your loader creates the
+expected number of records, and that it is idempotent.
+
+## Dev and Wip files
+
+Each loader can have one or more yaml files contained within its config directory. Most files will be aggregated
+together and only serve to provide a means of organization. However it is possible to add extra behaviour:
+
+`.dev.yml` files will only be loaded in development environments. This is useful for seeding data for quick testing, but
+which will not be needed in production environments. This may include test user accounts, dummy projects or quick
+start data.
+
+`.wip.yml` files will only be loaded if explicitly enabled via a WIP environmental variable. For example the file
+`my_feature.wip.yml` will run if the WIP env is set to `my_feature`. Multiple WIP flags can be set at the same time by
+providing a comma separated list. eg. `WIP=my_feature,other_feature`
+
+If you have an existing feature flag system you can use this instead by adding a `wip_list` method to
+`RecordLoader::ApplicationRecordLoader` which returns an array of enabled feature names. For example:
+
+```ruby
+  def wip_list
+    FeatureFlags.active.pluck(:name)
+  end
+```
+
+## Non Rails Environments
+
+In non-rails environments you can use the {RecordLoader::Adapter::Basic} adapter to avoid Rails specific functionality.
+This is the default adapter for {RecordLoader::Base}, although it is still recommended that you create an application
+specific class that inherits from this to allow for customization. Your custom record loaders can then inherit from this
+class instead.
+
+See {RecordLoader::Adapter} for information about custom adapters.
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can
+also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the
+version number in `version.rb`, ensure the CHANGELOG.md is updated and that everything is committed.
+Then run `bundle exec rake release`, which will create a git tag for the version,  push git commits and tags, and push
+the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at <https://github.com/sanger/record_loader>.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'bundler/gem_tasks'
+require 'rspec/core/rake_task'
+require 'rubocop/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+RuboCop::RakeTask.new
+
+task default: %i[rubocop spec]
+
+task release: :preflight
+
+desc 'Runs the preflight checklist before building a release'
+task :preflight do
+  puts '🛫 Preflight checklist'
+
+  current_version = RecordLoader::VERSION
+
+  puts 'This is still a manual process, but before we roll a release, lets confirm a few things:'
+  puts
+  puts "☐ lib/record_loader/version.rb updated? Currently contains #{current_version}"
+  puts '☐ CHANGELOG.md updated'
+  puts '☐ Commited, on master and up to date'
+  puts
+  print 'Proceed Y/N > '
+
+  proceed = $stdin.gets.chomp
+
+  if proceed.casecmp? 'Y'
+    puts 'Proceeding ...'
+  else
+    puts 'Canceling release process'
+    exit 1
+  end
+end

data/bin/console

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'record_loader'
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+require 'pry'
+Pry.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/generators/record_loader/USAGE

@@ -0,0 +1,36 @@
+Description:
+    Adds a new RecordLoader and all necessary files.
+
+    --record-class
+    Typically a loader is named after the class it creates, but if you want to
+    override this, you can specify a different ActiveRecord class name with the
+    record-class option. This functionality is purely for convenience, and will
+    change the class names used in the record loader, and associated specs. If
+    your loader is more complicated, and will create records of multiple classes
+    then you can ignore this setting and make the relevant modifications to
+    lib/record_loader/xxx_loader.rb and
+    spec/lib/record_loader/xxx_loader_spec.rb manually.
+
+    --record-key
+    By default the generator creates a simple loader that checks for a record
+    based on the value of a particular attribute, and it it doesn't exists,
+    creates it using the attributes specified in the yaml file. You can
+    specify which attribute gets used by specifying record-key, by default it
+    uses name.
+
+Example:
+    rails generate record_loader MyThing
+
+    This will generate a RecordLoader named MyThingLoader, a rake task to
+    trigger it, specs and example yaml files.
+
+    On first usage this will create:
+        lib/record_loader/application_record_loader.rb
+        lib/tasks/record_loader.rake
+
+    It will also create:
+      config/record_loader/my_things/default_records.yml
+      lib/record_loader/my_thing_loader.rb
+      lib/record_loader/tasks/record_loader/my_thing.rake
+      spec/data/record_loader/my_things/two_entry_example.yml
+      spec/lib/record_loader/my_thing_loader_spec.rb

data/lib/generators/record_loader/record_loader_generator.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+# Rails generator to automatically build record loaders
+# @see lib/generators/record_loader/USAGE
+class RecordLoaderGenerator < Rails::Generators::NamedBase
+  source_root File.expand_path('templates', __dir__)
+
+  class_option :record_class, type: :string, default: nil,
+                              desc: 'The name of the ActiveRecord class which will be created. eg. User'
+  class_option :record_key, type: :string, default: 'name',
+                            desc: 'The unique attribute by which record will be identified'
+
+  # Builds all templates in the templates directory
+  def create_directories
+    directory '.', './'
+  end
+
+  # Generally this will only run on generating the first loader
+  def initial_setup
+    # Copy across the application_record_loader.rb unless it already exists
+    copy_file '../static_files/application_record_loader.rb',
+              'lib/record_loader/application_record_loader.rb',
+              skip: true
+    # Copy across the record_loader.rake unless it already exists
+    copy_file '../static_files/record_loader.rake',
+              'lib/tasks/record_loader.rake',
+              skip: true
+  end
+
+  private
+
+  def loader_class_name
+    "#{name.camelcase}Loader"
+  end
+
+  def record_class
+    options['record_class'] || name.camelcase
+  end
+
+  def record_key
+    options['record_key']
+  end
+
+  def underscore_loader
+    "#{name.underscore}_loader"
+  end
+
+  def underscore
+    name.underscore
+  end
+
+  def underscores
+    underscore.pluralize
+  end
+end

data/lib/generators/record_loader/static_files/application_record_loader.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# This file was automatically generated by `rails g record_loader`
+module RecordLoader
+  # This forms the standard base class for record loaders in your
+  # application, allowing for easy configuration.
+  # @see https://rubydoc.info/github/sanger/record_loader/
+  class ApplicationRecordLoader < RecordLoader::Base
+    # Uses the standard RailsAdapter
+    # @see https://rubydoc.info/github/sanger/record_loader/RecordLoader/Adapter
+    adapter RecordLoader::Adapter::Rails.new
+  end
+end

data/lib/generators/record_loader/static_files/record_loader.rake

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# This file was automatically generated by `rails g record_loader`
+# It is not necessary to modify this file, instead individual tasks
+# will register themselves as a dependency.
+namespace :record_loader do
+  desc 'Automatically runs all record loader tasks'
+  task :all do
+    puts 'Loading all records'
+  end
+end

data/lib/generators/record_loader/templates/config/record_loader/%underscores%/default_records.yml

@@ -0,0 +1,11 @@
+---
+# Add entries to this file to generate them automatically upon running
+# rake record_loader:all or rake record_loader:<%= underscore %>
+# If you want entries which are only created in development mode then simply
+# create a file in this directory ending in '.dev.yml'
+# You can add as many additional .yml files to this directory as you wish.
+# Example:
+# <%= record_key %> A:
+#   other_attribute: 'Value'
+# <%= record_key %> B:
+#   other_attribute: 'Value'

data/lib/generators/record_loader/templates/lib/record_loader/%underscore_loader%.rb.tt

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+# This file was automatically generated via `rails g record_loader`
+
+# RecordLoader handles automatic population and updating of database records
+# across different environments
+# @see https://rubydoc.info/github/sanger/record_loader/
+module RecordLoader
+  # Creates the specified plate types if they are not present
+  class <%= loader_class_name %> < ApplicationRecordLoader
+    config_folder '<%= underscores %>'
+
+    def create_or_update!(<%= record_key %>, options)
+      <%= record_class %>.create_with(options).find_or_create_by!(<%= record_key %>: <%= record_key %>)
+    end
+  end
+end

data/lib/generators/record_loader/templates/lib/tasks/record_loader/%underscore%.rake.tt

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+# This file was automatically generated via `rails g record_loader`
+namespace :record_loader do
+  desc 'Automatically generate <%= record_class %> through <%= loader_class_name %>'
+  task <%= underscore %>: :environment do
+    RecordLoader::<%= loader_class_name %>.new.create!
+  end
+end
+
+# Automatically run this record loader as part of record_loader:all
+# Remove this line if the task should only run when invoked explicitly
+task 'record_loader:all' => 'record_loader:<%= underscore %>'

data/lib/generators/record_loader/templates/spec/data/record_loader/%underscores%/two_entry_example.yml

@@ -0,0 +1,5 @@
+---
+# This file was automatically generated via `rails g record_loader`
+# You should modify it to match your particular model.
+Unique attribute: {}
+Unique attribute 2: {}