sequel-data-migrate 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 16896608fde97a13bbe21af4275a06a3229f57c215719b2a569a9eb974ebdf89
+  data.tar.gz: 453e7f4a1afd9ed0bbd9ed3398858b366f81285b57ee5dd3e6decd552c2fd852
+SHA512:
+  metadata.gz: eac60cb9d571d250198ed6476b4a049af93652f5a00fc5b84ab02e534d4e0a78f60dad9b703d5d3dbf02c5e606d53400193c5b65460d83573a4dac81ebca5f54
+  data.tar.gz: e420debfc7d44f438ee5a5aa7a2447a952ebc02da7dadb2e1f5960d2cae103be93c68ba4d20b441319b5a2f3fbc970ccae5890dc647c2b5eba5596be5908f94c

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,28 @@
+require:
+  - rubocop-rspec
+
+AllCops:
+  TargetRubyVersion: 2.7
+  NewCops: enable
+
+Layout/LineLength:
+  Max: 120
+
+Metrics/AbcSize:
+  Max: 50
+Metrics/MethodLength:
+  Max: 20
+
+RSpec/ExampleLength:
+  Max: 20
+RSpec/MultipleExpectations:
+  Enabled: false
+
+Style/Documentation:
+  Enabled: false
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes

data/.ruby-version

@@ -0,0 +1 @@
+3.1.2

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-04-11
+
+- Initial release

data/Gemfile

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in sequel-data-migrate.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+gem "rbs"
+gem "sqlite3"
+
+gem "rspec"
+
+gem "rubocop"
+gem "rubocop-rspec"
+
+gem "byebug"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Hieu Nguyen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+# SequelData::Migrate
+
+Run data migrations along side schema migrations for Sequel. Inspired by
+https://github.com/ilyakatz/data-migrate.
+
+Data migrations are stored in db/data. They act like schema migrations,
+except they should be reserved for data migrations.
+For instance, if you realize you need to titleize all your titles, this is the place to do it.
+
+The code is ported from Sequel migration extension.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add sequel-data-migrate
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install sequel-data-migrate
+
+## Usage
+
+Configure your database:
+
+```ruby
+SequelData::Migrate.configure do |config|
+  config.db_configuration.host = ENV.fetch('DATABASE_URL')
+  config.migration_path = 'db/data_migration' # default value is db/data
+end
+```
+
+Add the rake task to `Rakelib` file in your project:
+
+```ruby
+require 'sequel_data/migrate/rake_task'
+task 'data:migrate' => :environment
+task 'data:rollback' => :environment
+```
+
+Now you can migrate using `rake db:migrate` and rollback using `rake
+db:rollback`.
+
+To create new migration file, you can use `data:create_migration` rake task:
+
+```shell
+bundle exec rake data:create_migration\[<file_name>\]
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies.
+Then, run `rake spec` to run the tests.
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+To release a new version, update the version number in `version.rb`,
+and then run `bundle exec rake release`, which will create a git tag for the version,
+push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/kaligo/sequel-data-migrate.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/lib/sequel_data/migrate/errors.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module SequelData
+  module Migrate
+    class Error < StandardError; end
+
+    class ConfigurationError < Error; end
+
+    class MigrationError < Error; end
+  end
+end

data/lib/sequel_data/migrate/generator.rb

@@ -0,0 +1,37 @@
+require 'fileutils'
+
+module SequelData
+  module Migrate
+    class Generator
+      VERSION_FORMAT = "%Y%m%d%H%M%S"
+
+      def initialize(config)
+        @config = config
+      end
+
+      def create_migration(name)
+        version = Time.now.utc.strftime(VERSION_FORMAT)
+        file_name = "#{version}_#{name}.rb"
+        klass_name = name.split("_").map(&:capitalize).join
+        content = <<~CONTENT
+          class #{klass_name} < SequelData::Migrate::Migration
+            def up
+            end
+
+            def down
+            end
+          end
+        CONTENT
+
+        File.join(config.migration_path, file_name).tap do |full_path|
+          FileUtils.mkdir_p(config.migration_path)
+          File.write(full_path, content)
+        end
+      end
+
+      private
+
+      attr_reader :config
+    end
+  end
+end

data/lib/sequel_data/migrate/migration.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module SequelData
+  module Migrate
+    class Migration
+      VALID_DIRECTION = %i[up down].freeze
+
+      # Returns the list of Migration descendants.
+      def self.descendants
+        @descendants ||= []
+      end
+
+      # Adds the new migration class to the list of Migration descendants.
+      def self.inherited(base)
+        descendants << base
+        super
+      end
+
+      def up
+        raise MigrationError, "No up method defined for migration #{self.class}"
+      end
+
+      def down
+        raise MigrationError, "No down method defined for migration #{self.class}"
+      end
+    end
+  end
+end

data/lib/sequel_data/migrate/migrator.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "sequel"
+require "sequel_data/migrate/migration"
+
+module SequelData
+  module Migrate
+    class Migrator
+      DATE_TIME_PATTERN = "[0-9]" * 14 # YYYYMMDDHHMMSS
+      FILE_NAME_PATTERN = "#{DATE_TIME_PATTERN}_*.rb"
+      MUTEX = Mutex.new
+
+      def initialize(config)
+        @config = config
+      end
+
+      def migrate
+        db = connect_database
+        dataset = ensure_table_exists(db)
+
+        already_migrated = dataset.select_map(column).to_set
+        migration_files = fetch_migration_files.reject { |file| already_migrated.include?(File.basename(file)) }.sort
+        migrations = fetch_migrations(migration_files)
+
+        migrations.zip(migration_files).each do |migration, file|
+          db.log_info("Begin applying migration file #{file}")
+          migration.new.up
+          set_migration_version(db, file)
+          db.log_info("Finished applying migration version #{file}")
+        end
+      end
+
+      def rollback(step = 1)
+        db = connect_database
+        dataset = ensure_table_exists(db)
+
+        already_migrated = dataset.select_map(column).to_set
+        migration_files = fetch_migration_files.select do |file|
+          already_migrated.include?(File.basename(file))
+        end.sort.reverse!
+        migrations = fetch_migrations(migration_files)
+
+        migrations.zip(migration_files).each do |migration, file|
+          step -= 1
+          break if step.negative?
+
+          db.log_info("Begin rolling back migration file #{file}")
+          migration.new.down
+          remove_migration_version(db, file)
+          db.log_info("Finished rolling back migration version #{file}")
+        end
+      end
+
+      private
+
+      attr_reader :config
+
+      def column
+        :version
+      end
+
+      def table
+        :data_migrations
+      end
+
+      def connect_database
+        raise ConfigurationError, "db_configuration is not set" if config.db_configuration.host.nil?
+
+        Sequel.connect(config.db_configuration.host)
+      end
+
+      def ensure_table_exists(db)
+        # we need to set this so that it can be used within create_table? block
+        c = column
+
+        db.from(table).tap do |dataset|
+          db.create_table?(table) do
+            String c, null: false, primary: true
+          end
+
+          unless dataset.columns.include?(c)
+            db.alter_table(table) do
+              add_column c, String, null: false, primary: true
+            end
+          end
+        end
+      end
+
+      def fetch_migration_files
+        Dir.glob("#{config.migration_path}/#{FILE_NAME_PATTERN}")
+      end
+
+      # Load the migration file, raising an exception if the file does not define
+      # a single migration.
+      def load_migration_file(file)
+        MUTEX.synchronize do
+          length = Migration.descendants.length
+
+          load(file)
+
+          if length != Migration.descendants.length - 1
+            raise MigrationError, "Migration file #{file.inspect} not containing a single migration detected"
+          end
+
+          klass = Migration.descendants.pop
+          if klass.is_a?(Class) && !klass.name.to_s.empty? && Object.const_defined?(klass.name)
+            Object.send(:remove_const, klass.name)
+          end
+
+          klass
+        end
+      end
+
+      # Returns a list of migration classes filtered for the migration range and
+      # ordered according to the migration direction.
+      def fetch_migrations(migration_files)
+        migration_files.map { |file| load_migration_file(file) }
+      end
+
+      def set_migration_version(db, file)
+        db.from(table).insert(column => File.basename(file))
+      end
+
+      def remove_migration_version(db, file)
+        db.from(table).where(column => File.basename(file)).delete
+      end
+    end
+  end
+end

data/lib/sequel_data/migrate/rake_task.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require "rake"
+load "sequel_data/migrate/tasks/data.rake"

data/lib/sequel_data/migrate/tasks/data.rake

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "sequel_data/migrate"
+
+namespace :data do
+  desc "Run data migrations"
+  task :migrate do
+    SequelData::Migrate.migrate
+  end
+
+  desc "Rollback data migrations"
+  task :rollback, %i[step] do |_, args|
+    SequelData::Migrate.rollback(args.fetch(:step, 1).to_i)
+  end
+
+  desc "Create a migration (parameters: NAME)"
+  task :create_migration, %i[name] do |_, args|
+    name = args[:name]
+
+    if name.nil?
+      puts "No NAME specified. Example usage:
+        `rake db:create_migration[backfill_users_email]`"
+      exit
+    end
+
+    path = SequelData::Migrate.create_migration(name)
+
+    puts "<= migration file created #{path}"
+  end
+end

data/lib/sequel_data/migrate/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module SequelData
+  module Migrate
+    VERSION = "0.1.0"
+  end
+end

data/lib/sequel_data/migrate.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "dry-configurable"
+require "sequel_data/migrate/version"
+require "sequel_data/migrate/errors"
+require "sequel_data/migrate/migrator"
+require "sequel_data/migrate/generator"
+
+module SequelData
+  module Migrate
+    extend Dry::Configurable
+
+    setting :db_configuration do
+      setting :host
+    end
+    setting :migration_path, default: "db/data"
+
+    def self.migrate
+      Migrator.new(config).migrate
+    end
+
+    def self.rollback(step = 1)
+      Migrator.new(config).rollback(step)
+    end
+
+    def self.create_migration(name)
+      Generator.new(config).create_migration(name)
+    end
+  end
+end

data/sig/sequel_data/migrate.rbs

@@ -0,0 +1,66 @@
+# TypeProf 0.21.2
+
+# Global variables
+$trace: false
+
+# Classes
+module SequelData
+  module Migrate
+    VERSION: String
+
+    def self.migrate: -> untyped
+    def self.rollback: (Integer? step) -> untyped
+    def self.create_migration: (String name) -> untyped
+
+    class Error < StandardError
+    end
+
+    class ConfigurationError < Error
+    end
+
+    class MigrationError < Error
+    end
+
+    class Generator
+      VERSION_FORMAT: String
+
+      def initialize: (untyped config) -> void
+      def create_migration: (String name) -> String
+
+      private
+      attr_reader config: untyped
+    end
+
+    class Migration
+      VALID_DIRECTION: [:up, :down]
+      self.@descendants: Array[untyped]
+
+      def self.descendants: -> Array[untyped]
+      def self.inherited: (untyped base) -> untyped
+      def up: -> untyped
+      def down: -> untyped
+    end
+
+    class Migrator
+      DATE_TIME_PATTERN: String
+      FILE_NAME_PATTERN: String
+      MUTEX: Thread::Mutex
+
+      def initialize: (untyped config) -> void
+      def migrate: -> untyped
+      def rollback: (?Integer step) -> untyped
+
+      private
+      attr_reader config: untyped
+      def column: -> :version
+      def table: -> :data_migrations
+      def connect_database: -> untyped
+      def ensure_table_exists: (untyped db) -> untyped
+      def fetch_migration_files: -> Array[String]
+      def load_migration_file: (String file) -> untyped
+      def fetch_migrations: (Array[String] migration_files) -> Array[untyped]
+      def set_migration_version: (untyped db, String? file) -> untyped
+      def remove_migration_version: (untyped db, String? file) -> untyped
+    end
+  end
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: sequel-data-migrate
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hieu Nguyen
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-04-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: dry-configurable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.13.0
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 4.0.0
+description: Migrate data with Sequel, based on Sequel migration extension
+email:
+- hieu.nguyen@ascendaloyalty.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".rubocop.yml"
+- ".ruby-version"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/sequel_data/migrate.rb
+- lib/sequel_data/migrate/errors.rb
+- lib/sequel_data/migrate/generator.rb
+- lib/sequel_data/migrate/migration.rb
+- lib/sequel_data/migrate/migrator.rb
+- lib/sequel_data/migrate/rake_task.rb
+- lib/sequel_data/migrate/tasks/data.rake
+- lib/sequel_data/migrate/version.rb
+- sig/sequel_data/migrate.rbs
+homepage: https://github.com/kaligo/sequel-data-migrate
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/kaligo/sequel-data-migrate
+  changelog_uri: https://github.com/kaligo/sequel-data-migrate/blob/master/CHANGELOG.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.7.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Migrate data with Sequel
+test_files: []