data-migration 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c310b238ffce7232c8d7e043b3bb05c22c7fc9feb1a202653172152e0eafe54f
+  data.tar.gz: c177a8be4fb4805a0cdac0683ea2b1711f8e78df14176f271d834869c9d7eb0f
+SHA512:
+  metadata.gz: 5c3653642986c3de1ac483875541c433f44fc604fe648dcf3b22a9065b16d0370b83c6a55ad20c7a41ed78dee646c14f69eacd9cf2f3f3fa49f7e9b8e944fb44
+  data.tar.gz: f604aa6ac2c1993c04b7f993eabf4f56f81c0db9b1c0a892e46f390a75fe4c11a0f8d237b2e80de190b497a07b68d0f7e3d08b162682245234120e1e40a6e58f

data/.editorconfig

@@ -0,0 +1,14 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+quote_type = double
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false

data/.github/dependabot.yml

@@ -0,0 +1,27 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: github-actions
+    directory: /
+    open-pull-requests-limit: 10
+    schedule:
+      interval: weekly
+  - package-ecosystem: bundler
+    directory: /
+    open-pull-requests-limit: 10
+    schedule:
+      interval: weekly
+  - package-ecosystem: npm
+    directory: /
+    open-pull-requests-limit: 10
+    schedule:
+      interval: weekly
+  - package-ecosystem: docker
+    directory: /
+    open-pull-requests-limit: 10
+    schedule:
+      interval: weekly

data/.github/workflows/_trunk_check.yml

@@ -0,0 +1,15 @@
+name: _trunk_check
+on: [pull_request]
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+permissions: read-all
+jobs:
+  run_trunk_action:
+    runs-on: ubuntu-latest
+    permissions:
+      checks: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - uses: trunk-io/trunk-action@v1

data/.github/workflows/test.yml

@@ -0,0 +1,42 @@
+name: test
+permissions: read-all
+on: [push, pull_request]
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: false
+jobs:
+  rspec:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        ruby: [ruby, jruby, truffleruby]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+      - run: bundle exec rspec
+  coverage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+      - run: bundle exec rspec
+      - uses: codecov/codecov-action@v5
+        continue-on-error: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+      - uses: codecov/test-results-action@v1
+        if: ${{ !cancelled() }}
+        continue-on-error: true
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: coverage/junit-coverage.xml
+      - uses: codacy/codacy-coverage-reporter-action@v1.3.0
+        continue-on-error: true
+        with:
+          project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
+          coverage-reports: coverage/coverage.xml

data/.gitignore

@@ -0,0 +1,24 @@
+# See https://help.github.com/articles/ignoring-files for more about ignoring files.
+#
+# If you find yourself ignoring temporary files generated by your text editor
+# or operating system, you probably want to add a global ignore instead:
+#   git config --global core.excludesfile '~/.gitignore_global'
+
+# Ignore bundler config.
+/.bundle
+/vendor/bundle
+
+.byebug_history
+
+/coverage
+/coverage/*
+
+/spec/examples.txt
+/tmp/*
+
+.DS_Store
+.idea
+.lh
+
+*.local
+*.gem

data/.ruby-version

@@ -0,0 +1 @@
+3.3.4

data/.trunk/.gitignore

@@ -0,0 +1,9 @@
+*out
+*logs
+*actions
+*notifications
+*tools
+plugins
+user_trunk.yaml
+user.yaml
+tmp

data/.trunk/configs/.markdownlint.yaml

@@ -0,0 +1,2 @@
+# Prettier friendly markdownlint config (all formatting rules disabled)
+extends: markdownlint/style/prettier

data/.trunk/configs/.shellcheckrc

@@ -0,0 +1,7 @@
+enable=all
+source-path=SCRIPTDIR
+disable=SC2154
+
+# If you're having issues with shellcheck following source, disable the errors via:
+# disable=SC1090
+# disable=SC1091

data/.trunk/configs/.yamllint.yaml

@@ -0,0 +1,7 @@
+rules:
+  quoted-strings:
+    required: only-when-needed
+    extra-allowed: ["{|}"]
+  key-duplicates: {}
+  octal-values:
+    forbid-implicit-octal: true

data/.trunk/trunk.yaml

@@ -0,0 +1,39 @@
+# This file controls the behavior of Trunk: https://docs.trunk.io/cli
+# To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
+version: 0.1
+cli:
+  version: 1.22.8
+# Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
+plugins:
+  sources:
+    - id: trunk
+      ref: v1.6.5
+      uri: https://github.com/trunk-io/plugins
+# Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
+runtimes:
+  enabled:
+    - ruby@3.1.4
+    - go@1.21.0
+    - node@18.12.1
+    - python@3.10.8
+# This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
+lint:
+  enabled:
+    - standardrb@1.3.0
+    - actionlint@1.7.4
+    - checkov@3.2.332
+    - git-diff-check
+    - markdownlint@0.43.0
+    - osv-scanner@1.9.1
+    - prettier@3.4.2
+    - shellcheck@0.10.0
+    - shfmt@3.6.0
+    - trufflehog@3.85.0
+    - yamllint@1.35.1
+actions:
+  disabled:
+    - trunk-announce
+    - trunk-check-pre-push
+    - trunk-fmt-pre-commit
+  enabled:
+    - trunk-upgrade-available

data/.vscode/extensions.json

@@ -0,0 +1,18 @@
+{
+  "recommendations": [
+    "donjayamanne.githistory",
+    "editorconfig.editorconfig",
+    "mikestead.dotenv",
+    "anweber.vscode-httpyac",
+    "shopify.ruby-lsp",
+    "bung87.vscode-gemfile",
+    "wayou.vscode-todo-highlight",
+    "vscode-icons-team.vscode-icons",
+    "koichisasada.vscode-rdbg",
+    "fooo.ruby-spec-runner",
+    "nhoizey.gremlins",
+    "streetsidesoftware.code-spell-checker",
+    "dewski.simplecov",
+    "trunk.io"
+  ]
+}

data/.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+  "editor.defaultFormatter": "trunk.io",
+  "editor.formatOnSave": false,
+  "editor.detectIndentation": false,
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true
+}

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.0
+
+- Initial version

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,246 @@
+PATH
+  remote: .
+  specs:
+    data-migration (1.0.0)
+      activejob (> 5)
+      activerecord (> 5)
+      activesupport (> 5)
+      rails (> 5)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    actioncable (8.0.0)
+      actionpack (= 8.0.0)
+      activesupport (= 8.0.0)
+      nio4r (~> 2.0)
+      websocket-driver (>= 0.6.1)
+      zeitwerk (~> 2.6)
+    actionmailbox (8.0.0)
+      actionpack (= 8.0.0)
+      activejob (= 8.0.0)
+      activerecord (= 8.0.0)
+      activestorage (= 8.0.0)
+      activesupport (= 8.0.0)
+      mail (>= 2.8.0)
+    actionmailer (8.0.0)
+      actionpack (= 8.0.0)
+      actionview (= 8.0.0)
+      activejob (= 8.0.0)
+      activesupport (= 8.0.0)
+      mail (>= 2.8.0)
+      rails-dom-testing (~> 2.2)
+    actionpack (8.0.0)
+      actionview (= 8.0.0)
+      activesupport (= 8.0.0)
+      nokogiri (>= 1.8.5)
+      rack (>= 2.2.4)
+      rack-session (>= 1.0.1)
+      rack-test (>= 0.6.3)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+      useragent (~> 0.16)
+    actiontext (8.0.0)
+      actionpack (= 8.0.0)
+      activerecord (= 8.0.0)
+      activestorage (= 8.0.0)
+      activesupport (= 8.0.0)
+      globalid (>= 0.6.0)
+      nokogiri (>= 1.8.5)
+    actionview (8.0.0)
+      activesupport (= 8.0.0)
+      builder (~> 3.1)
+      erubi (~> 1.11)
+      rails-dom-testing (~> 2.2)
+      rails-html-sanitizer (~> 1.6)
+    activejob (8.0.0)
+      activesupport (= 8.0.0)
+      globalid (>= 0.3.6)
+    activemodel (8.0.0)
+      activesupport (= 8.0.0)
+    activerecord (8.0.0)
+      activemodel (= 8.0.0)
+      activesupport (= 8.0.0)
+      timeout (>= 0.4.0)
+    activestorage (8.0.0)
+      actionpack (= 8.0.0)
+      activejob (= 8.0.0)
+      activerecord (= 8.0.0)
+      activesupport (= 8.0.0)
+      marcel (~> 1.0)
+    activesupport (8.0.0)
+      base64
+      benchmark (>= 0.3)
+      bigdecimal
+      concurrent-ruby (~> 1.0, >= 1.3.1)
+      connection_pool (>= 2.2.5)
+      drb
+      i18n (>= 1.6, < 2)
+      logger (>= 1.4.2)
+      minitest (>= 5.1)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
+    base64 (0.2.0)
+    benchmark (0.4.0)
+    bigdecimal (3.1.8)
+    bigdecimal (3.1.8-java)
+    builder (3.3.0)
+    concurrent-ruby (1.3.4)
+    connection_pool (2.4.1)
+    crass (1.0.6)
+    date (3.4.1)
+    date (3.4.1-java)
+    diff-lcs (1.5.1)
+    docile (1.4.1)
+    drb (2.2.1)
+    erubi (1.13.0)
+    globalid (1.2.1)
+      activesupport (>= 6.1)
+    i18n (1.14.6)
+      concurrent-ruby (~> 1.0)
+    io-console (0.8.0)
+    io-console (0.8.0-java)
+    irb (1.14.1)
+      rdoc (>= 4.0.0)
+      reline (>= 0.4.2)
+    jar-dependencies (0.5.0)
+    logger (1.6.2)
+    loofah (2.23.1)
+      crass (~> 1.0.2)
+      nokogiri (>= 1.12.0)
+    mail (2.8.1)
+      mini_mime (>= 0.1.1)
+      net-imap
+      net-pop
+      net-smtp
+    marcel (1.0.4)
+    mini_mime (1.1.5)
+    mini_portile2 (2.8.8)
+    minitest (5.25.4)
+    net-imap (0.5.1)
+      date
+      net-protocol
+    net-pop (0.1.2)
+      net-protocol
+    net-protocol (0.2.2)
+      timeout
+    net-smtp (0.5.0)
+      net-protocol
+    nio4r (2.7.4)
+    nio4r (2.7.4-java)
+    nokogiri (1.16.8-arm64-darwin)
+      racc (~> 1.4)
+    nokogiri (1.16.8-java)
+      racc (~> 1.4)
+    nokogiri (1.16.8-x86_64-linux)
+      racc (~> 1.4)
+    psych (5.2.1)
+      date
+      stringio
+    psych (5.2.1-java)
+      date
+      jar-dependencies (>= 0.1.7)
+    racc (1.8.1)
+    racc (1.8.1-java)
+    rack (3.1.8)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
+    rack-test (2.1.0)
+      rack (>= 1.3)
+    rackup (2.2.1)
+      rack (>= 3)
+    rails (8.0.0)
+      actioncable (= 8.0.0)
+      actionmailbox (= 8.0.0)
+      actionmailer (= 8.0.0)
+      actionpack (= 8.0.0)
+      actiontext (= 8.0.0)
+      actionview (= 8.0.0)
+      activejob (= 8.0.0)
+      activemodel (= 8.0.0)
+      activerecord (= 8.0.0)
+      activestorage (= 8.0.0)
+      activesupport (= 8.0.0)
+      bundler (>= 1.15.0)
+      railties (= 8.0.0)
+    rails-dom-testing (2.2.0)
+      activesupport (>= 5.0.0)
+      minitest
+      nokogiri (>= 1.6)
+    rails-html-sanitizer (1.6.1)
+      loofah (~> 2.21)
+      nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
+    railties (8.0.0)
+      actionpack (= 8.0.0)
+      activesupport (= 8.0.0)
+      irb (~> 1.13)
+      rackup (>= 1.0.0)
+      rake (>= 12.2)
+      thor (~> 1.0, >= 1.2.2)
+      zeitwerk (~> 2.6)
+    rake (13.2.1)
+    rdoc (6.8.1)
+      psych (>= 4.0.0)
+    reline (0.5.12)
+      io-console (~> 0.5)
+    rexml (3.3.9)
+    rspec (3.13.0)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.2)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.3)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.2)
+    rspec_junit_formatter (0.6.0)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    securerandom (0.4.0)
+    simplecov (0.22.0)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+      simplecov_json_formatter (~> 0.1)
+    simplecov-cobertura (2.1.0)
+      rexml
+      simplecov (~> 0.19)
+    simplecov-html (0.13.1)
+    simplecov_json_formatter (0.1.4)
+    sqlite3 (2.4.0)
+      mini_portile2 (~> 2.8.0)
+    sqlite3 (2.4.0-arm64-darwin)
+    sqlite3 (2.4.0-x86_64-linux-gnu)
+    stringio (3.1.2)
+    thor (1.3.2)
+    timeout (0.4.2)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    uri (1.0.2)
+    useragent (0.16.11)
+    websocket-driver (0.7.6)
+      websocket-extensions (>= 0.1.0)
+    websocket-driver (0.7.6-java)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.5)
+    zeitwerk (2.7.1)
+
+PLATFORMS
+  arm64-darwin
+  universal-java-11
+  x86_64-linux
+
+DEPENDENCIES
+  bundler (~> 2)
+  data-migration!
+  rspec (~> 3)
+  rspec_junit_formatter (~> 0.6)
+  simplecov (~> 0.21)
+  simplecov-cobertura (~> 2)
+  sqlite3 (~> 2.4)
+
+BUNDLED WITH
+   2.5.20

data/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 amkisko
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,141 @@
+# data-migration.rb
+
+[![Gem Version](https://badge.fury.io/rb/data-migration.svg)](https://badge.fury.io/rb/data-migration) [![Test Status](https://github.com/amkisko/data-migration.rb/actions/workflows/test.yml/badge.svg)](https://github.com/amkisko/data-migration.rb/actions/workflows/test.yml) [![codecov](https://codecov.io/gh/amkisko/data-migration.rb/graph/badge.svg?token=881AFPL643)](https://codecov.io/gh/amkisko/data-migration.rb)
+
+Data migrations kit for ActiveRecord and ActiveJob.
+
+Sponsored by [Kisko Labs](https://www.kiskolabs.com).
+
+## Data migrations concept
+
+- A short-living script that is manually applied to database
+- Not reversible
+- Can be applied multiple times
+- Accompanied by ActiveJob for background and batch operations
+- Accompanied by ActiveRecord to control and audit migrations progress
+- Operator's responsibility to ensure data consistency, notifications, monitoring and quality of implementation
+
+### Data migrations process
+
+1. Avoid implementing and running data migrations within schema migrations
+2. Data migrations should be planned beforehand, reserve time in the calendar
+3. Data migrations should be always controlled by operator
+4. Wrapping queries to transactions might lead to large memory consumption, unexpected exceptions and database unresponsiveness
+5. Large data migrations should have batching implemented which will lower memory consumption and database load
+6. Critical data migrations should be covered with tests, by finding consensus developers decide if migration is critical
+7. Before running critical data migrations, make sure that you have fresh backup of the database and you are ready to rollback in case of failure
+
+## Installation
+
+Using Bundler:
+
+```sh
+bundle add data-migration
+```
+
+Using RubyGems:
+
+```sh
+gem install data-migration
+```
+
+## Gemfile
+
+```ruby
+gem "data-migration"
+```
+
+## Usage
+
+### Run data migrations
+
+```sh
+bin/rails db:migrate:data 20241207120000_create_users
+```
+
+### Generate data migration job
+
+```sh
+bin/rails g data_migration create_users
+```
+
+## Configuration
+
+### Set data migrations directory
+
+Absolute path will be resolved by using `Rails.root`.
+
+```ruby
+DataMigration.config.data_migrations_path = "db/data_migrations"
+```
+
+### Turn off test script generation
+
+```ruby
+DataMigration.config.generate_spec = false
+```
+
+## Batch operations
+
+Batch operations are supported by using `enqueue` method, it will automatically enqueue or perform next job depending on `background` option.
+
+`enqueue` method calls are tracked within a single Thread, it should be used within a single job execution, also all `enqueue` calls rewrite each other and only last call will be used for enqueuing next job after the current job is completed.
+
+```ruby
+def perform(index: 1, background: true)
+  return if index > 2
+
+  User.find_or_create_by(email: "test_#{index}@example.com")
+
+  enqueue(index: index + 1, background:)
+end
+```
+
+## Specification checklist
+
+- [x] User can generate data migration file under `db/data_migrations` directory with common format
+- [x] User can generate data migration file with test script included
+- [x] User can run specific data migration using Rails console
+- [ ] User can run specific data migration using shell command
+- [x] User can run data migration in background
+- [x] User can run data migration in foreground
+- [x] User can specify operator for data migration
+- [x] User can specify monitoring context for data migration
+- [x] User can specify pause time for data migration
+- [x] User can specify jobs limit for data migration
+- [ ] User receives an error when data migration is applied within schema migration
+
+## Limitations & explanations
+
+- ActiveRecord migrations generator is used to generate data migration files
+- Data migrations are not reversible, it is operator's responsibility to ensure that data migration has correct effect
+- Keep migrations logic stable and predictable, e.g. by checking uniqueness of created/updated records
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at <https://github.com/amkisko/data-migration.rb>
+
+Contribution policy:
+
+- New features are not necessarily added to the gem
+- Pull request should have test coverage for affected parts
+- Pull request should have changelog entry
+- It might take up to 2 calendar weeks to review and merge critical fixes
+- It might take up to 6 calendar months to review and merge pull request
+- It might take up to 1 calendar year to review an issue
+
+## Publishing
+
+Prefer using script `usr/bin/release.sh`, it will ensure that repository is synced and after publishing gem will create a tag.
+
+```sh
+GEM_VERSION=$(grep -Eo "VERSION\s*=\s*\".+\"" lib/data_migration.rb  | grep -Eo "[0-9.]{5,}")
+rm data-migration-*.gem
+gem build data-migration.gemspec
+gem push data-migration-$GEM_VERSION.gem
+git tag $GEM_VERSION && git push --tags && gh release create $GEM_VERSION --generate-notes
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).