backhoe 0.10.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 77326a186f91c5bf3d2f47475d05bb64922b1df6039a8ee3d151779fafde9c85
4
- data.tar.gz: ce066cade2bb20be3d146ac30086116a7780619b8ecef4edb3a40f677b0f0857
3
+ metadata.gz: 1c3388209c28de44224013396cb7014a6353f046f3571bbbe14c9583e29c43c6
4
+ data.tar.gz: e19e932a0835eec574edd2d59b5c9b830a65f16acbab692aadad4fd2bea85c57
5
5
  SHA512:
6
- metadata.gz: e8f17f5ce5699cc9d4c8fd6eab4ab93881cfb6d01e723a3f7383c7506a483a29b8b170f0ff23c528e25f4b9f21b5d0b547208c5390a35403cbf7e0c7901ce5a9
7
- data.tar.gz: 6352b47008307ecd5f300c2f577a2ab1e92c4164af9f97ae9aefd7b03ab53f60161cc4c9c29b29bd8bef40fecf1f7ead833157838e3cf81f9532b83d2b837b7e
6
+ metadata.gz: 28348bb09ea2d40d4e2df95bb10f518602061b2be76b7301f9cfe43d67a4649259b0871dbd971243ab2887c56cc587498670d39fb4555b66744ad5ee32bc7532
7
+ data.tar.gz: b495b45a9bdcecf5f8d99458b984355360f555ee01a380c0d7b230619e60cf29d56206dbace91141383fc4e2b1cd0aa33ca2b8530947cfe8604886b9b7b4a196
@@ -5,10 +5,10 @@ jobs:
5
5
  strategy:
6
6
  fail-fast: false
7
7
  matrix:
8
- gemfile: [ activerecord_7.0, activerecord_7.1, activerecore_7.2 ]
9
- ruby: [ 3.1, 3.2, 3.3 ]
8
+ gemfile: [ activerecord_8.1 ]
9
+ ruby: [ 3.3, 3.4, "4.0" ]
10
10
 
11
- runs-on: ubuntu-22.04
11
+ runs-on: ubuntu-24.04
12
12
  env: # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps
13
13
  BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile
14
14
  POSTGRES_HOST_AUTH_METHOD: trust
data/Appraisals CHANGED
@@ -1,12 +1,3 @@
1
- appraise "activerecord-7.0" do
2
- gem "activerecord", "~>7.0.0"
1
+ appraise "activerecord-8.1" do
2
+ gem "activerecord", "~>8.1.0"
3
3
  end
4
-
5
- appraise "activerecord-7.1" do
6
- gem "activerecord", "~>7.1.0"
7
- end
8
-
9
- appraise "activerecord-7.2" do
10
- gem "activerecord", "~>7.2.0"
11
- end
12
-
data/CLAUDE.md ADDED
@@ -0,0 +1,36 @@
1
+ # CLAUDE.md
2
+
3
+ This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
4
+
5
+ ## Project
6
+
7
+ Backhoe is a Ruby gem for dumping and loading ActiveRecord database contents to/from files. Supports MySQL, PostgreSQL, and SQLite adapters with gzip compression and remote HTTP PUT.
8
+
9
+ ## Commands
10
+
11
+ ```bash
12
+ bundle exec rake # Run full test suite (RSpec)
13
+ bundle exec rspec # Run all specs
14
+ bundle exec rspec spec/backhoe/dump_spec.rb # Run a single spec file
15
+ bundle exec rspec spec/backhoe/dump_spec.rb:15 # Run a single example by line
16
+ ```
17
+
18
+ Tests require running MySQL and PostgreSQL services. Test database config is in `spec/support/database.yml`.
19
+
20
+ ## Architecture
21
+
22
+ The gem has two public methods: `Backhoe.dump(path, **options)` and `Backhoe.load(path, **options)`.
23
+
24
+ Three core classes implement the functionality:
25
+
26
+ - **Backhoe::Database** (`lib/backhoe/database.rb`) - Reads ActiveRecord connection config, detects the adapter type (`mysql?`, `postgresql?`, `sqlite?`), and builds CLI options for database tools.
27
+ - **Backhoe::Dump** (`lib/backhoe/dump.rb`) - Runs `mysqldump`, `pg_dump`, or file copy depending on adapter. Handles gzip, table skipping, column sanitization (via temp tables), and HTTP PUT for remote storage. Uses `Rake::DSL#sh` for shell commands.
28
+ - **Backhoe::Load** (`lib/backhoe/load.rb`) - Runs `mysql`, `psql`, or file copy to restore. Supports gzip decompression and optional drop/create database.
29
+
30
+ Shell commands use `set -o pipefail` in bash to catch piping errors.
31
+
32
+ ## Testing
33
+
34
+ The test suite uses Appraisals to test against ActiveRecord 7.0, 7.1, and 7.2. CI runs on GitHub Actions across Ruby 3.1–3.3 with that matrix.
35
+
36
+ Test support files in `spec/support/` include a `Database` helper class for creating/destroying test databases and fixture SQL files for both MySQL and PostgreSQL.
data/README.md CHANGED
@@ -10,6 +10,7 @@ Dump and load current ActiveRecord database to and from a file.
10
10
  Backhoe.dump "data.sql" # dumps db to db/data.sql
11
11
  Backhoe.dump "data.sql.gz" # => can also dump a gzipped sql file
12
12
  Backhoe.dump "data.sql", skip_tables: [:comments], skip_columns: { users: [:password] } # can skip whole tables or just specific columns
13
+ Backhoe.dump "data.sql", scrub_columns: { users: { email: "CONCAT('user', id, '@example.com')", phone: nil } } # or keep columns but overwrite them with SQL expressions (nil for NULL)
13
14
 
14
15
  # Load
15
16
  Backhoe.load "data.sql" # loads db from db/data.sql
data/backhoe.gemspec CHANGED
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
28
28
  spec.add_development_dependency "appraisal"
29
29
  spec.add_development_dependency "mysql2"
30
30
  spec.add_development_dependency "pg"
31
+ spec.add_development_dependency "sqlite3"
31
32
  spec.add_development_dependency "byebug"
32
33
  spec.add_development_dependency "timecop"
33
34
  spec.add_development_dependency "webrick"
@@ -2,6 +2,6 @@
2
2
 
3
3
  source "https://rubygems.org"
4
4
 
5
- gem "activerecord", "~>7.0.0"
5
+ gem "activerecord", "~>8.1.0"
6
6
 
7
7
  gemspec path: "../"
@@ -38,6 +38,15 @@ module Backhoe
38
38
  %w[mysql2 trilogy].include?(config["adapter"])
39
39
  end
40
40
 
41
+ def sqlite?
42
+ config["adapter"] == "sqlite3"
43
+ end
44
+
45
+ def path
46
+ raise unless sqlite?
47
+ config["database"]
48
+ end
49
+
41
50
  private
42
51
 
43
52
  def load_config
data/lib/backhoe/dump.rb CHANGED
@@ -1,37 +1,40 @@
1
1
  require "rake"
2
+ require "shellwords"
2
3
 
3
4
  module Backhoe
4
- class Dump < Struct.new(:database, :path, :skip_tables, :skip_columns)
5
+ class Dump < Struct.new(:database, :path, :skip_tables, :skip_columns, :scrub_columns)
5
6
  include Rake::DSL
6
7
 
7
8
  def initialize *args
8
9
  super
9
10
  self.skip_tables ||= []
10
11
  self.skip_columns ||= {}
12
+ self.scrub_columns ||= {}
11
13
  end
12
14
 
13
15
  def call
14
- if skip_tables.any?
16
+ if skip_tables.any? || skip_columns.any? || scrub_columns.any?
15
17
  raise NotImplementedError if database.postgresql?
16
18
  end
17
- if skip_columns.any?
18
- raise NotImplementedError if database.postgresql?
19
- SanitizedDatabase.new(skip_columns, path).dump do |tables|
20
- self.skip_tables += tables
21
- dump
19
+ if skip_columns.any? || scrub_columns.any?
20
+ SanitizedDatabase.new(skip_columns, scrub_columns).dump do |tables, trailer|
21
+ dump skip_tables + tables, trailer
22
22
  end
23
23
  else
24
- dump
24
+ dump skip_tables
25
25
  end
26
26
  end
27
27
 
28
28
  private
29
29
 
30
- def dump
30
+ def dump skipped, trailer = nil
31
31
  if database.mysql?
32
- bash_sh "#{mysqldump} --no-create-db --single-transaction --quick -e #{skip_table_options} #{database.to_mysql_options} #{database.name} | #{pipe} #{target}"
32
+ trailer = "; printf '\\n%s' #{Shellwords.escape(trailer)}" if trailer
33
+ bash_sh "{ #{mysqldump} --no-create-db --single-transaction --quick -e #{skip_table_options(skipped)} #{database.to_mysql_options} #{database.name}#{trailer}; } | #{pipe} #{target}"
33
34
  elsif database.postgresql?
34
35
  bash_sh "#{pg_dump} --column-inserts #{database.name} | #{pipe} #{target}"
36
+ elsif database.sqlite?
37
+ bash_sh "cat #{database.path} | #{pipe} #{target}"
35
38
  else
36
39
  raise "don't know how to dump #{database.adapter}"
37
40
  end
@@ -71,53 +74,54 @@ module Backhoe
71
74
  path =~ /\.gz\b/ ? "gzip -9f" : "cat"
72
75
  end
73
76
 
74
- def skip_table_options
75
- skip_tables.map do |table|
77
+ def skip_table_options skipped
78
+ skipped.map do |table|
76
79
  "--ignore-table=#{database.name}.#{table}"
77
80
  end.join(" ")
78
81
  end
79
82
 
80
- class SanitizedDatabase < Struct.new(:config, :path)
83
+ # Copies each table aside, strips the copy, dumps it in place of the original, and
84
+ # renames it back at the end of the dump.
85
+ class SanitizedDatabase
86
+ def initialize skip_columns, scrub_columns
87
+ @skip_columns = skip_columns.to_h { |table, columns| [table.to_s, columns] }
88
+ @scrub_columns = scrub_columns.to_h { |table, columns| [table.to_s, columns] }
89
+ end
90
+
81
91
  def dump
82
- with_sanitized_tables do
83
- yield skip_tables
84
- end
85
- skip_tables.each do |table|
86
- File.write path, "RENAME TABLE `sanitized_#{table}` TO `#{table}`;", mode: "a"
87
- end
92
+ tables.each { |table| copy table }
93
+ yield tables, tables.map { |table| "RENAME TABLE `sanitized_#{table}` TO `#{table}`;" }.join("\n")
94
+ ensure
95
+ tables.each { |table| sql "DROP TABLE IF EXISTS `sanitized_#{table}`" }
88
96
  end
89
97
 
90
98
  private
91
99
 
92
- def skip_tables
93
- config.keys
100
+ def tables
101
+ (@skip_columns.keys + @scrub_columns.keys).uniq
94
102
  end
95
103
 
96
- def with_sanitized_tables
97
- ActiveRecord::Base.transaction do
98
- config.each do |table, columns|
99
- sanitized_table = "sanitized_#{table}"
100
- sql <<-SQL
101
- DROP TABLE IF EXISTS `#{sanitized_table}`;
102
- CREATE TABLE `#{sanitized_table}` LIKE `#{table}`;
103
- INSERT INTO `#{sanitized_table}` SELECT * FROM `#{table}`;
104
- ALTER TABLE `#{sanitized_table}` #{columns.map { |column| "DROP `#{column}`" }.join(", ")};
105
- SQL
106
- end
107
-
108
- yield
109
-
110
- config.each do |table, _|
111
- sql "DROP TABLE `sanitized_#{table}`"
112
- end
104
+ def copy table
105
+ copy = "sanitized_#{table}"
106
+ sql "DROP TABLE IF EXISTS `#{copy}`"
107
+ sql "CREATE TABLE `#{copy}` LIKE `#{table}`"
108
+ sql "INSERT INTO `#{copy}` SELECT * FROM `#{table}`"
109
+
110
+ scrubs = @scrub_columns.fetch(table, {})
111
+ if scrubs.any?
112
+ sets = scrubs.map { |column, expression| "`#{column}` = #{expression.nil? ? "NULL" : expression}" }
113
+ sql "UPDATE `#{copy}` SET #{sets.join(", ")}"
113
114
  end
114
- end
115
115
 
116
- def sql queries
117
- queries.split(";").select(&:present?).each do |query|
118
- ActiveRecord::Base.connection.execute query
116
+ skips = @skip_columns.fetch(table, [])
117
+ if skips.any?
118
+ sql "ALTER TABLE `#{copy}` #{skips.map { |column| "DROP `#{column}`" }.join(", ")}"
119
119
  end
120
120
  end
121
+
122
+ def sql query
123
+ ActiveRecord::Base.connection.execute query
124
+ end
121
125
  end
122
126
  end
123
127
  end
data/lib/backhoe/load.rb CHANGED
@@ -5,11 +5,12 @@ module Backhoe
5
5
  include Rake::DSL
6
6
 
7
7
  def call
8
- case database.adapter
9
- when "mysql2", "trilogy"
8
+ if database.mysql?
10
9
  sh mysql_command
11
- when "postgresql"
10
+ elsif database.postgresql?
12
11
  sh psql_command
12
+ elsif database.sqlite?
13
+ sh sqlite_command
13
14
  else
14
15
  raise "don't know how to load #{database.adapter}"
15
16
  end
@@ -35,6 +36,10 @@ module Backhoe
35
36
  cmd
36
37
  end
37
38
 
39
+ def sqlite_command
40
+ "#{cat} #{path} > #{database.path}"
41
+ end
42
+
38
43
  def cat
39
44
  path =~ /\.gz$/ ? "zcat" : "cat"
40
45
  end
@@ -1,3 +1,3 @@
1
1
  module Backhoe
2
- VERSION = "0.10.1"
2
+ VERSION = "0.12.0"
3
3
  end
data/lib/backhoe.rb CHANGED
@@ -6,8 +6,8 @@ require "active_record"
6
6
 
7
7
  module Backhoe
8
8
  class << self
9
- def dump path, skip_tables: [], skip_columns: {}
10
- Dump.new(Database.new, path, skip_tables, skip_columns).call
9
+ def dump path, skip_tables: [], skip_columns: {}, scrub_columns: {}
10
+ Dump.new(Database.new, path, skip_tables, skip_columns, scrub_columns).call
11
11
  end
12
12
 
13
13
  def load path, drop_and_create: false
metadata CHANGED
@@ -1,14 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: backhoe
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.1
4
+ version: 0.12.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Micah Geisel
8
- autorequire:
9
8
  bindir: exe
10
9
  cert_chain: []
11
- date: 2024-09-26 00:00:00.000000000 Z
10
+ date: 1980-01-02 00:00:00.000000000 Z
12
11
  dependencies:
13
12
  - !ruby/object:Gem::Dependency
14
13
  name: activerecord
@@ -94,6 +93,20 @@ dependencies:
94
93
  - - ">="
95
94
  - !ruby/object:Gem::Version
96
95
  version: '0'
96
+ - !ruby/object:Gem::Dependency
97
+ name: sqlite3
98
+ requirement: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - ">="
101
+ - !ruby/object:Gem::Version
102
+ version: '0'
103
+ type: :development
104
+ prerelease: false
105
+ version_requirements: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - ">="
108
+ - !ruby/object:Gem::Version
109
+ version: '0'
97
110
  - !ruby/object:Gem::Dependency
98
111
  name: byebug
99
112
  requirement: !ruby/object:Gem::Requirement
@@ -147,6 +160,7 @@ files:
147
160
  - ".gitignore"
148
161
  - ".rspec"
149
162
  - Appraisals
163
+ - CLAUDE.md
150
164
  - Gemfile
151
165
  - LICENSE.txt
152
166
  - README.md
@@ -155,9 +169,7 @@ files:
155
169
  - bin/console
156
170
  - bin/setup
157
171
  - gemfiles/.bundle/config
158
- - gemfiles/activerecord_7.0.gemfile
159
- - gemfiles/activerecord_7.1.gemfile
160
- - gemfiles/activerecord_7.2.gemfile
172
+ - gemfiles/activerecord_8.1.gemfile
161
173
  - lib/backhoe.rb
162
174
  - lib/backhoe/database.rb
163
175
  - lib/backhoe/dump.rb
@@ -167,7 +179,6 @@ homepage: https://github.com/botandrose/backhoe
167
179
  licenses:
168
180
  - MIT
169
181
  metadata: {}
170
- post_install_message:
171
182
  rdoc_options: []
172
183
  require_paths:
173
184
  - lib
@@ -182,8 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
182
193
  - !ruby/object:Gem::Version
183
194
  version: '0'
184
195
  requirements: []
185
- rubygems_version: 3.5.1
186
- signing_key:
196
+ rubygems_version: 4.0.20
187
197
  specification_version: 4
188
198
  summary: Dump and load current database to and from a file.
189
199
  test_files: []
@@ -1,7 +0,0 @@
1
- # This file was generated by Appraisal
2
-
3
- source "https://rubygems.org"
4
-
5
- gem "activerecord", "~>7.1.0"
6
-
7
- gemspec path: "../"
@@ -1,7 +0,0 @@
1
- # This file was generated by Appraisal
2
-
3
- source "https://rubygems.org"
4
-
5
- gem "activerecord", "~>7.2.0"
6
-
7
- gemspec path: "../"