backhoe 0.10.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -3
- data/Appraisals +2 -11
- data/CLAUDE.md +36 -0
- data/README.md +1 -0
- data/backhoe.gemspec +1 -0
- data/gemfiles/{activerecord_7.0.gemfile → activerecord_8.1.gemfile} +1 -1
- data/lib/backhoe/database.rb +9 -0
- data/lib/backhoe/dump.rb +46 -42
- data/lib/backhoe/load.rb +8 -3
- data/lib/backhoe/version.rb +1 -1
- data/lib/backhoe.rb +2 -2
- metadata +19 -9
- data/gemfiles/activerecord_7.1.gemfile +0 -7
- data/gemfiles/activerecord_7.2.gemfile +0 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 1c3388209c28de44224013396cb7014a6353f046f3571bbbe14c9583e29c43c6
|
|
4
|
+
data.tar.gz: e19e932a0835eec574edd2d59b5c9b830a65f16acbab692aadad4fd2bea85c57
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 28348bb09ea2d40d4e2df95bb10f518602061b2be76b7301f9cfe43d67a4649259b0871dbd971243ab2887c56cc587498670d39fb4555b66744ad5ee32bc7532
|
|
7
|
+
data.tar.gz: b495b45a9bdcecf5f8d99458b984355360f555ee01a380c0d7b230619e60cf29d56206dbace91141383fc4e2b1cd0aa33ca2b8530947cfe8604886b9b7b4a196
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -5,10 +5,10 @@ jobs:
|
|
|
5
5
|
strategy:
|
|
6
6
|
fail-fast: false
|
|
7
7
|
matrix:
|
|
8
|
-
gemfile: [
|
|
9
|
-
ruby: [ 3.
|
|
8
|
+
gemfile: [ activerecord_8.1 ]
|
|
9
|
+
ruby: [ 3.3, 3.4, "4.0" ]
|
|
10
10
|
|
|
11
|
-
runs-on: ubuntu-
|
|
11
|
+
runs-on: ubuntu-24.04
|
|
12
12
|
env: # $BUNDLE_GEMFILE must be set at the job level, so it is set for all steps
|
|
13
13
|
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/${{ matrix.gemfile }}.gemfile
|
|
14
14
|
POSTGRES_HOST_AUTH_METHOD: trust
|
data/Appraisals
CHANGED
|
@@ -1,12 +1,3 @@
|
|
|
1
|
-
appraise "activerecord-
|
|
2
|
-
gem "activerecord", "~>
|
|
1
|
+
appraise "activerecord-8.1" do
|
|
2
|
+
gem "activerecord", "~>8.1.0"
|
|
3
3
|
end
|
|
4
|
-
|
|
5
|
-
appraise "activerecord-7.1" do
|
|
6
|
-
gem "activerecord", "~>7.1.0"
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
appraise "activerecord-7.2" do
|
|
10
|
-
gem "activerecord", "~>7.2.0"
|
|
11
|
-
end
|
|
12
|
-
|
data/CLAUDE.md
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# CLAUDE.md
|
|
2
|
+
|
|
3
|
+
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
|
4
|
+
|
|
5
|
+
## Project
|
|
6
|
+
|
|
7
|
+
Backhoe is a Ruby gem for dumping and loading ActiveRecord database contents to/from files. Supports MySQL, PostgreSQL, and SQLite adapters with gzip compression and remote HTTP PUT.
|
|
8
|
+
|
|
9
|
+
## Commands
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
bundle exec rake # Run full test suite (RSpec)
|
|
13
|
+
bundle exec rspec # Run all specs
|
|
14
|
+
bundle exec rspec spec/backhoe/dump_spec.rb # Run a single spec file
|
|
15
|
+
bundle exec rspec spec/backhoe/dump_spec.rb:15 # Run a single example by line
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Tests require running MySQL and PostgreSQL services. Test database config is in `spec/support/database.yml`.
|
|
19
|
+
|
|
20
|
+
## Architecture
|
|
21
|
+
|
|
22
|
+
The gem has two public methods: `Backhoe.dump(path, **options)` and `Backhoe.load(path, **options)`.
|
|
23
|
+
|
|
24
|
+
Three core classes implement the functionality:
|
|
25
|
+
|
|
26
|
+
- **Backhoe::Database** (`lib/backhoe/database.rb`) - Reads ActiveRecord connection config, detects the adapter type (`mysql?`, `postgresql?`, `sqlite?`), and builds CLI options for database tools.
|
|
27
|
+
- **Backhoe::Dump** (`lib/backhoe/dump.rb`) - Runs `mysqldump`, `pg_dump`, or file copy depending on adapter. Handles gzip, table skipping, column sanitization (via temp tables), and HTTP PUT for remote storage. Uses `Rake::DSL#sh` for shell commands.
|
|
28
|
+
- **Backhoe::Load** (`lib/backhoe/load.rb`) - Runs `mysql`, `psql`, or file copy to restore. Supports gzip decompression and optional drop/create database.
|
|
29
|
+
|
|
30
|
+
Shell commands use `set -o pipefail` in bash to catch piping errors.
|
|
31
|
+
|
|
32
|
+
## Testing
|
|
33
|
+
|
|
34
|
+
The test suite uses Appraisals to test against ActiveRecord 7.0, 7.1, and 7.2. CI runs on GitHub Actions across Ruby 3.1–3.3 with that matrix.
|
|
35
|
+
|
|
36
|
+
Test support files in `spec/support/` include a `Database` helper class for creating/destroying test databases and fixture SQL files for both MySQL and PostgreSQL.
|
data/README.md
CHANGED
|
@@ -10,6 +10,7 @@ Dump and load current ActiveRecord database to and from a file.
|
|
|
10
10
|
Backhoe.dump "data.sql" # dumps db to db/data.sql
|
|
11
11
|
Backhoe.dump "data.sql.gz" # => can also dump a gzipped sql file
|
|
12
12
|
Backhoe.dump "data.sql", skip_tables: [:comments], skip_columns: { users: [:password] } # can skip whole tables or just specific columns
|
|
13
|
+
Backhoe.dump "data.sql", scrub_columns: { users: { email: "CONCAT('user', id, '@example.com')", phone: nil } } # or keep columns but overwrite them with SQL expressions (nil for NULL)
|
|
13
14
|
|
|
14
15
|
# Load
|
|
15
16
|
Backhoe.load "data.sql" # loads db from db/data.sql
|
data/backhoe.gemspec
CHANGED
|
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
|
|
|
28
28
|
spec.add_development_dependency "appraisal"
|
|
29
29
|
spec.add_development_dependency "mysql2"
|
|
30
30
|
spec.add_development_dependency "pg"
|
|
31
|
+
spec.add_development_dependency "sqlite3"
|
|
31
32
|
spec.add_development_dependency "byebug"
|
|
32
33
|
spec.add_development_dependency "timecop"
|
|
33
34
|
spec.add_development_dependency "webrick"
|
data/lib/backhoe/database.rb
CHANGED
data/lib/backhoe/dump.rb
CHANGED
|
@@ -1,37 +1,40 @@
|
|
|
1
1
|
require "rake"
|
|
2
|
+
require "shellwords"
|
|
2
3
|
|
|
3
4
|
module Backhoe
|
|
4
|
-
class Dump < Struct.new(:database, :path, :skip_tables, :skip_columns)
|
|
5
|
+
class Dump < Struct.new(:database, :path, :skip_tables, :skip_columns, :scrub_columns)
|
|
5
6
|
include Rake::DSL
|
|
6
7
|
|
|
7
8
|
def initialize *args
|
|
8
9
|
super
|
|
9
10
|
self.skip_tables ||= []
|
|
10
11
|
self.skip_columns ||= {}
|
|
12
|
+
self.scrub_columns ||= {}
|
|
11
13
|
end
|
|
12
14
|
|
|
13
15
|
def call
|
|
14
|
-
if skip_tables.any?
|
|
16
|
+
if skip_tables.any? || skip_columns.any? || scrub_columns.any?
|
|
15
17
|
raise NotImplementedError if database.postgresql?
|
|
16
18
|
end
|
|
17
|
-
if skip_columns.any?
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
self.skip_tables += tables
|
|
21
|
-
dump
|
|
19
|
+
if skip_columns.any? || scrub_columns.any?
|
|
20
|
+
SanitizedDatabase.new(skip_columns, scrub_columns).dump do |tables, trailer|
|
|
21
|
+
dump skip_tables + tables, trailer
|
|
22
22
|
end
|
|
23
23
|
else
|
|
24
|
-
dump
|
|
24
|
+
dump skip_tables
|
|
25
25
|
end
|
|
26
26
|
end
|
|
27
27
|
|
|
28
28
|
private
|
|
29
29
|
|
|
30
|
-
def dump
|
|
30
|
+
def dump skipped, trailer = nil
|
|
31
31
|
if database.mysql?
|
|
32
|
-
|
|
32
|
+
trailer = "; printf '\\n%s' #{Shellwords.escape(trailer)}" if trailer
|
|
33
|
+
bash_sh "{ #{mysqldump} --no-create-db --single-transaction --quick -e #{skip_table_options(skipped)} #{database.to_mysql_options} #{database.name}#{trailer}; } | #{pipe} #{target}"
|
|
33
34
|
elsif database.postgresql?
|
|
34
35
|
bash_sh "#{pg_dump} --column-inserts #{database.name} | #{pipe} #{target}"
|
|
36
|
+
elsif database.sqlite?
|
|
37
|
+
bash_sh "cat #{database.path} | #{pipe} #{target}"
|
|
35
38
|
else
|
|
36
39
|
raise "don't know how to dump #{database.adapter}"
|
|
37
40
|
end
|
|
@@ -71,53 +74,54 @@ module Backhoe
|
|
|
71
74
|
path =~ /\.gz\b/ ? "gzip -9f" : "cat"
|
|
72
75
|
end
|
|
73
76
|
|
|
74
|
-
def skip_table_options
|
|
75
|
-
|
|
77
|
+
def skip_table_options skipped
|
|
78
|
+
skipped.map do |table|
|
|
76
79
|
"--ignore-table=#{database.name}.#{table}"
|
|
77
80
|
end.join(" ")
|
|
78
81
|
end
|
|
79
82
|
|
|
80
|
-
|
|
83
|
+
# Copies each table aside, strips the copy, dumps it in place of the original, and
|
|
84
|
+
# renames it back at the end of the dump.
|
|
85
|
+
class SanitizedDatabase
|
|
86
|
+
def initialize skip_columns, scrub_columns
|
|
87
|
+
@skip_columns = skip_columns.to_h { |table, columns| [table.to_s, columns] }
|
|
88
|
+
@scrub_columns = scrub_columns.to_h { |table, columns| [table.to_s, columns] }
|
|
89
|
+
end
|
|
90
|
+
|
|
81
91
|
def dump
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
File.write path, "RENAME TABLE `sanitized_#{table}` TO `#{table}`;", mode: "a"
|
|
87
|
-
end
|
|
92
|
+
tables.each { |table| copy table }
|
|
93
|
+
yield tables, tables.map { |table| "RENAME TABLE `sanitized_#{table}` TO `#{table}`;" }.join("\n")
|
|
94
|
+
ensure
|
|
95
|
+
tables.each { |table| sql "DROP TABLE IF EXISTS `sanitized_#{table}`" }
|
|
88
96
|
end
|
|
89
97
|
|
|
90
98
|
private
|
|
91
99
|
|
|
92
|
-
def
|
|
93
|
-
|
|
100
|
+
def tables
|
|
101
|
+
(@skip_columns.keys + @scrub_columns.keys).uniq
|
|
94
102
|
end
|
|
95
103
|
|
|
96
|
-
def
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
yield
|
|
109
|
-
|
|
110
|
-
config.each do |table, _|
|
|
111
|
-
sql "DROP TABLE `sanitized_#{table}`"
|
|
112
|
-
end
|
|
104
|
+
def copy table
|
|
105
|
+
copy = "sanitized_#{table}"
|
|
106
|
+
sql "DROP TABLE IF EXISTS `#{copy}`"
|
|
107
|
+
sql "CREATE TABLE `#{copy}` LIKE `#{table}`"
|
|
108
|
+
sql "INSERT INTO `#{copy}` SELECT * FROM `#{table}`"
|
|
109
|
+
|
|
110
|
+
scrubs = @scrub_columns.fetch(table, {})
|
|
111
|
+
if scrubs.any?
|
|
112
|
+
sets = scrubs.map { |column, expression| "`#{column}` = #{expression.nil? ? "NULL" : expression}" }
|
|
113
|
+
sql "UPDATE `#{copy}` SET #{sets.join(", ")}"
|
|
113
114
|
end
|
|
114
|
-
end
|
|
115
115
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
116
|
+
skips = @skip_columns.fetch(table, [])
|
|
117
|
+
if skips.any?
|
|
118
|
+
sql "ALTER TABLE `#{copy}` #{skips.map { |column| "DROP `#{column}`" }.join(", ")}"
|
|
119
119
|
end
|
|
120
120
|
end
|
|
121
|
+
|
|
122
|
+
def sql query
|
|
123
|
+
ActiveRecord::Base.connection.execute query
|
|
124
|
+
end
|
|
121
125
|
end
|
|
122
126
|
end
|
|
123
127
|
end
|
data/lib/backhoe/load.rb
CHANGED
|
@@ -5,11 +5,12 @@ module Backhoe
|
|
|
5
5
|
include Rake::DSL
|
|
6
6
|
|
|
7
7
|
def call
|
|
8
|
-
|
|
9
|
-
when "mysql2", "trilogy"
|
|
8
|
+
if database.mysql?
|
|
10
9
|
sh mysql_command
|
|
11
|
-
|
|
10
|
+
elsif database.postgresql?
|
|
12
11
|
sh psql_command
|
|
12
|
+
elsif database.sqlite?
|
|
13
|
+
sh sqlite_command
|
|
13
14
|
else
|
|
14
15
|
raise "don't know how to load #{database.adapter}"
|
|
15
16
|
end
|
|
@@ -35,6 +36,10 @@ module Backhoe
|
|
|
35
36
|
cmd
|
|
36
37
|
end
|
|
37
38
|
|
|
39
|
+
def sqlite_command
|
|
40
|
+
"#{cat} #{path} > #{database.path}"
|
|
41
|
+
end
|
|
42
|
+
|
|
38
43
|
def cat
|
|
39
44
|
path =~ /\.gz$/ ? "zcat" : "cat"
|
|
40
45
|
end
|
data/lib/backhoe/version.rb
CHANGED
data/lib/backhoe.rb
CHANGED
|
@@ -6,8 +6,8 @@ require "active_record"
|
|
|
6
6
|
|
|
7
7
|
module Backhoe
|
|
8
8
|
class << self
|
|
9
|
-
def dump path, skip_tables: [], skip_columns: {}
|
|
10
|
-
Dump.new(Database.new, path, skip_tables, skip_columns).call
|
|
9
|
+
def dump path, skip_tables: [], skip_columns: {}, scrub_columns: {}
|
|
10
|
+
Dump.new(Database.new, path, skip_tables, skip_columns, scrub_columns).call
|
|
11
11
|
end
|
|
12
12
|
|
|
13
13
|
def load path, drop_and_create: false
|
metadata
CHANGED
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: backhoe
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.12.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Micah Geisel
|
|
8
|
-
autorequire:
|
|
9
8
|
bindir: exe
|
|
10
9
|
cert_chain: []
|
|
11
|
-
date:
|
|
10
|
+
date: 1980-01-02 00:00:00.000000000 Z
|
|
12
11
|
dependencies:
|
|
13
12
|
- !ruby/object:Gem::Dependency
|
|
14
13
|
name: activerecord
|
|
@@ -94,6 +93,20 @@ dependencies:
|
|
|
94
93
|
- - ">="
|
|
95
94
|
- !ruby/object:Gem::Version
|
|
96
95
|
version: '0'
|
|
96
|
+
- !ruby/object:Gem::Dependency
|
|
97
|
+
name: sqlite3
|
|
98
|
+
requirement: !ruby/object:Gem::Requirement
|
|
99
|
+
requirements:
|
|
100
|
+
- - ">="
|
|
101
|
+
- !ruby/object:Gem::Version
|
|
102
|
+
version: '0'
|
|
103
|
+
type: :development
|
|
104
|
+
prerelease: false
|
|
105
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
106
|
+
requirements:
|
|
107
|
+
- - ">="
|
|
108
|
+
- !ruby/object:Gem::Version
|
|
109
|
+
version: '0'
|
|
97
110
|
- !ruby/object:Gem::Dependency
|
|
98
111
|
name: byebug
|
|
99
112
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -147,6 +160,7 @@ files:
|
|
|
147
160
|
- ".gitignore"
|
|
148
161
|
- ".rspec"
|
|
149
162
|
- Appraisals
|
|
163
|
+
- CLAUDE.md
|
|
150
164
|
- Gemfile
|
|
151
165
|
- LICENSE.txt
|
|
152
166
|
- README.md
|
|
@@ -155,9 +169,7 @@ files:
|
|
|
155
169
|
- bin/console
|
|
156
170
|
- bin/setup
|
|
157
171
|
- gemfiles/.bundle/config
|
|
158
|
-
- gemfiles/
|
|
159
|
-
- gemfiles/activerecord_7.1.gemfile
|
|
160
|
-
- gemfiles/activerecord_7.2.gemfile
|
|
172
|
+
- gemfiles/activerecord_8.1.gemfile
|
|
161
173
|
- lib/backhoe.rb
|
|
162
174
|
- lib/backhoe/database.rb
|
|
163
175
|
- lib/backhoe/dump.rb
|
|
@@ -167,7 +179,6 @@ homepage: https://github.com/botandrose/backhoe
|
|
|
167
179
|
licenses:
|
|
168
180
|
- MIT
|
|
169
181
|
metadata: {}
|
|
170
|
-
post_install_message:
|
|
171
182
|
rdoc_options: []
|
|
172
183
|
require_paths:
|
|
173
184
|
- lib
|
|
@@ -182,8 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
182
193
|
- !ruby/object:Gem::Version
|
|
183
194
|
version: '0'
|
|
184
195
|
requirements: []
|
|
185
|
-
rubygems_version:
|
|
186
|
-
signing_key:
|
|
196
|
+
rubygems_version: 4.0.20
|
|
187
197
|
specification_version: 4
|
|
188
198
|
summary: Dump and load current database to and from a file.
|
|
189
199
|
test_files: []
|