pgdump_scrambler 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +3 -0
- data/.rubocop.yml +2 -0
- data/.rubocop_todo.yml +173 -0
- data/.ruby-version +1 -0
- data/.travis.yml +5 -0
- data/.vscode/tasks.json +16 -0
- data/CHANGELOG.md +10 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +194 -0
- data/LICENSE.txt +21 -0
- data/README.md +79 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/pgdump-obfuscator +19 -0
- data/bin/setup +8 -0
- data/lib/config/table.rb +74 -0
- data/lib/pgdump_scrambler/config.rb +139 -0
- data/lib/pgdump_scrambler/dumper.rb +56 -0
- data/lib/pgdump_scrambler/railtie.rb +8 -0
- data/lib/pgdump_scrambler/s3_request.rb +106 -0
- data/lib/pgdump_scrambler/s3_uploader.rb +42 -0
- data/lib/pgdump_scrambler/version.rb +4 -0
- data/lib/pgdump_scrambler.rb +11 -0
- data/lib/tasks/pgdump_scrambler_tasks.rake +69 -0
- data/libexec/pgdump-obfuscator-darwin-amd64 +0 -0
- data/libexec/pgdump-obfuscator-darwin-arm64 +0 -0
- data/libexec/pgdump-obfuscator-linux-amd64 +0 -0
- data/libexec/pgdump-obfuscator-linux-arm64 +0 -0
- data/pgdump_scrambler.gemspec +37 -0
- metadata +130 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: '09085520af68283995aeb54b3ae886c303b2ec2a9123b7d76da9a201463bb0d9'
|
|
4
|
+
data.tar.gz: e7ee185201e95a18b84fa9b9eaafce6e98b0f0187125af647044ff4eb4789058
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 45cf30192e04b4af8f30d90aa753cd52b410f9d51597fd9f9eefbf050a6b15f16abd5c2c6ee3241e8261a1f63c22b4207f88e064e240523c371bec9db2a638b2
|
|
7
|
+
data.tar.gz: b7a5383078df617610d10eec93049f68194eea2aae21970752059dd632ff5b67b94e7a713a3701a57db3a8886991f86ec1c7d2a085b5d1a0ba97753fea99b52a
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.rubocop.yml
ADDED
data/.rubocop_todo.yml
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
# This configuration was generated by
|
|
2
|
+
# `rubocop --auto-gen-config`
|
|
3
|
+
# on 2018-03-08 00:50:23 +0900 using RuboCop version 0.52.1.
|
|
4
|
+
# The point is for the user to remove these configuration records
|
|
5
|
+
# one by one as the offenses are removed from the code base.
|
|
6
|
+
# Note that changes in the inspected code, or installation of new
|
|
7
|
+
# versions of RuboCop, may require this file to be generated again.
|
|
8
|
+
|
|
9
|
+
# Offense count: 1
|
|
10
|
+
# Cop supports --auto-correct.
|
|
11
|
+
# Configuration parameters: Include, TreatCommentsAsGroupSeparators.
|
|
12
|
+
# Include: **/*.gemspec
|
|
13
|
+
Gemspec/OrderedDependencies:
|
|
14
|
+
Exclude:
|
|
15
|
+
- 'pgdump_scrambler.gemspec'
|
|
16
|
+
|
|
17
|
+
# Offense count: 5
|
|
18
|
+
# Cop supports --auto-correct.
|
|
19
|
+
Layout/EmptyLineAfterMagicComment:
|
|
20
|
+
Exclude:
|
|
21
|
+
- 'lib/pgdump_scrambler.rb'
|
|
22
|
+
- 'lib/pgdump_scrambler/railtie.rb'
|
|
23
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
24
|
+
- 'lib/pgdump_scrambler/version.rb'
|
|
25
|
+
- 'lib/tasks/pgdump_scrambler_tasks.rake'
|
|
26
|
+
|
|
27
|
+
# Offense count: 1
|
|
28
|
+
# Cop supports --auto-correct.
|
|
29
|
+
# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
|
|
30
|
+
Layout/ExtraSpacing:
|
|
31
|
+
Exclude:
|
|
32
|
+
- 'pgdump_scrambler.gemspec'
|
|
33
|
+
|
|
34
|
+
# Offense count: 1
|
|
35
|
+
# Cop supports --auto-correct.
|
|
36
|
+
# Configuration parameters: EnforcedStyle.
|
|
37
|
+
# SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
|
|
38
|
+
Layout/IndentHeredoc:
|
|
39
|
+
Exclude:
|
|
40
|
+
- 'spec/pgdump_scrambler_spec.rb'
|
|
41
|
+
|
|
42
|
+
# Offense count: 1
|
|
43
|
+
# Cop supports --auto-correct.
|
|
44
|
+
# Configuration parameters: AllowForAlignment.
|
|
45
|
+
Layout/SpaceAroundOperators:
|
|
46
|
+
Exclude:
|
|
47
|
+
- 'pgdump_scrambler.gemspec'
|
|
48
|
+
|
|
49
|
+
# Offense count: 1
|
|
50
|
+
# Cop supports --auto-correct.
|
|
51
|
+
# Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces, SpaceBeforeBlockParameters.
|
|
52
|
+
# SupportedStyles: space, no_space
|
|
53
|
+
# SupportedStylesForEmptyBraces: space, no_space
|
|
54
|
+
Layout/SpaceInsideBlockBraces:
|
|
55
|
+
Exclude:
|
|
56
|
+
- 'Gemfile'
|
|
57
|
+
|
|
58
|
+
# Offense count: 2
|
|
59
|
+
# Cop supports --auto-correct.
|
|
60
|
+
# Configuration parameters: EnforcedStyle.
|
|
61
|
+
# SupportedStyles: final_newline, final_blank_line
|
|
62
|
+
Layout/TrailingBlankLines:
|
|
63
|
+
Exclude:
|
|
64
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
65
|
+
- 'lib/tasks/pgdump_scrambler_tasks.rake'
|
|
66
|
+
|
|
67
|
+
# Offense count: 4
|
|
68
|
+
# Cop supports --auto-correct.
|
|
69
|
+
Layout/TrailingWhitespace:
|
|
70
|
+
Exclude:
|
|
71
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
72
|
+
|
|
73
|
+
# Offense count: 1
|
|
74
|
+
# Configuration parameters: AllowSafeAssignment.
|
|
75
|
+
Lint/AssignmentInCondition:
|
|
76
|
+
Exclude:
|
|
77
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
78
|
+
|
|
79
|
+
# Offense count: 1
|
|
80
|
+
# Cop supports --auto-correct.
|
|
81
|
+
# Configuration parameters: IgnoreEmptyBlocks, AllowUnusedKeywordArguments.
|
|
82
|
+
Lint/UnusedBlockArgument:
|
|
83
|
+
Exclude:
|
|
84
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
85
|
+
|
|
86
|
+
# Offense count: 1
|
|
87
|
+
# Cop supports --auto-correct.
|
|
88
|
+
Security/YAMLLoad:
|
|
89
|
+
Exclude:
|
|
90
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
91
|
+
|
|
92
|
+
# Offense count: 5
|
|
93
|
+
Style/Documentation:
|
|
94
|
+
Exclude:
|
|
95
|
+
- 'spec/**/*'
|
|
96
|
+
- 'test/**/*'
|
|
97
|
+
- 'lib/pgdump_scrambler.rb'
|
|
98
|
+
- 'lib/pgdump_scrambler/railtie.rb'
|
|
99
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
100
|
+
|
|
101
|
+
# Offense count: 6
|
|
102
|
+
# Cop supports --auto-correct.
|
|
103
|
+
# Configuration parameters: EnforcedStyle.
|
|
104
|
+
# SupportedStyles: when_needed, always, never
|
|
105
|
+
Style/FrozenStringLiteralComment:
|
|
106
|
+
Exclude:
|
|
107
|
+
- 'Gemfile'
|
|
108
|
+
- 'Rakefile'
|
|
109
|
+
- 'bin/console'
|
|
110
|
+
- 'pgdump_scrambler.gemspec'
|
|
111
|
+
- 'spec/pgdump_scrambler_spec.rb'
|
|
112
|
+
- 'spec/spec_helper.rb'
|
|
113
|
+
|
|
114
|
+
# Offense count: 1
|
|
115
|
+
# Cop supports --auto-correct.
|
|
116
|
+
# Configuration parameters: EnforcedStyle, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
|
|
117
|
+
# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
|
|
118
|
+
Style/HashSyntax:
|
|
119
|
+
Exclude:
|
|
120
|
+
- 'Rakefile'
|
|
121
|
+
|
|
122
|
+
# Offense count: 1
|
|
123
|
+
# Cop supports --auto-correct.
|
|
124
|
+
Style/IfUnlessModifier:
|
|
125
|
+
Exclude:
|
|
126
|
+
- 'lib/pgdump_scrambler.rb'
|
|
127
|
+
|
|
128
|
+
# Offense count: 2
|
|
129
|
+
# Cop supports --auto-correct.
|
|
130
|
+
# Configuration parameters: PreferredDelimiters.
|
|
131
|
+
Style/PercentLiteralDelimiters:
|
|
132
|
+
Exclude:
|
|
133
|
+
- 'pgdump_scrambler.gemspec'
|
|
134
|
+
|
|
135
|
+
# Offense count: 1
|
|
136
|
+
# Cop supports --auto-correct.
|
|
137
|
+
Style/RescueModifier:
|
|
138
|
+
Exclude:
|
|
139
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
140
|
+
|
|
141
|
+
# Offense count: 32
|
|
142
|
+
# Cop supports --auto-correct.
|
|
143
|
+
# Configuration parameters: EnforcedStyle, ConsistentQuotesInMultiline.
|
|
144
|
+
# SupportedStyles: single_quotes, double_quotes
|
|
145
|
+
Style/StringLiterals:
|
|
146
|
+
Exclude:
|
|
147
|
+
- 'Gemfile'
|
|
148
|
+
- 'Rakefile'
|
|
149
|
+
- 'bin/console'
|
|
150
|
+
- 'lib/pgdump_scrambler.rb'
|
|
151
|
+
- 'lib/pgdump_scrambler/version.rb'
|
|
152
|
+
- 'pgdump_scrambler.gemspec'
|
|
153
|
+
- 'spec/spec_helper.rb'
|
|
154
|
+
|
|
155
|
+
# Offense count: 1
|
|
156
|
+
# Cop supports --auto-correct.
|
|
157
|
+
# Configuration parameters: EnforcedStyleForMultiline.
|
|
158
|
+
# SupportedStylesForMultiline: comma, consistent_comma, no_comma
|
|
159
|
+
Style/TrailingCommaInLiteral:
|
|
160
|
+
Exclude:
|
|
161
|
+
- 'lib/pgdump_scrambler/table.rb'
|
|
162
|
+
|
|
163
|
+
# Offense count: 2
|
|
164
|
+
# Cop supports --auto-correct.
|
|
165
|
+
Style/UnneededPercentQ:
|
|
166
|
+
Exclude:
|
|
167
|
+
- 'pgdump_scrambler.gemspec'
|
|
168
|
+
|
|
169
|
+
# Offense count: 11
|
|
170
|
+
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
|
171
|
+
# URISchemes: http, https
|
|
172
|
+
Metrics/LineLength:
|
|
173
|
+
Max: 96
|
data/.ruby-version
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.1.3
|
data/.travis.yml
ADDED
data/.vscode/tasks.json
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
// See https://go.microsoft.com/fwlink/?LinkId=733558
|
|
3
|
+
// for the documentation about the tasks.json format
|
|
4
|
+
"version": "2.0.0",
|
|
5
|
+
"tasks": [
|
|
6
|
+
{
|
|
7
|
+
"label": "rspec",
|
|
8
|
+
"type": "shell",
|
|
9
|
+
"command": "bundle exec rake",
|
|
10
|
+
"group": {
|
|
11
|
+
"kind": "build",
|
|
12
|
+
"isDefault": true
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
]
|
|
16
|
+
}
|
data/CHANGELOG.md
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: .
|
|
3
|
+
specs:
|
|
4
|
+
pgdump_scrambler (0.4.0)
|
|
5
|
+
|
|
6
|
+
GEM
|
|
7
|
+
remote: https://rubygems.org/
|
|
8
|
+
specs:
|
|
9
|
+
actioncable (7.0.4.3)
|
|
10
|
+
actionpack (= 7.0.4.3)
|
|
11
|
+
activesupport (= 7.0.4.3)
|
|
12
|
+
nio4r (~> 2.0)
|
|
13
|
+
websocket-driver (>= 0.6.1)
|
|
14
|
+
actionmailbox (7.0.4.3)
|
|
15
|
+
actionpack (= 7.0.4.3)
|
|
16
|
+
activejob (= 7.0.4.3)
|
|
17
|
+
activerecord (= 7.0.4.3)
|
|
18
|
+
activestorage (= 7.0.4.3)
|
|
19
|
+
activesupport (= 7.0.4.3)
|
|
20
|
+
mail (>= 2.7.1)
|
|
21
|
+
net-imap
|
|
22
|
+
net-pop
|
|
23
|
+
net-smtp
|
|
24
|
+
actionmailer (7.0.4.3)
|
|
25
|
+
actionpack (= 7.0.4.3)
|
|
26
|
+
actionview (= 7.0.4.3)
|
|
27
|
+
activejob (= 7.0.4.3)
|
|
28
|
+
activesupport (= 7.0.4.3)
|
|
29
|
+
mail (~> 2.5, >= 2.5.4)
|
|
30
|
+
net-imap
|
|
31
|
+
net-pop
|
|
32
|
+
net-smtp
|
|
33
|
+
rails-dom-testing (~> 2.0)
|
|
34
|
+
actionpack (7.0.4.3)
|
|
35
|
+
actionview (= 7.0.4.3)
|
|
36
|
+
activesupport (= 7.0.4.3)
|
|
37
|
+
rack (~> 2.0, >= 2.2.0)
|
|
38
|
+
rack-test (>= 0.6.3)
|
|
39
|
+
rails-dom-testing (~> 2.0)
|
|
40
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
|
41
|
+
actiontext (7.0.4.3)
|
|
42
|
+
actionpack (= 7.0.4.3)
|
|
43
|
+
activerecord (= 7.0.4.3)
|
|
44
|
+
activestorage (= 7.0.4.3)
|
|
45
|
+
activesupport (= 7.0.4.3)
|
|
46
|
+
globalid (>= 0.6.0)
|
|
47
|
+
nokogiri (>= 1.8.5)
|
|
48
|
+
actionview (7.0.4.3)
|
|
49
|
+
activesupport (= 7.0.4.3)
|
|
50
|
+
builder (~> 3.1)
|
|
51
|
+
erubi (~> 1.4)
|
|
52
|
+
rails-dom-testing (~> 2.0)
|
|
53
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
|
54
|
+
activejob (7.0.4.3)
|
|
55
|
+
activesupport (= 7.0.4.3)
|
|
56
|
+
globalid (>= 0.3.6)
|
|
57
|
+
activemodel (7.0.4.3)
|
|
58
|
+
activesupport (= 7.0.4.3)
|
|
59
|
+
activerecord (7.0.4.3)
|
|
60
|
+
activemodel (= 7.0.4.3)
|
|
61
|
+
activesupport (= 7.0.4.3)
|
|
62
|
+
activestorage (7.0.4.3)
|
|
63
|
+
actionpack (= 7.0.4.3)
|
|
64
|
+
activejob (= 7.0.4.3)
|
|
65
|
+
activerecord (= 7.0.4.3)
|
|
66
|
+
activesupport (= 7.0.4.3)
|
|
67
|
+
marcel (~> 1.0)
|
|
68
|
+
mini_mime (>= 1.1.0)
|
|
69
|
+
activesupport (7.0.4.3)
|
|
70
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
|
71
|
+
i18n (>= 1.6, < 2)
|
|
72
|
+
minitest (>= 5.1)
|
|
73
|
+
tzinfo (~> 2.0)
|
|
74
|
+
ast (2.4.2)
|
|
75
|
+
builder (3.2.4)
|
|
76
|
+
concurrent-ruby (1.2.2)
|
|
77
|
+
crass (1.0.6)
|
|
78
|
+
date (3.3.3)
|
|
79
|
+
diff-lcs (1.5.0)
|
|
80
|
+
erubi (1.12.0)
|
|
81
|
+
globalid (1.1.0)
|
|
82
|
+
activesupport (>= 5.0)
|
|
83
|
+
i18n (1.12.0)
|
|
84
|
+
concurrent-ruby (~> 1.0)
|
|
85
|
+
json (2.6.3)
|
|
86
|
+
loofah (2.20.0)
|
|
87
|
+
crass (~> 1.0.2)
|
|
88
|
+
nokogiri (>= 1.5.9)
|
|
89
|
+
mail (2.8.1)
|
|
90
|
+
mini_mime (>= 0.1.1)
|
|
91
|
+
net-imap
|
|
92
|
+
net-pop
|
|
93
|
+
net-smtp
|
|
94
|
+
marcel (1.0.2)
|
|
95
|
+
method_source (1.0.0)
|
|
96
|
+
mini_mime (1.1.2)
|
|
97
|
+
minitest (5.18.0)
|
|
98
|
+
net-imap (0.3.4)
|
|
99
|
+
date
|
|
100
|
+
net-protocol
|
|
101
|
+
net-pop (0.1.2)
|
|
102
|
+
net-protocol
|
|
103
|
+
net-protocol (0.2.1)
|
|
104
|
+
timeout
|
|
105
|
+
net-smtp (0.3.3)
|
|
106
|
+
net-protocol
|
|
107
|
+
nio4r (2.5.9)
|
|
108
|
+
nokogiri (1.14.3-x86_64-linux)
|
|
109
|
+
racc (~> 1.4)
|
|
110
|
+
parallel (1.22.1)
|
|
111
|
+
parser (3.2.2.0)
|
|
112
|
+
ast (~> 2.4.1)
|
|
113
|
+
racc (1.6.2)
|
|
114
|
+
rack (2.2.6.4)
|
|
115
|
+
rack-test (2.1.0)
|
|
116
|
+
rack (>= 1.3)
|
|
117
|
+
rails (7.0.4.3)
|
|
118
|
+
actioncable (= 7.0.4.3)
|
|
119
|
+
actionmailbox (= 7.0.4.3)
|
|
120
|
+
actionmailer (= 7.0.4.3)
|
|
121
|
+
actionpack (= 7.0.4.3)
|
|
122
|
+
actiontext (= 7.0.4.3)
|
|
123
|
+
actionview (= 7.0.4.3)
|
|
124
|
+
activejob (= 7.0.4.3)
|
|
125
|
+
activemodel (= 7.0.4.3)
|
|
126
|
+
activerecord (= 7.0.4.3)
|
|
127
|
+
activestorage (= 7.0.4.3)
|
|
128
|
+
activesupport (= 7.0.4.3)
|
|
129
|
+
bundler (>= 1.15.0)
|
|
130
|
+
railties (= 7.0.4.3)
|
|
131
|
+
rails-dom-testing (2.0.3)
|
|
132
|
+
activesupport (>= 4.2.0)
|
|
133
|
+
nokogiri (>= 1.6)
|
|
134
|
+
rails-html-sanitizer (1.5.0)
|
|
135
|
+
loofah (~> 2.19, >= 2.19.1)
|
|
136
|
+
railties (7.0.4.3)
|
|
137
|
+
actionpack (= 7.0.4.3)
|
|
138
|
+
activesupport (= 7.0.4.3)
|
|
139
|
+
method_source
|
|
140
|
+
rake (>= 12.2)
|
|
141
|
+
thor (~> 1.0)
|
|
142
|
+
zeitwerk (~> 2.5)
|
|
143
|
+
rainbow (3.1.1)
|
|
144
|
+
rake (13.0.6)
|
|
145
|
+
regexp_parser (2.7.0)
|
|
146
|
+
rexml (3.2.5)
|
|
147
|
+
rspec (3.12.0)
|
|
148
|
+
rspec-core (~> 3.12.0)
|
|
149
|
+
rspec-expectations (~> 3.12.0)
|
|
150
|
+
rspec-mocks (~> 3.12.0)
|
|
151
|
+
rspec-core (3.12.1)
|
|
152
|
+
rspec-support (~> 3.12.0)
|
|
153
|
+
rspec-expectations (3.12.2)
|
|
154
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
155
|
+
rspec-support (~> 3.12.0)
|
|
156
|
+
rspec-mocks (3.12.5)
|
|
157
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
158
|
+
rspec-support (~> 3.12.0)
|
|
159
|
+
rspec-support (3.12.0)
|
|
160
|
+
rubocop (1.50.1)
|
|
161
|
+
json (~> 2.3)
|
|
162
|
+
parallel (~> 1.10)
|
|
163
|
+
parser (>= 3.2.0.0)
|
|
164
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
165
|
+
regexp_parser (>= 1.8, < 3.0)
|
|
166
|
+
rexml (>= 3.2.5, < 4.0)
|
|
167
|
+
rubocop-ast (>= 1.28.0, < 2.0)
|
|
168
|
+
ruby-progressbar (~> 1.7)
|
|
169
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
|
170
|
+
rubocop-ast (1.28.0)
|
|
171
|
+
parser (>= 3.2.1.0)
|
|
172
|
+
ruby-progressbar (1.13.0)
|
|
173
|
+
thor (1.2.1)
|
|
174
|
+
timeout (0.3.2)
|
|
175
|
+
tzinfo (2.0.6)
|
|
176
|
+
concurrent-ruby (~> 1.0)
|
|
177
|
+
unicode-display_width (2.4.2)
|
|
178
|
+
websocket-driver (0.7.5)
|
|
179
|
+
websocket-extensions (>= 0.1.0)
|
|
180
|
+
websocket-extensions (0.1.5)
|
|
181
|
+
zeitwerk (2.6.7)
|
|
182
|
+
|
|
183
|
+
PLATFORMS
|
|
184
|
+
x86_64-linux
|
|
185
|
+
|
|
186
|
+
DEPENDENCIES
|
|
187
|
+
pgdump_scrambler!
|
|
188
|
+
rails (~> 7.0)
|
|
189
|
+
rake (~> 13.0)
|
|
190
|
+
rspec (~> 3.12)
|
|
191
|
+
rubocop
|
|
192
|
+
|
|
193
|
+
BUNDLED WITH
|
|
194
|
+
2.4.12
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2018 Shunichi Ikegami
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# PgdumpScrambler
|
|
2
|
+
|
|
3
|
+
Generate scrambled potgresql dump for rails application.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Add this line to your application's Gemfile:
|
|
8
|
+
|
|
9
|
+
```ruby
|
|
10
|
+
gem 'pgdump_scrambler', github: 'shunichi/pgdump_scrambler'
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
And then execute:
|
|
14
|
+
|
|
15
|
+
$ bundle
|
|
16
|
+
|
|
17
|
+
## Usage
|
|
18
|
+
|
|
19
|
+
Genarate config file.
|
|
20
|
+
|
|
21
|
+
```
|
|
22
|
+
bundle exec rake pgdump_scrambler:config_from_db
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Fix column scramble functions in config/pgdump_scrambler.yml
|
|
26
|
+
|
|
27
|
+
from:
|
|
28
|
+
|
|
29
|
+
```
|
|
30
|
+
tables:
|
|
31
|
+
users:
|
|
32
|
+
email: unspecified
|
|
33
|
+
name: unspecified
|
|
34
|
+
age: unspecified
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
to:
|
|
38
|
+
|
|
39
|
+
```
|
|
40
|
+
tables:
|
|
41
|
+
users:
|
|
42
|
+
email: uemail
|
|
43
|
+
name: sbytes
|
|
44
|
+
age: nop
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Dump the scrambled database.
|
|
48
|
+
|
|
49
|
+
```
|
|
50
|
+
bundle exec rake pgdump_scrambler:dump
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## scramble functions
|
|
54
|
+
|
|
55
|
+
- `bytes` random bytes (each byte is one of `0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ+-_`)
|
|
56
|
+
- `sbytes` random bytes (each byte is one of `0123456789abcdefghijklmnopqrstuvwxyz`)
|
|
57
|
+
- `digits` random digits
|
|
58
|
+
- `email` random email address
|
|
59
|
+
- `uemail` random unique email address
|
|
60
|
+
- `inet` random ip address
|
|
61
|
+
- `json` string value to random bytes, number value to random digits, keep data structure and key names
|
|
62
|
+
- `nullify` NULL
|
|
63
|
+
- `empty` empty string
|
|
64
|
+
- `const[VALUE]` constant value
|
|
65
|
+
- `nop` untouched
|
|
66
|
+
|
|
67
|
+
## Development
|
|
68
|
+
|
|
69
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
|
70
|
+
|
|
71
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
|
72
|
+
|
|
73
|
+
## Contributing
|
|
74
|
+
|
|
75
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/shunichi/pgdump_scrambler.
|
|
76
|
+
|
|
77
|
+
## License
|
|
78
|
+
|
|
79
|
+
The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "pgdump_scrambler"
|
|
5
|
+
|
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
|
8
|
+
|
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
|
10
|
+
# require "pry"
|
|
11
|
+
# Pry.start
|
|
12
|
+
|
|
13
|
+
require "irb"
|
|
14
|
+
IRB.start(__FILE__)
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
#!/usr/bin/env ruby
|
|
2
|
+
require 'shellwords'
|
|
3
|
+
|
|
4
|
+
arch = case RUBY_PLATFORM
|
|
5
|
+
when /aarch64-linux/
|
|
6
|
+
'linux-arm64'
|
|
7
|
+
when /x86_64-linux/
|
|
8
|
+
'linux-amd64'
|
|
9
|
+
when /x86_64-darwin/
|
|
10
|
+
'darwin-amd64'
|
|
11
|
+
when /arm64-darwin/
|
|
12
|
+
'darwin-arm64'
|
|
13
|
+
else
|
|
14
|
+
raise "Unsupported platform: #{RUBY_PLATFORM}"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
cmd = File.expand_path "#{File.dirname(__FILE__)}/../libexec/pgdump-obfuscator-#{arch}"
|
|
18
|
+
|
|
19
|
+
exec "#{cmd} #{Shellwords.join($*)}"
|
data/bin/setup
ADDED
data/lib/config/table.rb
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
module PgdumpScrambler
|
|
3
|
+
class Config
|
|
4
|
+
class Table
|
|
5
|
+
attr_reader :name
|
|
6
|
+
|
|
7
|
+
def initialize(name, columns)
|
|
8
|
+
@name = name
|
|
9
|
+
@column_hash = columns.sort_by(&:name).map { |column| [column.name, column] }.to_h
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def columns
|
|
13
|
+
@column_hash.values
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def [](column_name)
|
|
17
|
+
@column_hash[column_name]
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def update_with(other)
|
|
21
|
+
Table.new(name, other.column_hash.merge(@column_hash).values)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def options
|
|
25
|
+
columns.map(&:option).compact.map { |option| "-c #{name}:#{option}" }.join(' ')
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def unspecifiled_columns
|
|
29
|
+
@column_hash.map(&:second).select(&:unspecifiled?).map(&:name)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
protected
|
|
33
|
+
|
|
34
|
+
attr_reader :column_hash
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
class Column
|
|
38
|
+
SCRAMBLE_METHODS = %w[unspecified nop bytes sbytes digits email uemail inet json nullify empty].freeze
|
|
39
|
+
SCRAMBLE_CONST_REGEXP = /\Aconst\[.+\]\z/
|
|
40
|
+
NOP_METHODS = %w[unspecified nop].freeze
|
|
41
|
+
UNSPECIFIED = 'unspecified'
|
|
42
|
+
attr_reader :name
|
|
43
|
+
|
|
44
|
+
def initialize(name, scramble_method = UNSPECIFIED)
|
|
45
|
+
unless self.class.valid_scramble_method?(scramble_method)
|
|
46
|
+
raise ArgumentError, "invalid scramble_method: #{scramble_method}"
|
|
47
|
+
end
|
|
48
|
+
@name = name
|
|
49
|
+
@scramble_method = scramble_method
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def scramble_method
|
|
53
|
+
@scramble_method
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def unspecifiled?
|
|
57
|
+
@scramble_method == UNSPECIFIED
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def option
|
|
61
|
+
unless NOP_METHODS.member?(@scramble_method)
|
|
62
|
+
m = Shellwords.escape(scramble_method)
|
|
63
|
+
"#{@name}:#{m}"
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
class << self
|
|
68
|
+
def valid_scramble_method?(scramble_method)
|
|
69
|
+
SCRAMBLE_CONST_REGEXP.match?(scramble_method) || SCRAMBLE_METHODS.member?(scramble_method)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'yaml'
|
|
3
|
+
require 'erb'
|
|
4
|
+
require 'set'
|
|
5
|
+
require 'config/table'
|
|
6
|
+
|
|
7
|
+
module PgdumpScrambler
|
|
8
|
+
class Config
|
|
9
|
+
IGNORED_ACTIVE_RECORD_TABLES = %w[ar_internal_metadata schema_migrations].freeze
|
|
10
|
+
IGNORED_ACTIVE_RECORD_COLUMNS = %w[id created_at updated_at].to_set.freeze
|
|
11
|
+
KEY_DUMP_PATH = 'dump_path'
|
|
12
|
+
KEY_TABLES = 'tables'
|
|
13
|
+
KEY_EXCLUDE_TABLES = 'exclude_tables'
|
|
14
|
+
KEY_PGDUMP_ARGS = 'pgdump_args'
|
|
15
|
+
KEY_S3 = 's3'
|
|
16
|
+
DEFAULT_S3_PROPERTIES = {
|
|
17
|
+
'bucket' => 'YOUR_S3_BUCKET',
|
|
18
|
+
'region' => 'YOUR_S3_REGION',
|
|
19
|
+
'prefix' => 'YOUR_S3_PATH_PREFIX',
|
|
20
|
+
'access_key_id' => "<%= ENV['AWS_ACCESS_KEY_ID'] %>",
|
|
21
|
+
'secret_key' => "<%= ENV['AWS_SECRET_KEY'] %>"
|
|
22
|
+
}
|
|
23
|
+
attr_reader :dump_path, :s3, :resolved_s3, :exclude_tables, :pgdump_args
|
|
24
|
+
|
|
25
|
+
def initialize(tables, dump_path, s3, exclude_tables, pgdump_args)
|
|
26
|
+
@table_hash = tables.sort_by(&:name).map { |table| [table.name, table] }.to_h
|
|
27
|
+
@dump_path = dump_path
|
|
28
|
+
@s3 = s3
|
|
29
|
+
@resolved_s3 = s3.map { |k, v| [k, ERB.new(v).result] }.to_h if s3
|
|
30
|
+
@exclude_tables = exclude_tables
|
|
31
|
+
@pgdump_args = pgdump_args
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def table_names
|
|
35
|
+
@table_hash.keys
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def table(name)
|
|
39
|
+
@table_hash[name]
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def tables
|
|
43
|
+
@table_hash.values
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def update_with(other)
|
|
47
|
+
new_tables = @table_hash.map do |_, table|
|
|
48
|
+
if other_table = other.table(table.name)
|
|
49
|
+
table.update_with(other_table)
|
|
50
|
+
else
|
|
51
|
+
table
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
new_tables += (other.table_names - table_names).map { |table_name| other.table(table_name) }
|
|
55
|
+
Config.new(new_tables, @dump_path, @s3, @exclude_tables, @pgdump_args)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def unspecified_columns
|
|
59
|
+
@table_hash.map do |_, table|
|
|
60
|
+
columns = table.unspecifiled_columns
|
|
61
|
+
[table.name, columns] unless columns.empty?
|
|
62
|
+
end.compact.to_h
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def write(io)
|
|
66
|
+
yml = {}
|
|
67
|
+
yml[KEY_DUMP_PATH] = @dump_path
|
|
68
|
+
yml[KEY_S3] = @s3 if @s3
|
|
69
|
+
yml[KEY_EXCLUDE_TABLES] = @exclude_tables if @exclude_tables.size > 0
|
|
70
|
+
yml[KEY_TABLES] = @table_hash.map do |_, table|
|
|
71
|
+
columns = table.columns
|
|
72
|
+
unless columns.empty?
|
|
73
|
+
[
|
|
74
|
+
table.name,
|
|
75
|
+
columns.map { |column| [column.name, column.scramble_method] }.to_h,
|
|
76
|
+
]
|
|
77
|
+
end
|
|
78
|
+
end.compact.to_h
|
|
79
|
+
YAML.dump(yml, io)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def write_file(path)
|
|
83
|
+
File.open(path, 'w') do |io|
|
|
84
|
+
write(io)
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def obfuscator_options
|
|
89
|
+
tables.map(&:options).reject(&:empty?).join(' ')
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
class << self
|
|
93
|
+
def read(io)
|
|
94
|
+
yml = YAML.safe_load(io, permitted_classes: [], permitted_symbols: [], aliases: true)
|
|
95
|
+
if yml[KEY_TABLES]
|
|
96
|
+
tables = yml[KEY_TABLES].map do |table_name, columns|
|
|
97
|
+
Table.new(
|
|
98
|
+
table_name,
|
|
99
|
+
columns.map { |name, scramble_method| Column.new(name, scramble_method) }
|
|
100
|
+
)
|
|
101
|
+
end
|
|
102
|
+
else
|
|
103
|
+
tables = []
|
|
104
|
+
end
|
|
105
|
+
Config.new(tables, yml[KEY_DUMP_PATH], yml[KEY_S3], yml[KEY_EXCLUDE_TABLES] || [], yml[KEY_PGDUMP_ARGS])
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def read_file(path)
|
|
109
|
+
open(path, 'r') do |f|
|
|
110
|
+
read(f)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
if defined?(Rails)
|
|
115
|
+
def from_db
|
|
116
|
+
if defined?(Zeitwerk) && Rails.autoloaders.zeitwerk_enabled?
|
|
117
|
+
Zeitwerk::Loader.eager_load_all
|
|
118
|
+
else
|
|
119
|
+
Rails.application.eager_load!
|
|
120
|
+
end
|
|
121
|
+
klasses_by_table = ActiveRecord::Base.descendants.map { |klass| [klass.table_name, klass] }.to_h
|
|
122
|
+
table_names = ActiveRecord::Base.connection.tables.sort - IGNORED_ACTIVE_RECORD_TABLES
|
|
123
|
+
tables = table_names.map do |table_name|
|
|
124
|
+
klass = klasses_by_table[table_name]
|
|
125
|
+
if klass
|
|
126
|
+
columns = klass.columns.map(&:name).reject do |name|
|
|
127
|
+
IGNORED_ACTIVE_RECORD_COLUMNS.member?(name)
|
|
128
|
+
end.map do |name|
|
|
129
|
+
Column.new(name)
|
|
130
|
+
end
|
|
131
|
+
Table.new(table_name, columns)
|
|
132
|
+
end
|
|
133
|
+
end.compact
|
|
134
|
+
Config.new(tables, 'scrambled.dump.gz', Config::DEFAULT_S3_PROPERTIES, [], nil)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'open3'
|
|
3
|
+
module PgdumpScrambler
|
|
4
|
+
class Dumper
|
|
5
|
+
def initialize(config, db_config = {})
|
|
6
|
+
@db_config = db_config.empty? ? load_database_yml : config
|
|
7
|
+
@config = config
|
|
8
|
+
@output_path = config.dump_path
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def run
|
|
12
|
+
puts "executing pg_dump..."
|
|
13
|
+
puts full_command
|
|
14
|
+
if system(full_command)
|
|
15
|
+
puts "done!"
|
|
16
|
+
else
|
|
17
|
+
raise "pg_dump failed!"
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
private
|
|
22
|
+
|
|
23
|
+
def full_command
|
|
24
|
+
[pgdump_command, obfuscator_command, 'gzip -c'].compact.join(' | ') + "> #{@output_path}"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def obfuscator_command
|
|
28
|
+
if options = @config.obfuscator_options
|
|
29
|
+
command = File.expand_path('../../../bin/pgdump-obfuscator', __FILE__)
|
|
30
|
+
"#{command} #{options}"
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def pgdump_command
|
|
35
|
+
command = []
|
|
36
|
+
command << "PGPASSWORD=#{Shellwords.escape(@db_config['password'])}" if @db_config['password']
|
|
37
|
+
command << 'pg_dump'
|
|
38
|
+
command << @config.pgdump_args if @config.pgdump_args
|
|
39
|
+
command << "--username=#{Shellwords.escape(@db_config['username'])}" if @db_config['username']
|
|
40
|
+
command << "--host='#{@db_config['host']}'" if @db_config['host']
|
|
41
|
+
command << "--port='#{@db_config['port']}'" if @db_config['port']
|
|
42
|
+
command << @config.exclude_tables.map { |exclude_table| "--exclude-table-data=#{exclude_table}" }.join(' ') if @config.exclude_tables.present?
|
|
43
|
+
command << @db_config['database']
|
|
44
|
+
command.join(' ')
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def load_database_yml
|
|
48
|
+
if defined?(Rails)
|
|
49
|
+
db_config = open(Rails.root.join('config', 'database.yml'), 'r') do |f|
|
|
50
|
+
YAML.safe_load(f, permitted_classes: [], permitted_symbols: [], aliases: true)
|
|
51
|
+
end
|
|
52
|
+
db_config[Rails.env]
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'uri'
|
|
3
|
+
require 'digest'
|
|
4
|
+
require 'openssl'
|
|
5
|
+
|
|
6
|
+
module PgdumpScrambler
|
|
7
|
+
class S3Request
|
|
8
|
+
def initialize(s3_path:, verb:, region:, bucket:, access_key_id:, secret_key:, time: nil)
|
|
9
|
+
@s3_path = s3_path.start_with?('/') ? s3_path : "/#{s3_path}"
|
|
10
|
+
@verb = verb
|
|
11
|
+
@time = time || Time.now.utc
|
|
12
|
+
@region = region
|
|
13
|
+
@bucket = bucket
|
|
14
|
+
@access_key_id = access_key_id
|
|
15
|
+
@secret_key = secret_key
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def canonical_request
|
|
19
|
+
[
|
|
20
|
+
@verb,
|
|
21
|
+
URI.encode(@s3_path),
|
|
22
|
+
canonical_query_string,
|
|
23
|
+
"host:#{@bucket}.s3.amazonaws.com\n", # canonical headers
|
|
24
|
+
'host', # signed headers
|
|
25
|
+
'UNSIGNED-PAYLOAD'
|
|
26
|
+
].join("\n")
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def signature_key
|
|
30
|
+
date_key = hmac_sha256("AWS4#{@secret_key}", iso_date)
|
|
31
|
+
date_region_key = hmac_sha256(date_key, @region)
|
|
32
|
+
date_region_service_key = hmac_sha256(date_region_key, 's3')
|
|
33
|
+
hmac_sha256(date_region_service_key, 'aws4_request')
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def signature
|
|
37
|
+
hmac_sha256_hex(signature_key, string_to_sign)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def url
|
|
41
|
+
File.join("https://#{@bucket}.s3.amazonaws.com/", "#{@s3_path}?#{canonical_query_string}&X-Amz-Signature=#{signature}")
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def iso_time
|
|
47
|
+
@time.strftime("%Y%m%dT%H%M%SZ")
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def iso_date
|
|
51
|
+
@time.strftime('%Y%m%d')
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def hmac_sha256(key, message)
|
|
55
|
+
OpenSSL::HMAC.digest(OpenSSL::Digest::SHA256.new, key, message)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def hmac_sha256_hex(key, message)
|
|
59
|
+
OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, key, message)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def canonical_query_string
|
|
63
|
+
"X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=#{@access_key_id}%2F#{iso_date}%2F#{@region}%2Fs3%2Faws4_request&X-Amz-Date=#{iso_time}&X-Amz-Expires=86400&X-Amz-SignedHeaders=host"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def string_to_sign
|
|
67
|
+
[
|
|
68
|
+
'AWS4-HMAC-SHA256',
|
|
69
|
+
iso_time,
|
|
70
|
+
"#{iso_date}/#{@region}/s3/aws4_request",
|
|
71
|
+
Digest::SHA256.hexdigest(canonical_request),
|
|
72
|
+
].join("\n")
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
if $0 == __FILE__
|
|
78
|
+
# https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
|
|
79
|
+
require "minitest/autorun"
|
|
80
|
+
class TestS3Request < Minitest::Test
|
|
81
|
+
def setup
|
|
82
|
+
@s3_request = PgdumpScrambler::S3Request.new(verb: 'GET', s3_path: '/test.txt', region: 'us-east-1', bucket: 'examplebucket', access_key_id: 'AKIAIOSFODNN7EXAMPLE', secret_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', time: Time.utc(2013, 5, 24, 0, 0, 0))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def test_canonical_request
|
|
86
|
+
assert_equal <<~EOS.chomp, @s3_request.canonical_request
|
|
87
|
+
GET
|
|
88
|
+
/test.txt
|
|
89
|
+
X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host
|
|
90
|
+
host:examplebucket.s3.amazonaws.com
|
|
91
|
+
|
|
92
|
+
host
|
|
93
|
+
UNSIGNED-PAYLOAD
|
|
94
|
+
EOS
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def test_signature
|
|
98
|
+
assert_equal 'aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404', @s3_request.signature
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def test_url
|
|
102
|
+
exected_url = 'https://examplebucket.s3.amazonaws.com/test.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404'
|
|
103
|
+
assert_equal exected_url, @s3_request.url
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'net/http'
|
|
3
|
+
require 'uri'
|
|
4
|
+
require_relative './s3_request'
|
|
5
|
+
|
|
6
|
+
module PgdumpScrambler
|
|
7
|
+
class S3UploadError < StandardError
|
|
8
|
+
attr_reader :response
|
|
9
|
+
def initialize(response)
|
|
10
|
+
@response = response
|
|
11
|
+
super "S3 upload failed: #{response.body}"
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
class S3Uploader
|
|
16
|
+
def initialize(s3_path:, local_path:, region:, bucket:, access_key_id:, secret_key:)
|
|
17
|
+
raise 'missing access_key_id' if access_key_id.nil? || access_key_id.empty?
|
|
18
|
+
raise 'missing secret_key' if secret_key.nil? || secret_key.empty?
|
|
19
|
+
@s3_request = S3Request.new(s3_path: s3_path, verb: 'PUT', region: region, bucket: bucket, access_key_id: access_key_id, secret_key: secret_key)
|
|
20
|
+
@local_path = local_path
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def run
|
|
24
|
+
uri = URI.parse(@s3_request.url)
|
|
25
|
+
puts "upload #{@local_path} to #{uri.host}#{uri.path}"
|
|
26
|
+
open(@local_path, 'r') do |io|
|
|
27
|
+
uri_path = uri.path
|
|
28
|
+
uri_path += "?#{uri.query}" if uri.query
|
|
29
|
+
req = Net::HTTP::Put.new(uri_path)
|
|
30
|
+
req.body_stream = io
|
|
31
|
+
req.content_length = io.size
|
|
32
|
+
req.content_type = 'application/octet-stream'
|
|
33
|
+
http = Net::HTTP.new(uri.host, uri.port)
|
|
34
|
+
http.use_ssl = true
|
|
35
|
+
res = http.request(req)
|
|
36
|
+
if res.code != '200'
|
|
37
|
+
raise S3UploadError.new(res)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "pgdump_scrambler/version"
|
|
3
|
+
require "pgdump_scrambler/config"
|
|
4
|
+
require "pgdump_scrambler/dumper"
|
|
5
|
+
require "pgdump_scrambler/s3_uploader"
|
|
6
|
+
if defined?(Rails)
|
|
7
|
+
require 'pgdump_scrambler/railtie'
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
module PgdumpScrambler
|
|
11
|
+
end
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
namespace :pgdump_scrambler do
|
|
3
|
+
default_config_path = ENV['SCRAMBLER_CONFIG_PATH'] || 'config/pgdump_scrambler.yml'
|
|
4
|
+
|
|
5
|
+
desc 'create config from database'
|
|
6
|
+
task config_from_db: :environment do
|
|
7
|
+
config =
|
|
8
|
+
if File.exist?(default_config_path)
|
|
9
|
+
puts "#{default_config_path} found!\nmerge existing config with config from database"
|
|
10
|
+
PgdumpScrambler::Config
|
|
11
|
+
.read_file(default_config_path)
|
|
12
|
+
.update_with(PgdumpScrambler::Config.from_db)
|
|
13
|
+
else
|
|
14
|
+
puts "craete config from database"
|
|
15
|
+
PgdumpScrambler::Config.from_db
|
|
16
|
+
end
|
|
17
|
+
config.write_file(default_config_path)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
desc 'check if new columns exist'
|
|
21
|
+
task check: :environment do
|
|
22
|
+
config = PgdumpScrambler::Config
|
|
23
|
+
.read_file(default_config_path)
|
|
24
|
+
.update_with(PgdumpScrambler::Config.from_db)
|
|
25
|
+
unspecified_columns = config.unspecified_columns
|
|
26
|
+
count = unspecified_columns.sum { |_, columns| columns.size }
|
|
27
|
+
if count > 0
|
|
28
|
+
unspecified_columns.each_key do |table_name|
|
|
29
|
+
puts "#{table_name}:"
|
|
30
|
+
unspecified_columns[table_name].each do |column_name|
|
|
31
|
+
puts " #{column_name}"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
puts "#{count} unspecified columns found!"
|
|
35
|
+
exit 1
|
|
36
|
+
else
|
|
37
|
+
puts "No unspecified columns found."
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
desc 'create scrambled dump'
|
|
42
|
+
task dump: :environment do
|
|
43
|
+
config = PgdumpScrambler::Config.read_file(default_config_path)
|
|
44
|
+
PgdumpScrambler::Dumper.new(config).run
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
desc 'create scrambled dump'
|
|
48
|
+
task clear_dump: :environment do
|
|
49
|
+
config = PgdumpScrambler::Config.read_file(default_config_path)
|
|
50
|
+
if File.exists? config.dump_path
|
|
51
|
+
File.delete(config.dump_path)
|
|
52
|
+
puts "Dump file #{config.dump_path} has been deleted."
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
desc 'upload to s3'
|
|
57
|
+
task s3_upload: :environment do
|
|
58
|
+
config = PgdumpScrambler::Config.read_file(default_config_path)
|
|
59
|
+
uploader = PgdumpScrambler::S3Uploader.new(
|
|
60
|
+
s3_path: File.join(config.resolved_s3['prefix'], File::basename(config.dump_path)),
|
|
61
|
+
local_path: config.dump_path,
|
|
62
|
+
region: config.resolved_s3['region'],
|
|
63
|
+
bucket: config.resolved_s3['bucket'],
|
|
64
|
+
access_key_id: config.resolved_s3['access_key_id'],
|
|
65
|
+
secret_key: config.resolved_s3['secret_key']
|
|
66
|
+
)
|
|
67
|
+
uploader.run
|
|
68
|
+
end
|
|
69
|
+
end
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
|
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
|
+
require "pgdump_scrambler/version"
|
|
5
|
+
|
|
6
|
+
Gem::Specification.new do |spec|
|
|
7
|
+
spec.name = "pgdump_scrambler"
|
|
8
|
+
spec.version = PgdumpScrambler::VERSION
|
|
9
|
+
spec.authors = ["Shunichi Ikegami"]
|
|
10
|
+
spec.email = ["sike.tm@gmail.com"]
|
|
11
|
+
|
|
12
|
+
spec.summary = %q{scramble pg_dump columns}
|
|
13
|
+
spec.description = %q{scramble pg_dump columns.}
|
|
14
|
+
spec.homepage = 'https://github.com/shunichi/pgdump_scrambler'
|
|
15
|
+
spec.license = "MIT"
|
|
16
|
+
|
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
|
19
|
+
if spec.respond_to?(:metadata)
|
|
20
|
+
spec.metadata["allowed_push_host"] = 'https://rubygems.org'
|
|
21
|
+
else
|
|
22
|
+
raise "RubyGems 2.0 or newer is required to protect against " \
|
|
23
|
+
"public gem pushes."
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
27
|
+
f.match(%r{^(test|spec|features)/})
|
|
28
|
+
end
|
|
29
|
+
spec.bindir = "exe"
|
|
30
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
31
|
+
spec.require_paths = ["lib"]
|
|
32
|
+
|
|
33
|
+
spec.add_development_dependency "rake", "~> 13.0"
|
|
34
|
+
spec.add_development_dependency "rspec", "~> 3.12"
|
|
35
|
+
spec.add_development_dependency "rails", "~> 7.0"
|
|
36
|
+
spec.add_development_dependency "rubocop"
|
|
37
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: pgdump_scrambler
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.4.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Shunichi Ikegami
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: exe
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2023-10-19 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: rake
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '13.0'
|
|
20
|
+
type: :development
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
- - "~>"
|
|
25
|
+
- !ruby/object:Gem::Version
|
|
26
|
+
version: '13.0'
|
|
27
|
+
- !ruby/object:Gem::Dependency
|
|
28
|
+
name: rspec
|
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
|
30
|
+
requirements:
|
|
31
|
+
- - "~>"
|
|
32
|
+
- !ruby/object:Gem::Version
|
|
33
|
+
version: '3.12'
|
|
34
|
+
type: :development
|
|
35
|
+
prerelease: false
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
37
|
+
requirements:
|
|
38
|
+
- - "~>"
|
|
39
|
+
- !ruby/object:Gem::Version
|
|
40
|
+
version: '3.12'
|
|
41
|
+
- !ruby/object:Gem::Dependency
|
|
42
|
+
name: rails
|
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
|
+
requirements:
|
|
45
|
+
- - "~>"
|
|
46
|
+
- !ruby/object:Gem::Version
|
|
47
|
+
version: '7.0'
|
|
48
|
+
type: :development
|
|
49
|
+
prerelease: false
|
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
51
|
+
requirements:
|
|
52
|
+
- - "~>"
|
|
53
|
+
- !ruby/object:Gem::Version
|
|
54
|
+
version: '7.0'
|
|
55
|
+
- !ruby/object:Gem::Dependency
|
|
56
|
+
name: rubocop
|
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '0'
|
|
62
|
+
type: :development
|
|
63
|
+
prerelease: false
|
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
65
|
+
requirements:
|
|
66
|
+
- - ">="
|
|
67
|
+
- !ruby/object:Gem::Version
|
|
68
|
+
version: '0'
|
|
69
|
+
description: scramble pg_dump columns.
|
|
70
|
+
email:
|
|
71
|
+
- sike.tm@gmail.com
|
|
72
|
+
executables: []
|
|
73
|
+
extensions: []
|
|
74
|
+
extra_rdoc_files: []
|
|
75
|
+
files:
|
|
76
|
+
- ".gitignore"
|
|
77
|
+
- ".rspec"
|
|
78
|
+
- ".rubocop.yml"
|
|
79
|
+
- ".rubocop_todo.yml"
|
|
80
|
+
- ".ruby-version"
|
|
81
|
+
- ".travis.yml"
|
|
82
|
+
- ".vscode/tasks.json"
|
|
83
|
+
- CHANGELOG.md
|
|
84
|
+
- Gemfile
|
|
85
|
+
- Gemfile.lock
|
|
86
|
+
- LICENSE.txt
|
|
87
|
+
- README.md
|
|
88
|
+
- Rakefile
|
|
89
|
+
- bin/console
|
|
90
|
+
- bin/pgdump-obfuscator
|
|
91
|
+
- bin/setup
|
|
92
|
+
- lib/config/table.rb
|
|
93
|
+
- lib/pgdump_scrambler.rb
|
|
94
|
+
- lib/pgdump_scrambler/config.rb
|
|
95
|
+
- lib/pgdump_scrambler/dumper.rb
|
|
96
|
+
- lib/pgdump_scrambler/railtie.rb
|
|
97
|
+
- lib/pgdump_scrambler/s3_request.rb
|
|
98
|
+
- lib/pgdump_scrambler/s3_uploader.rb
|
|
99
|
+
- lib/pgdump_scrambler/version.rb
|
|
100
|
+
- lib/tasks/pgdump_scrambler_tasks.rake
|
|
101
|
+
- libexec/pgdump-obfuscator-darwin-amd64
|
|
102
|
+
- libexec/pgdump-obfuscator-darwin-arm64
|
|
103
|
+
- libexec/pgdump-obfuscator-linux-amd64
|
|
104
|
+
- libexec/pgdump-obfuscator-linux-arm64
|
|
105
|
+
- pgdump_scrambler.gemspec
|
|
106
|
+
homepage: https://github.com/shunichi/pgdump_scrambler
|
|
107
|
+
licenses:
|
|
108
|
+
- MIT
|
|
109
|
+
metadata:
|
|
110
|
+
allowed_push_host: https://rubygems.org
|
|
111
|
+
post_install_message:
|
|
112
|
+
rdoc_options: []
|
|
113
|
+
require_paths:
|
|
114
|
+
- lib
|
|
115
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
116
|
+
requirements:
|
|
117
|
+
- - ">="
|
|
118
|
+
- !ruby/object:Gem::Version
|
|
119
|
+
version: '0'
|
|
120
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
|
+
requirements:
|
|
122
|
+
- - ">="
|
|
123
|
+
- !ruby/object:Gem::Version
|
|
124
|
+
version: '0'
|
|
125
|
+
requirements: []
|
|
126
|
+
rubygems_version: 3.3.26
|
|
127
|
+
signing_key:
|
|
128
|
+
specification_version: 4
|
|
129
|
+
summary: scramble pg_dump columns
|
|
130
|
+
test_files: []
|