mask_sql 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +0 -1
- data/.rubocop.yml +30 -0
- data/.rubocop_todo.yml +2 -110
- data/CHANGELOG.md +26 -0
- data/README.md +72 -15
- data/Rakefile +1 -1
- data/lib/mask_sql/cli.rb +24 -1
- data/lib/mask_sql/converter.rb +139 -50
- data/lib/mask_sql/initializer.rb +14 -0
- data/lib/mask_sql/initializer/.mask.yml +24 -0
- data/lib/mask_sql/version.rb +1 -1
- data/mask_sql.gemspec +7 -4
- metadata +10 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 21e5f079a163803ee03c9655eaf54d546cc33aac
|
4
|
+
data.tar.gz: 0ca01caf8b69d017ef7c93b64db5bf8bc0b67ce8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b9b44ab04b03f09cb0bcf60f3a10e3d85110ca72626003508a55fe9fb3c8d63f1c12ce22d45de3d6a888e13dabbc6f8395a83347287dca554172da0f7fa12bb1
|
7
|
+
data.tar.gz: 13d3ac288396295a05b4480cb8d5e25a2b41a073d2735fb5b88961b9fa848a430e64b480be962b2c55e39ab46299b5b644f4cb2829420e9bfc77d74e3438cc48
|
data/.rspec
CHANGED
data/.rubocop.yml
CHANGED
@@ -2,3 +2,33 @@ inherit_from: .rubocop_todo.yml
|
|
2
2
|
|
3
3
|
AllCops:
|
4
4
|
DisplayCopNames: true
|
5
|
+
|
6
|
+
Layout/IndentHeredoc:
|
7
|
+
Exclude:
|
8
|
+
- 'spec/mask_sql/cli_spec.rb'
|
9
|
+
|
10
|
+
Layout/MultilineMethodCallIndentation:
|
11
|
+
EnforcedStyle: indented
|
12
|
+
|
13
|
+
Layout/TrailingWhitespace:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
Lint/NonLocalExitFromIterator:
|
17
|
+
Exclude:
|
18
|
+
- 'lib/mask_sql/converter.rb'
|
19
|
+
|
20
|
+
Metrics/AbcSize:
|
21
|
+
Max: 20
|
22
|
+
|
23
|
+
Metrics/BlockLength:
|
24
|
+
Exclude:
|
25
|
+
- 'spec/**/*'
|
26
|
+
|
27
|
+
Metrics/ClassLength:
|
28
|
+
Enabled: false
|
29
|
+
|
30
|
+
Metrics/LineLength:
|
31
|
+
Enabled: false
|
32
|
+
|
33
|
+
Metrics/MethodLength:
|
34
|
+
Max: 20
|
data/.rubocop_todo.yml
CHANGED
@@ -1,124 +1,16 @@
|
|
1
1
|
# This configuration was generated by
|
2
2
|
# `rubocop --auto-gen-config`
|
3
|
-
# on 2017-
|
3
|
+
# on 2017-05-09 22:17:28 +0900 using RuboCop version 0.48.1.
|
4
4
|
# The point is for the user to remove these configuration records
|
5
5
|
# one by one as the offenses are removed from the code base.
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
8
8
|
|
9
|
-
# Offense count: 2
|
10
|
-
Lint/NonLocalExitFromIterator:
|
11
|
-
Exclude:
|
12
|
-
- 'lib/mask_sql/converter.rb'
|
13
|
-
|
14
|
-
# Offense count: 1
|
15
|
-
Metrics/AbcSize:
|
16
|
-
Max: 44
|
17
|
-
|
18
|
-
# Offense count: 6
|
19
|
-
# Configuration parameters: CountComments, ExcludedMethods.
|
20
|
-
Metrics/BlockLength:
|
21
|
-
Max: 270
|
22
|
-
|
23
|
-
# Offense count: 2
|
24
|
-
Metrics/CyclomaticComplexity:
|
25
|
-
Max: 7
|
26
|
-
|
27
|
-
# Offense count: 49
|
28
|
-
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
29
|
-
# URISchemes: http, https
|
30
|
-
Metrics/LineLength:
|
31
|
-
Max: 153
|
32
|
-
|
33
9
|
# Offense count: 3
|
34
|
-
# Configuration parameters: CountComments.
|
35
|
-
Metrics/MethodLength:
|
36
|
-
Max: 29
|
37
|
-
|
38
|
-
# Offense count: 2
|
39
10
|
Style/Documentation:
|
40
11
|
Exclude:
|
41
12
|
- 'spec/**/*'
|
42
13
|
- 'test/**/*'
|
43
14
|
- 'lib/mask_sql/cli.rb'
|
44
15
|
- 'lib/mask_sql/converter.rb'
|
45
|
-
|
46
|
-
# Offense count: 1
|
47
|
-
# Cop supports --auto-correct.
|
48
|
-
Style/EmptyLineAfterMagicComment:
|
49
|
-
Exclude:
|
50
|
-
- 'mask_sql.gemspec'
|
51
|
-
|
52
|
-
# Offense count: 1
|
53
|
-
# Cop supports --auto-correct.
|
54
|
-
# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
|
55
|
-
Style/ExtraSpacing:
|
56
|
-
Exclude:
|
57
|
-
- 'mask_sql.gemspec'
|
58
|
-
|
59
|
-
# Offense count: 1
|
60
|
-
# Cop supports --auto-correct.
|
61
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
|
62
|
-
# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
|
63
|
-
Style/HashSyntax:
|
64
|
-
Exclude:
|
65
|
-
- 'Rakefile'
|
66
|
-
|
67
|
-
# Offense count: 4
|
68
|
-
# Cop supports --auto-correct.
|
69
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
70
|
-
# SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
|
71
|
-
Style/IndentHeredoc:
|
72
|
-
Exclude:
|
73
|
-
- 'spec/mask_sql/cli_spec.rb'
|
74
|
-
|
75
|
-
# Offense count: 2
|
76
|
-
# Cop supports --auto-correct.
|
77
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
|
78
|
-
# SupportedStyles: aligned, indented, indented_relative_to_receiver
|
79
|
-
Style/MultilineMethodCallIndentation:
|
80
|
-
Exclude:
|
81
|
-
- 'lib/mask_sql/converter.rb'
|
82
|
-
|
83
|
-
# Offense count: 1
|
84
|
-
# Cop supports --auto-correct.
|
85
|
-
Style/MutableConstant:
|
86
|
-
Exclude:
|
87
|
-
- 'lib/mask_sql/version.rb'
|
88
|
-
|
89
|
-
# Offense count: 1
|
90
|
-
# Cop supports --auto-correct.
|
91
|
-
# Configuration parameters: AutoCorrect, EnforcedStyle, SupportedStyles.
|
92
|
-
# SupportedStyles: predicate, comparison
|
93
|
-
Style/NumericPredicate:
|
94
|
-
Exclude:
|
95
|
-
- 'spec/**/*'
|
96
|
-
- 'lib/mask_sql/converter.rb'
|
97
|
-
|
98
|
-
# Offense count: 39
|
99
|
-
# Cop supports --auto-correct.
|
100
|
-
# Configuration parameters: PreferredDelimiters.
|
101
|
-
Style/PercentLiteralDelimiters:
|
102
|
-
Exclude:
|
103
|
-
- 'lib/mask_sql/cli.rb'
|
104
|
-
- 'mask_sql.gemspec'
|
105
|
-
- 'spec/mask_sql/cli_spec.rb'
|
106
|
-
|
107
|
-
# Offense count: 1
|
108
|
-
# Cop supports --auto-correct.
|
109
|
-
# Configuration parameters: AllowForAlignment.
|
110
|
-
Style/SpaceAroundOperators:
|
111
|
-
Exclude:
|
112
|
-
- 'mask_sql.gemspec'
|
113
|
-
|
114
|
-
# Offense count: 6
|
115
|
-
# Cop supports --auto-correct.
|
116
|
-
Style/TrailingWhitespace:
|
117
|
-
Exclude:
|
118
|
-
- 'spec/mask_sql/cli_spec.rb'
|
119
|
-
|
120
|
-
# Offense count: 2
|
121
|
-
# Cop supports --auto-correct.
|
122
|
-
Style/UnneededPercentQ:
|
123
|
-
Exclude:
|
124
|
-
- 'mask_sql.gemspec'
|
16
|
+
- 'lib/mask_sql/initializer.rb'
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
# Change Log
|
2
|
+
|
3
|
+
## 0.2.0 (2017-05-27)
|
4
|
+
|
5
|
+
### Features
|
6
|
+
|
7
|
+
* Add `init` command
|
8
|
+
* Add `group_indexes` option in config file
|
9
|
+
* Rename key in config file (`indexes` -> `dummy_values`)
|
10
|
+
* Support commas within string values
|
11
|
+
* Add validation for `--in` and `--out` options
|
12
|
+
* Retry when `Encoding::UndefinedConversionError` occurred
|
13
|
+
* Specify required Ruby version (`>= 2.0.0`)
|
14
|
+
|
15
|
+
### Performance Improvements
|
16
|
+
|
17
|
+
* Parse SQL without CSV library
|
18
|
+
|
19
|
+
## 0.1.0 (2017-04-17)
|
20
|
+
|
21
|
+
### Features
|
22
|
+
|
23
|
+
* Add `mask` command
|
24
|
+
* Support INSERT, REPLACE, and COPY statements
|
25
|
+
* Add `version`, `--version`, and `-v` commands
|
26
|
+
* Support various encodings
|
data/README.md
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# MaskSQL
|
2
2
|
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/mask_sql.svg)](https://badge.fury.io/rb/mask_sql)
|
3
4
|
[![Build Status](https://travis-ci.org/emsk/mask_sql.svg?branch=master)](https://travis-ci.org/emsk/mask_sql)
|
4
5
|
[![Coverage Status](https://coveralls.io/repos/github/emsk/mask_sql/badge.svg?branch=master)](https://coveralls.io/github/emsk/mask_sql)
|
5
6
|
[![Code Climate](https://codeclimate.com/github/emsk/mask_sql/badges/gpa.svg)](https://codeclimate.com/github/emsk/mask_sql)
|
@@ -11,36 +12,68 @@ MaskSQL is a command-line tool to mask sensitive values in a SQL file.
|
|
11
12
|
|
12
13
|
## Installation
|
13
14
|
|
14
|
-
|
15
|
+
Add this line to your application's Gemfile:
|
16
|
+
|
17
|
+
```ruby
|
18
|
+
gem 'mask_sql'
|
19
|
+
```
|
20
|
+
|
21
|
+
And then execute:
|
22
|
+
|
23
|
+
```sh
|
24
|
+
$ bundle
|
25
|
+
```
|
26
|
+
|
27
|
+
Or install it yourself as:
|
28
|
+
|
29
|
+
```sh
|
30
|
+
$ gem install mask_sql
|
31
|
+
```
|
15
32
|
|
16
33
|
## Usage
|
17
34
|
|
35
|
+
### Mask sensitive values in a SQL file
|
36
|
+
|
18
37
|
```sh
|
19
38
|
$ mask_sql --in dump.sql --out masked_dump.sql --config mask_config.yml
|
20
39
|
```
|
21
40
|
|
41
|
+
### Generate a config file
|
42
|
+
|
43
|
+
```sh
|
44
|
+
$ mask_sql init
|
45
|
+
```
|
46
|
+
|
22
47
|
## Command Options
|
23
48
|
|
24
49
|
| Option | Alias | Description | Default |
|
25
50
|
| :----- | :---- | :---------- | :------ |
|
26
|
-
| `--in` | `-i` | Input file path (Required) | |
|
27
|
-
| `--out` | `-o` | Output file path (Required) | |
|
28
|
-
| `--config` | `-c` | Config YAML file path | `.mask.yml` in the working directory |
|
29
|
-
| `--insert` | | `true` if mask `INSERT` SQL | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given |
|
30
|
-
| `--replace` | | `true` if mask `REPLACE` SQL | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given |
|
31
|
-
| `--copy` | | `true` if mask `COPY` SQL | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given |
|
51
|
+
| `--in` | `-i` | Input file path (Required). | |
|
52
|
+
| `--out` | `-o` | Output file path (Required). | |
|
53
|
+
| `--config` | `-c` | Config YAML file path. | `.mask.yml` in the working directory. |
|
54
|
+
| `--insert` | | `true` if mask `INSERT` SQL. | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given. |
|
55
|
+
| `--replace` | | `true` if mask `REPLACE` SQL. | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given. |
|
56
|
+
| `--copy` | | `true` if mask `COPY` SQL. | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given. |
|
32
57
|
|
33
58
|
## Config
|
34
59
|
|
35
|
-
The following keys are
|
60
|
+
The following keys are available in the config YAML file.
|
61
|
+
|
62
|
+
### Top level keys
|
63
|
+
|
64
|
+
| Key | Description | Type |
|
65
|
+
| :-- | :---------- | :--- |
|
66
|
+
| `mark` | Replacement text. | String |
|
67
|
+
| `targets` | Array of targets. | Array |
|
68
|
+
|
69
|
+
### Keys for `targets`
|
36
70
|
|
37
71
|
| Key | Description | Type |
|
38
72
|
| :-- | :---------- | :--- |
|
39
|
-
| `
|
40
|
-
| `
|
41
|
-
| `
|
42
|
-
| `
|
43
|
-
| `indexes` | Target column index (zero-based) and masking text | Hash |
|
73
|
+
| `table` | Target table name. | String |
|
74
|
+
| `columns` | Columns count of the table. | Integer |
|
75
|
+
| `dummy_values` | Target column index (zero-based) and dummy text. | Hash |
|
76
|
+
| `group_indexes` | Array of column indexes (zero-based).<br>Records that have the same values in these indexes are considered as the same numbering group. | Array |
|
44
77
|
|
45
78
|
## Examples
|
46
79
|
|
@@ -49,9 +82,11 @@ Input file (includes sensitive values):
|
|
49
82
|
```sql
|
50
83
|
INSERT INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','坂本龍馬','ryoma-sakamoto@example.com'),(2,'02','高杉晋作','shinsaku-takasugi@example.com'),(3,'03','沖田総司','soji-okita@example.com');
|
51
84
|
INSERT INTO `cats` (`code`, `name`) VALUES ('01','Sora'),('02','Hana'),('03','Leo');
|
85
|
+
INSERT INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('01','Pochi',1,1),('02','Rose',2,1),('03','Momo',1,1),('04','Sakura',1,2);
|
52
86
|
|
53
87
|
REPLACE INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','坂本龍馬','ryoma-sakamoto@example.com'),(2,'02','高杉晋作','shinsaku-takasugi@example.com'),(3,'03','沖田総司','soji-okita@example.com');
|
54
88
|
REPLACE INTO `cats` (`code`, `name`) VALUES ('01','Sora'),('02','Hana'),('03','Leo');
|
89
|
+
REPLACE INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('01','Pochi',1,1),('02','Rose',2,1),('03','Momo',1,1),('04','Sakura',1,2);
|
55
90
|
|
56
91
|
COPY people (id, code, name, email) FROM stdin;
|
57
92
|
1 01 坂本龍馬 ryoma-sakamoto@example.com
|
@@ -63,6 +98,12 @@ COPY cats (code, name) FROM stdin;
|
|
63
98
|
02 Hana
|
64
99
|
03 Leo
|
65
100
|
\.
|
101
|
+
COPY dogs (code, name, house_id, room_id) FROM stdin;
|
102
|
+
01 Pochi 1 1
|
103
|
+
02 Rose 2 1
|
104
|
+
03 Momo 1 1
|
105
|
+
04 Sakura 1 2
|
106
|
+
\.
|
66
107
|
```
|
67
108
|
|
68
109
|
Output file (the sensitive values are masked):
|
@@ -70,9 +111,11 @@ Output file (the sensitive values are masked):
|
|
70
111
|
```sql
|
71
112
|
INSERT INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','氏名1','email-1@example.com'),(2,'02','氏名2','email-2@example.com'),(3,'03','氏名3','email-3@example.com');
|
72
113
|
INSERT INTO `cats` (`code`, `name`) VALUES ('code-1','Cat name 1'),('code-2','Cat name 2'),('code-3','Cat name 3');
|
114
|
+
INSERT INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('code-1','Dog name 1',1,1),('code-1','Dog name 1',2,1),('code-2','Dog name 2',1,1),('code-1','Dog name 1',1,2);
|
73
115
|
|
74
116
|
REPLACE INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','氏名1','email-1@example.com'),(2,'02','氏名2','email-2@example.com'),(3,'03','氏名3','email-3@example.com');
|
75
117
|
REPLACE INTO `cats` (`code`, `name`) VALUES ('code-1','Cat name 1'),('code-2','Cat name 2'),('code-3','Cat name 3');
|
118
|
+
REPLACE INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('code-1','Dog name 1',1,1),('code-1','Dog name 1',2,1),('code-2','Dog name 2',1,1),('code-1','Dog name 1',1,2);
|
76
119
|
|
77
120
|
COPY people (id, code, name, email) FROM stdin;
|
78
121
|
1 01 氏名1 email-1@example.com
|
@@ -84,6 +127,12 @@ code-1 Cat name 1
|
|
84
127
|
code-2 Cat name 2
|
85
128
|
code-3 Cat name 3
|
86
129
|
\.
|
130
|
+
COPY dogs (code, name, house_id, room_id) FROM stdin;
|
131
|
+
code-1 Dog name 1 1 1
|
132
|
+
code-1 Dog name 1 2 1
|
133
|
+
code-2 Dog name 2 1 1
|
134
|
+
code-1 Dog name 1 1 2
|
135
|
+
\.
|
87
136
|
```
|
88
137
|
|
89
138
|
Config file:
|
@@ -93,14 +142,22 @@ mark: '[mask]'
|
|
93
142
|
targets:
|
94
143
|
- table: people
|
95
144
|
columns: 4
|
96
|
-
|
145
|
+
dummy_values:
|
97
146
|
2: 氏名[mask]
|
98
147
|
3: email-[mask]@example.com
|
99
148
|
- table: cats
|
100
149
|
columns: 2
|
101
|
-
|
150
|
+
dummy_values:
|
102
151
|
0: code-[mask]
|
103
152
|
1: Cat name [mask]
|
153
|
+
- table: dogs
|
154
|
+
columns: 4
|
155
|
+
dummy_values:
|
156
|
+
0: code-[mask]
|
157
|
+
1: Dog name [mask]
|
158
|
+
group_indexes:
|
159
|
+
- 2
|
160
|
+
- 3
|
104
161
|
```
|
105
162
|
|
106
163
|
## Supported Ruby Versions
|
data/Rakefile
CHANGED
data/lib/mask_sql/cli.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require 'thor'
|
2
2
|
require 'mask_sql/converter'
|
3
|
+
require 'mask_sql/initializer'
|
3
4
|
|
4
5
|
module MaskSQL
|
5
6
|
class CLI < Thor
|
@@ -14,6 +15,8 @@ module MaskSQL
|
|
14
15
|
option :copy, type: :boolean, banner: 'MASK `COPY` SQL'
|
15
16
|
|
16
17
|
def mask
|
18
|
+
return unless validate_options
|
19
|
+
|
17
20
|
converter_options = options.dup
|
18
21
|
|
19
22
|
if options[:config]
|
@@ -28,11 +31,31 @@ module MaskSQL
|
|
28
31
|
puts "\e[32mDone.\e[0m"
|
29
32
|
end
|
30
33
|
|
34
|
+
desc 'init', 'Generate a config file'
|
35
|
+
|
36
|
+
def init
|
37
|
+
puts Initializer.copy_template
|
38
|
+
end
|
39
|
+
|
31
40
|
desc 'version, -v, --version', 'Print the version'
|
32
|
-
map %w
|
41
|
+
map %w[-v --version] => :version
|
33
42
|
|
34
43
|
def version
|
35
44
|
puts "mask_sql #{MaskSQL::VERSION}"
|
36
45
|
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def validate_options
|
50
|
+
in_file = File.expand_path(options[:in])
|
51
|
+
out_file = File.expand_path(options[:out])
|
52
|
+
|
53
|
+
if in_file == out_file
|
54
|
+
warn "\e[31mOutput file is the same as input file.\e[0m"
|
55
|
+
return false
|
56
|
+
end
|
57
|
+
|
58
|
+
true
|
59
|
+
end
|
37
60
|
end
|
38
61
|
end
|
data/lib/mask_sql/converter.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
require 'csv'
|
2
1
|
require 'yaml'
|
3
2
|
require 'nkf'
|
4
3
|
|
@@ -20,69 +19,72 @@ module MaskSQL
|
|
20
19
|
@matched_copy = {}
|
21
20
|
end
|
22
21
|
|
23
|
-
def mask
|
24
|
-
encoding
|
22
|
+
def mask(encoding = nil)
|
23
|
+
encoding ||= NKF.guess(File.read(@options[:in])).name
|
25
24
|
|
26
25
|
File.open(@options[:out], "w:#{encoding}") do |out_file|
|
27
26
|
File.open(@options[:in], "r:#{encoding}") do |in_file|
|
28
27
|
in_file.each_line do |line|
|
29
|
-
@matched_copy.empty?
|
28
|
+
if @matched_copy.empty?
|
29
|
+
@counters = []
|
30
|
+
write_line(line, out_file)
|
31
|
+
else
|
32
|
+
write_copy_line(line, out_file)
|
33
|
+
end
|
30
34
|
end
|
31
35
|
end
|
32
36
|
end
|
37
|
+
rescue Encoding::UndefinedConversionError => e
|
38
|
+
raise Encoding::UndefinedConversionError, e.message if encoding == Encoding::UTF_8.name
|
39
|
+
mask(Encoding::UTF_8.name)
|
33
40
|
end
|
34
41
|
|
35
42
|
private
|
36
43
|
|
37
44
|
def write_line(line, output_file)
|
38
45
|
@targets.each do |target|
|
39
|
-
|
40
|
-
matched_line = match_line(line, table)
|
46
|
+
matched_line = match_line(line, target['table'])
|
41
47
|
next unless matched_line
|
42
48
|
|
43
49
|
if matched_line.names.include?('copy_sql')
|
44
50
|
output_file.puts line
|
45
|
-
|
46
|
-
@matched_copy[:indexes] = target['indexes']
|
47
|
-
@matched_copy[:record_index] = 1
|
51
|
+
init_matched_copy(target)
|
48
52
|
return
|
49
53
|
end
|
50
54
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
record_values.map!.with_index(1) do |values, record_index|
|
56
|
-
indexes.each do |mask_index|
|
57
|
-
before_value = values[mask_index]
|
58
|
-
values[mask_index] = target['indexes'][mask_index].gsub(@mark, record_index.to_s)
|
59
|
-
values[mask_index].insert(0, "'") if before_value.start_with?("'", "('")
|
60
|
-
values[mask_index].insert(-1, "'") if before_value.end_with?("'", "')")
|
61
|
-
values[mask_index].insert(0, '(') if mask_index == 0
|
62
|
-
values[mask_index].insert(-1, ')') if mask_index == columns - 1
|
63
|
-
end
|
64
|
-
values
|
65
|
-
end
|
55
|
+
all_values = parse_all_values(matched_line[:all_values])
|
56
|
+
|
57
|
+
record_values = get_record_values(all_values, target['columns'])
|
58
|
+
masked_values = mask_values(record_values, target)
|
66
59
|
|
67
|
-
output_file.puts "#{matched_line[:prefix]}#{
|
60
|
+
output_file.puts "#{matched_line[:prefix]}#{masked_values.join(',')}#{matched_line[:suffix]}"
|
68
61
|
return
|
69
62
|
end
|
70
63
|
|
71
64
|
output_file.puts line
|
72
65
|
end
|
73
66
|
|
67
|
+
def init_matched_copy(target)
|
68
|
+
@matched_copy[:dummy_values] = target['dummy_values']
|
69
|
+
@matched_copy[:group_indexes] = target['group_indexes'] || []
|
70
|
+
@matched_copy[:record_index] = 1
|
71
|
+
@counters = []
|
72
|
+
end
|
73
|
+
|
74
74
|
def write_copy_line(line, output_file)
|
75
|
-
if
|
75
|
+
if /^\\.$/ =~ line
|
76
76
|
output_file.puts line
|
77
77
|
@matched_copy.clear
|
78
78
|
return
|
79
79
|
end
|
80
80
|
|
81
|
-
record_values =
|
82
|
-
@matched_copy[:
|
83
|
-
|
84
|
-
|
85
|
-
|
81
|
+
record_values = line.split("\t")
|
82
|
+
count = get_current_count(record_values, @matched_copy[:record_index], @matched_copy[:group_indexes])
|
83
|
+
|
84
|
+
@matched_copy[:dummy_values].each do |dummy_index, dummy_value|
|
85
|
+
record_values[dummy_index] = dummy_value.sub(/^'/, '')
|
86
|
+
.sub(/'$/, '')
|
87
|
+
.gsub(@mark, count.to_s)
|
86
88
|
end
|
87
89
|
|
88
90
|
output_file.puts record_values.join("\t")
|
@@ -90,33 +92,120 @@ module MaskSQL
|
|
90
92
|
end
|
91
93
|
|
92
94
|
def match_line(line, table)
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
95
|
+
matched_line = match_insert(line, table)
|
96
|
+
return matched_line if matched_line
|
97
|
+
|
98
|
+
matched_line = match_replace(line, table)
|
99
|
+
return matched_line if matched_line
|
100
|
+
|
101
|
+
matched_line = match_copy(line, table)
|
102
|
+
return matched_line if matched_line
|
103
|
+
|
104
|
+
nil
|
105
|
+
end
|
97
106
|
|
98
|
-
|
99
|
-
|
100
|
-
|
107
|
+
def match_insert(line, table)
|
108
|
+
return unless @options[:insert]
|
109
|
+
/^(?<prefix>INSERT (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)$/i.match(line)
|
110
|
+
end
|
111
|
+
|
112
|
+
def match_replace(line, table)
|
113
|
+
return unless @options[:replace]
|
114
|
+
/^(?<prefix>REPLACE (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)$/i.match(line)
|
115
|
+
end
|
116
|
+
|
117
|
+
def match_copy(line, table)
|
118
|
+
return unless @options[:copy]
|
119
|
+
/^(?<copy_sql>COPY\s*`?#{table}`?.*FROM stdin;)$/i.match(line)
|
120
|
+
end
|
121
|
+
|
122
|
+
def parse_all_values(matched_all_values)
|
123
|
+
all_values = matched_all_values.chomp.split(',')
|
124
|
+
processing_index = 0
|
125
|
+
|
126
|
+
all_values.map!.with_index do |value, index|
|
127
|
+
next if index != 0 && index <= processing_index
|
128
|
+
|
129
|
+
if start_string?(value)
|
130
|
+
processing_value = value.dup
|
131
|
+
processing_index = index
|
132
|
+
|
133
|
+
until end_string?(processing_value)
|
134
|
+
processing_index += 1
|
135
|
+
processing_value += all_values[processing_index]
|
136
|
+
end
|
137
|
+
|
138
|
+
value = processing_value
|
139
|
+
end
|
140
|
+
|
141
|
+
value
|
101
142
|
end
|
102
143
|
|
103
|
-
|
104
|
-
|
105
|
-
|
144
|
+
all_values.compact
|
145
|
+
end
|
146
|
+
|
147
|
+
def start_string?(value)
|
148
|
+
value == "'" || value == "('" || (value.start_with?("'", "('") && !value.end_with?("'", "')"))
|
149
|
+
end
|
150
|
+
|
151
|
+
def end_string?(value)
|
152
|
+
value != "'" && value != "('" && value.end_with?("'", "')")
|
153
|
+
end
|
154
|
+
|
155
|
+
def get_record_values(all_values, columns)
|
156
|
+
all_values.each_slice(columns).to_a
|
157
|
+
end
|
158
|
+
|
159
|
+
def mask_values(record_values, target)
|
160
|
+
columns = target['columns']
|
161
|
+
dummy_values = target['dummy_values']
|
162
|
+
group_indexes = target['group_indexes'] || []
|
163
|
+
|
164
|
+
record_values.map!.with_index(1) do |values, record_index|
|
165
|
+
count = get_current_count(values, record_index, group_indexes)
|
166
|
+
|
167
|
+
dummy_values.each_key do |dummy_index|
|
168
|
+
original_value = values[dummy_index]
|
169
|
+
masked_value = dummy_values[dummy_index].gsub(@mark, count.to_s)
|
170
|
+
values[dummy_index] = mask_value(masked_value, original_value, dummy_index, columns)
|
171
|
+
end
|
172
|
+
|
173
|
+
values
|
106
174
|
end
|
107
175
|
|
108
|
-
|
176
|
+
record_values
|
177
|
+
end
|
178
|
+
|
179
|
+
def get_current_count(values, record_index, group_indexes)
|
180
|
+
return record_index if group_indexes.empty?
|
181
|
+
|
182
|
+
group_values = group_indexes.map do |group_index|
|
183
|
+
values[group_index]
|
184
|
+
end
|
185
|
+
increment_count(group_values)
|
109
186
|
end
|
110
187
|
|
111
|
-
def
|
112
|
-
|
113
|
-
|
114
|
-
/\A(?<prefix>INSERT (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)\Z/i
|
115
|
-
when :replace
|
116
|
-
/\A(?<prefix>REPLACE (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)\Z/i
|
117
|
-
when :copy
|
118
|
-
/(?<copy_sql>COPY\s*`?#{table}`?.*FROM stdin;)/i
|
188
|
+
def increment_count(group_values)
|
189
|
+
counter = @counters.find do |c|
|
190
|
+
c[:label] == group_values
|
119
191
|
end
|
192
|
+
|
193
|
+
if counter
|
194
|
+
counter[:count] += 1
|
195
|
+
else
|
196
|
+
counter = { label: group_values, count: 1 }
|
197
|
+
@counters.push(counter)
|
198
|
+
end
|
199
|
+
|
200
|
+
counter[:count]
|
201
|
+
end
|
202
|
+
|
203
|
+
def mask_value(masked_value, original_value, mask_index, columns)
|
204
|
+
masked_value.insert(0, "'") if original_value.start_with?("'", "('")
|
205
|
+
masked_value.insert(-1, "'") if original_value.end_with?("'", "')")
|
206
|
+
masked_value.insert(0, '(') if mask_index.zero?
|
207
|
+
masked_value.insert(-1, ')') if mask_index == columns - 1
|
208
|
+
masked_value
|
120
209
|
end
|
121
210
|
end
|
122
211
|
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module MaskSQL
|
2
|
+
class Initializer
|
3
|
+
TEMPLATE_CONFIG_FILE = '.mask.yml'.freeze
|
4
|
+
|
5
|
+
def self.copy_template
|
6
|
+
to = File.expand_path(TEMPLATE_CONFIG_FILE)
|
7
|
+
return "\e[33mexist #{to}\e[0m" if FileTest.exist?(to)
|
8
|
+
|
9
|
+
from = File.expand_path("../initializer/#{TEMPLATE_CONFIG_FILE}", __FILE__)
|
10
|
+
FileUtils.copy(from, to)
|
11
|
+
"\e[32mcreate #{to}\e[0m"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# This file was generated by the `mask_sql init` command.
|
2
|
+
# Edit this file to configure the masking behavior.
|
3
|
+
# See https://github.com/emsk/mask_sql for more details.
|
4
|
+
|
5
|
+
# mark: '[mask]'
|
6
|
+
# targets:
|
7
|
+
# - table: people
|
8
|
+
# columns: 4
|
9
|
+
# dummy_values:
|
10
|
+
# 2: 氏名[mask]
|
11
|
+
# 3: email-[mask]@example.com
|
12
|
+
# - table: cats
|
13
|
+
# columns: 2
|
14
|
+
# dummy_values:
|
15
|
+
# 0: code-[mask]
|
16
|
+
# 1: Cat name [mask]
|
17
|
+
# - table: dogs
|
18
|
+
# columns: 4
|
19
|
+
# dummy_values:
|
20
|
+
# 0: code-[mask]
|
21
|
+
# 1: Dog name [mask]
|
22
|
+
# group_indexes:
|
23
|
+
# - 2
|
24
|
+
# - 3
|
data/lib/mask_sql/version.rb
CHANGED
data/mask_sql.gemspec
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# coding: utf-8
|
2
|
+
|
2
3
|
lib = File.expand_path('../lib', __FILE__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'mask_sql/version'
|
@@ -9,8 +10,8 @@ Gem::Specification.new do |spec|
|
|
9
10
|
spec.authors = ['emsk']
|
10
11
|
spec.email = ['emsk1987@gmail.com']
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
-
spec.description =
|
13
|
+
spec.summary = 'Mask sensitive values in a SQL file'
|
14
|
+
spec.description = 'MaskSQL is a command-line tool to mask sensitive values in a SQL file'
|
14
15
|
spec.homepage = 'https://github.com/emsk/mask_sql'
|
15
16
|
spec.license = 'MIT'
|
16
17
|
|
@@ -21,11 +22,13 @@ Gem::Specification.new do |spec|
|
|
21
22
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
23
|
spec.require_paths = ['lib']
|
23
24
|
|
25
|
+
spec.required_ruby_version = '>= 2.0.0'
|
26
|
+
|
24
27
|
spec.add_runtime_dependency 'thor', '~> 0.19'
|
25
28
|
spec.add_development_dependency 'bundler', '~> 1.14'
|
26
29
|
spec.add_development_dependency 'coveralls', '~> 0.8'
|
27
30
|
spec.add_development_dependency 'rake', '~> 10.0'
|
28
|
-
spec.add_development_dependency 'rspec', '~> 3.
|
29
|
-
spec.add_development_dependency 'rubocop', '~> 0.
|
31
|
+
spec.add_development_dependency 'rspec', '~> 3.6'
|
32
|
+
spec.add_development_dependency 'rubocop', '~> 0.49'
|
30
33
|
spec.add_development_dependency 'simplecov', '~> 0.14'
|
31
34
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: mask_sql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- emsk
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-05-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -72,28 +72,28 @@ dependencies:
|
|
72
72
|
requirements:
|
73
73
|
- - "~>"
|
74
74
|
- !ruby/object:Gem::Version
|
75
|
-
version: '3.
|
75
|
+
version: '3.6'
|
76
76
|
type: :development
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
|
-
version: '3.
|
82
|
+
version: '3.6'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: rubocop
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
87
|
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version: '0.
|
89
|
+
version: '0.49'
|
90
90
|
type: :development
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version: '0.
|
96
|
+
version: '0.49'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: simplecov
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
@@ -121,6 +121,7 @@ files:
|
|
121
121
|
- ".rubocop.yml"
|
122
122
|
- ".rubocop_todo.yml"
|
123
123
|
- ".travis.yml"
|
124
|
+
- CHANGELOG.md
|
124
125
|
- Gemfile
|
125
126
|
- LICENSE.txt
|
126
127
|
- README.md
|
@@ -131,6 +132,8 @@ files:
|
|
131
132
|
- lib/mask_sql.rb
|
132
133
|
- lib/mask_sql/cli.rb
|
133
134
|
- lib/mask_sql/converter.rb
|
135
|
+
- lib/mask_sql/initializer.rb
|
136
|
+
- lib/mask_sql/initializer/.mask.yml
|
134
137
|
- lib/mask_sql/version.rb
|
135
138
|
- mask_sql.gemspec
|
136
139
|
homepage: https://github.com/emsk/mask_sql
|
@@ -145,7 +148,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
145
148
|
requirements:
|
146
149
|
- - ">="
|
147
150
|
- !ruby/object:Gem::Version
|
148
|
-
version:
|
151
|
+
version: 2.0.0
|
149
152
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
150
153
|
requirements:
|
151
154
|
- - ">="
|