mask_sql 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +0 -1
- data/.rubocop.yml +30 -0
- data/.rubocop_todo.yml +2 -110
- data/CHANGELOG.md +26 -0
- data/README.md +72 -15
- data/Rakefile +1 -1
- data/lib/mask_sql/cli.rb +24 -1
- data/lib/mask_sql/converter.rb +139 -50
- data/lib/mask_sql/initializer.rb +14 -0
- data/lib/mask_sql/initializer/.mask.yml +24 -0
- data/lib/mask_sql/version.rb +1 -1
- data/mask_sql.gemspec +7 -4
- metadata +10 -7
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 21e5f079a163803ee03c9655eaf54d546cc33aac
|
|
4
|
+
data.tar.gz: 0ca01caf8b69d017ef7c93b64db5bf8bc0b67ce8
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b9b44ab04b03f09cb0bcf60f3a10e3d85110ca72626003508a55fe9fb3c8d63f1c12ce22d45de3d6a888e13dabbc6f8395a83347287dca554172da0f7fa12bb1
|
|
7
|
+
data.tar.gz: 13d3ac288396295a05b4480cb8d5e25a2b41a073d2735fb5b88961b9fa848a430e64b480be962b2c55e39ab46299b5b644f4cb2829420e9bfc77d74e3438cc48
|
data/.rspec
CHANGED
data/.rubocop.yml
CHANGED
|
@@ -2,3 +2,33 @@ inherit_from: .rubocop_todo.yml
|
|
|
2
2
|
|
|
3
3
|
AllCops:
|
|
4
4
|
DisplayCopNames: true
|
|
5
|
+
|
|
6
|
+
Layout/IndentHeredoc:
|
|
7
|
+
Exclude:
|
|
8
|
+
- 'spec/mask_sql/cli_spec.rb'
|
|
9
|
+
|
|
10
|
+
Layout/MultilineMethodCallIndentation:
|
|
11
|
+
EnforcedStyle: indented
|
|
12
|
+
|
|
13
|
+
Layout/TrailingWhitespace:
|
|
14
|
+
Enabled: false
|
|
15
|
+
|
|
16
|
+
Lint/NonLocalExitFromIterator:
|
|
17
|
+
Exclude:
|
|
18
|
+
- 'lib/mask_sql/converter.rb'
|
|
19
|
+
|
|
20
|
+
Metrics/AbcSize:
|
|
21
|
+
Max: 20
|
|
22
|
+
|
|
23
|
+
Metrics/BlockLength:
|
|
24
|
+
Exclude:
|
|
25
|
+
- 'spec/**/*'
|
|
26
|
+
|
|
27
|
+
Metrics/ClassLength:
|
|
28
|
+
Enabled: false
|
|
29
|
+
|
|
30
|
+
Metrics/LineLength:
|
|
31
|
+
Enabled: false
|
|
32
|
+
|
|
33
|
+
Metrics/MethodLength:
|
|
34
|
+
Max: 20
|
data/.rubocop_todo.yml
CHANGED
|
@@ -1,124 +1,16 @@
|
|
|
1
1
|
# This configuration was generated by
|
|
2
2
|
# `rubocop --auto-gen-config`
|
|
3
|
-
# on 2017-
|
|
3
|
+
# on 2017-05-09 22:17:28 +0900 using RuboCop version 0.48.1.
|
|
4
4
|
# The point is for the user to remove these configuration records
|
|
5
5
|
# one by one as the offenses are removed from the code base.
|
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
|
8
8
|
|
|
9
|
-
# Offense count: 2
|
|
10
|
-
Lint/NonLocalExitFromIterator:
|
|
11
|
-
Exclude:
|
|
12
|
-
- 'lib/mask_sql/converter.rb'
|
|
13
|
-
|
|
14
|
-
# Offense count: 1
|
|
15
|
-
Metrics/AbcSize:
|
|
16
|
-
Max: 44
|
|
17
|
-
|
|
18
|
-
# Offense count: 6
|
|
19
|
-
# Configuration parameters: CountComments, ExcludedMethods.
|
|
20
|
-
Metrics/BlockLength:
|
|
21
|
-
Max: 270
|
|
22
|
-
|
|
23
|
-
# Offense count: 2
|
|
24
|
-
Metrics/CyclomaticComplexity:
|
|
25
|
-
Max: 7
|
|
26
|
-
|
|
27
|
-
# Offense count: 49
|
|
28
|
-
# Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, IgnoredPatterns.
|
|
29
|
-
# URISchemes: http, https
|
|
30
|
-
Metrics/LineLength:
|
|
31
|
-
Max: 153
|
|
32
|
-
|
|
33
9
|
# Offense count: 3
|
|
34
|
-
# Configuration parameters: CountComments.
|
|
35
|
-
Metrics/MethodLength:
|
|
36
|
-
Max: 29
|
|
37
|
-
|
|
38
|
-
# Offense count: 2
|
|
39
10
|
Style/Documentation:
|
|
40
11
|
Exclude:
|
|
41
12
|
- 'spec/**/*'
|
|
42
13
|
- 'test/**/*'
|
|
43
14
|
- 'lib/mask_sql/cli.rb'
|
|
44
15
|
- 'lib/mask_sql/converter.rb'
|
|
45
|
-
|
|
46
|
-
# Offense count: 1
|
|
47
|
-
# Cop supports --auto-correct.
|
|
48
|
-
Style/EmptyLineAfterMagicComment:
|
|
49
|
-
Exclude:
|
|
50
|
-
- 'mask_sql.gemspec'
|
|
51
|
-
|
|
52
|
-
# Offense count: 1
|
|
53
|
-
# Cop supports --auto-correct.
|
|
54
|
-
# Configuration parameters: AllowForAlignment, ForceEqualSignAlignment.
|
|
55
|
-
Style/ExtraSpacing:
|
|
56
|
-
Exclude:
|
|
57
|
-
- 'mask_sql.gemspec'
|
|
58
|
-
|
|
59
|
-
# Offense count: 1
|
|
60
|
-
# Cop supports --auto-correct.
|
|
61
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles, UseHashRocketsWithSymbolValues, PreferHashRocketsForNonAlnumEndingSymbols.
|
|
62
|
-
# SupportedStyles: ruby19, hash_rockets, no_mixed_keys, ruby19_no_mixed_keys
|
|
63
|
-
Style/HashSyntax:
|
|
64
|
-
Exclude:
|
|
65
|
-
- 'Rakefile'
|
|
66
|
-
|
|
67
|
-
# Offense count: 4
|
|
68
|
-
# Cop supports --auto-correct.
|
|
69
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles.
|
|
70
|
-
# SupportedStyles: auto_detection, squiggly, active_support, powerpack, unindent
|
|
71
|
-
Style/IndentHeredoc:
|
|
72
|
-
Exclude:
|
|
73
|
-
- 'spec/mask_sql/cli_spec.rb'
|
|
74
|
-
|
|
75
|
-
# Offense count: 2
|
|
76
|
-
# Cop supports --auto-correct.
|
|
77
|
-
# Configuration parameters: EnforcedStyle, SupportedStyles, IndentationWidth.
|
|
78
|
-
# SupportedStyles: aligned, indented, indented_relative_to_receiver
|
|
79
|
-
Style/MultilineMethodCallIndentation:
|
|
80
|
-
Exclude:
|
|
81
|
-
- 'lib/mask_sql/converter.rb'
|
|
82
|
-
|
|
83
|
-
# Offense count: 1
|
|
84
|
-
# Cop supports --auto-correct.
|
|
85
|
-
Style/MutableConstant:
|
|
86
|
-
Exclude:
|
|
87
|
-
- 'lib/mask_sql/version.rb'
|
|
88
|
-
|
|
89
|
-
# Offense count: 1
|
|
90
|
-
# Cop supports --auto-correct.
|
|
91
|
-
# Configuration parameters: AutoCorrect, EnforcedStyle, SupportedStyles.
|
|
92
|
-
# SupportedStyles: predicate, comparison
|
|
93
|
-
Style/NumericPredicate:
|
|
94
|
-
Exclude:
|
|
95
|
-
- 'spec/**/*'
|
|
96
|
-
- 'lib/mask_sql/converter.rb'
|
|
97
|
-
|
|
98
|
-
# Offense count: 39
|
|
99
|
-
# Cop supports --auto-correct.
|
|
100
|
-
# Configuration parameters: PreferredDelimiters.
|
|
101
|
-
Style/PercentLiteralDelimiters:
|
|
102
|
-
Exclude:
|
|
103
|
-
- 'lib/mask_sql/cli.rb'
|
|
104
|
-
- 'mask_sql.gemspec'
|
|
105
|
-
- 'spec/mask_sql/cli_spec.rb'
|
|
106
|
-
|
|
107
|
-
# Offense count: 1
|
|
108
|
-
# Cop supports --auto-correct.
|
|
109
|
-
# Configuration parameters: AllowForAlignment.
|
|
110
|
-
Style/SpaceAroundOperators:
|
|
111
|
-
Exclude:
|
|
112
|
-
- 'mask_sql.gemspec'
|
|
113
|
-
|
|
114
|
-
# Offense count: 6
|
|
115
|
-
# Cop supports --auto-correct.
|
|
116
|
-
Style/TrailingWhitespace:
|
|
117
|
-
Exclude:
|
|
118
|
-
- 'spec/mask_sql/cli_spec.rb'
|
|
119
|
-
|
|
120
|
-
# Offense count: 2
|
|
121
|
-
# Cop supports --auto-correct.
|
|
122
|
-
Style/UnneededPercentQ:
|
|
123
|
-
Exclude:
|
|
124
|
-
- 'mask_sql.gemspec'
|
|
16
|
+
- 'lib/mask_sql/initializer.rb'
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Change Log
|
|
2
|
+
|
|
3
|
+
## 0.2.0 (2017-05-27)
|
|
4
|
+
|
|
5
|
+
### Features
|
|
6
|
+
|
|
7
|
+
* Add `init` command
|
|
8
|
+
* Add `group_indexes` option in config file
|
|
9
|
+
* Rename key in config file (`indexes` -> `dummy_values`)
|
|
10
|
+
* Support commas within string values
|
|
11
|
+
* Add validation for `--in` and `--out` options
|
|
12
|
+
* Retry when `Encoding::UndefinedConversionError` occurred
|
|
13
|
+
* Specify required Ruby version (`>= 2.0.0`)
|
|
14
|
+
|
|
15
|
+
### Performance Improvements
|
|
16
|
+
|
|
17
|
+
* Parse SQL without CSV library
|
|
18
|
+
|
|
19
|
+
## 0.1.0 (2017-04-17)
|
|
20
|
+
|
|
21
|
+
### Features
|
|
22
|
+
|
|
23
|
+
* Add `mask` command
|
|
24
|
+
* Support INSERT, REPLACE, and COPY statements
|
|
25
|
+
* Add `version`, `--version`, and `-v` commands
|
|
26
|
+
* Support various encodings
|
data/README.md
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
# MaskSQL
|
|
2
2
|
|
|
3
|
+
[](https://badge.fury.io/rb/mask_sql)
|
|
3
4
|
[](https://travis-ci.org/emsk/mask_sql)
|
|
4
5
|
[](https://coveralls.io/github/emsk/mask_sql)
|
|
5
6
|
[](https://codeclimate.com/github/emsk/mask_sql)
|
|
@@ -11,36 +12,68 @@ MaskSQL is a command-line tool to mask sensitive values in a SQL file.
|
|
|
11
12
|
|
|
12
13
|
## Installation
|
|
13
14
|
|
|
14
|
-
|
|
15
|
+
Add this line to your application's Gemfile:
|
|
16
|
+
|
|
17
|
+
```ruby
|
|
18
|
+
gem 'mask_sql'
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
And then execute:
|
|
22
|
+
|
|
23
|
+
```sh
|
|
24
|
+
$ bundle
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
Or install it yourself as:
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
$ gem install mask_sql
|
|
31
|
+
```
|
|
15
32
|
|
|
16
33
|
## Usage
|
|
17
34
|
|
|
35
|
+
### Mask sensitive values in a SQL file
|
|
36
|
+
|
|
18
37
|
```sh
|
|
19
38
|
$ mask_sql --in dump.sql --out masked_dump.sql --config mask_config.yml
|
|
20
39
|
```
|
|
21
40
|
|
|
41
|
+
### Generate a config file
|
|
42
|
+
|
|
43
|
+
```sh
|
|
44
|
+
$ mask_sql init
|
|
45
|
+
```
|
|
46
|
+
|
|
22
47
|
## Command Options
|
|
23
48
|
|
|
24
49
|
| Option | Alias | Description | Default |
|
|
25
50
|
| :----- | :---- | :---------- | :------ |
|
|
26
|
-
| `--in` | `-i` | Input file path (Required) | |
|
|
27
|
-
| `--out` | `-o` | Output file path (Required) | |
|
|
28
|
-
| `--config` | `-c` | Config YAML file path | `.mask.yml` in the working directory |
|
|
29
|
-
| `--insert` | | `true` if mask `INSERT` SQL | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given |
|
|
30
|
-
| `--replace` | | `true` if mask `REPLACE` SQL | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given |
|
|
31
|
-
| `--copy` | | `true` if mask `COPY` SQL | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given |
|
|
51
|
+
| `--in` | `-i` | Input file path (Required). | |
|
|
52
|
+
| `--out` | `-o` | Output file path (Required). | |
|
|
53
|
+
| `--config` | `-c` | Config YAML file path. | `.mask.yml` in the working directory. |
|
|
54
|
+
| `--insert` | | `true` if mask `INSERT` SQL. | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given. |
|
|
55
|
+
| `--replace` | | `true` if mask `REPLACE` SQL. | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given. |
|
|
56
|
+
| `--copy` | | `true` if mask `COPY` SQL. | `false`, but `true` if `--insert`, `--replace`, and `--copy` options are not given. |
|
|
32
57
|
|
|
33
58
|
## Config
|
|
34
59
|
|
|
35
|
-
The following keys are
|
|
60
|
+
The following keys are available in the config YAML file.
|
|
61
|
+
|
|
62
|
+
### Top level keys
|
|
63
|
+
|
|
64
|
+
| Key | Description | Type |
|
|
65
|
+
| :-- | :---------- | :--- |
|
|
66
|
+
| `mark` | Replacement text. | String |
|
|
67
|
+
| `targets` | Array of targets. | Array |
|
|
68
|
+
|
|
69
|
+
### Keys for `targets`
|
|
36
70
|
|
|
37
71
|
| Key | Description | Type |
|
|
38
72
|
| :-- | :---------- | :--- |
|
|
39
|
-
| `
|
|
40
|
-
| `
|
|
41
|
-
| `
|
|
42
|
-
| `
|
|
43
|
-
| `indexes` | Target column index (zero-based) and masking text | Hash |
|
|
73
|
+
| `table` | Target table name. | String |
|
|
74
|
+
| `columns` | Columns count of the table. | Integer |
|
|
75
|
+
| `dummy_values` | Target column index (zero-based) and dummy text. | Hash |
|
|
76
|
+
| `group_indexes` | Array of column indexes (zero-based).<br>Records that have the same values in these indexes are considered as the same numbering group. | Array |
|
|
44
77
|
|
|
45
78
|
## Examples
|
|
46
79
|
|
|
@@ -49,9 +82,11 @@ Input file (includes sensitive values):
|
|
|
49
82
|
```sql
|
|
50
83
|
INSERT INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','坂本龍馬','ryoma-sakamoto@example.com'),(2,'02','高杉晋作','shinsaku-takasugi@example.com'),(3,'03','沖田総司','soji-okita@example.com');
|
|
51
84
|
INSERT INTO `cats` (`code`, `name`) VALUES ('01','Sora'),('02','Hana'),('03','Leo');
|
|
85
|
+
INSERT INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('01','Pochi',1,1),('02','Rose',2,1),('03','Momo',1,1),('04','Sakura',1,2);
|
|
52
86
|
|
|
53
87
|
REPLACE INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','坂本龍馬','ryoma-sakamoto@example.com'),(2,'02','高杉晋作','shinsaku-takasugi@example.com'),(3,'03','沖田総司','soji-okita@example.com');
|
|
54
88
|
REPLACE INTO `cats` (`code`, `name`) VALUES ('01','Sora'),('02','Hana'),('03','Leo');
|
|
89
|
+
REPLACE INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('01','Pochi',1,1),('02','Rose',2,1),('03','Momo',1,1),('04','Sakura',1,2);
|
|
55
90
|
|
|
56
91
|
COPY people (id, code, name, email) FROM stdin;
|
|
57
92
|
1 01 坂本龍馬 ryoma-sakamoto@example.com
|
|
@@ -63,6 +98,12 @@ COPY cats (code, name) FROM stdin;
|
|
|
63
98
|
02 Hana
|
|
64
99
|
03 Leo
|
|
65
100
|
\.
|
|
101
|
+
COPY dogs (code, name, house_id, room_id) FROM stdin;
|
|
102
|
+
01 Pochi 1 1
|
|
103
|
+
02 Rose 2 1
|
|
104
|
+
03 Momo 1 1
|
|
105
|
+
04 Sakura 1 2
|
|
106
|
+
\.
|
|
66
107
|
```
|
|
67
108
|
|
|
68
109
|
Output file (the sensitive values are masked):
|
|
@@ -70,9 +111,11 @@ Output file (the sensitive values are masked):
|
|
|
70
111
|
```sql
|
|
71
112
|
INSERT INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','氏名1','email-1@example.com'),(2,'02','氏名2','email-2@example.com'),(3,'03','氏名3','email-3@example.com');
|
|
72
113
|
INSERT INTO `cats` (`code`, `name`) VALUES ('code-1','Cat name 1'),('code-2','Cat name 2'),('code-3','Cat name 3');
|
|
114
|
+
INSERT INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('code-1','Dog name 1',1,1),('code-1','Dog name 1',2,1),('code-2','Dog name 2',1,1),('code-1','Dog name 1',1,2);
|
|
73
115
|
|
|
74
116
|
REPLACE INTO `people` (`id`, `code`, `name`, `email`) VALUES (1,'01','氏名1','email-1@example.com'),(2,'02','氏名2','email-2@example.com'),(3,'03','氏名3','email-3@example.com');
|
|
75
117
|
REPLACE INTO `cats` (`code`, `name`) VALUES ('code-1','Cat name 1'),('code-2','Cat name 2'),('code-3','Cat name 3');
|
|
118
|
+
REPLACE INTO `dogs` (`code`, `name`, `house_id`, `room_id`) VALUES ('code-1','Dog name 1',1,1),('code-1','Dog name 1',2,1),('code-2','Dog name 2',1,1),('code-1','Dog name 1',1,2);
|
|
76
119
|
|
|
77
120
|
COPY people (id, code, name, email) FROM stdin;
|
|
78
121
|
1 01 氏名1 email-1@example.com
|
|
@@ -84,6 +127,12 @@ code-1 Cat name 1
|
|
|
84
127
|
code-2 Cat name 2
|
|
85
128
|
code-3 Cat name 3
|
|
86
129
|
\.
|
|
130
|
+
COPY dogs (code, name, house_id, room_id) FROM stdin;
|
|
131
|
+
code-1 Dog name 1 1 1
|
|
132
|
+
code-1 Dog name 1 2 1
|
|
133
|
+
code-2 Dog name 2 1 1
|
|
134
|
+
code-1 Dog name 1 1 2
|
|
135
|
+
\.
|
|
87
136
|
```
|
|
88
137
|
|
|
89
138
|
Config file:
|
|
@@ -93,14 +142,22 @@ mark: '[mask]'
|
|
|
93
142
|
targets:
|
|
94
143
|
- table: people
|
|
95
144
|
columns: 4
|
|
96
|
-
|
|
145
|
+
dummy_values:
|
|
97
146
|
2: 氏名[mask]
|
|
98
147
|
3: email-[mask]@example.com
|
|
99
148
|
- table: cats
|
|
100
149
|
columns: 2
|
|
101
|
-
|
|
150
|
+
dummy_values:
|
|
102
151
|
0: code-[mask]
|
|
103
152
|
1: Cat name [mask]
|
|
153
|
+
- table: dogs
|
|
154
|
+
columns: 4
|
|
155
|
+
dummy_values:
|
|
156
|
+
0: code-[mask]
|
|
157
|
+
1: Dog name [mask]
|
|
158
|
+
group_indexes:
|
|
159
|
+
- 2
|
|
160
|
+
- 3
|
|
104
161
|
```
|
|
105
162
|
|
|
106
163
|
## Supported Ruby Versions
|
data/Rakefile
CHANGED
data/lib/mask_sql/cli.rb
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'thor'
|
|
2
2
|
require 'mask_sql/converter'
|
|
3
|
+
require 'mask_sql/initializer'
|
|
3
4
|
|
|
4
5
|
module MaskSQL
|
|
5
6
|
class CLI < Thor
|
|
@@ -14,6 +15,8 @@ module MaskSQL
|
|
|
14
15
|
option :copy, type: :boolean, banner: 'MASK `COPY` SQL'
|
|
15
16
|
|
|
16
17
|
def mask
|
|
18
|
+
return unless validate_options
|
|
19
|
+
|
|
17
20
|
converter_options = options.dup
|
|
18
21
|
|
|
19
22
|
if options[:config]
|
|
@@ -28,11 +31,31 @@ module MaskSQL
|
|
|
28
31
|
puts "\e[32mDone.\e[0m"
|
|
29
32
|
end
|
|
30
33
|
|
|
34
|
+
desc 'init', 'Generate a config file'
|
|
35
|
+
|
|
36
|
+
def init
|
|
37
|
+
puts Initializer.copy_template
|
|
38
|
+
end
|
|
39
|
+
|
|
31
40
|
desc 'version, -v, --version', 'Print the version'
|
|
32
|
-
map %w
|
|
41
|
+
map %w[-v --version] => :version
|
|
33
42
|
|
|
34
43
|
def version
|
|
35
44
|
puts "mask_sql #{MaskSQL::VERSION}"
|
|
36
45
|
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def validate_options
|
|
50
|
+
in_file = File.expand_path(options[:in])
|
|
51
|
+
out_file = File.expand_path(options[:out])
|
|
52
|
+
|
|
53
|
+
if in_file == out_file
|
|
54
|
+
warn "\e[31mOutput file is the same as input file.\e[0m"
|
|
55
|
+
return false
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
true
|
|
59
|
+
end
|
|
37
60
|
end
|
|
38
61
|
end
|
data/lib/mask_sql/converter.rb
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
require 'csv'
|
|
2
1
|
require 'yaml'
|
|
3
2
|
require 'nkf'
|
|
4
3
|
|
|
@@ -20,69 +19,72 @@ module MaskSQL
|
|
|
20
19
|
@matched_copy = {}
|
|
21
20
|
end
|
|
22
21
|
|
|
23
|
-
def mask
|
|
24
|
-
encoding
|
|
22
|
+
def mask(encoding = nil)
|
|
23
|
+
encoding ||= NKF.guess(File.read(@options[:in])).name
|
|
25
24
|
|
|
26
25
|
File.open(@options[:out], "w:#{encoding}") do |out_file|
|
|
27
26
|
File.open(@options[:in], "r:#{encoding}") do |in_file|
|
|
28
27
|
in_file.each_line do |line|
|
|
29
|
-
@matched_copy.empty?
|
|
28
|
+
if @matched_copy.empty?
|
|
29
|
+
@counters = []
|
|
30
|
+
write_line(line, out_file)
|
|
31
|
+
else
|
|
32
|
+
write_copy_line(line, out_file)
|
|
33
|
+
end
|
|
30
34
|
end
|
|
31
35
|
end
|
|
32
36
|
end
|
|
37
|
+
rescue Encoding::UndefinedConversionError => e
|
|
38
|
+
raise Encoding::UndefinedConversionError, e.message if encoding == Encoding::UTF_8.name
|
|
39
|
+
mask(Encoding::UTF_8.name)
|
|
33
40
|
end
|
|
34
41
|
|
|
35
42
|
private
|
|
36
43
|
|
|
37
44
|
def write_line(line, output_file)
|
|
38
45
|
@targets.each do |target|
|
|
39
|
-
|
|
40
|
-
matched_line = match_line(line, table)
|
|
46
|
+
matched_line = match_line(line, target['table'])
|
|
41
47
|
next unless matched_line
|
|
42
48
|
|
|
43
49
|
if matched_line.names.include?('copy_sql')
|
|
44
50
|
output_file.puts line
|
|
45
|
-
|
|
46
|
-
@matched_copy[:indexes] = target['indexes']
|
|
47
|
-
@matched_copy[:record_index] = 1
|
|
51
|
+
init_matched_copy(target)
|
|
48
52
|
return
|
|
49
53
|
end
|
|
50
54
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
record_values.map!.with_index(1) do |values, record_index|
|
|
56
|
-
indexes.each do |mask_index|
|
|
57
|
-
before_value = values[mask_index]
|
|
58
|
-
values[mask_index] = target['indexes'][mask_index].gsub(@mark, record_index.to_s)
|
|
59
|
-
values[mask_index].insert(0, "'") if before_value.start_with?("'", "('")
|
|
60
|
-
values[mask_index].insert(-1, "'") if before_value.end_with?("'", "')")
|
|
61
|
-
values[mask_index].insert(0, '(') if mask_index == 0
|
|
62
|
-
values[mask_index].insert(-1, ')') if mask_index == columns - 1
|
|
63
|
-
end
|
|
64
|
-
values
|
|
65
|
-
end
|
|
55
|
+
all_values = parse_all_values(matched_line[:all_values])
|
|
56
|
+
|
|
57
|
+
record_values = get_record_values(all_values, target['columns'])
|
|
58
|
+
masked_values = mask_values(record_values, target)
|
|
66
59
|
|
|
67
|
-
output_file.puts "#{matched_line[:prefix]}#{
|
|
60
|
+
output_file.puts "#{matched_line[:prefix]}#{masked_values.join(',')}#{matched_line[:suffix]}"
|
|
68
61
|
return
|
|
69
62
|
end
|
|
70
63
|
|
|
71
64
|
output_file.puts line
|
|
72
65
|
end
|
|
73
66
|
|
|
67
|
+
def init_matched_copy(target)
|
|
68
|
+
@matched_copy[:dummy_values] = target['dummy_values']
|
|
69
|
+
@matched_copy[:group_indexes] = target['group_indexes'] || []
|
|
70
|
+
@matched_copy[:record_index] = 1
|
|
71
|
+
@counters = []
|
|
72
|
+
end
|
|
73
|
+
|
|
74
74
|
def write_copy_line(line, output_file)
|
|
75
|
-
if
|
|
75
|
+
if /^\\.$/ =~ line
|
|
76
76
|
output_file.puts line
|
|
77
77
|
@matched_copy.clear
|
|
78
78
|
return
|
|
79
79
|
end
|
|
80
80
|
|
|
81
|
-
record_values =
|
|
82
|
-
@matched_copy[:
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
81
|
+
record_values = line.split("\t")
|
|
82
|
+
count = get_current_count(record_values, @matched_copy[:record_index], @matched_copy[:group_indexes])
|
|
83
|
+
|
|
84
|
+
@matched_copy[:dummy_values].each do |dummy_index, dummy_value|
|
|
85
|
+
record_values[dummy_index] = dummy_value.sub(/^'/, '')
|
|
86
|
+
.sub(/'$/, '')
|
|
87
|
+
.gsub(@mark, count.to_s)
|
|
86
88
|
end
|
|
87
89
|
|
|
88
90
|
output_file.puts record_values.join("\t")
|
|
@@ -90,33 +92,120 @@ module MaskSQL
|
|
|
90
92
|
end
|
|
91
93
|
|
|
92
94
|
def match_line(line, table)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
95
|
+
matched_line = match_insert(line, table)
|
|
96
|
+
return matched_line if matched_line
|
|
97
|
+
|
|
98
|
+
matched_line = match_replace(line, table)
|
|
99
|
+
return matched_line if matched_line
|
|
100
|
+
|
|
101
|
+
matched_line = match_copy(line, table)
|
|
102
|
+
return matched_line if matched_line
|
|
103
|
+
|
|
104
|
+
nil
|
|
105
|
+
end
|
|
97
106
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
107
|
+
def match_insert(line, table)
|
|
108
|
+
return unless @options[:insert]
|
|
109
|
+
/^(?<prefix>INSERT (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)$/i.match(line)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def match_replace(line, table)
|
|
113
|
+
return unless @options[:replace]
|
|
114
|
+
/^(?<prefix>REPLACE (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)$/i.match(line)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def match_copy(line, table)
|
|
118
|
+
return unless @options[:copy]
|
|
119
|
+
/^(?<copy_sql>COPY\s*`?#{table}`?.*FROM stdin;)$/i.match(line)
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def parse_all_values(matched_all_values)
|
|
123
|
+
all_values = matched_all_values.chomp.split(',')
|
|
124
|
+
processing_index = 0
|
|
125
|
+
|
|
126
|
+
all_values.map!.with_index do |value, index|
|
|
127
|
+
next if index != 0 && index <= processing_index
|
|
128
|
+
|
|
129
|
+
if start_string?(value)
|
|
130
|
+
processing_value = value.dup
|
|
131
|
+
processing_index = index
|
|
132
|
+
|
|
133
|
+
until end_string?(processing_value)
|
|
134
|
+
processing_index += 1
|
|
135
|
+
processing_value += all_values[processing_index]
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
value = processing_value
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
value
|
|
101
142
|
end
|
|
102
143
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
144
|
+
all_values.compact
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def start_string?(value)
|
|
148
|
+
value == "'" || value == "('" || (value.start_with?("'", "('") && !value.end_with?("'", "')"))
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def end_string?(value)
|
|
152
|
+
value != "'" && value != "('" && value.end_with?("'", "')")
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def get_record_values(all_values, columns)
|
|
156
|
+
all_values.each_slice(columns).to_a
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def mask_values(record_values, target)
|
|
160
|
+
columns = target['columns']
|
|
161
|
+
dummy_values = target['dummy_values']
|
|
162
|
+
group_indexes = target['group_indexes'] || []
|
|
163
|
+
|
|
164
|
+
record_values.map!.with_index(1) do |values, record_index|
|
|
165
|
+
count = get_current_count(values, record_index, group_indexes)
|
|
166
|
+
|
|
167
|
+
dummy_values.each_key do |dummy_index|
|
|
168
|
+
original_value = values[dummy_index]
|
|
169
|
+
masked_value = dummy_values[dummy_index].gsub(@mark, count.to_s)
|
|
170
|
+
values[dummy_index] = mask_value(masked_value, original_value, dummy_index, columns)
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
values
|
|
106
174
|
end
|
|
107
175
|
|
|
108
|
-
|
|
176
|
+
record_values
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def get_current_count(values, record_index, group_indexes)
|
|
180
|
+
return record_index if group_indexes.empty?
|
|
181
|
+
|
|
182
|
+
group_values = group_indexes.map do |group_index|
|
|
183
|
+
values[group_index]
|
|
184
|
+
end
|
|
185
|
+
increment_count(group_values)
|
|
109
186
|
end
|
|
110
187
|
|
|
111
|
-
def
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
/\A(?<prefix>INSERT (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)\Z/i
|
|
115
|
-
when :replace
|
|
116
|
-
/\A(?<prefix>REPLACE (INTO)?\s*`?#{table}`?.*VALUES\s*)(?<all_values>[^;]+)(?<suffix>;?)\Z/i
|
|
117
|
-
when :copy
|
|
118
|
-
/(?<copy_sql>COPY\s*`?#{table}`?.*FROM stdin;)/i
|
|
188
|
+
def increment_count(group_values)
|
|
189
|
+
counter = @counters.find do |c|
|
|
190
|
+
c[:label] == group_values
|
|
119
191
|
end
|
|
192
|
+
|
|
193
|
+
if counter
|
|
194
|
+
counter[:count] += 1
|
|
195
|
+
else
|
|
196
|
+
counter = { label: group_values, count: 1 }
|
|
197
|
+
@counters.push(counter)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
counter[:count]
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def mask_value(masked_value, original_value, mask_index, columns)
|
|
204
|
+
masked_value.insert(0, "'") if original_value.start_with?("'", "('")
|
|
205
|
+
masked_value.insert(-1, "'") if original_value.end_with?("'", "')")
|
|
206
|
+
masked_value.insert(0, '(') if mask_index.zero?
|
|
207
|
+
masked_value.insert(-1, ')') if mask_index == columns - 1
|
|
208
|
+
masked_value
|
|
120
209
|
end
|
|
121
210
|
end
|
|
122
211
|
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
module MaskSQL
|
|
2
|
+
class Initializer
|
|
3
|
+
TEMPLATE_CONFIG_FILE = '.mask.yml'.freeze
|
|
4
|
+
|
|
5
|
+
def self.copy_template
|
|
6
|
+
to = File.expand_path(TEMPLATE_CONFIG_FILE)
|
|
7
|
+
return "\e[33mexist #{to}\e[0m" if FileTest.exist?(to)
|
|
8
|
+
|
|
9
|
+
from = File.expand_path("../initializer/#{TEMPLATE_CONFIG_FILE}", __FILE__)
|
|
10
|
+
FileUtils.copy(from, to)
|
|
11
|
+
"\e[32mcreate #{to}\e[0m"
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# This file was generated by the `mask_sql init` command.
|
|
2
|
+
# Edit this file to configure the masking behavior.
|
|
3
|
+
# See https://github.com/emsk/mask_sql for more details.
|
|
4
|
+
|
|
5
|
+
# mark: '[mask]'
|
|
6
|
+
# targets:
|
|
7
|
+
# - table: people
|
|
8
|
+
# columns: 4
|
|
9
|
+
# dummy_values:
|
|
10
|
+
# 2: 氏名[mask]
|
|
11
|
+
# 3: email-[mask]@example.com
|
|
12
|
+
# - table: cats
|
|
13
|
+
# columns: 2
|
|
14
|
+
# dummy_values:
|
|
15
|
+
# 0: code-[mask]
|
|
16
|
+
# 1: Cat name [mask]
|
|
17
|
+
# - table: dogs
|
|
18
|
+
# columns: 4
|
|
19
|
+
# dummy_values:
|
|
20
|
+
# 0: code-[mask]
|
|
21
|
+
# 1: Dog name [mask]
|
|
22
|
+
# group_indexes:
|
|
23
|
+
# - 2
|
|
24
|
+
# - 3
|
data/lib/mask_sql/version.rb
CHANGED
data/mask_sql.gemspec
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
+
|
|
2
3
|
lib = File.expand_path('../lib', __FILE__)
|
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
|
4
5
|
require 'mask_sql/version'
|
|
@@ -9,8 +10,8 @@ Gem::Specification.new do |spec|
|
|
|
9
10
|
spec.authors = ['emsk']
|
|
10
11
|
spec.email = ['emsk1987@gmail.com']
|
|
11
12
|
|
|
12
|
-
spec.summary =
|
|
13
|
-
spec.description =
|
|
13
|
+
spec.summary = 'Mask sensitive values in a SQL file'
|
|
14
|
+
spec.description = 'MaskSQL is a command-line tool to mask sensitive values in a SQL file'
|
|
14
15
|
spec.homepage = 'https://github.com/emsk/mask_sql'
|
|
15
16
|
spec.license = 'MIT'
|
|
16
17
|
|
|
@@ -21,11 +22,13 @@ Gem::Specification.new do |spec|
|
|
|
21
22
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
22
23
|
spec.require_paths = ['lib']
|
|
23
24
|
|
|
25
|
+
spec.required_ruby_version = '>= 2.0.0'
|
|
26
|
+
|
|
24
27
|
spec.add_runtime_dependency 'thor', '~> 0.19'
|
|
25
28
|
spec.add_development_dependency 'bundler', '~> 1.14'
|
|
26
29
|
spec.add_development_dependency 'coveralls', '~> 0.8'
|
|
27
30
|
spec.add_development_dependency 'rake', '~> 10.0'
|
|
28
|
-
spec.add_development_dependency 'rspec', '~> 3.
|
|
29
|
-
spec.add_development_dependency 'rubocop', '~> 0.
|
|
31
|
+
spec.add_development_dependency 'rspec', '~> 3.6'
|
|
32
|
+
spec.add_development_dependency 'rubocop', '~> 0.49'
|
|
30
33
|
spec.add_development_dependency 'simplecov', '~> 0.14'
|
|
31
34
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: mask_sql
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- emsk
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-05-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: thor
|
|
@@ -72,28 +72,28 @@ dependencies:
|
|
|
72
72
|
requirements:
|
|
73
73
|
- - "~>"
|
|
74
74
|
- !ruby/object:Gem::Version
|
|
75
|
-
version: '3.
|
|
75
|
+
version: '3.6'
|
|
76
76
|
type: :development
|
|
77
77
|
prerelease: false
|
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
|
79
79
|
requirements:
|
|
80
80
|
- - "~>"
|
|
81
81
|
- !ruby/object:Gem::Version
|
|
82
|
-
version: '3.
|
|
82
|
+
version: '3.6'
|
|
83
83
|
- !ruby/object:Gem::Dependency
|
|
84
84
|
name: rubocop
|
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
|
86
86
|
requirements:
|
|
87
87
|
- - "~>"
|
|
88
88
|
- !ruby/object:Gem::Version
|
|
89
|
-
version: '0.
|
|
89
|
+
version: '0.49'
|
|
90
90
|
type: :development
|
|
91
91
|
prerelease: false
|
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
|
93
93
|
requirements:
|
|
94
94
|
- - "~>"
|
|
95
95
|
- !ruby/object:Gem::Version
|
|
96
|
-
version: '0.
|
|
96
|
+
version: '0.49'
|
|
97
97
|
- !ruby/object:Gem::Dependency
|
|
98
98
|
name: simplecov
|
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -121,6 +121,7 @@ files:
|
|
|
121
121
|
- ".rubocop.yml"
|
|
122
122
|
- ".rubocop_todo.yml"
|
|
123
123
|
- ".travis.yml"
|
|
124
|
+
- CHANGELOG.md
|
|
124
125
|
- Gemfile
|
|
125
126
|
- LICENSE.txt
|
|
126
127
|
- README.md
|
|
@@ -131,6 +132,8 @@ files:
|
|
|
131
132
|
- lib/mask_sql.rb
|
|
132
133
|
- lib/mask_sql/cli.rb
|
|
133
134
|
- lib/mask_sql/converter.rb
|
|
135
|
+
- lib/mask_sql/initializer.rb
|
|
136
|
+
- lib/mask_sql/initializer/.mask.yml
|
|
134
137
|
- lib/mask_sql/version.rb
|
|
135
138
|
- mask_sql.gemspec
|
|
136
139
|
homepage: https://github.com/emsk/mask_sql
|
|
@@ -145,7 +148,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
145
148
|
requirements:
|
|
146
149
|
- - ">="
|
|
147
150
|
- !ruby/object:Gem::Version
|
|
148
|
-
version:
|
|
151
|
+
version: 2.0.0
|
|
149
152
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
150
153
|
requirements:
|
|
151
154
|
- - ">="
|