masking 1.1.2.pre.alpha → 1.1.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8fff3fddd7077702babd4aeffafe86e80f79ab5151fb40ba92bba3ebf86513e3
4
- data.tar.gz: f9f7d5ed2dbd5c1ebb6789d154c6c6fe64b72d4528ee923a6c4d0b8575e869ba
3
+ metadata.gz: 369c9cd9c3b91a8d363815ea703220366b59cac10122c0de7e0300748ad07470
4
+ data.tar.gz: b033a8bf26ed0b4f1521173d8ac130be2f7624ec3a008c3c3e965c373413a80c
5
5
  SHA512:
6
- metadata.gz: 909c267ad406fc1bded87cb368cd5b130612270e2a41046c129cc529f9dd3fbb29b1d799a71613e404333463c7af6f2a33f323b0e3a29af204aae0c48c96e54d
7
- data.tar.gz: 618c7ff871b6ae32fc26095d0e409ffe67f1da6c4759a1a9ae45a93bc3336f1f775352a514d095955882c33bb874133df3d258af9c3040a4f6ed0d7bbf354789
6
+ metadata.gz: 06c178bec7f1a615cef67727159a46e5b8cdaaa4f98a5f773cb2cec8a3becf9647cdbde30f2471b579bc59fb55ffb4656b5ecbfab6d35e3cbbea5510d53bd0ff
7
+ data.tar.gz: be2c12a8cc0b91e4cfdc8371b33b4909f0cc828f24a5cd6ec355e7dd5197e9d8704c0bcbb702a02d204ffb24e523ae32a012886c71e64302d88bcb10c9d178ca
@@ -5,7 +5,30 @@ on:
5
5
  - main
6
6
  pull_request:
7
7
 
8
+ env:
9
+ DEBUG: ${{ vars.DEBUG }}
10
+
8
11
  jobs:
12
+ mariadb111:
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - uses: actions/checkout@v1
16
+ with:
17
+ fetch-depth: 1 # shallow clone
18
+ - name: run test
19
+ run: docker-compose/acceptance_test.sh mariadb111 docker-compose-ci.yml
20
+ env:
21
+ TRACE: ${{ vars.TRACE }}
22
+ mariadb110:
23
+ runs-on: ubuntu-latest
24
+ steps:
25
+ - uses: actions/checkout@v1
26
+ with:
27
+ fetch-depth: 1 # shallow clone
28
+ - name: run test
29
+ run: docker-compose/acceptance_test.sh mariadb110 docker-compose-ci.yml
30
+ env:
31
+ TRACE: ${{ vars.TRACE }}
9
32
  mariadb1011:
10
33
  runs-on: ubuntu-latest
11
34
  steps:
@@ -14,6 +37,8 @@ jobs:
14
37
  fetch-depth: 1 # shallow clone
15
38
  - name: run test
16
39
  run: docker-compose/acceptance_test.sh mariadb1011 docker-compose-ci.yml
40
+ env:
41
+ TRACE: ${{ vars.TRACE }}
17
42
  mariadb1010:
18
43
  runs-on: ubuntu-latest
19
44
  steps:
@@ -22,6 +47,8 @@ jobs:
22
47
  fetch-depth: 1 # shallow clone
23
48
  - name: run test
24
49
  run: docker-compose/acceptance_test.sh mariadb1010 docker-compose-ci.yml
50
+ env:
51
+ TRACE: ${{ vars.TRACE }}
25
52
  mariadb109:
26
53
  runs-on: ubuntu-latest
27
54
  steps:
@@ -30,6 +57,8 @@ jobs:
30
57
  fetch-depth: 1 # shallow clone
31
58
  - name: run test
32
59
  run: docker-compose/acceptance_test.sh mariadb109 docker-compose-ci.yml
60
+ env:
61
+ TRACE: ${{ vars.TRACE }}
33
62
  mariadb108:
34
63
  runs-on: ubuntu-latest
35
64
  steps:
@@ -38,6 +67,8 @@ jobs:
38
67
  fetch-depth: 1 # shallow clone
39
68
  - name: run test
40
69
  run: docker-compose/acceptance_test.sh mariadb108 docker-compose-ci.yml
70
+ env:
71
+ TRACE: ${{ vars.TRACE }}
41
72
  mariadb107:
42
73
  runs-on: ubuntu-latest
43
74
  steps:
@@ -46,6 +77,8 @@ jobs:
46
77
  fetch-depth: 1 # shallow clone
47
78
  - name: run test
48
79
  run: docker-compose/acceptance_test.sh mariadb107 docker-compose-ci.yml
80
+ env:
81
+ TRACE: ${{ vars.TRACE }}
49
82
  mariadb106:
50
83
  runs-on: ubuntu-latest
51
84
  steps:
@@ -54,6 +87,8 @@ jobs:
54
87
  fetch-depth: 1 # shallow clone
55
88
  - name: run test
56
89
  run: docker-compose/acceptance_test.sh mariadb106 docker-compose-ci.yml
90
+ env:
91
+ TRACE: ${{ vars.TRACE }}
57
92
  mariadb105:
58
93
  runs-on: ubuntu-latest
59
94
  steps:
@@ -62,6 +97,8 @@ jobs:
62
97
  fetch-depth: 1 # shallow clone
63
98
  - name: run test
64
99
  run: docker-compose/acceptance_test.sh mariadb105 docker-compose-ci.yml
100
+ env:
101
+ TRACE: ${{ vars.TRACE }}
65
102
  mariadb104:
66
103
  runs-on: ubuntu-latest
67
104
  steps:
@@ -70,6 +107,8 @@ jobs:
70
107
  fetch-depth: 1 # shallow clone
71
108
  - name: run test
72
109
  run: docker-compose/acceptance_test.sh mariadb104 docker-compose-ci.yml
110
+ env:
111
+ TRACE: ${{ vars.TRACE }}
73
112
  mariadb103:
74
113
  runs-on: ubuntu-latest
75
114
  steps:
@@ -78,6 +117,8 @@ jobs:
78
117
  fetch-depth: 1 # shallow clone
79
118
  - name: run test
80
119
  run: docker-compose/acceptance_test.sh mariadb103 docker-compose-ci.yml
120
+ env:
121
+ TRACE: ${{ vars.TRACE }}
81
122
  mariadb102:
82
123
  runs-on: ubuntu-latest
83
124
  steps:
@@ -86,27 +127,5 @@ jobs:
86
127
  fetch-depth: 1 # shallow clone
87
128
  - name: run test
88
129
  run: docker-compose/acceptance_test.sh mariadb102 docker-compose-ci.yml
89
- mariadb101:
90
- runs-on: ubuntu-latest
91
- steps:
92
- - uses: actions/checkout@v1
93
- with:
94
- fetch-depth: 1 # shallow clone
95
- - name: run test
96
- run: docker-compose/acceptance_test.sh mariadb101 docker-compose-ci.yml
97
- mariadb100:
98
- runs-on: ubuntu-latest
99
- steps:
100
- - uses: actions/checkout@v1
101
- with:
102
- fetch-depth: 1 # shallow clone
103
- - name: run test
104
- run: docker-compose/acceptance_test.sh mariadb100 docker-compose-ci.yml
105
- mariadb55:
106
- runs-on: ubuntu-latest
107
- steps:
108
- - uses: actions/checkout@v1
109
- with:
110
- fetch-depth: 1 # shallow clone
111
- - name: run test
112
- run: docker-compose/acceptance_test.sh mariadb55 docker-compose-ci.yml
130
+ env:
131
+ TRACE: ${{ vars.TRACE }}
@@ -6,35 +6,33 @@ on:
6
6
  pull_request:
7
7
 
8
8
  jobs:
9
- mysql80:
10
- runs-on: ubuntu-latest
11
- steps:
12
- - uses: actions/checkout@v1
13
- with:
14
- fetch-depth: 1 # shallow clone
15
- - name: run test
16
- run: docker-compose/acceptance_test.sh mysql80 docker-compose-ci.yml
17
- mysql57:
9
+ mysql81:
18
10
  runs-on: ubuntu-latest
19
11
  steps:
20
12
  - uses: actions/checkout@v1
21
13
  with:
22
14
  fetch-depth: 1 # shallow clone
23
15
  - name: run test
24
- run: docker-compose/acceptance_test.sh mysql57 docker-compose-ci.yml
25
- mysql56:
16
+ run: docker-compose/acceptance_test.sh mysql81 docker-compose-ci.yml
17
+ env:
18
+ TRACE: ${{ vars.TRACE }}
19
+ mysql80:
26
20
  runs-on: ubuntu-latest
27
21
  steps:
28
22
  - uses: actions/checkout@v1
29
23
  with:
30
24
  fetch-depth: 1 # shallow clone
31
25
  - name: run test
32
- run: docker-compose/acceptance_test.sh mysql56 docker-compose-ci.yml
33
- mysql55:
26
+ run: docker-compose/acceptance_test.sh mysql80 docker-compose-ci.yml
27
+ env:
28
+ TRACE: ${{ vars.TRACE }}
29
+ mysql57:
34
30
  runs-on: ubuntu-latest
35
31
  steps:
36
32
  - uses: actions/checkout@v1
37
33
  with:
38
34
  fetch-depth: 1 # shallow clone
39
35
  - name: run test
40
- run: docker-compose/acceptance_test.sh mysql55 docker-compose-ci.yml
36
+ run: docker-compose/acceptance_test.sh mysql57 docker-compose-ci.yml
37
+ env:
38
+ TRACE: ${{ vars.TRACE }}
data/CHANGELOG.md CHANGED
@@ -5,16 +5,22 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
- ## [Unreleased]
8
+ ## [v1.1.2] - 2023-09-24
9
9
 
10
10
  ### Added
11
11
 
12
12
  - adding dockerfile for github container images [#77](https://github.com/kibitan/masking/pull/77)
13
13
  - Update README.md about frequent asked question [#74](https://github.com/kibitan/masking/pull/74)
14
+ - enable codescene and add status badge [#80](https://github.com/kibitan/masking/pull/80)
15
+ - Drop support for MySQL 5.5, 5.6 and MariaDB 5.5, 10.0, 10.1 (EOL) [#90](https://github.com/kibitan/masking/pull/90) relates issue [#89](https://github.com/kibitan/masking/issues/89)
16
+ - Add support for MySQL 8.1 and MariaDB 11.0/11.1 [#91](https://github.com/kibitan/masking/pull/90) relates issue [#82](https://github.com/kibitan/masking/issues/89)
17
+ - add presentation and demo link [#87](https://github.com/kibitan/masking/pull/87)
18
+ - Add comment for complicated part of code [#83](https://github.com/kibitan/masking/pull/83)
14
19
 
15
20
  ### Fix
16
21
 
17
22
  - fix the error on the error message [#74](https://github.com/kibitan/masking/pull/74) relates issue [#73](https://github.com/kibitan/masking/issues/73)
23
+ - fix ci error [#74](https://github.com/kibitan/masking/pull/86)
18
24
 
19
25
  ## [v1.1.1] - 2023-04-03
20
26
 
@@ -86,7 +92,9 @@ $ bin/benchmark.rb
86
92
 
87
93
  Initial release version. 🎉
88
94
 
89
- [Unreleased]: https://github.com/kibitan/masking/compare/v1.1.0...HEAD
95
+ [Unreleased]: https://github.com/kibitan/masking/compare/v1.1.2...HEAD
96
+ [v1.1.2]: https://github.com/kibitan/masking/compare/v1.1.1...v1.1.2
97
+ [v1.1.1]: https://github.com/kibitan/masking/compare/v1.1.0...v1.1.1
90
98
  [v1.1.0]: https://github.com/kibitan/masking/compare/v1.0.1...v1.1.0
91
99
  [v1.0.1]: https://github.com/kibitan/masking/compare/v1.0.0...v1.0.1
92
100
  [v1.0.0]: https://github.com/kibitan/masking/compare/v0.0.3...v1.0.0
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- masking (1.1.2.pre.alpha)
4
+ masking (1.1.2)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
data/README.md CHANGED
@@ -6,7 +6,12 @@
6
6
 
7
7
  [![codecov](https://codecov.io/gh/kibitan/masking/branch/main/graph/badge.svg)](https://codecov.io/gh/kibitan/masking)
8
8
  [![Maintainability](https://api.codeclimate.com/v1/badges/290b3005ecc193a3d138/maintainability)](https://codeclimate.com/github/kibitan/masking/maintainability)
9
+ [![CodeScene Code Health](https://codescene.io/projects/38627/status-badges/code-health)](https://codescene.io/projects/38627)
9
10
  [![Gem Version](https://badge.fury.io/rb/masking.svg)](https://badge.fury.io/rb/masking)
11
+ <!--
12
+ [![CodeScene System Mastery](https://codescene.io/projects/38627/status-badges/system-mastery)](https://codescene.io/projects/38627)
13
+ [![CodeScene Missed Goals](https://codescene.io/projects/38627/status-badges/missed-goals)](https://codescene.io/projects/38627)
14
+ -->
10
15
 
11
16
  The command line tool for anonymizing database records by parsing a SQL dump file and build a new SQL dump file with masking sensitive/credential data.
12
17
 
@@ -22,8 +27,8 @@ gem install masking
22
27
 
23
28
  ## Supporting RDBMS
24
29
 
25
- * MySQL: 5.5<sup>[1](#footnote1)</sup>, 5.6, 5.7, 8.0
26
- * MariaDB: 5.5, 10.0<sup>[2](#footnote2)</sup>, 10.1, 10.2, 10.3, 10.4, 10.5, 10.6, 10.7, 10.8, 10.9, 10.10, 10.11
30
+ * MySQL: 5.7, 8.0, 8.1
31
+ * MariaDB: 10.2<sup>[1](#footnote1)</sup>, 10.3<sup>[1](#footnote1)</sup>, 10.4, 10.5, 10.6, 10.7<sup>[1](#footnote1)</sup>, 10.8<sup>[1](#footnote1)</sup>, 10.9<sup>[1](#footnote1)</sup>, 10.10, 10.11, 11.0, 11.1
27
32
 
28
33
  ## Usage
29
34
 
@@ -93,6 +98,12 @@ Usage: masking [options]
93
98
  -v, --version version
94
99
  ```
95
100
 
101
+ ## presentation / demo
102
+
103
+ [![IMAGE ALT TEXT HERE](https://img.youtube.com/vi/tnGLUhmHclI/0.jpg)](https://www.youtube.com/watch?v=tnGLUhmHclI)
104
+
105
+ presentation: [Generate anonymised database with MasKING](https://speakerdeck.com/kibitan/generate-anonymised-database-with-masking-2023-dot-09-dot-21-euruko-unconference-talk)
106
+
96
107
  ## Use case of anonymized (production) database
97
108
 
98
109
  * Analyzing production databases for BI, Machine Learning, troubleshooting with respecting GDPR
@@ -147,6 +158,8 @@ available option via environment variable:
147
158
  * `MYSQL_PASSWORD`: password for user(default: `password`)
148
159
  * `MYSQL_DBNAME`: database name(default: `mydb`)
149
160
 
161
+ NOTE: run with `TRACE=1` will show debug print. for the CI, `TRACE` environment variable on [setting field in the repository](https://github.com/kibitan/masking/settings/variables/actions/TRACE)
162
+
150
163
  ##### with docker-compose
151
164
 
152
165
  ```bash
@@ -161,23 +174,21 @@ docker-compose/acceptance_test.sh mysql80
161
174
 
162
175
  The docker-compose file names for other database versions, specify that file.
163
176
 
177
+ * MySQL 8.1: [`docker-compose/mysql80.yml`](./docker-compose/mysql81.yml)
164
178
  * MySQL 8.0: [`docker-compose/mysql80.yml`](./docker-compose/mysql80.yml)
165
179
  * MySQL 5.7: [`docker-compose/mysql57.yml`](./docker-compose/mysql57.yml)
166
- * MySQL 5.6: [`docker-compose/mysql56.yml`](./docker-compose/mysql56.yml)
167
- * MySQL 5.5<sup>[1](#footnote1)</sup>: [`docker-compose/mysql55.yml`](./docker-compose/mysql55.yml)
180
+ * MariaDB 11.1: [`docker-compose/mariadb111.yml`](./docker-compose/mariadb111.yml)
181
+ * MariaDB 11.0: [`docker-compose/mariadb110.yml`](./docker-compose/mariadb110.yml)
168
182
  * MariaDB 10.11: [`docker-compose/mariadb1011.yml`](./docker-compose/mariadb1011.yml)
169
183
  * MariaDB 10.10: [`docker-compose/mariadb1010.yml`](./docker-compose/mariadb1010.yml)
170
- * MariaDB 10.9: [`docker-compose/mariadb109.yml`](./docker-compose/mariadb109.yml)
171
- * MariaDB 10.8: [`docker-compose/mariadb108.yml`](./docker-compose/mariadb108.yml)
172
- * MariaDB 10.7: [`docker-compose/mariadb107.yml`](./docker-compose/mariadb107.yml)
184
+ * MariaDB 10.9<sup>[1](#footnote1): [`docker-compose/mariadb109.yml`](./docker-compose/mariadb109.yml)
185
+ * MariaDB 10.8<sup>[1](#footnote1): [`docker-compose/mariadb108.yml`](./docker-compose/mariadb108.yml)
186
+ * MariaDB 10.7<sup>[1](#footnote1): [`docker-compose/mariadb107.yml`](./docker-compose/mariadb107.yml)
173
187
  * MariaDB 10.6: [`docker-compose/mariadb106.yml`](./docker-compose/mariadb106.yml)
174
188
  * MariaDB 10.5: [`docker-compose/mariadb105.yml`](./docker-compose/mariadb105.yml)
175
189
  * MariaDB 10.4: [`docker-compose/mariadb104.yml`](./docker-compose/mariadb104.yml)
176
- * MariaDB 10.3: [`docker-compose/mariadb103.yml`](./docker-compose/mariadb103.yml)
177
- * MariaDB 10.2: [`docker-compose/mariadb102.yml`](./docker-compose/mariadb102.yml)
178
- * MariaDB 10.1: [`docker-compose/mariadb101.yml`](./docker-compose/mariadb101.yml)
179
- * MariaDB 10.0<sup>[2](#footnote2)</sup>: [`docker-compose/mariadb100.yml`](./docker-compose/mariadb100.yml)
180
- * MariaDB 5.5: [`docker-compose/mariadb55.yml`](./docker-compose/mariadb55.yml)
190
+ * MariaDB 10.3<sup>[1](#footnote1): [`docker-compose/mariadb103.yml`](./docker-compose/mariadb103.yml)
191
+ * MariaDB 10.2<sup>[1](#footnote1)</sup>: [`docker-compose/mariadb102.yml`](./docker-compose/mariadb102.yml)
181
192
 
182
193
  #### [Markdown lint](https://github.com/markdownlint/markdownlint)
183
194
 
@@ -252,6 +263,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
252
263
 
253
264
  Everyone interacting in the Masking project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/kibitan/masking/blob/main/CODE_OF_CONDUCT.md).
254
265
 
255
- <a name="footnote1">1</a>: <small> MySQL 5.5 is already not supported by [official](https://www.mysql.com/support/supportedplatforms/database.html)</small>
256
-
257
- <a name="footnote2">2</a>: <small> MariaDB 10.0 is already not supported by [official](https://mariadb.org/about/maintenance-policy/)</small>
266
+ <a name="footnote1">1</a>: <small> MariaDB 10.2, 10.3, 10.7, 10.8, 10.9 is already not supported by [official](https://mariadb.org/about/maintenance-policy/)</small>
@@ -1,37 +1,50 @@
1
1
  #!/bin/sh
2
+ set -eu -o pipefail
3
+ set -C # Prevent output redirection using ‘>’, ‘>&’, and ‘<>’ from overwriting existing files.
2
4
 
3
- set -Ceu -o pipefail
4
- # set -vx # for debug
5
+ if [[ "${TRACE-0}" == "1" ]]; then
6
+ set -vx
7
+ fi
5
8
 
6
9
  MYSQL_HOST=${MYSQL_HOST:-localhost}
7
- MYSQL_USER=${MYSQL_USER:-mysqluser}
8
- MYSQL_PASSWORD=${MYSQL_PASSWORD:-password}
10
+ MYSQL_USER=${MYSQL_USER:-root}
11
+ MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root_password}
12
+ MYSQL_PASSWORD=${MYSQL_PASSWORD:-MYSQL_ROOT_PASSWORD}
9
13
  MYSQL_DBNAME=${MYSQL_DBNAME:-mydb}
14
+ MYSQL_ANONYMIZED_DBNAME=${MYSQL_ANONYMIZED_DBNAME:-anonnymized_mydb}
10
15
 
11
16
  FILEDIR="$( cd "$( dirname "$0" )" && pwd )"
12
17
 
13
- # clear tmp file
14
- rm "$FILEDIR"/tmp/* || echo 'no tmp file'
15
-
16
- # import database
17
- mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/import_dumpfile.sql"
18
-
19
- # masking & restore
20
- mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
21
- mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/tmp/masking_dumpfile.sql"
22
- ## compare the result
23
- mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" -e 'SELECT * FROM users ORDER BY id;' --vertical > "$FILEDIR/tmp/query_result.txt"
24
- diff "$FILEDIR/expected_query_result.txt" "$FILEDIR/tmp/query_result.txt" || (echo 'test failed' && exit 1)
25
-
26
- # test errors
27
- set +e
28
- ## without masking.yml
29
- mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/no_file.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
30
- ## without `--complete-insert``
31
- mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" | exe/masking -c "$FILEDIR/masking.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
32
- set -e
33
- ### compare the result
34
- diff "$FILEDIR/expected_error_result.txt" "$FILEDIR/tmp/errors.txt" || (echo 'error output test failed' && exit 1)
35
-
36
- echo 'test passed!'
37
- exit 0
18
+ main() {
19
+ # clear tmp file
20
+ rm "$FILEDIR"/tmp/* || echo 'no tmp file'
21
+
22
+ # import database
23
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/import_dumpfile.sql"
24
+
25
+ # masking & restore
26
+ ## TODO: temporary add `--skip-extended-insert` as not working now
27
+ # mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
28
+ mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert --skip-extended-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
29
+
30
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "CREATE DATABASE $MYSQL_ANONYMIZED_DBNAME;"
31
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ANONYMIZED_DBNAME" < "$FILEDIR/tmp/masking_dumpfile.sql"
32
+ ## compare the result
33
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ANONYMIZED_DBNAME" -e 'SELECT * FROM users ORDER BY id;' --vertical > "$FILEDIR/tmp/query_result.txt"
34
+ diff "$FILEDIR/expected_query_result.txt" "$FILEDIR/tmp/query_result.txt" || (echo 'test failed' && exit 1)
35
+
36
+ # test errors
37
+ set +e
38
+ ## without masking.yml
39
+ mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/no_file.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
40
+ ## without `--complete-insert``
41
+ mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" | exe/masking -c "$FILEDIR/masking.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
42
+ set -e
43
+ ### compare the result
44
+ diff "$FILEDIR/expected_error_result.txt" "$FILEDIR/tmp/errors.txt" || (echo 'error output test failed' && exit 1)
45
+
46
+ echo 'test passed!'
47
+ exit 0
48
+ }
49
+
50
+ main "$@"
@@ -1,8 +1,19 @@
1
1
  #!/usr/bin/env bash
2
+ set -eu -o pipefail
3
+ set -C # Prevent output redirection using ‘>’, ‘>&’, and ‘<>’ from overwriting existing files.
2
4
 
3
- set -Ceu -o pipefail
4
- #set -vx # for debug
5
+ if [[ "${TRACE-0}" == "1" ]]; then
6
+ set -vx
7
+ fi
8
+
9
+ cd "$(dirname "$0")"
5
10
 
6
11
  MYSQL_VERSION=${1:-mysql80}
7
12
  DOCKER_COMPOSE_FILE=${2:-docker-compose.yml}
8
- docker-compose -f "$DOCKER_COMPOSE_FILE" -f "docker-compose/$MYSQL_VERSION.yml" run -e "MYSQL_HOST=$MYSQL_VERSION" app acceptance/run_test.sh
13
+ TRACE=${TRACE:-0}
14
+
15
+ main() {
16
+ docker-compose -f "../$DOCKER_COMPOSE_FILE" -f "./$MYSQL_VERSION.yml" run -e "MYSQL_HOST=$MYSQL_VERSION" -e "TRACE=$TRACE" app acceptance/run_test.sh
17
+ }
18
+
19
+ main "$@"
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb1010
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb1010
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb1010:
12
16
  image: mariadb:10.10
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb1011
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb1011
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb1011:
12
16
  image: mariadb:10.11
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb102
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb102
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb102:
12
16
  image: mariadb:10.2
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb103
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb103
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb103:
12
16
  image: mariadb:10.3
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb104
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb104
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb104:
12
16
  image: mariadb:10.4
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb105
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb105
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb105:
12
16
  image: mariadb:10.5
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb106
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb106
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb106:
12
16
  image: mariadb:10.6
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb107
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb107
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb107:
12
16
  image: mariadb:10.7
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb108
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb108
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb108:
12
16
  image: mariadb:10.8
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb109
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb109
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb109:
12
16
  image: mariadb:10.9
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -0,0 +1,20 @@
1
+ version: '3.7'
2
+
3
+ services:
4
+ app:
5
+ build:
6
+ target: with-mysql-client
7
+ depends_on:
8
+ - mariadb110
9
+ entrypoint: docker-compose/wait-for-mysql.sh mariadb110
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
14
+
15
+ mariadb110:
16
+ image: mariadb:11.0
17
+ restart: always
18
+ environment:
19
+ MYSQL_ROOT_PASSWORD: root_password
20
+ MYSQL_DATABASE: mydb
@@ -0,0 +1,20 @@
1
+ version: '3.7'
2
+
3
+ services:
4
+ app:
5
+ build:
6
+ target: with-mysql-client
7
+ depends_on:
8
+ - mariadb111
9
+ entrypoint: docker-compose/wait-for-mysql.sh mariadb111
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
14
+
15
+ mariadb111:
16
+ image: mariadb:11.1
17
+ restart: always
18
+ environment:
19
+ MYSQL_ROOT_PASSWORD: root_password
20
+ MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mysql57
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mysql57
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mysql57:
12
16
  image: mysql:5.7
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,13 +7,16 @@ services:
7
7
  depends_on:
8
8
  - mysql80
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mysql80
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mysql80:
12
16
  image: mysql:8.0
13
17
  # fix for: ERROR 1045 (28000): Plugin caching_sha2_password could not be loaded:
14
18
  command: --default-authentication-plugin=mysql_native_password
19
+ restart: always
15
20
  environment:
16
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
17
- MYSQL_USER: mysqluser
18
- MYSQL_PASSWORD: password
21
+ MYSQL_ROOT_PASSWORD: root_password
19
22
  MYSQL_DATABASE: mydb
@@ -0,0 +1,22 @@
1
+ version: '3.7'
2
+
3
+ services:
4
+ app:
5
+ build:
6
+ target: with-mysql-client
7
+ depends_on:
8
+ - mysql81
9
+ entrypoint: docker-compose/wait-for-mysql.sh mysql81
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
14
+
15
+ mysql81:
16
+ image: mysql:8.1
17
+ # fix for: ERROR 1045 (28000): Plugin caching_sha2_password could not be loaded:
18
+ command: --default-authentication-plugin=mysql_native_password
19
+ restart: always
20
+ environment:
21
+ MYSQL_ROOT_PASSWORD: root_password
22
+ MYSQL_DATABASE: mydb
@@ -3,7 +3,7 @@
3
3
 
4
4
  set -e
5
5
 
6
- MYSQL_USER=${MYSQL_USER:-mysqluser}
6
+ MYSQL_USER=${MYSQL_USER:-root}
7
7
  MYSQL_PASSWORD=${MYSQL_PASSWORD:-password}
8
8
 
9
9
  MYSQL_HOST="$1"
@@ -29,8 +29,12 @@ module Masking
29
29
  end
30
30
 
31
31
  def values
32
+ # NOTE: the reason to use `rows.each_with_index()`
33
+ # another simple implementations (e.g. `rows.count.time`) doesn't work.because during the block loop,
34
+ # the `rows` array object can be destructively changed by the #recursive_pattern_value_concat! method and
35
+ # it make different number of count to loop during the block loop, so it needs to be checked by the object size
32
36
  @values ||= values_section.split(VALUE_ROW_SPLITTER)
33
- .tap { |rows| rows.each_with_index { |_, i| recursive_pattern_value_concat(rows, i) } }
37
+ .tap { |rows| rows.each_with_index { |_, i| recursive_pattern_value_concat!(rows, i) } }
34
38
  .flat_map { |row| row.scan(values_regexp) }
35
39
  end
36
40
 
@@ -74,11 +78,12 @@ module Masking
74
78
  # if it's odd, concat with next row (it means a value contains "),(" pattern)
75
79
  # e.g. INSERT ... VALUES (123,'string ),( abc'),(456,'ab');
76
80
  # refs: implementation of parsing CSV on ruby standard library FasterCSV (ja): https://www.clear-code.com/blog/2018/12/25.html
77
- def recursive_pattern_value_concat(value_rows, index)
81
+ def recursive_pattern_value_concat!(value_rows, index)
78
82
  return if value_rows[index].gsub(/\\\\/, '').gsub(/\\'/, '').count(?').even?
79
83
 
84
+ # make destructive change for values_rows
80
85
  value_rows[index] += VALUE_ROW_SPLITTER + value_rows.delete_at(index + 1)
81
- recursive_pattern_value_concat(value_rows, index)
86
+ recursive_pattern_value_concat!(value_rows, index)
82
87
  end
83
88
  end
84
89
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Masking
4
- VERSION = '1.1.2-alpha'
4
+ VERSION = '1.1.2'
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: masking
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2.pre.alpha
4
+ version: 1.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chikahiro Tokoro
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-04-18 00:00:00.000000000 Z
11
+ date: 2023-09-24 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Command line tool for anonymizing database records by parsing a SQL dump
14
14
  file and build new SQL dump file with masking sensitive/credential data.
@@ -57,8 +57,6 @@ files:
57
57
  - docker-compose-ci.yml
58
58
  - docker-compose.yml
59
59
  - docker-compose/acceptance_test.sh
60
- - docker-compose/mariadb100.yml
61
- - docker-compose/mariadb101.yml
62
60
  - docker-compose/mariadb1010.yml
63
61
  - docker-compose/mariadb1011.yml
64
62
  - docker-compose/mariadb102.yml
@@ -69,11 +67,11 @@ files:
69
67
  - docker-compose/mariadb107.yml
70
68
  - docker-compose/mariadb108.yml
71
69
  - docker-compose/mariadb109.yml
72
- - docker-compose/mariadb55.yml
73
- - docker-compose/mysql55.yml
74
- - docker-compose/mysql56.yml
70
+ - docker-compose/mariadb110.yml
71
+ - docker-compose/mariadb111.yml
75
72
  - docker-compose/mysql57.yml
76
73
  - docker-compose/mysql80.yml
74
+ - docker-compose/mysql81.yml
77
75
  - docker-compose/wait-for-mysql.sh
78
76
  - exe/masking
79
77
  - lib/masking.rb
@@ -122,9 +120,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
122
120
  version: '2.6'
123
121
  required_rubygems_version: !ruby/object:Gem::Requirement
124
122
  requirements:
125
- - - ">"
123
+ - - ">="
126
124
  - !ruby/object:Gem::Version
127
- version: 1.3.1
125
+ version: '0'
128
126
  requirements: []
129
127
  rubygems_version: 3.4.6
130
128
  signing_key:
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mariadb100
9
- entrypoint: docker-compose/wait-for-mysql.sh mariadb100
10
-
11
- mariadb100:
12
- image: mariadb:10.0
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mariadb101
9
- entrypoint: docker-compose/wait-for-mysql.sh mariadb101
10
-
11
- mariadb101:
12
- image: mariadb:10.1
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mariadb55
9
- entrypoint: docker-compose/wait-for-mysql.sh mariadb55
10
-
11
- mariadb55:
12
- image: mariadb:5.5
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mysql55
9
- entrypoint: docker-compose/wait-for-mysql.sh mysql55
10
-
11
- mysql55:
12
- image: mysql:5.5
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mysql56
9
- entrypoint: docker-compose/wait-for-mysql.sh mysql56
10
-
11
- mysql56:
12
- image: mysql:5.6
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb