masking 1.1.2.pre.alpha → 1.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8fff3fddd7077702babd4aeffafe86e80f79ab5151fb40ba92bba3ebf86513e3
4
- data.tar.gz: f9f7d5ed2dbd5c1ebb6789d154c6c6fe64b72d4528ee923a6c4d0b8575e869ba
3
+ metadata.gz: 369c9cd9c3b91a8d363815ea703220366b59cac10122c0de7e0300748ad07470
4
+ data.tar.gz: b033a8bf26ed0b4f1521173d8ac130be2f7624ec3a008c3c3e965c373413a80c
5
5
  SHA512:
6
- metadata.gz: 909c267ad406fc1bded87cb368cd5b130612270e2a41046c129cc529f9dd3fbb29b1d799a71613e404333463c7af6f2a33f323b0e3a29af204aae0c48c96e54d
7
- data.tar.gz: 618c7ff871b6ae32fc26095d0e409ffe67f1da6c4759a1a9ae45a93bc3336f1f775352a514d095955882c33bb874133df3d258af9c3040a4f6ed0d7bbf354789
6
+ metadata.gz: 06c178bec7f1a615cef67727159a46e5b8cdaaa4f98a5f773cb2cec8a3becf9647cdbde30f2471b579bc59fb55ffb4656b5ecbfab6d35e3cbbea5510d53bd0ff
7
+ data.tar.gz: be2c12a8cc0b91e4cfdc8371b33b4909f0cc828f24a5cd6ec355e7dd5197e9d8704c0bcbb702a02d204ffb24e523ae32a012886c71e64302d88bcb10c9d178ca
@@ -5,7 +5,30 @@ on:
5
5
  - main
6
6
  pull_request:
7
7
 
8
+ env:
9
+ DEBUG: ${{ vars.DEBUG }}
10
+
8
11
  jobs:
12
+ mariadb111:
13
+ runs-on: ubuntu-latest
14
+ steps:
15
+ - uses: actions/checkout@v1
16
+ with:
17
+ fetch-depth: 1 # shallow clone
18
+ - name: run test
19
+ run: docker-compose/acceptance_test.sh mariadb111 docker-compose-ci.yml
20
+ env:
21
+ TRACE: ${{ vars.TRACE }}
22
+ mariadb110:
23
+ runs-on: ubuntu-latest
24
+ steps:
25
+ - uses: actions/checkout@v1
26
+ with:
27
+ fetch-depth: 1 # shallow clone
28
+ - name: run test
29
+ run: docker-compose/acceptance_test.sh mariadb110 docker-compose-ci.yml
30
+ env:
31
+ TRACE: ${{ vars.TRACE }}
9
32
  mariadb1011:
10
33
  runs-on: ubuntu-latest
11
34
  steps:
@@ -14,6 +37,8 @@ jobs:
14
37
  fetch-depth: 1 # shallow clone
15
38
  - name: run test
16
39
  run: docker-compose/acceptance_test.sh mariadb1011 docker-compose-ci.yml
40
+ env:
41
+ TRACE: ${{ vars.TRACE }}
17
42
  mariadb1010:
18
43
  runs-on: ubuntu-latest
19
44
  steps:
@@ -22,6 +47,8 @@ jobs:
22
47
  fetch-depth: 1 # shallow clone
23
48
  - name: run test
24
49
  run: docker-compose/acceptance_test.sh mariadb1010 docker-compose-ci.yml
50
+ env:
51
+ TRACE: ${{ vars.TRACE }}
25
52
  mariadb109:
26
53
  runs-on: ubuntu-latest
27
54
  steps:
@@ -30,6 +57,8 @@ jobs:
30
57
  fetch-depth: 1 # shallow clone
31
58
  - name: run test
32
59
  run: docker-compose/acceptance_test.sh mariadb109 docker-compose-ci.yml
60
+ env:
61
+ TRACE: ${{ vars.TRACE }}
33
62
  mariadb108:
34
63
  runs-on: ubuntu-latest
35
64
  steps:
@@ -38,6 +67,8 @@ jobs:
38
67
  fetch-depth: 1 # shallow clone
39
68
  - name: run test
40
69
  run: docker-compose/acceptance_test.sh mariadb108 docker-compose-ci.yml
70
+ env:
71
+ TRACE: ${{ vars.TRACE }}
41
72
  mariadb107:
42
73
  runs-on: ubuntu-latest
43
74
  steps:
@@ -46,6 +77,8 @@ jobs:
46
77
  fetch-depth: 1 # shallow clone
47
78
  - name: run test
48
79
  run: docker-compose/acceptance_test.sh mariadb107 docker-compose-ci.yml
80
+ env:
81
+ TRACE: ${{ vars.TRACE }}
49
82
  mariadb106:
50
83
  runs-on: ubuntu-latest
51
84
  steps:
@@ -54,6 +87,8 @@ jobs:
54
87
  fetch-depth: 1 # shallow clone
55
88
  - name: run test
56
89
  run: docker-compose/acceptance_test.sh mariadb106 docker-compose-ci.yml
90
+ env:
91
+ TRACE: ${{ vars.TRACE }}
57
92
  mariadb105:
58
93
  runs-on: ubuntu-latest
59
94
  steps:
@@ -62,6 +97,8 @@ jobs:
62
97
  fetch-depth: 1 # shallow clone
63
98
  - name: run test
64
99
  run: docker-compose/acceptance_test.sh mariadb105 docker-compose-ci.yml
100
+ env:
101
+ TRACE: ${{ vars.TRACE }}
65
102
  mariadb104:
66
103
  runs-on: ubuntu-latest
67
104
  steps:
@@ -70,6 +107,8 @@ jobs:
70
107
  fetch-depth: 1 # shallow clone
71
108
  - name: run test
72
109
  run: docker-compose/acceptance_test.sh mariadb104 docker-compose-ci.yml
110
+ env:
111
+ TRACE: ${{ vars.TRACE }}
73
112
  mariadb103:
74
113
  runs-on: ubuntu-latest
75
114
  steps:
@@ -78,6 +117,8 @@ jobs:
78
117
  fetch-depth: 1 # shallow clone
79
118
  - name: run test
80
119
  run: docker-compose/acceptance_test.sh mariadb103 docker-compose-ci.yml
120
+ env:
121
+ TRACE: ${{ vars.TRACE }}
81
122
  mariadb102:
82
123
  runs-on: ubuntu-latest
83
124
  steps:
@@ -86,27 +127,5 @@ jobs:
86
127
  fetch-depth: 1 # shallow clone
87
128
  - name: run test
88
129
  run: docker-compose/acceptance_test.sh mariadb102 docker-compose-ci.yml
89
- mariadb101:
90
- runs-on: ubuntu-latest
91
- steps:
92
- - uses: actions/checkout@v1
93
- with:
94
- fetch-depth: 1 # shallow clone
95
- - name: run test
96
- run: docker-compose/acceptance_test.sh mariadb101 docker-compose-ci.yml
97
- mariadb100:
98
- runs-on: ubuntu-latest
99
- steps:
100
- - uses: actions/checkout@v1
101
- with:
102
- fetch-depth: 1 # shallow clone
103
- - name: run test
104
- run: docker-compose/acceptance_test.sh mariadb100 docker-compose-ci.yml
105
- mariadb55:
106
- runs-on: ubuntu-latest
107
- steps:
108
- - uses: actions/checkout@v1
109
- with:
110
- fetch-depth: 1 # shallow clone
111
- - name: run test
112
- run: docker-compose/acceptance_test.sh mariadb55 docker-compose-ci.yml
130
+ env:
131
+ TRACE: ${{ vars.TRACE }}
@@ -6,35 +6,33 @@ on:
6
6
  pull_request:
7
7
 
8
8
  jobs:
9
- mysql80:
10
- runs-on: ubuntu-latest
11
- steps:
12
- - uses: actions/checkout@v1
13
- with:
14
- fetch-depth: 1 # shallow clone
15
- - name: run test
16
- run: docker-compose/acceptance_test.sh mysql80 docker-compose-ci.yml
17
- mysql57:
9
+ mysql81:
18
10
  runs-on: ubuntu-latest
19
11
  steps:
20
12
  - uses: actions/checkout@v1
21
13
  with:
22
14
  fetch-depth: 1 # shallow clone
23
15
  - name: run test
24
- run: docker-compose/acceptance_test.sh mysql57 docker-compose-ci.yml
25
- mysql56:
16
+ run: docker-compose/acceptance_test.sh mysql81 docker-compose-ci.yml
17
+ env:
18
+ TRACE: ${{ vars.TRACE }}
19
+ mysql80:
26
20
  runs-on: ubuntu-latest
27
21
  steps:
28
22
  - uses: actions/checkout@v1
29
23
  with:
30
24
  fetch-depth: 1 # shallow clone
31
25
  - name: run test
32
- run: docker-compose/acceptance_test.sh mysql56 docker-compose-ci.yml
33
- mysql55:
26
+ run: docker-compose/acceptance_test.sh mysql80 docker-compose-ci.yml
27
+ env:
28
+ TRACE: ${{ vars.TRACE }}
29
+ mysql57:
34
30
  runs-on: ubuntu-latest
35
31
  steps:
36
32
  - uses: actions/checkout@v1
37
33
  with:
38
34
  fetch-depth: 1 # shallow clone
39
35
  - name: run test
40
- run: docker-compose/acceptance_test.sh mysql55 docker-compose-ci.yml
36
+ run: docker-compose/acceptance_test.sh mysql57 docker-compose-ci.yml
37
+ env:
38
+ TRACE: ${{ vars.TRACE }}
data/CHANGELOG.md CHANGED
@@ -5,16 +5,22 @@ All notable changes to this project will be documented in this file.
5
5
  The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6
6
  and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7
7
 
8
- ## [Unreleased]
8
+ ## [v1.1.2] - 2023-09-24
9
9
 
10
10
  ### Added
11
11
 
12
12
  - adding dockerfile for github container images [#77](https://github.com/kibitan/masking/pull/77)
13
13
  - Update README.md about frequent asked question [#74](https://github.com/kibitan/masking/pull/74)
14
+ - enable codescene and add status badge [#80](https://github.com/kibitan/masking/pull/80)
15
+ - Drop support for MySQL 5.5, 5.6 and MariaDB 5.5, 10.0, 10.1 (EOL) [#90](https://github.com/kibitan/masking/pull/90) relates issue [#89](https://github.com/kibitan/masking/issues/89)
16
+ - Add support for MySQL 8.1 and MariaDB 11.0/11.1 [#91](https://github.com/kibitan/masking/pull/90) relates issue [#82](https://github.com/kibitan/masking/issues/89)
17
+ - add presentation and demo link [#87](https://github.com/kibitan/masking/pull/87)
18
+ - Add comment for complicated part of code [#83](https://github.com/kibitan/masking/pull/83)
14
19
 
15
20
  ### Fix
16
21
 
17
22
  - fix the error on the error message [#74](https://github.com/kibitan/masking/pull/74) relates issue [#73](https://github.com/kibitan/masking/issues/73)
23
+ - fix ci error [#74](https://github.com/kibitan/masking/pull/86)
18
24
 
19
25
  ## [v1.1.1] - 2023-04-03
20
26
 
@@ -86,7 +92,9 @@ $ bin/benchmark.rb
86
92
 
87
93
  Initial release version. 🎉
88
94
 
89
- [Unreleased]: https://github.com/kibitan/masking/compare/v1.1.0...HEAD
95
+ [Unreleased]: https://github.com/kibitan/masking/compare/v1.1.2...HEAD
96
+ [v1.1.2]: https://github.com/kibitan/masking/compare/v1.1.1...v1.1.2
97
+ [v1.1.1]: https://github.com/kibitan/masking/compare/v1.1.0...v1.1.1
90
98
  [v1.1.0]: https://github.com/kibitan/masking/compare/v1.0.1...v1.1.0
91
99
  [v1.0.1]: https://github.com/kibitan/masking/compare/v1.0.0...v1.0.1
92
100
  [v1.0.0]: https://github.com/kibitan/masking/compare/v0.0.3...v1.0.0
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- masking (1.1.2.pre.alpha)
4
+ masking (1.1.2)
5
5
 
6
6
  GEM
7
7
  remote: https://rubygems.org/
data/README.md CHANGED
@@ -6,7 +6,12 @@
6
6
 
7
7
  [![codecov](https://codecov.io/gh/kibitan/masking/branch/main/graph/badge.svg)](https://codecov.io/gh/kibitan/masking)
8
8
  [![Maintainability](https://api.codeclimate.com/v1/badges/290b3005ecc193a3d138/maintainability)](https://codeclimate.com/github/kibitan/masking/maintainability)
9
+ [![CodeScene Code Health](https://codescene.io/projects/38627/status-badges/code-health)](https://codescene.io/projects/38627)
9
10
  [![Gem Version](https://badge.fury.io/rb/masking.svg)](https://badge.fury.io/rb/masking)
11
+ <!--
12
+ [![CodeScene System Mastery](https://codescene.io/projects/38627/status-badges/system-mastery)](https://codescene.io/projects/38627)
13
+ [![CodeScene Missed Goals](https://codescene.io/projects/38627/status-badges/missed-goals)](https://codescene.io/projects/38627)
14
+ -->
10
15
 
11
16
  The command line tool for anonymizing database records by parsing a SQL dump file and build a new SQL dump file with masking sensitive/credential data.
12
17
 
@@ -22,8 +27,8 @@ gem install masking
22
27
 
23
28
  ## Supporting RDBMS
24
29
 
25
- * MySQL: 5.5<sup>[1](#footnote1)</sup>, 5.6, 5.7, 8.0
26
- * MariaDB: 5.5, 10.0<sup>[2](#footnote2)</sup>, 10.1, 10.2, 10.3, 10.4, 10.5, 10.6, 10.7, 10.8, 10.9, 10.10, 10.11
30
+ * MySQL: 5.7, 8.0, 8.1
31
+ * MariaDB: 10.2<sup>[1](#footnote1)</sup>, 10.3<sup>[1](#footnote1)</sup>, 10.4, 10.5, 10.6, 10.7<sup>[1](#footnote1)</sup>, 10.8<sup>[1](#footnote1)</sup>, 10.9<sup>[1](#footnote1)</sup>, 10.10, 10.11, 11.0, 11.1
27
32
 
28
33
  ## Usage
29
34
 
@@ -93,6 +98,12 @@ Usage: masking [options]
93
98
  -v, --version version
94
99
  ```
95
100
 
101
+ ## presentation / demo
102
+
103
+ [![IMAGE ALT TEXT HERE](https://img.youtube.com/vi/tnGLUhmHclI/0.jpg)](https://www.youtube.com/watch?v=tnGLUhmHclI)
104
+
105
+ presentation: [Generate anonymised database with MasKING](https://speakerdeck.com/kibitan/generate-anonymised-database-with-masking-2023-dot-09-dot-21-euruko-unconference-talk)
106
+
96
107
  ## Use case of anonymized (production) database
97
108
 
98
109
  * Analyzing production databases for BI, Machine Learning, troubleshooting with respecting GDPR
@@ -147,6 +158,8 @@ available option via environment variable:
147
158
  * `MYSQL_PASSWORD`: password for user(default: `password`)
148
159
  * `MYSQL_DBNAME`: database name(default: `mydb`)
149
160
 
161
+ NOTE: run with `TRACE=1` will show debug print. for the CI, `TRACE` environment variable on [setting field in the repository](https://github.com/kibitan/masking/settings/variables/actions/TRACE)
162
+
150
163
  ##### with docker-compose
151
164
 
152
165
  ```bash
@@ -161,23 +174,21 @@ docker-compose/acceptance_test.sh mysql80
161
174
 
162
175
  The docker-compose file names for other database versions, specify that file.
163
176
 
177
+ * MySQL 8.1: [`docker-compose/mysql80.yml`](./docker-compose/mysql81.yml)
164
178
  * MySQL 8.0: [`docker-compose/mysql80.yml`](./docker-compose/mysql80.yml)
165
179
  * MySQL 5.7: [`docker-compose/mysql57.yml`](./docker-compose/mysql57.yml)
166
- * MySQL 5.6: [`docker-compose/mysql56.yml`](./docker-compose/mysql56.yml)
167
- * MySQL 5.5<sup>[1](#footnote1)</sup>: [`docker-compose/mysql55.yml`](./docker-compose/mysql55.yml)
180
+ * MariaDB 11.1: [`docker-compose/mariadb111.yml`](./docker-compose/mariadb111.yml)
181
+ * MariaDB 11.0: [`docker-compose/mariadb110.yml`](./docker-compose/mariadb110.yml)
168
182
  * MariaDB 10.11: [`docker-compose/mariadb1011.yml`](./docker-compose/mariadb1011.yml)
169
183
  * MariaDB 10.10: [`docker-compose/mariadb1010.yml`](./docker-compose/mariadb1010.yml)
170
- * MariaDB 10.9: [`docker-compose/mariadb109.yml`](./docker-compose/mariadb109.yml)
171
- * MariaDB 10.8: [`docker-compose/mariadb108.yml`](./docker-compose/mariadb108.yml)
172
- * MariaDB 10.7: [`docker-compose/mariadb107.yml`](./docker-compose/mariadb107.yml)
184
+ * MariaDB 10.9<sup>[1](#footnote1): [`docker-compose/mariadb109.yml`](./docker-compose/mariadb109.yml)
185
+ * MariaDB 10.8<sup>[1](#footnote1): [`docker-compose/mariadb108.yml`](./docker-compose/mariadb108.yml)
186
+ * MariaDB 10.7<sup>[1](#footnote1): [`docker-compose/mariadb107.yml`](./docker-compose/mariadb107.yml)
173
187
  * MariaDB 10.6: [`docker-compose/mariadb106.yml`](./docker-compose/mariadb106.yml)
174
188
  * MariaDB 10.5: [`docker-compose/mariadb105.yml`](./docker-compose/mariadb105.yml)
175
189
  * MariaDB 10.4: [`docker-compose/mariadb104.yml`](./docker-compose/mariadb104.yml)
176
- * MariaDB 10.3: [`docker-compose/mariadb103.yml`](./docker-compose/mariadb103.yml)
177
- * MariaDB 10.2: [`docker-compose/mariadb102.yml`](./docker-compose/mariadb102.yml)
178
- * MariaDB 10.1: [`docker-compose/mariadb101.yml`](./docker-compose/mariadb101.yml)
179
- * MariaDB 10.0<sup>[2](#footnote2)</sup>: [`docker-compose/mariadb100.yml`](./docker-compose/mariadb100.yml)
180
- * MariaDB 5.5: [`docker-compose/mariadb55.yml`](./docker-compose/mariadb55.yml)
190
+ * MariaDB 10.3<sup>[1](#footnote1): [`docker-compose/mariadb103.yml`](./docker-compose/mariadb103.yml)
191
+ * MariaDB 10.2<sup>[1](#footnote1)</sup>: [`docker-compose/mariadb102.yml`](./docker-compose/mariadb102.yml)
181
192
 
182
193
  #### [Markdown lint](https://github.com/markdownlint/markdownlint)
183
194
 
@@ -252,6 +263,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
252
263
 
253
264
  Everyone interacting in the Masking project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/kibitan/masking/blob/main/CODE_OF_CONDUCT.md).
254
265
 
255
- <a name="footnote1">1</a>: <small> MySQL 5.5 is already not supported by [official](https://www.mysql.com/support/supportedplatforms/database.html)</small>
256
-
257
- <a name="footnote2">2</a>: <small> MariaDB 10.0 is already not supported by [official](https://mariadb.org/about/maintenance-policy/)</small>
266
+ <a name="footnote1">1</a>: <small> MariaDB 10.2, 10.3, 10.7, 10.8, 10.9 is already not supported by [official](https://mariadb.org/about/maintenance-policy/)</small>
@@ -1,37 +1,50 @@
1
1
  #!/bin/sh
2
+ set -eu -o pipefail
3
+ set -C # Prevent output redirection using ‘>’, ‘>&’, and ‘<>’ from overwriting existing files.
2
4
 
3
- set -Ceu -o pipefail
4
- # set -vx # for debug
5
+ if [[ "${TRACE-0}" == "1" ]]; then
6
+ set -vx
7
+ fi
5
8
 
6
9
  MYSQL_HOST=${MYSQL_HOST:-localhost}
7
- MYSQL_USER=${MYSQL_USER:-mysqluser}
8
- MYSQL_PASSWORD=${MYSQL_PASSWORD:-password}
10
+ MYSQL_USER=${MYSQL_USER:-root}
11
+ MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root_password}
12
+ MYSQL_PASSWORD=${MYSQL_PASSWORD:-MYSQL_ROOT_PASSWORD}
9
13
  MYSQL_DBNAME=${MYSQL_DBNAME:-mydb}
14
+ MYSQL_ANONYMIZED_DBNAME=${MYSQL_ANONYMIZED_DBNAME:-anonnymized_mydb}
10
15
 
11
16
  FILEDIR="$( cd "$( dirname "$0" )" && pwd )"
12
17
 
13
- # clear tmp file
14
- rm "$FILEDIR"/tmp/* || echo 'no tmp file'
15
-
16
- # import database
17
- mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/import_dumpfile.sql"
18
-
19
- # masking & restore
20
- mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
21
- mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/tmp/masking_dumpfile.sql"
22
- ## compare the result
23
- mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" -e 'SELECT * FROM users ORDER BY id;' --vertical > "$FILEDIR/tmp/query_result.txt"
24
- diff "$FILEDIR/expected_query_result.txt" "$FILEDIR/tmp/query_result.txt" || (echo 'test failed' && exit 1)
25
-
26
- # test errors
27
- set +e
28
- ## without masking.yml
29
- mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/no_file.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
30
- ## without `--complete-insert``
31
- mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" | exe/masking -c "$FILEDIR/masking.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
32
- set -e
33
- ### compare the result
34
- diff "$FILEDIR/expected_error_result.txt" "$FILEDIR/tmp/errors.txt" || (echo 'error output test failed' && exit 1)
35
-
36
- echo 'test passed!'
37
- exit 0
18
+ main() {
19
+ # clear tmp file
20
+ rm "$FILEDIR"/tmp/* || echo 'no tmp file'
21
+
22
+ # import database
23
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/import_dumpfile.sql"
24
+
25
+ # masking & restore
26
+ ## TODO: temporary add `--skip-extended-insert` as not working now
27
+ # mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
28
+ mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert --skip-extended-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
29
+
30
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "CREATE DATABASE $MYSQL_ANONYMIZED_DBNAME;"
31
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ANONYMIZED_DBNAME" < "$FILEDIR/tmp/masking_dumpfile.sql"
32
+ ## compare the result
33
+ mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ANONYMIZED_DBNAME" -e 'SELECT * FROM users ORDER BY id;' --vertical > "$FILEDIR/tmp/query_result.txt"
34
+ diff "$FILEDIR/expected_query_result.txt" "$FILEDIR/tmp/query_result.txt" || (echo 'test failed' && exit 1)
35
+
36
+ # test errors
37
+ set +e
38
+ ## without masking.yml
39
+ mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/no_file.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
40
+ ## without `--complete-insert``
41
+ mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" | exe/masking -c "$FILEDIR/masking.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
42
+ set -e
43
+ ### compare the result
44
+ diff "$FILEDIR/expected_error_result.txt" "$FILEDIR/tmp/errors.txt" || (echo 'error output test failed' && exit 1)
45
+
46
+ echo 'test passed!'
47
+ exit 0
48
+ }
49
+
50
+ main "$@"
@@ -1,8 +1,19 @@
1
1
  #!/usr/bin/env bash
2
+ set -eu -o pipefail
3
+ set -C # Prevent output redirection using ‘>’, ‘>&’, and ‘<>’ from overwriting existing files.
2
4
 
3
- set -Ceu -o pipefail
4
- #set -vx # for debug
5
+ if [[ "${TRACE-0}" == "1" ]]; then
6
+ set -vx
7
+ fi
8
+
9
+ cd "$(dirname "$0")"
5
10
 
6
11
  MYSQL_VERSION=${1:-mysql80}
7
12
  DOCKER_COMPOSE_FILE=${2:-docker-compose.yml}
8
- docker-compose -f "$DOCKER_COMPOSE_FILE" -f "docker-compose/$MYSQL_VERSION.yml" run -e "MYSQL_HOST=$MYSQL_VERSION" app acceptance/run_test.sh
13
+ TRACE=${TRACE:-0}
14
+
15
+ main() {
16
+ docker-compose -f "../$DOCKER_COMPOSE_FILE" -f "./$MYSQL_VERSION.yml" run -e "MYSQL_HOST=$MYSQL_VERSION" -e "TRACE=$TRACE" app acceptance/run_test.sh
17
+ }
18
+
19
+ main "$@"
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb1010
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb1010
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb1010:
12
16
  image: mariadb:10.10
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb1011
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb1011
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb1011:
12
16
  image: mariadb:10.11
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb102
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb102
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb102:
12
16
  image: mariadb:10.2
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb103
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb103
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb103:
12
16
  image: mariadb:10.3
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb104
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb104
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb104:
12
16
  image: mariadb:10.4
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb105
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb105
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb105:
12
16
  image: mariadb:10.5
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb106
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb106
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb106:
12
16
  image: mariadb:10.6
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb107
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb107
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb107:
12
16
  image: mariadb:10.7
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb108
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb108
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb108:
12
16
  image: mariadb:10.8
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mariadb109
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mariadb109
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mariadb109:
12
16
  image: mariadb:10.9
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -0,0 +1,20 @@
1
+ version: '3.7'
2
+
3
+ services:
4
+ app:
5
+ build:
6
+ target: with-mysql-client
7
+ depends_on:
8
+ - mariadb110
9
+ entrypoint: docker-compose/wait-for-mysql.sh mariadb110
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
14
+
15
+ mariadb110:
16
+ image: mariadb:11.0
17
+ restart: always
18
+ environment:
19
+ MYSQL_ROOT_PASSWORD: root_password
20
+ MYSQL_DATABASE: mydb
@@ -0,0 +1,20 @@
1
+ version: '3.7'
2
+
3
+ services:
4
+ app:
5
+ build:
6
+ target: with-mysql-client
7
+ depends_on:
8
+ - mariadb111
9
+ entrypoint: docker-compose/wait-for-mysql.sh mariadb111
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
14
+
15
+ mariadb111:
16
+ image: mariadb:11.1
17
+ restart: always
18
+ environment:
19
+ MYSQL_ROOT_PASSWORD: root_password
20
+ MYSQL_DATABASE: mydb
@@ -7,11 +7,14 @@ services:
7
7
  depends_on:
8
8
  - mysql57
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mysql57
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mysql57:
12
16
  image: mysql:5.7
17
+ restart: always
13
18
  environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
19
+ MYSQL_ROOT_PASSWORD: root_password
17
20
  MYSQL_DATABASE: mydb
@@ -7,13 +7,16 @@ services:
7
7
  depends_on:
8
8
  - mysql80
9
9
  entrypoint: docker-compose/wait-for-mysql.sh mysql80
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
10
14
 
11
15
  mysql80:
12
16
  image: mysql:8.0
13
17
  # fix for: ERROR 1045 (28000): Plugin caching_sha2_password could not be loaded:
14
18
  command: --default-authentication-plugin=mysql_native_password
19
+ restart: always
15
20
  environment:
16
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
17
- MYSQL_USER: mysqluser
18
- MYSQL_PASSWORD: password
21
+ MYSQL_ROOT_PASSWORD: root_password
19
22
  MYSQL_DATABASE: mydb
@@ -0,0 +1,22 @@
1
+ version: '3.7'
2
+
3
+ services:
4
+ app:
5
+ build:
6
+ target: with-mysql-client
7
+ depends_on:
8
+ - mysql81
9
+ entrypoint: docker-compose/wait-for-mysql.sh mysql81
10
+ restart: always
11
+ environment:
12
+ MYSQL_USER: root
13
+ MYSQL_PASSWORD: root_password
14
+
15
+ mysql81:
16
+ image: mysql:8.1
17
+ # fix for: ERROR 1045 (28000): Plugin caching_sha2_password could not be loaded:
18
+ command: --default-authentication-plugin=mysql_native_password
19
+ restart: always
20
+ environment:
21
+ MYSQL_ROOT_PASSWORD: root_password
22
+ MYSQL_DATABASE: mydb
@@ -3,7 +3,7 @@
3
3
 
4
4
  set -e
5
5
 
6
- MYSQL_USER=${MYSQL_USER:-mysqluser}
6
+ MYSQL_USER=${MYSQL_USER:-root}
7
7
  MYSQL_PASSWORD=${MYSQL_PASSWORD:-password}
8
8
 
9
9
  MYSQL_HOST="$1"
@@ -29,8 +29,12 @@ module Masking
29
29
  end
30
30
 
31
31
  def values
32
+ # NOTE: the reason to use `rows.each_with_index()`
33
+ # another simple implementations (e.g. `rows.count.time`) doesn't work.because during the block loop,
34
+ # the `rows` array object can be destructively changed by the #recursive_pattern_value_concat! method and
35
+ # it make different number of count to loop during the block loop, so it needs to be checked by the object size
32
36
  @values ||= values_section.split(VALUE_ROW_SPLITTER)
33
- .tap { |rows| rows.each_with_index { |_, i| recursive_pattern_value_concat(rows, i) } }
37
+ .tap { |rows| rows.each_with_index { |_, i| recursive_pattern_value_concat!(rows, i) } }
34
38
  .flat_map { |row| row.scan(values_regexp) }
35
39
  end
36
40
 
@@ -74,11 +78,12 @@ module Masking
74
78
  # if it's odd, concat with next row (it means a value contains "),(" pattern)
75
79
  # e.g. INSERT ... VALUES (123,'string ),( abc'),(456,'ab');
76
80
  # refs: implementation of parsing CSV on ruby standard library FasterCSV (ja): https://www.clear-code.com/blog/2018/12/25.html
77
- def recursive_pattern_value_concat(value_rows, index)
81
+ def recursive_pattern_value_concat!(value_rows, index)
78
82
  return if value_rows[index].gsub(/\\\\/, '').gsub(/\\'/, '').count(?').even?
79
83
 
84
+ # make destructive change for values_rows
80
85
  value_rows[index] += VALUE_ROW_SPLITTER + value_rows.delete_at(index + 1)
81
- recursive_pattern_value_concat(value_rows, index)
86
+ recursive_pattern_value_concat!(value_rows, index)
82
87
  end
83
88
  end
84
89
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Masking
4
- VERSION = '1.1.2-alpha'
4
+ VERSION = '1.1.2'
5
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: masking
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.2.pre.alpha
4
+ version: 1.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Chikahiro Tokoro
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2023-04-18 00:00:00.000000000 Z
11
+ date: 2023-09-24 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Command line tool for anonymizing database records by parsing a SQL dump
14
14
  file and build new SQL dump file with masking sensitive/credential data.
@@ -57,8 +57,6 @@ files:
57
57
  - docker-compose-ci.yml
58
58
  - docker-compose.yml
59
59
  - docker-compose/acceptance_test.sh
60
- - docker-compose/mariadb100.yml
61
- - docker-compose/mariadb101.yml
62
60
  - docker-compose/mariadb1010.yml
63
61
  - docker-compose/mariadb1011.yml
64
62
  - docker-compose/mariadb102.yml
@@ -69,11 +67,11 @@ files:
69
67
  - docker-compose/mariadb107.yml
70
68
  - docker-compose/mariadb108.yml
71
69
  - docker-compose/mariadb109.yml
72
- - docker-compose/mariadb55.yml
73
- - docker-compose/mysql55.yml
74
- - docker-compose/mysql56.yml
70
+ - docker-compose/mariadb110.yml
71
+ - docker-compose/mariadb111.yml
75
72
  - docker-compose/mysql57.yml
76
73
  - docker-compose/mysql80.yml
74
+ - docker-compose/mysql81.yml
77
75
  - docker-compose/wait-for-mysql.sh
78
76
  - exe/masking
79
77
  - lib/masking.rb
@@ -122,9 +120,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
122
120
  version: '2.6'
123
121
  required_rubygems_version: !ruby/object:Gem::Requirement
124
122
  requirements:
125
- - - ">"
123
+ - - ">="
126
124
  - !ruby/object:Gem::Version
127
- version: 1.3.1
125
+ version: '0'
128
126
  requirements: []
129
127
  rubygems_version: 3.4.6
130
128
  signing_key:
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mariadb100
9
- entrypoint: docker-compose/wait-for-mysql.sh mariadb100
10
-
11
- mariadb100:
12
- image: mariadb:10.0
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mariadb101
9
- entrypoint: docker-compose/wait-for-mysql.sh mariadb101
10
-
11
- mariadb101:
12
- image: mariadb:10.1
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mariadb55
9
- entrypoint: docker-compose/wait-for-mysql.sh mariadb55
10
-
11
- mariadb55:
12
- image: mariadb:5.5
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mysql55
9
- entrypoint: docker-compose/wait-for-mysql.sh mysql55
10
-
11
- mysql55:
12
- image: mysql:5.5
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb
@@ -1,17 +0,0 @@
1
- version: '3.7'
2
-
3
- services:
4
- app:
5
- build:
6
- target: with-mysql-client
7
- depends_on:
8
- - mysql56
9
- entrypoint: docker-compose/wait-for-mysql.sh mysql56
10
-
11
- mysql56:
12
- image: mysql:5.6
13
- environment:
14
- MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
15
- MYSQL_USER: mysqluser
16
- MYSQL_PASSWORD: password
17
- MYSQL_DATABASE: mydb