masking 1.1.1 → 1.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/acceptance_test_mariadb.yml +43 -24
- data/.github/workflows/acceptance_test_mysql.yml +12 -14
- data/CHANGELOG.md +19 -2
- data/CODEOWNERS +1 -0
- data/Dockerfile +3 -3
- data/Dockerfile.ghcr +10 -0
- data/Gemfile.lock +1 -1
- data/README.md +47 -36
- data/acceptance/expected_error_result.txt +2 -0
- data/acceptance/run_test.sh +39 -14
- data/docker-compose/acceptance_test.sh +14 -3
- data/docker-compose/mariadb1010.yml +6 -3
- data/docker-compose/mariadb1011.yml +6 -3
- data/docker-compose/mariadb102.yml +6 -3
- data/docker-compose/mariadb103.yml +6 -3
- data/docker-compose/mariadb104.yml +6 -3
- data/docker-compose/mariadb105.yml +6 -3
- data/docker-compose/mariadb106.yml +6 -3
- data/docker-compose/mariadb107.yml +6 -3
- data/docker-compose/mariadb108.yml +6 -3
- data/docker-compose/mariadb109.yml +6 -3
- data/docker-compose/mariadb110.yml +20 -0
- data/docker-compose/mariadb111.yml +20 -0
- data/docker-compose/mysql57.yml +6 -3
- data/docker-compose/mysql80.yml +6 -3
- data/docker-compose/mysql81.yml +22 -0
- data/docker-compose/wait-for-mysql.sh +1 -1
- data/lib/masking/cli/error_message.rb +15 -8
- data/lib/masking/insert_statement.rb +8 -3
- data/lib/masking/version.rb +1 -1
- metadata +8 -8
- data/docker-compose/mariadb100.yml +0 -17
- data/docker-compose/mariadb101.yml +0 -17
- data/docker-compose/mariadb55.yml +0 -17
- data/docker-compose/mysql55.yml +0 -17
- data/docker-compose/mysql56.yml +0 -17
- data/lib/masking/cli/error_messages.yml +0 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 369c9cd9c3b91a8d363815ea703220366b59cac10122c0de7e0300748ad07470
|
4
|
+
data.tar.gz: b033a8bf26ed0b4f1521173d8ac130be2f7624ec3a008c3c3e965c373413a80c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 06c178bec7f1a615cef67727159a46e5b8cdaaa4f98a5f773cb2cec8a3becf9647cdbde30f2471b579bc59fb55ffb4656b5ecbfab6d35e3cbbea5510d53bd0ff
|
7
|
+
data.tar.gz: be2c12a8cc0b91e4cfdc8371b33b4909f0cc828f24a5cd6ec355e7dd5197e9d8704c0bcbb702a02d204ffb24e523ae32a012886c71e64302d88bcb10c9d178ca
|
@@ -5,7 +5,30 @@ on:
|
|
5
5
|
- main
|
6
6
|
pull_request:
|
7
7
|
|
8
|
+
env:
|
9
|
+
DEBUG: ${{ vars.DEBUG }}
|
10
|
+
|
8
11
|
jobs:
|
12
|
+
mariadb111:
|
13
|
+
runs-on: ubuntu-latest
|
14
|
+
steps:
|
15
|
+
- uses: actions/checkout@v1
|
16
|
+
with:
|
17
|
+
fetch-depth: 1 # shallow clone
|
18
|
+
- name: run test
|
19
|
+
run: docker-compose/acceptance_test.sh mariadb111 docker-compose-ci.yml
|
20
|
+
env:
|
21
|
+
TRACE: ${{ vars.TRACE }}
|
22
|
+
mariadb110:
|
23
|
+
runs-on: ubuntu-latest
|
24
|
+
steps:
|
25
|
+
- uses: actions/checkout@v1
|
26
|
+
with:
|
27
|
+
fetch-depth: 1 # shallow clone
|
28
|
+
- name: run test
|
29
|
+
run: docker-compose/acceptance_test.sh mariadb110 docker-compose-ci.yml
|
30
|
+
env:
|
31
|
+
TRACE: ${{ vars.TRACE }}
|
9
32
|
mariadb1011:
|
10
33
|
runs-on: ubuntu-latest
|
11
34
|
steps:
|
@@ -14,6 +37,8 @@ jobs:
|
|
14
37
|
fetch-depth: 1 # shallow clone
|
15
38
|
- name: run test
|
16
39
|
run: docker-compose/acceptance_test.sh mariadb1011 docker-compose-ci.yml
|
40
|
+
env:
|
41
|
+
TRACE: ${{ vars.TRACE }}
|
17
42
|
mariadb1010:
|
18
43
|
runs-on: ubuntu-latest
|
19
44
|
steps:
|
@@ -22,6 +47,8 @@ jobs:
|
|
22
47
|
fetch-depth: 1 # shallow clone
|
23
48
|
- name: run test
|
24
49
|
run: docker-compose/acceptance_test.sh mariadb1010 docker-compose-ci.yml
|
50
|
+
env:
|
51
|
+
TRACE: ${{ vars.TRACE }}
|
25
52
|
mariadb109:
|
26
53
|
runs-on: ubuntu-latest
|
27
54
|
steps:
|
@@ -30,6 +57,8 @@ jobs:
|
|
30
57
|
fetch-depth: 1 # shallow clone
|
31
58
|
- name: run test
|
32
59
|
run: docker-compose/acceptance_test.sh mariadb109 docker-compose-ci.yml
|
60
|
+
env:
|
61
|
+
TRACE: ${{ vars.TRACE }}
|
33
62
|
mariadb108:
|
34
63
|
runs-on: ubuntu-latest
|
35
64
|
steps:
|
@@ -38,6 +67,8 @@ jobs:
|
|
38
67
|
fetch-depth: 1 # shallow clone
|
39
68
|
- name: run test
|
40
69
|
run: docker-compose/acceptance_test.sh mariadb108 docker-compose-ci.yml
|
70
|
+
env:
|
71
|
+
TRACE: ${{ vars.TRACE }}
|
41
72
|
mariadb107:
|
42
73
|
runs-on: ubuntu-latest
|
43
74
|
steps:
|
@@ -46,6 +77,8 @@ jobs:
|
|
46
77
|
fetch-depth: 1 # shallow clone
|
47
78
|
- name: run test
|
48
79
|
run: docker-compose/acceptance_test.sh mariadb107 docker-compose-ci.yml
|
80
|
+
env:
|
81
|
+
TRACE: ${{ vars.TRACE }}
|
49
82
|
mariadb106:
|
50
83
|
runs-on: ubuntu-latest
|
51
84
|
steps:
|
@@ -54,6 +87,8 @@ jobs:
|
|
54
87
|
fetch-depth: 1 # shallow clone
|
55
88
|
- name: run test
|
56
89
|
run: docker-compose/acceptance_test.sh mariadb106 docker-compose-ci.yml
|
90
|
+
env:
|
91
|
+
TRACE: ${{ vars.TRACE }}
|
57
92
|
mariadb105:
|
58
93
|
runs-on: ubuntu-latest
|
59
94
|
steps:
|
@@ -62,6 +97,8 @@ jobs:
|
|
62
97
|
fetch-depth: 1 # shallow clone
|
63
98
|
- name: run test
|
64
99
|
run: docker-compose/acceptance_test.sh mariadb105 docker-compose-ci.yml
|
100
|
+
env:
|
101
|
+
TRACE: ${{ vars.TRACE }}
|
65
102
|
mariadb104:
|
66
103
|
runs-on: ubuntu-latest
|
67
104
|
steps:
|
@@ -70,6 +107,8 @@ jobs:
|
|
70
107
|
fetch-depth: 1 # shallow clone
|
71
108
|
- name: run test
|
72
109
|
run: docker-compose/acceptance_test.sh mariadb104 docker-compose-ci.yml
|
110
|
+
env:
|
111
|
+
TRACE: ${{ vars.TRACE }}
|
73
112
|
mariadb103:
|
74
113
|
runs-on: ubuntu-latest
|
75
114
|
steps:
|
@@ -78,6 +117,8 @@ jobs:
|
|
78
117
|
fetch-depth: 1 # shallow clone
|
79
118
|
- name: run test
|
80
119
|
run: docker-compose/acceptance_test.sh mariadb103 docker-compose-ci.yml
|
120
|
+
env:
|
121
|
+
TRACE: ${{ vars.TRACE }}
|
81
122
|
mariadb102:
|
82
123
|
runs-on: ubuntu-latest
|
83
124
|
steps:
|
@@ -86,27 +127,5 @@ jobs:
|
|
86
127
|
fetch-depth: 1 # shallow clone
|
87
128
|
- name: run test
|
88
129
|
run: docker-compose/acceptance_test.sh mariadb102 docker-compose-ci.yml
|
89
|
-
|
90
|
-
|
91
|
-
steps:
|
92
|
-
- uses: actions/checkout@v1
|
93
|
-
with:
|
94
|
-
fetch-depth: 1 # shallow clone
|
95
|
-
- name: run test
|
96
|
-
run: docker-compose/acceptance_test.sh mariadb101 docker-compose-ci.yml
|
97
|
-
mariadb100:
|
98
|
-
runs-on: ubuntu-latest
|
99
|
-
steps:
|
100
|
-
- uses: actions/checkout@v1
|
101
|
-
with:
|
102
|
-
fetch-depth: 1 # shallow clone
|
103
|
-
- name: run test
|
104
|
-
run: docker-compose/acceptance_test.sh mariadb100 docker-compose-ci.yml
|
105
|
-
mariadb55:
|
106
|
-
runs-on: ubuntu-latest
|
107
|
-
steps:
|
108
|
-
- uses: actions/checkout@v1
|
109
|
-
with:
|
110
|
-
fetch-depth: 1 # shallow clone
|
111
|
-
- name: run test
|
112
|
-
run: docker-compose/acceptance_test.sh mariadb55 docker-compose-ci.yml
|
130
|
+
env:
|
131
|
+
TRACE: ${{ vars.TRACE }}
|
@@ -6,35 +6,33 @@ on:
|
|
6
6
|
pull_request:
|
7
7
|
|
8
8
|
jobs:
|
9
|
-
|
10
|
-
runs-on: ubuntu-latest
|
11
|
-
steps:
|
12
|
-
- uses: actions/checkout@v1
|
13
|
-
with:
|
14
|
-
fetch-depth: 1 # shallow clone
|
15
|
-
- name: run test
|
16
|
-
run: docker-compose/acceptance_test.sh mysql80 docker-compose-ci.yml
|
17
|
-
mysql57:
|
9
|
+
mysql81:
|
18
10
|
runs-on: ubuntu-latest
|
19
11
|
steps:
|
20
12
|
- uses: actions/checkout@v1
|
21
13
|
with:
|
22
14
|
fetch-depth: 1 # shallow clone
|
23
15
|
- name: run test
|
24
|
-
run: docker-compose/acceptance_test.sh
|
25
|
-
|
16
|
+
run: docker-compose/acceptance_test.sh mysql81 docker-compose-ci.yml
|
17
|
+
env:
|
18
|
+
TRACE: ${{ vars.TRACE }}
|
19
|
+
mysql80:
|
26
20
|
runs-on: ubuntu-latest
|
27
21
|
steps:
|
28
22
|
- uses: actions/checkout@v1
|
29
23
|
with:
|
30
24
|
fetch-depth: 1 # shallow clone
|
31
25
|
- name: run test
|
32
|
-
run: docker-compose/acceptance_test.sh
|
33
|
-
|
26
|
+
run: docker-compose/acceptance_test.sh mysql80 docker-compose-ci.yml
|
27
|
+
env:
|
28
|
+
TRACE: ${{ vars.TRACE }}
|
29
|
+
mysql57:
|
34
30
|
runs-on: ubuntu-latest
|
35
31
|
steps:
|
36
32
|
- uses: actions/checkout@v1
|
37
33
|
with:
|
38
34
|
fetch-depth: 1 # shallow clone
|
39
35
|
- name: run test
|
40
|
-
run: docker-compose/acceptance_test.sh
|
36
|
+
run: docker-compose/acceptance_test.sh mysql57 docker-compose-ci.yml
|
37
|
+
env:
|
38
|
+
TRACE: ${{ vars.TRACE }}
|
data/CHANGELOG.md
CHANGED
@@ -5,7 +5,22 @@ All notable changes to this project will be documented in this file.
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
7
7
|
|
8
|
-
## [
|
8
|
+
## [v1.1.2] - 2023-09-24
|
9
|
+
|
10
|
+
### Added
|
11
|
+
|
12
|
+
- adding dockerfile for github container images [#77](https://github.com/kibitan/masking/pull/77)
|
13
|
+
- Update README.md about frequent asked question [#74](https://github.com/kibitan/masking/pull/74)
|
14
|
+
- enable codescene and add status badge [#80](https://github.com/kibitan/masking/pull/80)
|
15
|
+
- Drop support for MySQL 5.5, 5.6 and MariaDB 5.5, 10.0, 10.1 (EOL) [#90](https://github.com/kibitan/masking/pull/90) relates issue [#89](https://github.com/kibitan/masking/issues/89)
|
16
|
+
- Add support for MySQL 8.1 and MariaDB 11.0/11.1 [#91](https://github.com/kibitan/masking/pull/90) relates issue [#82](https://github.com/kibitan/masking/issues/89)
|
17
|
+
- add presentation and demo link [#87](https://github.com/kibitan/masking/pull/87)
|
18
|
+
- Add comment for complicated part of code [#83](https://github.com/kibitan/masking/pull/83)
|
19
|
+
|
20
|
+
### Fix
|
21
|
+
|
22
|
+
- fix the error on the error message [#74](https://github.com/kibitan/masking/pull/74) relates issue [#73](https://github.com/kibitan/masking/issues/73)
|
23
|
+
- fix ci error [#74](https://github.com/kibitan/masking/pull/86)
|
9
24
|
|
10
25
|
## [v1.1.1] - 2023-04-03
|
11
26
|
|
@@ -77,7 +92,9 @@ $ bin/benchmark.rb
|
|
77
92
|
|
78
93
|
Initial release version. 🎉
|
79
94
|
|
80
|
-
[Unreleased]: https://github.com/kibitan/masking/compare/v1.1.
|
95
|
+
[Unreleased]: https://github.com/kibitan/masking/compare/v1.1.2...HEAD
|
96
|
+
[v1.1.2]: https://github.com/kibitan/masking/compare/v1.1.1...v1.1.2
|
97
|
+
[v1.1.1]: https://github.com/kibitan/masking/compare/v1.1.0...v1.1.1
|
81
98
|
[v1.1.0]: https://github.com/kibitan/masking/compare/v1.0.1...v1.1.0
|
82
99
|
[v1.0.1]: https://github.com/kibitan/masking/compare/v1.0.0...v1.0.1
|
83
100
|
[v1.0.0]: https://github.com/kibitan/masking/compare/v0.0.3...v1.0.0
|
data/CODEOWNERS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* @kibitan
|
data/Dockerfile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
ARG
|
1
|
+
ARG RUBY_VERSION=3.2
|
2
2
|
|
3
|
-
FROM ruby:$
|
3
|
+
FROM ruby:${RUBY_VERSION}-alpine AS builder
|
4
4
|
RUN apk add --no-cache build-base git
|
5
5
|
RUN addgroup -S app && adduser -S -G app app
|
6
6
|
USER app
|
@@ -13,7 +13,7 @@ USER root
|
|
13
13
|
RUN apk add --no-cache mysql-client
|
14
14
|
USER app
|
15
15
|
|
16
|
-
FROM ruby:$
|
16
|
+
FROM ruby:${RUBY_VERSION}-alpine
|
17
17
|
# TODO: remove dependecy of `git` from masking.gemspec:L19
|
18
18
|
RUN apk add --no-cache git
|
19
19
|
RUN addgroup -S app && adduser -S -G app app
|
data/Dockerfile.ghcr
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
# `docker run ghcr.io/kibitan/masking`
|
2
|
+
ARG RUBY_VERSION=3.2
|
3
|
+
|
4
|
+
FROM ruby:$RUBY_VERSION-alpine
|
5
|
+
RUN addgroup -S app && adduser -S -G app app
|
6
|
+
USER app
|
7
|
+
WORKDIR /app
|
8
|
+
ARG MASKING_VERSION=1.1.1
|
9
|
+
RUN gem install masking -v "${MASKING_VERSION}"
|
10
|
+
ENTRYPOINT ["masking"]
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -6,7 +6,12 @@
|
|
6
6
|
|
7
7
|
[![codecov](https://codecov.io/gh/kibitan/masking/branch/main/graph/badge.svg)](https://codecov.io/gh/kibitan/masking)
|
8
8
|
[![Maintainability](https://api.codeclimate.com/v1/badges/290b3005ecc193a3d138/maintainability)](https://codeclimate.com/github/kibitan/masking/maintainability)
|
9
|
+
[![CodeScene Code Health](https://codescene.io/projects/38627/status-badges/code-health)](https://codescene.io/projects/38627)
|
9
10
|
[![Gem Version](https://badge.fury.io/rb/masking.svg)](https://badge.fury.io/rb/masking)
|
11
|
+
<!--
|
12
|
+
[![CodeScene System Mastery](https://codescene.io/projects/38627/status-badges/system-mastery)](https://codescene.io/projects/38627)
|
13
|
+
[![CodeScene Missed Goals](https://codescene.io/projects/38627/status-badges/missed-goals)](https://codescene.io/projects/38627)
|
14
|
+
-->
|
10
15
|
|
11
16
|
The command line tool for anonymizing database records by parsing a SQL dump file and build a new SQL dump file with masking sensitive/credential data.
|
12
17
|
|
@@ -22,40 +27,42 @@ gem install masking
|
|
22
27
|
|
23
28
|
## Supporting RDBMS
|
24
29
|
|
25
|
-
* MySQL: 5.
|
26
|
-
* MariaDB:
|
30
|
+
* MySQL: 5.7, 8.0, 8.1
|
31
|
+
* MariaDB: 10.2<sup>[1](#footnote1)</sup>, 10.3<sup>[1](#footnote1)</sup>, 10.4, 10.5, 10.6, 10.7<sup>[1](#footnote1)</sup>, 10.8<sup>[1](#footnote1)</sup>, 10.9<sup>[1](#footnote1)</sup>, 10.10, 10.11, 11.0, 11.1
|
27
32
|
|
28
33
|
## Usage
|
29
34
|
|
30
35
|
1. Setup configuration for anonymizing target tables/columns to `masking.yml`
|
31
36
|
|
37
|
+
*NOTE: the columns which doesn't mention here will be NOT anonymized, it stays as it is.*
|
38
|
+
|
32
39
|
```yaml
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
40
|
+
# table_name:
|
41
|
+
# column_name: masked_value
|
42
|
+
|
43
|
+
users:
|
44
|
+
string: anonymized string
|
45
|
+
email: anonymized+%{n}@example.com # %{n} will be replaced with sequential number
|
46
|
+
integer: 12345
|
47
|
+
float: 123.45
|
48
|
+
boolean: true
|
49
|
+
null_column: null
|
50
|
+
date: 2018-08-24
|
51
|
+
time: 2018-08-24 15:54:06
|
52
|
+
binary_or_blob: !binary | # Binary Data Language-Independent Type for YAML™ Version 1.1: http://yaml.org/type/binary.html
|
53
|
+
R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5
|
54
|
+
OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+
|
55
|
+
+f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC
|
56
|
+
AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs=
|
57
|
+
# When a column name is suffixed with `?`, the original NULL value will not be anonymized.
|
58
|
+
# This option can be beneficial for simulating SQL execution that closely resembles the original data.
|
59
|
+
nullable_string?: anonymized nullable %{n} string
|
53
60
|
```
|
54
61
|
|
55
62
|
A value will be implicitly converted to a compatible type. If you prefer to explicitly convert, you could use a tag as defined in [YAML Version 1.1](http://yaml.org/spec/current.html#id2503753)
|
56
63
|
|
57
64
|
```yaml
|
58
|
-
|
65
|
+
not-date: !!str 2002-04-28
|
59
66
|
```
|
60
67
|
|
61
68
|
String should be matched with [MySQL String Type]( https://dev.mysql.com/doc/refman/8.0/en/string-type-overview.html). Integer/Float should be matched with [MySQL Numeric Type](https://dev.mysql.com/doc/refman/8.0/en/numeric-type-overview.html). Date/Time should be matched with [MySQL Date and Time Type](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-type-overview.html).
|
@@ -91,6 +98,12 @@ Usage: masking [options]
|
|
91
98
|
-v, --version version
|
92
99
|
```
|
93
100
|
|
101
|
+
## presentation / demo
|
102
|
+
|
103
|
+
[![IMAGE ALT TEXT HERE](https://img.youtube.com/vi/tnGLUhmHclI/0.jpg)](https://www.youtube.com/watch?v=tnGLUhmHclI)
|
104
|
+
|
105
|
+
presentation: [Generate anonymised database with MasKING](https://speakerdeck.com/kibitan/generate-anonymised-database-with-masking-2023-dot-09-dot-21-euruko-unconference-talk)
|
106
|
+
|
94
107
|
## Use case of anonymized (production) database
|
95
108
|
|
96
109
|
* Analyzing production databases for BI, Machine Learning, troubleshooting with respecting GDPR
|
@@ -145,6 +158,8 @@ available option via environment variable:
|
|
145
158
|
* `MYSQL_PASSWORD`: password for user(default: `password`)
|
146
159
|
* `MYSQL_DBNAME`: database name(default: `mydb`)
|
147
160
|
|
161
|
+
NOTE: run with `TRACE=1` will show debug print. for the CI, `TRACE` environment variable on [setting field in the repository](https://github.com/kibitan/masking/settings/variables/actions/TRACE)
|
162
|
+
|
148
163
|
##### with docker-compose
|
149
164
|
|
150
165
|
```bash
|
@@ -159,23 +174,21 @@ docker-compose/acceptance_test.sh mysql80
|
|
159
174
|
|
160
175
|
The docker-compose file names for other database versions, specify that file.
|
161
176
|
|
177
|
+
* MySQL 8.1: [`docker-compose/mysql80.yml`](./docker-compose/mysql81.yml)
|
162
178
|
* MySQL 8.0: [`docker-compose/mysql80.yml`](./docker-compose/mysql80.yml)
|
163
179
|
* MySQL 5.7: [`docker-compose/mysql57.yml`](./docker-compose/mysql57.yml)
|
164
|
-
*
|
165
|
-
*
|
180
|
+
* MariaDB 11.1: [`docker-compose/mariadb111.yml`](./docker-compose/mariadb111.yml)
|
181
|
+
* MariaDB 11.0: [`docker-compose/mariadb110.yml`](./docker-compose/mariadb110.yml)
|
166
182
|
* MariaDB 10.11: [`docker-compose/mariadb1011.yml`](./docker-compose/mariadb1011.yml)
|
167
183
|
* MariaDB 10.10: [`docker-compose/mariadb1010.yml`](./docker-compose/mariadb1010.yml)
|
168
|
-
* MariaDB 10.9: [`docker-compose/mariadb109.yml`](./docker-compose/mariadb109.yml)
|
169
|
-
* MariaDB 10.8: [`docker-compose/mariadb108.yml`](./docker-compose/mariadb108.yml)
|
170
|
-
* MariaDB 10.7: [`docker-compose/mariadb107.yml`](./docker-compose/mariadb107.yml)
|
184
|
+
* MariaDB 10.9<sup>[1](#footnote1): [`docker-compose/mariadb109.yml`](./docker-compose/mariadb109.yml)
|
185
|
+
* MariaDB 10.8<sup>[1](#footnote1): [`docker-compose/mariadb108.yml`](./docker-compose/mariadb108.yml)
|
186
|
+
* MariaDB 10.7<sup>[1](#footnote1): [`docker-compose/mariadb107.yml`](./docker-compose/mariadb107.yml)
|
171
187
|
* MariaDB 10.6: [`docker-compose/mariadb106.yml`](./docker-compose/mariadb106.yml)
|
172
188
|
* MariaDB 10.5: [`docker-compose/mariadb105.yml`](./docker-compose/mariadb105.yml)
|
173
189
|
* MariaDB 10.4: [`docker-compose/mariadb104.yml`](./docker-compose/mariadb104.yml)
|
174
|
-
* MariaDB 10.3: [`docker-compose/mariadb103.yml`](./docker-compose/mariadb103.yml)
|
175
|
-
* MariaDB 10.2
|
176
|
-
* MariaDB 10.1: [`docker-compose/mariadb101.yml`](./docker-compose/mariadb101.yml)
|
177
|
-
* MariaDB 10.0<sup>[2](#footnote2)</sup>: [`docker-compose/mariadb100.yml`](./docker-compose/mariadb100.yml)
|
178
|
-
* MariaDB 5.5: [`docker-compose/mariadb55.yml`](./docker-compose/mariadb55.yml)
|
190
|
+
* MariaDB 10.3<sup>[1](#footnote1): [`docker-compose/mariadb103.yml`](./docker-compose/mariadb103.yml)
|
191
|
+
* MariaDB 10.2<sup>[1](#footnote1)</sup>: [`docker-compose/mariadb102.yml`](./docker-compose/mariadb102.yml)
|
179
192
|
|
180
193
|
#### [Markdown lint](https://github.com/markdownlint/markdownlint)
|
181
194
|
|
@@ -250,6 +263,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
|
|
250
263
|
|
251
264
|
Everyone interacting in the Masking project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/kibitan/masking/blob/main/CODE_OF_CONDUCT.md).
|
252
265
|
|
253
|
-
<a name="footnote1">1</a>: <small>
|
254
|
-
|
255
|
-
<a name="footnote2">2</a>: <small> MariaDB 10.0 is already not supported by [official](https://mariadb.org/about/maintenance-policy/)</small>
|
266
|
+
<a name="footnote1">1</a>: <small> MariaDB 10.2, 10.3, 10.7, 10.8, 10.9 is already not supported by [official](https://mariadb.org/about/maintenance-policy/)</small>
|
data/acceptance/run_test.sh
CHANGED
@@ -1,25 +1,50 @@
|
|
1
1
|
#!/bin/sh
|
2
|
+
set -eu -o pipefail
|
3
|
+
set -C # Prevent output redirection using ‘>’, ‘>&’, and ‘<>’ from overwriting existing files.
|
2
4
|
|
3
|
-
|
4
|
-
|
5
|
+
if [[ "${TRACE-0}" == "1" ]]; then
|
6
|
+
set -vx
|
7
|
+
fi
|
5
8
|
|
6
9
|
MYSQL_HOST=${MYSQL_HOST:-localhost}
|
7
|
-
MYSQL_USER=${MYSQL_USER:-
|
8
|
-
|
10
|
+
MYSQL_USER=${MYSQL_USER:-root}
|
11
|
+
MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD:-root_password}
|
12
|
+
MYSQL_PASSWORD=${MYSQL_PASSWORD:-MYSQL_ROOT_PASSWORD}
|
9
13
|
MYSQL_DBNAME=${MYSQL_DBNAME:-mydb}
|
14
|
+
MYSQL_ANONYMIZED_DBNAME=${MYSQL_ANONYMIZED_DBNAME:-anonnymized_mydb}
|
10
15
|
|
11
16
|
FILEDIR="$( cd "$( dirname "$0" )" && pwd )"
|
12
17
|
|
13
|
-
|
14
|
-
|
18
|
+
main() {
|
19
|
+
# clear tmp file
|
20
|
+
rm "$FILEDIR"/tmp/* || echo 'no tmp file'
|
15
21
|
|
16
|
-
|
17
|
-
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/import_dumpfile.sql"
|
22
|
+
# import database
|
23
|
+
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/import_dumpfile.sql"
|
18
24
|
|
19
|
-
|
20
|
-
|
21
|
-
|
25
|
+
# masking & restore
|
26
|
+
## TODO: temporary add `--skip-extended-insert` as not working now
|
27
|
+
# mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
|
28
|
+
mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert --skip-extended-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
|
22
29
|
|
23
|
-
|
24
|
-
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$
|
25
|
-
|
30
|
+
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" -e "CREATE DATABASE $MYSQL_ANONYMIZED_DBNAME;"
|
31
|
+
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ANONYMIZED_DBNAME" < "$FILEDIR/tmp/masking_dumpfile.sql"
|
32
|
+
## compare the result
|
33
|
+
mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_ANONYMIZED_DBNAME" -e 'SELECT * FROM users ORDER BY id;' --vertical > "$FILEDIR/tmp/query_result.txt"
|
34
|
+
diff "$FILEDIR/expected_query_result.txt" "$FILEDIR/tmp/query_result.txt" || (echo 'test failed' && exit 1)
|
35
|
+
|
36
|
+
# test errors
|
37
|
+
set +e
|
38
|
+
## without masking.yml
|
39
|
+
mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/no_file.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
|
40
|
+
## without `--complete-insert``
|
41
|
+
mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" | exe/masking -c "$FILEDIR/masking.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
|
42
|
+
set -e
|
43
|
+
### compare the result
|
44
|
+
diff "$FILEDIR/expected_error_result.txt" "$FILEDIR/tmp/errors.txt" || (echo 'error output test failed' && exit 1)
|
45
|
+
|
46
|
+
echo 'test passed!'
|
47
|
+
exit 0
|
48
|
+
}
|
49
|
+
|
50
|
+
main "$@"
|
@@ -1,8 +1,19 @@
|
|
1
1
|
#!/usr/bin/env bash
|
2
|
+
set -eu -o pipefail
|
3
|
+
set -C # Prevent output redirection using ‘>’, ‘>&’, and ‘<>’ from overwriting existing files.
|
2
4
|
|
3
|
-
|
4
|
-
|
5
|
+
if [[ "${TRACE-0}" == "1" ]]; then
|
6
|
+
set -vx
|
7
|
+
fi
|
8
|
+
|
9
|
+
cd "$(dirname "$0")"
|
5
10
|
|
6
11
|
MYSQL_VERSION=${1:-mysql80}
|
7
12
|
DOCKER_COMPOSE_FILE=${2:-docker-compose.yml}
|
8
|
-
|
13
|
+
TRACE=${TRACE:-0}
|
14
|
+
|
15
|
+
main() {
|
16
|
+
docker-compose -f "../$DOCKER_COMPOSE_FILE" -f "./$MYSQL_VERSION.yml" run -e "MYSQL_HOST=$MYSQL_VERSION" -e "TRACE=$TRACE" app acceptance/run_test.sh
|
17
|
+
}
|
18
|
+
|
19
|
+
main "$@"
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb1010
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb1010
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb1010:
|
12
16
|
image: mariadb:10.10
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb1011
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb1011
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb1011:
|
12
16
|
image: mariadb:10.11
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb102
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb102
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb102:
|
12
16
|
image: mariadb:10.2
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb103
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb103
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb103:
|
12
16
|
image: mariadb:10.3
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb104
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb104
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb104:
|
12
16
|
image: mariadb:10.4
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb105
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb105
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb105:
|
12
16
|
image: mariadb:10.5
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb106
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb106
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb106:
|
12
16
|
image: mariadb:10.6
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb107
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb107
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb107:
|
12
16
|
image: mariadb:10.7
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb108
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb108
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb108:
|
12
16
|
image: mariadb:10.8
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mariadb109
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mariadb109
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mariadb109:
|
12
16
|
image: mariadb:10.9
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
@@ -0,0 +1,20 @@
|
|
1
|
+
version: '3.7'
|
2
|
+
|
3
|
+
services:
|
4
|
+
app:
|
5
|
+
build:
|
6
|
+
target: with-mysql-client
|
7
|
+
depends_on:
|
8
|
+
- mariadb110
|
9
|
+
entrypoint: docker-compose/wait-for-mysql.sh mariadb110
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
14
|
+
|
15
|
+
mariadb110:
|
16
|
+
image: mariadb:11.0
|
17
|
+
restart: always
|
18
|
+
environment:
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
20
|
+
MYSQL_DATABASE: mydb
|
@@ -0,0 +1,20 @@
|
|
1
|
+
version: '3.7'
|
2
|
+
|
3
|
+
services:
|
4
|
+
app:
|
5
|
+
build:
|
6
|
+
target: with-mysql-client
|
7
|
+
depends_on:
|
8
|
+
- mariadb111
|
9
|
+
entrypoint: docker-compose/wait-for-mysql.sh mariadb111
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
14
|
+
|
15
|
+
mariadb111:
|
16
|
+
image: mariadb:11.1
|
17
|
+
restart: always
|
18
|
+
environment:
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
20
|
+
MYSQL_DATABASE: mydb
|
data/docker-compose/mysql57.yml
CHANGED
@@ -7,11 +7,14 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mysql57
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mysql57
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mysql57:
|
12
16
|
image: mysql:5.7
|
17
|
+
restart: always
|
13
18
|
environment:
|
14
|
-
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
19
|
+
MYSQL_ROOT_PASSWORD: root_password
|
17
20
|
MYSQL_DATABASE: mydb
|
data/docker-compose/mysql80.yml
CHANGED
@@ -7,13 +7,16 @@ services:
|
|
7
7
|
depends_on:
|
8
8
|
- mysql80
|
9
9
|
entrypoint: docker-compose/wait-for-mysql.sh mysql80
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
10
14
|
|
11
15
|
mysql80:
|
12
16
|
image: mysql:8.0
|
13
17
|
# fix for: ERROR 1045 (28000): Plugin caching_sha2_password could not be loaded:
|
14
18
|
command: --default-authentication-plugin=mysql_native_password
|
19
|
+
restart: always
|
15
20
|
environment:
|
16
|
-
|
17
|
-
MYSQL_USER: mysqluser
|
18
|
-
MYSQL_PASSWORD: password
|
21
|
+
MYSQL_ROOT_PASSWORD: root_password
|
19
22
|
MYSQL_DATABASE: mydb
|
@@ -0,0 +1,22 @@
|
|
1
|
+
version: '3.7'
|
2
|
+
|
3
|
+
services:
|
4
|
+
app:
|
5
|
+
build:
|
6
|
+
target: with-mysql-client
|
7
|
+
depends_on:
|
8
|
+
- mysql81
|
9
|
+
entrypoint: docker-compose/wait-for-mysql.sh mysql81
|
10
|
+
restart: always
|
11
|
+
environment:
|
12
|
+
MYSQL_USER: root
|
13
|
+
MYSQL_PASSWORD: root_password
|
14
|
+
|
15
|
+
mysql81:
|
16
|
+
image: mysql:8.1
|
17
|
+
# fix for: ERROR 1045 (28000): Plugin caching_sha2_password could not be loaded:
|
18
|
+
command: --default-authentication-plugin=mysql_native_password
|
19
|
+
restart: always
|
20
|
+
environment:
|
21
|
+
MYSQL_ROOT_PASSWORD: root_password
|
22
|
+
MYSQL_DATABASE: mydb
|
@@ -1,8 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'pathname'
|
4
3
|
require 'erb'
|
5
4
|
require 'ostruct'
|
5
|
+
require 'masking/errors'
|
6
6
|
|
7
7
|
module Masking
|
8
8
|
class Cli
|
@@ -19,19 +19,26 @@ module Masking
|
|
19
19
|
|
20
20
|
attr_reader :error_class
|
21
21
|
|
22
|
-
YAML_FILE_PATH = Pathname('lib/masking/cli/error_messages.yml')
|
23
|
-
|
24
|
-
def error_messages
|
25
|
-
@error_messages = YAML.safe_load(YAML_FILE_PATH.read)
|
26
|
-
end
|
27
|
-
|
28
22
|
def error_message(keyword_args)
|
29
23
|
ERB.new(
|
30
|
-
|
24
|
+
ERROR_MESSAGES.fetch(error_class.to_s)
|
31
25
|
).result(
|
32
26
|
OpenStruct.new(keyword_args).instance_eval { binding } # rubocop:disable Style/OpenStructUse
|
33
27
|
)
|
34
28
|
end
|
29
|
+
|
30
|
+
ERROR_MESSAGES = {
|
31
|
+
'Masking::Error::ConfigFileDoesNotExist' =>
|
32
|
+
'ERROR: config file (<%= config_file_path %>) does not exist',
|
33
|
+
'Masking::Error::ConfigFileIsNotFile' =>
|
34
|
+
'ERROR: config file (<%= config_file_path %>) is not file',
|
35
|
+
'Masking::Error::ConfigFileIsNotValidYaml' =>
|
36
|
+
'ERROR: config file (<%= config_file_path %>) is not valid yaml format',
|
37
|
+
'Masking::Error::ConfigFileContainsNullAsColumnName' =>
|
38
|
+
'ERROR: config file (<%= config_file_path %>) is not valid, column name contains `null`',
|
39
|
+
'Masking::Error::InsertStatementParseError' =>
|
40
|
+
'ERROR: cannot parse SQL dump file. you may forget to put `--complete-insert` option in mysqldump?'
|
41
|
+
}.freeze
|
35
42
|
end
|
36
43
|
end
|
37
44
|
end
|
@@ -29,8 +29,12 @@ module Masking
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def values
|
32
|
+
# NOTE: the reason to use `rows.each_with_index()`
|
33
|
+
# another simple implementations (e.g. `rows.count.time`) doesn't work.because during the block loop,
|
34
|
+
# the `rows` array object can be destructively changed by the #recursive_pattern_value_concat! method and
|
35
|
+
# it make different number of count to loop during the block loop, so it needs to be checked by the object size
|
32
36
|
@values ||= values_section.split(VALUE_ROW_SPLITTER)
|
33
|
-
.tap { |rows| rows.each_with_index { |_, i| recursive_pattern_value_concat(rows, i) } }
|
37
|
+
.tap { |rows| rows.each_with_index { |_, i| recursive_pattern_value_concat!(rows, i) } }
|
34
38
|
.flat_map { |row| row.scan(values_regexp) }
|
35
39
|
end
|
36
40
|
|
@@ -74,11 +78,12 @@ module Masking
|
|
74
78
|
# if it's odd, concat with next row (it means a value contains "),(" pattern)
|
75
79
|
# e.g. INSERT ... VALUES (123,'string ),( abc'),(456,'ab');
|
76
80
|
# refs: implementation of parsing CSV on ruby standard library FasterCSV (ja): https://www.clear-code.com/blog/2018/12/25.html
|
77
|
-
def recursive_pattern_value_concat(value_rows, index)
|
81
|
+
def recursive_pattern_value_concat!(value_rows, index)
|
78
82
|
return if value_rows[index].gsub(/\\\\/, '').gsub(/\\'/, '').count(?').even?
|
79
83
|
|
84
|
+
# make destructive change for values_rows
|
80
85
|
value_rows[index] += VALUE_ROW_SPLITTER + value_rows.delete_at(index + 1)
|
81
|
-
recursive_pattern_value_concat(value_rows, index)
|
86
|
+
recursive_pattern_value_concat!(value_rows, index)
|
82
87
|
end
|
83
88
|
end
|
84
89
|
end
|
data/lib/masking/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: masking
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Chikahiro Tokoro
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-
|
11
|
+
date: 2023-09-24 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Command line tool for anonymizing database records by parsing a SQL dump
|
14
14
|
file and build new SQL dump file with masking sensitive/credential data.
|
@@ -31,13 +31,16 @@ files:
|
|
31
31
|
- ".rubocop_todo.yml"
|
32
32
|
- ".ruby-version"
|
33
33
|
- CHANGELOG.md
|
34
|
+
- CODEOWNERS
|
34
35
|
- CODE_OF_CONDUCT.md
|
35
36
|
- Dockerfile
|
37
|
+
- Dockerfile.ghcr
|
36
38
|
- Gemfile
|
37
39
|
- Gemfile.lock
|
38
40
|
- LICENSE.txt
|
39
41
|
- README.md
|
40
42
|
- Rakefile
|
43
|
+
- acceptance/expected_error_result.txt
|
41
44
|
- acceptance/expected_query_result.txt
|
42
45
|
- acceptance/import_dumpfile.sql
|
43
46
|
- acceptance/masking.yml
|
@@ -54,8 +57,6 @@ files:
|
|
54
57
|
- docker-compose-ci.yml
|
55
58
|
- docker-compose.yml
|
56
59
|
- docker-compose/acceptance_test.sh
|
57
|
-
- docker-compose/mariadb100.yml
|
58
|
-
- docker-compose/mariadb101.yml
|
59
60
|
- docker-compose/mariadb1010.yml
|
60
61
|
- docker-compose/mariadb1011.yml
|
61
62
|
- docker-compose/mariadb102.yml
|
@@ -66,17 +67,16 @@ files:
|
|
66
67
|
- docker-compose/mariadb107.yml
|
67
68
|
- docker-compose/mariadb108.yml
|
68
69
|
- docker-compose/mariadb109.yml
|
69
|
-
- docker-compose/
|
70
|
-
- docker-compose/
|
71
|
-
- docker-compose/mysql56.yml
|
70
|
+
- docker-compose/mariadb110.yml
|
71
|
+
- docker-compose/mariadb111.yml
|
72
72
|
- docker-compose/mysql57.yml
|
73
73
|
- docker-compose/mysql80.yml
|
74
|
+
- docker-compose/mysql81.yml
|
74
75
|
- docker-compose/wait-for-mysql.sh
|
75
76
|
- exe/masking
|
76
77
|
- lib/masking.rb
|
77
78
|
- lib/masking/cli.rb
|
78
79
|
- lib/masking/cli/error_message.rb
|
79
|
-
- lib/masking/cli/error_messages.yml
|
80
80
|
- lib/masking/config.rb
|
81
81
|
- lib/masking/config/target_columns.rb
|
82
82
|
- lib/masking/config/target_columns/column.rb
|
@@ -1,17 +0,0 @@
|
|
1
|
-
version: '3.7'
|
2
|
-
|
3
|
-
services:
|
4
|
-
app:
|
5
|
-
build:
|
6
|
-
target: with-mysql-client
|
7
|
-
depends_on:
|
8
|
-
- mariadb100
|
9
|
-
entrypoint: docker-compose/wait-for-mysql.sh mariadb100
|
10
|
-
|
11
|
-
mariadb100:
|
12
|
-
image: mariadb:10.0
|
13
|
-
environment:
|
14
|
-
MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
17
|
-
MYSQL_DATABASE: mydb
|
@@ -1,17 +0,0 @@
|
|
1
|
-
version: '3.7'
|
2
|
-
|
3
|
-
services:
|
4
|
-
app:
|
5
|
-
build:
|
6
|
-
target: with-mysql-client
|
7
|
-
depends_on:
|
8
|
-
- mariadb101
|
9
|
-
entrypoint: docker-compose/wait-for-mysql.sh mariadb101
|
10
|
-
|
11
|
-
mariadb101:
|
12
|
-
image: mariadb:10.1
|
13
|
-
environment:
|
14
|
-
MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
17
|
-
MYSQL_DATABASE: mydb
|
@@ -1,17 +0,0 @@
|
|
1
|
-
version: '3.7'
|
2
|
-
|
3
|
-
services:
|
4
|
-
app:
|
5
|
-
build:
|
6
|
-
target: with-mysql-client
|
7
|
-
depends_on:
|
8
|
-
- mariadb55
|
9
|
-
entrypoint: docker-compose/wait-for-mysql.sh mariadb55
|
10
|
-
|
11
|
-
mariadb55:
|
12
|
-
image: mariadb:5.5
|
13
|
-
environment:
|
14
|
-
MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
17
|
-
MYSQL_DATABASE: mydb
|
data/docker-compose/mysql55.yml
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
version: '3.7'
|
2
|
-
|
3
|
-
services:
|
4
|
-
app:
|
5
|
-
build:
|
6
|
-
target: with-mysql-client
|
7
|
-
depends_on:
|
8
|
-
- mysql55
|
9
|
-
entrypoint: docker-compose/wait-for-mysql.sh mysql55
|
10
|
-
|
11
|
-
mysql55:
|
12
|
-
image: mysql:5.5
|
13
|
-
environment:
|
14
|
-
MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
17
|
-
MYSQL_DATABASE: mydb
|
data/docker-compose/mysql56.yml
DELETED
@@ -1,17 +0,0 @@
|
|
1
|
-
version: '3.7'
|
2
|
-
|
3
|
-
services:
|
4
|
-
app:
|
5
|
-
build:
|
6
|
-
target: with-mysql-client
|
7
|
-
depends_on:
|
8
|
-
- mysql56
|
9
|
-
entrypoint: docker-compose/wait-for-mysql.sh mysql56
|
10
|
-
|
11
|
-
mysql56:
|
12
|
-
image: mysql:5.6
|
13
|
-
environment:
|
14
|
-
MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
|
15
|
-
MYSQL_USER: mysqluser
|
16
|
-
MYSQL_PASSWORD: password
|
17
|
-
MYSQL_DATABASE: mydb
|
@@ -1,10 +0,0 @@
|
|
1
|
-
Masking::Error::ConfigFileDoesNotExist:
|
2
|
-
"ERROR: config file (<%= config_file_path %>) does not exist"
|
3
|
-
Masking::Error::ConfigFileIsNotFile:
|
4
|
-
"ERROR: config file (<%= config_file_path %>) is not file"
|
5
|
-
Masking::Error::ConfigFileIsNotValidYaml:
|
6
|
-
"ERROR: config file (<%= config_file_path %>) is not valid yaml format"
|
7
|
-
Masking::Error::ConfigFileContainsNullAsColumnName:
|
8
|
-
"ERROR: config file (<%= config_file_path %>) is not valid, column name contains `null`"
|
9
|
-
Masking::Error::InsertStatementParseError:
|
10
|
-
"ERROR: cannot parse SQL dump file. you may forget to put `--complete-insert` option in mysqldump?"
|