masking 1.1.0 → 1.1.2.pre.alpha

data/acceptance/expected_query_result.txt

@@ -1,33 +1,39 @@
 *************************** 1. row ***************************
-            id: 1
-        string: anonymized string
-         email: anonymized+1@example.com
-       integer: 12345
-         float: 123.45
-       boolean: 1
-          null: NULL
-          date: 2018-08-24
-          time: 2018-08-24 15:54:06
-binary_or_blob: \x92       
+              id: 1
+          string: anonymized string
+           email: anonymized+1@example.com
+         integer: 12345
+           float: 123.45
+         boolean: 1
+            null: NULL
+            date: 2018-08-24
+            time: 2018-08-24 15:54:06
+  binary_or_blob: \x92       
+ nullable_string: string1
+nullable_integer: NULL
 *************************** 2. row ***************************
-            id: 2
-        string: anonymized string
-         email: anonymized+2@example.com
-       integer: 12345
-         float: 123.45
-       boolean: 1
-          null: NULL
-          date: 2018-08-24
-          time: 2018-08-24 15:54:06
-binary_or_blob: \x92\x92   
+              id: 2
+          string: anonymized string
+           email: anonymized+2@example.com
+         integer: 12345
+           float: 123.45
+         boolean: 1
+            null: NULL
+            date: 2018-08-24
+            time: 2018-08-24 15:54:06
+  binary_or_blob: \x92\x92   
+ nullable_string: NULL
+nullable_integer: 33333
 *************************** 3. row ***************************
-            id: 3
-        string: anonymized string
-         email: anonymized+3@example.com
-       integer: 12345
-         float: 123.45
-       boolean: 1
-          null: NULL
-          date: 2018-08-24
-          time: 2018-08-24 15:54:06
-binary_or_blob: NULL
+              id: 3
+          string: anonymized string
+           email: anonymized+3@example.com
+         integer: 12345
+           float: 123.45
+         boolean: 1
+            null: NULL
+            date: 2018-08-24
+            time: 2018-08-24 15:54:06
+  binary_or_blob: NULL
+ nullable_string: string3
+nullable_integer: 33333

data/acceptance/import_dumpfile.sql

@@ -33,6 +33,8 @@ CREATE TABLE `users` (
   `date` date DEFAULT NULL,
   `time` timestamp NULL DEFAULT NULL,
   `binary_or_blob` binary(11) DEFAULT NULL,
+  `nullable_string` varchar(20) DEFAULT '',
+  `nullable_integer` int(11) DEFAULT NULL,
   PRIMARY KEY (`id`)
 ) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8;
 /*!40101 SET character_set_client = @saved_cs_client */;
@@ -43,7 +45,7 @@ CREATE TABLE `users` (
 
 LOCK TABLES `users` WRITE;
 /*!40000 ALTER TABLE `users` DISABLE KEYS */;
-INSERT INTO `users` (`id`, `string`, `email`, `integer`, `float`, `boolean`, `null`, `date`, `time`, `binary_or_blob`) VALUES (1,'example@exa','test@example.com',1245,2.1,1,-321,NULL,NULL,_binary '\\x92\0\0\0\0\0\0\0'),(2,'あいうえお','invalid@email',0,-23.4422,0,321,NULL,NULL,_binary '\\x92\\x92\0\0\0'),(3,'+-l;a*&^%$','chikahiro@test.com',-1,21.2321,NULL,-231321,'2019-10-31','2019-10-31 16:27:21',NULL);
+INSERT INTO `users` (`id`, `string`, `email`, `integer`, `float`, `boolean`, `null`, `date`, `time`, `binary_or_blob`, `nullable_string`, `nullable_integer`) VALUES (1,'example@exa','test@example.com',1245,2.1,1,-321,NULL,NULL,_binary '\\x92\0\0\0\0\0\0\0','abcde',NULL),(2,'あいうえお','invalid@email',0,-23.4422,0,321,NULL,NULL,_binary '\\x92\\x92\0\0\0',NULL,12),(3,'+-l;a*&^%$','chikahiro@test.com',-1,21.2321,NULL,-231321,'2019-10-31','2019-10-31 16:27:21',NULL,'abc123',-123);
 /*!40000 ALTER TABLE `users` ENABLE KEYS */;
 UNLOCK TABLES;
 /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

data/acceptance/masking.yml

@@ -7,6 +7,8 @@ users:
   'null': NULL
   date: 2018-08-24
   time: 2018-08-24 15:54:06
+  nullable_string?: string%{n}
+  nullable_integer?: 33333
   ## TODO: something not working well with binary...
   # binary_or_blob: !binary |
   #   R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5

data/acceptance/run_test.sh

@@ -10,16 +10,28 @@ MYSQL_DBNAME=${MYSQL_DBNAME:-mydb}
 
 FILEDIR="$( cd "$( dirname "$0" )" && pwd )"
 
-## clear tmp file
+# clear tmp file
 rm "$FILEDIR"/tmp/* || echo 'no tmp file'
 
-## import database
+# import database
 mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/import_dumpfile.sql"
 
-## masking
+#  masking & restore
 mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/masking.yml" > "$FILEDIR/tmp/masking_dumpfile.sql"
 mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" < "$FILEDIR/tmp/masking_dumpfile.sql"
-
-## compare
+## compare the result
 mysql -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" -e 'SELECT * FROM users ORDER BY id;' --vertical > "$FILEDIR/tmp/query_result.txt"
-diff "$FILEDIR/tmp/query_result.txt" "$FILEDIR/expected_query_result.txt" && echo 'test passed!'
+diff "$FILEDIR/expected_query_result.txt" "$FILEDIR/tmp/query_result.txt" || (echo 'test failed' && exit 1)
+
+# test errors
+set +e
+## without masking.yml
+mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" --complete-insert | exe/masking -c "$FILEDIR/no_file.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
+## without `--complete-insert``
+mysqldump -h "$MYSQL_HOST" -u "$MYSQL_USER" -p"$MYSQL_PASSWORD" "$MYSQL_DBNAME" | exe/masking -c "$FILEDIR/masking.yml" 2>> "$FILEDIR/tmp/errors.txt" 1> /dev/null
+set -e
+### compare the result
+diff "$FILEDIR/expected_error_result.txt" "$FILEDIR/tmp/errors.txt" || (echo 'error output test failed' && exit 1)
+
+echo 'test passed!'
+exit 0

data/docker-compose/mariadb1010.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+  app:
+    build:
+      target: with-mysql-client
+    depends_on:
+      - mariadb1010
+    entrypoint: docker-compose/wait-for-mysql.sh mariadb1010
+
+  mariadb1010:
+    image: mariadb:10.10
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: password
+      MYSQL_DATABASE: mydb

data/docker-compose/mariadb1011.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+  app:
+    build:
+      target: with-mysql-client
+    depends_on:
+      - mariadb1011
+    entrypoint: docker-compose/wait-for-mysql.sh mariadb1011
+
+  mariadb1011:
+    image: mariadb:10.11
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: password
+      MYSQL_DATABASE: mydb

data/docker-compose/mariadb105.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+  app:
+    build:
+      target: with-mysql-client
+    depends_on:
+      - mariadb105
+    entrypoint: docker-compose/wait-for-mysql.sh mariadb105
+
+  mariadb105:
+    image: mariadb:10.5
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: password
+      MYSQL_DATABASE: mydb

data/docker-compose/mariadb106.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+  app:
+    build:
+      target: with-mysql-client
+    depends_on:
+      - mariadb106
+    entrypoint: docker-compose/wait-for-mysql.sh mariadb106
+
+  mariadb106:
+    image: mariadb:10.6
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: password
+      MYSQL_DATABASE: mydb

data/docker-compose/mariadb107.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+  app:
+    build:
+      target: with-mysql-client
+    depends_on:
+      - mariadb107
+    entrypoint: docker-compose/wait-for-mysql.sh mariadb107
+
+  mariadb107:
+    image: mariadb:10.7
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: password
+      MYSQL_DATABASE: mydb

data/docker-compose/mariadb108.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+  app:
+    build:
+      target: with-mysql-client
+    depends_on:
+      - mariadb108
+    entrypoint: docker-compose/wait-for-mysql.sh mariadb108
+
+  mariadb108:
+    image: mariadb:10.8
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: password
+      MYSQL_DATABASE: mydb

data/docker-compose/mariadb109.yml

@@ -0,0 +1,17 @@
+version: '3.7'
+
+services:
+  app:
+    build:
+      target: with-mysql-client
+    depends_on:
+      - mariadb109
+    entrypoint: docker-compose/wait-for-mysql.sh mariadb109
+
+  mariadb109:
+    image: mariadb:10.9
+    environment:
+      MYSQL_RANDOM_ROOT_PASSWORD: 'yes'
+      MYSQL_USER: mysqluser
+      MYSQL_PASSWORD: password
+      MYSQL_DATABASE: mydb

data/lib/masking/cli/error_message.rb

@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'pathname'
 require 'erb'
 require 'ostruct'
+require 'masking/errors'
 
 module Masking
   class Cli
@@ -19,19 +19,26 @@ module Masking
 
       attr_reader :error_class
 
-      YAML_FILE_PATH = Pathname('lib/masking/cli/error_messages.yml')
-
-      def error_messages
-        @error_messages = YAML.safe_load(YAML_FILE_PATH.read)
-      end
-
       def error_message(keyword_args)
         ERB.new(
-          error_messages.fetch(error_class.to_s)
+          ERROR_MESSAGES.fetch(error_class.to_s)
         ).result(
-          OpenStruct.new(keyword_args).instance_eval { binding }
+          OpenStruct.new(keyword_args).instance_eval { binding } # rubocop:disable Style/OpenStructUse
         )
       end
+
+      ERROR_MESSAGES = {
+        'Masking::Error::ConfigFileDoesNotExist' =>
+          'ERROR: config file (<%= config_file_path %>) does not exist',
+        'Masking::Error::ConfigFileIsNotFile' =>
+          'ERROR: config file (<%= config_file_path %>) is not file',
+        'Masking::Error::ConfigFileIsNotValidYaml' =>
+          'ERROR: config file (<%= config_file_path %>) is not valid yaml format',
+        'Masking::Error::ConfigFileContainsNullAsColumnName' =>
+          'ERROR: config file (<%= config_file_path %>) is not valid, column name contains `null`',
+        'Masking::Error::InsertStatementParseError' =>
+          'ERROR: cannot parse SQL dump file. you may forget to put `--complete-insert` option in mysqldump?'
+      }.freeze
     end
   end
 end

data/lib/masking/config/target_columns/column.rb

@@ -6,22 +6,36 @@ module Masking
   class Config
     class TargetColumns
       class Column
-        attr_reader :name, :table_name, :method_value, :method
+        attr_reader :table_name, :method_value, :method
 
         def initialize(name, table_name:, method_value:)
           raise ColumnNameIsNil if name.nil?
 
-          @name         = name.to_sym
-          @table_name   = table_name.to_sym
-          @method_value = method_value
-          @method       = Method.new(method_value)
+          @original_name = name.to_sym
+          @table_name    = table_name.to_sym
+          @method_value  = method_value
+          @method        = Method.new(method_value, ignore_null: ignore_null?)
         end
 
         def ==(other)
           name == other.name && table_name == other.table_name && method_value == other.method_value
         end
 
+        def name
+          @name ||= ignore_null? ? original_name.to_s.chomp(IGNORE_NULL_SUFFIX).to_sym : original_name
+        end
+
+        def ignore_null?
+          @ignore_null ||= original_name.to_s.end_with?(IGNORE_NULL_SUFFIX)
+        end
+
         class ColumnNameIsNil < StandardError; end
+
+        private
+
+        attr_reader :original_name
+
+        IGNORE_NULL_SUFFIX = '?'
       end
     end
   end

data/lib/masking/config/target_columns/method/string_binary_distinctor.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
-require 'masking/config/target_columns/method/binary'
-require 'masking/config/target_columns/method/string'
+require 'masking/config/target_columns/method/type/binary'
+require 'masking/config/target_columns/method/type/string'
 
 module Masking
   class Config
@@ -10,7 +10,7 @@ module Masking
         module StringBinaryDistinctor
           class << self
             def new(value)
-              binary?(value) ? Binary.new(value) : String.new(value)
+              binary?(value) ? Type::Binary.new(value) : Type::String.new(value)
             end
 
             private

data/lib/masking/config/target_columns/method/type/base.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class Base
+            def initialize(value)
+              @value = value
+            end
+
+            def call(_sql_value)
+              raise NotImplementedError
+            end
+
+            private
+
+            attr_reader :value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/type/binary.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'masking/config/target_columns/method/type/base'
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class Binary < Base
+            def call(_sql_value)
+              "_binary '#{value}'".b
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/type/boolean.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'masking/config/target_columns/method/type/base'
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class Boolean < Base
+            def call(_sql_value)
+              boolean_format.to_s
+            end
+
+            private
+
+            # NOTE: 11.1.1 Numeric Type Overview, chapter BOOL, BOOLEAN
+            #       https://dev.mysql.com/doc/refman/8.0/en/numeric-type-overview.html
+            def boolean_format
+              value ? 1 : 0
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/type/date.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'masking/config/target_columns/method/type/base'
+require 'date'
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class Date < Base
+            def call(_sql_value)
+              "'#{date_format}'"
+            end
+
+            private
+
+            FORMAT = '%Y-%m-%d'
+
+            def date_format
+              value.strftime(FORMAT)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/type/extension/ignore_null.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          module Extension
+            module IgnoreNull
+              def call(sql_value)
+                if sql_value == 'NULL'
+                  sequence! if respond_to?(:sequence!, true)
+                  return 'NULL'
+                end
+
+                super
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/type/float.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'masking/config/target_columns/method/type/base'
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class Float < Base
+            def call(_sql_value)
+              value.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/type/integer.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'masking/config/target_columns/method/type/base'
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class Integer < Base
+            def call(_sql_value)
+              value.to_s
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/{null.rb → type/null.rb}

@@ -4,11 +4,11 @@ module Masking
   class Config
     class TargetColumns
       class Method
-        class Null
-          def initialize(*); end
-
-          def call
-            'NULL'
+        module Type
+          class Null < Base
+            def call(_sql_value)
+              'NULL'
+            end
           end
         end
       end

data/lib/masking/config/target_columns/method/type/string.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'masking/config/target_columns/method/type/base'
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class String < Base
+            def initialize(value)
+              super(value)
+              @sequence = 0
+            end
+
+            def call(_sql_value)
+              sequence!
+              "'#{output}'".b
+            end
+
+            private
+
+            SEQUENTIAL_NUMBER_PLACEHOLDER = '%{n}' # rubocop:disable Style/FormatStringToken
+
+            def output
+              value.sub(SEQUENTIAL_NUMBER_PLACEHOLDER, @sequence.to_s)
+            end
+
+            def sequence!
+              @sequence += 1
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method/type/time.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'masking/config/target_columns/method/type/base'
+
+module Masking
+  class Config
+    class TargetColumns
+      class Method
+        module Type
+          class Time < Base
+            def call(_sql_value)
+              "'#{time_format}'"
+            end
+
+            private
+
+            FORMAT = '%Y-%m-%d %H:%M:%S'
+            def time_format
+              value.strftime(FORMAT)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/masking/config/target_columns/method.rb

@@ -2,7 +2,9 @@
 
 require 'pathname'
 require 'forwardable'
-Dir[Pathname(__FILE__).dirname.join('method/*.rb').to_s].each(&method(:require))
+require 'masking/config/target_columns/method/type/extension/ignore_null'
+require 'masking/config/target_columns/method/string_binary_distinctor'
+Dir[Pathname(__FILE__).dirname.join('method/type/*.rb').to_s].sort.each(&method(:require))
 
 module Masking
   class Config
@@ -10,8 +12,10 @@ module Masking
       class Method
         extend Forwardable
 
-        def initialize(method)
-          @method_type = mapping(method.class).new(method)
+        def initialize(method, ignore_null: false)
+          @method_type = mapping(method.class).new(method).tap do |obj|
+            obj.singleton_class.prepend(Type::Extension::IgnoreNull) if ignore_null
+          end
         end
 
         def_delegator :@method_type, :call
@@ -21,13 +25,13 @@ module Masking
         # rubocop:disable Layout/HashAlignment
         MAPPING = {
           ::String     => StringBinaryDistinctor,
-          ::Integer    => Integer,
-          ::Float      => Float,
-          ::Date       => Date,
-          ::Time       => Time,
-          ::TrueClass  => Boolean,
-          ::FalseClass => Boolean,
-          ::NilClass   => Null
+          ::Integer    => Type::Integer,
+          ::Float      => Type::Float,
+          ::Date       => Type::Date,
+          ::Time       => Type::Time,
+          ::TrueClass  => Type::Boolean,
+          ::FalseClass => Type::Boolean,
+          ::NilClass   => Type::Null
         }.freeze
         # rubocop:enable Layout/HashAlignment