masking 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad1fbd60375e405bad8295cdbb8c947139a7ecdb9f0eb3f01d88b97fad571584
-  data.tar.gz: 00e5a59479399c80a8d7d08f3f1b75ae301bff53a9ebb85bb9086de768a0d139
+  metadata.gz: 8a65d39a6c96c94bf473883d1ef9877f1abdd16cfcae97da27e99f8c7afa9963
+  data.tar.gz: 109965dbb6e2ee880f9fd955e4f0d05f6a3dfbc8cbecd153315f9bcad275bf86
 SHA512:
-  metadata.gz: afe2010c052a9c8e1bb6e6948c933020d5868a35d796f0b0e109360b2e522a566bc6062e89e9e74a2216fdeadbdfb37e8a7d2e81b1d4cfda9de34453e719d4c8
-  data.tar.gz: 98743faed783497adbba56e39fdb1bff0e757b734f1b1aa11814da40fc42479c88594f556da451026b17187f042f09669f29ed92bd2623cc997236c0106c522e
+  metadata.gz: 03010bd6d5fb1c90ed1c6dca9dff8f7d84bfe934d65bf48fab28feb3aa6c67381c0c1c6d85ba055b21334919ad1b075a166e23ab3d01fc76f09575f08c1ccac0
+  data.tar.gz: ff9ddd10f8f30c72427815177bb37d68e46fedd7d104d1141d5655449c7ae3811e4d2d0135113b2145b4b809d5774752b9c0100337927d6a59949a9c4fe35580

data/.codeclimate.yml

@@ -5,10 +5,12 @@ plugins:
   markdownlint:
     enabled: true
     checks:
-      # below 3 checks are disabled because Codeclimate's Markdownlint is not latest version
+      # below 3 checks are disabled because Codeclimate's Markdownlint is not latest version (0.5.0)
       MD023:
         enabled: false
       MD034:
         enabled: false
       MD036:
         enabled: false
+    exclude_patterns:
+    - ".chglog/*.tpl.md"

data/CHANGELOG.md

@@ -0,0 +1,41 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [v0.0.3] - 2019-07-07
+
+### Changed
+
+- Update Document [#25](https://github.com/kibitan/masking/pull/25)
+- Performance Tuning [#28](https://github.com/kibitan/masking/pull/28) special thanks [@isaiah](https://github.com/isaiah)
+
+```
+$ bin/benchmark.rb
+(v0.0.2)
+       user     system      total        real
+   2.197755   0.239222   2.436977 (  2.460922)
+
+(v0.0.3)
+       user     system      total        real
+   1.136621   0.198741   1.335362 (  1.355499)
+```
+
+## [v0.0.2] - 2019-01-06
+
+### Fix
+
+- fix bug about NoMethodError on some environment
+
+## [v0.0.1] - 2019-01-05
+
+Initial release version. 🎉
+
+[Unreleased]: https://github.com/kibitan/masking/compare/v0.0.3...HEAD
+[v0.0.3]: https://github.com/kibitan/masking/compare/v0.0.2...v0.0.3
+[v0.0.2]: https://github.com/kibitan/masking/compare/v0.0.1...v0.0.2
+[v0.0.1]: https://github.com/kibitan/masking/tree/v0.0.1

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    masking (0.0.2)
+    masking (0.0.3)
 
 GEM
   remote: https://rubygems.org/

data/README.md

@@ -3,18 +3,12 @@
 [![Build Status](https://travis-ci.org/kibitan/masking.svg?branch=master)](https://travis-ci.org/kibitan/masking)
 [![Coverage Status](https://coveralls.io/repos/github/kibitan/masking/badge.svg?branch=master)](https://coveralls.io/github/kibitan/masking?branch=master)
 [![Maintainability](https://api.codeclimate.com/v1/badges/290b3005ecc193a3d138/maintainability)](https://codeclimate.com/github/kibitan/masking/maintainability)
+[![Gem Version](https://badge.fury.io/rb/masking.svg)](https://badge.fury.io/rb/masking)
 
 The command line tool for anonymizing database records by parsing a SQL dump file and build new SQL dump file with masking sensitive/credential data.
 
 ## Installation
 
-```bash
-git clone git@github.com:kibitan/masking.git
-bin/setup
-```
-
-or install it yourself as:
-
 ```bash
 gem install masking
 ```
@@ -29,51 +23,57 @@ gem install masking
 
 ## Usage
 
-1. setup configuration of target columns to `masking.yml`
-
-  ```yaml
-  # table_name:
-  #   column_name: masked_value
-
-  users:
-    string: anonymized string
-    email: anonymized+%{n}@example.com # %{n} will be replaced with sequential number
-    integer: 12345
-    float: 123.45
-    boolean: true
-    null: null
-    date: 2018-08-24
-    time: 2018-08-24 15:54:06
-    binary_or_blob: !binary | # Binary Data Language-Independent Type for YAML™ Version 1.1: http://yaml.org/type/binary.html
-      R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5
-      OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+
-      +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC
-      AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs=
-  ```
-
-A value will be implicitly converted to compatible type. If you prefer to explicitly convert, you could use a tag as defined in [YAML Version 1.1](http://yaml.org/spec/current.html#id2503753)
-
-```yaml
-not-date: !!str 2002-04-28
-```
+1. Setup configuration for anonymizing target tables/columns to `masking.yml`
+
+    ```yaml
+      # table_name:
+      #   column_name: masked_value
+
+      users:
+        string: anonymized string
+        email: anonymized+%{n}@example.com # %{n} will be replaced with sequential number
+        integer: 12345
+        float: 123.45
+        boolean: true
+        null: null
+        date: 2018-08-24
+        time: 2018-08-24 15:54:06
+        binary_or_blob: !binary | # Binary Data Language-Independent Type for YAML™ Version 1.1: http://yaml.org/type/binary.html
+          R0lGODlhDAAMAIQAAP//9/X17unp5WZmZgAAAOfn515eXvPz7Y6OjuDg4J+fn5
+          OTk6enp56enmlpaWNjY6Ojo4SEhP/++f/++f/++f/++f/++f/++f/++f/++f/+
+          +f/++f/++f/++f/++f/++SH+Dk1hZGUgd2l0aCBHSU1QACwAAAAADAAMAAAFLC
+          AgjoEwnuNAFOhpEMTRiggcz4BNJHrv/zCFcLiwMWYNG84BwwEeECcgggoBADs=
+    ```
+
+    A value will be implicitly converted to compatible type. If you prefer to explicitly convert, you could use a tag as defined in [YAML Version 1.1](http://yaml.org/spec/current.html#id2503753)
+
+    ```yaml
+      not-date: !!str 2002-04-28
+    ```
+
+    String should be matched with [MySQL String Type]( https://dev.mysql.com/doc/refman/8.0/en/string-type-overview.html). Integer/Float should be matched with [MySQL Numeric Type](https://dev.mysql.com/doc/refman/8.0/en/numeric-type-overview.html). Date/Time should be matched with [MySQL Date and Time Type](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-type-overview.html).
 
-String should be matched with [MySQL String Type]( https://dev.mysql.com/doc/refman/8.0/en/string-type-overview.html). Integer/Float should be matched with [MySQL Numeric Type](https://dev.mysql.com/doc/refman/8.0/en/numeric-type-overview.html). Date/Time should be matched with [MySQL Date and Time Type](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-type-overview.html).
+    *NOTE: MasKING doesn't check actual schema's type from dump. If you put uncomaptible value it will cause error during restoring to database.*
 
-*NOTE: MasKING doesn't check actual schema's type from dump. If you put uncomaptible value it will cause error during restoring to database.*
+1. Dump database with anonymizing
 
-1. dump with mask
+    MasKING works with `mysqldump --complete-insert`
 
-  MasKING works with `mysqldump --complete-insert`
+    ```bash
+      mysqldump --complete-insert -u USERNAME DATABASE_NAME | masking > anonymized_dump.sql
+    ```
 
-  ```bash
-    mysqldump --complete-insert -u USERNAME DATABASE_NAME | masking > masked_dump.sql
-  ```
+1. Restore from anonymized dump file
 
-1. restore
+    ```bash
+      mysql -u USERNAME ANONYMIZED_DATABASE_NAME < anonymized_dump.sql
+    ```
 
-  ```bash
-    mysql -u USERNAME MASKED_DATABASE_NAME < masked_dump.sql
-  ```
+    Tip: If you don't need to have anonymized dump file, you can directly insert from stream. It can be faster because it has less IO interaction.
+
+      ```bash
+        mysqldump --complete-insert -u USERNAME DATABASE_NAME | masking | mysql -u USERNAME ANONYMIZED_DATABASE_NAME
+      ```
 
 ### options
 
@@ -83,13 +83,48 @@ Usage: masking [options]
     -c, --config=FILE_PATH           specify config file. default: masking.yml
 ```
 
-## Run test & rubocop & notes
+## Use case of annonymized (production) database
+
+* Simulate for database migration and find a problem before release
+
+Some schema changing statement will lock table and it will cause trouble during the migration. But, without having a large number of record such as production, a migration will finish at the moment and easy to overlook.
+
+* Performance optimization of database queries
+
+Some database query can be slow, but some query isn't reproducible until you have similar amount of records/cardinality.
+
+* Finding bug before release on production
+
+Some bugs are related to unexpected data in production (for instance so long text, invalid/not-well formatted data) and it might be noticed after releasing in production.
+
+* Better development/demo of a feature
+
+Using similar data with real one will be good to make a good view of how feature looks like. It makes easy to find out the things to be changed/fixed before release/check the feature in production.
+
+* Analyze metrics on our production data with respecting GDPR
+
+We can use this database for BI and some trouble shooting.
+
+* And… your idea here!
+
+## Development
+
+```bash
+git clone git@github.com:kibitan/masking.git
+bin/setup
+```
+
+You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`.
+
+### Run test & rubocop & notes
 
 ```bash
   bundle exec rake
 ```
 
-### Protip
+#### Protip
 
 It's useful that set `rake` on [Git hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks).
 
@@ -100,18 +135,12 @@ bundle exec rake
 EOF
 ```
 
-### [Markdown lint](https://github.com/markdownlint/markdownlint)
+#### [Markdown lint](https://github.com/markdownlint/markdownlint)
 
 ```bash
 bundle exec mdl *.md
 ```
 
-## Development
-
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
-
-To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
-
 ### Profiling
 
 use `bin/masking_profile`
@@ -127,6 +156,16 @@ graph html is saved at /your/repo/profile/graph.html
 
 see also: [ruby-prof/ruby-prof: ruby-prof: a code profiler for MRI rubies](https://github.com/ruby-prof/ruby-prof)
 
+### Benchmark
+
+use `bin/benchmark.rb`
+
+```bash
+$ bin/benchmark.rb
+       user     system      total        real
+   1.152776   0.207064   1.359840 (  1.375090)
+```
+
 ## Design Concept
 
 ### KISS ~ keep it simple, stupid ~
@@ -137,10 +176,6 @@ No connection to database, No handling file, Only dealing with stdin/stdout. ~ D
 
 Depend on only pure language standard libraries, no external libraries. (except development/test environment)
 
-### High Code Quality
-
-100% of code coverage [![Coverage Status](https://coveralls.io/repos/github/kibitan/masking/badge.svg?branch=master)](https://coveralls.io/github/kibitan/masking?branch=master) and low complexity [![Maintainability](https://api.codeclimate.com/v1/badges/290b3005ecc193a3d138/maintainability)](https://codeclimate.com/github/kibitan/masking/maintainability)
-
 ## Future Todo
 
 * Pluguable/customizable for a mask way  e.g. integrate with [Faker](https://github.com/stympy/faker)

data/bin/benchmark.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift('./lib')
+require 'benchmark'
+require 'masking'
+
+n = 10_000
+
+Masking.configure do |config|
+  config.target_columns_file_path = 'spec/fixtures/config/masking.yml'
+end
+
+fixture = File.open('spec/fixtures/insert_statement/sample.sql')
+
+Benchmark.bm do |x|
+  x.report do
+    n.times do
+      Masking::Main.new(input: fixture, output: File.open(File::NULL, 'w')).run
+      fixture.rewind
+    end
+  end
+end

data/bin/masking_profile

@@ -37,11 +37,11 @@ RubyProf::GraphHtmlPrinter.new(result).tap do |graph_html_printer|
 end
 
 # # this is quite slow so commented out. but it's useful
-# call_stack_html_file = Pathname(__dir__).join('../profile/call_stack.html')
-# RubyProf::CallStackPrinter.new(result).tap do |call_stack_html_printer|
-#   call_stack_html_printer.print(File.new(call_stack_html_file, 'w'))
-#   puts "call stack html is saved at #{call_stack_html_file}"
-# end
+call_stack_html_file = Pathname(__dir__).join('../profile/call_stack.html')
+RubyProf::CallStackPrinter.new(result).tap do |call_stack_html_printer|
+  call_stack_html_printer.print(File.new(call_stack_html_file, 'w'))
+  puts "call stack html is saved at #{call_stack_html_file}"
+end
 
 # # I'm not sure how to read this
 # call_graph_dot_file = Pathname(__dir__).join('../profile/call_graph.dot')

data/lib/masking/config/target_columns.rb

@@ -24,7 +24,7 @@ module Masking
 
       # TODO: refactoring
       def columns(table_name:)
-        tables.find { |table| table.name == table_name.to_sym }&.columns
+        tables[table_name.to_sym]&.columns
       end
 
       def ==(other)
@@ -41,10 +41,8 @@ module Masking
 
       # TODO: extract to other class
       def tables
-        @tables ||= [].tap do |arr|
-          data.each do |table_name, columns|
-            arr << Table.new(table_name, columns: columns)
-          end
+        @tables ||= data.each_with_object({}) do |kv, r|
+          r[kv[0].to_sym] = Table.new(kv[0], columns: kv[1])
         end
       end
     end

data/lib/masking/config/target_columns/column.rb

@@ -7,6 +7,7 @@ module Masking
     class TargetColumns
       class Column
         attr_reader :name, :table_name, :method_value
+        attr_accessor :index
 
         def initialize(name, table_name:, method_value:)
           @name         = name.to_sym

data/lib/masking/config/target_columns/method/date.rb

@@ -8,7 +8,7 @@ module Masking
       class Method
         class Date
           def initialize(value)
-            @date = value
+            @date = value.strftime(FORMAT)
           end
 
           def call
@@ -21,7 +21,7 @@ module Masking
           FORMAT = '%Y-%m-%d'
 
           def date_format
-            date.strftime(FORMAT)
+            date
           end
         end
       end

data/lib/masking/config/target_columns/method/float.rb

@@ -6,11 +6,11 @@ module Masking
       class Method
         class Float
           def initialize(value)
-            @float = value
+            @float = value.to_s
           end
 
           def call
-            float.to_s
+            float
           end
 
           private

data/lib/masking/config/target_columns/method/integer.rb

@@ -6,11 +6,11 @@ module Masking
       class Method
         class Integer
           def initialize(value)
-            @integer = value
+            @integer = value.to_s
           end
 
           def call
-            integer.to_s
+            integer
           end
 
           private

data/lib/masking/config/target_columns/method/string.rb

@@ -11,13 +11,13 @@ module Masking
           end
 
           def call
-            "'#{output}'".b
+            ("'" + output + "'").b
           end
 
           private
 
+          SEQUENTIAL_NUMBER_PLACEHOLDER = '%{n}' # rubocop:disable Style/FormatStringToken
           attr_reader :string
-          SEQUENTIAL_NUMBER_PLACEHOLDER = /%{n}/.freeze
 
           def output
             string.sub(SEQUENTIAL_NUMBER_PLACEHOLDER, sequence.to_s)

data/lib/masking/config/target_columns/method/time.rb

@@ -6,7 +6,7 @@ module Masking
       class Method
         class Time
           def initialize(value)
-            @time = value
+            @time = value.strftime(FORMAT)
           end
 
           def call
@@ -19,7 +19,7 @@ module Masking
           FORMAT = '%Y-%m-%d %H:%M:%S'
 
           def time_format
-            time.strftime(FORMAT)
+            time
           end
         end
       end

data/lib/masking/data_mask_processor.rb

@@ -23,19 +23,26 @@ module Masking
     end
 
     # TODO: define insert_statement.mask_values(column, mask_method) method & refactoring
-    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
     def process
       return raw_line unless target_table?
 
-      target_columns.columns(table_name: insert_statement.table).each do |target_column|
-        insert_statement.values.map do |value|
-          value[target_column.name] = target_column.masked_value if value.column?(target_column.name)
+      columns = target_columns.columns(table_name: insert_statement.table)
+      if columns.first.index.nil?
+        columns.each do |target_column|
+          target_column.index = insert_statement.columns.index(target_column.name)
+        end
+      end
+
+      insert_statement.values.each do |values|
+        columns.each do |target_column|
+          values[target_column.index] = target_column.masked_value unless target_column.index.nil?
         end
       end
 
       insert_statement.sql
     end
-    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
 
     def target_table?
       target_columns.contains?(table_name: insert_statement.table)

data/lib/masking/insert_statement.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'masking/insert_statement/value'
 require 'masking/insert_statement/sql_builder'
 
 module Masking
@@ -26,8 +25,7 @@ module Masking
       # NOTE: define and extract to ValueSet class?
       @values ||= values_section.split(VALUE_ROW_SPLITTER)
                                 .tap { |rows| rows.each_with_index { |_, i| recursive_pattern_value_concat(rows, i) } }
-                                .map { |row| row.scan(values_regexp).flatten }
-                                .map { |data| Value.new(columns: columns, data: data) }
+                                .flat_map { |row| row.scan(values_regexp) }
     end
 
     def sql
@@ -65,7 +63,7 @@ module Masking
     #   e.g. INSERT ... VALUES (123,'string ),( abc'),(456,'ab');
     # refs: implementation of parsing CSV on ruby standard library FasterCSV (ja): https://www.clear-code.com/blog/2018/12/25.html
     def recursive_pattern_value_concat(value_rows, index)
-      return if value_rows[index].gsub(/\\\\/, '').gsub(/\\'/, '').scan(/'/).count.even?
+      return if value_rows[index].gsub(/\\\\/, '').gsub(/\\'/, '').count(?').even?
 
       value_rows[index] += VALUE_ROW_SPLITTER + value_rows.delete_at(index + 1)
       recursive_pattern_value_concat(value_rows, index)

data/lib/masking/insert_statement/sql_builder.rb

@@ -27,7 +27,7 @@ module Masking
       end
 
       def values_section
-        values.map(&:phrase).join(?,)
+        values.map { |value| '(' + value.join(',') + ')' }.join(',')
       end
     end
   end

data/lib/masking/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Masking
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: masking
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Chikahiro Tokoro
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-01-05 00:00:00.000000000 Z
+date: 2019-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -193,12 +193,14 @@ files:
 - ".rubocop.yml"
 - ".ruby-version"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/benchmark.rb
 - bin/console
 - bin/masking_profile
 - bin/setup
@@ -226,7 +228,6 @@ files:
 - lib/masking/errors.rb
 - lib/masking/insert_statement.rb
 - lib/masking/insert_statement/sql_builder.rb
-- lib/masking/insert_statement/value.rb
 - lib/masking/sql_dump_line.rb
 - lib/masking/version.rb
 - masking.gemspec

data/lib/masking/insert_statement/value.rb

@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-require 'delegate'
-
-module Masking
-  class InsertStatement
-    class Value < ::SimpleDelegator
-      def initialize(columns:, data:)
-        @columns = columns
-        @data    = Struct.new(*columns).new(*data)
-        # NOTE: is it better to get rid of SimpleDelegator, store data in instance variable and define accesor for it?
-        super(@data)
-      end
-
-      def phrase
-        '(' + to_a.join(?,) + ')'
-      end
-
-      # override for make comparable
-      # NOTE: original #== method comapares struct subclass
-      def ==(other)
-        to_h == other.to_h
-      end
-
-      def column?(column_name)
-        @columns.include?(column_name.to_sym)
-      end
-    end
-  end
-end