simple_rewriter 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a52b7ed209e661abcd1b72ab09401597cda25e6e34ea3af81d91302de7daeb63
+  data.tar.gz: 0f61df165fe4a75b5a680da636dae30072f3f29068b7ba5df19037b49f3e9a42
+SHA512:
+  metadata.gz: 682070d30384de78cb82ef0b56e3b1caf03ccb8e49109f9fe887a3b8831adaf04ed7183c186e6ba6697d7cd21e41bb5a0a7516abe53920d9d912f85d44196240
+  data.tar.gz: 8f5200373fa9e75467137586c4b4d2c7e8454ec800afa42e1997e1ceed1e1612bdc4bcd0db628a955d1baeef3e6450f8cefae8ecb0c873b8596f32685f4a48b7

data/.circleci/config.yml

@@ -0,0 +1,57 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: circleci/ruby:2.5.3-node-browsers
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+            - v1-dependencies-{{ checksum "Gemfile.lock" }}
+            # fallback to using the latest cache if no exact match is found
+            - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+
+      # run tests!
+      - run:
+          name: linter
+          command: |
+            rubocop --parallel
+
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | \
+              circleci tests split --split-by=timings)"
+
+            bin/rspec \
+              --format progress \
+              --out /tmp/test-results/rspec.xml \
+              $TEST_FILES
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--require spec_helper

data/.rubocop.yml

@@ -0,0 +1,25 @@
+inherit_gem:
+  ragnarson-stylecheck: config/rubocop.yml
+
+AllCops:
+  TargetRubyVersion: 2.5
+
+Metrics/BlockLength:
+  Exclude:
+    - spec/**/*
+Rails:
+  Enabled: false
+Style/Documentation:
+  Enabled: false
+Style/FrozenStringLiteralComment:
+  Enabled: false
+Naming/MemoizedInstanceVariableName:
+  EnforcedStyleForLeadingUnderscores: required
+Metrics/MethodLength:
+  Max: 15
+Metrics/ClassLength:
+  Max: 150
+Metrics/ModuleLength:
+  Max: 180
+Metrics/CyclomaticComplexity:
+  Max: 7

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+ruby '2.5.3'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,70 @@
+PATH
+  remote: .
+  specs:
+    simple_rewriter (0.0.1)
+      faker (~> 1.9.3)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    coderay (1.1.2)
+    concurrent-ruby (1.1.5)
+    diff-lcs (1.3)
+    faker (1.9.3)
+      i18n (>= 0.7)
+    i18n (1.6.0)
+      concurrent-ruby (~> 1.0)
+    jaro_winkler (1.5.2)
+    method_source (0.9.2)
+    parallel (1.14.0)
+    parser (2.6.0.0)
+      ast (~> 2.4.0)
+    powerpack (0.1.2)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    ragnarson-stylecheck (0.7.0)
+      rake (>= 10.0)
+      rubocop (~> 0.58.2)
+    rainbow (3.0.0)
+    rake (12.3.2)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+    rubocop (0.58.2)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.5, != 2.5.1.1)
+      powerpack (~> 0.1)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (~> 1.0, >= 1.0.1)
+    ruby-progressbar (1.10.0)
+    unicode-display_width (1.5.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  pry (~> 0.12.2)
+  ragnarson-stylecheck (~> 0.7.0)
+  rake (~> 12.3.2)
+  rspec (~> 3.8.0)
+  simple_rewriter!
+
+RUBY VERSION
+   ruby 2.5.3p105
+
+BUNDLED WITH
+   1.16.6

data/LICENSE.md

@@ -0,0 +1,8 @@
+Copyright 2019 Verivox GmbH
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,115 @@
+
+[![Gem Version](https://badge.fury.io/rb/simple_rewriter.svg)](https://badge.fury.io/rb/simple_rewriter)
+[![CircleCI](https://circleci.com/gh/Verivox/simple_rewriter.svg?style=svg)](https://circleci.com/gh/Verivox/simple_rewriter)
+
+## Simple Rewriter
+   As usually staging and production databases are different from each other, so testing new features on staging database can be painful because of lack of data, and diferrences of data.
+
+   Simple Rewriter enables ability to anonymize whole database with simply mappings, you can use dump production data to have production-like data on staging database without worrying about personal data.
+
+### Instalation
+```
+gem install simple_rewriter
+```
+Or in gemfile
+
+```ruby
+gem "simple_rewriter", "~> 0.0.1"
+```
+
+### Sample usage:
+```ruby
+SimpleRewriter.call(
+  record: user, 
+  attributes: [
+    :created_at, 
+    { token: :hex },
+    { email_field: :email },
+    { address_field: :address },
+    { hash_field: [:deep, :detect_type] },
+    { remote_file_urls: { sample_pdf: true, default: [] } # carrierwave support
+  ]
+)
+```
+
+| Options         | Expected Output |
+|-----------------|-----------------|
+|   :email	      | eac94f-example@domain.com |
+|   :address	    | {:street=>\"137 Fahey Street\", :city=>\"New Terryville\", :zip=>\"30752\"} |
+|   :hex	        | 6fe6a2485164f0 |
+|   :name	        | Evie Upton |
+|   :first_name	  | Evie |
+|   :last_name	  | Upton |
+|   :phonenumber	| 333-333-3333 |
+|   :zip	        | 58517 |
+|   :street	      | 282 Kevin Brook |
+|   :city	        | Imogeneborough |
+|   :date	        | 1972-01-02 01:23:21 +0100 | 
+|   :sample_pdf	  | http://www.africau.edu/images/default/sample.pdf |
+|   :deep	        | nil _Only used to trigger rewriter to go deeper into record_ |
+|   :deep_attributes_map | nil _Used to map deeper attributes of record_ |
+
+Options can be joined for e.g. passing `[:first_name, :last_name]` will give the same as `[:name]`
+
+### Configuration
+
+Prevent rewriter from saving record
+```ruby
+
+SimpleRewriter.configure do |config|
+  config.save = false
+end
+```
+
+Custom seeds
+```ruby
+SimpleRewriter.configure do |config|
+  config.integer_range = 0..10
+  config.hash_key_length = 5
+  config.rand_num_max = 10
+  config.address = nil # override default address generator as hash with values
+end
+```
+
+Custom readers/writers
+```ruby
+SimpleRewriter.configure do |config|
+  config.reader = SimpleRewriter::Reader
+  config.writer = SimpleRewriter::Writer
+end
+```
+
+Sample Pdf url
+```ruby
+SimpleRewriter.configure do |config|
+  config.sample_pdf_url = "http://www.africau.edu/images/default/sample.pdf"
+end
+```
+
+### Examples
+Real [anonymizer](/examples/DB_ANONYMIZER.md) configured with sample mappings for batch updates.
+
+### Contributing
+We use PORO to keep dependieces clear and simple.
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+Running tests
+```ruby
+bundle install
+bin/rspec
+```
+
+### Authors
+
+> [jcapt](https://github.com/jcapt)  · 
+> [PiotrMisiurek](https://github.com/PiotrMisiurek)  · 
+> [michalsz](https://github.com/michalsz)
+
+---
+
+Licensed under MIT by the Verivox GmbH

data/Rakefile

@@ -0,0 +1,4 @@
+task :console do
+  exec 'bin/pry -r simple_rewriter -I ./lib'
+end
+task c: :console

data/bin/pry

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'pry' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("pry", "pry")

data/bin/rake

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rake", "rake")

data/bin/rspec

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rspec' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rspec-core", "rspec")

data/bin/rubocop

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rubocop' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load Gem.bin_path("rubocop", "rubocop")

data/examples/DB_ANONYMIZER.md

@@ -0,0 +1,57 @@
+### Db Anonymizer
+
+```ruby
+class DbAnonymizer
+  include Mappings
+
+  ProductionRestrictedError = Class.new(StandardError)
+
+  def initialize(bulk_import: true, mappings: CLASSES_WITH_ATTRIBUTES)
+    raise_production_error if Rails.env.production?
+
+    @bulk_import = bulk_import
+    @mappings = mappings
+  end
+
+  def call
+    rewrite_attributes
+  end
+
+  private
+
+  attr_reader :bulk_import, :mappings
+
+  def rewrite_attributes
+    each_batch do |klass, objects, attributes, valid_objects|
+      objects.each do |object|
+        result = SimpleRewriter.call(record: object, attributes: attributes)
+        valid_objects << result.record if result.success?
+      end
+
+      klass.bulk_import(valid_objects, on_duplicate_key_update: :all, validate: false) if bulk_import
+    end
+  end
+
+  def each_batch
+    klass_with_config do |klass, attributes|
+      klass.find_in_batches(batch_size: 100) do |objects|
+        yield(klass, objects, attributes, [])
+      end
+    end
+  end
+
+  def klass_with_config
+    mappings.each do |klass, attribute|
+      yield(klass.to_s.safe_constantize, attribute)
+    end
+  end
+
+  def raise_production_error
+    raise ProductionRestrictedError, "User force_production option to run it instead"
+  end
+end
+```
+
+1. `klass_with_config` iterates over mappings and yields record class, and attribute to rewrite
+2. `each_batch` yields 100 records to rewrite on one batch
+3. `rewrite_attributes` calls `SimpleRewriter` with options and does import on 100 records

data/examples/db_anonymizer/mappings.rb

@@ -0,0 +1,13 @@
+module DbAnonymizer::Mappings
+  CLASSES_WITH_ATTRIBUTES = {
+    User: [
+      :name,
+      :place,
+      :birthdate,
+      { email: :email },
+      { token: :hex },
+      { jsonb_field: %i(deep detect_type) },
+      { remote_attachments_urls: { sample_pdf: true, default: [] } }
+    ]
+  }.freeze
+end

data/examples/db_anonymizer.rb

@@ -0,0 +1,52 @@
+require "activerecord-import"
+require "activerecord-import/active_record/adapters/postgresql_adapter"
+
+class DbAnonymizer
+  include Mappings
+
+  ProductionRestrictedError = Class.new(StandardError)
+
+  def initialize(bulk_import: true, mappings: CLASSES_WITH_ATTRIBUTES)
+    raise_production_error if Rails.env.production?
+
+    @bulk_import = bulk_import
+    @mappings = mappings
+  end
+
+  def call
+    rewrite_attributes
+  end
+
+  private
+
+  attr_reader :bulk_import, :mappings
+
+  def rewrite_attributes
+    each_batch do |klass, objects, attributes, valid_objects|
+      objects.each do |object|
+        result = SimpleRewriter.call(record: object, attributes: attributes)
+        valid_objects << result.record if result.success?
+      end
+
+      klass.bulk_import(valid_objects, on_duplicate_key_update: :all, validate: false) if bulk_import
+    end
+  end
+
+  def each_batch
+    klass_with_config do |klass, attributes|
+      klass.find_in_batches(batch_size: 100) do |objects|
+        yield(klass, objects, attributes, [])
+      end
+    end
+  end
+
+  def klass_with_config
+    mappings.each do |klass, attribute|
+      yield(klass.to_s.safe_constantize, attribute)
+    end
+  end
+
+  def raise_production_error
+    raise ProductionRestrictedError, "User force_production option to run it instead"
+  end
+end

data/lib/simple_rewriter/attribute.rb

@@ -0,0 +1,24 @@
+require "forwardable"
+
+class SimpleRewriter::Attribute
+  extend Forwardable
+
+  def initialize(attr_value, options)
+    @attr_value = attr_value
+    @options = options
+  end
+
+  def_delegator :generator, :generate
+
+  private
+
+  attr_reader :attr_value, :options, :generator_klass
+
+  def generator
+    @_generator ||= generator_class.new(attr_value, options)
+  end
+
+  def generator_class
+    SimpleRewriter::ClassResolver.call(attr_value, "SimpleRewriter::Rewriters::%{klass}Rewriter", options)
+  end
+end

data/lib/simple_rewriter/attribute_resolver.rb

@@ -0,0 +1,51 @@
+module SimpleRewriter
+  class AttributeResolver < BaseService
+    def initialize(attribute:)
+      @attribute = attribute
+    end
+
+    def call
+      finalize
+    end
+
+    private
+
+    INDEX_OF_ATTRIBUTE_NAME = 0
+    private_constant :INDEX_OF_ATTRIBUTE_NAME
+
+    attr_reader :attribute
+
+    def options
+      return extract_options if attribute.is_a?(Hash)
+    end
+
+    def extract_options
+      options = attribute.values
+
+      return options[0] if options[0].is_a?(Hash)
+      return convert_array_options(options.flatten) if options.is_a?(Array)
+
+      options.flatten
+    end
+
+    def convert_array_options(array)
+      array.each_with_object({}) { |option, obj| converted_option(option, obj) }
+    end
+
+    def converted_option(option, obj)
+      return obj[option.keys[0]] = option.values[0] if option.is_a?(Hash)
+
+      obj[option] = true
+    end
+
+    def extracted_attribute
+      return attribute.keys[INDEX_OF_ATTRIBUTE_NAME] if attribute.is_a?(Hash)
+
+      attribute
+    end
+
+    def finalize
+      OpenStruct.new(attribute: extracted_attribute, options: options)
+    end
+  end
+end

data/lib/simple_rewriter/base_service.rb

@@ -0,0 +1,5 @@
+class SimpleRewriter::BaseService
+  def self.call(*args)
+    new(*args).call
+  end
+end