mongo_masker 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f217269ec66fb6a591ab9892b713668ef49cd761030f0f1daf3fb8f806c928f5
+  data.tar.gz: c3720d536bd94c8adc2c2a5fbd07446ebb54e9a192d34e93b0be7fea36ee34f3
+SHA512:
+  metadata.gz: 16e4d2ca4b1b9ef37005beebe1bfb44431b8ba4bfdd93758c15f55178042da1f1ccee353cec9f70823ddc7928d6619b1c0c32a82a13f42ceb95824ef0e5c8e2a
+  data.tar.gz: cd977e2df4ecc121785d3c28158020883f895f0922b3973c21de401c54e0f6937f52d0655af6b26d3ccf6bc84d3641968b4a41972cbe22e98a30a20cb15da71e

data/.dockerignore

@@ -0,0 +1 @@
+/data/

data/.gitignore

@@ -0,0 +1,53 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc
+
+# import data
+/data/

data/Dockerfile

@@ -0,0 +1,12 @@
+FROM ruby:2.6
+
+ENV LANG C.UTF-8
+
+WORKDIR /masking
+
+ADD Gemfile /masking/Gemfile
+ADD Gemfile.lock /masking/Gemfile.lock
+
+RUN bundle install
+
+COPY . .

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gem 'ffaker'
+gem 'mongo'
+gem 'irb'
+gem 'thor'

data/Gemfile.lock

@@ -0,0 +1,21 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    bson (4.4.2)
+    ffaker (2.10.0)
+    irb (1.0.0)
+    mongo (2.7.0)
+      bson (>= 4.4.2, < 5.0.0)
+    thor (0.20.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ffaker
+  irb
+  mongo
+  thor
+
+BUNDLED WITH
+   1.17.1

data/README.md

@@ -0,0 +1,41 @@
+# Mongodb-data-masking
+Masking your data in mongodb
+
+# Usage
+```
+bundle
+ruby masker.rb mask mask.yml
+```
+
+# Example mask.yml
+```yaml
+version: 1
+db_url: mongodb://mongodb:27017/development
+models:
+  - name: users
+    condition:
+      email:
+        "$not": !ruby/regexp '/@basicinc\.jp$/'
+    fields:
+      email: FFaker::Internet.safe_email
+  - name: users
+    fields:
+      reset_password_token: String.new
+      confirmation_token: String.new
+  - name: sitesconta
+    fields:
+      title: FFaker::NameJA.name
+      description: FFaker::LoremJA.sentence
+      domain: FFaker::Internet.domain_name
+      external_service: :external_services
+  - name: external_services
+    fields:
+      _type: "'ExternalService'"
+      facebook: nil
+      google: nil
+  - name: contacts
+    condition:
+      site_id:
+        "$ne": BSON::ObjectId('12312345346456456456sdff')
+    delete: true
+```

data/bin/mongo_masker

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'mongo_masker'
+require 'thor'
+
+class MongoMasker::Cli < Thor
+  include Thor::Actions
+
+  desc 'mask MASKER_FILE', 'Mask database'
+  def mask(masker_file)
+    masker = MongoMasker::Main.new(masker_file)
+    masker.mask
+  end
+end
+
+MongoMasker::Cli.start

data/docker-compose.yml

@@ -0,0 +1,20 @@
+version: '3'
+services:
+  mongodb:
+    image: mongo:3.4.2
+    volumes:
+      - mongo-data:/data/db
+      - ./data:/tmp/data
+
+  app:
+    build: ./
+    links:
+      - mongodb
+    volumes:
+      - ./lib:/masking/lib
+      - ./bin:/masking/bin
+    stdin_open: true
+    tty: true
+
+volumes:
+  mongo-data:

data/lib/mongo_masker.rb

@@ -0,0 +1,141 @@
+require 'yaml'
+require 'ffaker'
+require 'mongo'
+
+Mongo::Logger.logger.level = Logger::FATAL
+
+module MongoMasker
+  class Main
+    def initialize(config = nil)
+      configure config unless config.nil?
+      @sequence = 0
+    end
+
+    def db
+      @db ||= Mongo::Client.new(@config['db_url'] || 'mongodb://mongodb:27017/development')
+    end
+
+    def mask(config = nil)
+      configure config unless config.nil?
+      raise 'Please provide mask' if @config.nil?
+
+      track_time do
+        @config['models'].each do |model|
+          mask_document model
+        end
+        puts 'Done!' unless @config['silent']
+      end
+    end
+
+    def configure(config)
+      @config = config.is_a?(String) ? load_from_yaml(config) : config
+    end
+
+    def seq
+      @sequence += 1
+    end
+
+    private
+
+    def track_time
+      start_at = Time.now
+      yield
+      finish_at = Time.now
+      puts "Elapsed time: #{format_time_diff(start_at, finish_at)}" unless @config['silent']
+    end
+
+    def format_time_diff(start_at, finish_at)
+      output = Time.at(finish_at - start_at).strftime '%H hours %M minutes %S seconds'
+      output.gsub(/^0+ hours /, '').gsub(/^0+ minutes /, '')
+    end
+
+    def mask_document(model)
+      scope = prepair_scope model
+
+      if model['delete']
+        delete_documents scope, model
+      else
+        mask_each_document scope, model
+      end
+    rescue StandardError => e
+      puts "\nCan't mask #{model['name']}" unless @config['silent']
+      raise e
+    ensure
+      puts '' unless @config['silent']
+    end
+
+    def delete_documents(scope, model)
+      puts "Deleting #{model['name']}"
+      scope.delete_many
+    end
+
+    def mask_each_document(scope, model)
+      total = scope.count()
+
+      scope.each_with_index do |document, index|
+        print "Masking #{model['name']} (#{index + 1}/#{total})\r" unless @config['silent']
+        mask = create_mask model['fields']
+
+        apply_mask(document, mask)
+        db[model['name']].find({_id: document['_id']}).update_one(document)
+      end
+    end
+
+    def prepair_scope(model)
+      scope = db[model['name']]
+      scope.find(parse_condition(model['condition']))
+    end
+
+    def parse_condition condition
+      if condition.is_a?(String) && condition.match?(/^BSON::ObjectId\('[A-Za-z0-9]+'\)$/)
+        return eval(condition)
+      end
+
+      if condition.is_a?(Hash)
+        condition.each do |op, value|
+          condition[op] = parse_condition value
+        end
+        return condition
+      end
+
+      if condition.is_a?(Array)
+        return condition.map do |value|
+          parse_condition value
+        end
+      end
+
+      condition
+    end
+
+    def create_mask(fields)
+      mask = {}
+      fields.each do |field, value|
+        if value.is_a?(Symbol)
+          sub_mask = create_mask @config['models'].find{|model| model['name'] == value.to_s}['fields']
+          mask[field] = sub_mask
+          next
+        end
+
+        mask[field] = evalute_field_value(value)
+      end
+      mask
+    end
+
+    def apply_mask(document, mask)
+      mask = mask.reject do |field, _value|
+        document[field].nil?
+      end
+      document.update(mask)
+    end
+
+    def evalute_field_value(value)
+      eval(value)
+    rescue StandardError
+      raise "Can't eval `#{value}`"
+    end
+
+    def load_from_yaml(config_path)
+      YAML.load_file config_path
+    end
+  end
+end

data/mongo_masker.gemspec

@@ -0,0 +1,23 @@
+Gem::Specification.new do |s|
+  s.name        = 'mongo_masker'
+  s.version     = '1.0.0'
+  s.date        = '2019-02-19'
+  s.summary     = 'Mongo masker'
+  s.description = 'Masking production data mongodb for testing/development'
+  s.authors     = ['Clicia Scarlet']
+  s.email       = ['pnvduc@gmail.com']
+  s.files    = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.bindir   = 'bin'
+  s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.require_paths = ['lib']
+  s.homepage    = 'https://github.com/basicinc/Masker'
+  s.license     = 'MIT'
+
+  s.add_runtime_dependency 'thor'
+  s.add_runtime_dependency 'mongo'
+  s.add_runtime_dependency 'ffaker'
+  s.add_runtime_dependency 'irb'
+
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'rake'
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: mongo_masker
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Clicia Scarlet
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-02-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ffaker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: irb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Masking production data mongodb for testing/development
+email:
+- pnvduc@gmail.com
+executables:
+- mongo_masker
+extensions: []
+extra_rdoc_files: []
+files:
+- ".dockerignore"
+- ".gitignore"
+- Dockerfile
+- Gemfile
+- Gemfile.lock
+- README.md
+- bin/mongo_masker
+- docker-compose.yml
+- lib/mongo_masker.rb
+- mongo_masker.gemspec
+homepage: https://github.com/basicinc/Masker
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Mongo masker
+test_files: []