privacy 0.1.0 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e43f59881052bff4df200d87ea0058f3c1400235d7bbc947dcde15727b1cd48
-  data.tar.gz: 57fd3cf9d93c96980faa238f0a2611ceffed53782acaf34d981b76d3b6552877
+  metadata.gz: 70f75e3ceaea1bcf698189b550f06b19898d813fb1f6c69ce47bb3c2770f7eb9
+  data.tar.gz: d234953c1819d6ae303cabc4239be5b06be7945b0fdf3de2b8091859a440e961
 SHA512:
-  metadata.gz: 70766c9d2aa81bdd286cdbae07f8080ecb19996b50285b66d0a1e846f8c457409f425ba01d16d9f213ce76604e0d244fb151c17572399806fa3b047b4b137161
-  data.tar.gz: 334375cc60576731aaf10c28a7409d61fd3f35f39dd645515fffba4f85cca2cd5d29ee8fcd210e18996099b7e53b10007c7cdbfa1b98983eb186f20300447f1e
+  metadata.gz: 542f23027312376c876cdccab5fe24de7e7168becf3f842ce143eb5bf892c876089845168dc2fd99033f59edf0b018125a2f5e1fe386f59844f876f2c6330644
+  data.tar.gz: 7d1a71839a9b8abe462579f73eaf91ac0da26ae12cad07940d495d56f35b697d1b1a4290d3a92e55c4e13ed73a016ab12a6414543971e7dcb5dd9ae12f0396aa

data/.gitignore

@@ -6,3 +6,6 @@
 /pkg/
 /spec/reports/
 /tmp/
+*.xlsx
+*.xls
+*.csv

data/Gemfile.lock

@@ -1,12 +1,28 @@
 PATH
   remote: .
   specs:
-    privacy (0.1.0)
+    privacy (0.1.4)
+      creek
+      thor
+      write_xlsx
 
 GEM
   remote: https://rubygems.org/
   specs:
+    creek (2.5.2)
+      nokogiri (>= 1.10.0)
+      rubyzip (>= 1.0.0)
+    mini_portile2 (2.4.0)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
     rake (12.3.3)
+    rubyzip (2.3.0)
+    thor (1.0.1)
+    write_xlsx (0.85.7)
+      rubyzip (>= 1.0.0)
+      zip-zip
+    zip-zip (0.3)
+      rubyzip (>= 1.0.0)
 
 PLATFORMS
   ruby

data/README.md

@@ -1,8 +1,6 @@
 # Privacy
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/privacy`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-TODO: Delete this and the text above, and describe your gem
+Privacy removes personal data from a xls file.
 
 ## Installation
 
@@ -22,9 +20,9 @@ Or install it yourself as:
 
 ## Usage
 
-```
-privacy process ./file.xlsx
-```
+Process all excel files in current directory `privacy process`.
+
+Process a specific file in current directory `privacy process /file.xlsx`.
 
 ## Development
 
@@ -32,6 +30,10 @@ After checking out the repo, run `bin/setup` to install dependencies. You can al
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
+To release a new version, change version number and `rake release`.
+
+To work on the code, use `bundle exec privacy process`.
+
 ## Contributing
 
 Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/privacy. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/privacy/blob/master/CODE_OF_CONDUCT.md).

data/lib/privacy.rb

@@ -1,11 +1,13 @@
 require 'privacy/version'
+require 'privacy/processor'
 require 'thor'
 
 module Privacy
   class CLI < Thor
     desc "process [file]", "Make data private"
-    def process(file)
-      puts file
+    def process(file = nil)
+      file.nil? ? Dir.glob("*.xlsx") { |file| Processor.new(file) }
+                : Processor.new(file)
     end
   end
 end

data/lib/privacy/person.rb

@@ -0,0 +1,133 @@
+class Privacy::Person
+  attr_reader :identifier, :row, :index, :columns, :data, :combinations, :initials
+
+  COLUMNS_TO_ANONYMIZE = [
+    'Nom _identifier_',
+    'Prénom _identifier_',
+    'Date de naissance _identifier_',
+    'Lieu naissance Pays _identifier_',
+    'Lieu naissance Ville _identifier_',
+    'Lieu naissance Département _identifier_',
+    'Sexe _identifier_',
+    'Nationalité _identifier_',
+    'Adresse _identifier_ [Date]',
+    'Adresse _identifier_ [Département]',
+    'Adresse _identifier_ [Ville]',
+    'Adresse _identifier_ [Pays]',
+    'Profession _identifier_',
+    'Date profession _identifier_'
+  ]
+
+  INITIALS_SEPARATORS = [
+    '',
+    '.'
+  ]
+
+  def initialize(identifier)
+    @identifier = identifier
+    puts "Identify #{identifier}"
+  end
+
+  def identify_columns(row)
+    @columns = {}
+    row.each do |key, value|
+      COLUMNS_TO_ANONYMIZE.each do |column|
+        name = column_with_identifier column
+        if value == name
+          letter = key.gsub '1', ''
+          @columns[letter] = name
+          puts "  #{name} on column #{letter}"
+        end
+      end
+    end
+  end
+
+  def anonymize(row, index)
+    @row = row
+    @index = index + 1 # Excel is 1-indexed
+    anonymize_personal_data
+    build_combinations
+    remove_combinations
+  end
+
+  protected
+
+  def anonymize_personal_data
+    @data = {}
+    puts
+    puts "Anonymize row #{index}"
+    puts "  #{identifier}"
+    @columns.each do |letter, column|
+      cell = "#{letter}#{index}"
+      value = row[cell]
+      next if value.nil?
+      @data[column] = value
+      puts "      #{value} (#{column})"
+      row[cell] = "[#{column}]"
+    end
+  end
+
+  def build_combinations
+    puts 'build_combinations'
+    @combinations = []
+    first_name_column = column_with_identifier 'Prénom _identifier_'
+    last_name_column = column_with_identifier 'Nom _identifier_'
+    first_name = data[first_name_column]
+    last_name = data[last_name_column]
+    if !first_name.nil? && !last_name.nil?
+      initials = []
+      first_name.split(' ').each { |l| initials << l[0].upcase }
+      last_name.split(' ').each { |l| initials << l[0].upcase }
+      @combinations << "#{first_name} #{last_name}"
+      @combinations << " #{first_name} #{last_name}"
+      @combinations << "#{first_name} #{last_name} "
+      @combinations << "#{last_name} #{first_name}"
+      @combinations << " #{last_name} #{first_name}"
+      @combinations << "#{last_name} #{first_name} "
+      @combinations += initials_combinations(initials)
+    end
+    puts "Combinations: #{@combinations.join(', ')}"
+  end
+
+  def column_with_identifier(column)
+    column.gsub '_identifier_', identifier
+  end
+
+  def initials_combinations(initials)
+    puts "Initials for #{initials}"
+    combinations = []
+    initials.each do |letter|
+      initials.shift
+      recursive_combinations = initials_combinations initials
+      INITIALS_SEPARATORS.each do |separator|
+        initial_with_separator = "#{letter}#{separator}"
+        if recursive_combinations.none?
+          combinations << "#{initial_with_separator}"
+          combinations << "#{initial_with_separator} "
+          combinations << " #{initial_with_separator}"
+        else
+          recursive_combinations.each do |c|
+            combinations << "#{initial_with_separator}#{c}"
+            combinations << " #{initial_with_separator}#{c}"
+            combinations << "#{initial_with_separator} #{c}"
+            combinations << "#{initial_with_separator} #{c} "
+            combinations << "#{initial_with_separator}#{c} "
+          end
+        end
+      end
+    end
+    combinations.uniq
+  end
+
+  def remove_combinations
+    row.each do |key, value|
+      anonymized_value = value
+      if value.is_a? String
+        combinations.each do |token|
+          anonymized_value = anonymized_value.gsub(token, "[#{identifier}]")
+        end
+      end
+      row[key] = anonymized_value
+    end
+  end
+end

data/lib/privacy/processor.rb

@@ -0,0 +1,79 @@
+require 'creek'
+require 'write_xlsx'
+require 'privacy/person'
+
+class Privacy::Processor
+  attr_reader :file
+
+  DIRECTORY = 'processed'
+
+  def initialize(file)
+    @file = file
+    process unless @file.start_with? '~'
+  end
+
+  protected
+
+  def process
+    puts "Processing #{@file}"
+    make_directory
+    filter_data
+    write_file
+  end
+
+  def make_directory
+    Dir.mkdir DIRECTORY unless File.exists? DIRECTORY
+  end
+
+  def filter_data
+    @ppmec = Privacy::Person.new 'ppmec'
+    @ppmec.identify_columns sheet.rows.first
+    @ppvic = Privacy::Person.new 'ppvic'
+    @ppvic.identify_columns sheet.rows.first
+    sheet.rows.first.each do |key, value|
+      worksheet.write key, value
+    end
+    sheet.rows.each_with_index do |row, index|
+      next if index.zero?
+      @ppmec.anonymize row, index
+      @ppvic.anonymize row, index
+      row.each do |key, value|
+        worksheet.write key, value
+      end
+    end
+  end
+
+  def write_file
+    workbook.close
+  end
+
+  # New file
+  # http://cxn03651.github.io/write_xlsx/index.html
+
+  def workbook
+    @workbook ||= WriteXLSX.new processed_file_name
+  end
+
+  def worksheet
+    @worksheet ||= workbook.add_worksheet
+  end
+
+  def processed_file_name
+    "#{DIRECTORY}/#{file}"
+  end
+
+  # Current file
+  # https://github.com/pythonicrubyist/creek
+
+  def path
+    @path ||= File.expand_path file
+  end
+
+  def data
+    @data ||= Creek::Book.new path, with_headers: true
+  end
+
+  def sheet
+    @sheet ||= data.sheets[0]
+  end
+end

data/lib/privacy/version.rb

@@ -1,3 +1,3 @@
 module Privacy
-  VERSION = "0.1.0"
+  VERSION = "0.1.5"
 end

data/privacy.gemspec

@@ -1,29 +1,31 @@
 require_relative 'lib/privacy/version'
 
-Gem::Specification.new do |spec|
-  spec.name          = "privacy"
-  spec.version       = Privacy::VERSION
-  spec.authors       = ["Arnaud Levy"]
-  spec.email         = ["contact@arnaudlevy.com"]
+Gem::Specification.new do |s|
+  s.name          = "privacy"
+  s.version       = Privacy::VERSION
+  s.authors       = ["Arnaud Levy"]
+  s.email         = ["contact@arnaudlevy.com"]
 
-  spec.summary       = "Privacy removes personal data from a xls file"
-  spec.description   = "Remove first name and last name from xls files, replace them with an anonymous placeholder"
-  spec.homepage      = "https://github.com/arnaudlevy/privacy"
-  spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  s.summary       = "Privacy removes personal data from a xls file"
+  s.description   = "Remove first name and last name from xls files, replace them with an anonymous placeholder"
+  s.homepage      = "https://github.com/arnaudlevy/privacy"
+  s.license       = "MIT"
+  s.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
 
-  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+  # s.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = "https://github.com/arnaudlevy/privacy"
+  s.metadata["homepage_uri"] = s.homepage
+  s.metadata["source_code_uri"] = "https://github.com/arnaudlevy/privacy"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
-  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
-    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  s.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|s|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-  spec.add_dependency "thor", "~> 0.20"
+  s.bindir        = "exe"
+  s.executables   = s.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency "thor"
+  s.add_dependency "creek"
+  s.add_dependency "write_xlsx"
 end

metadata

@@ -1,29 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: privacy
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.5
 platform: ruby
 authors:
 - Arnaud Levy
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-27 00:00:00.000000000 Z
+date: 2020-06-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.20'
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: creek
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: write_xlsx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Remove first name and last name from xls files, replace them with an
   anonymous placeholder
 email:
@@ -33,6 +61,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".DS_Store"
 - ".gitignore"
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -44,6 +73,8 @@ files:
 - bin/setup
 - exe/privacy
 - lib/privacy.rb
+- lib/privacy/person.rb
+- lib/privacy/processor.rb
 - lib/privacy/version.rb
 - privacy.gemspec
 homepage: https://github.com/arnaudlevy/privacy