anon 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b58dd8e96a5488b0ae05d43be7b9a9cc2162db34
+  data.tar.gz: e8ae73279bd357ab92e6aa258426d63206db3734
+SHA512:
+  metadata.gz: 162ebe61b83c353976ff81da3d2882d32cbf9b0b464a3b6b8a67e3ad9bdce20abdaa7cadbdcd91ce0b3ac7838f1f72ef98dfe21b38ed6b104c7605e105f0c046
+  data.tar.gz: ab5a70456e30880db8e2a238454ddc003813c339218ccbd152ee3e7d5dac656cfd5a215948dab083d02bbad08b9552695641afb9f162b1402ea5245a784438c3

data/bin/anon

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+require 'anon/cli'
+Anon::CLI.start(ARGV)

data/lib/anon/base.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+
+require 'time_difference'
+
+module Anon
+  # Anonymiser base class
+  class Base
+
+    # Performs anonymisation
+    def self.anonymise!(*args)
+      new(*args).anonymise!
+    end
+
+    protected
+
+    # Helper method that replaces a personal e-mail
+    # with an anonymous one.
+    #
+    # The same personal e-mail will be replaced
+    # with the same anonymous e-mail.
+    def anonymous_email(personal_email)
+      @anonymised_emails ||= {}
+
+      unless @anonymised_emails.key? personal_email
+        next_count = @anonymised_emails.count + 1
+        @anonymised_emails[personal_email] = "anon#{next_count}@anon.com"
+      end
+
+      @anonymised_emails[personal_email]
+    end
+
+    # Initializes progress tracking.
+    def start_progress
+      @progress = 0
+      @started = Time.now
+      update_progress
+    end
+
+    # Adds 1 to the progress count.
+    def increment_progress
+      @progress += 1
+      update_progress
+    end
+
+    # End progress tracking and output the results.
+    def complete_progress
+      stopped = Time.now
+      duration = TimeDifference.between(@started, stopped).in_seconds
+      if duration == 0
+        average = @progress
+      else
+        average = (@progress.to_f / duration.to_f).round
+      end
+
+      $stderr.puts "Read #{@progress} lines in #{duration} seconds (#{average} lines/s)"
+      $stderr.puts "#{@anonymised_emails.count} unique e-mails replaced"
+    end
+
+    private
+
+    def update_progress
+      output_progress if @progress % 100 == 0
+    end
+
+    def output_progress
+      $stderr.print "Working... #{@progress}\r"
+      $stderr.flush
+    end
+  end
+end

data/lib/anon/cli.rb

@@ -0,0 +1,57 @@
+require 'thor'
+
+module Anon
+  # Command Line Interface for Anon
+  class CLI < Thor
+
+    desc 'csv [OPTIONS]', 'Anonymise a csv file'
+
+    option :infile,
+           aliases: [:i],
+           desc: 'input filename to read from, reads from STDIN if ommited'
+
+    option :outfile,
+           aliases: [:o],
+           desc: 'output filename write to, writes to STDOUT if ommited'
+
+    option :columns,
+           aliases: [:c],
+           desc: 'columns to anonymise, by index or name
+e.g. 0,1,5 or email-address,other_email, guesses based on header if ommited'
+
+    option :header,
+           type: :boolean,
+           default: true,
+           desc: 'if the csv file to be processed has a header row'
+
+    def csv
+      require 'anon/csv'
+      Anon::CSV.anonymise!(input, output, options[:columns], options[:header])
+    end
+
+    desc 'text [OPTIONS]', 'Anonymise a text file'
+
+    option :infile,
+           aliases: [:i],
+           desc: 'input filename to read from, reads from STDIN if ommited'
+
+    option :outfile,
+           aliases: [:o],
+           desc: 'output filename write to, writes to STDOUT if ommited'
+
+    def text
+      require 'anon/text'
+      Anon::Text.anonymise!(input, output)
+    end
+
+    private
+
+    def input
+      options[:infile] ? File.open(options[:infile]) : $stdin
+    end
+
+    def output
+      options[:outfile] ? File.open(options[:outfile], 'w') : $stdout
+    end
+  end
+end

data/lib/anon/csv/columns.rb

@@ -0,0 +1,33 @@
+require 'anon/base'
+
+module Anon
+  class CSV < Base
+    class Columns
+      def initialize(columns, headers)
+        @columns, @headers = columns, headers
+      end
+
+      def to_anonymise
+        @_to_anonymise ||= indexes || columns || best_guess
+      end
+
+      private
+
+      attr_reader :headers
+
+      def indexes
+        columns.map { |c| Integer(c) } if columns
+      rescue ArgumentError
+        nil
+      end
+
+      def columns
+        @columns.split(',') if @columns
+      end
+
+      def best_guess
+        headers.select { |h| h.match(/e.*mail/i) }
+      end
+    end
+  end
+end

data/lib/anon/csv.rb

@@ -0,0 +1,63 @@
+# encoding: utf-8
+
+require 'anon/base'
+require 'anon/csv/columns'
+require 'csv'
+
+module Anon
+  # Replaces the contents of a set of columns with anonymous e-mails.
+  class CSV < Base
+    def initialize(input, output, columns_to_anonymise, has_header = true)
+      @input = input
+      @output = output
+      @columns_to_anonymise = columns_to_anonymise
+      @has_header = has_header
+    end
+
+    # Anonymises all content of the columns set in the initializer
+    def anonymise!
+      start_progress
+      map_lines do |line|
+        anonymise(line)
+        increment_progress
+        line
+      end
+      complete_progress
+    end
+
+    private
+
+    attr_reader :has_header
+
+    def anonymise(line)
+      columns.to_anonymise.each do |anon_index|
+        line[anon_index] = anonymous_email(line[anon_index])
+      end
+    end
+
+    def input
+      @_input ||= ::CSV.new(@input, headers: has_header)
+    end
+
+    def output
+      @_output ||= ::CSV.new(@output, write_headers: has_header, headers: headers)
+    end
+
+
+    def columns
+      @_columns ||= Columns.new(@columns_to_anonymise, input.headers)
+    end
+
+    # Reads each line from the incoming file, processes it using the block
+    # and saves the return value of the block to the outgoing file.
+    def map_lines
+      while (inline = input.gets)
+        output.puts yield(inline)
+      end
+    end
+
+    def headers
+      input.headers
+    end
+  end
+end

data/lib/anon/text.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+require 'anon/base'
+
+module Anon
+  # Anonymises any detected e-mail address in a text file.
+  class Text < Base
+    # From the email regex research: http://fightingforalostcause.net/misc/2006/compare-email-regex.php
+    # Authors: James Watts and Francisco Jose Martin Moreno
+    EMAIL_REGEX = /([\w\!\#\z\%\&\'\*\+\-\/\=\?\\A\`{\|\}\~]+\.)*[\w\+-]+@((((([a-z0-9]{1}[a-z0-9\-]{0,62}[a-z0-9]{1})|[a-z])\.)+[a-z]{2,6})|(\d{1,3}\.){3}\d{1,3}(\:\d{1,5})?)/i # rubocop:disable Metrics/LineLength
+
+    def initialize(incoming_filename, outgoing_filename)
+      @input = incoming_filename
+      @output = outgoing_filename
+    end
+
+    # Anonymises any e-mail addresses found in the text
+    def anonymise!
+      start_progress
+      map_lines do |line|
+        line = anonymise_line(line)
+        increment_progress
+        line
+      end
+      complete_progress
+    end
+
+    private
+
+    attr_reader :input, :output
+
+    # Reads each line from the incoming file, processes it using the block
+    # and saves the return value of the block to the outgoing file.
+    def map_lines
+      while (inline = input.gets)
+        output.puts yield(inline)
+      end
+    end
+
+    def anonymise_line(line)
+      line.gsub(EMAIL_REGEX) { |email| anonymous_email(email) }
+    end
+  end
+end

data/lib/anon.rb

@@ -0,0 +1,3 @@
+require 'anon/base'
+require 'anon/csv'
+require 'anon/text'

data/spec/intergration/cli_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+require 'fileutils'
+
+describe 'The comand line interface' do
+  let(:expected_output) do
+    'pgid_sku,contract start date,e---mail,post_code,product_category,transmission_type
+F^u4bgebdfds, 15/07/1756,anon1@anon.com, XY1 TH7, jokes > joke books, oily
+FNY7dujh, 15/11/1856,anon2@anon.com, CZ1 NJ7, books > dusty, automatic
+'
+  end
+
+  describe 'text' do
+
+    describe 'reading from a file' do
+      let(:output) { `bin/anon text -i spec/fixture/csv_with_headers.csv` }
+      specify { expect(output).to eq expected_output }
+    end
+
+    describe 'reading from stdin' do
+      let(:output) { `cat spec/fixture/csv_with_headers.csv | bin/anon text` }
+      specify { expect(output).to eq expected_output }
+    end
+
+    describe 'writing to a file' do
+      let(:output) do
+        FileUtils.rm_rf('spec/fixture/test.out')
+        `cat spec/fixture/csv_with_headers.csv | bin/anon text -o spec/fixture/test.out`
+        File.read('spec/fixture/test.out')
+      end
+      specify { expect(output).to eq expected_output }
+    end
+
+  end
+
+  describe 'csv' do
+    describe 'reading from a file' do
+      let(:output) { `bin/anon csv -i spec/fixture/csv_with_headers.csv -c 2` }
+      specify { expect(output).to eq expected_output }
+    end
+
+    describe 'reading from stdin' do
+      let(:output) { `cat spec/fixture/csv_with_headers.csv | bin/anon csv -c 2` }
+      specify { expect(output).to eq expected_output }
+    end
+
+    describe 'writing to a file' do
+      let(:output) do
+        FileUtils.rm_rf('spec/fixture/test.out')
+        `cat spec/fixture/csv_with_headers.csv | bin/anon csv -c 2 -o spec/fixture/test.out`
+        File.read('spec/fixture/test.out')
+      end
+      specify { expect(output).to eq expected_output }
+    end
+
+    describe 'a headerless file' do
+      let(:expected_output) do
+        'F^u4bgebdfds, 15/07/1756,anon1@anon.com, XY1 TH7, jokes > joke books, oily
+FNY7dujh, 15/11/1856,anon2@anon.com, CZ1 NJ7, books > dusty, automatic
+'
+      end
+
+      let(:output) { `bin/anon csv -c 2 -i spec/fixture/csv_without_headers.csv --no-header` }
+      specify { expect(output).to eq expected_output }
+    end
+
+    describe 'named headers' do
+      let(:output) { `bin/anon csv -i spec/fixture/csv_with_headers.csv -c e---mail` }
+      specify { expect(output).to eq expected_output }
+    end
+
+    describe 'automatic header detection' do
+      let(:output) { `bin/anon csv -i spec/fixture/csv_with_headers.csv` }
+      specify { expect(output).to eq expected_output }
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+require 'simplecov'
+SimpleCov.start
+
+def capture_stdout
+  real_stdout = $stdout
+  $stdout = StringIO.new
+  yield
+  $stdout.string
+ensure
+  $stdout = real_stdout
+end

data/spec/unit/cli_spec.rb

@@ -0,0 +1,46 @@
+require 'spec_helper'
+
+require 'anon/cli'
+require 'anon/csv'
+require 'anon/text'
+
+describe Anon::CLI do
+  describe '#text' do
+    context 'without options' do
+      it 'calls the text anonomiser with standard in and out' do
+        expect(Anon::Text).to receive(:anonymise!).with($stdin, $stdout)
+        subject.text
+      end
+    end
+
+    context 'specifying an input file' do
+      it 'calls the text anonomiser with the file' do
+        expect(Anon::Text).to receive(:anonymise!) do |input, _output|
+          expect(input).to be_a File
+          expect(input.path).to eq 'spec/fixture/test.in'
+        end
+        subject.options = { infile: 'spec/fixture/test.in' }
+        subject.text
+      end
+    end
+
+    context 'specifying an output file' do
+      it 'calls the text anonomiser with the file' do
+        expect(Anon::Text).to receive(:anonymise!) do |input, _output|
+          expect(input).to be_a File
+          expect(input.path).to eq 'spec/fixture/test.in'
+        end
+        subject.options = { infile: 'spec/fixture/test.in' }
+        subject.text
+      end
+    end
+  end
+
+  describe '#csv' do
+    it 'calls the csv anonomiser with the correct columns' do
+      expect(Anon::CSV).to receive(:anonymise!).with($stdin, $stdout, '1,4,7', nil)
+      subject.options = { columns: '1,4,7' }
+      subject.csv
+    end
+  end
+end

data/spec/unit/columns_spec.rb

@@ -0,0 +1,93 @@
+require 'spec_helper'
+require 'anon/csv/columns'
+
+describe Anon::CSV::Columns do
+  describe '#to_anonymise' do
+    context 'index columns' do
+      subject { described_class.new('1,2,3', double) }
+
+      it 'converts the indexes to Integers' do
+        expect(subject.to_anonymise).to eq [1, 2, 3]
+      end
+    end
+
+    context 'named columns' do
+      subject { described_class.new('email_address,name,foo', double) }
+
+      it 'returns the named columns' do
+        expect(subject.to_anonymise).to eq %w(email_address name foo)
+      end
+    end
+
+    context 'automatic column detection' do
+      let(:email_headers) do
+        [
+          'email',
+          'zemailaddress',
+          'contact email address',
+          'emailaddress',
+          'e-mail',
+          'personal email',
+          'email address',
+          'ct_email_addr',
+          'e_mail_address',
+          'student_e_mail',
+          'e-mail address',
+          'parent email address',
+          'customer email address',
+          'email_address',
+          'contact address.email',
+          'user email string',
+          'email client',
+          'client email',
+          'ot ship-to email address',
+          'customer email',
+          'clnp email address1',
+          'default_email',
+          'email_addr',
+          'sender_email',
+          'webemail',
+        ]
+      end
+
+      let(:non_email_headers) do
+        [
+          'pgid_sku',
+          'contract start date',
+          'post_code',
+          'product_category',
+          'transmission_type',
+          'policy_start_date',
+          'category-genericname',
+          'order id',
+          'unique_model',
+          'start',
+          'image',
+          'orderdate',
+          'model',
+          'homebound date',
+          'pid-cid',
+          'ct_name_last',
+          'booking ref.',
+          'a5',
+          'id',
+          'bazaarvoice id',
+          'questionnaire_name',
+          'deeplink_id',
+          'hotel_name',
+          'series_identifier',
+          'registration',
+          'zmailname',
+        ]
+      end
+
+      let(:headers) { email_headers + non_email_headers }
+
+      subject { described_class.new(nil, headers.shuffle) }
+
+      it 'returns the email related headers' do
+        expect(subject.to_anonymise.sort).to eq email_headers.sort
+      end
+    end
+  end
+end

data/spec/unit/csv_spec.rb

@@ -0,0 +1,76 @@
+require 'spec_helper'
+require 'anon/csv'
+
+describe Anon::CSV do
+
+  describe '.anonymise!' do
+
+    before do
+      described_class.anonymise!(input_stream, output_stream, columns, headers)
+    end
+
+    let(:output_stream) { StringIO.new }
+    let(:columns) { '0' }
+
+    context 'with headers' do
+      let(:headers) { true }
+
+      let(:input_stream) do
+        StringIO.new "email,foo,bar
+        foo@bar.com,34545,bannas
+        foopface@fooofoo.co.uk,124353,apples"
+      end
+
+      it 'anonymises the correct column' do
+        output_stream.rewind
+        expect(output_stream.gets).to eq "email,foo,bar\n"
+        expect(output_stream.gets).to eq "anon1@anon.com,34545,bannas\n"
+        expect(output_stream.gets).to eq "anon2@anon.com,124353,apples\n"
+        expect(output_stream.gets).to eq nil
+      end
+
+      context 'with named columns' do
+        let(:columns) { 'email,bar' }
+
+        it 'anonymises the correct columns' do
+          output_stream.rewind
+          expect(output_stream.gets).to eq "email,foo,bar\n"
+          expect(output_stream.gets).to eq "anon1@anon.com,34545,anon2@anon.com\n"
+          expect(output_stream.gets).to eq "anon3@anon.com,124353,anon4@anon.com\n"
+          expect(output_stream.gets).to eq nil
+        end
+      end
+
+      context 'without named columns' do
+        let(:columns) { nil }
+
+        it 'guesses the columns to anonymise' do
+          output_stream.rewind
+          expect(output_stream.gets).to eq "email,foo,bar\n"
+          expect(output_stream.gets).to eq "anon1@anon.com,34545,bannas\n"
+          expect(output_stream.gets).to eq "anon2@anon.com,124353,apples\n"
+          expect(output_stream.gets).to eq nil
+        end
+      end
+
+    end
+
+    context 'without headers' do
+      let(:headers) { false }
+
+      let(:input_stream) do
+        StringIO.new "looloo@example.com,2456,satsuma
+        foo@bar.com,34545,bannas
+        foopface@fooofoo.co.uk,124353,apples"
+      end
+
+      it 'anonymises the correct column' do
+        output_stream.rewind
+        expect(output_stream.gets).to eq "anon1@anon.com,2456,satsuma\n"
+        expect(output_stream.gets).to eq "anon2@anon.com,34545,bannas\n"
+        expect(output_stream.gets).to eq "anon3@anon.com,124353,apples\n"
+        expect(output_stream.gets).to eq nil
+      end
+    end
+  end
+end

data/spec/unit/text_spec.rb

@@ -0,0 +1,24 @@
+require 'spec_helper'
+require 'anon/text'
+
+describe Anon::Text do
+  let(:output_stream) { StringIO.new }
+  subject { described_class.new(input_stream, output_stream) }
+
+  describe '#anonymise!' do
+    let(:input_stream) do
+      StringIO.new ' someone@foo.com then some other interesting text another@icecream.museum'
+    end
+
+    it 'anonymises any email addresss' do
+      subject.anonymise!
+      expect(output_stream.string).to_not include 'someone@foo.com'
+      expect(output_stream.string).to_not include 'another@icecream.museum'
+    end
+
+    it 'leaves the other text alone' do
+      subject.anonymise!
+      expect(output_stream.string).to eq " anon1@anon.com then some other interesting text anon2@anon.com\n"
+    end
+  end
+end

metadata

@@ -0,0 +1,142 @@
+--- !ruby/object:Gem::Specification
+name: anon
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Reevoo Engineering
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: time_difference
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: reevoocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Replaces personal data with fake data
+email: developers@reevoo.com
+executables:
+- anon
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/anon
+- lib/anon.rb
+- lib/anon/base.rb
+- lib/anon/cli.rb
+- lib/anon/csv.rb
+- lib/anon/csv/columns.rb
+- lib/anon/text.rb
+- spec/intergration/cli_spec.rb
+- spec/spec_helper.rb
+- spec/unit/cli_spec.rb
+- spec/unit/columns_spec.rb
+- spec/unit/csv_spec.rb
+- spec/unit/text_spec.rb
+homepage: https://github.com/reevoo/anon
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Replaces personal data with fake data
+test_files: []
+has_rdoc: