security_guard 0.0.3 → 0.0.4

data/.gitignore

@@ -15,4 +15,5 @@ rdoc
 spec/reports
 test/tmp
 test/version_tmp
+spec/tmp
 tmp

data/README.md

@@ -36,7 +36,7 @@ country_ips.ips = ['4.4.4.4', '8.8.8.8', '203.206.0.1']
 # the above is equivalent to:
 
 country_ips = SecurityGuard::CountryIps.new(
-  :countries => ['Australia', 'United Kingdom']
+  :countries => ['Australia', 'United Kingdom'],
   :ips       => ['4.4.4.4', '8.8.8.8', '203.206.0.1']
 )
 ```
@@ -46,9 +46,10 @@ country_ips = SecurityGuard::CountryIps.new(
 Returns a list of the IPs from given country and IP dictionaries. Useful for auditing IPs from higher risk nations.
 
 ```ruby
-country_ips = SecurityGuard::CountryIps.new
-country_ips.countries = ['Australia', 'United Kingdom']
-country_ips.ips = ['4.4.4.4', '8.8.8.8', '203.206.0.1']
+country_ips = SecurityGuard::CountryIps.new(
+  :countries => ['Australia'],
+  :ips       => ['4.4.4.4', '8.8.8.8', '203.206.0.1']
+)
 country_ips.result # => ['203.206.0.1']
 ```
 
@@ -59,8 +60,24 @@ country_ips.countries_from_file = '/path/to/the/file'
 country_ips.ips_from_file = '/path/to/the/file'
 ```
 
+### Deduplication
+
+Deduplicates content contained within a list of files. Useful for deduplicating email newsletter subscription lists.
+
+```ruby
+dedupe = SecurityGuard::Deduplication.new(
+  :input_folder  => '/path/to/the/input/folder',
+  :output_folder => '/path/to/the/output/folder'
+).process
+```
+
 ## Changelog
 
+### v0.0.4 [2012-02-08]
+
+- Fixed Concerns::Initializable for CountryIps
+- Added Deduplication
+
 ### v0.0.3 [2012-01-20]
 
 - Added Concerns::Initializable

data/bin/sguard

@@ -12,10 +12,10 @@ module SecurityGuard
     parameter '[output_path]', 'output the results in a text file'
 
     def execute
-      country_ips = SecurityGuard::CountryIps.new
-
-      country_ips.ips_from_file       = ip_addresses_path
-      country_ips.countries_from_file = countries_path
+      country_ips = SecurityGuard::CountryIps.new(
+        :ips_from_file       => ip_addresses_path,
+        :countries_from_file => countries_path
+      )
 
       if output_path
         MainCommand.output_result(country_ips.result, output_path)
@@ -25,6 +25,20 @@ module SecurityGuard
     end
   end
 
+  class DeduplicationCommand < Clamp::Command
+    parameter 'input_folder', 'folder containing the input files'
+    parameter 'output_folder', 'folder for outputing deduped files'
+
+    def execute
+      SecurityGuard::Deduplication.new(
+        :input_folder  => input_folder,
+        :output_folder => output_folder
+      ).process
+
+      ap "Deduplicated results published to '#{output_folder}'."
+    end
+  end
+
   class MainCommand < Clamp::Command
     def self.output_result(result, output_path)
       `echo #{result} > #{output_path}`
@@ -34,6 +48,10 @@ module SecurityGuard
     subcommand 'country_ips',
                'Returns a list of the IPs from given country and IP dictionaries.',
                SecurityGuard::CountryIpsCommand
+
+    subcommand 'deduplication',
+               'Deduplicates content contained within a list of files.',
+               SecurityGuard::DeduplicationCommand
   end
 end
 

data/lib/security_guard.rb

@@ -4,3 +4,4 @@ require 'security_guard/utils/geo_ips'
 require 'security_guard/concerns/accepts_from_file'
 require 'security_guard/concerns/initializable'
 require 'security_guard/country_ips'
+require 'security_guard/deduplication'

data/lib/security_guard/concerns/initializable.rb

@@ -5,10 +5,10 @@ module SecurityGuard
         klass.extend ClassMethods
       end
 
-      def initializable_attrs(attributes)
+      def initializable_attrs(attributes = nil)
         attributes.each do |name, value|
           self.send "#{name}=", value
-        end
+        end if attributes
       end
 
       module ClassMethods

data/lib/security_guard/country_ips.rb

@@ -1,11 +1,14 @@
 module SecurityGuard
   class CountryIps
     include Concerns::AcceptsFromFile
+    include Concerns::Initializable
 
     accepts_from_file :countries, :ips
+    initializable     :countries, :ips
 
-    def initialize
+    def initialize(args = nil)
       @geoip ||= Utils::GeoIps.new(Utils::Files.load('GeoIP.dat'))
+      initializable_attrs args
     end
 
     def result

data/lib/security_guard/deduplication.rb

@@ -0,0 +1,64 @@
+module SecurityGuard
+  class Deduplication
+    include Concerns::Initializable
+
+    initializable :input_folder, :output_folder
+    attr_accessor :source_data, :deduped_data
+
+    def initialize(args = nil)
+      @source_data  ||= []
+      @deduped_data ||= []
+      initializable_attrs args
+    end
+
+    def process
+      read_data_from input_folder
+      dedupe source_data
+      write_data_to output_folder
+    end
+
+    private
+
+    def read_data_from(folder)
+      files = Dir["#{folder}/*"]
+
+      raise Exception.new('Input folder is invalid or is empty.') if files.empty?
+
+      files.each do |file|
+        source_data << File.readlines(file).map{ |line| line.downcase.strip }
+      end
+    end
+
+    def write_data_to(folder)
+      deduped_data.each_with_index do |array, index|
+        File.open("#{folder}/#{index+1}.txt", 'w') do |f|
+          f.puts array
+        end
+      end
+    end
+
+    def dedupe(data)
+      # start from the lowest array (in terms of dedupe priority)
+      data.reverse!
+      data_original = data.clone
+      data_original.each do |array|
+        data.shift
+        deduped_data << _deduped_multi(array, data)
+      end
+      # the top array doesn't need to be compared, just needs to be unique
+      deduped_data.last.uniq!
+      deduped_data.reverse!
+    end
+
+    def _deduped_multi(target, others)
+      others.each do |array|
+        target = _deduped(target, array)
+      end
+      target
+    end
+
+    def _deduped(target, other)
+      (target - other & target).uniq
+    end
+  end
+end

data/lib/security_guard/version.rb

@@ -1,3 +1,3 @@
 module SecurityGuard
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

data/specs/concerns/initializable_spec.rb

@@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 class TestInitializableWithInitializer
   include SecurityGuard::Concerns::Initializable
   initializable :attr1, :attr2
-  def initialize(args)
+  def initialize(args = nil)
     initializable_attrs args
   end
 end
@@ -21,4 +21,12 @@ describe SecurityGuard::Concerns::Initializable do
       test.attr2.must_equal 'test2'
     end
   end
+
+  it 'without any initialiser arguments' do
+    test = TestInitializableWithInitializer.new
+    test.attr1 = 'test1'
+    test.attr2 = 'test2'
+    test.attr1.must_equal 'test1'
+    test.attr2.must_equal 'test2'
+  end
 end

data/specs/country_ips_spec.rb

@@ -2,9 +2,10 @@ require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
 describe SecurityGuard::CountryIps do
   before do
-    @country_ips = SecurityGuard::CountryIps.new
-    @country_ips.countries = ['Australia']
-    @country_ips.ips = ['4.4.4.4', '8.8.8.8', '203.206.0.1']
+    @country_ips = SecurityGuard::CountryIps.new(
+      :countries => ['Australia'],
+      :ips       => ['4.4.4.4', '8.8.8.8', '203.206.0.1']
+    )
   end
 
   it 'contains GeoIP data' do

data/specs/deduplication_spec.rb

@@ -0,0 +1,68 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe SecurityGuard::Deduplication do
+  before do
+    @dedupe = SecurityGuard::Deduplication.new(
+      :input_folder  => fixture_file('dedupe_lists/'),
+      :output_folder => fixture_file('../tmp/')
+    )
+
+    @source_data = [
+      ['a@example.com', 'b@example.com', 'c@example.com', 'c@example.com'],
+      ['b@example.com', 'd@example.com'],
+      ['c@example.com', 'd@example.com', 'e@example.com'],
+    ]
+
+    `rm -rf #{fixture_file('../tmp/*')}`
+  end
+
+  it 'reads data from the input folder' do
+    @dedupe.send :read_data_from, fixture_file('dedupe_lists/')
+    @dedupe.source_data.must_equal @source_data
+  end
+
+  it 'writes data to the output folder' do
+    @dedupe.deduped_data = [
+      [1, 2, 3],
+      [4, 5, 6],
+    ]
+    @dedupe.send :write_data_to, fixture_file('../tmp/')
+
+    File.read(fixture_file('../tmp/1.txt')).must_equal "1\n2\n3\n"
+    File.read(fixture_file('../tmp/2.txt')).must_equal "4\n5\n6\n"
+  end
+
+  it 'reads input and writes output' do
+    @dedupe.process
+
+    File.read(fixture_file('../tmp/1.txt')).must_equal "a@example.com\nb@example.com\nc@example.com\n"
+    File.read(fixture_file('../tmp/2.txt')).must_equal "d@example.com\n"
+    File.read(fixture_file('../tmp/3.txt')).must_equal "e@example.com\n"
+  end
+
+  it 'dedupes an array with another array' do
+    @dedupe.send(:_deduped,
+      ['a@example.com', 'b@example.com', 'c@example.com', 'c@example.com'],
+      ['b@example.com', 'd@example.com'],
+    ).must_equal ['a@example.com', 'c@example.com']
+  end
+
+  it 'dedupes an array against multiple arrays' do
+    @dedupe.send(:_deduped_multi,
+      ['c@example.com', 'd@example.com', 'e@example.com'],
+      [
+        ['a@example.com', 'b@example.com', 'c@example.com', 'c@example.com'],
+        ['b@example.com', 'd@example.com'],
+      ]
+    ).must_equal ['e@example.com']
+  end
+
+  it 'dedupes the source data' do
+    @dedupe.send :dedupe, @source_data
+    @dedupe.deduped_data.must_equal [
+      ['a@example.com', 'b@example.com', 'c@example.com'],
+      ['d@example.com'],
+      ['e@example.com'],
+    ]
+  end
+end

data/specs/fixtures/dedupe_lists/a.txt

@@ -0,0 +1,4 @@
+a@example.com
+b@example.com
+c@example.com
+c@example.com

data/specs/fixtures/dedupe_lists/b.txt

@@ -0,0 +1,2 @@
+b@example.com
+d@example.com

data/specs/fixtures/dedupe_lists/c.txt

@@ -0,0 +1,3 @@
+c@example.com
+d@example.com
+e@example.com

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: security_guard
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-01-20 00:00:00.000000000 Z
+date: 2012-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: clamp
-  requirement: &70250519727080 !ruby/object:Gem::Requirement
+  requirement: &70315493707920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70250519727080
+  version_requirements: *70315493707920
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70250519724280 !ruby/object:Gem::Requirement
+  requirement: &70315493707500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70250519724280
+  version_requirements: *70315493707500
 - !ruby/object:Gem::Dependency
   name: geoip
-  requirement: &70250519722640 !ruby/object:Gem::Requirement
+  requirement: &70315493707080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70250519722640
+  version_requirements: *70315493707080
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70250519721680 !ruby/object:Gem::Requirement
+  requirement: &70315493706660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70250519721680
+  version_requirements: *70315493706660
 - !ruby/object:Gem::Dependency
   name: minitest-colorize
-  requirement: &70250519715500 !ruby/object:Gem::Requirement
+  requirement: &70315493706240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70250519715500
+  version_requirements: *70315493706240
 description: A collection of useful tools for auditing data and performing security
   checks.
 email:
@@ -86,6 +86,7 @@ files:
 - lib/security_guard/concerns/accepts_from_file.rb
 - lib/security_guard/concerns/initializable.rb
 - lib/security_guard/country_ips.rb
+- lib/security_guard/deduplication.rb
 - lib/security_guard/utils/files.rb
 - lib/security_guard/utils/geo_ips.rb
 - lib/security_guard/version.rb
@@ -93,7 +94,11 @@ files:
 - specs/concerns/accepts_from_file_spec.rb
 - specs/concerns/initializable_spec.rb
 - specs/country_ips_spec.rb
+- specs/deduplication_spec.rb
 - specs/fixtures/countries.txt
+- specs/fixtures/dedupe_lists/a.txt
+- specs/fixtures/dedupe_lists/b.txt
+- specs/fixtures/dedupe_lists/c.txt
 - specs/fixtures/ip_addresses.txt
 - specs/spec_helper.rb
 - specs/utils/files_spec.rb