nfcollector 3.2.0

data/test/input/input-no-tags.csv

@@ -0,0 +1,3 @@
+10.1.1.56,A,www.google.com,/search,macbook01.local,2011-07-11 14:56:10.000,ddraper,200,text/html,5576,1
+10.1.1.56,A,www.google.com,/search,macbook01.local,2011-07-11 14:56:10.000,ddraper,200,text/html,5576,1
+10.1.1.56,A,www.google.com,/search,macbook01.local,2011-07-12 14:56:10.000,ddraper,200,text/html,5576,1

data/test/input/input-no-tags.dat

@@ -0,0 +1,3 @@
+eJzVzUEKgzAURdF5V9EFPL//iYngrEtJY1Bowi9pIF1+3UKHcod3cKjCM+fx
+QO9ddrM9J4lWMH5SqPFACfFp9lJKthgyJiUHXQbyznl1fqWKqmLbaninen5F
+S982Hq1kOLd48MYLONMfzg8+4Fmo

data/test/test_helper.rb

@@ -0,0 +1,15 @@
+require 'byebug'
+require 'stringio'
+require 'shoulda'
+require 'timecop'
+require 'differ'
+
+require "test/unit"
+require "mocha"
+
+require File.dirname(__FILE__) + '/../lib/nfcollector'
+
+def assert_strings_equal(expected, actual)
+  message = "Expected:\n#{expected}\n#{"-"*20}\nActual:\n#{actual}\n#{"-"*20}\nDiff:\n#{Differ.diff_by_line(actual, expected).format_as(:color)}"
+  assert(expected == actual, message)
+end

data/test/tester.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/ruby
+
+# Ensure local load path
+$: << "."
+
+require "rubygems"
+require "bundler/setup"
+
+require 'logger'
+require 'zlib'
+
+require 'payload_processor'
+
+logger = Logger.new(STDOUT)
+
+begin
+  processor = PayloadProcessor.new
+  
+  processor.account_id = 40
+  processor.process_attribute_codes(">a,Rr,Rh,Rp,>A,t,Un,Hs,mt,<s,Cs,Rc")
+  
+  contents = File.new('input/input-100000.csv').read
+  contents = Base64.encode64(Zlib::Deflate::deflate(contents)).tr("+/", "-_")
+  processor.io = StringIO.new(contents)
+  
+  start_time = Time.now
+  processor.process_payload
+  end_time = Time.now
+  puts "Processing Time: #{end_time - start_time}"
+rescue
+  logger.error $!
+end

metadata

@@ -0,0 +1,252 @@
+--- !ruby/object:Gem::Specification
+name: nfcollector
+version: !ruby/object:Gem::Version
+  version: 3.2.0
+platform: ruby
+authors:
+- Dan Draper
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: shoulda
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: differ
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: domainatrix
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.10
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.10
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: Data collection system for NetFox
+email:
+- daniel@codefire.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".rvmrc"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/nfcollector.rb
+- lib/nfcollector/attribute_validator.rb
+- lib/nfcollector/attributes.rb
+- lib/nfcollector/categoriser.rb
+- lib/nfcollector/category_partition.rb
+- lib/nfcollector/configuration.rb
+- lib/nfcollector/copy_file_writer.rb
+- lib/nfcollector/domain_parser.rb
+- lib/nfcollector/input_definition.rb
+- lib/nfcollector/mapping.rb
+- lib/nfcollector/mapping/categories_processor.rb
+- lib/nfcollector/mapping/column_transpiler.rb
+- lib/nfcollector/mapping/default_output.rb
+- lib/nfcollector/mapping/effective_tld_names.dat
+- lib/nfcollector/mapping/indexer.rb
+- lib/nfcollector/mapping/mapped_row.rb
+- lib/nfcollector/mapping/output.rb
+- lib/nfcollector/mapping/transpiler.rb
+- lib/nfcollector/nfcollector_exception.rb
+- lib/nfcollector/partition.rb
+- lib/nfcollector/partitioner.rb
+- lib/nfcollector/payload_processor.rb
+- lib/nfcollector/sequence_generator.rb
+- lib/nfcollector/version.rb
+- lib/nfcollector/weblog_partition.rb
+- nfcollector.gemspec
+- spec/attribute_validator_spec.rb
+- spec/attributes_spec.rb
+- spec/command_parser_spec.rb
+- spec/copy_file_writer_spec.rb
+- spec/input_definition_spec.rb
+- spec/nfcollector/category_partitioner_spec.rb
+- spec/nfcollector/date_partitioner_spec.rb
+- spec/nfcollector/input_definition_spec.rb
+- spec/nfcollector/mapping/column_transpiler_spec.rb
+- spec/nfcollector/mapping/output_spec.rb
+- spec/nfcollector/mapping/transpiler_spec.rb
+- spec/payload_job_spec.rb
+- spec/payload_processor_spec.rb
+- spec/spec_helper.rb
+- test/domains_hosts
+- test/generate_input.rb
+- test/input/input-1000.csv
+- test/input/input-100000.csv
+- test/input/input-100000.dat
+- test/input/input-no-tags.csv
+- test/input/input-no-tags.dat
+- test/input/input-no-tags.gz
+- test/input/input-with-tags.csv.gz
+- test/test_helper.rb
+- test/tester.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Data Collection
+test_files:
+- spec/attribute_validator_spec.rb
+- spec/attributes_spec.rb
+- spec/command_parser_spec.rb
+- spec/copy_file_writer_spec.rb
+- spec/input_definition_spec.rb
+- spec/nfcollector/category_partitioner_spec.rb
+- spec/nfcollector/date_partitioner_spec.rb
+- spec/nfcollector/input_definition_spec.rb
+- spec/nfcollector/mapping/column_transpiler_spec.rb
+- spec/nfcollector/mapping/output_spec.rb
+- spec/nfcollector/mapping/transpiler_spec.rb
+- spec/payload_job_spec.rb
+- spec/payload_processor_spec.rb
+- spec/spec_helper.rb
+- test/domains_hosts
+- test/generate_input.rb
+- test/input/input-1000.csv
+- test/input/input-100000.csv
+- test/input/input-100000.dat
+- test/input/input-no-tags.csv
+- test/input/input-no-tags.dat
+- test/input/input-no-tags.gz
+- test/input/input-with-tags.csv.gz
+- test/test_helper.rb
+- test/tester.rb