nfcollector 3.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 00d69574752ea8a74595a80a94dbbe8120d05fd1
+  data.tar.gz: 1411de4b5124e5b1841543b6dade0461b9b7e57e
+SHA512:
+  metadata.gz: 6a4ddd1387607a78ba9f8b3c86e83e73b2c7aff2ecdf0f927e171e968571d68ac3d76c814583873abdf81a17adf931f7c797089b5942dd6c40efead201c6bc36
+  data.tar.gz: 602f74db253c3f55c60984dc4f3286bc8224ee071ea07f4aace9a78c6c8fc911ff2cff2b5acfc8126c65f8867d65b2ac2ba86e72b626f041fc916519765f9685

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--warnings
+--require spec_helper

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 2.1.2@nfcollector2 --create

data/Gemfile

@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in nfcollector.gemspec
+gemspec
+
+gem 'rspec'
+
+gem 'rspecify'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Dan Draper
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# Nfcollector
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'nfcollector'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install nfcollector
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/nfcollector.rb

@@ -0,0 +1,41 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'rubygems'
+require "bundler/setup"
+require 'socket'
+require 'csv'
+require 'zlib'
+require 'time'
+require 'domainatrix'
+require 'base64'
+require 'active_support/core_ext/string/conversions'
+require 'active_support/core_ext/object/with_options'
+require 'active_support/core_ext/class/attribute'
+
+require 'nfcollector/configuration'
+require 'nfcollector/input_definition'
+
+require 'nfcollector/mapping'
+require 'nfcollector/sequence_generator'
+require 'nfcollector/nfcollector_exception'
+require 'nfcollector/attribute_validator'
+require 'nfcollector/attributes'
+require 'nfcollector/categoriser'
+require 'nfcollector/copy_file_writer'
+require 'nfcollector/domain_parser'
+require 'nfcollector/partitioner'
+require 'nfcollector/partition'
+require 'nfcollector/category_partition'
+require 'nfcollector/weblog_partition'
+require 'nfcollector/payload_processor'
+
+module Nfcollector
+  def self.config
+    Configuration.tap do |config|
+      yield config if block_given?
+    end
+  end
+end
+
+Nfcollector.config.set_defaults!

data/lib/nfcollector/attribute_validator.rb

@@ -0,0 +1,59 @@
+module Nfcollector
+  class MissingRequiredAttribute < NfcollectorException
+    attr_reader :attr
+
+    def initialize(attr)
+      @attr = attr
+      super
+    end
+
+    def message
+      "Missing attribute '#{attr}'"
+    end
+  end
+
+  class DuplicateAttribute < NfcollectorException
+    def message
+      "Duplicate attribute detected"
+    end
+  end
+
+  class UnknownAttribute < NfcollectorException
+    def message
+      "Unknown Attribute #{super}"
+    end
+  end
+
+  class AttributeValidator
+    def initialize(attributes)
+      @attributes = attributes
+    end
+
+    # Takes the Attribute String
+    def validate!
+      attrs = @attributes.split(",")
+
+      # Known Attrs
+      attrs.each do |attr|
+        unless Attributes::MAPPINGS.keys.include?(attr)
+          raise UnknownAttribute.new(attr)
+        end
+      end
+
+      # Dupes
+      attrs.inject([]) do |accum, attr|
+        raise DuplicateAttribute.new(attr) if accum.include?(attr)
+        accum << attr
+      end
+
+      # Required Attrs
+      Attributes::REQUIRED.each do |attr|
+        raise MissingRequiredAttribute.new(attr) unless attrs.include?(attr)
+      end
+    end
+
+    def self.validate!(attributes)
+      self.new(attributes).validate!
+    end
+  end
+end

data/lib/nfcollector/attributes.rb

@@ -0,0 +1,99 @@
+module Nfcollector
+  class Attributes
+
+    ATTR_CREATED_AT        = 't'
+    ATTR_CLIENT_IP         = '>a'
+    ATTR_CACHED            = 'Cs'
+    ATTR_HTTP_RESP         = 'Hs'
+    ATTR_BYTES             = '<s'
+    ATTR_HOST              = 'Rh'
+    ATTR_PATH              = 'Rp'
+    ATTR_USERNAME          = 'Un'
+    ATTR_MIME_TYPE         = 'mt'
+    ATTR_REQUEST_RESPONSE  = 'Rr'
+    ATTR_USER_GROUP        = 'Ug'
+    ATTR_CLIENT_FQDN       = '>A'
+    ATTR_COMPUTER_GROUP    = '>G'
+    ATTR_CATEGORY          = 'Rc'
+
+    # Input Attributes mapped to their column or map reduce names
+    MAPPINGS = {
+      ATTR_CREATED_AT        => :created_at,
+      ATTR_CLIENT_IP         => :client_ip,
+      ATTR_CACHED            => :cached,
+      ATTR_HTTP_RESP         => :http_resp_code,
+      ATTR_BYTES             => :bytes,
+      ATTR_HOST              => :host,
+      ATTR_PATH              => :path,
+      ATTR_USERNAME          => :username,
+      ATTR_MIME_TYPE         => :mime_type,
+      ATTR_REQUEST_RESPONSE  => :request_response,
+      ATTR_USER_GROUP        => :user_group,
+      ATTR_CLIENT_FQDN       => :client_fqdn,
+      ATTR_COMPUTER_GROUP    => :computer_group,
+      ATTR_CATEGORY          => :category
+    }
+
+    COPY_FILE_COLUMNS = [
+      :created_at,
+      :client_ip,
+      :cached,
+      :http_resp_code,
+      :bytes,
+      :host,
+      :path,
+      :username,
+      :mime_type,
+      :request_response,
+      :user_group,
+      :client_fqdn,
+      :computer_group,
+      :domain
+    ] 
+      
+    MAP_REDUCE_COLUMNS = [
+      :created_at,
+      :bytes,
+      :host,
+      :username,
+      :domain,
+      :category
+    ]
+      
+    REQUIRED = [
+      ATTR_CREATED_AT,
+      ATTR_CLIENT_IP,
+      ATTR_CACHED,
+      ATTR_HTTP_RESP,
+      ATTR_BYTES,
+      ATTR_HOST,
+      ATTR_PATH,
+      ATTR_USERNAME,
+      ATTR_MIME_TYPE 
+    ]
+
+    # Selects from an array of column names
+    # only the ones that can be used for copy files
+    def self.for_copy_file(arr)
+      # TODO: No longer needed?
+      arr.select { |item|
+        COPY_FILE_COLUMNS.include?(item.to_sym)
+      }.map(&:to_sym)
+    end
+
+    # Parses a comma separated list of attrs into
+    # an array of column names as symbols
+    def self.parse(attributes_string)
+      InputDefinition.new.tap do |definition|
+        attributes_string.split(',').each_with_index do |attr, index|
+          definition.set(index, MAPPINGS[attr])
+        end
+      end
+    end
+
+    # Returns the column name associated with the given attr code
+    def self.map(attr_code)
+      MAPPINGS[attr_code.to_s]
+    end
+  end
+end

data/lib/nfcollector/categoriser.rb

@@ -0,0 +1,43 @@
+module Nfcollector
+  class Categoriser
+    class Entry
+      attr_reader :last_seen
+      attr_reader :hits
+      attr_reader :categories
+
+      def initialize
+        @hits       = 0
+        @last_seen  = Time.at(0)
+        @categories = Set.new
+      end
+
+      def update(last_seen, category_ids)
+        @hits += 1
+        @last_seen = last_seen if @last_seen < last_seen
+        @categories.merge(category_ids)
+      end
+    end
+
+    def initialize(account_id)
+      @account_id = account_id
+      @domains    = {}
+      @hosts      = {}
+    end
+
+    #def perform(domain, host, last_seen)
+    def perform(mapped_row, indicies)
+      domain, host, last_seen = mapped_row.values_at(*indicies)
+      @domains[domain] ||= Entry.new
+      @hosts[host]     ||= Entry.new
+      # TODO: This is doing TWO lookups
+      @domains[domain].update(last_seen, mapped_row.category_ids)
+      @hosts[host].update(last_seen, mapped_row.category_ids)
+    end
+
+    def commit!
+      # TODO: This feels a bit clunky...
+      Configuration.categorisation_domains_committer.commit(@account_id, @domains)
+      Configuration.categorisation_hosts_committer.commit(@account_id, @hosts)
+    end
+  end
+end

data/lib/nfcollector/category_partition.rb

@@ -0,0 +1,17 @@
+
+module Nfcollector
+  class CategoryPartition < Partition
+    # Calculates the ID for this partition
+    # Think of it as a statistical 'bin'
+    #
+    # In this case we just use the category id
+    #
+    def self.partition_id(category_id)
+      category_id
+    end
+
+    def keys
+      { account_id: account_id, category_id: partition_id }
+    end
+  end
+end

data/lib/nfcollector/configuration.rb

@@ -0,0 +1,24 @@
+module Nfcollector
+  class Configuration
+    class_attribute :ar_class
+    class_attribute :output_dir
+    class_attribute :categories_lookup
+    class_attribute :logger
+
+    # Set the class or object that will be used to commit domain categorisations (must respond to commit and take a hash)
+    class_attribute :categorisation_domains_committer
+
+    # Set the class or object that will be used to commit host categorisations (must respond to commit and take a hash)
+    class_attribute :categorisation_hosts_committer
+
+    # Set the class or object that will be used to commit users
+    class_attribute :user_committer
+
+    # Delete files after processing (default true)
+    class_attribute :delete_files
+
+    def self.set_defaults!
+      self.delete_files = true
+    end
+  end
+end

data/lib/nfcollector/copy_file_writer.rb

@@ -0,0 +1,47 @@
+
+module Nfcollector
+  class UnexpectedRowLength < NfcollectorException; end
+  class FileEmpty < NfcollectorException; end
+
+  class CopyFileWriter
+    attr_reader :partition
+    delegate :file_name,  to: :partition
+    delegate :table_name, to: :partition
+    delegate :headers,    to: :partition
+
+    def initialize(partition)
+      @partition = partition
+      prepare
+    end
+
+    def self.open(partition)
+      cfw = self.new(partition)
+      yield cfw if block_given?
+      cfw
+    end
+
+    def commit!
+      begin
+        csv = CSV.new(@file, :force_quotes => true, :skip_blanks => true)
+        partition.rows.each { |row| csv << row }
+      rescue => ex
+        @error = ex
+      ensure
+        @file.close
+      end
+      handle_error!(@error) if @error
+    end
+
+    private
+      def prepare
+        @file = File.open(file_name, 'w')
+        @file.puts "-- Created at: #{Time.now.utc}"
+        @file.puts "COPY #{table_name} (account_id,#{headers.join(',')}) FROM stdin WITH csv;"
+      end
+
+      def handle_error!(ex)
+        File.delete(file_name) if File.file?(file_name)
+        raise ex
+      end
+  end
+end

data/lib/nfcollector/domain_parser.rb

@@ -0,0 +1,49 @@
+
+class Domainatrix::Url
+  attr_accessor :query
+
+  def toplevel
+    [ domain, public_suffix ].compact.join(".")
+  end
+end
+
+class NFODomainParser < Domainatrix::DomainParser
+
+  def parse(url)
+    uri = URI.parse(url)
+    Domainatrix::Url.new(parse_domains_from_host(uri.host).merge({
+      :scheme => uri.scheme,
+      :host   => uri.host,
+      :path   => uri.path,
+      :query  => uri.query,
+      :url    => url
+    }))
+  end
+
+  # TODO: This is a big monkey patch - we should be forking and fixing this
+  def parse_domains_from_host(host)
+    parts = host.split(".").reverse
+    public_suffix = []
+    domain = ""
+    subdomains = []
+    sub_hash = @public_suffixes
+    parts.each_index do |i|
+      part = parts[i]
+      sub_parts = sub_hash[part]
+      sub_hash = sub_parts
+      if sub_parts.empty? || !sub_parts.has_key?(parts[i+1])
+        public_suffix << part
+        domain = parts[i+1]
+        subdomains = parts.slice(i+2, parts.size)
+        break
+      else
+        public_suffix << part
+      end
+    end
+    {:public_suffix => public_suffix.reverse.join("."), :domain => domain, :subdomain => subdomains.reverse.join(".")}
+  rescue
+    # Applies to IP Addresses here too
+    {:public_suffix => nil, :domain => host, :subdomain => nil}
+  end
+
+end