squiggle 0.0.1

data/History.txt

@@ -0,0 +1,4 @@
+=== 0.0.1 2011-01-31
+
+* 1 major enhancement:
+  * Initial release

data/Manifest.txt

@@ -0,0 +1,15 @@
+History.txt
+Manifest.txt
+PostInstall.txt
+README.rdoc
+Rakefile
+lib/squiggle.rb
+lib/squiggle/base.rb
+lib/squiggle/chunk_parser.rb
+lib/squiggle/domain_parser.rb
+lib/squiggle/log_line.rb
+lib/squiggle/squid_standard_parser.rb
+script/console
+script/destroy
+script/generate
+test/test_helper.rb

data/PostInstall.txt

@@ -0,0 +1,7 @@
+
+For more information on squiggle, see http://squiggle.rubyforge.org
+
+NOTE: Change this information in PostInstall.txt 
+You can also delete it if you don't want it.
+
+

data/README.rdoc

@@ -0,0 +1,48 @@
+= squiggle
+
+* http://github.com/#{github_username}/#{project_name}
+
+== DESCRIPTION:
+
+Log line parser
+
+== FEATURES/PROBLEMS:
+
+Used for NetFox Agent
+
+== SYNOPSIS:
+
+TODO
+
+== REQUIREMENTS:
+
+* Active Support >= 3.0.3
+
+== INSTALL:
+
+* sudo gem install
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2011 Daniel Draper, NetFox
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+gem 'activesupport', '>= 3.0.3'
+gem 'hoe', '>= 2.1.0'
+require 'hoe'
+require 'fileutils'
+
+require 'active_support/core_ext/time/zones'
+
+require './lib/squiggle'
+
+Hoe.plugin :newgem
+# Hoe.plugin :website
+# Hoe.plugin :cucumberfeatures
+
+# Generate all the Rake tasks
+# Run 'rake -T' to see list of generated tasks (from gem root directory)
+$hoe = Hoe.spec 'squiggle' do
+  self.developer 'Daniel Draper', 'daniel@netfox.com'
+  self.post_install_message = 'PostInstall.txt' # TODO remove if post-install message not required
+  self.rubyforge_name       = self.name # TODO this is default value
+  self.extra_deps           = [['domainatrix'], ['activesupport', "~> 3.0.3"]]
+end
+
+require 'newgem/tasks'
+Dir['tasks/**/*.rake'].each { |t| load t }
+
+# TODO - want other tests/tasks run by default? Add them to the list
+# remove_task :default
+# task :default => [:spec, :features]

data/lib/squiggle.rb

@@ -0,0 +1,15 @@
+$:.unshift(File.dirname(__FILE__)) unless
+  $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
+
+require 'active_support/core_ext/time/conversions'
+require 'active_support/core_ext/time/zones'
+
+require 'squiggle/base'
+require 'squiggle/chunk_parser'
+require 'squiggle/domain_parser'
+require 'squiggle/log_line'
+require 'squiggle/squid_standard_parser'
+
+module Squiggle
+  VERSION = '0.0.1'
+end

data/lib/squiggle/base.rb

@@ -0,0 +1,39 @@
+
+module Squiggle
+  class Base
+    def initialize(time_zone)
+      @time_zone = time_zone
+    end
+
+    def parse(line)
+      logline = process(line)
+      if logline.valid?
+        logline.cached = cached?(logline)
+        logline.pageview = pageview?(logline)
+      else
+        STDERR.puts("INVALID LINE (#{logline.errors}): '#{line}'") unless line.blank?
+      end
+      logline
+    end
+    
+    def pageview?(logline)
+      case logline.mime_type
+        when /html/,/text/,/pdf/ then true
+        else false
+      end
+    end
+
+    def cached?(logline)
+      # TODO Implement this
+      false
+    end
+
+    # Return the time in the client's time zone
+    def parse_timestamp(str)
+      # Parse the epoch seconds into UTC (assumes Time.zone set to UTC in environment)
+      t = Time.at(str.gsub(/^L/, '').to_i)
+      # Return the time zone in the clients TZ
+      t.in_time_zone(@time_zone)
+    end
+  end
+end

data/lib/squiggle/chunk_parser.rb

@@ -0,0 +1,30 @@
+module Squiggle
+  class ChunkParser
+    def initialize(chunk, options = {})
+      options[:parser] = SquidStandardParser
+      @parser = options[:parser].new(options[:time_zone])
+      @lines = chunk.split("\n").select { |e| e.length > 5 }
+      @current_line = next_line
+      Rails.logger.info("Chunk Parser received: #{@lines.size} lines")
+    end
+
+    def has_lines?
+      !@lines.empty?
+    end
+
+    def current_line
+      @current_line
+    end
+
+    def next_line
+      return nil if @lines.empty?
+      to_parse = @lines.shift
+      logline = @parser.parse(to_parse)
+      if logline.invalid?
+        STDERR.puts "Line is INVALID"
+        logline = self.next_line # recurse
+      end
+      @current_line = logline
+    end
+  end
+end

data/lib/squiggle/domain_parser.rb

@@ -0,0 +1,51 @@
+require 'domainatrix'
+
+class Domainatrix::Url
+  attr_accessor :query
+
+  def toplevel
+    [ domain, public_suffix ].compact.join(".")
+  end
+end
+
+module Squiggle
+  class DomainParser < Domainatrix::DomainParser
+
+    def parse(url)
+      uri = URI.parse(url)
+      Domainatrix::Url.new(parse_domains_from_host(uri.host).merge({
+        :scheme => uri.scheme,
+        :host   => uri.host,
+        :path   => uri.path,
+        :query  => uri.query,
+        :url    => url
+      }))
+    end
+
+    # TODO: This is a big monkey patch - we should be forking and fixing this
+    def parse_domains_from_host(host)
+      parts = host.split(".").reverse
+      public_suffix = []
+      domain = ""
+      subdomains = []
+      sub_hash = @public_suffixes
+      parts.each_index do |i|
+        part = parts[i]
+        sub_parts = sub_hash[part]
+        sub_hash = sub_parts
+        if sub_parts.empty? || !sub_parts.has_key?(parts[i+1])
+          public_suffix << part
+          domain = parts[i+1]
+          subdomains = parts.slice(i+2, parts.size)
+          break
+        else
+          public_suffix << part
+        end
+      end
+      {:public_suffix => public_suffix.reverse.join("."), :domain => domain, :subdomain => subdomains.reverse.join(".")}
+    rescue
+      # Applies to IP Addresses here too
+      {:public_suffix => nil, :domain => host, :subdomain => nil}
+    end
+  end
+end

data/lib/squiggle/log_line.rb

@@ -0,0 +1,105 @@
+
+require 'uri'
+
+module Squiggle
+  class LogLine
+
+    @@domain_parser = DomainParser.new("lib/effective_tld_names.dat")
+
+    attr_accessor :bytes
+    attr_accessor :cache_status
+    attr_accessor :cache_sibling
+    attr_accessor :cached
+    attr_accessor :client_ip
+    attr_accessor :created_at
+    attr_reader   :http_resp_code
+    attr_reader   :mime_type
+    attr_accessor :pageview
+    attr_reader   :uri
+    attr_reader   :username
+    attr_accessor :original_line
+    attr_reader   :errors
+
+    alias :cached? :cached
+    alias :pageview? :pageview
+
+    def initialize
+      @errors = {}
+      @invalid = false
+      # Set defaults
+      self.pageview = false
+      self.cached = false
+      yield self if block_given?
+      class << @errors
+        def to_s
+          self.map { |(k,v)| "#{k} => #{v}" }.join(", ")
+        end
+      end
+    end
+
+    def invalid?
+      # TODO: Run more checks and log if a check fails
+      return true if @invalid
+      if @uri.nil?
+        @errors[:uri] = "Missing URL FROM #{@original_line}"
+        return true
+      end
+      unless http_resp_code =~ /\A[+-]?\d+\Z/
+        @errors[:http_resp_code] = "Invalid HTTP Response Code"
+        return true
+      end
+      if http_resp_code && http_resp_code.to_i == 407
+        @errors[:http_resp_code] = "407 code is ignored so setting to invalid"
+        return true
+      end
+    end
+
+    def valid?
+      !invalid?
+    end
+
+    def http_resp_code=(code)
+      if code == "-"
+        code = "200"
+      end
+      @http_resp_code = code
+    end
+
+    def mime_type=(mt)
+      if mt == "-"
+        mt = "Unknown"
+      end
+      @mime_type = mt
+    end
+
+    def username=(uname)
+      @username = URI.decode(uname || '').gsub(/\"/, '')
+    end
+
+    def uri=(uri)
+      raise "No Domain Parser Set" unless @@domain_parser
+      # CONNECT Requests
+      unless uri =~ /^http/
+        uri = "https://#{uri}"
+      end
+      @uri = @@domain_parser.parse(uri)
+    rescue
+      @invalid = true
+      @errors[:uri] = "FAILED URL: '#{uri}' (#{$!})"
+    end
+
+    # Returns the cost for this line as a float
+    # TODO: Make the logserver an event machine and use EM::Deferrable here??
+    def cost
+      pc = PolicyClient.new(self)
+      pc.cost
+    end
+
+    def copy_line
+      arr = [ bytes, cached, client_ip, created_at, uri.toplevel, uri.host, http_resp_code, mime_type, (pageview ? 1 : 0), uri.path, uri.scheme, username ]
+      arr.map { |entry| "\"#{entry}\"" }.join(",")
+    end
+
+      # TODO: URI Escape? quotes around commas?
+  end
+end

data/lib/squiggle/squid_standard_parser.rb

@@ -0,0 +1,36 @@
+module Squiggle
+  # Based on standard squid log format
+  # %ts.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
+  # Example:
+  # 1253604221.678  19 127.0.0.1 TCP_REFRESH_FAIL_HIT/302 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico - NONE/- text/html
+  #
+  # TODO: Write a log format parser like squid's
+  class SquidStandardParser < Base
+    def process(line)
+      if line.nil?
+        return LogLine.new
+      end
+      line.strip!
+      if line.empty?
+        return LogLine.new
+      end
+      toks = line.split(/\s+/)
+      return LogLine.new do |ll|
+        ll.original_line = line
+        ll.created_at = parse_timestamp(toks[0])
+        ll.client_ip = toks[2]
+        ll.cache_status, ll.http_resp_code = (toks[3] || "").split("/")
+        ll.bytes = toks[4].to_i
+        ll.uri = toks[6]
+        ll.username = toks[7]
+        ll.cache_sibling = toks[8].try(:split, "/").try(:[], 0)
+        ll.mime_type = toks[9]
+      end
+    end
+
+    def cached?(logline)
+      # TODO Implement this
+      false
+    end
+  end
+end

data/script/console

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+# File: script/console
+irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
+
+libs =  " -r irb/completion"
+# Perhaps use a console_lib to store any extra methods I may want available in the cosole
+# libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/console_logger.rb'}"
+libs <<  " -r #{File.dirname(__FILE__) + '/../lib/squiggle.rb'}"
+puts "Loading squiggle gem"
+exec "#{irb} #{libs} --simple-prompt"

data/script/destroy

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/destroy'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Destroy.new.run(ARGV)

data/script/generate

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path(File.join(File.dirname(__FILE__), '..'))
+
+begin
+  require 'rubigen'
+rescue LoadError
+  require 'rubygems'
+  require 'rubigen'
+end
+require 'rubigen/scripts/generate'
+
+ARGV.shift if ['--help', '-h'].include?(ARGV[0])
+RubiGen::Base.use_component_sources! [:rubygems, :newgem, :newgem_theme, :test_unit]
+RubiGen::Scripts::Generate.new.run(ARGV)

data/test/test_domain_parser.rb

@@ -0,0 +1,26 @@
+require 'active_support'
+
+class DomainParserTest < ActiveSupport::TestCase
+  test "ip address form" do
+    assert_extracted_domain "192.168.10.100", "http://192.168.10.100/foo/bar"
+  end
+
+  test "internal server with no FQDN" do
+    assert_extracted_domain "internal", "http://internal/foo/bar"
+  end
+
+  test "internal server with FQDN" do
+    assert_extracted_domain "internal.highschool", "http://internal.highschool/foo/bar"
+  end
+
+  test "valid domains" do
+    suffixes = read_dat_file("lib/effective_tld_names.dat")
+    suffixes.each do |(k,v)|
+      suffix(k,v) do |entry|
+        assert_extracted_domain("website.#{entry}", "http://website.#{entry}/foo/bar?q=abc123")
+        assert_extracted_domain("website.#{entry}", "http://www.website.#{entry}/foo/bar?q=abc123")
+        assert_extracted_domain("website.#{entry}", "http://subdomain.website.#{entry}/foo/bar?q=abc123")
+      end
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,54 @@
+require 'active_support'
+require 'stringio'
+require 'test/unit'
+require 'factory_girl'
+require 'faker'
+require File.dirname(__FILE__) + '/../lib/squiggle'
+
+Factory.define(:logline, :class => Squiggle::LogLine) do |f|
+  f.bytes { rand(1000) }
+  f.cache_status { '' }
+  f.cache_sibling { '' }
+  f.created_at { Time.now }
+  f.client_ip { (1..4).to_a.map { rand(254) + 1 }.join(".") }
+  f.uri { "http://#{Faker::Internet.domain_name}" }
+  f.username { Faker::Internet.user_name }
+  f.mime_type { "text/html" }
+  f.http_resp_code "200"
+end
+
+class ActiveSupport::TestCase
+  def assert_extracted_domain(result, source)
+    @parser ||= Squiggle::DomainParser.new("lib/effective_tld_names.dat") 
+    assert_equal result, @parser.parse(source).toplevel
+  end
+
+  def suffix(key, values)
+    if values.empty? or values == '*'
+      yield(key.gsub(/\!/, ''))
+    else
+      values.each do |k,v|
+        unless k == "*"
+          suffix("#{k}.#{key}", v) { |e| yield e.gsub(/\!/, '') }
+        end
+      end
+    end
+  end
+
+  protected
+    def read_dat_file(file_name)
+      @public_suffixes = {}
+      File.readlines(file_name).each do |line|
+        line = line.strip
+        unless (line =~ /\/\//) || line.empty?
+          parts = line.split(".").reverse
+
+          sub_hash = @public_suffixes
+          parts.each do |part|
+            sub_hash = (sub_hash[part] ||= {})
+          end
+        end
+      end
+      @public_suffixes
+    end
+end

data/test/test_log_line.rb

@@ -0,0 +1,66 @@
+
+class LogLineTest < ActiveSupport::TestCase
+  test "basic log line" do
+    logline = Factory.build(:logline)
+    assert logline.valid?
+    assert logline.errors.empty?
+  end
+
+  test "non-integer http response code" do
+    logline = Factory.build(:logline, :http_resp_code => "gooby")
+    assert !logline.valid?
+    assert !logline.errors.empty?
+    assert logline.errors.has_key?(:http_resp_code)
+  end
+
+  test "407 http response code is invalid" do
+    logline = Factory.build(:logline, :http_resp_code => "407")
+    assert !logline.valid?
+    assert !logline.errors.empty?
+    assert logline.errors.has_key?(:http_resp_code)
+  end
+
+  test "URL is valid" do
+    logline = Factory.build(:logline, :uri => "http://www.google.com.au/search?q=bar")
+    assert logline.uri
+    assert_equal "www.google.com.au", logline.uri.host
+    assert_equal "/search", logline.uri.path
+    assert_equal "http", logline.uri.scheme
+    assert_equal "google.com.au", logline.uri.toplevel
+  end
+
+  test "CONNECT request is valid" do
+    logline = Factory.build(:logline, :uri => "www.westpac.com.au:443")
+    assert logline.uri
+    assert_equal "www.westpac.com.au", logline.uri.host
+    assert_equal "", logline.uri.path
+    assert_equal "https", logline.uri.scheme
+    assert_equal "westpac.com.au", logline.uri.toplevel
+  end
+
+  test "ssl URI is valid" do
+    logline = Factory.build(:logline, :uri => "https://edsuite.decs.sa.edu.au/login.php")
+    assert logline.uri
+    assert_equal "edsuite.decs.sa.edu.au", logline.uri.host
+    assert_equal "/login.php", logline.uri.path
+    assert_equal "https", logline.uri.scheme
+    assert_equal "decs.sa.edu.au", logline.uri.toplevel
+  end
+
+  test "blank username is valid" do
+    logline = Factory.build(:logline, :username => '-')
+    assert logline.valid?
+    assert_equal logline.username, "-"
+  end
+
+  test "digest parsed username is valid" do
+    logline = Factory.build(:logline, :username => '%22daniel%20draper%22')
+    assert logline.valid?
+    assert_equal logline.username, "daniel draper"
+  end
+
+  test "copy line" do
+    logline = Factory.build(:logline)
+    assert logline.copy_line
+  end
+end

data/test/test_squid_parser.rb

@@ -0,0 +1,30 @@
+
+class ParserTest < ActiveSupport::TestCase
+  # TODO: Timestamp parsing
+  # TODO: Rename this to SquidParserTest
+
+  test "basic parsing" do
+    raw = "1253604221  19 127.0.0.1 TCP_MISS/200 562 GET http://www.gravatar.com/blavatar/e81cfb9068d04d1cfd598533bb380e1f?s=16&d=http://s.wordpress.com/favicon.ico - NONE/- text/html"
+    parser = Squiggle::SquidStandardParser.new('Adelaide')
+    logline = parser.parse(raw)
+    assert logline.valid?
+    assert_equal "2009-09-22 16:53:41 +0930", logline.created_at.to_s
+    assert_equal "127.0.0.1", logline.client_ip
+    assert_equal false, logline.cached
+    assert_equal false, logline.cached? # Method alias
+    assert_equal "200", logline.http_resp_code
+    assert_equal 562, logline.bytes
+  end
+
+  test "cached status" do
+
+  end
+
+  test "page view status" do
+
+  end
+
+  test "blocked status" do
+
+  end
+end

data/test/test_squiggle.rb

@@ -0,0 +1,6 @@
+
+class SquiggleTest < ActiveSupport::TestCase
+  test "just return true" do
+    assert true
+  end
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification 
+name: squiggle
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Daniel Draper
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-03 00:00:00 +10:30
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: domainatrix
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 3
+        - 0
+        - 3
+        version: 3.0.3
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rubyforge
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 0
+        - 4
+        version: 2.0.4
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 2
+        - 6
+        - 0
+        version: 2.6.0
+  type: :development
+  version_requirements: *id004
+description: Log line parser
+email: 
+- daniel@netfox.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+files: 
+- History.txt
+- Manifest.txt
+- PostInstall.txt
+- README.rdoc
+- Rakefile
+- lib/squiggle.rb
+- lib/squiggle/base.rb
+- lib/squiggle/chunk_parser.rb
+- lib/squiggle/domain_parser.rb
+- lib/squiggle/log_line.rb
+- lib/squiggle/squid_standard_parser.rb
+- script/console
+- script/destroy
+- script/generate
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/#{github_username}/#{project_name}
+licenses: []
+
+post_install_message: PostInstall.txt
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: squiggle
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Log line parser
+test_files: 
+- test/test_domain_parser.rb
+- test/test_helper.rb
+- test/test_log_line.rb
+- test/test_squid_parser.rb
+- test/test_squiggle.rb