stella 0.5.3 → 0.5.4

data/lib/utils/domainutil.rb

@@ -0,0 +1,47 @@
+
+require 'net/dns/packet'
+
+module DomainUtil
+
+  def DomainUtil.parse_domain_request(data=[])
+    return unless data && !data.empty?
+    data = data.split(/\r?\n/) unless data.kind_of? Array
+    data.shift while (data[0].empty? || data[0].nil?)   # Remove leading empties
+    
+    dns_data = Net::DNS::Packet.parse( data.join($/) )
+    return unless dns_data.header.query? 
+    domain_name = dns_data.question[0].qName
+    return dns_data, domain_name, dns_data.header
+  end
+  
+  def DomainUtil.parse_domain_response(data=[])
+    return unless data && !data.empty?
+    data = data.split(/\r?\n/) unless data.kind_of? Array
+    data.shift while (data[0].empty? || data[0].nil?)   # Remove leading empties
+    
+    # This is the heavy lifting. 
+    dns_data = Net::DNS::Packet.parse( data.join($/) )
+    
+    # We don't want queries or empty answers
+    return if dns_data.header.query? || dns_data.answer.nil? || dns_data.answer.empty?
+    
+    domain_name = dns_data.answer[0].name
+    
+    # Empty the lists if they are already populated
+    addresses = []
+    cnames = []
+
+    # Store the CNAMEs associated to this domain. Can be empty. 
+    dns_data.each_cname do |cname|
+      cnames << cname.to_s
+    end
+
+    # Store the IP address for this domain. If empty, the lookup was unsuccessful. 
+    dns_data.each_address do |ip|
+      addresses << ip.to_s
+    end
+    
+    return dns_data, domain_name, dns_data.header, addresses, cnames
+  end
+
+end

data/lib/utils/fileutil.rb

@@ -1,4 +1,4 @@
-require 'ftools'
+require 'fileutils'
 
 module FileUtil
   
@@ -50,10 +50,29 @@ module FileUtil
     File.chmod(0600, path)
   end
   
-  def FileUtil.create_dir(dirpath)
+  def FileUtil.create_file(filepath, perm='w', file_perms=nil, force=false)
+    raise Exception.new("File #{filepath} already exists!") if File.exists?(filepath) && !force
+    
+    newfile = File.new(filepath, perm)
+    begin
+    if file_perms && File.exists?(file_perms)
+      File.chown(File.stat(file_perms).uid.to_i, File.stat(file_perms).gid.to_i, filepath)
+    end
+rescue NotImplementedError => ex
+end
+
+    newfile
+  end
+
+  def FileUtil.create_dir(dirpath, dir_perms=nil)
     return if File.directory?(dirpath)
     
     #STDERR.puts "Creating #{ dirpath }"
-    File.makedirs(dirpath)
+    FileUtils.makedirs(dirpath)
+    
+    if dir_perms && File.exists?(dir_perms)
+      File.chown(File.stat(dir_perms).uid.to_i, File.stat(dir_perms).gid.to_i, dirpath)
+    end
+    
   end
 end

data/lib/utils/httputil.rb

@@ -1,172 +1,210 @@
-require 'net/http'
+
 require 'uri'
 require 'timeout'
+require 'net/http'
 
   module HTTPUtil
     VALID_METHODS = %w{GET HEAD POST PUT DELETE}
     @@timeout = 20
     
-    def HTTPUtil.hostname(tmp_uri)
-      return if tmp_uri.empty?
-      uri = URI.parse(tmp_uri) if tmp_uri.is_a? String
-      
-      #STDERR.puts "Hostname for #{ uri.port }"
-      uri.host
+    # Takes a string. See WEBrick::parse_header(string).
+    def HTTPUtil.parse_header(raw)
+      header = Hash.new([].freeze)
+      raw.each_line do |line|
+        case line
+        when /\A(.+?):\s+(.+)\z/om
+          name, value = $1, $2 
+          name = name.tr('-', '_').to_sym
+          value.strip!
+          
+          header[name] = [] unless header.has_key?(name)
+          header[name] << value
+        end
+      end
+      header
     end
     
-    # Normalize all URIs before they are used for anything else
-    def HTTPUtil.normalize(uri_str, scheme = true) 
-      
-      
-      #STDERR.puts "  BEFORE: " << uri_str
-      if (!uri_str.index(/^https?:\/\//))
-        uri_str = 'http://' << uri_str
-      end
-      #STDERR.puts "   AFTER: " << uri_str
+    # Takes a string or array. See parse_header_body for further info.
+    # Returns +method+, +http_version+, +uri+, +header+, +body+
+    def HTTPUtil.parse_http_request(data, host=:unknown, port=80)
+      return unless data && !data.empty?
+      data = data.split(/\r?\n/) unless data.kind_of? Array
+      data.shift while (data[0].empty? || data[0].nil?)   # Remove leading empties
+      request_line = data.shift                           # i.e. GET /path HTTP/1.1
+      method, path, http_version = nil
       
-      uri_str.gsub!(/\s/, '%20')
-      
-      uri = URI.parse(uri_str)
-      
-      uri_clean = ""
-      
-      # TODO: use URI.to_s instead of manually creating the string
-      
-      if (scheme)
-        uri_clean << uri.scheme.to_s + '://'  
+      # With WEBrick and other proxies, the entire URI is included in HTTP requests. 
+      # i.e. GET http://stellaaahhhh.com/streetcar.png HTTP/1.1
+      # The parser is expecting just the absolute path. 
+      if request_line =~ /^(\S+)\s+(http:\/\/.+)\s+(HTTP.+)?/mo
+        uri = URI.parse($2)
+        request_line = "#{$1} #{uri.request_uri} #{$3}"
+        host = uri.host
       end
       
+      if request_line =~ /^(\S+)\s+(\S+)(?:\s+HTTP\/(\d+\.\d+))?/mo
+        method = $1
+        http_version = $3 # Comes before $2 b/c the split resets the numbered vars
+        path, query_string   = $2.split('?')
         
-      if (!uri.userinfo.nil?)
-        uri_clean << uri.userinfo.to_s
-        uri_clean << '@'
-      end
-      
-      #uri.host.gsub!(/^www\./, '')
-      
-      uri_clean << uri.host.to_s
-      
-      if (!uri.port.nil? && uri.port != 80 && uri.port != 443)
-        uri_clean << ':' + uri.port.to_s
-      end
-      
-      
-      
-      if (!uri.path.nil? && !uri.path.empty?)
-        uri_clean << uri.path
-      elsif
-        uri_clean << '/'
-      end
-      
-      
-      if (!uri.query.nil? && !uri.path.empty?)
-        uri_clean << "?" << uri.query
+        # We only process the header and body data when we know we're
+        # starting from the beginning of a request string. We don't
+        # want no partials. 
+        header, body = HTTPUtil.parse_header_body(data)
+        query = HTTPUtil.parse_query(method, query_string)
+        
+        
+        # TODO: Parse username/password
+        uri = URI::HTTP.build({
+          :scheme => 'http',
+          :host => header[:Host][0] || host.to_s, 
+          :port => port, 
+          :path => path, 
+          :query => query_string
+        })
+        
+      else
+        rl = request_line.sub(/\x0d?\x0a\z/o, '')
+        raise "Bad Request-Line `#{rl}'."
       end
       
-      #STDERR.puts "IN: " << uri_str
-      #STDERR.puts "OUT: " << uri_clean
-      
-      uri_clean
+      return method, http_version, uri, header, body
     end
     
-    def HTTPUtil.fetch_content(uri, limit = 10)
-      res = self.fetch(uri,limit)
-      return (res) ? res.body : ""
-    end
     
-    def HTTPUtil.fetch(uri, limit = 10)
-       
-      # You should choose better exception.
-      raise ArgumentError, 'HTTP redirect too deep' if limit == 0
-      STDERR.puts "URL: #{uri.to_s}"
-      uri = URI.parse(uri) if uri.is_a? String
+    # Takes a string or array. See parse_header_body for further info. 
+    # Returns +status+, +http_version+, +message+, +header+, +body+
+    def HTTPUtil.parse_http_response(data=[])
+      return unless data && !data.empty?
+      data = data.split(/\r?\n/) unless data.kind_of? Array
+      data.shift while (data[0].empty? || data[0].nil?)   # Remove leading empties
+      status_line = data.shift                            # ie. HTTP/1.1 200 OK
+      http_version, status, message = nil
       
-      begin
-        timeout(@@timeout) do
-           response = Net::HTTP.get_response(uri)
+      if status_line =~ /^HTTP\/(\d.+?)(\s+(\d\d\d)\s+(.+))?$/mo
+        http_version = $1
+        status   = $2
+        message   = $4
         
+        header, body, query = HTTPUtil.parse_header_body(data)
         
-          case response
-          when Net::HTTPSuccess     then response
-          when Net::HTTPRedirection then fetch(response['location'], limit - 1)
-          else
-            STDERR.puts "Not found: " << uri.to_s
-          end
-        end
-      rescue TimeoutError
-        STDERR.puts "Net::HTTP timed out for " << uri.to_s
-        return
-      rescue => ex
-        STDERR.puts "Error: #{ex.message}"
+      else  
+        raise "Bad Response-Line `#{status_line}'."
       end
-    
+      
+      return status, http_version, message, header, body
     end
     
-    def HTTPUtil.post(uri, params = {}, limit = 10)
-       
-      # You should choose better exception.
-      raise ArgumentError, 'HTTP redirect too deep' if limit == 0
+    # Process everything after the first line of an HTTP request or response:
+    # GET / HTTP/1.1
+    # HTTP/1.1 200 OK
+    # etc...
+    # Used by parse_http_request and parse_http_response but can be used separately. 
+    # Takes a string or array of strings. A string should be formatted like an HTTP 
+    # request or response. If a body is present it should be separated by two newlines.
+    # An array of string should contain an empty or nil element between the header 
+    # and body content. This will happen naturally if the raw lines were split by 
+    # a single line terminator. (i.e. /\n/ rather than /\n\n/)
+    # Returns header (hash), body (string)
+    def HTTPUtil.parse_header_body(data=[])
+      header, body = {}, nil
+      data = data.split(/\r?\n/) unless data.kind_of? Array
+      data.shift while (data[0].nil? || data[0].empty?)   # Remove leading empties
       
-      uri = URI.parse(uri) if uri.is_a? String
+      return header, body unless data && !data.empty?
       
-      begin
-        timeout(@@timeout) do
-           response = Net::HTTP.post_form(uri, params)
-        
-          case response
-          when Net::HTTPSuccess     then response
-          when Net::HTTPRedirection then fetch(response['location'], limit - 1)
-          else
-            STDERR.puts "Error for " << uri.to_s
-            STDERR.puts response.body
-          end
-        end
-      rescue TimeoutError
-        STDERR.puts "Net::HTTP timed out for " << uri.to_s
-        return
-      end
+      #puts data.to_yaml
       
+      # Skip that first line if it exists
+      data.shift if data[0].match(/\AHTTP|GET|POST|DELETE|PUT|HEAD/mo)
       
+      header_lines = []
+      header_lines << data.shift while (!data[0].nil? && !data[0].empty?)
+      header = HTTPUtil::parse_header(header_lines.join($/))
       
+      # We omit the blank line that delimits the header from the body
+      body = data[1..-1].join($/) unless data.empty? 
+      
+      return header, body
     end
     
-    def HTTPUtil.exists(uri)
-      
-      begin
-        response = fetch(uri)
-        case response
-        when Net::HTTPSuccess     then true
-        when Net::HTTPRedirection then fetch(response['location'], limit - 1)
+    def HTTPUtil.parse_query(request_method, query_string, content_type='', body='')
+      query = Hash.new([].freeze)
+
+        if request_method == "GET" || request_method == "HEAD"
+          query = HTTPUtil::parse_query_from_string(query_string)
+        elsif content_type =~ /^application\/x-www-form-urlencoded/
+          query = HTTPUtil::parse_query_from_string(body)
+        elsif content_type =~ /^multipart\/form-data; boundary=(.+)/
+          boundary = $1.tr('"', '')
+          query = HTTPUtil::parse_form_data(body, boundary)
         else
-          false
+          query
         end
-        
-      rescue Exception => e
-        STDERR.puts "Problem: " + e.message
-        false
-      end
-      
+
+      query
     end
     
-    def HTTPUtil.parse_query(query)
-      params = Hash.new([].freeze)
-
-      query.split(/[&;]/n).each do |pairs|
-        key, value = pairs.split('=',2).collect{|v| URI.unescape(v) }
-        if params.has_key?(key)
-          params[key].push(value)
+    def HTTPUtil.validate_method(meth='GET')
+      (VALID_METHODS.member? meth.upcase) ? meth : VALID_METHODS[0]
+    end
+    
+    # Parses a query string by breaking it up at the '&'
+    # and ';' characters.  You can also use this to parse
+    # cookies by changing the characters used in the second
+    # parameter (which defaults to '&;'.
+    # Stolen from Mongrel
+    def HTTPUtil.parse_query_from_string(qs, d = '&;')
+      params = {}
+      (qs||'').split(/[#{d}] */n).inject(params) { |h,p|
+        k, v=unescape(p).split('=',2)
+        next unless k
+        k = k.tr('-', '_').to_sym
+        if cur = params[k]
+          if cur.class == Array
+            params[k] << v
+          else
+            params[k] = [cur, v]
+          end
         else
-          params[key] = [value]
+          params[k] = v
         end
-      end
+      }
 
-      params
+      return params
     end
+
     
-    def HTTPUtil.validate_method(meth='GET')
-      meth = (VALID_METHODS.member? meth.upcase) ? meth : VALID_METHODS[0]
+    
+    # Based on WEBrick::HTTPutils::parse_form_data
+    def HTTPUtil.parse_form_data(io, boundary)
+      boundary_regexp = /\A--#{boundary}(--)?#{$/}\z/
+      form_data = Hash.new
+      return form_data unless io
+      data = nil
+      io.each_line{|line|
+        if boundary_regexp =~ line
+          if data
+            data.chop!
+            key = data.name.tr('-', '_').to_sym
+            if form_data.has_key?(key)
+              form_data[key].append_data(data)
+            else
+              form_data[key] = data 
+            end
+          end
+          data = FormData.new
+          next
+        else
+          if data
+            data << line
+          end
+        end
+      }
+      return form_data
     end
     
+    
     # Extend the basic query string parser provided by the cgi module.
     # converts single valued params (the most common case) to
     # objects instead of arrays
@@ -178,7 +216,6 @@ require 'timeout'
     # hash of parameters, contains arrays for multivalued parameters
     # (multiselect, checkboxes , etc)
     # If no query string is provided (nil or "") returns an empty hash.
-
     def HTTPUtil.query_to_hash(query_string)
       return {} unless query_string
 
@@ -205,6 +242,25 @@ require 'timeout'
       #return pairs.join(";")
     end
     
+
+    
+    # Performs URI escaping so that you can construct proper
+    # query strings faster.  Use this rather than the cgi.rb
+    # version since it's faster.  (Stolen from Mongrel/Camping).
+    def HTTPUtil.escape(s)
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+        '%'+$1.unpack('H2'*$1.size).join('%').upcase
+      }.tr(' ', '+')
+    end
+
+
+    # Unescapes a URI escaped string. (Stolen from Mongrel/Camping).
+    def HTTPUtil.unescape(s)
+      s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      }
+    end
+    
     
   end
 

data/lib/utils/mathutil.rb

@@ -6,6 +6,7 @@ module MathUtil
     n = 0
     mean = 0.0
     s = 0.0
+    
     population.each { |x|
       n = n + 1
       delta = (x - mean).to_f
@@ -13,7 +14,9 @@ module MathUtil
       s = (s + delta * (x - mean)).to_f
     }
     
-    return s / n
+    s / n
+  rescue => ex
+    0.0
   end
 
   # calculate the standard deviation of a population
@@ -43,7 +46,8 @@ module Enumerable
     # Sum of all the elements of the Enumerable
 
     def sum
-        return self.inject(0) { |acc, i| acc + i }
+        return 0 if !self || self.empty?
+        self.inject(0) { |acc, i| acc.to_f + i.to_f }
     end
 
     ##
@@ -52,7 +56,10 @@ module Enumerable
     # The Enumerable must respond to #length
 
     def average
-        return self.sum / self.length.to_f
+      return 0 unless self
+      self.sum / self.length.to_f
+    rescue => ex
+      0.0
     end
 
     ##
@@ -61,9 +68,12 @@ module Enumerable
     # The Enumerable must respond to #length
 
     def sample_variance
-        avg = self.average
-        sum = self.inject(0) { |acc, i| acc + (i - avg) ** 2 }
-        return (1 / self.length.to_f * sum)
+      return 0 unless self
+      avg = self.average
+      sum = self.sum
+      (1 / self.length.to_f * sum)
+    rescue => ex
+      0.0
     end
 
     ##
@@ -72,7 +82,10 @@ module Enumerable
     # The Enumerable must respond to #length
 
     def standard_deviation
-        return Math.sqrt(self.sample_variance)
+      return 0 unless self
+      Math.sqrt(self.sample_variance)
+    rescue => ex
+      0.0
     end
 
 end