lwac 0.2.0

data/lib/lwac/shared/data_types.rb

@@ -0,0 +1,283 @@
+# Sets are used to hold links in an unordered, non-duplicated fashion.
+require 'set'
+
+
+
+module LWAC
+
+  # -----------------------------------------------------------------------------
+  # Holds a datapoint, which is the return value from querying a link
+  # Immutable.
+  class DataPoint
+    attr_reader :link, :headers, :head, :body, :response_properties, :client_id, :error
+
+    # Methods to extract from Curl::Easy
+    INTERESTING_CURL_EASY_METHODS = %w{
+        body_str
+        cacert
+        cert
+        cert_key
+        certtype
+        connect_time
+        connect_timeout
+        content_type
+        cookiefile
+        cookiejar
+        cookies
+        dns_cache_timeout
+        download_speed
+        downloaded_bytes
+        downloaded_content_length
+        enable_cookies?
+        encoding
+        fetch_file_time?
+        file_time
+        follow_location?
+        ftp_commands
+        ftp_entry_path
+        ftp_filemethod
+        ftp_response_timeout
+        header_in_body?
+        header_size
+        header_str
+        headers
+        http_auth_types
+        http_connect_code
+        ignore_content_length?
+        interface
+        last_effective_url
+        local_port
+        local_port_range
+        low_speed_limit
+        low_speed_time
+        max_redirects
+        multipart_form_post?
+        name_lookup_time
+        num_connects
+        os_errno
+        password
+        post_body
+        pre_transfer_time
+        primary_ip
+        proxy_auth_types
+        proxy_port
+        proxy_tunnel?
+        proxy_type
+        proxy_url
+        proxypwd
+        redirect_count
+        redirect_time
+        redirect_url
+        request_size
+        resolve_mode
+        response_code
+        ssl_verify_host
+        ssl_verify_peer?
+        ssl_version
+        start_transfer_time
+        status
+        timeout
+        total_time
+        unrestricted_auth?
+        upload_speed
+        uploaded_bytes
+        uploaded_content_length
+        url
+        use_netrc?
+        use_ssl
+        useragent
+        username
+        userpwd
+        verbose?
+    }
+
+
+    def initialize(link, headers, head, body, response_properties, client_id, error=nil)
+      @link                 = link
+      @headers              = headers
+      @headers              = DataPoint.headers_to_hash(@headers) if not @headers.is_a?(Hash)
+      @head                 = head
+      @body                 = body
+      @response_properties  = response_properties
+      @error                = error
+      @client_id            = client_id
+    end
+
+    def to_s
+      "<DataPoint #{@link.to_s}>"
+    end
+
+    # Turns HTTP headers into a ruby hash, by parsing
+    # them as a string
+    def self.headers_to_hash(header_string)
+      headers = {}
+      header_string.each_line do |ln|
+        if ln.index(':')
+          key = ln[0..(ln.index(':') - 1)].strip
+          val = ln[(ln.index(':') + 1)..-1].strip
+          headers[key] = val
+        end
+      end
+      return headers
+    end
+
+    # Converts a Curl result and an originating link
+    # into a datapoint with a standard character encoding, etc.
+    def self.from_request(config, link, res, client_id, error)
+      # DataPoint.new(link, headers, head, body, response_properties, @client_id, nil)
+      require 'curl'  # 
+
+      # Fix encoding of head if required
+      $log.debug "Fixing header encoding..."
+      body                = fix_encoding(res.body_str.to_s, config)
+
+      # Fix encoding of head if required
+      $log.debug "Fixing header encoding..."
+      head                = fix_encoding(res.header_str.to_s, config)
+
+      # Generate a hash of headers
+      $log.debug "Parsing headers..."
+      header_hash = DataPoint.headers_to_hash(head)
+
+
+      # Per-regex MIME handling 
+      $log.debug "Passing MIME filter in #{config[:mimes][:policy]} mode..."
+      allow_mime = (config[:mimes][:policy] == :blacklist)
+      encoding   = header_hash["Content-Type"].to_s
+      config[:mimes][:list].each{|mime_rx|
+        if encoding.to_s =~ Regexp.new(mime_rx, config[:mimes][:ignore_case]) then
+          allow_mime = (config[:mimes][:policy] == :whitelist)
+          $log.debug "Link #{link.id} matched MIME regex #{mime_rx}"
+        end
+      }
+      body                  = '' unless allow_mime
+
+      # Load stuff out of response object.
+      $log.debug "Extracting #{INTERESTING_CURL_EASY_METHODS.length} details from result..."
+      response_properties = {}
+      INTERESTING_CURL_EASY_METHODS.map { |m| response_properties[m.to_sym] = res.send(m.to_sym) }
+      response_properties[:mime_allowed] = allow_mime
+
+      DataPoint.new(link,
+                    header_hash,
+                    head,
+                    body,
+                    response_properties,
+                    client_id,
+                    error
+                   )
+    end
+
+    private
+
+    # On user request, set the string encoding to something and provide policy for its fixes
+    def self.fix_encoding(str, config)
+      return str if not config[:fix_encoding]
+      return str.encode(config[:target_encoding], config[:encoding_options])
+    end
+
+  end
+
+
+
+  # -----------------------------------------------------------------------------
+  # Holds a link.  Immutable.
+  class Link
+    attr_reader :id, :uri
+
+    def initialize(id, uri)
+      @id = id
+      @uri = uri
+    end
+
+    def to_s
+      "<#{@id}|#{@uri}>"
+    end
+  end
+
+
+
+  # -----------------------------------------------------------------------------
+  # Holds all data on a given sample, which covers:
+  #  * A list of Links that are in the sample
+  #  * Sample start/end times
+  #
+  # Will throw errors if one tries to edit it whilst closed.
+  class Sample
+    attr_reader :id, :sample_start_time, :size, :progress
+    attr_accessor :sample_end_time, :last_dp_id, :pending, :approx_filesize
+
+    def initialize(id, size, start_id=0, pending_links=Set.new, permit_sampling=false, sample_start_time=Time.now)
+      @id                 = id
+      
+      @size               = size.to_i       # Number of datapoints in sample (read from db)
+      @progress           = 0               # How many links have been done in total
+
+      @pending            = pending_links   # links read from db non-contiguously and simply not used yet
+      @last_dp_id         = start_id        # Where to start reading next IDs
+
+      # cumulative filesize of all data in sample
+      @approx_filesize    = 0
+
+      @permit_sampling    = permit_sampling 
+      @sample_start_time  = sample_start_time
+      @sample_end_time    = nil
+    end
+      
+    # Start sampling.
+    def open_sample 
+      @permit_sampling    = true
+      @sample_start_time  = Time.now
+    end
+
+    def close_sample
+      @permit_sampling    = false
+      @sample_end_time    = Time.now
+    end
+
+    # Has this sample got any links pending?
+    def complete?
+      @progress >= @size
+    end
+
+    # Has the sample been opened?
+    def open?
+      @permit_sampling
+    end
+
+    def link_complete(filesize)
+      @approx_filesize += (filesize || 0)
+      @progress += 1
+    end
+
+    def remaining
+      @size - @progress
+    end
+
+    # Nicer output
+    def to_s
+      "<Sample #{@id}, #{@progress}/#{@size} [#{open? ? "open":"closed"}, #{complete? ? "complete":"incomplete"}]>"
+    end
+  end
+
+
+
+  # -----------------------------------------------------------------------------
+  # Holds time-dependent parameters of the server, meaning:
+  #  * The last sample that was completed, and its duration
+  #  * The current sample in progress
+  #  * The time when the next sample is due
+  class ServerState
+    attr_accessor :last_sample_id, :current_sample, :next_sample_due, :last_sample_duration
+    attr_reader :version
+
+    def initialize(version, last_sample_id=-1, current_sample=nil, next_sample_due=nil)
+      @version                    = version
+      @last_sample_id             = last_sample_id
+      @current_sample             = current_sample        || Sample.new(-1, 0)
+      @next_sample_due            = next_sample_due       || Time.now
+      @last_sample_duration       = last_sample_duration  || 1
+    end
+  end
+
+end

data/lib/lwac/shared/identity.rb

@@ -0,0 +1,44 @@
+# Contains methods for reporting and testing versioning.
+#
+# Actual version numbers are held in lib/lwac.rb
+module LWAC::Identity
+
+  # Versions that may be loaded by the storage manager
+  # If it ain't in this list, it ain't coming off disk into RAM.
+  COMPATIBLE_CORPUS_VERSIONS  = [LWAC::VERSION]
+  COMPATIBLE_NETWORK_VERSIONS = [LWAC::VERSION]
+
+  # Print the author string?
+  POMPOUS_MODE = true 
+
+  # Checks if a given version of a corpus is compatible 
+  def self.storage_is_compatible?(ver)
+    COMPATIBLE_CORPUS_VERSIONS.include?(ver)
+  end
+  
+  # Checks if a given version of a client is compatible
+  def self.network_is_compatible?(ver)
+    COMPATIBLE_NETWORK_VERSIONS.include?(ver)
+  end
+
+  # Present the version to the log
+  def self.announce_version
+    msgs = []
+    msgs << "LWAC v#{LWAC::VERSION} (#{LWAC::DATE})"
+
+    if POMPOUS_MODE
+      auth_string = "by #{LWAC::AUTHORS[0..3].map{|a| "#{a[:name]} <#{a[:contact]}>"}.join(", ")}"
+      auth_string += " and #{LWAC::AUTHORS.length - 5} more." if LWAC::AUTHORS.length > 5
+      msgs << auth_string
+    end
+
+    if $log
+      msgs.each{|m| $log.info(m) }
+    else
+      msgs.each{|m| puts m }
+    end
+  end
+end
+
+
+

data/lib/lwac/shared/launch_tools.rb

@@ -0,0 +1,87 @@
+# -----------------------------------------------------------------------------
+# These procedures are designed as helpers for launching the various utilities
+# in LWAC.  They cover:
+#  * Loading configs
+#  * Checking dependencies at runtime for helpful error output
+#  * Instantiating global log objects
+
+require 'lwac/shared/multilog'
+
+module LWAC
+
+  def self.print_usage
+    $stderr.puts "USAGE: #{$PROGRAM_NAME} TOOL CONFIG [IMPORT_FILE]"
+    $stderr.puts ""
+    $stderr.puts " TOOL        : one of 'server', 'client', 'import' or 'export'"
+    $stderr.puts " CONFIG      : A path to the config file for the tool"
+    $stderr.puts " IMPORT_FILE : A URL list to import"
+    $stderr.puts ""
+  end
+
+
+  # -----------------------------------------------------------------------------
+  # Load configs from ARGV[0] and output usage info.
+  def self.load_config 
+
+    # First, check arguments are fine.
+    if ARGV.length < 2 or not File.readable?(ARGV[1]) then
+      print_usage()
+      exit(1)
+    end
+
+    # Check the tool is a valid one
+    if not %w{server client import export}.include?(ARGV[0]) then
+      $stderr.puts "Not a valid command: #{ARGV[0]}"
+      print_usage()
+      exit(1)
+    end
+
+    # Require things we need for the below
+    require 'yaml'
+    require 'logger'
+
+
+
+    # Then check filesystem is in shape
+    #require_relative ...
+
+
+    # Then load the config
+    tool   = ARGV[0].to_sym
+    config = YAML.load_file(ARGV[1])
+
+
+
+    # Then, create global log
+    logdevs = []
+    if config[:logging] and config[:logging][:logs].is_a?(Hash)
+      config[:logging][:logs].each{|name, ldopts| 
+        # Construct the log
+        ld            = {:name => name}
+        ld[:dev]      = %w{STDOUT STDERR}.include?(ldopts[:dev]) ? eval(ldopts[:dev]) : ldopts[:dev] || STDOUT
+        ld[:level]    = ldopts[:level]
+
+        # Add to the list of logs
+        logdevs << ld
+      }
+    end
+    $log = MultiOutputLogger.new(logdevs, config[:logging][:progname].to_s)
+
+    # Apply nicer log output format
+    $log.formatter = proc do |severity, datetime, progname, msg|
+      "#{severity.to_s[0]} #{progname} [#{datetime.strftime('%y-%m-%d %H:%M:%S')}] #{msg}\n"
+    end
+    
+    
+    # Handle signals nicely.
+    $log.debug "Installing signal handlers..."
+    %w{INT HUP KILL ABRT}.each{|s|
+      trap(s) { raise SignalException.new(s) }
+    }
+
+
+    # Return the config we've loaded.
+    return tool, config
+  end
+
+end

data/lib/lwac/shared/multilog.rb

@@ -0,0 +1,158 @@
+require 'logger'
+
+module LWAC
+  # Add the ability to log to many devices, one for posterity and one for cron.
+  class MultiOutputLogger < Logger
+
+    # Default log level
+    DEFAULT_LEVEL = Logger::UNKNOWN
+
+    # Create a simple log object with one log level and one device
+    def initialize(logdevs = {}, progname=nil, shift_age = 0, shift_size = 1048576)
+      super(nil, shift_age, shift_size)
+      @progname     = progname
+      @shift_age    = shift_age
+      @shift_size   = shift_size
+      @lowest_level = DEFAULT_LEVEL
+      configure_logs(logdevs)
+    end
+
+    def configure_logs(logdevs = {})
+      # Remove all exsiting logs
+      @logdevs.each{|name, ld| remove_log(name)} if @logdevs
+
+      # Parse logdevs hash options
+      @logdevs      = {}
+      logdevs       = [logdevs] if logdevs.class == Hash
+
+      # If the user provides a device then set up a single log as :log
+      if not logdevs.class == Array then
+        @logdevs[:default]    = {:dev => logdevs, :level => DEFAULT_LEVEL}
+        @lowest_level         = @logdevs[:default][:level]
+        return
+      end
+        
+      # If the user provides a hash, check each arg
+      logdevs.each{|ld|
+        name        = ld[:name]         ||= :default
+        dev         = ld[:dev]          ||= $stdout
+        level       = ld[:level]        ||= DEFAULT_LEVEL
+        shift_age   = ld[:shift_age]    ||= @shift_age
+        shift_size  = ld[:shift_size]   ||= @shift_size
+        level       = MultiOutputLogger.string_to_level(level) if level.class != Fixnum 
+
+        # Add to the name deely.
+        add_log(name, dev, level, shift_age, shift_size)
+      } 
+    end
+
+    # Add a log.
+    def add_log(name, destination, level, shift_age = 0, shift_size = 1048576)
+      dev = LogDevice.new(destination, :shift_age => shift_age, :shift_size => shift_size)
+
+      @logdevs[name] = {:dev => dev, :level => level}
+      @lowest_level = level if (not @lowest_level) or level < @lowest_level
+    end
+
+    # Stop logging to one of the logs
+    def remove_log(name)
+      if(@logdevs[name])
+        # Back up old level
+        old_level = @logdevs[name][:level]
+        
+        # Remove
+        @logdevs.delete(name)
+        
+        # Update lowest level if we need to
+        @lowest_level = @logdevs.values.map{|x| x[:level] }.min if old_level == @lowest_level
+      end
+    end
+
+    # Print a summary of log output devices
+    def summarise_logging
+      add(@lowest_level, "Summary of logs:")
+      if(@logdevs.length > 0)
+        c = 0
+        @logdevs.each{|name, ld|
+          msg = " (#{c+=1}/#{@logdevs.length}) #{name} (level: #{MultiOutputLogger.level_to_string(ld[:level])}, device: fd=#{ld[:dev].dev.fileno}#{ld[:dev].dev.tty? ? " TTY" : ""}#{ld[:dev].filename ? " filename=#{ld[:dev].filename}" : ""})"
+          add(@lowest_level,  msg)
+        } 
+      else
+        add(@lowest_level, " *** No logs!") # Amusingly, this can never output
+      end
+    end
+
+    # set the log level of one of the logs
+    def set_level(name, level=nil)
+      # Default
+      if not level then
+        level = name
+        name = nil 
+      end
+
+      # Look up the level if the user provided a :symbol or "string"
+      level = MultiOutputLogger.string_to_level(level.to_s) if level.class != Fixnum
+
+      if name
+        # Set a specific one
+        raise "No log by the name '#{name}'" if not @logdevs[name]
+        @logdevs[name][:level] = level
+      else
+        # Set them all by default 
+        @logdevs.each{|name, logdev| logdev[:level] = level }
+      end
+    end
+
+    # Returns the log level of a log
+    def get_level(name = nil)
+      name = :default if not name
+      return nil if not @logdevs[name]
+      return @logdevs[name][:level]
+    end
+
+    # Overrides the basic internal add in Logger
+    def add(severity, message = nil, progname = nil, &block)
+      severity ||= UNKNOWN
+
+      # give up if no logdevs or if too low a severity
+      return true if severity < @lowest_level or (not @logdevs.values.map{|ld| ld[:dev].nil?}.include?(false))
+      
+      # Set progname to nil unless it is explicitly specified
+      progname ||= @progname
+      if message.nil?
+        if block_given?
+          message   = yield
+        else
+          message   = progname
+          progname  = @progname
+        end
+      end
+
+      # Sync time across the logs and output only if above the log level for that device
+      msg = format_message(format_severity(severity), Time.now, progname, message)
+      @logdevs.each{ |name, ld|
+        ld[:dev].write(msg) if not ld[:dev].nil? and ld[:level] <= severity
+      }
+      return true
+    end
+
+    # convert a level to a string
+    def self.level_to_string(lvl) 
+      labels = %w(DEBUG INFO WARN ERROR FATAL)
+      return labels[lvl] || "UNKNOWN"
+    end
+
+    # Convert a string to a logger level number
+    def self.string_to_level(str)
+      labels = %w(DEBUG INFO WARN ERROR FATAL)
+      return labels.index(str.to_s.upcase) || Logger::UNKNOWN
+    end
+
+    def close
+      @logdevs.each{|name, ld|
+        ld[:dev].close
+      }
+    end
+  end
+
+end