lwac 0.2.0

data/lib/lwac/import.rb

@@ -0,0 +1,152 @@
+require 'lwac/server/storage_manager'
+require 'lwac/server/db_conn'
+
+module LWAC
+
+  # Handles the importing of links to a database
+  class Importer
+
+    # Create a new Importer object with a given config.  See the import_config docs page for details
+    # on the form of this config hash.
+    def initialize(config)
+      @config = config
+      load_server_config
+
+      @dbclass = case(@server_config[:storage][:database][:engine])
+        when :mysql
+          MySQLDatabaseConnection
+        else
+          SQLite3DatabaseConnection
+        end
+
+      find_schemata
+      @enc = @server_config[:client_policy]
+
+    end
+
+    # Create a database at the given path
+    def create_db(db_conf)
+
+      # Nice output
+      case( db_conf[:engine] )
+      when :mysql
+        $log.info "Creating MySQL db at using schema from #{@config[:schemata_path]}..."
+      else
+        $log.info "Creating SQLite3 db at #{db_conf[:engine_conf][:filename]} using schema from #{@config[:schemata_path]}..."
+      end
+
+      # Actual stuff---create the db
+      begin
+        @dbclass.create_database( db_conf[:engine_conf] )
+      rescue StandardError => e
+        if db_conf[:engine] == :sqlite
+          $log.fatal "Failed to create database.  Does the parent directory exist?"
+        else
+          $log.fatal "Failed to create database.  Is the server running?"
+        end
+        raise e
+      end
+
+      # Apply schema
+      db = @dbclass.new( db_conf[:engine_conf] )
+      @schemata.each{|s|
+        $log.debug "Schema: #{s}"
+        schema = File.read(s)
+        db.execute(schema, false)
+      }
+      db.close
+
+      # reporting
+      $log.info "Done!"
+    end
+
+    # Import links from a filename
+    def import(list)
+      begin
+        $log.info "Connecting to database..."
+        db = connect_to_db
+        $log.info "Importing links..."
+        count = 0
+        last_notify = Time.now
+        File.read(list).force_encoding('UTF-8').each_line{|line|
+
+          # Fix encoding based on config
+          line = fix_encoding(line)
+
+          line.chomp!
+          if line.length > 0 then
+            count += 1
+            db.insert_link(line)
+          end
+
+          # Print some progress
+          if (count % @config[:notify]) == 0
+            notify_progress(count, Time.now - last_notify)
+            last_notify = Time.now
+          end
+
+        }
+        print "\n" if $stdout.tty?
+        $log.info "Added #{count} link[s]."
+      rescue StandardError => e
+        $log.fatal "#{e}"
+        $log.debug "#{e.backtrace.join("\n")}"
+      ensure
+        db.close if db
+      end
+    end
+
+    # Notify the user of progress
+    def notify_progress(count, time_since_last)
+      str = "#{count} (#{(@config[:notify].to_f / time_since_last).round}/s)"
+
+      if $stdout.tty? 
+        print "\r#{str}" 
+      else
+        $log.info str
+      end
+    end
+
+  private
+
+    # On user request, set the string encoding to something and provide policy for its fixes
+    def fix_encoding(str)
+      return str if not @enc[:fix_encoding]
+      return str.encode(@enc[:target_encoding], @enc[:encoding_options])
+    end
+
+    # Load server configuration file into ram
+    def load_server_config
+      # Attempt to load server config
+      if not File.exist?(@config[:server_config]) then
+        raise "Server config file does not exist at #{@config[:server_config]}"
+      end
+      @server_config = YAML.load_file( File.open(@config[:server_config]) )
+    end
+
+    # Looks in the schema directory and finds SQL files
+    def find_schemata
+      @config[:schemata_path] = File.join(LWAC::RESOURCE_DIR, 'schemata', @server_config[:storage][:database][:engine].to_s) if not @config[:schemata_path]
+      @schemata     = Dir.glob(File.join(@config[:schemata_path], "*.sql"))
+    end
+
+    # Connect to the database with a high level object manager
+    def connect_to_db
+      # Create db if not already there
+      # FIXME: make this conditional work on mysql
+      if not @dbclass.database_exists?( @server_config[:storage][:database][:engine_conf] )
+        if @config[:create_db] then
+          create_db(@server_config[:storage][:database])
+        else
+          raise "Database does not exist, and current settings do not allow creating it."
+        end
+      end
+
+      # Create new storage manager with config in read-write mode
+      return DatabaseStorageManager.new(@server_config[:storage][:database], false) 
+    end
+
+  end
+
+
+end

data/lib/lwac/server.rb

@@ -0,0 +1,294 @@
+
+require 'lwac/shared/multilog'
+require 'lwac/shared/identity'
+require 'lwac/server/consistency_manager'
+require 'lwac/server/storage_manager'
+
+
+module LWAC
+
+  # Handles link allocation to clients within the LWAC system
+  class DownloadServer
+    def initialize(config)
+      @config       = config
+      @dispatched   = {}  # links checked out to clients
+      @cm           = ConsistencyManager.new(config)
+
+      @timeouts     = {}  # timeout threads for clients
+      @rates        = {}  # estimates for how fast clients are
+    end
+
+    # Returns the nonce given, designed as a quick ping when the client starts up
+    def ping(client_id, nonce)
+      $log.info "Client #{client_id} pinged the server."
+      return nonce
+    end
+
+    # Returns either a list of Link objects or a delay to wait for (FixNum)
+    def check_out(client_id, request)
+      links = nil
+
+      $log.info "Client #{client_id} wishes to check out #{request} links."
+
+      # Tell the client to wait if the CM says to wait.
+      if(@cm.wait > 0)
+        $log.info "Telling client to wait #{@cm.wait + @config[:client_management][:delay_overestimate]} seconds."
+        return @cm.wait + @config[:client_management][:delay_overestimate]
+      end
+
+      # Check it has a hash to make everything else easier
+      @dispatched[client_id]  = {} if not @dispatched[client_id]  
+
+      # If the client has already been allocated links
+      # if(@dispatched[client_id].values.length > 0)
+      #   $log.debug "Client #{client_id} already has some links checked out.  Will re-issue these instead."
+      #   links = @dispatched[client_id].values
+      # else
+        # Else, check out some new ones
+        links = @cm.check_out(request)
+        links.each{|l|
+          # puts "####################### #{l.id}" if @dispatched[client_id][l.id]
+          @dispatched[client_id][l.id] = l
+        }
+      # end
+
+      # If we found no links
+      if(links.length == 0)
+        # We found no links, so tell the client to wait until one of the others may have failed.
+        $log.info "Found no links for the client.  Told it to wait #{@config[:client_management][:empty_client_backoff]}s."
+        return @config[:client_management][:empty_client_backoff]
+      end
+
+
+      # Kill any old timeouts if the client tries to check out twice
+      @timeouts[client_id].kill if @timeouts[client_id]
+
+      # Register the new timeout and start a thread to call its cancel method
+      timeout = estimate_client_timeout(client_id, @dispatched[client_id].length)
+        # (@config[:client_management][:time_per_link] * @dispatched[client_id].length)
+      @timeouts[client_id] = Thread.new{ 
+        sleep(timeout)
+        cancel_timeout(client_id)
+      }
+
+      # Ensure the rate computer knows it's got work
+      register_checkout_rate(client_id)
+
+      $log.info "Dispatched #{@dispatched[client_id].length} link[s], timeout #{timeout.round(1)}s (#{Time.now + timeout})"
+
+      summary
+
+      return [@config[:client_policy], links]
+    end
+
+    # Returns either a list of link objects or nil to delete them
+    def check_in(client_id, datapoints)
+      $log.info"Client #{client_id} checking in #{datapoints.length} datapoint[s]..."
+
+      # Check we have actually checked them out
+      check_in_list = []
+      erroneous     = 0
+      datapoints.each do |dp|
+        if @dispatched[client_id] && @dispatched[client_id].values.map{ |l| l.id }.include?(dp.link.id)
+          $log.debug "Adding #{dp} to check-in list"
+          check_in_list << dp
+          @dispatched[client_id].delete(dp.link.id)
+        else
+          # puts "***************************** #{dp}"
+          erroneous += 1
+        end
+      end
+
+      $log.error "Failed to check in #{erroneous} datapoint[s] which were not checked out to him." if erroneous > 0
+
+      # Prevent the timeout firing
+      if(@dispatched[client_id] and @dispatched[client_id].length == 0) then
+        @timeouts[client_id].kill if @timeouts[client_id]
+        @timeouts[client_id] = nil
+      end
+
+      # Estimate client's work rate based on the amount it's done.
+      rate = compute_client_rate(client_id, check_in_list.length)
+      $log.debug "Client #{client_id} is working at #{@rates[client_id].round(2)} links/s" if rate
+
+      # then check them in
+      @cm.check_in(check_in_list)
+
+      $log.debug "Check in complete"
+      
+      summary
+
+      # Report errors checking in, and client rate
+      return [erroneous, rate]
+    end
+
+    # Returns nil
+    def cancel(client_id)
+      if(@dispatched[client_id]) then
+        $log.info "Client #{client_id} is cancelling #{@dispatched[client_id].values.length} link[s]..." 
+
+        # Uncheck the item from the consistency manager
+        @cm.uncheck(@dispatched[client_id].values) if(@dispatched[client_id])
+
+        # Then blank this client's list
+        @dispatched[client_id] = {} 
+
+        # Prevent any timeout firing
+        @timeouts[client_id].kill if @timeouts[client_id]
+        @timeouts[client_id] = nil
+
+
+        $log.debug "Cancel complete"
+      else
+        $log.error "Client #{client_id} attempted to cancel links it does not have checked out."
+      end
+      
+      summary
+    end
+
+    # Close all resources and get ready to quit
+    def close
+      $log.fatal "Closing DownloadServer cleanly..."
+      @cm.close
+      $log.fatal "Done."
+    end
+
+  private
+
+    # Record the last time the client asked for work
+    def register_checkout_rate(client_id)
+      @rates[client_id] = Time.now
+    end
+
+    # Transform the time in the rates listing to a rate,
+    # based on the time the client last asked for work
+    def compute_client_rate(client_id, num_links)
+      if @rates[client_id].is_a?(Time) then
+        @rates[client_id] = num_links / (Time.now - @rates[client_id]).to_f
+        return @rates[client_id]
+      end
+      return nil
+    end
+
+    # Use past experience to compute a timeout for a given client
+    def estimate_client_timeout(client_id, link_count)
+      $log.debug "Estimating client timeout..."
+      if @rates[client_id].is_a?(Numeric) then
+        return (@rates[client_id] * link_count) * @config[:client_management][:dynamic_time_overestimate].to_f
+      end
+
+      # Fall back on the old system
+      return (@config[:client_management][:time_per_link] * link_count)
+    end
+
+    # The client has not got back to us, so revoke its links
+    def cancel_timeout(client_id)
+      if(@dispatched[client_id]) then
+        # Alert the user
+        $log.warn "Client #{client_id} hasn't been heard from for a while..."
+        $log.warn "Cleaning up link assignments for dead client #{client_id}."
+
+        # Uncheck the item from the consistency manager
+        @cm.uncheck(@dispatched[client_id].values) if(@dispatched[client_id])
+
+        # Then blank this client's list
+        @dispatched[client_id] = {} 
+        $log.debug "Done."
+      else
+        $log.warn "Client #{client_id} cleaned its own links before disconnecting.  This is usually a sign it has caught a signal."
+      end
+    
+      @timeouts[client_id] = nil
+    end
+
+    # Present a list of clients and their checked out links.
+    def summary
+      co, sample, done, stime, cached = @cm.counts
+      remain = sample - done
+
+
+      # Debug info
+      str = ["CM: #{co}/#{sample} checked out (#{remain} remaining)."]
+      str << "Summary of Clients:"
+      c = 0
+      @dispatched.each{|client, links|
+        str << "  (#{c+=1}/#{@dispatched.keys.length}) #{client} => #{links.values.length} links."
+      }
+      str.each{|s| $log.debug s }
+
+
+      # Say progress
+      $log.info "#{co} / #{cached} / #{done} / #{sample} links checked out/cached/complete/total (#{((done).to_f/sample.to_f * 100.0).round(2)}%)."
+
+      # Compute ETA
+      if stime and done > 0
+        tdiff = Time.now.to_i - (stime || Time.at(0)).to_i
+        if tdiff > 0 then
+          rate  = done.to_f / tdiff.to_f 
+          eta   = Time.now + (remain / rate).to_i
+          $log.info "ETA for this sample: #{eta} (#{rate.round(1)} links/s, #{(rate * 60*60).round} links/hr)"
+        end
+      end
+
+    end
+
+  end
+
+
+  # Mediates client access to the server by acting as a web service API.
+  #
+  # Also handles thread safety, though of 0.2.0b SimpleRPC does that for us.
+  class DownloadService 
+
+    # Ensure we handle only one thing at once
+    MUTEX = Mutex.new
+   
+    # Create a new DownloadService object with a given DownloadServer
+    def initialize(server)
+      @server = server
+    end
+
+    # Send links to a user, and keep track of who asked for them
+    def ping(version, client_id, nonce)
+      version_check(version)
+      MUTEX.synchronize{
+        @server.ping(client_id, nonce)
+      }
+    end
+
+    # Send links to a user, and keep track of who asked for them
+    def check_out(version, client_id, number_requested)
+      version_check(version)
+      MUTEX.synchronize{
+        @server.check_out(client_id, number_requested)
+      }
+    end
+    
+    # Accept datapoints back from the user
+    def check_in(version, client_id, datapoints)
+      version_check(version)
+      MUTEX.synchronize{
+        @server.check_in(client_id, datapoints)
+      }
+    end
+
+    # Cancel links ahead of time
+    def cancel(version, client_id)
+      version_check(version)
+      MUTEX.synchronize{
+        @server.cancel(client_id)
+      }
+    end
+
+  private
+
+    # Check version is compatible
+    def version_check(ver)
+      raise "Client rejected: incompatible version '#{ver}'" if not Identity::network_is_compatible?(ver)
+    end
+  end
+
+
+
+end
+

data/lib/lwac/server/consistency_manager.rb

@@ -0,0 +1,265 @@
+require 'yaml'
+require 'thread'
+require 'set'
+require "lwac/shared/data_types"
+require "lwac/shared/multilog"
+require "lwac/server/storage_manager"
+
+module LWAC
+
+  # Wraps storage and link policies to enforce efficient workflow
+  # with regards links
+  #
+  # Provides facilities for the following:
+  #
+  # 1) Read current state from files
+  # 2) Create a new sample.  Read links from the db for that sample
+  # 3) Write datapoints and whilst keeping track of the link IDs to ensure all are done.
+  # 4) Close a sample and ensure everything is complete before opening another
+  #
+  #
+  # This can be thought of as the server's API.  It wraps all other server functions.
+  class ConsistencyManager
+
+    def initialize(config)
+      @storage    = StorageManager.new(config[:storage])
+      @state      = @storage.state
+      @mutex      = Mutex.new
+      @config     = config[:sampling_policy]
+
+      # Two lists to handle link checkout
+      @links      = @state.current_sample.pending
+      @checked_out_links = {}
+
+
+      # Print handy messages to people
+      if(@state.last_sample_id == -1)
+        $log.info "No sampling has occurred yet, this is a new deployment."
+        open_sample # Bootstrap the sample
+      end
+
+      # Print more handy messages to people
+      if(not @state.current_sample.open? and @state.current_sample.complete?)
+        $log.info "Current sample is closed and complete.  Opening a new one..."
+        open_sample
+      end
+
+      $log.info "Current sample: #{@state.current_sample}."
+      if(@state.current_sample.open?) 
+        # Prevents the server completing a sample even if already open...
+        # check_sample_limit
+        $log.info "Sample opened at #{@state.current_sample.sample_start_time}, resuming..."
+      else
+        if(wait <= 0)
+          $log.info "Sample is closed but ready to open."
+        else
+          $log.info "Sample closed: wait #{wait}s before sampling until #{Time.now + wait}."
+        end
+      end
+
+    end
+
+    def counts
+      start_time = (@state.current_sample) ? @state.current_sample.sample_start_time : nil
+      return @checked_out_links.values.length, 
+             @state.current_sample.size, 
+             @state.current_sample.progress, 
+             start_time,
+             @links.length
+    end
+
+
+    # Retrieve links
+    def check_out(number = :all)
+      raise "Cannot check out links.  Wait #{wait}s until #{Time.now + wait}." if wait > 0
+      if not @state.current_sample.open? then
+        @state.current_sample.open_sample 
+        @storage.write_sample 
+      end
+
+      links = []
+      @mutex.synchronize{
+        number = @state.remaining if number == :all
+
+        # Check out links and reserve them
+        $log.debug "Checking out #{number}/#{@state.current_sample.remaining} links."
+
+        # If the cache isn't large enough, read more from the DB
+        if @links.length < number then
+          $log.debug "Reading #{number-@links.length} links from database (id > #{@state.current_sample.last_dp_id})"
+
+          # Read from DB
+          ids = @storage.read_link_ids(@state.current_sample.last_dp_id.to_i, (number - @links.length))
+
+          # increment the last count or keep it the same if there were no returns
+          @state.current_sample.last_dp_id = (ids.max || @state.current_sample.last_dp_id)
+
+          # put in the links list
+          @links += ids
+        end
+
+        # then assign from @links
+        count = 0
+        select = @links.classify{ ((count+=1) <= number) }
+
+        # put back the ones we don't want
+        @links = select[false] || Set.new
+
+        # grab the ones we do and get them from the db
+        links = @storage.read_links( select[true].to_a )
+
+        # then pop them in the checkout list
+        links.each{|l|
+          @checked_out_links[l.id] = l
+        }
+
+        # @links.each{|id|
+        #   break if (count+=1) > number
+        #   # Read from DB
+        #   link = @storage.read_link(id)
+        #   # Add to the list of recorded checkec out ones
+        #   @checked_out_links[id] = link
+        #   # add to the list to return
+        #   links << link
+        #   # and delete from the pending list
+        #   @links.delete(id)
+        # }
+
+        $log.debug "Done."
+      }
+
+      $log.debug "Total memory cache usage: #{@checked_out_links.length + @links.length} links"
+
+      # TODO: exception handling.
+      return links
+    end 
+
+    # Check links in without converting them to datapoints.  This doesn't
+    # affect data consistency beyond making it possible to guarantee
+    # that we don't duplicate or omit
+    def uncheck(links = [])
+      @mutex.synchronize{
+        links.each{|l|
+          id = l.id if l.class == Link
+          
+          raise "Attempt to uncheck a link that is not checked out" if not @checked_out_links.delete(id)
+          @links << id 
+        }
+      }
+    end
+
+    # Check links in, write the return to disk
+    def check_in(datapoints = [])
+      raise "Cannot check in whilst waiting.  Wait #{wait}s until #{Time.now + wait}." if wait > 0
+
+      @mutex.synchronize{
+        # Check in each datapoint
+        $log.debug "Checking in #{datapoints.length} datapoints."
+        datapoints.each{|dp|
+          if(@checked_out_links.delete(dp.link.id))
+            @storage.write_datapoint(dp)
+
+            # increment the progress counter
+            @state.current_sample.link_complete( dp.response_properties[:downloaded_bytes] || 0 )
+            
+            # They shouldn't even be in the list below, hence it being commented out.
+            #@links.delete(dp.link.id)
+          else
+            $log.warn "Attempted to check in link with ID #{dp.link.id}, but the sample says it's already been done."
+          end
+        }
+
+        # Close the sample if we detect that we're done
+        if(@state.current_sample.complete?)
+          $log.info "Current sample complete."
+          close_sample
+        end
+      }
+    end
+
+    # Calculate how long we have until the sample is "openable"
+    def wait
+      @mutex.synchronize{
+        (@state.next_sample_due - Time.now.to_i).ceil
+      }
+    end
+
+    # Close the resource neatly.
+    def close
+      $log.debug "Closing consistency manager by unchecking #{@checked_out_links.values.length} links."
+
+      # un-check-out all checked-out links
+      uncheck(@checked_out_links.values)
+      @state.current_sample.pending = @links
+
+
+      # Close storage manager
+      @storage.close
+    end
+
+  private
+
+    # Compute the next sample time
+    def compute_next_sample_time
+      # First, round down to whatever period people want
+      time = Time.at(((Time.now.to_i / @config[:sample_time]).floor * @config[:sample_time]) + @config[:sample_alignment])
+
+      # Then jump forward until the next point in the future
+      while(time < Time.now)
+        time += (@config[:sample_time])
+      end
+      return time.to_i
+    end
+
+    # Close a sample and open a new one.
+    def close_sample
+      # Write sample end time
+      @state.last_sample_duration             = (Time.now - @state.current_sample.sample_start_time).round
+      @state.current_sample.close_sample                     
+
+      $log.info "*** Closing sample #{@state.current_sample}"
+      $log.info "Sample duration: #{@state.last_sample_duration.round}s, size: #{(@state.current_sample.approx_filesize / 1024 / 1024).round(2)}MB"
+
+      # Write sample to disk
+      @storage.write_sample(@state.current_sample)
+
+      # Open the next sample.
+      open_sample()
+    end
+
+    # Open a new sample with or without closing the old one (used as bootstrap)
+    def open_sample
+      check_sample_limit
+
+      # Increment sample
+      @state.last_sample_id         = @state.current_sample.id
+      @state.current_sample         = Sample.new(@state.current_sample.id.to_i + 1, @storage.count_links)
+      @links                        = @state.current_sample.pending  # XXX why?... Ensure we take a copy, don't go editing the sample
+      @state.next_sample_due        = compute_next_sample_time
+      
+      # Tell people
+      $log.info "*** Opened new sample to commence on #{Time.at(@state.next_sample_due)}"
+      $log.info "Estimated completion time: #{Time.at(@state.next_sample_due.to_i + @state.last_sample_duration.to_i)}"
+     
+      # Ensure we don't lose it if we're forced to close 
+      @storage.update_state(@state)
+    end
+
+    # Check against the config's sample limit, and raise sigint to stop if so.
+    def check_sample_limit
+      if @state.current_sample and @config[:sample_limit] and @config[:sample_limit].to_i > 0 and (@state.current_sample.id.to_i + 1) > @config[:sample_limit]  then
+        $log.fatal "*** Sample limit (#{@config[:sample_limit]}) reached.  Shutting down..."
+        raise SignalException.new('SIGTERM')
+      end
+    end
+  end
+  # 
+  # # Test script.
+  # if(__FILE__ == $0) then
+  #   $log = MultiOutputLogger.new($stdout)
+  #   $log.set_level(:debug)
+  #   config = YAML.load_file("./config/server.yml")
+  #   cm = ConsistencyManager.new(config)
+  # end
+  # 
+end