mturk 1.8.1

data/lib/amazon/util/lazy_results.rb

@@ -0,0 +1,59 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'amazon/util/paginated_iterator'
+
+module Amazon
+module Util
+
+# This class provides a wrapper for lazy evaluation of results.
+# The constructor takes a block which should accept a pagenumber
+# and return a page worth of results.
+class LazyResults
+  include Enumerable
+
+  def initialize( &feeder )
+    @iterator = PaginatedIterator.new( &feeder )
+    flush
+  end
+
+  # clear the result set and start over again
+  def flush
+    @truth = []
+    @iterator.restart
+  end
+
+  # iterate over entire result set, loading lazily
+  def each( &block ) # :yields: item
+    @truth.each {|e| yield e }
+    @iterator.each {|e| @truth << e ; yield e }
+  end
+
+  # index into the array set.  if requested index has not been loaded, will load up to that index
+  def []( index )
+    feedme while !@iterator.done and index >= @truth.size
+    return @truth[index]
+  end
+
+  # fully populate the result set and return a true array
+  def to_a
+    feedme until @iterator.done
+    return @truth.dup
+  end
+
+  def inspect # :nodoc:
+    "#<Amazon::Util::LazyResults truth_size=#{@truth.size} page=#{@page} done=#{@done}>"
+  end
+
+  private
+
+  # fetch the next item from the iterator and stick it in @truth
+  def feedme
+    item = @iterator.next
+    @truth << item unless item.nil?
+  end
+
+end # LazyResults
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/logging.rb

@@ -0,0 +1,23 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'logger'
+
+module Amazon
+module Util
+module Logging
+
+  @@AmazonLogger = nil
+
+  def set_log( filename )
+    @@AmazonLogger = Logger.new filename
+  end
+
+  def log( str )
+    set_log 'mturk.log' if @@AmazonLogger.nil?
+    @@AmazonLogger.debug str
+  end
+
+end
+end
+end

data/lib/amazon/util/paginated_iterator.rb

@@ -0,0 +1,70 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+module Amazon
+module Util
+
+# PaginatedIterator provides an iterator interface to a paginated
+# dataset, buffering the current page. It can be used to stream
+# large result sets which would not fit into memory or only need
+# to be processed in a single pass.
+class PaginatedIterator
+
+  # feeder should be a block that accepts a pagenumber and 
+  # returns an array containing the corresponding page
+  # worth of results. It should return an empty array when
+  # there are no more results in the dataset.
+  def initialize( &feeder )
+    @feeder = feeder
+    restart
+  end
+
+  # resets the iterator to start pulling from the first page
+  def restart
+    @buffer = []
+    @page = 1
+    @done = false
+  end
+
+  # returns the next item, or nil if there are no more items
+  def next
+    fetchpage if @buffer.empty?
+    @buffer.shift
+  end
+
+  # checks if we have another item available
+  def hasNext
+    fetchpage if @buffer.empty?
+    return !@buffer.empty?
+  end
+
+  # iterates over the remaining items
+  def each( &block ) # :yields: item
+    until @done
+      item = self.next
+      yield item unless item.nil?
+    end
+  end
+
+  attr_reader :done
+
+  private
+
+  def fetchpage
+    return [] if @done
+    res = @feeder.call @page
+    res = [res].flatten - [nil]
+    if res.nil? or res.empty?
+      @done = true
+      return []
+    else
+      @page += 1
+      @buffer += res
+      return res
+    end
+  end
+
+end # PaginatedIterator
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/proactive_results.rb

@@ -0,0 +1,116 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'monitor'
+require 'amazon/util/threadpool'
+
+module Amazon
+module Util
+
+# ProactiveResults is not as lazy as LazyResults
+# The constructor takes a block which should accept a pagenumber
+# and return a page worth of results.
+class ProactiveResults
+  include Enumerable
+
+  THREADPOOL_SIZE = 3
+
+  def initialize( exception_handler=nil, &feeder )
+    @feeder = feeder
+    @eh = exception_handler
+    @tp = nil
+    self.flush
+  end
+
+  # clear the result set and start over again
+  def flush
+    @tp.finish unless @tp.nil?
+    @tp = ThreadPool.new(THREADPOOL_SIZE, @eh)
+    @done = false
+    @inflight = [].extend(MonitorMixin)
+    @current_page = 1
+    @truth = []
+    1.upto(THREADPOOL_SIZE) do |page|
+      getPage(page)
+    end
+  end
+
+  # iterate over entire result set, waiting for
+  # threads to finish where necessary
+  def each( &block ) # :yields: item
+    index = 0
+    while true
+      if index >= @truth.size
+        break if @done
+        feedme
+      else
+        yield @truth[index]
+        index += 1
+      end
+    end
+  end
+
+  # index into the result set. if we haven't
+  # loaded enough, will wait until we have
+  def []( index )
+    feedme while !@done and index >= @truth.size
+    return @truth[index]
+  end
+
+  # wait for the entire results set to be populated,
+  # then return an array of the results
+  def to_a
+    feedme until @done
+    return @truth.dup
+  end
+
+  def inspect # :nodoc:
+    "#<Amazon::Util::ProactiveResults truth_size=#{@truth.size} pending_pages=#{@pending.size}>"
+  end
+
+  private
+
+  def getPage(num)
+    @inflight.synchronize do
+      workitem = @tp.addWork(num) { |n| worker(n) }
+      @inflight[num] = workitem
+    end
+  end
+
+  def worker(page)
+    res = []
+    begin
+      res = @feeder.call( page )
+    ensure
+      getPage( page + THREADPOOL_SIZE ) unless (res.nil? || res.empty?)
+    end
+  end
+
+  def feedme
+    return if @done
+    item = nil
+    @inflight.synchronize do
+      if @inflight[@current_page].nil?
+        raise "This should be the last page! #{@current_page} #{@inflight.inspect}" unless [] == ( @inflight - [nil] )
+        @done = true
+        return
+      end
+      item = @inflight[@current_page]
+      @inflight[@current_page] = nil # clear out our references
+      @current_page += 1
+    end
+    res = item.getResult
+    case res
+    when Array
+      @truth += res
+    when Exception, NilClass
+      # ignore
+    else
+      raise "Unexpected result type: #{res.class}"
+    end
+  end
+
+end
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/threadpool.rb

@@ -0,0 +1,129 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'thread'
+require 'set'
+
+module Amazon
+module Util
+
+# ThreadPool is a generic threadpooling class that enables
+# easier multithreaded workflows.  Initialize with a thread count,
+# then addWork to queue up tasks.  You can +sync+ to ensure the current
+# workload is complete, or +finish+ to flush the threads when you're done.
+class ThreadPool
+
+  # First arg is the thread count.  Threads will be created once and wait
+  # for work ( no performance penalty, since they're waiting on a Queue.
+  # Second arg (optional) is a proc to be used as an exception handler. If
+  # this argument is passed in and the thread encounters an uncaught
+  # exception, the proc will be called with the exception as the only argument.
+  def initialize( num_threads, exception_handler=nil )
+    @work = Queue.new
+    @threads = ThreadGroup.new
+    num_threads.times do
+      worker_thread = Thread.new { workerProcess(exception_handler) }
+      @threads.add worker_thread
+    end
+  end
+
+  # add work to the queue
+  # pass any number of arguments, they will be passed on to the block.
+  def addWork( *args, &block )
+    item = WorkItem.new( args, &block )
+    @work.push( item )
+    item
+  end
+
+  # how many worker threads are there?
+  def threadcount
+    @threads.list.length
+  end
+
+  # request thread completion
+  # No more work will be performed
+  def noMoreWork
+    threadcount.times { @work << :Finish }
+  end
+
+  # request thread completion and wait for them to finish
+  def finish
+    noMoreWork
+    @threads.list.each do |t|
+      t.join
+    end
+  end
+
+  # wait for the currently queued work to finish
+  # (This freezes up the entire pool, temporarily)
+  def sync
+    t = threadcount
+
+    if t < 2
+      item = addWork { :sync }
+      return item.getResult
+    end
+
+    q = Queue.new
+    items = []
+
+    items << addWork do
+      q.pop
+    end
+
+    (t-2).times do |z|
+      items << addWork(z) do |i|
+        items[i].getResult
+      end
+    end
+
+    addWork do
+      q.push :sync
+    end
+
+    items.last.getResult
+  end
+
+  private
+
+  def workerProcess( exception_handler=nil )
+    while true
+      workitem = @work.pop
+      return if workitem == :Finish
+      begin
+        workitem.run
+      rescue Exception => e
+        if exception_handler.nil?
+          print "Worker thread has thrown an exception: "+e.to_s+"\n"
+        else
+          exception_handler.call(workitem)
+        end
+      end
+    end
+  end
+
+  class WorkItem
+    attr_reader :args, :block
+    def initialize( args, &block )
+      @args = args
+      @block = block
+      @result = Queue.new
+    end
+    def run
+      res = @block.call( *@args)
+      @result.push res
+    rescue Exception => e
+      @result.push e
+      raise e
+    end
+    def getResult
+      value = @result.pop
+      @result = [value]
+      value
+    end
+  end
+
+end # ThreadPool
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/user_data_store.rb

@@ -0,0 +1,100 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'amazon/util/data_reader'
+
+module Amazon
+module Util
+
+# The UserDataStore is a platform-independent class intended to store application configuration information in a human-readable per-user location.
+class UserDataStore
+
+  def initialize(app_name)
+    @app = sanitizeKey(app_name)
+    @base = findBaseStore(@app)
+    @dirty = []
+    @data = Hash.new {|h,a| h[a] = {} }
+    loadConfig
+  end
+
+  def get(namespace,property)
+    ns = sanitizeKey(namespace)
+    @data[ns][property]
+  end
+
+  def set(namespace,property,value)
+    ns = sanitizeKey(namespace)
+    @dirty << ns unless @dirty.member? ns
+    @data[ns][property] = value
+  end
+
+  def clear(namespace,property = nil)
+    ns = sanitizeKey(namespace)
+    @dirty << ns unless @dirty.member? ns
+    if property.nil?
+      @data[ns] = {}
+    else
+      @data[ns].delete_if {|k,v| k == property }
+    end
+  end
+
+  def save
+    @dirty.delete_if do |name|
+      saveNamespace( name )
+    end
+  end
+
+  private
+
+  def loadConfig
+    Dir.open(@base).each do |filename|
+      next if filename =~ /^\./
+      loadNamespace( filename )
+    end
+  end
+ 
+  def sanitizeKey(ns)
+    ns.to_s.downcase
+  end
+
+  def loadNamespace(name)
+    @data[name] = DataReader.load( File.join( @base, name ), :Properties )
+  end
+
+  def saveNamespace(name)
+    filename = File.join( @base, name )
+    # kill old config before saving
+    File.delete filename if File.exist? filename
+    # now save out the data
+    DataReader.save( filename, @data[name], :Properties ) unless @data[name].keys.empty?
+  end
+
+  def findBaseStore(app_name)
+    home = findHomeDir
+    folder = findAppFolderName(app_name)
+    base = File.join( home, folder )
+    Dir.open( home ) do |d|
+      unless d.member? folder
+        Dir.mkdir base
+      end
+    end
+    base
+  end
+
+  def findHomeDir
+    return ENV['TEST_HOME_OVERRIDE'] unless ENV['TEST_HOME_OVERRIDE'].nil?
+    return Gem::user_home if defined? Gem
+    return ENV['HOME'] unless ENV['HOME'].nil?
+    return ENV['USERPROFILE'] unless ENV['USERPROFILE'].nil?
+    return ENV['HOMEDRIVE'] + ENV['HOMEPATH'] if PLATFORM =~ /win32/
+    return '.'
+  end
+
+  def findAppFolderName(app_name)
+    "." + app_name
+  end
+
+end # UserDataStore
+
+end # Amazon::Util
+end # Amazon