ruby-aws 1.7.1 → 1.7.2

data/lib/amazon/util/hash_nesting.rb

@@ -1,93 +0,0 @@
-# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
-# License::   Apache License, Version 2.0
-
-module Amazon
-module Util
-  
-module HashNesting
-  
-  def nest
-    result = {}.extend HashNesting
-    primaryKeys.each { |key| traverse_nest( "#{key}", self[key] ) { |k,v| result[k] = v } }
-    result
-  end
-  
-  def nest!
-    keys = primaryKeys
-    tmp = self.dup
-    self.keys.each { |k| self.delete k}
-    keys.each { |key| traverse_nest( "#{key}", tmp[key] ) { |k,v| self[k] = v} }
-    self
-  end
-  
-  def unnest
-    result = {}.extend HashNesting
-    for key in primaryKeys
-      true_keys = key.to_s.split('.')
-      resolve_nesting( result, self[key], *true_keys)
-    end
-    result
-  end
-  
-  def unnest!
-    for key in primaryKeys
-      true_keys = key.to_s.split('.')
-      value = self[key]
-      self.delete key
-      resolve_nesting( self, value, *true_keys)
-    end
-    self
-  end
-
-  private
-  
-  # if hash has both string and symbol keys, symbol wins
-  def primaryKeys
-    sym_keys = []
-    str_keys = []
-    self.keys.each { |k|
-      case k
-      when Symbol
-        sym_keys << k
-      when String
-        str_keys << k
-      else
-        str_keys << k
-      end
-    }
-    str_keys.delete_if {|k| sym_keys.member? k.to_s.to_sym }
-    sym_keys + str_keys
-  end
-  
-  def resolve_nesting( dest, data, *keys )
-    return data if keys.empty?
-    dest ||= {}
-    key = keys.shift.to_sym
-    if keys.first.to_i.to_s == keys.first
-      # array
-      index = keys.shift.to_i - 1
-      raise "illegal index: #{keys.join '.'}  index must be >= 1" if index < 0
-      dest[key] ||= []
-      dest[key][index] = resolve_nesting( dest[key][index], data, *keys )
-    else
-      # hash
-      dest[key] = resolve_nesting( dest[key], data, *keys )
-    end
-    dest
-  end
-  
-  def traverse_nest( namespace, data, &block )
-    case data.class.to_s
-    when 'Array'
-      data.each_with_index { |v,i| traverse_nest( "#{namespace}.#{i+1}", v, &block ) }
-    when 'Hash'
-      data.each { |k,v| traverse_nest( "#{namespace}.#{k}", v, &block ) }
-    else
-      yield namespace, data.to_s
-    end
-  end
-  
-end # HashNesting
-
-end # Amazon::Util
-end # Amazon

data/lib/amazon/util/lazy_results.rb

@@ -1,59 +0,0 @@
-# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
-# License::   Apache License, Version 2.0
-
-require 'amazon/util/paginated_iterator'
-
-module Amazon
-module Util
-
-# This class provides a wrapper for lazy evaluation of results.
-# The constructor takes a block which should accept a pagenumber
-# and return a page worth of results.
-class LazyResults
-  include Enumerable
-
-  def initialize( &feeder )
-    @iterator = PaginatedIterator.new( &feeder )
-    flush
-  end
-
-  # clear the result set and start over again
-  def flush
-    @truth = []
-    @iterator.restart
-  end
-
-  # iterate over entire result set, loading lazily
-  def each( &block ) # :yields: item
-    @truth.each {|e| yield e }
-    @iterator.each {|e| @truth << e ; yield e }
-  end
-
-  # index into the array set.  if requested index has not been loaded, will load up to that index
-  def []( index )
-    feedme while !@iterator.done and index >= @truth.size
-    return @truth[index]
-  end
-
-  # fully populate the result set and return a true array
-  def to_a
-    feedme until @iterator.done
-    return @truth.dup
-  end
-
-  def inspect # :nodoc:
-    "#<Amazon::Util::LazyResults truth_size=#{@truth.size} page=#{@page} done=#{@done}>"
-  end
-
-  private
-
-  # fetch the next item from the iterator and stick it in @truth
-  def feedme
-    item = @iterator.next
-    @truth << item unless item.nil?
-  end
-
-end # LazyResults
-
-end # Amazon::Util
-end # Amazon

data/lib/amazon/util/logging.rb

@@ -1,23 +0,0 @@
-# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
-# License::   Apache License, Version 2.0
-
-require 'logger'
-
-module Amazon
-module Util
-module Logging
-
-  @@AmazonLogger = nil
-
-  def set_log( filename )
-    @@AmazonLogger = Logger.new filename
-  end
-
-  def log( str )
-    set_log 'ruby-aws.log' if @@AmazonLogger.nil?
-    @@AmazonLogger.debug str
-  end
-
-end
-end
-end

data/lib/amazon/util/paginated_iterator.rb

@@ -1,70 +0,0 @@
-# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
-# License::   Apache License, Version 2.0
-
-module Amazon
-module Util
-
-# PaginatedIterator provides an iterator interface to a paginated
-# dataset, buffering the current page. It can be used to stream
-# large result sets which would not fit into memory or only need
-# to be processed in a single pass.
-class PaginatedIterator
-
-  # feeder should be a block that accepts a pagenumber and 
-  # returns an array containing the corresponding page
-  # worth of results. It should return an empty array when
-  # there are no more results in the dataset.
-  def initialize( &feeder )
-    @feeder = feeder
-    restart
-  end
-
-  # resets the iterator to start pulling from the first page
-  def restart
-    @buffer = []
-    @page = 1
-    @done = false
-  end
-
-  # returns the next item, or nil if there are no more items
-  def next
-    fetchpage if @buffer.empty?
-    @buffer.shift
-  end
-
-  # checks if we have another item available
-  def hasNext
-    fetchpage if @buffer.empty?
-    return !@buffer.empty?
-  end
-
-  # iterates over the remaining items
-  def each( &block ) # :yields: item
-    until @done
-      item = self.next
-      yield item unless item.nil?
-    end
-  end
-
-  attr_reader :done
-
-  private
-
-  def fetchpage
-    return [] if @done
-    res = @feeder.call @page
-    res = [res].flatten - [nil]
-    if res.nil? or res.empty?
-      @done = true
-      return []
-    else
-      @page += 1
-      @buffer += res
-      return res
-    end
-  end
-
-end # PaginatedIterator
-
-end # Amazon::Util
-end # Amazon

data/lib/amazon/util/proactive_results.rb

@@ -1,116 +0,0 @@
-# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
-# License::   Apache License, Version 2.0
-
-require 'monitor'
-require 'amazon/util/threadpool'
-
-module Amazon
-module Util
-
-# ProactiveResults is not as lazy as LazyResults
-# The constructor takes a block which should accept a pagenumber
-# and return a page worth of results.
-class ProactiveResults
-  include Enumerable
-
-  THREADPOOL_SIZE = 3
-
-  def initialize( exception_handler=nil, &feeder )
-    @feeder = feeder
-    @eh = exception_handler
-    @tp = nil
-    self.flush
-  end
-
-  # clear the result set and start over again
-  def flush
-    @tp.finish unless @tp.nil?
-    @tp = ThreadPool.new(THREADPOOL_SIZE, @eh)
-    @done = false
-    @inflight = [].extend(MonitorMixin)
-    @current_page = 1
-    @truth = []
-    1.upto(THREADPOOL_SIZE) do |page|
-      getPage(page)
-    end
-  end
-
-  # iterate over entire result set, waiting for
-  # threads to finish where necessary
-  def each( &block ) # :yields: item
-    index = 0
-    while true
-      if index >= @truth.size
-        break if @done
-        feedme
-      else
-        yield @truth[index]
-        index += 1
-      end
-    end
-  end
-
-  # index into the result set. if we haven't
-  # loaded enough, will wait until we have
-  def []( index )
-    feedme while !@done and index >= @truth.size
-    return @truth[index]
-  end
-
-  # wait for the entire results set to be populated,
-  # then return an array of the results
-  def to_a
-    feedme until @done
-    return @truth.dup
-  end
-
-  def inspect # :nodoc:
-    "#<Amazon::Util::ProactiveResults truth_size=#{@truth.size} pending_pages=#{@pending.size}>"
-  end
-
-  private
-
-  def getPage(num)
-    @inflight.synchronize do
-      workitem = @tp.addWork(num) { |n| worker(n) }
-      @inflight[num] = workitem
-    end
-  end
-
-  def worker(page)
-    res = []
-    begin
-      res = @feeder.call( page )
-    ensure
-      getPage( page + THREADPOOL_SIZE ) unless (res.nil? || res.empty?)
-    end
-  end
-
-  def feedme
-    return if @done
-    item = nil
-    @inflight.synchronize do
-      if @inflight[@current_page].nil?
-        raise "This should be the last page! #{@current_page} #{@inflight.inspect}" unless [] == ( @inflight - [nil] )
-        @done = true
-        return
-      end
-      item = @inflight[@current_page]
-      @inflight[@current_page] = nil # clear out our references
-      @current_page += 1
-    end
-    res = item.getResult
-    case res
-    when Array
-      @truth += res
-    when Exception, NilClass
-      # ignore
-    else
-      raise "Unexpected result type: #{res.class}"
-    end
-  end
-
-end
-
-end # Amazon::Util
-end # Amazon

data/lib/amazon/util/threadpool.rb

@@ -1,129 +0,0 @@
-# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
-# License::   Apache License, Version 2.0
-
-require 'thread'
-require 'set'
-
-module Amazon
-module Util
-
-# ThreadPool is a generic threadpooling class that enables
-# easier multithreaded workflows.  Initialize with a thread count,
-# then addWork to queue up tasks.  You can +sync+ to ensure the current
-# workload is complete, or +finish+ to flush the threads when you're done.
-class ThreadPool
-
-  # First arg is the thread count.  Threads will be created once and wait
-  # for work ( no performance penalty, since they're waiting on a Queue.
-  # Second arg (optional) is a proc to be used as an exception handler. If
-  # this argument is passed in and the thread encounters an uncaught
-  # exception, the proc will be called with the exception as the only argument.
-  def initialize( num_threads, exception_handler=nil )
-    @work = Queue.new
-    @threads = ThreadGroup.new
-    num_threads.times do
-      worker_thread = Thread.new { workerProcess(exception_handler) }
-      @threads.add worker_thread
-    end
-  end
-
-  # add work to the queue
-  # pass any number of arguments, they will be passed on to the block.
-  def addWork( *args, &block )
-    item = WorkItem.new( args, &block )
-    @work.push( item )
-    item
-  end
-
-  # how many worker threads are there?
-  def threadcount
-    @threads.list.length
-  end
-
-  # request thread completion
-  # No more work will be performed
-  def noMoreWork
-    threadcount.times { @work << :Finish }
-  end
-
-  # request thread completion and wait for them to finish
-  def finish
-    noMoreWork
-    @threads.list.each do |t|
-      t.join
-    end
-  end
-
-  # wait for the currently queued work to finish
-  # (This freezes up the entire pool, temporarily)
-  def sync
-    t = threadcount
-
-    if t < 2
-      item = addWork { :sync }
-      return item.getResult
-    end
-
-    q = Queue.new
-    items = []
-
-    items << addWork do
-      q.pop
-    end
-
-    (t-2).times do |z|
-      items << addWork(z) do |i|
-        items[i].getResult
-      end
-    end
-
-    addWork do
-      q.push :sync
-    end
-
-    items.last.getResult
-  end
-
-  private
-
-  def workerProcess( exception_handler=nil )
-    while true
-      workitem = @work.pop
-      return if workitem == :Finish
-      begin
-        workitem.run
-      rescue Exception => e
-        if exception_handler.nil?
-          print "Worker thread has thrown an exception: "+e.to_s+"\n"
-        else
-          exception_handler.call(workitem)
-        end
-      end
-    end
-  end
-
-  class WorkItem
-    attr_reader :args, :block
-    def initialize( args, &block )
-      @args = args
-      @block = block
-      @result = Queue.new
-    end
-    def run
-      res = @block.call( *@args)
-      @result.push res
-    rescue Exception => e
-      @result.push e
-      raise e
-    end
-    def getResult
-      value = @result.pop
-      @result = [value]
-      value
-    end
-  end
-
-end # ThreadPool
-
-end # Amazon::Util
-end # Amazon