dbgrandi-ruby-aws 1.2.0

data/lib/amazon/util.rb

@@ -0,0 +1,10 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'amazon/util/logging'
+require 'amazon/util/paginated_iterator'
+require 'amazon/util/lazy_results'
+require 'amazon/util/proactive_results'
+require 'amazon/util/binder'
+require 'amazon/util/data_reader'
+require 'amazon/util/threadpool'

data/lib/amazon/util/binder.rb

@@ -0,0 +1,44 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'erb'
+
+module Amazon
+module Util
+
+# Simple class for holding eval information
+# useful for feeding to ERB templates and the like
+class Binder
+
+  def initialize(initial={},&block) # :yields: self 
+    initial.each {|k,v| set(k,v) }
+    yield self unless block.nil?
+  end
+
+  def merge(hash)
+    hash.each {|k,v| set(k,v) }
+  end
+
+  def set(k,v)
+    self.instance_variable_set "@#{k.to_s}", "#{v.to_s}"
+  end
+
+  def bind
+    binding
+  end
+  
+  # Helper method to simplify ERB evaluation
+  def erb_eval( template )
+    buffer = ""
+    c = ERB::Compiler.new("")
+    c.put_cmd = "buffer <<" if c.respond_to? :put_cmd=
+    c.insert_cmd = "buffer <<" if c.respond_to? :insert_cmd=
+    compiled = c.compile template
+    eval compiled
+    return buffer
+  end
+
+end # Binder
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/data_reader.rb

@@ -0,0 +1,157 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'yaml'
+require 'csv'
+require 'amazon/util/hash_nesting'
+
+module Amazon
+module Util
+
+# DataReader is a class for loading in data files.  It is used to support bulk file-based operations.
+# DataReader supports a number of different formats:
+# * YAML
+# * Tabular
+# * CSV
+# * Java Properties
+# By default, DataReader assumes Tabular, but load and save both support your choice of format
+class DataReader
+
+  attr_accessor :data
+
+  def initialize(data=[])
+    @data = data
+  end
+  
+  def [](index)
+    return @data[index]
+  end
+  
+  def []=(index)
+    return @data[index]
+  end
+  
+  def load( filename, format=:Tabular )
+    return {} unless File.exists? filename
+    raw_data = File.read( filename )
+    case format
+    when :Tabular
+      @data = parse_csv( raw_data, "\t" )
+    when :YAML
+      @data = YAML.load( raw_data ) || {}
+    when :CSV
+      @data = parse_csv( raw_data )
+    when :Properties
+      @data = parse_properties( raw_data )
+    else
+      raise "invalid format.  options are :Tabular, :YAML, :CSV, :Properties"
+    end
+  end
+  
+  def save( filename, format=:Tabular, force_headers=false )
+    return if @data.nil? or @data.empty?
+    existing = File.exists?( filename ) && File.size( filename ) > 0
+    File.open( filename, 'a+' ) {|f| 
+      f << case format
+      when :Tabular
+        generate_csv( @data, force_headers || !existing, "\t" )
+      when :YAML
+        YAML.dump( @data )
+      when :CSV
+        generate_csv( @data, force_headers || !existing )
+      when :Properties
+        generate_properties( @data )
+      end
+      f << "\n" # adding a newline on the end, so appending is happy
+    }
+  end
+
+  def self.load( filename, format=:Tabular )
+    reader = DataReader.new()
+    reader.load( filename, format )
+  end
+  
+  def self.save( filename, data, format=:Tabular, force_headers=false )
+    reader = DataReader.new( data )
+    reader.save( filename, format, force_headers )
+  end
+  
+  private
+  
+  def parse_csv( raw_data, delim=nil )
+    processed = []
+    rows = CSV.parse( raw_data, delim )
+    return parse_rows( rows )
+  end
+
+  def parse_rows( rows )
+    processed = []
+    headers = rows.shift
+    for row in rows
+      item = {}
+      headers.each_index do |i|
+        item[headers[i].to_sym] = correct_type(row[i]) unless row[i].nil? or row[i].empty?
+      end
+      item.extend HashNesting
+      processed << item.unnest
+    end
+    return processed
+  end
+  
+  def split_data( data )
+    data = data.collect {|d| d.extend(HashNesting).nest }
+    headers = data[0].keys.sort
+    rows = data.collect { |item|
+      row = []
+      item.keys.each {|k|
+        headers << k unless headers.include? k
+        index = headers.index k
+        row[index] = item[k].to_s
+      }
+      row
+    }
+    return headers, rows
+  end
+  
+  def generate_csv( data, dump_header, delim=nil )
+    return "" if data.nil? or data.empty?
+    headers, rows = split_data( data )
+    return generate_rows( headers, rows, dump_header, delim )
+  end
+  
+  def generate_rows( headers, rows, dump_header, record_seperator=nil )
+    rows.unshift headers if dump_header
+    buff = rows.collect { |row|
+      CSV.generate_line( row, record_seperator )
+    }
+    return buff.join("\n")
+  end
+
+  def parse_properties( raw_data )
+    processed = {}
+    for line in raw_data.split(/\n\r?/)
+      next if line =~ /^\W*(#.*)?$/ # ignore lines beginning w/ comments
+      if md = /^([^:=]+)[=:](.*)/.match(line)
+        processed[md[1].strip] = correct_type(md[2].strip)
+      end
+    end
+    processed.extend HashNesting
+    return processed.unnest
+  end
+  
+  def generate_properties( raw_data )
+    raw_data.extend HashNesting
+    (raw_data.nest.collect {|k,v| "#{k}:#{v}" }).join("\n")
+  end
+  
+  # convert to integer if possible
+  def correct_type( str )
+    return str.to_f if str =~ /^\d+\.\d+$/ unless str =~ /^0\d/
+    return str.to_i if str =~ /^\d+$/ unless str =~ /^0\d/
+    return str
+  end
+  
+end # DataReader
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/filter_chain.rb

@@ -0,0 +1,79 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+module Amazon
+module Util
+
+# A class for managing around style interceptors,
+# which can be used to implement a decorator design pattern.
+class FilterChain
+
+  class Filter
+   
+    attr_reader :name, :filter_params, :filter_block
+ 
+    def initialize( name, filter_params, filter_block )
+      @name = name
+      @filter_params = filter_params
+      @filter_block = filter_block
+    end
+
+    def execute( chain, block_params )
+      @filter_block.call( chain, block_params, *@filter_params )
+    end
+
+  end
+
+  attr_reader :filters
+
+  def initialize()
+    @filters = []
+  end
+
+  def execute( *block_params, &block )
+    if @filters.size == 0
+      block.call( *block_params )
+    else
+      create_chain( @filters, 0, block, block_params ).call
+    end 
+  end
+
+  def add( name=nil, *filter_params, &filter_block )
+    add_filter( Filter.new( name, filter_params, filter_block ) )
+  end
+
+  def add_filter( filter )
+    if !filter.name.nil?
+      @filters.each_with_index { |existing_filter,i|
+        if filter.name == existing_filter.name
+          @filters[i] = filter
+          return
+        end
+      }
+    end
+    @filters << filter
+  end
+
+  def remove( name )
+    @filters.delete_if { |filter| name == filter.name }
+  end
+
+  def remove_all()
+    @filters.clear
+  end
+
+  private
+
+  def create_chain( filters, pos, block, block_params )
+    if pos >= filters.size
+      return proc{ block.call( *block_params ) }
+    else
+      chain = create_chain( filters, pos+1, block, block_params ) 
+      return proc { filters[pos].execute( chain, block_params ) }
+    end
+  end
+
+end
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/hash_nesting.rb

@@ -0,0 +1,93 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+module Amazon
+module Util
+  
+module HashNesting
+  
+  def nest
+    result = {}.extend HashNesting
+    primaryKeys.each { |key| traverse_nest( "#{key}", self[key] ) { |k,v| result[k] = v } }
+    result
+  end
+  
+  def nest!
+    keys = primaryKeys
+    tmp = self.dup
+    self.keys.each { |k| self.delete k}
+    keys.each { |key| traverse_nest( "#{key}", tmp[key] ) { |k,v| self[k] = v} }
+    self
+  end
+  
+  def unnest
+    result = {}.extend HashNesting
+    for key in primaryKeys
+      true_keys = key.to_s.split('.')
+      resolve_nesting( result, self[key], *true_keys)
+    end
+    result
+  end
+  
+  def unnest!
+    for key in primaryKeys
+      true_keys = key.to_s.split('.')
+      value = self[key]
+      self.delete key
+      resolve_nesting( self, value, *true_keys)
+    end
+    self
+  end
+
+  private
+  
+  # if hash has both string and symbol keys, symbol wins
+  def primaryKeys
+    sym_keys = []
+    str_keys = []
+    self.keys.each { |k|
+      case k
+      when Symbol
+        sym_keys << k
+      when String
+        str_keys << k
+      else
+        str_keys << k
+      end
+    }
+    str_keys.delete_if {|k| sym_keys.member? k.to_s.to_sym }
+    sym_keys + str_keys
+  end
+  
+  def resolve_nesting( dest, data, *keys )
+    return data if keys.empty?
+    dest ||= {}
+    key = keys.shift.to_sym
+    if keys.first.to_i.to_s == keys.first
+      # array
+      index = keys.shift.to_i - 1
+      raise "illegal index: #{keys.join '.'}  index must be >= 1" if index < 0
+      dest[key] ||= []
+      dest[key][index] = resolve_nesting( dest[key][index], data, *keys )
+    else
+      # hash
+      dest[key] = resolve_nesting( dest[key], data, *keys )
+    end
+    dest
+  end
+  
+  def traverse_nest( namespace, data, &block )
+    case data.class.to_s
+    when 'Array'
+      data.each_with_index { |v,i| traverse_nest( "#{namespace}.#{i+1}", v, &block ) }
+    when 'Hash'
+      data.each { |k,v| traverse_nest( "#{namespace}.#{k}", v, &block ) }
+    else
+      yield namespace, data.to_s
+    end
+  end
+  
+end # HashNesting
+
+end # Amazon::Util
+end # Amazon

data/lib/amazon/util/lazy_results.rb

@@ -0,0 +1,59 @@
+# Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
+# License::   Apache License, Version 2.0
+
+require 'amazon/util/paginated_iterator'
+
+module Amazon
+module Util
+
+# This class provides a wrapper for lazy evaluation of results.
+# The constructor takes a block which should accept a pagenumber
+# and return a page worth of results.
+class LazyResults
+  include Enumerable
+
+  def initialize( &feeder )
+    @iterator = PaginatedIterator.new( &feeder )
+    flush
+  end
+
+  # clear the result set and start over again
+  def flush
+    @truth = []
+    @iterator.restart
+  end
+
+  # iterate over entire result set, loading lazily
+  def each( &block ) # :yields: item
+    @truth.each {|e| yield e }
+    @iterator.each {|e| @truth << e ; yield e }
+  end
+
+  # index into the array set.  if requested index has not been loaded, will load up to that index
+  def []( index )
+    feedme while !@iterator.done and index >= @truth.size
+    return @truth[index]
+  end
+
+  # fully populate the result set and return a true array
+  def to_a
+    feedme until @iterator.done
+    return @truth.dup
+  end
+
+  def inspect # :nodoc:
+    "#<Amazon::Util::LazyResults truth_size=#{@truth.size} page=#{@page} done=#{@done}>"
+  end
+
+  private
+
+  # fetch the next item from the iterator and stick it in @truth
+  def feedme
+    item = @iterator.next
+    @truth << item unless item.nil?
+  end
+
+end # LazyResults
+
+end # Amazon::Util
+end # Amazon