dbgrandi-ruby-aws 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. data/History.txt +38 -0
  2. data/LICENSE.txt +202 -0
  3. data/Manifest.txt +69 -0
  4. data/NOTICE.txt +4 -0
  5. data/README.txt +105 -0
  6. data/Rakefile +20 -0
  7. data/bin/ruby-aws +9 -0
  8. data/lib/amazon/util.rb +10 -0
  9. data/lib/amazon/util/binder.rb +44 -0
  10. data/lib/amazon/util/data_reader.rb +157 -0
  11. data/lib/amazon/util/filter_chain.rb +79 -0
  12. data/lib/amazon/util/hash_nesting.rb +93 -0
  13. data/lib/amazon/util/lazy_results.rb +59 -0
  14. data/lib/amazon/util/logging.rb +23 -0
  15. data/lib/amazon/util/paginated_iterator.rb +70 -0
  16. data/lib/amazon/util/proactive_results.rb +116 -0
  17. data/lib/amazon/util/threadpool.rb +129 -0
  18. data/lib/amazon/util/user_data_store.rb +100 -0
  19. data/lib/amazon/webservices/mechanical_turk.rb +117 -0
  20. data/lib/amazon/webservices/mechanical_turk_requester.rb +261 -0
  21. data/lib/amazon/webservices/mturk/mechanical_turk_error_handler.rb +136 -0
  22. data/lib/amazon/webservices/mturk/question_generator.rb +58 -0
  23. data/lib/amazon/webservices/util/amazon_authentication_relay.rb +64 -0
  24. data/lib/amazon/webservices/util/command_line.rb +156 -0
  25. data/lib/amazon/webservices/util/convenience_wrapper.rb +90 -0
  26. data/lib/amazon/webservices/util/filter_proxy.rb +45 -0
  27. data/lib/amazon/webservices/util/mock_transport.rb +70 -0
  28. data/lib/amazon/webservices/util/request_signer.rb +42 -0
  29. data/lib/amazon/webservices/util/rest_transport.rb +108 -0
  30. data/lib/amazon/webservices/util/soap_simplifier.rb +48 -0
  31. data/lib/amazon/webservices/util/soap_transport.rb +38 -0
  32. data/lib/amazon/webservices/util/soap_transport_header_handler.rb +27 -0
  33. data/lib/amazon/webservices/util/unknown_result_exception.rb +27 -0
  34. data/lib/amazon/webservices/util/validation_exception.rb +55 -0
  35. data/lib/amazon/webservices/util/xml_simplifier.rb +61 -0
  36. data/lib/ruby-aws.rb +21 -0
  37. data/lib/ruby-aws/version.rb +8 -0
  38. data/samples/mturk/best_image/BestImage.rb +61 -0
  39. data/samples/mturk/best_image/best_image.properties +39 -0
  40. data/samples/mturk/best_image/best_image.question +82 -0
  41. data/samples/mturk/blank_slate/BlankSlate.rb +63 -0
  42. data/samples/mturk/blank_slate/BlankSlate_multithreaded.rb +67 -0
  43. data/samples/mturk/helloworld/MTurkHelloWorld.rb +56 -0
  44. data/samples/mturk/helloworld/mturk.yml +8 -0
  45. data/samples/mturk/reviewer/Reviewer.rb +103 -0
  46. data/samples/mturk/reviewer/mturk.yml +8 -0
  47. data/samples/mturk/simple_survey/SimpleSurvey.rb +90 -0
  48. data/samples/mturk/simple_survey/simple_survey.question +30 -0
  49. data/samples/mturk/site_category/SiteCategory.rb +87 -0
  50. data/samples/mturk/site_category/externalpage.htm +71 -0
  51. data/samples/mturk/site_category/site_category.input +6 -0
  52. data/samples/mturk/site_category/site_category.properties +45 -0
  53. data/samples/mturk/site_category/site_category.question +9 -0
  54. data/test/mturk/test_changehittypeofhit.rb +130 -0
  55. data/test/mturk/test_error_handler.rb +135 -0
  56. data/test/mturk/test_mechanical_turk_requester.rb +178 -0
  57. data/test/mturk/test_mock_mechanical_turk_requester.rb +205 -0
  58. data/test/test_ruby-aws.rb +22 -0
  59. data/test/unit/test_binder.rb +89 -0
  60. data/test/unit/test_data_reader.rb +135 -0
  61. data/test/unit/test_exceptions.rb +32 -0
  62. data/test/unit/test_hash_nesting.rb +93 -0
  63. data/test/unit/test_lazy_results.rb +89 -0
  64. data/test/unit/test_mock_transport.rb +132 -0
  65. data/test/unit/test_paginated_iterator.rb +58 -0
  66. data/test/unit/test_proactive_results.rb +108 -0
  67. data/test/unit/test_question_generator.rb +54 -0
  68. data/test/unit/test_threadpool.rb +50 -0
  69. data/test/unit/test_user_data_store.rb +80 -0
  70. metadata +158 -0
@@ -0,0 +1,64 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'amazon/webservices/util/command_line'
5
+ require 'amazon/webservices/util/request_signer'
6
+
7
+ module Amazon
8
+ module WebServices
9
+ module Util
10
+
11
+ class AmazonAuthenticationRelay
12
+
13
+ REQUIRED_PARAMETERS = [:Name,:Transport]
14
+
15
+ def initialize( args )
16
+ missing_parameters = REQUIRED_PARAMETERS - args.keys
17
+ raise "Missing paramters: #{missing_parameters.join(',')}" unless missing_parameters.empty?
18
+ @name = args[:Name]
19
+ @transport = args[:Transport]
20
+ @keyId, @key = findAuthInfo( args )
21
+ end
22
+
23
+ def withCredential(credential,method,*request)
24
+ time = Time.now.gmtime.strftime('%Y-%m-%dT%H:%M:%S.123Z')
25
+ request[0] ||= {}
26
+ args = { :AWSAccessKeyId => @keyId,
27
+ :Timestamp => time,
28
+ :Signature => RequestSigner.sign(@name,method,time,@key),
29
+ :Credential => credential,
30
+ :Request => request }
31
+ @transport.send( method, args )
32
+ end
33
+
34
+ def method_missing(method, *request)
35
+ time = Time.now.gmtime.strftime('%Y-%m-%dT%H:%M:%S.321Z')
36
+ request[0] ||= {}
37
+ args = { :AWSAccessKeyId => @keyId,
38
+ :Timestamp => time,
39
+ :Signature => RequestSigner.sign(@name,method,time,@key),
40
+ :Request => request }
41
+ @transport.send( method, args )
42
+ end
43
+
44
+ private
45
+
46
+ def findAuthInfo( args )
47
+ if args.has_key? :AWSAccessKey or args.has_key? :AWSAccessKeyId
48
+ # user tried to pass authentication information in, so just use that
49
+ raise "Missing AWSAccessKeyId" if args[:AWSAccessKeyId].nil?
50
+ raise "Missing AWSAccessKey" if args[:AWSAccessKey].nil?
51
+ return args[:AWSAccessKeyId], args[:AWSAccessKey]
52
+ else
53
+ # didn't get passed in, so load from config or go interactive
54
+ cmd = CommandLine.new
55
+ cmd.checkAuthConfig
56
+ return cmd.authId, cmd.authKey
57
+ end
58
+ end
59
+
60
+ end # AmazonAuthenticationRelay
61
+
62
+ end # Amazon::WebServices::Util
63
+ end # Amazon::WebServices
64
+ end # Amazon
@@ -0,0 +1,156 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'optparse'
5
+ require 'highline'
6
+ require 'ruby-aws'
7
+ require 'amazon/util/user_data_store'
8
+
9
+ module Amazon
10
+ module WebServices
11
+ module Util
12
+
13
+ class CommandLine
14
+
15
+ SDKS = ['Mechanical Turk']
16
+
17
+ MISSING_AUTH = "You are missing authentication information required to utilize Amazon Web Services. You will now be prompted for your <%= color('Access Key ID',BOLD) %> and your <%= color('Secret Access Key',BOLD) %>. If you do not have this information, please log into http://www.amazonaws.com/ \n\n"
18
+ RESUME_AUTH = "Authentication information has been initialized. Continuing... \n\n"
19
+
20
+ MAIN_HELP = <<EOF
21
+
22
+ This is the <%= color('RubyAWS', RED, BOLD) %> Command Line Application's Interactive mode.
23
+ You got here by typing:
24
+
25
+ <%= color('$ ruby-aws',GREEN) %>
26
+
27
+ This application currently supports the following functionality:
28
+
29
+ <%= list( ['Amazon Web Services Authentication Configuration'] ) %>
30
+ You can also invoke this tool with commandline parameters. For more information:
31
+
32
+ <%= color('$ ruby-aws --help',GREEN) %>
33
+
34
+ Thanks for using Amazon Web Services!
35
+
36
+ EOF
37
+
38
+ def initialize( interactive = true )
39
+ @interactive = interactive
40
+ @h = HighLine.new
41
+ @h.wrap_at = :auto
42
+ @store = Amazon::Util::UserDataStore.new :AWS
43
+ HighLine.use_color = useColor?
44
+ HighLine.track_eof = false # disabling because it misbehaves
45
+ end
46
+
47
+ def checkAuthConfig
48
+ if @store.get(:Auth,:AccessKeyId).nil? or @store.get(:Auth,:SecretAccessKey).nil?
49
+ @h.say MISSING_AUTH
50
+ getAuthConfig
51
+ @h.say RESUME_AUTH
52
+ end
53
+ end
54
+
55
+ def getAuthConfig( default={} )
56
+ id = @h.ask( 'What is your Access Key ID?' ) {|q| q.validate = /^[A-Z0-9]{20}$/ ; q.default = authId.to_s ; q.first_answer = default[:ID] }
57
+ key = @h.ask( 'What is your Secret Access Key?' ) {|q| q.validate = /^[\w\/+]{40}$/ ; q.default = authKey.to_s ; q.first_answer = default[:Key] }
58
+
59
+ @store.set(:Auth,:AccessKeyId,id)
60
+ @store.set(:Auth,:SecretAccessKey,key)
61
+ @store.save
62
+ rescue
63
+ raise "Unable to retrieve authentication information from the Console"
64
+ end
65
+
66
+ def authKey
67
+ @store.get(:Auth,:SecretAccessKey)
68
+ end
69
+ def authId
70
+ @store.get(:Auth,:AccessKeyId)
71
+ end
72
+
73
+ def useColor?
74
+ if @store.get(:Misc,:ColorTerminal).nil?
75
+ if @interactive and @h.agree( "Should the console application use color? (y/n)" )
76
+ @store.set(:Misc,:ColorTerminal,true)
77
+ else
78
+ @store.set(:Misc,:ColorTerminal,false)
79
+ end
80
+ @store.save
81
+ end
82
+ return @store.get(:Misc,:ColorTerminal)
83
+ end
84
+
85
+ def toggleColor
86
+ value = !useColor?
87
+ @store.set(:Misc,:ColorTerminal,value)
88
+ HighLine.use_color = value
89
+ end
90
+
91
+ def default_menu
92
+ loop do
93
+ @h.choose do |menu|
94
+ menu.header = "\n" + @h.color('RubyAWS',HighLine::BOLD,HighLine::RED) + " " + @h.color('Command Line Application',HighLine::BOLD) + " " + @h.color('[Interactive Mode]',HighLine::GREEN,HighLine::BOLD)
95
+ menu.select_by = :index
96
+ menu.choice( 'Configure Amazon Web Services Authentication' ) do
97
+ if @h.agree( "\nCurrent ID: #{@h.color(authId,HighLine::BOLD)}\nCurrent Key: #{@h.color(authKey,HighLine::BOLD)}\nDo you want to change?" )
98
+ getAuthConfig
99
+ end
100
+ end
101
+ menu.choice( 'Toggle Color' ) { toggleColor }
102
+ menu.choice( :help ) do
103
+ @h.say MAIN_HELP
104
+ end
105
+ menu.choice( 'Save and Quit' ) { @store.save ; exit }
106
+ menu.prompt = "\nWhat would you like to do? "
107
+ end
108
+ end
109
+ end
110
+
111
+ def parseOptions
112
+ res = {}
113
+ opts = OptionParser.new
114
+
115
+ opts.on( '-i', '--interactive', 'Load Interactive Mode' ) { res[:Interactive] = true }
116
+ opts.on( '-a', '--authenticate', 'Configure Authentication Options' ) { res[:Auth] = true }
117
+ opts.on( '--id=ID', 'Set Access Key ID (requires "-a")' ) { |id| res[:ID] = id }
118
+ opts.on( '--key=KEY', 'Set Secret Access Key (requires "-a")' ) { |key| res[:Key] = key }
119
+
120
+ begin
121
+ opts.parse(ARGV)
122
+ raise "-i and -a are exclusive options. Please pick one." if res[:Interactive] and res[:Auth]
123
+ raise "--id requires -a" if res[:ID] and !res[:Auth]
124
+ raise "--key requires -a" if res[:Key] and !res[:Auth]
125
+ res[:Mode] = res[:Auth] ? :Auth : :Interactive
126
+ rescue => e
127
+ p e.message
128
+ end
129
+ res
130
+ end
131
+
132
+ def run
133
+
134
+ opts = parseOptions
135
+
136
+ case opts[:Mode]
137
+ when :Interactive
138
+
139
+ @h.say "\n<%= color('Welcome to',BOLD) %> <%= color('RubyAWS #{RubyAWS::VERSION}',BOLD,RED) %>"
140
+ @h.say "This version includes SDK extensions for: <%= list (#{SDKS.inspect}.collect {|a| color(a,BOLD)}), :inline, ' and ' %>\n\n"
141
+
142
+ checkAuthConfig
143
+
144
+ default_menu
145
+ when :Auth
146
+
147
+ getAuthConfig( :Key => opts[:Key], :ID => opts[:ID] )
148
+
149
+ end
150
+ end
151
+
152
+ end
153
+
154
+ end # Amazon::WebServices::Util
155
+ end # Amazon::WebServices
156
+ end # Amazon::WebServices
@@ -0,0 +1,90 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'amazon/util'
5
+
6
+ module Amazon
7
+ module WebServices
8
+ module Util
9
+
10
+ class ConvenienceWrapper
11
+ include Amazon::Util::Logging
12
+
13
+ REQUIRED_PARAMETERS = [:ServiceClass]
14
+
15
+ def initialize(args)
16
+ missing_parameters = REQUIRED_PARAMETERS - args.keys
17
+ raise "Missing paramters: #{missing_parameters.join(',')}" unless missing_parameters.empty?
18
+ @service = args[:ServiceClass].new( args )
19
+ end
20
+
21
+ def callService( method, *args )
22
+ @service.send( method, *args )
23
+ end
24
+
25
+ def method_missing( method, *args )
26
+ if @service.respond_to? method
27
+ callService( method, *args )
28
+ else
29
+ callService( ConvenienceWrapper.real_method( method ), *args )
30
+ end
31
+ end
32
+
33
+ def self.serviceCall( method, responseTag, defaultArgs={} )
34
+ method = method.to_s
35
+ name = ( method[0..0].downcase + method[1..-1] ).to_sym
36
+ rawName = ( name.to_s + "Raw" ).to_sym
37
+ method = real_method( method )
38
+
39
+ raise 'Stop redifining service methods!' if self.instance_methods.include? name.to_s
40
+
41
+ define_method( rawName ) do |args|
42
+ log "Sending service request '#{name}' with args: #{args.inspect}"
43
+ result = callService( method, args )
44
+ return result[responseTag]
45
+ end
46
+
47
+ define_method( name ) do |*params|
48
+ userArgs = params[0] || {}
49
+ args = defaultArgs.merge( userArgs )
50
+ self.send rawName, args
51
+ end
52
+ end
53
+
54
+ def self.paginate( method, elementTag, pageSize=25 )
55
+ method = method.to_s
56
+ all_name = ( method[0..0].downcase + method[1..-1] + "All" ).to_sym
57
+ iterator_name = ( method[0..0].downcase + method[1..-1] + "Iterator" ).to_sym
58
+ proactive_name = ( method[0..0].downcase + method[1..-1] + "AllProactive" ).to_sym
59
+ method = ( method[0..0].downcase + method[1..-1] ).to_sym
60
+
61
+ raise 'Stop redifining service methods!' if self.instance_methods.include? name.to_s
62
+
63
+ processors = { all_name => Amazon::Util::LazyResults,
64
+ iterator_name => Amazon::Util::PaginatedIterator,
65
+ proactive_name => Amazon::Util::ProactiveResults }
66
+
67
+ processors.each do |name,processor|
68
+ define_method( name ) do |*params|
69
+ userArgs = params[0] || {}
70
+ args = {:PageSize => pageSize}.merge( userArgs ) # allow user to override page size
71
+ return processor.new do |pageNumber|
72
+ pageArgs = args.merge({:PageNumber => pageNumber}) # don't allow override of page number
73
+ res = self.send( method, pageArgs)[elementTag]
74
+ res.nil? ? nil : [res].flatten
75
+ end
76
+ end
77
+ end
78
+
79
+ end
80
+
81
+ def self.real_method( method )
82
+ method = method.to_s
83
+ method = ( method[0..0].upcase + method[1..-1] ).to_sym
84
+ end
85
+
86
+ end # ConvenienceWrapper
87
+
88
+ end # Amazon::WebServices::Util
89
+ end # Amazon::WebServices
90
+ end # Amazon
@@ -0,0 +1,45 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'amazon/util/filter_chain'
5
+
6
+ module Amazon
7
+ module WebServices
8
+ module Util
9
+
10
+ # Filter proxy is a class that can filter argument input before passing onto a proxy object
11
+ # == Usage
12
+ # Initialize with standard argument block, pass the class or pre-initialized object as the :FilterProxy parameter.
13
+ # All parameters will be passed along to the contructor of the passed class.
14
+ # FilterProxy exposes a FilterChain object, with all the magic which that class provides.
15
+ class FilterProxy
16
+
17
+ attr_accessor :filter_chain
18
+
19
+ def initialize(args)
20
+ @filter_chain = Amazon::Util::FilterChain.new
21
+ end
22
+
23
+ def configure(args)
24
+ @proxy = case args[:FilterProxy]
25
+ when Class
26
+ args[:FilterProxy].new( args )
27
+ when nil
28
+ raise "No FilterProxy or DefaultOverride defined!" unless args[:DefaultOverride].is_a? Proc
29
+ args[:DefaultOverride].call( args )
30
+ else
31
+ args[:FilterProxy]
32
+ end
33
+ end
34
+
35
+ def method_missing(method, *args)
36
+ @filter_chain.execute(method, args) { |method,args|
37
+ @proxy.send(method,*args)
38
+ }
39
+ end
40
+
41
+ end
42
+
43
+ end # Amazon::WebServices::Util
44
+ end # Amazon::WebServices
45
+ end # Amazon
@@ -0,0 +1,70 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class MockTransport
9
+ include Enumerable
10
+
11
+ class MethodCall
12
+
13
+ def initialize( name, args )
14
+ @name = name
15
+ @args = args
16
+ @request = args[:Request][0]
17
+ end
18
+
19
+ attr_reader :name, :args, :request
20
+
21
+ end
22
+
23
+ def initialize( args={} )
24
+ @call_buffer = []
25
+ @index = 0
26
+ @listener = nil
27
+ @mock_reply = args[:MockReply] || { :MockResult => { :Mock => true, :Request => {} }, :OperationRequest => {} }
28
+ @listener = args[:MockListener] if args[:MockListener].is_a? Proc
29
+ end
30
+
31
+ attr_reader :call_buffer
32
+ attr_accessor :mock_reply
33
+
34
+ def flush
35
+ @call_buffer = []
36
+ @index = 0
37
+ end
38
+
39
+ def next
40
+ return nil if @index >= @call_buffer.size
41
+ ret = @call_buffer[@index]
42
+ @index += 1
43
+ ret
44
+ end
45
+
46
+ def each(&block) # :yields: method_call
47
+ @call_buffer[@index..-1].each { |item| yield item }
48
+ # yield self.next until @index >= @call_buffer.size
49
+ end
50
+
51
+ def listen(&block) # :yields: method_call
52
+ @listener = block
53
+ end
54
+
55
+ def method_missing(method,*args)
56
+ raise "only support one parameter" unless args.size <= 1
57
+ the_method = MethodCall.new( method, args[0] )
58
+ @call_buffer << the_method
59
+ listen_result = @listener.call( the_method ) unless @listener.nil?
60
+ response = @mock_reply.dup
61
+ response[:MockResult][:Request] = the_method.request unless response[:MockResult].nil?
62
+ response.merge!(listen_result) if listen_result.is_a? Hash
63
+ response
64
+ end
65
+
66
+ end
67
+
68
+ end # Amazon::Webservices::Util
69
+ end # Amazon::Webservices
70
+ end # Amazon
@@ -0,0 +1,42 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'base64'
5
+ require 'digest/sha1'
6
+
7
+ module Amazon
8
+ module WebServices
9
+ module Util
10
+
11
+ module RequestSigner
12
+
13
+ def RequestSigner.sign(service,method,time,key)
14
+ msg = "#{service}#{method}#{time}"
15
+ return hmac_sha1( key, msg )
16
+ end
17
+
18
+
19
+ private
20
+
21
+ def RequestSigner.hmac_sha1(key, s)
22
+ ipad = [].fill(0x36, 0, 64)
23
+ opad = [].fill(0x5C, 0, 64)
24
+ key = key.unpack("C*")
25
+ key += [].fill(0, 0, 64-key.length) if key.length < 64
26
+
27
+ inner = []
28
+ 64.times { |i| inner.push(key[i] ^ ipad[i]) }
29
+ inner += s.unpack("C*")
30
+
31
+ outer = []
32
+ 64.times { |i| outer.push(key[i] ^ opad[i]) }
33
+ outer = outer.pack("c*")
34
+ outer += Digest::SHA1.digest(inner.pack("c*"))
35
+
36
+ return Base64::encode64(Digest::SHA1.digest(outer)).chomp
37
+ end
38
+
39
+ end # Amazon::MTS::Util::RequestSigner
40
+ end # Amazon::MTS::Util
41
+ end # Amazon::MTS
42
+ end # Amazon