ruby-aws 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. data.tar.gz.sig +0 -0
  2. data/.gemtest +0 -0
  3. data/History.txt +75 -0
  4. data/LICENSE.txt +202 -0
  5. data/Manifest.txt +72 -0
  6. data/NOTICE.txt +4 -0
  7. data/README.txt +105 -0
  8. data/Rakefile +33 -0
  9. data/bin/ruby-aws +9 -0
  10. data/lib/amazon/util.rb +10 -0
  11. data/lib/amazon/util/binder.rb +48 -0
  12. data/lib/amazon/util/data_reader.rb +169 -0
  13. data/lib/amazon/util/filter_chain.rb +79 -0
  14. data/lib/amazon/util/hash_nesting.rb +93 -0
  15. data/lib/amazon/util/lazy_results.rb +59 -0
  16. data/lib/amazon/util/logging.rb +23 -0
  17. data/lib/amazon/util/paginated_iterator.rb +70 -0
  18. data/lib/amazon/util/proactive_results.rb +116 -0
  19. data/lib/amazon/util/threadpool.rb +129 -0
  20. data/lib/amazon/util/user_data_store.rb +100 -0
  21. data/lib/amazon/webservices/mechanical_turk.rb +123 -0
  22. data/lib/amazon/webservices/mechanical_turk_requester.rb +274 -0
  23. data/lib/amazon/webservices/mturk/mechanical_turk_error_handler.rb +150 -0
  24. data/lib/amazon/webservices/mturk/question_generator.rb +58 -0
  25. data/lib/amazon/webservices/util/amazon_authentication_relay.rb +72 -0
  26. data/lib/amazon/webservices/util/command_line.rb +157 -0
  27. data/lib/amazon/webservices/util/convenience_wrapper.rb +90 -0
  28. data/lib/amazon/webservices/util/filter_proxy.rb +45 -0
  29. data/lib/amazon/webservices/util/mock_transport.rb +70 -0
  30. data/lib/amazon/webservices/util/request_signer.rb +42 -0
  31. data/lib/amazon/webservices/util/rest_transport.rb +120 -0
  32. data/lib/amazon/webservices/util/soap_simplifier.rb +48 -0
  33. data/lib/amazon/webservices/util/soap_transport.rb +20 -0
  34. data/lib/amazon/webservices/util/soap_transport_header_handler.rb +27 -0
  35. data/lib/amazon/webservices/util/unknown_result_exception.rb +27 -0
  36. data/lib/amazon/webservices/util/validation_exception.rb +55 -0
  37. data/lib/amazon/webservices/util/xml_simplifier.rb +61 -0
  38. data/lib/ruby-aws.rb +19 -0
  39. data/lib/ruby-aws/version.rb +6 -0
  40. data/run_rcov.sh +1 -0
  41. data/samples/mturk/best_image/BestImage.rb +61 -0
  42. data/samples/mturk/best_image/best_image.properties +39 -0
  43. data/samples/mturk/best_image/best_image.question +82 -0
  44. data/samples/mturk/blank_slate/BlankSlate.rb +63 -0
  45. data/samples/mturk/blank_slate/BlankSlate_multithreaded.rb +67 -0
  46. data/samples/mturk/helloworld/MTurkHelloWorld.rb +56 -0
  47. data/samples/mturk/helloworld/mturk.yml +8 -0
  48. data/samples/mturk/review_policy/ReviewPolicy.rb +139 -0
  49. data/samples/mturk/review_policy/review_policy.question +30 -0
  50. data/samples/mturk/reviewer/Reviewer.rb +103 -0
  51. data/samples/mturk/reviewer/mturk.yml +8 -0
  52. data/samples/mturk/simple_survey/SimpleSurvey.rb +90 -0
  53. data/samples/mturk/simple_survey/simple_survey.question +30 -0
  54. data/samples/mturk/site_category/SiteCategory.rb +87 -0
  55. data/samples/mturk/site_category/externalpage.htm +71 -0
  56. data/samples/mturk/site_category/site_category.input +6 -0
  57. data/samples/mturk/site_category/site_category.properties +45 -0
  58. data/samples/mturk/site_category/site_category.question +9 -0
  59. data/test/mturk/test_changehittypeofhit.rb +130 -0
  60. data/test/mturk/test_error_handler.rb +137 -0
  61. data/test/mturk/test_mechanical_turk_requester.rb +178 -0
  62. data/test/mturk/test_mock_mechanical_turk_requester.rb +205 -0
  63. data/test/test_ruby-aws.rb +24 -0
  64. data/test/unit/test_binder.rb +89 -0
  65. data/test/unit/test_data_reader.rb +135 -0
  66. data/test/unit/test_exceptions.rb +32 -0
  67. data/test/unit/test_hash_nesting.rb +99 -0
  68. data/test/unit/test_lazy_results.rb +89 -0
  69. data/test/unit/test_mock_transport.rb +132 -0
  70. data/test/unit/test_paginated_iterator.rb +58 -0
  71. data/test/unit/test_proactive_results.rb +108 -0
  72. data/test/unit/test_question_generator.rb +55 -0
  73. data/test/unit/test_threadpool.rb +50 -0
  74. data/test/unit/test_user_data_store.rb +80 -0
  75. metadata +238 -0
  76. metadata.gz.sig +0 -0
@@ -0,0 +1,58 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'rexml/element'
5
+
6
+ module Amazon
7
+ module WebServices
8
+ module MTurk
9
+
10
+ class QuestionGenerator
11
+
12
+ def self.build(type=:Basic)
13
+ question = self.new(type)
14
+ yield question
15
+ return question.to_xml
16
+ end
17
+
18
+ def initialize(type=:Basic)
19
+ @overview = nil
20
+ @questions = []
21
+ @type = type
22
+ end
23
+
24
+ def ask(*args)
25
+ case @type
26
+ when :Basic
27
+ askBasic( args.join )
28
+ end
29
+ end
30
+
31
+ def askBasic(text)
32
+ id = "BasicQuestion#{@questions.size+1}"
33
+ question = REXML::Element.new 'Text'
34
+ question.text = text
35
+ answerSpec = "<FreeTextAnswer/>"
36
+ @questions << { :Id => id, :Question => question.to_s, :AnswerSpec => answerSpec }
37
+ end
38
+
39
+ def to_xml
40
+ components = [PREAMBLE]
41
+ components << OVERVIEW % @overview unless @overview.nil?
42
+ for question in @questions
43
+ components << QUESTION % [ question[:Id], question[:Question], question[:AnswerSpec] ]
44
+ end
45
+ components << [TAIL]
46
+ return components.join
47
+ end
48
+
49
+ PREAMBLE = '<?xml version="1.0" encoding="UTF-8"?>'+"\n"+'<QuestionForm xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd">'
50
+ OVERVIEW = '<Overview>%s</Overview>'
51
+ QUESTION = '<Question><QuestionIdentifier>%s</QuestionIdentifier><QuestionContent>%s</QuestionContent><AnswerSpecification>%s</AnswerSpecification></Question>'
52
+ TAIL = '</QuestionForm>'
53
+
54
+ end # QuestionGenerator
55
+
56
+ end # Amazon::WebServices::MTurk
57
+ end # Amazon::WebServices
58
+ end # Amazon
@@ -0,0 +1,72 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'amazon/webservices/util/command_line'
5
+ require 'amazon/webservices/util/request_signer'
6
+
7
+ module Amazon
8
+ module WebServices
9
+ module Util
10
+
11
+ class AmazonAuthenticationRelay
12
+
13
+ REQUIRED_PARAMETERS = [:Name,:Transport]
14
+
15
+ def initialize( args )
16
+ missing_parameters = REQUIRED_PARAMETERS - args.keys
17
+ raise "Missing paramters: #{missing_parameters.join(',')}" unless missing_parameters.empty?
18
+ @name = args[:Name]
19
+ @transport = args[:Transport]
20
+ @keyId, @key = findAuthInfo( args )
21
+ end
22
+
23
+ def withCredential(credential,method,*request)
24
+ time = Time.now.gmtime.strftime('%Y-%m-%dT%H:%M:%S.123Z')
25
+ request[0] ||= {}
26
+ args = { :AWSAccessKeyId => @keyId,
27
+ :Timestamp => time,
28
+ :Signature => RequestSigner.sign(@name,method,time,@key),
29
+ :Credential => credential,
30
+ :Request => request }
31
+ @transport.send( method, args )
32
+ end
33
+
34
+ def method_missing(method, *request)
35
+ time = Time.now.gmtime.strftime('%Y-%m-%dT%H:%M:%S.321Z')
36
+ request[0] ||= {}
37
+ args = { :AWSAccessKeyId => @keyId,
38
+ :Timestamp => time,
39
+ :Signature => RequestSigner.sign(@name,method,time,@key),
40
+ :Request => request }
41
+ @transport.send( method, args )
42
+ end
43
+
44
+ def to_s
45
+ "AmazonAuthenticationRelay[name:#{@name} transport:#{@transport}]>"
46
+ end
47
+
48
+ def inspect
49
+ "#<Amazon::WebServices::Util::AmazonAuthenticationRelay:#{object_id} name:#{@name} transport:#{@transport.inspect}>"
50
+ end
51
+
52
+ private
53
+
54
+ def findAuthInfo( args )
55
+ if args.has_key? :AWSAccessKey or args.has_key? :AWSAccessKeyId
56
+ # user tried to pass authentication information in, so just use that
57
+ raise "Missing AWSAccessKeyId" if args[:AWSAccessKeyId].nil?
58
+ raise "Missing AWSAccessKey" if args[:AWSAccessKey].nil?
59
+ return args[:AWSAccessKeyId], args[:AWSAccessKey]
60
+ else
61
+ # didn't get passed in, so load from config or go interactive
62
+ cmd = CommandLine.new
63
+ cmd.checkAuthConfig
64
+ return cmd.authId, cmd.authKey
65
+ end
66
+ end
67
+
68
+ end # AmazonAuthenticationRelay
69
+
70
+ end # Amazon::WebServices::Util
71
+ end # Amazon::WebServices
72
+ end # Amazon
@@ -0,0 +1,157 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'optparse'
5
+ require 'highline'
6
+ require 'amazon/util/user_data_store'
7
+
8
+ module Amazon
9
+ module WebServices
10
+ module Util
11
+
12
+ class CommandLine
13
+
14
+ SDKS = ['Mechanical Turk']
15
+
16
+ MISSING_AUTH = "You are missing authentication information required to utilize Amazon Web Services. You will now be prompted for your <%= color('Access Key ID',BOLD) %> and your <%= color('Secret Access Key',BOLD) %>. If you do not have this information, please log into http://www.amazonaws.com/ \n\n"
17
+ RESUME_AUTH = "Authentication information has been initialized. Continuing... \n\n"
18
+
19
+ MAIN_HELP = <<EOF
20
+
21
+ This is the <%= color('RubyAWS', RED, BOLD) %> Command Line Application's Interactive mode.
22
+ You got here by typing:
23
+
24
+ <%= color('$ ruby-aws',GREEN) %>
25
+
26
+ This application currently supports the following functionality:
27
+
28
+ <%= list( ['Amazon Web Services Authentication Configuration'] ) %>
29
+ You can also invoke this tool with commandline parameters. For more information:
30
+
31
+ <%= color('$ ruby-aws --help',GREEN) %>
32
+
33
+ Thanks for using Amazon Web Services!
34
+
35
+ EOF
36
+
37
+ def initialize( interactive = true )
38
+ @interactive = interactive
39
+ @h = HighLine.new
40
+ @h.wrap_at = :auto
41
+ @store = Amazon::Util::UserDataStore.new :AWS
42
+ HighLine.use_color = useColor?
43
+ HighLine.track_eof = false # disabling because it misbehaves
44
+ end
45
+
46
+ def checkAuthConfig
47
+ if @store.get(:Auth,:AccessKeyId).nil? or @store.get(:Auth,:SecretAccessKey).nil?
48
+ @h.say MISSING_AUTH
49
+ getAuthConfig
50
+ @h.say RESUME_AUTH
51
+ end
52
+ end
53
+
54
+ def getAuthConfig( default={} )
55
+ id = @h.ask( 'What is your Access Key ID?' ) {|q| q.validate = /^[A-Z0-9]{20}$/ ; q.default = authId.to_s ; q.first_answer = default[:ID] }
56
+ key = @h.ask( 'What is your Secret Access Key?' ) {|q| q.validate = /^[\w\/+]{40}$/ ; q.default = authKey.to_s ; q.first_answer = default[:Key] }
57
+
58
+ @store.set(:Auth,:AccessKeyId,id)
59
+ @store.set(:Auth,:SecretAccessKey,key)
60
+
61
+ should_save = @h.agree( 'Would you like to save your authentication information?' )
62
+ @store.save if should_save
63
+ rescue
64
+ raise "Unable to retrieve authentication information from the Console"
65
+ end
66
+
67
+ def authKey
68
+ @store.get(:Auth,:SecretAccessKey)
69
+ end
70
+ def authId
71
+ @store.get(:Auth,:AccessKeyId)
72
+ end
73
+
74
+ def useColor?
75
+ if @store.get(:Misc,:ColorTerminal).nil?
76
+ if @interactive and @h.agree( "Should the console application use color? (y/n)" )
77
+ @store.set(:Misc,:ColorTerminal,true)
78
+ else
79
+ @store.set(:Misc,:ColorTerminal,false)
80
+ end
81
+ @store.save
82
+ end
83
+ return @store.get(:Misc,:ColorTerminal)
84
+ end
85
+
86
+ def toggleColor
87
+ value = !useColor?
88
+ @store.set(:Misc,:ColorTerminal,value)
89
+ HighLine.use_color = value
90
+ end
91
+
92
+ def default_menu
93
+ loop do
94
+ @h.choose do |menu|
95
+ menu.header = "\n" + @h.color('RubyAWS',HighLine::BOLD,HighLine::RED) + " " + @h.color('Command Line Application',HighLine::BOLD) + " " + @h.color('[Interactive Mode]',HighLine::GREEN,HighLine::BOLD)
96
+ menu.select_by = :index
97
+ menu.choice( 'Configure Amazon Web Services Authentication' ) do
98
+ if @h.agree( "\nCurrent ID: #{@h.color(authId,HighLine::BOLD)}\nCurrent Key: #{@h.color(authKey,HighLine::BOLD)}\nDo you want to change?" )
99
+ getAuthConfig
100
+ end
101
+ end
102
+ menu.choice( 'Toggle Color' ) { toggleColor }
103
+ menu.choice( :help ) do
104
+ @h.say MAIN_HELP
105
+ end
106
+ menu.choice( 'Save and Quit' ) { @store.save ; exit }
107
+ menu.prompt = "\nWhat would you like to do? "
108
+ end
109
+ end
110
+ end
111
+
112
+ def parseOptions
113
+ res = {}
114
+ opts = OptionParser.new
115
+
116
+ opts.on( '-i', '--interactive', 'Load Interactive Mode' ) { res[:Interactive] = true }
117
+ opts.on( '-a', '--authenticate', 'Configure Authentication Options' ) { res[:Auth] = true }
118
+ opts.on( '--id=ID', 'Set Access Key ID (requires "-a")' ) { |id| res[:ID] = id }
119
+ opts.on( '--key=KEY', 'Set Secret Access Key (requires "-a")' ) { |key| res[:Key] = key }
120
+
121
+ begin
122
+ opts.parse(ARGV)
123
+ raise "-i and -a are exclusive options. Please pick one." if res[:Interactive] and res[:Auth]
124
+ raise "--id requires -a" if res[:ID] and !res[:Auth]
125
+ raise "--key requires -a" if res[:Key] and !res[:Auth]
126
+ res[:Mode] = res[:Auth] ? :Auth : :Interactive
127
+ rescue => e
128
+ p e.message
129
+ end
130
+ res
131
+ end
132
+
133
+ def run
134
+
135
+ opts = parseOptions
136
+
137
+ case opts[:Mode]
138
+ when :Interactive
139
+
140
+ @h.say "\n<%= color('Welcome to',BOLD) %> <%= color('RubyAWS #{RubyAWS::VERSION}',BOLD,RED) %>"
141
+ @h.say "This version includes SDK extensions for: <%= list (#{SDKS.inspect}.collect {|a| color(a,BOLD)}), :inline, ' and ' %>\n\n"
142
+
143
+ checkAuthConfig
144
+
145
+ default_menu
146
+ when :Auth
147
+
148
+ getAuthConfig( :Key => opts[:Key], :ID => opts[:ID] )
149
+
150
+ end
151
+ end
152
+
153
+ end
154
+
155
+ end # Amazon::WebServices::Util
156
+ end # Amazon::WebServices
157
+ end # Amazon::WebServices
@@ -0,0 +1,90 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'amazon/util'
5
+
6
+ module Amazon
7
+ module WebServices
8
+ module Util
9
+
10
+ class ConvenienceWrapper
11
+ include Amazon::Util::Logging
12
+
13
+ REQUIRED_PARAMETERS = [:ServiceClass]
14
+
15
+ def initialize(args)
16
+ missing_parameters = REQUIRED_PARAMETERS - args.keys
17
+ raise "Missing paramters: #{missing_parameters.join(',')}" unless missing_parameters.empty?
18
+ @service = args[:ServiceClass].new( args )
19
+ end
20
+
21
+ def callService( method, *args )
22
+ @service.send( method, *args )
23
+ end
24
+
25
+ def method_missing( method, *args )
26
+ if @service.respond_to? method
27
+ callService( method, *args )
28
+ else
29
+ callService( ConvenienceWrapper.real_method( method ), *args )
30
+ end
31
+ end
32
+
33
+ def self.serviceCall( method, responseTag, defaultArgs={} )
34
+ method = method.to_s
35
+ name = ( method[0..0].downcase + method[1..-1] ).to_sym
36
+ rawName = ( name.to_s + "Raw" ).to_sym
37
+ method = real_method( method )
38
+
39
+ raise 'Stop redifining service methods!' if self.instance_methods.include? name.to_s
40
+
41
+ define_method( rawName ) do |args|
42
+ log "Sending service request '#{name}' with args: #{args.inspect}"
43
+ result = callService( method, args )
44
+ return result[responseTag]
45
+ end
46
+
47
+ define_method( name ) do |*params|
48
+ userArgs = params[0] || {}
49
+ args = defaultArgs.merge( userArgs )
50
+ self.send rawName, args
51
+ end
52
+ end
53
+
54
+ def self.paginate( method, elementTag, pageSize=25 )
55
+ method = method.to_s
56
+ all_name = ( method[0..0].downcase + method[1..-1] + "All" ).to_sym
57
+ iterator_name = ( method[0..0].downcase + method[1..-1] + "Iterator" ).to_sym
58
+ proactive_name = ( method[0..0].downcase + method[1..-1] + "AllProactive" ).to_sym
59
+ method = ( method[0..0].downcase + method[1..-1] ).to_sym
60
+
61
+ raise 'Stop redifining service methods!' if self.instance_methods.include? name.to_s
62
+
63
+ processors = { all_name => Amazon::Util::LazyResults,
64
+ iterator_name => Amazon::Util::PaginatedIterator,
65
+ proactive_name => Amazon::Util::ProactiveResults }
66
+
67
+ processors.each do |name,processor|
68
+ define_method( name ) do |*params|
69
+ userArgs = params[0] || {}
70
+ args = {:PageSize => pageSize}.merge( userArgs ) # allow user to override page size
71
+ return processor.new do |pageNumber|
72
+ pageArgs = args.merge({:PageNumber => pageNumber}) # don't allow override of page number
73
+ res = self.send( method, pageArgs)[elementTag]
74
+ res.nil? ? nil : [res].flatten
75
+ end
76
+ end
77
+ end
78
+
79
+ end
80
+
81
+ def self.real_method( method )
82
+ method = method.to_s
83
+ method = ( method[0..0].upcase + method[1..-1] ).to_sym
84
+ end
85
+
86
+ end # ConvenienceWrapper
87
+
88
+ end # Amazon::WebServices::Util
89
+ end # Amazon::WebServices
90
+ end # Amazon
@@ -0,0 +1,45 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'amazon/util/filter_chain'
5
+
6
+ module Amazon
7
+ module WebServices
8
+ module Util
9
+
10
+ # Filter proxy is a class that can filter argument input before passing onto a proxy object
11
+ # == Usage
12
+ # Initialize with standard argument block, pass the class or pre-initialized object as the :FilterProxy parameter.
13
+ # All parameters will be passed along to the contructor of the passed class.
14
+ # FilterProxy exposes a FilterChain object, with all the magic which that class provides.
15
+ class FilterProxy
16
+
17
+ attr_accessor :filter_chain
18
+
19
+ def initialize(args)
20
+ @filter_chain = Amazon::Util::FilterChain.new
21
+ end
22
+
23
+ def configure(args)
24
+ @proxy = case args[:FilterProxy]
25
+ when Class
26
+ args[:FilterProxy].new( args )
27
+ when nil
28
+ raise "No FilterProxy or DefaultOverride defined!" unless args[:DefaultOverride].is_a? Proc
29
+ args[:DefaultOverride].call( args )
30
+ else
31
+ args[:FilterProxy]
32
+ end
33
+ end
34
+
35
+ def method_missing(method, *args)
36
+ @filter_chain.execute(method, args) { |method,args|
37
+ @proxy.send(method,*args)
38
+ }
39
+ end
40
+
41
+ end
42
+
43
+ end # Amazon::WebServices::Util
44
+ end # Amazon::WebServices
45
+ end # Amazon
@@ -0,0 +1,70 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class MockTransport
9
+ include Enumerable
10
+
11
+ class MethodCall
12
+
13
+ def initialize( name, args )
14
+ @name = name
15
+ @args = args
16
+ @request = args[:Request][0]
17
+ end
18
+
19
+ attr_reader :name, :args, :request
20
+
21
+ end
22
+
23
+ def initialize( args={} )
24
+ @call_buffer = []
25
+ @index = 0
26
+ @listener = nil
27
+ @mock_reply = args[:MockReply] || { :MockResult => { :Mock => true, :Request => {} }, :OperationRequest => {} }
28
+ @listener = args[:MockListener] if args[:MockListener].is_a? Proc
29
+ end
30
+
31
+ attr_reader :call_buffer
32
+ attr_accessor :mock_reply
33
+
34
+ def flush
35
+ @call_buffer = []
36
+ @index = 0
37
+ end
38
+
39
+ def next
40
+ return nil if @index >= @call_buffer.size
41
+ ret = @call_buffer[@index]
42
+ @index += 1
43
+ ret
44
+ end
45
+
46
+ def each(&block) # :yields: method_call
47
+ @call_buffer[@index..-1].each { |item| yield item }
48
+ # yield self.next until @index >= @call_buffer.size
49
+ end
50
+
51
+ def listen(&block) # :yields: method_call
52
+ @listener = block
53
+ end
54
+
55
+ def method_missing(method,*args)
56
+ raise "only support one parameter" unless args.size <= 1
57
+ the_method = MethodCall.new( method, args[0] )
58
+ @call_buffer << the_method
59
+ listen_result = @listener.call( the_method ) unless @listener.nil?
60
+ response = @mock_reply.dup
61
+ response[:MockResult][:Request] = the_method.request unless response[:MockResult].nil?
62
+ response.merge!(listen_result) if listen_result.is_a? Hash
63
+ response
64
+ end
65
+
66
+ end
67
+
68
+ end # Amazon::Webservices::Util
69
+ end # Amazon::Webservices
70
+ end # Amazon