mturk 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +2 -0
  4. data/.gemtest +0 -0
  5. data/History.md +105 -0
  6. data/LICENSE.txt +202 -0
  7. data/Manifest.txt +72 -0
  8. data/NOTICE.txt +4 -0
  9. data/README.md +100 -0
  10. data/Rakefile +33 -0
  11. data/bin/mturk +9 -0
  12. data/lib/amazon/util.rb +10 -0
  13. data/lib/amazon/util/binder.rb +48 -0
  14. data/lib/amazon/util/data_reader.rb +169 -0
  15. data/lib/amazon/util/filter_chain.rb +79 -0
  16. data/lib/amazon/util/hash_nesting.rb +93 -0
  17. data/lib/amazon/util/lazy_results.rb +59 -0
  18. data/lib/amazon/util/logging.rb +23 -0
  19. data/lib/amazon/util/paginated_iterator.rb +70 -0
  20. data/lib/amazon/util/proactive_results.rb +116 -0
  21. data/lib/amazon/util/threadpool.rb +129 -0
  22. data/lib/amazon/util/user_data_store.rb +100 -0
  23. data/lib/amazon/webservices/mechanical_turk.rb +123 -0
  24. data/lib/amazon/webservices/mechanical_turk_requester.rb +285 -0
  25. data/lib/amazon/webservices/mturk/mechanical_turk_error_handler.rb +153 -0
  26. data/lib/amazon/webservices/mturk/question_generator.rb +58 -0
  27. data/lib/amazon/webservices/util/amazon_authentication_relay.rb +72 -0
  28. data/lib/amazon/webservices/util/command_line.rb +155 -0
  29. data/lib/amazon/webservices/util/convenience_wrapper.rb +90 -0
  30. data/lib/amazon/webservices/util/filter_proxy.rb +45 -0
  31. data/lib/amazon/webservices/util/mock_transport.rb +70 -0
  32. data/lib/amazon/webservices/util/request_signer.rb +42 -0
  33. data/lib/amazon/webservices/util/rest_transport.rb +120 -0
  34. data/lib/amazon/webservices/util/soap_simplifier.rb +48 -0
  35. data/lib/amazon/webservices/util/soap_transport.rb +20 -0
  36. data/lib/amazon/webservices/util/soap_transport_header_handler.rb +27 -0
  37. data/lib/amazon/webservices/util/unknown_result_exception.rb +27 -0
  38. data/lib/amazon/webservices/util/validation_exception.rb +55 -0
  39. data/lib/amazon/webservices/util/xml_simplifier.rb +61 -0
  40. data/lib/mturk.rb +19 -0
  41. data/lib/mturk/version.rb +6 -0
  42. data/run_rcov.sh +1 -0
  43. data/samples/best_image/BestImage.rb +61 -0
  44. data/samples/best_image/best_image.properties +39 -0
  45. data/samples/best_image/best_image.question +82 -0
  46. data/samples/blank_slate/BlankSlate.rb +63 -0
  47. data/samples/blank_slate/BlankSlate_multithreaded.rb +67 -0
  48. data/samples/helloworld/MTurkHelloWorld.rb +56 -0
  49. data/samples/helloworld/mturk.yml +8 -0
  50. data/samples/review_policy/ReviewPolicy.rb +139 -0
  51. data/samples/review_policy/review_policy.question +30 -0
  52. data/samples/reviewer/Reviewer.rb +103 -0
  53. data/samples/reviewer/mturk.yml +8 -0
  54. data/samples/simple_survey/SimpleSurvey.rb +98 -0
  55. data/samples/simple_survey/simple_survey.question +30 -0
  56. data/samples/site_category/SiteCategory.rb +87 -0
  57. data/samples/site_category/externalpage.htm +71 -0
  58. data/samples/site_category/site_category.input +6 -0
  59. data/samples/site_category/site_category.properties +56 -0
  60. data/samples/site_category/site_category.question +9 -0
  61. data/test/mturk/test_changehittypeofhit.rb +130 -0
  62. data/test/mturk/test_error_handler.rb +403 -0
  63. data/test/mturk/test_mechanical_turk_requester.rb +178 -0
  64. data/test/mturk/test_mock_mechanical_turk_requester.rb +205 -0
  65. data/test/test_mturk.rb +21 -0
  66. data/test/unit/test_binder.rb +89 -0
  67. data/test/unit/test_data_reader.rb +135 -0
  68. data/test/unit/test_exceptions.rb +32 -0
  69. data/test/unit/test_hash_nesting.rb +99 -0
  70. data/test/unit/test_lazy_results.rb +89 -0
  71. data/test/unit/test_mock_transport.rb +132 -0
  72. data/test/unit/test_paginated_iterator.rb +58 -0
  73. data/test/unit/test_proactive_results.rb +108 -0
  74. data/test/unit/test_question_generator.rb +55 -0
  75. data/test/unit/test_threadpool.rb +50 -0
  76. data/test/unit/test_user_data_store.rb +80 -0
  77. metadata +225 -0
  78. metadata.gz.sig +0 -0
@@ -0,0 +1,42 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'base64'
5
+ require 'digest/sha1'
6
+
7
+ module Amazon
8
+ module WebServices
9
+ module Util
10
+
11
+ module RequestSigner
12
+
13
+ def RequestSigner.sign(service,method,time,key)
14
+ msg = "#{service}#{method}#{time}"
15
+ return hmac_sha1( key, msg )
16
+ end
17
+
18
+
19
+ private
20
+
21
+ def RequestSigner.hmac_sha1(key, s)
22
+ ipad = [].fill(0x36, 0, 64)
23
+ opad = [].fill(0x5C, 0, 64)
24
+ key = key.unpack("C*")
25
+ key += [].fill(0, 0, 64-key.length) if key.length < 64
26
+
27
+ inner = []
28
+ 64.times { |i| inner.push(key[i] ^ ipad[i]) }
29
+ inner += s.unpack("C*")
30
+
31
+ outer = []
32
+ 64.times { |i| outer.push(key[i] ^ opad[i]) }
33
+ outer = outer.pack("c*")
34
+ outer += Digest::SHA1.digest(inner.pack("c*"))
35
+
36
+ return Base64::encode64(Digest::SHA1.digest(outer)).chomp
37
+ end
38
+
39
+ end # Amazon::MTS::Util::RequestSigner
40
+ end # Amazon::MTS::Util
41
+ end # Amazon::MTS
42
+ end # Amazon
@@ -0,0 +1,120 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'cgi'
5
+ require 'net/https'
6
+ require 'rubygems'
7
+ require 'nokogiri'
8
+ require 'amazon/webservices/util/xml_simplifier'
9
+
10
+ module Amazon
11
+ module WebServices
12
+ module Util
13
+
14
+ class RESTTransport
15
+
16
+ REQUIRED_PARAMETERS = [:Endpoint]
17
+
18
+ def self.canPost?
19
+ Net::HTTP.respond_to? :post_form
20
+ end
21
+
22
+ def initialize( args )
23
+ missing_parameters = REQUIRED_PARAMETERS - args.keys
24
+ raise "Missing paramters: #{missing_parameters.join(',')}" unless missing_parameters.empty?
25
+ @uri = URI.parse( args[:Endpoint] )
26
+ @httpMethod = resolveHTTPMethod( args[:RestStyle] )
27
+ @version = args[:Version]
28
+ @ssl = (@uri.scheme == 'https') || (@uri.port == 443) || args[:UseSSL]
29
+ @skip_ssl_verify = args[:SkipSSLCheck]
30
+
31
+ agent = ::MTurk::agent( args[:SoftwareName] )
32
+ @headers = {
33
+ 'User-Agent' => agent,
34
+ 'X-Amazon-Software' => agent,
35
+ 'Content-Type' => 'application/x-www-form-urlencoded; charset=utf-8',
36
+ }
37
+ end
38
+
39
+ def resolveHTTPMethod( method )
40
+ case method.to_s.upcase
41
+ when "GET"
42
+ return :GET
43
+ when "POST"
44
+ raise "Your version of Ruby does not support HTTP Post" unless RESTTransport.canPost?
45
+ return :POST
46
+ else
47
+ return ( RESTTransport.canPost? ? :POST : :GET )
48
+ end
49
+ end
50
+
51
+ def method_missing( method, *args )
52
+ params = { :Operation => method, :Version => @version }
53
+ params.merge!( args[0].delete( :Request )[0] )
54
+ params.merge!( args[0] )
55
+
56
+ http = Net::HTTP.new( @uri.host, @uri.port )
57
+ if @ssl
58
+ http.use_ssl = true
59
+ if @skip_ssl_verify
60
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
61
+ else
62
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
63
+ end
64
+ end
65
+
66
+ req = nil
67
+ if @httpMethod == :GET
68
+ req = Net::HTTP::Get.new( @uri.request_uri + toQueryString(params), @headers )
69
+ else
70
+ req = Net::HTTP::Post.new( @uri.request_uri, @headers )
71
+ req.form_data = toPostParams( params )
72
+ req['Content-Type'] = @headers['Content-Type'] # necessary because req.form_data resets Content-Type header
73
+ end
74
+
75
+ res = http.start { |conn|
76
+ conn.request(req)
77
+ }.body
78
+
79
+ xml = Nokogiri::XML( res )
80
+ XMLSimplifier.simplify xml
81
+ end
82
+
83
+ private
84
+
85
+ def toQueryString(params)
86
+ queryString = ""
87
+ each_http_param(params) { |key,value|
88
+ queryString << ( '&' + key + '=' + CGI.escape(value) )
89
+ }
90
+ return queryString
91
+ end
92
+
93
+ def toPostParams(params)
94
+ postParams = {}
95
+ each_http_param(params) { |key,value|
96
+ postParams[key] = value }
97
+ return postParams
98
+ end
99
+
100
+ def each_http_param(params,&block)
101
+ params.each {|k,v| each_http_param_helper( k, v, false, &block ) unless v.nil? }
102
+ end
103
+
104
+ def each_http_param_helper(key,value,num=false,&block)
105
+ key = key.to_s
106
+ case value.class.to_s
107
+ when 'Array'
108
+ value.each_with_index { |v,i| each_http_param_helper( "#{key}.#{i+1}", v, true, &block ) unless v.nil? }
109
+ when 'Hash'
110
+ value.each { |k,v| each_http_param_helper( "#{key}#{num ? '.' : '.1.'}#{k}", v, false, &block ) unless v.nil? }
111
+ else
112
+ yield key, value.to_s
113
+ end
114
+ end
115
+
116
+ end # RESTTransport
117
+
118
+ end # Amazon::WebServices::Util
119
+ end # Amazon::WebServices
120
+ end # Amazon
@@ -0,0 +1,48 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class SOAPSimplifier
9
+
10
+ # simplify(item) -- convert a soap object into a simple nested hash
11
+ def self.simplify(item)
12
+ case item.class.to_s
13
+ when 'SOAP::Mapping::Object'
14
+ simple = {}
15
+ item.__xmlattr.each {|name,at| simple["*#{name}*"] = simplify(at)}
16
+ item.__xmlele.each { |element|
17
+ # element consists of a QName and a payload
18
+ name = element[0].name
19
+ payload = simplify(element[1])
20
+ simple[name.to_sym] = payload
21
+ }
22
+ simple
23
+ when 'Array'
24
+ item.collect {|i| simplify(i) }
25
+ else
26
+ str = item.to_s
27
+ case str
28
+ when /^(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z$/
29
+ Time.gm($1,$2,$3,$4,$5,$6)
30
+ when /^\d+$/
31
+ if str.to_i.to_s == str
32
+ str.to_i
33
+ else
34
+ str
35
+ end
36
+ when /^\d+\.\d+$/
37
+ str.to_f
38
+ else
39
+ str
40
+ end
41
+ end
42
+ end
43
+
44
+ end # Amazon::WebServices::Util::SoapSimplifier
45
+
46
+ end # Amazon::WebServices::Util
47
+ end # Amazon::WebServices
48
+ end # Amazon
@@ -0,0 +1,20 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class SOAPTransport
9
+
10
+ def initialize(args)
11
+ raise 'SOAP no longer supported'
12
+ end
13
+
14
+ def self.canSOAP? ; false ; end
15
+
16
+ end # SOAPTransport
17
+
18
+ end # Amazon::WebServices::Util
19
+ end # Amazon::WebServices
20
+ end # Amazon
@@ -0,0 +1,27 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'soap/header/simplehandler.rb'
5
+ require 'xsd/qname.rb'
6
+
7
+ module Amazon
8
+ module WebServices
9
+ module Util
10
+
11
+ class SOAPTransportHeaderHandler < SOAP::Header::SimpleHandler
12
+
13
+ def initialize(ns, tag, value)
14
+ super(XSD::QName.new(ns, tag))
15
+ @tag = tag
16
+ @value = value
17
+ end
18
+
19
+ def on_simple_outbound
20
+ @value
21
+ end
22
+
23
+ end # SOAPTransportHeaderHandler
24
+
25
+ end # Amazon::WebServices::Util
26
+ end # Amazon::WebServices
27
+ end # Amazon
@@ -0,0 +1,27 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ # This exception is thrown when we don't know if a service call succeeded or not
9
+ class UnknownResultException < RuntimeError
10
+
11
+ attr_reader :method, :args, :exception
12
+
13
+ def initialize( exception, method, args={} )
14
+ @method = method
15
+ @args = args
16
+ @exception = exception
17
+ end
18
+
19
+ def to_s
20
+ "UnknownResultException: got #{@exception} calling #{@method}"
21
+ end
22
+
23
+ end
24
+
25
+ end # Amazon::WebServices::Util
26
+ end # Amazon::WebServices
27
+ end # Amazon
@@ -0,0 +1,55 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class ValidationException < RuntimeError
9
+
10
+ attr_reader :message, :description, :result
11
+
12
+ def initialize( result, message=nil )
13
+ @result = result
14
+
15
+ @message = [
16
+ message,
17
+ get_nested_key( result, :OperationRequest, :Errors, :Error, :Code ),
18
+ get_nested_key( result, :Request, :Errors, :Error, :Code ),
19
+ get_nested_key( result, :Errors, :Error, :Code ),
20
+ ].detect { |v| !v.nil? }
21
+
22
+ @description = [
23
+ get_nested_key( result, :OperationRequest, :Errors, :Error, :Message ),
24
+ get_nested_key( result, :Request, :Errors, :Error, :Message ),
25
+ get_nested_key( result, :Errors, :Error, :Message ),
26
+ ].detect { |v| !v.nil? }
27
+
28
+ end
29
+
30
+ def to_s
31
+ "ValidationException: #{message}"
32
+ end
33
+
34
+ private
35
+
36
+ def get_nested_key( hash, *keys )
37
+ return nil unless hash.kind_of?(Hash)
38
+ result = hash
39
+ if hash.key? keys.first
40
+ keys.each do |key|
41
+ return nil unless result.kind_of?(Hash)
42
+ result = result[key]
43
+ end
44
+ return result
45
+ else
46
+ nested = hash.collect { |k,v| get_nested_key( v, *keys ) }
47
+ return ([nested].flatten - [nil]).first
48
+ end
49
+ end
50
+
51
+ end
52
+
53
+ end # Amazon::WebServices::Util
54
+ end # Amazon::WebServices
55
+ end # Amazon
@@ -0,0 +1,61 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'rexml/document'
5
+
6
+ module Amazon
7
+ module WebServices
8
+ module Util
9
+
10
+ class XMLSimplifier
11
+
12
+ # simplify(xml) -- convert an xml document into a simple nested hash
13
+ def self.simplify(xml)
14
+ case xml.class.to_s
15
+ when /Text/
16
+ {}
17
+ when /Document/
18
+ xml.root.children.inject({}) {|data,child| self.merge( data, simplify(child) ) }
19
+ when /Element/
20
+ if xml.children.size > 1 || xml.children.first.class.to_s !~ /Text/
21
+ value = xml.children.inject({}) { |data,child| self.merge( data, simplify(child) ) }
22
+ { xml.name.to_sym => value }
23
+ else
24
+ str = xml.text
25
+ value = case str
26
+ when /^(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z$/
27
+ Time.gm($1,$2,$3,$4,$5,$6)
28
+ when /^-?\d+$/
29
+ if str.to_i.to_s == str
30
+ str.to_i
31
+ else
32
+ str
33
+ end
34
+ when /^-?\d+\.\d+$/
35
+ str.to_f
36
+ else
37
+ str
38
+ end
39
+ { xml.name.to_sym => value }
40
+ end
41
+ else
42
+ raise "XMLSimplifier -- failed to simplify: #{xml.inspect}"
43
+ end
44
+ end
45
+
46
+ def self.merge(hash1, hash2)
47
+ hash2.each_key { |key|
48
+ if hash1[key]
49
+ hash1[key] = [hash1[key], hash2[key]].flatten
50
+ else
51
+ hash1[key] = hash2[key]
52
+ end
53
+ }
54
+ hash1
55
+ end
56
+
57
+ end # Amazon::WebServices::Util::XMLSimplifier
58
+
59
+ end # Amazon::WebServices::Util
60
+ end # Amazon::WebServices
61
+ end # Amazon
@@ -0,0 +1,19 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'mturk/version'
5
+
6
+ module MTurk
7
+
8
+ def self.agent(software_name="")
9
+ version = "ruby-mturk/#{MTurk::VERSION}"
10
+ if software_name.to_s == ""
11
+ version
12
+ else
13
+ "#{version} #{software_name}"
14
+ end
15
+ end
16
+
17
+ end
18
+
19
+ require 'amazon/webservices/mechanical_turk_requester'
@@ -0,0 +1,6 @@
1
+ # Copyright:: Copyright (c) 2007-2015 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module MTurk
5
+ VERSION = '1.8.1'.freeze
6
+ end
@@ -0,0 +1 @@
1
+ find test/ -iname '*.rb' | xargs rcov -Ilib -t
@@ -0,0 +1,61 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
4
+ # License:: Apache License, Version 2.0
5
+
6
+ begin ; require 'rubygems' ; rescue LoadError ; end
7
+
8
+ # The Best Image sample application will create a HIT asking a worker
9
+ # to choose the best of three images, given a set of criteria.
10
+ #
11
+ # The following concepts are covered:
12
+ # - Using the <FormattedContent> functionality in QuestionForm
13
+ # - File-based QuestionForm and HIT properties HIT loading
14
+ # - Using a basic system qualification
15
+
16
+ require 'mturk'
17
+ @mturk = Amazon::WebServices::MechanicalTurkRequester.new :Host => :Sandbox
18
+
19
+ # Use this line instead if you want to talk to Prod
20
+ #@mturk = Amazon::WebServices::MechanicalTurkRequester.new :Host => :Production
21
+
22
+
23
+ # Check to see if your account has sufficient funds
24
+ def hasEnoughFunds?
25
+ available = @mturk.availableFunds
26
+ puts "Got account balance: %.2f" % available
27
+ return available > 0.055
28
+ end
29
+
30
+ def getHITUrl( hitTypeId )
31
+ if @mturk.host =~ /sandbox/
32
+ "http://workersandbox.mturk.com/mturk/preview?groupId=#{hitTypeId}" # Sandbox Url
33
+ else
34
+ "http://mturk.com/mturk/preview?groupId=#{hitTypeId}" # Production Url
35
+ end
36
+ end
37
+
38
+ # Create the BestImage HIT
39
+ def createBestImage
40
+
41
+ # Defining the location of the file containing the QuestionForm and the properties of the HIT
42
+ rootDir = File.dirname $0;
43
+ questionFile = rootDir + "/best_image.question";
44
+ propertiesFile = rootDir + "/best_image.properties";
45
+
46
+ # Loading configuration properties from a HIT properties file.
47
+ # In this sample, the qualification is defined in the properties file.
48
+ props = Amazon::Util::DataReader.load( propertiesFile, :Properties )
49
+ props[:Reward] = { :Amount => 0.05, :CurrencyCode => 'USD'}
50
+ # Loading the question (QuestionForm) file.
51
+ question = File.read( questionFile )
52
+ # no validation
53
+ result = @mturk.createHIT( {:Question => question}.merge(props) )
54
+ puts "Created HIT: #{result[:HITId]}"
55
+ puts "Url: #{getHITUrl( result[:HITTypeId] )}"
56
+
57
+ # save the HIT Id to a file so we don't lose it...
58
+ Amazon::Util::DataReader.save( File.join( rootDir, "hits_created" ), [{:HITId => result[:HITId] }], :Tabular )
59
+ end
60
+
61
+ createBestImage if hasEnoughFunds?