mturk 1.8.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +2 -0
  4. data/.gemtest +0 -0
  5. data/History.md +105 -0
  6. data/LICENSE.txt +202 -0
  7. data/Manifest.txt +72 -0
  8. data/NOTICE.txt +4 -0
  9. data/README.md +100 -0
  10. data/Rakefile +33 -0
  11. data/bin/mturk +9 -0
  12. data/lib/amazon/util.rb +10 -0
  13. data/lib/amazon/util/binder.rb +48 -0
  14. data/lib/amazon/util/data_reader.rb +169 -0
  15. data/lib/amazon/util/filter_chain.rb +79 -0
  16. data/lib/amazon/util/hash_nesting.rb +93 -0
  17. data/lib/amazon/util/lazy_results.rb +59 -0
  18. data/lib/amazon/util/logging.rb +23 -0
  19. data/lib/amazon/util/paginated_iterator.rb +70 -0
  20. data/lib/amazon/util/proactive_results.rb +116 -0
  21. data/lib/amazon/util/threadpool.rb +129 -0
  22. data/lib/amazon/util/user_data_store.rb +100 -0
  23. data/lib/amazon/webservices/mechanical_turk.rb +123 -0
  24. data/lib/amazon/webservices/mechanical_turk_requester.rb +285 -0
  25. data/lib/amazon/webservices/mturk/mechanical_turk_error_handler.rb +153 -0
  26. data/lib/amazon/webservices/mturk/question_generator.rb +58 -0
  27. data/lib/amazon/webservices/util/amazon_authentication_relay.rb +72 -0
  28. data/lib/amazon/webservices/util/command_line.rb +155 -0
  29. data/lib/amazon/webservices/util/convenience_wrapper.rb +90 -0
  30. data/lib/amazon/webservices/util/filter_proxy.rb +45 -0
  31. data/lib/amazon/webservices/util/mock_transport.rb +70 -0
  32. data/lib/amazon/webservices/util/request_signer.rb +42 -0
  33. data/lib/amazon/webservices/util/rest_transport.rb +120 -0
  34. data/lib/amazon/webservices/util/soap_simplifier.rb +48 -0
  35. data/lib/amazon/webservices/util/soap_transport.rb +20 -0
  36. data/lib/amazon/webservices/util/soap_transport_header_handler.rb +27 -0
  37. data/lib/amazon/webservices/util/unknown_result_exception.rb +27 -0
  38. data/lib/amazon/webservices/util/validation_exception.rb +55 -0
  39. data/lib/amazon/webservices/util/xml_simplifier.rb +61 -0
  40. data/lib/mturk.rb +19 -0
  41. data/lib/mturk/version.rb +6 -0
  42. data/run_rcov.sh +1 -0
  43. data/samples/best_image/BestImage.rb +61 -0
  44. data/samples/best_image/best_image.properties +39 -0
  45. data/samples/best_image/best_image.question +82 -0
  46. data/samples/blank_slate/BlankSlate.rb +63 -0
  47. data/samples/blank_slate/BlankSlate_multithreaded.rb +67 -0
  48. data/samples/helloworld/MTurkHelloWorld.rb +56 -0
  49. data/samples/helloworld/mturk.yml +8 -0
  50. data/samples/review_policy/ReviewPolicy.rb +139 -0
  51. data/samples/review_policy/review_policy.question +30 -0
  52. data/samples/reviewer/Reviewer.rb +103 -0
  53. data/samples/reviewer/mturk.yml +8 -0
  54. data/samples/simple_survey/SimpleSurvey.rb +98 -0
  55. data/samples/simple_survey/simple_survey.question +30 -0
  56. data/samples/site_category/SiteCategory.rb +87 -0
  57. data/samples/site_category/externalpage.htm +71 -0
  58. data/samples/site_category/site_category.input +6 -0
  59. data/samples/site_category/site_category.properties +56 -0
  60. data/samples/site_category/site_category.question +9 -0
  61. data/test/mturk/test_changehittypeofhit.rb +130 -0
  62. data/test/mturk/test_error_handler.rb +403 -0
  63. data/test/mturk/test_mechanical_turk_requester.rb +178 -0
  64. data/test/mturk/test_mock_mechanical_turk_requester.rb +205 -0
  65. data/test/test_mturk.rb +21 -0
  66. data/test/unit/test_binder.rb +89 -0
  67. data/test/unit/test_data_reader.rb +135 -0
  68. data/test/unit/test_exceptions.rb +32 -0
  69. data/test/unit/test_hash_nesting.rb +99 -0
  70. data/test/unit/test_lazy_results.rb +89 -0
  71. data/test/unit/test_mock_transport.rb +132 -0
  72. data/test/unit/test_paginated_iterator.rb +58 -0
  73. data/test/unit/test_proactive_results.rb +108 -0
  74. data/test/unit/test_question_generator.rb +55 -0
  75. data/test/unit/test_threadpool.rb +50 -0
  76. data/test/unit/test_user_data_store.rb +80 -0
  77. metadata +225 -0
  78. metadata.gz.sig +0 -0
@@ -0,0 +1,42 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'base64'
5
+ require 'digest/sha1'
6
+
7
+ module Amazon
8
+ module WebServices
9
+ module Util
10
+
11
+ module RequestSigner
12
+
13
+ def RequestSigner.sign(service,method,time,key)
14
+ msg = "#{service}#{method}#{time}"
15
+ return hmac_sha1( key, msg )
16
+ end
17
+
18
+
19
+ private
20
+
21
+ def RequestSigner.hmac_sha1(key, s)
22
+ ipad = [].fill(0x36, 0, 64)
23
+ opad = [].fill(0x5C, 0, 64)
24
+ key = key.unpack("C*")
25
+ key += [].fill(0, 0, 64-key.length) if key.length < 64
26
+
27
+ inner = []
28
+ 64.times { |i| inner.push(key[i] ^ ipad[i]) }
29
+ inner += s.unpack("C*")
30
+
31
+ outer = []
32
+ 64.times { |i| outer.push(key[i] ^ opad[i]) }
33
+ outer = outer.pack("c*")
34
+ outer += Digest::SHA1.digest(inner.pack("c*"))
35
+
36
+ return Base64::encode64(Digest::SHA1.digest(outer)).chomp
37
+ end
38
+
39
+ end # Amazon::MTS::Util::RequestSigner
40
+ end # Amazon::MTS::Util
41
+ end # Amazon::MTS
42
+ end # Amazon
@@ -0,0 +1,120 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'cgi'
5
+ require 'net/https'
6
+ require 'rubygems'
7
+ require 'nokogiri'
8
+ require 'amazon/webservices/util/xml_simplifier'
9
+
10
+ module Amazon
11
+ module WebServices
12
+ module Util
13
+
14
+ class RESTTransport
15
+
16
+ REQUIRED_PARAMETERS = [:Endpoint]
17
+
18
+ def self.canPost?
19
+ Net::HTTP.respond_to? :post_form
20
+ end
21
+
22
+ def initialize( args )
23
+ missing_parameters = REQUIRED_PARAMETERS - args.keys
24
+ raise "Missing paramters: #{missing_parameters.join(',')}" unless missing_parameters.empty?
25
+ @uri = URI.parse( args[:Endpoint] )
26
+ @httpMethod = resolveHTTPMethod( args[:RestStyle] )
27
+ @version = args[:Version]
28
+ @ssl = (@uri.scheme == 'https') || (@uri.port == 443) || args[:UseSSL]
29
+ @skip_ssl_verify = args[:SkipSSLCheck]
30
+
31
+ agent = ::MTurk::agent( args[:SoftwareName] )
32
+ @headers = {
33
+ 'User-Agent' => agent,
34
+ 'X-Amazon-Software' => agent,
35
+ 'Content-Type' => 'application/x-www-form-urlencoded; charset=utf-8',
36
+ }
37
+ end
38
+
39
+ def resolveHTTPMethod( method )
40
+ case method.to_s.upcase
41
+ when "GET"
42
+ return :GET
43
+ when "POST"
44
+ raise "Your version of Ruby does not support HTTP Post" unless RESTTransport.canPost?
45
+ return :POST
46
+ else
47
+ return ( RESTTransport.canPost? ? :POST : :GET )
48
+ end
49
+ end
50
+
51
+ def method_missing( method, *args )
52
+ params = { :Operation => method, :Version => @version }
53
+ params.merge!( args[0].delete( :Request )[0] )
54
+ params.merge!( args[0] )
55
+
56
+ http = Net::HTTP.new( @uri.host, @uri.port )
57
+ if @ssl
58
+ http.use_ssl = true
59
+ if @skip_ssl_verify
60
+ http.verify_mode = OpenSSL::SSL::VERIFY_NONE
61
+ else
62
+ http.verify_mode = OpenSSL::SSL::VERIFY_PEER
63
+ end
64
+ end
65
+
66
+ req = nil
67
+ if @httpMethod == :GET
68
+ req = Net::HTTP::Get.new( @uri.request_uri + toQueryString(params), @headers )
69
+ else
70
+ req = Net::HTTP::Post.new( @uri.request_uri, @headers )
71
+ req.form_data = toPostParams( params )
72
+ req['Content-Type'] = @headers['Content-Type'] # necessary because req.form_data resets Content-Type header
73
+ end
74
+
75
+ res = http.start { |conn|
76
+ conn.request(req)
77
+ }.body
78
+
79
+ xml = Nokogiri::XML( res )
80
+ XMLSimplifier.simplify xml
81
+ end
82
+
83
+ private
84
+
85
+ def toQueryString(params)
86
+ queryString = ""
87
+ each_http_param(params) { |key,value|
88
+ queryString << ( '&' + key + '=' + CGI.escape(value) )
89
+ }
90
+ return queryString
91
+ end
92
+
93
+ def toPostParams(params)
94
+ postParams = {}
95
+ each_http_param(params) { |key,value|
96
+ postParams[key] = value }
97
+ return postParams
98
+ end
99
+
100
+ def each_http_param(params,&block)
101
+ params.each {|k,v| each_http_param_helper( k, v, false, &block ) unless v.nil? }
102
+ end
103
+
104
+ def each_http_param_helper(key,value,num=false,&block)
105
+ key = key.to_s
106
+ case value.class.to_s
107
+ when 'Array'
108
+ value.each_with_index { |v,i| each_http_param_helper( "#{key}.#{i+1}", v, true, &block ) unless v.nil? }
109
+ when 'Hash'
110
+ value.each { |k,v| each_http_param_helper( "#{key}#{num ? '.' : '.1.'}#{k}", v, false, &block ) unless v.nil? }
111
+ else
112
+ yield key, value.to_s
113
+ end
114
+ end
115
+
116
+ end # RESTTransport
117
+
118
+ end # Amazon::WebServices::Util
119
+ end # Amazon::WebServices
120
+ end # Amazon
@@ -0,0 +1,48 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class SOAPSimplifier
9
+
10
+ # simplify(item) -- convert a soap object into a simple nested hash
11
+ def self.simplify(item)
12
+ case item.class.to_s
13
+ when 'SOAP::Mapping::Object'
14
+ simple = {}
15
+ item.__xmlattr.each {|name,at| simple["*#{name}*"] = simplify(at)}
16
+ item.__xmlele.each { |element|
17
+ # element consists of a QName and a payload
18
+ name = element[0].name
19
+ payload = simplify(element[1])
20
+ simple[name.to_sym] = payload
21
+ }
22
+ simple
23
+ when 'Array'
24
+ item.collect {|i| simplify(i) }
25
+ else
26
+ str = item.to_s
27
+ case str
28
+ when /^(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z$/
29
+ Time.gm($1,$2,$3,$4,$5,$6)
30
+ when /^\d+$/
31
+ if str.to_i.to_s == str
32
+ str.to_i
33
+ else
34
+ str
35
+ end
36
+ when /^\d+\.\d+$/
37
+ str.to_f
38
+ else
39
+ str
40
+ end
41
+ end
42
+ end
43
+
44
+ end # Amazon::WebServices::Util::SoapSimplifier
45
+
46
+ end # Amazon::WebServices::Util
47
+ end # Amazon::WebServices
48
+ end # Amazon
@@ -0,0 +1,20 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class SOAPTransport
9
+
10
+ def initialize(args)
11
+ raise 'SOAP no longer supported'
12
+ end
13
+
14
+ def self.canSOAP? ; false ; end
15
+
16
+ end # SOAPTransport
17
+
18
+ end # Amazon::WebServices::Util
19
+ end # Amazon::WebServices
20
+ end # Amazon
@@ -0,0 +1,27 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'soap/header/simplehandler.rb'
5
+ require 'xsd/qname.rb'
6
+
7
+ module Amazon
8
+ module WebServices
9
+ module Util
10
+
11
+ class SOAPTransportHeaderHandler < SOAP::Header::SimpleHandler
12
+
13
+ def initialize(ns, tag, value)
14
+ super(XSD::QName.new(ns, tag))
15
+ @tag = tag
16
+ @value = value
17
+ end
18
+
19
+ def on_simple_outbound
20
+ @value
21
+ end
22
+
23
+ end # SOAPTransportHeaderHandler
24
+
25
+ end # Amazon::WebServices::Util
26
+ end # Amazon::WebServices
27
+ end # Amazon
@@ -0,0 +1,27 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ # This exception is thrown when we don't know if a service call succeeded or not
9
+ class UnknownResultException < RuntimeError
10
+
11
+ attr_reader :method, :args, :exception
12
+
13
+ def initialize( exception, method, args={} )
14
+ @method = method
15
+ @args = args
16
+ @exception = exception
17
+ end
18
+
19
+ def to_s
20
+ "UnknownResultException: got #{@exception} calling #{@method}"
21
+ end
22
+
23
+ end
24
+
25
+ end # Amazon::WebServices::Util
26
+ end # Amazon::WebServices
27
+ end # Amazon
@@ -0,0 +1,55 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module Amazon
5
+ module WebServices
6
+ module Util
7
+
8
+ class ValidationException < RuntimeError
9
+
10
+ attr_reader :message, :description, :result
11
+
12
+ def initialize( result, message=nil )
13
+ @result = result
14
+
15
+ @message = [
16
+ message,
17
+ get_nested_key( result, :OperationRequest, :Errors, :Error, :Code ),
18
+ get_nested_key( result, :Request, :Errors, :Error, :Code ),
19
+ get_nested_key( result, :Errors, :Error, :Code ),
20
+ ].detect { |v| !v.nil? }
21
+
22
+ @description = [
23
+ get_nested_key( result, :OperationRequest, :Errors, :Error, :Message ),
24
+ get_nested_key( result, :Request, :Errors, :Error, :Message ),
25
+ get_nested_key( result, :Errors, :Error, :Message ),
26
+ ].detect { |v| !v.nil? }
27
+
28
+ end
29
+
30
+ def to_s
31
+ "ValidationException: #{message}"
32
+ end
33
+
34
+ private
35
+
36
+ def get_nested_key( hash, *keys )
37
+ return nil unless hash.kind_of?(Hash)
38
+ result = hash
39
+ if hash.key? keys.first
40
+ keys.each do |key|
41
+ return nil unless result.kind_of?(Hash)
42
+ result = result[key]
43
+ end
44
+ return result
45
+ else
46
+ nested = hash.collect { |k,v| get_nested_key( v, *keys ) }
47
+ return ([nested].flatten - [nil]).first
48
+ end
49
+ end
50
+
51
+ end
52
+
53
+ end # Amazon::WebServices::Util
54
+ end # Amazon::WebServices
55
+ end # Amazon
@@ -0,0 +1,61 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'rexml/document'
5
+
6
+ module Amazon
7
+ module WebServices
8
+ module Util
9
+
10
+ class XMLSimplifier
11
+
12
+ # simplify(xml) -- convert an xml document into a simple nested hash
13
+ def self.simplify(xml)
14
+ case xml.class.to_s
15
+ when /Text/
16
+ {}
17
+ when /Document/
18
+ xml.root.children.inject({}) {|data,child| self.merge( data, simplify(child) ) }
19
+ when /Element/
20
+ if xml.children.size > 1 || xml.children.first.class.to_s !~ /Text/
21
+ value = xml.children.inject({}) { |data,child| self.merge( data, simplify(child) ) }
22
+ { xml.name.to_sym => value }
23
+ else
24
+ str = xml.text
25
+ value = case str
26
+ when /^(\d+)-(\d+)-(\d+)T(\d+):(\d+):(\d+)Z$/
27
+ Time.gm($1,$2,$3,$4,$5,$6)
28
+ when /^-?\d+$/
29
+ if str.to_i.to_s == str
30
+ str.to_i
31
+ else
32
+ str
33
+ end
34
+ when /^-?\d+\.\d+$/
35
+ str.to_f
36
+ else
37
+ str
38
+ end
39
+ { xml.name.to_sym => value }
40
+ end
41
+ else
42
+ raise "XMLSimplifier -- failed to simplify: #{xml.inspect}"
43
+ end
44
+ end
45
+
46
+ def self.merge(hash1, hash2)
47
+ hash2.each_key { |key|
48
+ if hash1[key]
49
+ hash1[key] = [hash1[key], hash2[key]].flatten
50
+ else
51
+ hash1[key] = hash2[key]
52
+ end
53
+ }
54
+ hash1
55
+ end
56
+
57
+ end # Amazon::WebServices::Util::XMLSimplifier
58
+
59
+ end # Amazon::WebServices::Util
60
+ end # Amazon::WebServices
61
+ end # Amazon
@@ -0,0 +1,19 @@
1
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ require 'mturk/version'
5
+
6
+ module MTurk
7
+
8
+ def self.agent(software_name="")
9
+ version = "ruby-mturk/#{MTurk::VERSION}"
10
+ if software_name.to_s == ""
11
+ version
12
+ else
13
+ "#{version} #{software_name}"
14
+ end
15
+ end
16
+
17
+ end
18
+
19
+ require 'amazon/webservices/mechanical_turk_requester'
@@ -0,0 +1,6 @@
1
+ # Copyright:: Copyright (c) 2007-2015 Amazon Technologies, Inc.
2
+ # License:: Apache License, Version 2.0
3
+
4
+ module MTurk
5
+ VERSION = '1.8.1'.freeze
6
+ end
@@ -0,0 +1 @@
1
+ find test/ -iname '*.rb' | xargs rcov -Ilib -t
@@ -0,0 +1,61 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
4
+ # License:: Apache License, Version 2.0
5
+
6
+ begin ; require 'rubygems' ; rescue LoadError ; end
7
+
8
+ # The Best Image sample application will create a HIT asking a worker
9
+ # to choose the best of three images, given a set of criteria.
10
+ #
11
+ # The following concepts are covered:
12
+ # - Using the <FormattedContent> functionality in QuestionForm
13
+ # - File-based QuestionForm and HIT properties HIT loading
14
+ # - Using a basic system qualification
15
+
16
+ require 'mturk'
17
+ @mturk = Amazon::WebServices::MechanicalTurkRequester.new :Host => :Sandbox
18
+
19
+ # Use this line instead if you want to talk to Prod
20
+ #@mturk = Amazon::WebServices::MechanicalTurkRequester.new :Host => :Production
21
+
22
+
23
+ # Check to see if your account has sufficient funds
24
+ def hasEnoughFunds?
25
+ available = @mturk.availableFunds
26
+ puts "Got account balance: %.2f" % available
27
+ return available > 0.055
28
+ end
29
+
30
+ def getHITUrl( hitTypeId )
31
+ if @mturk.host =~ /sandbox/
32
+ "http://workersandbox.mturk.com/mturk/preview?groupId=#{hitTypeId}" # Sandbox Url
33
+ else
34
+ "http://mturk.com/mturk/preview?groupId=#{hitTypeId}" # Production Url
35
+ end
36
+ end
37
+
38
+ # Create the BestImage HIT
39
+ def createBestImage
40
+
41
+ # Defining the location of the file containing the QuestionForm and the properties of the HIT
42
+ rootDir = File.dirname $0;
43
+ questionFile = rootDir + "/best_image.question";
44
+ propertiesFile = rootDir + "/best_image.properties";
45
+
46
+ # Loading configuration properties from a HIT properties file.
47
+ # In this sample, the qualification is defined in the properties file.
48
+ props = Amazon::Util::DataReader.load( propertiesFile, :Properties )
49
+ props[:Reward] = { :Amount => 0.05, :CurrencyCode => 'USD'}
50
+ # Loading the question (QuestionForm) file.
51
+ question = File.read( questionFile )
52
+ # no validation
53
+ result = @mturk.createHIT( {:Question => question}.merge(props) )
54
+ puts "Created HIT: #{result[:HITId]}"
55
+ puts "Url: #{getHITUrl( result[:HITTypeId] )}"
56
+
57
+ # save the HIT Id to a file so we don't lose it...
58
+ Amazon::Util::DataReader.save( File.join( rootDir, "hits_created" ), [{:HITId => result[:HITId] }], :Tabular )
59
+ end
60
+
61
+ createBestImage if hasEnoughFunds?