mturk 1.8.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (78) hide show
  1. checksums.yaml +7 -0
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +2 -0
  4. data/.gemtest +0 -0
  5. data/History.md +105 -0
  6. data/LICENSE.txt +202 -0
  7. data/Manifest.txt +72 -0
  8. data/NOTICE.txt +4 -0
  9. data/README.md +100 -0
  10. data/Rakefile +33 -0
  11. data/bin/mturk +9 -0
  12. data/lib/amazon/util.rb +10 -0
  13. data/lib/amazon/util/binder.rb +48 -0
  14. data/lib/amazon/util/data_reader.rb +169 -0
  15. data/lib/amazon/util/filter_chain.rb +79 -0
  16. data/lib/amazon/util/hash_nesting.rb +93 -0
  17. data/lib/amazon/util/lazy_results.rb +59 -0
  18. data/lib/amazon/util/logging.rb +23 -0
  19. data/lib/amazon/util/paginated_iterator.rb +70 -0
  20. data/lib/amazon/util/proactive_results.rb +116 -0
  21. data/lib/amazon/util/threadpool.rb +129 -0
  22. data/lib/amazon/util/user_data_store.rb +100 -0
  23. data/lib/amazon/webservices/mechanical_turk.rb +123 -0
  24. data/lib/amazon/webservices/mechanical_turk_requester.rb +285 -0
  25. data/lib/amazon/webservices/mturk/mechanical_turk_error_handler.rb +153 -0
  26. data/lib/amazon/webservices/mturk/question_generator.rb +58 -0
  27. data/lib/amazon/webservices/util/amazon_authentication_relay.rb +72 -0
  28. data/lib/amazon/webservices/util/command_line.rb +155 -0
  29. data/lib/amazon/webservices/util/convenience_wrapper.rb +90 -0
  30. data/lib/amazon/webservices/util/filter_proxy.rb +45 -0
  31. data/lib/amazon/webservices/util/mock_transport.rb +70 -0
  32. data/lib/amazon/webservices/util/request_signer.rb +42 -0
  33. data/lib/amazon/webservices/util/rest_transport.rb +120 -0
  34. data/lib/amazon/webservices/util/soap_simplifier.rb +48 -0
  35. data/lib/amazon/webservices/util/soap_transport.rb +20 -0
  36. data/lib/amazon/webservices/util/soap_transport_header_handler.rb +27 -0
  37. data/lib/amazon/webservices/util/unknown_result_exception.rb +27 -0
  38. data/lib/amazon/webservices/util/validation_exception.rb +55 -0
  39. data/lib/amazon/webservices/util/xml_simplifier.rb +61 -0
  40. data/lib/mturk.rb +19 -0
  41. data/lib/mturk/version.rb +6 -0
  42. data/run_rcov.sh +1 -0
  43. data/samples/best_image/BestImage.rb +61 -0
  44. data/samples/best_image/best_image.properties +39 -0
  45. data/samples/best_image/best_image.question +82 -0
  46. data/samples/blank_slate/BlankSlate.rb +63 -0
  47. data/samples/blank_slate/BlankSlate_multithreaded.rb +67 -0
  48. data/samples/helloworld/MTurkHelloWorld.rb +56 -0
  49. data/samples/helloworld/mturk.yml +8 -0
  50. data/samples/review_policy/ReviewPolicy.rb +139 -0
  51. data/samples/review_policy/review_policy.question +30 -0
  52. data/samples/reviewer/Reviewer.rb +103 -0
  53. data/samples/reviewer/mturk.yml +8 -0
  54. data/samples/simple_survey/SimpleSurvey.rb +98 -0
  55. data/samples/simple_survey/simple_survey.question +30 -0
  56. data/samples/site_category/SiteCategory.rb +87 -0
  57. data/samples/site_category/externalpage.htm +71 -0
  58. data/samples/site_category/site_category.input +6 -0
  59. data/samples/site_category/site_category.properties +56 -0
  60. data/samples/site_category/site_category.question +9 -0
  61. data/test/mturk/test_changehittypeofhit.rb +130 -0
  62. data/test/mturk/test_error_handler.rb +403 -0
  63. data/test/mturk/test_mechanical_turk_requester.rb +178 -0
  64. data/test/mturk/test_mock_mechanical_turk_requester.rb +205 -0
  65. data/test/test_mturk.rb +21 -0
  66. data/test/unit/test_binder.rb +89 -0
  67. data/test/unit/test_data_reader.rb +135 -0
  68. data/test/unit/test_exceptions.rb +32 -0
  69. data/test/unit/test_hash_nesting.rb +99 -0
  70. data/test/unit/test_lazy_results.rb +89 -0
  71. data/test/unit/test_mock_transport.rb +132 -0
  72. data/test/unit/test_paginated_iterator.rb +58 -0
  73. data/test/unit/test_proactive_results.rb +108 -0
  74. data/test/unit/test_question_generator.rb +55 -0
  75. data/test/unit/test_threadpool.rb +50 -0
  76. data/test/unit/test_user_data_store.rb +80 -0
  77. metadata +225 -0
  78. metadata.gz.sig +0 -0
@@ -0,0 +1,39 @@
1
+ ######################################
2
+ ## Basic HIT Properties
3
+ ######################################
4
+
5
+ Title:Select the best image for a given topic
6
+ Description:The task is to select one image out of three that best represents the provided topic given a set of evaluation criteria.
7
+ Keywords:sample, SDK, best, image
8
+ Reward.Amount:0
9
+ Reward.CurrencyCode:USD
10
+ MaxAssignments:1
11
+ RequesterAnnotation:sample#image
12
+
13
+ ######################################
14
+ ## HIT Timing Properties
15
+ ######################################
16
+
17
+ # this Assignment Duration value is 60 * 60 = 1 hour
18
+ AssignmentDurationInSeconds:3600
19
+
20
+ # this HIT Lifetime value is 60*60 = 1 hour
21
+ LifetimeInSeconds:3600
22
+
23
+ # this Auto Approval period is 60*60 = 1 hour
24
+ AutoApprovalDelayInSeconds:3600
25
+
26
+ ######################################
27
+ ## Qualification Properties
28
+ ######################################
29
+
30
+ # Qualifications can be defined in the properties file instead of in code.
31
+ # You can add multiple qualifications for this HIT by simply increasing the # suffix.
32
+ # i.e. qualification.2: XXXXX
33
+ # qualification.comparator.2:greaterthan
34
+
35
+ # this is a built-in qualification -- user must have an approval rate of 25% or greater
36
+ QualificationRequirement.1.QualificationTypeId:000000000000000000L0
37
+ QualificationRequirement.1.Comparator:GreaterThan
38
+ QualificationRequirement.1.IntegerValue:25
39
+ QualificationRequirement.1.RequiredToPreview:false
@@ -0,0 +1,82 @@
1
+ <?xml version="1.0"?>
2
+ <QuestionForm xmlns="http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd">
3
+ <Overview>
4
+ <FormattedContent><![CDATA[
5
+ <h1 align="center">Select the best image</h1>
6
+ <h2>Instructions</h2>
7
+ Your task is to select <strong>one</strong> image out of <strong>three</strong> that best represents the provided topic given the evaluation criteria defined below.
8
+ <br/>It is possible that none of the images can satisfy all the evaluation criteria. Your goal is to select the best image of the three.
9
+ <br/>
10
+ <h3>Evaluation Criteria</h3>
11
+ <ul>
12
+ <li>The image should be a representation of the provided topic (i.e. If the topic is Ferris Wheel, you should not select images that are not related to a Ferris Wheel.)</li>
13
+ <li>The image should be in focus. (i.e. not blurry)</li>
14
+ <li>The image should have good color balance. (i.e. not tinted)</li>
15
+ <li>The image should be properly exposed. (i.e. not overexposed/underexposed)</li>
16
+ <li>The image should be composed well. (i.e. not skewed)</li>
17
+ <li>Please refer to the following <a href="http://s3.amazonaws.com/mturk/samples/bestimage/best-image-instructions.html" target="_blank" >link</a> for visual examples of the above criteria.</li>
18
+ </ul>
19
+ <h2>Task</h2>
20
+ ]]></FormattedContent>
21
+ </Overview>
22
+ <Question>
23
+ <QuestionIdentifier>best_image</QuestionIdentifier>
24
+ <IsRequired>true</IsRequired>
25
+ <QuestionContent>
26
+ <FormattedContent><![CDATA[
27
+ <font size="4" color="darkblue" >Select the image below that best represents: Houses of Parliament, London, England</font>
28
+ ]]></FormattedContent>
29
+ </QuestionContent>
30
+ <AnswerSpecification>
31
+ <SelectionAnswer>
32
+ <StyleSuggestion>radiobutton</StyleSuggestion>
33
+ <Selections>
34
+ <Selection>
35
+ <SelectionIdentifier>sample-bestimage-image1.jpg</SelectionIdentifier>
36
+ <Binary>
37
+ <MimeType>
38
+ <Type>image</Type>
39
+ <SubType>jpg</SubType>
40
+ </MimeType>
41
+ <DataURL>http://s3.amazonaws.com/mturk/samples/bestimage/sample-bestimage-image1.jpg</DataURL>
42
+ <AltText>sample-bestimage-image1.jpg</AltText>
43
+ </Binary>
44
+ </Selection>
45
+ <Selection>
46
+ <SelectionIdentifier>sample-bestimage-image2.jpg</SelectionIdentifier>
47
+ <Binary>
48
+ <MimeType>
49
+ <Type>image</Type>
50
+ <SubType>jpg</SubType>
51
+ </MimeType>
52
+ <DataURL>http://s3.amazonaws.com/mturk/samples/bestimage/sample-bestimage-image2.jpg</DataURL>
53
+ <AltText>sample-bestimage-image2.jpg</AltText>
54
+ </Binary>
55
+ </Selection>
56
+ <Selection>
57
+ <SelectionIdentifier>sample-bestimage-image3.jpg</SelectionIdentifier>
58
+ <Binary>
59
+ <MimeType>
60
+ <Type>image</Type>
61
+ <SubType>jpg</SubType>
62
+ </MimeType>
63
+ <DataURL>http://s3.amazonaws.com/mturk/samples/bestimage/sample-bestimage-image3.jpg</DataURL>
64
+ <AltText>sample-bestimage-image3.jpg</AltText>
65
+ </Binary>
66
+ </Selection>
67
+ </Selections>
68
+ </SelectionAnswer>
69
+ </AnswerSpecification>
70
+ </Question>
71
+ <Question>
72
+ <QuestionIdentifier>comments</QuestionIdentifier>
73
+ <QuestionContent>
74
+ <Text>Please help us improve this HIT by including any Questions and/or Comments (optional):</Text>
75
+ </QuestionContent>
76
+ <AnswerSpecification>
77
+ <FreeTextAnswer>
78
+ <NumberOfLinesSuggestion>10</NumberOfLinesSuggestion>
79
+ </FreeTextAnswer>
80
+ </AnswerSpecification>
81
+ </Question>
82
+ </QuestionForm>
@@ -0,0 +1,63 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
4
+ # License:: Apache License, Version 2.0
5
+
6
+ begin ; require 'rubygems' ; rescue LoadError ; end
7
+
8
+ # The BlankSlate sample application disposes all of your HITs on sandbox
9
+
10
+ require 'mturk'
11
+ @mturk = Amazon::WebServices::MechanicalTurkRequester.new :Host => :Sandbox
12
+
13
+ def forceExpire(id)
14
+ print "Ensuring HIT #{id} is expired: "
15
+ begin
16
+ @mturk.forceExpireHIT( :HITId => id )
17
+ rescue => e
18
+ raise e unless e.message == 'AWS.MechanicalTurk.InvalidHITState'
19
+ end
20
+ puts "OK"
21
+ end
22
+
23
+ def approveRemainingAssignments(id)
24
+ print "Approving remaining assignments for HIT #{id}: "
25
+ count = 0
26
+ @mturk.getAssignmentsForHITAll( :HITId => id ).each do |assignment|
27
+ @mturk.approveAssignment :AssignmentId => assignment[:AssignmentId] if assignment[:AssignmentStatus] == 'Submitted'
28
+ count += 1
29
+ end
30
+ puts "OK (Approved #{count})"
31
+ end
32
+
33
+ def dispose(id)
34
+ print "Disposing HIT #{id}: "
35
+ @mturk.disposeHIT( :HITId => id )
36
+ puts "OK"
37
+ end
38
+
39
+ def purge
40
+ hit_ids = @mturk.searchHITsAll.collect {|hit| hit[:HITId] }
41
+ puts "Found #{hit_ids.size} HITs"
42
+
43
+ return false if hit_ids.size == 0
44
+
45
+ hit_ids.each do |id|
46
+ begin
47
+ forceExpire id
48
+ approveRemainingAssignments id
49
+ dispose id
50
+ rescue Exception => e
51
+ raise e if e.is_a? Interrupt
52
+ puts e.inspect
53
+ end
54
+ end
55
+
56
+ return true
57
+ end
58
+
59
+ while purge
60
+ puts 'Ensuring there are no more hits...'
61
+ end
62
+
63
+ puts 'You now have a blank slate'
@@ -0,0 +1,67 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
4
+ # License:: Apache License, Version 2.0
5
+
6
+ begin ; require 'rubygems' ; rescue LoadError ; end
7
+
8
+ # The BlankSlate sample application disposes all of your HITs on sandbox
9
+
10
+ require 'mturk'
11
+ @mturk = Amazon::WebServices::MechanicalTurkRequester.new :Host => :Sandbox
12
+ require 'amazon/util/threadpool'
13
+
14
+ def forceExpire(id)
15
+ @mturk.forceExpireHIT( :HITId => id )
16
+ rescue
17
+ end
18
+
19
+ def approveRemainingAssignments(id)
20
+ @mturk.getAssignmentsForHITAll( :HITId => id ).each do |assignment|
21
+ begin
22
+ @mturk.approveAssignment :AssignmentId => assignment[:AssignmentId] if assignment[:AssignmentStatus] == 'Submitted'
23
+ rescue
24
+ end
25
+ end
26
+ end
27
+
28
+ def dispose(id)
29
+ @mturk.disposeHIT( :HITId => id )
30
+ end
31
+
32
+ def purge
33
+ puts "*** starting purge ***"
34
+ hit_ids = @mturk.searchHITsAllProactive.collect {|hit| hit[:HITId] }
35
+ puts "Found #{hit_ids.size} HITs"
36
+
37
+ return false if hit_ids.size == 0
38
+
39
+ threadpool = Amazon::Util::ThreadPool.new(12)
40
+
41
+ puts "*** loading work ***"
42
+ hit_ids.each do |hid|
43
+ threadpool.addWork(hid) do |id|
44
+ begin
45
+ puts "starting #{id}"
46
+ forceExpire id
47
+ approveRemainingAssignments id
48
+ dispose id
49
+ puts "cleared #{id}"
50
+ rescue Exception => e
51
+ raise e if e.is_a? Interrupt
52
+ puts e.inspect
53
+ end
54
+ end
55
+ end
56
+ puts "*** finished adding to queue ***"
57
+
58
+ threadpool.finish
59
+
60
+ return true
61
+ end
62
+
63
+ while purge
64
+ puts 'Ensuring there are no more hits...'
65
+ end
66
+
67
+ puts 'You now have a blank slate'
@@ -0,0 +1,56 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
4
+ # License:: Apache License, Version 2.0
5
+
6
+ begin ; require 'rubygems' ; rescue LoadError ; end
7
+
8
+ # The MTurk Hello World sample application creates a simple HIT via the Mechanical Turk Ruby Libraries.
9
+
10
+ require 'mturk'
11
+ @mturk = Amazon::WebServices::MechanicalTurkRequester.new :Config => File.join( File.dirname(__FILE__), 'mturk.yml' )
12
+
13
+ require 'amazon/webservices/mturk/question_generator'
14
+ include Amazon::WebServices::MTurk
15
+
16
+ def createHelloWorld
17
+ title = "Answer a question"
18
+ desc = "This is a HIT created by the Amazon Mechanical Turk SDK for Ruby. Please answer the question."
19
+ keywords = "sample, SDK, hello"
20
+ numAssignments = 1
21
+ rewardAmount = 0.05 # 5 cents
22
+
23
+ question = QuestionGenerator.build(:Basic) do |q|
24
+ q.ask "What is the weather like right now in Seattle, WA?"
25
+ end
26
+
27
+ puts question
28
+
29
+ result = @mturk.createHIT( :Title => title,
30
+ :Description => desc,
31
+ :MaxAssignments => numAssignments,
32
+ :Reward => { :Amount => rewardAmount, :CurrencyCode => 'USD' },
33
+ :Question => question,
34
+ :Keywords => keywords )
35
+
36
+ puts "Created HIT: #{result[:HITId]}"
37
+ puts "Url: #{getHITUrl( result[:HITTypeId] )}"
38
+ end
39
+
40
+ def getHITUrl( hitTypeId )
41
+ if @mturk.host =~ /sandbox/
42
+ "http://workersandbox.mturk.com/mturk/preview?groupId=#{hitTypeId}" # Sandbox Url
43
+ else
44
+ "http://mturk.com/mturk/preview?groupId=#{hitTypeId}" # Production Url
45
+ end
46
+ end
47
+
48
+ # Check to see if your account has sufficient funds
49
+ def hasEnoughFunds?
50
+ available = @mturk.availableFunds
51
+ puts "Got account balance: %.2f" % available
52
+ return available > 0.055
53
+ end
54
+
55
+
56
+ createHelloWorld if hasEnoughFunds?
@@ -0,0 +1,8 @@
1
+ # Your AWSAccessKeyId ( leave commented to use global default )
2
+ # AWSAccessKeyId: <need access key id>
3
+
4
+ # Your AWSAccessKey ( leave commented to use global default )
5
+ # AWSAccessKey: <need access key>
6
+
7
+ # Host to talk to ( Prod or Sandbox )
8
+ Host: Sandbox
@@ -0,0 +1,139 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ # Copyright:: Copyright (c) 2007 Amazon Technologies, Inc.
4
+ # License:: Apache License, Version 2.0
5
+
6
+ begin ; require 'rubygems' ; rescue LoadError ; end
7
+
8
+ # The Review Policy sample application will create a HIT which utilizes review policies
9
+
10
+ require 'mturk'
11
+ @mturk = Amazon::WebServices::MechanicalTurkRequester.new
12
+
13
+ # Check to see if your account has sufficient funds
14
+ def hasEnoughFunds?
15
+ available = @mturk.availableFunds
16
+ puts "Got account balance: %.2f" % available
17
+ return available > 0.055
18
+ end
19
+
20
+ def getHITUrl( hitTypeId )
21
+ if @mturk.host =~ /sandbox/
22
+ "http://workersandbox.mturk.com/mturk/preview?groupId=#{hitTypeId}" # Sandbox Url
23
+ else
24
+ "http://mturk.com/mturk/preview?groupId=#{hitTypeId}" # Production Url
25
+ end
26
+ end
27
+
28
+ def questionXml
29
+ rootDir = File.dirname $0
30
+ questionFile = rootDir + "/review_policy.question"
31
+ question = File.read( questionFile )
32
+ end
33
+
34
+ # Helper function to convert policy parameters
35
+ def kv_map(hash)
36
+ hash.collect do |k,v|
37
+ case v
38
+ when Hash
39
+ { :Key => k, :MapEntry => kv_map(v) }
40
+ else
41
+ { :Key => k, :Value => v }
42
+ end
43
+ end
44
+ end
45
+
46
+ def createMyHit
47
+ scoreKnownAnswersPolicy = {
48
+ :PolicyName => 'ScoreMyKnownAnswers/2011-09-01',
49
+ :Parameter => kv_map({
50
+ 'ApproveIfKnownAnswerScoreIsAtLeast' => 100,
51
+ 'ApproveReason' => 'You can count',
52
+ 'RejectIfKnownAnswerScoreIsLessThan' => 50,
53
+ 'RejectReason' => 'You flunked math',
54
+ 'ExtendIfKnownAnswerScoreIsLessThan' => 50,
55
+ 'AnswerKey' => { 'q1' => 2, 'q2' => 4 }
56
+ }),
57
+ }
58
+
59
+ pluralityHitReviewPolicy = {
60
+ :PolicyName => 'SimplePlurality/2011-09-01',
61
+ :Parameter => kv_map({
62
+ 'QuestionIds' => ['q3'],
63
+ 'QuestionAgreementThreshold' => 49,
64
+ 'ExtendIfHITAgreementScoreIsLessThan' => 100,
65
+ 'ExtendAssignments' => 1,
66
+ 'ExtendMaximumAssignments' => 10,
67
+ 'ApproveIfWorkerAgreementScoreIsAtLeast' => 100,
68
+ 'RejectIfWorkerAgreementScoreIsLessThan' => 100,
69
+ }),
70
+ }
71
+
72
+ hit_properties = {
73
+ :Title => 'Answer some questions',
74
+ :Description => 'This is a HIT created by the Mechanical Turk SDK. Please answer the provided questions.',
75
+ :Keywords => '',
76
+ :Reward => {
77
+ :CurrencyCode => 'USD',
78
+ :Amount => 0.00
79
+ },
80
+ :RequesterAnnotation => 'Test Hit',
81
+ :AssignmentDurationInSeconds => 60 * 60,
82
+ :AutoApprovalDelayInSeconds => 60 * 60 * 10,
83
+ :MaxAssignments => 3, # review policy requires multiple assignments per hit
84
+ :LifetimeInSeconds => 60 * 60,
85
+ :Question => questionXml,
86
+ :AssignmentReviewPolicy => scoreKnownAnswersPolicy,
87
+ :HITReviewPolicy => pluralityHitReviewPolicy,
88
+ }
89
+ hit = @mturk.createHIT( hit_properties )
90
+
91
+ puts "Created HIT: #{hit[:HITId]}"
92
+ puts "Url: #{getHITUrl( hit[:HITTypeId] )}"
93
+ end
94
+
95
+ def displayPolicy(name,report)
96
+ puts "Results for policy #{name}:"
97
+ [report[:ReviewResult]].flatten.each do |result|
98
+ q = result[:Key]
99
+ q += " for #{result[:QuestionId]}" unless result[:QuestionId].to_s == ""
100
+ puts "- %s %s: %s is %s" % [result[:SubjectType],result[:SubjectId],q,result[:Value]]
101
+ end unless report[:ReviewResult].nil?
102
+ puts "Actions for policy #{name}:"
103
+ [report[:ReviewAction]].flatten.each do |action|
104
+ puts " - Action %s on %s %s: %s" % [action[:ActionName],action[:ObjectType],action[:ObjectId],action[:ActionStatus]]
105
+ puts " Result: #{action[:Result]}"
106
+ end unless report[:ReviewAction].nil?
107
+ end
108
+
109
+ def getMyHitResults(hitId)
110
+ puts "Getting review policy results for HIT #{hitId}"
111
+ rez = @mturk.getReviewResultsForHIT( :HITId => hitId,
112
+ :PolicyLevel => ['Assignment','HIT'],
113
+ :RetrieveActions => true,
114
+ :RetrieveResults => true,
115
+ :PageNumber => 1,
116
+ :PageSize => 1000 )
117
+ displayPolicy(rez[:HITReviewPolicy][:PolicyName],rez[:HITReviewReport]) unless rez[:HITReviewPolicy].nil?
118
+ displayPolicy(rez[:AssignmentReviewPolicy][:PolicyName],rez[:AssignmentReviewReport]) unless rez[:AssignmentReviewPolicy].nil?
119
+ end
120
+
121
+ def usage
122
+ puts <<EOF
123
+ Usage:
124
+ ReviewPolicy.rb
125
+ This will load a hit configured with sample review policies
126
+ ReviewPolicy.rb [HitId]
127
+ This will retrieve review policy information about the identified HIT
128
+ EOF
129
+ end
130
+
131
+
132
+ case ARGV.size
133
+ when 0
134
+ createMyHit if hasEnoughFunds?
135
+ when 1
136
+ getMyHitResults(ARGV[0])
137
+ else
138
+ usage
139
+ end