typingpool 0.7.0

data/Rakefile

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'rake'
+require 'rake/testtask'
+
+task :default => [:test]
+
+desc "Run all tests"
+task :test => [:test_unit, :test_integration]
+
+desc "Run unit tests"
+Rake::TestTask.new('test_unit') do |t|
+  t.test_files = FileList[
+                          'test/test_unit*'
+                         ]
+end
+
+desc "Run integration tests"
+Rake::TestTask.new('test_integration') do |t|
+  t.test_files = FileList[ 
+                          (1..6).map{|n| "test/test_integration_script_#{n}*" }
+                         ]
+end

data/bin/tp-assign

@@ -0,0 +1,240 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'typingpool'
+require 'highline/import'
+include Typingpool::App::FriendlyExceptions
+include Typingpool::App::CLI::Formatter
+
+options = {
+  :keyword => [],
+  :qualify => []
+}
+OptionParser.new do |opts|
+  options[:banner] = "USAGE: #{File.basename($PROGRAM_NAME)} PROJECT TEMPLATE [--reward 0.75]\n"
+  options[:banner] += "  [--deadline 3h] [--lifetime 2d] [--approval 1d] \n"
+  options[:banner] += "  [--qualify 'approval_rate >= 95' --qualify 'hits_approved > 10'...]\n"
+  options[:banner] += "  [--keyword transcription --keyword mp3...]\n"
+  options[:banner] += "  [--confirm] [--sandbox] [--config PATH]\n"
+  opts.banner = options[:banner]
+
+  opts.on('--project=PROJECT',
+          "Required. Path or name within", 
+          "$config_file:transcripts. Also accepted as",
+          "first argument to script or via STDIN") do |project|
+    options[:project] = project
+  end
+
+  opts.on('--template=TEMPLATE',
+          "Required. Path or name within dir",
+          "$config_file:app/templates/assign. Also", 
+          "accepted as first argument to script") do |template|
+    options[:template] = template
+  end
+
+  opts.on('--reward=DOLLARS',
+          "Default: $config_file:assign:reward.",
+          "Always in USD (Amazon only supports USD).",
+          "Per chunk. Format N.NN") do |reward|
+    options[:reward] = reward
+  end
+
+  opts.on('--keyword=WORD', 
+          "Default: $config_file:assign:keywords.",
+          "Repeatable") do |keyword|
+    options[:keyword].push(keyword)
+  end
+
+  timespec = 'N[.N]y|M(onths)|d|h|m(inutes)|s'
+
+  opts.on('--deadline=TIMESPEC',
+          'Default: $config:assign:deadline. Worker',
+          "time to transcribe.",
+          timespec) do |timespec|
+    options[:deadline] = timespec
+  end
+
+  opts.on('--lifetime=TIMESPEC',
+          'Default: $config:assign:lifetime.',
+          'Assignment time to expire.',
+          timespec) do |timespec|
+    options[:lifetime] = timespec
+  end
+
+  opts.on('--approval=TIMESPEC',
+          'Default: $config:assign:approval.',
+          'Submission time to auto approve.',
+          timespec) do |timespec|
+    options[:approval] = timespec
+  end
+
+  opts.on('--qualify=QUALIFICATION',
+          "Default: $config:assign:qualify.",
+          "Repeatable.",
+          "An RTurk::Qualifications::TYPES +",
+          ">|<|==|!=|true|exists|>=|<= [+ INT]") do |qualification|
+    options[:qualify].push(qualification)
+  end
+
+  opts.on('--confirm',
+          "Confirm the total cost of the assignments",
+          "before assigning even if",
+          "$config:assign:confirm is set to 'no'") do
+    options[:confirm] = true
+  end
+
+  opts.on('--sandbox',
+          "Test in Mechanical Turk's sandbox") do
+    options[:sandbox] = true
+  end
+
+  opts.on('--config=PATH',
+          'Default: ~/.typingpool') do |path|
+    options[:config] = path
+  end
+
+  opts.on('--help',
+          'Display this screen') do
+    STDERR.puts opts
+    exit
+  end
+end.parse!
+
+config = Typingpool::App::CLI.config_from_arg(options[:config]) or abort "No config file at '#{options[:config]}'"
+
+if options[:keyword].count > 0
+  config.assign.keywords = []
+  config.assign.keywords.push(*options[:keyword])
+end
+
+[:reward, :deadline, :lifetime, :approval].each do |param|
+  with_friendly_exceptions("--#{param} argument", options[param]) do
+    config.assign.send("#{param.to_s}=", options[param]) if options[param]
+  end
+end #[:deadline, :lifetime, :approval].each...
+
+if options[:qualify].count > 0
+  config.assign['qualify'] = []
+  options[:qualify].each do |qualification|
+    with_friendly_exceptions('--qualify argument', qualification) do
+      config.assign.add_qualification(qualification)
+    end
+  end #options[:qualify].each...
+end #if options[:qualify].count > 0 
+
+options[:banner] += "\n#{Typingpool::App::CLI.help_arg_explanation}\n"
+
+positional = %w(project template)
+if Typingpool::Utility.stdin_has_content? && project = STDIN.gets
+  project.chomp!
+  abort "Duplicate project values (STDIN and --project)" if options[:project]
+  options[:project] = project
+  positional.shift
+end
+
+positional.each do |name|
+  arg = ARGV.shift
+  if options[name.to_sym]
+    abort "Duplicate values for #{name} (argument and --#{name})" if arg
+  else
+    options[name.to_sym] = arg or abort "Missing required arg #{name}\n\n#{options[:banner]}"
+  end
+end
+abort "Unexpected argument(s): #{ARGV.join(';')}" if ARGV.count > 0
+
+project = Typingpool::App::CLI.project_from_arg_and_config(options[:project], config)
+
+begin
+  template = Typingpool::Template::Assignment.from_config(options[:template], config)
+rescue Typingpool::Error::File::NotExists => e
+  abort "Couldn't find the template dir in your config file: #{e}"
+rescue Typingpool::Error => e
+  abort "Couldn't find your template: #{e}"
+end
+
+#always upload assignment html (can't re-use old ones because params
+#may have changed, affecting html output) #
+STDERR.puts "Figuring out what needs to be assigned"
+assignments = Typingpool::App.assignments_file_for_sandbox_status(options[:sandbox], project)
+needed_assignments = []
+unneeded_assignments = {
+  :complete => 0,
+  :outstanding => 0
+}
+assignments.each do |assignment|
+  if assignment['transcript']
+    unneeded_assignments[:complete] += 1
+    next
+  end
+  if assignment['hit_expires_at'].to_s.match(/\S/) #has been assigned previously
+    if ((Time.parse(assignment['hit_expires_at']) + assignment['hit_assignments_duration'].to_i) > Time.now)
+      #unexpired active HIT - do not reassign
+      unneeded_assignments[:outstanding] += 1
+      next
+    end
+  end
+  needed_assignments << assignment
+end
+STDERR.puts "#{assignments.count} assignments total"
+STDERR.puts "#{unneeded_assignments[:complete]} assignments completed" if unneeded_assignments[:complete] > 0
+STDERR.puts "#{unneeded_assignments[:outstanding]} assignments outstanding" if unneeded_assignments[:outstanding] > 0
+if needed_assignments.empty?
+  STDERR.puts "Nothing to assign"
+  exit
+end
+STDERR.puts "#{needed_assignments.count} assignments to assign"
+Typingpool::Amazon.setup(:sandbox => options[:sandbox], :config => config)
+
+#Are there enough funds for this assignment?
+cost = 0
+cost_string = '0.00'
+unless options[:sandbox]
+  STDERR.puts "Checking your balance"
+  balance = RTurk.GetAccountBalance.amount
+  #"* 1.1" reflects 10% Amazon service surcharge
+  cost = (needed_assignments.count * options[:reward].to_f * 1.1).round(2)
+  cost_string = sprintf("%.2f", cost)
+  abort "Anticipated assignment cost of $#{cost_string} would exceed available balance of $#{balance}" if cost > balance
+  if options[:confirm] || config.assign.confirm
+    begin
+      input = ask("Assignment cost is $#{cost_string}. Proceed? [#{cli_bold('y')}es]/#{cli_bold('n')}o:") 
+    end until input.to_s.match(/(^y)|(^n)|(^\s*$)/i)
+    exit if input.match(/^n/i)
+  end
+end
+
+#we'll need to re-upload audio if we ran tp-finish on the project
+#or if our last attempt to upload audio failed partway through
+Typingpool::App.upload_audio_for_project(project) do |file, as|
+  puts "Uploading #{File.basename(file)} to #{project.remote.host}/#{project.remote.path} as #{as}"
+end
+
+#Delete old assignment html for assignments to be re-assigned. We
+#can't re-use old assignment HTML because new params (e.g. reward)
+#might cause the new HTML to be different
+STDERR.puts "Deleting old assignment HTML from #{project.remote.host}"
+Typingpool::App.updelete_assignment_assets(project, assignments, needed_assignments, ['assignment'])
+
+STDERR.puts "Uploading assignment HTML to #{project.remote.host}"
+urls = Typingpool::App.upload_html_for_project_assignments(project, assignments, needed_assignments, template)
+
+STDERR.puts (options[:sandbox] ? 'Assigning (in sandbox)' : 'Assigning')
+needed_assignments.each_with_index do |assignment, i|
+  question = Typingpool::Amazon::Question.new(urls[i], template.render(assignment))
+  begin
+    hit = Typingpool::Amazon::HIT.create(question, config.assign)
+    Typingpool::App.record_assigned_hits_in_assignments_file(assignments, [hit])
+  rescue RTurk::RTurkError => e
+    goodbye = "Mechanical Turk error: #{e}\n\n"
+    if i == 0
+      goodbye += "To retry, run tp-assign again with the same arguments."
+    else
+      goodbye += "To cancel #{i} successful assignments, run `tp-finish '#{options[:project]}'`.\n\n"
+      goodbye += "To try and assign remaining #{neeeded_assignments.count - i} jobs, run tp-assign again with the same arguments."
+    end #if i == 0
+    abort goodbye
+  end #begin
+  STDERR.puts "  Assigned transcription job for '#{project.class.local_basename_from_url(assignment['audio_url'])}'"
+end #needed_assignments.each_with_index do...
+STDERR.puts "Assigned #{needed_assignments.count} transcription jobs for $#{cost_string}"
+STDERR.puts "Remaining balance: $#{RTurk.GetAccountBalance.amount}" unless options[:sandbox]

data/bin/tp-collect

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+require 'typingpool'
+require 'optparse'
+
+options = {}
+OptionParser.new do |commands|
+  commands.banner =  "USAGE: #{File.basename($PROGRAM_NAME)} [--config PATH] [--sandbox]\n"
+  commands.on('--sandbox',
+              "Collect from the Mechanical Turk test sandbox") do
+    options[:sandbox] = true
+  end
+  commands.on('--config=PATH',
+              "Default: ~/.typingpool",
+              "A config file") do |path|
+    options[:config] = path
+  end
+  commands.on('--fixture=PATH',
+              "Optional. For testing purposes only.",
+              "A VCR ficture for running with mock data.") do |fixture|
+    options[:fixture] = fixture
+  end
+  commands.on('--help',
+              "Display this screen") do
+    STDERR.puts commands
+    exit
+  end
+end.parse!
+
+config = Typingpool::App::CLI.config_from_arg(options[:config]) or abort "No config file at '#{options[:config]}'"
+
+if options[:fixture]
+  Typingpool::App.vcr_record(options[:fixture], config)
+end
+
+STDERR.puts "Collecting results from Amazon"
+Typingpool::Amazon.setup(:sandbox => options[:sandbox], :config => config)
+hits = Typingpool::Amazon::HIT.all_approved
+
+STDERR.puts "Looking for local project folders to receive results" unless hits.empty?
+Typingpool::App.find_projects_waiting_for_hits(hits, config) do |project, hits|
+  assignments_file = Typingpool::App.assignments_file_for_sandbox_status(options[:sandbox], project)
+  Typingpool::App.record_approved_hits_in_assignments_file(assignments_file, hits)
+  out_file = Typingpool::App.create_transcript(project, assignments_file)
+  STDERR.puts "Wrote #{out_file} to local folder #{project.name}."
+end
+
+if options[:fixture]
+  Typingpool::App.vcr_stop
+end

data/bin/tp-config

@@ -0,0 +1,114 @@
+#!/usr/bin/env ruby
+
+require 'typingpool'
+require 'highline'
+require 'highline/import'
+require 'securerandom'
+require 'fileutils'
+require 'optparse'
+require 'open3'
+
+options = {
+  :testing => false
+}
+OptionParser.new do |opts|
+  opts.banner = "USAGE: #{File.basename($PROGRAM_NAME)} [CONFIGFILE=#{Typingpool::Config.default_file}]\n\n"
+  opts.banner += "Installs or updates a Typingpool config file, prompting the user for\n"
+  opts.banner += "the minimal information to get up and running."
+  opts.on('--help',
+          'Display this screen.') do
+    puts opts
+    exit
+  end
+  opts.on('--test',
+          "Used by automated tests. Ignore.") do
+    options[:testing] = true
+  end
+end.parse!
+
+config_path = ARGV.first || Typingpool::Config.default_file
+config_path_full = File.expand_path(config_path)
+config = nil
+if File.exists? config_path_full
+  abort "Not a file: #{config_path}" unless File.file? config_path_full
+  STDERR.puts "Editing existing config file '#{config_path}'"
+  begin
+    config = Typingpool::Config.file(config_path_full) 
+  rescue ArgumentError, Psych::SyntaxError
+    abort "The specified config file is not valid YAML"
+  end #begin
+else
+  abort "Invalid path '#{config_path}'" unless File.dirname(config_path_full) && File.directory?(File.dirname(config_path_full))
+  STDERR.puts "Making a new config file at '#{config_path}'"
+  config = Typingpool::Config.from_bundled_template
+end #if File.exists? config_path
+
+config.amazon ||= {}
+config.amazon.key = ask('Your Amazon Web Services "Access Key ID"? '){|q| q.default = config.amazon.key if config.amazon.key }.to_s.chomp
+abort "Cannot proceed without an Amazon Access Key ID" if config.amazon.key.empty?
+config.amazon.secret = ask('Your Amazon Web Services "Secret Access Key"? '){|q| q.default = config.amazon.secret if config.amazon.secret }.to_s.chomp
+abort "Cannot proceed without an Amazon Secret Access Key" if config.amazon.secret.empty?
+
+unless options[:testing]
+  begin
+    RTurk.setup(config.amazon.key, config.amazon.secret, :sandbox => true)
+    RTurk.GetAccountBalance
+    STDERR.puts "Verified your new Amazon credentials"
+  rescue RTurk::InvalidRequest
+    abort "Your Amazon credentials do not appear to work. Please check them and run #{File.basename($PROGRAM_NAME)} again."
+  end #begin
+end
+
+if not(config.amazon.bucket || (config.sftp && config.sftp.user))
+  config.amazon.bucket = ['typingpool', SecureRandom.hex(8)].join('-')
+end
+
+unless config.transcripts
+  desktop_path = File.expand_path(File.join('~', 'Desktop'))
+  if File.exists?(desktop_path) && File.directory?(desktop_path)
+    config.transcripts = File.join(desktop_path, 'Transcripts')
+  else
+    config.transcripts = File.join('~', 'transcripts')
+  end 
+end
+
+transcripts = nil
+loop do
+  transcripts = ask('Working directory/folder for transcripts? '){|q| q.default = config['transcripts'] }.to_s.chomp
+  abort "Cannot proceed without a transcripts directory" if transcripts.empty?
+  if File.exists?(File.expand_path(transcripts))
+    if File.directory?(File.expand_path(transcripts))
+      break
+    else
+      STDERR.puts "Location #{transcripts} already taken by a file"
+    end #if File.directory?...
+  else
+    FileUtils.mkdir(File.expand_path(transcripts))
+    break
+  end #if File.exists?...
+end #loop do
+config.transcripts = transcripts
+
+unless config.templates
+  transcripts_dir = File.basename(File.expand_path(config.transcripts))
+  transcripts_dir_capitalized = (transcripts_dir[0].upcase == transcripts_dir[0])
+  templates =  transcripts_dir_capitalized ? 'Templates' : 'templates'
+  config.templates = File.join(config['transcripts'], templates)
+  FileUtils.mkdir(config.templates) unless File.exists? config.templates
+end
+
+unless config.cache
+  config.cache = File.join('~', '.typingpool.cache')
+end
+
+File.open(config_path_full, 'w') do |out|
+  out << YAML.dump(config.to_hash)
+end
+
+STDERR.puts "Successfully wrote config to '#{config_path}'."
+
+Typingpool::App.if_missing_dependencies do |missing|
+  missing.map!{|cmd| "`#{cmd}`" }
+  them = missing.count > 1 ? 'them' : 'it'
+  STDERR.puts "By the way, it looks like you're missing #{Typingpool::Utility.join_in_english(missing)}. You'll need to install #{them} before Typingpool can run."
+end

data/bin/tp-finish

@@ -0,0 +1,101 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'typingpool'
+require 'fileutils'
+
+options = {}
+OptionParser.new do |commands|
+  options[:banner] =  "USAGE: #{File.basename($PROGRAM_NAME)} PROJECT | --dead\n"
+  options[:banner] += "  [--sandbox]\n  [--config PATH]\n"
+  commands.banner = options[:banner]
+  commands.on('--dead',
+              "Remove ALL expired and rejected results, regardless of project.",
+              "Use in leiu of PROJECT.",
+              "Removes only Mechanical Turk HITs, not remote audio files") do
+    options[:dead] = true
+  end
+  commands.on('--sandbox',
+              "Test in Mechanical Turk's sandbox") do
+    options[:sandbox] = true
+  end
+  commands.on('--config=PATH',
+              "Default: #{Typingpool::Config.default_file}.",
+              "A config file") do |path|
+    options[:config] = path
+  end
+  commands.on('--help',
+              'Display this screen') do
+    STDERR.puts commands 
+    exit
+  end
+end.parse!
+options[:banner] += "\n#{Typingpool::App::CLI.help_arg_explanation}\n"
+config = Typingpool::App::CLI.config_from_arg(options[:config]) or abort "No config file at '#{options[:config]}'"
+
+options[:project] = ARGV.shift
+abort "Can't specify both PROJECT and --dead" if options[:project] && options[:dead]
+abort "No PROJECT specified (and no --dead option)\n#{options[:banner]}" unless (options[:project] || options[:dead])
+
+project=nil
+assignments=nil
+if options[:project]
+  project = Typingpool::App::CLI.project_from_arg_and_config(options[:project], config)
+  project.local.id or abort "Can't find project id in #{project.local.path}"
+  assignments = Typingpool::App.assignments_file_for_sandbox_status(options[:sandbox], project) 
+end
+
+Typingpool::Amazon.setup(:sandbox => options[:sandbox], :config => config)
+STDERR.puts "Removing from Amazon"
+STDERR.puts "  Collecting all results"
+#Set up result set, depending on options
+results = if project
+            Typingpool::Amazon::HIT.all_for_project(project.local.id)
+          elsif options[:dead]
+            Typingpool::Amazon::HIT.all.select do |result|
+              dead = ((result.full.expired_and_overdue? || result.rejected?) && result.ours?)
+              result.to_cache
+              dead
+            end
+          end
+
+#Remove the results from Mechanical Turk
+fails = []
+results.each do |result| 
+  STDERR.puts "  Removing HIT #{result.id} (#{result.full.status})"
+  begin
+    result.remove_from_amazon 
+  rescue Typingpool::Error::Amazon::UnreviewedContent => e
+    fails.push(e)
+  else
+    if project
+      STDERR.puts "  Removing from data/assignment.csv"
+      Typingpool::App.unrecord_hits_details_in_assignments_file(assignments, [result])  
+    end
+    STDERR.puts "  Removing from local cache"
+    Typingpool::Amazon::HIT.delete_cache(result.id)
+  end
+end
+if fails.count > 0
+  STDERR.puts "  Removed " + (results.size - fails.size) + " HITs from Amazon"
+  abort "#{fails.size} transcriptions are submitted but unprocessed (#{fails.join('; ')})"
+end
+#Remove the remote audio files associated with the results and update
+#one of the assignment.csvs associated with the project
+if project 
+  #Don't want to delete audio if there's another assignments file
+  #relying on it
+  delete_types = if (options[:sandbox] || (File.exists? project.local.file('data', 'sandbox-assignment.csv')))
+                   ['assignment']
+                 else
+                   ['audio', 'assignment']
+                 end #if options[:sandbox] ||...
+  deleted = Typingpool::App.updelete_assignment_assets(project, assignments, assignments, delete_types) do |file|
+    STDERR.puts "Removing #{file} from #{project.remote.host}"
+  end
+  if options[:sandbox]
+    #delete sandbox-assignment.csv so we know it's safe to delete
+    #audio files when we finish using regular assignment.csv
+    FileUtils.remove_entry_secure project.local.file('data', 'sandbox-assignment.csv')
+  end
+end #if project