typingpool 0.7.0

data/bin/tp-make

@@ -0,0 +1,169 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'typingpool'
+require 'fileutils'
+require 'tempfile'
+require 'tmpdir'
+include Typingpool::App::FriendlyExceptions
+
+options = {
+  :files => [],
+  :voices => [],
+  :unusual => [],
+  :chunk => '1:00',
+}
+OptionParser.new do |opts|
+  options[:banner] = "USAGE: #{File.basename($PROGRAM_NAME)} PROJECTNAME FILE [FILE [FILE...]]\n"
+  options[:banner] += "  [--chunks 1:00] [--subtitle 'Hack Day interview']\n"
+  options[:banner] += "  [--voice 'John' --voice 'Pat Foo, British female'...]\n"
+  options[:banner] += "  [--unusual 'Hack Day' --unusual 'Sunnyvale, Chad Dickerson'...]\n"
+  options[:banner] += "  [--bitrate 256] [--config PATH]\n"
+  options[:banner] += "  [--title PROJECTNAME] [--file foo.mp3 [--file bar.mp3...]]\n"
+  opts.banner = options[:banner]
+
+  opts.on('--title TITLE', 
+          'Required. For file names and transcript.', 
+          'Also accepted as first argument') do |title|
+    options[:title] = title
+  end
+
+  opts.on('--file FILE', 
+          'Required. Audio for transcribing.', 
+          'Repeatable (sorting is by name).', 
+          'Also accepted as second and later arguments') do |file|
+    options[:files].push(file)
+  end
+
+  opts.on('--subtitle SUBTITLE',
+          'For transcript') do |subtitle|
+    options[:subtitle] = subtitle
+  end
+
+  opts.on('--chunks MM:SS', 
+          'Default: 1:00. Audio divided thusly for',
+          'transcribing. Try also HH:MM:SS and SSS') do |chunk|
+    options[:chunk] = chunk
+  end
+
+  opts.on('--voice "NAME[, DESCR]"', 
+          'Name, optional description of recorded', 
+          'person, to aid transcriber. Repeatable') do |voice|
+    options[:voices].push(voice)
+  end
+
+  opts.on('--unusual WORD[,WORD,]', 
+          'Unusual word within recording, to aid', 
+          'transcriber. Commas for multiple.',
+          'Repeatable') do |word|
+    options[:unusual].push(word)
+  end
+
+  opts.on('--config PATH',
+          'Default: ~/.typingpool. A config file') do |path|
+    options[:config] = path
+  end
+
+  opts.on('--bitrate KBPS',
+          'Default: Mirror input. Output bitrate in',
+          'kb/s. Only applies if/when converting to',
+          'MP3') do |kbps|
+    options[:bitrate] = kbps
+  end
+
+  opts.on('--help',
+          'Display this screen.') do
+    puts opts
+    exit
+  end
+end.parse!
+
+Typingpool::App.if_missing_dependencies do |missing|
+  missing.map!{|cmd| "`#{cmd}`" }
+  them = missing.count > 1 ? 'them' : 'it'
+  abort "It looks like you're missing #{Typingpool::Utility.join_in_english(missing)}. You'll need to install #{them} before Typingpool can run."
+end
+
+options[:title] ||= ARGV.shift if ARGV.count > 0
+options[:files].push(ARGV).flatten! if ARGV.count > 0
+options[:banner] += "\n#{Typingpool::App::CLI.help_arg_explanation}\n"
+abort "No files specified\n\n#{options[:banner]}" if options[:files].empty?
+abort "No title specified\n\n#{options[:banner]}" if options[:title].to_s.empty?
+options[:files].sort!
+options[:files].each do |file|
+  File.extname(file) or abort "You need a file extension on the file '#{file}'"
+  File.exists?(file) or abort "There is no file '#{file}'"
+  File.file?(file) or abort "Not a file: '#{file}'"
+end
+options[:unusual].map!{|unusual| unusual.split(/\s*,\s*/)}.flatten!
+options[:voices].map! do |voice| 
+  name, description = voice.split(/\s*,\s*/)
+  {
+    :name => name,
+    :description => (description || '')
+  }
+end
+
+config = Typingpool::App::CLI.config_from_arg(options[:config]) or abort "No config file at '#{options[:config]}'"
+
+project = with_friendly_exceptions('project title', options[:title]) do
+  Typingpool::Project.new(options[:title], config) 
+end
+
+with_friendly_exceptions('--chunk argument', options[:chunk]) do
+  project.interval = options[:chunk] if options[:chunk] 
+end
+
+with_friendly_exceptions('--bitrate argument', options[:bitrate]) do
+  project.bitrate = options[:bitrate] if options[:bitrate]
+end
+
+if project.local
+  if (File.exists?(project.local.file('data', 'assignment.csv')) &&
+      project.local.file('data', 'assignment.csv').as(:csv).read.select{|assignment| assignment['audio_uploaded'] == 'maybe' }.count > 0)
+    #project where the upload died partway through
+    STDERR.puts "Fixing incomplete project"
+    STDERR.puts "Determining which mp3s need uploading"
+  else
+    abort "The title '#{options[:title]}' is taken"
+  end #if(File.exists(project.local.file('data', 'assignment.csv') &&...
+else
+  project.create_local
+  project.local.subtitle = options[:subtitle] if options[:subtitle]
+  options[:files].each{|path| FileUtils.cp(path, project.local.subdir('audio', 'originals')) }
+
+  temp_conversion_dir = Typingpool::Filer::Dir.new(Dir.mktmpdir)
+  converted_files = project.local.subdir('audio', 'originals').as(:audio).to_mp3(temp_conversion_dir, project.bitrate) do |file| 
+    STDERR.puts "Converting #{File.basename(file) } to mp3" 
+  end
+
+  STDERR.puts "Merging audio" if converted_files.count > 1
+  temp_merge_file = Tempfile.new(["#{project.name}.all.", ".mp3"])
+  temp_merge_filer = Typingpool::Filer::Audio.new(temp_merge_file.path)
+  converted_files.merge(temp_merge_filer)
+
+  STDERR.puts "Splitting audio into uniform bits"
+  chunks = temp_merge_filer.split(project.interval_as_min_dot_sec, project.name, project.local.subdir('audio','chunks'))
+  urls = project.create_remote_names(chunks).map{|file| project.remote.file_to_url(file) }
+
+  #Write mp3 URLs before uploading, so we can rollback if uploading
+  #dies halfway through
+  assignment_path = project.create_assignment_csv(:path => ['data', 'assignment.csv'], :urls => urls, :unusual => options[:unusual], :voices => options[:voices])
+
+  temp_merge_file.close
+  temp_merge_file.unlink
+  FileUtils.remove_entry_secure(temp_conversion_dir)
+end #if project.local
+
+Typingpool::App.upload_audio_for_project(project) do |file, as|
+  STDERR.puts "Uploading #{File.basename(file)} to #{project.remote.host}/#{project.remote.path} as #{as}"
+end
+
+if STDOUT.tty? && Typingpool::Utility.os_x?
+  STDERR.puts "Opening project folder #{project.local.path}"
+  project.local.finder_open 
+end
+
+STDERR.puts "Done. Project at:"
+puts project.local.path
+

data/bin/tp-review

@@ -0,0 +1,175 @@
+#!/usr/bin/env ruby
+
+require 'typingpool'
+require 'highline/import'
+require 'optparse'
+include Typingpool::App::CLI::Formatter
+
+options = {}
+OptionParser.new do |commands|
+  options[:banner] =  "USAGE: #{File.basename($PROGRAM_NAME) } [PROJECT]\n"
+  options[:banner] += "  [--config PATH] [--sandbox]\n"
+  commands.banner = options[:banner]
+
+  commands.on('--project=PROJECT',
+              'Path or name within dir $config_file:transcripts.',
+              'Also accepted as first argument to script.',
+              'If not specified, reviews pending results',
+              'for ALL projects') do |project|
+    options[:project] = project
+  end
+  commands.on('--config=PATH',
+              'Default: ~/.typingpool') do |path|
+    options[:config] = path
+  end
+  commands.on('--sandbox',
+              'Test in Mechanical Turk\'s sandbox') do |sandbox|
+    options[:sandbox] = true
+  end
+  commands.on('--fixture=PATH',
+              "Optional. For testing purposes only.",
+              "A VCR fixture for running with mock data") do |fixture|
+    options[:fixture] = fixture
+  end
+  commands.on('--help',
+              "Display this screen") do
+    STDERR.puts commands
+    exit
+  end
+end.parse!
+
+config = Typingpool::App::CLI.config_from_arg(options[:config]) or abort "No config file at '#{options[:config]}'"
+options[:banner] += "\n#{Typingpool::App::CLI.help_arg_explanation}\n"
+
+if ARGV.count > 0
+  if options[:project]
+    abort "Duplicate project values ('#{ARGV[0]}' and '#{options[:project]}')"
+  end
+  options[:project] = ARGV.shift
+end
+abort "Unexpected argument(s): #{ARGV.join(';')}" if ARGV.count > 0
+
+if options[:project]
+  options[:project] = Typingpool::App::CLI.project_from_arg_and_config(options[:project], config)
+end
+
+if options[:fixture]
+  Typingpool::App.vcr_record(options[:fixture], config)
+end
+
+Typingpool::Amazon.setup(:sandbox => options[:sandbox], :config => config)
+
+assignments = Typingpool::App.assignments_file_for_sandbox_status(options[:sandbox], options[:project]) if options[:project]
+STDERR.puts "Gathering submissions from Amazon"
+hits = if options[:project] && ((hit_ids = assignments.map{|assignment| assignment['hit_id'] }.select{|hit_id| hit_id }).count > 0)
+         Typingpool::Amazon::HIT.with_ids(hit_ids).select{|hit| hit.submitted? }
+       else
+         hits = Typingpool::Amazon::HIT.all_reviewable{|hit| hit.submitted? && hit.ours? }
+         if options[:project]
+           #code path for projects assigned through RUI
+           hits.select!{|hit| hit.project_id == options[:project].local.id }
+         end
+         hits
+       end
+
+STDERR.puts "Matching submissions with local projects"
+choices = %w(approve reject quit skip)
+catch :quitting do
+  project_hits = Typingpool::App.find_projects_waiting_for_hits(hits, config) 
+  all_hits = project_hits.map{|hash| hash[:hits] }.flatten
+  i = 0
+  project_hits.each do |hash|
+    hits = hash[:hits]
+    project = hash[:project]
+    assignments = Typingpool::App.assignments_file_for_sandbox_status(options[:sandbox], project)
+    hits.sort!{|a, b| a.url <=> b.url }
+    hits.each do |hit|
+      i += 1
+      say(cli_bold("\nTranscript for: ") + hit.url)
+      project_info = cli_bold("Project: ") + project.name
+      if project.local.subtitle
+        project_info += ": #{project.local.subtitle}"
+      end
+      say(project_info)
+      say(cli_bold("Submitted at: ") +
+          hit.assignment.submitted_at.localtime.to_s +
+          cli_bold(" by ") +
+          hit.assignment.worker_id)
+      say(hit.transcript.body_as_text(4, 68))
+      prompt = choices.map do |c| 
+        cli_reverse('(') +
+          cli_reverse(cli_bold(c.slice(0).upcase)) +
+          cli_reverse(")#{c.slice(1, c.size)}") 
+      end
+      prompt << cli_reverse('[') + prompt.pop + cli_reverse(']')
+      prompt = prompt.join(cli_reverse(', ')) 
+      prompt += cli_reverse('? ')
+      prompt += cli_reverse("(#{i}/#{all_hits.count}) ")
+
+      choice=nil
+      until choice
+        input = ask(prompt)
+        if input.to_s.match(/^\s*$/)
+          choice = choices.last
+        elsif not(choice = choices.detect{|possible| possible[0] == input.downcase[0] })
+          say("Invalid selection '#{input}'.")
+        end
+      end #until choice
+
+      case choice
+      when 'approve'
+        begin
+          hit.assignment.at_amazon.approve!
+        rescue RTurk::InvalidRequest => exception
+            #assignment may have passed its approval deadline since
+            #the session started and has been auto approved. In that
+            #case, add to the transcript; no need to tell the user.
+          unless exception.message.match(/AWS.MechanicalTurk.InvalidAssignmentState\b.+\bstatus of:\s*Submitted/i)
+            #different issue than auto approval; raise exception
+            raise exception
+          end
+        end #begin
+        Typingpool::App.record_approved_hits_in_assignments_file(assignments, [hit])
+        Typingpool::App.create_transcript(project, assignments)
+        say(cli_bold("Approved. ") + "#{project.name} transcript updated.\n")
+      when 'reject'
+        reason=nil
+        until reason.to_s.match(/\S/)
+          reason = ask("Rejection reason, for worker: ")
+        end
+        begin
+          hit.assignment.at_amazon.reject!(reason)
+          hit.at_amazon.dispose!
+        rescue RTurk::InvalidRequest => exception
+          #see comment under 'approve' case above
+          if exception.message.match(/AWS.MechanicalTurk.InvalidAssignmentState\b.+\bstatus of:\s*Submitted/i)
+            say(cli_bold("Not rejected:") + " This assignment passed its approval deadline since this session began and the assignment has been auto-approved. It will be deleted, but you've been charged for this assignment and your rejection message will not be delivered.")
+            hit.remove_from_amazon
+          else
+            raise exception
+          end
+        else
+          say(cli_bold("Rejected\n"))
+        end #begin
+        Typingpool::App.unrecord_hits_in_assignments_file(assignments, [hit])
+      when 'quit'
+        say(cli_bold("Quitting"))
+        throw :quitting
+      when 'skip'
+        say(cli_bold("Skipping\n"))
+        next
+      end #case choice
+    end #hits.each_with_index
+
+  end #find_projects_waiting_for(...) do
+end #catch :quitting do
+
+#Put project path on STDOUT so script can be pipelined, e.g.
+#`tp-review Foo | tp-finish`
+if options[:project]
+  puts options[:project].local.path
+end
+
+if options[:fixture]
+  Typingpool::App.vcr_stop
+end