typingpool 0.7.0

data/test/make_transcript_chunks_fixture.rb

@@ -0,0 +1,53 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+
+lines_each = 2
+if ARGV.last.to_s.match(/^\d+$/)
+  lines_each = ARGV.pop.to_i
+end
+
+if ARGV.empty?
+  abort "USAGE: #{File.basename($PROGRAM_NAME)} PROJECT_DIR [PROJECT_DIR...] [LINES_EACH=2]"
+end
+
+data_files = ARGV.map do |path| 
+  data_file = nil
+  expanded = File.expand_path(path)
+  if not(File.exists? expanded)
+    abort "No such dir #{path}"
+  end
+  if not(File.directory? expanded)
+    abort "Not a dir: #{path}"
+  end
+  %w(data csv).each do |data_dir_name|
+    possible_data_file = File.join(expanded, data_dir_name, 'assignment.csv')
+    if File.exists? possible_data_file
+      data_file = possible_data_file
+    end
+  end #%w().each do...
+  if not(data_file)
+    abort "Dir #{path} has no data/assignment.csv or csv/assignment.csv"
+  end
+  data_file
+end #ARGV.map
+
+assignments=[]
+data_files.each do |path|
+  csv = Typingpool::Filer::CSV.new(path)
+  with_transcripts = csv.reject{|assignment| assignment['transcript'].to_s.empty? }
+  next if with_transcripts.empty?
+  seeking = with_transcripts.count
+  if seeking > lines_each
+    seeking = lines_each
+  end
+  assignments.push(*with_transcripts.sample(seeking))
+end #data_files.each do...
+
+if assignments.empty?
+  abort "No transcripts found"
+end
+fixtures_dir = File.join(Typingpool::Utility.lib_dir, 'test', 'fixtures')
+Typingpool::Filer::CSV.new(File.join(fixtures_dir, 'transcript-chunks.csv')).write(assignments)

data/test/test_integration_script_1_tp_config.rb

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+
+class TestTpConfig < Typingpool::Test::Script
+
+def test_abort_with_invalid_file
+  exception = assert_raise(Typingpool::Error::Shell) do
+    tp_config(File.join(fixtures_dir, 'not_yaml.txt'))
+  end
+  assert_match(exception.message, /not valid yaml/i)
+end
+
+def test_abort_with_directory_path
+  dir = File.join(fixtures_dir, 'vcr')
+  assert(File.exists? dir)
+  assert(File.directory? dir)
+  exception = assert_raise(Typingpool::Error::Shell) do
+    tp_config(dir)
+  end
+  assert_match(exception.message, /not a file/i)
+end
+
+def test_abort_with_invalid_path
+  path = '/jksdljs/euwiroeuw'
+  refute(File.exists? path)
+  exception = assert_raise(Typingpool::Error::Shell) do
+    tp_config(path)
+  end
+  assert_match(exception.message, /valid path/i)
+end
+
+def test_usage_message
+  assert(output = tp_config('--help'))
+  assert_match(output, /\bUSAGE:/)
+end
+
+def test_new_config_creation
+  in_temp_dir do |dir|
+    path = {
+      :config => File.join(dir, 'config.yml'),
+      :transcript_dir => File.join(dir, 'transcriptionz')
+    }
+    path.values.each{|path| refute(File.exists? path) }
+    assert(output = tp_config_with_input([path[:config], '--test'], ['keykey', 'secretsecret', path[:transcript_dir]]))
+    assert_match(output[:err], /wrote config to/i)
+    path.values.each{|path| assert(File.exists? path) }
+    assert(File.file? path[:config] )
+    assert(File.directory? path[:transcript_dir] )
+    assert(config = Typingpool::Config.file(path[:config]))
+    assert_equal(path[:transcript_dir], config.transcripts)
+    assert_equal(File.join(path[:transcript_dir], 'templates').downcase, config.templates.downcase)
+    assert_equal('keykey', config.amazon.key)
+    assert_equal('secretsecret', config.amazon.secret)
+    refute_empty(config.amazon.bucket.to_s)
+  end #in_temp_dir do |dir|
+end
+
+def test_config_editing
+  in_temp_dir do |dir|
+    path = {
+      :config => File.join(dir, 'config.yml'),
+      :fixture => File.join(fixtures_dir, 'config-1'),
+      :transcript_dir => File.join(dir, 'transcriptionz')
+    }
+    assert(File.exists? path[:fixture])
+    refute(File.exists? path[:config])
+    FileUtils.cp(path[:fixture], path[:config])
+    assert(File.exists? path[:config])
+    assert(original_config = Typingpool::Config.file(path[:config]))
+    [:key, :secret, :bucket].each{|param| refute_empty(original_config.amazon.send(param).to_s) }
+    [:transcripts, :templates, :cache].each{|param| refute_empty(original_config.send(param).to_s) } 
+    assert(output = tp_config_with_input([path[:config], '--test'], ['keykey', 'secretsecret', path[:transcript_dir]]))
+    assert(edited_config = Typingpool::Config.file(path[:config]))
+    [:key, :secret].each{|param| refute_equal(original_config.amazon.send(param), edited_config.amazon.send(param)) }
+    assert_equal(original_config.amazon.bucket, edited_config.amazon.bucket)
+    [:templates, :cache].each{|param| assert_equal(original_config.send(param), edited_config.send(param)) }
+    refute_equal(original_config.transcripts, edited_config.transcripts)
+    assert_equal('keykey', edited_config.amazon.key)
+    assert_equal('secretsecret', edited_config.amazon.secret)
+    assert_equal(path[:transcript_dir], edited_config.transcripts)
+  end #in_temp_dir |dir| do
+end
+
+def test_skips_bucket_when_sftp_params_exist
+  in_temp_dir do |dir|
+    path = {
+      :config => File.join(dir, 'config.yml'),
+      :fixture => File.join(fixtures_dir, 'config-2'),
+      :transcript_dir => File.join(dir, 'transcriptionz')
+    }
+    assert(File.exists? path[:fixture])
+    refute(File.exists? path[:config])
+    FileUtils.cp(path[:fixture], path[:config])
+    assert(File.exists? path[:config])
+    assert(original_config = Typingpool::Config.file(path[:config]))
+    assert_empty(original_config.amazon.bucket.to_s)
+    assert(output = tp_config_with_input([path[:config], '--test'], ['keykey', 'secretsecret', path[:transcript_dir]]))
+    assert_match(output[:err], /wrote config to/i)
+    assert(edited_config = Typingpool::Config.file(path[:config]))
+    assert_empty(edited_config.amazon.bucket.to_s)
+  end #in_temp_dir do |dir|
+end
+
+end #class TestTpConfig

data/test/test_integration_script_2_tp_make.rb

@@ -0,0 +1,119 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+require 'csv'
+
+class TestTpMake < Typingpool::Test::Script                   
+  def test_abort_with_no_files
+    assert_tp_make_abort_match(['--title', 'Foo', '--chunks', '0:20'], /no files/i)
+  end
+
+  def test_abort_with_no_title
+    assert_tp_make_abort_match(['--file', audio_files[0]], /no title/i)
+  end
+
+  def test_abort_with_invalid_title
+    assert_tp_make_abort_match(['--file', audio_files[0], '--title', 'Foo/Bar'], /illegal character/i)
+  end
+
+  def test_abort_with_no_args
+    assert_tp_make_abort_match([], /\bUSAGE:/)
+  end
+
+  def assert_tp_make_abort_match(args, regex)
+    assert_script_abort_match(args, regex) do |args|
+      call_tp_make(*args)
+    end
+  end
+
+  def tp_make_with(dir, config_path, subdir='mp3')
+    begin
+      tp_make(dir, config_path, subdir)
+      assert(project = temp_tp_dir_project(dir, Typingpool::Config.file(config_path)))
+      assert_not_nil(project.local)
+      assert_not_nil(project.local.id)
+      assert(project.local.subdir('audio','chunks').to_a.size <= 7)
+      assert(project.local.subdir('audio','chunks').to_a.size >= 6)
+      assert_equal(project_default[:subtitle], project.local.subtitle)
+      assignments = project.local.file('data', 'assignment.csv').as(:csv)
+      assert_equal(project.local.subdir('audio','chunks').to_a.size, assignments.count)
+      assert_all_assets_have_upload_status(assignments, ['audio'], 'yes')
+      assignments.each do |assignment|
+        assert_not_nil(assignment['audio_url'])
+        assert(working_url? assignment['audio_url'])
+        assert_equal(assignment['project_id'], project.local.id)
+        assert_equal(assignment['unusual'].split(/\s*,\s*/), project_default[:unusual])
+        project_default[:voice].each_with_index do |voice, i|
+          name, description = voice.split(/\s*,\s*/)
+          assert_equal(name, assignment["voice#{i+1}"])
+          if not(description.to_s.empty?)
+            assert_equal(description, assignment["voice#{i+1}title"])
+          end
+        end #project_default[:voice].each_with_index...
+      end #assignments.each d0....
+    ensure
+      tp_finish_outside_sandbox(dir, config_path)
+    end #begin
+    assert_all_assets_have_upload_status(assignments, ['audio'], 'no')
+  end
+
+  def test_tp_make
+    Dir.entries(audio_dir).select{|entry| File.directory?(File.join(audio_dir, entry))}.reject{|entry| entry.match(/^\./) }.each do |subdir|
+      in_temp_tp_dir do |dir|
+        config_path = self.config_path(dir)
+        skip_if_no_upload_credentials('tp-make integration test', Typingpool::Config.file(config_path))
+        tp_make_with(dir, config_path, subdir)
+      end #in_temp_tp_dir
+    end #Dir.entries
+  end
+
+  def test_tp_make_s3
+    in_temp_tp_dir do |dir|
+      skip_if_no_s3_credentials('tp-make S3 integration test', config)
+      config_path = setup_s3_config(dir)
+      tp_make_with(dir, config_path)
+    end #in_temp_tp_dir do...
+  end 
+
+  def test_fixing_failed_tp_make
+    in_temp_tp_dir do |dir|
+      config = config_from_dir(dir)
+      skip_if_no_s3_credentials('tp-make failed upload integration test', config)
+      good_config_path = setup_s3_config(dir)
+      bad_config_path = setup_s3_config_with_bad_password(dir)
+      assert_raises(Typingpool::Error::Shell) do
+        tp_make(dir, bad_config_path, 'mp3')
+      end
+      project_dir = temp_tp_dir_project_dir(dir)
+      assert(File.exists? project_dir)
+      assert(File.directory? project_dir)
+      assert(File.exists? File.join(project_dir, 'data', 'assignment.csv'))
+      originals_dir = File.join(project_dir, 'audio', 'originals')
+      refute_empty(Dir.entries(originals_dir).reject{|entry| entry.match(/^\./) }.map{|entry| File.join(originals_dir, entry) }.select{|path| File.file? path })
+      assert(project = temp_tp_dir_project(dir))
+      assert(assignment_csv = project.local.file('data', 'assignment.csv').as(:csv))
+      refute_empty(assignment_csv.to_a)
+      assert_all_assets_have_upload_status(assignment_csv, ['audio'], 'maybe')
+      assert(audio_urls = assignment_csv.map{|assignment| assignment['audio_url'] })
+      refute_empty(audio_urls)
+      assert_empty(audio_urls.select{|url| working_url? url })
+      begin
+        tp_make(dir, good_config_path, 'mp3')
+        refute_empty(assignment_csv.read)
+        assert_all_assets_have_upload_status(assignment_csv, ['audio'], 'yes')
+        refute_empty(audio_urls2 = assignment_csv.map{|assignment| assignment['audio_url'] })
+        audio_urls.each_with_index do |original_url, i|
+          assert_equal(original_url, audio_urls2[i])
+        end
+        assert_equal(audio_urls.count, audio_urls2.select{|url| working_url? url }.count)
+      ensure
+        tp_finish_outside_sandbox(dir, good_config_path)
+      end #begin
+      assert_all_assets_have_upload_status(assignment_csv, ['audio'], 'no')
+    end #in_temp_tp_dir do...
+  end
+
+end #TestTpMake

data/test/test_integration_script_3_tp_assign.rb

@@ -0,0 +1,152 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+
+class TestTpAssign < Typingpool::Test::Script
+  #TODO: test that qualifications are sent (will need heroic effort
+  #(or at least some xml parsing) since rturk doesn't provide an
+  #easy way to look at HIT qualifications)
+  def test_abort_with_no_input
+    assert_raise(Typingpool::Error::Shell){call_tp_assign}
+  end
+
+  def test_abort_with_no_template
+    assert_tp_assign_abort_match([project_default[:title]], /Missing\b[^\n\r\f]*\btemplate/i)
+  end
+
+  def test_abort_with_bad_timespec
+    assert_tp_assign_abort_match([project_default[:title], assign_default[:template], '--lifetime', '4u'], /can't convert/i)
+  end
+
+  def test_abort_with_bad_qualification
+    assert_tp_assign_abort_match([project_default[:title], assign_default[:template], '--qualify', 'approval_rate &= 8'], /\bsense of --qualify.+\bunknown comparator\b/i)
+    assert_tp_assign_abort_match([project_default[:title], assign_default[:template], '--qualify', 'fake_rate > 8'], /\bsense of --qualify\b.+unknown\b[^\n\r\f]*\btype\b/i)
+  end
+
+  def test_abort_with_bad_reward
+    assert_tp_assign_abort_match([project_default[:title], assign_default[:template], '--reward', 'foo'], /sense of --reward/i)
+  end
+
+  def assert_tp_assign_abort_match(args, regex)
+    assert_script_abort_match(args, regex) do |args|
+      call_tp_assign(*args)
+    end
+  end
+
+  def test_tp_assign
+    skip_if_no_amazon_credentials('tp-assign integration test')
+    skip_if_no_upload_credentials('tp-assign integration test')
+    in_temp_tp_dir do |dir|
+      tp_make(dir)
+      begin
+        tp_assign(dir)
+        assign_time = Time.now
+        config = config_from_dir(dir)
+        project = temp_tp_dir_project(dir)
+        setup_amazon(dir)
+        results = nil
+        refute_empty(results = Typingpool::Amazon::HIT.all_for_project(project.local.id))
+        assert_equal(project.local.subdir('audio','chunks').to_a.size, results.size)
+        assert_equal(Typingpool::Utility.timespec_to_seconds(assign_default[:deadline]), results[0].full.assignments_duration.to_i)
+        #These numbers will be apart due to clock differences and
+        #timing vagaries of the assignment.
+        assert_in_delta((assign_time + Typingpool::Utility.timespec_to_seconds(assign_default[:lifetime])).to_f, results[0].full.expires_at.to_f, 60)
+        keywords = results[0].at_amazon.keywords
+        assign_default[:keyword].each{|keyword| assert_includes(keywords, keyword)}
+        sandbox_csv = project.local.file('data', 'sandbox-assignment.csv').as(:csv)
+        refute_empty(assignment_urls = sandbox_csv.map{|assignment| assignment['assignment_url'] })
+        assert(assignment_html = fetch_url(assignment_urls.first).body)
+        assert_match(assignment_html, /\b20[\s-]+second\b/)
+        assert_all_assets_have_upload_status(sandbox_csv, ['assignment'], 'yes')
+      ensure
+        tp_finish(dir)
+      end #begin
+      assert_empty(Typingpool::Amazon::HIT.all_for_project(project.local.id))
+    end # in_temp_tp_dir
+  end
+
+   def test_uploads_audio_when_needed
+     skip_if_no_amazon_credentials('tp-assign unuploaded audio integration test')
+     skip_if_no_s3_credentials('tp-assign unuploaded audio integration test')
+     in_temp_tp_dir do |dir|
+       good_config_path = setup_s3_config(dir)
+       bad_config_path = setup_s3_config_with_bad_password(dir)
+       assert_raises(Typingpool::Error::Shell) do
+         tp_make(dir, bad_config_path, 'mp3')
+       end
+       project_dir = temp_tp_dir_project_dir(dir)
+       assert(File.exists? project_dir)
+       assert(File.directory? project_dir)
+       assert(project = temp_tp_dir_project(dir, Typingpool::Config.file(bad_config_path)))
+       csv = project.local.file('data', 'assignment.csv').as(:csv)
+       assert_empty(csv.select{|assignment| working_url? assignment['audio_url']})
+       assert_all_assets_have_upload_status(csv, ['audio'], 'maybe')
+       begin
+         tp_assign(dir, good_config_path)
+         sandbox_csv = project.local.file('data', 'sandbox-assignment.csv').as(:csv)
+         assert_equal(csv.count, sandbox_csv.count)
+         assert_equal(sandbox_csv.count, sandbox_csv.select{|assignment| working_url? assignment['audio_url'] }.count)
+        assert_all_assets_have_upload_status(csv, ['audio'], 'yes')
+       ensure
+         tp_finish(dir, good_config_path)
+       end #begin
+     end # in_temp_tp_dir do...
+   end
+
+def test_fixing_failed_assignment_html_upload
+  skip_if_no_amazon_credentials('tp-assign failed assignment upload integration test')
+  skip_if_no_s3_credentials('tp-assign failed assignment upload integration test')
+  in_temp_tp_dir do |dir|
+    good_config_path = setup_s3_config(dir)
+    bad_config_path = setup_s3_config_with_bad_password(dir)
+    tp_make(dir, good_config_path, 'mp3')
+    begin
+      assert(project = temp_tp_dir_project(dir, Typingpool::Config.file(good_config_path)))
+      assert(project.local)
+      get_assignment_urls = lambda{|csv| csv.map{|assignment| assignment['assignment_url'] }.select{|url| url } }
+      assert_empty(get_assignment_urls.call(project.local.file('data', 'assignment.csv').as(:csv)))
+      exception = assert_raises(Typingpool::Error::Shell) do
+        tp_assign(dir, bad_config_path)
+      end #assert_raises...
+      assert_match(exception.message, /s3 operation fail/i)
+      sandbox_csv = project.local.file('data', 'sandbox-assignment.csv').as(:csv)
+      refute_empty(get_assignment_urls.call(sandbox_csv))
+      check_assignment_urls = lambda{ get_assignment_urls.call(sandbox_csv).map{|url| Typingpool::Utility.working_url? url } }
+      check_assignment_urls.call.each{|checked_out| refute(checked_out) }
+      assert_all_assets_have_upload_status(sandbox_csv, ['assignment'], 'maybe')
+      tp_assign(dir, good_config_path)
+      check_assignment_urls.call.each{|checked_out| assert(checked_out) }
+      assert_all_assets_have_upload_status(sandbox_csv, ['assignment'], 'yes')
+    ensure
+      tp_finish(dir, good_config_path)
+    end #begin
+  end #in_temp_tp_dir do...
+end
+
+def test_abort_on_config_mismatch
+  skip_if_no_s3_credentials('tp-assign abort on config mismatch test')
+  in_temp_tp_dir do |dir|
+    config = config_from_dir(dir)
+    good_config_path = setup_s3_config(dir, config, '.config_s3_good')
+    tp_make(dir, good_config_path)
+    begin
+      tp_finish_outside_sandbox(dir, good_config_path)
+      assert(config.amazon.bucket)
+      new_bucket = 'configmismatch-test'
+      refute_equal(new_bucket, config.amazon.bucket)
+      config.amazon.bucket = new_bucket
+      bad_config_path = setup_s3_config(dir, config, '.config_s3_bad')
+      exception = assert_raises(Typingpool::Error::Shell) do
+        tp_assign(dir, bad_config_path)
+      end #assert_raises...
+      assert_match(exception.message, /\burls don't look right\b/i)
+    ensure
+      tp_finish(dir, good_config_path)
+    end #begin
+  end #in_temp_tp_dir do...
+end
+
+end #TestTpAssign

data/test/test_integration_script_4_tp_review.rb

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+
+class TestTpReview < Typingpool::Test::Script
+
+  def test_tp_review
+    in_temp_tp_dir do |dir|
+      skip_if_no_upload_credentials('tp-review integration test')
+      skip_if_no_amazon_credentials('tp-review integration test')
+      tp_make(dir)
+      copy_fixtures_to_temp_tp_dir(dir, 'tp_review_')
+      assert(File.exists? File.join(temp_tp_dir_project_dir(dir), 'data','sandbox-assignment.csv'))
+      project = temp_tp_dir_project(dir)
+        assert_equal(7, project.local.file('data','sandbox-assignment.csv').as(:csv).reject{|assignment| assignment['hit_id'].to_s.empty? }.count)
+      begin
+        output = nil
+        assert_nothing_raised do
+          output = tp_review_with_fixture(dir, File.join(fixtures_dir, 'vcr', 'tp-review-1'), %w(a r a r s q))
+        end
+        assert_equal(0, output[:status].to_i, "Bad exit code: #{output[:status]} err: #{output[:err]}")
+        assert_equal(5, project.local.file('data','sandbox-assignment.csv').as(:csv).reject{|assignment| assignment['hit_id'].to_s.empty? }.count)
+        reviews = split_reviews(output[:out])
+        assert_match(reviews[1], /Interview\.00\.00/)
+        #we can't specify leading \b boundaries because the ansi
+        #escape sequences mess that up
+        assert_match(reviews[1], /Approved\b/i)
+        assert_match(reviews[2], /Interview\.00\.20/)
+        assert_match(reviews[2], /reason\b/i)
+        assert_match(reviews[2], /Rejected\b/i)
+        assert_match(reviews[3], /Interview\.00\.40/)
+        assert_match(reviews[3], /Approved\b/i)
+        assert_match(reviews[4], /Interview\.01\.00/)
+        assert_match(reviews[4], /reason\b/i)
+        assert_match(reviews[4], /Rejected\b/i)
+        assert_match(reviews[5], /Interview\.01\.20/)
+        assert_match(reviews[5], /Skipping\b/i)
+        assert_match(reviews[6], /Interview\.02\.00/)
+        assert_match(reviews[6], /Quitting\b/i)
+        transcript = assert_has_partial_transcript(dir)
+        assert_html_has_audio_count(2, transcript)
+        assert_assignment_csv_has_transcription_count(2, project, 'sandbox-assignment.csv')
+
+        assert_nothing_raised do
+          output = tp_review_with_fixture(dir, File.join(fixtures_dir, 'vcr', 'tp-review-2'), %w(a r))
+        end
+        assert_equal(0, output[:status].to_i, "Bad exit code: #{output[:status]} err: #{output[:err]}")
+        assert_equal(4, project.local.file('data','sandbox-assignment.csv').as(:csv).reject{|assignment| assignment['hit_id'].to_s.empty? }.count)
+        reviews = split_reviews(output[:out])
+        assert_match(reviews[1], /Interview\.01\.20/)
+        assert_match(reviews[1], /Approved\b/i)
+        assert_match(reviews[2], /Interview\.02\.00/)
+        assert_match(reviews[2], /reason\b/i)
+        assert_match(reviews[2], /Rejected\b/i)
+        transcript = assert_has_partial_transcript(dir)
+        assert_html_has_audio_count(3, transcript)
+        assert_assignment_csv_has_transcription_count(3, project, 'sandbox-assignment.csv')
+      ensure
+        rm_fixtures_from_temp_tp_dir(dir, 'tp_review_')
+        tp_finish(dir)
+      end #begin
+    end #in_temp_tp_dir
+  end
+
+  def split_reviews(output)
+    output.split(/Transcript for\b/)
+  end
+
+end #class TestTpReview

data/test/test_integration_script_5_tp_collect.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+
+class TestTpCollect < Typingpool::Test::Script
+  require 'fileutils'
+
+  def test_tp_collect
+    in_temp_tp_dir do |dir|
+      skip_if_no_upload_credentials('tp-collect integration test')
+      skip_if_no_amazon_credentials('tp-collect integration test')
+      tp_make(dir)
+      copy_fixtures_to_temp_tp_dir(dir, 'tp_collect_')
+      begin
+        project = temp_tp_dir_project(dir)
+        assert_nothing_raised do
+          tp_collect_with_fixture(dir, File.join(vcr_dir, 'tp-collect-1'))
+        end
+        transcript = assert_has_partial_transcript(dir)
+        assert_html_has_audio_count(2, transcript)
+        assert_assignment_csv_has_transcription_count(2, project, 'sandbox-assignment.csv')
+        assert_nothing_raised do
+          tp_collect_with_fixture(dir, File.join(vcr_dir, 'tp-collect-2'))
+        end
+        transcript = assert_has_partial_transcript(dir)
+        assert_html_has_audio_count(4, transcript)
+        assert_assignment_csv_has_transcription_count(4, project, 'sandbox-assignment.csv')
+        assert_nothing_raised do
+          tp_collect_with_fixture(dir, File.join(vcr_dir, 'tp-collect-3'))
+        end
+#       transcript = assert_has_transcript(dir) || assert_has_partial_transcript(dir)
+#        assert_html_has_audio_count(7, transcript)
+#        assert_assignment_csv_has_transcription_count(7, project, 'sandbox-assignment.csv')
+      ensure
+        rm_fixtures_from_temp_tp_dir(dir, 'tp_collect_')
+        tp_finish(dir)
+      end #begin
+    end #in_temp_tp_dir
+  end
+
+end #TestTpCollect

data/test/test_integration_script_6_tp_finish.rb

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+
+class TestTpFinish < Typingpool::Test::Script
+  def tp_finish_on_audio_files_with(dir, config_path)
+    skip_if_no_amazon_credentials('tp-finish audio test')
+    skip_if_no_upload_credentials('tp-finish audio test')
+    tp_make(dir, config_path)
+    project = temp_tp_dir_project(dir, Typingpool::Config.file(config_path))
+    csv = project.local.file('data', 'assignment.csv').as(:csv)
+    urls = csv.map{|assignment| assignment['audio_url'] }
+    refute_empty(urls)
+    assert_all_assets_have_upload_status(csv, ['audio'], 'yes')
+    assert_equal(urls.size, urls.select{|url| working_url? url}.size)
+    assert_nothing_raised do
+      tp_finish_outside_sandbox(dir, config_path)
+    end
+    assert_empty(urls.select{|url| working_url? url })
+    assert_all_assets_have_upload_status(csv, ['audio'], 'no')
+  end
+
+  def test_tp_finish_on_audio_files
+    in_temp_tp_dir do |dir|
+      config_path = self.config_path(dir)
+      tp_finish_on_audio_files_with(dir, config_path)
+    end
+  end
+
+  def test_tp_finish_on_audio_files_with_s3
+    in_temp_tp_dir do |dir|
+      config = config_from_dir(dir)
+      config.to_hash.delete('sftp')
+      config_path = write_config(config, dir)
+      tp_finish_on_audio_files_with(dir, config_path)
+    end
+  end
+
+  def test_tp_finish_on_amazon_hits
+    skip_if_no_amazon_credentials('tp-finish Amazon test')
+    skip_if_no_upload_credentials('tp-finish Amazon test')
+    in_temp_tp_dir do |dir|
+      tp_make(dir)
+      tp_assign(dir)
+      project = temp_tp_dir_project(dir)
+      sandbox_csv = project.local.file('data', 'sandbox-assignment.csv').as(:csv)
+      assert_all_assets_have_upload_status(sandbox_csv, ['audio', 'assignment'], 'yes')
+      setup_amazon(dir)
+      results = Typingpool::Amazon::HIT.all_for_project(project.local.id)
+      refute_empty(results)
+      assert_nothing_raised do
+        tp_finish(dir)
+      end
+      assert_empty(Typingpool::Amazon::HIT.all_for_project(project.local.id))
+      results.each do |result|
+        #The original HIT might be gone, or there and marked
+        #'disposed', depending whether Amazon has swept the server for
+        #dead HITs yet
+        begin 
+          hit = RTurk::Hit.find(result.id)
+          assert_match(hit.status, /^dispos/i)
+        rescue RTurk::InvalidRequest => exception
+          assert_match(exception.message, /HITDoesNotExist/i)
+        end #begin
+      end #results.each...
+      refute(File.exists? sandbox_csv)
+      assert_all_assets_have_upload_status(project.local.file('data', 'assignment.csv').as(:csv), ['audio'], 'no')
+    end #in_temp_tp_dir
+  end
+
+  def test_tp_finish_with_missing_files
+    skip_if_no_amazon_credentials('tp-finish missing files test')
+    skip_if_no_upload_credentials('tp-finish missing files test')
+    in_temp_tp_dir do |dir|
+      project = nil
+      tp_make(dir)
+      begin
+        project = temp_tp_dir_project(dir)
+        assignments = project.local.file('data', 'assignment.csv').as(:csv).read
+        urls = assignments.map{|assignment| assignment['audio_url'] }
+        assert_empty(urls.reject{|url| working_url? url })
+        bogus_url = urls.first.sub(/\.mp3/, '.foo.mp3')
+        refute_equal(urls.first, bogus_url)
+        refute(working_url? bogus_url)
+        bogus_assignment = assignments.first.dup
+        bogus_assignment['audio_url'] = bogus_url
+        assignments.insert(1, bogus_assignment)
+        project.local.file('data', 'assignment.csv').as(:csv).write(assignments)
+        assert_equal(1, project.local.file('data', 'assignment.csv').as(:csv).reject{|assignment| working_url? assignment['audio_url'] }.count)
+      ensure
+        tp_finish_outside_sandbox(dir)
+      end #begin
+      assert_empty(project.local.file('data', 'assignment.csv').as(:csv).select{|assignment| working_url? assignment['audio_url'] })
+    end #in_temp_tp_dir...
+  end
+
+def test_abort_on_config_mismatch
+  skip_if_no_s3_credentials('tp-finish abort on config mismatch test')
+  in_temp_tp_dir do |dir|
+    config = config_from_dir(dir)
+    good_config_path = setup_s3_config(dir, config, '.config_s3_good')
+    tp_make(dir, good_config_path)
+    begin
+      assert(config.amazon.bucket)
+      new_bucket = 'configmismatch-test'
+      refute_equal(new_bucket, config.amazon.bucket)
+      config.amazon.bucket = new_bucket
+      bad_config_path = setup_s3_config(dir, config, '.config_s3_bad')
+      exception = assert_raises(Typingpool::Error::Shell) do
+        tp_finish_outside_sandbox(dir, bad_config_path)
+      end #assert_raises...
+      assert_match(exception.message, /\burls don't look right\b/i)
+    ensure
+      tp_finish_outside_sandbox(dir, good_config_path)
+    end #begin
+  end #in_temp_tp_dir do...
+
+end
+
+end #TestTpFinish