typingpool 0.7.0

data/test/test_unit_project_remote.rb

@@ -0,0 +1,157 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+require 'stringio'
+
+class TestProjectRemote < Typingpool::Test
+  def test_project_remote_from_config
+    assert(remote = Typingpool::Project::Remote.from_config(project_default[:title], dummy_config(1)))
+    assert_instance_of(Typingpool::Project::Remote::S3, remote)
+    assert(remote = Typingpool::Project::Remote.from_config(project_default[:title], dummy_config(2)))
+    assert_instance_of(Typingpool::Project::Remote::SFTP, remote)
+    config = dummy_config(2)
+    config.to_hash.delete('sftp')
+    assert_raises(Typingpool::Error) do
+      Typingpool::Project::Remote.from_config(project_default[:title], config)
+    end #assert_raises
+  end
+
+  def test_project_remote_s3_base
+    config = dummy_config(1)
+    assert(remote = Typingpool::Project::Remote::S3.new(project_default[:title], config.amazon))
+    %w(key secret bucket).each do |param|
+      refute_nil(remote.send(param.to_sym))
+      assert_equal(config.amazon.send(param.to_sym), remote.send(param.to_sym))
+    end #%w().each do...
+    assert_nil(config.amazon.url)
+    assert_includes(remote.url, config.amazon.bucket)
+    custom_url = 'http://tp.example.com/tp-test/1/2/3'
+    config.amazon.url = custom_url
+    assert(remote = Typingpool::Project::Remote::S3.new(project_default[:title], config.amazon))
+    refute_nil(remote.url)
+    refute_includes(remote.url, config.amazon.bucket)
+    assert_includes(remote.url, custom_url)
+    assert_equal('tp.example.com', remote.host)
+    assert_equal('/tp-test/1/2/3', remote.path)
+  end
+
+  def test_project_remote_s3_networked
+    assert(config = self.config)
+    skip_if_no_s3_credentials('Project::Remote::S3 upload and delete tests', config)
+    config.to_hash.delete('sftp')
+    assert(project = Typingpool::Project.new(project_default[:title], config))
+    assert_instance_of(Typingpool::Project::Remote::S3, remote = project.remote)
+    standard_put_remove_tests(remote)
+  end
+
+  def test_project_remote_sftp_base
+    config = dummy_config(2)
+    assert(remote = Typingpool::Project::Remote::SFTP.new(project_default[:title], config.sftp))
+    %w(host path user url).each do |param|
+      refute_nil(remote.send(param.to_sym))
+      assert_equal(config.sftp.send(param.to_sym), remote.send(param.to_sym))
+    end #%w().each do...
+    assert_equal('example.com', remote.host)
+    assert_equal('public_html/transfer', remote.path)
+  end
+
+  def test_project_remote_sftp_networked
+    assert(config = self.config)
+    skip_if_no_sftp_credentials('Project::Remote::SFTP upload and delete tests', config)
+    assert(project = Typingpool::Project.new(project_default[:title], config))
+    assert_instance_of(Typingpool::Project::Remote::SFTP, remote = project.remote)
+    standard_put_remove_tests(remote)
+  end
+
+  def standard_put_remove_tests(remote)
+    basenames = ['amazon-question-html.html', 'amazon-question-url.txt']
+    local_files = basenames.map{|basename| File.join(fixtures_dir, basename) }
+    local_files.each{|path| assert(File.exists? path) }
+    strings = local_files.map{|path| File.read(path) }
+    strings.each{|string| refute_empty(string) }
+
+    #with default basenames
+    put_remove_test(
+                    :remote => remote, 
+                    :streams => local_files.map{|path| File.new(path) },
+                    :test_with => lambda{|urls| urls.each_with_index{|url, i| assert_includes(url, basenames[i]) } }
+                    )
+
+    #now with different basenames
+    remote_basenames = basenames.map{|name| [File.basename(name, '.*'), pseudo_random_chars, File.extname(name)].join }
+    base_args = {
+      :remote => remote,
+      :as => remote_basenames,
+      :test_with => lambda{|urls| urls.each_with_index{|url, i| assert_includes(url, remote_basenames[i]) }}
+    }
+
+    put_remove_test(
+                    base_args.merge(
+                                    :streams => local_files.map{|path| File.new(path) },
+                                    )
+                    )
+
+    #now using remove_urls for removal
+    put_remove_test(
+                    base_args.merge(
+                                    :streams => local_files.map{|path| File.new(path) },
+                                    :remove_with => lambda{|urls|  base_args[:remote].remove_urls(urls) }
+                                    )
+                    )
+
+    #now with stringio streams
+    put_remove_test(
+                    base_args.merge( 
+                                    :streams => strings.map{|string| StringIO.new(string) },
+                                    )
+                    )
+
+
+  end
+
+  #Uploads and then deletes streams to a remote server, running some
+  #basic tests along the way, along with some optional lambdas.
+  # ==== Params
+  # args[:remote]      Required. A Project::Remote instance to use for
+  #                    putting and removing.  
+  # args[:streams]     Required. An enumerable collection of IO streams to
+  #                    put and remove.
+  # args[:as]          Optional. An array of basenames to use to name the
+  #                    streams remotely. Default is to call
+  #                    Project::Remote#put with no 'as' param.
+  # args[:test_with]   Optional. Lambda to call after putting the
+  #                    streams and after running the standard tests.
+  # args[:remove_with] Optional. Lambda to call to remove the remote
+  #                    files. Default lambda calls
+  #                    args[:remote].remove(args[:as])
+  # ==== Returns
+  #URLs of the former remote files (non functioning/404)
+  def put_remove_test(args)
+    args[:remote] or raise Error::Argument, "Must supply a Project::Remote instance as :remote"
+    args[:streams] or raise Error::Argument, "Must supply an array of IO streams as :streams"
+    args[:remove_with] ||= lambda do |urls| 
+      args[:remote].remove(urls.map{|url| Typingpool::Utility.url_basename(url) })
+    end #lambda do...
+    put_args = [args[:streams]]
+    put_args.push(args[:as]) if args[:as]
+    assert(urls = args[:remote].put(*put_args))
+    begin
+      assert_equal(args[:streams].count, urls.count)
+      urls.each{|url| assert(working_url?(url)) }
+      args[:test_with].call(urls) if args[:test_with]
+    ensure
+      args[:remove_with].call(urls)
+    end #begin
+    urls.each{|url| refute(working_url?(url)) }
+    urls
+  end
+
+  #Copy-pasted from Project::Remote so we don't have to make that a public method
+  def pseudo_random_chars(length=6)
+    (0...length).map{(65 + rand(25)).chr}.join
+  end
+
+end #TestProjectRemote

data/test/test_unit_template.rb

@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+require 'fileutils'
+
+class TestTemplate < Typingpool::Test
+  def test_template_base
+    assert(template1 = Typingpool::Template.new('template', [fixtures_dir]))
+    has_template_in_fixtures_dir_test(template1)
+    assert(template2 = Typingpool::Template.new('template.html.erb', [fixtures_dir]))
+    has_template_in_fixtures_dir_test(template2)
+    signature = 'ffffffffff'
+    assert_match(template1.render({:title => signature}), /#{signature}/)
+    refute_match(template1.read, /#{signature}/)
+    assert_match(template1.read, /<h1><%= title/i)
+    in_temp_dir do |dir1|
+      assert(template = Typingpool::Template.new('template', [dir1, fixtures_dir]))
+      assert(template.look_in.detect{|path| path == dir1})
+      has_template_in_fixtures_dir_test(template)
+      in_temp_dir do |dir2|
+        copy_template_fixture_into(dir2, '-2') do |path|
+          assert(template = Typingpool::Template.new('template', [dir1, dir2, fixtures_dir]))
+          assert_equal(path, template.full_path)
+          assert_match(template.render({:title => signature}), /#{signature}/)
+          refute_match(template.read, /#{signature}/)
+          assert_match(template.read, /<h2><%= title/i)
+        end #copy_template_fixture_into do...
+      end #in_temp_dir do |dir2|
+    end #in_temp_dir do |dir1|
+  end
+
+  def test_template_from_config
+    in_temp_dir do |dir1|
+      in_temp_dir do |dir2|
+        copy_template_fixture_into(dir2, '-2') do |path|
+          config = dummy_config
+          config.templates = dir2
+          assert(template = Typingpool::Template.from_config('template', config))
+          assert_equal(path, template.full_path)
+          assert_match(template.read, /<h2><%= title/i)
+          config.templates = dir1
+          exception = assert_raise(Typingpool::Error) do 
+            template = Typingpool::Template.from_config('template', config)
+          end #assert_raises() do...
+          assert_match(exception.message, /could not find/i)
+        end #copy_template_into do...
+      end #in_temp_dir do |dir2|
+    end #in_temp_dir do |dir1|
+  end
+
+  def test_template_assignment
+    in_temp_dir do |dir|
+      copy_template_fixture_into(dir) do |path|
+        assignment_subdir = File.join(dir, 'assignment')
+        FileUtils.mkdir(assignment_subdir)
+        copy_template_fixture_into(assignment_subdir, '-2') do |assignment_path|
+          config = dummy_config
+          config.templates = dir
+          assert(template = Typingpool::Template::Assignment.from_config('template', config))
+          assert_equal(assignment_path, template.full_path)
+          assert_match(template.read, /<h2><%= title/i)
+        end #copy_template_fixture_into(assignment_subdir) do...
+      end #copy_template_fixture_into(dir, '') do...
+    end #in_temp_dir do...
+  end
+
+  def test_template_env
+    assert(template = Typingpool::Template.new('template-3', [fixtures_dir]))
+    signatures = [('g' * 9), ('h' * 11)]
+    assert(rendered = template.render(:title => signatures[0], :new_title => signatures[1]))
+    assert_match(rendered, /<h1><%= title/i)
+    assert_match(rendered, /<h1>#{signatures[0]}/i)
+    assert_match(rendered, /<h1>#{signatures[1]}/i)
+    in_temp_dir do |dir|
+      copy_template_fixture_into(dir) do |template_path|
+        subdir = File.join(dir, 'closer')
+        FileUtils.mkdir(subdir)
+        copy_template_fixture_into(subdir, '-2') do |closer_template_oath|
+          FileUtils.cp(File.join(fixtures_dir, 'template-3.html.erb'), subdir)
+          FileUtils.cp(File.join(fixtures_dir, 'template-2.html.erb'), subdir)
+          look_in = [dir, subdir]
+          assert(template = Typingpool::Template.new('template', look_in))
+          assert_match(template.read, /<h1><%= title/i)
+          refute_match(template.read, /<h2><%= title/i)
+          assert(calling_template = Typingpool::Template.new('template-3', look_in))
+          assert(rendered = calling_template.render(:title => signatures[0], :new_title => signatures[1]))
+          assert_match(rendered, /<h2>/)
+          refute_match(rendered, /<h1>/)
+        end #copy_template_fixture_into do...
+      end #copy_template_fixture_into do...
+    end #in_temp_dir do...
+  end
+
+  def copy_template_fixture_into(dir, which_fixture='')
+    dest = File.join(dir, 'template.html.erb')
+    FileUtils.cp(File.join(fixtures_dir, "template#{which_fixture}.html.erb"), dest)
+    begin
+      yield(dest)
+    ensure
+      File.delete(dest)
+    end #begin
+  end
+
+  def has_template_in_fixtures_dir_test(template)
+    assert_equal(File.join(fixtures_dir, 'template.html.erb'), template.full_path)
+  end
+
+end #TestTemplate

data/test/test_unit_transcript.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
+
+require 'typingpool'
+require 'typingpool/test'
+
+class TestTranscript < Typingpool::Test
+
+  def test_transcript_base
+    assert(transcript = Typingpool::Transcript.new(project_default[:title]))
+    assert_equal(project_default[:title], transcript.title)
+    assert_equal(0, transcript.count)
+    dummy_chunks = self.dummy_chunks
+    assert(transcript = Typingpool::Transcript.new(project_default[:title], dummy_chunks))
+    assert_equal(dummy_chunks.count, transcript.count)
+    assert_respond_to(transcript, :each)
+    transcript.each_with_index do |chunk, i| 
+      assert_respond_to(chunk, :body)
+      assert_equal(dummy_chunks[i].body, chunk.body)
+      assert_equal(chunk.body, transcript[i].body)
+    end #transcript.each_with_index do...
+    assert(transcript = Typingpool::Transcript.new(project_default[:title]))
+    assert_equal(0, transcript.count)
+    assert_respond_to(transcript, :add_chunk)
+    dummy_chunks.each{|chunk| transcript.add_chunk(chunk) }
+    assert_equal(dummy_chunks.count, transcript.count)
+    random_index = rand(dummy_chunks.count)
+    assert_equal(dummy_chunks[random_index].body, transcript[random_index].body)
+  end
+
+  def test_transcript_chunks
+    dummy_assignments.each do |assignment|
+      assert(chunk = Typingpool::Transcript::Chunk.new(assignment['transcript']))
+      assert_equal(assignment['transcript'], chunk.body)
+      chunk.worker = assignment['worker']
+      assert_equal(assignment['worker'], chunk.worker)
+      chunk.project = assignment['project_id']
+      assert_equal(assignment['project_id'], chunk.project)
+      chunk.hit = assignment['hit_id']
+      assert_equal(assignment['hit_id'], chunk.hit)
+      url_populated = %w(offset offset_seconds filename filename_local)
+      url_populated.each{|attr| assert_nil(chunk.send(attr.to_sym)) }
+      chunk.url = assignment['audio_url']
+      assert_equal(assignment['audio_url'], chunk.url)
+      url_populated.each{|attr| refute_nil(chunk.send(attr.to_sym)) }
+      matches = chunk.offset.match(/^(\d+):(\d+)$/)
+      assert_equal(((matches[1].to_i * 60) + matches[2].to_i), chunk.offset_seconds)
+      refute_equal(chunk.filename, chunk.filename_local)
+      original_newline_count = chunk.body_as_text.scan(/\n/).count
+      chunk.body = chunk.body + "\n\r\f" + ('foo bar baz' * 100) 
+      refute_match(chunk.body_as_text, /\r/) unless $/.match(/\r/)
+      assert_equal(original_newline_count + 2, chunk.body_as_text.scan(/\n/).count)
+      original_p_count = chunk.body_as_html.scan(/<p>/i).count
+      chunk.body = chunk.body + "One & two\n\n...and 3 < 4."
+      refute_match(chunk.body_as_html, /\s&\s/)
+      refute_match(chunk.body_as_html, /\s<\s/)
+      assert_equal(original_p_count + 1, chunk.body_as_html.scan(/<p>/i).count)
+    end
+  end
+
+  def dummy_chunks
+    dummy_assignments.map do |assignment|
+      chunk = Typingpool::Transcript::Chunk.new(assignment['transcript'])
+      chunk.worker = assignment['worker']
+      chunk.project = assignment['project_id']
+      chunk.hit = assignment['hit_id']
+      chunk.url = assignment['audio_url']
+      chunk
+    end #dummy_assignments.map do...
+  end
+
+  def dummy_assignments
+    Typingpool::Filer::CSV.new(File.join(fixtures_dir, 'transcript-chunks.csv')).read
+  end
+
+end #TestTranscript

metadata

@@ -0,0 +1,234 @@
+--- !ruby/object:Gem::Specification 
+name: typingpool
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.7.0
+platform: ruby
+authors: 
+- Ryan Tate
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-09-09 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rturk
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "2.9"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: highline
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "1.6"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "1.5"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: aws-s3
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: "0.6"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: net-sftp
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.0.5
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: vcr
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: webmock
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id007
+description: An app for transcribing audio using Mechanical Turk
+email: ryantate@ryantate.com
+executables: 
+- tp-config
+- tp-make
+- tp-assign
+- tp-review
+- tp-collect
+- tp-finish
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- bin/tp-config
+- bin/tp-make
+- bin/tp-assign
+- bin/tp-review
+- bin/tp-collect
+- bin/tp-finish
+- lib/typingpool.rb
+- lib/typingpool/amazon.rb
+- lib/typingpool/app.rb
+- lib/typingpool/config.rb
+- lib/typingpool/error.rb
+- lib/typingpool/filer.rb
+- lib/typingpool/project.rb
+- lib/typingpool/template.rb
+- lib/typingpool/test.rb
+- lib/typingpool/transcript.rb
+- lib/typingpool/utility.rb
+- lib/typingpool/templates/assignment/amazon-init.js
+- lib/typingpool/templates/assignment/interview.html.erb
+- lib/typingpool/templates/assignment/interview/nameless.html.erb
+- lib/typingpool/templates/assignment/interview/noisy.html.erb
+- lib/typingpool/templates/assignment/interview/partials/voices.html.erb
+- lib/typingpool/templates/assignment/interview/phone.html.erb
+- lib/typingpool/templates/assignment/main.css
+- lib/typingpool/templates/assignment/partials/entry.html.erb
+- lib/typingpool/templates/assignment/partials/footer.html.erb
+- lib/typingpool/templates/assignment/partials/header.html.erb
+- lib/typingpool/templates/assignment/partials/labeling-example.html.erb
+- lib/typingpool/templates/assignment/partials/labeling.html.erb
+- lib/typingpool/templates/assignment/partials/length-description.html.erb
+- lib/typingpool/templates/assignment/partials/voices.html.erb
+- lib/typingpool/templates/assignment/speech.html.erb
+- lib/typingpool/templates/config.yml
+- lib/typingpool/templates/project/audio/chunks/.empty_directory
+- lib/typingpool/templates/project/audio/originals/.empty_directory
+- lib/typingpool/templates/project/data/.empty_directory
+- lib/typingpool/templates/project/etc/ About these files - read me.txt
+- lib/typingpool/templates/project/etc/audio-compat.js
+- lib/typingpool/templates/project/etc/player/audio-player.js
+- lib/typingpool/templates/project/etc/player/license.txt
+- lib/typingpool/templates/project/etc/player/player.swf
+- lib/typingpool/templates/project/etc/transcript.css
+- lib/typingpool/templates/transcript.html.erb
+- lib/typingpool/test/fixtures/amazon-question-html.html
+- lib/typingpool/test/fixtures/amazon-question-url.txt
+- lib/typingpool/test/fixtures/audio/mp3/interview.1.mp3
+- lib/typingpool/test/fixtures/audio/mp3/interview.2.mp3
+- lib/typingpool/test/fixtures/audio/wma/VN620007.WMA
+- lib/typingpool/test/fixtures/audio/wma/VN620052.WMA
+- lib/typingpool/test/fixtures/config-1
+- lib/typingpool/test/fixtures/config-2
+- lib/typingpool/test/fixtures/not_yaml.txt
+- lib/typingpool/test/fixtures/template-2.html.erb
+- lib/typingpool/test/fixtures/template-3.html.erb
+- lib/typingpool/test/fixtures/template.html.erb
+- lib/typingpool/test/fixtures/tp_collect_id.txt
+- lib/typingpool/test/fixtures/tp_collect_sandbox-assignment.csv
+- lib/typingpool/test/fixtures/tp_review_id.txt
+- lib/typingpool/test/fixtures/tp_review_sandbox-assignment.csv
+- lib/typingpool/test/fixtures/transcript-chunks.csv
+- lib/typingpool/test/fixtures/utf8_transcript.txt
+- lib/typingpool/test/fixtures/vcr/tp-collect-1.yml
+- lib/typingpool/test/fixtures/vcr/tp-collect-2.yml
+- lib/typingpool/test/fixtures/vcr/tp-collect-3.yml
+- lib/typingpool/test/fixtures/vcr/tp-review-1.yml
+- lib/typingpool/test/fixtures/vcr/tp-review-2.yml
+- test/make_amazon_question_fixture.rb
+- test/make_tp_collect_fixture_1.rb
+- test/make_tp_collect_fixture_2.rb
+- test/make_tp_collect_fixture_3.rb
+- test/make_tp_collect_fixture_4.rb
+- test/make_tp_review_fixture_1.rb
+- test/make_tp_review_fixture_2.rb
+- test/make_transcript_chunks_fixture.rb
+- test/test_integration_script_1_tp_config.rb
+- test/test_integration_script_2_tp_make.rb
+- test/test_integration_script_3_tp_assign.rb
+- test/test_integration_script_4_tp_review.rb
+- test/test_integration_script_5_tp_collect.rb
+- test/test_integration_script_6_tp_finish.rb
+- test/test_unit_amazon.rb
+- test/test_unit_config.rb
+- test/test_unit_filer.rb
+- test/test_unit_project.rb
+- test/test_unit_project_local.rb
+- test/test_unit_project_remote.rb
+- test/test_unit_template.rb
+- test/test_unit_transcript.rb
+has_rdoc: true
+homepage: http://github.com/ryantate/typingpool
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.9.2
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: 
+- ffmpeg
+- mp3splt
+- mp3wrap
+rubyforge_project: 
+rubygems_version: 1.5.0
+signing_key: 
+specification_version: 3
+summary: An app for transcribing audio using Mechanical Turk
+test_files: 
+- test/test_unit_amazon.rb
+- test/test_unit_config.rb
+- test/test_unit_filer.rb
+- test/test_unit_project.rb
+- test/test_unit_project_local.rb
+- test/test_unit_project_remote.rb
+- test/test_unit_template.rb
+- test/test_unit_transcript.rb