typingpool 0.7.0

data/lib/typingpool/test.rb

@@ -0,0 +1,418 @@
+module Typingpool
+  require 'test/unit' 
+
+  class Test < ::Test::Unit::TestCase 
+    require 'nokogiri'
+    require 'fileutils'
+
+    def MiniTest.filter_backtrace(bt)
+      bt
+    end
+
+    def self.app_dir
+      File.dirname(File.dirname(File.dirname(__FILE__)))
+    end
+
+    def fixtures_dir
+      File.join(Utility.lib_dir, 'test', 'fixtures')
+    end
+
+    def audio_dir
+      File.join(fixtures_dir, 'audio')
+    end
+
+    def config
+      if File.exists?(File.expand_path(Config.default_file))
+        Config.file
+      else
+        Config.from_bundled_template
+      end
+    end
+
+    def amazon_credentials?(config=self.config)
+      config.amazon && config.amazon.key && config.amazon.secret
+    end
+
+    def skip_with_message(reason, skipping_what='')
+      skipping_what = " #{skipping_what}" if not(skipping_what.empty?)
+      skip ("Skipping#{skipping_what}: #{reason}")
+      true
+    end
+
+    def skip_if_no_amazon_credentials(skipping_what='', config=self.config)
+      if not (amazon_credentials?(config))
+        skip_with_message('Missing or incomplete Amazon credentials', skipping_what)
+      end
+    end
+
+    def s3_credentials?(config)
+      amazon_credentials?(config) && config.amazon.bucket
+    end
+
+    def skip_if_no_s3_credentials(skipping_what='', config=self.config)
+      if not (skip_if_no_amazon_credentials(skipping_what, config))
+        if not(s3_credentials?(config))
+          skip_with_message('No Amazon S3 credentials', skipping_what)
+        end #if not(s3_credentials?...)
+      end #if not(skip_if_no_amazon_credentials...)
+    end
+
+    def sftp_credentials?(config)
+      config.sftp && config.sftp.user && config.sftp.host && config.sftp.url
+    end
+
+    def skip_if_no_sftp_credentials(skipping_what='', config=self.config)
+      if not(sftp_credentials?(config))
+        skip_with_message('No SFTP credentials', skipping_what)
+      end #if not(sftp_credentials?...
+    end
+
+    def skip_if_no_upload_credentials(skipping_what='', config=self.config)
+      if not(s3_credentials?(config) || sftp_credentials?(config))
+        skip_with_message("No S3 or SFTP credentials in config", skipping_what)
+      end #if not(s3_credentials?...
+    end
+
+    def add_goodbye_message(msg)
+      at_exit do
+        STDERR.puts msg
+      end
+    end
+
+    def dummy_config(number=1)
+      Typingpool::Config.file(File.join(fixtures_dir, "config-#{number}"))
+    end
+
+
+    def project_default
+      Hash[
+           :config_filename => '.config',
+           :subtitle => "Typingpool's test interview transcription",
+           :title => "Typingpool's Test & Interview",
+           :chunks => '0:20',
+           :unusual => ['Hack Day', 'Sunnyvale', 'Chad D'],
+           :voice => ['Ryan', 'Havi, hacker'],
+          ]
+    end
+
+
+    def in_temp_dir
+      Typingpool::Utility.in_temp_dir{|dir| yield(dir) }
+    end
+
+    def working_url?(*args)
+      Typingpool::Utility.working_url?(*args)
+    end
+
+    def fetch_url(*args)
+      Typingpool::Utility.fetch_url(*args)
+    end
+
+    class Script < Test 
+      require 'typingpool'
+      require 'yaml'
+      require 'open3'
+
+
+      def audio_files(subdir='mp3')
+        dir = File.join(audio_dir, subdir)
+        Dir.entries(dir).reject{|entry| entry.match(/^\./) }.map{|entry| File.join(dir, entry)}.select{|path| File.file?(path) }
+      end
+
+      def config_path(dir)
+        ::File.join(dir, project_default[:config_filename])   
+      end
+
+      def config_from_dir(dir)
+        Config.file(config_path(dir))
+      end
+
+
+      def setup_amazon(dir)
+        Amazon.setup(:sandbox => true, :config => config_from_dir(dir))
+      end
+
+
+      def in_temp_tp_dir
+        ::Dir.mktmpdir('typingpool_') do |dir|
+          setup_temp_tp_dir(dir)
+          yield(dir)
+        end
+      end
+
+      def setup_temp_tp_dir(dir)
+        make_temp_tp_dir_config(dir)
+        Dir.mkdir(File.join(dir, 'projects'))
+      end
+
+      def setup_s3_config(dir, config=config_from_dir(dir), filename='.config_s3')
+        return unless s3_credentials?(config)
+        config.to_hash.delete('sftp')
+        write_config(config, dir, filename)
+      end
+
+      def setup_s3_config_with_bad_password(dir, config=config_from_dir(dir))
+        bad_password = 'f'
+        refute_equal(config.to_hash['amazon']['secret'], bad_password)
+        config.to_hash['amazon']['secret'] = bad_password
+        setup_s3_config(dir, config, '.config_s3_bad')
+      end
+
+      def make_temp_tp_dir_config(dir, config=self.config)
+        config.transcripts = File.join(dir, 'projects')
+        config.cache = File.join(dir, '.cache')
+        config['assign']['reward'] = '0.02'
+        config.assign.to_hash.delete('qualify')
+        write_config(config, dir, project_default[:config_filename])   
+      end
+
+      def write_config(config, dir, filename=project_default[:config_filename])
+        path = ::File.join(dir, filename)
+        ::File.open(path, 'w') do |out|
+          out << YAML.dump(config.to_hash)
+        end
+        path
+      end
+
+      def temp_tp_dir_project_dir(temp_tp_dir)
+        ::File.join(temp_tp_dir, 'projects', project_default[:title])
+      end
+
+      def temp_tp_dir_project(dir, config=config_from_dir(dir))
+        Project.new(project_default[:title], config)
+      end
+
+      def call_script(*args)
+        Utility.system_quietly(*args)
+      end
+
+      def path_to_tp_make
+        ::File.join(self.class.app_dir, 'bin', 'tp-make')
+      end
+
+      def call_tp_make(*args)
+        call_script(path_to_tp_make, *args)
+      end
+
+      def tp_make(in_dir, config=config_path(in_dir), audio_subdir='mp3')
+        call_tp_make(
+                     '--config', config, 
+                     '--chunks', project_default[:chunks],
+                     *[:title, :subtitle].map{|param| ["--#{param}", project_default[param]] }.flatten,
+                     *[:voice, :unusual].map{|param| project_default[param].map{|value| ["--#{param}", value] } }.flatten,
+                     *audio_files(audio_subdir).map{|path| ['--file', path]}.flatten
+                     )
+      end
+
+      def path_to_tp_finish
+        ::File.join(self.class.app_dir, 'bin', 'tp-finish')
+      end
+
+      def call_tp_finish(*args)
+        call_script(path_to_tp_finish, *args)
+      end
+
+      def tp_finish(dir, config_path=self.config_path(dir))
+        tp_finish_inside_sandbox(dir, config_path)
+        tp_finish_outside_sandbox(dir, config_path)
+      end
+
+
+      def tp_finish_inside_sandbox(dir, config_path=self.config_path(dir))
+        tp_finish_outside_sandbox(dir, config_path, '--sandbox')
+      end
+
+      def tp_finish_outside_sandbox(dir, config_path=self.config_path(dir), *args)
+        call_tp_finish(
+                       project_default[:title],
+                       '--config', config_path, 
+                       *args
+                       )
+      end
+
+      def path_to_tp_assign
+        File.join(self.class.app_dir, 'bin', 'tp-assign')
+      end
+
+      def call_tp_assign(*args)
+        call_script(path_to_tp_assign, '--sandbox', *args)
+      end
+
+      def assign_default
+        Hash[
+             :template => 'interview/phone',
+             :deadline => '5h',
+             :lifetime => '10h',
+             :approval => '10h',
+             :qualify => ['approval_rate >= 90', 'hits_approved > 10'],
+             :keyword => ['test', 'mp3', 'typingpooltest']
+            ]
+      end
+
+      def tp_assign(dir, config_path=config_path(dir))
+        call_tp_assign(
+                       project_default[:title],
+                       assign_default[:template],
+                       '--config', config_path,
+                       *[:deadline, :lifetime, :approval].map{|param| ["--#{param}", assign_default[param]] }.flatten,
+                       *[:qualify, :keyword].map{|param| assign_default[param].map{|value| ["--#{param}", value] } }.flatten
+                       )
+      end
+
+      def path_to_tp_collect
+        File.join(self.class.app_dir, 'bin', 'tp-collect')
+      end
+
+      def call_tp_collect(fixture_path, *args)
+        call_script(path_to_tp_collect, '--sandbox', '--fixture', fixture_path, *args)
+      end
+
+      def tp_collect_with_fixture(dir, fixture_path)
+        call_tp_collect(
+                        fixture_path,
+                        '--config', config_path(dir)
+                        )
+      end
+
+
+      def path_to_tp_review
+        File.join(self.class.app_dir, 'bin', 'tp-review')
+      end
+
+      def tp_review_with_fixture(dir, fixture_path, choices)
+        output = {}
+        Open3.popen3(path_to_tp_review, '--sandbox', '--fixture', fixture_path, '--config', config_path(dir), project_default[:title]) do |stdin, stdout, stderr, wait_thr|
+          choices.each do |choice|
+            stdin.puts(choice)
+            if choice.strip.match(/^r/i)
+              stdin.puts("No reason - this is a test")
+            end
+          end
+          output[:out] = stdout.gets(nil)
+          output[:err] = stderr.gets(nil)
+          [stdin, stdout, stderr].each{|stream| stream.close }
+          output[:status] = wait_thr.value
+        end
+        output
+      end
+
+      def path_to_tp_config
+        File.join(self.class.app_dir, 'bin', 'tp-config')
+      end
+
+      def tp_config(*args)
+        call_script(path_to_tp_config, *args)
+      end
+
+      def tp_config_with_input(args, input)
+        output = {}
+        Open3.popen3(path_to_tp_config, *args) do |stdin, stdout, stderr, wait_thr|
+          input.each do |sending|
+            stdin.puts(sending)
+          end
+          output[:out] = stdout.gets(nil)
+          output[:err] = stderr.gets(nil)
+          [stdin, stdout, stderr].each{|stream| stream.close }
+          output[:status] = wait_thr.value
+        end #Open3.popen3...
+        output
+      end
+
+      def fixture_project_dir(name)
+        File.join(fixtures_dir, name)
+      end
+
+      def make_fixture_project_dir(name)
+        dir = fixture_project_dir(name)
+        if File.exists? dir
+          raise Error::Test, "Fixture project already exists for #{name} at #{dir}"
+        end
+        ::Dir.mkdir(dir)
+        dir
+      end
+
+      def remove_fixture_project_dir(name)
+        FileUtils.remove_entry_secure(fixture_project_dir(name), :secure => true)
+      end
+
+      def with_fixtures_in_temp_tp_dir(dir, fixture_prefix)
+        fixtures = Dir.entries(fixtures_dir).select{|entry| entry.include?(fixture_prefix) && entry.index(fixture_prefix) == 0 }.select{|entry| File.file?(File.join(fixtures_dir, entry)) }
+        fixtures.map!{|fixture| fixture[fixture_prefix.size .. -1] }
+        fixtures.each do |fixture|
+          project_path = File.join(temp_tp_dir_project_dir(dir), 'data', fixture)
+          fixture_path = File.join(fixtures_dir, [fixture_prefix, fixture].join )
+          yield(fixture_path, project_path)
+        end
+      end
+
+      def copy_fixtures_to_temp_tp_dir(dir, fixture_prefix)
+        with_fixtures_in_temp_tp_dir(dir, fixture_prefix) do |fixture_path, project_path|
+          if File.exists? project_path
+            FileUtils.mv(project_path, File.join(File.dirname(project_path), "orig_#{File.basename(project_path)}"))
+          end
+          FileUtils.cp(fixture_path, project_path)
+        end
+      end
+
+      def rm_fixtures_from_temp_tp_dir(dir, fixture_prefix)
+        with_fixtures_in_temp_tp_dir(dir, fixture_prefix) do |fixture_path, project_path|
+          FileUtils.rm(project_path)
+          path_to_orig = File.join(File.dirname(project_path), "orig_#{File.basename(project_path)}")
+          if File.exists?(path_to_orig)
+            FileUtils.mv(path_to_orig, project_path)
+          end
+        end
+      end
+
+      def assert_has_transcript(dir, transcript_file='transcript.html')
+        transcript_path = File.join(temp_tp_dir_project_dir(dir), transcript_file)
+        assert(File.exists?(transcript_path))
+        assert(not((transcript = IO.read(transcript_path)).empty?))
+        transcript
+      end
+
+      def assert_has_partial_transcript(dir)
+        assert_has_transcript(dir, 'transcript_in_progress.html')
+      end
+
+      def assert_assignment_csv_has_transcription_count(count, project, which_csv='assignment.csv')
+        assert_equal(count, project.local.file('data', which_csv).as(:csv).reject{|assignment| assignment['transcript'].to_s.empty?}.size)
+      end
+
+      def assert_html_has_audio_count(count, html)
+        assert_equal(count, noko(html).css('audio').size)
+      end
+
+      def assert_all_assets_have_upload_status(assignment_csv, types, status)
+        types.each do |type|
+          recorded_uploads = assignment_csv.map{|assignment| assignment["#{type}_uploaded"] }
+          refute_empty(recorded_uploads)
+          assert_equal(recorded_uploads.count, recorded_uploads.select{|uploaded| uploaded == status }.count)
+        end
+      end
+
+      def assert_shell_error_match(regex)
+        exception = assert_raise(Typingpool::Error::Shell) do
+          yield
+        end
+        assert_match(exception.message, regex)
+      end
+
+      def assert_script_abort_match(args, regex)
+        in_temp_tp_dir do |dir|
+          assert_shell_error_match(regex) do 
+            yield([*args, '--config', config_path(dir)])
+          end
+        end #in_temp_tp_dir do...
+      end
+
+      def noko(html)
+        Nokogiri::HTML(html) 
+      end
+
+      def vcr_dir
+        File.join(fixtures_dir, 'vcr')
+      end
+    end #Script
+  end #Test
+end #Typingpool

data/lib/typingpool/transcript.rb

@@ -0,0 +1,181 @@
+module Typingpool
+  #This is the model class for Typingpool's final and most important
+  #output, a transcript of the Project audio in HTML format, with
+  #embedded audio. A Transcript instance is actually an enumerable
+  #container for Transcript::Chunk instances. Each Transcript::Chunk
+  #corresponds to an Amazon::HIT and to an audio "chunk" (file) that
+  #has been transcribed and which is part of a larger recording.
+  #
+  #This class is likey to be done away with in the next few point
+  #versions of Typingpool. Functionality and data unique to
+  #Transcipt::Chunk can probably be rolled into
+  #Amazon::HIT. Transcript itself can probably be folded into Project,
+  #which would become a HIT container, and then we'd pass Project
+  #instances to the output template.
+  class Transcript
+    include Enumerable
+
+    #Get/set the title of the transcript, typically corresponds to the name of the
+    #associated Project
+    attr_accessor :title
+
+    #Get/set the subtitle of the transcript, corresponds to Project#local#subtitle
+    #(a.k.a data/subtitle.txt in the project dir)
+    attr_accessor :subtitle
+
+    #Constructor. Takes an optional title (see above for explanation
+    #of title) and an optional array of Transcript::Chunk instances.
+    def initialize(title=nil, chunks=[])
+      @title = title
+      @chunks = chunks
+    end
+
+    #Iterate of the Transcript::Chunk instances
+    def each
+      @chunks.each do |chunk|
+        yield chunk
+      end
+    end
+
+    #Takes an index, returns the Transcript::Chunk at that index.
+    def [](index)
+      @chunks[index]
+    end
+
+    #Returns chunks joined by double newlines
+    def to_s
+      @chunks.join("\n\n")
+    end
+    
+    #Takes a Transcript::Chunk instance and adds it to the Transcript instance.
+    def add_chunk(chunk)
+      @chunks.push(chunk)
+    end
+
+    #Transcript::Chunk is the model class for one transcription by one
+    #Mechanical Turk worker of one "chunk" (a file) of audio, which in
+    #turn is a portion of a larger recording (for example, one minute
+    #of a 60 minute interview). It is basically parallel and similar
+    #to an Amazon::HIT instance. Transcript is a container for these
+    #chunks, which know how to render themselves as text and HTML.
+    class Chunk
+      require 'cgi'
+      require 'rubygems/text'
+      include Gem::Text
+
+      #Get/set the raw text of the transcript
+      attr_accessor :body
+
+      #Get/set the Amazon ID of the Mechanical Turk worker who
+      #transcribed the audio into text
+      attr_accessor :worker
+
+      #Get/set the id of the Amazon::HIT associated with this chunk
+      attr_accessor :hit
+
+      #Get/set the id of the Project#local associated with this chunk
+      attr_accessor :project
+
+      #Return the offset associated with the chunk, in MM:SS
+      #format. This corresponds to the associated audio file, which is
+      #a chunk of a larger recording and which starts at a particular
+      #time offset, for example from 1:00 (the offset) to 2:00 (the
+      #next offset).
+      #
+      #
+      #This should be updated to return HH:MM:SS and MM:SS.sss when
+      #appropriate, since in Project#interval we use that format and
+      #allow audio to be divided into such units. (TODO)
+      attr_reader :offset
+
+      #Returns the offset in seconds. So for an offset of 1:00 would return 60.
+      attr_reader :offset_seconds
+
+      #Returns the name of the remote audio file corresponding to this
+      #chunk. The remote file has the project ID and pseudo random
+      #characters added to it.
+      attr_reader :filename
+
+      #Returns the name of the local audio file corresponding to this
+      #chunk.
+      attr_reader :filename_local
+
+      #Returns the URL of the remote audio transcribed in the body of
+      #this chunk.
+      attr_reader :url
+
+      #Constructor. Takes the raw text of the transcription.
+      def initialize(body)
+        @body = body
+      end
+
+      #Sorts by offset seconds.
+      def <=>(other)
+        self.offset_seconds <=> other.offset_seconds
+      end
+
+      #Takes an URL. As an important side effect, sets various
+      #attributes, including url, filename, filename_local, offset and
+      #offset_seconds. So setting Chunk#url= http://whateverwhatever
+      #is an important step in populating the instance.
+      def url=(url)
+        #http://ryantate.com/transfer/Speech.01.00.ede9b0f2aed0d35a26cef7160bc9e35e.ISEAOM.mp3
+        matches = Project.url_regex.match(url) or raise Error::Argument::Format, "Unexpected format to url '#{url}'"
+        @url = matches[0]
+        @filename = matches[1]
+        @filename_local = Project.local_basename_from_url(@url)
+        @offset = "#{matches[3]}:#{matches[4]}"
+        @offset_seconds = (matches[3].to_i * 60) + matches[4].to_i
+      end
+
+      #Takes an optional specification of how many spaces to indent
+      #the text by (default 0) and an optional specification of how
+      #many characters to wrap at (default no wrapping).
+      #
+      #Returns the text with newlines normalized to Unix format, runs
+      #of newlines shortened to a maximum of two newlines, leading and
+      #trailing whitespace removed from each line, and the text
+      #wrapped/indented as specified.
+      def body_as_text(indent=nil, wrap=nil)
+        text = self.body
+        text = Utility.normalize_newlines(text)
+        text.gsub!(/\n\n+/, "\n\n")
+        text = text.split("\n").map{|line| line.strip }.join("\n")
+        text = wrap_text(text, wrap) if wrap
+        text = indent_text(text, indent) if indent
+        text
+      end
+      alias :to_s :body_as_text
+      alias :to_str :body_as_text
+
+      #Takes an optional count of how many characters to wrap at
+      #(default 72). Returns the body, presumed to be raw text, as
+      #HTML. Any HTML tags in the body are escaped. Text blocks
+      #separated by double newlines are converted to HTML paragraphs,
+      #while single newlines are converted to HTML BR tags. Newlines
+      #are normalized as in body_as_text, and lines in the HTML source
+      #are automatically wrapped as specified.
+      def body_as_html(wrap=72)
+        text = body_as_text
+        text = CGI::escapeHTML(text)
+        text = Utility.newlines_to_html(text)
+        text = text.split("\n").map do |line| 
+          wrap_text(line, 72).chomp
+        end.join("\n") 
+        text
+      end
+
+      protected
+
+      def indent_text(text, indent)
+        text.gsub!(/^/, " " * indent)
+        text
+      end
+
+      def wrap_text(text, wrap=72)
+        format_text(text, wrap)
+      end
+
+    end #Chunk 
+  end #Transcript 
+end #Typingpool