typingpool 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Rakefile +23 -0
- data/bin/tp-assign +240 -0
- data/bin/tp-collect +50 -0
- data/bin/tp-config +114 -0
- data/bin/tp-finish +101 -0
- data/bin/tp-make +169 -0
- data/bin/tp-review +175 -0
- data/lib/typingpool/amazon.rb +732 -0
- data/lib/typingpool/app.rb +634 -0
- data/lib/typingpool/config.rb +344 -0
- data/lib/typingpool/error.rb +22 -0
- data/lib/typingpool/filer.rb +396 -0
- data/lib/typingpool/project.rb +593 -0
- data/lib/typingpool/template.rb +175 -0
- data/lib/typingpool/templates/assignment/amazon-init.js +38 -0
- data/lib/typingpool/templates/assignment/interview/nameless.html.erb +13 -0
- data/lib/typingpool/templates/assignment/interview/noisy.html.erb +12 -0
- data/lib/typingpool/templates/assignment/interview/partials/voices.html.erb +10 -0
- data/lib/typingpool/templates/assignment/interview/phone.html.erb +12 -0
- data/lib/typingpool/templates/assignment/interview.html.erb +11 -0
- data/lib/typingpool/templates/assignment/main.css +20 -0
- data/lib/typingpool/templates/assignment/partials/entry.html.erb +19 -0
- data/lib/typingpool/templates/assignment/partials/footer.html.erb +3 -0
- data/lib/typingpool/templates/assignment/partials/header.html.erb +11 -0
- data/lib/typingpool/templates/assignment/partials/labeling-example.html.erb +4 -0
- data/lib/typingpool/templates/assignment/partials/labeling.html.erb +5 -0
- data/lib/typingpool/templates/assignment/partials/length-description.html.erb +6 -0
- data/lib/typingpool/templates/assignment/partials/voices.html.erb +10 -0
- data/lib/typingpool/templates/assignment/speech.html.erb +11 -0
- data/lib/typingpool/templates/config.yml +21 -0
- data/lib/typingpool/templates/project/audio/chunks/.empty_directory +0 -0
- data/lib/typingpool/templates/project/audio/originals/.empty_directory +0 -0
- data/lib/typingpool/templates/project/data/.empty_directory +0 -0
- data/lib/typingpool/templates/project/etc/ About these files - read me.txt +8 -0
- data/lib/typingpool/templates/project/etc/audio-compat.js +25 -0
- data/lib/typingpool/templates/project/etc/player/audio-player.js +4 -0
- data/lib/typingpool/templates/project/etc/player/license.txt +19 -0
- data/lib/typingpool/templates/project/etc/player/player.swf +0 -0
- data/lib/typingpool/templates/project/etc/transcript.css +49 -0
- data/lib/typingpool/templates/transcript.html.erb +23 -0
- data/lib/typingpool/test/fixtures/amazon-question-html.html +95 -0
- data/lib/typingpool/test/fixtures/amazon-question-url.txt +1 -0
- data/lib/typingpool/test/fixtures/audio/mp3/interview.1.mp3 +0 -0
- data/lib/typingpool/test/fixtures/audio/mp3/interview.2.mp3 +0 -0
- data/lib/typingpool/test/fixtures/audio/wma/VN620007.WMA +0 -0
- data/lib/typingpool/test/fixtures/audio/wma/VN620052.WMA +0 -0
- data/lib/typingpool/test/fixtures/config-1 +20 -0
- data/lib/typingpool/test/fixtures/config-2 +25 -0
- data/lib/typingpool/test/fixtures/not_yaml.txt +4 -0
- data/lib/typingpool/test/fixtures/template-2.html.erb +10 -0
- data/lib/typingpool/test/fixtures/template-3.html.erb +22 -0
- data/lib/typingpool/test/fixtures/template.html.erb +10 -0
- data/lib/typingpool/test/fixtures/tp_collect_id.txt +1 -0
- data/lib/typingpool/test/fixtures/tp_collect_sandbox-assignment.csv +8 -0
- data/lib/typingpool/test/fixtures/tp_review_id.txt +1 -0
- data/lib/typingpool/test/fixtures/tp_review_sandbox-assignment.csv +8 -0
- data/lib/typingpool/test/fixtures/transcript-chunks.csv +226 -0
- data/lib/typingpool/test/fixtures/utf8_transcript.txt +7 -0
- data/lib/typingpool/test/fixtures/vcr/tp-collect-1.yml +2712 -0
- data/lib/typingpool/test/fixtures/vcr/tp-collect-2.yml +2718 -0
- data/lib/typingpool/test/fixtures/vcr/tp-collect-3.yml +2768 -0
- data/lib/typingpool/test/fixtures/vcr/tp-review-1.yml +570 -0
- data/lib/typingpool/test/fixtures/vcr/tp-review-2.yml +351 -0
- data/lib/typingpool/test.rb +418 -0
- data/lib/typingpool/transcript.rb +181 -0
- data/lib/typingpool/utility.rb +272 -0
- data/lib/typingpool.rb +500 -0
- data/test/make_amazon_question_fixture.rb +24 -0
- data/test/make_tp_collect_fixture_1.rb +26 -0
- data/test/make_tp_collect_fixture_2.rb +16 -0
- data/test/make_tp_collect_fixture_3.rb +15 -0
- data/test/make_tp_collect_fixture_4.rb +17 -0
- data/test/make_tp_review_fixture_1.rb +26 -0
- data/test/make_tp_review_fixture_2.rb +30 -0
- data/test/make_transcript_chunks_fixture.rb +53 -0
- data/test/test_integration_script_1_tp_config.rb +108 -0
- data/test/test_integration_script_2_tp_make.rb +119 -0
- data/test/test_integration_script_3_tp_assign.rb +152 -0
- data/test/test_integration_script_4_tp_review.rb +72 -0
- data/test/test_integration_script_5_tp_collect.rb +44 -0
- data/test/test_integration_script_6_tp_finish.rb +123 -0
- data/test/test_unit_amazon.rb +153 -0
- data/test/test_unit_config.rb +94 -0
- data/test/test_unit_filer.rb +202 -0
- data/test/test_unit_project.rb +168 -0
- data/test/test_unit_project_local.rb +68 -0
- data/test/test_unit_project_remote.rb +157 -0
- data/test/test_unit_template.rb +111 -0
- data/test/test_unit_transcript.rb +77 -0
- metadata +234 -0
@@ -0,0 +1,153 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
|
4
|
+
|
5
|
+
require 'typingpool'
|
6
|
+
require 'typingpool/test'
|
7
|
+
require 'uri'
|
8
|
+
require 'cgi'
|
9
|
+
|
10
|
+
class TestAmazon < Typingpool::Test
|
11
|
+
|
12
|
+
def test_amazon_base
|
13
|
+
setup_result = Typingpool::Amazon.setup(:sandbox => true, :config => dummy_config)
|
14
|
+
assert_match(setup_result, /amazonaws/)
|
15
|
+
assert(Typingpool::Amazon.cache)
|
16
|
+
assert_instance_of(PStore, Typingpool::Amazon.cache)
|
17
|
+
assert_equal(dummy_config.cache, Typingpool::Amazon.cache.path)
|
18
|
+
assert(full_rturk_hit = Typingpool::Amazon.rturk_hit_full('test'))
|
19
|
+
assert_instance_of(RTurk::Hit, full_rturk_hit)
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_amazon_question
|
23
|
+
assert(question = dummy_question)
|
24
|
+
assert_instance_of(Typingpool::Amazon::Question, question)
|
25
|
+
assert_equal(question_url, question.url)
|
26
|
+
assert_equal(question_html, question.html)
|
27
|
+
assert_match(question.title, /Transcribe MP3 of/i)
|
28
|
+
assert_match(question.description, /telephone conversation/i)
|
29
|
+
assert_match(question.annotation, /\S/)
|
30
|
+
assert(decoded_annotation = URI.decode_www_form(CGI.unescapeHTML(question.annotation)))
|
31
|
+
decoded_annotation = Hash[*decoded_annotation.flatten]
|
32
|
+
assert_match(decoded_annotation[Typingpool::Amazon::HIT.url_at], /^http/i)
|
33
|
+
assert_match(decoded_annotation[Typingpool::Amazon::HIT.id_at], /\S/)
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_amazon_hit_create
|
37
|
+
with_dummy_hit_or_skip('test_amazon_hit_create') do |hit, config|
|
38
|
+
assert_equal(hit.full.external_question_url, dummy_question.url)
|
39
|
+
assert_equal(config.assign.deadline, hit.full.assignments_duration.to_i)
|
40
|
+
assert(rturk_hit = hit.at_amazon)
|
41
|
+
assert_equal(dummy_question.annotation.to_s, CGI.escapeHTML(rturk_hit.annotation.to_s))
|
42
|
+
assert_equal(dummy_question.title.strip, rturk_hit.title.strip)
|
43
|
+
assert_equal(dummy_question.description.strip, rturk_hit.description.strip)
|
44
|
+
assert_equal(config.assign.reward.to_f, rturk_hit.reward.to_f)
|
45
|
+
assert_equal(config.assign.keywords.first.to_s, rturk_hit.keywords.first.to_s)
|
46
|
+
end #with_dummy_hit
|
47
|
+
end
|
48
|
+
|
49
|
+
#fails to test all_reviewable or all_approved - those require a VCR fixture (TODO)
|
50
|
+
def test_amazon_hit_retrievers
|
51
|
+
with_dummy_hit_or_skip('test_amazon_hit_retrievers') do |hit, config|
|
52
|
+
assert(result = Typingpool::Amazon::HIT.with_ids([hit.id]))
|
53
|
+
assert_equal(1, result.count)
|
54
|
+
assert_equal(hit.id, result.first.id)
|
55
|
+
assert(result = Typingpool::Amazon::HIT.all_for_project(hit.project_id))
|
56
|
+
assert_equal(1, result.count)
|
57
|
+
assert_equal(hit.id, result.first.id)
|
58
|
+
assert(results = Typingpool::Amazon::HIT.all)
|
59
|
+
assert(results.count > 0)
|
60
|
+
assert(results = Typingpool::Amazon::HIT.all{|incoming_hit| incoming_hit.id == hit.id })
|
61
|
+
assert_equal(1, results.count)
|
62
|
+
assert_equal(hit.id, result.first.id)
|
63
|
+
end #with_dummy_hit
|
64
|
+
end
|
65
|
+
|
66
|
+
#fails to properly test approved?, rejected?, submitted?, assignment - those require a VCR ficture (TODO)
|
67
|
+
def test_amazon_hit_base
|
68
|
+
with_dummy_hit_or_skip('test_amazon_hit_base') do |hit, config|
|
69
|
+
assert_instance_of(Typingpool::Amazon::HIT, hit)
|
70
|
+
assert_match(hit.id, /\S/)
|
71
|
+
assert_match(hit.url, /^http/i)
|
72
|
+
assert_match(hit.project_id, /\S/)
|
73
|
+
assert_match(hit.project_title_from_url, /\S/)
|
74
|
+
assert(not(hit.approved?))
|
75
|
+
assert(not(hit.rejected?))
|
76
|
+
assert(not(hit.submitted?))
|
77
|
+
assert(hit.ours?)
|
78
|
+
assert_instance_of(Typingpool::Transcript::Chunk, hit.transcript)
|
79
|
+
assert_kind_of(RTurk::Hit, hit.at_amazon)
|
80
|
+
assert_instance_of(Typingpool::Amazon::HIT::Full, hit.full)
|
81
|
+
assert_instance_of(Typingpool::Amazon::HIT::Assignment::Empty, hit.assignment)
|
82
|
+
end #with_dummy_hit_or_skip
|
83
|
+
end
|
84
|
+
|
85
|
+
#fails to test external_question* methods
|
86
|
+
def test_amazon_hit_full
|
87
|
+
with_dummy_hit_or_skip('test_amazon_hit_full') do |hit, config|
|
88
|
+
assert(full = hit.full)
|
89
|
+
assert_instance_of(Typingpool::Amazon::HIT::Full, full)
|
90
|
+
[:id, :type_id].each{|attr| assert_match(full.send(attr), /\S/) }
|
91
|
+
assert(not(full.expired?))
|
92
|
+
assert(not(full.expired_and_overdue?))
|
93
|
+
assert_equal('Assignable', full.status)
|
94
|
+
assert_match(full.external_question_url, /^http/i)
|
95
|
+
[:assignments_completed, :assignments_pending].each{|attr| assert_match(full.send(attr).to_s, /^\d+$/) }
|
96
|
+
assert_kind_of(Time, full.expires_at)
|
97
|
+
assert_instance_of(Hash, full.annotation)
|
98
|
+
assert_match(full.annotation[Typingpool::Amazon::HIT.url_at], /^http/i)
|
99
|
+
assert_match(full.annotation[Typingpool::Amazon::HIT.id_at], /\S/)
|
100
|
+
end #with_dummy_hit_or_skip
|
101
|
+
end
|
102
|
+
|
103
|
+
def test_amazon_hit_full_fromsearchhits
|
104
|
+
with_dummy_hit_or_skip('test_amazon_hit_full_fromsearchhits') do |hit, config|
|
105
|
+
assert(full = hit.full)
|
106
|
+
assert_instance_of(Typingpool::Amazon::HIT::Full, full)
|
107
|
+
assert(hit2 = Typingpool::Amazon::HIT.all{|incoming_hit| incoming_hit.id == hit.id }.first)
|
108
|
+
assert_equal(hit.id, hit2.id)
|
109
|
+
assert(full2 = hit2.full)
|
110
|
+
assert_instance_of(Typingpool::Amazon::HIT::Full::FromSearchHITs, full2)
|
111
|
+
assert_equal(full.annotation.to_s, full2.annotation.to_s)
|
112
|
+
[:assignments_completed, :assignments_pending, :id, :type_id, :status, :expires_at, :assignments_duration, :external_question_url].each{|attr| assert_equal(full.send(attr).to_s, full2.send(attr).to_s) }
|
113
|
+
end #with_dummy_hit_or_skip
|
114
|
+
end
|
115
|
+
|
116
|
+
#Lacks test for HIT::Assignment - needs VCR fixture (TODO)
|
117
|
+
|
118
|
+
def question_html
|
119
|
+
File.read(File.join(fixtures_dir, 'amazon-question-html.html'))
|
120
|
+
end
|
121
|
+
|
122
|
+
def question_url
|
123
|
+
File.read(File.join(fixtures_dir, 'amazon-question-url.txt')).strip
|
124
|
+
end
|
125
|
+
|
126
|
+
def dummy_question
|
127
|
+
Typingpool::Amazon::Question.new(question_url, question_html)
|
128
|
+
end
|
129
|
+
|
130
|
+
def dummy_hit(config)
|
131
|
+
Typingpool::Amazon::HIT.create(dummy_question, config.assign)
|
132
|
+
end
|
133
|
+
|
134
|
+
def with_dummy_hit_or_skip(skipping_what)
|
135
|
+
config = self.config
|
136
|
+
skip_if_no_amazon_credentials(skipping_what, config)
|
137
|
+
config.assign.reward = '0.01'
|
138
|
+
cache = Tempfile.new('typingpool_cache')
|
139
|
+
begin
|
140
|
+
config.cache = cache.path
|
141
|
+
Typingpool::Amazon.setup(:sandbox => true, :config => config)
|
142
|
+
hit = dummy_hit(config)
|
143
|
+
begin
|
144
|
+
yield(hit, config)
|
145
|
+
ensure
|
146
|
+
hit.remove_from_amazon
|
147
|
+
end #begin
|
148
|
+
ensure
|
149
|
+
cache.close
|
150
|
+
cache.unlink
|
151
|
+
end #begin
|
152
|
+
end
|
153
|
+
end #TestAmazon
|
@@ -0,0 +1,94 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
|
4
|
+
|
5
|
+
require 'typingpool'
|
6
|
+
require 'typingpool/test'
|
7
|
+
|
8
|
+
class TestConfig < Typingpool::Test
|
9
|
+
|
10
|
+
def test_config_regular_file
|
11
|
+
assert(config = Typingpool::Config.file(File.join(fixtures_dir, 'config-1')))
|
12
|
+
assert_equal('~/Documents/Transcripts/', config['transcripts'])
|
13
|
+
assert_match(config.transcripts, /Transcripts$/)
|
14
|
+
refute_match(config.transcripts, /~/)
|
15
|
+
%w(key secret).each do |param|
|
16
|
+
regex = /test101010/
|
17
|
+
assert_match(config.amazon.send(param), regex)
|
18
|
+
assert_match(config.amazon[param], regex)
|
19
|
+
assert_match(config.amazon.to_hash[param], regex)
|
20
|
+
end
|
21
|
+
assert_equal(0.75, config.assign.reward.to_f)
|
22
|
+
assert_equal(3*60*60, config.assign.deadline.to_i)
|
23
|
+
assert_equal('3h', config.assign['deadline'])
|
24
|
+
assert_equal(60*60*24*2, config.assign.lifetime.to_i)
|
25
|
+
assert_equal('2d', config.assign['lifetime'])
|
26
|
+
assert_equal(3, config.assign.keywords.count)
|
27
|
+
assert_kind_of(Typingpool::Config::Root::Assign::Qualification, config.assign.qualify.first)
|
28
|
+
assert_equal(:approval_rate, config.assign.qualify.first.to_arg[0])
|
29
|
+
assert_equal(:gte, config.assign.qualify.first.to_arg[1].keys.first)
|
30
|
+
assert_equal('95', config.assign.qualify.first.to_arg[1].values.first.to_s)
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_config_sftp
|
34
|
+
assert(config = Typingpool::Config.file(File.join(fixtures_dir, 'config-2')))
|
35
|
+
assert_equal('ryan', config.sftp.user)
|
36
|
+
assert_equal('public_html/transfer/', config.sftp['path'])
|
37
|
+
assert_equal('public_html/transfer', config.sftp.path)
|
38
|
+
assert_equal('http://example.com/mturk/', config.sftp['url'])
|
39
|
+
assert_equal('http://example.com/mturk', config.sftp.url)
|
40
|
+
end
|
41
|
+
|
42
|
+
def test_config_screwy_file
|
43
|
+
assert(config = Typingpool::Config.file(File.join(fixtures_dir, 'config-2')))
|
44
|
+
|
45
|
+
exception = assert_raises(Typingpool::Error::Argument) do
|
46
|
+
config.assign.qualify
|
47
|
+
end
|
48
|
+
assert_match(exception.message, /Unknown qualification type/i)
|
49
|
+
|
50
|
+
config.assign['qualify'] = [config.assign['qualify'].pop]
|
51
|
+
exception = assert_raises(Typingpool::Error::Argument) do
|
52
|
+
config.assign.qualify
|
53
|
+
end
|
54
|
+
assert_match(exception.message, /Unknown comparator/i)
|
55
|
+
|
56
|
+
assert_equal('3z', config.assign['deadline'])
|
57
|
+
exception = assert_raises(Typingpool::Error::Argument::Format) do
|
58
|
+
config.assign.deadline
|
59
|
+
end
|
60
|
+
assert_match(exception.message, /can't convert/i)
|
61
|
+
|
62
|
+
config.assign['reward'] = 'foo'
|
63
|
+
exception = assert_raises(Typingpool::Error::Argument::Format) do
|
64
|
+
config.assign.reward
|
65
|
+
end
|
66
|
+
assert_match(exception.message, /\bformat should\b/i)
|
67
|
+
end
|
68
|
+
|
69
|
+
def test_config_regular_input
|
70
|
+
assert(config = Typingpool::Config.file(File.join(fixtures_dir, 'config-1')))
|
71
|
+
new_reward = '0.80'
|
72
|
+
refute_equal(config.assign.reward, new_reward)
|
73
|
+
assert(config.assign.reward = new_reward)
|
74
|
+
assert_equal(new_reward, config.assign.reward)
|
75
|
+
|
76
|
+
new_time = '11d'
|
77
|
+
refute_equal(new_time, config.assign.approval)
|
78
|
+
assert(config.assign.approval = new_time)
|
79
|
+
assert_equal(950400, config.assign.approval)
|
80
|
+
end
|
81
|
+
|
82
|
+
def test_config_screwy_input
|
83
|
+
exception = assert_raises(Typingpool::Error::Argument::Format) do
|
84
|
+
config.assign.reward = 'foo'
|
85
|
+
end
|
86
|
+
assert_match(exception.message, /\bformat should\b/i)
|
87
|
+
|
88
|
+
exception = assert_raises(Typingpool::Error::Argument::Format) do
|
89
|
+
config.assign.approval = '11f'
|
90
|
+
end
|
91
|
+
assert_match(exception.message, /can't convert/i)
|
92
|
+
|
93
|
+
end
|
94
|
+
end #TestConfig
|
@@ -0,0 +1,202 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
|
4
|
+
|
5
|
+
require 'typingpool'
|
6
|
+
require 'typingpool/test'
|
7
|
+
require 'fileutils'
|
8
|
+
|
9
|
+
class TestFiler < Typingpool::Test
|
10
|
+
|
11
|
+
def test_filer_base
|
12
|
+
path = File.join(fixtures_dir, 'config-1')
|
13
|
+
assert(filer = Typingpool::Filer.new(path))
|
14
|
+
assert_equal(path, "#{filer}")
|
15
|
+
assert(text = filer.read)
|
16
|
+
assert_match(text, /^amazon:\n/)
|
17
|
+
assert_match(text, /transcripts: ~\/Documents\/Transcripts\//)
|
18
|
+
assert_match(text, /- mp3\s*$/)
|
19
|
+
assert_equal(fixtures_dir, filer.dir.path)
|
20
|
+
in_temp_dir do |dir|
|
21
|
+
path = File.join(dir, 'filer-temp')
|
22
|
+
assert(filer = Typingpool::Filer.new(path))
|
23
|
+
assert_instance_of(Typingpool::Filer::CSV, filer_csv = filer.as(:csv))
|
24
|
+
assert_equal(filer.path, filer_csv.path)
|
25
|
+
assert_nil(filer.read)
|
26
|
+
data = "foo\nbar\nbaz."
|
27
|
+
assert(filer.write(data))
|
28
|
+
assert_equal(data, filer.read)
|
29
|
+
assert(path = filer.mv!(File.join(dir, 'filer-temp-2')))
|
30
|
+
assert(File.exists? filer.path)
|
31
|
+
assert_equal('filer-temp-2', File.basename(filer.path))
|
32
|
+
end #in_temp_dir
|
33
|
+
end
|
34
|
+
|
35
|
+
def test_filer_csv
|
36
|
+
path = File.join(fixtures_dir, 'tp_review_sandbox-assignment.csv')
|
37
|
+
assert(filer = Typingpool::Filer::CSV.new(path))
|
38
|
+
assert_equal(path, "#{filer}")
|
39
|
+
assert(data = filer.read)
|
40
|
+
assert_instance_of(Array, data)
|
41
|
+
assert_instance_of(Hash, data.first)
|
42
|
+
assert_respond_to(filer, :each)
|
43
|
+
assert_respond_to(filer, :map)
|
44
|
+
assert_respond_to(filer, :select)
|
45
|
+
assert(data.first['audio_url'])
|
46
|
+
assert_match(data.first['audio_url'], /^https?:\/\/\w/)
|
47
|
+
assert(filer.select{|r| r['audio_url'] }.count > 0)
|
48
|
+
in_temp_dir do |dir|
|
49
|
+
path = File.join(dir, 'filer-temp')
|
50
|
+
assert(filer2 = Typingpool::Filer::CSV.new(path))
|
51
|
+
assert_equal([], filer2.read)
|
52
|
+
assert(filer2.write(data))
|
53
|
+
assert_equal(Typingpool::Filer.new(filer.path).read, Typingpool::Filer.new(filer2.path).read)
|
54
|
+
assert_equal(filer.count, filer2.count)
|
55
|
+
filer2.each! do |row|
|
56
|
+
row['audio_url'] = row['audio_url'].reverse
|
57
|
+
end
|
58
|
+
rewritten = Typingpool::Filer.new(filer2.path).read
|
59
|
+
assert(Typingpool::Filer.new(filer.path).read != rewritten)
|
60
|
+
keys = filer2.first.keys
|
61
|
+
filer2.write_arrays(filer2.map{|row| keys.map{|key| row[key] } }, keys)
|
62
|
+
assert_equal(rewritten, Typingpool::Filer.new(filer2.path).read)
|
63
|
+
end #in_temp_dir
|
64
|
+
end
|
65
|
+
|
66
|
+
#This might be more comprehensive if it looped and ran once with
|
67
|
+
#Encoding set to 'US-ASCII' - but we'd have to be carefult to reset
|
68
|
+
#Encoding back to orig value. For now I just run ruby -E 'US-ASCII'
|
69
|
+
#test_unit_filer.rb now and again.
|
70
|
+
def test_filer_csv_utf8
|
71
|
+
path = File.join(fixtures_dir, 'tp_review_sandbox-assignment.csv')
|
72
|
+
assert(filer = Typingpool::Filer::CSV.new(path))
|
73
|
+
assert(data = filer.read)
|
74
|
+
assert_instance_of(Array, data)
|
75
|
+
assert_instance_of(Hash, data.first)
|
76
|
+
in_temp_dir do |dir|
|
77
|
+
path = File.join(dir, 'filer-temp')
|
78
|
+
assert(filer2 = Typingpool::Filer::CSV.new(path))
|
79
|
+
assert_equal([], filer2.read)
|
80
|
+
assert(filer2.write(data))
|
81
|
+
assert_equal(Typingpool::Filer.new(filer.path).read, Typingpool::Filer.new(filer2.path).read)
|
82
|
+
refute_empty(assignments = filer2.read)
|
83
|
+
assert(assignment = assignments.pop)
|
84
|
+
assert(assignment['transcript'] = File.read(File.join(fixtures_dir, 'utf8_transcript.txt'), :encoding => 'UTF-8'))
|
85
|
+
assignments.push(assignment)
|
86
|
+
assert(filer2.write(assignments))
|
87
|
+
refute_empty(filer2.read) #will throw ArgumentError: invalid byte sequence in US-ASCII in degenerate case
|
88
|
+
end #in_temp_dir
|
89
|
+
end
|
90
|
+
|
91
|
+
def test_filer_audio
|
92
|
+
mp3 = Typingpool::Filer::Audio.new(files_from(File.join(audio_dir, 'mp3')).first)
|
93
|
+
wma = Typingpool::Filer::Audio.new(files_from(File.join(audio_dir, 'wma')).first)
|
94
|
+
assert(mp3.mp3?)
|
95
|
+
assert(not(wma.mp3?))
|
96
|
+
in_temp_dir do |dir|
|
97
|
+
[mp3, wma].each do |file|
|
98
|
+
FileUtils.cp(file, dir)
|
99
|
+
end
|
100
|
+
mp3 = Typingpool::Filer::Audio.new(File.join(dir, File.basename(mp3)))
|
101
|
+
wma = Typingpool::Filer::Audio.new(File.join(dir, File.basename(wma)))
|
102
|
+
dest = Typingpool::Filer::Audio.new(File.join(dir, 'filer-temp.mp3'))
|
103
|
+
assert(converted = wma.to_mp3(dest))
|
104
|
+
assert_equal(dest.path, converted.path)
|
105
|
+
assert(wma.bitrate >= 30)
|
106
|
+
assert(wma.bitrate <= 40)
|
107
|
+
assert(converted.bitrate)
|
108
|
+
assert(converted.mp3?)
|
109
|
+
assert(chunks = mp3.split('0.25', 'filer-temp', Typingpool::Filer::Dir.new(dir)))
|
110
|
+
assert(not(chunks.to_a.empty?))
|
111
|
+
assert_equal(3, chunks.count)
|
112
|
+
chunks.each{|chunk| assert(File.exists? chunk) }
|
113
|
+
assert(chunks.first.offset)
|
114
|
+
assert_match(chunks.first.offset, /0\.00\b/)
|
115
|
+
assert_match(chunks.to_a[1].offset, /0\.25\b/)
|
116
|
+
end #in_temp_dir
|
117
|
+
end
|
118
|
+
|
119
|
+
def files_from(dir)
|
120
|
+
Dir.entries(dir).map{|entry| File.join(dir, entry) }.select{|path| File.file? path }.reject{|path| path.match(/^\./) }
|
121
|
+
end
|
122
|
+
|
123
|
+
def test_filer_files_base
|
124
|
+
file_selector = /tp[_-]collect/
|
125
|
+
dir = fixtures_dir
|
126
|
+
files = files_from(dir).select{|path| path.match(file_selector) }
|
127
|
+
dir = File.join(fixtures_dir, 'vcr')
|
128
|
+
files.push(*files_from(dir).select{|path| path.match(file_selector) })
|
129
|
+
assert(files.count > 0)
|
130
|
+
assert(filer = Typingpool::Filer::Files.new(files.map{|path| Typingpool::Filer.new(path) }))
|
131
|
+
assert_equal(filer.files.count, files.count)
|
132
|
+
assert_respond_to(filer, :each)
|
133
|
+
assert_respond_to(filer, :select)
|
134
|
+
assert_respond_to(filer, :map)
|
135
|
+
assert_instance_of(Typingpool::Filer::Files::Audio, filer.as(:audio))
|
136
|
+
end
|
137
|
+
|
138
|
+
def test_filer_files_audio
|
139
|
+
mp3s = files_from(File.join(audio_dir, 'mp3')).map{|path| Typingpool::Filer::Audio.new(path) }
|
140
|
+
wmas = files_from(File.join(audio_dir, 'wma')).map{|path| Typingpool::Filer::Audio.new(path) }
|
141
|
+
assert(mp3s.count > 0)
|
142
|
+
assert(wmas.count > 0)
|
143
|
+
assert(filer_mp3 = Typingpool::Filer::Files::Audio.new(mp3s))
|
144
|
+
assert(filer_wma = Typingpool::Filer::Files::Audio.new(wmas))
|
145
|
+
assert_equal(mp3s.count, filer_mp3.files.count)
|
146
|
+
assert_equal(wmas.count, filer_wma.files.count)
|
147
|
+
in_temp_dir do |dir|
|
148
|
+
dest_filer = Typingpool::Filer::Dir.new(dir)
|
149
|
+
assert(filer_conversion = filer_wma.to_mp3(dest_filer))
|
150
|
+
assert_equal(filer_wma.files.count, filer_conversion.files.count)
|
151
|
+
assert_equal(filer_wma.files.count, filer_conversion.select{|file| File.exists? file }.count)
|
152
|
+
assert_equal(filer_wma.files.count, filer_conversion.select{|file| file.mp3? }.count)
|
153
|
+
assert_equal(filer_conversion.files.count, dest_filer.files.count)
|
154
|
+
temp_path = File.join(dir, 'temp.mp3')
|
155
|
+
assert(filer_merged = filer_mp3.merge(Typingpool::Filer.new(temp_path)))
|
156
|
+
assert(File.size(filer_merged) > File.size(filer_mp3.first))
|
157
|
+
assert(filer_merged.mp3?)
|
158
|
+
assert(filer_merged.path != filer_mp3.first.path)
|
159
|
+
assert(filer_merged.path != filer_mp3.to_a[1].path)
|
160
|
+
end #in_temp_dir
|
161
|
+
end
|
162
|
+
|
163
|
+
def test_filer_dir
|
164
|
+
assert(dir = Typingpool::Filer::Dir.new(fixtures_dir))
|
165
|
+
assert_equal(fixtures_dir, dir.path)
|
166
|
+
dir2_path = File.join(fixtures_dir, 'doesntexist')
|
167
|
+
assert(not(File.exists? dir2_path))
|
168
|
+
assert(dir2 = Typingpool::Filer::Dir.new(dir2_path))
|
169
|
+
in_temp_dir do |dir|
|
170
|
+
dir3_path = File.join(dir, 'filer-dir-temp')
|
171
|
+
assert(not(File.exists? dir3_path))
|
172
|
+
assert(dir3 = Typingpool::Filer::Dir.create(dir3_path))
|
173
|
+
assert(File.exists? dir3_path)
|
174
|
+
assert_instance_of(Typingpool::Filer::Dir, dir3)
|
175
|
+
assert_nil(dir2 = Typingpool::Filer::Dir.named(File.basename(dir2_path), File.dirname(dir2_path)))
|
176
|
+
assert(dir3 = Typingpool::Filer::Dir.named(File.basename(dir3_path), File.dirname(dir3_path)))
|
177
|
+
assert_instance_of(Typingpool::Filer::Dir, dir3)
|
178
|
+
assert_equal(dir3_path, dir3.to_s)
|
179
|
+
assert_equal(dir3_path, dir3.to_str)
|
180
|
+
assert(filer = dir3.file('doesntexist'))
|
181
|
+
end #in_temp_dir
|
182
|
+
assert(filer = dir.file('vcr', 'tp-collect-1.yml'))
|
183
|
+
assert(File.exists? filer.path)
|
184
|
+
assert_instance_of(Typingpool::Filer, filer)
|
185
|
+
assert(csv = dir.file('tp_collect_sandbox-assignment.csv').as(:csv))
|
186
|
+
assert(File.exists? csv.path)
|
187
|
+
assert_instance_of(Typingpool::Filer::CSV, csv)
|
188
|
+
dir4 = Typingpool::Filer::Dir.new(audio_dir)
|
189
|
+
assert(audio = dir4.file('mp3', 'interview.1.mp3').as(:audio))
|
190
|
+
assert(File.exists? audio.path)
|
191
|
+
assert_instance_of(Typingpool::Filer::Audio, audio)
|
192
|
+
assert(filers = dir.files)
|
193
|
+
assert(not(filers.empty?))
|
194
|
+
assert_kind_of(Typingpool::Filer, filers.first)
|
195
|
+
assert(File.exists? filers.first.path)
|
196
|
+
dir_files = Dir.entries(dir.path).map{|entry| File.join(dir.path, entry)}.select{|path| File.file?(path) }.reject{|path| File.basename(path).match(/^\./) }
|
197
|
+
assert_equal(dir_files.count, filers.count)
|
198
|
+
assert(dir5 = dir.subdir('vcr'))
|
199
|
+
assert(File.exists? dir5.path)
|
200
|
+
assert_instance_of(Typingpool::Filer::Dir, dir5)
|
201
|
+
end
|
202
|
+
end #TestFiler
|
@@ -0,0 +1,168 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
|
4
|
+
|
5
|
+
require 'typingpool'
|
6
|
+
require 'typingpool/test'
|
7
|
+
require 'tmpdir'
|
8
|
+
require 'fileutils'
|
9
|
+
require 'uri'
|
10
|
+
|
11
|
+
class TestProject < Typingpool::Test
|
12
|
+
def test_project_base_new
|
13
|
+
assert(project = Typingpool::Project.new(project_default[:title], dummy_config))
|
14
|
+
assert_instance_of(Typingpool::Project, project)
|
15
|
+
assert_equal(project_default[:title], project.name)
|
16
|
+
assert_equal(dummy_config.to_hash.to_s, project.config.to_hash.to_s)
|
17
|
+
assert_raise(Typingpool::Error::Argument::Format) do
|
18
|
+
Typingpool::Project.new('one/two', dummy_config)
|
19
|
+
end #assert_raise...
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_project_base_bitrate
|
23
|
+
assert(project = Typingpool::Project.new(project_default[:title], dummy_config))
|
24
|
+
bitrate = rand(999) + 1
|
25
|
+
assert(project.bitrate = bitrate)
|
26
|
+
assert_equal(bitrate, project.bitrate)
|
27
|
+
bitrate = rand
|
28
|
+
assert_raises(Typingpool::Error::Argument::Format) do
|
29
|
+
project.bitrate = bitrate
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def test_project_base_interval
|
34
|
+
assert(project = Typingpool::Project.new(project_default[:title], dummy_config))
|
35
|
+
assert(project.interval = 120)
|
36
|
+
assert_equal(120, project.interval)
|
37
|
+
assert_equal(1, set_and_return_interval(project, '01'))
|
38
|
+
assert_equal(1, set_and_return_interval(project, 1))
|
39
|
+
assert_equal("0:01", project.send(:interval_as_time_string))
|
40
|
+
assert_equal(2, set_and_return_interval(project, 2))
|
41
|
+
assert_equal("0:02", project.send(:interval_as_time_string))
|
42
|
+
assert_raises(Typingpool::Error::Argument::Format) do
|
43
|
+
set_and_return_interval(project, 2.1)
|
44
|
+
end
|
45
|
+
assert_equal(11, set_and_return_interval(project,11))
|
46
|
+
assert_equal("0:11", project.send(:interval_as_time_string))
|
47
|
+
assert_equal(1, set_and_return_interval(project, '00:01'))
|
48
|
+
assert_equal(60, set_and_return_interval(project, '01:00'))
|
49
|
+
assert_equal("1:00", project.send(:interval_as_time_string))
|
50
|
+
assert_equal(60, set_and_return_interval(project, '1:00'))
|
51
|
+
assert_equal(3552, set_and_return_interval(project, '59:12'))
|
52
|
+
assert_equal("59:12", project.send(:interval_as_time_string))
|
53
|
+
assert_equal(3680, set_and_return_interval(project, '61:20'))
|
54
|
+
assert_equal("1:01:20", project.send(:interval_as_time_string))
|
55
|
+
assert_raises(Typingpool::Error::Argument::Format) do
|
56
|
+
set_and_return_interval(project, '61:20.1')
|
57
|
+
end
|
58
|
+
assert_equal(7152, set_and_return_interval(project, '01:59:12'))
|
59
|
+
assert_equal("1:59:12", project.send(:interval_as_time_string))
|
60
|
+
assert_equal(7152, set_and_return_interval(project, '1:59:12'))
|
61
|
+
assert_raises(Typingpool::Error::Argument::Format) do
|
62
|
+
set_and_return_interval(project, '01:59:12.01')
|
63
|
+
end
|
64
|
+
assert_equal(43152, set_and_return_interval(project, '11:59:12'))
|
65
|
+
assert_equal("11:59:12", project.send(:interval_as_time_string))
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_project_base_interval_as_mds
|
69
|
+
assert(project = Typingpool::Project.new(project_default[:title], dummy_config))
|
70
|
+
assert_equal('0.1', set_and_return_interval_as_mds(project, 1))
|
71
|
+
assert_equal('0.2', set_and_return_interval_as_mds(project, 2))
|
72
|
+
assert_equal('0.11', set_and_return_interval_as_mds(project,11))
|
73
|
+
assert_equal('1.0', set_and_return_interval_as_mds(project, '01:00'))
|
74
|
+
assert_equal('59.12', set_and_return_interval_as_mds(project, '59:12'))
|
75
|
+
assert_equal('61.20', set_and_return_interval_as_mds(project, '61:20'))
|
76
|
+
assert_equal('119.12', set_and_return_interval_as_mds(project, '1:59:12'))
|
77
|
+
assert_equal('719.12', set_and_return_interval_as_mds(project, '11:59:12'))
|
78
|
+
end
|
79
|
+
|
80
|
+
def test_project_base_local
|
81
|
+
config = dummy_config
|
82
|
+
config.transcripts = fixtures_dir
|
83
|
+
assert(project = Typingpool::Project.new('project', config))
|
84
|
+
assert_nil(project.local)
|
85
|
+
valid_transcript_dir = File.join(Typingpool::Utility.lib_dir, 'templates')
|
86
|
+
assert_kind_of(Typingpool::Project::Local, project.local(valid_transcript_dir))
|
87
|
+
config.transcripts = valid_transcript_dir
|
88
|
+
assert(project = Typingpool::Project.new('project', config))
|
89
|
+
assert_kind_of(Typingpool::Project::Local, project.local)
|
90
|
+
end
|
91
|
+
|
92
|
+
def test_project_base_remote
|
93
|
+
assert(project = Typingpool::Project.new(project_default[:title], dummy_config(1)))
|
94
|
+
assert_instance_of(Typingpool::Project::Remote::S3, project.remote)
|
95
|
+
assert(project = Typingpool::Project.new(project_default[:title], dummy_config(2)))
|
96
|
+
assert_instance_of(Typingpool::Project::Remote::SFTP, project.remote)
|
97
|
+
config = dummy_config(2)
|
98
|
+
config.to_hash.delete('sftp')
|
99
|
+
assert(project = Typingpool::Project.new(project_default[:title], config))
|
100
|
+
assert_raises(Typingpool::Error) do
|
101
|
+
project.remote
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
def test_project_base_create_local
|
106
|
+
config = dummy_config
|
107
|
+
in_temp_dir do |dir|
|
108
|
+
config.transcripts = dir
|
109
|
+
assert(project = Typingpool::Project.new(project_default[:title], config))
|
110
|
+
assert_nil(project.local)
|
111
|
+
assert_kind_of(Typingpool::Project::Local, project.create_local)
|
112
|
+
assert_kind_of(Typingpool::Project::Local, project.local)
|
113
|
+
end #in_temp_dir do
|
114
|
+
end
|
115
|
+
|
116
|
+
def test_project_base_create_assignment_csv
|
117
|
+
config = dummy_config
|
118
|
+
in_temp_dir do |dir|
|
119
|
+
config.transcripts = dir
|
120
|
+
assert(project = Typingpool::Project.new(project_default[:title], config))
|
121
|
+
project.create_local
|
122
|
+
assert(project.interval = '1:00')
|
123
|
+
assert_kind_of(Typingpool::Project::Local, project.local)
|
124
|
+
dummy_remote_files = (1..5).map{|n| "#{project_default[:title]}.#{n}" }
|
125
|
+
relative_path = ['data', 'assignment.csv']
|
126
|
+
voices = project_default[:voice].map do |voice|
|
127
|
+
spec = voice.split(/,\s*/)
|
128
|
+
hash = {:name => spec[0]}
|
129
|
+
hash[:description] = spec[1] if spec[1]
|
130
|
+
hash
|
131
|
+
end
|
132
|
+
assert(result = project.create_assignment_csv(:path => relative_path, :urls => dummy_remote_files, :unusual => project_default[:unusual], :voices => voices, ))
|
133
|
+
assert_includes(result, dir)
|
134
|
+
csv_file = File.join(dir, project_default[:title], *relative_path)
|
135
|
+
assert(File.exists? csv_file)
|
136
|
+
assert(File.file? csv_file)
|
137
|
+
assert(parsed = CSV.read(csv_file))
|
138
|
+
assignment_headers = parsed.shift
|
139
|
+
assert_equal(dummy_remote_files.count, parsed.count)
|
140
|
+
assert(chunk_index = assignment_headers.find_index('chunk'))
|
141
|
+
assert_equal('1:00', parsed.first[chunk_index].to_s)
|
142
|
+
assert(chunk_minutes_index = assignment_headers.find_index('chunk_minutes'))
|
143
|
+
assert_equal(1, parsed.first[chunk_minutes_index].to_i)
|
144
|
+
assert(chunk_seconds_index = assignment_headers.find_index('chunk_seconds'))
|
145
|
+
assert_empty(parsed.first[chunk_seconds_index].to_s)
|
146
|
+
assert(voices_count_index = assignment_headers.find_index('voices_count'))
|
147
|
+
assert_equal(voices.count, parsed.first[voices_count_index].to_i)
|
148
|
+
assert(voice_2_title_index = assignment_headers.find_index('voice2title'))
|
149
|
+
refute_empty(parsed.first[voice_2_title_index].to_s)
|
150
|
+
end #in_temp_dir do
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_local_basename_from_url
|
154
|
+
url = ['http://example.com/dir/', URI.escape('Example Title With Spaces & Ampersand.html')].join
|
155
|
+
assert_match(url, /%20/)
|
156
|
+
#assert(basename = Typingpool::Project.local_basename_from_url.u)
|
157
|
+
end
|
158
|
+
|
159
|
+
def set_and_return_interval(project, interval)
|
160
|
+
project.interval = interval
|
161
|
+
project.interval
|
162
|
+
end
|
163
|
+
|
164
|
+
def set_and_return_interval_as_mds(project, interval)
|
165
|
+
project.interval = interval
|
166
|
+
project.interval_as_min_dot_sec
|
167
|
+
end
|
168
|
+
end #TestProject
|
@@ -0,0 +1,68 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.join(File.dirname(File.dirname($0)), 'lib')
|
4
|
+
|
5
|
+
require 'typingpool'
|
6
|
+
require 'typingpool/test'
|
7
|
+
|
8
|
+
class TestProjectLocal < Typingpool::Test
|
9
|
+
def test_project_local_ours
|
10
|
+
assert(File.exists?(non_project_dir))
|
11
|
+
assert(File.directory?(non_project_dir))
|
12
|
+
refute(Typingpool::Project::Local.ours?(Typingpool::Filer::Dir.new(non_project_dir)))
|
13
|
+
assert(File.exists?(project_template_dir))
|
14
|
+
assert(Typingpool::Project::Local.ours?(Typingpool::Filer::Dir.new(project_template_dir)))
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_project_local_named
|
18
|
+
assert_nil(Typingpool::Project::Local.named(project_default[:title], fixtures_dir))
|
19
|
+
assert_kind_of(Typingpool::Project::Local, local = Typingpool::Project::Local.named('project', project_template_dir_parent))
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_project_local_valid_name
|
23
|
+
assert(Typingpool::Project::Local.valid_name?('hello, world'))
|
24
|
+
refute(Typingpool::Project::Local.valid_name?('hello / world'))
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_project_local_create
|
28
|
+
in_temp_dir do |dir|
|
29
|
+
assert(local = create_project_local(dir))
|
30
|
+
assert(File.exists?(local.path))
|
31
|
+
assert(File.directory?(local.path))
|
32
|
+
assert_kind_of(Typingpool::Project::Local, local)
|
33
|
+
refute_nil(local.id)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def test_project_instance
|
38
|
+
in_temp_dir do |dir|
|
39
|
+
assert(local = create_project_local(dir))
|
40
|
+
refute_nil(local.id)
|
41
|
+
assert_raise(Typingpool::Error) do
|
42
|
+
local.create_id
|
43
|
+
end
|
44
|
+
assert_nil(local.subtitle)
|
45
|
+
[:subtitle].each do |accessor|
|
46
|
+
text = 'hello, world'
|
47
|
+
assert(local.send("#{accessor.to_s}=".to_sym, text))
|
48
|
+
assert_equal(text, local.send(accessor))
|
49
|
+
end #[].each do...
|
50
|
+
end #in_temp_dir do..
|
51
|
+
end
|
52
|
+
|
53
|
+
def create_project_local(dir)
|
54
|
+
Typingpool::Project::Local.create(project_default[:title], dir, project_template_dir)
|
55
|
+
end
|
56
|
+
|
57
|
+
def project_template_dir
|
58
|
+
File.join(project_template_dir_parent, 'project')
|
59
|
+
end
|
60
|
+
|
61
|
+
def project_template_dir_parent
|
62
|
+
File.join(Typingpool::Utility.lib_dir, 'templates')
|
63
|
+
end
|
64
|
+
|
65
|
+
def non_project_dir
|
66
|
+
File.join(fixtures_dir, 'vcr')
|
67
|
+
end
|
68
|
+
end #TestProjectLocal
|