typingpool 0.8.10 → 0.8.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/bin/tp-assign +9 -5
- data/bin/tp-config +6 -6
- data/bin/tp-finish +1 -1
- data/bin/tp-make +9 -3
- data/lib/typingpool/amazon/hit/full/fromsearchhits.rb +2 -1
- data/lib/typingpool/amazon/hit/full.rb +3 -2
- data/lib/typingpool/amazon/hit.rb +19 -1
- data/lib/typingpool/amazon.rb +1 -2
- data/lib/typingpool/app/cli.rb +3 -3
- data/lib/typingpool/app/friendlyexceptions.rb +1 -1
- data/lib/typingpool/app.rb +24 -6
- data/lib/typingpool/config/root.rb +1 -0
- data/lib/typingpool/config.rb +6 -5
- data/lib/typingpool/filer/audio.rb +2 -2
- data/lib/typingpool/filer/dir.rb +1 -1
- data/lib/typingpool/filer.rb +1 -1
- data/lib/typingpool/project/local.rb +3 -3
- data/lib/typingpool/project/remote.rb +4 -3
- data/lib/typingpool/project.rb +2 -1
- data/lib/typingpool/template.rb +2 -2
- data/lib/typingpool/test/fixtures/amazon-question-url.txt +1 -1
- data/lib/typingpool/test/fixtures/config-2 +1 -1
- data/lib/typingpool/test/fixtures/test_amazon_hit_full_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_1_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_1_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_1_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_3_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_3_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_3_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_4_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_4_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_4_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_6_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_6_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_6_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_finish_3_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_finish_3_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_finish_3_time.txt +1 -1
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_base.yml +35 -29
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_create.yml +52 -44
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_full.yml +36 -30
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_full_fromsearchhits.yml +69 -57
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_retrievers.yml +94 -76
- data/lib/typingpool/test/fixtures/vcr/test_handles_hits_with_broken_external_question.yml +51 -53
- data/lib/typingpool/test/fixtures/vcr/tp_assign_1.yml +628 -628
- data/lib/typingpool/test/fixtures/vcr/tp_assign_2.yml +265 -3044
- data/lib/typingpool/test/fixtures/vcr/tp_assign_3.yml +629 -629
- data/lib/typingpool/test/fixtures/vcr/tp_assign_4.yml +65 -62
- data/lib/typingpool/test/fixtures/vcr/tp_assign_5.yml +623 -623
- data/lib/typingpool/test/fixtures/vcr/tp_assign_6.yml +592 -588
- data/lib/typingpool/test/fixtures/vcr/tp_assign_7.yml +195 -195
- data/lib/typingpool/test/fixtures/vcr/tp_finish_1.yml +91 -91
- data/lib/typingpool/test/fixtures/vcr/tp_finish_2.yml +76 -3989
- data/lib/typingpool/test/fixtures/vcr/tp_finish_3.yml +713 -695
- data/lib/typingpool/test/fixtures/vcr/tp_finish_4.yml +527 -3276
- data/lib/typingpool/test/fixtures/vcr/tp_finish_5.yml +338 -5602
- data/lib/typingpool/test/fixtures/vcr/tp_finish_6.yml +91 -91
- data/lib/typingpool/test/fixtures/vcr/tp_finish_7.yml +86 -3999
- data/lib/typingpool/test/fixtures/vcr/tp_finish_8.yml +51 -0
- data/lib/typingpool/test/script.rb +4 -6
- data/lib/typingpool/test.rb +3 -2
- data/lib/typingpool/utility.rb +6 -5
- data/lib/typingpool/version.rb +1 -1
- data/test/test_integration_script_1_tp_config.rb +12 -12
- data/test/test_integration_script_2_tp_make.rb +24 -8
- data/test/test_integration_script_3_tp_assign.rb +34 -19
- data/test/test_integration_script_4_tp_review.rb +1 -1
- data/test/test_integration_script_6_tp_finish.rb +5 -5
- data/test/test_unit_amazon.rb +12 -4
- data/test/test_unit_config.rb +2 -2
- data/test/test_unit_filer.rb +14 -14
- data/test/test_unit_project.rb +3 -3
- data/test/test_unit_project_local.rb +4 -4
- data/test/test_unit_project_remote.rb +1 -1
- data/test/test_unit_test.rb +2 -2
- data/typingpool.gemspec +3 -2
- metadata +24 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 554974b6b35f179d3f50deb682ab0e5f9fd0964a
|
4
|
+
data.tar.gz: 0b67e2cdbb110c4f05bb820f2ad06241ead2fd7c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8463b4059f24d13d4d6245e155a0db3d7d6dde459e1612d698b6a12a95a57c1fa15d240763530e9b883948fd7f04e2d41bab968675af6e25eb072b476b87bfb1
|
7
|
+
data.tar.gz: 4848802256545cb9b3149f57ef0d7f9a63b251fb3ed90d05db1b13f8a396353e0600fd3f65eb3be2ac9d1ffbc855c0b770d0ba8c7deffbe88a2308bfd2449f1c
|
data/.gitignore
ADDED
data/bin/tp-assign
CHANGED
@@ -123,6 +123,12 @@ end.parse!
|
|
123
123
|
|
124
124
|
config = Typingpool::App::CLI.config_from_arg(options[:config]) or abort "No config file at '#{options[:config]}'"
|
125
125
|
|
126
|
+
begin
|
127
|
+
Typingpool::App.validate_sftp(config)
|
128
|
+
rescue Typingpool::Error => e
|
129
|
+
abort e.message
|
130
|
+
end
|
131
|
+
|
126
132
|
if options[:keyword].count > 0
|
127
133
|
config.assign.keywords = []
|
128
134
|
config.assign.keywords.push(*options[:keyword])
|
@@ -217,17 +223,15 @@ end
|
|
217
223
|
|
218
224
|
#Are there enough funds for this assignment?
|
219
225
|
cost = 0
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
cost = (needed_assignments.count * (config.assign.reward.to_f + amazon_fee_per_assignment) * 100).ceil.to_f / 100
|
226
|
+
#add in the Amazon commission
|
227
|
+
cost_per_assignment = Typingpool::Amazon::HIT.reward_to_total_cost(config.assign.reward.to_f)
|
228
|
+
cost = (needed_assignments.count * cost_per_assignment * 100).ceil.to_f / 100
|
224
229
|
cost_string = sprintf("%.2f", cost)
|
225
230
|
if options[:sandbox]
|
226
231
|
STDERR.puts "(Cost is $0 (nothing) since you're in the sandbox. Outside the sandbox this assignment would cost $#{cost_string}.)"
|
227
232
|
else
|
228
233
|
STDERR.puts "Checking your balance"
|
229
234
|
balance = RTurk.GetAccountBalance.amount
|
230
|
-
#"* 1.1" reflects 10% Amazon service surcharge
|
231
235
|
abort "Anticipated assignment cost of $#{cost_string} would exceed available balance of $#{balance}" if cost > balance
|
232
236
|
if options[:confirm] || config.assign.confirm
|
233
237
|
begin
|
data/bin/tp-config
CHANGED
@@ -29,7 +29,7 @@ end.parse!
|
|
29
29
|
config_path = ARGV.first || Typingpool::Config.default_file
|
30
30
|
config_path_full = File.expand_path(config_path)
|
31
31
|
config = nil
|
32
|
-
if File.
|
32
|
+
if File.exist? config_path_full
|
33
33
|
abort "Not a file: #{config_path}" unless File.file? config_path_full
|
34
34
|
STDERR.puts "Editing existing config file '#{config_path}'"
|
35
35
|
begin
|
@@ -41,7 +41,7 @@ else
|
|
41
41
|
abort "Invalid path '#{config_path}'" unless File.dirname(config_path_full) && File.directory?(File.dirname(config_path_full))
|
42
42
|
STDERR.puts "Making a new config file at '#{config_path}'"
|
43
43
|
config = Typingpool::Config.from_bundled_template
|
44
|
-
end #if File.
|
44
|
+
end #if File.exist? config_path
|
45
45
|
|
46
46
|
config.amazon ||= {}
|
47
47
|
config.amazon.key = ask('Your Amazon Web Services "Access Key ID"? '){|q| q.default = config.amazon.key if config.amazon.key }.to_s.chomp
|
@@ -65,7 +65,7 @@ end
|
|
65
65
|
|
66
66
|
unless config.transcripts
|
67
67
|
desktop_path = File.expand_path(File.join('~', 'Desktop'))
|
68
|
-
if File.
|
68
|
+
if File.exist?(desktop_path) && File.directory?(desktop_path)
|
69
69
|
config.transcripts = File.join(desktop_path, 'Transcripts')
|
70
70
|
else
|
71
71
|
config.transcripts = File.join('~', 'transcripts')
|
@@ -76,7 +76,7 @@ transcripts = nil
|
|
76
76
|
loop do
|
77
77
|
transcripts = ask('Working directory/folder for transcripts? '){|q| q.default = config['transcripts'] }.to_s.chomp
|
78
78
|
abort "Cannot proceed without a transcripts directory" if transcripts.empty?
|
79
|
-
if File.
|
79
|
+
if File.exist?(File.expand_path(transcripts))
|
80
80
|
if File.directory?(File.expand_path(transcripts))
|
81
81
|
break
|
82
82
|
else
|
@@ -85,7 +85,7 @@ loop do
|
|
85
85
|
else
|
86
86
|
FileUtils.mkdir(File.expand_path(transcripts))
|
87
87
|
break
|
88
|
-
end #if File.
|
88
|
+
end #if File.exist?...
|
89
89
|
end #loop do
|
90
90
|
config.transcripts = transcripts
|
91
91
|
|
@@ -94,7 +94,7 @@ unless config.templates
|
|
94
94
|
transcripts_dir_capitalized = (transcripts_dir[0].upcase == transcripts_dir[0])
|
95
95
|
templates = transcripts_dir_capitalized ? 'Templates' : 'templates'
|
96
96
|
config.templates = File.join(config['transcripts'], templates)
|
97
|
-
FileUtils.mkdir(config.templates) unless File.
|
97
|
+
FileUtils.mkdir(config.templates) unless File.exist? config.templates
|
98
98
|
end
|
99
99
|
|
100
100
|
unless config.cache
|
data/bin/tp-finish
CHANGED
@@ -110,7 +110,7 @@ end
|
|
110
110
|
if project
|
111
111
|
#Don't want to delete audio if there's another assignments file
|
112
112
|
#relying on it
|
113
|
-
delete_types = if (options[:sandbox] || (File.
|
113
|
+
delete_types = if (options[:sandbox] || (File.exist? project.local.file('data', 'sandbox-assignment.csv')))
|
114
114
|
['assignment']
|
115
115
|
else
|
116
116
|
['audio', 'assignment']
|
data/bin/tp-make
CHANGED
@@ -120,7 +120,7 @@ abort "No title specified\n\n#{options[:banner]}" if options[:title].to_s.empty?
|
|
120
120
|
options[:files].sort!
|
121
121
|
options[:files].each do |file|
|
122
122
|
File.extname(file) or abort "You need a file extension on the file '#{file}'"
|
123
|
-
File.
|
123
|
+
File.exist?(file) or abort "There is no file '#{file}'"
|
124
124
|
File.file?(file) or abort "Not a file: '#{file}'"
|
125
125
|
end
|
126
126
|
options[:unusual].map!{|unusual| unusual.split(/\s*,\s*/)}.flatten!
|
@@ -138,6 +138,12 @@ project = with_friendly_exceptions('project title', options[:title]) do
|
|
138
138
|
Typingpool::Project.new(options[:title], config)
|
139
139
|
end
|
140
140
|
|
141
|
+
begin
|
142
|
+
Typingpool::App.validate_sftp(config)
|
143
|
+
rescue Typingpool::Error => e
|
144
|
+
abort e.message
|
145
|
+
end
|
146
|
+
|
141
147
|
with_friendly_exceptions('--chunk argument', options[:chunk]) do
|
142
148
|
project.interval = options[:chunk] if options[:chunk]
|
143
149
|
end
|
@@ -147,14 +153,14 @@ with_friendly_exceptions('--bitrate argument', options[:bitrate]) do
|
|
147
153
|
end
|
148
154
|
|
149
155
|
if project.local
|
150
|
-
if (File.
|
156
|
+
if (File.exist?(project.local.file('data', 'assignment.csv')) &&
|
151
157
|
project.local.file('data', 'assignment.csv').as(:csv).read.select{|assignment| assignment['audio_uploaded'] == 'maybe' }.count > 0)
|
152
158
|
#project where the upload died partway through
|
153
159
|
STDERR.puts "Fixing incomplete project"
|
154
160
|
STDERR.puts "Determining which mp3s need uploading"
|
155
161
|
else
|
156
162
|
abort "The title '#{options[:title]}' is taken"
|
157
|
-
end #if(File.
|
163
|
+
end #if(File.exist?(project.local.file('data', 'assignment.csv') &&...
|
158
164
|
else
|
159
165
|
project.create_local
|
160
166
|
project.local.subtitle = options[:subtitle] if options[:subtitle]
|
@@ -22,7 +22,8 @@ module Typingpool
|
|
22
22
|
@assignments_completed = rturk_hit.completed_assignments
|
23
23
|
@assignments_pending = rturk_hit.pending_assignments
|
24
24
|
self.annotation = annotation
|
25
|
-
|
25
|
+
@checked_question = nil
|
26
|
+
end
|
26
27
|
|
27
28
|
def external_question_url
|
28
29
|
unless @checked_question
|
@@ -53,7 +53,8 @@ module Typingpool
|
|
53
53
|
#return something. In first use, must make an HTTP request to
|
54
54
|
#obtain the HTML.
|
55
55
|
def external_question
|
56
|
-
|
56
|
+
@external_question ||= nil
|
57
|
+
unless @external_question
|
57
58
|
if external_question_url && external_question_url.match(/^http/)
|
58
59
|
#expensive, obviously:
|
59
60
|
begin
|
@@ -66,7 +67,7 @@ module Typingpool
|
|
66
67
|
raise e unless e.message.match(/\b40[34]\b/)
|
67
68
|
end #begin
|
68
69
|
end #if external_question_url && external_question_url.match...
|
69
|
-
end #
|
70
|
+
end #unless @external_question
|
70
71
|
@external_question
|
71
72
|
end
|
72
73
|
|
@@ -227,6 +227,21 @@ module Typingpool
|
|
227
227
|
selected
|
228
228
|
end
|
229
229
|
end
|
230
|
+
|
231
|
+
def reward_to_total_cost(reward)
|
232
|
+
amazon_fee = reward.to_f * commission_rate
|
233
|
+
amazon_fee = minimum_commission if amazon_fee < minimum_commission
|
234
|
+
reward + amazon_fee
|
235
|
+
end
|
236
|
+
|
237
|
+
def minimum_commission
|
238
|
+
0.01
|
239
|
+
end
|
240
|
+
|
241
|
+
def commission_rate
|
242
|
+
0.2
|
243
|
+
end
|
244
|
+
|
230
245
|
end #class << self
|
231
246
|
|
232
247
|
#Corresponds to the Amazon Mechanical Turk HIT#HITId
|
@@ -235,6 +250,9 @@ module Typingpool
|
|
235
250
|
#Constructor. Takes an RTurk::Hit instance.
|
236
251
|
def initialize(rturk_hit)
|
237
252
|
@id = rturk_hit.id
|
253
|
+
@full = nil
|
254
|
+
@assignment = nil
|
255
|
+
@ours = nil
|
238
256
|
end
|
239
257
|
|
240
258
|
#URL of the audio file associated with this HIT (the audio file
|
@@ -256,7 +274,7 @@ module Typingpool
|
|
256
274
|
#by parsing the #url. May be dropped in a future release.
|
257
275
|
def project_title_from_url(url=self.url)
|
258
276
|
matches = Project.url_regex.match(url) or raise Error::Argument::Format, "Unexpected format to url '#{url}'"
|
259
|
-
URI.
|
277
|
+
URI.decode_www_form_component(matches[2])
|
260
278
|
end
|
261
279
|
|
262
280
|
#Returns true if this HIT has an approved assignment associated
|
data/lib/typingpool/amazon.rb
CHANGED
data/lib/typingpool/app/cli.rb
CHANGED
@@ -12,7 +12,7 @@ module Typingpool
|
|
12
12
|
def config_from_arg(arg=nil)
|
13
13
|
if arg
|
14
14
|
path = File.expand_path(arg)
|
15
|
-
return unless File.
|
15
|
+
return unless File.exist?(path) && File.file?(path)
|
16
16
|
Config.file(path)
|
17
17
|
else
|
18
18
|
Config.file
|
@@ -40,13 +40,13 @@ module Typingpool
|
|
40
40
|
# ==== Returns
|
41
41
|
# A Project instance.
|
42
42
|
def project_from_arg_and_config(arg, config)
|
43
|
-
path = if (File.
|
43
|
+
path = if (File.exist?(arg) && File.directory?(arg))
|
44
44
|
config.transcripts = File.dirname(arg)
|
45
45
|
arg
|
46
46
|
else
|
47
47
|
abort "No 'transcripts' dir specified in your config file and '#{arg}' is not a valid path" unless config.transcripts
|
48
48
|
path = File.join(config.transcripts, arg)
|
49
|
-
abort "No such project '#{arg}' in dir '#{config.transcripts}'" unless File.
|
49
|
+
abort "No such project '#{arg}' in dir '#{config.transcripts}'" unless File.exist? path
|
50
50
|
abort "'#{arg}' is not a directory at '#{path}'" unless File.directory? path
|
51
51
|
path
|
52
52
|
end
|
@@ -23,7 +23,7 @@ module Typingpool
|
|
23
23
|
yield(*input)
|
24
24
|
rescue Typingpool::Error::Argument => exception
|
25
25
|
goodbye = "Could not make sense of #{name.to_s} "
|
26
|
-
goodbye += input.map{|
|
26
|
+
goodbye += input.map{|input_chunk| "'#{input_chunk}'" }.join(', ')
|
27
27
|
goodbye += ". #{exception.message}"
|
28
28
|
goodbye += '.' unless goodbye.match(/\.$/)
|
29
29
|
abort goodbye
|
data/lib/typingpool/app.rb
CHANGED
@@ -61,7 +61,7 @@ module Typingpool
|
|
61
61
|
project.remote.put(files.to_streams, remote_files) do |file, as|
|
62
62
|
yield(file, as) if block_given?
|
63
63
|
end
|
64
|
-
assignments_files = [assignments_file]
|
64
|
+
# assignments_files = [assignments_file] #why did we make this var and never use?
|
65
65
|
record_assignment_upload_status(assignments_file, uploading, ['audio'], 'yes')
|
66
66
|
uploading.map{|assignment| assignment['audio_url'] }
|
67
67
|
end
|
@@ -219,7 +219,7 @@ module Typingpool
|
|
219
219
|
need[hit.project_id] = false
|
220
220
|
project = Typingpool::Project.new(hit.project_title_from_url, config)
|
221
221
|
next unless project.local && (project.local.id == hit.project_id)
|
222
|
-
next if File.
|
222
|
+
next if File.exist? project.local.file(transcript_filename[:done])
|
223
223
|
by_project_id[hit.project_id] = {
|
224
224
|
:project => project,
|
225
225
|
:hits => [hit]
|
@@ -271,7 +271,7 @@ module Typingpool
|
|
271
271
|
rescue Error => e
|
272
272
|
abort "There was a fatal error with the transcript template: #{e}"
|
273
273
|
end #begin
|
274
|
-
File.delete(project.local.file(transcript_filename[:working])) if File.
|
274
|
+
File.delete(project.local.file(transcript_filename[:working])) if File.exist?(project.local.file(transcript_filename[:working]))
|
275
275
|
File.open(project.local.file(out_file), 'w') do |out|
|
276
276
|
out << template.render({:transcript => transcript})
|
277
277
|
end #File.open...
|
@@ -288,8 +288,8 @@ module Typingpool
|
|
288
288
|
#sandbox-assignmens.csv.
|
289
289
|
def ensure_sandbox_assignment_csv(project)
|
290
290
|
csv = project.local.file('data', 'sandbox-assignment.csv').as(:csv)
|
291
|
-
return csv if File.
|
292
|
-
raise Error, "No assignment CSV to copy" unless File.
|
291
|
+
return csv if File.exist? csv
|
292
|
+
raise Error, "No assignment CSV to copy" unless File.exist? project.local.file('data', 'assignment.csv')
|
293
293
|
csv.write(
|
294
294
|
project.local.file('data', 'assignment.csv').as(:csv).map do |assignment|
|
295
295
|
unrecord_hit_in_csv_row(assignment)
|
@@ -415,13 +415,31 @@ module Typingpool
|
|
415
415
|
missing = []
|
416
416
|
[['ffmpeg','-version'], ['mp3splt', '-v'], ['mp3wrap']].each do |cmdline|
|
417
417
|
begin
|
418
|
-
|
418
|
+
Open3.capture3(*cmdline)
|
419
419
|
rescue
|
420
420
|
missing.push(cmdline.first)
|
421
421
|
end #begin
|
422
422
|
end #...].each do |cmdline|
|
423
423
|
yield(missing) unless missing.empty?
|
424
424
|
end
|
425
|
+
|
426
|
+
|
427
|
+
#Checks if SFTP arguments are valid in case we want to abort
|
428
|
+
#before attempting upload
|
429
|
+
def validate_sftp(config)
|
430
|
+
if config.sftp
|
431
|
+
[:user, :host, :url].each do |param|
|
432
|
+
if (not(config.sftp.send(param)) || config.sftp.send(param).to_s.empty?)
|
433
|
+
raise Error, "Config file has an SFTP section but section is missing required parameter #{param}"
|
434
|
+
end
|
435
|
+
end #[:user, :host, :url].each...
|
436
|
+
unless config.sftp.url.match(/^https:\/\//i)
|
437
|
+
raise Error, "URL specified in the SFTP section of your config file must begin with 'https' per Amazon policy"
|
438
|
+
end
|
439
|
+
end #if config.sftp
|
440
|
+
end
|
441
|
+
|
442
|
+
|
425
443
|
#protected
|
426
444
|
|
427
445
|
def with_abort_on_url_mismatch(url_type='')
|
data/lib/typingpool/config.rb
CHANGED
@@ -4,7 +4,7 @@ module Typingpool
|
|
4
4
|
#often modified at runtime, for example in response to script flags.
|
5
5
|
#
|
6
6
|
#==Fields
|
7
|
-
# All listed defaults are populated when you run tp-
|
7
|
+
# All listed defaults are populated when you run tp-config.
|
8
8
|
#===Required
|
9
9
|
# [transcripts] Unexpanded path to working directory for
|
10
10
|
# transcripts. This is where tp-make creates new
|
@@ -19,7 +19,7 @@ module Typingpool
|
|
19
19
|
# [bucket] The name of the "bucket" on Amazon S3 where your uploads
|
20
20
|
# will be stored. Not required if you specify SFTP config
|
21
21
|
# instead (see below). Default: Generated for you when you
|
22
|
-
# run tp-
|
22
|
+
# run tp-config.
|
23
23
|
#
|
24
24
|
#===Optional
|
25
25
|
# [cache] Unexpanded path to the cache file (pstore). Default:
|
@@ -31,7 +31,8 @@ module Typingpool
|
|
31
31
|
#====amazon
|
32
32
|
# [url] Base URL to use when linking to files uploaded to S3. You
|
33
33
|
# may want to use this if you do custom domain mapping on
|
34
|
-
# S3. Default is https://$bucket.s3.amazonaws.com.
|
34
|
+
# S3. Default is https://$bucket.s3.amazonaws.com. MUST BE
|
35
|
+
# HTTPS URL PER AMAZON POLICY.
|
35
36
|
#====sftp
|
36
37
|
#If you provide SFTP config, the specified SFTP server will be used
|
37
38
|
#to host remote mp3 and html files rather than Amazon S3. At
|
@@ -41,8 +42,8 @@ module Typingpool
|
|
41
42
|
# [user] SFTP username
|
42
43
|
# [host] SFTP server
|
43
44
|
# [path] Files will be uploaded into this path. Optional.
|
44
|
-
# [url]
|
45
|
-
# preceding config.
|
45
|
+
# [url] Base URL to use when linking to files uploaded using the
|
46
|
+
# preceding config. MUST BE HTTPS URL PER AMAZON POLICY.
|
46
47
|
#====assign
|
47
48
|
#Defaults for tp-assign.
|
48
49
|
# [reward] Pay per transcription chunk in U.S. dollars. Default: 0.75.
|
@@ -26,7 +26,7 @@ module Typingpool
|
|
26
26
|
def to_mp3(dest=self.dir.file("#{File.basename(@path, '.*') }.mp3"), bitrate=nil)
|
27
27
|
bitrate ||= self.bitrate || 192
|
28
28
|
Utility.system_quietly('ffmpeg', '-i', @path, '-acodec', 'libmp3lame', '-ab', "#{bitrate}k", '-ac', '2', dest)
|
29
|
-
File.
|
29
|
+
File.exist?(dest) or raise Error::Shell, "Could not found output from `ffmpeg` on #{path}"
|
30
30
|
self.class.new(dest.path)
|
31
31
|
end
|
32
32
|
|
@@ -34,7 +34,7 @@ module Typingpool
|
|
34
34
|
#integer corresponding to kb/s, or nil if the bitrate could not
|
35
35
|
#be determined.
|
36
36
|
def bitrate
|
37
|
-
|
37
|
+
_, err, _ = Open3.capture3('ffmpeg', '-i', @path)
|
38
38
|
bitrate = err.match(/(\d+) kb\/s/)
|
39
39
|
return bitrate ? bitrate[1].to_i : nil
|
40
40
|
end
|
data/lib/typingpool/filer/dir.rb
CHANGED
@@ -28,7 +28,7 @@ module Typingpool
|
|
28
28
|
#the parent directory, returns it. If not, returns nil.
|
29
29
|
def named(name, in_dir)
|
30
30
|
path = File.join(in_dir, name)
|
31
|
-
if File.
|
31
|
+
if File.exist?(path) && File.directory?(path)
|
32
32
|
new(path)
|
33
33
|
end
|
34
34
|
end
|
data/lib/typingpool/filer.rb
CHANGED
@@ -51,7 +51,7 @@ module Typingpool
|
|
51
51
|
#the file layout inside the dir indicates it is a
|
52
52
|
#Project::Local instance.
|
53
53
|
def ours?(dir)
|
54
|
-
File.
|
54
|
+
File.exist?(dir.subdir('audio')) && File.exist?(dir.subdir('audio', 'originals'))
|
55
55
|
end
|
56
56
|
|
57
57
|
#Takes the name of a project and returns true if it is a valid
|
@@ -63,7 +63,7 @@ module Typingpool
|
|
63
63
|
rescue Errno::ENOENT
|
64
64
|
return false
|
65
65
|
end #begin
|
66
|
-
return File.
|
66
|
+
return File.exist?(File.join(dir, name))
|
67
67
|
end #Utility.in_temp_dir do...
|
68
68
|
end
|
69
69
|
|
@@ -86,7 +86,7 @@ module Typingpool
|
|
86
86
|
file('data',"#{sym.to_s}.txt").write(value)
|
87
87
|
end
|
88
88
|
define_method("delete_#{sym.to_s}".to_sym) do
|
89
|
-
if File.
|
89
|
+
if File.exist? file('data',"#{sym.to_s}.txt")
|
90
90
|
File.delete(file('data',"#{sym.to_s}.txt"))
|
91
91
|
end
|
92
92
|
end
|
@@ -19,7 +19,8 @@ module Typingpool
|
|
19
19
|
class Remote
|
20
20
|
require 'typingpool/project/remote/s3'
|
21
21
|
require 'typingpool/project/remote/sftp'
|
22
|
-
|
22
|
+
require 'erb'
|
23
|
+
|
23
24
|
#Constructor. Takes a Config
|
24
25
|
#instance. Returns a Project::Remote::S3 or
|
25
26
|
#Project::Remote::SFTP instance, depending on the particulars of
|
@@ -47,14 +48,14 @@ module Typingpool
|
|
47
48
|
#Given a file path, returns the URL to the file path were it to
|
48
49
|
#be uploaded by this instance.
|
49
50
|
def file_to_url(file)
|
50
|
-
"#{url}/#{
|
51
|
+
"#{url}/#{ERB::Util.url_encode(file)}"
|
51
52
|
end
|
52
53
|
|
53
54
|
#Given an URL, returns the file portion of the path, given the
|
54
55
|
#configuration of this instance.
|
55
56
|
def url_basename(url)
|
56
57
|
basename = url.split("#{self.url}/")[1] or raise Error, "Could not find base url '#{self.url}' within longer url '#{url}'"
|
57
|
-
URI.
|
58
|
+
URI.decode_www_form_component(basename)
|
58
59
|
end
|
59
60
|
|
60
61
|
|
data/lib/typingpool/project.rb
CHANGED
@@ -79,6 +79,7 @@ module Typingpool
|
|
79
79
|
seconds = seconds.round(2)
|
80
80
|
end
|
81
81
|
min_dot_sec = "#{(@interval.to_i / 60).floor}.#{seconds}"
|
82
|
+
min_dot_sec
|
82
83
|
end
|
83
84
|
|
84
85
|
#Takes an integer for setting the project.bitrate. The integer
|
@@ -165,7 +166,7 @@ module Typingpool
|
|
165
166
|
#Make this unneccesary.)
|
166
167
|
def self.local_basename_from_url(url)
|
167
168
|
matches = Project.url_regex.match(url) or raise Error::Argument::Format, "Unexpected format to url '#{url}'"
|
168
|
-
URI.
|
169
|
+
URI.decode_www_form_component([matches[2..4].join('.'), matches[5]].join)
|
169
170
|
end
|
170
171
|
|
171
172
|
protected
|
data/lib/typingpool/template.rb
CHANGED
@@ -25,7 +25,7 @@ module Typingpool
|
|
25
25
|
|
26
26
|
def validate_config(config)
|
27
27
|
if config.templates
|
28
|
-
File.
|
28
|
+
File.exist?(config.templates) or raise Error::File::NotExists, "No such templates dir: #{config.templates}"
|
29
29
|
File.directory?(config.templates) or raise Error::File::NotExists, "Templates dir not a directory: #{config.templates}"
|
30
30
|
end
|
31
31
|
end
|
@@ -71,7 +71,7 @@ module Typingpool
|
|
71
71
|
look_in.each do |dir|
|
72
72
|
extensions.each do |ext|
|
73
73
|
path = File.join(dir, [@path, ext].join)
|
74
|
-
if File.
|
74
|
+
if File.exist?(path) && File.file?(path)
|
75
75
|
return path
|
76
76
|
end
|
77
77
|
end
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
https://example.com/assignments/101.html
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
1483150167
|
@@ -1,7 +1,7 @@
|
|
1
1
|
audio_url,project_id,unusual,chunk,chunk_hours,chunk_minutes,chunk_seconds,voices_count,voice1,voice1title,voice2,voice2title
|
2
|
-
https://typingpool-tokyo-test-ffa60-
|
3
|
-
https://typingpool-tokyo-test-ffa60-
|
4
|
-
https://typingpool-tokyo-test-ffa60-
|
5
|
-
https://typingpool-tokyo-test-ffa60-
|
6
|
-
https://typingpool-tokyo-test-ffa60-
|
7
|
-
https://typingpool-tokyo-test-ffa60-
|
2
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.00.964909765946472311a24ef40288b4cc.EQUTOD.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
3
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.22.964909765946472311a24ef40288b4cc.JOJQTX.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
4
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.44.964909765946472311a24ef40288b4cc.SHAJNA.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
5
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.06.964909765946472311a24ef40288b4cc.EQAINL.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
6
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.28.964909765946472311a24ef40288b4cc.EUTSIT.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
7
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.50.964909765946472311a24ef40288b4cc.QHVPEM.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
964909765946472311a24ef40288b4cc
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
1387336464
|
@@ -1,7 +1,7 @@
|
|
1
1
|
audio_url,project_id,unusual,chunk,chunk_hours,chunk_minutes,chunk_seconds,voices_count,voice1,voice1title,voice2,voice2title
|
2
|
-
https://typingpool-tokyo-test-ffa60-
|
3
|
-
https://typingpool-tokyo-test-ffa60-
|
4
|
-
https://typingpool-tokyo-test-ffa60-
|
5
|
-
https://typingpool-tokyo-test-ffa60-
|
6
|
-
https://typingpool-tokyo-test-ffa60-
|
7
|
-
https://typingpool-tokyo-test-ffa60-
|
2
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.00.83c0903b4536978722782651e2a5396f.LGUMHC.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
3
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.22.83c0903b4536978722782651e2a5396f.BFFHKE.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
4
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.44.83c0903b4536978722782651e2a5396f.JJGXSM.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
5
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.06.83c0903b4536978722782651e2a5396f.KRAUEE.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
6
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.28.83c0903b4536978722782651e2a5396f.CXCPGN.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
7
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.50.83c0903b4536978722782651e2a5396f.HDDKDX.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
83c0903b4536978722782651e2a5396f
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
1387336426
|
@@ -1,7 +1,7 @@
|
|
1
1
|
audio_url,project_id,unusual,chunk,chunk_hours,chunk_minutes,chunk_seconds,voices_count,voice1,voice1title,voice2,voice2title
|
2
|
-
https://typingpool-tokyo-test-ffa60-
|
3
|
-
https://typingpool-tokyo-test-ffa60-
|
4
|
-
https://typingpool-tokyo-test-ffa60-
|
5
|
-
https://typingpool-tokyo-test-ffa60-
|
6
|
-
https://typingpool-tokyo-test-ffa60-
|
7
|
-
https://typingpool-tokyo-test-ffa60-
|
2
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.00.4e27cfaef75990a462d514fb1144a6fe.YNAPWI.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
3
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.22.4e27cfaef75990a462d514fb1144a6fe.XHCIUL.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
4
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.44.4e27cfaef75990a462d514fb1144a6fe.MMPOHE.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
5
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.06.4e27cfaef75990a462d514fb1144a6fe.FHHCOU.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
6
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.28.4e27cfaef75990a462d514fb1144a6fe.CWKBFG.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
7
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.50.4e27cfaef75990a462d514fb1144a6fe.XUAOTS.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
4e27cfaef75990a462d514fb1144a6fe
|
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
1387336366
|