typingpool 0.8.10 → 0.8.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +3 -0
- data/bin/tp-assign +9 -5
- data/bin/tp-config +6 -6
- data/bin/tp-finish +1 -1
- data/bin/tp-make +9 -3
- data/lib/typingpool/amazon/hit/full/fromsearchhits.rb +2 -1
- data/lib/typingpool/amazon/hit/full.rb +3 -2
- data/lib/typingpool/amazon/hit.rb +19 -1
- data/lib/typingpool/amazon.rb +1 -2
- data/lib/typingpool/app/cli.rb +3 -3
- data/lib/typingpool/app/friendlyexceptions.rb +1 -1
- data/lib/typingpool/app.rb +24 -6
- data/lib/typingpool/config/root.rb +1 -0
- data/lib/typingpool/config.rb +6 -5
- data/lib/typingpool/filer/audio.rb +2 -2
- data/lib/typingpool/filer/dir.rb +1 -1
- data/lib/typingpool/filer.rb +1 -1
- data/lib/typingpool/project/local.rb +3 -3
- data/lib/typingpool/project/remote.rb +4 -3
- data/lib/typingpool/project.rb +2 -1
- data/lib/typingpool/template.rb +2 -2
- data/lib/typingpool/test/fixtures/amazon-question-url.txt +1 -1
- data/lib/typingpool/test/fixtures/config-2 +1 -1
- data/lib/typingpool/test/fixtures/test_amazon_hit_full_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_1_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_1_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_1_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_3_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_3_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_3_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_4_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_4_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_4_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_6_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_assign_6_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_assign_6_time.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_finish_3_assignment.csv +6 -6
- data/lib/typingpool/test/fixtures/tp_finish_3_id.txt +1 -1
- data/lib/typingpool/test/fixtures/tp_finish_3_time.txt +1 -1
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_base.yml +35 -29
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_create.yml +52 -44
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_full.yml +36 -30
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_full_fromsearchhits.yml +69 -57
- data/lib/typingpool/test/fixtures/vcr/test_amazon_hit_retrievers.yml +94 -76
- data/lib/typingpool/test/fixtures/vcr/test_handles_hits_with_broken_external_question.yml +51 -53
- data/lib/typingpool/test/fixtures/vcr/tp_assign_1.yml +628 -628
- data/lib/typingpool/test/fixtures/vcr/tp_assign_2.yml +265 -3044
- data/lib/typingpool/test/fixtures/vcr/tp_assign_3.yml +629 -629
- data/lib/typingpool/test/fixtures/vcr/tp_assign_4.yml +65 -62
- data/lib/typingpool/test/fixtures/vcr/tp_assign_5.yml +623 -623
- data/lib/typingpool/test/fixtures/vcr/tp_assign_6.yml +592 -588
- data/lib/typingpool/test/fixtures/vcr/tp_assign_7.yml +195 -195
- data/lib/typingpool/test/fixtures/vcr/tp_finish_1.yml +91 -91
- data/lib/typingpool/test/fixtures/vcr/tp_finish_2.yml +76 -3989
- data/lib/typingpool/test/fixtures/vcr/tp_finish_3.yml +713 -695
- data/lib/typingpool/test/fixtures/vcr/tp_finish_4.yml +527 -3276
- data/lib/typingpool/test/fixtures/vcr/tp_finish_5.yml +338 -5602
- data/lib/typingpool/test/fixtures/vcr/tp_finish_6.yml +91 -91
- data/lib/typingpool/test/fixtures/vcr/tp_finish_7.yml +86 -3999
- data/lib/typingpool/test/fixtures/vcr/tp_finish_8.yml +51 -0
- data/lib/typingpool/test/script.rb +4 -6
- data/lib/typingpool/test.rb +3 -2
- data/lib/typingpool/utility.rb +6 -5
- data/lib/typingpool/version.rb +1 -1
- data/test/test_integration_script_1_tp_config.rb +12 -12
- data/test/test_integration_script_2_tp_make.rb +24 -8
- data/test/test_integration_script_3_tp_assign.rb +34 -19
- data/test/test_integration_script_4_tp_review.rb +1 -1
- data/test/test_integration_script_6_tp_finish.rb +5 -5
- data/test/test_unit_amazon.rb +12 -4
- data/test/test_unit_config.rb +2 -2
- data/test/test_unit_filer.rb +14 -14
- data/test/test_unit_project.rb +3 -3
- data/test/test_unit_project_local.rb +4 -4
- data/test/test_unit_project_remote.rb +1 -1
- data/test/test_unit_test.rb +2 -2
- data/typingpool.gemspec +3 -2
- metadata +24 -8
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 554974b6b35f179d3f50deb682ab0e5f9fd0964a
|
|
4
|
+
data.tar.gz: 0b67e2cdbb110c4f05bb820f2ad06241ead2fd7c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8463b4059f24d13d4d6245e155a0db3d7d6dde459e1612d698b6a12a95a57c1fa15d240763530e9b883948fd7f04e2d41bab968675af6e25eb072b476b87bfb1
|
|
7
|
+
data.tar.gz: 4848802256545cb9b3149f57ef0d7f9a63b251fb3ed90d05db1b13f8a396353e0600fd3f65eb3be2ac9d1ffbc855c0b770d0ba8c7deffbe88a2308bfd2449f1c
|
data/.gitignore
ADDED
data/bin/tp-assign
CHANGED
|
@@ -123,6 +123,12 @@ end.parse!
|
|
|
123
123
|
|
|
124
124
|
config = Typingpool::App::CLI.config_from_arg(options[:config]) or abort "No config file at '#{options[:config]}'"
|
|
125
125
|
|
|
126
|
+
begin
|
|
127
|
+
Typingpool::App.validate_sftp(config)
|
|
128
|
+
rescue Typingpool::Error => e
|
|
129
|
+
abort e.message
|
|
130
|
+
end
|
|
131
|
+
|
|
126
132
|
if options[:keyword].count > 0
|
|
127
133
|
config.assign.keywords = []
|
|
128
134
|
config.assign.keywords.push(*options[:keyword])
|
|
@@ -217,17 +223,15 @@ end
|
|
|
217
223
|
|
|
218
224
|
#Are there enough funds for this assignment?
|
|
219
225
|
cost = 0
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
cost = (needed_assignments.count * (config.assign.reward.to_f + amazon_fee_per_assignment) * 100).ceil.to_f / 100
|
|
226
|
+
#add in the Amazon commission
|
|
227
|
+
cost_per_assignment = Typingpool::Amazon::HIT.reward_to_total_cost(config.assign.reward.to_f)
|
|
228
|
+
cost = (needed_assignments.count * cost_per_assignment * 100).ceil.to_f / 100
|
|
224
229
|
cost_string = sprintf("%.2f", cost)
|
|
225
230
|
if options[:sandbox]
|
|
226
231
|
STDERR.puts "(Cost is $0 (nothing) since you're in the sandbox. Outside the sandbox this assignment would cost $#{cost_string}.)"
|
|
227
232
|
else
|
|
228
233
|
STDERR.puts "Checking your balance"
|
|
229
234
|
balance = RTurk.GetAccountBalance.amount
|
|
230
|
-
#"* 1.1" reflects 10% Amazon service surcharge
|
|
231
235
|
abort "Anticipated assignment cost of $#{cost_string} would exceed available balance of $#{balance}" if cost > balance
|
|
232
236
|
if options[:confirm] || config.assign.confirm
|
|
233
237
|
begin
|
data/bin/tp-config
CHANGED
|
@@ -29,7 +29,7 @@ end.parse!
|
|
|
29
29
|
config_path = ARGV.first || Typingpool::Config.default_file
|
|
30
30
|
config_path_full = File.expand_path(config_path)
|
|
31
31
|
config = nil
|
|
32
|
-
if File.
|
|
32
|
+
if File.exist? config_path_full
|
|
33
33
|
abort "Not a file: #{config_path}" unless File.file? config_path_full
|
|
34
34
|
STDERR.puts "Editing existing config file '#{config_path}'"
|
|
35
35
|
begin
|
|
@@ -41,7 +41,7 @@ else
|
|
|
41
41
|
abort "Invalid path '#{config_path}'" unless File.dirname(config_path_full) && File.directory?(File.dirname(config_path_full))
|
|
42
42
|
STDERR.puts "Making a new config file at '#{config_path}'"
|
|
43
43
|
config = Typingpool::Config.from_bundled_template
|
|
44
|
-
end #if File.
|
|
44
|
+
end #if File.exist? config_path
|
|
45
45
|
|
|
46
46
|
config.amazon ||= {}
|
|
47
47
|
config.amazon.key = ask('Your Amazon Web Services "Access Key ID"? '){|q| q.default = config.amazon.key if config.amazon.key }.to_s.chomp
|
|
@@ -65,7 +65,7 @@ end
|
|
|
65
65
|
|
|
66
66
|
unless config.transcripts
|
|
67
67
|
desktop_path = File.expand_path(File.join('~', 'Desktop'))
|
|
68
|
-
if File.
|
|
68
|
+
if File.exist?(desktop_path) && File.directory?(desktop_path)
|
|
69
69
|
config.transcripts = File.join(desktop_path, 'Transcripts')
|
|
70
70
|
else
|
|
71
71
|
config.transcripts = File.join('~', 'transcripts')
|
|
@@ -76,7 +76,7 @@ transcripts = nil
|
|
|
76
76
|
loop do
|
|
77
77
|
transcripts = ask('Working directory/folder for transcripts? '){|q| q.default = config['transcripts'] }.to_s.chomp
|
|
78
78
|
abort "Cannot proceed without a transcripts directory" if transcripts.empty?
|
|
79
|
-
if File.
|
|
79
|
+
if File.exist?(File.expand_path(transcripts))
|
|
80
80
|
if File.directory?(File.expand_path(transcripts))
|
|
81
81
|
break
|
|
82
82
|
else
|
|
@@ -85,7 +85,7 @@ loop do
|
|
|
85
85
|
else
|
|
86
86
|
FileUtils.mkdir(File.expand_path(transcripts))
|
|
87
87
|
break
|
|
88
|
-
end #if File.
|
|
88
|
+
end #if File.exist?...
|
|
89
89
|
end #loop do
|
|
90
90
|
config.transcripts = transcripts
|
|
91
91
|
|
|
@@ -94,7 +94,7 @@ unless config.templates
|
|
|
94
94
|
transcripts_dir_capitalized = (transcripts_dir[0].upcase == transcripts_dir[0])
|
|
95
95
|
templates = transcripts_dir_capitalized ? 'Templates' : 'templates'
|
|
96
96
|
config.templates = File.join(config['transcripts'], templates)
|
|
97
|
-
FileUtils.mkdir(config.templates) unless File.
|
|
97
|
+
FileUtils.mkdir(config.templates) unless File.exist? config.templates
|
|
98
98
|
end
|
|
99
99
|
|
|
100
100
|
unless config.cache
|
data/bin/tp-finish
CHANGED
|
@@ -110,7 +110,7 @@ end
|
|
|
110
110
|
if project
|
|
111
111
|
#Don't want to delete audio if there's another assignments file
|
|
112
112
|
#relying on it
|
|
113
|
-
delete_types = if (options[:sandbox] || (File.
|
|
113
|
+
delete_types = if (options[:sandbox] || (File.exist? project.local.file('data', 'sandbox-assignment.csv')))
|
|
114
114
|
['assignment']
|
|
115
115
|
else
|
|
116
116
|
['audio', 'assignment']
|
data/bin/tp-make
CHANGED
|
@@ -120,7 +120,7 @@ abort "No title specified\n\n#{options[:banner]}" if options[:title].to_s.empty?
|
|
|
120
120
|
options[:files].sort!
|
|
121
121
|
options[:files].each do |file|
|
|
122
122
|
File.extname(file) or abort "You need a file extension on the file '#{file}'"
|
|
123
|
-
File.
|
|
123
|
+
File.exist?(file) or abort "There is no file '#{file}'"
|
|
124
124
|
File.file?(file) or abort "Not a file: '#{file}'"
|
|
125
125
|
end
|
|
126
126
|
options[:unusual].map!{|unusual| unusual.split(/\s*,\s*/)}.flatten!
|
|
@@ -138,6 +138,12 @@ project = with_friendly_exceptions('project title', options[:title]) do
|
|
|
138
138
|
Typingpool::Project.new(options[:title], config)
|
|
139
139
|
end
|
|
140
140
|
|
|
141
|
+
begin
|
|
142
|
+
Typingpool::App.validate_sftp(config)
|
|
143
|
+
rescue Typingpool::Error => e
|
|
144
|
+
abort e.message
|
|
145
|
+
end
|
|
146
|
+
|
|
141
147
|
with_friendly_exceptions('--chunk argument', options[:chunk]) do
|
|
142
148
|
project.interval = options[:chunk] if options[:chunk]
|
|
143
149
|
end
|
|
@@ -147,14 +153,14 @@ with_friendly_exceptions('--bitrate argument', options[:bitrate]) do
|
|
|
147
153
|
end
|
|
148
154
|
|
|
149
155
|
if project.local
|
|
150
|
-
if (File.
|
|
156
|
+
if (File.exist?(project.local.file('data', 'assignment.csv')) &&
|
|
151
157
|
project.local.file('data', 'assignment.csv').as(:csv).read.select{|assignment| assignment['audio_uploaded'] == 'maybe' }.count > 0)
|
|
152
158
|
#project where the upload died partway through
|
|
153
159
|
STDERR.puts "Fixing incomplete project"
|
|
154
160
|
STDERR.puts "Determining which mp3s need uploading"
|
|
155
161
|
else
|
|
156
162
|
abort "The title '#{options[:title]}' is taken"
|
|
157
|
-
end #if(File.
|
|
163
|
+
end #if(File.exist?(project.local.file('data', 'assignment.csv') &&...
|
|
158
164
|
else
|
|
159
165
|
project.create_local
|
|
160
166
|
project.local.subtitle = options[:subtitle] if options[:subtitle]
|
|
@@ -22,7 +22,8 @@ module Typingpool
|
|
|
22
22
|
@assignments_completed = rturk_hit.completed_assignments
|
|
23
23
|
@assignments_pending = rturk_hit.pending_assignments
|
|
24
24
|
self.annotation = annotation
|
|
25
|
-
|
|
25
|
+
@checked_question = nil
|
|
26
|
+
end
|
|
26
27
|
|
|
27
28
|
def external_question_url
|
|
28
29
|
unless @checked_question
|
|
@@ -53,7 +53,8 @@ module Typingpool
|
|
|
53
53
|
#return something. In first use, must make an HTTP request to
|
|
54
54
|
#obtain the HTML.
|
|
55
55
|
def external_question
|
|
56
|
-
|
|
56
|
+
@external_question ||= nil
|
|
57
|
+
unless @external_question
|
|
57
58
|
if external_question_url && external_question_url.match(/^http/)
|
|
58
59
|
#expensive, obviously:
|
|
59
60
|
begin
|
|
@@ -66,7 +67,7 @@ module Typingpool
|
|
|
66
67
|
raise e unless e.message.match(/\b40[34]\b/)
|
|
67
68
|
end #begin
|
|
68
69
|
end #if external_question_url && external_question_url.match...
|
|
69
|
-
end #
|
|
70
|
+
end #unless @external_question
|
|
70
71
|
@external_question
|
|
71
72
|
end
|
|
72
73
|
|
|
@@ -227,6 +227,21 @@ module Typingpool
|
|
|
227
227
|
selected
|
|
228
228
|
end
|
|
229
229
|
end
|
|
230
|
+
|
|
231
|
+
def reward_to_total_cost(reward)
|
|
232
|
+
amazon_fee = reward.to_f * commission_rate
|
|
233
|
+
amazon_fee = minimum_commission if amazon_fee < minimum_commission
|
|
234
|
+
reward + amazon_fee
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def minimum_commission
|
|
238
|
+
0.01
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
def commission_rate
|
|
242
|
+
0.2
|
|
243
|
+
end
|
|
244
|
+
|
|
230
245
|
end #class << self
|
|
231
246
|
|
|
232
247
|
#Corresponds to the Amazon Mechanical Turk HIT#HITId
|
|
@@ -235,6 +250,9 @@ module Typingpool
|
|
|
235
250
|
#Constructor. Takes an RTurk::Hit instance.
|
|
236
251
|
def initialize(rturk_hit)
|
|
237
252
|
@id = rturk_hit.id
|
|
253
|
+
@full = nil
|
|
254
|
+
@assignment = nil
|
|
255
|
+
@ours = nil
|
|
238
256
|
end
|
|
239
257
|
|
|
240
258
|
#URL of the audio file associated with this HIT (the audio file
|
|
@@ -256,7 +274,7 @@ module Typingpool
|
|
|
256
274
|
#by parsing the #url. May be dropped in a future release.
|
|
257
275
|
def project_title_from_url(url=self.url)
|
|
258
276
|
matches = Project.url_regex.match(url) or raise Error::Argument::Format, "Unexpected format to url '#{url}'"
|
|
259
|
-
URI.
|
|
277
|
+
URI.decode_www_form_component(matches[2])
|
|
260
278
|
end
|
|
261
279
|
|
|
262
280
|
#Returns true if this HIT has an approved assignment associated
|
data/lib/typingpool/amazon.rb
CHANGED
data/lib/typingpool/app/cli.rb
CHANGED
|
@@ -12,7 +12,7 @@ module Typingpool
|
|
|
12
12
|
def config_from_arg(arg=nil)
|
|
13
13
|
if arg
|
|
14
14
|
path = File.expand_path(arg)
|
|
15
|
-
return unless File.
|
|
15
|
+
return unless File.exist?(path) && File.file?(path)
|
|
16
16
|
Config.file(path)
|
|
17
17
|
else
|
|
18
18
|
Config.file
|
|
@@ -40,13 +40,13 @@ module Typingpool
|
|
|
40
40
|
# ==== Returns
|
|
41
41
|
# A Project instance.
|
|
42
42
|
def project_from_arg_and_config(arg, config)
|
|
43
|
-
path = if (File.
|
|
43
|
+
path = if (File.exist?(arg) && File.directory?(arg))
|
|
44
44
|
config.transcripts = File.dirname(arg)
|
|
45
45
|
arg
|
|
46
46
|
else
|
|
47
47
|
abort "No 'transcripts' dir specified in your config file and '#{arg}' is not a valid path" unless config.transcripts
|
|
48
48
|
path = File.join(config.transcripts, arg)
|
|
49
|
-
abort "No such project '#{arg}' in dir '#{config.transcripts}'" unless File.
|
|
49
|
+
abort "No such project '#{arg}' in dir '#{config.transcripts}'" unless File.exist? path
|
|
50
50
|
abort "'#{arg}' is not a directory at '#{path}'" unless File.directory? path
|
|
51
51
|
path
|
|
52
52
|
end
|
|
@@ -23,7 +23,7 @@ module Typingpool
|
|
|
23
23
|
yield(*input)
|
|
24
24
|
rescue Typingpool::Error::Argument => exception
|
|
25
25
|
goodbye = "Could not make sense of #{name.to_s} "
|
|
26
|
-
goodbye += input.map{|
|
|
26
|
+
goodbye += input.map{|input_chunk| "'#{input_chunk}'" }.join(', ')
|
|
27
27
|
goodbye += ". #{exception.message}"
|
|
28
28
|
goodbye += '.' unless goodbye.match(/\.$/)
|
|
29
29
|
abort goodbye
|
data/lib/typingpool/app.rb
CHANGED
|
@@ -61,7 +61,7 @@ module Typingpool
|
|
|
61
61
|
project.remote.put(files.to_streams, remote_files) do |file, as|
|
|
62
62
|
yield(file, as) if block_given?
|
|
63
63
|
end
|
|
64
|
-
assignments_files = [assignments_file]
|
|
64
|
+
# assignments_files = [assignments_file] #why did we make this var and never use?
|
|
65
65
|
record_assignment_upload_status(assignments_file, uploading, ['audio'], 'yes')
|
|
66
66
|
uploading.map{|assignment| assignment['audio_url'] }
|
|
67
67
|
end
|
|
@@ -219,7 +219,7 @@ module Typingpool
|
|
|
219
219
|
need[hit.project_id] = false
|
|
220
220
|
project = Typingpool::Project.new(hit.project_title_from_url, config)
|
|
221
221
|
next unless project.local && (project.local.id == hit.project_id)
|
|
222
|
-
next if File.
|
|
222
|
+
next if File.exist? project.local.file(transcript_filename[:done])
|
|
223
223
|
by_project_id[hit.project_id] = {
|
|
224
224
|
:project => project,
|
|
225
225
|
:hits => [hit]
|
|
@@ -271,7 +271,7 @@ module Typingpool
|
|
|
271
271
|
rescue Error => e
|
|
272
272
|
abort "There was a fatal error with the transcript template: #{e}"
|
|
273
273
|
end #begin
|
|
274
|
-
File.delete(project.local.file(transcript_filename[:working])) if File.
|
|
274
|
+
File.delete(project.local.file(transcript_filename[:working])) if File.exist?(project.local.file(transcript_filename[:working]))
|
|
275
275
|
File.open(project.local.file(out_file), 'w') do |out|
|
|
276
276
|
out << template.render({:transcript => transcript})
|
|
277
277
|
end #File.open...
|
|
@@ -288,8 +288,8 @@ module Typingpool
|
|
|
288
288
|
#sandbox-assignmens.csv.
|
|
289
289
|
def ensure_sandbox_assignment_csv(project)
|
|
290
290
|
csv = project.local.file('data', 'sandbox-assignment.csv').as(:csv)
|
|
291
|
-
return csv if File.
|
|
292
|
-
raise Error, "No assignment CSV to copy" unless File.
|
|
291
|
+
return csv if File.exist? csv
|
|
292
|
+
raise Error, "No assignment CSV to copy" unless File.exist? project.local.file('data', 'assignment.csv')
|
|
293
293
|
csv.write(
|
|
294
294
|
project.local.file('data', 'assignment.csv').as(:csv).map do |assignment|
|
|
295
295
|
unrecord_hit_in_csv_row(assignment)
|
|
@@ -415,13 +415,31 @@ module Typingpool
|
|
|
415
415
|
missing = []
|
|
416
416
|
[['ffmpeg','-version'], ['mp3splt', '-v'], ['mp3wrap']].each do |cmdline|
|
|
417
417
|
begin
|
|
418
|
-
|
|
418
|
+
Open3.capture3(*cmdline)
|
|
419
419
|
rescue
|
|
420
420
|
missing.push(cmdline.first)
|
|
421
421
|
end #begin
|
|
422
422
|
end #...].each do |cmdline|
|
|
423
423
|
yield(missing) unless missing.empty?
|
|
424
424
|
end
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
#Checks if SFTP arguments are valid in case we want to abort
|
|
428
|
+
#before attempting upload
|
|
429
|
+
def validate_sftp(config)
|
|
430
|
+
if config.sftp
|
|
431
|
+
[:user, :host, :url].each do |param|
|
|
432
|
+
if (not(config.sftp.send(param)) || config.sftp.send(param).to_s.empty?)
|
|
433
|
+
raise Error, "Config file has an SFTP section but section is missing required parameter #{param}"
|
|
434
|
+
end
|
|
435
|
+
end #[:user, :host, :url].each...
|
|
436
|
+
unless config.sftp.url.match(/^https:\/\//i)
|
|
437
|
+
raise Error, "URL specified in the SFTP section of your config file must begin with 'https' per Amazon policy"
|
|
438
|
+
end
|
|
439
|
+
end #if config.sftp
|
|
440
|
+
end
|
|
441
|
+
|
|
442
|
+
|
|
425
443
|
#protected
|
|
426
444
|
|
|
427
445
|
def with_abort_on_url_mismatch(url_type='')
|
data/lib/typingpool/config.rb
CHANGED
|
@@ -4,7 +4,7 @@ module Typingpool
|
|
|
4
4
|
#often modified at runtime, for example in response to script flags.
|
|
5
5
|
#
|
|
6
6
|
#==Fields
|
|
7
|
-
# All listed defaults are populated when you run tp-
|
|
7
|
+
# All listed defaults are populated when you run tp-config.
|
|
8
8
|
#===Required
|
|
9
9
|
# [transcripts] Unexpanded path to working directory for
|
|
10
10
|
# transcripts. This is where tp-make creates new
|
|
@@ -19,7 +19,7 @@ module Typingpool
|
|
|
19
19
|
# [bucket] The name of the "bucket" on Amazon S3 where your uploads
|
|
20
20
|
# will be stored. Not required if you specify SFTP config
|
|
21
21
|
# instead (see below). Default: Generated for you when you
|
|
22
|
-
# run tp-
|
|
22
|
+
# run tp-config.
|
|
23
23
|
#
|
|
24
24
|
#===Optional
|
|
25
25
|
# [cache] Unexpanded path to the cache file (pstore). Default:
|
|
@@ -31,7 +31,8 @@ module Typingpool
|
|
|
31
31
|
#====amazon
|
|
32
32
|
# [url] Base URL to use when linking to files uploaded to S3. You
|
|
33
33
|
# may want to use this if you do custom domain mapping on
|
|
34
|
-
# S3. Default is https://$bucket.s3.amazonaws.com.
|
|
34
|
+
# S3. Default is https://$bucket.s3.amazonaws.com. MUST BE
|
|
35
|
+
# HTTPS URL PER AMAZON POLICY.
|
|
35
36
|
#====sftp
|
|
36
37
|
#If you provide SFTP config, the specified SFTP server will be used
|
|
37
38
|
#to host remote mp3 and html files rather than Amazon S3. At
|
|
@@ -41,8 +42,8 @@ module Typingpool
|
|
|
41
42
|
# [user] SFTP username
|
|
42
43
|
# [host] SFTP server
|
|
43
44
|
# [path] Files will be uploaded into this path. Optional.
|
|
44
|
-
# [url]
|
|
45
|
-
# preceding config.
|
|
45
|
+
# [url] Base URL to use when linking to files uploaded using the
|
|
46
|
+
# preceding config. MUST BE HTTPS URL PER AMAZON POLICY.
|
|
46
47
|
#====assign
|
|
47
48
|
#Defaults for tp-assign.
|
|
48
49
|
# [reward] Pay per transcription chunk in U.S. dollars. Default: 0.75.
|
|
@@ -26,7 +26,7 @@ module Typingpool
|
|
|
26
26
|
def to_mp3(dest=self.dir.file("#{File.basename(@path, '.*') }.mp3"), bitrate=nil)
|
|
27
27
|
bitrate ||= self.bitrate || 192
|
|
28
28
|
Utility.system_quietly('ffmpeg', '-i', @path, '-acodec', 'libmp3lame', '-ab', "#{bitrate}k", '-ac', '2', dest)
|
|
29
|
-
File.
|
|
29
|
+
File.exist?(dest) or raise Error::Shell, "Could not found output from `ffmpeg` on #{path}"
|
|
30
30
|
self.class.new(dest.path)
|
|
31
31
|
end
|
|
32
32
|
|
|
@@ -34,7 +34,7 @@ module Typingpool
|
|
|
34
34
|
#integer corresponding to kb/s, or nil if the bitrate could not
|
|
35
35
|
#be determined.
|
|
36
36
|
def bitrate
|
|
37
|
-
|
|
37
|
+
_, err, _ = Open3.capture3('ffmpeg', '-i', @path)
|
|
38
38
|
bitrate = err.match(/(\d+) kb\/s/)
|
|
39
39
|
return bitrate ? bitrate[1].to_i : nil
|
|
40
40
|
end
|
data/lib/typingpool/filer/dir.rb
CHANGED
|
@@ -28,7 +28,7 @@ module Typingpool
|
|
|
28
28
|
#the parent directory, returns it. If not, returns nil.
|
|
29
29
|
def named(name, in_dir)
|
|
30
30
|
path = File.join(in_dir, name)
|
|
31
|
-
if File.
|
|
31
|
+
if File.exist?(path) && File.directory?(path)
|
|
32
32
|
new(path)
|
|
33
33
|
end
|
|
34
34
|
end
|
data/lib/typingpool/filer.rb
CHANGED
|
@@ -51,7 +51,7 @@ module Typingpool
|
|
|
51
51
|
#the file layout inside the dir indicates it is a
|
|
52
52
|
#Project::Local instance.
|
|
53
53
|
def ours?(dir)
|
|
54
|
-
File.
|
|
54
|
+
File.exist?(dir.subdir('audio')) && File.exist?(dir.subdir('audio', 'originals'))
|
|
55
55
|
end
|
|
56
56
|
|
|
57
57
|
#Takes the name of a project and returns true if it is a valid
|
|
@@ -63,7 +63,7 @@ module Typingpool
|
|
|
63
63
|
rescue Errno::ENOENT
|
|
64
64
|
return false
|
|
65
65
|
end #begin
|
|
66
|
-
return File.
|
|
66
|
+
return File.exist?(File.join(dir, name))
|
|
67
67
|
end #Utility.in_temp_dir do...
|
|
68
68
|
end
|
|
69
69
|
|
|
@@ -86,7 +86,7 @@ module Typingpool
|
|
|
86
86
|
file('data',"#{sym.to_s}.txt").write(value)
|
|
87
87
|
end
|
|
88
88
|
define_method("delete_#{sym.to_s}".to_sym) do
|
|
89
|
-
if File.
|
|
89
|
+
if File.exist? file('data',"#{sym.to_s}.txt")
|
|
90
90
|
File.delete(file('data',"#{sym.to_s}.txt"))
|
|
91
91
|
end
|
|
92
92
|
end
|
|
@@ -19,7 +19,8 @@ module Typingpool
|
|
|
19
19
|
class Remote
|
|
20
20
|
require 'typingpool/project/remote/s3'
|
|
21
21
|
require 'typingpool/project/remote/sftp'
|
|
22
|
-
|
|
22
|
+
require 'erb'
|
|
23
|
+
|
|
23
24
|
#Constructor. Takes a Config
|
|
24
25
|
#instance. Returns a Project::Remote::S3 or
|
|
25
26
|
#Project::Remote::SFTP instance, depending on the particulars of
|
|
@@ -47,14 +48,14 @@ module Typingpool
|
|
|
47
48
|
#Given a file path, returns the URL to the file path were it to
|
|
48
49
|
#be uploaded by this instance.
|
|
49
50
|
def file_to_url(file)
|
|
50
|
-
"#{url}/#{
|
|
51
|
+
"#{url}/#{ERB::Util.url_encode(file)}"
|
|
51
52
|
end
|
|
52
53
|
|
|
53
54
|
#Given an URL, returns the file portion of the path, given the
|
|
54
55
|
#configuration of this instance.
|
|
55
56
|
def url_basename(url)
|
|
56
57
|
basename = url.split("#{self.url}/")[1] or raise Error, "Could not find base url '#{self.url}' within longer url '#{url}'"
|
|
57
|
-
URI.
|
|
58
|
+
URI.decode_www_form_component(basename)
|
|
58
59
|
end
|
|
59
60
|
|
|
60
61
|
|
data/lib/typingpool/project.rb
CHANGED
|
@@ -79,6 +79,7 @@ module Typingpool
|
|
|
79
79
|
seconds = seconds.round(2)
|
|
80
80
|
end
|
|
81
81
|
min_dot_sec = "#{(@interval.to_i / 60).floor}.#{seconds}"
|
|
82
|
+
min_dot_sec
|
|
82
83
|
end
|
|
83
84
|
|
|
84
85
|
#Takes an integer for setting the project.bitrate. The integer
|
|
@@ -165,7 +166,7 @@ module Typingpool
|
|
|
165
166
|
#Make this unneccesary.)
|
|
166
167
|
def self.local_basename_from_url(url)
|
|
167
168
|
matches = Project.url_regex.match(url) or raise Error::Argument::Format, "Unexpected format to url '#{url}'"
|
|
168
|
-
URI.
|
|
169
|
+
URI.decode_www_form_component([matches[2..4].join('.'), matches[5]].join)
|
|
169
170
|
end
|
|
170
171
|
|
|
171
172
|
protected
|
data/lib/typingpool/template.rb
CHANGED
|
@@ -25,7 +25,7 @@ module Typingpool
|
|
|
25
25
|
|
|
26
26
|
def validate_config(config)
|
|
27
27
|
if config.templates
|
|
28
|
-
File.
|
|
28
|
+
File.exist?(config.templates) or raise Error::File::NotExists, "No such templates dir: #{config.templates}"
|
|
29
29
|
File.directory?(config.templates) or raise Error::File::NotExists, "Templates dir not a directory: #{config.templates}"
|
|
30
30
|
end
|
|
31
31
|
end
|
|
@@ -71,7 +71,7 @@ module Typingpool
|
|
|
71
71
|
look_in.each do |dir|
|
|
72
72
|
extensions.each do |ext|
|
|
73
73
|
path = File.join(dir, [@path, ext].join)
|
|
74
|
-
if File.
|
|
74
|
+
if File.exist?(path) && File.file?(path)
|
|
75
75
|
return path
|
|
76
76
|
end
|
|
77
77
|
end
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
https://example.com/assignments/101.html
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
1483150167
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
audio_url,project_id,unusual,chunk,chunk_hours,chunk_minutes,chunk_seconds,voices_count,voice1,voice1title,voice2,voice2title
|
|
2
|
-
https://typingpool-tokyo-test-ffa60-
|
|
3
|
-
https://typingpool-tokyo-test-ffa60-
|
|
4
|
-
https://typingpool-tokyo-test-ffa60-
|
|
5
|
-
https://typingpool-tokyo-test-ffa60-
|
|
6
|
-
https://typingpool-tokyo-test-ffa60-
|
|
7
|
-
https://typingpool-tokyo-test-ffa60-
|
|
2
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.00.964909765946472311a24ef40288b4cc.EQUTOD.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
3
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.22.964909765946472311a24ef40288b4cc.JOJQTX.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
4
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.44.964909765946472311a24ef40288b4cc.SHAJNA.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
5
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.06.964909765946472311a24ef40288b4cc.EQAINL.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
6
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.28.964909765946472311a24ef40288b4cc.EUTSIT.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
7
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.50.964909765946472311a24ef40288b4cc.QHVPEM.mp3,964909765946472311a24ef40288b4cc,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
964909765946472311a24ef40288b4cc
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
1387336464
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
audio_url,project_id,unusual,chunk,chunk_hours,chunk_minutes,chunk_seconds,voices_count,voice1,voice1title,voice2,voice2title
|
|
2
|
-
https://typingpool-tokyo-test-ffa60-
|
|
3
|
-
https://typingpool-tokyo-test-ffa60-
|
|
4
|
-
https://typingpool-tokyo-test-ffa60-
|
|
5
|
-
https://typingpool-tokyo-test-ffa60-
|
|
6
|
-
https://typingpool-tokyo-test-ffa60-
|
|
7
|
-
https://typingpool-tokyo-test-ffa60-
|
|
2
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.00.83c0903b4536978722782651e2a5396f.LGUMHC.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
3
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.22.83c0903b4536978722782651e2a5396f.BFFHKE.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
4
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.44.83c0903b4536978722782651e2a5396f.JJGXSM.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
5
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.06.83c0903b4536978722782651e2a5396f.KRAUEE.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
6
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.28.83c0903b4536978722782651e2a5396f.CXCPGN.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
7
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.50.83c0903b4536978722782651e2a5396f.HDDKDX.mp3,83c0903b4536978722782651e2a5396f,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
83c0903b4536978722782651e2a5396f
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
1387336426
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
audio_url,project_id,unusual,chunk,chunk_hours,chunk_minutes,chunk_seconds,voices_count,voice1,voice1title,voice2,voice2title
|
|
2
|
-
https://typingpool-tokyo-test-ffa60-
|
|
3
|
-
https://typingpool-tokyo-test-ffa60-
|
|
4
|
-
https://typingpool-tokyo-test-ffa60-
|
|
5
|
-
https://typingpool-tokyo-test-ffa60-
|
|
6
|
-
https://typingpool-tokyo-test-ffa60-
|
|
7
|
-
https://typingpool-tokyo-test-ffa60-
|
|
2
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.00.4e27cfaef75990a462d514fb1144a6fe.YNAPWI.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
3
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.22.4e27cfaef75990a462d514fb1144a6fe.XHCIUL.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
4
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.00.44.4e27cfaef75990a462d514fb1144a6fe.MMPOHE.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
5
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.06.4e27cfaef75990a462d514fb1144a6fe.FHHCOU.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
6
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.28.4e27cfaef75990a462d514fb1144a6fe.CWKBFG.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
7
|
+
https://typingpool-tokyo-test-ffa60-zzzrt.s3.amazonaws.com/Typingpool's%20Test%20&%20Interview.01.50.4e27cfaef75990a462d514fb1144a6fe.XUAOTS.mp3,4e27cfaef75990a462d514fb1144a6fe,"Hack Day, Sunnyvale, Chad D",0:22,,,22,2,Ryan,"",Havi,hacker
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
4e27cfaef75990a462d514fb1144a6fe
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
1387336366
|