typingpool 0.7.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/LICENSE +20 -0
- data/README.markdown +452 -0
- data/lib/typingpool/amazon/hit/assignment/empty.rb +19 -0
- data/lib/typingpool/amazon/hit/assignment.rb +43 -0
- data/lib/typingpool/amazon/hit/full/fromsearchhits.rb +44 -0
- data/lib/typingpool/amazon/hit/full.rb +105 -0
- data/lib/typingpool/amazon/hit.rb +458 -0
- data/lib/typingpool/amazon/question.rb +45 -0
- data/lib/typingpool/amazon.rb +3 -677
- data/lib/typingpool/app/cli/formatter.rb +16 -0
- data/lib/typingpool/app/cli.rb +64 -0
- data/lib/typingpool/app/friendlyexceptions.rb +34 -0
- data/lib/typingpool/app.rb +2 -97
- data/lib/typingpool/config/root.rb +114 -0
- data/lib/typingpool/config.rb +13 -119
- data/lib/typingpool/filer/audio.rb +84 -0
- data/lib/typingpool/filer/csv.rb +57 -0
- data/lib/typingpool/filer/dir.rb +76 -0
- data/lib/typingpool/filer/files/audio.rb +63 -0
- data/lib/typingpool/filer/files.rb +55 -0
- data/lib/typingpool/filer.rb +4 -313
- data/lib/typingpool/project/local.rb +117 -0
- data/lib/typingpool/project/remote/s3.rb +135 -0
- data/lib/typingpool/project/remote/sftp.rb +100 -0
- data/lib/typingpool/project/remote.rb +65 -0
- data/lib/typingpool/project.rb +2 -396
- data/lib/typingpool/template/assignment.rb +17 -0
- data/lib/typingpool/template/env.rb +77 -0
- data/lib/typingpool/template.rb +2 -87
- data/lib/typingpool/test/script.rb +310 -0
- data/lib/typingpool/test.rb +1 -306
- data/lib/typingpool/transcript/chunk.rb +129 -0
- data/lib/typingpool/transcript.rb +1 -125
- data/lib/typingpool/utility/castable.rb +65 -0
- data/lib/typingpool/utility.rb +1 -61
- data/test/test_integration_script_6_tp_finish.rb +1 -0
- metadata +135 -81
@@ -0,0 +1,135 @@
|
|
1
|
+
module Typingpool
|
2
|
+
class Project
|
3
|
+
class Remote
|
4
|
+
|
5
|
+
#Subclass for storing remote files on Amazon Simple Storage
|
6
|
+
#Service (S3)
|
7
|
+
class S3 < Remote
|
8
|
+
require 'aws/s3'
|
9
|
+
|
10
|
+
#An Amazon Web Services "Access Key ID." Set from the
|
11
|
+
#Config#amazon value passed to Project::Remote::S3.new, but
|
12
|
+
#changeable.
|
13
|
+
attr_accessor :key
|
14
|
+
|
15
|
+
#An Amazon Web Services "Secret Access Key." Set from the
|
16
|
+
#Config#amazon value passed to Project::Remote::S3.new, but
|
17
|
+
#changeable.
|
18
|
+
attr_accessor :secret
|
19
|
+
|
20
|
+
#The S3 "bucket" where uploads will be stores. Set from the
|
21
|
+
#Config#amazon value passed to Project::Remote::S3.new, but
|
22
|
+
#changeable.
|
23
|
+
attr_accessor :bucket
|
24
|
+
|
25
|
+
#Returns the base URL, which is prepended to the remote
|
26
|
+
#files. This is either the 'url' attribute of the
|
27
|
+
#Config#amazon value passed to Project::Remote::S3.new or, if
|
28
|
+
#that attribute is not set, the value returned by
|
29
|
+
#'default_url' (e.g. "https://bucketname.s3.amazonaws.com").
|
30
|
+
attr_reader :url
|
31
|
+
|
32
|
+
#Constructor. Takes the project name and the result of calling
|
33
|
+
#the 'amazon' method on a Config instance (i.e. the amazon
|
34
|
+
#section of a Config file).
|
35
|
+
def initialize(name, amazon_config)
|
36
|
+
@name = name
|
37
|
+
@config = amazon_config
|
38
|
+
@key = @config.key or raise Error::File::Remote::S3, "Missing Amazon key in config"
|
39
|
+
@secret = @config.secret or raise Error::File::Remote::S3, "Missing Amazon secret in config"
|
40
|
+
@bucket = @config.bucket or raise Error::File::Remote::S3, "Missing Amazon bucket in config"
|
41
|
+
@url = @config.url || default_url
|
42
|
+
end
|
43
|
+
|
44
|
+
#The remote host (server) name, parsed from #url
|
45
|
+
def host
|
46
|
+
URI.parse(@url).host
|
47
|
+
end
|
48
|
+
|
49
|
+
#The remote path (directory), pased from #url
|
50
|
+
def path
|
51
|
+
URI.parse(@url).path
|
52
|
+
end
|
53
|
+
|
54
|
+
#Upload files/strings to S3, optionally changing the names in the process.
|
55
|
+
# ==== Params
|
56
|
+
#[io_streams] Enumerable collection of IO objects, like a File
|
57
|
+
# or StringIO instance.
|
58
|
+
#[as] Optional if the io_streams are File instances. Array of
|
59
|
+
# file basenames, used to name the destination
|
60
|
+
# files. Default is the basename of the Files
|
61
|
+
# passed in as io_streams.
|
62
|
+
# ==== Returns
|
63
|
+
#Array of URLs corresponding to the uploaded files.
|
64
|
+
def put(io_streams, as=io_streams.map{|file| File.basename(file)})
|
65
|
+
batch(io_streams) do |stream, i|
|
66
|
+
dest = as[i]
|
67
|
+
yield(stream, dest) if block_given?
|
68
|
+
begin
|
69
|
+
AWS::S3::S3Object.store(dest, stream, @bucket, :access => :public_read)
|
70
|
+
rescue AWS::S3::NoSuchBucket
|
71
|
+
make_bucket
|
72
|
+
retry
|
73
|
+
end
|
74
|
+
file_to_url(dest)
|
75
|
+
end #batch
|
76
|
+
end
|
77
|
+
|
78
|
+
#Delete objects from S3.
|
79
|
+
# ==== Params
|
80
|
+
#[files] Enumerable collection of file names. Should NOT
|
81
|
+
# include the bucket name (path).
|
82
|
+
# ==== Returns
|
83
|
+
#Array of booleans corresponding to whether the delete call
|
84
|
+
#succeeded.
|
85
|
+
def remove(files)
|
86
|
+
batch(files) do |file, i|
|
87
|
+
yield(file) if block_given?
|
88
|
+
AWS::S3::S3Object.delete(file, @bucket)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
protected
|
93
|
+
|
94
|
+
def batch(io_streams)
|
95
|
+
results = []
|
96
|
+
io_streams.each_with_index do |stream, i|
|
97
|
+
connect if i == 0
|
98
|
+
begin
|
99
|
+
results.push(yield(stream, i))
|
100
|
+
rescue AWS::S3::S3Exception => e
|
101
|
+
if e.message.match(/AWS::S3::SignatureDoesNotMatch/)
|
102
|
+
raise Error::File::Remote::S3::Credentials, "S3 operation failed with a signature error. This likely means your AWS key or secret is wrong. Error: #{e}"
|
103
|
+
else
|
104
|
+
raise Error::File::Remote::S3, "Your S3 operation failed with an Amazon error: #{e}"
|
105
|
+
end #if
|
106
|
+
end #begin
|
107
|
+
end #files.each
|
108
|
+
disconnect unless io_streams.empty?
|
109
|
+
results
|
110
|
+
end
|
111
|
+
|
112
|
+
def connect
|
113
|
+
AWS::S3::Base.establish_connection!(
|
114
|
+
:access_key_id => @key,
|
115
|
+
:secret_access_key => @secret,
|
116
|
+
:persistent => true,
|
117
|
+
:use_ssl => true
|
118
|
+
)
|
119
|
+
end
|
120
|
+
|
121
|
+
def disconnect
|
122
|
+
AWS::S3::Base.disconnect
|
123
|
+
end
|
124
|
+
|
125
|
+
def make_bucket
|
126
|
+
AWS::S3::Bucket.create(@bucket)
|
127
|
+
end
|
128
|
+
|
129
|
+
def default_url
|
130
|
+
"https://#{@bucket}.s3.amazonaws.com"
|
131
|
+
end
|
132
|
+
end #S3
|
133
|
+
end #Remote
|
134
|
+
end #Project
|
135
|
+
end #Typingpool
|
@@ -0,0 +1,100 @@
|
|
1
|
+
module Typingpool
|
2
|
+
class Project
|
3
|
+
class Remote
|
4
|
+
|
5
|
+
#Subclass for storing remote files on an SFTP server. Only
|
6
|
+
#public/private key authentication has been tested. There is not
|
7
|
+
#yet any provision for password-based authentication, though
|
8
|
+
#adding it should be trivial.
|
9
|
+
class SFTP < Remote
|
10
|
+
require 'net/sftp'
|
11
|
+
|
12
|
+
#Returns the remote host (server) name. This is set from
|
13
|
+
#Config#sftp#host.
|
14
|
+
attr_reader :host
|
15
|
+
|
16
|
+
#Returns the remote path (directory). This is set from
|
17
|
+
#Config#sftp#path.
|
18
|
+
attr_reader :path
|
19
|
+
|
20
|
+
#Returns the name of the user used to log in to the SFTP
|
21
|
+
#server. This is et from Config#sftp#user.
|
22
|
+
attr_reader :user
|
23
|
+
|
24
|
+
#Returns the base URL, which is prepended to the remote
|
25
|
+
#files. This is set from Config#sftp#url.
|
26
|
+
attr_reader :url
|
27
|
+
|
28
|
+
#Constructor. Takes the project name and a Config#sftp.
|
29
|
+
def initialize(name, sftp_config)
|
30
|
+
@name = name
|
31
|
+
@config = sftp_config
|
32
|
+
@user = @config.user or raise Error::File::Remote::SFTP, "No SFTP user specified in config"
|
33
|
+
@host = @config.host or raise Error::File::Remote::SFTP, "No SFTP host specified in config"
|
34
|
+
@url = @config.url or raise Error::File::Remote::SFTP, "No SFTP url specified in config"
|
35
|
+
@path = @config.path || ''
|
36
|
+
end
|
37
|
+
|
38
|
+
#See docs for Project::Remote::S3#put.
|
39
|
+
def put(io_streams, as=io_streams.map{|file| File.basename(file)})
|
40
|
+
begin
|
41
|
+
i = 0
|
42
|
+
batch(io_streams) do |stream, connection|
|
43
|
+
dest = as[i]
|
44
|
+
i += 1
|
45
|
+
yield(stream, dest) if block_given?
|
46
|
+
connection.upload(stream, join_with_path(dest))
|
47
|
+
file_to_url(dest)
|
48
|
+
end
|
49
|
+
rescue Net::SFTP::StatusException => e
|
50
|
+
raise Error::File::Remote::SFTP, "SFTP upload failed: #{e.description}"
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
#See docs for Project::Remote::S3#remove.
|
55
|
+
def remove(files)
|
56
|
+
requests = batch(files) do |file, connection|
|
57
|
+
yield(file) if block_given?
|
58
|
+
connection.remove(join_with_path(file))
|
59
|
+
end
|
60
|
+
failures = requests.reject{|request| request.response.ok?}
|
61
|
+
if not(failures.empty?)
|
62
|
+
summary = failures.map{|request| request.response.to_s}.join('; ')
|
63
|
+
raise Error::File::Remote::SFTP, "SFTP removal failed: #{summary}"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
protected
|
68
|
+
|
69
|
+
def connection
|
70
|
+
begin
|
71
|
+
Net::SFTP.start(@host, @user) do |connection|
|
72
|
+
yield(connection)
|
73
|
+
connection.loop
|
74
|
+
end
|
75
|
+
rescue Net::SSH::AuthenticationFailed
|
76
|
+
raise Error::File::Remote::SFTP, "SFTP authentication failed: #{$?}"
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def batch(files)
|
81
|
+
results = []
|
82
|
+
connection do |connection|
|
83
|
+
files.each do |file|
|
84
|
+
results.push(yield(file, connection))
|
85
|
+
end
|
86
|
+
end
|
87
|
+
return results
|
88
|
+
end
|
89
|
+
|
90
|
+
def join_with_path(file)
|
91
|
+
if @path
|
92
|
+
[@path, file].join('/')
|
93
|
+
else
|
94
|
+
file
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end #SFTP
|
98
|
+
end #Remote
|
99
|
+
end #Project
|
100
|
+
end #Typingpool
|
@@ -0,0 +1,65 @@
|
|
1
|
+
module Typingpool
|
2
|
+
class Project
|
3
|
+
#Representation of the Project instance on remote servers. This is
|
4
|
+
#basically a collection of audio files to be transcribed and HTML
|
5
|
+
#files containing instructions and a form for the
|
6
|
+
#transcribers. The backend can be Amazon S3 (the default) or an
|
7
|
+
#SFTP server. Each backend is encapsulated in its own subclass. A
|
8
|
+
#backend subclass must provide a 'put' method, which takes an
|
9
|
+
#array of IO streams and an optional array of remote file
|
10
|
+
#basenames; a 'remove' method, which takes an array of remote file
|
11
|
+
#basenames; and the methods 'host' and 'path', which return the
|
12
|
+
#location of the destination server and destination directory,
|
13
|
+
#respectively.
|
14
|
+
#
|
15
|
+
#Thus, there will always be 'put', 'remove', 'host' and 'path'
|
16
|
+
#methods available, in addition to the Project::Remote methods
|
17
|
+
#outlined below.
|
18
|
+
class Remote
|
19
|
+
require 'typingpool/project/remote/s3'
|
20
|
+
require 'typingpool/project/remote/sftp'
|
21
|
+
|
22
|
+
#The project name
|
23
|
+
attr_accessor :name
|
24
|
+
|
25
|
+
#Constructor. Takes the project name and a Config
|
26
|
+
#instance. Returns a Project::Remote::S3 or
|
27
|
+
#Project::Remote::SFTP instance, depending on the particulars of
|
28
|
+
#the Config. If there are sufficient config params to return
|
29
|
+
#EITHER an S3 or SFTP subclass, it will prefer the SFTP
|
30
|
+
#subclass.
|
31
|
+
def self.from_config(name, config)
|
32
|
+
if config.sftp
|
33
|
+
SFTP.new(name, config.sftp)
|
34
|
+
elsif config.amazon && config.amazon.bucket
|
35
|
+
S3.new(name, config.amazon)
|
36
|
+
else
|
37
|
+
raise Error, "No valid upload params found in config file (SFTP or Amazon info)"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
#Like project.remote.remove, except it takes an array of URLs
|
42
|
+
#instead an array of remote basenames, saving you from having to
|
43
|
+
#manually extract basenames from the URL.
|
44
|
+
def remove_urls(urls)
|
45
|
+
basenames = urls.map{|url| url_basename(url) }
|
46
|
+
remove(basenames){|file| yield(file) if block_given? }
|
47
|
+
end
|
48
|
+
|
49
|
+
#Given a file path, returns the URL to the file path were it to
|
50
|
+
#be uploaded by this instance.
|
51
|
+
def file_to_url(file)
|
52
|
+
"#{@url}/#{URI.escape(file)}"
|
53
|
+
end
|
54
|
+
|
55
|
+
#Given an URL, returns the file portion of the path, given the
|
56
|
+
#configuration of this instance.
|
57
|
+
def url_basename(url)
|
58
|
+
basename = url.split("#{self.url}/")[1] or raise Error, "Could not find base url '#{self.url}' within longer url '#{url}'"
|
59
|
+
URI.unescape(basename)
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
end #Remote
|
64
|
+
end #Project
|
65
|
+
end #Typingpool
|