typingpool 0.7.0 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. data/LICENSE +20 -0
  2. data/README.markdown +452 -0
  3. data/lib/typingpool/amazon/hit/assignment/empty.rb +19 -0
  4. data/lib/typingpool/amazon/hit/assignment.rb +43 -0
  5. data/lib/typingpool/amazon/hit/full/fromsearchhits.rb +44 -0
  6. data/lib/typingpool/amazon/hit/full.rb +105 -0
  7. data/lib/typingpool/amazon/hit.rb +458 -0
  8. data/lib/typingpool/amazon/question.rb +45 -0
  9. data/lib/typingpool/amazon.rb +3 -677
  10. data/lib/typingpool/app/cli/formatter.rb +16 -0
  11. data/lib/typingpool/app/cli.rb +64 -0
  12. data/lib/typingpool/app/friendlyexceptions.rb +34 -0
  13. data/lib/typingpool/app.rb +2 -97
  14. data/lib/typingpool/config/root.rb +114 -0
  15. data/lib/typingpool/config.rb +13 -119
  16. data/lib/typingpool/filer/audio.rb +84 -0
  17. data/lib/typingpool/filer/csv.rb +57 -0
  18. data/lib/typingpool/filer/dir.rb +76 -0
  19. data/lib/typingpool/filer/files/audio.rb +63 -0
  20. data/lib/typingpool/filer/files.rb +55 -0
  21. data/lib/typingpool/filer.rb +4 -313
  22. data/lib/typingpool/project/local.rb +117 -0
  23. data/lib/typingpool/project/remote/s3.rb +135 -0
  24. data/lib/typingpool/project/remote/sftp.rb +100 -0
  25. data/lib/typingpool/project/remote.rb +65 -0
  26. data/lib/typingpool/project.rb +2 -396
  27. data/lib/typingpool/template/assignment.rb +17 -0
  28. data/lib/typingpool/template/env.rb +77 -0
  29. data/lib/typingpool/template.rb +2 -87
  30. data/lib/typingpool/test/script.rb +310 -0
  31. data/lib/typingpool/test.rb +1 -306
  32. data/lib/typingpool/transcript/chunk.rb +129 -0
  33. data/lib/typingpool/transcript.rb +1 -125
  34. data/lib/typingpool/utility/castable.rb +65 -0
  35. data/lib/typingpool/utility.rb +1 -61
  36. data/test/test_integration_script_6_tp_finish.rb +1 -0
  37. metadata +135 -81
@@ -0,0 +1,135 @@
1
+ module Typingpool
2
+ class Project
3
+ class Remote
4
+
5
+ #Subclass for storing remote files on Amazon Simple Storage
6
+ #Service (S3)
7
+ class S3 < Remote
8
+ require 'aws/s3'
9
+
10
+ #An Amazon Web Services "Access Key ID." Set from the
11
+ #Config#amazon value passed to Project::Remote::S3.new, but
12
+ #changeable.
13
+ attr_accessor :key
14
+
15
+ #An Amazon Web Services "Secret Access Key." Set from the
16
+ #Config#amazon value passed to Project::Remote::S3.new, but
17
+ #changeable.
18
+ attr_accessor :secret
19
+
20
+ #The S3 "bucket" where uploads will be stores. Set from the
21
+ #Config#amazon value passed to Project::Remote::S3.new, but
22
+ #changeable.
23
+ attr_accessor :bucket
24
+
25
+ #Returns the base URL, which is prepended to the remote
26
+ #files. This is either the 'url' attribute of the
27
+ #Config#amazon value passed to Project::Remote::S3.new or, if
28
+ #that attribute is not set, the value returned by
29
+ #'default_url' (e.g. "https://bucketname.s3.amazonaws.com").
30
+ attr_reader :url
31
+
32
+ #Constructor. Takes the project name and the result of calling
33
+ #the 'amazon' method on a Config instance (i.e. the amazon
34
+ #section of a Config file).
35
+ def initialize(name, amazon_config)
36
+ @name = name
37
+ @config = amazon_config
38
+ @key = @config.key or raise Error::File::Remote::S3, "Missing Amazon key in config"
39
+ @secret = @config.secret or raise Error::File::Remote::S3, "Missing Amazon secret in config"
40
+ @bucket = @config.bucket or raise Error::File::Remote::S3, "Missing Amazon bucket in config"
41
+ @url = @config.url || default_url
42
+ end
43
+
44
+ #The remote host (server) name, parsed from #url
45
+ def host
46
+ URI.parse(@url).host
47
+ end
48
+
49
+ #The remote path (directory), pased from #url
50
+ def path
51
+ URI.parse(@url).path
52
+ end
53
+
54
+ #Upload files/strings to S3, optionally changing the names in the process.
55
+ # ==== Params
56
+ #[io_streams] Enumerable collection of IO objects, like a File
57
+ # or StringIO instance.
58
+ #[as] Optional if the io_streams are File instances. Array of
59
+ # file basenames, used to name the destination
60
+ # files. Default is the basename of the Files
61
+ # passed in as io_streams.
62
+ # ==== Returns
63
+ #Array of URLs corresponding to the uploaded files.
64
+ def put(io_streams, as=io_streams.map{|file| File.basename(file)})
65
+ batch(io_streams) do |stream, i|
66
+ dest = as[i]
67
+ yield(stream, dest) if block_given?
68
+ begin
69
+ AWS::S3::S3Object.store(dest, stream, @bucket, :access => :public_read)
70
+ rescue AWS::S3::NoSuchBucket
71
+ make_bucket
72
+ retry
73
+ end
74
+ file_to_url(dest)
75
+ end #batch
76
+ end
77
+
78
+ #Delete objects from S3.
79
+ # ==== Params
80
+ #[files] Enumerable collection of file names. Should NOT
81
+ # include the bucket name (path).
82
+ # ==== Returns
83
+ #Array of booleans corresponding to whether the delete call
84
+ #succeeded.
85
+ def remove(files)
86
+ batch(files) do |file, i|
87
+ yield(file) if block_given?
88
+ AWS::S3::S3Object.delete(file, @bucket)
89
+ end
90
+ end
91
+
92
+ protected
93
+
94
+ def batch(io_streams)
95
+ results = []
96
+ io_streams.each_with_index do |stream, i|
97
+ connect if i == 0
98
+ begin
99
+ results.push(yield(stream, i))
100
+ rescue AWS::S3::S3Exception => e
101
+ if e.message.match(/AWS::S3::SignatureDoesNotMatch/)
102
+ raise Error::File::Remote::S3::Credentials, "S3 operation failed with a signature error. This likely means your AWS key or secret is wrong. Error: #{e}"
103
+ else
104
+ raise Error::File::Remote::S3, "Your S3 operation failed with an Amazon error: #{e}"
105
+ end #if
106
+ end #begin
107
+ end #files.each
108
+ disconnect unless io_streams.empty?
109
+ results
110
+ end
111
+
112
+ def connect
113
+ AWS::S3::Base.establish_connection!(
114
+ :access_key_id => @key,
115
+ :secret_access_key => @secret,
116
+ :persistent => true,
117
+ :use_ssl => true
118
+ )
119
+ end
120
+
121
+ def disconnect
122
+ AWS::S3::Base.disconnect
123
+ end
124
+
125
+ def make_bucket
126
+ AWS::S3::Bucket.create(@bucket)
127
+ end
128
+
129
+ def default_url
130
+ "https://#{@bucket}.s3.amazonaws.com"
131
+ end
132
+ end #S3
133
+ end #Remote
134
+ end #Project
135
+ end #Typingpool
@@ -0,0 +1,100 @@
1
+ module Typingpool
2
+ class Project
3
+ class Remote
4
+
5
+ #Subclass for storing remote files on an SFTP server. Only
6
+ #public/private key authentication has been tested. There is not
7
+ #yet any provision for password-based authentication, though
8
+ #adding it should be trivial.
9
+ class SFTP < Remote
10
+ require 'net/sftp'
11
+
12
+ #Returns the remote host (server) name. This is set from
13
+ #Config#sftp#host.
14
+ attr_reader :host
15
+
16
+ #Returns the remote path (directory). This is set from
17
+ #Config#sftp#path.
18
+ attr_reader :path
19
+
20
+ #Returns the name of the user used to log in to the SFTP
21
+ #server. This is et from Config#sftp#user.
22
+ attr_reader :user
23
+
24
+ #Returns the base URL, which is prepended to the remote
25
+ #files. This is set from Config#sftp#url.
26
+ attr_reader :url
27
+
28
+ #Constructor. Takes the project name and a Config#sftp.
29
+ def initialize(name, sftp_config)
30
+ @name = name
31
+ @config = sftp_config
32
+ @user = @config.user or raise Error::File::Remote::SFTP, "No SFTP user specified in config"
33
+ @host = @config.host or raise Error::File::Remote::SFTP, "No SFTP host specified in config"
34
+ @url = @config.url or raise Error::File::Remote::SFTP, "No SFTP url specified in config"
35
+ @path = @config.path || ''
36
+ end
37
+
38
+ #See docs for Project::Remote::S3#put.
39
+ def put(io_streams, as=io_streams.map{|file| File.basename(file)})
40
+ begin
41
+ i = 0
42
+ batch(io_streams) do |stream, connection|
43
+ dest = as[i]
44
+ i += 1
45
+ yield(stream, dest) if block_given?
46
+ connection.upload(stream, join_with_path(dest))
47
+ file_to_url(dest)
48
+ end
49
+ rescue Net::SFTP::StatusException => e
50
+ raise Error::File::Remote::SFTP, "SFTP upload failed: #{e.description}"
51
+ end
52
+ end
53
+
54
+ #See docs for Project::Remote::S3#remove.
55
+ def remove(files)
56
+ requests = batch(files) do |file, connection|
57
+ yield(file) if block_given?
58
+ connection.remove(join_with_path(file))
59
+ end
60
+ failures = requests.reject{|request| request.response.ok?}
61
+ if not(failures.empty?)
62
+ summary = failures.map{|request| request.response.to_s}.join('; ')
63
+ raise Error::File::Remote::SFTP, "SFTP removal failed: #{summary}"
64
+ end
65
+ end
66
+
67
+ protected
68
+
69
+ def connection
70
+ begin
71
+ Net::SFTP.start(@host, @user) do |connection|
72
+ yield(connection)
73
+ connection.loop
74
+ end
75
+ rescue Net::SSH::AuthenticationFailed
76
+ raise Error::File::Remote::SFTP, "SFTP authentication failed: #{$?}"
77
+ end
78
+ end
79
+
80
+ def batch(files)
81
+ results = []
82
+ connection do |connection|
83
+ files.each do |file|
84
+ results.push(yield(file, connection))
85
+ end
86
+ end
87
+ return results
88
+ end
89
+
90
+ def join_with_path(file)
91
+ if @path
92
+ [@path, file].join('/')
93
+ else
94
+ file
95
+ end
96
+ end
97
+ end #SFTP
98
+ end #Remote
99
+ end #Project
100
+ end #Typingpool
@@ -0,0 +1,65 @@
1
+ module Typingpool
2
+ class Project
3
+ #Representation of the Project instance on remote servers. This is
4
+ #basically a collection of audio files to be transcribed and HTML
5
+ #files containing instructions and a form for the
6
+ #transcribers. The backend can be Amazon S3 (the default) or an
7
+ #SFTP server. Each backend is encapsulated in its own subclass. A
8
+ #backend subclass must provide a 'put' method, which takes an
9
+ #array of IO streams and an optional array of remote file
10
+ #basenames; a 'remove' method, which takes an array of remote file
11
+ #basenames; and the methods 'host' and 'path', which return the
12
+ #location of the destination server and destination directory,
13
+ #respectively.
14
+ #
15
+ #Thus, there will always be 'put', 'remove', 'host' and 'path'
16
+ #methods available, in addition to the Project::Remote methods
17
+ #outlined below.
18
+ class Remote
19
+ require 'typingpool/project/remote/s3'
20
+ require 'typingpool/project/remote/sftp'
21
+
22
+ #The project name
23
+ attr_accessor :name
24
+
25
+ #Constructor. Takes the project name and a Config
26
+ #instance. Returns a Project::Remote::S3 or
27
+ #Project::Remote::SFTP instance, depending on the particulars of
28
+ #the Config. If there are sufficient config params to return
29
+ #EITHER an S3 or SFTP subclass, it will prefer the SFTP
30
+ #subclass.
31
+ def self.from_config(name, config)
32
+ if config.sftp
33
+ SFTP.new(name, config.sftp)
34
+ elsif config.amazon && config.amazon.bucket
35
+ S3.new(name, config.amazon)
36
+ else
37
+ raise Error, "No valid upload params found in config file (SFTP or Amazon info)"
38
+ end
39
+ end
40
+
41
+ #Like project.remote.remove, except it takes an array of URLs
42
+ #instead an array of remote basenames, saving you from having to
43
+ #manually extract basenames from the URL.
44
+ def remove_urls(urls)
45
+ basenames = urls.map{|url| url_basename(url) }
46
+ remove(basenames){|file| yield(file) if block_given? }
47
+ end
48
+
49
+ #Given a file path, returns the URL to the file path were it to
50
+ #be uploaded by this instance.
51
+ def file_to_url(file)
52
+ "#{@url}/#{URI.escape(file)}"
53
+ end
54
+
55
+ #Given an URL, returns the file portion of the path, given the
56
+ #configuration of this instance.
57
+ def url_basename(url)
58
+ basename = url.split("#{self.url}/")[1] or raise Error, "Could not find base url '#{self.url}' within longer url '#{url}'"
59
+ URI.unescape(basename)
60
+ end
61
+
62
+
63
+ end #Remote
64
+ end #Project
65
+ end #Typingpool