typingpool 0.7.0 → 0.7.1

data/lib/typingpool/project/remote/s3.rb

@@ -0,0 +1,135 @@
+module Typingpool
+  class Project
+    class Remote
+
+      #Subclass for storing remote files on Amazon Simple Storage
+      #Service (S3)
+      class S3 < Remote
+        require 'aws/s3'
+
+        #An Amazon Web Services "Access Key ID." Set from the
+        #Config#amazon value passed to Project::Remote::S3.new, but
+        #changeable.
+        attr_accessor :key
+
+        #An Amazon Web Services "Secret Access Key." Set from the
+        #Config#amazon value passed to Project::Remote::S3.new, but
+        #changeable.
+        attr_accessor :secret
+
+        #The S3 "bucket" where uploads will be stores. Set from the
+        #Config#amazon value passed to Project::Remote::S3.new, but
+        #changeable.
+        attr_accessor :bucket
+
+        #Returns the base URL, which is prepended to the remote
+        #files. This is either the 'url' attribute of the
+        #Config#amazon value passed to Project::Remote::S3.new or, if
+        #that attribute is not set, the value returned by
+        #'default_url' (e.g. "https://bucketname.s3.amazonaws.com").
+        attr_reader :url
+
+        #Constructor. Takes the project name and the result of calling
+        #the 'amazon' method on a Config instance (i.e. the amazon
+        #section of a Config file).
+        def initialize(name, amazon_config)
+          @name = name
+          @config = amazon_config
+          @key = @config.key or raise Error::File::Remote::S3, "Missing Amazon key in config"
+          @secret = @config.secret or raise Error::File::Remote::S3, "Missing Amazon secret in config"
+          @bucket = @config.bucket or raise Error::File::Remote::S3, "Missing Amazon bucket in config"
+          @url = @config.url || default_url
+        end
+
+        #The remote host (server) name, parsed from #url
+        def host
+          URI.parse(@url).host
+        end
+
+        #The remote path (directory), pased from #url
+        def path
+          URI.parse(@url).path
+        end
+
+        #Upload files/strings to S3, optionally changing the names in the process.
+        # ==== Params
+        #[io_streams] Enumerable collection of IO objects, like a File
+        #             or StringIO instance.
+        #[as]         Optional if the io_streams are File instances. Array of
+        #             file basenames, used to name the destination
+        #             files. Default is the basename of the Files
+        #             passed in as io_streams.
+        # ==== Returns
+        #Array of URLs corresponding to the uploaded files.
+        def put(io_streams, as=io_streams.map{|file| File.basename(file)})
+          batch(io_streams) do |stream, i|
+            dest = as[i]
+            yield(stream, dest) if block_given?
+            begin
+              AWS::S3::S3Object.store(dest, stream, @bucket,  :access => :public_read)
+            rescue AWS::S3::NoSuchBucket
+              make_bucket
+              retry
+            end
+            file_to_url(dest)
+          end #batch
+        end
+
+        #Delete objects from S3.
+        # ==== Params
+        #[files] Enumerable collection of file names. Should NOT
+        #        include the bucket name (path).
+        # ==== Returns
+        #Array of booleans corresponding to whether the delete call
+        #succeeded.
+        def remove(files)
+          batch(files) do |file, i|
+            yield(file) if block_given?
+            AWS::S3::S3Object.delete(file, @bucket)
+          end
+        end
+
+        protected
+
+        def batch(io_streams)
+          results = []
+          io_streams.each_with_index do |stream, i|
+            connect if i == 0
+            begin
+              results.push(yield(stream, i))
+            rescue AWS::S3::S3Exception => e
+              if e.message.match(/AWS::S3::SignatureDoesNotMatch/)
+                raise Error::File::Remote::S3::Credentials, "S3 operation failed with a signature error. This likely means your AWS key or secret is wrong. Error: #{e}"
+              else
+                raise Error::File::Remote::S3, "Your S3 operation failed with an Amazon error: #{e}"
+              end #if    
+            end #begin
+          end #files.each
+          disconnect unless io_streams.empty?
+          results
+        end
+
+        def connect
+          AWS::S3::Base.establish_connection!(
+                                              :access_key_id => @key,
+                                              :secret_access_key => @secret,
+                                              :persistent => true,
+                                              :use_ssl => true
+                                              )
+        end
+
+        def disconnect
+          AWS::S3::Base.disconnect
+        end
+
+        def make_bucket
+          AWS::S3::Bucket.create(@bucket)
+        end
+
+        def default_url
+          "https://#{@bucket}.s3.amazonaws.com"
+        end
+      end #S3
+    end #Remote
+  end #Project
+end #Typingpool

data/lib/typingpool/project/remote/sftp.rb

@@ -0,0 +1,100 @@
+module Typingpool
+  class Project
+    class Remote
+
+      #Subclass for storing remote files on an SFTP server. Only
+      #public/private key authentication has been tested. There is not
+      #yet any provision for password-based authentication, though
+      #adding it should be trivial.
+      class SFTP < Remote
+        require 'net/sftp'
+
+        #Returns the remote host (server) name. This is set from
+        #Config#sftp#host.
+        attr_reader :host
+
+        #Returns the remote path (directory). This is set from
+        #Config#sftp#path.
+        attr_reader :path
+
+        #Returns the name of the user used to log in to the SFTP
+        #server. This is et from Config#sftp#user.
+        attr_reader :user
+
+        #Returns the base URL, which is prepended to the remote
+        #files. This is set from Config#sftp#url.
+        attr_reader :url
+
+        #Constructor. Takes the project name and a Config#sftp.
+        def initialize(name, sftp_config)
+          @name = name
+          @config = sftp_config   
+          @user = @config.user or raise Error::File::Remote::SFTP, "No SFTP user specified in config"
+          @host = @config.host or raise Error::File::Remote::SFTP, "No SFTP host specified in config"
+          @url = @config.url or raise Error::File::Remote::SFTP, "No SFTP url specified in config"
+          @path = @config.path || ''
+        end
+
+        #See docs for Project::Remote::S3#put.
+        def put(io_streams, as=io_streams.map{|file| File.basename(file)})
+          begin
+            i = 0
+            batch(io_streams) do |stream, connection|
+              dest = as[i]
+              i += 1
+              yield(stream, dest) if block_given?
+              connection.upload(stream, join_with_path(dest))
+              file_to_url(dest)
+            end
+          rescue Net::SFTP::StatusException => e
+            raise Error::File::Remote::SFTP, "SFTP upload failed: #{e.description}"
+          end
+        end
+
+        #See docs for Project::Remote::S3#remove.
+        def remove(files)
+          requests = batch(files) do |file, connection|
+            yield(file) if block_given?
+            connection.remove(join_with_path(file))
+          end
+          failures = requests.reject{|request| request.response.ok?}
+          if not(failures.empty?)
+            summary = failures.map{|request| request.response.to_s}.join('; ')
+            raise Error::File::Remote::SFTP, "SFTP removal failed: #{summary}"
+          end
+        end
+
+        protected
+
+        def connection
+          begin
+            Net::SFTP.start(@host, @user) do |connection|
+              yield(connection)
+              connection.loop
+            end
+          rescue Net::SSH::AuthenticationFailed
+            raise Error::File::Remote::SFTP, "SFTP authentication failed: #{$?}"
+          end
+        end
+
+        def batch(files)
+          results = []
+          connection do |connection|
+            files.each do |file|
+              results.push(yield(file, connection))
+            end
+          end
+          return results
+        end
+
+        def join_with_path(file)
+          if @path
+            [@path, file].join('/')
+          else
+            file
+          end
+        end
+      end #SFTP
+    end #Remote
+  end #Project
+end #Typingpool

data/lib/typingpool/project/remote.rb

@@ -0,0 +1,65 @@
+module Typingpool
+  class Project
+    #Representation of the Project instance on remote servers. This is
+    #basically a collection of audio files to be transcribed and HTML
+    #files containing instructions and a form for the
+    #transcribers. The backend can be Amazon S3 (the default) or an
+    #SFTP server. Each backend is encapsulated in its own subclass. A
+    #backend subclass must provide a 'put' method, which takes an
+    #array of IO streams and an optional array of remote file
+    #basenames; a 'remove' method, which takes an array of remote file
+    #basenames; and the methods 'host' and 'path', which return the
+    #location of the destination server and destination directory,
+    #respectively.
+    #
+    #Thus, there will always be 'put', 'remove', 'host' and 'path'
+    #methods available, in addition to the Project::Remote methods
+    #outlined below.
+    class Remote
+      require 'typingpool/project/remote/s3'
+      require 'typingpool/project/remote/sftp'
+
+      #The project name
+      attr_accessor :name
+
+      #Constructor. Takes the project name and a Config
+      #instance. Returns a Project::Remote::S3 or
+      #Project::Remote::SFTP instance, depending on the particulars of
+      #the Config. If there are sufficient config params to return
+      #EITHER an S3 or SFTP subclass, it will prefer the SFTP
+      #subclass.
+      def self.from_config(name, config)
+        if config.sftp
+          SFTP.new(name, config.sftp)
+        elsif config.amazon && config.amazon.bucket
+          S3.new(name, config.amazon)
+        else
+          raise Error, "No valid upload params found in config file (SFTP or Amazon info)"
+        end
+      end
+
+      #Like project.remote.remove, except it takes an array of URLs
+      #instead an array of remote basenames, saving you from having to
+      #manually extract basenames from the URL.
+      def remove_urls(urls)
+        basenames = urls.map{|url| url_basename(url) } 
+        remove(basenames){|file| yield(file) if block_given? }
+      end
+
+      #Given a file path, returns the URL to the file path were it to
+      #be uploaded by this instance.
+      def file_to_url(file)
+        "#{@url}/#{URI.escape(file)}"
+      end
+
+      #Given an URL, returns the file portion of the path, given the
+      #configuration of this instance.
+      def url_basename(url)
+        basename = url.split("#{self.url}/")[1] or raise Error, "Could not find base url '#{self.url}' within longer url '#{url}'"
+        URI.unescape(basename)
+      end
+
+
+    end #Remote
+  end #Project
+end #Typingpool