worochi 0.0.7 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5aa10b9332e68182b3a9dbed21aff7f1fcc79c78
-  data.tar.gz: 451929c1f40471004dc786839f1897965f04a803
+  metadata.gz: 0ba8b523ca7164198a72a0cb2b02230f6a394012
+  data.tar.gz: 30d3cba96de58d8c04c6c2498f23e747ee320eef
 SHA512:
-  metadata.gz: 1c76eb196eb3a1c759ea9fbde0c75e23e3a34af09176086c2f91ee65aadb69de98df0ba5eb9e7cff769187d061d92f864f31dc72d83f458cd376bd0c25eb7e49
-  data.tar.gz: 740a453516871166e8fe34b7f28b8e31d351d8d3f4bd796f4217978720a6bca2ccf0c9f763a39d711de9ea11e74eb3610d71e7ba506845de9ef40b18cb57ad8c
+  metadata.gz: 91d81134ec1ebd38502aadf22f4a90e19d648ae4cb8be0c15712961ad124ea8a57ef2ead0334af2d62a0dfb89270225f5a395880f10ab747238b2d8147806c25
+  data.tar.gz: 2f1665102e34f54b1f29f17c40da921854ebede6cadd1662fa5f8198f4d79c965f2bb1032b14700f98171f98a15d63c9b618e58c995f14c6bcd259793734f535

data/lib/worochi/agent/{sample.rb → #example.rb}

@@ -1,19 +1,7 @@
 class Worochi
-  # The {Agent} for a sample API.
+  # The {Agent} for an example API.
   # This is a sample of methods that should be implemented by a service agent.
-  class Agent::Sample < Agent
-
-    # This is the minimum set of default options that should be specified.
-    #
-    # @return [Hash] default options for sample API
-    # @see Agent#set_options
-    def default_options
-      {
-        service: :sample, # service name
-        dir: '/' # remote directory to act on
-      }
-    end
-
+  class Agent::Example < Agent
     # This is where the service-specific API client should be initialized.
     #
     # @return [ApiClient]

data/lib/worochi/agent/dropbox.rb

@@ -4,16 +4,6 @@ class Worochi
   # The {Agent} for Dropbox API. This wraps around the `dropbox-sdk` gem.
   # @see https://www.dropbox.com/developers/core/start/ruby
   class Agent::Dropbox < Agent
-    # @return [Hash] default options for Dropbox
-    def default_options
-      {
-        service: :dropbox,
-        chunk_size: 2*1024*1024,
-        overwrite: true,
-        dir: '/'
-      }
-    end
-
     # Initializes Dropbox SDK client. Refer to
     # {https://www.dropbox.com/developers/core/start/ruby
     # official Dropbox documentation}.
@@ -27,16 +17,16 @@ class Worochi
     # Push a single {Item} to Dropbox.
     #
     # @param item [Item]
-    # @return [nil]
+    # @return [Boolean] true if chunk uploader was used
     def push_item(item)
       Worochi::Log.debug "Uploading #{item.path} (#{item.size} bytes) to Dropbox..."
-      if item.size > options[:chunk_size]
+      if chunked = item.size > options[:chunk_size]
         push_item_chunked(item)
       else
         @client.put_file(full_path(item), item.content, options[:overwrite])
       end
       Worochi::Log.debug "Uploaded"
-      nil
+      chunked
     end
 
     # Returns a list of files and subdirectories at the remote path specified

data/lib/worochi/agent/github.rb

@@ -1,25 +1,10 @@
 require 'octokit'
 require 'base64'
-require 'worochi/helper/github'
 
 class Worochi
   # The {Agent} for GitHub API. This wraps around the `octokit` gem.
   # @see https://github.com/octokit/octokit.rb
   class Agent::Github < Agent
-
-    # @return [Hash] default options for GitHub
-    def default_options
-      {
-        service: :github,
-        source: 'worochi',
-        target: 'worochi',
-        repo: 'darkmirage/test',
-        block_size: Worochi::Helper::Github::BLOCK_SIZE,
-        commit_msg: 'Empty commit message',
-        dir: '/'
-      }
-    end
-
     # Initializes Octokit client. Refer to
     # {https://github.com/octokit/octokit.rb
     # octokit.rb documentation}.
@@ -68,12 +53,14 @@ class Worochi
         x.path.split('/').size <=> y.path.split('/').size
       end
 
-      # Checks that folders are at the requested path and not at a lower or
-      # higher level
+      # Filters for folders containing the specified path
+      result.reject! { |elem| !elem.path.match(remote_path + '($|\/.+)') }
+      raise Error, 'Invalid GitHub path specified' if result.empty?
+
+      # Filters out lower levels
       result.reject! do |elem|
-        !elem.path.match(remote_path + '($|\/.+)') ||
-        (File.join(remote_path,
-            elem.path.split('/').last).sub(/^\//, '') != elem.path)
+        filename = elem.path.split('/').last
+        File.join(remote_path, filename).sub(/^\//, '') != elem.path
       end
 
       result.map do |elem|
@@ -142,7 +129,7 @@ class Worochi
     # @return [String] SHA1 checksum of the created blob
     def stream_blob(item)
       Worochi::Log.debug "Using JSON streaming..."
-      post_stream = Worochi::Helper::Github::StreamIO.new(item)
+      post_stream = Helper::StreamIO.new(item)
 
       uri = URI("https://api.github.com/repos/#{repo}/git/blobs")
       request = Net::HTTP::Post.new(uri.path)
@@ -156,8 +143,6 @@ class Worochi
       http.request request do |response|
         return JSON.parse(response.body)['sha']
       end
-
-      raise Error, 'Failed to upload file to GitHub'
     end
 
     # @return [Hash] repo information

data/lib/worochi/agent.rb

@@ -1,17 +1,15 @@
+require 'yaml'
+
 class Worochi
   # The parent class for all service agents.
   class Agent
-    # Service name.
-    # @return [Symbol]
-    attr_reader :type
     # Service options.
-    # @return [Hash]
+    # @return [Hashie::Mash]
     attr_accessor :options
 
     # @param opts [Hash] service options
     def initialize(opts={})
       set_options(opts)
-      @type = options[:service]
       init_client
     end
 
@@ -92,6 +90,11 @@ class Worochi
       list_helper(:folders, args)
     end
 
+    # Returns a list of files and folders at the remote path specified by
+    # `options[:dir]`. Relies on the service-specific implementation of
+    # `#list`. Refer to {#files} for overloaded prototypes.
+    #
+    # @return [Array<String>, Array<Hash>] list of files and folders
     def files_and_folders(*args)
       list_helper(:both, args)
     end
@@ -99,29 +102,35 @@ class Worochi
     # Updates {.options} using `opts`.
     #
     # @param opts [Hash] new options
-    # @return [Hash] the updated options
+    # @return [Hashie::Mash] the updated options
     def set_options(opts={})
       self.options ||= default_options
-      sym_opts = {}
-      opts.each { |k, v| sym_opts[k.to_sym] = v }
-      options.merge!(sym_opts)
+      opts = Hashie::Mash.new(opts)
+      options.merge!(opts)
     end
 
     # Sets the remote target directory path. This is the same as modifying
     # `options[:dir]`.
     #
     # @param path [String] the new path
-    # @return [Hash] the updated options
+    # @return [Hashie::Mash] the updated options
     def set_dir(path)
-      options[:dir] = path
+      options.dir = path
       options
     end
 
+    # Returns the service type for the agent.
+    #
+    # @return [Symbol] service type
+    def type
+      options.service
+    end
+
     # Returns the display name for the agent's service.
     #
     # @return [String] display name
     def name
-      Worochi::Config.service_display_name(type)
+      Worochi::Config.service_display_name(options.service)
     end
 
   private
@@ -129,7 +138,7 @@ class Worochi
     #
     # @param mode [Symbol] display files, folders, or both
     # @param args [Array] argument list
-    # @return [Array<String>, Array<Hash>] list of files or folders
+    # @return [Array<String>, Array<Hashie::Mash>] list of files or folders
     def list_helper(mode, args)
       details = true if args.first == true
       if args.first.kind_of?(String)
@@ -147,20 +156,39 @@ class Worochi
       end
 
       result = list(path).reject { |elem| elem[:type] == excluded }
-      result.map! { |elem| elem[:name] } unless details
+      if details
+        result.map! { |elem| Hashie::Mash.new(elem) }
+      else
+        result.map! { |elem| elem[:name] }
+      end
       result
     end
 
     # @return [String] full path combining remote directory and item path
     def full_path(item)
-      File.join(options[:dir], item.path)
+      File.join(options.dir, item.path)
     end
 
-    # Agents should override this.
+    # Agents should either override this or have a YAML config file at
+    # config/service_name.yml.
     #
-    # @return [nil]
+    # @return [Hashie::Mash] options parsed from YAML config
     def default_options
-      raise Error, 'Default options not specified.'
+      service = service_name.to_sym
+      Worochi::Config.service_opts(service)
+    end
+
+    # Returns the service name based on the class name.
+    #
+    # @return [Symbol] service name
+    def service_name
+      name = self.class.to_s.split( '::' ).last
+      name.gsub!(/::/, '/')
+      name.gsub!(/([A-Z]+)([A-Z][a-z])/,'\1_\2')
+      name.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
+      name.tr!("-", "_")
+      name.downcase!
+      name.to_sym
     end
 
     class << self

data/lib/worochi/config/#example.yml

@@ -0,0 +1,35 @@
+# Use this as a template for configuring new services
+
+# General configurations (must be present)
+# id must be unique across all services
+id: 0
+display_name: Sample Agent
+dir: /
+
+# Required if using OAuth2 functionality
+oauth:
+  # These are the ENV names and not the actual ID and secret. If not defined,
+  # then assumed to be SERVICE_NAME_ID and SERVICE_NAME_SECRET.
+  id_env: SAMPLE_ID
+  secret_env: SAMPLE_SECRET
+
+  # API endpoints for OAuth2
+  token_url: /1/oauth2/token
+  authorize_url: /1/oauth2/authorize
+
+  # Some services (e.g. Dropbox) have a different domain for the second step
+  # of the OAuth flow. In those cases, :site will be used as the endpoint for
+  # the start of the authorization flow while :token_site will be used as the
+  # endpoint for token retrieval.
+  site: https://www.sample.com
+  token_site: https://api.sample.com
+
+  # Scope if required by the service
+  scope: gallery
+
+# Service specific (optional; example is for GitHub)
+repo: darkmirage/test
+source: master
+target: worochi
+block_size: 12288
+commit_msg: Empty commit message

data/lib/worochi/config/dropbox.yml

@@ -0,0 +1,14 @@
+id: 2
+
+display_name: Dropbox
+dir: /
+oauth:
+  id_env: DROPBOX_ID
+  secret_env: DROPBOX_SECRET
+  token_url: /1/oauth2/token
+  authorize_url: /1/oauth2/authorize
+  site: https://www.dropbox.com
+  token_site: https://api.dropbox.com
+
+chunk_size: 2097152
+overwrite: true

data/lib/worochi/config/github.yml

@@ -0,0 +1,17 @@
+id: 1
+
+display_name: GitHub
+dir: /
+oauth:
+  id_env: GITHUB_ID
+  secret_env: GITHUB_SECRET
+  token_url: /login/oauth/access_token
+  authorize_url: /login/oauth/authorize
+  site: https://github.com
+  scope: repo
+
+repo: darkmirage/test
+source: master
+target: worochi
+block_size: 12288
+commit_msg: Empty commit message

data/lib/worochi/config.rb

@@ -1,85 +1,10 @@
-class Worochi
-  # Configurations for Worochi.
-  module Config
-    @services = {
-      github: [1, 'GitHub'],
-      dropbox: [2, 'Dropbox']
-    }
-    @s3_bucket = 'data-pixelapse'
-    @s3_prefix = 's3'
-
-    class << self
-      # Array of service names. 
-      #
-      # @return [Array<Symbol>]
-      def services
-        @services.keys
-      end
-
-      # Returns display name for the service.
-      #
-      # @overload service_display_name(service)
-      #   @param service [Symbol]
-      # @overload service_display_name(service_id)
-      #   @param service_id [Integer]
-      # @return [String] display name
-      def service_display_name(arg)
-        service = arg.to_sym if arg.respond_to?(:to_sym)
-        service = service_name(arg) unless @services.include?(service)
-        if service.nil?
-          nil
-        else
-          @services[service][1]
-        end
-      end
-
-      # Returns the service ID for the service, which can be used as a
-      # primary key for databases.
-      #
-      # @param service [Symbol]
-      # @return [Integer] service ID
-      def service_id(service)
-        if @services[service.to_sym].nil?
-          nil
-        else
-          @services[service.to_sym][0]
-        end
-      end
-
-      # Returns the service name given the service ID.
-      #
-      # @param service_id [Integer]
-      # @return [Symbol] if service exists
-      # @return [nil] if service does not exist
-      def service_name(service_id)
-        @service_ids ||= {}
-        return @service_ids[service_id] if @service_ids.include?(service_id)
-        @services.each do |key, value|
-          if value[0] == service_id
-            @service_ids[value[0]] = key
-            return key
-          end
-        end
-        nil
-      end
-
-      # Name of S3 bucket.
-      #
-      # @return [String]
-      attr_reader :s3_bucket
-
-      # Prefix for S3 resource paths.
-      #
-      # @return [String]
-      attr_reader :s3_prefix
-
-      # Disable debug and error messages if `true`.
-      #
-      # @return [Boolean]
-      attr_accessor :silent
-
-      alias_method :silent?, :silent
-    end
-  end
-end
+require 'worochi/configurator'
 
+class Worochi
+  # Default configurations
+  Config.s3_enabled = true
+  Config.s3_bucket = 'worochi'
+  Config.s3_prefix = 's3'
+  Config.silent = true
+  Config.logdev = $stdout
+end

data/lib/worochi/configurator.rb

@@ -0,0 +1,120 @@
+class Worochi
+  # Configuration methods for loading and modifying options.
+  module Config
+    class << self
+      # Loads options from YAML configuration files.
+      #
+      # @return [nil]
+      def load_yaml
+        @options = {}
+        services.each do |service|
+          path = File.join(File.dirname(__FILE__), "config/#{service}.yml")
+          raise Error, "Missing config for #{service}" unless File.file?(path)
+          @options[service] = Hashie::Mash.new(YAML.load_file(path))
+          @options[service].service = service
+        end
+      end
+
+      # Returns the service configurations that was loaded from YAML.
+      #
+      # @return [Hashie::Mash]
+      def service_opts(service)
+        if service.nil? || @options.nil? || @options[service].nil?
+          raise Error, "Invalid service (#{service}) specified"
+        end
+        @options[service].clone
+      end
+
+      # Array of service names. Parsed from the file names of any .rb files in
+      # the worochi/agent directory that does not contain the `#` character.
+      #
+      # @return [Array<Symbol>]
+      def services
+        return @services if @services
+        files = Dir[File.join(File.dirname(__FILE__), 'agent/[^#]*.rb')]
+        @services = files.map do |file|
+          File.basename(file, '.rb').to_sym
+        end
+        @services
+      end
+
+      # Returns display name for the service.
+      #
+      # @overload service_display_name(service)
+      #   @param service [Symbol]
+      # @overload service_display_name(service_id)
+      #   @param service_id [Integer]
+      # @return [String] display name
+      def service_display_name(arg)
+        service = arg.to_sym if arg.respond_to?(:to_sym)
+        service = service_name(arg) unless @options.include?(service)
+        if service.nil?
+          nil
+        else
+          @options[service].display_name
+        end
+      end
+
+      # Returns the service ID for the service, which can be used as a
+      # primary key for databases.
+      #
+      # @param service [Symbol]
+      # @return [Integer] service ID
+      def service_id(service)
+        if @options[service.to_sym].nil?
+          nil
+        else
+          @options[service.to_sym].id
+        end
+      end
+
+      # Returns the service name given the service ID.
+      #
+      # @param id [Integer]
+      # @return [Symbol] if service exists
+      # @return [nil] if service does not exist
+      def service_name(id)
+        @service_names ||= {}
+        return @service_names[id] if @service_names.include?(id)
+        @options.each do |key, value|
+          if value.id == id
+            @service_names[value.id] = key
+            return key
+          end
+        end
+        nil
+      end
+
+      # Enables AWS S3 support. AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID
+      # should be present in ENV.
+      #
+      # @return [Boolean]
+      attr_accessor :s3_enabled
+
+      # Name of S3 bucket.
+      #
+      # @return [String]
+      attr_accessor :s3_bucket
+
+      # Prefix for S3 resource paths.
+      #
+      # @return [String]
+      attr_accessor :s3_prefix
+
+      # Logging device.
+      #
+      # @return [IO]
+      attr_accessor :logdev
+
+      # Disable debug and error messages if `true`.
+      #
+      # @return [Boolean]
+      attr_accessor :silent
+
+      alias_method :silent?, :silent
+      alias_method :s3_enabled?, :s3_enabled
+
+    end
+  end
+end
+

data/lib/worochi/helper/github_helper.rb

@@ -0,0 +1,111 @@
+require 'base64'
+
+class Worochi
+  class Agent::Github
+    # Helper classes for GitHub JSON streaming.
+    module Helper
+      # Size to read in. Must be a multiple of 3 for Base64 streaming.
+      BLOCK_SIZE = 12288
+
+      # This is a wrapper that produces a JSON stream that works with
+      # `Net::HTTP::Post#body_stream`.
+      class StreamIO
+        def initialize(item)
+          item.content.rewind
+          @parts = [
+            StringIO.new('{"content":"'),
+            Base64IO.new(item.content),
+            StringIO.new('","encoding":"base64"}')
+          ]
+          @part_no = 0
+          @pos = 0
+          @size = @parts.inject(0) {|sum, p| sum + p.size}
+        end
+
+        # Rewind each component of the stream.
+        #
+        # @return [nil]
+        def rewind
+          @parts.each { |part| part.rewind }
+          @part_no = 0
+          @pos = 0
+          nil
+        end
+
+        # @return [Boolean] file has been fully read.
+        def eof?
+          @pos >= size
+        end
+
+        # @param length [Integer]
+        # @param outbuf [IO]
+        # @return [String]
+        def read(length=nil, outbuf=nil)
+          if eof?
+            outbuf.clear if outbuf
+            return length.nil? ? '' : nil
+          end
+          
+          length ||= size
+
+          output = ''
+          while output.length < length
+            curr = @parts[@part_no]
+            output += curr.read(length - output.length).to_s
+            @part_no += 1 if curr.eof?
+            break if @part_no == @parts.size
+          end
+          @pos += output.length
+
+          unless outbuf.nil?
+            outbuf.clear
+            outbuf.insert(0, output)
+          end
+
+          output
+        end
+
+        # @return [Integer] size of the JSON
+        attr_reader :size
+      end
+
+      # This is a wrapper around the file content that streams the file as a
+      # Base64 encoded string.
+      class Base64IO
+        def initialize(file)
+          file.rewind
+          @file = file
+          @encoded_size = (@file.size / 3.0).ceil * 4
+          @buffer = ''
+        end
+
+        # @return [Integer] size of the JSON
+        def size
+          @encoded_size
+        end
+
+        # @return [Boolean] file has been fully read.
+        def eof?
+          @buffer.empty? && @file.eof?
+        end
+
+        # Rewind the stream.
+        def rewind
+          @file.rewind
+          @buffer = ''
+          nil
+        end
+
+        # @param length [Integer]
+        # @param outbuf [IO]
+        # @return [String]
+        def read(length=size, outbuf=nil)
+          while @buffer.length < length and not @file.eof?
+            @buffer += Base64.strict_encode64 @file.read(BLOCK_SIZE)
+          end
+          @buffer.empty? ? nil : @buffer.slice!(0, length)
+        end
+      end
+    end
+  end
+end