pione 0.3.2 → 0.4.0

data/lib/pione/location/data-location.rb

@@ -3,6 +3,8 @@ module Pione
     class DataLocation < BasicLocation
       location_type :data
 
+      KNOWN_ATTRS = [:need_caching, :real_appendable, :writable]
+
       class << self
         # @return [String]
         #   location's scheme name
@@ -17,16 +19,27 @@ module Pione
           SCHEMES[name] = self
         end
 
-        def set_real_appendable(b)
-          @appendable = b
+
+        # Define a location's attribute.
+        #
+        # - need_caching : whether the location needs to be cached or not
+        # - real_appendable : whether the location can appendable or not
+        # - writable : whether the location is writable or not
+        def define(name, val)
+          if DataLocation::KNOWN_ATTRS.include?(name)
+            (@attr ||= Hash.new)[name] = val
+          else
+            raise ArgumentError.new(name)
+          end
         end
 
-        def real_appendable?
-          @appendable
+        # Return true if the location needs caching.
+        def need_caching?
+          @attr[:need_caching]
         end
 
-        def set_writable(b)
-          @writable = b
+        def real_appendable?
+          @attr[:real_appendable]
         end
 
         def writable?
@@ -34,7 +47,7 @@ module Pione
         end
       end
 
-      forward! :class, :scheme, :real_appendable?, :writable?
+      forward! :class, :scheme, :real_appendable?, :writable?, :need_caching?
       forward :@uri, :host
 
       # @return [URI]
@@ -62,7 +75,7 @@ module Pione
         if scheme == "local"
           self
         else
-          Location[Temppath.create].tap {|tmp| copy(tmp)}
+          Location[Temppath.create].tap {|tmp| copy(tmp) if exist?}
         end
       end
 
@@ -164,7 +177,13 @@ module Pione
       #   data content
       # @return [void]
       def append(data)
-        raise NotImplmentedError
+        if real_appendable?
+          raise NotImplmentedError
+        else
+          _local = local
+          _local.append(data)
+          _local.copy(self)
+        end
       end
 
       # Read location data.
@@ -264,22 +283,31 @@ module Pione
         raise NotImplementedError
       end
 
-      # Return true if data in the location is a file.
+      # Return true if data at the location is a file. When there exists no
+      # files and no directories, then return false.
       #
       # @return [Boolean]
-      #   true if data in the location is a file
+      #   true if data at the location is a file
       def file?
         raise NotImplementedError
       end
 
-      # Return true if data in the location is a directory.
+      # Return true if data at the location is a directory. When there exists no
+      # files and no direcotries, then return false.
       #
       # @return [Boolean]
-      #   true if data in the location is a directory
+      #   true if data at the location is a directory
       def directory?
         raise NotImplementedError
       end
 
+      # Make the path a directory.
+      #
+      # @return [void]
+      def mkdir
+        raise NotImplementedError
+      end
+
       # Move to the destination.
       #
       # @param dest [BasicLocation]

data/lib/pione/location/dropbox-location.rb

@@ -4,99 +4,183 @@ module Pione
     class DropboxLocation < DataLocation
       set_scheme "dropbox"
 
+      define(:need_caching, true)
+      define(:real_appendable, false)
+      define(:writable, true)
+
       class << self
-        attr_reader :session
+        attr_reader :client
 
-        # Initialize dropbox settings.
+        # Return true if Dropbox's access token cache exists.
         #
-        # @param tuple_space_server [TupleSpaceServer]
-        #   tuple space server
+        # @return [Boolean]
+        #   true if Dropbox's access token cache exists
+        def cached?
+          cache = Pathname.new("~/.pione/dropbox_api.cache").expand_path
+          return (cache.exist? and cache.read.size > 0)
+        end
+
+        # Setup Dropbox location for CUI client. This method gets Dropbox's
+        # access token from cache file or OAuth2.
+        #
+        # @param tuple_space [TupleSpaceServer]
+        #   tuple space
         # @return [void]
-        def self.init(tuple_space_server)
-          tuple_consumer_key = TupleSapce::AttributeTuple.new("dropbox_consumer_key", nil)
-          tuple_consumer_secret = TupleSpace::AttributeTuple.new("dropbox_consumer_secret", nil)
-          tuple_access_token_key = TupleSpace::AttributeTuple.new("dropbox_access_token_key", nil)
-          tuple_access_token_secret = TupleSpace::AttributeTuple.new("dropbox_access_token_secret", nil)
+        def setup_for_cui_client(tuple_space)
+          return if @client
 
-          consumer_key = tuple_space_server.read(tuple_consumer_key, 0).value rescue nil
-          consumer_secret = tuple_space_server.read(tuple_consumer_secret, 0).value rescue nil
-          access_token_key = tuple_space_server.read(tuple_access_token_key, 0).value rescue nil
-          access_token_secret = tuple_space_server.read(tuple_access_token_secret, 0).value rescue nil
+          access_token = nil
+          cache = Pathname.new("~/.pione/dropbox_api.cache").expand_path
+
+          if cache.exist?
+            # load access token from cache file
+            access_token = cache.read
+          else
+            # get access token by OAtuh2
+            setup_consumer_key_and_secret
+            flow = auth_by_oauth2flow_no_redirect
+            access_token, user_id = get_code_for_cui_client(flow)
+
+            # cache session
+            cache.open("w+") {|c| c.write access_token}
+          end
+
+          # make a client
+          @client = DropboxClient.new(access_token)
+
+          # share access token in tuple space
+          share_access_token(tuple_space, access_token)
+        end
 
-          @session = DropboxSession.new(consumer_key, consumer_secret)
-          @session.set_access_token(access_token_key, access_token_secret)
+        # Enable Dropbox locations. This method get an access token from the
+        # tuple space and make a Dropbox client. This assumes to be called from
+        # task worker agents.
+        #
+        # @param tuple_space_server [TupleSpaceServer]
+        #   tuple space server
+        # @return [void]
+        def enable(tuple_space)
+          tuple = TupleSpace::AttributeTuple.new("dropbox_access_token", nil)
+          if tuple_access_token = tuple_space.read!(tuple)
+            @client = DropboxClient.new(tuple_access_token.value)
+          else
+            raise DropboxLocationUnavailable.new("There is no access token.")
+          end
         end
 
         def rebuild(path)
           Location["dropbox:%s" % path]
         end
 
-        # Share dropbox's access token with PIONE agents.
+        # Setup Dropbox's consumer key and secret. They are loaded from a YAML
+        # file "dropbox_api.yml" at PIONE's home directory.
         #
-        # @param tuple_space_server [TupleSpaceServer]
-        #   tuple space server
-        # @param consumer_key [String]
-        #   consumer key
-        # @param consumer_secret [String]
-        #   consumer secret
         # @return [void]
-        def share_access_token(tuple_space_server, consumer_key, consumer_secret)
-          access_token = session.get_access_token
-          [ TupleSpace::AttributeTuple.new("dropbox_consumer_key", consumer_key),
-            TupleSpace::AttributeTuple.new("dropbox_consumer_secret", consumer_secret),
-            TupleSpace::AttributeTuple.new("dropbox_access_token_key", access_token.key),
-            TupleSpace::AttributeTuple.new("dropbox_access_token_secret", access_token.secret)
-          ].each {|tuple| tuple_space_server.write(tuple) }
+        def setup_consumer_key_and_secret
+          path = Pathname.new("~/.pione/dropbox_api.yml").expand_path
+          if File.exist?(path)
+            api = YAML.load(path.read)
+            @consumer_key = api["key"]
+            @consumer_secret = api["secret"]
+          else
+            raise DropboxLocationUnavailable.new("There are no consumer key and consumer secret.")
+          end
         end
 
-        # Return true if dropbox session is authorized.
+        # Authorize dropbox account by `DropboxOAuth2FlowNoRedirect`.
         #
-        # @return [Boolean]
-        #   true if dropbox session is authorized
-        def ready?
-          @session.authorized?
+        # @return [String]
+        #   authorize URL
+        def auth_by_oauth2flow_no_redirect
+          if @consumer_key and @consumer_secret
+            return DropboxOAuth2FlowNoRedirect.new(@consumer_key, @consumer_secret)
+          else
+            raise DropboxLocationUnavailable.new("There are no consumer key and consumer secret.")
+          end
         end
 
-        # Set the session.
+        # Authorize dropbox account by `DropboxOAuth2Flow`.
         #
-        # @param session [String]
-        #    dropbox session
+        # @param redirect [String]
+        #   redirect URL
+        # @param session [Hash]
+        #   session
+        # @return [String]
+        #   authorize URL
+        def auth_by_oauth2flow(redirect, session)
+          if @consumer_key and @consumer_secret
+            return DropboxOAuth2Flow.new(@consumer_key, @consumer_secret, redirect, session, :dropbox_auth_csrf_token)
+          else
+            raise DropboxLocationUnavailabel.new("There are no consumer key and consumer secret.")
+          end
+        end
+
+        # Share dropbox's access token with PIONE agents.
+        #
+        # @param tuple_space_server [TupleSpaceServer]
+        #   tuple space server
+        # @param access_token [String]
+        #   access token
         # @return [void]
-        def self.set_session(session)
-          @session = session
+        def share_access_token(tuple_space_server, access_token)
+          tuple = TupleSpace::AttributeTuple.new("dropbox_access_token", access_token)
+          tuple_space_server.write(tuple)
+        end
+
+        # Get code for CUI client.
+        #
+        # @return [Array]
+        #   access token and Dropbox user ID
+        def get_code_for_cui_client(flow)
+          puts '1. Go to: %s' % flow.start
+          puts '2. Click "Allow"'
+          puts '3. Copy the authorization code'
+          print 'Enter the authorization code here: '
+          code = STDIN.gets.strip
+          flow.finish(code)
         end
       end
 
       def initialize(uri)
         super(uri)
-        @client = DropboxClient.new(self.class.session, "app_folder")
       end
 
       def create(data)
-        @client.put_file(@path, StringIO.new(data))
+        if exist?
+          raise ExistAlready.new(self)
+        else
+          client.put_file(@path.to_s, StringIO.new(data))
+        end
         return self
       end
 
       def read
-        @client.get_file(@path)
+        exist? ? client.get_file(@path.to_s) : (raise NotFound.new(self))
       end
 
       def update(data)
-        @client.put_file(@path, StringIO.new(data), true)
+        client.put_file(@path.to_s, StringIO.new(data), true)
         return self
       end
 
       def delete
-        @client.file_delete(@path)
+        if exist?
+          client.file_delete(@path.to_s)
+        end
       end
 
       # dropbox have "modified" time only, therefore ctime is not implemented
 
       def mtime
-        metadata = @client.metadata(@path)
+        metadata = client.metadata(@path.to_s)
         Time.parse(metadata["modified"])
       end
 
+      def size
+        metadata = client.metadata(@path.to_s)
+        return metadata["bytes"]
+      end
+
       def entries(option={})
         rel_entries(option).map {|entry| rebuild(@path + entry)}
       end
@@ -104,10 +188,11 @@ module Pione
       def rel_entries(option={})
         list = []
         raise NotFound.new(self) if not(directory?)
-        metadata = @client.metadata(@path)
+
+        metadata = client.metadata(@path.to_s)
         metadata["contents"].select{|entry| not(entry["is_deleted"])}.each do |entry|
-          list << entry["path"][0, @path.size]
-          entry_location = rebuild(@path + entry)
+          list << entry["path"].sub(@path.to_s, "").sub(/^\//, "")
+          entry_location = rebuild(@path + File.basename(entry["path"]))
           if option[:rec] and entry_location.directory?
             _list = entry_location.rel_entries(option).map {|subentry| entry + subentry}
             list = list + _list
@@ -117,25 +202,43 @@ module Pione
       end
 
       def exist?
-        metadata = @client.metadata(@path)
+        metadata = client.metadata(@path.to_s)
         return not(metadata["is_deleted"])
+      rescue DropboxLocationUnavailable
+        raise
       rescue DropboxError
         return false
       end
 
       def file?
-        metadata = @client.metadata(@path)
-        return (not(metadata["is_dir"]) and not(metadata["is_deleted"]))
+        begin
+          metadata = client.metadata(@path.to_s)
+          return (not(metadata["is_dir"]) and not(metadata["is_deleted"]))
+        rescue DropboxError
+          # when there exists no files and no directories
+          false
+        end
       end
 
       def directory?
-        metadata = @client.metadata(@path)
-        return (metadata["is_dir"] and not(metadata["is_deleted"]))
+        begin
+          metadata = client.metadata(@path.to_s)
+          return (metadata["is_dir"] and not(metadata["is_deleted"]))
+        rescue DropboxError
+          # when there exists no files and no directories
+          false
+        end
+      end
+
+      def mkdir
+        unless exist?
+          client.file_create_folder(@path.to_s)
+        end
       end
 
       def move(dest)
         if dest.scheme == scheme
-          @client.file_move(@path, dest.path)
+          client.file_move(@path.to_s, dest.path)
         else
           copy(dest)
           delete
@@ -143,10 +246,12 @@ module Pione
       end
 
       def copy(dest)
+        raise NotFound.new(self) unless exist?
+
         if dest.scheme == scheme
-          @client.file_copy(@path, dest.path)
+          client.file_copy(@path.to_s, dest.path)
         else
-          dest.update(read)
+          dest.write(read)
         end
       end
 
@@ -161,6 +266,20 @@ module Pione
       def turn(dest)
         copy(dest)
       end
+
+      private
+
+      # Check availability of Dropbox's access token.
+      #
+      # @return [void]
+      def client
+        if self.class.client.nil?
+          # raise an exception when Dropbox client isn't enabled
+          raise DropboxLocationUnavailable.new("There is no access token.")
+        else
+          self.class.client
+        end
+      end
     end
   end
 end

data/lib/pione/location/ftp-location.rb

@@ -3,7 +3,10 @@ module Pione
     # FTPLocation represents locations on FTP server.
     class FTPLocation < DataLocation
       set_scheme "ftp"
-      set_real_appendable false
+
+      define(:need_caching, true)
+      define(:real_appendable, false)
+      define(:writable, true)
 
       # for myftp scheme
       SCHEMES["myftp"] = self

data/lib/pione/location/http-location.rb

@@ -2,8 +2,10 @@ module Pione
   module Location
     class HTTPLocation < DataLocation
       set_scheme "http"
-      set_real_appendable false
-      set_writable false
+
+      define(:need_caching, true)
+      define(:real_appendable, false)
+      define(:writable, false)
 
       def read
         http_get {|res| res.body}
@@ -33,7 +35,7 @@ module Pione
 
       def copy(dest, option={})
         # setup options
-        option[:keep_mtime] ||= true
+        option[:keep_mtime] = true if option[:keep_mtime].nil?
 
         # copy
         http_get {|rec| dest.write rec.body}