mode 0.0.5 → 0.0.7

data/lib/mode/commands/helpers.rb

@@ -5,6 +5,60 @@ module Mode
         started_at = Time.now
         return yield, (Time.now.to_f - started_at.to_f)
       end
+
+      def config_dir
+        Mode::Config.default_dir
+      end
+
+      def config_path
+        Mode::Config.full_path(config_dir)
+      end
+
+      def config_exists?
+        Mode::Config.exists?(config_dir)
+      end
+
+      def connect_config_exists?
+        Mode::Connector::Config.exists?(config_dir)
+      end
+
+      def require_config!
+        unless config_exists?
+          raise "Couldn't find Mode configuration. Please use `mode login` before continuing."
+        end
+      end
+
+      def require_connect_config!
+        unless connect_config_exists?
+          raise "No connect configuration file found."
+        end
+      end
+
+      def require_credentials!
+        message = "Couldn't find Mode login credentials. Please use `mode login` before continuing."
+
+        if config_exists?
+          config = Mode::Config.new(config_dir)
+          unless config.username && config.access_token
+            raise message
+          end
+        else
+          raise message
+        end
+      end
+
+      def configure_api_requests!
+        config = Mode::Config.new(config_dir)
+
+        Mode::API::Request.configure(
+          config.environment, {            
+            :credentials => {
+              :username => config.username,
+              :access_token => config.access_token
+            }
+          }
+        )
+      end
     end
   end
 end

data/lib/mode/commands/import.rb

@@ -1,36 +1,225 @@
 module Mode
   module Commands
-    class Import < Thor
-      attr_accessor :account
-      attr_accessor :table_name
-      attr_accessor :package
-      attr_accessor :resource_name
+    class Import
+      include Mode::Commands::Helpers
+      
+      attr_reader :source
+      attr_reader :account
+      attr_reader :table_name
 
-      def initialize(account, table_name, package, resource_name)
+      def initialize(source, account, table_name)
+        @source = source
         @account = account
         @table_name = table_name
-        @package = package
-        @resource_name = resource_name
+
+        require_config!
+        require_credentials!
+        configure_api_requests!
       end
 
-      no_commands do
-        def execute
+      def execute
+        package = make_package
+        uploaded = upload_package(package)
+        imported = import_package(uploaded)
+      end
 
-          puts "Importing #{package.name}/#{resource_name} to #{account}/#{table_name}"
-          # We need to check the datapackage.md5
-          # We need to check the md5 of the source
-          # If either of these have changed the package needs to be reverified
+      def make_package
+        find_or_build_package(source, :name => table_name)
+      end
 
-          # 1. Compress to temporary dir
-          # 2. POST /imports with name and zipfile
-          # 3. Poll for execution status until finished or timeout (what's the timeout?)
-        end
+      def upload_package(package)
+        resource = package.resources.first
+        file_path = File.join(resource.base_path, resource.path)
+
+        replace_resource_path!(resource)
+
+        Mode::API::Request.post(upload_path, :package => {
+          :contents => {
+            'datapackage.json' => package.to_json,
+            resource.path => Faraday::UploadIO.new(file_path, 'application/octet-stream')
+          }, :name => table_name
+        })
+      end
+
+      def import_package(uploaded)
+        resource = parse_uploaded_response(uploaded)
+        submit_import_form(resource, { 'table_name' => table_name })
+      end
+
+      def target_resource_path(path)
+        self.class.target_resource_path(path)
       end
 
       private
 
-      def valid_package?(package)
-        # package.valid?
+      def upload_path
+        Mode::API::Request.packages_path
+      end
+
+      def find_or_build_package(source, options = {})
+        self.class.find_or_build_package(source, options)
+      end
+
+      # We replace the resource path with the desired path on the remote end
+      def replace_resource_path!(resource)
+        resource.path = target_resource_path(resource.path)
+      end
+
+      def submit_import_form(resource, params)
+        resource.forms('import').submit!('import' => params)
+      end
+
+      def parse_uploaded_response(content)
+        resource = Mode::API::Resource.new(content)
+      end
+
+      class << self
+        #
+        # Utilities that can be useful elsewhere
+        #
+
+        def find_package(source, path = nil)
+          path = path || search_path(source)
+
+          if path =~ /\/data$/
+            parent = parent_path(path)
+            find_package(source, parent)
+          elsif package_exists?(path)
+            DataPackage::Package.open(path)
+          else
+            nil
+          end
+        end
+
+        def build_package(source, options = {})
+          name = options[:name] || 'data'
+          dest = options[:dest] || tmpdir
+
+          if File.exist?(source)
+            converter = convert(source)
+            DataPackage::Package.new(dest).tap do |package|
+              package.name = name
+              package.version = '0.0.1'
+              package.resources << build_resource(converter)
+            end
+          else
+            nil
+          end
+        end
+
+        def find_or_build_package(source, options = {})
+          package = find_package(source)
+
+          if package.nil?
+            build_opts = {
+              :name => options[:name],
+              :dest => options[:dest]
+            }
+            package = build_package(source, build_opts)
+          else
+            package = prune_package(source, package)
+          end
+
+          package
+        end
+
+        def target_resource_path(path = nil)
+          "data/index#{path.nil? ? '.csv' : File.extname(path)}"
+        end
+
+        private
+
+        #
+        # Package Building
+        #
+
+        def build_resource(converter)
+          DataPackage::Resource.new(tmpdir, {
+            'name' => 'index.csv',
+            'path' => target_resource_path,
+            'size' => File.size?(target),
+            'schema' => build_schema(converter)
+          })
+        end
+
+        def build_schema(converter)
+          {
+            'fields' => build_fields(converter),
+            'dialect' => DataPackage::Dialect.new
+          }
+        end
+
+        def build_fields(converter)
+          converter.field_types.inject([]) do |fields, (field_name, field_type)|
+            fields << {'name' => field_name, 'type' => field_type.to_s}
+          end
+        end
+
+        #
+        # Data Conversion
+        #
+
+        def convert(source)
+          csv = DataKit::CSV::Parser.new(source)
+          analysis = DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => 1.0)
+          DataKit::CSV::Converter.convert(csv, analysis, target)
+        end
+
+        def tmpdir
+          @tmpdir ||= Dir.mktmpdir
+        end
+
+        def tmppath(path)
+          File.join(tmpdir, path)
+        end
+
+        def target(path = 'data', name = 'index.csv')
+          full_path = tmppath(path)
+          FileUtils.mkdir_p(full_path)
+          @target ||= File.join(full_path, name)
+        end
+
+        #
+        # Checks and Validations
+        #
+
+        def search_path(source)
+          File.file?(source) ? File.dirname(source) : source
+        end
+
+        def package_exists?(path)
+          DataPackage::Package.exist?(path)
+        end
+
+        def parent_path(path)
+          File.expand_path(File.join(path, '..'))
+        end
+
+        def resource_path(path)
+          matched = path.match(/((data\/)?[^\/]+)$/)
+          matched.nil? ? matched : matched.to_a.first
+        end
+
+        def first_resource(package)
+          package.resources.find{ |r| not r.path.nil? }
+        end
+
+        def find_resource(package, path)
+          package.resources.find{ |r| r.path == path }
+        end
+
+        def prune_package(source, package)
+          if File.directory?(source)
+            resource = first_resource(package)
+          elsif File.file?(source)
+            resource_path = resource_path(source)
+            resource = find_resource(package, resource_path)
+          end
+
+          package.resources = []
+          package.resources << resource
+          package
+        end
       end
     end
   end

data/lib/mode/commands/login.rb

@@ -0,0 +1,111 @@
+module Mode
+  module Commands
+    class Login < Thor
+      attr_reader :options
+      attr_reader :environment
+
+      def initialize(options = {})
+        @options = options
+
+        if options[:test]
+          @environment = :test
+        elsif options[:staging]
+          @environment = :staging
+        elsif options[:development]
+          @environment = :development
+        else
+          @environment = :production
+        end
+      end
+
+      no_commands do
+        include Mode::Commands::Helpers
+
+        def execute
+          username, password = configure_credentials
+          chosen_token       = choose_access_token.token
+          update_configuration(environment, username, chosen_token)
+        rescue => err
+          say err.message
+          Mode::Logger.instance.error(
+            "Commands::Login", err.message, err.backtrace)
+        end
+      end
+
+      private
+
+      def configure_api(username, password)
+        Mode::API::Request.configure(environment, {
+          :credentials => {
+            :username => username,
+            :access_token => password
+          }
+        })
+      end
+
+      def configure_credentials
+        say "Enter your Mode credentials:"
+        username = ask "Username:"
+        password = ask "Password:", :echo => false
+
+        # go ahead and configure API
+        configure_api(username, password)
+
+        return username, password
+      end
+
+      def fetch_access_tokens
+        resource = Mode::API::Request.get(:access_tokens)
+
+        if resource.is_a?(Mode::API::Resource)
+          resource.embedded('access_tokens').map do |access_token|
+            Mode::Auth::AccessToken.new(access_token)
+          end
+        elsif resource.status == 401
+          raise "Login failed, credentials invalid"
+        end
+      end
+
+      def prompt_access_tokens(access_tokens)
+        say "\nPlease choose an access token to use:\n\n"
+
+        choices = []
+        access_tokens.each_with_index do |token, index|
+          choices << (index + 1).to_s
+          say "#{index + 1}. #{token.name}"
+        end
+        choice = ask "\nWhich token would you like to use?:", :limited_to => choices
+      end
+
+      def choose_access_token
+        access_tokens = fetch_access_tokens
+
+        if access_tokens.length == 1
+          access_tokens.first
+        else
+          chosen = prompt_access_tokens(access_tokens)
+          access_tokens[chosen.to_i - 1]
+        end
+      end
+
+      def update_configuration(environment, username, access_token)        
+        config = find_or_create_config
+        config.username     = username
+        config.access_token = access_token
+        config.environment  = environment.to_s
+        config.save
+
+        say "Updated configuration at #{config.path}"
+      end
+
+      def find_or_create_config
+        if config_exists?
+          Mode::Config.new(config_dir)
+        else
+          FileUtils.mkdir_p(config_dir)
+          Mode::Config.init(config_dir)
+        end
+      end
+    end
+  end
+end

data/lib/mode/config.rb

@@ -2,52 +2,32 @@ require 'yaml'
 
 module Mode
   class Config
-    FILENAME = '.mode.yml'
+    include Mode::Configurable
 
-    attr_accessor :path
-
-    attr_accessor :username, :access_token
-
-    def initialize(path, filename = nil)
-      @path = self.class.full_path(path, filename)
-
-      if File.exist?(@path)
-        configure YAML.load_file(@path)
-      else
-        raise "Could not load configuration file from #{@path}"
-      end
-    end
-
-    def save
-      File.open(path, 'w+') do |file|
-        file.write(to_yaml)
-      end
-    end
+    # Config Variables
+    attr_accessor :username
+    attr_accessor :access_token
+    attr_accessor :environment
 
     class << self
-      def init(path, filename = nil)
-        File.open(full_path(path, filename), 'w+') do |file|
-          file.write({}.to_yaml)
-        end
-
-        new(path, filename)
-      end
-
-      def full_path(path, filename = nil)
-        File.expand_path(File.join(path, filename || Mode::Config::FILENAME))
+      def default_filename
+        'config.yml'
       end
     end
 
-  private
-    def configure(config)
+    private
+    
+    def configure(config = {})
       @username = config['username']
       @access_token = config['access_token']
+      @environment = config['environment'].to_s
     end
 
     def to_yaml
       {
         'username' => username,
-        'access_token' => access_token
+        'access_token' => access_token,
+        'environment' => environment
       }.to_yaml
     end
   end