mode 0.0.3

data/lib/mode/cli/base.rb

@@ -0,0 +1,15 @@
+module Mode
+  module CLI
+    class Base < Thor
+      #
+      # CLI commands are split out into separate files that just reopen this class
+      #  An alternative method for doing this would be to extend this class on include
+      #  And then I'd have an explicit list of the commands we were including
+      #
+
+      private
+      
+      include Mode::CLI::Helpers
+    end
+  end
+end

data/lib/mode/cli/helpers.rb

@@ -0,0 +1,22 @@
+module Mode
+  module CLI
+    module Helpers
+      def valid_file?(file)
+        !file.nil? && File.exist?(file)
+      end
+
+      def valid_table?(table)
+        table =~ /[\w\d\_\-]+\/[\w\d\_\-]+/
+      end
+
+      def pkg_name(path)
+        path.split('/').last.split('.').first
+      end
+
+      def sample_rate(path)
+        file_size = File.size(path)
+        DataKit::CSV::Analyzer.sample_rate(file_size)
+      end
+    end
+  end
+end

data/lib/mode/cli/import.rb

@@ -0,0 +1,62 @@
+module Mode
+  module CLI
+    class Base < Thor
+      desc "import SOURCE ACCOUNT/TABLENAME [--update | --replace]", "Import a flat file into the Mode data warehouse"
+      long_desc <<-LONGDESC
+        The import commands allows you to create, update and
+        replace tables in the Mode data warehouse with data from
+        flat files and data packages. The default action is create
+        with optional flags to update or replace a table.
+
+        Data can be imported from one of three source types
+
+        1. CSV File
+        \x5> $ mode import gdp_quarterly.csv besquared/gdp_quarterly
+
+        2. JSON File (LD-JSON)
+        \x5> $ mode import gdp_quarterly.json besquared/gdp_quarterly 
+
+        3. Data Package
+        \x5> $ mode import gdp_us/quarterly besquared/gdp_quarterly
+
+
+        Note: If you do not specify a data package resource name we'll attempt to use the first resource in the package.
+      LONGDESC
+      option :update, :type => :boolean
+      option :replace, :type => :boolean
+      option :primary_key, :banner => 'pos1[,pos2,...] (ex: 0,2)'
+      def import(source, table)
+        unless valid_table?(table)
+          error "Error: Invalid account or table name given"
+          return
+        end
+
+        if File.directory?(source)
+          unless Mode::Package::Base.exist?(source)
+            error "Error: Invalid package given"
+            return
+          end
+
+          package, resource_name = *source.split('/')
+          package = Mode::Package::Base.open(source)
+        else
+          unless valid_file?(source)
+            error "Error: Invalid source file given"
+            return
+          end
+
+          dst_path = Dir.mktmpdir
+          src_data = Mode::CSV::Parser.new(source)
+          builder = Mode::Package::Builder.new(src_data, dst_path, pkg_name(source), sample_rate(source))
+
+          package = builder.execute # make the package
+        end
+
+        account, table_name = *table.split('/')
+        resource_name = package.resources.first.name
+        
+        Mode::Commands::Import.new(account, table_name, package, resource_name).execute
+      end
+    end
+  end
+end

data/lib/mode/cli/package.rb

@@ -0,0 +1,13 @@
+module Mode
+  module CLI
+    class Base < Thor
+      desc "package <csv path> <package path>", "Creates a new data package from a csv file"
+      option :sample, :banner => '<sampling rate> (ex: 0.5)'
+      option :keys, :banner => '<positions> (ex: 0,2)', :default => String.new
+      def package(csv, path)
+        keys = options[:keys].split(',').collect(&:strip).collect(&:to_i)
+        Mode::Commands::Package.new(csv, path, options.merge(keys: keys)).execute
+      end
+    end
+  end
+end

data/lib/mode/cli/setup.rb

@@ -0,0 +1,12 @@
+module Mode
+  module CLI
+    class Base < Thor
+      desc "setup", "Setup a new mode configuration in the given directory (defaults to home)"
+      option :dir, :default => '~'
+      option :host, :default => 'www.modeanalytics.com'
+      def setup
+        Mode::Commands::Init.new(options).execute
+      end
+    end
+  end
+end

data/lib/mode/commands/analyze_field.rb

@@ -0,0 +1,70 @@
+require 'terminal-table'
+
+module Mode
+  module Commands
+    class AnalyzeField < Thor
+      attr_accessor :path
+      attr_accessor :field_pos
+      attr_accessor :options
+
+      def initialize(path, field_pos, options = {})
+        @path = path
+        @field_pos = field_pos.to_i
+        @options = options
+      end
+
+      no_commands do
+        include Mode::Commands::Helpers
+
+        def execute
+          if path.nil? || !File.exist?(path)
+            error "Couldn't find file at #{path}"
+            return
+          end
+
+          csv = DataKit::CSV::Parser.new(path)
+
+          field_name = csv.headers[field_pos]
+
+          say "Analyzing #{field_name} at #{path || 'input'} (Sampling #{'%.2f' % (100 * sampling_rate)}%)..."
+
+          analysis, total_time = timer_block do
+            DataKit::CSV::FieldAnalyzer.analyze(csv, field_pos, {
+              :match_type => match_type,
+              :sampling_rate => sampling_rate
+            })
+          end
+
+          say "Analyzed #{analysis.sample_count} of #{analysis.row_count} rows in #{'%.2f' % total_time} seconds\n"
+
+          display(analysis)
+        end
+      end
+
+      private
+      def display(analysis)
+        table = Terminal::Table.new(:headings => [
+          'Row No.', 'Type', 'Value'
+        ])
+
+        analysis.types.each do |type, rows|
+          rows.each do |row_num|
+            table.add_row [row_num, type, analysis.value_at(row_num)]
+          end
+        end
+
+        say table
+      end
+
+      def match_type
+        options[:match_type] ? options[:match_type].to_sym : :any
+      end
+
+      def sampling_rate
+        1
+        # file_size = File.size(path)
+        # options[:sample].to_f || DataKit::CSV::SchemaAnalyzer.sampling_rate(file_size)
+      end
+    end
+  end
+end

data/lib/mode/commands/analyze_schema.rb

@@ -0,0 +1,88 @@
+require 'terminal-table'
+
+module Mode
+  module Commands
+    class AnalyzeSchema < Thor
+      attr_accessor :path
+      attr_accessor :options
+
+      def initialize(path, options = {})
+        @path = path
+        @options = options
+      end
+
+      no_commands do
+        include Mode::Commands::Helpers
+
+        def execute
+          if path.nil? || !File.exist?(path)
+            error "Couldn't find file at #{path}"
+            return
+          end
+
+          csv = DataKit::CSV::Parser.new(path)
+
+          say "Analyzing #{path || 'input'} (Sampling #{'%.2f' % (100 * sample_rate)}%)..."
+
+          analyzer, total_time = timer_block do
+            DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => sample_rate)
+          end
+
+          say "Analyzed #{analyzer.sample_count} of #{analyzer.row_count} rows in #{'%.2f' % total_time} seconds\n"
+
+          display(analyzer)
+        end
+      end
+
+      private
+
+      def format_percentage_cell(numerator, denominator)
+        cell = { :alignment => :right }
+
+        if numerator == 0
+          cell[:value] = nil
+        else
+          cell[:value] = '%.2f' % (100 * (numerator / denominator.to_f)) + '%'
+        end
+
+        cell
+      end
+
+      def display(analysis)
+        table = Terminal::Table.new(:headings => [
+          'Field No.', 'Field', 'Type', 
+          'String (%)', 'Integer (%)', 'Number (%)',
+          'Date/Time (%)', 'Boolean (%)', 'Empty (%)'
+        ])
+
+        analysis.fields.each_with_index do |field_name, index|
+          row = [index, field_name]
+
+          field_type = analysis.type?(field_name)
+
+          if analysis.has_single_type?(field_name)
+            row << field_type
+          elsif analysis.has_only_numeric_types?(field_name)
+            row << field_type
+          else
+            row << '** ' + field_type
+          end
+
+          DataKit::Dataset::Field::Types.each do |type|
+            type_count = analysis.type_count(field_name, type)
+            row << format_percentage_cell(type_count, analysis.sample_count)
+          end
+
+          table.add_row(row)
+        end
+
+        say table
+      end
+
+      def sample_rate
+        file_size = File.size(path)
+        options[:sample] || DataKit::CSV::SchemaAnalyzer.sampling_rate(file_size)
+      end
+    end
+  end
+end

data/lib/mode/commands/helpers.rb

@@ -0,0 +1,10 @@
+module Mode
+  module Commands
+    module Helpers
+      def timer_block(&block)
+        started_at = Time.now
+        return yield, (Time.now.to_f - started_at.to_f)
+      end
+    end
+  end
+end

data/lib/mode/commands/import.rb

@@ -0,0 +1,37 @@
+module Mode
+  module Commands
+    class Import < Thor
+      attr_accessor :account
+      attr_accessor :table_name
+      attr_accessor :package
+      attr_accessor :resource_name
+
+      def initialize(account, table_name, package, resource_name)
+        @account = account
+        @table_name = table_name
+        @package = package
+        @resource_name = resource_name
+      end
+
+      no_commands do
+        def execute
+
+          puts "Importing #{package.name}/#{resource_name} to #{account}/#{table_name}"
+          # We need to check the datapackage.md5
+          # We need to check the md5 of the source
+          # If either of these have changed the package needs to be reverified
+
+          # 1. Compress to temporary dir
+          # 2. POST /imports with name and zipfile
+          # 3. Poll for execution status until finished or timeout (what's the timeout?)
+        end
+      end
+
+      private
+
+      def valid_package?(package)
+        # package.valid?
+      end
+    end
+  end
+end

data/lib/mode/commands/package.rb

@@ -0,0 +1,60 @@
+require 'digest/md5'
+
+module Mode
+  module Commands
+    class Package < Thor
+      attr_accessor :data
+      attr_accessor :path
+      attr_accessor :name
+      attr_accessor :options
+
+      def initialize(data, name, options = {})
+        @data = data
+        @path = name
+        @name = name
+        @options = options
+      end
+
+      no_commands do
+        def execute
+          unless valid_data?(data)
+            error "Error: Couldn't find valid data file at #{data}"
+            return
+          end
+
+          unless valid_name?(name)
+            error "Error: Data package names can only contain letters, numbers, hyphens and underscores"
+            return
+          end
+
+          if Mode::Package::Base.exist?(name)
+            error "Error: A data package already exists at #{name}"
+            return
+          else
+            FileUtils.mkdir_p(name)
+          end
+
+          csv = Mode::CSV::Parser.new(data)
+          package = Mode::Package::Builder.new(csv, path, name, sample_rate).execute
+
+          say "Finished packaging #{name}!"
+          say "Use `mode table create <account>/<tablename> #{name}` to create a table in the public data warehouse"
+        end
+      end
+
+      private
+      
+      def valid_name?(name)
+        name =~ /\A[\w\d\-\_]+\z/
+      end
+
+      def valid_data?(data)
+        !data.nil? && File.exist?(data)
+      end
+
+      def sample_rate
+        Mode::CSV::Analyzer.sample_rate(File.size(data))
+      end
+    end
+  end
+end

data/lib/mode/commands/setup.rb

@@ -0,0 +1,36 @@
+module Mode
+  module Commands
+    class Setup < Thor
+      attr_accessor :options
+
+      def initialize(options = {})
+        @options = options
+      end
+
+      no_commands do 
+        def execute
+          config_path = Mode::Config.full_path(options[:dir])
+          
+          if File.exist?(config_path)
+            say "Configuration at #{config_path} already exists"
+            unless yes? "Would you like to modify the existing configuration? (y/n):"
+              return
+            end
+          end
+
+          say "Initializing configuration at #{config_path}"
+
+          config = Mode::Config.init(options[:dir])
+
+          config.username = ask "Mode username:"
+
+          say "Your can view your access tokens at http://#{options[:host]}/accounts/#{config.username}/access_tokens"
+
+          config.access_token = ask "Access token for #{config.username}:"
+
+          config.save
+        end
+      end
+    end
+  end
+end

data/lib/mode/config.rb

@@ -0,0 +1,54 @@
+require 'yaml'
+
+module Mode
+  class Config
+    FILENAME = '.mode.yml'
+
+    attr_accessor :path
+
+    attr_accessor :username, :access_token
+
+    def initialize(path, filename = nil)
+      @path = self.class.full_path(path, filename)
+
+      if File.exist?(@path)
+        configure YAML.load_file(@path)
+      else
+        raise "Could not load configuration file from #{@path}"
+      end
+    end
+
+    def save
+      File.open(path, 'w+') do |file|
+        file.write(to_yaml)
+      end
+    end
+
+    class << self
+      def init(path, filename = nil)
+        File.open(full_path(path, filename), 'w+') do |file|
+          file.write({}.to_yaml)
+        end
+
+        new(path, filename)
+      end
+
+      def full_path(path, filename = nil)
+        File.expand_path(File.join(path, filename || Mode::Config::FILENAME))
+      end
+    end
+
+  private
+    def configure(config)
+      @username = config['username']
+      @access_token = config['access_token']
+    end
+
+    def to_yaml
+      {
+        'username' => username,
+        'access_token' => access_token
+      }.to_yaml
+    end
+  end
+end