mode 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: efdff2f30299e26fcb53225a2ecc9d067b276d00
-  data.tar.gz: c7d5c3e22b7cde0a38f51cf2e01d65d60f199542
+  metadata.gz: 6bfab7e5e64907e822459c3d0750038e99ea0ce7
+  data.tar.gz: 6dfeb3b3138921cf3b95439d3b5d7775cee76025
 SHA512:
-  metadata.gz: ba7761578619bfb884531acfc5d836b26b493db2135b496b98f8b52294b84a495251ad27cd9616a489d2618a2f59791defe381987199fa245b67697a5df77e90
-  data.tar.gz: 49f72b8cdc4fdc5e48299ac4806e7b4eb2e1cae25cedfb29063558001874c9395affa99048e040f65933520924f362dac6f5db374aaed1da6100b0a37b988020
+  metadata.gz: e69bfae6c5ba1335c3755c376871e2406a1e1615427d5578913fd41799c9808d606034727691bf87f71b6affe0c82a137f6e665e52e32c640256c12e4c6b7c18
+  data.tar.gz: 38a4242c9208533c02730ce09382810104b00769c9eec976f137de5a1accb5e7f2f53d55a1f692c154b9ad52be591efa81c027b6638bc0f2cd2302081023cbb7

data/.gitignore

@@ -1,5 +1,6 @@
 *.gem
 *.rbc
+*.ruby-version
 .bundle
 .config
 .yardoc

data/README.md

@@ -6,23 +6,77 @@ This package provides command line tools for managing datasets and connecting da
 * Personal data warehouse connectivity
 * Dataset formatting and importing (CSV)
 
-## Installation
+## Prerequisites
 
-### Standalone
+This package requires at least Ruby 1.9 and Ruby 2.0 is recommended.
 
-From your terminal run:
+If you don't have an up to date version of Ruby or you're not sure then use the instructions below to get going.
 
-    $ gem install mode
+### Mac OSX
 
-### Bundler
+To install Ruby on OSX complete the 4 steps below.
 
-Add this line to your application's Gemfile:
+1\. Install Homebrew
 
-    gem 'mode'
+```
+ruby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)"
+```
+
+2\. Install rbenv
+
+```
+brew update
+brew install rbenv ruby-build
+echo 'eval "$(rbenv init -)"' >> ~/.bash_profile
+source ~/.bash_profile
+```
+
+3\. Install Ruby 2.0
+
+Note: This usually takes several minutes
+
+```
+rbenv install 2.0.0-p353
+rbenv global 2.0.0-p353
+rbenv rehash
+```
+
+4\. Install the mode gem
+
+```
+gem install mode
+rbenv rehash
+```
+
+### OSX Combined
+
+For convenience you can just copy and paste all the lines at once into your terminal
+
+```
+ruby -e "$(curl -fsSL https://raw.github.com/Homebrew/homebrew/go/install)"
+brew update
+brew install rbenv ruby-build
+echo 'eval "$(rbenv init -)"' >> ~/.bash_profile
+source ~/.bash_profile
+rbenv install 2.0.0-p353
+rbenv global 2.0.0-p353
+gem install mode
+rbenv rehash
+```
 
-And then execute:
+### Windows
 
-    $ bundle
+To install a current version of Ruby on windows complete the 2 steps below.
+
+1\. Install Ruby 2.0
+
+You can download the latest ruby version from [RubyInstaller](http://rubyinstaller.org/downloads/).
+
+2\. Install the mode gem
+
+```
+$ gem install mode
+```
 
 ## Setup
 

data/lib/mode.rb

@@ -1,13 +1,19 @@
 require 'thor'
 require 'data_kit'
+require 'data_package'
 
-# Base
-require "mode/version"
+require 'mode/version'
+
+# Config
 require 'mode/config'
 
 # Commands
 require 'mode/commands/helpers'
 require 'mode/commands/setup'
 require 'mode/commands/import'
+require 'mode/commands/package'
 require 'mode/commands/analyze_field'
-require 'mode/commands/analyze_schema'
+require 'mode/commands/analyze_schema'
+
+# Utilities
+require 'mode/package_builder' 

data/lib/mode/cli.rb

@@ -3,4 +3,5 @@ require 'mode/cli/helpers'
 require 'mode/cli/base'
 require 'mode/cli/setup'
 require 'mode/cli/import'
-require 'mode/cli/analyze'
+require 'mode/cli/analyze'
+require 'mode/cli/package'

data/lib/mode/cli/analyze.rb

@@ -2,9 +2,9 @@ module Mode
   module CLI
     class Base < Thor
       desc "analyze PATH [--field=POSITION] [--match-type=TYPE]", "Analyzes a dataset or a field "
-      option :sample, :banner => 'RATE (Example: 0.5)'
-      option :field, :banner => 'POSITION  (Beginning at 0, Example: 10)'
-      option :match_type, :banner => 'TYPE (One of: string, number, integer, datetime, boolean)'
+      option :sample, :banner => 'RATE', :desc => "Proportion of rows to inspect. Example: 0.5"
+      option :field, :banner => 'POSITION', :desc => "Field positions begin at 0. Example: 10"
+      option :match_type, :banner => 'TYPE', :desc => "One of the following: string, number, integer, datetime, boolean"
       # option :keys, :banner => 'POSITIONS (ex: 0,2)', :default => String.new
       def analyze(path = nil)
         #keys = options[:keys].split(',').collect(&:strip).collect(&:to_i)

data/lib/mode/cli/package.rb

@@ -1,12 +1,15 @@
 module Mode
   module CLI
     class Base < Thor
-      desc "package <csv path> <package path>", "Creates a new data package from a csv file"
-      option :sample, :banner => '<sampling rate> (ex: 0.5)'
-      option :keys, :banner => '<positions> (ex: 0,2)', :default => String.new
-      def package(csv, path)
-        keys = options[:keys].split(',').collect(&:strip).collect(&:to_i)
-        Mode::Commands::Package.new(csv, path, options.merge(keys: keys)).execute
+      desc "package SOURCE PACKAGEPATH [--name=NAME] [--resource-name=NAME]", "Creates a new data package from a csv file"
+      option :name, :desc => 'The name of the package'
+      option :resource_name, :default => 'data', :desc => 'The name of the resource in package'
+      def package(source, dest)
+        parts = dest.split(File::Separator)
+
+        name = options[:name] || parts.last
+        package_path = parts.length == 1 ? parts.first : parts[0, parts.length - 1]
+        Mode::Commands::Package.new(source, package_path, name, options[:resource_name]).execute
       end
     end
   end

data/lib/mode/commands/analyze_schema.rb

@@ -22,10 +22,10 @@ module Mode
 
           csv = DataKit::CSV::Parser.new(path)
 
-          say "Analyzing #{path || 'input'} (Sampling #{'%.2f' % (100 * sample_rate)}%)..."
+          say "Analyzing #{path || 'input'} (Sampling #{'%.2f' % (100 * sampling_rate)}%)..."
 
           analysis, total_time = timer_block do
-            DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => sample_rate)
+            DataKit::CSV::SchemaAnalyzer.analyze(csv, :sampling_rate => sampling_rate)
           end
 
           puts analysis.use_type_hints
@@ -67,7 +67,7 @@ module Mode
           elsif analysis.has_only_numeric_types?(field_name)
             row << field_type
           else
-            row << '** ' + field_type
+            row << '** ' + field_type.to_s
           end
 
           DataKit::Dataset::Field::Types.each do |type|
@@ -81,9 +81,9 @@ module Mode
         say table
       end
 
-      def sample_rate
+      def sampling_rate
         file_size = File.size(path)
-        options[:sample].to_f || DataKit::CSV::SchemaAnalyzer.sampling_rate(file_size)
+        (options[:sample] || DataKit::CSV::SchemaAnalyzer.sampling_rate(file_size)).to_f
       end
     end
   end

data/lib/mode/commands/package.rb

@@ -4,15 +4,15 @@ module Mode
   module Commands
     class Package < Thor
       attr_accessor :data
-      attr_accessor :path
-      attr_accessor :name
-      attr_accessor :options
+      attr_accessor :base_path
+      attr_accessor :package_name
+      attr_accessor :resource_name
 
-      def initialize(data, name, options = {})
+      def initialize(data, base_path, package_name, resource_name)
         @data = data
-        @path = name
-        @name = name
-        @options = options
+        @base_path = base_path
+        @package_name = package_name
+        @resource_name = resource_name
       end
 
       no_commands do
@@ -22,20 +22,20 @@ module Mode
             return
           end
 
-          unless valid_name?(name)
+          unless valid_name?(package_name)
             error "Error: Data package names can only contain letters, numbers, hyphens and underscores"
             return
           end
 
-          if Mode::Package::Base.exist?(name)
-            error "Error: A data package already exists at #{name}"
+          if DataPackage::Package.exist?(base_path)
+            error "Error: A data package already exists at #{base_path}"
             return
           else
-            FileUtils.mkdir_p(name)
+            FileUtils.mkdir_p(File.join(base_path, package_name))
           end
 
-          csv = Mode::CSV::Parser.new(data)
-          package = Mode::Package::Builder.new(csv, path, name, sample_rate).execute
+          csv = DataKit::CSV::Parser.new(data)
+          package = Mode::PackageBuilder.new(csv, base_path, package_name, resource_name).execute
 
           say "Finished packaging #{name}!"
           say "Use `mode table create <account>/<tablename> #{name}` to create a table in the public data warehouse"
@@ -51,10 +51,6 @@ module Mode
       def valid_data?(data)
         !data.nil? && File.exist?(data)
       end
-
-      def sample_rate
-        Mode::CSV::Analyzer.sample_rate(File.size(data))
-      end
     end
   end
 end

data/lib/mode/package_builder.rb

@@ -0,0 +1,57 @@
+module Mode
+  class PackageBuilder
+    attr_accessor :data
+    attr_accessor :base_path
+    attr_accessor :package_name
+    attr_accessor :resource_name
+
+    def initialize(data, base_path, package_name, resource_name)
+      @data = data
+      @base_path = base_path
+      @package_name = package_name
+      @resource_name = resource_name
+    end
+
+    def execute
+      package = init_package
+      analysis = build_analysis(data)
+      converter = convert_data(data, analysis, package)
+      package.resources << build_resource(analysis, package, resource_name, "#{resource_name}.csv")
+      package.dump && package
+    end
+
+    private
+    
+    def valid_name?(name)
+      name =~ /\A[\w\d\-\_]+\z/
+    end
+
+    def md5hash(path)
+      Digest::MD5.hexdigest(File.read(data_path))
+    end
+
+    def init_package
+      DataPackage::Package.init(base_path, package_name)
+    end
+
+    def build_analysis(data)
+      DataKit::CSV::Analyzer.analyze(data, :sampling_rate => 1)
+    end
+
+    def convert_data(data, analysis, package)
+      # say "Converting data into standard format..."
+      dest_path = File.join(package.base_path, "#{resource_name}.csv")
+      converter = DataKit::CSV::Converter.convert(data, analysis, dest_path)
+    end
+
+    def build_resource(analysis, package, name)
+      Mode::Package::Resource.build(
+        analysis.schema.to_package, path, {
+          'name' => name, 'format' => 'csv',
+          'dialect' => Mode::Package::Dialect.new,
+          'hash' => md5hash(File.join(package.base_path, "#{name}.csv"))
+        }
+      )
+    end
+  end
+end

data/lib/mode/version.rb

@@ -1,3 +1,3 @@
 module Mode
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mode
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Mode Analytics
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-17 00:00:00.000000000 Z
+date: 2013-12-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -181,6 +181,7 @@ files:
 - lib/mode/commands/package.rb
 - lib/mode/commands/setup.rb
 - lib/mode/config.rb
+- lib/mode/package_builder.rb
 - lib/mode/version.rb
 - mode.gemspec
 - spec/commands/analyze_schema_spec.rb