datapimp 1.2.1 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4f7ee3a50c4372896b49f7ac03339154fcba987d
-  data.tar.gz: cb2eac6337c4596b18f15ca04962eba1fce5ebe2
+  metadata.gz: 347b281ae0e4e748f903410bdf0218a1b24a00a5
+  data.tar.gz: f497dcd87a2a07bc2bbec8966e2c86cde589aaf6
 SHA512:
-  metadata.gz: 2eab363452cee9b6f3b0b0d426ecc84255d27d98c424273dec232babc472c0a9657a770f6374f0819a89bddf81fdafa7b17ae27d1a0fdb031efd05e626847336
-  data.tar.gz: 2053ec83701f4c235bcb6f78a376a9f52d3d8b640d3713703b8cb0214c4a78ea214e1045884a8b9a853d8606cb8ffd061108a91faeb2e23309a57f9ab153714b
+  metadata.gz: c37075bc592d3c1e55246f6b83341f466cef1f08faaea0734b4ba04b6e84187aa33302721101b94ed6d6fa7e8edc703094922ba4789c3a48b68b0d9d90284cae
+  data.tar.gz: 7e8a151dc77e82ca4a01f5705fe7f20bf718bf332c1f91037a4d7a486ed42f2fbb3555a6e4dc15321ef1676c804870ad5afa5b2c852ecabc9122d3e697def470

data/bin/console

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "datapimp"
+
+require "pry"
+Pry.start

data/bin/datapimp

@@ -9,6 +9,7 @@ require "commander/import"
 require 'datapimp'
 require 'datapimp/cli'
 
+$datapimp_cli = "datapimp"
 $terminal.wrap_at = 120
 
 program :name, 'Datapimp'

data/datapimp.gemspec

@@ -20,11 +20,11 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'pry', '~> 0.10'
   spec.add_dependency 'hashie', '>= 3.0.4' 
-  spec.add_dependency 'commander', '~> 4.3'
+  spec.add_dependency 'commander', '>= 4.3'
   spec.add_dependency 'terminal-table'
   spec.add_dependency 'fog-aws', '>= 0.1'
   spec.add_dependency 'dropbox-api', '>= 0.4.7'
-  spec.add_dependency 'google_drive', '~> 1.0'
+  spec.add_dependency 'google_drive', '>= 1.0'
   spec.add_dependency 'google-api-client', '>= 0.8'
   spec.add_dependency 'rack-contrib', '~> 1.2'
   spec.add_dependency 'uri_template', '~> 0.7'
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'github-fs', '~> 0'
   spec.add_dependency 'colored', '> 0.0'
   spec.add_dependency 'multi_json', '~> 1.10'
-  spec.add_dependency 'pivotal-tracker', '~> 0.5.13'
+  spec.add_dependency 'tracker_api', '>= 0.2.10'
   spec.add_dependency 'keen', '~> 0.9.0'
 
   # these are locked to specific versions so that 
@@ -46,6 +46,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'nokogiri', '~> 1.6'
   spec.add_dependency 'unf', '0.1.4'
   spec.add_dependency 'unf_ext', '0.0.6'
+  spec.add_dependency 'git-version-bump'
+
 
   spec.add_development_dependency "rake", '~> 0'
   spec.add_development_dependency 'rspec', '~> 3.3.0'

data/lib/datapimp/cli.rb

@@ -3,6 +3,7 @@ require 'launchy'
 module Datapimp
   module Cli
     def self.load_commands(from_path=nil)
+      $datapimp_cli ||= "datapimp"
       from_path ||= Datapimp.lib.join("datapimp","cli")
       Dir[from_path.join("**/*.rb")].each {|f| require(f) }
     end
@@ -44,6 +45,16 @@ module Datapimp
         c.option '--aws-access-key-id', String, 'AWS Access Key ID'
       end
 
+      if services.include?(:keen)
+        c.option '--keen-project-id', String, 'Keen Project ID'
+        c.option '--keen-read-key', String, 'Keen Read Key'
+        c.option '--keen-write-key', String, 'Keen Write Key'
+      end
+
+      if services.include?(:pivotal)
+        c.option '--pivotal-access-token', String, 'Pivotal Tracker Access Token'
+      end
+
     end
   end
 end

data/lib/datapimp/cli/config.rb

@@ -1,5 +1,5 @@
 command 'config' do |c|
-  c.syntax = "datapimp config [OPTIONS]"
+  c.syntax = "#{$datapimp_cli} config [OPTIONS]"
   c.description = "Shows the configuration options being used"
 
   c.option '--env', "Output compatible with .env files"
@@ -16,7 +16,7 @@ command 'config' do |c|
 end
 
 command 'config set' do |c|
-  c.syntax = 'datapimp config set KEY=VALUE KEY=VALUE [options]'
+  c.syntax = "#{$datapimp_cli} config set KEY=VALUE KEY=VALUE [options]"
   c.description = 'manipulate configuration settings'
 
   c.option '--global', 'Set the configuration globally'

data/lib/datapimp/cli/create.rb

@@ -1,5 +1,5 @@
 command 'create cache invalidations' do |c|
-  c.syntax = 'datapimp create cache invalidations'
+  c.syntax = "#{$datapimp_cli} create cache invalidations"
   c.description = 'invalidate remote cache layers (i.e. cloudfront after a s3 deploy)'
 
   Datapimp::Cli.accepts_keys_for(c, :amazon)
@@ -75,7 +75,7 @@ command 'create cache invalidations' do |c|
 end
 
 command 'create s3 bucket' do |c|
-  c.syntax = 'datapimp create s3 bucket BUCKETNAME'
+  c.syntax = "#{$datapimp_cli} create s3 bucket BUCKETNAME"
   c.description = 'create an s3 bucket to use for website hosting'
 
   Datapimp::Cli.accepts_keys_for(c, :amazon)
@@ -90,7 +90,7 @@ command 'create s3 bucket' do |c|
 end
 
 command 'create cloudfront distribution' do |c|
-  c.syntax = "datapimp create cloudfront distribution"
+  c.syntax = "#{$datapimp_cli} create cloudfront distribution"
   c.description = "create a cloudfront distribution to link to a specific bucket"
 
   Datapimp::Cli.accepts_keys_for(c, :amazon)
@@ -99,6 +99,8 @@ command 'create cloudfront distribution' do |c|
   c.option '--domains DOMAINS', Array, 'What domains will be pointing to this bucket?'
 
   c.action do |args, options|
+    options.default(bucket: args.first)
+
     bucket = Datapimp::Sync::S3Bucket.new(remote: options.bucket)
 
     cdn_options = {

data/lib/datapimp/cli/list.rb

@@ -1,5 +1,5 @@
 command "list spreadsheets" do |c|
-  c.syntax = "datapimp list spreadsheets"
+  c.syntax = "#{$datapimp_cli} list spreadsheets"
   c.description = "list the spreadsheets which can be used as datasources"
 
   c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
@@ -21,14 +21,14 @@ command "list spreadsheets" do |c|
     if lines.length > 0
       puts "\n\nExample:"
       puts "====="
-      puts "datapimp sync data #{ lines.first.split(/\t/).first } --type google-spreadsheet"
+      puts "#{$datapimp_cli} sync data #{ lines.first.split(/\t/).first } --type google-spreadsheet"
       puts "\n\n"
     end
   end
 end
 
 command "list folders" do |c|
-  c.syntax= "datapimp list folders [OPTIONS]"
+  c.syntax= "#{$datapimp_cli} list folders [OPTIONS]"
   c.description= "lists folders in a remote service"
 
   c.option '--type SERVICE', String, 'Which service to search: dropbox, google, amazon'

data/lib/datapimp/cli/run.rb

@@ -1,5 +1,5 @@
 command "run" do |c|
-  c.syntax = "datapimp run FILE"
+  c.syntax = "#{$datapimp_cli} run FILE"
   c.description = "runs a script in the context of the datapimp config"
 
   c.option '--format FORMAT', String, 'which format should we serialize the result? json default'

data/lib/datapimp/cli/setup.rb

@@ -1,5 +1,5 @@
 command "setup google" do |c|
-  c.syntax = "datapimp set up google"
+  c.syntax = "#{$datapimp_cli} set up google"
   c.description = "setup integration with google drive"
 
   c.action do |args, options|
@@ -8,7 +8,7 @@ command "setup google" do |c|
 end
 
 command "setup amazon" do |c|
-  c.syntax = "datapimp setup amazon"
+  c.syntax = "#{$datapimp_cli} setup amazon"
   c.description = "setup integration with amazon"
 
   c.action do |args, options|
@@ -17,7 +17,7 @@ command "setup amazon" do |c|
 end
 
 command "setup dropbox" do |c|
-  c.syntax = "datapimp set up dropbox"
+  c.syntax = "#{$datapimp_cli} set up dropbox"
   c.description = "setup integration with dropbox"
 
   c.action do |args, options|
@@ -26,10 +26,19 @@ command "setup dropbox" do |c|
 end
 
 command "setup github" do |c|
-  c.syntax = "datapimp set up github"
+  c.syntax = "#{$datapimp_cli} set up github"
   c.description = "setup integration with github"
 
   c.action do |args, options|
     Datapimp::Sync.github.setup()
   end
 end
+
+command "setup pivotal" do |c|
+  c.syntax = "#{$datapimp_cli} set up github"
+  c.description = "setup integration with github"
+
+  c.action do |args, options|
+    Datapimp::Sync.pivotal.setup()
+  end
+end

data/lib/datapimp/cli/sync.rb

@@ -1,6 +1,6 @@
 command "sync folder" do |c|
   c.description = "Synchronize the contents of a local folder with a file sharing service"
-  c.syntax = "datapimp sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
+  c.syntax = "#{$datapimp_cli} sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
 
   c.option '--type TYPE', String, "Which service is hosting the folder"
   c.option '--action ACTION', String, "Which sync action to run? push, pull"
@@ -18,7 +18,7 @@ end
 
 command "sync data" do |c|
   c.description = "Synchronize the contents of a local data store with its remote source"
-  c.syntax = "datapimp sync data [OPTIONS]"
+  c.syntax = "#{$datapimp_cli} sync data [OPTIONS]"
 
   c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
   c.option '--output FILE', String, "Write the output to a file"
@@ -30,17 +30,19 @@ command "sync data" do |c|
   c.option '--limit LIMIT', Integer, "Limit the number of results for Pivotal resources"
   c.option '--offset OFFSET', Integer, "Offset applied when using the limit option for Pivotal resources"
 
-  c.example "Syncing an excel file from dropbox ", "datapimp sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
-  c.example "Syncing a google spreadsheet", "datapimp sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
-  c.example "Syncing Pivotal Tracker data, user activity", "datapimp sync data --type pivotal --view user-activity"
-  c.example "Syncing Pivotal Tracker data, project activity", "datapimp sync data --type pivotal --view project-activity PROJECT_ID"
-  c.example "Syncing Pivotal Tracker data, project stories", "datapimp sync data --type pivotal --view project-stories PROJECT_ID"
-  c.example "Syncing Pivotal Tracker data, project story notes", "datapimp sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
-  c.example "Syncing keen.io data, extraction from an event_collection", "datapimp sync data --type keen EVENT_COLLECTION"
-  c.example "Syncing Github Issues", "datapimp sync data --type github --view issues REPOSITORY"
-  c.example "Syncing Github Issue Comments", "datapimp sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
-
-  Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox)
+  c.example "Syncing an excel file from dropbox ", "#{$datapimp_cli} sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
+  c.example "Syncing a google spreadsheet", "#{$datapimp_cli} sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
+  c.example "Syncing keen.io data, extraction from an event_collection", "#{$datapimp_cli} sync data --type keen EVENT_COLLECTION"
+
+  c.example "Syncing Github Issues", "#{$datapimp_cli} sync data --type github --view issues REPOSITORY"
+  c.example "Syncing Github Issue Comments", "#{$datapimp_cli} sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
+
+  c.example "Syncing Pivotal Tracker data, user activity", "#{$datapimp_cli} sync data --type pivotal --view user-activity"
+  c.example "Syncing Pivotal Tracker data, project activity", "#{$datapimp_cli} sync data --type pivotal --view project-activity PROJECT_ID"
+  c.example "Syncing Pivotal Tracker data, project stories", "#{$datapimp_cli} sync data --type pivotal --view project-stories PROJECT_ID"
+  c.example "Syncing Pivotal Tracker data, project story notes", "#{$datapimp_cli} sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
+
+  Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox, :keen, :pivotal)
 
   c.action do |args, options|
     options.default(view:"to_s")

data/lib/datapimp/cli/view.rb

@@ -1,5 +1,5 @@
 command 'view amazon setup' do |c|
-  c.syntax = 'datapimp view amazon setup'
+  c.syntax = "#{$datapimp_cli} view amazon setup"
   c.description = 'view the amazon (s3 + cloudfront) setup'
 
   Datapimp::Cli.accepts_keys_for(c, :amazon)

data/lib/datapimp/clients/keen.rb

@@ -0,0 +1,48 @@
+module Datapimp
+  module Clients
+    class Keen
+      include Singleton
+
+      def self.method_missing(meth, *args, &block)
+        if client.respond_to?(meth)
+          return client.send(meth, *args, &block)
+        end
+
+        super
+      end
+
+      def self.client(options={})
+        Keen
+      end
+
+      def options
+        @options ||= {}
+      end
+
+      def with_options(opts={})
+        options.merge!(opts)
+        self
+      end
+
+      def setup(options={})
+        access_token = options[:keen_read_key] || Datapimp.config.keen_read_key
+        project_id = options[:keen_project_id] || Datapimp.config.keen_project_id
+
+        unless access_token.to_s.length > 1
+          if respond_to?(:ask)
+            access_token = ask("Enter a keen read key when  you have one", String)
+          end
+        end
+
+        unless project_id.to_s.length > 1
+          if respond_to?(:ask)
+            project_id = ask("Enter a keen read key when  you have one", String)
+          end
+        end
+
+        Datapimp.config.set(:keen_read_key, access_token) if access_token.to_s.length > 1
+        Datapimp.config.set(:keen_project_id, project_id) if project_id.to_s.length > 1
+      end
+    end
+  end
+end

data/lib/datapimp/configuration.rb

@@ -15,8 +15,10 @@ module Datapimp
       github_access_token: '',
 
       pivotal_access_token: '',
-      keen_project_id: ENV.fetch('KEEN_PROJECT_ID', ''),
-      keen_read_key: ENV.fetch('KEEN_READ_KEY', ''),
+
+      keen_project_id: '',
+      keen_read_key: '',
+      keen_write_key: '',
 
       dnsimple_api_token: '',
       dnsimple_username: '',

data/lib/datapimp/sources/google.rb

@@ -1,212 +1 @@
-module Datapimp::Sources
-  class GoogleSpreadsheet < Datapimp::Sources::Base
-    requires :key
-
-    attr_accessor :key,
-                  :session,
-                  :name
-
-    def initialize name, options={}
-      @options  = options
-
-      if name.is_a?(GoogleDrive::Spreadsheet)
-        @spreadsheet = name
-        @name = @spreadsheet.title
-        @key = @spreadsheet.key
-      end
-
-      @key      ||= options[:key]
-      @session  ||= options.fetch(:session) { Datapimp::Sync.google.api }
-
-      ensure_valid_options!
-    end
-
-    def self.create_from_file(path, title)
-      if find_by_title(title)
-        raise 'Spreadsheet with this title already exists'
-      end
-
-      session.upload_from_file(path, title, :content_type => "text/csv")
-
-      find_by_title(title)
-    end
-
-    def self.[](key_or_title)
-      find_by_key(key_or_title) || find_by_title(key_or_title)
-    end
-
-    def self.find_by_key(key)
-      sheet = session_spreadsheets.detect do |spreadsheet|
-        spreadsheet.key == key
-      end
-
-      sheet && new(sheet, session: Datapimp::Sync.google.session)
-    end
-
-    def self.find_by_title title
-      sheet = session_spreadsheets.detect do |spreadsheet|
-        spreadsheet.title.match(title)
-      end
-
-      sheet && new(sheet, session: Datapimp::Sync.google.session)
-    end
-
-    def self.session_spreadsheets
-      @session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
-    end
-
-    def self.create_from_data(data, options={})
-      require 'csv'
-
-      headers = Array(options[:headers]).map(&:to_s)
-
-      tmpfile = "tmp-csv.csv"
-
-      CSV.open(tmpfile, "wb") do |csv|
-        csv << headers
-
-        data.each do |row|
-          csv << headers.map do |header|
-            row = row.stringify_keys
-            row[header.to_s]
-          end
-        end
-      end
-
-      spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
-
-      new(spreadsheet.title, key: spreadsheet.key)
-    end
-
-
-    def title
-      @name ||= spreadsheet.try(:title)
-    end
-
-    def edit_url
-      spreadsheet.human_url
-    end
-
-    def share_write_access_with *emails
-      acl = spreadsheet.acl
-
-      Array(emails).flatten.each do |email|
-        acl.push scope_type: "user",
-                 with_key: false,
-                 role: "writer",
-                 scope: email
-      end
-    end
-
-    def share_read_access_with *emails
-      acl = spreadsheet.acl
-
-      Array(emails).flatten.each do |email|
-        acl.push scope_type: "user",
-                 with_key: false,
-                 role: "reader",
-                 scope: email
-      end
-    end
-
-    def add_to_collection collection_title
-      collection = if collection_title.is_a?(GoogleDrive::Collection)
-        collection_title
-      else
-        session.collections.find do |c|
-          c.title == collection_title
-        end
-      end
-
-      if !collection
-        collection_names = session.collections.map(&:title)
-        raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
-      end
-    end
-
-    def spreadsheet_key
-      key
-    end
-
-    def stale?
-      (!need_to_refresh? && (age > max_age)) || fresh_on_server?
-    end
-
-    def fresh_on_server?
-      refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
-    end
-
-    def last_updated_at
-      if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
-        DateTime.parse(value).to_i
-      else
-        Time.now.to_i
-      end
-    end
-
-    def fetch
-      self.raw = process_worksheets
-    end
-
-    def preprocess
-      single? ? raw.values.flatten : raw
-    end
-
-    protected
-
-      def process_worksheets
-        worksheets.inject({}.to_mash) do |memo, parts|
-          k, ws = parts
-          header_row = Array(ws.rows[0])
-          column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
-          rows = ws.rows.slice(1, ws.rows.length)
-
-          row_index = 1
-          memo[k] = rows.map do |row|
-            col_index = 0
-
-            _record = column_names.inject({}) do |record, field|
-              record[field] = "#{ row[col_index] }".strip
-              record["_id"] = row_index
-              col_index += 1
-              record
-            end
-
-            row_index += 1
-
-            _record
-          end
-
-          memo
-        end
-      end
-
-      def single?
-        worksheets.length == 1
-      end
-
-      def header_rows_for_worksheet key
-        if key.is_a?(Fixnum)
-          _worksheets[key]
-        else
-          worksheets.fetch(key)
-        end
-      end
-
-      def worksheets
-        @worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
-          key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
-          memo[key] = ws
-          memo
-        end
-      end
-
-      def _worksheets
-        @_worksheets ||= spreadsheet.worksheets
-      end
-
-      def spreadsheet
-        @spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
-      end
-  end
-end
+require 'datapimp/sources/google_spreadsheet'

data/lib/datapimp/sources/google_spreadsheet.rb

@@ -0,0 +1,212 @@
+module Datapimp::Sources
+  class GoogleSpreadsheet < Datapimp::Sources::Base
+    requires :key
+
+    attr_accessor :key,
+                  :session,
+                  :name
+
+    def initialize name, options={}
+      @options  = options
+
+      if name.is_a?(GoogleDrive::Spreadsheet)
+        @spreadsheet = name
+        @name = @spreadsheet.title
+        @key = @spreadsheet.key
+      end
+
+      @key      ||= options[:key]
+      @session  ||= options.fetch(:session) { Datapimp::Sync.google.api }
+
+      ensure_valid_options!
+    end
+
+    def self.create_from_file(path, title)
+      if find_by_title(title)
+        raise 'Spreadsheet with this title already exists'
+      end
+
+      session.upload_from_file(path, title, :content_type => "text/csv")
+
+      find_by_title(title)
+    end
+
+    def self.[](key_or_title)
+      find_by_key(key_or_title) || find_by_title(key_or_title)
+    end
+
+    def self.find_by_key(key)
+      sheet = session_spreadsheets.detect do |spreadsheet|
+        spreadsheet.key == key
+      end
+
+      sheet && new(sheet, session: Datapimp::Sync.google.session)
+    end
+
+    def self.find_by_title title
+      sheet = session_spreadsheets.detect do |spreadsheet|
+        spreadsheet.title.match(title)
+      end
+
+      sheet && new(sheet, session: Datapimp::Sync.google.session)
+    end
+
+    def self.session_spreadsheets
+      @session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
+    end
+
+    def self.create_from_data(data, options={})
+      require 'csv'
+
+      headers = Array(options[:headers]).map(&:to_s)
+
+      tmpfile = "tmp-csv.csv"
+
+      CSV.open(tmpfile, "wb") do |csv|
+        csv << headers
+
+        data.each do |row|
+          csv << headers.map do |header|
+            row = row.stringify_keys
+            row[header.to_s]
+          end
+        end
+      end
+
+      spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
+
+      new(spreadsheet.title, key: spreadsheet.key)
+    end
+
+
+    def title
+      @name ||= spreadsheet.try(:title)
+    end
+
+    def edit_url
+      spreadsheet.human_url
+    end
+
+    def share_write_access_with *emails
+      acl = spreadsheet.acl
+
+      Array(emails).flatten.each do |email|
+        acl.push scope_type: "user",
+                 with_key: false,
+                 role: "writer",
+                 scope: email
+      end
+    end
+
+    def share_read_access_with *emails
+      acl = spreadsheet.acl
+
+      Array(emails).flatten.each do |email|
+        acl.push scope_type: "user",
+                 with_key: false,
+                 role: "reader",
+                 scope: email
+      end
+    end
+
+    def add_to_collection collection_title
+      collection = if collection_title.is_a?(GoogleDrive::Collection)
+        collection_title
+      else
+        session.collections.find do |c|
+          c.title == collection_title
+        end
+      end
+
+      if !collection
+        collection_names = session.collections.map(&:title)
+        raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
+      end
+    end
+
+    def spreadsheet_key
+      key
+    end
+
+    def stale?
+      (!need_to_refresh? && (age > max_age)) || fresh_on_server?
+    end
+
+    def fresh_on_server?
+      refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
+    end
+
+    def last_updated_at
+      if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
+        DateTime.parse(value).to_i
+      else
+        Time.now.to_i
+      end
+    end
+
+    def fetch
+      self.raw = process_worksheets
+    end
+
+    def preprocess
+      single? ? raw.values.flatten : raw
+    end
+
+    protected
+
+      def process_worksheets
+        worksheets.inject({}.to_mash) do |memo, parts|
+          k, ws = parts
+          header_row = Array(ws.rows[0])
+          column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
+          rows = ws.rows.slice(1, ws.rows.length)
+
+          row_index = 1
+          memo[k] = rows.map do |row|
+            col_index = 0
+
+            _record = column_names.inject({}) do |record, field|
+              record[field] = "#{ row[col_index] }".strip
+              record["_id"] = row_index
+              col_index += 1
+              record
+            end
+
+            row_index += 1
+
+            _record
+          end
+
+          memo
+        end
+      end
+
+      def single?
+        worksheets.length == 1
+      end
+
+      def header_rows_for_worksheet key
+        if key.is_a?(Fixnum)
+          _worksheets[key]
+        else
+          worksheets.fetch(key)
+        end
+      end
+
+      def worksheets
+        @worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
+          key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
+          memo[key] = ws
+          memo
+        end
+      end
+
+      def _worksheets
+        @_worksheets ||= spreadsheet.worksheets
+      end
+
+      def spreadsheet
+        @spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
+      end
+  end
+end

data/lib/datapimp/sync.rb

@@ -4,7 +4,7 @@
 module Datapimp
   module Sync
     def self.data_source_types
-      %w(dropbox amazon github google json excel nokogiri)
+      %w(dropbox amazon github google pivotal json excel nokogiri)
     end
 
     def self.dispatch_sync_data_action(args, options)
@@ -13,7 +13,7 @@ module Datapimp
 
       result = case type
                when "github"
-                 Datapimp::Sources::GithubRepository.new(source, options)
+                 Datapimp::Sources::GithubRepository.new(args, options)
                when "google", "google-spreadsheet"
                  require 'google_drive'
                  Datapimp::Sources::GoogleSpreadsheet.new(nil, key: source)

data/lib/datapimp/version.rb

@@ -1,3 +1,5 @@
+require 'git-version-bump'
+
 module Datapimp
-  VERSION = "1.2.1"
+  VERSION = GVB.version
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: datapimp
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.2.2
 platform: ruby
 authors:
 - Jonathan Soeder
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-06-30 00:00:00.000000000 Z
+date: 2015-07-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pry
@@ -42,14 +42,14 @@ dependencies:
   name: commander
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '4.3'
 - !ruby/object:Gem::Dependency
@@ -98,14 +98,14 @@ dependencies:
   name: google_drive
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
@@ -291,19 +291,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
-  name: pivotal-tracker
+  name: tracker_api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.13
+        version: 0.2.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.5.13
+        version: 0.2.10
 - !ruby/object:Gem::Dependency
   name: keen
   requirement: !ruby/object:Gem::Requirement
@@ -360,6 +360,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.0.6
+- !ruby/object:Gem::Dependency
+  name: git-version-bump
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -434,6 +448,7 @@ description: Your rails app in a custom tailored suit.
 email:
 - jonathan.soeder@gmail.com
 executables:
+- console
 - datapimp
 extensions: []
 extra_rdoc_files: []
@@ -446,6 +461,7 @@ files:
 - README.md
 - ROADMAP.md
 - Rakefile
+- bin/console
 - bin/datapimp
 - datapimp.gemspec
 - lib/datapimp.rb
@@ -462,6 +478,7 @@ files:
 - lib/datapimp/clients/dropbox.rb
 - lib/datapimp/clients/github.rb
 - lib/datapimp/clients/google.rb
+- lib/datapimp/clients/keen.rb
 - lib/datapimp/configuration.rb
 - lib/datapimp/core_ext.rb
 - lib/datapimp/logging.rb
@@ -470,6 +487,7 @@ files:
 - lib/datapimp/sources/excel.rb
 - lib/datapimp/sources/github_repository.rb
 - lib/datapimp/sources/google.rb
+- lib/datapimp/sources/google_spreadsheet.rb
 - lib/datapimp/sources/json.rb
 - lib/datapimp/sources/keen.rb
 - lib/datapimp/sources/nokogiri.rb