datapimp 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cc151c26a29a3f7a68150caf0f4258a69abf5fa9
-  data.tar.gz: b44748c88eb3506558cad01abedb499c67821a7e
+  metadata.gz: 3f321f490c14ab95399fc552046f520a1d338a0b
+  data.tar.gz: 0b9fb145a1a21baa0073f28df92c868fb2fa1fe5
 SHA512:
-  metadata.gz: 919a2a9fb5b86f6863d4ed13c22d2b038eda4af404f6284fc5970d9a48aa77e62842524e4180c6beffa6f6e1471229614dbdb0ebe137aff1927bd30b02f3a8f2
-  data.tar.gz: 8e2ae2c62e9aadc7839f8a54f929a0ea612fc6f4ac5b1f112748af0a00425969a54473911ae7efdbb5765c2a125b819257b9a22926df08fae2d3436d1d098ecc
+  metadata.gz: eb2289f7c16a39284b1f46f83336033364e73b18d4c9b776fc5112a36287e547f5bc64fe313629e40d706f67045e965fbeb3498351a49b31a275f4049b1671c8
+  data.tar.gz: b5720aaf745175c305b79c288c84dd6b711bd7ce82da825d5c8935e2dd95881c3123c6a8e68f036280172ee51b4caa2ec0d58b83a61b7ea18ac80e2b08435de4

data/Rakefile

@@ -1,20 +1,8 @@
-Dir[File.join(Dir.pwd, 'tasks', '**', '*.rb')].each { |f| require f }
-Dir[File.join(Dir.pwd, 'tasks', '*.rake')].each { |f| load f }
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
-require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
 
-#Distribution.configure do |config|
-#  config.package_name = 'datapimp'
-#  config.version = Datapimp::VERSION
-#  config.rb_version = '20150210-2.1.5'
-#  config.packaging_dir = File.expand_path 'packaging'
-#  config.native_extensions = [
-#    'escape_utils-1.0.1',
-#    'nokogiri-1.6.5',
-#    'unf_ext-1.0.6'
-#  ]
-#end
-
-task :default do
-  puts "Sup?"
-end
+task :test => :spec
+task :default => :spec

data/datapimp.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
   spec.files         = `git ls-files`.split("\n")
   spec.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   spec.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  
+
   spec.add_dependency 'pry', '~> 0.10'
   spec.add_dependency 'hashie', '>= 3.0.4' 
   spec.add_dependency 'commander', '~> 4.3'
@@ -38,16 +38,20 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'github-fs', '~> 0'
   spec.add_dependency 'colored', '> 0.0'
   spec.add_dependency 'multi_json', '~> 1.10'
+  spec.add_dependency 'pivotal-tracker', '~> 0.5.13'
+  spec.add_dependency 'keen', '~> 0.9.0'
 
   # these are locked to specific versions so that 
   # we can use the native extensions for traveling ruby
   spec.add_dependency 'nokogiri', '~> 1.6'
   spec.add_dependency 'unf', '0.1.4'
   spec.add_dependency 'unf_ext', '0.0.6'
-  
+
   spec.add_development_dependency "rake", '~> 0'
-  spec.add_development_dependency "rack-test", '~> 0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
+  spec.add_development_dependency 'rspec', '~> 3.3.0'
+  spec.add_development_dependency 'webmock', '~> 1.21.0'
+  spec.add_development_dependency 'vcr', '~> 2.9.3'
+  spec.add_development_dependency 'byebug', '~> 5.0.0'
 
   spec.require_paths = ["lib"]
 

data/lib/datapimp/cli/sync.rb

@@ -5,13 +5,14 @@ command "sync folder" do |c|
   c.option '--type TYPE', String, "Which service is hosting the folder"
   c.option '--action ACTION', String, "Which sync action to run? push, pull"
   c.option '--reset', nil, "Reset the local path (if supported by the syncable folder)"
+  c.option '--acl', String, "Which acl to use? private, public-read, authenticated-read"
 
   Datapimp::Cli.accepts_keys_for(c, :amazon, :google, :github, :dropbox)
 
   c.action do |args, options|
-    options.default(action:"pull", type: "dropbox", reset: false)
+    options.default(action:"pull", type: "dropbox", acl: "public-read", reset: false)
     local, remote = args
-    Datapimp::Sync.dispatch_sync_folder_action(local, remote, options.to_hash)
+    Datapimp::Sync.dispatch_sync_folder_action(local, remote, options.to_hash.to_mash)
   end
 end
 
@@ -26,15 +27,25 @@ command "sync data" do |c|
   c.option '--columns NAMES', Array, "Extract only these columns"
   c.option '--relations NAMES', Array, "Also fetch these relationships on the object if applicable"
 
+  c.option '--limit LIMIT', Integer, "Limit the number of results for Pivotal resources"
+  c.option '--offset OFFSET', Integer, "Offset applied when using the limit option for Pivotal resources"
+
   c.example "Syncing an excel file from dropbox ", "datapimp sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
   c.example "Syncing a google spreadsheet", "datapimp sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
+  c.example "Syncing Pivotal Tracker data, user activity", "datapimp sync data --type pivotal --view user-activity"
+  c.example "Syncing Pivotal Tracker data, project activity", "datapimp sync data --type pivotal --view project-activity PROJECT_ID"
+  c.example "Syncing Pivotal Tracker data, project stories", "datapimp sync data --type pivotal --view project-stories PROJECT_ID"
+  c.example "Syncing Pivotal Tracker data, project story notes", "datapimp sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
+  c.example "Syncing keen.io data, extraction from an event_collection", "datapimp sync data --type keen EVENT_COLLECTION"
+  c.example "Syncing Github Issues", "datapimp sync data --type github --view issues REPOSITORY"
+  c.example "Syncing Github Issue Comments", "datapimp sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
 
   Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox)
 
   c.action do |args, options|
     options.default(view:"to_s")
 
-    data = Datapimp::Sync.dispatch_sync_data_action(args.first, options.to_hash)
+    data = Datapimp::Sync.dispatch_sync_data_action(args, options.to_hash)
 
     result = data.send(options.view)
     result = JSON.generate(result) if options.format == "json" && options.type != "google-spreadsheet"

data/lib/datapimp/configuration.rb

@@ -14,6 +14,10 @@ module Datapimp
       github_app_secret: '',
       github_access_token: '',
 
+      pivotal_access_token: '',
+      keen_project_id: ENV.fetch('KEEN_PROJECT_ID', ''),
+      keen_read_key: ENV.fetch('KEEN_READ_KEY', ''),
+
       dnsimple_api_token: '',
       dnsimple_username: '',
 

data/lib/datapimp/sources/keen.rb

@@ -0,0 +1,29 @@
+require 'keen'
+
+module Datapimp::Sources
+  class Keen < Datapimp::Sources::Base
+    attr_reader :options
+
+    def initialize(args, options)
+      @collection = args.first
+      @options    = options.to_mash
+    end
+
+    def to_s
+      extraction(@collection)
+    end
+
+    def extraction(event_collection)
+      jsonify(client.extraction(event_collection))
+    end
+
+    private
+
+    def client
+      @_client ||= ::Keen::Client.new(
+        project_id: Datapimp.config.keen_project_id,
+        read_key:   Datapimp.config.keen_read_key
+      )
+    end
+  end
+end

data/lib/datapimp/sources/pivotal.rb

@@ -0,0 +1,60 @@
+require 'pivotal-tracker'
+
+module Datapimp::Sources
+  class Pivotal < Datapimp::Sources::Base
+    def initialize(args, options)
+      @project_id = args.shift
+      @story_id   = args.shift
+      @options    = options.to_mash
+
+      PivotalTracker::Client.token = Datapimp.config.pivotal_access_token
+    end
+
+    def all
+      %w(user_activity project_activity project_stories).each_with_object({}) do |slice, memo|
+        memo[slice] = send(slice)
+      end
+    end
+
+    def to_s
+      all
+    end
+
+    def user_activity
+      @_user_activity ||= PivotalTracker::Activity.all(nil, limit_params).map {|a| jsonify(a) }
+    end
+
+    def project_activity
+      project.activities.all(limit_params).map {|a| jsonify(a) }
+    end
+
+    def project_stories
+      stories = project.stories.all(limit_params)
+
+      # add notes for each story and convert the objects to hashes
+      stories.map do |story|
+        story_hash = jsonify(story)
+        story_hash[:notes] = story.notes.all(limit_params).map {|a| jsonify(a) }
+        story_hash
+      end
+    end
+
+    def project_story_notes
+      notes = project.stories.find(@story_id).notes.all(limit_params)
+      notes.map {|a| jsonify(a) }
+    end
+
+    private
+
+    def project
+      @_project ||= PivotalTracker::Project.find(@project_id)
+    end
+
+    def limit_params
+      Hash.new.tap do |h|
+        h[:limit]   = @options.limit  if @options.limit
+        h[:offset]  = @options.offset if @options.offset
+      end
+    end
+  end
+end

data/lib/datapimp/sources.rb

@@ -191,6 +191,31 @@ module Datapimp
       def path_to_file
         Pathname(path).join("#{ file }")
       end
+
+      def jsonify(value)
+        case value
+        when String, Numeric, NilClass, TrueClass, FalseClass
+          value
+        when Hash
+          Hash[value.map { |k, v| [jsonify(k), jsonify(v)] }]
+        when Array
+          value.map { |v| jsonify(v) }
+        when HappyMapper
+          value.instance_variables.each_with_object({}) do |var_name, memo|
+            key       = var_name.to_s.sub(/^@/, '').to_sym
+            val       = value.instance_variable_get(var_name)
+            memo[key] = jsonify(val)
+          end
+        else
+          if value.respond_to?(:to_attrs)
+            value.to_attrs
+          elsif value.respond_to?(:as_json)
+            value.as_json
+          else
+            value.to_s
+          end
+        end
+      end
     end
   end
 end

data/lib/datapimp/sync/s3_bucket.rb

@@ -98,7 +98,7 @@ module Datapimp
               log "Skipping #{ destination }: similar etag"
             else
               existing.body = entry.read
-              existing.acl = 'public-read'
+              existing.acl = options.acl || 'public-read'
               existing.content_type = content_type
               log "Updated #{ destination }; content-type: #{ content_type }"
               uploaded << destination

data/lib/datapimp/sync.rb

@@ -7,17 +7,21 @@ module Datapimp
       %w(dropbox amazon github google json excel nokogiri)
     end
 
-    def self.dispatch_sync_data_action(source, options)
-      type = options[:type]
+    def self.dispatch_sync_data_action(args, options)
+      source  = args.first
+      type    = options[:type]
 
-      result = case
-               when type == "github"
+      result = case type
+               when "github"
                  Datapimp::Sources::GithubRepository.new(source, options)
-               when type == "google" || type == "google-spreadsheet"
+               when "google" || "google-spreadsheet"
                  require 'google_drive'
                  Datapimp::Sources::GoogleSpreadsheet.new(nil, key: source)
+               when "pivotal" then
+                 Datapimp::Sources::Pivotal.new(args, options)
+               when "keen" then
+                 Datapimp::Sources::Keen.new(args, options)
                end
-
       result
     end
 

data/lib/datapimp/version.rb

@@ -1,3 +1,3 @@
 module Datapimp
-  VERSION = "1.1.1"
+  VERSION = "1.2.0"
 end

data/spec/datapimp/sync/github_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+require 'datapimp/sources/github_repository'
+
+describe Datapimp::Sources::GithubRepository do
+  let(:options)     { double('options', limit: 5, offset: nil, format: nil, output: nil, relations: ["comments"]).as_null_object }
+  let(:repository)  { "architects/githubfs-test" }
+
+  describe "issues" do
+    it "should return an array of issues" do
+      service = described_class.new(repository, options)
+
+      VCR.use_cassette(:github_issues) do
+        output = service.to_s
+        expect(output).to be_kind_of(Hash)
+        expect(output).to have_key('issues')
+      end
+    end
+  end
+end

data/spec/datapimp/sync/keen_spec.rb

@@ -0,0 +1,17 @@
+require 'spec_helper'
+require 'datapimp/sources/keen'
+
+describe Datapimp::Sources::Keen do
+  let(:options) { double('options', limit: 5, offset: nil, format: nil, output: nil).as_null_object }
+
+  describe "Extraction" do
+    it "should return an array with all property values" do
+      service = described_class.new(['purchases'], options)
+
+      VCR.use_cassette(:keen_extraction) do
+        output = service.to_s
+        expect(output).to be_kind_of(Array)
+      end
+    end
+  end
+end

data/spec/datapimp/sync/pivotal_spec.rb

@@ -0,0 +1,22 @@
+require 'spec_helper'
+require 'datapimp/sources/pivotal'
+
+describe Datapimp::Sources::Pivotal do
+  let(:options) { double('options', limit: 5, offset: nil, format: nil, output: nil).as_null_object }
+  let(:project) { '442903' }
+
+  describe "user activity" do
+    it "should return an array of activities" do
+      service = described_class.new([project], options)
+
+      VCR.use_cassette(:pivotal_user_activity) do
+        output = service.to_s
+
+        expect(output).to be_kind_of(Hash)
+        %w(user_activity project_activity project_stories).each do |key|
+          expect(output).to have_key(key)
+        end
+      end
+    end
+  end
+end