datapimp 1.1.1 → 1.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +6 -18
- data/datapimp.gemspec +8 -4
- data/lib/datapimp/cli/sync.rb +14 -3
- data/lib/datapimp/configuration.rb +4 -0
- data/lib/datapimp/sources/keen.rb +29 -0
- data/lib/datapimp/sources/pivotal.rb +60 -0
- data/lib/datapimp/sources.rb +25 -0
- data/lib/datapimp/sync/s3_bucket.rb +1 -1
- data/lib/datapimp/sync.rb +10 -6
- data/lib/datapimp/version.rb +1 -1
- data/spec/datapimp/sync/github_spec.rb +19 -0
- data/spec/datapimp/sync/keen_spec.rb +17 -0
- data/spec/datapimp/sync/pivotal_spec.rb +22 -0
- data/spec/fixtures/cassettes/github_issues.yml +592 -0
- data/spec/fixtures/cassettes/keen_extraction.yml +59 -0
- data/spec/fixtures/cassettes/pivotal_user_activity.yml +646 -0
- data/spec/spec_helper.rb +10 -7
- metadata +78 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3f321f490c14ab95399fc552046f520a1d338a0b
|
4
|
+
data.tar.gz: 0b9fb145a1a21baa0073f28df92c868fb2fa1fe5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eb2289f7c16a39284b1f46f83336033364e73b18d4c9b776fc5112a36287e547f5bc64fe313629e40d706f67045e965fbeb3498351a49b31a275f4049b1671c8
|
7
|
+
data.tar.gz: b5720aaf745175c305b79c288c84dd6b711bd7ce82da825d5c8935e2dd95881c3123c6a8e68f036280172ee51b4caa2ec0d58b83a61b7ea18ac80e2b08435de4
|
data/Rakefile
CHANGED
@@ -1,20 +1,8 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
require 'bundler'
|
2
|
+
Bundler::GemHelper.install_tasks
|
3
3
|
|
4
|
-
require
|
4
|
+
require 'rspec/core/rake_task'
|
5
|
+
RSpec::Core::RakeTask.new(:spec)
|
5
6
|
|
6
|
-
|
7
|
-
|
8
|
-
# config.version = Datapimp::VERSION
|
9
|
-
# config.rb_version = '20150210-2.1.5'
|
10
|
-
# config.packaging_dir = File.expand_path 'packaging'
|
11
|
-
# config.native_extensions = [
|
12
|
-
# 'escape_utils-1.0.1',
|
13
|
-
# 'nokogiri-1.6.5',
|
14
|
-
# 'unf_ext-1.0.6'
|
15
|
-
# ]
|
16
|
-
#end
|
17
|
-
|
18
|
-
task :default do
|
19
|
-
puts "Sup?"
|
20
|
-
end
|
7
|
+
task :test => :spec
|
8
|
+
task :default => :spec
|
data/datapimp.gemspec
CHANGED
@@ -17,7 +17,7 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.files = `git ls-files`.split("\n")
|
18
18
|
spec.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
19
|
spec.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
-
|
20
|
+
|
21
21
|
spec.add_dependency 'pry', '~> 0.10'
|
22
22
|
spec.add_dependency 'hashie', '>= 3.0.4'
|
23
23
|
spec.add_dependency 'commander', '~> 4.3'
|
@@ -38,16 +38,20 @@ Gem::Specification.new do |spec|
|
|
38
38
|
spec.add_dependency 'github-fs', '~> 0'
|
39
39
|
spec.add_dependency 'colored', '> 0.0'
|
40
40
|
spec.add_dependency 'multi_json', '~> 1.10'
|
41
|
+
spec.add_dependency 'pivotal-tracker', '~> 0.5.13'
|
42
|
+
spec.add_dependency 'keen', '~> 0.9.0'
|
41
43
|
|
42
44
|
# these are locked to specific versions so that
|
43
45
|
# we can use the native extensions for traveling ruby
|
44
46
|
spec.add_dependency 'nokogiri', '~> 1.6'
|
45
47
|
spec.add_dependency 'unf', '0.1.4'
|
46
48
|
spec.add_dependency 'unf_ext', '0.0.6'
|
47
|
-
|
49
|
+
|
48
50
|
spec.add_development_dependency "rake", '~> 0'
|
49
|
-
spec.add_development_dependency
|
50
|
-
spec.add_development_dependency '
|
51
|
+
spec.add_development_dependency 'rspec', '~> 3.3.0'
|
52
|
+
spec.add_development_dependency 'webmock', '~> 1.21.0'
|
53
|
+
spec.add_development_dependency 'vcr', '~> 2.9.3'
|
54
|
+
spec.add_development_dependency 'byebug', '~> 5.0.0'
|
51
55
|
|
52
56
|
spec.require_paths = ["lib"]
|
53
57
|
|
data/lib/datapimp/cli/sync.rb
CHANGED
@@ -5,13 +5,14 @@ command "sync folder" do |c|
|
|
5
5
|
c.option '--type TYPE', String, "Which service is hosting the folder"
|
6
6
|
c.option '--action ACTION', String, "Which sync action to run? push, pull"
|
7
7
|
c.option '--reset', nil, "Reset the local path (if supported by the syncable folder)"
|
8
|
+
c.option '--acl', String, "Which acl to use? private, public-read, authenticated-read"
|
8
9
|
|
9
10
|
Datapimp::Cli.accepts_keys_for(c, :amazon, :google, :github, :dropbox)
|
10
11
|
|
11
12
|
c.action do |args, options|
|
12
|
-
options.default(action:"pull", type: "dropbox", reset: false)
|
13
|
+
options.default(action:"pull", type: "dropbox", acl: "public-read", reset: false)
|
13
14
|
local, remote = args
|
14
|
-
Datapimp::Sync.dispatch_sync_folder_action(local, remote, options.to_hash)
|
15
|
+
Datapimp::Sync.dispatch_sync_folder_action(local, remote, options.to_hash.to_mash)
|
15
16
|
end
|
16
17
|
end
|
17
18
|
|
@@ -26,15 +27,25 @@ command "sync data" do |c|
|
|
26
27
|
c.option '--columns NAMES', Array, "Extract only these columns"
|
27
28
|
c.option '--relations NAMES', Array, "Also fetch these relationships on the object if applicable"
|
28
29
|
|
30
|
+
c.option '--limit LIMIT', Integer, "Limit the number of results for Pivotal resources"
|
31
|
+
c.option '--offset OFFSET', Integer, "Offset applied when using the limit option for Pivotal resources"
|
32
|
+
|
29
33
|
c.example "Syncing an excel file from dropbox ", "datapimp sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
|
30
34
|
c.example "Syncing a google spreadsheet", "datapimp sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
|
35
|
+
c.example "Syncing Pivotal Tracker data, user activity", "datapimp sync data --type pivotal --view user-activity"
|
36
|
+
c.example "Syncing Pivotal Tracker data, project activity", "datapimp sync data --type pivotal --view project-activity PROJECT_ID"
|
37
|
+
c.example "Syncing Pivotal Tracker data, project stories", "datapimp sync data --type pivotal --view project-stories PROJECT_ID"
|
38
|
+
c.example "Syncing Pivotal Tracker data, project story notes", "datapimp sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
|
39
|
+
c.example "Syncing keen.io data, extraction from an event_collection", "datapimp sync data --type keen EVENT_COLLECTION"
|
40
|
+
c.example "Syncing Github Issues", "datapimp sync data --type github --view issues REPOSITORY"
|
41
|
+
c.example "Syncing Github Issue Comments", "datapimp sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
|
31
42
|
|
32
43
|
Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox)
|
33
44
|
|
34
45
|
c.action do |args, options|
|
35
46
|
options.default(view:"to_s")
|
36
47
|
|
37
|
-
data = Datapimp::Sync.dispatch_sync_data_action(args
|
48
|
+
data = Datapimp::Sync.dispatch_sync_data_action(args, options.to_hash)
|
38
49
|
|
39
50
|
result = data.send(options.view)
|
40
51
|
result = JSON.generate(result) if options.format == "json" && options.type != "google-spreadsheet"
|
@@ -14,6 +14,10 @@ module Datapimp
|
|
14
14
|
github_app_secret: '',
|
15
15
|
github_access_token: '',
|
16
16
|
|
17
|
+
pivotal_access_token: '',
|
18
|
+
keen_project_id: ENV.fetch('KEEN_PROJECT_ID', ''),
|
19
|
+
keen_read_key: ENV.fetch('KEEN_READ_KEY', ''),
|
20
|
+
|
17
21
|
dnsimple_api_token: '',
|
18
22
|
dnsimple_username: '',
|
19
23
|
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'keen'
|
2
|
+
|
3
|
+
module Datapimp::Sources
|
4
|
+
class Keen < Datapimp::Sources::Base
|
5
|
+
attr_reader :options
|
6
|
+
|
7
|
+
def initialize(args, options)
|
8
|
+
@collection = args.first
|
9
|
+
@options = options.to_mash
|
10
|
+
end
|
11
|
+
|
12
|
+
def to_s
|
13
|
+
extraction(@collection)
|
14
|
+
end
|
15
|
+
|
16
|
+
def extraction(event_collection)
|
17
|
+
jsonify(client.extraction(event_collection))
|
18
|
+
end
|
19
|
+
|
20
|
+
private
|
21
|
+
|
22
|
+
def client
|
23
|
+
@_client ||= ::Keen::Client.new(
|
24
|
+
project_id: Datapimp.config.keen_project_id,
|
25
|
+
read_key: Datapimp.config.keen_read_key
|
26
|
+
)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,60 @@
|
|
1
|
+
require 'pivotal-tracker'
|
2
|
+
|
3
|
+
module Datapimp::Sources
|
4
|
+
class Pivotal < Datapimp::Sources::Base
|
5
|
+
def initialize(args, options)
|
6
|
+
@project_id = args.shift
|
7
|
+
@story_id = args.shift
|
8
|
+
@options = options.to_mash
|
9
|
+
|
10
|
+
PivotalTracker::Client.token = Datapimp.config.pivotal_access_token
|
11
|
+
end
|
12
|
+
|
13
|
+
def all
|
14
|
+
%w(user_activity project_activity project_stories).each_with_object({}) do |slice, memo|
|
15
|
+
memo[slice] = send(slice)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
def to_s
|
20
|
+
all
|
21
|
+
end
|
22
|
+
|
23
|
+
def user_activity
|
24
|
+
@_user_activity ||= PivotalTracker::Activity.all(nil, limit_params).map {|a| jsonify(a) }
|
25
|
+
end
|
26
|
+
|
27
|
+
def project_activity
|
28
|
+
project.activities.all(limit_params).map {|a| jsonify(a) }
|
29
|
+
end
|
30
|
+
|
31
|
+
def project_stories
|
32
|
+
stories = project.stories.all(limit_params)
|
33
|
+
|
34
|
+
# add notes for each story and convert the objects to hashes
|
35
|
+
stories.map do |story|
|
36
|
+
story_hash = jsonify(story)
|
37
|
+
story_hash[:notes] = story.notes.all(limit_params).map {|a| jsonify(a) }
|
38
|
+
story_hash
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def project_story_notes
|
43
|
+
notes = project.stories.find(@story_id).notes.all(limit_params)
|
44
|
+
notes.map {|a| jsonify(a) }
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def project
|
50
|
+
@_project ||= PivotalTracker::Project.find(@project_id)
|
51
|
+
end
|
52
|
+
|
53
|
+
def limit_params
|
54
|
+
Hash.new.tap do |h|
|
55
|
+
h[:limit] = @options.limit if @options.limit
|
56
|
+
h[:offset] = @options.offset if @options.offset
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
data/lib/datapimp/sources.rb
CHANGED
@@ -191,6 +191,31 @@ module Datapimp
|
|
191
191
|
def path_to_file
|
192
192
|
Pathname(path).join("#{ file }")
|
193
193
|
end
|
194
|
+
|
195
|
+
def jsonify(value)
|
196
|
+
case value
|
197
|
+
when String, Numeric, NilClass, TrueClass, FalseClass
|
198
|
+
value
|
199
|
+
when Hash
|
200
|
+
Hash[value.map { |k, v| [jsonify(k), jsonify(v)] }]
|
201
|
+
when Array
|
202
|
+
value.map { |v| jsonify(v) }
|
203
|
+
when HappyMapper
|
204
|
+
value.instance_variables.each_with_object({}) do |var_name, memo|
|
205
|
+
key = var_name.to_s.sub(/^@/, '').to_sym
|
206
|
+
val = value.instance_variable_get(var_name)
|
207
|
+
memo[key] = jsonify(val)
|
208
|
+
end
|
209
|
+
else
|
210
|
+
if value.respond_to?(:to_attrs)
|
211
|
+
value.to_attrs
|
212
|
+
elsif value.respond_to?(:as_json)
|
213
|
+
value.as_json
|
214
|
+
else
|
215
|
+
value.to_s
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
194
219
|
end
|
195
220
|
end
|
196
221
|
end
|
@@ -98,7 +98,7 @@ module Datapimp
|
|
98
98
|
log "Skipping #{ destination }: similar etag"
|
99
99
|
else
|
100
100
|
existing.body = entry.read
|
101
|
-
existing.acl = 'public-read'
|
101
|
+
existing.acl = options.acl || 'public-read'
|
102
102
|
existing.content_type = content_type
|
103
103
|
log "Updated #{ destination }; content-type: #{ content_type }"
|
104
104
|
uploaded << destination
|
data/lib/datapimp/sync.rb
CHANGED
@@ -7,17 +7,21 @@ module Datapimp
|
|
7
7
|
%w(dropbox amazon github google json excel nokogiri)
|
8
8
|
end
|
9
9
|
|
10
|
-
def self.dispatch_sync_data_action(
|
11
|
-
|
10
|
+
def self.dispatch_sync_data_action(args, options)
|
11
|
+
source = args.first
|
12
|
+
type = options[:type]
|
12
13
|
|
13
|
-
result = case
|
14
|
-
when
|
14
|
+
result = case type
|
15
|
+
when "github"
|
15
16
|
Datapimp::Sources::GithubRepository.new(source, options)
|
16
|
-
when
|
17
|
+
when "google" || "google-spreadsheet"
|
17
18
|
require 'google_drive'
|
18
19
|
Datapimp::Sources::GoogleSpreadsheet.new(nil, key: source)
|
20
|
+
when "pivotal" then
|
21
|
+
Datapimp::Sources::Pivotal.new(args, options)
|
22
|
+
when "keen" then
|
23
|
+
Datapimp::Sources::Keen.new(args, options)
|
19
24
|
end
|
20
|
-
|
21
25
|
result
|
22
26
|
end
|
23
27
|
|
data/lib/datapimp/version.rb
CHANGED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'datapimp/sources/github_repository'
|
3
|
+
|
4
|
+
describe Datapimp::Sources::GithubRepository do
|
5
|
+
let(:options) { double('options', limit: 5, offset: nil, format: nil, output: nil, relations: ["comments"]).as_null_object }
|
6
|
+
let(:repository) { "architects/githubfs-test" }
|
7
|
+
|
8
|
+
describe "issues" do
|
9
|
+
it "should return an array of issues" do
|
10
|
+
service = described_class.new(repository, options)
|
11
|
+
|
12
|
+
VCR.use_cassette(:github_issues) do
|
13
|
+
output = service.to_s
|
14
|
+
expect(output).to be_kind_of(Hash)
|
15
|
+
expect(output).to have_key('issues')
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'datapimp/sources/keen'
|
3
|
+
|
4
|
+
describe Datapimp::Sources::Keen do
|
5
|
+
let(:options) { double('options', limit: 5, offset: nil, format: nil, output: nil).as_null_object }
|
6
|
+
|
7
|
+
describe "Extraction" do
|
8
|
+
it "should return an array with all property values" do
|
9
|
+
service = described_class.new(['purchases'], options)
|
10
|
+
|
11
|
+
VCR.use_cassette(:keen_extraction) do
|
12
|
+
output = service.to_s
|
13
|
+
expect(output).to be_kind_of(Array)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'datapimp/sources/pivotal'
|
3
|
+
|
4
|
+
describe Datapimp::Sources::Pivotal do
|
5
|
+
let(:options) { double('options', limit: 5, offset: nil, format: nil, output: nil).as_null_object }
|
6
|
+
let(:project) { '442903' }
|
7
|
+
|
8
|
+
describe "user activity" do
|
9
|
+
it "should return an array of activities" do
|
10
|
+
service = described_class.new([project], options)
|
11
|
+
|
12
|
+
VCR.use_cassette(:pivotal_user_activity) do
|
13
|
+
output = service.to_s
|
14
|
+
|
15
|
+
expect(output).to be_kind_of(Hash)
|
16
|
+
%w(user_activity project_activity project_stories).each do |key|
|
17
|
+
expect(output).to have_key(key)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|