datapimp 1.1.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/datapimp/cli/sync.rb +12 -26
- data/lib/datapimp/sources/github_repository.rb +67 -0
- data/lib/datapimp/sync.rb +14 -0
- data/lib/datapimp/version.rb +1 -1
- metadata +2 -3
- data/lib/datapimp/cli/support/data_sync.rb +0 -92
- data/lib/datapimp/sources/github.rb +0 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cc151c26a29a3f7a68150caf0f4258a69abf5fa9
|
4
|
+
data.tar.gz: b44748c88eb3506558cad01abedb499c67821a7e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 919a2a9fb5b86f6863d4ed13c22d2b038eda4af404f6284fc5970d9a48aa77e62842524e4180c6beffa6f6e1471229614dbdb0ebe137aff1927bd30b02f3a8f2
|
7
|
+
data.tar.gz: 8e2ae2c62e9aadc7839f8a54f929a0ea612fc6f4ac5b1f112748af0a00425969a54473911ae7efdbb5765c2a125b819257b9a22926df08fae2d3436d1d098ecc
|
data/lib/datapimp/cli/sync.rb
CHANGED
@@ -21,6 +21,7 @@ command "sync data" do |c|
|
|
21
21
|
|
22
22
|
c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
|
23
23
|
c.option '--output FILE', String, "Write the output to a file"
|
24
|
+
c.option '--view NAME', String, "Which view should we display?"
|
24
25
|
c.option '--format FORMAT', String, "Which format to serialize the output in? valid options are JSON"
|
25
26
|
c.option '--columns NAMES', Array, "Extract only these columns"
|
26
27
|
c.option '--relations NAMES', Array, "Also fetch these relationships on the object if applicable"
|
@@ -31,32 +32,17 @@ command "sync data" do |c|
|
|
31
32
|
Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox)
|
32
33
|
|
33
34
|
c.action do |args, options|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
repository = args.shift
|
46
|
-
|
47
|
-
service = Datapimp::DataSync::Github.new(repository, options)
|
48
|
-
service.sync_milestones
|
49
|
-
elsif options.type == "github-releases"
|
50
|
-
repository = args.shift
|
51
|
-
|
52
|
-
service = Datapimp::DataSync::Github.new(repository, options)
|
53
|
-
service.sync_releases
|
54
|
-
elsif options.type == "github-issue-comments"
|
55
|
-
repository = args.shift
|
56
|
-
issue = args.shift
|
57
|
-
|
58
|
-
service = Datapimp::DataSync::Github.new(repository, options)
|
59
|
-
service.sync_issue_comments(issue)
|
35
|
+
options.default(view:"to_s")
|
36
|
+
|
37
|
+
data = Datapimp::Sync.dispatch_sync_data_action(args.first, options.to_hash)
|
38
|
+
|
39
|
+
result = data.send(options.view)
|
40
|
+
result = JSON.generate(result) if options.format == "json" && options.type != "google-spreadsheet"
|
41
|
+
|
42
|
+
if options.output
|
43
|
+
Pathname(options.output).open("w+") {|fh| fh.write(result) }
|
44
|
+
else
|
45
|
+
puts result
|
60
46
|
end
|
61
47
|
end
|
62
48
|
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
module Datapimp::Sources
|
2
|
+
class GithubRepository < Datapimp::Sources::Base
|
3
|
+
attr_reader :options, :repository
|
4
|
+
|
5
|
+
def initialize(repository, options)
|
6
|
+
@repository = repository
|
7
|
+
@options = options.to_mash
|
8
|
+
end
|
9
|
+
|
10
|
+
def all
|
11
|
+
%w(issues milestones commits releases).reduce({}) do |memo, slice|
|
12
|
+
memo[slice] = send(slice)
|
13
|
+
memo
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def to_s
|
18
|
+
all
|
19
|
+
end
|
20
|
+
|
21
|
+
def issues
|
22
|
+
issues = client.issues(repository, filter: "all")
|
23
|
+
issues.map! do |issue|
|
24
|
+
%w(comments events labels).each do |rel|
|
25
|
+
issue[rel] = issue.rels[rel].get.data if relations.include?(rel)
|
26
|
+
end
|
27
|
+
issue
|
28
|
+
end
|
29
|
+
serve_output(issues)
|
30
|
+
end
|
31
|
+
|
32
|
+
def milestones
|
33
|
+
milestones = client.milestones(repository)
|
34
|
+
serve_output(milestones)
|
35
|
+
end
|
36
|
+
|
37
|
+
def releases
|
38
|
+
releases = client.releases(repository)
|
39
|
+
serve_output(releases)
|
40
|
+
end
|
41
|
+
|
42
|
+
def commits
|
43
|
+
commits = client.commits(repository)
|
44
|
+
serve_output(commits)
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def client
|
50
|
+
@_client ||= Datapimp::Sync.github.api
|
51
|
+
end
|
52
|
+
|
53
|
+
def relations
|
54
|
+
@_relations ||= Array(@options.relations)
|
55
|
+
end
|
56
|
+
|
57
|
+
def serve_output(output)
|
58
|
+
if output.is_a?(Array)
|
59
|
+
output.map! do |o|
|
60
|
+
o.respond_to?(:to_attrs) ? o.send(:to_attrs) : o
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
output
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
data/lib/datapimp/sync.rb
CHANGED
@@ -7,6 +7,20 @@ module Datapimp
|
|
7
7
|
%w(dropbox amazon github google json excel nokogiri)
|
8
8
|
end
|
9
9
|
|
10
|
+
def self.dispatch_sync_data_action(source, options)
|
11
|
+
type = options[:type]
|
12
|
+
|
13
|
+
result = case
|
14
|
+
when type == "github"
|
15
|
+
Datapimp::Sources::GithubRepository.new(source, options)
|
16
|
+
when type == "google" || type == "google-spreadsheet"
|
17
|
+
require 'google_drive'
|
18
|
+
Datapimp::Sources::GoogleSpreadsheet.new(nil, key: source)
|
19
|
+
end
|
20
|
+
|
21
|
+
result
|
22
|
+
end
|
23
|
+
|
10
24
|
# Create any type of syncable folder and dispatch a run call to it
|
11
25
|
# with whatever options you want.
|
12
26
|
#
|
data/lib/datapimp/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datapimp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan Soeder
|
@@ -400,7 +400,6 @@ files:
|
|
400
400
|
- lib/datapimp/cli/list.rb
|
401
401
|
- lib/datapimp/cli/run.rb
|
402
402
|
- lib/datapimp/cli/setup.rb
|
403
|
-
- lib/datapimp/cli/support/data_sync.rb
|
404
403
|
- lib/datapimp/cli/sync.rb
|
405
404
|
- lib/datapimp/cli/view.rb
|
406
405
|
- lib/datapimp/clients/amazon.rb
|
@@ -413,7 +412,7 @@ files:
|
|
413
412
|
- lib/datapimp/sources.rb
|
414
413
|
- lib/datapimp/sources/dropbox.rb
|
415
414
|
- lib/datapimp/sources/excel.rb
|
416
|
-
- lib/datapimp/sources/
|
415
|
+
- lib/datapimp/sources/github_repository.rb
|
417
416
|
- lib/datapimp/sources/google.rb
|
418
417
|
- lib/datapimp/sources/json.rb
|
419
418
|
- lib/datapimp/sources/nokogiri.rb
|
@@ -1,92 +0,0 @@
|
|
1
|
-
module Datapimp::DataSync
|
2
|
-
def self.sync_google_spreadsheet(options, *args)
|
3
|
-
require 'google_drive' unless defined?(::GoogleDrive)
|
4
|
-
|
5
|
-
raise 'Must setup google client' unless Datapimp::Sync.google.spreadsheets
|
6
|
-
|
7
|
-
key = args.shift
|
8
|
-
name = args.shift || "Spreadsheet"
|
9
|
-
|
10
|
-
raise 'Must supply a spreadsheet key' unless key
|
11
|
-
|
12
|
-
spreadsheet = Datapimp::Sources::GoogleSpreadsheet.new(name, key: key)
|
13
|
-
|
14
|
-
if options.output
|
15
|
-
Pathname(options.output).open("w+") do |fh|
|
16
|
-
fh.write(spreadsheet.to_s)
|
17
|
-
end
|
18
|
-
else
|
19
|
-
puts spreadsheet.to_s
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
class Github
|
24
|
-
attr_reader :options, :repository
|
25
|
-
|
26
|
-
def initialize(repository, options)
|
27
|
-
@repository = repository
|
28
|
-
@options = options
|
29
|
-
end
|
30
|
-
|
31
|
-
def sync_issues
|
32
|
-
issues = client.issues(repository, filter: "all")
|
33
|
-
issues.map! do |issue|
|
34
|
-
%w(comments events labels).each do |rel|
|
35
|
-
issue[rel] = issue.rels[rel].get.data if relations.include?(rel)
|
36
|
-
end
|
37
|
-
issue
|
38
|
-
end
|
39
|
-
serve_output(issues)
|
40
|
-
end
|
41
|
-
|
42
|
-
def sync_issue_comments(issue_id)
|
43
|
-
comments = client.issue_comments(repository, issue_id)
|
44
|
-
serve_output(comments)
|
45
|
-
end
|
46
|
-
|
47
|
-
def sync_milestones
|
48
|
-
milestones = client.milestones(repository)
|
49
|
-
serve_output(milestones)
|
50
|
-
end
|
51
|
-
|
52
|
-
def sync_releases
|
53
|
-
releases = client.releases(repository)
|
54
|
-
serve_output(releases)
|
55
|
-
end
|
56
|
-
|
57
|
-
def sync_commits
|
58
|
-
commits = client.commits(repository)
|
59
|
-
serve_output(commits)
|
60
|
-
end
|
61
|
-
|
62
|
-
private
|
63
|
-
|
64
|
-
def client
|
65
|
-
@_client ||= Datapimp::Sync.github.api
|
66
|
-
end
|
67
|
-
|
68
|
-
def relations
|
69
|
-
@_relations ||= @options.relations.to_a
|
70
|
-
end
|
71
|
-
|
72
|
-
def serve_output(output)
|
73
|
-
if output.is_a?(Array)
|
74
|
-
output.map! do |o|
|
75
|
-
o.respond_to?(:to_attrs) ? o.send(:to_attrs) : o
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
if @options.format && @options.format == "json"
|
80
|
-
output = JSON.generate(output)
|
81
|
-
end
|
82
|
-
|
83
|
-
if @options.output
|
84
|
-
Pathname(options.output).open("w+") do |f|
|
85
|
-
f.write(output)
|
86
|
-
end
|
87
|
-
else
|
88
|
-
puts output.to_s
|
89
|
-
end
|
90
|
-
end
|
91
|
-
end
|
92
|
-
end
|