datapimp 1.2.1 → 1.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/console +7 -0
- data/bin/datapimp +1 -0
- data/datapimp.gemspec +5 -3
- data/lib/datapimp/cli.rb +11 -0
- data/lib/datapimp/cli/config.rb +2 -2
- data/lib/datapimp/cli/create.rb +5 -3
- data/lib/datapimp/cli/list.rb +3 -3
- data/lib/datapimp/cli/run.rb +1 -1
- data/lib/datapimp/cli/setup.rb +13 -4
- data/lib/datapimp/cli/sync.rb +15 -13
- data/lib/datapimp/cli/view.rb +1 -1
- data/lib/datapimp/clients/keen.rb +48 -0
- data/lib/datapimp/configuration.rb +4 -2
- data/lib/datapimp/sources/google.rb +1 -212
- data/lib/datapimp/sources/google_spreadsheet.rb +212 -0
- data/lib/datapimp/sync.rb +2 -2
- data/lib/datapimp/version.rb +3 -1
- metadata +29 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 347b281ae0e4e748f903410bdf0218a1b24a00a5
|
4
|
+
data.tar.gz: f497dcd87a2a07bc2bbec8966e2c86cde589aaf6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c37075bc592d3c1e55246f6b83341f466cef1f08faaea0734b4ba04b6e84187aa33302721101b94ed6d6fa7e8edc703094922ba4789c3a48b68b0d9d90284cae
|
7
|
+
data.tar.gz: 7e8a151dc77e82ca4a01f5705fe7f20bf718bf332c1f91037a4d7a486ed42f2fbb3555a6e4dc15321ef1676c804870ad5afa5b2c852ecabc9122d3e697def470
|
data/bin/console
ADDED
data/bin/datapimp
CHANGED
data/datapimp.gemspec
CHANGED
@@ -20,11 +20,11 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_dependency 'pry', '~> 0.10'
|
22
22
|
spec.add_dependency 'hashie', '>= 3.0.4'
|
23
|
-
spec.add_dependency 'commander', '
|
23
|
+
spec.add_dependency 'commander', '>= 4.3'
|
24
24
|
spec.add_dependency 'terminal-table'
|
25
25
|
spec.add_dependency 'fog-aws', '>= 0.1'
|
26
26
|
spec.add_dependency 'dropbox-api', '>= 0.4.7'
|
27
|
-
spec.add_dependency 'google_drive', '
|
27
|
+
spec.add_dependency 'google_drive', '>= 1.0'
|
28
28
|
spec.add_dependency 'google-api-client', '>= 0.8'
|
29
29
|
spec.add_dependency 'rack-contrib', '~> 1.2'
|
30
30
|
spec.add_dependency 'uri_template', '~> 0.7'
|
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
|
|
38
38
|
spec.add_dependency 'github-fs', '~> 0'
|
39
39
|
spec.add_dependency 'colored', '> 0.0'
|
40
40
|
spec.add_dependency 'multi_json', '~> 1.10'
|
41
|
-
spec.add_dependency '
|
41
|
+
spec.add_dependency 'tracker_api', '>= 0.2.10'
|
42
42
|
spec.add_dependency 'keen', '~> 0.9.0'
|
43
43
|
|
44
44
|
# these are locked to specific versions so that
|
@@ -46,6 +46,8 @@ Gem::Specification.new do |spec|
|
|
46
46
|
spec.add_dependency 'nokogiri', '~> 1.6'
|
47
47
|
spec.add_dependency 'unf', '0.1.4'
|
48
48
|
spec.add_dependency 'unf_ext', '0.0.6'
|
49
|
+
spec.add_dependency 'git-version-bump'
|
50
|
+
|
49
51
|
|
50
52
|
spec.add_development_dependency "rake", '~> 0'
|
51
53
|
spec.add_development_dependency 'rspec', '~> 3.3.0'
|
data/lib/datapimp/cli.rb
CHANGED
@@ -3,6 +3,7 @@ require 'launchy'
|
|
3
3
|
module Datapimp
|
4
4
|
module Cli
|
5
5
|
def self.load_commands(from_path=nil)
|
6
|
+
$datapimp_cli ||= "datapimp"
|
6
7
|
from_path ||= Datapimp.lib.join("datapimp","cli")
|
7
8
|
Dir[from_path.join("**/*.rb")].each {|f| require(f) }
|
8
9
|
end
|
@@ -44,6 +45,16 @@ module Datapimp
|
|
44
45
|
c.option '--aws-access-key-id', String, 'AWS Access Key ID'
|
45
46
|
end
|
46
47
|
|
48
|
+
if services.include?(:keen)
|
49
|
+
c.option '--keen-project-id', String, 'Keen Project ID'
|
50
|
+
c.option '--keen-read-key', String, 'Keen Read Key'
|
51
|
+
c.option '--keen-write-key', String, 'Keen Write Key'
|
52
|
+
end
|
53
|
+
|
54
|
+
if services.include?(:pivotal)
|
55
|
+
c.option '--pivotal-access-token', String, 'Pivotal Tracker Access Token'
|
56
|
+
end
|
57
|
+
|
47
58
|
end
|
48
59
|
end
|
49
60
|
end
|
data/lib/datapimp/cli/config.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command 'config' do |c|
|
2
|
-
c.syntax = "
|
2
|
+
c.syntax = "#{$datapimp_cli} config [OPTIONS]"
|
3
3
|
c.description = "Shows the configuration options being used"
|
4
4
|
|
5
5
|
c.option '--env', "Output compatible with .env files"
|
@@ -16,7 +16,7 @@ command 'config' do |c|
|
|
16
16
|
end
|
17
17
|
|
18
18
|
command 'config set' do |c|
|
19
|
-
c.syntax =
|
19
|
+
c.syntax = "#{$datapimp_cli} config set KEY=VALUE KEY=VALUE [options]"
|
20
20
|
c.description = 'manipulate configuration settings'
|
21
21
|
|
22
22
|
c.option '--global', 'Set the configuration globally'
|
data/lib/datapimp/cli/create.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command 'create cache invalidations' do |c|
|
2
|
-
c.syntax =
|
2
|
+
c.syntax = "#{$datapimp_cli} create cache invalidations"
|
3
3
|
c.description = 'invalidate remote cache layers (i.e. cloudfront after a s3 deploy)'
|
4
4
|
|
5
5
|
Datapimp::Cli.accepts_keys_for(c, :amazon)
|
@@ -75,7 +75,7 @@ command 'create cache invalidations' do |c|
|
|
75
75
|
end
|
76
76
|
|
77
77
|
command 'create s3 bucket' do |c|
|
78
|
-
c.syntax =
|
78
|
+
c.syntax = "#{$datapimp_cli} create s3 bucket BUCKETNAME"
|
79
79
|
c.description = 'create an s3 bucket to use for website hosting'
|
80
80
|
|
81
81
|
Datapimp::Cli.accepts_keys_for(c, :amazon)
|
@@ -90,7 +90,7 @@ command 'create s3 bucket' do |c|
|
|
90
90
|
end
|
91
91
|
|
92
92
|
command 'create cloudfront distribution' do |c|
|
93
|
-
c.syntax = "
|
93
|
+
c.syntax = "#{$datapimp_cli} create cloudfront distribution"
|
94
94
|
c.description = "create a cloudfront distribution to link to a specific bucket"
|
95
95
|
|
96
96
|
Datapimp::Cli.accepts_keys_for(c, :amazon)
|
@@ -99,6 +99,8 @@ command 'create cloudfront distribution' do |c|
|
|
99
99
|
c.option '--domains DOMAINS', Array, 'What domains will be pointing to this bucket?'
|
100
100
|
|
101
101
|
c.action do |args, options|
|
102
|
+
options.default(bucket: args.first)
|
103
|
+
|
102
104
|
bucket = Datapimp::Sync::S3Bucket.new(remote: options.bucket)
|
103
105
|
|
104
106
|
cdn_options = {
|
data/lib/datapimp/cli/list.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command "list spreadsheets" do |c|
|
2
|
-
c.syntax = "
|
2
|
+
c.syntax = "#{$datapimp_cli} list spreadsheets"
|
3
3
|
c.description = "list the spreadsheets which can be used as datasources"
|
4
4
|
|
5
5
|
c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
|
@@ -21,14 +21,14 @@ command "list spreadsheets" do |c|
|
|
21
21
|
if lines.length > 0
|
22
22
|
puts "\n\nExample:"
|
23
23
|
puts "====="
|
24
|
-
puts "
|
24
|
+
puts "#{$datapimp_cli} sync data #{ lines.first.split(/\t/).first } --type google-spreadsheet"
|
25
25
|
puts "\n\n"
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
30
|
command "list folders" do |c|
|
31
|
-
c.syntax= "
|
31
|
+
c.syntax= "#{$datapimp_cli} list folders [OPTIONS]"
|
32
32
|
c.description= "lists folders in a remote service"
|
33
33
|
|
34
34
|
c.option '--type SERVICE', String, 'Which service to search: dropbox, google, amazon'
|
data/lib/datapimp/cli/run.rb
CHANGED
data/lib/datapimp/cli/setup.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command "setup google" do |c|
|
2
|
-
c.syntax = "
|
2
|
+
c.syntax = "#{$datapimp_cli} set up google"
|
3
3
|
c.description = "setup integration with google drive"
|
4
4
|
|
5
5
|
c.action do |args, options|
|
@@ -8,7 +8,7 @@ command "setup google" do |c|
|
|
8
8
|
end
|
9
9
|
|
10
10
|
command "setup amazon" do |c|
|
11
|
-
c.syntax = "
|
11
|
+
c.syntax = "#{$datapimp_cli} setup amazon"
|
12
12
|
c.description = "setup integration with amazon"
|
13
13
|
|
14
14
|
c.action do |args, options|
|
@@ -17,7 +17,7 @@ command "setup amazon" do |c|
|
|
17
17
|
end
|
18
18
|
|
19
19
|
command "setup dropbox" do |c|
|
20
|
-
c.syntax = "
|
20
|
+
c.syntax = "#{$datapimp_cli} set up dropbox"
|
21
21
|
c.description = "setup integration with dropbox"
|
22
22
|
|
23
23
|
c.action do |args, options|
|
@@ -26,10 +26,19 @@ command "setup dropbox" do |c|
|
|
26
26
|
end
|
27
27
|
|
28
28
|
command "setup github" do |c|
|
29
|
-
c.syntax = "
|
29
|
+
c.syntax = "#{$datapimp_cli} set up github"
|
30
30
|
c.description = "setup integration with github"
|
31
31
|
|
32
32
|
c.action do |args, options|
|
33
33
|
Datapimp::Sync.github.setup()
|
34
34
|
end
|
35
35
|
end
|
36
|
+
|
37
|
+
command "setup pivotal" do |c|
|
38
|
+
c.syntax = "#{$datapimp_cli} set up github"
|
39
|
+
c.description = "setup integration with github"
|
40
|
+
|
41
|
+
c.action do |args, options|
|
42
|
+
Datapimp::Sync.pivotal.setup()
|
43
|
+
end
|
44
|
+
end
|
data/lib/datapimp/cli/sync.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
command "sync folder" do |c|
|
2
2
|
c.description = "Synchronize the contents of a local folder with a file sharing service"
|
3
|
-
c.syntax = "
|
3
|
+
c.syntax = "#{$datapimp_cli} sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
|
4
4
|
|
5
5
|
c.option '--type TYPE', String, "Which service is hosting the folder"
|
6
6
|
c.option '--action ACTION', String, "Which sync action to run? push, pull"
|
@@ -18,7 +18,7 @@ end
|
|
18
18
|
|
19
19
|
command "sync data" do |c|
|
20
20
|
c.description = "Synchronize the contents of a local data store with its remote source"
|
21
|
-
c.syntax = "
|
21
|
+
c.syntax = "#{$datapimp_cli} sync data [OPTIONS]"
|
22
22
|
|
23
23
|
c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
|
24
24
|
c.option '--output FILE', String, "Write the output to a file"
|
@@ -30,17 +30,19 @@ command "sync data" do |c|
|
|
30
30
|
c.option '--limit LIMIT', Integer, "Limit the number of results for Pivotal resources"
|
31
31
|
c.option '--offset OFFSET', Integer, "Offset applied when using the limit option for Pivotal resources"
|
32
32
|
|
33
|
-
c.example "Syncing an excel file from dropbox ", "
|
34
|
-
c.example "Syncing a google spreadsheet", "
|
35
|
-
c.example "Syncing
|
36
|
-
|
37
|
-
c.example "Syncing
|
38
|
-
c.example "Syncing
|
39
|
-
|
40
|
-
c.example "Syncing
|
41
|
-
c.example "Syncing
|
42
|
-
|
43
|
-
|
33
|
+
c.example "Syncing an excel file from dropbox ", "#{$datapimp_cli} sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
|
34
|
+
c.example "Syncing a google spreadsheet", "#{$datapimp_cli} sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
|
35
|
+
c.example "Syncing keen.io data, extraction from an event_collection", "#{$datapimp_cli} sync data --type keen EVENT_COLLECTION"
|
36
|
+
|
37
|
+
c.example "Syncing Github Issues", "#{$datapimp_cli} sync data --type github --view issues REPOSITORY"
|
38
|
+
c.example "Syncing Github Issue Comments", "#{$datapimp_cli} sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
|
39
|
+
|
40
|
+
c.example "Syncing Pivotal Tracker data, user activity", "#{$datapimp_cli} sync data --type pivotal --view user-activity"
|
41
|
+
c.example "Syncing Pivotal Tracker data, project activity", "#{$datapimp_cli} sync data --type pivotal --view project-activity PROJECT_ID"
|
42
|
+
c.example "Syncing Pivotal Tracker data, project stories", "#{$datapimp_cli} sync data --type pivotal --view project-stories PROJECT_ID"
|
43
|
+
c.example "Syncing Pivotal Tracker data, project story notes", "#{$datapimp_cli} sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
|
44
|
+
|
45
|
+
Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox, :keen, :pivotal)
|
44
46
|
|
45
47
|
c.action do |args, options|
|
46
48
|
options.default(view:"to_s")
|
data/lib/datapimp/cli/view.rb
CHANGED
@@ -0,0 +1,48 @@
|
|
1
|
+
module Datapimp
|
2
|
+
module Clients
|
3
|
+
class Keen
|
4
|
+
include Singleton
|
5
|
+
|
6
|
+
def self.method_missing(meth, *args, &block)
|
7
|
+
if client.respond_to?(meth)
|
8
|
+
return client.send(meth, *args, &block)
|
9
|
+
end
|
10
|
+
|
11
|
+
super
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.client(options={})
|
15
|
+
Keen
|
16
|
+
end
|
17
|
+
|
18
|
+
def options
|
19
|
+
@options ||= {}
|
20
|
+
end
|
21
|
+
|
22
|
+
def with_options(opts={})
|
23
|
+
options.merge!(opts)
|
24
|
+
self
|
25
|
+
end
|
26
|
+
|
27
|
+
def setup(options={})
|
28
|
+
access_token = options[:keen_read_key] || Datapimp.config.keen_read_key
|
29
|
+
project_id = options[:keen_project_id] || Datapimp.config.keen_project_id
|
30
|
+
|
31
|
+
unless access_token.to_s.length > 1
|
32
|
+
if respond_to?(:ask)
|
33
|
+
access_token = ask("Enter a keen read key when you have one", String)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
unless project_id.to_s.length > 1
|
38
|
+
if respond_to?(:ask)
|
39
|
+
project_id = ask("Enter a keen read key when you have one", String)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
Datapimp.config.set(:keen_read_key, access_token) if access_token.to_s.length > 1
|
44
|
+
Datapimp.config.set(:keen_project_id, project_id) if project_id.to_s.length > 1
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -15,8 +15,10 @@ module Datapimp
|
|
15
15
|
github_access_token: '',
|
16
16
|
|
17
17
|
pivotal_access_token: '',
|
18
|
-
|
19
|
-
|
18
|
+
|
19
|
+
keen_project_id: '',
|
20
|
+
keen_read_key: '',
|
21
|
+
keen_write_key: '',
|
20
22
|
|
21
23
|
dnsimple_api_token: '',
|
22
24
|
dnsimple_username: '',
|
@@ -1,212 +1 @@
|
|
1
|
-
|
2
|
-
class GoogleSpreadsheet < Datapimp::Sources::Base
|
3
|
-
requires :key
|
4
|
-
|
5
|
-
attr_accessor :key,
|
6
|
-
:session,
|
7
|
-
:name
|
8
|
-
|
9
|
-
def initialize name, options={}
|
10
|
-
@options = options
|
11
|
-
|
12
|
-
if name.is_a?(GoogleDrive::Spreadsheet)
|
13
|
-
@spreadsheet = name
|
14
|
-
@name = @spreadsheet.title
|
15
|
-
@key = @spreadsheet.key
|
16
|
-
end
|
17
|
-
|
18
|
-
@key ||= options[:key]
|
19
|
-
@session ||= options.fetch(:session) { Datapimp::Sync.google.api }
|
20
|
-
|
21
|
-
ensure_valid_options!
|
22
|
-
end
|
23
|
-
|
24
|
-
def self.create_from_file(path, title)
|
25
|
-
if find_by_title(title)
|
26
|
-
raise 'Spreadsheet with this title already exists'
|
27
|
-
end
|
28
|
-
|
29
|
-
session.upload_from_file(path, title, :content_type => "text/csv")
|
30
|
-
|
31
|
-
find_by_title(title)
|
32
|
-
end
|
33
|
-
|
34
|
-
def self.[](key_or_title)
|
35
|
-
find_by_key(key_or_title) || find_by_title(key_or_title)
|
36
|
-
end
|
37
|
-
|
38
|
-
def self.find_by_key(key)
|
39
|
-
sheet = session_spreadsheets.detect do |spreadsheet|
|
40
|
-
spreadsheet.key == key
|
41
|
-
end
|
42
|
-
|
43
|
-
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
44
|
-
end
|
45
|
-
|
46
|
-
def self.find_by_title title
|
47
|
-
sheet = session_spreadsheets.detect do |spreadsheet|
|
48
|
-
spreadsheet.title.match(title)
|
49
|
-
end
|
50
|
-
|
51
|
-
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
52
|
-
end
|
53
|
-
|
54
|
-
def self.session_spreadsheets
|
55
|
-
@session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
|
56
|
-
end
|
57
|
-
|
58
|
-
def self.create_from_data(data, options={})
|
59
|
-
require 'csv'
|
60
|
-
|
61
|
-
headers = Array(options[:headers]).map(&:to_s)
|
62
|
-
|
63
|
-
tmpfile = "tmp-csv.csv"
|
64
|
-
|
65
|
-
CSV.open(tmpfile, "wb") do |csv|
|
66
|
-
csv << headers
|
67
|
-
|
68
|
-
data.each do |row|
|
69
|
-
csv << headers.map do |header|
|
70
|
-
row = row.stringify_keys
|
71
|
-
row[header.to_s]
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
|
77
|
-
|
78
|
-
new(spreadsheet.title, key: spreadsheet.key)
|
79
|
-
end
|
80
|
-
|
81
|
-
|
82
|
-
def title
|
83
|
-
@name ||= spreadsheet.try(:title)
|
84
|
-
end
|
85
|
-
|
86
|
-
def edit_url
|
87
|
-
spreadsheet.human_url
|
88
|
-
end
|
89
|
-
|
90
|
-
def share_write_access_with *emails
|
91
|
-
acl = spreadsheet.acl
|
92
|
-
|
93
|
-
Array(emails).flatten.each do |email|
|
94
|
-
acl.push scope_type: "user",
|
95
|
-
with_key: false,
|
96
|
-
role: "writer",
|
97
|
-
scope: email
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
def share_read_access_with *emails
|
102
|
-
acl = spreadsheet.acl
|
103
|
-
|
104
|
-
Array(emails).flatten.each do |email|
|
105
|
-
acl.push scope_type: "user",
|
106
|
-
with_key: false,
|
107
|
-
role: "reader",
|
108
|
-
scope: email
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
def add_to_collection collection_title
|
113
|
-
collection = if collection_title.is_a?(GoogleDrive::Collection)
|
114
|
-
collection_title
|
115
|
-
else
|
116
|
-
session.collections.find do |c|
|
117
|
-
c.title == collection_title
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
if !collection
|
122
|
-
collection_names = session.collections.map(&:title)
|
123
|
-
raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
|
124
|
-
end
|
125
|
-
end
|
126
|
-
|
127
|
-
def spreadsheet_key
|
128
|
-
key
|
129
|
-
end
|
130
|
-
|
131
|
-
def stale?
|
132
|
-
(!need_to_refresh? && (age > max_age)) || fresh_on_server?
|
133
|
-
end
|
134
|
-
|
135
|
-
def fresh_on_server?
|
136
|
-
refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
|
137
|
-
end
|
138
|
-
|
139
|
-
def last_updated_at
|
140
|
-
if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
|
141
|
-
DateTime.parse(value).to_i
|
142
|
-
else
|
143
|
-
Time.now.to_i
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
def fetch
|
148
|
-
self.raw = process_worksheets
|
149
|
-
end
|
150
|
-
|
151
|
-
def preprocess
|
152
|
-
single? ? raw.values.flatten : raw
|
153
|
-
end
|
154
|
-
|
155
|
-
protected
|
156
|
-
|
157
|
-
def process_worksheets
|
158
|
-
worksheets.inject({}.to_mash) do |memo, parts|
|
159
|
-
k, ws = parts
|
160
|
-
header_row = Array(ws.rows[0])
|
161
|
-
column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
|
162
|
-
rows = ws.rows.slice(1, ws.rows.length)
|
163
|
-
|
164
|
-
row_index = 1
|
165
|
-
memo[k] = rows.map do |row|
|
166
|
-
col_index = 0
|
167
|
-
|
168
|
-
_record = column_names.inject({}) do |record, field|
|
169
|
-
record[field] = "#{ row[col_index] }".strip
|
170
|
-
record["_id"] = row_index
|
171
|
-
col_index += 1
|
172
|
-
record
|
173
|
-
end
|
174
|
-
|
175
|
-
row_index += 1
|
176
|
-
|
177
|
-
_record
|
178
|
-
end
|
179
|
-
|
180
|
-
memo
|
181
|
-
end
|
182
|
-
end
|
183
|
-
|
184
|
-
def single?
|
185
|
-
worksheets.length == 1
|
186
|
-
end
|
187
|
-
|
188
|
-
def header_rows_for_worksheet key
|
189
|
-
if key.is_a?(Fixnum)
|
190
|
-
_worksheets[key]
|
191
|
-
else
|
192
|
-
worksheets.fetch(key)
|
193
|
-
end
|
194
|
-
end
|
195
|
-
|
196
|
-
def worksheets
|
197
|
-
@worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
|
198
|
-
key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
|
199
|
-
memo[key] = ws
|
200
|
-
memo
|
201
|
-
end
|
202
|
-
end
|
203
|
-
|
204
|
-
def _worksheets
|
205
|
-
@_worksheets ||= spreadsheet.worksheets
|
206
|
-
end
|
207
|
-
|
208
|
-
def spreadsheet
|
209
|
-
@spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
|
210
|
-
end
|
211
|
-
end
|
212
|
-
end
|
1
|
+
require 'datapimp/sources/google_spreadsheet'
|
@@ -0,0 +1,212 @@
|
|
1
|
+
module Datapimp::Sources
|
2
|
+
class GoogleSpreadsheet < Datapimp::Sources::Base
|
3
|
+
requires :key
|
4
|
+
|
5
|
+
attr_accessor :key,
|
6
|
+
:session,
|
7
|
+
:name
|
8
|
+
|
9
|
+
def initialize name, options={}
|
10
|
+
@options = options
|
11
|
+
|
12
|
+
if name.is_a?(GoogleDrive::Spreadsheet)
|
13
|
+
@spreadsheet = name
|
14
|
+
@name = @spreadsheet.title
|
15
|
+
@key = @spreadsheet.key
|
16
|
+
end
|
17
|
+
|
18
|
+
@key ||= options[:key]
|
19
|
+
@session ||= options.fetch(:session) { Datapimp::Sync.google.api }
|
20
|
+
|
21
|
+
ensure_valid_options!
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.create_from_file(path, title)
|
25
|
+
if find_by_title(title)
|
26
|
+
raise 'Spreadsheet with this title already exists'
|
27
|
+
end
|
28
|
+
|
29
|
+
session.upload_from_file(path, title, :content_type => "text/csv")
|
30
|
+
|
31
|
+
find_by_title(title)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.[](key_or_title)
|
35
|
+
find_by_key(key_or_title) || find_by_title(key_or_title)
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.find_by_key(key)
|
39
|
+
sheet = session_spreadsheets.detect do |spreadsheet|
|
40
|
+
spreadsheet.key == key
|
41
|
+
end
|
42
|
+
|
43
|
+
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.find_by_title title
|
47
|
+
sheet = session_spreadsheets.detect do |spreadsheet|
|
48
|
+
spreadsheet.title.match(title)
|
49
|
+
end
|
50
|
+
|
51
|
+
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.session_spreadsheets
|
55
|
+
@session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.create_from_data(data, options={})
|
59
|
+
require 'csv'
|
60
|
+
|
61
|
+
headers = Array(options[:headers]).map(&:to_s)
|
62
|
+
|
63
|
+
tmpfile = "tmp-csv.csv"
|
64
|
+
|
65
|
+
CSV.open(tmpfile, "wb") do |csv|
|
66
|
+
csv << headers
|
67
|
+
|
68
|
+
data.each do |row|
|
69
|
+
csv << headers.map do |header|
|
70
|
+
row = row.stringify_keys
|
71
|
+
row[header.to_s]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
|
77
|
+
|
78
|
+
new(spreadsheet.title, key: spreadsheet.key)
|
79
|
+
end
|
80
|
+
|
81
|
+
|
82
|
+
def title
|
83
|
+
@name ||= spreadsheet.try(:title)
|
84
|
+
end
|
85
|
+
|
86
|
+
def edit_url
|
87
|
+
spreadsheet.human_url
|
88
|
+
end
|
89
|
+
|
90
|
+
def share_write_access_with *emails
|
91
|
+
acl = spreadsheet.acl
|
92
|
+
|
93
|
+
Array(emails).flatten.each do |email|
|
94
|
+
acl.push scope_type: "user",
|
95
|
+
with_key: false,
|
96
|
+
role: "writer",
|
97
|
+
scope: email
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def share_read_access_with *emails
|
102
|
+
acl = spreadsheet.acl
|
103
|
+
|
104
|
+
Array(emails).flatten.each do |email|
|
105
|
+
acl.push scope_type: "user",
|
106
|
+
with_key: false,
|
107
|
+
role: "reader",
|
108
|
+
scope: email
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def add_to_collection collection_title
|
113
|
+
collection = if collection_title.is_a?(GoogleDrive::Collection)
|
114
|
+
collection_title
|
115
|
+
else
|
116
|
+
session.collections.find do |c|
|
117
|
+
c.title == collection_title
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
if !collection
|
122
|
+
collection_names = session.collections.map(&:title)
|
123
|
+
raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def spreadsheet_key
|
128
|
+
key
|
129
|
+
end
|
130
|
+
|
131
|
+
def stale?
|
132
|
+
(!need_to_refresh? && (age > max_age)) || fresh_on_server?
|
133
|
+
end
|
134
|
+
|
135
|
+
def fresh_on_server?
|
136
|
+
refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
|
137
|
+
end
|
138
|
+
|
139
|
+
def last_updated_at
|
140
|
+
if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
|
141
|
+
DateTime.parse(value).to_i
|
142
|
+
else
|
143
|
+
Time.now.to_i
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def fetch
|
148
|
+
self.raw = process_worksheets
|
149
|
+
end
|
150
|
+
|
151
|
+
def preprocess
|
152
|
+
single? ? raw.values.flatten : raw
|
153
|
+
end
|
154
|
+
|
155
|
+
protected
|
156
|
+
|
157
|
+
def process_worksheets
|
158
|
+
worksheets.inject({}.to_mash) do |memo, parts|
|
159
|
+
k, ws = parts
|
160
|
+
header_row = Array(ws.rows[0])
|
161
|
+
column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
|
162
|
+
rows = ws.rows.slice(1, ws.rows.length)
|
163
|
+
|
164
|
+
row_index = 1
|
165
|
+
memo[k] = rows.map do |row|
|
166
|
+
col_index = 0
|
167
|
+
|
168
|
+
_record = column_names.inject({}) do |record, field|
|
169
|
+
record[field] = "#{ row[col_index] }".strip
|
170
|
+
record["_id"] = row_index
|
171
|
+
col_index += 1
|
172
|
+
record
|
173
|
+
end
|
174
|
+
|
175
|
+
row_index += 1
|
176
|
+
|
177
|
+
_record
|
178
|
+
end
|
179
|
+
|
180
|
+
memo
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def single?
|
185
|
+
worksheets.length == 1
|
186
|
+
end
|
187
|
+
|
188
|
+
def header_rows_for_worksheet key
|
189
|
+
if key.is_a?(Fixnum)
|
190
|
+
_worksheets[key]
|
191
|
+
else
|
192
|
+
worksheets.fetch(key)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def worksheets
|
197
|
+
@worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
|
198
|
+
key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
|
199
|
+
memo[key] = ws
|
200
|
+
memo
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def _worksheets
|
205
|
+
@_worksheets ||= spreadsheet.worksheets
|
206
|
+
end
|
207
|
+
|
208
|
+
def spreadsheet
|
209
|
+
@spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
data/lib/datapimp/sync.rb
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
module Datapimp
|
5
5
|
module Sync
|
6
6
|
def self.data_source_types
|
7
|
-
%w(dropbox amazon github google json excel nokogiri)
|
7
|
+
%w(dropbox amazon github google pivotal json excel nokogiri)
|
8
8
|
end
|
9
9
|
|
10
10
|
def self.dispatch_sync_data_action(args, options)
|
@@ -13,7 +13,7 @@ module Datapimp
|
|
13
13
|
|
14
14
|
result = case type
|
15
15
|
when "github"
|
16
|
-
Datapimp::Sources::GithubRepository.new(
|
16
|
+
Datapimp::Sources::GithubRepository.new(args, options)
|
17
17
|
when "google", "google-spreadsheet"
|
18
18
|
require 'google_drive'
|
19
19
|
Datapimp::Sources::GoogleSpreadsheet.new(nil, key: source)
|
data/lib/datapimp/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datapimp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan Soeder
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|
@@ -42,14 +42,14 @@ dependencies:
|
|
42
42
|
name: commander
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '4.3'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '4.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
@@ -98,14 +98,14 @@ dependencies:
|
|
98
98
|
name: google_drive
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - "
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '1.0'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - "
|
108
|
+
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '1.0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
@@ -291,19 +291,19 @@ dependencies:
|
|
291
291
|
- !ruby/object:Gem::Version
|
292
292
|
version: '1.10'
|
293
293
|
- !ruby/object:Gem::Dependency
|
294
|
-
name:
|
294
|
+
name: tracker_api
|
295
295
|
requirement: !ruby/object:Gem::Requirement
|
296
296
|
requirements:
|
297
|
-
- - "
|
297
|
+
- - ">="
|
298
298
|
- !ruby/object:Gem::Version
|
299
|
-
version: 0.
|
299
|
+
version: 0.2.10
|
300
300
|
type: :runtime
|
301
301
|
prerelease: false
|
302
302
|
version_requirements: !ruby/object:Gem::Requirement
|
303
303
|
requirements:
|
304
|
-
- - "
|
304
|
+
- - ">="
|
305
305
|
- !ruby/object:Gem::Version
|
306
|
-
version: 0.
|
306
|
+
version: 0.2.10
|
307
307
|
- !ruby/object:Gem::Dependency
|
308
308
|
name: keen
|
309
309
|
requirement: !ruby/object:Gem::Requirement
|
@@ -360,6 +360,20 @@ dependencies:
|
|
360
360
|
- - '='
|
361
361
|
- !ruby/object:Gem::Version
|
362
362
|
version: 0.0.6
|
363
|
+
- !ruby/object:Gem::Dependency
|
364
|
+
name: git-version-bump
|
365
|
+
requirement: !ruby/object:Gem::Requirement
|
366
|
+
requirements:
|
367
|
+
- - ">="
|
368
|
+
- !ruby/object:Gem::Version
|
369
|
+
version: '0'
|
370
|
+
type: :runtime
|
371
|
+
prerelease: false
|
372
|
+
version_requirements: !ruby/object:Gem::Requirement
|
373
|
+
requirements:
|
374
|
+
- - ">="
|
375
|
+
- !ruby/object:Gem::Version
|
376
|
+
version: '0'
|
363
377
|
- !ruby/object:Gem::Dependency
|
364
378
|
name: rake
|
365
379
|
requirement: !ruby/object:Gem::Requirement
|
@@ -434,6 +448,7 @@ description: Your rails app in a custom tailored suit.
|
|
434
448
|
email:
|
435
449
|
- jonathan.soeder@gmail.com
|
436
450
|
executables:
|
451
|
+
- console
|
437
452
|
- datapimp
|
438
453
|
extensions: []
|
439
454
|
extra_rdoc_files: []
|
@@ -446,6 +461,7 @@ files:
|
|
446
461
|
- README.md
|
447
462
|
- ROADMAP.md
|
448
463
|
- Rakefile
|
464
|
+
- bin/console
|
449
465
|
- bin/datapimp
|
450
466
|
- datapimp.gemspec
|
451
467
|
- lib/datapimp.rb
|
@@ -462,6 +478,7 @@ files:
|
|
462
478
|
- lib/datapimp/clients/dropbox.rb
|
463
479
|
- lib/datapimp/clients/github.rb
|
464
480
|
- lib/datapimp/clients/google.rb
|
481
|
+
- lib/datapimp/clients/keen.rb
|
465
482
|
- lib/datapimp/configuration.rb
|
466
483
|
- lib/datapimp/core_ext.rb
|
467
484
|
- lib/datapimp/logging.rb
|
@@ -470,6 +487,7 @@ files:
|
|
470
487
|
- lib/datapimp/sources/excel.rb
|
471
488
|
- lib/datapimp/sources/github_repository.rb
|
472
489
|
- lib/datapimp/sources/google.rb
|
490
|
+
- lib/datapimp/sources/google_spreadsheet.rb
|
473
491
|
- lib/datapimp/sources/json.rb
|
474
492
|
- lib/datapimp/sources/keen.rb
|
475
493
|
- lib/datapimp/sources/nokogiri.rb
|