datapimp 1.2.1 → 1.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/console +7 -0
- data/bin/datapimp +1 -0
- data/datapimp.gemspec +5 -3
- data/lib/datapimp/cli.rb +11 -0
- data/lib/datapimp/cli/config.rb +2 -2
- data/lib/datapimp/cli/create.rb +5 -3
- data/lib/datapimp/cli/list.rb +3 -3
- data/lib/datapimp/cli/run.rb +1 -1
- data/lib/datapimp/cli/setup.rb +13 -4
- data/lib/datapimp/cli/sync.rb +15 -13
- data/lib/datapimp/cli/view.rb +1 -1
- data/lib/datapimp/clients/keen.rb +48 -0
- data/lib/datapimp/configuration.rb +4 -2
- data/lib/datapimp/sources/google.rb +1 -212
- data/lib/datapimp/sources/google_spreadsheet.rb +212 -0
- data/lib/datapimp/sync.rb +2 -2
- data/lib/datapimp/version.rb +3 -1
- metadata +29 -11
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 347b281ae0e4e748f903410bdf0218a1b24a00a5
|
4
|
+
data.tar.gz: f497dcd87a2a07bc2bbec8966e2c86cde589aaf6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c37075bc592d3c1e55246f6b83341f466cef1f08faaea0734b4ba04b6e84187aa33302721101b94ed6d6fa7e8edc703094922ba4789c3a48b68b0d9d90284cae
|
7
|
+
data.tar.gz: 7e8a151dc77e82ca4a01f5705fe7f20bf718bf332c1f91037a4d7a486ed42f2fbb3555a6e4dc15321ef1676c804870ad5afa5b2c852ecabc9122d3e697def470
|
data/bin/console
ADDED
data/bin/datapimp
CHANGED
data/datapimp.gemspec
CHANGED
@@ -20,11 +20,11 @@ Gem::Specification.new do |spec|
|
|
20
20
|
|
21
21
|
spec.add_dependency 'pry', '~> 0.10'
|
22
22
|
spec.add_dependency 'hashie', '>= 3.0.4'
|
23
|
-
spec.add_dependency 'commander', '
|
23
|
+
spec.add_dependency 'commander', '>= 4.3'
|
24
24
|
spec.add_dependency 'terminal-table'
|
25
25
|
spec.add_dependency 'fog-aws', '>= 0.1'
|
26
26
|
spec.add_dependency 'dropbox-api', '>= 0.4.7'
|
27
|
-
spec.add_dependency 'google_drive', '
|
27
|
+
spec.add_dependency 'google_drive', '>= 1.0'
|
28
28
|
spec.add_dependency 'google-api-client', '>= 0.8'
|
29
29
|
spec.add_dependency 'rack-contrib', '~> 1.2'
|
30
30
|
spec.add_dependency 'uri_template', '~> 0.7'
|
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
|
|
38
38
|
spec.add_dependency 'github-fs', '~> 0'
|
39
39
|
spec.add_dependency 'colored', '> 0.0'
|
40
40
|
spec.add_dependency 'multi_json', '~> 1.10'
|
41
|
-
spec.add_dependency '
|
41
|
+
spec.add_dependency 'tracker_api', '>= 0.2.10'
|
42
42
|
spec.add_dependency 'keen', '~> 0.9.0'
|
43
43
|
|
44
44
|
# these are locked to specific versions so that
|
@@ -46,6 +46,8 @@ Gem::Specification.new do |spec|
|
|
46
46
|
spec.add_dependency 'nokogiri', '~> 1.6'
|
47
47
|
spec.add_dependency 'unf', '0.1.4'
|
48
48
|
spec.add_dependency 'unf_ext', '0.0.6'
|
49
|
+
spec.add_dependency 'git-version-bump'
|
50
|
+
|
49
51
|
|
50
52
|
spec.add_development_dependency "rake", '~> 0'
|
51
53
|
spec.add_development_dependency 'rspec', '~> 3.3.0'
|
data/lib/datapimp/cli.rb
CHANGED
@@ -3,6 +3,7 @@ require 'launchy'
|
|
3
3
|
module Datapimp
|
4
4
|
module Cli
|
5
5
|
def self.load_commands(from_path=nil)
|
6
|
+
$datapimp_cli ||= "datapimp"
|
6
7
|
from_path ||= Datapimp.lib.join("datapimp","cli")
|
7
8
|
Dir[from_path.join("**/*.rb")].each {|f| require(f) }
|
8
9
|
end
|
@@ -44,6 +45,16 @@ module Datapimp
|
|
44
45
|
c.option '--aws-access-key-id', String, 'AWS Access Key ID'
|
45
46
|
end
|
46
47
|
|
48
|
+
if services.include?(:keen)
|
49
|
+
c.option '--keen-project-id', String, 'Keen Project ID'
|
50
|
+
c.option '--keen-read-key', String, 'Keen Read Key'
|
51
|
+
c.option '--keen-write-key', String, 'Keen Write Key'
|
52
|
+
end
|
53
|
+
|
54
|
+
if services.include?(:pivotal)
|
55
|
+
c.option '--pivotal-access-token', String, 'Pivotal Tracker Access Token'
|
56
|
+
end
|
57
|
+
|
47
58
|
end
|
48
59
|
end
|
49
60
|
end
|
data/lib/datapimp/cli/config.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command 'config' do |c|
|
2
|
-
c.syntax = "
|
2
|
+
c.syntax = "#{$datapimp_cli} config [OPTIONS]"
|
3
3
|
c.description = "Shows the configuration options being used"
|
4
4
|
|
5
5
|
c.option '--env', "Output compatible with .env files"
|
@@ -16,7 +16,7 @@ command 'config' do |c|
|
|
16
16
|
end
|
17
17
|
|
18
18
|
command 'config set' do |c|
|
19
|
-
c.syntax =
|
19
|
+
c.syntax = "#{$datapimp_cli} config set KEY=VALUE KEY=VALUE [options]"
|
20
20
|
c.description = 'manipulate configuration settings'
|
21
21
|
|
22
22
|
c.option '--global', 'Set the configuration globally'
|
data/lib/datapimp/cli/create.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command 'create cache invalidations' do |c|
|
2
|
-
c.syntax =
|
2
|
+
c.syntax = "#{$datapimp_cli} create cache invalidations"
|
3
3
|
c.description = 'invalidate remote cache layers (i.e. cloudfront after a s3 deploy)'
|
4
4
|
|
5
5
|
Datapimp::Cli.accepts_keys_for(c, :amazon)
|
@@ -75,7 +75,7 @@ command 'create cache invalidations' do |c|
|
|
75
75
|
end
|
76
76
|
|
77
77
|
command 'create s3 bucket' do |c|
|
78
|
-
c.syntax =
|
78
|
+
c.syntax = "#{$datapimp_cli} create s3 bucket BUCKETNAME"
|
79
79
|
c.description = 'create an s3 bucket to use for website hosting'
|
80
80
|
|
81
81
|
Datapimp::Cli.accepts_keys_for(c, :amazon)
|
@@ -90,7 +90,7 @@ command 'create s3 bucket' do |c|
|
|
90
90
|
end
|
91
91
|
|
92
92
|
command 'create cloudfront distribution' do |c|
|
93
|
-
c.syntax = "
|
93
|
+
c.syntax = "#{$datapimp_cli} create cloudfront distribution"
|
94
94
|
c.description = "create a cloudfront distribution to link to a specific bucket"
|
95
95
|
|
96
96
|
Datapimp::Cli.accepts_keys_for(c, :amazon)
|
@@ -99,6 +99,8 @@ command 'create cloudfront distribution' do |c|
|
|
99
99
|
c.option '--domains DOMAINS', Array, 'What domains will be pointing to this bucket?'
|
100
100
|
|
101
101
|
c.action do |args, options|
|
102
|
+
options.default(bucket: args.first)
|
103
|
+
|
102
104
|
bucket = Datapimp::Sync::S3Bucket.new(remote: options.bucket)
|
103
105
|
|
104
106
|
cdn_options = {
|
data/lib/datapimp/cli/list.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command "list spreadsheets" do |c|
|
2
|
-
c.syntax = "
|
2
|
+
c.syntax = "#{$datapimp_cli} list spreadsheets"
|
3
3
|
c.description = "list the spreadsheets which can be used as datasources"
|
4
4
|
|
5
5
|
c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
|
@@ -21,14 +21,14 @@ command "list spreadsheets" do |c|
|
|
21
21
|
if lines.length > 0
|
22
22
|
puts "\n\nExample:"
|
23
23
|
puts "====="
|
24
|
-
puts "
|
24
|
+
puts "#{$datapimp_cli} sync data #{ lines.first.split(/\t/).first } --type google-spreadsheet"
|
25
25
|
puts "\n\n"
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
30
30
|
command "list folders" do |c|
|
31
|
-
c.syntax= "
|
31
|
+
c.syntax= "#{$datapimp_cli} list folders [OPTIONS]"
|
32
32
|
c.description= "lists folders in a remote service"
|
33
33
|
|
34
34
|
c.option '--type SERVICE', String, 'Which service to search: dropbox, google, amazon'
|
data/lib/datapimp/cli/run.rb
CHANGED
data/lib/datapimp/cli/setup.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
command "setup google" do |c|
|
2
|
-
c.syntax = "
|
2
|
+
c.syntax = "#{$datapimp_cli} set up google"
|
3
3
|
c.description = "setup integration with google drive"
|
4
4
|
|
5
5
|
c.action do |args, options|
|
@@ -8,7 +8,7 @@ command "setup google" do |c|
|
|
8
8
|
end
|
9
9
|
|
10
10
|
command "setup amazon" do |c|
|
11
|
-
c.syntax = "
|
11
|
+
c.syntax = "#{$datapimp_cli} setup amazon"
|
12
12
|
c.description = "setup integration with amazon"
|
13
13
|
|
14
14
|
c.action do |args, options|
|
@@ -17,7 +17,7 @@ command "setup amazon" do |c|
|
|
17
17
|
end
|
18
18
|
|
19
19
|
command "setup dropbox" do |c|
|
20
|
-
c.syntax = "
|
20
|
+
c.syntax = "#{$datapimp_cli} set up dropbox"
|
21
21
|
c.description = "setup integration with dropbox"
|
22
22
|
|
23
23
|
c.action do |args, options|
|
@@ -26,10 +26,19 @@ command "setup dropbox" do |c|
|
|
26
26
|
end
|
27
27
|
|
28
28
|
command "setup github" do |c|
|
29
|
-
c.syntax = "
|
29
|
+
c.syntax = "#{$datapimp_cli} set up github"
|
30
30
|
c.description = "setup integration with github"
|
31
31
|
|
32
32
|
c.action do |args, options|
|
33
33
|
Datapimp::Sync.github.setup()
|
34
34
|
end
|
35
35
|
end
|
36
|
+
|
37
|
+
command "setup pivotal" do |c|
|
38
|
+
c.syntax = "#{$datapimp_cli} set up github"
|
39
|
+
c.description = "setup integration with github"
|
40
|
+
|
41
|
+
c.action do |args, options|
|
42
|
+
Datapimp::Sync.pivotal.setup()
|
43
|
+
end
|
44
|
+
end
|
data/lib/datapimp/cli/sync.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
command "sync folder" do |c|
|
2
2
|
c.description = "Synchronize the contents of a local folder with a file sharing service"
|
3
|
-
c.syntax = "
|
3
|
+
c.syntax = "#{$datapimp_cli} sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
|
4
4
|
|
5
5
|
c.option '--type TYPE', String, "Which service is hosting the folder"
|
6
6
|
c.option '--action ACTION', String, "Which sync action to run? push, pull"
|
@@ -18,7 +18,7 @@ end
|
|
18
18
|
|
19
19
|
command "sync data" do |c|
|
20
20
|
c.description = "Synchronize the contents of a local data store with its remote source"
|
21
|
-
c.syntax = "
|
21
|
+
c.syntax = "#{$datapimp_cli} sync data [OPTIONS]"
|
22
22
|
|
23
23
|
c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
|
24
24
|
c.option '--output FILE', String, "Write the output to a file"
|
@@ -30,17 +30,19 @@ command "sync data" do |c|
|
|
30
30
|
c.option '--limit LIMIT', Integer, "Limit the number of results for Pivotal resources"
|
31
31
|
c.option '--offset OFFSET', Integer, "Offset applied when using the limit option for Pivotal resources"
|
32
32
|
|
33
|
-
c.example "Syncing an excel file from dropbox ", "
|
34
|
-
c.example "Syncing a google spreadsheet", "
|
35
|
-
c.example "Syncing
|
36
|
-
|
37
|
-
c.example "Syncing
|
38
|
-
c.example "Syncing
|
39
|
-
|
40
|
-
c.example "Syncing
|
41
|
-
c.example "Syncing
|
42
|
-
|
43
|
-
|
33
|
+
c.example "Syncing an excel file from dropbox ", "#{$datapimp_cli} sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
|
34
|
+
c.example "Syncing a google spreadsheet", "#{$datapimp_cli} sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
|
35
|
+
c.example "Syncing keen.io data, extraction from an event_collection", "#{$datapimp_cli} sync data --type keen EVENT_COLLECTION"
|
36
|
+
|
37
|
+
c.example "Syncing Github Issues", "#{$datapimp_cli} sync data --type github --view issues REPOSITORY"
|
38
|
+
c.example "Syncing Github Issue Comments", "#{$datapimp_cli} sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
|
39
|
+
|
40
|
+
c.example "Syncing Pivotal Tracker data, user activity", "#{$datapimp_cli} sync data --type pivotal --view user-activity"
|
41
|
+
c.example "Syncing Pivotal Tracker data, project activity", "#{$datapimp_cli} sync data --type pivotal --view project-activity PROJECT_ID"
|
42
|
+
c.example "Syncing Pivotal Tracker data, project stories", "#{$datapimp_cli} sync data --type pivotal --view project-stories PROJECT_ID"
|
43
|
+
c.example "Syncing Pivotal Tracker data, project story notes", "#{$datapimp_cli} sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
|
44
|
+
|
45
|
+
Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox, :keen, :pivotal)
|
44
46
|
|
45
47
|
c.action do |args, options|
|
46
48
|
options.default(view:"to_s")
|
data/lib/datapimp/cli/view.rb
CHANGED
@@ -0,0 +1,48 @@
|
|
1
|
+
module Datapimp
|
2
|
+
module Clients
|
3
|
+
class Keen
|
4
|
+
include Singleton
|
5
|
+
|
6
|
+
def self.method_missing(meth, *args, &block)
|
7
|
+
if client.respond_to?(meth)
|
8
|
+
return client.send(meth, *args, &block)
|
9
|
+
end
|
10
|
+
|
11
|
+
super
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.client(options={})
|
15
|
+
Keen
|
16
|
+
end
|
17
|
+
|
18
|
+
def options
|
19
|
+
@options ||= {}
|
20
|
+
end
|
21
|
+
|
22
|
+
def with_options(opts={})
|
23
|
+
options.merge!(opts)
|
24
|
+
self
|
25
|
+
end
|
26
|
+
|
27
|
+
def setup(options={})
|
28
|
+
access_token = options[:keen_read_key] || Datapimp.config.keen_read_key
|
29
|
+
project_id = options[:keen_project_id] || Datapimp.config.keen_project_id
|
30
|
+
|
31
|
+
unless access_token.to_s.length > 1
|
32
|
+
if respond_to?(:ask)
|
33
|
+
access_token = ask("Enter a keen read key when you have one", String)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
unless project_id.to_s.length > 1
|
38
|
+
if respond_to?(:ask)
|
39
|
+
project_id = ask("Enter a keen read key when you have one", String)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
Datapimp.config.set(:keen_read_key, access_token) if access_token.to_s.length > 1
|
44
|
+
Datapimp.config.set(:keen_project_id, project_id) if project_id.to_s.length > 1
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
@@ -15,8 +15,10 @@ module Datapimp
|
|
15
15
|
github_access_token: '',
|
16
16
|
|
17
17
|
pivotal_access_token: '',
|
18
|
-
|
19
|
-
|
18
|
+
|
19
|
+
keen_project_id: '',
|
20
|
+
keen_read_key: '',
|
21
|
+
keen_write_key: '',
|
20
22
|
|
21
23
|
dnsimple_api_token: '',
|
22
24
|
dnsimple_username: '',
|
@@ -1,212 +1 @@
|
|
1
|
-
|
2
|
-
class GoogleSpreadsheet < Datapimp::Sources::Base
|
3
|
-
requires :key
|
4
|
-
|
5
|
-
attr_accessor :key,
|
6
|
-
:session,
|
7
|
-
:name
|
8
|
-
|
9
|
-
def initialize name, options={}
|
10
|
-
@options = options
|
11
|
-
|
12
|
-
if name.is_a?(GoogleDrive::Spreadsheet)
|
13
|
-
@spreadsheet = name
|
14
|
-
@name = @spreadsheet.title
|
15
|
-
@key = @spreadsheet.key
|
16
|
-
end
|
17
|
-
|
18
|
-
@key ||= options[:key]
|
19
|
-
@session ||= options.fetch(:session) { Datapimp::Sync.google.api }
|
20
|
-
|
21
|
-
ensure_valid_options!
|
22
|
-
end
|
23
|
-
|
24
|
-
def self.create_from_file(path, title)
|
25
|
-
if find_by_title(title)
|
26
|
-
raise 'Spreadsheet with this title already exists'
|
27
|
-
end
|
28
|
-
|
29
|
-
session.upload_from_file(path, title, :content_type => "text/csv")
|
30
|
-
|
31
|
-
find_by_title(title)
|
32
|
-
end
|
33
|
-
|
34
|
-
def self.[](key_or_title)
|
35
|
-
find_by_key(key_or_title) || find_by_title(key_or_title)
|
36
|
-
end
|
37
|
-
|
38
|
-
def self.find_by_key(key)
|
39
|
-
sheet = session_spreadsheets.detect do |spreadsheet|
|
40
|
-
spreadsheet.key == key
|
41
|
-
end
|
42
|
-
|
43
|
-
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
44
|
-
end
|
45
|
-
|
46
|
-
def self.find_by_title title
|
47
|
-
sheet = session_spreadsheets.detect do |spreadsheet|
|
48
|
-
spreadsheet.title.match(title)
|
49
|
-
end
|
50
|
-
|
51
|
-
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
52
|
-
end
|
53
|
-
|
54
|
-
def self.session_spreadsheets
|
55
|
-
@session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
|
56
|
-
end
|
57
|
-
|
58
|
-
def self.create_from_data(data, options={})
|
59
|
-
require 'csv'
|
60
|
-
|
61
|
-
headers = Array(options[:headers]).map(&:to_s)
|
62
|
-
|
63
|
-
tmpfile = "tmp-csv.csv"
|
64
|
-
|
65
|
-
CSV.open(tmpfile, "wb") do |csv|
|
66
|
-
csv << headers
|
67
|
-
|
68
|
-
data.each do |row|
|
69
|
-
csv << headers.map do |header|
|
70
|
-
row = row.stringify_keys
|
71
|
-
row[header.to_s]
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
|
76
|
-
spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
|
77
|
-
|
78
|
-
new(spreadsheet.title, key: spreadsheet.key)
|
79
|
-
end
|
80
|
-
|
81
|
-
|
82
|
-
def title
|
83
|
-
@name ||= spreadsheet.try(:title)
|
84
|
-
end
|
85
|
-
|
86
|
-
def edit_url
|
87
|
-
spreadsheet.human_url
|
88
|
-
end
|
89
|
-
|
90
|
-
def share_write_access_with *emails
|
91
|
-
acl = spreadsheet.acl
|
92
|
-
|
93
|
-
Array(emails).flatten.each do |email|
|
94
|
-
acl.push scope_type: "user",
|
95
|
-
with_key: false,
|
96
|
-
role: "writer",
|
97
|
-
scope: email
|
98
|
-
end
|
99
|
-
end
|
100
|
-
|
101
|
-
def share_read_access_with *emails
|
102
|
-
acl = spreadsheet.acl
|
103
|
-
|
104
|
-
Array(emails).flatten.each do |email|
|
105
|
-
acl.push scope_type: "user",
|
106
|
-
with_key: false,
|
107
|
-
role: "reader",
|
108
|
-
scope: email
|
109
|
-
end
|
110
|
-
end
|
111
|
-
|
112
|
-
def add_to_collection collection_title
|
113
|
-
collection = if collection_title.is_a?(GoogleDrive::Collection)
|
114
|
-
collection_title
|
115
|
-
else
|
116
|
-
session.collections.find do |c|
|
117
|
-
c.title == collection_title
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
if !collection
|
122
|
-
collection_names = session.collections.map(&:title)
|
123
|
-
raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
|
124
|
-
end
|
125
|
-
end
|
126
|
-
|
127
|
-
def spreadsheet_key
|
128
|
-
key
|
129
|
-
end
|
130
|
-
|
131
|
-
def stale?
|
132
|
-
(!need_to_refresh? && (age > max_age)) || fresh_on_server?
|
133
|
-
end
|
134
|
-
|
135
|
-
def fresh_on_server?
|
136
|
-
refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
|
137
|
-
end
|
138
|
-
|
139
|
-
def last_updated_at
|
140
|
-
if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
|
141
|
-
DateTime.parse(value).to_i
|
142
|
-
else
|
143
|
-
Time.now.to_i
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
def fetch
|
148
|
-
self.raw = process_worksheets
|
149
|
-
end
|
150
|
-
|
151
|
-
def preprocess
|
152
|
-
single? ? raw.values.flatten : raw
|
153
|
-
end
|
154
|
-
|
155
|
-
protected
|
156
|
-
|
157
|
-
def process_worksheets
|
158
|
-
worksheets.inject({}.to_mash) do |memo, parts|
|
159
|
-
k, ws = parts
|
160
|
-
header_row = Array(ws.rows[0])
|
161
|
-
column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
|
162
|
-
rows = ws.rows.slice(1, ws.rows.length)
|
163
|
-
|
164
|
-
row_index = 1
|
165
|
-
memo[k] = rows.map do |row|
|
166
|
-
col_index = 0
|
167
|
-
|
168
|
-
_record = column_names.inject({}) do |record, field|
|
169
|
-
record[field] = "#{ row[col_index] }".strip
|
170
|
-
record["_id"] = row_index
|
171
|
-
col_index += 1
|
172
|
-
record
|
173
|
-
end
|
174
|
-
|
175
|
-
row_index += 1
|
176
|
-
|
177
|
-
_record
|
178
|
-
end
|
179
|
-
|
180
|
-
memo
|
181
|
-
end
|
182
|
-
end
|
183
|
-
|
184
|
-
def single?
|
185
|
-
worksheets.length == 1
|
186
|
-
end
|
187
|
-
|
188
|
-
def header_rows_for_worksheet key
|
189
|
-
if key.is_a?(Fixnum)
|
190
|
-
_worksheets[key]
|
191
|
-
else
|
192
|
-
worksheets.fetch(key)
|
193
|
-
end
|
194
|
-
end
|
195
|
-
|
196
|
-
def worksheets
|
197
|
-
@worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
|
198
|
-
key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
|
199
|
-
memo[key] = ws
|
200
|
-
memo
|
201
|
-
end
|
202
|
-
end
|
203
|
-
|
204
|
-
def _worksheets
|
205
|
-
@_worksheets ||= spreadsheet.worksheets
|
206
|
-
end
|
207
|
-
|
208
|
-
def spreadsheet
|
209
|
-
@spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
|
210
|
-
end
|
211
|
-
end
|
212
|
-
end
|
1
|
+
require 'datapimp/sources/google_spreadsheet'
|
@@ -0,0 +1,212 @@
|
|
1
|
+
module Datapimp::Sources
|
2
|
+
class GoogleSpreadsheet < Datapimp::Sources::Base
|
3
|
+
requires :key
|
4
|
+
|
5
|
+
attr_accessor :key,
|
6
|
+
:session,
|
7
|
+
:name
|
8
|
+
|
9
|
+
def initialize name, options={}
|
10
|
+
@options = options
|
11
|
+
|
12
|
+
if name.is_a?(GoogleDrive::Spreadsheet)
|
13
|
+
@spreadsheet = name
|
14
|
+
@name = @spreadsheet.title
|
15
|
+
@key = @spreadsheet.key
|
16
|
+
end
|
17
|
+
|
18
|
+
@key ||= options[:key]
|
19
|
+
@session ||= options.fetch(:session) { Datapimp::Sync.google.api }
|
20
|
+
|
21
|
+
ensure_valid_options!
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.create_from_file(path, title)
|
25
|
+
if find_by_title(title)
|
26
|
+
raise 'Spreadsheet with this title already exists'
|
27
|
+
end
|
28
|
+
|
29
|
+
session.upload_from_file(path, title, :content_type => "text/csv")
|
30
|
+
|
31
|
+
find_by_title(title)
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.[](key_or_title)
|
35
|
+
find_by_key(key_or_title) || find_by_title(key_or_title)
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.find_by_key(key)
|
39
|
+
sheet = session_spreadsheets.detect do |spreadsheet|
|
40
|
+
spreadsheet.key == key
|
41
|
+
end
|
42
|
+
|
43
|
+
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
44
|
+
end
|
45
|
+
|
46
|
+
def self.find_by_title title
|
47
|
+
sheet = session_spreadsheets.detect do |spreadsheet|
|
48
|
+
spreadsheet.title.match(title)
|
49
|
+
end
|
50
|
+
|
51
|
+
sheet && new(sheet, session: Datapimp::Sync.google.session)
|
52
|
+
end
|
53
|
+
|
54
|
+
def self.session_spreadsheets
|
55
|
+
@session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.create_from_data(data, options={})
|
59
|
+
require 'csv'
|
60
|
+
|
61
|
+
headers = Array(options[:headers]).map(&:to_s)
|
62
|
+
|
63
|
+
tmpfile = "tmp-csv.csv"
|
64
|
+
|
65
|
+
CSV.open(tmpfile, "wb") do |csv|
|
66
|
+
csv << headers
|
67
|
+
|
68
|
+
data.each do |row|
|
69
|
+
csv << headers.map do |header|
|
70
|
+
row = row.stringify_keys
|
71
|
+
row[header.to_s]
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
|
77
|
+
|
78
|
+
new(spreadsheet.title, key: spreadsheet.key)
|
79
|
+
end
|
80
|
+
|
81
|
+
|
82
|
+
def title
|
83
|
+
@name ||= spreadsheet.try(:title)
|
84
|
+
end
|
85
|
+
|
86
|
+
def edit_url
|
87
|
+
spreadsheet.human_url
|
88
|
+
end
|
89
|
+
|
90
|
+
def share_write_access_with *emails
|
91
|
+
acl = spreadsheet.acl
|
92
|
+
|
93
|
+
Array(emails).flatten.each do |email|
|
94
|
+
acl.push scope_type: "user",
|
95
|
+
with_key: false,
|
96
|
+
role: "writer",
|
97
|
+
scope: email
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def share_read_access_with *emails
|
102
|
+
acl = spreadsheet.acl
|
103
|
+
|
104
|
+
Array(emails).flatten.each do |email|
|
105
|
+
acl.push scope_type: "user",
|
106
|
+
with_key: false,
|
107
|
+
role: "reader",
|
108
|
+
scope: email
|
109
|
+
end
|
110
|
+
end
|
111
|
+
|
112
|
+
def add_to_collection collection_title
|
113
|
+
collection = if collection_title.is_a?(GoogleDrive::Collection)
|
114
|
+
collection_title
|
115
|
+
else
|
116
|
+
session.collections.find do |c|
|
117
|
+
c.title == collection_title
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
if !collection
|
122
|
+
collection_names = session.collections.map(&:title)
|
123
|
+
raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
def spreadsheet_key
|
128
|
+
key
|
129
|
+
end
|
130
|
+
|
131
|
+
def stale?
|
132
|
+
(!need_to_refresh? && (age > max_age)) || fresh_on_server?
|
133
|
+
end
|
134
|
+
|
135
|
+
def fresh_on_server?
|
136
|
+
refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
|
137
|
+
end
|
138
|
+
|
139
|
+
def last_updated_at
|
140
|
+
if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
|
141
|
+
DateTime.parse(value).to_i
|
142
|
+
else
|
143
|
+
Time.now.to_i
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def fetch
|
148
|
+
self.raw = process_worksheets
|
149
|
+
end
|
150
|
+
|
151
|
+
def preprocess
|
152
|
+
single? ? raw.values.flatten : raw
|
153
|
+
end
|
154
|
+
|
155
|
+
protected
|
156
|
+
|
157
|
+
def process_worksheets
|
158
|
+
worksheets.inject({}.to_mash) do |memo, parts|
|
159
|
+
k, ws = parts
|
160
|
+
header_row = Array(ws.rows[0])
|
161
|
+
column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
|
162
|
+
rows = ws.rows.slice(1, ws.rows.length)
|
163
|
+
|
164
|
+
row_index = 1
|
165
|
+
memo[k] = rows.map do |row|
|
166
|
+
col_index = 0
|
167
|
+
|
168
|
+
_record = column_names.inject({}) do |record, field|
|
169
|
+
record[field] = "#{ row[col_index] }".strip
|
170
|
+
record["_id"] = row_index
|
171
|
+
col_index += 1
|
172
|
+
record
|
173
|
+
end
|
174
|
+
|
175
|
+
row_index += 1
|
176
|
+
|
177
|
+
_record
|
178
|
+
end
|
179
|
+
|
180
|
+
memo
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
def single?
|
185
|
+
worksheets.length == 1
|
186
|
+
end
|
187
|
+
|
188
|
+
def header_rows_for_worksheet key
|
189
|
+
if key.is_a?(Fixnum)
|
190
|
+
_worksheets[key]
|
191
|
+
else
|
192
|
+
worksheets.fetch(key)
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
def worksheets
|
197
|
+
@worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
|
198
|
+
key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
|
199
|
+
memo[key] = ws
|
200
|
+
memo
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def _worksheets
|
205
|
+
@_worksheets ||= spreadsheet.worksheets
|
206
|
+
end
|
207
|
+
|
208
|
+
def spreadsheet
|
209
|
+
@spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
end
|
data/lib/datapimp/sync.rb
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
module Datapimp
|
5
5
|
module Sync
|
6
6
|
def self.data_source_types
|
7
|
-
%w(dropbox amazon github google json excel nokogiri)
|
7
|
+
%w(dropbox amazon github google pivotal json excel nokogiri)
|
8
8
|
end
|
9
9
|
|
10
10
|
def self.dispatch_sync_data_action(args, options)
|
@@ -13,7 +13,7 @@ module Datapimp
|
|
13
13
|
|
14
14
|
result = case type
|
15
15
|
when "github"
|
16
|
-
Datapimp::Sources::GithubRepository.new(
|
16
|
+
Datapimp::Sources::GithubRepository.new(args, options)
|
17
17
|
when "google", "google-spreadsheet"
|
18
18
|
require 'google_drive'
|
19
19
|
Datapimp::Sources::GoogleSpreadsheet.new(nil, key: source)
|
data/lib/datapimp/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: datapimp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan Soeder
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: pry
|
@@ -42,14 +42,14 @@ dependencies:
|
|
42
42
|
name: commander
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - "
|
45
|
+
- - ">="
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '4.3'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - "
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '4.3'
|
55
55
|
- !ruby/object:Gem::Dependency
|
@@ -98,14 +98,14 @@ dependencies:
|
|
98
98
|
name: google_drive
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - "
|
101
|
+
- - ">="
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '1.0'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - "
|
108
|
+
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '1.0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
@@ -291,19 +291,19 @@ dependencies:
|
|
291
291
|
- !ruby/object:Gem::Version
|
292
292
|
version: '1.10'
|
293
293
|
- !ruby/object:Gem::Dependency
|
294
|
-
name:
|
294
|
+
name: tracker_api
|
295
295
|
requirement: !ruby/object:Gem::Requirement
|
296
296
|
requirements:
|
297
|
-
- - "
|
297
|
+
- - ">="
|
298
298
|
- !ruby/object:Gem::Version
|
299
|
-
version: 0.
|
299
|
+
version: 0.2.10
|
300
300
|
type: :runtime
|
301
301
|
prerelease: false
|
302
302
|
version_requirements: !ruby/object:Gem::Requirement
|
303
303
|
requirements:
|
304
|
-
- - "
|
304
|
+
- - ">="
|
305
305
|
- !ruby/object:Gem::Version
|
306
|
-
version: 0.
|
306
|
+
version: 0.2.10
|
307
307
|
- !ruby/object:Gem::Dependency
|
308
308
|
name: keen
|
309
309
|
requirement: !ruby/object:Gem::Requirement
|
@@ -360,6 +360,20 @@ dependencies:
|
|
360
360
|
- - '='
|
361
361
|
- !ruby/object:Gem::Version
|
362
362
|
version: 0.0.6
|
363
|
+
- !ruby/object:Gem::Dependency
|
364
|
+
name: git-version-bump
|
365
|
+
requirement: !ruby/object:Gem::Requirement
|
366
|
+
requirements:
|
367
|
+
- - ">="
|
368
|
+
- !ruby/object:Gem::Version
|
369
|
+
version: '0'
|
370
|
+
type: :runtime
|
371
|
+
prerelease: false
|
372
|
+
version_requirements: !ruby/object:Gem::Requirement
|
373
|
+
requirements:
|
374
|
+
- - ">="
|
375
|
+
- !ruby/object:Gem::Version
|
376
|
+
version: '0'
|
363
377
|
- !ruby/object:Gem::Dependency
|
364
378
|
name: rake
|
365
379
|
requirement: !ruby/object:Gem::Requirement
|
@@ -434,6 +448,7 @@ description: Your rails app in a custom tailored suit.
|
|
434
448
|
email:
|
435
449
|
- jonathan.soeder@gmail.com
|
436
450
|
executables:
|
451
|
+
- console
|
437
452
|
- datapimp
|
438
453
|
extensions: []
|
439
454
|
extra_rdoc_files: []
|
@@ -446,6 +461,7 @@ files:
|
|
446
461
|
- README.md
|
447
462
|
- ROADMAP.md
|
448
463
|
- Rakefile
|
464
|
+
- bin/console
|
449
465
|
- bin/datapimp
|
450
466
|
- datapimp.gemspec
|
451
467
|
- lib/datapimp.rb
|
@@ -462,6 +478,7 @@ files:
|
|
462
478
|
- lib/datapimp/clients/dropbox.rb
|
463
479
|
- lib/datapimp/clients/github.rb
|
464
480
|
- lib/datapimp/clients/google.rb
|
481
|
+
- lib/datapimp/clients/keen.rb
|
465
482
|
- lib/datapimp/configuration.rb
|
466
483
|
- lib/datapimp/core_ext.rb
|
467
484
|
- lib/datapimp/logging.rb
|
@@ -470,6 +487,7 @@ files:
|
|
470
487
|
- lib/datapimp/sources/excel.rb
|
471
488
|
- lib/datapimp/sources/github_repository.rb
|
472
489
|
- lib/datapimp/sources/google.rb
|
490
|
+
- lib/datapimp/sources/google_spreadsheet.rb
|
473
491
|
- lib/datapimp/sources/json.rb
|
474
492
|
- lib/datapimp/sources/keen.rb
|
475
493
|
- lib/datapimp/sources/nokogiri.rb
|