datapimp 1.2.1 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4f7ee3a50c4372896b49f7ac03339154fcba987d
4
- data.tar.gz: cb2eac6337c4596b18f15ca04962eba1fce5ebe2
3
+ metadata.gz: 347b281ae0e4e748f903410bdf0218a1b24a00a5
4
+ data.tar.gz: f497dcd87a2a07bc2bbec8966e2c86cde589aaf6
5
5
  SHA512:
6
- metadata.gz: 2eab363452cee9b6f3b0b0d426ecc84255d27d98c424273dec232babc472c0a9657a770f6374f0819a89bddf81fdafa7b17ae27d1a0fdb031efd05e626847336
7
- data.tar.gz: 2053ec83701f4c235bcb6f78a376a9f52d3d8b640d3713703b8cb0214c4a78ea214e1045884a8b9a853d8606cb8ffd061108a91faeb2e23309a57f9ab153714b
6
+ metadata.gz: c37075bc592d3c1e55246f6b83341f466cef1f08faaea0734b4ba04b6e84187aa33302721101b94ed6d6fa7e8edc703094922ba4789c3a48b68b0d9d90284cae
7
+ data.tar.gz: 7e8a151dc77e82ca4a01f5705fe7f20bf718bf332c1f91037a4d7a486ed42f2fbb3555a6e4dc15321ef1676c804870ad5afa5b2c852ecabc9122d3e697def470
data/bin/console ADDED
@@ -0,0 +1,7 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "datapimp"
5
+
6
+ require "pry"
7
+ Pry.start
data/bin/datapimp CHANGED
@@ -9,6 +9,7 @@ require "commander/import"
9
9
  require 'datapimp'
10
10
  require 'datapimp/cli'
11
11
 
12
+ $datapimp_cli = "datapimp"
12
13
  $terminal.wrap_at = 120
13
14
 
14
15
  program :name, 'Datapimp'
data/datapimp.gemspec CHANGED
@@ -20,11 +20,11 @@ Gem::Specification.new do |spec|
20
20
 
21
21
  spec.add_dependency 'pry', '~> 0.10'
22
22
  spec.add_dependency 'hashie', '>= 3.0.4'
23
- spec.add_dependency 'commander', '~> 4.3'
23
+ spec.add_dependency 'commander', '>= 4.3'
24
24
  spec.add_dependency 'terminal-table'
25
25
  spec.add_dependency 'fog-aws', '>= 0.1'
26
26
  spec.add_dependency 'dropbox-api', '>= 0.4.7'
27
- spec.add_dependency 'google_drive', '~> 1.0'
27
+ spec.add_dependency 'google_drive', '>= 1.0'
28
28
  spec.add_dependency 'google-api-client', '>= 0.8'
29
29
  spec.add_dependency 'rack-contrib', '~> 1.2'
30
30
  spec.add_dependency 'uri_template', '~> 0.7'
@@ -38,7 +38,7 @@ Gem::Specification.new do |spec|
38
38
  spec.add_dependency 'github-fs', '~> 0'
39
39
  spec.add_dependency 'colored', '> 0.0'
40
40
  spec.add_dependency 'multi_json', '~> 1.10'
41
- spec.add_dependency 'pivotal-tracker', '~> 0.5.13'
41
+ spec.add_dependency 'tracker_api', '>= 0.2.10'
42
42
  spec.add_dependency 'keen', '~> 0.9.0'
43
43
 
44
44
  # these are locked to specific versions so that
@@ -46,6 +46,8 @@ Gem::Specification.new do |spec|
46
46
  spec.add_dependency 'nokogiri', '~> 1.6'
47
47
  spec.add_dependency 'unf', '0.1.4'
48
48
  spec.add_dependency 'unf_ext', '0.0.6'
49
+ spec.add_dependency 'git-version-bump'
50
+
49
51
 
50
52
  spec.add_development_dependency "rake", '~> 0'
51
53
  spec.add_development_dependency 'rspec', '~> 3.3.0'
data/lib/datapimp/cli.rb CHANGED
@@ -3,6 +3,7 @@ require 'launchy'
3
3
  module Datapimp
4
4
  module Cli
5
5
  def self.load_commands(from_path=nil)
6
+ $datapimp_cli ||= "datapimp"
6
7
  from_path ||= Datapimp.lib.join("datapimp","cli")
7
8
  Dir[from_path.join("**/*.rb")].each {|f| require(f) }
8
9
  end
@@ -44,6 +45,16 @@ module Datapimp
44
45
  c.option '--aws-access-key-id', String, 'AWS Access Key ID'
45
46
  end
46
47
 
48
+ if services.include?(:keen)
49
+ c.option '--keen-project-id', String, 'Keen Project ID'
50
+ c.option '--keen-read-key', String, 'Keen Read Key'
51
+ c.option '--keen-write-key', String, 'Keen Write Key'
52
+ end
53
+
54
+ if services.include?(:pivotal)
55
+ c.option '--pivotal-access-token', String, 'Pivotal Tracker Access Token'
56
+ end
57
+
47
58
  end
48
59
  end
49
60
  end
@@ -1,5 +1,5 @@
1
1
  command 'config' do |c|
2
- c.syntax = "datapimp config [OPTIONS]"
2
+ c.syntax = "#{$datapimp_cli} config [OPTIONS]"
3
3
  c.description = "Shows the configuration options being used"
4
4
 
5
5
  c.option '--env', "Output compatible with .env files"
@@ -16,7 +16,7 @@ command 'config' do |c|
16
16
  end
17
17
 
18
18
  command 'config set' do |c|
19
- c.syntax = 'datapimp config set KEY=VALUE KEY=VALUE [options]'
19
+ c.syntax = "#{$datapimp_cli} config set KEY=VALUE KEY=VALUE [options]"
20
20
  c.description = 'manipulate configuration settings'
21
21
 
22
22
  c.option '--global', 'Set the configuration globally'
@@ -1,5 +1,5 @@
1
1
  command 'create cache invalidations' do |c|
2
- c.syntax = 'datapimp create cache invalidations'
2
+ c.syntax = "#{$datapimp_cli} create cache invalidations"
3
3
  c.description = 'invalidate remote cache layers (i.e. cloudfront after a s3 deploy)'
4
4
 
5
5
  Datapimp::Cli.accepts_keys_for(c, :amazon)
@@ -75,7 +75,7 @@ command 'create cache invalidations' do |c|
75
75
  end
76
76
 
77
77
  command 'create s3 bucket' do |c|
78
- c.syntax = 'datapimp create s3 bucket BUCKETNAME'
78
+ c.syntax = "#{$datapimp_cli} create s3 bucket BUCKETNAME"
79
79
  c.description = 'create an s3 bucket to use for website hosting'
80
80
 
81
81
  Datapimp::Cli.accepts_keys_for(c, :amazon)
@@ -90,7 +90,7 @@ command 'create s3 bucket' do |c|
90
90
  end
91
91
 
92
92
  command 'create cloudfront distribution' do |c|
93
- c.syntax = "datapimp create cloudfront distribution"
93
+ c.syntax = "#{$datapimp_cli} create cloudfront distribution"
94
94
  c.description = "create a cloudfront distribution to link to a specific bucket"
95
95
 
96
96
  Datapimp::Cli.accepts_keys_for(c, :amazon)
@@ -99,6 +99,8 @@ command 'create cloudfront distribution' do |c|
99
99
  c.option '--domains DOMAINS', Array, 'What domains will be pointing to this bucket?'
100
100
 
101
101
  c.action do |args, options|
102
+ options.default(bucket: args.first)
103
+
102
104
  bucket = Datapimp::Sync::S3Bucket.new(remote: options.bucket)
103
105
 
104
106
  cdn_options = {
@@ -1,5 +1,5 @@
1
1
  command "list spreadsheets" do |c|
2
- c.syntax = "datapimp list spreadsheets"
2
+ c.syntax = "#{$datapimp_cli} list spreadsheets"
3
3
  c.description = "list the spreadsheets which can be used as datasources"
4
4
 
5
5
  c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
@@ -21,14 +21,14 @@ command "list spreadsheets" do |c|
21
21
  if lines.length > 0
22
22
  puts "\n\nExample:"
23
23
  puts "====="
24
- puts "datapimp sync data #{ lines.first.split(/\t/).first } --type google-spreadsheet"
24
+ puts "#{$datapimp_cli} sync data #{ lines.first.split(/\t/).first } --type google-spreadsheet"
25
25
  puts "\n\n"
26
26
  end
27
27
  end
28
28
  end
29
29
 
30
30
  command "list folders" do |c|
31
- c.syntax= "datapimp list folders [OPTIONS]"
31
+ c.syntax= "#{$datapimp_cli} list folders [OPTIONS]"
32
32
  c.description= "lists folders in a remote service"
33
33
 
34
34
  c.option '--type SERVICE', String, 'Which service to search: dropbox, google, amazon'
@@ -1,5 +1,5 @@
1
1
  command "run" do |c|
2
- c.syntax = "datapimp run FILE"
2
+ c.syntax = "#{$datapimp_cli} run FILE"
3
3
  c.description = "runs a script in the context of the datapimp config"
4
4
 
5
5
  c.option '--format FORMAT', String, 'which format should we serialize the result? json default'
@@ -1,5 +1,5 @@
1
1
  command "setup google" do |c|
2
- c.syntax = "datapimp set up google"
2
+ c.syntax = "#{$datapimp_cli} set up google"
3
3
  c.description = "setup integration with google drive"
4
4
 
5
5
  c.action do |args, options|
@@ -8,7 +8,7 @@ command "setup google" do |c|
8
8
  end
9
9
 
10
10
  command "setup amazon" do |c|
11
- c.syntax = "datapimp setup amazon"
11
+ c.syntax = "#{$datapimp_cli} setup amazon"
12
12
  c.description = "setup integration with amazon"
13
13
 
14
14
  c.action do |args, options|
@@ -17,7 +17,7 @@ command "setup amazon" do |c|
17
17
  end
18
18
 
19
19
  command "setup dropbox" do |c|
20
- c.syntax = "datapimp set up dropbox"
20
+ c.syntax = "#{$datapimp_cli} set up dropbox"
21
21
  c.description = "setup integration with dropbox"
22
22
 
23
23
  c.action do |args, options|
@@ -26,10 +26,19 @@ command "setup dropbox" do |c|
26
26
  end
27
27
 
28
28
  command "setup github" do |c|
29
- c.syntax = "datapimp set up github"
29
+ c.syntax = "#{$datapimp_cli} set up github"
30
30
  c.description = "setup integration with github"
31
31
 
32
32
  c.action do |args, options|
33
33
  Datapimp::Sync.github.setup()
34
34
  end
35
35
  end
36
+
37
+ command "setup pivotal" do |c|
38
+ c.syntax = "#{$datapimp_cli} set up github"
39
+ c.description = "setup integration with github"
40
+
41
+ c.action do |args, options|
42
+ Datapimp::Sync.pivotal.setup()
43
+ end
44
+ end
@@ -1,6 +1,6 @@
1
1
  command "sync folder" do |c|
2
2
  c.description = "Synchronize the contents of a local folder with a file sharing service"
3
- c.syntax = "datapimp sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
3
+ c.syntax = "#{$datapimp_cli} sync folder LOCAL_PATH REMOTE_PATH [OPTIONS]"
4
4
 
5
5
  c.option '--type TYPE', String, "Which service is hosting the folder"
6
6
  c.option '--action ACTION', String, "Which sync action to run? push, pull"
@@ -18,7 +18,7 @@ end
18
18
 
19
19
  command "sync data" do |c|
20
20
  c.description = "Synchronize the contents of a local data store with its remote source"
21
- c.syntax = "datapimp sync data [OPTIONS]"
21
+ c.syntax = "#{$datapimp_cli} sync data [OPTIONS]"
22
22
 
23
23
  c.option '--type TYPE', String, "What type of source data is this? #{ Datapimp::Sync.data_source_types.join(", ") }"
24
24
  c.option '--output FILE', String, "Write the output to a file"
@@ -30,17 +30,19 @@ command "sync data" do |c|
30
30
  c.option '--limit LIMIT', Integer, "Limit the number of results for Pivotal resources"
31
31
  c.option '--offset OFFSET', Integer, "Offset applied when using the limit option for Pivotal resources"
32
32
 
33
- c.example "Syncing an excel file from dropbox ", "datapimp sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
34
- c.example "Syncing a google spreadsheet", "datapimp sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
35
- c.example "Syncing Pivotal Tracker data, user activity", "datapimp sync data --type pivotal --view user-activity"
36
- c.example "Syncing Pivotal Tracker data, project activity", "datapimp sync data --type pivotal --view project-activity PROJECT_ID"
37
- c.example "Syncing Pivotal Tracker data, project stories", "datapimp sync data --type pivotal --view project-stories PROJECT_ID"
38
- c.example "Syncing Pivotal Tracker data, project story notes", "datapimp sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
39
- c.example "Syncing keen.io data, extraction from an event_collection", "datapimp sync data --type keen EVENT_COLLECTION"
40
- c.example "Syncing Github Issues", "datapimp sync data --type github --view issues REPOSITORY"
41
- c.example "Syncing Github Issue Comments", "datapimp sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
42
-
43
- Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox)
33
+ c.example "Syncing an excel file from dropbox ", "#{$datapimp_cli} sync data --type dropbox --columns name,description --dropbox-app-key ABC --dropbox-app-secret DEF --dropbox-client-token HIJ --dropbox-client-secret JKL spreadsheets/test.xslx"
34
+ c.example "Syncing a google spreadsheet", "#{$datapimp_cli} sync data --type google-spreadsheet WHATEVER_THE_KEY_IS"
35
+ c.example "Syncing keen.io data, extraction from an event_collection", "#{$datapimp_cli} sync data --type keen EVENT_COLLECTION"
36
+
37
+ c.example "Syncing Github Issues", "#{$datapimp_cli} sync data --type github --view issues REPOSITORY"
38
+ c.example "Syncing Github Issue Comments", "#{$datapimp_cli} sync data --type github --view issue-comments REPOSITORY ISSUE_ID"
39
+
40
+ c.example "Syncing Pivotal Tracker data, user activity", "#{$datapimp_cli} sync data --type pivotal --view user-activity"
41
+ c.example "Syncing Pivotal Tracker data, project activity", "#{$datapimp_cli} sync data --type pivotal --view project-activity PROJECT_ID"
42
+ c.example "Syncing Pivotal Tracker data, project stories", "#{$datapimp_cli} sync data --type pivotal --view project-stories PROJECT_ID"
43
+ c.example "Syncing Pivotal Tracker data, project story notes", "#{$datapimp_cli} sync data --type pivotal --view project-story-notes PROJECT_ID STORY_ID"
44
+
45
+ Datapimp::Cli.accepts_keys_for(c, :google, :github, :dropbox, :keen, :pivotal)
44
46
 
45
47
  c.action do |args, options|
46
48
  options.default(view:"to_s")
@@ -1,5 +1,5 @@
1
1
  command 'view amazon setup' do |c|
2
- c.syntax = 'datapimp view amazon setup'
2
+ c.syntax = "#{$datapimp_cli} view amazon setup"
3
3
  c.description = 'view the amazon (s3 + cloudfront) setup'
4
4
 
5
5
  Datapimp::Cli.accepts_keys_for(c, :amazon)
@@ -0,0 +1,48 @@
1
+ module Datapimp
2
+ module Clients
3
+ class Keen
4
+ include Singleton
5
+
6
+ def self.method_missing(meth, *args, &block)
7
+ if client.respond_to?(meth)
8
+ return client.send(meth, *args, &block)
9
+ end
10
+
11
+ super
12
+ end
13
+
14
+ def self.client(options={})
15
+ Keen
16
+ end
17
+
18
+ def options
19
+ @options ||= {}
20
+ end
21
+
22
+ def with_options(opts={})
23
+ options.merge!(opts)
24
+ self
25
+ end
26
+
27
+ def setup(options={})
28
+ access_token = options[:keen_read_key] || Datapimp.config.keen_read_key
29
+ project_id = options[:keen_project_id] || Datapimp.config.keen_project_id
30
+
31
+ unless access_token.to_s.length > 1
32
+ if respond_to?(:ask)
33
+ access_token = ask("Enter a keen read key when you have one", String)
34
+ end
35
+ end
36
+
37
+ unless project_id.to_s.length > 1
38
+ if respond_to?(:ask)
39
+ project_id = ask("Enter a keen read key when you have one", String)
40
+ end
41
+ end
42
+
43
+ Datapimp.config.set(:keen_read_key, access_token) if access_token.to_s.length > 1
44
+ Datapimp.config.set(:keen_project_id, project_id) if project_id.to_s.length > 1
45
+ end
46
+ end
47
+ end
48
+ end
@@ -15,8 +15,10 @@ module Datapimp
15
15
  github_access_token: '',
16
16
 
17
17
  pivotal_access_token: '',
18
- keen_project_id: ENV.fetch('KEEN_PROJECT_ID', ''),
19
- keen_read_key: ENV.fetch('KEEN_READ_KEY', ''),
18
+
19
+ keen_project_id: '',
20
+ keen_read_key: '',
21
+ keen_write_key: '',
20
22
 
21
23
  dnsimple_api_token: '',
22
24
  dnsimple_username: '',
@@ -1,212 +1 @@
1
- module Datapimp::Sources
2
- class GoogleSpreadsheet < Datapimp::Sources::Base
3
- requires :key
4
-
5
- attr_accessor :key,
6
- :session,
7
- :name
8
-
9
- def initialize name, options={}
10
- @options = options
11
-
12
- if name.is_a?(GoogleDrive::Spreadsheet)
13
- @spreadsheet = name
14
- @name = @spreadsheet.title
15
- @key = @spreadsheet.key
16
- end
17
-
18
- @key ||= options[:key]
19
- @session ||= options.fetch(:session) { Datapimp::Sync.google.api }
20
-
21
- ensure_valid_options!
22
- end
23
-
24
- def self.create_from_file(path, title)
25
- if find_by_title(title)
26
- raise 'Spreadsheet with this title already exists'
27
- end
28
-
29
- session.upload_from_file(path, title, :content_type => "text/csv")
30
-
31
- find_by_title(title)
32
- end
33
-
34
- def self.[](key_or_title)
35
- find_by_key(key_or_title) || find_by_title(key_or_title)
36
- end
37
-
38
- def self.find_by_key(key)
39
- sheet = session_spreadsheets.detect do |spreadsheet|
40
- spreadsheet.key == key
41
- end
42
-
43
- sheet && new(sheet, session: Datapimp::Sync.google.session)
44
- end
45
-
46
- def self.find_by_title title
47
- sheet = session_spreadsheets.detect do |spreadsheet|
48
- spreadsheet.title.match(title)
49
- end
50
-
51
- sheet && new(sheet, session: Datapimp::Sync.google.session)
52
- end
53
-
54
- def self.session_spreadsheets
55
- @session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
56
- end
57
-
58
- def self.create_from_data(data, options={})
59
- require 'csv'
60
-
61
- headers = Array(options[:headers]).map(&:to_s)
62
-
63
- tmpfile = "tmp-csv.csv"
64
-
65
- CSV.open(tmpfile, "wb") do |csv|
66
- csv << headers
67
-
68
- data.each do |row|
69
- csv << headers.map do |header|
70
- row = row.stringify_keys
71
- row[header.to_s]
72
- end
73
- end
74
- end
75
-
76
- spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
77
-
78
- new(spreadsheet.title, key: spreadsheet.key)
79
- end
80
-
81
-
82
- def title
83
- @name ||= spreadsheet.try(:title)
84
- end
85
-
86
- def edit_url
87
- spreadsheet.human_url
88
- end
89
-
90
- def share_write_access_with *emails
91
- acl = spreadsheet.acl
92
-
93
- Array(emails).flatten.each do |email|
94
- acl.push scope_type: "user",
95
- with_key: false,
96
- role: "writer",
97
- scope: email
98
- end
99
- end
100
-
101
- def share_read_access_with *emails
102
- acl = spreadsheet.acl
103
-
104
- Array(emails).flatten.each do |email|
105
- acl.push scope_type: "user",
106
- with_key: false,
107
- role: "reader",
108
- scope: email
109
- end
110
- end
111
-
112
- def add_to_collection collection_title
113
- collection = if collection_title.is_a?(GoogleDrive::Collection)
114
- collection_title
115
- else
116
- session.collections.find do |c|
117
- c.title == collection_title
118
- end
119
- end
120
-
121
- if !collection
122
- collection_names = session.collections.map(&:title)
123
- raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
124
- end
125
- end
126
-
127
- def spreadsheet_key
128
- key
129
- end
130
-
131
- def stale?
132
- (!need_to_refresh? && (age > max_age)) || fresh_on_server?
133
- end
134
-
135
- def fresh_on_server?
136
- refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
137
- end
138
-
139
- def last_updated_at
140
- if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
141
- DateTime.parse(value).to_i
142
- else
143
- Time.now.to_i
144
- end
145
- end
146
-
147
- def fetch
148
- self.raw = process_worksheets
149
- end
150
-
151
- def preprocess
152
- single? ? raw.values.flatten : raw
153
- end
154
-
155
- protected
156
-
157
- def process_worksheets
158
- worksheets.inject({}.to_mash) do |memo, parts|
159
- k, ws = parts
160
- header_row = Array(ws.rows[0])
161
- column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
162
- rows = ws.rows.slice(1, ws.rows.length)
163
-
164
- row_index = 1
165
- memo[k] = rows.map do |row|
166
- col_index = 0
167
-
168
- _record = column_names.inject({}) do |record, field|
169
- record[field] = "#{ row[col_index] }".strip
170
- record["_id"] = row_index
171
- col_index += 1
172
- record
173
- end
174
-
175
- row_index += 1
176
-
177
- _record
178
- end
179
-
180
- memo
181
- end
182
- end
183
-
184
- def single?
185
- worksheets.length == 1
186
- end
187
-
188
- def header_rows_for_worksheet key
189
- if key.is_a?(Fixnum)
190
- _worksheets[key]
191
- else
192
- worksheets.fetch(key)
193
- end
194
- end
195
-
196
- def worksheets
197
- @worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
198
- key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
199
- memo[key] = ws
200
- memo
201
- end
202
- end
203
-
204
- def _worksheets
205
- @_worksheets ||= spreadsheet.worksheets
206
- end
207
-
208
- def spreadsheet
209
- @spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
210
- end
211
- end
212
- end
1
+ require 'datapimp/sources/google_spreadsheet'
@@ -0,0 +1,212 @@
1
+ module Datapimp::Sources
2
+ class GoogleSpreadsheet < Datapimp::Sources::Base
3
+ requires :key
4
+
5
+ attr_accessor :key,
6
+ :session,
7
+ :name
8
+
9
+ def initialize name, options={}
10
+ @options = options
11
+
12
+ if name.is_a?(GoogleDrive::Spreadsheet)
13
+ @spreadsheet = name
14
+ @name = @spreadsheet.title
15
+ @key = @spreadsheet.key
16
+ end
17
+
18
+ @key ||= options[:key]
19
+ @session ||= options.fetch(:session) { Datapimp::Sync.google.api }
20
+
21
+ ensure_valid_options!
22
+ end
23
+
24
+ def self.create_from_file(path, title)
25
+ if find_by_title(title)
26
+ raise 'Spreadsheet with this title already exists'
27
+ end
28
+
29
+ session.upload_from_file(path, title, :content_type => "text/csv")
30
+
31
+ find_by_title(title)
32
+ end
33
+
34
+ def self.[](key_or_title)
35
+ find_by_key(key_or_title) || find_by_title(key_or_title)
36
+ end
37
+
38
+ def self.find_by_key(key)
39
+ sheet = session_spreadsheets.detect do |spreadsheet|
40
+ spreadsheet.key == key
41
+ end
42
+
43
+ sheet && new(sheet, session: Datapimp::Sync.google.session)
44
+ end
45
+
46
+ def self.find_by_title title
47
+ sheet = session_spreadsheets.detect do |spreadsheet|
48
+ spreadsheet.title.match(title)
49
+ end
50
+
51
+ sheet && new(sheet, session: Datapimp::Sync.google.session)
52
+ end
53
+
54
+ def self.session_spreadsheets
55
+ @session_spreadsheets ||= Datapimp::Sync.google.api.spreadsheets
56
+ end
57
+
58
+ def self.create_from_data(data, options={})
59
+ require 'csv'
60
+
61
+ headers = Array(options[:headers]).map(&:to_s)
62
+
63
+ tmpfile = "tmp-csv.csv"
64
+
65
+ CSV.open(tmpfile, "wb") do |csv|
66
+ csv << headers
67
+
68
+ data.each do |row|
69
+ csv << headers.map do |header|
70
+ row = row.stringify_keys
71
+ row[header.to_s]
72
+ end
73
+ end
74
+ end
75
+
76
+ spreadsheet = Datapimp::Sync.google.api.upload_from_file(tmpfile, options[:title], :content_type => "text/csv")
77
+
78
+ new(spreadsheet.title, key: spreadsheet.key)
79
+ end
80
+
81
+
82
+ def title
83
+ @name ||= spreadsheet.try(:title)
84
+ end
85
+
86
+ def edit_url
87
+ spreadsheet.human_url
88
+ end
89
+
90
+ def share_write_access_with *emails
91
+ acl = spreadsheet.acl
92
+
93
+ Array(emails).flatten.each do |email|
94
+ acl.push scope_type: "user",
95
+ with_key: false,
96
+ role: "writer",
97
+ scope: email
98
+ end
99
+ end
100
+
101
+ def share_read_access_with *emails
102
+ acl = spreadsheet.acl
103
+
104
+ Array(emails).flatten.each do |email|
105
+ acl.push scope_type: "user",
106
+ with_key: false,
107
+ role: "reader",
108
+ scope: email
109
+ end
110
+ end
111
+
112
+ def add_to_collection collection_title
113
+ collection = if collection_title.is_a?(GoogleDrive::Collection)
114
+ collection_title
115
+ else
116
+ session.collections.find do |c|
117
+ c.title == collection_title
118
+ end
119
+ end
120
+
121
+ if !collection
122
+ collection_names = session.collections.map(&:title)
123
+ raise 'Could not find collection in Google drive. Maybe you mean: ' + collection_names.join(', ')
124
+ end
125
+ end
126
+
127
+ def spreadsheet_key
128
+ key
129
+ end
130
+
131
+ def stale?
132
+ (!need_to_refresh? && (age > max_age)) || fresh_on_server?
133
+ end
134
+
135
+ def fresh_on_server?
136
+ refreshed_at.to_i > 0 && (last_updated_at > refreshed_at)
137
+ end
138
+
139
+ def last_updated_at
140
+ if value = spreadsheet.document_feed_entry_internal.css('updated').try(:text) rescue nil
141
+ DateTime.parse(value).to_i
142
+ else
143
+ Time.now.to_i
144
+ end
145
+ end
146
+
147
+ def fetch
148
+ self.raw = process_worksheets
149
+ end
150
+
151
+ def preprocess
152
+ single? ? raw.values.flatten : raw
153
+ end
154
+
155
+ protected
156
+
157
+ def process_worksheets
158
+ worksheets.inject({}.to_mash) do |memo, parts|
159
+ k, ws = parts
160
+ header_row = Array(ws.rows[0])
161
+ column_names = header_row.map {|cell| "#{ cell }".parameterize.underscore }
162
+ rows = ws.rows.slice(1, ws.rows.length)
163
+
164
+ row_index = 1
165
+ memo[k] = rows.map do |row|
166
+ col_index = 0
167
+
168
+ _record = column_names.inject({}) do |record, field|
169
+ record[field] = "#{ row[col_index] }".strip
170
+ record["_id"] = row_index
171
+ col_index += 1
172
+ record
173
+ end
174
+
175
+ row_index += 1
176
+
177
+ _record
178
+ end
179
+
180
+ memo
181
+ end
182
+ end
183
+
184
+ def single?
185
+ worksheets.length == 1
186
+ end
187
+
188
+ def header_rows_for_worksheet key
189
+ if key.is_a?(Fixnum)
190
+ _worksheets[key]
191
+ else
192
+ worksheets.fetch(key)
193
+ end
194
+ end
195
+
196
+ def worksheets
197
+ @worksheets ||= _worksheets.inject({}.to_mash) do |memo,ws|
198
+ key = ws.title.strip.downcase.underscore.gsub(/\s+/,'_')
199
+ memo[key] = ws
200
+ memo
201
+ end
202
+ end
203
+
204
+ def _worksheets
205
+ @_worksheets ||= spreadsheet.worksheets
206
+ end
207
+
208
+ def spreadsheet
209
+ @spreadsheet ||= session.spreadsheet_by_key(spreadsheet_key)
210
+ end
211
+ end
212
+ end
data/lib/datapimp/sync.rb CHANGED
@@ -4,7 +4,7 @@
4
4
  module Datapimp
5
5
  module Sync
6
6
  def self.data_source_types
7
- %w(dropbox amazon github google json excel nokogiri)
7
+ %w(dropbox amazon github google pivotal json excel nokogiri)
8
8
  end
9
9
 
10
10
  def self.dispatch_sync_data_action(args, options)
@@ -13,7 +13,7 @@ module Datapimp
13
13
 
14
14
  result = case type
15
15
  when "github"
16
- Datapimp::Sources::GithubRepository.new(source, options)
16
+ Datapimp::Sources::GithubRepository.new(args, options)
17
17
  when "google", "google-spreadsheet"
18
18
  require 'google_drive'
19
19
  Datapimp::Sources::GoogleSpreadsheet.new(nil, key: source)
@@ -1,3 +1,5 @@
1
+ require 'git-version-bump'
2
+
1
3
  module Datapimp
2
- VERSION = "1.2.1"
4
+ VERSION = GVB.version
3
5
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: datapimp
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.2.1
4
+ version: 1.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jonathan Soeder
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-06-30 00:00:00.000000000 Z
11
+ date: 2015-07-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: pry
@@ -42,14 +42,14 @@ dependencies:
42
42
  name: commander
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - "~>"
45
+ - - ">="
46
46
  - !ruby/object:Gem::Version
47
47
  version: '4.3'
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - "~>"
52
+ - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: '4.3'
55
55
  - !ruby/object:Gem::Dependency
@@ -98,14 +98,14 @@ dependencies:
98
98
  name: google_drive
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - "~>"
101
+ - - ">="
102
102
  - !ruby/object:Gem::Version
103
103
  version: '1.0'
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - "~>"
108
+ - - ">="
109
109
  - !ruby/object:Gem::Version
110
110
  version: '1.0'
111
111
  - !ruby/object:Gem::Dependency
@@ -291,19 +291,19 @@ dependencies:
291
291
  - !ruby/object:Gem::Version
292
292
  version: '1.10'
293
293
  - !ruby/object:Gem::Dependency
294
- name: pivotal-tracker
294
+ name: tracker_api
295
295
  requirement: !ruby/object:Gem::Requirement
296
296
  requirements:
297
- - - "~>"
297
+ - - ">="
298
298
  - !ruby/object:Gem::Version
299
- version: 0.5.13
299
+ version: 0.2.10
300
300
  type: :runtime
301
301
  prerelease: false
302
302
  version_requirements: !ruby/object:Gem::Requirement
303
303
  requirements:
304
- - - "~>"
304
+ - - ">="
305
305
  - !ruby/object:Gem::Version
306
- version: 0.5.13
306
+ version: 0.2.10
307
307
  - !ruby/object:Gem::Dependency
308
308
  name: keen
309
309
  requirement: !ruby/object:Gem::Requirement
@@ -360,6 +360,20 @@ dependencies:
360
360
  - - '='
361
361
  - !ruby/object:Gem::Version
362
362
  version: 0.0.6
363
+ - !ruby/object:Gem::Dependency
364
+ name: git-version-bump
365
+ requirement: !ruby/object:Gem::Requirement
366
+ requirements:
367
+ - - ">="
368
+ - !ruby/object:Gem::Version
369
+ version: '0'
370
+ type: :runtime
371
+ prerelease: false
372
+ version_requirements: !ruby/object:Gem::Requirement
373
+ requirements:
374
+ - - ">="
375
+ - !ruby/object:Gem::Version
376
+ version: '0'
363
377
  - !ruby/object:Gem::Dependency
364
378
  name: rake
365
379
  requirement: !ruby/object:Gem::Requirement
@@ -434,6 +448,7 @@ description: Your rails app in a custom tailored suit.
434
448
  email:
435
449
  - jonathan.soeder@gmail.com
436
450
  executables:
451
+ - console
437
452
  - datapimp
438
453
  extensions: []
439
454
  extra_rdoc_files: []
@@ -446,6 +461,7 @@ files:
446
461
  - README.md
447
462
  - ROADMAP.md
448
463
  - Rakefile
464
+ - bin/console
449
465
  - bin/datapimp
450
466
  - datapimp.gemspec
451
467
  - lib/datapimp.rb
@@ -462,6 +478,7 @@ files:
462
478
  - lib/datapimp/clients/dropbox.rb
463
479
  - lib/datapimp/clients/github.rb
464
480
  - lib/datapimp/clients/google.rb
481
+ - lib/datapimp/clients/keen.rb
465
482
  - lib/datapimp/configuration.rb
466
483
  - lib/datapimp/core_ext.rb
467
484
  - lib/datapimp/logging.rb
@@ -470,6 +487,7 @@ files:
470
487
  - lib/datapimp/sources/excel.rb
471
488
  - lib/datapimp/sources/github_repository.rb
472
489
  - lib/datapimp/sources/google.rb
490
+ - lib/datapimp/sources/google_spreadsheet.rb
473
491
  - lib/datapimp/sources/json.rb
474
492
  - lib/datapimp/sources/keen.rb
475
493
  - lib/datapimp/sources/nokogiri.rb