datapimp 0.0.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +2 -0
  3. data/Gemfile +6 -0
  4. data/Guardfile +5 -0
  5. data/LICENSE.txt +4 -18
  6. data/README.md +199 -17
  7. data/Rakefile +24 -0
  8. data/bin/datapimp +18 -0
  9. data/datapimp.gemspec +27 -7
  10. data/lib/datapimp/cli/01_extensions.rb +25 -0
  11. data/lib/datapimp/cli/config.rb +24 -0
  12. data/lib/datapimp/cli/setup.rb +35 -0
  13. data/lib/datapimp/cli/sync.rb +29 -0
  14. data/lib/datapimp/cli.rb +49 -0
  15. data/lib/datapimp/clients/amazon.rb +172 -0
  16. data/lib/datapimp/clients/dropbox.rb +178 -0
  17. data/lib/datapimp/clients/github.rb +59 -0
  18. data/lib/datapimp/clients/google.rb +145 -0
  19. data/lib/datapimp/configuration.rb +210 -0
  20. data/lib/datapimp/core_ext.rb +5 -0
  21. data/lib/datapimp/data_sources/dropbox.rb +8 -0
  22. data/lib/datapimp/data_sources/excel.rb +5 -0
  23. data/lib/datapimp/data_sources/github.rb +5 -0
  24. data/lib/datapimp/data_sources/google.rb +5 -0
  25. data/lib/datapimp/data_sources/json.rb +5 -0
  26. data/lib/datapimp/data_sources/nokogiri.rb +5 -0
  27. data/lib/datapimp/data_sources.rb +10 -0
  28. data/lib/datapimp/sync/dropbox_delta.rb +67 -0
  29. data/lib/datapimp/sync/dropbox_folder.rb +10 -0
  30. data/lib/datapimp/sync/google_drive_folder.rb +5 -0
  31. data/lib/datapimp/sync.rb +30 -0
  32. data/lib/datapimp/version.rb +1 -1
  33. data/lib/datapimp.rb +30 -2
  34. data/packaging/wrapper.sh +32 -0
  35. data/spec/spec_helper.rb +29 -0
  36. data/spec/support/test_helpers.rb +7 -0
  37. data/tasks/distribution/configuration.rb +15 -0
  38. data/tasks/distribution/executable.rb +28 -0
  39. data/tasks/distribution/package.rb +85 -0
  40. data/tasks/distribution/package_helpers.rb +12 -0
  41. data/tasks/distribution/release.rb +49 -0
  42. data/tasks/distribution/release_notes.erb +14 -0
  43. data/tasks/distribution/release_notes.rb +62 -0
  44. data/tasks/distribution/tarball.rb +47 -0
  45. data/tasks/distribution/travelling_ruby.rb +87 -0
  46. data/tasks/package.rake +41 -0
  47. data/tasks/upload.rake +40 -0
  48. metadata +300 -26
@@ -0,0 +1,172 @@
1
+ module Datapimp
2
+ module Clients
3
+ class Amazon
4
+ include Singleton
5
+
6
+ require 'fog/aws'
7
+
8
+ def aws_access_key_id
9
+ options[:aws_access_key_id] || options[:access_key_id] || Datapimp.config.aws_access_key_id
10
+ end
11
+
12
+ def aws_secret_access_key
13
+ options[:aws_secret_access_key] || options[:secret_access_key] || Datapimp.config.aws_secret_access_key
14
+ end
15
+
16
+ def storage
17
+ return @storage if @storage
18
+
19
+ # Silence fog warnings
20
+ Fog::Logger.define_singleton_method(:warning) do |*args|
21
+ nil
22
+ end
23
+
24
+ @storage = Fog::Storage.new({
25
+ provider: 'AWS',
26
+ aws_access_key_id: aws_access_key_id,
27
+ aws_secret_access_key: aws_secret_access_key,
28
+ path_style: true
29
+ })
30
+
31
+
32
+ @storage
33
+ end
34
+
35
+ def cdn
36
+ @cdn ||= Fog::CDN.new({
37
+ provider: 'AWS',
38
+ aws_access_key_id: aws_access_key_id,
39
+ aws_secret_access_key: aws_secret_access_key
40
+ })
41
+ end
42
+
43
+ def s3_bucket_website_url
44
+ if s3_bucket.is_a?(Fog::Storage::AWS::Directory)
45
+ website_url_for(s3_bucket)
46
+ end
47
+ end
48
+
49
+ def site_description
50
+ options[:description] || options[:site_name] || s3_bucket.key
51
+ end
52
+
53
+ # the domain, and the domain with www
54
+ def site_domain_aliases
55
+ options[:aliases]
56
+ end
57
+
58
+ def s3_bucket
59
+ if bucket_name = options[:bucket_name] || Datapimp.config.get("bucket_name")
60
+ if bucket = find_bucket_by_name(bucket_name)
61
+ return bucket
62
+ else
63
+ "There is no bucket named: #{ bucket_name }. You can create one by running 'datapimp setup amazon --create-bucket=BUCKET_NAME"
64
+ end
65
+ else
66
+ raise 'Could not determine bucketname for Datapimp.amazon.s3_bucket'
67
+ end
68
+ end
69
+
70
+ def create_cdn_for(website_url, comment, aliases)
71
+ aliases = aliases.join(",") if aliases.is_a?(Array)
72
+
73
+ existing = cdn.distributions.find do |distribution|
74
+ distribution.comment == comment
75
+ end
76
+
77
+ return existing if existing
78
+
79
+ cdn.distributions.create(cdn_options(website_url: website_url, comment: comment, aliases: aliases))
80
+ end
81
+
82
+ def cdn_options(o={})
83
+ {
84
+ enabled: true,
85
+ custom_origin: {
86
+ 'DNSName'=> o.fetch(:website_url) { s3_bucket_website_url },
87
+ 'OriginProtocolPolicy'=>'http-only'
88
+ },
89
+ comment: o.fetch(:comment) { site_description },
90
+ caller_reference: Time.now.to_i.to_s,
91
+ cname: o.fetch(:aliases) { site_domain_aliases },
92
+ default_root_object: 'index.html'
93
+ }
94
+ end
95
+
96
+ def self.client(options={})
97
+ @client ||= begin
98
+ instance.with_options(options)
99
+ end
100
+ end
101
+
102
+
103
+ def self.method_missing(meth, *args, &block)
104
+ if client.respond_to?(meth)
105
+ return client.send(meth, *args, &block)
106
+ end
107
+
108
+ super
109
+ end
110
+
111
+ def website_host_for(bucket_or_bucket_name)
112
+ URI.parse(website_url_for(bucket_or_bucket_name)).host
113
+ end
114
+
115
+ def website_url_for(bucket_or_bucket_name)
116
+ bucket = bucket_or_bucket_name
117
+
118
+ if bucket_or_bucket_name.is_a?(String)
119
+ bucket = storage.directories.get(bucket_or_bucket_name)
120
+ end
121
+
122
+ if bucket
123
+ "http://#{bucket.key}.s3-website-#{ bucket.location }.amazonaws.com"
124
+ end
125
+ end
126
+
127
+ def find_or_create_bucket(bucket_name)
128
+ find_bucket_by_name(bucket_name) || create_bucket(bucket_name)
129
+ end
130
+
131
+ def find_bucket_by_name(bucket_name)
132
+ storage.directories.get(bucket_name) rescue nil
133
+ end
134
+
135
+ def create_bucket(bucket_name)
136
+ storage.directories.create(key: bucket_name, public: true).tap do |bucket|
137
+ storage.put_bucket_website(bucket_name, 'index.html', key: 'error.html')
138
+ #storage.put_bucket_cors(bucket_name, {"AllowedOrigin"=>"*","AllowedMethod"=>"GET","AllowedHeader"=>"Authorization"})
139
+ end
140
+ end
141
+
142
+ def create_redirect_bucket(bucket_name, redirect_to_bucket_name)
143
+ create_bucket(redirect_to_bucket_name) unless find_bucket_by_name(redirect_to_bucket_name)
144
+ create_bucket(bucket_name)
145
+ end
146
+
147
+ def with_options(opts={})
148
+ options.merge!(opts)
149
+ self
150
+ end
151
+
152
+ def options
153
+ @options ||= {}
154
+ end
155
+
156
+ def has_application_keys?
157
+ (Datapimp.config.aws_access_key_id.to_s.length > 0 && Datapimp.config.aws_secret_access_key.to_s.length > 0)
158
+ end
159
+
160
+ def interactive_setup(options={})
161
+ secret_key = Datapimp.config.aws_secret_access_key.to_s
162
+ access_key_id = Datapimp.config.aws_access_key_id.to_s
163
+
164
+ secret_key = ask("What is the AWS Secret Access Key?") unless secret_key.length > 8
165
+ access_key_id = ask("What is the AWS Access Key ID?") unless access_key_id.length > 8
166
+
167
+ Datapimp.config.set(:aws_access_key_id, access_key_id) if access_key_id.length > 8
168
+ Datapimp.config.set(:aws_secret_access_key, secret_key) if secret_key.length > 8
169
+ end
170
+ end
171
+ end
172
+ end
@@ -0,0 +1,178 @@
1
+ module Datapimp
2
+ module Clients
3
+ class Dropbox
4
+ include Singleton
5
+
6
+ def self.method_missing(meth, *args, &block)
7
+ if client.respond_to?(meth)
8
+ return client.send(meth, *args, &block)
9
+ end
10
+
11
+ super
12
+ end
13
+
14
+ def self.client(options={})
15
+ require 'dropbox-api' unless defined?(::Dropbox::API)
16
+ @client ||= begin
17
+ ::Dropbox::API::Config.app_key = options.fetch(:dropbox_app_key) { Datapimp.config.dropbox_app_key }
18
+ ::Dropbox::API::Config.app_secret = options.fetch(:dropbox_app_secret) { Datapimp.config.dropbox_app_secret }
19
+ ::Dropbox::API::Config.mode = options.fetch(:dropbox_app_type) { Datapimp.config.dropbox_app_type }
20
+ instance.with_options(options)
21
+ end
22
+ end
23
+
24
+ def sandboxed_app?
25
+ entry = api.ls.first
26
+ entry && entry.root == "app_folder"
27
+ end
28
+
29
+ def api
30
+ @api ||= begin
31
+ token = options.fetch(:token) { Datapimp.config.dropbox_client_token }
32
+ secret = options.fetch(:secret) { Datapimp.config.dropbox_client_secret }
33
+ ::Dropbox::API::Client.new(token: token , secret: secret)
34
+ end
35
+ end
36
+
37
+ def find(path)
38
+ api.find(path)
39
+ rescue ::Dropbox::API::Error::NotFound
40
+ nil
41
+ end
42
+
43
+ def push_from(local_path, options={})
44
+ path_prefix = options.fetch(:prefix)
45
+ root = options[:root]
46
+
47
+ unless find(path_prefix)
48
+ api.mkdir(path_prefix)
49
+ end
50
+
51
+ uploader = lambda do |node|
52
+ next if node.to_s == ".DS_Store"
53
+
54
+ if node.directory?
55
+ Array(node.children).each(&uploader)
56
+ elsif node.file?
57
+ relative = node.relative_path_from(local_path)
58
+ target = "#{path_prefix}/#{relative}"
59
+ target = target.gsub(/^\//,'')
60
+ api.upload(target, node.read)
61
+ end
62
+ end
63
+
64
+ Pathname(local_path).children.each(&uploader)
65
+ end
66
+
67
+ def create_site_folder(folder, allow_overwrite=false)
68
+ found = find(folder)
69
+
70
+ unless (!found || (found && !allow_overwrite))
71
+ api.mkdir("/#{folder}")
72
+ end
73
+ end
74
+
75
+ def sync(local_path, remote_path, options={})
76
+ sync_folders(local_path, remote_path, options)
77
+ end
78
+
79
+ # Instructs Datapimp to create a syncable folder
80
+ # between the local and remote path, and optionally
81
+ # modifies the middleman config for the site
82
+ def sync_folders(local_path, remote_path, options={})
83
+ app = options[:app]
84
+
85
+ folder = Datapimp::Sync::Folder.new(local_path: local_path,
86
+ remote_path: remote_path,
87
+ app: app).synced
88
+
89
+ if !!options[:append_config] == true
90
+ Datapimp.append_config(folder.config_line)
91
+ end
92
+
93
+ folder
94
+ end
95
+
96
+ def method_missing meth, *args, &block
97
+ if api.respond_to?(meth)
98
+ return api.send(meth, *args, &block)
99
+ end
100
+
101
+ super
102
+ end
103
+
104
+ def authorize(token, secret)
105
+ @api = nil if @api
106
+ options[:token] = token
107
+ options[:secret] = secret
108
+ self
109
+ end
110
+
111
+ def options
112
+ @options ||= {}
113
+ end
114
+
115
+ def with_options(opts={})
116
+ options.merge!(opts)
117
+ self
118
+ end
119
+
120
+ def requires_setup?
121
+ !(dropbox_app_key.length > 0 && dropbox_app_secret.length > 0)
122
+ end
123
+
124
+ def dropbox_app_key
125
+ Datapimp.config.dropbox_app_key.to_s
126
+ end
127
+
128
+ def dropbox_app_secret
129
+ Datapimp.config.dropbox_app_secret.to_s
130
+ end
131
+
132
+ def setup(options={})
133
+ interactive_setup(options)
134
+ end
135
+
136
+ def interactive_setup(options={})
137
+ if requires_setup?
138
+ if dropbox_app_key.length == 0
139
+ if answer = options[:dropbox_app_key] || ask("What is the dropbox app key?", String)
140
+ Datapimp.config.set("dropbox_app_key", answer)
141
+ end
142
+ if answer = options[:dropbox_app_secret] || ask("What is the dropbox app secret?", String)
143
+ Datapimp.config.set("dropbox_app_secret", answer)
144
+ end
145
+ end
146
+ end
147
+
148
+ raise 'Missing dropbox application values' if requires_setup?
149
+
150
+ ::Dropbox::API::Config.app_key = Datapimp.config.dropbox_app_key
151
+ ::Dropbox::API::Config.app_secret = Datapimp.config.dropbox_app_secret
152
+
153
+ consumer = ::Dropbox::API::OAuth.consumer(:authorize)
154
+ request_token = consumer.get_request_token
155
+ puts "\nGo to this url and click 'Authorize' to get the token:"
156
+ puts request_token.authorize_url
157
+ Launchy.open(request_token.authorize_url)
158
+
159
+ query = request_token.authorize_url.split('?').last
160
+ params = CGI.parse(query)
161
+ token = params['oauth_token'].first
162
+ print "\nOnce you authorize the app on Dropbox, press enter... "
163
+ STDIN.gets.chomp
164
+
165
+ access_token = request_token.get_access_token(:oauth_verifier => token)
166
+
167
+ Datapimp.config.set 'dropbox_client_token', access_token.token
168
+ Datapimp.config.set 'dropbox_client_secret', access_token.secret
169
+
170
+ puts "\nAuthorization complete!:\n\n"
171
+ puts " Dropbox::API::Config.app_key = '#{consumer.key}'"
172
+ puts " Dropbox::API::Config.app_secret = '#{consumer.secret}'"
173
+ puts " client = Dropbox::API::Client.new(:token => '#{access_token.token}', :secret => '#{access_token.secret}')"
174
+ puts "\n"
175
+ end
176
+ end
177
+ end
178
+ end
@@ -0,0 +1,59 @@
1
+ module Datapimp
2
+ module Clients
3
+ class Github
4
+ include Singleton
5
+
6
+ def self.method_missing(meth, *args, &block)
7
+ if client.respond_to?(meth)
8
+ return client.send(meth, *args, &block)
9
+ end
10
+
11
+ super
12
+ end
13
+
14
+ def self.client(options={})
15
+ require 'octokit' unless defined?(::Oktokit)
16
+
17
+ @client ||= begin
18
+ instance.with_options(options)
19
+ end
20
+ end
21
+
22
+ def options
23
+ @options ||= {}
24
+ end
25
+
26
+ def with_options(opts={})
27
+ options.merge!(opts)
28
+ self
29
+ end
30
+
31
+ def api
32
+ @api ||= begin
33
+ Octokit::Client.new(access_token: Datapimp.config.github_access_token)
34
+ end
35
+ end
36
+
37
+ def setup(options={})
38
+ access_token = options[:github_access_token] || Datapimp.config.github_access_token
39
+
40
+ unless access_token.to_s.length == 40
41
+ puts "You should generate an access token to use with the Github client."
42
+ puts "Access tokens allow you to revoke and/or limit access if needed."
43
+ puts "To learn more about access tokens, and how to generate them, visit: https://help.github.com/articles/creating-an-access-token-for-command-line-use/"
44
+
45
+ if respond_to?(:ask)
46
+ access_token = ask("Enter a 40 character access token when you have one", String)
47
+ end
48
+ end
49
+
50
+ unless access_token.to_s.length == 40
51
+ puts "Can not proceed without a valid access token: error code #{ access_token.length }"
52
+ return
53
+ end
54
+
55
+ Datapimp.config.set(:github_access_token, access_token)
56
+ end
57
+ end
58
+ end
59
+ end
@@ -0,0 +1,145 @@
1
+ module Datapimp
2
+ module Clients
3
+ class Google
4
+ include Singleton
5
+
6
+ def self.method_missing(meth, *args, &block)
7
+ if client.respond_to?(meth)
8
+ return client.send(meth, *args, &block)
9
+ end
10
+
11
+ super
12
+ end
13
+
14
+ def self.client(options={})
15
+ require 'google_drive' unless defined?(::GoogleDrive)
16
+
17
+ @client ||= begin
18
+ instance.with_options(options)
19
+ end
20
+ end
21
+
22
+ def refreshable?
23
+ has_application_keys? && has_refresh_token?
24
+ end
25
+
26
+ # Runs through an interactive session where we get the
27
+ # necessary tokens needed to integrate with google drive.
28
+ def setup(options={})
29
+ get_application_keys unless has_application_keys?
30
+
31
+ if options[:client_id]
32
+ Datapimp.config.set "google_client_id", options[:client_id]
33
+ end
34
+
35
+ if options[:client_secret]
36
+ Datapimp.config.set "google_client_secret", options[:client_secret]
37
+ end
38
+
39
+ if has_refresh_token?
40
+ refresh_access_token!
41
+ elsif respond_to?(:ask)
42
+ Launchy.open(auth_client.authorization_uri)
43
+ say("\n1. Open this page:\n%s\n\n" % auth_client.authorization_uri)
44
+ auth_client.code = ask("2. Enter the authorization code shown in the page: ", String)
45
+ auth_client.fetch_access_token!
46
+ Datapimp.config.set "google_refresh_token", auth_client.refresh_token
47
+ Datapimp.config.set "google_access_token", auth_client.access_token
48
+ end
49
+ end
50
+
51
+ def session
52
+ api
53
+ end
54
+
55
+ def api
56
+ @api ||= begin
57
+ refresh_access_token!
58
+ GoogleDrive.login_with_oauth(Datapimp.config.google_access_token)
59
+ end
60
+ end
61
+
62
+ def method_missing meth, *args, &block
63
+ if api.respond_to?(meth)
64
+ return api.send(meth, *args, &block)
65
+ end
66
+
67
+ super
68
+ end
69
+
70
+ def authorize(token, secret)
71
+ @api = nil if @api
72
+ options[:token] = token
73
+ options[:secret] = secret
74
+ self
75
+ end
76
+
77
+ def options
78
+ @options ||= {}
79
+ end
80
+
81
+ def with_options(opts={})
82
+ options.merge!(opts)
83
+ self
84
+ end
85
+
86
+ def has_application_keys?
87
+ (Datapimp.config.google_client_id.to_s.length > 0 && Datapimp.config.google_client_secret.to_s.length > 0)
88
+ end
89
+
90
+ def get_application_keys
91
+ unless Datapimp.config.google_client_id.to_s.length > 0
92
+ google_client_id = ask("What is the Google Client ID?", String)
93
+ Datapimp.config.set "google_client_id", google_client_id
94
+ end
95
+
96
+ unless Datapimp.config.google_client_secret.to_s.length > 0
97
+ google_client_secret = ask("What is the Google Client Secret?", String)
98
+ Datapimp.config.set "google_client_secret", google_client_secret
99
+ end
100
+ end
101
+
102
+ def auth_client
103
+ return @auth_client if @auth_client
104
+
105
+ client = ::Google::APIClient.new(
106
+ :application_name => "google_drive Ruby library",
107
+ :application_version => "0.3.11"
108
+ )
109
+
110
+ client_id = "452925651630-egr1f18o96acjjvphpbbd1qlsevkho1d.apps.googleusercontent.com"
111
+ client_secret = "1U3-Krii5x1oLPrwD5zgn-ry"
112
+
113
+ @auth_client = auth = client.authorization
114
+ auth.client_id = client_id #Datapimp.config.google_client_id
115
+ auth.client_secret = client_secret #Datapimp.config.google_client_secret
116
+ auth.scope =
117
+ "https://www.googleapis.com/auth/drive " +
118
+ "https://spreadsheets.google.com/feeds/ " +
119
+ "https://docs.google.com/feeds/ " +
120
+ "https://docs.googleusercontent.com/"
121
+
122
+ auth.redirect_uri = "urn:ietf:wg:oauth:2.0:oob"
123
+
124
+ auth
125
+ end
126
+
127
+ def refresh_token
128
+ Datapimp.config.google_refresh_token.to_s
129
+ end
130
+
131
+ def has_refresh_token?
132
+ refresh_token.length > 0
133
+ end
134
+
135
+ def refresh_access_token!
136
+ if has_refresh_token?
137
+ auth_client.refresh_token = refresh_token
138
+ auth_client.fetch_access_token!
139
+ Datapimp.config.set "google_access_token", auth_client.access_token
140
+ end
141
+ end
142
+
143
+ end
144
+ end
145
+ end