browse-everything 1.1.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.circleci/config.yml +37 -15
- data/.rubocop.yml +43 -35
- data/Gemfile +3 -6
- data/README.md +1 -0
- data/app/assets/javascripts/browse_everything/behavior.js +16 -8
- data/app/assets/javascripts/treetable.webpack.js +687 -0
- data/app/controllers/browse_everything_controller.rb +60 -60
- data/app/helpers/browse_everything_helper.rb +4 -0
- data/app/views/browse_everything/_files.html.erb +3 -2
- data/browse-everything.gemspec +2 -2
- data/lib/browse_everything/auth/google/credentials.rb +5 -5
- data/lib/browse_everything/auth/google/request_parameters.rb +38 -38
- data/lib/browse_everything/driver/base.rb +14 -14
- data/lib/browse_everything/driver/box.rb +56 -56
- data/lib/browse_everything/driver/dropbox.rb +39 -39
- data/lib/browse_everything/driver/file_system.rb +17 -17
- data/lib/browse_everything/driver/google_drive.rb +38 -38
- data/lib/browse_everything/driver/s3.rb +61 -61
- data/lib/browse_everything/file_entry.rb +1 -1
- data/lib/browse_everything/retriever.rb +69 -69
- data/lib/browse_everything/version.rb +1 -1
- data/spec/features/test_compiling_stylesheets_spec.rb +1 -1
- data/spec/lib/browse_everything/driver_spec.rb +43 -3
- data/spec/spec_helper.rb +1 -1
- data/spec/test_app_templates/Gemfile.extra +9 -0
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +54 -3
- metadata +13 -12
@@ -16,14 +16,14 @@ module BrowseEverything
|
|
16
16
|
class << self
|
17
17
|
private
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
19
|
+
def klass_for(metadata)
|
20
|
+
case metadata
|
21
|
+
when DropboxApi::Metadata::File
|
22
|
+
FileFactory
|
23
|
+
else
|
24
|
+
ResourceFactory
|
26
25
|
end
|
26
|
+
end
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
@@ -138,44 +138,44 @@ module BrowseEverything
|
|
138
138
|
|
139
139
|
private
|
140
140
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
141
|
+
def session
|
142
|
+
AuthenticationFactory.new(
|
143
|
+
self.class.authentication_klass,
|
144
|
+
config[:client_id],
|
145
|
+
config[:client_secret]
|
146
|
+
)
|
147
|
+
end
|
148
148
|
|
149
|
-
|
150
|
-
|
151
|
-
|
149
|
+
def authenticate
|
150
|
+
session.authenticate
|
151
|
+
end
|
152
152
|
|
153
|
-
|
154
|
-
|
155
|
-
|
153
|
+
def authenticator
|
154
|
+
@authenticator ||= authenticate
|
155
|
+
end
|
156
156
|
|
157
|
-
|
158
|
-
|
159
|
-
|
157
|
+
def client
|
158
|
+
DropboxApi::Client.new(token)
|
159
|
+
end
|
160
160
|
|
161
|
-
|
162
|
-
|
163
|
-
|
161
|
+
def redirect_uri(url_options)
|
162
|
+
connector_response_url(**url_options)
|
163
|
+
end
|
164
164
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
165
|
+
# Ensures that the "tmp" directory is used if there is no default download
|
166
|
+
# directory specified in the configuration
|
167
|
+
# @return [String]
|
168
|
+
def default_download_directory
|
169
|
+
Rails.root.join('tmp')
|
170
|
+
end
|
171
171
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
172
|
+
# Retrieves the directory path for downloads used when retrieving the
|
173
|
+
# resource from Dropbox
|
174
|
+
# @return [String]
|
175
|
+
def download_directory_path
|
176
|
+
dir_path = config[:download_directory] || default_download_directory
|
177
|
+
File.expand_path(dir_path)
|
178
|
+
end
|
179
179
|
end
|
180
180
|
end
|
181
181
|
end
|
@@ -55,25 +55,25 @@ module BrowseEverything
|
|
55
55
|
|
56
56
|
private
|
57
57
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
58
|
+
# Construct an array of FileEntry objects for the contents of a
|
59
|
+
# directory
|
60
|
+
# @param real_path [String] path to the file system directory
|
61
|
+
# @return [Array<BrowseEverything::FileEntry>]
|
62
|
+
def make_directory_entry(real_path)
|
63
|
+
entries = []
|
64
|
+
entries + Dir[File.join(real_path, '*')].collect { |f| details(f) }
|
65
|
+
end
|
66
66
|
|
67
|
-
|
68
|
-
|
69
|
-
|
67
|
+
def make_pathname(path)
|
68
|
+
Pathname.new(File.expand_path(path)).relative_path_from(Pathname.new(config[:home]))
|
69
|
+
end
|
70
70
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
71
|
+
def file_size(path)
|
72
|
+
File.size(path).to_i
|
73
|
+
rescue StandardError => error
|
74
|
+
Rails.logger.error "Failed to find the file size for #{path}: #{error}"
|
75
|
+
0
|
76
|
+
end
|
77
77
|
end
|
78
78
|
end
|
79
79
|
end
|
@@ -191,50 +191,50 @@ module BrowseEverything
|
|
191
191
|
|
192
192
|
private
|
193
193
|
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
}
|
194
|
+
def client_secrets
|
195
|
+
{
|
196
|
+
Google::Auth::ClientId::WEB_APP => {
|
197
|
+
Google::Auth::ClientId::CLIENT_ID => config[:client_id],
|
198
|
+
Google::Auth::ClientId::CLIENT_SECRET => config[:client_secret]
|
200
199
|
}
|
201
|
-
|
200
|
+
}
|
201
|
+
end
|
202
202
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
203
|
+
# This is required for using the googleauth Gem
|
204
|
+
# @see http://www.rubydoc.info/gems/googleauth/Google/Auth/Stores/FileTokenStore FileTokenStore for googleauth
|
205
|
+
# @return [Tempfile] temporary file within which to cache credentials
|
206
|
+
def file_token_store_path
|
207
|
+
Tempfile.new('gdrive.yaml')
|
208
|
+
end
|
209
209
|
|
210
|
-
|
211
|
-
|
212
|
-
|
210
|
+
def scope
|
211
|
+
Google::Apis::DriveV3::AUTH_DRIVE
|
212
|
+
end
|
213
213
|
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
214
|
+
# Provides the user ID for caching access tokens
|
215
|
+
# (This is a hack which attempts to anonymize the access tokens)
|
216
|
+
# @return [String] the ID for the user
|
217
|
+
def user_id
|
218
|
+
'current_user'
|
219
|
+
end
|
220
220
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
221
|
+
# Please see https://developers.google.com/drive/v3/web/manage-downloads
|
222
|
+
# @param id [String] the ID for the Google Drive File
|
223
|
+
# @return [String] the URL for the file download
|
224
|
+
def download_url(id)
|
225
|
+
"https://www.googleapis.com/drive/v3/files/#{id}?alt=media"
|
226
|
+
end
|
227
227
|
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
228
|
+
# Restore the credentials for the Google API
|
229
|
+
# @param access_token [String] the access token redeemed using an authorization code
|
230
|
+
# @return Credentials credentials restored from a cached access token
|
231
|
+
def restore_credentials(access_token)
|
232
|
+
client = Auth::Google::Credentials.new
|
233
|
+
client.client_id = client_id.id
|
234
|
+
client.client_secret = client_id.secret
|
235
|
+
client.update_token!('access_token' => access_token)
|
236
|
+
@credentials = client
|
237
|
+
end
|
238
238
|
end
|
239
239
|
end
|
240
240
|
end
|
@@ -79,79 +79,79 @@ module BrowseEverything
|
|
79
79
|
|
80
80
|
private
|
81
81
|
|
82
|
-
|
83
|
-
|
84
|
-
|
82
|
+
def strip(path)
|
83
|
+
path.sub %r{^/?(.+?)/?$}, '\1'
|
84
|
+
end
|
85
85
|
|
86
|
-
|
87
|
-
|
88
|
-
|
86
|
+
def from_base(key)
|
87
|
+
Pathname.new(key).relative_path_from(Pathname.new(config[:base].to_s)).to_s
|
88
|
+
end
|
89
89
|
|
90
|
-
|
91
|
-
|
92
|
-
|
90
|
+
def full_path(path)
|
91
|
+
config[:base].present? ? File.join(config[:base], path) : path
|
92
|
+
end
|
93
93
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
94
|
+
def aws_config
|
95
|
+
result = {}
|
96
|
+
result[:credentials] = Aws::Credentials.new(config[:app_key], config[:app_secret]) if config[:app_key].present?
|
97
|
+
result[:region] = config[:region] if config.key?(:region)
|
98
|
+
result
|
99
|
+
end
|
100
100
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
101
|
+
def session
|
102
|
+
AuthenticationFactory.new(
|
103
|
+
self.class.authentication_klass,
|
104
|
+
aws_config
|
105
|
+
)
|
106
|
+
end
|
107
107
|
|
108
|
-
|
109
|
-
|
110
|
-
|
108
|
+
def authenticate
|
109
|
+
session.authenticate
|
110
|
+
end
|
111
111
|
|
112
|
-
|
113
|
-
|
114
|
-
|
112
|
+
def client
|
113
|
+
@client ||= authenticate
|
114
|
+
end
|
115
115
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
116
|
+
# Construct a BrowseEverything::FileEntry object
|
117
|
+
# @param name [String]
|
118
|
+
# @param size [String]
|
119
|
+
# @param date [DateTime]
|
120
|
+
# @param dir [String]
|
121
|
+
# @return [BrowseEverything::FileEntry]
|
122
|
+
def entry_for(name, size, date, dir)
|
123
|
+
BrowseEverything::FileEntry.new(name, [key, name].join(':'), File.basename(name), size, date, dir)
|
124
|
+
end
|
125
125
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
end
|
126
|
+
# Populate the entries with FileEntry objects from an S3 listing
|
127
|
+
# @param listing [Seahorse::Client::Response]
|
128
|
+
def add_directories(listing)
|
129
|
+
listing.common_prefixes.each do |prefix|
|
130
|
+
new_entry = entry_for(from_base(prefix.prefix), 0, Time.current, true)
|
131
|
+
@entries << new_entry unless new_entry.nil?
|
133
132
|
end
|
133
|
+
end
|
134
134
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
end
|
135
|
+
# Given a listing and a S3 listing and path, populate the entries
|
136
|
+
# @param listing [Seahorse::Client::Response]
|
137
|
+
# @param path [String]
|
138
|
+
def add_files(listing, path)
|
139
|
+
listing.contents.each do |entry|
|
140
|
+
key = from_base(entry.key)
|
141
|
+
new_entry = entry_for(key, entry.size, entry.last_modified, false)
|
142
|
+
@entries << new_entry unless strip(key) == strip(path) || new_entry.nil?
|
144
143
|
end
|
144
|
+
end
|
145
145
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
146
|
+
# For a given path to a S3 resource, retrieve the listing object and
|
147
|
+
# construct the file entries
|
148
|
+
# @param path [String]
|
149
|
+
def generate_listing(path)
|
150
|
+
client
|
151
|
+
listing = client.list_objects(bucket: config[:bucket], delimiter: '/', prefix: full_path(path))
|
152
|
+
add_directories(listing)
|
153
|
+
add_files(listing, path)
|
154
|
+
end
|
155
155
|
end
|
156
156
|
end
|
157
157
|
end
|
@@ -89,81 +89,81 @@ module BrowseEverything
|
|
89
89
|
|
90
90
|
private
|
91
91
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
# This avoids the potential for a KeyError
|
99
|
-
headers = options.fetch('headers', {}) || {}
|
100
|
-
|
101
|
-
file_size_value = options.fetch('file_size', 0)
|
102
|
-
file_size = file_size_value.to_i
|
103
|
-
|
104
|
-
output = {
|
105
|
-
url: ::Addressable::URI.parse(url),
|
106
|
-
headers: headers,
|
107
|
-
file_size: file_size
|
108
|
-
}
|
109
|
-
|
110
|
-
output[:file_size] = get_file_size(output) if output[:file_size] < 1
|
111
|
-
output
|
112
|
-
end
|
92
|
+
# Extract and parse options used to download a file or resource from an HTTP API
|
93
|
+
# @param options [Hash]
|
94
|
+
# @return [Hash]
|
95
|
+
def extract_download_options(options)
|
96
|
+
url = options.fetch('url')
|
113
97
|
|
114
|
-
#
|
115
|
-
|
116
|
-
def retrieve_file(options)
|
117
|
-
file_uri = options.fetch(:url)
|
118
|
-
file_size = options.fetch(:file_size)
|
119
|
-
|
120
|
-
retrieved = 0
|
121
|
-
File.open(file_uri.path, 'rb') do |f|
|
122
|
-
until f.eof?
|
123
|
-
chunk = f.read(chunk_size)
|
124
|
-
retrieved += chunk.length
|
125
|
-
yield(chunk, retrieved, file_size)
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|
98
|
+
# This avoids the potential for a KeyError
|
99
|
+
headers = options.fetch('headers', {}) || {}
|
129
100
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
101
|
+
file_size_value = options.fetch('file_size', 0)
|
102
|
+
file_size = file_size_value.to_i
|
103
|
+
|
104
|
+
output = {
|
105
|
+
url: ::Addressable::URI.parse(url),
|
106
|
+
headers: headers,
|
107
|
+
file_size: file_size
|
108
|
+
}
|
109
|
+
|
110
|
+
output[:file_size] = get_file_size(output) if output[:file_size] < 1
|
111
|
+
output
|
112
|
+
end
|
113
|
+
|
114
|
+
# Retrieve the file from the file system
|
115
|
+
# @param options [Hash]
|
116
|
+
def retrieve_file(options)
|
117
|
+
file_uri = options.fetch(:url)
|
118
|
+
file_size = options.fetch(:file_size)
|
119
|
+
|
120
|
+
retrieved = 0
|
121
|
+
File.open(file_uri.path, 'rb') do |f|
|
122
|
+
until f.eof?
|
123
|
+
chunk = f.read(chunk_size)
|
124
|
+
retrieved += chunk.length
|
144
125
|
yield(chunk, retrieved, file_size)
|
145
126
|
end
|
146
|
-
request.run
|
147
127
|
end
|
128
|
+
end
|
148
129
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
when /https?/
|
161
|
-
response = Typhoeus.head(url.to_s, headers: headers)
|
162
|
-
length_value = response.headers['Content-Length'] || file_size
|
163
|
-
length_value.to_i
|
164
|
-
else
|
165
|
-
raise URI::BadURIError, "Unknown URI scheme: #{url.scheme}"
|
166
|
-
end
|
130
|
+
# Retrieve a resource over the HTTP
|
131
|
+
# @param options [Hash]
|
132
|
+
def retrieve_http(options)
|
133
|
+
file_size = options.fetch(:file_size)
|
134
|
+
headers = options.fetch(:headers)
|
135
|
+
url = options.fetch(:url)
|
136
|
+
retrieved = 0
|
137
|
+
|
138
|
+
request = Typhoeus::Request.new(url.to_s, method: :get, headers: headers)
|
139
|
+
request.on_headers do |response|
|
140
|
+
raise DownloadError.new("#{self.class}: Failed to download #{url}: Status Code: #{response.code}", response) unless response.code == 200
|
167
141
|
end
|
142
|
+
request.on_body do |chunk|
|
143
|
+
retrieved += chunk.bytesize
|
144
|
+
yield(chunk, retrieved, file_size)
|
145
|
+
end
|
146
|
+
request.run
|
147
|
+
end
|
148
|
+
|
149
|
+
# Retrieve the file size
|
150
|
+
# @param options [Hash]
|
151
|
+
# @return [Integer] the size of the requested file
|
152
|
+
def get_file_size(options)
|
153
|
+
url = options.fetch(:url)
|
154
|
+
headers = options.fetch(:headers)
|
155
|
+
file_size = options.fetch(:file_size)
|
156
|
+
|
157
|
+
case url.scheme
|
158
|
+
when 'file'
|
159
|
+
File.size(url.path)
|
160
|
+
when /https?/
|
161
|
+
response = Typhoeus.head(url.to_s, headers: headers)
|
162
|
+
length_value = response.headers['Content-Length'] || file_size
|
163
|
+
length_value.to_i
|
164
|
+
else
|
165
|
+
raise URI::BadURIError, "Unknown URI scheme: #{url.scheme}"
|
166
|
+
end
|
167
|
+
end
|
168
168
|
end
|
169
169
|
end
|