browse-everything 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +37 -15
- data/.rubocop.yml +43 -35
- data/Gemfile +3 -6
- data/README.md +1 -0
- data/app/assets/javascripts/browse_everything/behavior.js +16 -8
- data/app/assets/javascripts/treetable.webpack.js +687 -0
- data/app/controllers/browse_everything_controller.rb +60 -60
- data/app/helpers/browse_everything_helper.rb +4 -0
- data/app/views/browse_everything/_files.html.erb +3 -2
- data/browse-everything.gemspec +2 -2
- data/lib/browse_everything/auth/google/credentials.rb +5 -5
- data/lib/browse_everything/auth/google/request_parameters.rb +38 -38
- data/lib/browse_everything/driver/base.rb +14 -14
- data/lib/browse_everything/driver/box.rb +56 -56
- data/lib/browse_everything/driver/dropbox.rb +39 -39
- data/lib/browse_everything/driver/file_system.rb +17 -17
- data/lib/browse_everything/driver/google_drive.rb +38 -38
- data/lib/browse_everything/driver/s3.rb +61 -61
- data/lib/browse_everything/file_entry.rb +1 -1
- data/lib/browse_everything/retriever.rb +69 -69
- data/lib/browse_everything/version.rb +1 -1
- data/spec/features/test_compiling_stylesheets_spec.rb +1 -1
- data/spec/lib/browse_everything/driver_spec.rb +43 -3
- data/spec/spec_helper.rb +1 -1
- data/spec/test_app_templates/Gemfile.extra +9 -0
- data/spec/test_app_templates/lib/generators/test_app_generator.rb +54 -3
- metadata +13 -12
@@ -16,14 +16,14 @@ module BrowseEverything
|
|
16
16
|
class << self
|
17
17
|
private
|
18
18
|
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
end
|
19
|
+
def klass_for(metadata)
|
20
|
+
case metadata
|
21
|
+
when DropboxApi::Metadata::File
|
22
|
+
FileFactory
|
23
|
+
else
|
24
|
+
ResourceFactory
|
26
25
|
end
|
26
|
+
end
|
27
27
|
end
|
28
28
|
end
|
29
29
|
|
@@ -138,44 +138,44 @@ module BrowseEverything
|
|
138
138
|
|
139
139
|
private
|
140
140
|
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
141
|
+
def session
|
142
|
+
AuthenticationFactory.new(
|
143
|
+
self.class.authentication_klass,
|
144
|
+
config[:client_id],
|
145
|
+
config[:client_secret]
|
146
|
+
)
|
147
|
+
end
|
148
148
|
|
149
|
-
|
150
|
-
|
151
|
-
|
149
|
+
def authenticate
|
150
|
+
session.authenticate
|
151
|
+
end
|
152
152
|
|
153
|
-
|
154
|
-
|
155
|
-
|
153
|
+
def authenticator
|
154
|
+
@authenticator ||= authenticate
|
155
|
+
end
|
156
156
|
|
157
|
-
|
158
|
-
|
159
|
-
|
157
|
+
def client
|
158
|
+
DropboxApi::Client.new(token)
|
159
|
+
end
|
160
160
|
|
161
|
-
|
162
|
-
|
163
|
-
|
161
|
+
def redirect_uri(url_options)
|
162
|
+
connector_response_url(**url_options)
|
163
|
+
end
|
164
164
|
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
165
|
+
# Ensures that the "tmp" directory is used if there is no default download
|
166
|
+
# directory specified in the configuration
|
167
|
+
# @return [String]
|
168
|
+
def default_download_directory
|
169
|
+
Rails.root.join('tmp')
|
170
|
+
end
|
171
171
|
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
172
|
+
# Retrieves the directory path for downloads used when retrieving the
|
173
|
+
# resource from Dropbox
|
174
|
+
# @return [String]
|
175
|
+
def download_directory_path
|
176
|
+
dir_path = config[:download_directory] || default_download_directory
|
177
|
+
File.expand_path(dir_path)
|
178
|
+
end
|
179
179
|
end
|
180
180
|
end
|
181
181
|
end
|
@@ -55,25 +55,25 @@ module BrowseEverything
|
|
55
55
|
|
56
56
|
private
|
57
57
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
58
|
+
# Construct an array of FileEntry objects for the contents of a
|
59
|
+
# directory
|
60
|
+
# @param real_path [String] path to the file system directory
|
61
|
+
# @return [Array<BrowseEverything::FileEntry>]
|
62
|
+
def make_directory_entry(real_path)
|
63
|
+
entries = []
|
64
|
+
entries + Dir[File.join(real_path, '*')].collect { |f| details(f) }
|
65
|
+
end
|
66
66
|
|
67
|
-
|
68
|
-
|
69
|
-
|
67
|
+
def make_pathname(path)
|
68
|
+
Pathname.new(File.expand_path(path)).relative_path_from(Pathname.new(config[:home]))
|
69
|
+
end
|
70
70
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
71
|
+
def file_size(path)
|
72
|
+
File.size(path).to_i
|
73
|
+
rescue StandardError => error
|
74
|
+
Rails.logger.error "Failed to find the file size for #{path}: #{error}"
|
75
|
+
0
|
76
|
+
end
|
77
77
|
end
|
78
78
|
end
|
79
79
|
end
|
@@ -191,50 +191,50 @@ module BrowseEverything
|
|
191
191
|
|
192
192
|
private
|
193
193
|
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
}
|
194
|
+
def client_secrets
|
195
|
+
{
|
196
|
+
Google::Auth::ClientId::WEB_APP => {
|
197
|
+
Google::Auth::ClientId::CLIENT_ID => config[:client_id],
|
198
|
+
Google::Auth::ClientId::CLIENT_SECRET => config[:client_secret]
|
200
199
|
}
|
201
|
-
|
200
|
+
}
|
201
|
+
end
|
202
202
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
203
|
+
# This is required for using the googleauth Gem
|
204
|
+
# @see http://www.rubydoc.info/gems/googleauth/Google/Auth/Stores/FileTokenStore FileTokenStore for googleauth
|
205
|
+
# @return [Tempfile] temporary file within which to cache credentials
|
206
|
+
def file_token_store_path
|
207
|
+
Tempfile.new('gdrive.yaml')
|
208
|
+
end
|
209
209
|
|
210
|
-
|
211
|
-
|
212
|
-
|
210
|
+
def scope
|
211
|
+
Google::Apis::DriveV3::AUTH_DRIVE
|
212
|
+
end
|
213
213
|
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
214
|
+
# Provides the user ID for caching access tokens
|
215
|
+
# (This is a hack which attempts to anonymize the access tokens)
|
216
|
+
# @return [String] the ID for the user
|
217
|
+
def user_id
|
218
|
+
'current_user'
|
219
|
+
end
|
220
220
|
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
221
|
+
# Please see https://developers.google.com/drive/v3/web/manage-downloads
|
222
|
+
# @param id [String] the ID for the Google Drive File
|
223
|
+
# @return [String] the URL for the file download
|
224
|
+
def download_url(id)
|
225
|
+
"https://www.googleapis.com/drive/v3/files/#{id}?alt=media"
|
226
|
+
end
|
227
227
|
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
228
|
+
# Restore the credentials for the Google API
|
229
|
+
# @param access_token [String] the access token redeemed using an authorization code
|
230
|
+
# @return Credentials credentials restored from a cached access token
|
231
|
+
def restore_credentials(access_token)
|
232
|
+
client = Auth::Google::Credentials.new
|
233
|
+
client.client_id = client_id.id
|
234
|
+
client.client_secret = client_id.secret
|
235
|
+
client.update_token!('access_token' => access_token)
|
236
|
+
@credentials = client
|
237
|
+
end
|
238
238
|
end
|
239
239
|
end
|
240
240
|
end
|
@@ -79,79 +79,79 @@ module BrowseEverything
|
|
79
79
|
|
80
80
|
private
|
81
81
|
|
82
|
-
|
83
|
-
|
84
|
-
|
82
|
+
def strip(path)
|
83
|
+
path.sub %r{^/?(.+?)/?$}, '\1'
|
84
|
+
end
|
85
85
|
|
86
|
-
|
87
|
-
|
88
|
-
|
86
|
+
def from_base(key)
|
87
|
+
Pathname.new(key).relative_path_from(Pathname.new(config[:base].to_s)).to_s
|
88
|
+
end
|
89
89
|
|
90
|
-
|
91
|
-
|
92
|
-
|
90
|
+
def full_path(path)
|
91
|
+
config[:base].present? ? File.join(config[:base], path) : path
|
92
|
+
end
|
93
93
|
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
94
|
+
def aws_config
|
95
|
+
result = {}
|
96
|
+
result[:credentials] = Aws::Credentials.new(config[:app_key], config[:app_secret]) if config[:app_key].present?
|
97
|
+
result[:region] = config[:region] if config.key?(:region)
|
98
|
+
result
|
99
|
+
end
|
100
100
|
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
101
|
+
def session
|
102
|
+
AuthenticationFactory.new(
|
103
|
+
self.class.authentication_klass,
|
104
|
+
aws_config
|
105
|
+
)
|
106
|
+
end
|
107
107
|
|
108
|
-
|
109
|
-
|
110
|
-
|
108
|
+
def authenticate
|
109
|
+
session.authenticate
|
110
|
+
end
|
111
111
|
|
112
|
-
|
113
|
-
|
114
|
-
|
112
|
+
def client
|
113
|
+
@client ||= authenticate
|
114
|
+
end
|
115
115
|
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
116
|
+
# Construct a BrowseEverything::FileEntry object
|
117
|
+
# @param name [String]
|
118
|
+
# @param size [String]
|
119
|
+
# @param date [DateTime]
|
120
|
+
# @param dir [String]
|
121
|
+
# @return [BrowseEverything::FileEntry]
|
122
|
+
def entry_for(name, size, date, dir)
|
123
|
+
BrowseEverything::FileEntry.new(name, [key, name].join(':'), File.basename(name), size, date, dir)
|
124
|
+
end
|
125
125
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
end
|
126
|
+
# Populate the entries with FileEntry objects from an S3 listing
|
127
|
+
# @param listing [Seahorse::Client::Response]
|
128
|
+
def add_directories(listing)
|
129
|
+
listing.common_prefixes.each do |prefix|
|
130
|
+
new_entry = entry_for(from_base(prefix.prefix), 0, Time.current, true)
|
131
|
+
@entries << new_entry unless new_entry.nil?
|
133
132
|
end
|
133
|
+
end
|
134
134
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
end
|
135
|
+
# Given a listing and a S3 listing and path, populate the entries
|
136
|
+
# @param listing [Seahorse::Client::Response]
|
137
|
+
# @param path [String]
|
138
|
+
def add_files(listing, path)
|
139
|
+
listing.contents.each do |entry|
|
140
|
+
key = from_base(entry.key)
|
141
|
+
new_entry = entry_for(key, entry.size, entry.last_modified, false)
|
142
|
+
@entries << new_entry unless strip(key) == strip(path) || new_entry.nil?
|
144
143
|
end
|
144
|
+
end
|
145
145
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
146
|
+
# For a given path to a S3 resource, retrieve the listing object and
|
147
|
+
# construct the file entries
|
148
|
+
# @param path [String]
|
149
|
+
def generate_listing(path)
|
150
|
+
client
|
151
|
+
listing = client.list_objects(bucket: config[:bucket], delimiter: '/', prefix: full_path(path))
|
152
|
+
add_directories(listing)
|
153
|
+
add_files(listing, path)
|
154
|
+
end
|
155
155
|
end
|
156
156
|
end
|
157
157
|
end
|
@@ -89,81 +89,81 @@ module BrowseEverything
|
|
89
89
|
|
90
90
|
private
|
91
91
|
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
# This avoids the potential for a KeyError
|
99
|
-
headers = options.fetch('headers', {}) || {}
|
100
|
-
|
101
|
-
file_size_value = options.fetch('file_size', 0)
|
102
|
-
file_size = file_size_value.to_i
|
103
|
-
|
104
|
-
output = {
|
105
|
-
url: ::Addressable::URI.parse(url),
|
106
|
-
headers: headers,
|
107
|
-
file_size: file_size
|
108
|
-
}
|
109
|
-
|
110
|
-
output[:file_size] = get_file_size(output) if output[:file_size] < 1
|
111
|
-
output
|
112
|
-
end
|
92
|
+
# Extract and parse options used to download a file or resource from an HTTP API
|
93
|
+
# @param options [Hash]
|
94
|
+
# @return [Hash]
|
95
|
+
def extract_download_options(options)
|
96
|
+
url = options.fetch('url')
|
113
97
|
|
114
|
-
#
|
115
|
-
|
116
|
-
def retrieve_file(options)
|
117
|
-
file_uri = options.fetch(:url)
|
118
|
-
file_size = options.fetch(:file_size)
|
119
|
-
|
120
|
-
retrieved = 0
|
121
|
-
File.open(file_uri.path, 'rb') do |f|
|
122
|
-
until f.eof?
|
123
|
-
chunk = f.read(chunk_size)
|
124
|
-
retrieved += chunk.length
|
125
|
-
yield(chunk, retrieved, file_size)
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|
98
|
+
# This avoids the potential for a KeyError
|
99
|
+
headers = options.fetch('headers', {}) || {}
|
129
100
|
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
101
|
+
file_size_value = options.fetch('file_size', 0)
|
102
|
+
file_size = file_size_value.to_i
|
103
|
+
|
104
|
+
output = {
|
105
|
+
url: ::Addressable::URI.parse(url),
|
106
|
+
headers: headers,
|
107
|
+
file_size: file_size
|
108
|
+
}
|
109
|
+
|
110
|
+
output[:file_size] = get_file_size(output) if output[:file_size] < 1
|
111
|
+
output
|
112
|
+
end
|
113
|
+
|
114
|
+
# Retrieve the file from the file system
|
115
|
+
# @param options [Hash]
|
116
|
+
def retrieve_file(options)
|
117
|
+
file_uri = options.fetch(:url)
|
118
|
+
file_size = options.fetch(:file_size)
|
119
|
+
|
120
|
+
retrieved = 0
|
121
|
+
File.open(file_uri.path, 'rb') do |f|
|
122
|
+
until f.eof?
|
123
|
+
chunk = f.read(chunk_size)
|
124
|
+
retrieved += chunk.length
|
144
125
|
yield(chunk, retrieved, file_size)
|
145
126
|
end
|
146
|
-
request.run
|
147
127
|
end
|
128
|
+
end
|
148
129
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
when /https?/
|
161
|
-
response = Typhoeus.head(url.to_s, headers: headers)
|
162
|
-
length_value = response.headers['Content-Length'] || file_size
|
163
|
-
length_value.to_i
|
164
|
-
else
|
165
|
-
raise URI::BadURIError, "Unknown URI scheme: #{url.scheme}"
|
166
|
-
end
|
130
|
+
# Retrieve a resource over the HTTP
|
131
|
+
# @param options [Hash]
|
132
|
+
def retrieve_http(options)
|
133
|
+
file_size = options.fetch(:file_size)
|
134
|
+
headers = options.fetch(:headers)
|
135
|
+
url = options.fetch(:url)
|
136
|
+
retrieved = 0
|
137
|
+
|
138
|
+
request = Typhoeus::Request.new(url.to_s, method: :get, headers: headers)
|
139
|
+
request.on_headers do |response|
|
140
|
+
raise DownloadError.new("#{self.class}: Failed to download #{url}: Status Code: #{response.code}", response) unless response.code == 200
|
167
141
|
end
|
142
|
+
request.on_body do |chunk|
|
143
|
+
retrieved += chunk.bytesize
|
144
|
+
yield(chunk, retrieved, file_size)
|
145
|
+
end
|
146
|
+
request.run
|
147
|
+
end
|
148
|
+
|
149
|
+
# Retrieve the file size
|
150
|
+
# @param options [Hash]
|
151
|
+
# @return [Integer] the size of the requested file
|
152
|
+
def get_file_size(options)
|
153
|
+
url = options.fetch(:url)
|
154
|
+
headers = options.fetch(:headers)
|
155
|
+
file_size = options.fetch(:file_size)
|
156
|
+
|
157
|
+
case url.scheme
|
158
|
+
when 'file'
|
159
|
+
File.size(url.path)
|
160
|
+
when /https?/
|
161
|
+
response = Typhoeus.head(url.to_s, headers: headers)
|
162
|
+
length_value = response.headers['Content-Length'] || file_size
|
163
|
+
length_value.to_i
|
164
|
+
else
|
165
|
+
raise URI::BadURIError, "Unknown URI scheme: #{url.scheme}"
|
166
|
+
end
|
167
|
+
end
|
168
168
|
end
|
169
169
|
end
|