filestack 2.6.7 → 2.9.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.travis.yml +3 -0
- data/CHANGELOG.md +15 -0
- data/README.md +65 -10
- data/VERSION +1 -1
- data/filestack-ruby.gemspec +1 -1
- data/lib/filestack/config.rb +21 -6
- data/lib/filestack/models/filestack_client.rb +12 -17
- data/lib/filestack/ruby/version.rb +1 -1
- data/lib/filestack/utils/multipart_upload_utils.rb +65 -63
- data/lib/filestack/utils/utils.rb +58 -45
- metadata +8 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: c8df62167708408191c6712610f423f11ca60cf7fc8efd6210c9c1c5ae675468
|
4
|
+
data.tar.gz: 9f1699ac9e3850e932e44a09883397b5ebea98546e5bca835de90526555086d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a5808e35c807e81789324f38f59385bc2a96cb539b22fb6c083b5f2ffe99e0a0eaf443e25d8b5ed9371838461791991c66d95e0db22c94dc6e848ebd386489e0
|
7
|
+
data.tar.gz: b24184548151630522ced35dda7c645e382f241fe276732f92c0fea16cb62ec906d5ed7618476546ebea864feab6f35de90f344387abc73c95455fddd5db3588
|
data/.travis.yml
CHANGED
@@ -9,6 +9,9 @@ rvm:
|
|
9
9
|
before_install:
|
10
10
|
- gem uninstall -v '>= 2' -i $(rvm gemdir)@global -ax bundler || true
|
11
11
|
- gem install bundler -v '< 2'
|
12
|
+
- sudo apt-get update -y
|
13
|
+
- sudo apt-get install -y shared-mime-info
|
14
|
+
- mkdir -p ./usr/share/mime/packages && cp -a /usr/share/mime/packages/freedesktop.org.xml ./usr/share/mime/packages/
|
12
15
|
|
13
16
|
after_success:
|
14
17
|
- coveralls
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,20 @@
|
|
1
1
|
# Filestack-Ruby Changelog
|
2
2
|
|
3
|
+
## 2.9.1 (March 26, 2021)
|
4
|
+
- Update version of mimemagic gem to 0.3.9
|
5
|
+
|
6
|
+
## 2.9.0 (March 25, 2021)
|
7
|
+
- Update version of mimemagic gem to 0.3.8
|
8
|
+
|
9
|
+
## 2.8.1 (October 1, 2020)
|
10
|
+
- Add rescue and raises body message on upload
|
11
|
+
|
12
|
+
## 2.8.0 (September 29, 2020)
|
13
|
+
- Add IO object upload
|
14
|
+
|
15
|
+
## 2.7.0 (September 28, 2020)
|
16
|
+
- Add workflows
|
17
|
+
|
3
18
|
## 2.6.7 (August 4, 2020)
|
4
19
|
- Add content disposition task
|
5
20
|
|
data/README.md
CHANGED
@@ -37,6 +37,19 @@ Or install it yourself as:
|
|
37
37
|
|
38
38
|
$ gem install filestack
|
39
39
|
|
40
|
+
## Dependencies
|
41
|
+
|
42
|
+
We use the gem `mimemagic` which requires a copy of the Freedesktop.org shared-mime-info database.
|
43
|
+
|
44
|
+
macOS users can install the database via Homebrew with `brew install shared-mime-info`.
|
45
|
+
|
46
|
+
If you are unable to use a package manager, you can obtain a copy of the needed file by extracting it from the Debian package. This process will also work on a Windows machine.
|
47
|
+
|
48
|
+
1. Download the package from https://packages.debian.org/sid/amd64/shared-mime-info/download
|
49
|
+
2. Ensure the command line version of 7-Zip is installed
|
50
|
+
3. `7z x -so shared-mime-info_2.0-1_amd64.deb data.tar | 7z e -sidata.tar "./usr/share/mime/packages/freedesktop.org.xml"`
|
51
|
+
4. Place the file `freedesktop.org.xml` in an appropriate location, and then set the environment variable `FREEDESKTOP_MIME_TYPES_PATH` to that path. Once that has been done the gem should install successfully. Please note that the gem will depend upon the file remaining in that location at run time.
|
52
|
+
|
40
53
|
## Usage
|
41
54
|
|
42
55
|
### Import
|
@@ -44,47 +57,89 @@ Or install it yourself as:
|
|
44
57
|
require 'filestack'
|
45
58
|
```
|
46
59
|
Intialize the client using your API key, and security if you are using it.
|
60
|
+
|
47
61
|
```ruby
|
48
62
|
client = FilestackClient.new('YOUR_API_KEY', security: security_object)
|
49
63
|
```
|
50
64
|
### Uploading
|
51
|
-
Filestack uses multipart uploading by default, which is faster for larger files. This can be turned off by passing in ```multipart: false```. Multipart is disabled when uploading external URLs.
|
52
65
|
```ruby
|
53
|
-
filelink = client.upload(filepath: '/path/to/
|
66
|
+
filelink = client.upload(filepath: '/path/to/localfile')
|
54
67
|
|
55
|
-
|
68
|
+
# OR
|
69
|
+
|
70
|
+
filelink = client.upload(external_url: 'http://domain.com/image.png')
|
56
71
|
|
57
72
|
# OR
|
58
73
|
|
59
|
-
|
74
|
+
file = StringIO.new
|
75
|
+
filelink = client.upload(io: file)
|
60
76
|
```
|
61
77
|
|
62
|
-
To upload a local and an external file with
|
78
|
+
To upload a local, an IO object and an external file with following optional options:
|
79
|
+
|
63
80
|
```ruby
|
64
|
-
|
81
|
+
options = {
|
82
|
+
filename: 'string',
|
83
|
+
location: 'string',
|
84
|
+
path: 'string',
|
85
|
+
container: 'string',
|
86
|
+
mimetype: 'string',
|
87
|
+
region: 'string',
|
88
|
+
workflows: ['workflow-id-1', 'workflow-id-2'],
|
89
|
+
upload_tags: {
|
90
|
+
key: 'value',
|
91
|
+
key2: 'value'
|
92
|
+
}
|
93
|
+
}
|
65
94
|
|
66
|
-
filelink = client.upload(
|
95
|
+
filelink = client.upload(filepath: '/path/to/localfile', options: { mimetype: 'image/png', filename: 'custom_filename.png' })
|
96
|
+
|
97
|
+
filelink = client.upload(external_url: 'http://domain.com/image.png', options: { mimetype: 'image/jpeg', filename: 'custom_filename.png' })
|
67
98
|
```
|
68
99
|
|
69
100
|
To store file on `dropbox`, `azure`, `gcs` or `rackspace`, you must have the chosen provider configured in the developer portal to enable this feature. By default the file is stored on `s3`. You can add more details of the storage in `options`.
|
70
101
|
|
71
102
|
```ruby
|
72
|
-
filelink = client.upload(filepath: '/path/to/file', storage: '
|
103
|
+
filelink = client.upload(filepath: '/path/to/file', storage: 's3', options: { path: 'folder_name/', container: 'container_name', location: 's3', region: 'region_name' })
|
73
104
|
|
74
|
-
filelink = client.upload(external_url: 'http://someurl.com/image.png',
|
105
|
+
filelink = client.upload(external_url: 'http://someurl.com/image.png', options: { location: 'dropbox', path: 'folder_name' })
|
106
|
+
```
|
107
|
+
|
108
|
+
### Workflows
|
109
|
+
Workflows allow you to wire up conditional logic and image processing to enforce business processes, automate ingest, and save valuable development time. In order to trigger the workflow job for each upload:
|
110
|
+
|
111
|
+
```ruby
|
112
|
+
filelink = client.upload(filepath: '/path/to/file', options: { workflows: ["workflow_id_1", "workflow_id_2"] })
|
113
|
+
|
114
|
+
#OR
|
115
|
+
|
116
|
+
filelink = client.upload(external_url: 'http://someurl.com/image.png', options: { workflows: ["workflow_id_1"] })
|
75
117
|
```
|
76
118
|
|
77
119
|
### Security
|
78
120
|
If security is enabled on your account, or if you are using certain actions that require security (delete, overwrite and certain transformations), you will need to create a security object and pass it into the client on instantiation.
|
79
121
|
|
80
122
|
```ruby
|
81
|
-
security = FilestackSecurity.new('YOUR_APP_SECRET', options: {call: %w[read store pick]})
|
123
|
+
security = FilestackSecurity.new('YOUR_APP_SECRET', options: {call: %w[read store pick runWorkflow]})
|
82
124
|
client = FilestackClient.new('YOUR_API_KEY', security: security)
|
83
125
|
```
|
84
126
|
|
85
127
|
### Using FilestackFilelinks
|
86
128
|
FilestackFilelink objects are representation of a file handle. You can download, get raw file content, delete and overwrite file handles directly. Security is required for overwrite and delete methods.
|
87
129
|
|
130
|
+
Initialize the filelink using the file handle, your API key, and security if required. The file handle is the string following the last slash `/` in the file URL.
|
131
|
+
|
132
|
+
```ruby
|
133
|
+
filelink = FilestackFilelink.new(handle, apikey: 'YOUR_API_KEY', security: security_object)
|
134
|
+
```
|
135
|
+
|
136
|
+
### Deletion
|
137
|
+
You can delete a file by simply calling `delete` on the filelink.
|
138
|
+
|
139
|
+
```ruby
|
140
|
+
filelink.delete
|
141
|
+
```
|
142
|
+
|
88
143
|
### Transformations
|
89
144
|
Transforms can be initiated one of two ways. The first, by calling ```transform``` on a filelink:
|
90
145
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.9.1
|
data/filestack-ruby.gemspec
CHANGED
@@ -24,7 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
|
25
25
|
spec.add_dependency "typhoeus", "~> 1.1"
|
26
26
|
spec.add_dependency "parallel", "~> 1.11", ">= 1.11.2"
|
27
|
-
spec.add_dependency "mimemagic", "~> 0.3.
|
27
|
+
spec.add_dependency "mimemagic", "~> 0.3.9"
|
28
28
|
spec.add_dependency "progress_bar"
|
29
29
|
|
30
30
|
spec.add_development_dependency "bundler", "~> 1.7"
|
data/lib/filestack/config.rb
CHANGED
@@ -7,11 +7,6 @@ class FilestackConfig
|
|
7
7
|
CDN_URL = 'https://cdn.filestackcontent.com'.freeze
|
8
8
|
PROCESS_URL = 'https://process.filestackapi.com'.freeze
|
9
9
|
|
10
|
-
MULTIPART_START_URL = 'https://upload.filestackapi.com/multipart/start'.freeze
|
11
|
-
MULTIPART_UPLOAD_URL = 'https://upload.filestackapi.com/multipart/upload'.freeze
|
12
|
-
MULTIPART_COMMIT_URL = 'https://upload.filestackapi.com/multipart/commit'.freeze
|
13
|
-
MULTIPART_COMPLETE_URL = 'https://upload.filestackapi.com/multipart/complete'.freeze
|
14
|
-
|
15
10
|
MULTIPART_PARAMS = %w[
|
16
11
|
store_location store_region store_container
|
17
12
|
store_path store_access
|
@@ -22,10 +17,30 @@ class FilestackConfig
|
|
22
17
|
VERSION = Filestack::Ruby::VERSION
|
23
18
|
HEADERS = {
|
24
19
|
'User-Agent' => "filestack-ruby #{VERSION}",
|
25
|
-
'Filestack-Source' => "Ruby-#{VERSION}"
|
20
|
+
'Filestack-Source' => "Ruby-#{VERSION}",
|
21
|
+
'Content-Type' => "application/json",
|
22
|
+
'Accept-Encoding' => "application/json"
|
26
23
|
}.freeze
|
27
24
|
|
25
|
+
DEFAULT_UPLOAD_MIMETYPE = 'application/octet-stream'
|
26
|
+
|
28
27
|
INTELLIGENT_ERROR_MESSAGES = ['BACKEND_SERVER', 'BACKEND_NETWORK', 'S3_SERVER', 'S3_NETWORK']
|
28
|
+
|
29
|
+
def self.multipart_start_url
|
30
|
+
"https://upload.filestackapi.com/multipart/start"
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.multipart_upload_url(base_url)
|
34
|
+
"https://#{base_url}/multipart/upload"
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.multipart_commit_url(base_url)
|
38
|
+
"https://#{base_url}/multipart/commit"
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.multipart_complete_url(base_url)
|
42
|
+
"https://#{base_url}/multipart/complete"
|
43
|
+
end
|
29
44
|
end
|
30
45
|
|
31
46
|
class TransformConfig
|
@@ -23,30 +23,25 @@ class FilestackClient
|
|
23
23
|
@security = security
|
24
24
|
end
|
25
25
|
|
26
|
-
# Upload a local file or
|
26
|
+
# Upload a local file, external url or IO object
|
27
27
|
# @param [String] filepath The path of a local file
|
28
28
|
# @param [String] external_url An external URL
|
29
|
-
# @param [
|
30
|
-
# (Default: true)
|
29
|
+
# @param [StringIO] io The IO object
|
31
30
|
# @param [Hash] options User-supplied upload options
|
31
|
+
# @param [Boolean] intelligent Upload file using Filestack Intelligent Ingestion
|
32
|
+
# @param [String] storage Default storage to be used for uploads
|
32
33
|
#
|
33
34
|
# return [Filestack::FilestackFilelink]
|
34
|
-
def upload(filepath: nil, external_url: nil,
|
35
|
-
if filepath && external_url
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
multipart_upload(@apikey, filepath, @security, options, timeout, storage, intelligent: intelligent)
|
35
|
+
def upload(filepath: nil, external_url: nil, io: nil, options: {}, intelligent: false, timeout: 60, storage: 'S3')
|
36
|
+
return 'You cannot upload a URL and file at the same time' if (filepath || io) && external_url
|
37
|
+
|
38
|
+
response = if external_url
|
39
|
+
send_upload(@apikey, external_url, @security, options)
|
40
40
|
else
|
41
|
-
|
42
|
-
|
43
|
-
filepath: filepath,
|
44
|
-
external_url: external_url,
|
45
|
-
options: options,
|
46
|
-
security: @security,
|
47
|
-
storage: storage
|
48
|
-
)
|
41
|
+
return 'You cannot upload IO object and file at the same time' if io && filepath
|
42
|
+
multipart_upload(@apikey, filepath, io, @security, options, timeout, storage, intelligent)
|
49
43
|
end
|
44
|
+
|
50
45
|
FilestackFilelink.new(response['handle'], security: @security, apikey: @apikey)
|
51
46
|
end
|
52
47
|
# Transform an external URL
|
@@ -13,24 +13,23 @@ include UploadUtils
|
|
13
13
|
include IntelligentUtils
|
14
14
|
# Includes all the utility functions for Filestack multipart uploads
|
15
15
|
module MultipartUploadUtils
|
16
|
-
|
17
|
-
|
16
|
+
|
17
|
+
def get_file_attributes(file, options = {})
|
18
|
+
filename = options[:filename] || File.basename(file)
|
19
|
+
mimetype = options[:mimetype] || MimeMagic.by_magic(File.open(file)) || FilestackConfig::DEFAULT_UPLOAD_MIMETYPE
|
18
20
|
filesize = File.size(file)
|
19
|
-
|
20
|
-
if mimetype.nil?
|
21
|
-
mimetype = 'application/octet-stream'
|
22
|
-
end
|
21
|
+
|
23
22
|
[filename, filesize, mimetype.to_s]
|
24
23
|
end
|
25
24
|
|
26
|
-
def
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
25
|
+
def get_io_attributes(io, options = {})
|
26
|
+
filename = options[:filename] || 'unnamed_file'
|
27
|
+
mimetype = options[:mimetype] || FilestackConfig::DEFAULT_UPLOAD_MIMETYPE
|
28
|
+
|
29
|
+
io.seek(0, IO::SEEK_END)
|
30
|
+
filesize = io.tell
|
31
|
+
|
32
|
+
[filename, filesize, mimetype.to_s]
|
34
33
|
end
|
35
34
|
|
36
35
|
# Send start response to multipart endpoint
|
@@ -41,8 +40,10 @@ module MultipartUploadUtils
|
|
41
40
|
# @param [String] mimetype Mimetype of incoming file
|
42
41
|
# @param [FilestackSecurity] security Security object with
|
43
42
|
# policy/signature
|
43
|
+
# @param [String] storage Default storage to be used for uploads
|
44
44
|
# @param [Hash] options User-defined options for
|
45
45
|
# multipart uploads
|
46
|
+
# @param [Bool] intelligent Upload file using Filestack Intelligent Ingestion
|
46
47
|
#
|
47
48
|
# @return [Typhoeus::Response]
|
48
49
|
def multipart_start(apikey, filename, filesize, mimetype, security, storage, options = {}, intelligent)
|
@@ -51,23 +52,21 @@ module MultipartUploadUtils
|
|
51
52
|
filename: filename,
|
52
53
|
mimetype: mimetype,
|
53
54
|
size: filesize,
|
54
|
-
|
55
|
-
|
56
|
-
multipart: intelligent
|
55
|
+
store: { location: storage },
|
56
|
+
fii: intelligent
|
57
57
|
}
|
58
58
|
|
59
|
-
options
|
60
|
-
params = params.merge!(options) if options
|
59
|
+
params[:store].merge!(options) if options
|
61
60
|
|
62
61
|
unless security.nil?
|
63
62
|
params[:policy] = security.policy
|
64
63
|
params[:signature] = security.signature
|
65
64
|
end
|
66
65
|
|
67
|
-
response = Typhoeus.post(
|
68
|
-
|
69
|
-
|
70
|
-
|
66
|
+
response = Typhoeus.post(FilestackConfig.multipart_start_url,
|
67
|
+
body: params.to_json,
|
68
|
+
headers: FilestackConfig::HEADERS)
|
69
|
+
|
71
70
|
if response.code == 200
|
72
71
|
JSON.parse(response.body)
|
73
72
|
else
|
@@ -79,22 +78,21 @@ module MultipartUploadUtils
|
|
79
78
|
#
|
80
79
|
# @param [String] apikey Filestack API key
|
81
80
|
# @param [String] filename Name of incoming file
|
82
|
-
# @param [String] filepath Local path to file
|
83
81
|
# @param [Int] filesize Size of incoming file
|
84
82
|
# @param [Typhoeus::Response] start_response Response body from
|
85
83
|
# multipart_start
|
84
|
+
# @param [String] storage Default storage to be used for uploads
|
86
85
|
# @param [Hash] options User-defined options for
|
87
86
|
# multipart uploads
|
88
87
|
#
|
89
88
|
# @return [Array]
|
90
|
-
def create_upload_jobs(apikey, filename,
|
89
|
+
def create_upload_jobs(apikey, filename, filesize, start_response, storage, options)
|
91
90
|
jobs = []
|
92
91
|
part = 1
|
93
92
|
seek_point = 0
|
94
93
|
while seek_point < filesize
|
95
94
|
part_info = {
|
96
|
-
|
97
|
-
filepath: filepath,
|
95
|
+
seek_point: seek_point,
|
98
96
|
filename: filename,
|
99
97
|
apikey: apikey,
|
100
98
|
part: part,
|
@@ -104,10 +102,10 @@ module MultipartUploadUtils
|
|
104
102
|
upload_id: start_response['upload_id'],
|
105
103
|
location_url: start_response['location_url'],
|
106
104
|
start_response: start_response,
|
107
|
-
|
105
|
+
store: { location: storage },
|
108
106
|
}
|
109
|
-
|
110
|
-
part_info
|
107
|
+
|
108
|
+
part_info[:store].merge!(options) if options
|
111
109
|
|
112
110
|
if seek_point + FilestackConfig::DEFAULT_CHUNK_SIZE > filesize
|
113
111
|
size = filesize - (seek_point)
|
@@ -128,16 +126,17 @@ module MultipartUploadUtils
|
|
128
126
|
# @param [Hash] job Hash of options needed
|
129
127
|
# to upload a chunk
|
130
128
|
# @param [String] apikey Filestack API key
|
131
|
-
# @param [String]
|
129
|
+
# @param [String] filepath Location url given back
|
132
130
|
# from endpoint
|
133
|
-
# @param [
|
131
|
+
# @param [StringIO] io The IO object
|
134
132
|
# @param [Hash] options User-defined options for
|
135
133
|
# multipart uploads
|
134
|
+
# @param [String] storage Default storage to be used for uploads
|
136
135
|
#
|
137
136
|
# @return [Typhoeus::Response]
|
138
|
-
def upload_chunk(job, apikey, filepath, options)
|
139
|
-
file = File.open(filepath)
|
140
|
-
file.seek(job[:
|
137
|
+
def upload_chunk(job, apikey, filepath, io, options, storage)
|
138
|
+
file = filepath ? File.open(filepath) : io
|
139
|
+
file.seek(job[:seek_point])
|
141
140
|
chunk = file.read(FilestackConfig::DEFAULT_CHUNK_SIZE)
|
142
141
|
|
143
142
|
md5 = Digest::MD5.new
|
@@ -150,14 +149,13 @@ module MultipartUploadUtils
|
|
150
149
|
uri: job[:uri],
|
151
150
|
region: job[:region],
|
152
151
|
upload_id: job[:upload_id],
|
153
|
-
|
154
|
-
file: Tempfile.new(job[:filename])
|
152
|
+
store: { location: storage },
|
155
153
|
}
|
156
154
|
data = data.merge!(options) if options
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
155
|
+
|
156
|
+
fs_response = Typhoeus.post(FilestackConfig.multipart_upload_url(job[:location_url]),
|
157
|
+
body: data.to_json,
|
158
|
+
headers: FilestackConfig::HEADERS).body
|
161
159
|
fs_response = JSON.parse(fs_response)
|
162
160
|
Typhoeus.put(
|
163
161
|
fs_response['url'], headers: fs_response['headers'], body: chunk
|
@@ -170,19 +168,20 @@ module MultipartUploadUtils
|
|
170
168
|
# @param [String] filepath Local path to file
|
171
169
|
# @param [Hash] options User-defined options for
|
172
170
|
# multipart uploads
|
171
|
+
# @param [String] storage Default storage to be used for uploads
|
173
172
|
#
|
174
173
|
# @return [Array] Array of parts/etags strings
|
175
|
-
def run_uploads(jobs, apikey, filepath, options)
|
174
|
+
def run_uploads(jobs, apikey, filepath, io, options, storage)
|
176
175
|
bar = ProgressBar.new(jobs.length)
|
177
176
|
results = Parallel.map(jobs, in_threads: 4) do |job|
|
178
177
|
response = upload_chunk(
|
179
|
-
job, apikey, filepath, options
|
178
|
+
job, apikey, filepath, io, options, storage
|
180
179
|
)
|
181
180
|
if response.code == 200
|
182
181
|
bar.increment!
|
183
182
|
part = job[:part]
|
184
183
|
etag = response.headers[:etag]
|
185
|
-
|
184
|
+
{ part_number: part, etag: etag }
|
186
185
|
end
|
187
186
|
end
|
188
187
|
results
|
@@ -202,6 +201,8 @@ module MultipartUploadUtils
|
|
202
201
|
# part numbers
|
203
202
|
# @param [Hash] options User-defined options for
|
204
203
|
# multipart uploads
|
204
|
+
# @param [String] storage Default storage to be used for uploads
|
205
|
+
# @param [Boolean] intelligent Upload file using Filestack Intelligent Ingestion
|
205
206
|
#
|
206
207
|
# @return [Typhoeus::Response]
|
207
208
|
def multipart_complete(apikey, filename, filesize, mimetype, start_response, parts_and_etags, options, storage, intelligent = false)
|
@@ -213,52 +214,53 @@ module MultipartUploadUtils
|
|
213
214
|
filename: filename,
|
214
215
|
size: filesize,
|
215
216
|
mimetype: mimetype,
|
216
|
-
|
217
|
-
file: Tempfile.new(filename)
|
217
|
+
store: { location: storage },
|
218
218
|
}
|
219
|
-
options
|
220
|
-
data.merge!(
|
221
|
-
data.merge!(intelligent ? { multipart: intelligent } : { parts: parts_and_etags.join(';') })
|
219
|
+
data[:store].merge!(options) if options
|
220
|
+
data.merge!(intelligent ? { fii: intelligent } : { parts: parts_and_etags })
|
222
221
|
|
223
|
-
Typhoeus.post(
|
224
|
-
|
225
|
-
|
226
|
-
)
|
222
|
+
Typhoeus.post(FilestackConfig.multipart_complete_url(start_response['location_url']),
|
223
|
+
body: data.to_json,
|
224
|
+
headers: FilestackConfig::HEADERS)
|
227
225
|
end
|
228
226
|
|
229
227
|
# Run entire multipart process through with file and options
|
230
228
|
#
|
231
229
|
# @param [String] apikey Filestack API key
|
232
230
|
# @param [String] filename Name of incoming file
|
231
|
+
# @param [StringIO] io The IO object
|
233
232
|
# @param [FilestackSecurity] security Security object with
|
234
233
|
# policy/signature
|
235
234
|
# @param [Hash] options User-defined options for
|
236
235
|
# multipart uploads
|
236
|
+
# @param [String] storage Default storage to be used for uploads
|
237
|
+
# @param [Boolean] intelligent Upload file using Filestack Intelligent Ingestion
|
237
238
|
#
|
238
239
|
# @return [Hash]
|
239
|
-
def multipart_upload(apikey, filepath, security, options, timeout, storage, intelligent
|
240
|
-
filename, filesize, mimetype =
|
240
|
+
def multipart_upload(apikey, filepath, io, security, options, timeout, storage, intelligent = false)
|
241
|
+
filename, filesize, mimetype = if filepath
|
242
|
+
get_file_attributes(filepath, options)
|
243
|
+
else
|
244
|
+
get_io_attributes(io, options)
|
245
|
+
end
|
246
|
+
|
241
247
|
start_response = multipart_start(
|
242
248
|
apikey, filename, filesize, mimetype, security, storage, options, intelligent
|
243
249
|
)
|
244
250
|
|
245
|
-
unless start_response['upload_type'].nil?
|
246
|
-
intelligent_enabled = ((start_response['upload_type'].include? 'intelligent_ingestion')) && intelligent
|
247
|
-
end
|
248
|
-
|
249
251
|
jobs = create_upload_jobs(
|
250
|
-
apikey, filename,
|
252
|
+
apikey, filename, filesize, start_response, storage, options
|
251
253
|
)
|
252
254
|
|
253
|
-
if
|
255
|
+
if intelligent
|
254
256
|
state = IntelligentState.new
|
255
|
-
run_intelligent_upload_flow(jobs, state)
|
257
|
+
run_intelligent_upload_flow(jobs, filepath, io, state, storage)
|
256
258
|
response_complete = multipart_complete(
|
257
259
|
apikey, filename, filesize, mimetype,
|
258
260
|
start_response, nil, options, storage, intelligent
|
259
261
|
)
|
260
262
|
else
|
261
|
-
parts_and_etags = run_uploads(jobs, apikey, filepath, options)
|
263
|
+
parts_and_etags = run_uploads(jobs, apikey, filepath, io, options, storage)
|
262
264
|
response_complete = multipart_complete(
|
263
265
|
apikey, filename, filesize, mimetype,
|
264
266
|
start_response, parts_and_etags, options, storage
|
@@ -62,41 +62,51 @@ module UploadUtils
|
|
62
62
|
)
|
63
63
|
end
|
64
64
|
|
65
|
+
def build_store_task(options = {})
|
66
|
+
return 'store' if options.nil? || options.empty?
|
67
|
+
tasks = []
|
68
|
+
options.each do |key, value|
|
69
|
+
value = case key
|
70
|
+
when :workflows
|
71
|
+
[value.join('","')]
|
72
|
+
when :path
|
73
|
+
"\"#{value}\""
|
74
|
+
else
|
75
|
+
value
|
76
|
+
end
|
77
|
+
tasks.push("#{key}:#{value.to_s.downcase}")
|
78
|
+
end
|
79
|
+
"store=#{tasks.join(',')}"
|
80
|
+
end
|
81
|
+
|
65
82
|
# Uploads to v1 REST API (for external URLs or if multipart is turned off)
|
66
83
|
#
|
67
84
|
# @param [String] apikey Filestack API key
|
68
|
-
# @param [String] filepath Local path to file
|
69
85
|
# @param [String] external_url External URL to be uploaded
|
70
86
|
# @param [FilestackSecurity] security Security object with
|
71
87
|
# policy/signature
|
72
88
|
# @param [Hash] options User-defined options for
|
73
89
|
# multipart uploads
|
74
|
-
# @param [String] storage Storage destination
|
75
|
-
# (s3, rackspace, etc)
|
76
90
|
# @return [Hash]
|
77
|
-
def send_upload(apikey,
|
78
|
-
|
79
|
-
{ fileUpload: File.open(filepath) }
|
80
|
-
else
|
81
|
-
{ url: external_url }
|
82
|
-
end
|
83
|
-
|
84
|
-
# adds any user-defined upload options to request payload
|
85
|
-
data = data.merge!(options) unless options.nil?
|
86
|
-
base = "#{FilestackConfig::API_URL}/store/#{storage}?key=#{apikey}"
|
91
|
+
def send_upload(apikey, external_url = nil, security = nil, options = nil)
|
92
|
+
base = "#{FilestackConfig::CDN_URL}/#{apikey}/#{build_store_task(options)}"
|
87
93
|
|
88
94
|
if security
|
89
95
|
policy = security.policy
|
90
96
|
signature = security.signature
|
91
|
-
base = "#{base}
|
97
|
+
base = "#{base}/security=s:#{signature},p:#{policy}"
|
92
98
|
end
|
93
99
|
|
94
|
-
response = Typhoeus.post(base,
|
100
|
+
response = Typhoeus.post("#{base}/#{external_url}", headers: FilestackConfig::HEADERS)
|
95
101
|
|
96
102
|
if response.code == 200
|
97
|
-
|
98
|
-
|
99
|
-
|
103
|
+
begin
|
104
|
+
response_body = JSON.parse(response.body)
|
105
|
+
handle = response_body['url'].split('/').last
|
106
|
+
return { 'handle' => handle }
|
107
|
+
rescue
|
108
|
+
raise response.body
|
109
|
+
end
|
100
110
|
end
|
101
111
|
raise response.body
|
102
112
|
end
|
@@ -193,7 +203,7 @@ module IntelligentUtils
|
|
193
203
|
# @param [IntelligentState] state An IntelligentState object
|
194
204
|
#
|
195
205
|
# @return [Array]
|
196
|
-
def run_intelligent_upload_flow(jobs, state)
|
206
|
+
def run_intelligent_upload_flow(jobs, filepath, io, state, storage)
|
197
207
|
bar = ProgressBar.new(jobs.length)
|
198
208
|
generator = create_intelligent_generator(jobs)
|
199
209
|
working_offset = FilestackConfig::DEFAULT_OFFSET_SIZE
|
@@ -201,7 +211,7 @@ module IntelligentUtils
|
|
201
211
|
batch = get_generator_batch(generator)
|
202
212
|
# run parts
|
203
213
|
Parallel.map(batch, in_threads: 4) do |part|
|
204
|
-
state = run_intelligent_uploads(part, state)
|
214
|
+
state = run_intelligent_uploads(part, filepath, io, state, storage)
|
205
215
|
# condition: a chunk has failed but we have not reached the maximum retries
|
206
216
|
while bad_state(state)
|
207
217
|
# condition: timeout to S3, requiring offset size to be changed
|
@@ -213,7 +223,7 @@ module IntelligentUtils
|
|
213
223
|
sleep(state.backoff)
|
214
224
|
end
|
215
225
|
state.add_retry
|
216
|
-
state = run_intelligent_uploads(part, state)
|
226
|
+
state = run_intelligent_uploads(part, filepath, io, state, storage)
|
217
227
|
end
|
218
228
|
raise "Upload has failed. Please try again later." unless state.ok
|
219
229
|
bar.increment!
|
@@ -269,14 +279,14 @@ module IntelligentUtils
|
|
269
279
|
# multipart_start
|
270
280
|
#
|
271
281
|
# @return [Dict]
|
272
|
-
def chunk_job(job, state, apikey, filename,
|
282
|
+
def chunk_job(job, state, apikey, filename, filesize, start_response, storage)
|
273
283
|
offset = 0
|
274
|
-
seek_point = job[:
|
284
|
+
seek_point = job[:seek_point]
|
275
285
|
chunk_list = []
|
286
|
+
|
276
287
|
while (offset < FilestackConfig::DEFAULT_CHUNK_SIZE) && (seek_point + offset) < filesize
|
277
288
|
chunk_list.push(
|
278
|
-
|
279
|
-
filepath: filepath,
|
289
|
+
seek_point: seek_point,
|
280
290
|
filename: filename,
|
281
291
|
apikey: apikey,
|
282
292
|
part: job[:part],
|
@@ -285,7 +295,7 @@ module IntelligentUtils
|
|
285
295
|
region: start_response['region'],
|
286
296
|
upload_id: start_response['upload_id'],
|
287
297
|
location_url: start_response['location_url'],
|
288
|
-
|
298
|
+
store: { location: storage },
|
289
299
|
offset: offset
|
290
300
|
)
|
291
301
|
offset += state.offset
|
@@ -300,15 +310,14 @@ module IntelligentUtils
|
|
300
310
|
# @param [IntelligentState] state An IntelligentState object
|
301
311
|
#
|
302
312
|
# @return [IntelligentState]
|
303
|
-
def run_intelligent_uploads(part, state)
|
313
|
+
def run_intelligent_uploads(part, filepath, io, state, storage)
|
304
314
|
failed = false
|
305
315
|
chunks = chunk_job(
|
306
|
-
part, state, part[:apikey], part[:filename], part[:
|
307
|
-
part[:filesize], part[:start_response]
|
316
|
+
part, state, part[:apikey], part[:filename], part[:filesize], part[:start_response], storage
|
308
317
|
)
|
309
318
|
Parallel.map(chunks, in_threads: 3) do |chunk|
|
310
319
|
begin
|
311
|
-
upload_chunk_intelligently(chunk, state, part[:apikey],
|
320
|
+
upload_chunk_intelligently(chunk, state, part[:apikey], filepath, io, part[:options], storage)
|
312
321
|
rescue => e
|
313
322
|
state.error_type = e.message
|
314
323
|
failed = true
|
@@ -322,6 +331,7 @@ module IntelligentUtils
|
|
322
331
|
else
|
323
332
|
state.ok = true
|
324
333
|
end
|
334
|
+
|
325
335
|
commit_params = {
|
326
336
|
apikey: part[:apikey],
|
327
337
|
uri: part[:uri],
|
@@ -329,12 +339,14 @@ module IntelligentUtils
|
|
329
339
|
upload_id: part[:upload_id],
|
330
340
|
size: part[:filesize],
|
331
341
|
part: part[:part],
|
332
|
-
location_url: part[:location_url],
|
333
|
-
|
334
|
-
file: Tempfile.new(part[:filename])
|
342
|
+
location_url: part[:start_response]['location_url'],
|
343
|
+
store: { location: storage }
|
335
344
|
}
|
336
|
-
|
337
|
-
|
345
|
+
|
346
|
+
response = Typhoeus.post(FilestackConfig.multipart_commit_url(commit_params[:location_url]),
|
347
|
+
body: commit_params.to_json,
|
348
|
+
headers: FilestackConfig::HEADERS)
|
349
|
+
|
338
350
|
if response.code == 200
|
339
351
|
state.reset
|
340
352
|
else
|
@@ -354,9 +366,10 @@ module IntelligentUtils
|
|
354
366
|
# multipart uploads
|
355
367
|
#
|
356
368
|
# @return [Typhoeus::Response]
|
357
|
-
def upload_chunk_intelligently(job, state, apikey, filepath, options)
|
358
|
-
file = File.open(filepath)
|
359
|
-
file.seek(job[:
|
369
|
+
def upload_chunk_intelligently(job, state, apikey, filepath, io, options, storage)
|
370
|
+
file = filepath ? File.open(filepath) : io
|
371
|
+
file.seek(job[:seek_point] + job[:offset])
|
372
|
+
|
360
373
|
chunk = file.read(state.offset)
|
361
374
|
md5 = Digest::MD5.new
|
362
375
|
md5 << chunk
|
@@ -368,17 +381,17 @@ module IntelligentUtils
|
|
368
381
|
uri: job[:uri],
|
369
382
|
region: job[:region],
|
370
383
|
upload_id: job[:upload_id],
|
371
|
-
|
384
|
+
store: { location: storage },
|
372
385
|
offset: job[:offset],
|
373
|
-
|
374
|
-
'multipart' => 'true'
|
386
|
+
fii: true
|
375
387
|
}
|
376
388
|
|
377
389
|
data = data.merge!(options) if options
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
390
|
+
|
391
|
+
fs_response = Typhoeus.post(FilestackConfig.multipart_upload_url(job[:location_url]),
|
392
|
+
body: data.to_json,
|
393
|
+
headers: FilestackConfig::HEADERS)
|
394
|
+
|
382
395
|
# POST to multipart/upload
|
383
396
|
begin
|
384
397
|
unless fs_response.code == 200
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filestack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.9.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Filestack
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-03-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: typhoeus
|
@@ -50,14 +50,14 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: 0.3.
|
53
|
+
version: 0.3.9
|
54
54
|
type: :runtime
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: 0.3.
|
60
|
+
version: 0.3.9
|
61
61
|
- !ruby/object:Gem::Dependency
|
62
62
|
name: progress_bar
|
63
63
|
requirement: !ruby/object:Gem::Requirement
|
@@ -201,7 +201,7 @@ homepage: https://github.com/filestack/filestack-ruby
|
|
201
201
|
licenses:
|
202
202
|
- MIT
|
203
203
|
metadata: {}
|
204
|
-
post_install_message:
|
204
|
+
post_install_message:
|
205
205
|
rdoc_options: []
|
206
206
|
require_paths:
|
207
207
|
- lib
|
@@ -216,9 +216,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
216
216
|
- !ruby/object:Gem::Version
|
217
217
|
version: '0'
|
218
218
|
requirements: []
|
219
|
-
|
220
|
-
|
221
|
-
signing_key:
|
219
|
+
rubygems_version: 3.0.8
|
220
|
+
signing_key:
|
222
221
|
specification_version: 4
|
223
222
|
summary: Official Ruby SDK for the Filestack API
|
224
223
|
test_files: []
|