filestack 2.6.5 → 2.8.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/CHANGELOG.md +15 -0
- data/README.md +48 -12
- data/VERSION +1 -1
- data/docs/TransformConfig.html +2 -1
- data/lib/filestack/config.rb +22 -6
- data/lib/filestack/models/filestack_client.rb +12 -17
- data/lib/filestack/ruby/version.rb +1 -1
- data/lib/filestack/utils/multipart_upload_utils.rb +65 -63
- data/lib/filestack/utils/utils.rb +58 -45
- metadata +6 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 77f23edf489898b7e03305bca017f6eb519cf3995aecd120908a6d01042ca60d
|
4
|
+
data.tar.gz: bcdc3b18c9ecd5ebbc886cfd518a08b2ba0eccdc994f24e673095fdfdc84945f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 596a72ca7995f8ba53a63d78f6ae2b0994f5c02b5b8e38fb273f0b6aefb8f657638544ee2775320d7d8c472dee8879bf5e7765de0f8ecf96ed93c973eeb1707a
|
7
|
+
data.tar.gz: e55b56cfefdfeaa5bffbd52f83af5d9d92ebf7952150c02fec12a43f256a3f8b2e8575627593f12fe826ac1004104e4c42bf45c1e7c8748609730faaf6ced218
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,20 @@
|
|
1
1
|
# Filestack-Ruby Changelog
|
2
2
|
|
3
|
+
## 2.8.1 (October 1, 2020)
|
4
|
+
- Add rescue and raises body message on upload
|
5
|
+
|
6
|
+
## 2.8.0 (September 29, 2020)
|
7
|
+
- Add IO object upload
|
8
|
+
|
9
|
+
## 2.7.0 (September 28, 2020)
|
10
|
+
- Add workflows
|
11
|
+
|
12
|
+
## 2.6.7 (August 4, 2020)
|
13
|
+
- Add content disposition task
|
14
|
+
|
15
|
+
## 2.6.6 (June 19, 2020)
|
16
|
+
- Add compress task to transformations
|
17
|
+
|
3
18
|
## 2.6.5 (January 9, 2020)
|
4
19
|
- Updated version of the parallel gem
|
5
20
|
|
data/README.md
CHANGED
@@ -44,41 +44,70 @@ Or install it yourself as:
|
|
44
44
|
require 'filestack'
|
45
45
|
```
|
46
46
|
Intialize the client using your API key, and security if you are using it.
|
47
|
+
|
47
48
|
```ruby
|
48
49
|
client = FilestackClient.new('YOUR_API_KEY', security: security_object)
|
49
50
|
```
|
50
51
|
### Uploading
|
51
|
-
Filestack uses multipart uploading by default, which is faster for larger files. This can be turned off by passing in ```multipart: false```. Multipart is disabled when uploading external URLs.
|
52
52
|
```ruby
|
53
|
-
filelink = client.upload(filepath: '/path/to/
|
53
|
+
filelink = client.upload(filepath: '/path/to/localfile')
|
54
|
+
|
55
|
+
# OR
|
54
56
|
|
55
|
-
filelink = client.upload(
|
57
|
+
filelink = client.upload(external_url: 'http://domain.com/image.png')
|
56
58
|
|
57
59
|
# OR
|
58
60
|
|
59
|
-
|
61
|
+
file = StringIO.new
|
62
|
+
filelink = client.upload(io: file)
|
60
63
|
```
|
61
64
|
|
62
|
-
To upload a local and an external file with
|
65
|
+
To upload a local, an IO object and an external file with following optional options:
|
66
|
+
|
63
67
|
```ruby
|
64
|
-
|
68
|
+
options = {
|
69
|
+
filename: 'string',
|
70
|
+
location: 'string',
|
71
|
+
path: 'string',
|
72
|
+
container: 'string',
|
73
|
+
mimetype: 'string',
|
74
|
+
region: 'string',
|
75
|
+
workflows: ['workflow-id-1', 'workflow-id-2'],
|
76
|
+
upload_tags: {
|
77
|
+
key: 'value',
|
78
|
+
key2: 'value'
|
79
|
+
}
|
80
|
+
}
|
81
|
+
|
82
|
+
filelink = client.upload(filepath: '/path/to/localfile', options: { mimetype: 'image/png', filename: 'custom_filename.png' })
|
65
83
|
|
66
|
-
filelink = client.upload(external_url: 'http://
|
84
|
+
filelink = client.upload(external_url: 'http://domain.com/image.png', options: { mimetype: 'image/jpeg', filename: 'custom_filename.png' })
|
67
85
|
```
|
68
86
|
|
69
87
|
To store file on `dropbox`, `azure`, `gcs` or `rackspace`, you must have the chosen provider configured in the developer portal to enable this feature. By default the file is stored on `s3`. You can add more details of the storage in `options`.
|
70
88
|
|
71
89
|
```ruby
|
72
|
-
filelink = client.upload(filepath: '/path/to/file', storage: '
|
90
|
+
filelink = client.upload(filepath: '/path/to/file', storage: 's3', options: { path: 'folder_name/', container: 'container_name', location: 's3', region: 'region_name' })
|
91
|
+
|
92
|
+
filelink = client.upload(external_url: 'http://someurl.com/image.png', options: { location: 'dropbox', path: 'folder_name' })
|
93
|
+
```
|
94
|
+
|
95
|
+
### Workflows
|
96
|
+
Workflows allow you to wire up conditional logic and image processing to enforce business processes, automate ingest, and save valuable development time. In order to trigger the workflow job for each upload:
|
97
|
+
|
98
|
+
```ruby
|
99
|
+
filelink = client.upload(filepath: '/path/to/file', options: { workflows: ["workflow_id_1", "workflow_id_2"] })
|
100
|
+
|
101
|
+
#OR
|
73
102
|
|
74
|
-
filelink = client.upload(external_url: 'http://someurl.com/image.png',
|
103
|
+
filelink = client.upload(external_url: 'http://someurl.com/image.png', options: { workflows: ["workflow_id_1"] })
|
75
104
|
```
|
76
105
|
|
77
106
|
### Security
|
78
107
|
If security is enabled on your account, or if you are using certain actions that require security (delete, overwrite and certain transformations), you will need to create a security object and pass it into the client on instantiation.
|
79
108
|
|
80
109
|
```ruby
|
81
|
-
security = FilestackSecurity.new('YOUR_APP_SECRET', options: {call: %w[read store pick]})
|
110
|
+
security = FilestackSecurity.new('YOUR_APP_SECRET', options: {call: %w[read store pick runWorkflow]})
|
82
111
|
client = FilestackClient.new('YOUR_API_KEY', security: security)
|
83
112
|
```
|
84
113
|
|
@@ -123,13 +152,20 @@ Return `default` file if the source of the transformation does not work or the t
|
|
123
152
|
To use fallback, you should provide `handle` of the file that should be returned. Optionally, you can add `cache`, which means number of seconds fallback response should be cached in CDN.
|
124
153
|
|
125
154
|
```ruby
|
126
|
-
transform = client.transform_external('https://someurl.com').fallback(
|
155
|
+
transform = client.transform_external('https://someurl.com/file.png').fallback(file: 'DEFAULT_HANDLE_OR_FILEPATH')
|
127
156
|
```
|
128
157
|
|
129
158
|
If you are using fallback handle that belongs to different application than the one which runs transformation (APIKEY) and it is secured with security policy, appropriate signature and policy with read call should be used:
|
130
159
|
|
131
160
|
```ruby
|
132
|
-
transform = client.transform_external('https://someurl.com').fallback(
|
161
|
+
transform = client.transform_external('https://someurl.com/file.png').fallback(file: 'DEFAULT_HANDLE_OR_FILEPATH?policy=HANDLE_APIKEY_POLICY&signature=HANDLE_APIKEY_SIGNATURE', cache: 10)
|
162
|
+
```
|
163
|
+
|
164
|
+
### Content
|
165
|
+
Sets `Content-Disposition` header for given file.
|
166
|
+
|
167
|
+
```ruby
|
168
|
+
transform = filelink.transform.content(filename: 'DEFAULT_FILENAME', type: 'TYPE')
|
133
169
|
```
|
134
170
|
|
135
171
|
### Tagging
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
2.
|
1
|
+
2.8.1
|
data/docs/TransformConfig.html
CHANGED
@@ -114,7 +114,8 @@
|
|
114
114
|
</span><span class='tstring_content'> partial_pixelate</span><span class='words_sep'> </span><span class='tstring_content'>partial_blur</span><span class='words_sep'> </span><span class='tstring_content'>collage</span><span class='words_sep'> </span><span class='tstring_content'>upscale</span><span class='words_sep'> </span><span class='tstring_content'>enhance</span><span class='words_sep'>
|
115
115
|
</span><span class='tstring_content'> redeye</span><span class='words_sep'> </span><span class='tstring_content'>ascii</span><span class='words_sep'> </span><span class='tstring_content'>filetype_conversion</span><span class='words_sep'> </span><span class='tstring_content'>quality</span><span class='words_sep'> </span><span class='tstring_content'>urlscreenshot</span><span class='words_sep'>
|
116
116
|
</span><span class='tstring_content'> no_metadata</span><span class='words_sep'></span><span class='tstring_content'> fallback</span><span class='words_sep'></span><span class='tstring_content'> pdfinfo</span><span class='words_sep'></span><span class='tstring_content'> pdfconvert</span><span class='words_sep'></span><span class='tstring_content'> cache</span><span class='words_sep'></span><span class='tstring_content'> auto_image</span><span class='words_sep'>
|
117
|
-
</span><span class='tstring_content'> minify_js</span><span class='words_sep'></span><span class='tstring_content'> minify_css</span><span class='words_sep'></span><span class='tstring_content'> animate</span><span class='words_sep'></span><span class='tstring_content'> video_convert</span><span class='words_sep'></span><span class='tstring_content'> video_playlist</span><span class='words_sep'
|
117
|
+
</span><span class='tstring_content'> minify_js</span><span class='words_sep'></span><span class='tstring_content'> minify_css</span><span class='words_sep'></span><span class='tstring_content'> animate</span><span class='words_sep'></span><span class='tstring_content'> video_convert</span><span class='words_sep'></span><span class='tstring_content'> video_playlist</span><span class='words_sep'>
|
118
|
+
</span><span class='tstring_content'> compress</span><span class='words_sep'></span><span class='tstring_content'> content</span><span class='words_sep'></span>
|
118
119
|
<span class='tstring_end'>]</span></span><span class='period'>.</span><span class='id identifier rubyid_freeze'>freeze</span></pre></dd>
|
119
120
|
|
120
121
|
</dl>
|
data/lib/filestack/config.rb
CHANGED
@@ -7,11 +7,6 @@ class FilestackConfig
|
|
7
7
|
CDN_URL = 'https://cdn.filestackcontent.com'.freeze
|
8
8
|
PROCESS_URL = 'https://process.filestackapi.com'.freeze
|
9
9
|
|
10
|
-
MULTIPART_START_URL = 'https://upload.filestackapi.com/multipart/start'.freeze
|
11
|
-
MULTIPART_UPLOAD_URL = 'https://upload.filestackapi.com/multipart/upload'.freeze
|
12
|
-
MULTIPART_COMMIT_URL = 'https://upload.filestackapi.com/multipart/commit'.freeze
|
13
|
-
MULTIPART_COMPLETE_URL = 'https://upload.filestackapi.com/multipart/complete'.freeze
|
14
|
-
|
15
10
|
MULTIPART_PARAMS = %w[
|
16
11
|
store_location store_region store_container
|
17
12
|
store_path store_access
|
@@ -22,10 +17,30 @@ class FilestackConfig
|
|
22
17
|
VERSION = Filestack::Ruby::VERSION
|
23
18
|
HEADERS = {
|
24
19
|
'User-Agent' => "filestack-ruby #{VERSION}",
|
25
|
-
'Filestack-Source' => "Ruby-#{VERSION}"
|
20
|
+
'Filestack-Source' => "Ruby-#{VERSION}",
|
21
|
+
'Content-Type' => "application/json",
|
22
|
+
'Accept-Encoding' => "application/json"
|
26
23
|
}.freeze
|
27
24
|
|
25
|
+
DEFAULT_UPLOAD_MIMETYPE = 'application/octet-stream'
|
26
|
+
|
28
27
|
INTELLIGENT_ERROR_MESSAGES = ['BACKEND_SERVER', 'BACKEND_NETWORK', 'S3_SERVER', 'S3_NETWORK']
|
28
|
+
|
29
|
+
def self.multipart_start_url
|
30
|
+
"https://upload.filestackapi.com/multipart/start"
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.multipart_upload_url(base_url)
|
34
|
+
"https://#{base_url}/multipart/upload"
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.multipart_commit_url(base_url)
|
38
|
+
"https://#{base_url}/multipart/commit"
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.multipart_complete_url(base_url)
|
42
|
+
"https://#{base_url}/multipart/complete"
|
43
|
+
end
|
29
44
|
end
|
30
45
|
|
31
46
|
class TransformConfig
|
@@ -38,5 +53,6 @@ class TransformConfig
|
|
38
53
|
redeye ascii filetype_conversion quality urlscreenshot
|
39
54
|
no_metadata fallback pdfinfo pdfconvert cache auto_image
|
40
55
|
minify_js minify_css animate video_convert video_playlist
|
56
|
+
compress content
|
41
57
|
].freeze
|
42
58
|
end
|
@@ -23,30 +23,25 @@ class FilestackClient
|
|
23
23
|
@security = security
|
24
24
|
end
|
25
25
|
|
26
|
-
# Upload a local file or
|
26
|
+
# Upload a local file, external url or IO object
|
27
27
|
# @param [String] filepath The path of a local file
|
28
28
|
# @param [String] external_url An external URL
|
29
|
-
# @param [
|
30
|
-
# (Default: true)
|
29
|
+
# @param [StringIO] io The IO object
|
31
30
|
# @param [Hash] options User-supplied upload options
|
31
|
+
# @param [Boolean] intelligent Upload file using Filestack Intelligent Ingestion
|
32
|
+
# @param [String] storage Default storage to be used for uploads
|
32
33
|
#
|
33
34
|
# return [Filestack::FilestackFilelink]
|
34
|
-
def upload(filepath: nil, external_url: nil,
|
35
|
-
if filepath && external_url
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
multipart_upload(@apikey, filepath, @security, options, timeout, storage, intelligent: intelligent)
|
35
|
+
def upload(filepath: nil, external_url: nil, io: nil, options: {}, intelligent: false, timeout: 60, storage: 'S3')
|
36
|
+
return 'You cannot upload a URL and file at the same time' if (filepath || io) && external_url
|
37
|
+
|
38
|
+
response = if external_url
|
39
|
+
send_upload(@apikey, external_url, @security, options)
|
40
40
|
else
|
41
|
-
|
42
|
-
|
43
|
-
filepath: filepath,
|
44
|
-
external_url: external_url,
|
45
|
-
options: options,
|
46
|
-
security: @security,
|
47
|
-
storage: storage
|
48
|
-
)
|
41
|
+
return 'You cannot upload IO object and file at the same time' if io && filepath
|
42
|
+
multipart_upload(@apikey, filepath, io, @security, options, timeout, storage, intelligent)
|
49
43
|
end
|
44
|
+
|
50
45
|
FilestackFilelink.new(response['handle'], security: @security, apikey: @apikey)
|
51
46
|
end
|
52
47
|
# Transform an external URL
|
@@ -13,24 +13,23 @@ include UploadUtils
|
|
13
13
|
include IntelligentUtils
|
14
14
|
# Includes all the utility functions for Filestack multipart uploads
|
15
15
|
module MultipartUploadUtils
|
16
|
-
|
17
|
-
|
16
|
+
|
17
|
+
def get_file_attributes(file, options = {})
|
18
|
+
filename = options[:filename] || File.basename(file)
|
19
|
+
mimetype = options[:mimetype] || MimeMagic.by_magic(File.open(file)) || FilestackConfig::DEFAULT_UPLOAD_MIMETYPE
|
18
20
|
filesize = File.size(file)
|
19
|
-
|
20
|
-
if mimetype.nil?
|
21
|
-
mimetype = 'application/octet-stream'
|
22
|
-
end
|
21
|
+
|
23
22
|
[filename, filesize, mimetype.to_s]
|
24
23
|
end
|
25
24
|
|
26
|
-
def
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
25
|
+
def get_io_attributes(io, options = {})
|
26
|
+
filename = options[:filename] || 'unnamed_file'
|
27
|
+
mimetype = options[:mimetype] || FilestackConfig::DEFAULT_UPLOAD_MIMETYPE
|
28
|
+
|
29
|
+
io.seek(0, IO::SEEK_END)
|
30
|
+
filesize = io.tell
|
31
|
+
|
32
|
+
[filename, filesize, mimetype.to_s]
|
34
33
|
end
|
35
34
|
|
36
35
|
# Send start response to multipart endpoint
|
@@ -41,8 +40,10 @@ module MultipartUploadUtils
|
|
41
40
|
# @param [String] mimetype Mimetype of incoming file
|
42
41
|
# @param [FilestackSecurity] security Security object with
|
43
42
|
# policy/signature
|
43
|
+
# @param [String] storage Default storage to be used for uploads
|
44
44
|
# @param [Hash] options User-defined options for
|
45
45
|
# multipart uploads
|
46
|
+
# @param [Bool] intelligent Upload file using Filestack Intelligent Ingestion
|
46
47
|
#
|
47
48
|
# @return [Typhoeus::Response]
|
48
49
|
def multipart_start(apikey, filename, filesize, mimetype, security, storage, options = {}, intelligent)
|
@@ -51,23 +52,21 @@ module MultipartUploadUtils
|
|
51
52
|
filename: filename,
|
52
53
|
mimetype: mimetype,
|
53
54
|
size: filesize,
|
54
|
-
|
55
|
-
|
56
|
-
multipart: intelligent
|
55
|
+
store: { location: storage },
|
56
|
+
fii: intelligent
|
57
57
|
}
|
58
58
|
|
59
|
-
options
|
60
|
-
params = params.merge!(options) if options
|
59
|
+
params[:store].merge!(options) if options
|
61
60
|
|
62
61
|
unless security.nil?
|
63
62
|
params[:policy] = security.policy
|
64
63
|
params[:signature] = security.signature
|
65
64
|
end
|
66
65
|
|
67
|
-
response = Typhoeus.post(
|
68
|
-
|
69
|
-
|
70
|
-
|
66
|
+
response = Typhoeus.post(FilestackConfig.multipart_start_url,
|
67
|
+
body: params.to_json,
|
68
|
+
headers: FilestackConfig::HEADERS)
|
69
|
+
|
71
70
|
if response.code == 200
|
72
71
|
JSON.parse(response.body)
|
73
72
|
else
|
@@ -79,22 +78,21 @@ module MultipartUploadUtils
|
|
79
78
|
#
|
80
79
|
# @param [String] apikey Filestack API key
|
81
80
|
# @param [String] filename Name of incoming file
|
82
|
-
# @param [String] filepath Local path to file
|
83
81
|
# @param [Int] filesize Size of incoming file
|
84
82
|
# @param [Typhoeus::Response] start_response Response body from
|
85
83
|
# multipart_start
|
84
|
+
# @param [String] storage Default storage to be used for uploads
|
86
85
|
# @param [Hash] options User-defined options for
|
87
86
|
# multipart uploads
|
88
87
|
#
|
89
88
|
# @return [Array]
|
90
|
-
def create_upload_jobs(apikey, filename,
|
89
|
+
def create_upload_jobs(apikey, filename, filesize, start_response, storage, options)
|
91
90
|
jobs = []
|
92
91
|
part = 1
|
93
92
|
seek_point = 0
|
94
93
|
while seek_point < filesize
|
95
94
|
part_info = {
|
96
|
-
|
97
|
-
filepath: filepath,
|
95
|
+
seek_point: seek_point,
|
98
96
|
filename: filename,
|
99
97
|
apikey: apikey,
|
100
98
|
part: part,
|
@@ -104,10 +102,10 @@ module MultipartUploadUtils
|
|
104
102
|
upload_id: start_response['upload_id'],
|
105
103
|
location_url: start_response['location_url'],
|
106
104
|
start_response: start_response,
|
107
|
-
|
105
|
+
store: { location: storage },
|
108
106
|
}
|
109
|
-
|
110
|
-
part_info
|
107
|
+
|
108
|
+
part_info[:store].merge!(options) if options
|
111
109
|
|
112
110
|
if seek_point + FilestackConfig::DEFAULT_CHUNK_SIZE > filesize
|
113
111
|
size = filesize - (seek_point)
|
@@ -128,16 +126,17 @@ module MultipartUploadUtils
|
|
128
126
|
# @param [Hash] job Hash of options needed
|
129
127
|
# to upload a chunk
|
130
128
|
# @param [String] apikey Filestack API key
|
131
|
-
# @param [String]
|
129
|
+
# @param [String] filepath Location url given back
|
132
130
|
# from endpoint
|
133
|
-
# @param [
|
131
|
+
# @param [StringIO] io The IO object
|
134
132
|
# @param [Hash] options User-defined options for
|
135
133
|
# multipart uploads
|
134
|
+
# @param [String] storage Default storage to be used for uploads
|
136
135
|
#
|
137
136
|
# @return [Typhoeus::Response]
|
138
|
-
def upload_chunk(job, apikey, filepath, options)
|
139
|
-
file = File.open(filepath)
|
140
|
-
file.seek(job[:
|
137
|
+
def upload_chunk(job, apikey, filepath, io, options, storage)
|
138
|
+
file = filepath ? File.open(filepath) : io
|
139
|
+
file.seek(job[:seek_point])
|
141
140
|
chunk = file.read(FilestackConfig::DEFAULT_CHUNK_SIZE)
|
142
141
|
|
143
142
|
md5 = Digest::MD5.new
|
@@ -150,14 +149,13 @@ module MultipartUploadUtils
|
|
150
149
|
uri: job[:uri],
|
151
150
|
region: job[:region],
|
152
151
|
upload_id: job[:upload_id],
|
153
|
-
|
154
|
-
file: Tempfile.new(job[:filename])
|
152
|
+
store: { location: storage },
|
155
153
|
}
|
156
154
|
data = data.merge!(options) if options
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
155
|
+
|
156
|
+
fs_response = Typhoeus.post(FilestackConfig.multipart_upload_url(job[:location_url]),
|
157
|
+
body: data.to_json,
|
158
|
+
headers: FilestackConfig::HEADERS).body
|
161
159
|
fs_response = JSON.parse(fs_response)
|
162
160
|
Typhoeus.put(
|
163
161
|
fs_response['url'], headers: fs_response['headers'], body: chunk
|
@@ -170,19 +168,20 @@ module MultipartUploadUtils
|
|
170
168
|
# @param [String] filepath Local path to file
|
171
169
|
# @param [Hash] options User-defined options for
|
172
170
|
# multipart uploads
|
171
|
+
# @param [String] storage Default storage to be used for uploads
|
173
172
|
#
|
174
173
|
# @return [Array] Array of parts/etags strings
|
175
|
-
def run_uploads(jobs, apikey, filepath, options)
|
174
|
+
def run_uploads(jobs, apikey, filepath, io, options, storage)
|
176
175
|
bar = ProgressBar.new(jobs.length)
|
177
176
|
results = Parallel.map(jobs, in_threads: 4) do |job|
|
178
177
|
response = upload_chunk(
|
179
|
-
job, apikey, filepath, options
|
178
|
+
job, apikey, filepath, io, options, storage
|
180
179
|
)
|
181
180
|
if response.code == 200
|
182
181
|
bar.increment!
|
183
182
|
part = job[:part]
|
184
183
|
etag = response.headers[:etag]
|
185
|
-
|
184
|
+
{ part_number: part, etag: etag }
|
186
185
|
end
|
187
186
|
end
|
188
187
|
results
|
@@ -202,6 +201,8 @@ module MultipartUploadUtils
|
|
202
201
|
# part numbers
|
203
202
|
# @param [Hash] options User-defined options for
|
204
203
|
# multipart uploads
|
204
|
+
# @param [String] storage Default storage to be used for uploads
|
205
|
+
# @param [Boolean] intelligent Upload file using Filestack Intelligent Ingestion
|
205
206
|
#
|
206
207
|
# @return [Typhoeus::Response]
|
207
208
|
def multipart_complete(apikey, filename, filesize, mimetype, start_response, parts_and_etags, options, storage, intelligent = false)
|
@@ -213,52 +214,53 @@ module MultipartUploadUtils
|
|
213
214
|
filename: filename,
|
214
215
|
size: filesize,
|
215
216
|
mimetype: mimetype,
|
216
|
-
|
217
|
-
file: Tempfile.new(filename)
|
217
|
+
store: { location: storage },
|
218
218
|
}
|
219
|
-
options
|
220
|
-
data.merge!(
|
221
|
-
data.merge!(intelligent ? { multipart: intelligent } : { parts: parts_and_etags.join(';') })
|
219
|
+
data[:store].merge!(options) if options
|
220
|
+
data.merge!(intelligent ? { fii: intelligent } : { parts: parts_and_etags })
|
222
221
|
|
223
|
-
Typhoeus.post(
|
224
|
-
|
225
|
-
|
226
|
-
)
|
222
|
+
Typhoeus.post(FilestackConfig.multipart_complete_url(start_response['location_url']),
|
223
|
+
body: data.to_json,
|
224
|
+
headers: FilestackConfig::HEADERS)
|
227
225
|
end
|
228
226
|
|
229
227
|
# Run entire multipart process through with file and options
|
230
228
|
#
|
231
229
|
# @param [String] apikey Filestack API key
|
232
230
|
# @param [String] filename Name of incoming file
|
231
|
+
# @param [StringIO] io The IO object
|
233
232
|
# @param [FilestackSecurity] security Security object with
|
234
233
|
# policy/signature
|
235
234
|
# @param [Hash] options User-defined options for
|
236
235
|
# multipart uploads
|
236
|
+
# @param [String] storage Default storage to be used for uploads
|
237
|
+
# @param [Boolean] intelligent Upload file using Filestack Intelligent Ingestion
|
237
238
|
#
|
238
239
|
# @return [Hash]
|
239
|
-
def multipart_upload(apikey, filepath, security, options, timeout, storage, intelligent
|
240
|
-
filename, filesize, mimetype =
|
240
|
+
def multipart_upload(apikey, filepath, io, security, options, timeout, storage, intelligent = false)
|
241
|
+
filename, filesize, mimetype = if filepath
|
242
|
+
get_file_attributes(filepath, options)
|
243
|
+
else
|
244
|
+
get_io_attributes(io, options)
|
245
|
+
end
|
246
|
+
|
241
247
|
start_response = multipart_start(
|
242
248
|
apikey, filename, filesize, mimetype, security, storage, options, intelligent
|
243
249
|
)
|
244
250
|
|
245
|
-
unless start_response['upload_type'].nil?
|
246
|
-
intelligent_enabled = ((start_response['upload_type'].include? 'intelligent_ingestion')) && intelligent
|
247
|
-
end
|
248
|
-
|
249
251
|
jobs = create_upload_jobs(
|
250
|
-
apikey, filename,
|
252
|
+
apikey, filename, filesize, start_response, storage, options
|
251
253
|
)
|
252
254
|
|
253
|
-
if
|
255
|
+
if intelligent
|
254
256
|
state = IntelligentState.new
|
255
|
-
run_intelligent_upload_flow(jobs, state)
|
257
|
+
run_intelligent_upload_flow(jobs, filepath, io, state, storage)
|
256
258
|
response_complete = multipart_complete(
|
257
259
|
apikey, filename, filesize, mimetype,
|
258
260
|
start_response, nil, options, storage, intelligent
|
259
261
|
)
|
260
262
|
else
|
261
|
-
parts_and_etags = run_uploads(jobs, apikey, filepath, options)
|
263
|
+
parts_and_etags = run_uploads(jobs, apikey, filepath, io, options, storage)
|
262
264
|
response_complete = multipart_complete(
|
263
265
|
apikey, filename, filesize, mimetype,
|
264
266
|
start_response, parts_and_etags, options, storage
|
@@ -62,41 +62,51 @@ module UploadUtils
|
|
62
62
|
)
|
63
63
|
end
|
64
64
|
|
65
|
+
def build_store_task(options = {})
|
66
|
+
return 'store' if options.nil? || options.empty?
|
67
|
+
tasks = []
|
68
|
+
options.each do |key, value|
|
69
|
+
value = case key
|
70
|
+
when :workflows
|
71
|
+
[value.join('","')]
|
72
|
+
when :path
|
73
|
+
"\"#{value}\""
|
74
|
+
else
|
75
|
+
value
|
76
|
+
end
|
77
|
+
tasks.push("#{key}:#{value.to_s.downcase}")
|
78
|
+
end
|
79
|
+
"store=#{tasks.join(',')}"
|
80
|
+
end
|
81
|
+
|
65
82
|
# Uploads to v1 REST API (for external URLs or if multipart is turned off)
|
66
83
|
#
|
67
84
|
# @param [String] apikey Filestack API key
|
68
|
-
# @param [String] filepath Local path to file
|
69
85
|
# @param [String] external_url External URL to be uploaded
|
70
86
|
# @param [FilestackSecurity] security Security object with
|
71
87
|
# policy/signature
|
72
88
|
# @param [Hash] options User-defined options for
|
73
89
|
# multipart uploads
|
74
|
-
# @param [String] storage Storage destination
|
75
|
-
# (s3, rackspace, etc)
|
76
90
|
# @return [Hash]
|
77
|
-
def send_upload(apikey,
|
78
|
-
|
79
|
-
{ fileUpload: File.open(filepath) }
|
80
|
-
else
|
81
|
-
{ url: external_url }
|
82
|
-
end
|
83
|
-
|
84
|
-
# adds any user-defined upload options to request payload
|
85
|
-
data = data.merge!(options) unless options.nil?
|
86
|
-
base = "#{FilestackConfig::API_URL}/store/#{storage}?key=#{apikey}"
|
91
|
+
def send_upload(apikey, external_url = nil, security = nil, options = nil)
|
92
|
+
base = "#{FilestackConfig::CDN_URL}/#{apikey}/#{build_store_task(options)}"
|
87
93
|
|
88
94
|
if security
|
89
95
|
policy = security.policy
|
90
96
|
signature = security.signature
|
91
|
-
base = "#{base}
|
97
|
+
base = "#{base}/security=s:#{signature},p:#{policy}"
|
92
98
|
end
|
93
99
|
|
94
|
-
response = Typhoeus.post(base,
|
100
|
+
response = Typhoeus.post("#{base}/#{external_url}", headers: FilestackConfig::HEADERS)
|
95
101
|
|
96
102
|
if response.code == 200
|
97
|
-
|
98
|
-
|
99
|
-
|
103
|
+
begin
|
104
|
+
response_body = JSON.parse(response.body)
|
105
|
+
handle = response_body['url'].split('/').last
|
106
|
+
return { 'handle' => handle }
|
107
|
+
rescue
|
108
|
+
raise response.body
|
109
|
+
end
|
100
110
|
end
|
101
111
|
raise response.body
|
102
112
|
end
|
@@ -193,7 +203,7 @@ module IntelligentUtils
|
|
193
203
|
# @param [IntelligentState] state An IntelligentState object
|
194
204
|
#
|
195
205
|
# @return [Array]
|
196
|
-
def run_intelligent_upload_flow(jobs, state)
|
206
|
+
def run_intelligent_upload_flow(jobs, filepath, io, state, storage)
|
197
207
|
bar = ProgressBar.new(jobs.length)
|
198
208
|
generator = create_intelligent_generator(jobs)
|
199
209
|
working_offset = FilestackConfig::DEFAULT_OFFSET_SIZE
|
@@ -201,7 +211,7 @@ module IntelligentUtils
|
|
201
211
|
batch = get_generator_batch(generator)
|
202
212
|
# run parts
|
203
213
|
Parallel.map(batch, in_threads: 4) do |part|
|
204
|
-
state = run_intelligent_uploads(part, state)
|
214
|
+
state = run_intelligent_uploads(part, filepath, io, state, storage)
|
205
215
|
# condition: a chunk has failed but we have not reached the maximum retries
|
206
216
|
while bad_state(state)
|
207
217
|
# condition: timeout to S3, requiring offset size to be changed
|
@@ -213,7 +223,7 @@ module IntelligentUtils
|
|
213
223
|
sleep(state.backoff)
|
214
224
|
end
|
215
225
|
state.add_retry
|
216
|
-
state = run_intelligent_uploads(part, state)
|
226
|
+
state = run_intelligent_uploads(part, filepath, io, state, storage)
|
217
227
|
end
|
218
228
|
raise "Upload has failed. Please try again later." unless state.ok
|
219
229
|
bar.increment!
|
@@ -269,14 +279,14 @@ module IntelligentUtils
|
|
269
279
|
# multipart_start
|
270
280
|
#
|
271
281
|
# @return [Dict]
|
272
|
-
def chunk_job(job, state, apikey, filename,
|
282
|
+
def chunk_job(job, state, apikey, filename, filesize, start_response, storage)
|
273
283
|
offset = 0
|
274
|
-
seek_point = job[:
|
284
|
+
seek_point = job[:seek_point]
|
275
285
|
chunk_list = []
|
286
|
+
|
276
287
|
while (offset < FilestackConfig::DEFAULT_CHUNK_SIZE) && (seek_point + offset) < filesize
|
277
288
|
chunk_list.push(
|
278
|
-
|
279
|
-
filepath: filepath,
|
289
|
+
seek_point: seek_point,
|
280
290
|
filename: filename,
|
281
291
|
apikey: apikey,
|
282
292
|
part: job[:part],
|
@@ -285,7 +295,7 @@ module IntelligentUtils
|
|
285
295
|
region: start_response['region'],
|
286
296
|
upload_id: start_response['upload_id'],
|
287
297
|
location_url: start_response['location_url'],
|
288
|
-
|
298
|
+
store: { location: storage },
|
289
299
|
offset: offset
|
290
300
|
)
|
291
301
|
offset += state.offset
|
@@ -300,15 +310,14 @@ module IntelligentUtils
|
|
300
310
|
# @param [IntelligentState] state An IntelligentState object
|
301
311
|
#
|
302
312
|
# @return [IntelligentState]
|
303
|
-
def run_intelligent_uploads(part, state)
|
313
|
+
def run_intelligent_uploads(part, filepath, io, state, storage)
|
304
314
|
failed = false
|
305
315
|
chunks = chunk_job(
|
306
|
-
part, state, part[:apikey], part[:filename], part[:
|
307
|
-
part[:filesize], part[:start_response]
|
316
|
+
part, state, part[:apikey], part[:filename], part[:filesize], part[:start_response], storage
|
308
317
|
)
|
309
318
|
Parallel.map(chunks, in_threads: 3) do |chunk|
|
310
319
|
begin
|
311
|
-
upload_chunk_intelligently(chunk, state, part[:apikey],
|
320
|
+
upload_chunk_intelligently(chunk, state, part[:apikey], filepath, io, part[:options], storage)
|
312
321
|
rescue => e
|
313
322
|
state.error_type = e.message
|
314
323
|
failed = true
|
@@ -322,6 +331,7 @@ module IntelligentUtils
|
|
322
331
|
else
|
323
332
|
state.ok = true
|
324
333
|
end
|
334
|
+
|
325
335
|
commit_params = {
|
326
336
|
apikey: part[:apikey],
|
327
337
|
uri: part[:uri],
|
@@ -329,12 +339,14 @@ module IntelligentUtils
|
|
329
339
|
upload_id: part[:upload_id],
|
330
340
|
size: part[:filesize],
|
331
341
|
part: part[:part],
|
332
|
-
location_url: part[:location_url],
|
333
|
-
|
334
|
-
file: Tempfile.new(part[:filename])
|
342
|
+
location_url: part[:start_response]['location_url'],
|
343
|
+
store: { location: storage }
|
335
344
|
}
|
336
|
-
|
337
|
-
|
345
|
+
|
346
|
+
response = Typhoeus.post(FilestackConfig.multipart_commit_url(commit_params[:location_url]),
|
347
|
+
body: commit_params.to_json,
|
348
|
+
headers: FilestackConfig::HEADERS)
|
349
|
+
|
338
350
|
if response.code == 200
|
339
351
|
state.reset
|
340
352
|
else
|
@@ -354,9 +366,10 @@ module IntelligentUtils
|
|
354
366
|
# multipart uploads
|
355
367
|
#
|
356
368
|
# @return [Typhoeus::Response]
|
357
|
-
def upload_chunk_intelligently(job, state, apikey, filepath, options)
|
358
|
-
file = File.open(filepath)
|
359
|
-
file.seek(job[:
|
369
|
+
def upload_chunk_intelligently(job, state, apikey, filepath, io, options, storage)
|
370
|
+
file = filepath ? File.open(filepath) : io
|
371
|
+
file.seek(job[:seek_point] + job[:offset])
|
372
|
+
|
360
373
|
chunk = file.read(state.offset)
|
361
374
|
md5 = Digest::MD5.new
|
362
375
|
md5 << chunk
|
@@ -368,17 +381,17 @@ module IntelligentUtils
|
|
368
381
|
uri: job[:uri],
|
369
382
|
region: job[:region],
|
370
383
|
upload_id: job[:upload_id],
|
371
|
-
|
384
|
+
store: { location: storage },
|
372
385
|
offset: job[:offset],
|
373
|
-
|
374
|
-
'multipart' => 'true'
|
386
|
+
fii: true
|
375
387
|
}
|
376
388
|
|
377
389
|
data = data.merge!(options) if options
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
390
|
+
|
391
|
+
fs_response = Typhoeus.post(FilestackConfig.multipart_upload_url(job[:location_url]),
|
392
|
+
body: data.to_json,
|
393
|
+
headers: FilestackConfig::HEADERS)
|
394
|
+
|
382
395
|
# POST to multipart/upload
|
383
396
|
begin
|
384
397
|
unless fs_response.code == 200
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: filestack
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Filestack
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: typhoeus
|
@@ -201,7 +201,7 @@ homepage: https://github.com/filestack/filestack-ruby
|
|
201
201
|
licenses:
|
202
202
|
- MIT
|
203
203
|
metadata: {}
|
204
|
-
post_install_message:
|
204
|
+
post_install_message:
|
205
205
|
rdoc_options: []
|
206
206
|
require_paths:
|
207
207
|
- lib
|
@@ -216,9 +216,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
216
216
|
- !ruby/object:Gem::Version
|
217
217
|
version: '0'
|
218
218
|
requirements: []
|
219
|
-
|
220
|
-
|
221
|
-
signing_key:
|
219
|
+
rubygems_version: 3.0.8
|
220
|
+
signing_key:
|
222
221
|
specification_version: 4
|
223
222
|
summary: Official Ruby SDK for the Filestack API
|
224
223
|
test_files: []
|