filestack 2.1.0 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/README.md +2 -2
- data/VERSION +1 -1
- data/docs/AV.html +557 -0
- data/docs/Filestack.html +115 -0
- data/docs/Filestack/Ruby.html +115 -0
- data/docs/FilestackClient.html +702 -0
- data/docs/FilestackCommon.html +884 -0
- data/docs/FilestackConfig.html +197 -0
- data/docs/FilestackFilelink.html +1277 -0
- data/docs/FilestackSecurity.html +654 -0
- data/docs/IntelligentState.html +729 -0
- data/docs/IntelligentUtils.html +1639 -0
- data/docs/MultipartUploadUtils.html +1543 -0
- data/docs/Transform.html +1152 -0
- data/docs/TransformConfig.html +138 -0
- data/docs/TransformUtils.html +272 -0
- data/docs/UploadUtils.html +673 -0
- data/docs/_index.html +227 -0
- data/docs/class_list.html +51 -0
- data/docs/css/common.css +1 -0
- data/docs/css/full_list.css +58 -0
- data/docs/css/style.css +492 -0
- data/docs/file.README.html +210 -0
- data/docs/file_list.html +56 -0
- data/docs/frames.html +17 -0
- data/docs/index.html +210 -0
- data/docs/js/app.js +248 -0
- data/docs/js/full_list.js +216 -0
- data/docs/js/jquery.js +4 -0
- data/docs/method_list.html +643 -0
- data/docs/top-level-namespace.html +145 -0
- data/examples/file_transformations.rb +6 -0
- data/examples/intelligent_upload.rb +7 -0
- data/examples/normal_upload.rb +5 -0
- data/examples/upload_external_url.rb +7 -0
- data/examples/upload_with_security.rb +6 -0
- data/examples/video_convert.rb +10 -0
- data/filestack-ruby.gemspec +1 -0
- data/lib/filestack/config.rb +9 -5
- data/lib/filestack/mixins/filestack_common.rb +17 -1
- data/lib/filestack/models/filelink.rb +8 -1
- data/lib/filestack/models/filestack_client.rb +17 -3
- data/lib/filestack/ruby/version.rb +1 -1
- data/lib/filestack/utils/multipart_upload_utils.rb +91 -36
- data/lib/filestack/utils/utils.rb +309 -1
- metadata +52 -3
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="utf-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
6
|
+
<title>
|
|
7
|
+
Top Level Namespace
|
|
8
|
+
|
|
9
|
+
— Documentation by YARD 0.9.9
|
|
10
|
+
|
|
11
|
+
</title>
|
|
12
|
+
|
|
13
|
+
<link rel="stylesheet" href="css/style.css" type="text/css" charset="utf-8" />
|
|
14
|
+
|
|
15
|
+
<link rel="stylesheet" href="css/common.css" type="text/css" charset="utf-8" />
|
|
16
|
+
|
|
17
|
+
<script type="text/javascript" charset="utf-8">
|
|
18
|
+
pathId = "";
|
|
19
|
+
relpath = '';
|
|
20
|
+
</script>
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
<script type="text/javascript" charset="utf-8" src="js/jquery.js"></script>
|
|
24
|
+
|
|
25
|
+
<script type="text/javascript" charset="utf-8" src="js/app.js"></script>
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
</head>
|
|
29
|
+
<body>
|
|
30
|
+
<div class="nav_wrap">
|
|
31
|
+
<iframe id="nav" src="class_list.html?1"></iframe>
|
|
32
|
+
<div id="resizer"></div>
|
|
33
|
+
</div>
|
|
34
|
+
|
|
35
|
+
<div id="main" tabindex="-1">
|
|
36
|
+
<div id="header">
|
|
37
|
+
<div id="menu">
|
|
38
|
+
|
|
39
|
+
<a href="_index.html">Index</a> »
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
<span class="title">Top Level Namespace</span>
|
|
43
|
+
|
|
44
|
+
</div>
|
|
45
|
+
|
|
46
|
+
<div id="search">
|
|
47
|
+
|
|
48
|
+
<a class="full_list_link" id="class_list_link"
|
|
49
|
+
href="class_list.html">
|
|
50
|
+
|
|
51
|
+
<svg width="24" height="24">
|
|
52
|
+
<rect x="0" y="4" width="24" height="4" rx="1" ry="1"></rect>
|
|
53
|
+
<rect x="0" y="12" width="24" height="4" rx="1" ry="1"></rect>
|
|
54
|
+
<rect x="0" y="20" width="24" height="4" rx="1" ry="1"></rect>
|
|
55
|
+
</svg>
|
|
56
|
+
</a>
|
|
57
|
+
|
|
58
|
+
</div>
|
|
59
|
+
<div class="clear"></div>
|
|
60
|
+
</div>
|
|
61
|
+
|
|
62
|
+
<div id="content"><h1>Top Level Namespace
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
</h1>
|
|
67
|
+
<div class="box_info">
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
<dl>
|
|
75
|
+
<dt>Includes:</dt>
|
|
76
|
+
<dd><span class='object_link'><a href="Filestack.html" title="Filestack (module)">Filestack</a></span>, <span class='object_link'><a href="IntelligentUtils.html" title="IntelligentUtils (module)">IntelligentUtils</a></span>, <span class='object_link'><a href="UploadUtils.html" title="UploadUtils (module)">UploadUtils</a></span></dd>
|
|
77
|
+
</dl>
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
</div>
|
|
85
|
+
|
|
86
|
+
<h2>Defined Under Namespace</h2>
|
|
87
|
+
<p class="children">
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
<strong class="modules">Modules:</strong> <span class='object_link'><a href="Filestack.html" title="Filestack (module)">Filestack</a></span>, <span class='object_link'><a href="FilestackCommon.html" title="FilestackCommon (module)">FilestackCommon</a></span>, <span class='object_link'><a href="IntelligentUtils.html" title="IntelligentUtils (module)">IntelligentUtils</a></span>, <span class='object_link'><a href="MultipartUploadUtils.html" title="MultipartUploadUtils (module)">MultipartUploadUtils</a></span>, <span class='object_link'><a href="TransformUtils.html" title="TransformUtils (module)">TransformUtils</a></span>, <span class='object_link'><a href="UploadUtils.html" title="UploadUtils (module)">UploadUtils</a></span>
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
<strong class="classes">Classes:</strong> <span class='object_link'><a href="AV.html" title="AV (class)">AV</a></span>, <span class='object_link'><a href="FilestackClient.html" title="FilestackClient (class)">FilestackClient</a></span>, <span class='object_link'><a href="FilestackConfig.html" title="FilestackConfig (class)">FilestackConfig</a></span>, <span class='object_link'><a href="FilestackFilelink.html" title="FilestackFilelink (class)">FilestackFilelink</a></span>, <span class='object_link'><a href="FilestackSecurity.html" title="FilestackSecurity (class)">FilestackSecurity</a></span>, <span class='object_link'><a href="IntelligentState.html" title="IntelligentState (class)">IntelligentState</a></span>, <span class='object_link'><a href="Transform.html" title="Transform (class)">Transform</a></span>, <span class='object_link'><a href="TransformConfig.html" title="TransformConfig (class)">TransformConfig</a></span>
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
</p>
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
<h2>Method Summary</h2>
|
|
113
|
+
|
|
114
|
+
<h3 class="inherited">Methods included from <span class='object_link'><a href="IntelligentUtils.html" title="IntelligentUtils (module)">IntelligentUtils</a></span></h3>
|
|
115
|
+
<p class="inherited"><span class='object_link'><a href="IntelligentUtils.html#bad_state-instance_method" title="IntelligentUtils#bad_state (method)">#bad_state</a></span>, <span class='object_link'><a href="IntelligentUtils.html#change_offset-instance_method" title="IntelligentUtils#change_offset (method)">#change_offset</a></span>, <span class='object_link'><a href="IntelligentUtils.html#chunk_job-instance_method" title="IntelligentUtils#chunk_job (method)">#chunk_job</a></span>, <span class='object_link'><a href="IntelligentUtils.html#create_intelligent_generator-instance_method" title="IntelligentUtils#create_intelligent_generator (method)">#create_intelligent_generator</a></span>, <span class='object_link'><a href="IntelligentUtils.html#create_upload_job_chunks-instance_method" title="IntelligentUtils#create_upload_job_chunks (method)">#create_upload_job_chunks</a></span>, <span class='object_link'><a href="IntelligentUtils.html#get_generator_batch-instance_method" title="IntelligentUtils#get_generator_batch (method)">#get_generator_batch</a></span>, <span class='object_link'><a href="IntelligentUtils.html#run_intelligent_upload_flow-instance_method" title="IntelligentUtils#run_intelligent_upload_flow (method)">#run_intelligent_upload_flow</a></span>, <span class='object_link'><a href="IntelligentUtils.html#run_intelligent_uploads-instance_method" title="IntelligentUtils#run_intelligent_uploads (method)">#run_intelligent_uploads</a></span>, <span class='object_link'><a href="IntelligentUtils.html#upload_chunk_intelligently-instance_method" title="IntelligentUtils#upload_chunk_intelligently (method)">#upload_chunk_intelligently</a></span></p>
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
<h3 class="inherited">Methods included from <span class='object_link'><a href="UploadUtils.html" title="UploadUtils (module)">UploadUtils</a></span></h3>
|
|
126
|
+
<p class="inherited"><span class='object_link'><a href="UploadUtils.html#get_url-instance_method" title="UploadUtils#get_url (method)">#get_url</a></span>, <span class='object_link'><a href="UploadUtils.html#make_call-instance_method" title="UploadUtils#make_call (method)">#make_call</a></span>, <span class='object_link'><a href="UploadUtils.html#send_upload-instance_method" title="UploadUtils#send_upload (method)">#send_upload</a></span></p>
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
</div>
|
|
136
|
+
|
|
137
|
+
<div id="footer">
|
|
138
|
+
Generated on Thu Aug 17 10:04:38 2017 by
|
|
139
|
+
<a href="http://yardoc.org" title="Yay! A Ruby Documentation Tool" target="_parent">yard</a>
|
|
140
|
+
0.9.9 (ruby-2.4.0).
|
|
141
|
+
</div>
|
|
142
|
+
|
|
143
|
+
</div>
|
|
144
|
+
</body>
|
|
145
|
+
</html>
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
require 'filestack'
|
|
2
|
+
|
|
3
|
+
# intelligent upload flow requires you to pass in the intelligent parameter
|
|
4
|
+
# and have the service enabled on your application
|
|
5
|
+
client = FilestackClient.new('APIKEY')
|
|
6
|
+
filelink = client.upload(filepath: 'test-files/doom.mp4', intelligent: true)
|
|
7
|
+
puts filelink.url
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
require 'filestack'
|
|
2
|
+
|
|
3
|
+
# intelligent upload flow requires you to pass in the intelligent parameter
|
|
4
|
+
# and have the service enabled on your application
|
|
5
|
+
client = FilestackClient.new('APIKEY')
|
|
6
|
+
filelink = client.upload(external_url: 'http://trailers.divx.com/divx_prod/profiles/WiegelesHeliSki_DivXPlus_19Mbps.mkv')
|
|
7
|
+
puts filelink.url
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
require 'filestack'
|
|
2
|
+
|
|
3
|
+
client = FilestackClient.new('APIKEY')
|
|
4
|
+
filelink = client.upload(filepath: 'test-files/doom.mp4')
|
|
5
|
+
video = filelink.transform.av_convert(width: 100, height: 100)
|
|
6
|
+
while video.status != 'completed'
|
|
7
|
+
puts video.status
|
|
8
|
+
end
|
|
9
|
+
filelink = video.to_filelink
|
|
10
|
+
puts filelink.url
|
data/filestack-ruby.gemspec
CHANGED
|
@@ -25,6 +25,7 @@ Gem::Specification.new do |spec|
|
|
|
25
25
|
spec.add_dependency "unirest", "~> 1.1.2"
|
|
26
26
|
spec.add_dependency "parallel", "~> 1.11.2"
|
|
27
27
|
spec.add_dependency "mimemagic", "~> 0.3.2"
|
|
28
|
+
spec.add_dependency "progress_bar"
|
|
28
29
|
|
|
29
30
|
spec.add_development_dependency "bundler", "~> 1.7"
|
|
30
31
|
spec.add_development_dependency "rake", "~> 10.0"
|
data/lib/filestack/config.rb
CHANGED
|
@@ -7,21 +7,25 @@ class FilestackConfig
|
|
|
7
7
|
CDN_URL = 'https://cdn.filestackcontent.com'.freeze
|
|
8
8
|
PROCESS_URL = 'https://process.filestackapi.com'.freeze
|
|
9
9
|
|
|
10
|
-
MULTIPART_START_URL = 'https://upload.filestackapi.com/multipart/start'.freeze
|
|
11
|
-
MULTIPART_UPLOAD_URL = 'https://upload.filestackapi.com/multipart/upload'.freeze
|
|
12
|
-
|
|
10
|
+
MULTIPART_START_URL = 'https://rc-upload.filestackapi.com/multipart/start'.freeze
|
|
11
|
+
MULTIPART_UPLOAD_URL = 'https://rc-upload.filestackapi.com/multipart/upload'.freeze
|
|
12
|
+
MULTIPART_COMMIT_URL = 'https://rc-upload.filestackapi.com/multipart/commit'.freeze
|
|
13
|
+
MULTIPART_COMPLETE_URL = 'https://rc-upload.filestackapi.com/multipart/complete'.freeze
|
|
13
14
|
|
|
14
15
|
MULTIPART_PARAMS = %w[
|
|
15
16
|
store_location store_region store_container
|
|
16
17
|
store_path store_access
|
|
17
18
|
].freeze
|
|
18
19
|
|
|
19
|
-
DEFAULT_CHUNK_SIZE =
|
|
20
|
+
DEFAULT_CHUNK_SIZE = 8 * 1024**2
|
|
21
|
+
DEFAULT_OFFSET_SIZE = 1 * 1024**2
|
|
20
22
|
VERSION = Filestack::Ruby::VERSION
|
|
21
23
|
HEADERS = {
|
|
22
24
|
'User-Agent' => "filestack-ruby #{VERSION}",
|
|
23
25
|
'Filestack-Source' => "Ruby-#{VERSION}"
|
|
24
26
|
}.freeze
|
|
27
|
+
|
|
28
|
+
INTELLIGENT_ERROR_MESSAGES = ['BACKEND_SERVER', 'BACKEND_NETWORK', 'S3_SERVER', 'S3_NETWORK']
|
|
25
29
|
end
|
|
26
30
|
|
|
27
31
|
class TransformConfig
|
|
@@ -31,6 +35,6 @@ class TransformConfig
|
|
|
31
35
|
torn_edges shadow circle border sharpen blur monochrome
|
|
32
36
|
blackwhite sepia pixelate oil_paint negative modulate
|
|
33
37
|
partial_pixelate partial_blur collage upscale enhance
|
|
34
|
-
redeye ascii filetype_conversion quality
|
|
38
|
+
redeye ascii filetype_conversion quality urlscreenshot
|
|
35
39
|
].freeze
|
|
36
40
|
end
|
|
@@ -23,7 +23,7 @@ module FilestackCommon
|
|
|
23
23
|
# @param [String] filepath Local path of file to be written
|
|
24
24
|
#
|
|
25
25
|
# @return [Int] Size of file written
|
|
26
|
-
def send_download(filepath)
|
|
26
|
+
def send_download(url, filepath)
|
|
27
27
|
content = send_get_content(url)
|
|
28
28
|
File.write(filepath, content.body)
|
|
29
29
|
end
|
|
@@ -90,4 +90,20 @@ module FilestackCommon
|
|
|
90
90
|
"security=signature:#{signature},policy:#{policy}/#{handle}"
|
|
91
91
|
UploadUtils.make_call(url, 'get').body[task]
|
|
92
92
|
end
|
|
93
|
+
|
|
94
|
+
def send_metadata(handle, security = nil, params)
|
|
95
|
+
if security
|
|
96
|
+
policy = security.policy
|
|
97
|
+
signature = security.signature
|
|
98
|
+
url = "#{FilestackConfig::CDN_URL}/#{handle}/metadata?signature=#{signature}&policy=#{policy}"
|
|
99
|
+
else
|
|
100
|
+
url = "#{FilestackConfig::CDN_URL}/#{handle}/metadata"
|
|
101
|
+
end
|
|
102
|
+
response = UploadUtils.make_call(url, 'get', parameters: params)
|
|
103
|
+
|
|
104
|
+
if response.code == 200
|
|
105
|
+
return response.body
|
|
106
|
+
end
|
|
107
|
+
raise response.body
|
|
108
|
+
end
|
|
93
109
|
end
|
|
@@ -37,7 +37,7 @@ class FilestackFilelink
|
|
|
37
37
|
# downloaded filelink
|
|
38
38
|
# @return [Unirest::Response]
|
|
39
39
|
def download(filepath)
|
|
40
|
-
send_download(filepath)
|
|
40
|
+
send_download(url, filepath)
|
|
41
41
|
end
|
|
42
42
|
|
|
43
43
|
# Delete filelink
|
|
@@ -71,6 +71,13 @@ class FilestackFilelink
|
|
|
71
71
|
send_tags('tags', @handle, @security)
|
|
72
72
|
end
|
|
73
73
|
|
|
74
|
+
# Return metadata for file handle
|
|
75
|
+
#
|
|
76
|
+
# @return [Unirest::Response]
|
|
77
|
+
def metadata(params = {})
|
|
78
|
+
send_metadata(@handle, @security, params)
|
|
79
|
+
end
|
|
80
|
+
|
|
74
81
|
# Return true (SFW) or false (NSFW)
|
|
75
82
|
#
|
|
76
83
|
# @return [Bool]
|
|
@@ -1,6 +1,8 @@
|
|
|
1
|
+
require 'filestack/config'
|
|
1
2
|
require 'filestack/utils/multipart_upload_utils'
|
|
2
3
|
require 'filestack/models/filestack_transform'
|
|
3
4
|
require 'filestack/utils/utils'
|
|
5
|
+
require 'json'
|
|
4
6
|
|
|
5
7
|
# The Filestack FilestackClient class acts as a hub for all
|
|
6
8
|
# Filestack actions that do not require a file handle, including
|
|
@@ -29,12 +31,12 @@ class FilestackClient
|
|
|
29
31
|
# @param [Hash] options User-supplied upload options
|
|
30
32
|
#
|
|
31
33
|
# return [Filestack::FilestackFilelink]
|
|
32
|
-
def upload(filepath: nil, external_url: nil, multipart: true, options: nil, storage: 's3')
|
|
34
|
+
def upload(filepath: nil, external_url: nil, multipart: true, options: nil, storage: 's3', intelligent: false, timeout: 60)
|
|
33
35
|
if filepath && external_url
|
|
34
36
|
return 'You cannot upload a URL and file at the same time'
|
|
35
37
|
end
|
|
36
38
|
response = if filepath && multipart
|
|
37
|
-
multipart_upload(@apikey, filepath, @security, options)
|
|
39
|
+
multipart_upload(@apikey, filepath, @security, options, timeout, intelligent: intelligent)
|
|
38
40
|
else
|
|
39
41
|
send_upload(
|
|
40
42
|
@apikey,
|
|
@@ -47,8 +49,20 @@ class FilestackClient
|
|
|
47
49
|
end
|
|
48
50
|
FilestackFilelink.new(response['handle'], security: @security, apikey: @apikey)
|
|
49
51
|
end
|
|
50
|
-
|
|
52
|
+
# Transform an external URL
|
|
53
|
+
#
|
|
54
|
+
# @param [string] external_url A valid URL
|
|
55
|
+
#
|
|
56
|
+
# @return [Filestack::Transform]
|
|
51
57
|
def transform_external(external_url)
|
|
52
58
|
Transform.new(external_url: external_url, security: @security, apikey: @apikey)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def zip(destination, files)
|
|
62
|
+
encoded_files = JSON.generate(files).gsub('"', '')
|
|
63
|
+
zip_url = "#{FilestackConfig::CDN_URL}/#{@apikey}/zip/#{encoded_files}"
|
|
64
|
+
escaped_zip_url = zip_url.gsub("[","%5B").gsub("]","%5D")
|
|
65
|
+
response = UploadUtils.make_call(escaped_zip_url, 'get')
|
|
66
|
+
File.write(destination, response.body)
|
|
53
67
|
end
|
|
54
68
|
end
|
|
@@ -4,18 +4,22 @@ require 'mimemagic'
|
|
|
4
4
|
require 'json'
|
|
5
5
|
require 'parallel'
|
|
6
6
|
require 'unirest'
|
|
7
|
-
|
|
7
|
+
require 'progress_bar'
|
|
8
8
|
require 'filestack/config'
|
|
9
9
|
require 'filestack/utils/utils'
|
|
10
10
|
|
|
11
11
|
include UploadUtils
|
|
12
|
-
|
|
12
|
+
include IntelligentUtils
|
|
13
|
+
Unirest.timeout(30)
|
|
13
14
|
# Includes all the utility functions for Filestack multipart uploads
|
|
14
15
|
module MultipartUploadUtils
|
|
15
16
|
def get_file_info(file)
|
|
16
17
|
filename = File.basename(file)
|
|
17
18
|
filesize = File.size(file)
|
|
18
19
|
mimetype = MimeMagic.by_magic(File.open(file))
|
|
20
|
+
if mimetype.nil?
|
|
21
|
+
mimetype = 'application/octet-stream'
|
|
22
|
+
end
|
|
19
23
|
[filename, filesize, mimetype.to_s]
|
|
20
24
|
end
|
|
21
25
|
|
|
@@ -38,9 +42,10 @@ module MultipartUploadUtils
|
|
|
38
42
|
mimetype: mimetype,
|
|
39
43
|
size: filesize,
|
|
40
44
|
store_location: options.nil? ? 's3' : options[:store_location],
|
|
41
|
-
file: Tempfile.new(filename)
|
|
45
|
+
file: Tempfile.new(filename),
|
|
46
|
+
options: options,
|
|
47
|
+
'multipart' => 'true'
|
|
42
48
|
}
|
|
43
|
-
|
|
44
49
|
params = params.merge!(options) if options
|
|
45
50
|
|
|
46
51
|
unless security.nil?
|
|
@@ -55,7 +60,7 @@ module MultipartUploadUtils
|
|
|
55
60
|
if response.code == 200
|
|
56
61
|
response.body
|
|
57
62
|
else
|
|
58
|
-
raise
|
|
63
|
+
raise RuntimeError.new(response.body)
|
|
59
64
|
end
|
|
60
65
|
end
|
|
61
66
|
|
|
@@ -67,8 +72,6 @@ module MultipartUploadUtils
|
|
|
67
72
|
# @param [Int] filesize Size of incoming file
|
|
68
73
|
# @param [Unirest::Response] start_response Response body from
|
|
69
74
|
# multipart_start
|
|
70
|
-
# @param [FilestackSecurity] security Security object with
|
|
71
|
-
# policy/signature
|
|
72
75
|
# @param [Hash] options User-defined options for
|
|
73
76
|
# multipart uploads
|
|
74
77
|
#
|
|
@@ -78,24 +81,35 @@ module MultipartUploadUtils
|
|
|
78
81
|
part = 1
|
|
79
82
|
seek_point = 0
|
|
80
83
|
while seek_point < filesize
|
|
81
|
-
|
|
84
|
+
part_info = {
|
|
82
85
|
seek: seek_point,
|
|
83
86
|
filepath: filepath,
|
|
84
87
|
filename: filename,
|
|
85
88
|
apikey: apikey,
|
|
86
89
|
part: part,
|
|
90
|
+
filesize: filesize,
|
|
87
91
|
uri: start_response['uri'],
|
|
88
92
|
region: start_response['region'],
|
|
89
93
|
upload_id: start_response['upload_id'],
|
|
90
94
|
location_url: start_response['location_url'],
|
|
95
|
+
start_response: start_response,
|
|
96
|
+
options: options,
|
|
91
97
|
store_location: options.nil? ? 's3' : options[:store_location]
|
|
92
|
-
|
|
98
|
+
}
|
|
99
|
+
if seek_point + FilestackConfig::DEFAULT_CHUNK_SIZE > filesize
|
|
100
|
+
size = filesize - (seek_point)
|
|
101
|
+
else
|
|
102
|
+
size = FilestackConfig::DEFAULT_CHUNK_SIZE
|
|
103
|
+
end
|
|
104
|
+
part_info[:size] = size
|
|
105
|
+
jobs.push(part_info)
|
|
93
106
|
part += 1
|
|
94
107
|
seek_point += FilestackConfig::DEFAULT_CHUNK_SIZE
|
|
95
108
|
end
|
|
96
109
|
jobs
|
|
97
110
|
end
|
|
98
111
|
|
|
112
|
+
|
|
99
113
|
# Uploads one chunk of the file
|
|
100
114
|
#
|
|
101
115
|
# @param [Hash] job Hash of options needed
|
|
@@ -112,6 +126,7 @@ module MultipartUploadUtils
|
|
|
112
126
|
file = File.open(filepath)
|
|
113
127
|
file.seek(job[:seek])
|
|
114
128
|
chunk = file.read(FilestackConfig::DEFAULT_CHUNK_SIZE)
|
|
129
|
+
|
|
115
130
|
md5 = Digest::MD5.new
|
|
116
131
|
md5 << chunk
|
|
117
132
|
data = {
|
|
@@ -122,7 +137,7 @@ module MultipartUploadUtils
|
|
|
122
137
|
uri: job[:uri],
|
|
123
138
|
region: job[:region],
|
|
124
139
|
upload_id: job[:upload_id],
|
|
125
|
-
store_location:
|
|
140
|
+
store_location: job[:store_location],
|
|
126
141
|
file: Tempfile.new(job[:filename])
|
|
127
142
|
}
|
|
128
143
|
data = data.merge!(options) if options
|
|
@@ -134,7 +149,6 @@ module MultipartUploadUtils
|
|
|
134
149
|
fs_response['url'], headers: fs_response['headers'], parameters: chunk
|
|
135
150
|
)
|
|
136
151
|
end
|
|
137
|
-
|
|
138
152
|
# Runs all jobs in parallel
|
|
139
153
|
#
|
|
140
154
|
# @param [Array] jobs Array of jobs to be run
|
|
@@ -145,17 +159,20 @@ module MultipartUploadUtils
|
|
|
145
159
|
#
|
|
146
160
|
# @return [Array] Array of parts/etags strings
|
|
147
161
|
def run_uploads(jobs, apikey, filepath, options)
|
|
148
|
-
|
|
162
|
+
bar = ProgressBar.new(jobs.length)
|
|
163
|
+
results = Parallel.map(jobs, in_threads: 4) do |job|
|
|
149
164
|
response = upload_chunk(
|
|
150
165
|
job, apikey, filepath, options
|
|
151
166
|
)
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
167
|
+
if response.code == 200
|
|
168
|
+
bar.increment!
|
|
169
|
+
part = job[:part]
|
|
170
|
+
etag = response.headers[:etag]
|
|
171
|
+
"#{part}:#{etag}"
|
|
172
|
+
end
|
|
155
173
|
end
|
|
156
174
|
results
|
|
157
175
|
end
|
|
158
|
-
|
|
159
176
|
# Send complete call to multipart endpoint
|
|
160
177
|
#
|
|
161
178
|
# @param [String] apikey Filestack API key
|
|
@@ -173,20 +190,34 @@ module MultipartUploadUtils
|
|
|
173
190
|
# multipart uploads
|
|
174
191
|
#
|
|
175
192
|
# @return [Unirest::Response]
|
|
176
|
-
def multipart_complete(apikey, filename, filesize, mimetype, start_response, parts_and_etags, options)
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
193
|
+
def multipart_complete(apikey, filename, filesize, mimetype, start_response, parts_and_etags, options, intelligent = false)
|
|
194
|
+
if !intelligent
|
|
195
|
+
data = {
|
|
196
|
+
apikey: apikey,
|
|
197
|
+
uri: start_response['uri'],
|
|
198
|
+
region: start_response['region'],
|
|
199
|
+
upload_id: start_response['upload_id'],
|
|
200
|
+
filename: filename,
|
|
201
|
+
size: filesize,
|
|
202
|
+
mimetype: mimetype,
|
|
203
|
+
parts: parts_and_etags.join(';'),
|
|
204
|
+
store_location: options.nil? ? 's3' : options[:store_location],
|
|
205
|
+
file: Tempfile.new(filename)
|
|
206
|
+
}
|
|
207
|
+
else
|
|
208
|
+
data = {
|
|
209
|
+
apikey: apikey,
|
|
210
|
+
uri: start_response['uri'],
|
|
211
|
+
region: start_response['region'],
|
|
212
|
+
upload_id: start_response['upload_id'],
|
|
213
|
+
filename: filename,
|
|
214
|
+
size: filesize,
|
|
215
|
+
mimetype: mimetype,
|
|
216
|
+
store_location: options.nil? ? 's3' : options[:store_location],
|
|
217
|
+
file: Tempfile.new(filename),
|
|
218
|
+
'multipart' => 'true'
|
|
219
|
+
}
|
|
220
|
+
end
|
|
190
221
|
data = data.merge!(options) if options
|
|
191
222
|
|
|
192
223
|
Unirest.post(
|
|
@@ -205,19 +236,43 @@ module MultipartUploadUtils
|
|
|
205
236
|
# multipart uploads
|
|
206
237
|
#
|
|
207
238
|
# @return [Unirest::Response]
|
|
208
|
-
def multipart_upload(apikey, filepath, security, options)
|
|
239
|
+
def multipart_upload(apikey, filepath, security, options, timeout, intelligent: false)
|
|
209
240
|
filename, filesize, mimetype = get_file_info(filepath)
|
|
210
241
|
start_response = multipart_start(
|
|
211
242
|
apikey, filename, filesize, mimetype, security, options
|
|
212
243
|
)
|
|
244
|
+
unless start_response['upload_type'].nil?
|
|
245
|
+
intelligent_enabled = ((start_response['upload_type'].include? 'intelligent_ingestion')) && intelligent
|
|
246
|
+
end
|
|
213
247
|
jobs = create_upload_jobs(
|
|
214
248
|
apikey, filename, filepath, filesize, start_response, options
|
|
215
249
|
)
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
250
|
+
if intelligent_enabled
|
|
251
|
+
state = IntelligentState.new
|
|
252
|
+
run_intelligent_upload_flow(jobs, state)
|
|
253
|
+
response_complete = multipart_complete(
|
|
254
|
+
apikey, filename, filesize, mimetype,
|
|
255
|
+
start_response, nil, options, intelligent
|
|
256
|
+
)
|
|
257
|
+
else
|
|
258
|
+
parts_and_etags = run_uploads(jobs, apikey, filepath, options)
|
|
259
|
+
response_complete = multipart_complete(
|
|
260
|
+
apikey, filename, filesize, mimetype,
|
|
261
|
+
start_response, parts_and_etags, options
|
|
262
|
+
)
|
|
263
|
+
end
|
|
264
|
+
begin
|
|
265
|
+
Timeout::timeout(timeout){
|
|
266
|
+
while response_complete.code == 202
|
|
267
|
+
response_complete = multipart_complete(
|
|
268
|
+
apikey, filename, filesize, mimetype,
|
|
269
|
+
start_response, nil, options, intelligent
|
|
270
|
+
)
|
|
271
|
+
end
|
|
272
|
+
}
|
|
273
|
+
rescue
|
|
274
|
+
raise "Upload timed out upon completion. Please try again later"
|
|
275
|
+
end
|
|
221
276
|
response_complete.body
|
|
222
277
|
end
|
|
223
278
|
end
|