cloudinary 1.29.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +0 -1
- data/CHANGELOG.md +94 -0
- data/README.md +5 -3
- data/cloudinary.gemspec +25 -50
- data/lib/active_storage/service/cloudinary_service.rb +61 -8
- data/lib/cloudinary/account_api.rb +39 -8
- data/lib/cloudinary/analytics.rb +157 -0
- data/lib/cloudinary/api.rb +68 -10
- data/lib/cloudinary/auth_token.rb +1 -5
- data/lib/cloudinary/base_api.rb +36 -31
- data/lib/cloudinary/carrier_wave/storage.rb +2 -1
- data/lib/cloudinary/carrier_wave.rb +0 -5
- data/lib/cloudinary/helper.rb +3 -11
- data/lib/cloudinary/migrator.rb +70 -71
- data/lib/cloudinary/uploader.rb +70 -90
- data/lib/cloudinary/utils.rb +32 -55
- data/lib/cloudinary/version.rb +1 -1
- data/lib/cloudinary.rb +3 -9
- data/lib/tasks/cloudinary/fetch_assets.rake +9 -3
- data/vendor/assets/javascripts/cloudinary/jquery.cloudinary.js +3 -3
- metadata +179 -38
- data/lib/cloudinary/ostruct2.rb +0 -284
data/lib/cloudinary/api.rb
CHANGED
@@ -14,6 +14,22 @@ class Cloudinary::Api
|
|
14
14
|
call_api(:get, "ping", {}, options)
|
15
15
|
end
|
16
16
|
|
17
|
+
# Retrieves account configuration details.
|
18
|
+
#
|
19
|
+
# @param [Hash] options The optional parameters.
|
20
|
+
#
|
21
|
+
# @return [Cloudinary::Api::Response]
|
22
|
+
#
|
23
|
+
# @raise [Cloudinary::Api::Error]
|
24
|
+
#
|
25
|
+
# @see https://cloudinary.com/documentation/admin_api#config
|
26
|
+
def self.config(options={})
|
27
|
+
uri = "config"
|
28
|
+
params = only(options, :settings)
|
29
|
+
|
30
|
+
call_api(:get, uri, params, options)
|
31
|
+
end
|
32
|
+
|
17
33
|
# Gets cloud usage details.
|
18
34
|
#
|
19
35
|
# Returns a report detailing your current Cloudinary cloud usage details, including
|
@@ -649,7 +665,7 @@ class Cloudinary::Api
|
|
649
665
|
#
|
650
666
|
# @see https://cloudinary.com/documentation/admin_api#update_an_upload_preset
|
651
667
|
def self.update_upload_preset(name, options={})
|
652
|
-
params = Cloudinary::Uploader.build_upload_params(options)
|
668
|
+
params = Cloudinary::Uploader.build_upload_params(options, true)
|
653
669
|
call_api(:put, "upload_presets/#{name}", params.merge(only(options, :unsigned, :disallow_public_id, :live)), options)
|
654
670
|
end
|
655
671
|
|
@@ -664,7 +680,7 @@ class Cloudinary::Api
|
|
664
680
|
#
|
665
681
|
# @see https://cloudinary.com/documentation/admin_api#create_an_upload_preset
|
666
682
|
def self.create_upload_preset(options={})
|
667
|
-
params = Cloudinary::Uploader.build_upload_params(options)
|
683
|
+
params = Cloudinary::Uploader.build_upload_params(options, true)
|
668
684
|
call_api(:post, "upload_presets", params.merge(only(options, :name, :unsigned, :disallow_public_id, :live)), options)
|
669
685
|
end
|
670
686
|
|
@@ -731,6 +747,21 @@ class Cloudinary::Api
|
|
731
747
|
call_api(:post, "folders/#{folder_name}", {}, options)
|
732
748
|
end
|
733
749
|
|
750
|
+
# Renames existing asset folder.
|
751
|
+
#
|
752
|
+
# @param [String] from_path The full path of an existing asset folder.
|
753
|
+
# @param [String] to_path The full path of the new asset folder.
|
754
|
+
# @param [Hash] options The optional parameters.
|
755
|
+
#
|
756
|
+
# @return [Cloudinary::Api::Response]
|
757
|
+
#
|
758
|
+
# @raise [Cloudinary::Api::Error]
|
759
|
+
#
|
760
|
+
# @see https://cloudinary.com/documentation/admin_api#rename_folder
|
761
|
+
def self.rename_folder(from_path, to_path, options={})
|
762
|
+
call_api(:put, "folders/#{from_path}", {:to_folder => to_path}, options)
|
763
|
+
end
|
764
|
+
|
734
765
|
# Lists upload mappings by folder and its mapped template (URL).
|
735
766
|
#
|
736
767
|
# @param [Hash] options The optional parameters. See the
|
@@ -1006,9 +1037,7 @@ class Cloudinary::Api
|
|
1006
1037
|
#
|
1007
1038
|
# @see https://cloudinary.com/documentation/admin_api#create_a_metadata_field
|
1008
1039
|
def self.add_metadata_field(field, options = {})
|
1009
|
-
|
1010
|
-
|
1011
|
-
call_metadata_api(:post, [], params, options)
|
1040
|
+
call_metadata_api(:post, [], prepare_metadata_field_params(field), options)
|
1012
1041
|
end
|
1013
1042
|
|
1014
1043
|
# Updates a metadata field by external ID.
|
@@ -1025,10 +1054,19 @@ class Cloudinary::Api
|
|
1025
1054
|
#
|
1026
1055
|
# @see https://cloudinary.com/documentation/admin_api#update_a_metadata_field_by_external_id
|
1027
1056
|
def self.update_metadata_field(field_external_id, field, options = {})
|
1028
|
-
uri
|
1029
|
-
params = only(field, :label, :mandatory, :default_value, :validation)
|
1057
|
+
uri = [field_external_id]
|
1030
1058
|
|
1031
|
-
call_metadata_api(:put, uri,
|
1059
|
+
call_metadata_api(:put, uri, prepare_metadata_field_params(field), options)
|
1060
|
+
end
|
1061
|
+
|
1062
|
+
# Prepares optional parameters for add/update_metadata_field API calls.
|
1063
|
+
# @param [Hash] options Additional options
|
1064
|
+
# @return [Object] Optional parameters
|
1065
|
+
def self.prepare_metadata_field_params(field)
|
1066
|
+
only(field,
|
1067
|
+
:type, :external_id, :label, :mandatory, :restrictions, :default_value, :default_disabled,
|
1068
|
+
:validation, :datasource, :allow_dynamic_list_values
|
1069
|
+
)
|
1032
1070
|
end
|
1033
1071
|
|
1034
1072
|
# Deletes a metadata field definition by external ID.
|
@@ -1222,6 +1260,25 @@ class Cloudinary::Api
|
|
1222
1260
|
call_metadata_rules_api(:delete, uri, {}, options)
|
1223
1261
|
end
|
1224
1262
|
|
1263
|
+
# Analyzes an asset with the requested analysis type.
|
1264
|
+
#
|
1265
|
+
# @param [Object] input_type The type of input for the asset to analyze ('uri').
|
1266
|
+
# @param [Object] analysis_type The type of analysis to run ('google_tagging', 'captioning', 'fashion').
|
1267
|
+
# @param [Hash] options The optional parameters.
|
1268
|
+
#
|
1269
|
+
# @return [Cloudinary::Api::Response]
|
1270
|
+
#
|
1271
|
+
# @raise [Cloudinary::Api::Error]
|
1272
|
+
def self.analyze(input_type, analysis_type, options = {})
|
1273
|
+
api_uri = ["analysis", "analyze", input_type]
|
1274
|
+
params = only(options, :uri, :parameters)
|
1275
|
+
params["analysis_type"] = analysis_type
|
1276
|
+
|
1277
|
+
options[:api_version] = 'v2'
|
1278
|
+
|
1279
|
+
call_api(:post, api_uri, params, options)
|
1280
|
+
end
|
1281
|
+
|
1225
1282
|
protected
|
1226
1283
|
|
1227
1284
|
# Execute a call api for input params.
|
@@ -1236,13 +1293,14 @@ class Cloudinary::Api
|
|
1236
1293
|
api_key = options[:api_key] || Cloudinary.config.api_key
|
1237
1294
|
api_secret = options[:api_secret] || Cloudinary.config.api_secret
|
1238
1295
|
oauth_token = options[:oauth_token] || Cloudinary.config.oauth_token
|
1296
|
+
api_version = options[:api_version] || Cloudinary.config.api_version || 'v1_1'
|
1239
1297
|
|
1240
1298
|
validate_authorization(api_key, api_secret, oauth_token)
|
1241
1299
|
|
1242
1300
|
auth = { :key => api_key, :secret => api_secret, :oauth_token => oauth_token }
|
1243
1301
|
|
1244
1302
|
call_cloudinary_api(method, uri, auth, params, options) do |cloudinary, inner_uri|
|
1245
|
-
[cloudinary,
|
1303
|
+
[cloudinary, api_version, cloud_name, inner_uri]
|
1246
1304
|
end
|
1247
1305
|
end
|
1248
1306
|
|
@@ -1254,7 +1312,7 @@ class Cloudinary::Api
|
|
1254
1312
|
return Cloudinary::Utils.json_decode(response.body)
|
1255
1313
|
rescue => e
|
1256
1314
|
# Error is parsing json
|
1257
|
-
raise GeneralError.new("Error parsing server response (#{response.
|
1315
|
+
raise GeneralError.new("Error parsing server response (#{response.status}) - #{response.body}. Got - #{e}")
|
1258
1316
|
end
|
1259
1317
|
|
1260
1318
|
# Protected function that assists with performing an API call to the metadata_fields part of the Admin API.
|
data/lib/cloudinary/base_api.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
-
require "
|
1
|
+
require "faraday"
|
2
2
|
require "json"
|
3
3
|
|
4
4
|
module Cloudinary::BaseApi
|
5
|
+
@adapter = nil
|
5
6
|
class Error < CloudinaryException; end
|
6
7
|
class NotFound < Error; end
|
7
8
|
class NotAllowed < Error; end
|
@@ -15,17 +16,19 @@ module Cloudinary::BaseApi
|
|
15
16
|
attr_reader :rate_limit_reset_at, :rate_limit_remaining, :rate_limit_allowed
|
16
17
|
|
17
18
|
def initialize(response=nil)
|
18
|
-
|
19
|
-
|
20
|
-
|
19
|
+
unless response
|
20
|
+
return
|
21
|
+
end
|
21
22
|
|
22
|
-
|
23
|
-
|
23
|
+
# This sets the instantiated self as the response Hash
|
24
|
+
update Cloudinary::Api.parse_json_response response
|
24
25
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
26
|
+
# According to RFC 2616, header names are case-insensitive.
|
27
|
+
lc_headers = response.headers.transform_keys(&:downcase)
|
28
|
+
|
29
|
+
@rate_limit_allowed = lc_headers["x-featureratelimit-limit"].to_i if lc_headers["x-featureratelimit-limit"]
|
30
|
+
@rate_limit_reset_at = Time.parse(lc_headers["x-featureratelimit-reset"]) if lc_headers["x-featureratelimit-reset"]
|
31
|
+
@rate_limit_remaining = lc_headers["x-featureratelimit-remaining"].to_i if lc_headers["x-featureratelimit-remaining"]
|
29
32
|
end
|
30
33
|
end
|
31
34
|
|
@@ -36,28 +39,30 @@ module Cloudinary::BaseApi
|
|
36
39
|
end
|
37
40
|
|
38
41
|
def call_json_api(method, api_url, payload, timeout, headers, proxy = nil, user = nil, password = nil)
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
headers: headers,
|
44
|
-
proxy: proxy,
|
45
|
-
user: user,
|
46
|
-
password: password) do |response|
|
47
|
-
return Response.new(response) if response.code == 200
|
48
|
-
exception_class = case response.code
|
49
|
-
when 400 then BadRequest
|
50
|
-
when 401 then AuthorizationRequired
|
51
|
-
when 403 then NotAllowed
|
52
|
-
when 404 then NotFound
|
53
|
-
when 409 then AlreadyExists
|
54
|
-
when 420 then RateLimited
|
55
|
-
when 500 then GeneralError
|
56
|
-
else raise GeneralError.new("Server returned unexpected status code - #{response.code} - #{response.body}")
|
57
|
-
end
|
58
|
-
json = Cloudinary::Api.parse_json_response(response)
|
59
|
-
raise exception_class.new(json["error"]["message"])
|
42
|
+
conn = Faraday.new(url: api_url) do |faraday|
|
43
|
+
faraday.proxy = proxy if proxy
|
44
|
+
faraday.request :json
|
45
|
+
faraday.adapter @adapter || Faraday.default_adapter
|
60
46
|
end
|
47
|
+
|
48
|
+
response = conn.run_request(method.downcase.to_sym, nil, payload, headers) do |req|
|
49
|
+
req.options.timeout = timeout if timeout
|
50
|
+
req.basic_auth(user, password) if user && password
|
51
|
+
end
|
52
|
+
|
53
|
+
return Response.new(response) if response.status == 200
|
54
|
+
exception_class = case response.status
|
55
|
+
when 400 then BadRequest
|
56
|
+
when 401 then AuthorizationRequired
|
57
|
+
when 403 then NotAllowed
|
58
|
+
when 404 then NotFound
|
59
|
+
when 409 then AlreadyExists
|
60
|
+
when 420, 429 then RateLimited
|
61
|
+
when 500 then GeneralError
|
62
|
+
else raise GeneralError.new("Server returned unexpected status code - #{response.status} - #{response.body}")
|
63
|
+
end
|
64
|
+
json = Cloudinary::Api.parse_json_response(response)
|
65
|
+
raise exception_class.new(json["error"]["message"])
|
61
66
|
end
|
62
67
|
|
63
68
|
private
|
@@ -36,7 +36,8 @@ class Cloudinary::CarrierWave::Storage < ::CarrierWave::Storage::Abstract
|
|
36
36
|
params[:type]=uploader.class.storage_type
|
37
37
|
|
38
38
|
params[:resource_type] ||= :auto
|
39
|
-
uploader.
|
39
|
+
upload_method = uploader.respond_to?(:upload_chunked?) && uploader.upload_chunked? ? "upload_large" : "upload"
|
40
|
+
uploader.metadata = Cloudinary::Uploader.send(upload_method, data, params)
|
40
41
|
if uploader.metadata["error"]
|
41
42
|
raise Cloudinary::CarrierWave::UploadError.new(uploader.metadata["error"]["message"], uploader.metadata["error"]["http_code"])
|
42
43
|
end
|
data/lib/cloudinary/helper.rb
CHANGED
@@ -179,7 +179,7 @@ module CloudinaryHelper
|
|
179
179
|
version_store = options.delete(:version_store)
|
180
180
|
if options[:version].blank? && (version_store == :file) && defined?(Rails) && defined?(Rails.root)
|
181
181
|
file_name = "#{Rails.root}/tmp/cloudinary/cloudinary_sprite_#{source.sub(/\..*/, '')}.version"
|
182
|
-
if File.
|
182
|
+
if File.exist?(file_name)
|
183
183
|
options[:version] = File.read(file_name).chomp
|
184
184
|
end
|
185
185
|
end
|
@@ -286,13 +286,6 @@ module CloudinaryHelper
|
|
286
286
|
Cloudinary::Utils.private_download_url(public_id, format, options)
|
287
287
|
end
|
288
288
|
|
289
|
-
# Helper method that uses the deprecated ZIP download API.
|
290
|
-
# Replaced by cl_download_zip_url that uses the more advanced and robust archive generation and download API
|
291
|
-
# @deprecated
|
292
|
-
def cl_zip_download_url(tag, options = {})
|
293
|
-
Cloudinary::Utils.zip_download_url(tag, options)
|
294
|
-
end
|
295
|
-
|
296
289
|
# @see {Cloudinary::Utils.download_archive_url}
|
297
290
|
def cl_download_archive_url(options = {})
|
298
291
|
Cloudinary::Utils.download_archive_url(options)
|
@@ -304,13 +297,13 @@ module CloudinaryHelper
|
|
304
297
|
end
|
305
298
|
|
306
299
|
def cl_signed_download_url(public_id, options = {})
|
307
|
-
Cloudinary::Utils.
|
300
|
+
Cloudinary::Utils.cloudinary_url(public_id, options)
|
308
301
|
end
|
309
302
|
|
310
303
|
def self.included(base)
|
311
304
|
ActionView::Helpers::FormBuilder.send(:include, Cloudinary::FormBuilder)
|
312
305
|
base.class_eval do
|
313
|
-
|
306
|
+
unless method_defined?(:image_tag)
|
314
307
|
include ActionView::Helpers::AssetTagHelper
|
315
308
|
end
|
316
309
|
alias_method :image_tag_without_cloudinary, :image_tag unless public_method_defined? :image_tag_without_cloudinary
|
@@ -325,7 +318,6 @@ module CloudinaryHelper
|
|
325
318
|
private
|
326
319
|
|
327
320
|
def cloudinary_url_internal(source, options = {})
|
328
|
-
options[:ssl_detected] = request.ssl? if defined?(request) && request && request.respond_to?(:ssl?)
|
329
321
|
if defined?(CarrierWave::Uploader::Base) && source.is_a?(CarrierWave::Uploader::Base)
|
330
322
|
if source.version_name.present?
|
331
323
|
options[:transformation] = Cloudinary::Utils.build_array(source.transformation) + Cloudinary::Utils.build_array(options[:transformation])
|
data/lib/cloudinary/migrator.rb
CHANGED
@@ -6,29 +6,29 @@ class Cloudinary::Migrator
|
|
6
6
|
attr_reader :db
|
7
7
|
attr_reader :work, :results, :mutex
|
8
8
|
attr_reader :extra_options
|
9
|
-
|
10
|
-
|
9
|
+
|
10
|
+
|
11
11
|
@@init = false
|
12
12
|
def self.init
|
13
13
|
return if @@init
|
14
14
|
@@init = true
|
15
15
|
|
16
|
-
begin
|
16
|
+
begin
|
17
17
|
require 'sqlite3'
|
18
18
|
rescue LoadError
|
19
|
-
raise "Please add sqlite3 to your Gemfile"
|
19
|
+
raise "Please add sqlite3 to your Gemfile"
|
20
20
|
end
|
21
21
|
require 'tempfile'
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
def json_decode(str)
|
25
25
|
Cloudinary::Utils.json_decode(str)
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def initialize(options={})
|
29
29
|
self.class.init
|
30
|
-
|
31
|
-
options[:db_file] = "tmp/migration#{$$}.db" if options[:private_database] && !options[:db_file]
|
30
|
+
|
31
|
+
options[:db_file] = "tmp/migration#{$$}.db" if options[:private_database] && !options[:db_file]
|
32
32
|
@dbfile = options[:db_file] || "tmp/migration.db"
|
33
33
|
FileUtils.mkdir_p(File.dirname(@dbfile))
|
34
34
|
@db = SQLite3::Database.new @dbfile, :results_as_hash=>true
|
@@ -37,7 +37,6 @@ class Cloudinary::Migrator
|
|
37
37
|
@debug = options[:debug] || false
|
38
38
|
@ignore_duplicates = options[:ignore_duplicates]
|
39
39
|
@threads = [options[:threads] || 10, 100].min
|
40
|
-
@threads = 1 if RUBY_VERSION < "1.9"
|
41
40
|
@extra_options = {:api_key=>options[:api_key], :api_secret=>options[:api_secret]}
|
42
41
|
@delete_after_done = options[:delete_after_done] || options[:private_database]
|
43
42
|
@max_processing = @threads * 10
|
@@ -77,7 +76,7 @@ class Cloudinary::Migrator
|
|
77
76
|
@db.execute("delete from queue")
|
78
77
|
end
|
79
78
|
end
|
80
|
-
|
79
|
+
|
81
80
|
def register_retrieve(&block)
|
82
81
|
@retrieve = block
|
83
82
|
end
|
@@ -85,26 +84,26 @@ class Cloudinary::Migrator
|
|
85
84
|
def register_complete(&block)
|
86
85
|
@complete = block
|
87
86
|
end
|
88
|
-
|
89
|
-
def process(options={})
|
87
|
+
|
88
|
+
def process(options={})
|
90
89
|
raise CloudinaryException, "url not given and no retrieve callback given" if options[:url].nil? && self.retrieve.nil?
|
91
90
|
raise CloudinaryException, "id not given and retrieve or complete callback given" if options[:id].nil? && (!self.retrieve.nil? || !self.complete.nil?)
|
92
91
|
|
93
92
|
debug("Process: #{options.inspect}")
|
94
93
|
start
|
95
|
-
process_results
|
94
|
+
process_results
|
96
95
|
wait_for_queue
|
97
96
|
options = options.dup
|
98
97
|
id = options.delete(:id)
|
99
98
|
url = options.delete(:url)
|
100
99
|
public_id = options.delete(:public_id)
|
101
100
|
row = {
|
102
|
-
"internal_id"=>id,
|
103
|
-
"url"=>url,
|
104
|
-
"public_id"=>public_id,
|
101
|
+
"internal_id"=>id,
|
102
|
+
"url"=>url,
|
103
|
+
"public_id"=>public_id,
|
105
104
|
"metadata"=>options.to_json,
|
106
|
-
"status"=>"processing"
|
107
|
-
}
|
105
|
+
"status"=>"processing"
|
106
|
+
}
|
108
107
|
begin
|
109
108
|
insert_row(row)
|
110
109
|
add_to_work_queue(row)
|
@@ -112,7 +111,7 @@ class Cloudinary::Migrator
|
|
112
111
|
raise if !@ignore_duplicates
|
113
112
|
end
|
114
113
|
end
|
115
|
-
|
114
|
+
|
116
115
|
def done
|
117
116
|
start
|
118
117
|
process_all_pending
|
@@ -122,37 +121,37 @@ class Cloudinary::Migrator
|
|
122
121
|
@db.close
|
123
122
|
File.delete(@dbfile) if @delete_after_done
|
124
123
|
end
|
125
|
-
|
124
|
+
|
126
125
|
def max_given_id
|
127
126
|
db.get_first_value("select max(internal_id) from queue").to_i
|
128
127
|
end
|
129
|
-
|
128
|
+
|
130
129
|
def close_if_needed(file)
|
131
130
|
if file.nil?
|
132
131
|
# Do nothing.
|
133
|
-
elsif file.respond_to?(:close!)
|
134
|
-
file.close!
|
132
|
+
elsif file.respond_to?(:close!)
|
133
|
+
file.close!
|
135
134
|
elsif file.respond_to?(:close)
|
136
135
|
file.close
|
137
136
|
end
|
138
137
|
rescue
|
139
138
|
# Ignore errors in closing files
|
140
|
-
end
|
139
|
+
end
|
141
140
|
|
142
|
-
def temporary_file(data, filename)
|
143
|
-
file =
|
141
|
+
def temporary_file(data, filename)
|
142
|
+
file = Tempfile.new('cloudinary', :encoding => 'ascii-8bit')
|
144
143
|
file.unlink
|
145
144
|
file.write(data)
|
146
145
|
file.rewind
|
147
|
-
# Tempfile return path == nil after unlink, which break rest-client
|
146
|
+
# Tempfile return path == nil after unlink, which break rest-client
|
148
147
|
class << file
|
149
148
|
attr_accessor :original_filename
|
150
149
|
def content_type
|
151
|
-
"application/octet-stream"
|
150
|
+
"application/octet-stream"
|
152
151
|
end
|
153
152
|
end
|
154
153
|
file.original_filename = filename
|
155
|
-
file
|
154
|
+
file
|
156
155
|
end
|
157
156
|
|
158
157
|
private
|
@@ -160,21 +159,21 @@ class Cloudinary::Migrator
|
|
160
159
|
def update_all(values)
|
161
160
|
@db.execute("update queue set #{values.keys.map{|key| "#{key}=?"}.join(",")}", *values.values)
|
162
161
|
end
|
163
|
-
|
164
|
-
def update_row(row, values)
|
162
|
+
|
163
|
+
def update_row(row, values)
|
165
164
|
values.merge!("updated_at"=>Time.now.to_i)
|
166
165
|
query = ["update queue set #{values.keys.map{|key| "#{key}=?"}.join(",")} where id=?"] + values.values + [row["id"]]
|
167
166
|
@db.execute(*query)
|
168
167
|
values.each{|key, value| row[key.to_s] = value}
|
169
|
-
row
|
168
|
+
row
|
170
169
|
end
|
171
|
-
|
170
|
+
|
172
171
|
def insert_row(values)
|
173
172
|
values.merge!("updated_at"=>Time.now.to_i)
|
174
173
|
@db.execute("insert into queue (#{values.keys.join(",")}) values (#{values.keys.map{"?"}.join(",")})", *values.values)
|
175
174
|
values["id"] = @db.last_insert_row_id
|
176
175
|
end
|
177
|
-
|
176
|
+
|
178
177
|
def refill_queue(last_id)
|
179
178
|
@db.execute("select * from queue where status in ('error', 'processing') and id > ? limit ?", last_id, 10000) do
|
180
179
|
|row|
|
@@ -183,25 +182,25 @@ class Cloudinary::Migrator
|
|
183
182
|
add_to_work_queue(row)
|
184
183
|
end
|
185
184
|
last_id
|
186
|
-
end
|
185
|
+
end
|
187
186
|
|
188
187
|
def process_results
|
189
188
|
while self.results.length > 0
|
190
189
|
row = self.results.pop
|
191
|
-
result = json_decode(row["result"])
|
190
|
+
result = json_decode(row["result"])
|
192
191
|
debug("Done ID=#{row['internal_id']}, result=#{result.inspect}")
|
193
|
-
complete.call(row["internal_id"], result) if complete
|
194
|
-
if result["error"]
|
192
|
+
complete.call(row["internal_id"], result) if complete
|
193
|
+
if result["error"]
|
195
194
|
status = case result["error"]["http_code"]
|
196
195
|
when 400, 404 then "fatal" # Problematic request. Not a server problem.
|
197
196
|
else "error"
|
198
197
|
end
|
199
198
|
else
|
200
199
|
status = "completed"
|
201
|
-
end
|
200
|
+
end
|
202
201
|
updates = {:status=>status, :result=>row["result"]}
|
203
202
|
updates["public_id"] = result["public_id"] if result["public_id"] && !row["public_id"]
|
204
|
-
begin
|
203
|
+
begin
|
205
204
|
update_row(row, updates)
|
206
205
|
rescue SQLite3::ConstraintException
|
207
206
|
updates = {:status=>"error", :result=>{:error=>{:message=>"public_id already exists"}}.to_json}
|
@@ -219,16 +218,16 @@ class Cloudinary::Migrator
|
|
219
218
|
raise if retry_count > tries
|
220
219
|
sleep rand * 3
|
221
220
|
retry
|
222
|
-
end
|
221
|
+
end
|
223
222
|
end
|
224
|
-
|
223
|
+
|
225
224
|
def start
|
226
225
|
return if @started
|
227
226
|
@started = true
|
228
227
|
@terminate = false
|
229
|
-
|
228
|
+
|
230
229
|
self.work.clear
|
231
|
-
|
230
|
+
|
232
231
|
main = self
|
233
232
|
Thread.abort_on_exception = true
|
234
233
|
1.upto(@threads) do
|
@@ -251,8 +250,8 @@ class Cloudinary::Migrator
|
|
251
250
|
elsif defined?(::CarrierWave) && defined?(Cloudinary::CarrierWave) && data.is_a?(Cloudinary::CarrierWave)
|
252
251
|
cw = true
|
253
252
|
begin
|
254
|
-
data.model.save!
|
255
|
-
rescue Cloudinary::CarrierWave::UploadError
|
253
|
+
data.model.save!
|
254
|
+
rescue Cloudinary::CarrierWave::UploadError
|
256
255
|
# upload errors will be handled by the result values.
|
257
256
|
end
|
258
257
|
result = data.metadata
|
@@ -264,22 +263,22 @@ class Cloudinary::Migrator
|
|
264
263
|
elsif data.match(/^https?:/)
|
265
264
|
url = data
|
266
265
|
else
|
267
|
-
file = main.temporary_file(data, row["public_id"] || "cloudinaryfile")
|
266
|
+
file = main.temporary_file(data, row["public_id"] || "cloudinaryfile")
|
268
267
|
end
|
269
268
|
end
|
270
|
-
|
269
|
+
|
271
270
|
if url || file
|
272
271
|
options = main.extra_options.merge(:public_id=>row["public_id"])
|
273
272
|
json_decode(row["metadata"]).each do
|
274
273
|
|key, value|
|
275
274
|
options[key.to_sym] = value
|
276
275
|
end
|
277
|
-
|
276
|
+
|
278
277
|
result = Cloudinary::Uploader.upload(url || file, options.merge(:return_error=>true)) || ({:error=>{:message=>"Received nil from uploader!"}})
|
279
278
|
elsif cw
|
280
279
|
result ||= {"status" => "saved"}
|
281
280
|
else
|
282
|
-
result = {"error" => {"message" => "Empty data and url", "http_code"=>404}}
|
281
|
+
result = {"error" => {"message" => "Empty data and url", "http_code"=>404}}
|
283
282
|
end
|
284
283
|
main.results << {"id"=>row["id"], "internal_id"=>row["internal_id"], "result"=>result.to_json}
|
285
284
|
rescue => e
|
@@ -289,14 +288,14 @@ class Cloudinary::Migrator
|
|
289
288
|
ensure
|
290
289
|
main.mutex.synchronize{main.in_process -= 1}
|
291
290
|
main.close_if_needed(file)
|
292
|
-
end
|
291
|
+
end
|
293
292
|
end
|
294
293
|
end
|
295
|
-
end
|
296
|
-
|
294
|
+
end
|
295
|
+
|
297
296
|
retry_previous_queue # Retry all work from previous iteration before we start processing this one.
|
298
297
|
end
|
299
|
-
|
298
|
+
|
300
299
|
def debug(message)
|
301
300
|
if @debug
|
302
301
|
mutex.synchronize{
|
@@ -310,60 +309,60 @@ class Cloudinary::Migrator
|
|
310
309
|
begin
|
311
310
|
prev_last_id, last_id = last_id, refill_queue(last_id)
|
312
311
|
end while last_id > prev_last_id
|
313
|
-
process_results
|
312
|
+
process_results
|
314
313
|
end
|
315
|
-
|
314
|
+
|
316
315
|
def process_all_pending
|
317
316
|
# Waiting for work to finish. While we are at it, process results.
|
318
|
-
while self.in_process > 0
|
317
|
+
while self.in_process > 0
|
319
318
|
process_results
|
320
319
|
sleep 0.1
|
321
320
|
end
|
322
321
|
# Make sure we processed all the results
|
323
322
|
process_results
|
324
323
|
end
|
325
|
-
|
324
|
+
|
326
325
|
def add_to_work_queue(row)
|
327
326
|
self.work << row
|
328
327
|
mutex.synchronize{self.in_process += 1}
|
329
328
|
end
|
330
|
-
|
329
|
+
|
331
330
|
def wait_for_queue
|
332
331
|
# Waiting f
|
333
332
|
while self.work.length > @max_processing
|
334
|
-
process_results
|
333
|
+
process_results
|
335
334
|
sleep 0.1
|
336
|
-
end
|
335
|
+
end
|
337
336
|
end
|
338
337
|
|
339
|
-
def self.sample
|
338
|
+
def self.sample
|
340
339
|
migrator = Cloudinary::Migrator.new(
|
341
|
-
:retrieve=>proc{|id| Post.find(id).data},
|
340
|
+
:retrieve=>proc{|id| Post.find(id).data},
|
342
341
|
:complete=>proc{|id, result| a}
|
343
342
|
)
|
344
|
-
|
343
|
+
|
345
344
|
Post.find_each(:conditions=>["id > ?", migrator.max_given_id], :select=>"id") do
|
346
345
|
|post|
|
347
346
|
migrator.process(:id=>post.id, :public_id=>"post_#{post.id}")
|
348
347
|
end
|
349
348
|
migrator.done
|
350
|
-
end
|
351
|
-
|
352
|
-
def self.test
|
349
|
+
end
|
350
|
+
|
351
|
+
def self.test
|
353
352
|
posts = {}
|
354
|
-
done = {}
|
353
|
+
done = {}
|
355
354
|
migrator = Cloudinary::Migrator.new(
|
356
|
-
:retrieve=>proc{|id| posts[id]},
|
355
|
+
:retrieve=>proc{|id| posts[id]},
|
357
356
|
:complete=>proc{|id, result| $stderr.print "done #{id} #{result}\n"; done[id] = result}
|
358
357
|
)
|
359
358
|
start = migrator.max_given_id + 1
|
360
359
|
(start..1000).each{|i| posts[i] = "hello#{i}"}
|
361
|
-
|
360
|
+
|
362
361
|
posts.each do
|
363
362
|
|id, data|
|
364
363
|
migrator.process(:id=>id, :public_id=>"post_#{id}")
|
365
364
|
end
|
366
365
|
migrator.done
|
367
366
|
pp [done.length, start]
|
368
|
-
end
|
367
|
+
end
|
369
368
|
end
|