kithe 2.0.0.pre.alpha2 → 2.0.0.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/jobs/kithe/create_derivatives_job.rb +2 -2
- data/app/models/kithe/asset.rb +82 -154
- data/app/models/kithe/asset/derivative_creator.rb +32 -62
- data/app/models/kithe/asset/derivative_definition.rb +12 -13
- data/app/models/kithe/asset/set_shrine_uploader.rb +64 -0
- data/app/models/kithe/collection.rb +0 -6
- data/app/models/kithe/model.rb +0 -21
- data/app/models/kithe/work.rb +0 -5
- data/app/uploaders/kithe/asset_uploader.rb +15 -78
- data/lib/kithe/version.rb +4 -1
- data/lib/shrine/plugins/kithe_checksum_signatures.rb +41 -0
- data/lib/shrine/plugins/kithe_controllable_backgrounding.rb +53 -0
- data/lib/shrine/plugins/kithe_derivative_definitions.rb +101 -0
- data/lib/shrine/plugins/kithe_derivatives.rb +54 -0
- data/lib/shrine/plugins/kithe_determine_mime_type.rb +39 -0
- data/lib/shrine/plugins/kithe_persisted_derivatives.rb +161 -0
- data/lib/shrine/plugins/kithe_promotion_callbacks.rb +4 -0
- data/lib/shrine/plugins/kithe_promotion_directives.rb +33 -3
- data/lib/shrine/plugins/kithe_storage_location.rb +53 -4
- data/lib/tasks/kithe_tasks.rake +22 -15
- data/spec/dummy/log/development.log +867 -0
- data/spec/dummy/log/test.log +26005 -0
- data/spec/models/kithe/asset/asset_derivatives_spec.rb +137 -0
- data/spec/models/kithe/asset/asset_promotion_hooks_spec.rb +26 -5
- data/spec/models/kithe/asset/set_shrine_uploader_spec.rb +39 -0
- data/spec/models/kithe/asset_spec.rb +9 -59
- data/spec/models/kithe/model_spec.rb +0 -32
- data/spec/shrine/kithe_accept_remote_url_spec.rb +49 -0
- data/spec/shrine/kithe_checksum_signatures_spec.rb +63 -0
- data/spec/shrine/kithe_derivative_definitions_spec.rb +303 -0
- data/spec/shrine/kithe_persisted_derivatives_spec.rb +424 -0
- data/spec/shrine/kithe_storage_location_spec.rb +43 -15
- data/spec/spec_helper.rb +7 -6
- data/spec/test_support/images/3x3_pixel.jpg +0 -0
- data/spec/test_support/shrine_spec_support.rb +2 -1
- metadata +23 -23
- data/app/models/kithe/asset/derivative_updater.rb +0 -119
- data/app/models/kithe/derivative.rb +0 -15
- data/app/uploaders/kithe/derivative_uploader.rb +0 -48
- data/spec/models/kithe/asset/asset_create_derivatives_spec.rb +0 -320
- data/spec/models/kithe/derivative_spec.rb +0 -168
@@ -0,0 +1,54 @@
|
|
1
|
+
require 'mini_mime'
|
2
|
+
|
3
|
+
class Shrine
|
4
|
+
module Plugins
|
5
|
+
# Includes the Shrine `derivatives` plugin with some configuration, and
|
6
|
+
# extra features. The metadata for shrine derivatives is stored in the same
|
7
|
+
# JSON as the main file.
|
8
|
+
#
|
9
|
+
# * default kithe storage location of :kithe_derivatives
|
10
|
+
#
|
11
|
+
# * nice metadata["filename"] for derivatives, instead of default shrine fairly
|
12
|
+
# random (filename ends up used by default in content-disposition headers when delivered)
|
13
|
+
#
|
14
|
+
# * Includes kithe_persisted_derivatives with #add_persisted_derivatives
|
15
|
+
# and #create_persisted_derivatives methods for concurrency-safe
|
16
|
+
# derivative persisting.
|
17
|
+
#
|
18
|
+
# ## Shrine derivatives references
|
19
|
+
#
|
20
|
+
# https://shrinerb.com/docs/plugins/derivatives
|
21
|
+
# https://shrinerb.com/docs/processing
|
22
|
+
class KitheDerivatives
|
23
|
+
def self.load_dependencies(uploader, *)
|
24
|
+
uploader.plugin :derivatives, storage: -> (derivative) do
|
25
|
+
# default derivatives storage to
|
26
|
+
:kithe_derivatives
|
27
|
+
end
|
28
|
+
|
29
|
+
uploader.plugin :kithe_persisted_derivatives
|
30
|
+
uploader.plugin :kithe_derivative_definitions
|
31
|
+
end
|
32
|
+
|
33
|
+
module InstanceMethods
|
34
|
+
|
35
|
+
# Override to fix "filename" metadata to be something reasonable, regardless
|
36
|
+
# of what if anything was the filename of the IO being attached. shrine S3 will
|
37
|
+
# insist on setting a default content-disposition with this filename.
|
38
|
+
def extract_metadata(io, derivative:nil, **context)
|
39
|
+
result = super
|
40
|
+
|
41
|
+
if derivative && context[:record] && result["mime_type"]
|
42
|
+
extension = MiniMime.lookup_by_content_type(result["mime_type"] || "")&.extension || "bin"
|
43
|
+
result["filename"] = "#{context[:record].friendlier_id}_#{derivative}.#{extension}"
|
44
|
+
end
|
45
|
+
|
46
|
+
result
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
end
|
51
|
+
register_plugin(:kithe_derivatives, KitheDerivatives)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
@@ -0,0 +1,39 @@
|
|
1
|
+
class Shrine
|
2
|
+
module Plugins
|
3
|
+
# Custom kithe logic for determining mime type, using the shrine mime_type plugin.
|
4
|
+
#
|
5
|
+
# We start out using the `marcel` analyzer.
|
6
|
+
# Marcel analyzer is pure-ruby and fast. It's from Basecamp and is what
|
7
|
+
# ActiveStorage uses. It is very similar to :mimemagic (and uses mimemagic
|
8
|
+
# under the hood), but mimemagic seems not to be maintained with up to date
|
9
|
+
# magic db? https://github.com/minad/mimemagic/pull/66
|
10
|
+
#
|
11
|
+
# But marcel is not able to catch some of our MP3s as audio/mpeg. The
|
12
|
+
# `mediainfo` CLI is, and is one of the tools Harvard FITS uses.
|
13
|
+
# If marcel came up blank, AND we are configured to use mediainfo CLI
|
14
|
+
# (which by default we will be if it's available), we will try
|
15
|
+
# shelling out to mediainfo command line.
|
16
|
+
#
|
17
|
+
# https://github.com/MediaArea/MediaInfo
|
18
|
+
#
|
19
|
+
# Ensure that if mime-type can't be otherwise determined, it is assigned
|
20
|
+
# "application/octet-stream", basically the type for generic binary.
|
21
|
+
class KitheDetermineMimeType
|
22
|
+
def self.load_dependencies(uploader, *)
|
23
|
+
uploader.plugin :determine_mime_type, analyzer: -> (io, analyzers) do
|
24
|
+
mime_type = analyzers[:marcel].call(io)
|
25
|
+
|
26
|
+
|
27
|
+
if Kithe.use_mediainfo && mime_type == "application/octet-stream" || mime_type.blank?
|
28
|
+
mime_type = Kithe::MediainfoAnalyzer.new.call(io)
|
29
|
+
end
|
30
|
+
|
31
|
+
mime_type = "application/octet-stream" if mime_type.blank?
|
32
|
+
|
33
|
+
mime_type
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
register_plugin(:kithe_determine_mime_type, KitheDetermineMimeType)
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,161 @@
|
|
1
|
+
class Shrine
|
2
|
+
module Plugins
|
3
|
+
# Some convenience methods for adding/changing derivatives in
|
4
|
+
# concurrency-safe manner:
|
5
|
+
#
|
6
|
+
# * Won't make a change if the underlying original has changed
|
7
|
+
# so doesn't match the one you wanted to remove.
|
8
|
+
# * Won't over-write changes made concurrently in the db by other processes
|
9
|
+
# * Will always make sure to clean up any temporary files on all error
|
10
|
+
# and falure conditions.
|
11
|
+
#
|
12
|
+
# Shrine has some building blocks for this, which we use, but it's a bit tricky
|
13
|
+
# to put them together to be generically reliable, as we think we've done here.
|
14
|
+
#
|
15
|
+
# All these methods will cause your Asset model to be saved, because of how
|
16
|
+
# the shrine atomic helpers work. So these methods will by default raise
|
17
|
+
# a TypeError if your Asset model has any unrelated outstanding changes,
|
18
|
+
# but you can tell it to save anyway with `allow_other_changes: true`.
|
19
|
+
#
|
20
|
+
# ## Shrine references:
|
21
|
+
#
|
22
|
+
# https://shrinerb.com/docs/plugins/derivatives
|
23
|
+
# https://shrinerb.com/docs/processing
|
24
|
+
class KithePersistedDerivatives
|
25
|
+
module AttacherMethods
|
26
|
+
# Like the shrine `add_derivatives` method, but also *persists* the
|
27
|
+
# derivatives (saves to db), in a realiably concurrency-safe way.
|
28
|
+
#
|
29
|
+
# Generally can take any options that shrine `add_derivatives`
|
30
|
+
# can take, including custom `storage` or `metadata` arguments.
|
31
|
+
#
|
32
|
+
# Like shrine add_derivatives, it will assume the files passed in are
|
33
|
+
# temporary, and delete them for you. If you want to disable this behavior:
|
34
|
+
#
|
35
|
+
# attacher.add_persisted_derivatives({key: io}, delete: false)
|
36
|
+
#
|
37
|
+
# In some cases the derivatives can't be persisted because the underlying
|
38
|
+
# database has changed such that they would not be applicable. In those
|
39
|
+
# cases `false` will be return value, otherwise returns the new derivatives
|
40
|
+
# just as shrine `add_derivatives`
|
41
|
+
#
|
42
|
+
# Because the concurrent-safe persistence method will save the associated model --
|
43
|
+
# and save without ActiveRecord validation -- it is not safe to
|
44
|
+
# add_persisted_derivatives on a model with other unsaved changes. The
|
45
|
+
# method will by default refuse to do so, throwing a TypeError. If you'd
|
46
|
+
# like to force it, pass `allow_other_changes: true` as an argument.
|
47
|
+
#
|
48
|
+
# Also takes care of deleting any replaced derivative files, that are no longer
|
49
|
+
# referenced by the model. Shrine by default does not do this:
|
50
|
+
# https://github.com/shrinerb/shrine/issues/468
|
51
|
+
#
|
52
|
+
# All deletions are inline. In general this could be a fairly expensive operation,
|
53
|
+
# it can be wise to do it in a bg job.
|
54
|
+
def add_persisted_derivatives(local_files, **options)
|
55
|
+
other_changes_allowed = !!options.delete(:allow_other_changes)
|
56
|
+
if record && !other_changes_allowed && record.changed?
|
57
|
+
raise TypeError.new("Can't safely add_persisted_derivatives on model with unsaved changes. Pass `allow_other_changes: true` to force.")
|
58
|
+
end
|
59
|
+
|
60
|
+
existing_derivative_files = nil
|
61
|
+
|
62
|
+
# upload to storage
|
63
|
+
new_derivatives = upload_derivatives(local_files, **options)
|
64
|
+
|
65
|
+
begin
|
66
|
+
atomic_persist do |reloaded_attacher|
|
67
|
+
# record so we can delete any replaced ones...
|
68
|
+
existing_derivative_files = map_derivative(reloaded_attacher.derivatives).collect { |path, file| file }
|
69
|
+
|
70
|
+
# make sure we don't override derivatives created in other jobs, by
|
71
|
+
# first using the current up-to-date derivatives from db,
|
72
|
+
# then merging our changes in on top.
|
73
|
+
set_derivatives(reloaded_attacher.derivatives)
|
74
|
+
merge_derivatives(new_derivatives)
|
75
|
+
end
|
76
|
+
rescue Shrine::AttachmentChanged, ActiveRecord::RecordNotFound => e
|
77
|
+
# underlying file has changed or model has been deleted, inappropriate
|
78
|
+
# to add the derivatives, we can just silently drop them, but clean
|
79
|
+
# up after ourselves.
|
80
|
+
delete_derivatives(local_files) unless options[:delete] == false
|
81
|
+
delete_derivatives(new_derivatives)
|
82
|
+
|
83
|
+
return false
|
84
|
+
rescue StandardError => e
|
85
|
+
# unexpected error, clean up our files and re-raise
|
86
|
+
delete_derivatives(local_files) unless options[:delete] == false
|
87
|
+
delete_derivatives(new_derivatives)
|
88
|
+
raise e
|
89
|
+
end
|
90
|
+
|
91
|
+
# Take care of deleting from storage any derivatives that were replaced.
|
92
|
+
current_derivative_files = map_derivative(derivatives).collect { |path, file| file }
|
93
|
+
replaced_files = existing_derivative_files - current_derivative_files
|
94
|
+
delete_derivatives(replaced_files)
|
95
|
+
|
96
|
+
new_derivatives
|
97
|
+
end
|
98
|
+
|
99
|
+
# Like the shrine `create_derivatives` method, but persists the created derivatives
|
100
|
+
# to the database in a concurrency-safe way.
|
101
|
+
#
|
102
|
+
# Can take all options that shrine `create_derivatives` can take, including custom
|
103
|
+
# processors, custom storage key, and arbitrary custom processor arguments.
|
104
|
+
#
|
105
|
+
# asset.file_attacher.create_persisted_derivatives
|
106
|
+
# asset.file_attacher.create_persisted_derivatives(storage: :custom_key)
|
107
|
+
# asset.file_attacher.create_persisted_derivatives(:kithe_derivatives)
|
108
|
+
# asset.file_attacher.create_persisted_derivatives(:kithe_derivatives, some_arg: "value")
|
109
|
+
# asset.file_attacher.create_persisted_derivatives(:kithe_derivatives, alternate_source_file)
|
110
|
+
#
|
111
|
+
# Also has an `allow_other_changes` argument, see #add_persisted_derivatives.
|
112
|
+
def create_persisted_derivatives(*args, storage: nil, allow_other_changes: false, **options)
|
113
|
+
return false unless file
|
114
|
+
|
115
|
+
local_files = process_derivatives(*args, **options)
|
116
|
+
add_persisted_derivatives(local_files, storage: storage, allow_other_changes: allow_other_changes)
|
117
|
+
end
|
118
|
+
|
119
|
+
# Kind of like built-in Shrine #remove_derivatives, but also takes care of
|
120
|
+
# persisting AND deleting the removed derivative file from storage --
|
121
|
+
# all in concurrency-safe way, including not making sure to overwrite
|
122
|
+
# any unrelated derivatives someone else was adding.
|
123
|
+
#
|
124
|
+
# Can take the same sorts of path arguments as Shrine derivative #remove_derivatives
|
125
|
+
#
|
126
|
+
# asset.file_attacher.remove_persisted_derivatives(:small_thumb)
|
127
|
+
# asset.file_attacher.remove_persisted_derivatives(:small_thumb, :large_thumb)
|
128
|
+
# asset.file_attacher.remove_persisted_derivatives(:small_thumb, :large_thumb, allow_other_changes: true)
|
129
|
+
def remove_persisted_derivatives(*paths, **options)
|
130
|
+
return if paths.empty?
|
131
|
+
|
132
|
+
other_changes_allowed = !!options.delete(:allow_other_changes)
|
133
|
+
if record && !other_changes_allowed && record.changed?
|
134
|
+
raise TypeError.new("Can't safely add_persisted_derivatives on model with unsaved changes. Pass `allow_other_changes: true` to force.")
|
135
|
+
end
|
136
|
+
|
137
|
+
removed_derivatives = nil
|
138
|
+
atomic_persist do |reloaded_attacher|
|
139
|
+
set_derivatives(reloaded_attacher.derivatives)
|
140
|
+
removed_derivatives = remove_derivatives(*paths, delete: false)
|
141
|
+
end
|
142
|
+
|
143
|
+
if removed_derivatives
|
144
|
+
map_derivative(removed_derivatives) do |_, derivative|
|
145
|
+
derivative.delete if derivative
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
removed_derivatives
|
150
|
+
rescue Shrine::AttachmentChanged, ActiveRecord::RecordNotFound
|
151
|
+
# original was already deleted or changed, the derivatives wer'e trying to delete.
|
152
|
+
# It should be fine to do nothing, the process that deleted or changed
|
153
|
+
# the model should already have deleted all these derivatives.
|
154
|
+
# But we'll return false as a signel.
|
155
|
+
return false
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
register_plugin(:kithe_persisted_derivatives, KithePersistedDerivatives)
|
160
|
+
end
|
161
|
+
end
|
@@ -24,6 +24,10 @@ class Shrine
|
|
24
24
|
# was convenient and avoided confusion to isolate wrapping in a class method that can be used
|
25
25
|
# anywhere, and only depends on args passed in, no implicit state anywhere.
|
26
26
|
class KithePromotionCallbacks
|
27
|
+
def self.load_dependencies(uploader, *)
|
28
|
+
uploader.plugin :kithe_promotion_directives
|
29
|
+
end
|
30
|
+
|
27
31
|
# promotion logic differs somewhat in different modes of use (bg or inline promotion),
|
28
32
|
# so we extract the wrapping logic here. Exactly what the logic wrapped is can
|
29
33
|
# differ.
|
@@ -89,15 +89,15 @@ class Shrine
|
|
89
89
|
# some_model.save!
|
90
90
|
def set_promotion_directives(hash)
|
91
91
|
# ActiveJob sometimes has trouble if there are symbols in there, somewhat
|
92
|
-
# unpredictably.
|
93
|
-
hash = hash.collect { |k, v| [k.to_s, v
|
92
|
+
# unpredictably. And for other reasons, standardize on everything a string.
|
93
|
+
hash = hash.collect { |k, v| [k.to_s, v.to_s]}.to_h
|
94
94
|
|
95
95
|
unrecognized = hash.keys.collect(&:to_sym) - KithePromotionDirectives.allowed_promotion_directives
|
96
96
|
unless unrecognized.length == 0
|
97
97
|
raise ArgumentError.new("Unrecognized promotion directive key: #{unrecognized.join('')}")
|
98
98
|
end
|
99
99
|
|
100
|
-
promotion_directives.merge
|
100
|
+
context[:promotion_directives] = promotion_directives.merge(hash).freeze
|
101
101
|
end
|
102
102
|
|
103
103
|
# context[:promotion_directives], lazily initializing to hash for convenience.
|
@@ -105,6 +105,36 @@ class Shrine
|
|
105
105
|
context[:promotion_directives] ||= {}
|
106
106
|
end
|
107
107
|
end
|
108
|
+
|
109
|
+
# VERY hacky way to try to preserve promotion_directives on Asset.reload.
|
110
|
+
#
|
111
|
+
# This may not be necessary in a future shrine version if shrine resolves
|
112
|
+
# issue. See: https://github.com/shrinerb/shrine/issues/463
|
113
|
+
#
|
114
|
+
# It is the activerecord plugin implementation that erases all shrine context
|
115
|
+
# (and thus our promotion directives) on reload.
|
116
|
+
# https://github.com/shrinerb/shrine/blob/b5fc2e1432e51e6fde87c120bc6cf6abeb286c68/lib/shrine/plugins/activerecord.rb#L56-L60
|
117
|
+
#
|
118
|
+
# It is quite tricky to override the activerecord plugin's own override, because
|
119
|
+
# of the way shrine does these overrides. We've figured out a pretty crazy way
|
120
|
+
# below.
|
121
|
+
module AttachmentMethods
|
122
|
+
def included(model)
|
123
|
+
super
|
124
|
+
|
125
|
+
original_reload = instance_method(:reload)
|
126
|
+
|
127
|
+
define_method :reload do |*args|
|
128
|
+
previous_promotion_directives = file_attacher.promotion_directives
|
129
|
+
|
130
|
+
result = original_reload.bind(self).call(*args)
|
131
|
+
file_attacher.set_promotion_directives(previous_promotion_directives)
|
132
|
+
|
133
|
+
result
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
108
138
|
end
|
109
139
|
register_plugin(:kithe_promotion_directives, KithePromotionDirectives)
|
110
140
|
end
|
@@ -2,25 +2,74 @@ require 'shrine/storage/url'
|
|
2
2
|
|
3
3
|
class Shrine
|
4
4
|
module Plugins
|
5
|
-
# Set
|
6
|
-
#
|
5
|
+
# Set custom storage locations/paths for both the original file which is the main
|
6
|
+
# file in the shrine attachment at Asset#file, and any shrine derivatives.
|
7
|
+
#
|
8
|
+
# Shrine's default is to just put both of these at top-level `[randomID].suffix`. We
|
9
|
+
# instead:
|
10
|
+
#
|
11
|
+
# ## Original file
|
12
|
+
#
|
13
|
+
# Stored at `asset/#{asset_uuid_id}/#{unique_file_id}.suffix` -- regardless of
|
14
|
+
# asset sub-class, since they all have unique ids, just all under asset/. (In retrospect,
|
15
|
+
# maybe shoudl have left `asset/` off, and let consumer specify a prefix when configuring
|
16
|
+
# storage).
|
7
17
|
#
|
8
18
|
# If no Asset pk is available (direct upload or unsaved Asset), will be stored just
|
9
19
|
# under "asset/#{unique_file_id}.#{suffix}"
|
10
20
|
#
|
11
21
|
# We are choosing to store under Asset UUID PK instead of friendlier_id, friendlier_id
|
12
22
|
# is good for public URLs and UI, but actual PK is more reliable/immutable.
|
23
|
+
#
|
24
|
+
# ## Derivatives
|
25
|
+
#
|
26
|
+
# Stored at `#{asset_uuid_id}/derivative_key/#{unique_file_id}.suffix`.
|
27
|
+
#
|
28
|
+
# If asset uuid pk is not available, will raise a TypeError and refuse to store
|
29
|
+
# derivative. (This may have to be thought through more.)
|
30
|
+
#
|
31
|
+
# If you want an additional prefix, supply it hwen configuring kithe_derivatives
|
32
|
+
# storage.
|
13
33
|
module KitheStorageLocation
|
14
34
|
module InstanceMethods
|
15
|
-
def generate_location(io, context)
|
35
|
+
def generate_location(io, derivative: nil, **context)
|
36
|
+
original = super
|
37
|
+
|
38
|
+
if derivative
|
39
|
+
_kithe_generate_derivative_location(io, original: original, derivative: derivative, **context)
|
40
|
+
else
|
41
|
+
_kithe_generate_main_location(io, original: original, **context)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def _kithe_generate_main_location(io, original:, **context)
|
16
48
|
# If it doesn't have a id, we're probably storing in cache, possibly as part
|
17
49
|
# of direct upload endpoint. A better path will be created on store.
|
18
50
|
id = context[:record].id if context[:record].respond_to?(:id)
|
19
51
|
|
20
|
-
basename =
|
52
|
+
basename = original
|
21
53
|
|
22
54
|
["asset", id, basename].compact.join("/")
|
23
55
|
end
|
56
|
+
|
57
|
+
# Usually NOT in the same bucket/prefix as the originals/main attachments.
|
58
|
+
# You can set a prefix yourself in your shrine storage config if you want them
|
59
|
+
# on the same bucket, and probably should.
|
60
|
+
def _kithe_generate_derivative_location(io, original:, derivative:, record:, **context)
|
61
|
+
# for now to be save, insist the record exist and have an id so we can get the
|
62
|
+
# correct derivative location. This is consistent with kithe 1.x behavior. We can
|
63
|
+
# enhance later maybe.
|
64
|
+
unless record && record.id
|
65
|
+
raise TypeError.new("Can't determine correct derivative location without a persisted record. Record: #{record}")
|
66
|
+
end
|
67
|
+
unless derivative && original
|
68
|
+
raise ArgumentError.new("Missing required argument")
|
69
|
+
end
|
70
|
+
|
71
|
+
[record.id, derivative, original].join("/")
|
72
|
+
end
|
24
73
|
end
|
25
74
|
end
|
26
75
|
register_plugin(:kithe_storage_location, KitheStorageLocation)
|
data/lib/tasks/kithe_tasks.rake
CHANGED
@@ -8,11 +8,10 @@ namespace :kithe do
|
|
8
8
|
options = {}
|
9
9
|
OptionParser.new do |opts|
|
10
10
|
opts.banner = "Usage: ./bin/rake kithe:create_derivatives -- [options]"
|
11
|
-
opts.on("--derivatives
|
12
|
-
opts.on("--lazy","Lazy create") { options[:lazy] = true }
|
13
|
-
opts.on("--asset-id
|
14
|
-
opts.on("--work-id
|
15
|
-
opts.on("--mark-derivatives-created", "set derivatives_created? flag on assets") { |ids| options[:mark_derivatives_created] = true }
|
11
|
+
opts.on("--derivatives=TYPES", "comma-seperated list of type keys") { |ids| options[:derivative_keys] = ids.split(",")}
|
12
|
+
opts.on("--lazy", "Lazy create") { options[:lazy] = true }
|
13
|
+
opts.on("--asset-id=FRIENDLIER_IDS", "comma-seperated list of asset (friendlier) ids") { |ids| options[:asset_ids] = ids.split(",") }
|
14
|
+
opts.on("--work-id=FRIENDLIER_IDS", "comma-seperated list of work (friendlier) ids") { |ids| options[:work_ids] = ids.split(",") }
|
16
15
|
end.tap do |parser|
|
17
16
|
parser.parse!(parser.order(ARGV) {})
|
18
17
|
end
|
@@ -22,17 +21,20 @@ namespace :kithe do
|
|
22
21
|
scope = scope.joins(:parent).where("parents_kithe_models.friendlier_id": options[:work_ids])
|
23
22
|
end
|
24
23
|
scope = scope.where(friendlier_id: options[:asset_ids]) if options[:asset_ids]
|
25
|
-
scope = scope.includes(:derivatives) if options[:lazy]
|
26
24
|
|
27
25
|
progress_bar = ProgressBar.create(total: scope.count, format: Kithe::STANDARD_PROGRESS_BAR_FORMAT)
|
28
26
|
|
29
27
|
scope.find_each do |asset|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
28
|
+
begin
|
29
|
+
progress_bar.title = asset.friendlier_id
|
30
|
+
asset.create_derivatives(
|
31
|
+
only: options[:derivative_keys],
|
32
|
+
lazy: !!options[:lazy]
|
33
|
+
)
|
34
|
+
rescue Shrine::FileNotFound => e
|
35
|
+
progress_bar.log("original missing for #{asset.friendlier_id}")
|
36
|
+
# it's cool, skip it
|
37
|
+
end
|
36
38
|
progress_bar.increment
|
37
39
|
end
|
38
40
|
end
|
@@ -43,9 +45,14 @@ namespace :kithe do
|
|
43
45
|
task :lazy_defaults => :environment do
|
44
46
|
progress_bar = ProgressBar.create(total: Kithe::Asset.count, format: Kithe::STANDARD_PROGRESS_BAR_FORMAT)
|
45
47
|
|
46
|
-
Kithe::Asset.
|
47
|
-
|
48
|
-
|
48
|
+
Kithe::Asset.find_each do |asset|
|
49
|
+
begin
|
50
|
+
progress_bar.title = asset.friendlier_id
|
51
|
+
asset.create_derivatives(lazy: true)
|
52
|
+
rescue Shrine::FileNotFound => e
|
53
|
+
progress_bar.log("original missing for #{asset.friendlier_id}")
|
54
|
+
# it's cool, skip it
|
55
|
+
end
|
49
56
|
progress_bar.increment
|
50
57
|
end
|
51
58
|
end
|