refile 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +27 -0
- data/.rspec +1 -0
- data/.travis.yml +8 -0
- data/Gemfile +3 -0
- data/LICENSE.txt +22 -0
- data/README.md +476 -0
- data/Rakefile +11 -0
- data/app/assets/javascripts/refile.js +50 -0
- data/app/helpers/attachment_helper.rb +52 -0
- data/config.ru +8 -0
- data/config/locales/en.yml +5 -0
- data/config/routes.rb +3 -0
- data/lib/refile.rb +72 -0
- data/lib/refile/app.rb +97 -0
- data/lib/refile/attachment.rb +89 -0
- data/lib/refile/attachment/active_record.rb +24 -0
- data/lib/refile/backend/file_system.rb +70 -0
- data/lib/refile/backend/s3.rb +129 -0
- data/lib/refile/file.rb +65 -0
- data/lib/refile/image_processing.rb +73 -0
- data/lib/refile/rails.rb +36 -0
- data/lib/refile/random_hasher.rb +5 -0
- data/lib/refile/version.rb +3 -0
- data/refile.gemspec +34 -0
- data/spec/refile/app_spec.rb +151 -0
- data/spec/refile/attachment_spec.rb +141 -0
- data/spec/refile/backend/file_system_spec.rb +30 -0
- data/spec/refile/backend/s3_spec.rb +11 -0
- data/spec/refile/backend_examples.rb +215 -0
- data/spec/refile/features/direct_upload_spec.rb +29 -0
- data/spec/refile/features/normal_upload_spec.rb +36 -0
- data/spec/refile/features/presigned_upload_spec.rb +29 -0
- data/spec/refile/fixtures/hello.txt +1 -0
- data/spec/refile/fixtures/large.txt +44 -0
- data/spec/refile/spec_helper.rb +58 -0
- data/spec/refile/test_app.rb +46 -0
- data/spec/refile/test_app/app/assets/javascripts/application.js +40 -0
- data/spec/refile/test_app/app/controllers/application_controller.rb +2 -0
- data/spec/refile/test_app/app/controllers/direct_posts_controller.rb +15 -0
- data/spec/refile/test_app/app/controllers/home_controller.rb +4 -0
- data/spec/refile/test_app/app/controllers/normal_posts_controller.rb +19 -0
- data/spec/refile/test_app/app/controllers/presigned_posts_controller.rb +30 -0
- data/spec/refile/test_app/app/models/post.rb +5 -0
- data/spec/refile/test_app/app/views/direct_posts/new.html.erb +16 -0
- data/spec/refile/test_app/app/views/home/index.html.erb +1 -0
- data/spec/refile/test_app/app/views/layouts/application.html.erb +14 -0
- data/spec/refile/test_app/app/views/normal_posts/new.html.erb +20 -0
- data/spec/refile/test_app/app/views/normal_posts/show.html.erb +9 -0
- data/spec/refile/test_app/app/views/presigned_posts/new.html.erb +16 -0
- data/spec/refile/test_app/config/database.yml +7 -0
- data/spec/refile/test_app/config/routes.rb +17 -0
- data/spec/refile/test_app/public/favicon.ico +0 -0
- data/spec/refile_spec.rb +35 -0
- metadata +294 -0
data/Rakefile
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
"use strict";
|
2
|
+
|
3
|
+
document.addEventListener("change", function(e) {
|
4
|
+
if(e.target.tagName === "INPUT" && e.target.type === "file" && e.target.dataset.direct) {
|
5
|
+
var input = e.target;
|
6
|
+
var file = input.files[0];
|
7
|
+
if(file) {
|
8
|
+
var url = e.target.dataset.url;
|
9
|
+
if(e.target.dataset.fields) {
|
10
|
+
var fields = JSON.parse(e.target.dataset.fields);
|
11
|
+
}
|
12
|
+
|
13
|
+
var data = new FormData();
|
14
|
+
|
15
|
+
if(fields) {
|
16
|
+
Object.keys(fields).forEach(function(key) {
|
17
|
+
data.append(key, fields[key]);
|
18
|
+
});
|
19
|
+
}
|
20
|
+
data.append(input.dataset.as, file);
|
21
|
+
|
22
|
+
var xhr = new XMLHttpRequest();
|
23
|
+
xhr.addEventListener("load", function(e) {
|
24
|
+
input.classList.remove("uploading")
|
25
|
+
input.dispatchEvent(new CustomEvent("upload:complete", { detail: xhr.responseText, bubbles: true }));
|
26
|
+
if((xhr.status >= 200 && xhr.status < 300) || xhr.status === 304) {
|
27
|
+
var id = input.dataset.id || JSON.parse(xhr.responseText).id;
|
28
|
+
input.dispatchEvent(new CustomEvent("upload:success", { detail: xhr.responseText, bubbles: true }));
|
29
|
+
input.previousSibling.value = id;
|
30
|
+
input.removeAttribute("name");
|
31
|
+
} else {
|
32
|
+
input.dispatchEvent(new CustomEvent("upload:failure", { detail: xhr.responseText, bubbles: true }));
|
33
|
+
}
|
34
|
+
});
|
35
|
+
|
36
|
+
xhr.addEventListener("progress", function(e) {
|
37
|
+
if (e.lengthComputable) {
|
38
|
+
input.dispatchEvent(new CustomEvent("upload:progress", { detail: e, bubbles: true }));
|
39
|
+
}
|
40
|
+
});
|
41
|
+
|
42
|
+
xhr.open("POST", url, true);
|
43
|
+
xhr.send(data);
|
44
|
+
|
45
|
+
input.classList.add("uploading")
|
46
|
+
input.dispatchEvent(new CustomEvent("upload:start", { bubbles: true }));
|
47
|
+
}
|
48
|
+
}
|
49
|
+
});
|
50
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module AttachmentHelper
|
2
|
+
def attachment_url(record, name, *args, filename: nil, format: nil)
|
3
|
+
file = record.send(name)
|
4
|
+
|
5
|
+
filename ||= name.to_s
|
6
|
+
|
7
|
+
backend_name = Refile.backends.key(file.backend)
|
8
|
+
host = Refile.host || root_url
|
9
|
+
|
10
|
+
File.join(host, refile_app_path, backend_name, *args.map(&:to_s), file.id, filename.parameterize("_"))
|
11
|
+
end
|
12
|
+
|
13
|
+
def attachment_image_tag(record, name, *args, fallback: nil, format: nil, **options)
|
14
|
+
file = record.send(name)
|
15
|
+
classes = ["attachment", record.class.model_name.singular, name, *options[:class]]
|
16
|
+
|
17
|
+
if file
|
18
|
+
image_tag(attachment_url(record, name, *args, format: format), options.merge(class: classes))
|
19
|
+
elsif fallback
|
20
|
+
classes << "fallback"
|
21
|
+
image_tag(fallback, options.merge(class: classes))
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def attachment_field(object_name, method, options = {})
|
26
|
+
if options[:object]
|
27
|
+
cache = options[:object].send(:"#{method}_attachment").cache
|
28
|
+
|
29
|
+
if options[:direct]
|
30
|
+
host = Refile.host || root_url
|
31
|
+
backend_name = Refile.backends.key(cache)
|
32
|
+
|
33
|
+
options[:data] ||= {}
|
34
|
+
options[:data][:direct] = true
|
35
|
+
options[:data][:as] = "file"
|
36
|
+
options[:data][:url] = File.join(host, refile_app_path, backend_name)
|
37
|
+
end
|
38
|
+
|
39
|
+
if options[:presigned] and cache.respond_to?(:presign)
|
40
|
+
signature = cache.presign
|
41
|
+
options[:data] ||= {}
|
42
|
+
options[:data][:direct] = true
|
43
|
+
options[:data][:id] = signature.id
|
44
|
+
options[:data][:url] = signature.url
|
45
|
+
options[:data][:fields] = signature.fields
|
46
|
+
options[:data][:as] = signature.as
|
47
|
+
end
|
48
|
+
end
|
49
|
+
hidden_field(object_name, :"#{method}_cache_id", options.slice(:object)) +
|
50
|
+
file_field(object_name, method, options)
|
51
|
+
end
|
52
|
+
end
|
data/config.ru
ADDED
data/config/routes.rb
ADDED
data/lib/refile.rb
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
require "uri"
|
2
|
+
require "fileutils"
|
3
|
+
require "tempfile"
|
4
|
+
|
5
|
+
module Refile
|
6
|
+
class Invalid < StandardError; end
|
7
|
+
|
8
|
+
class << self
|
9
|
+
attr_accessor :read_chunk_size, :app, :host, :direct_upload
|
10
|
+
attr_writer :store, :cache
|
11
|
+
|
12
|
+
def backends
|
13
|
+
@backends ||= {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def processors
|
17
|
+
@processors ||= {}
|
18
|
+
end
|
19
|
+
|
20
|
+
def processor(name, processor = nil, &block)
|
21
|
+
processor ||= block
|
22
|
+
processors[name.to_s] = processor
|
23
|
+
end
|
24
|
+
|
25
|
+
def store
|
26
|
+
backends["store"]
|
27
|
+
end
|
28
|
+
|
29
|
+
def store=(backend)
|
30
|
+
backends["store"] = backend
|
31
|
+
end
|
32
|
+
|
33
|
+
def cache
|
34
|
+
backends["cache"]
|
35
|
+
end
|
36
|
+
|
37
|
+
def cache=(backend)
|
38
|
+
backends["cache"] = backend
|
39
|
+
end
|
40
|
+
|
41
|
+
def configure
|
42
|
+
yield self
|
43
|
+
end
|
44
|
+
|
45
|
+
def verify_uploadable(uploadable, max_size)
|
46
|
+
[:size, :read, :eof?, :close].each do |m|
|
47
|
+
unless uploadable.respond_to?(m)
|
48
|
+
raise ArgumentError, "does not respond to `#{m}`."
|
49
|
+
end
|
50
|
+
end
|
51
|
+
if max_size and uploadable.size > max_size
|
52
|
+
raise Refile::Invalid, "#{uploadable.inspect} is too large"
|
53
|
+
end
|
54
|
+
true
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
require "refile/version"
|
59
|
+
require "refile/attachment"
|
60
|
+
require "refile/random_hasher"
|
61
|
+
require "refile/file"
|
62
|
+
require "refile/app"
|
63
|
+
require "refile/backend/file_system"
|
64
|
+
end
|
65
|
+
|
66
|
+
Refile.configure do |config|
|
67
|
+
# FIXME: what is a sane default here? This is a little less than a
|
68
|
+
# memory page, which seemed like a good default, is there a better
|
69
|
+
# one?
|
70
|
+
config.read_chunk_size = 3000
|
71
|
+
config.direct_upload = ["cache"]
|
72
|
+
end
|
data/lib/refile/app.rb
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
require "logger"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
module Refile
|
5
|
+
class App
|
6
|
+
def initialize(logger: nil, allow_origin: nil)
|
7
|
+
@logger = logger
|
8
|
+
@logger ||= ::Logger.new(nil)
|
9
|
+
@allow_origin = allow_origin
|
10
|
+
end
|
11
|
+
|
12
|
+
class Proxy
|
13
|
+
def initialize(peek, file)
|
14
|
+
@peek = peek
|
15
|
+
@file = file
|
16
|
+
end
|
17
|
+
|
18
|
+
def close
|
19
|
+
@file.close
|
20
|
+
end
|
21
|
+
|
22
|
+
def each(&block)
|
23
|
+
block.call(@peek)
|
24
|
+
@file.each(&block)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def call(env)
|
29
|
+
@logger.info { "Refile: #{env["REQUEST_METHOD"]} #{env["PATH_INFO"]}" }
|
30
|
+
if env["REQUEST_METHOD"] == "GET"
|
31
|
+
backend_name, *process_args, id, filename = env["PATH_INFO"].sub(/^\//, "").split("/")
|
32
|
+
backend = Refile.backends[backend_name]
|
33
|
+
|
34
|
+
if backend and id
|
35
|
+
@logger.debug { "Refile: serving #{id.inspect} from #{backend_name} backend which is of type #{backend.class}" }
|
36
|
+
|
37
|
+
file = backend.get(id)
|
38
|
+
|
39
|
+
unless process_args.empty?
|
40
|
+
name = process_args.shift
|
41
|
+
unless Refile.processors[name]
|
42
|
+
@logger.debug { "Refile: no such processor #{name.inspect}" }
|
43
|
+
return not_found
|
44
|
+
end
|
45
|
+
file = Refile.processors[name].call(file, *process_args)
|
46
|
+
end
|
47
|
+
|
48
|
+
peek = begin
|
49
|
+
file.read(Refile.read_chunk_size)
|
50
|
+
rescue => e
|
51
|
+
log_error(e)
|
52
|
+
return not_found
|
53
|
+
end
|
54
|
+
|
55
|
+
headers = {}
|
56
|
+
headers["Access-Control-Allow-Origin"] = @allow_origin if @allow_origin
|
57
|
+
|
58
|
+
[200, headers, Proxy.new(peek, file)]
|
59
|
+
else
|
60
|
+
@logger.debug { "Refile: must specify backend and id" }
|
61
|
+
not_found
|
62
|
+
end
|
63
|
+
elsif env["REQUEST_METHOD"] == "POST"
|
64
|
+
backend_name, *rest = env["PATH_INFO"].sub(/^\//, "").split("/")
|
65
|
+
backend = Refile.backends[backend_name]
|
66
|
+
|
67
|
+
return not_found unless rest.empty?
|
68
|
+
return not_found unless backend and Refile.direct_upload.include?(backend_name)
|
69
|
+
|
70
|
+
file = backend.upload(Rack::Request.new(env).params.fetch("file").fetch(:tempfile))
|
71
|
+
[200, { "Content-Type" => "application/json" }, [{ id: file.id }.to_json]]
|
72
|
+
else
|
73
|
+
@logger.debug { "Refile: request methods other than GET and POST are not allowed" }
|
74
|
+
not_found
|
75
|
+
end
|
76
|
+
rescue => e
|
77
|
+
log_error(e)
|
78
|
+
[500, {}, ["error"]]
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def not_found
|
84
|
+
[404, {}, ["not found"]]
|
85
|
+
end
|
86
|
+
|
87
|
+
def log_error(e)
|
88
|
+
if @logger.debug?
|
89
|
+
@logger.debug "Refile: unable to read file"
|
90
|
+
@logger.debug "#{e.class}: #{e.message}"
|
91
|
+
e.backtrace.each do |line|
|
92
|
+
@logger.debug " #{line}"
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module Refile
|
2
|
+
module Attachment
|
3
|
+
IMAGE_TYPES = %w[jpg jpeg gif png]
|
4
|
+
|
5
|
+
class Attachment
|
6
|
+
attr_reader :record, :name, :cache, :store, :cache_id, :options
|
7
|
+
|
8
|
+
def initialize(record, name, **options)
|
9
|
+
@record = record
|
10
|
+
@name = name
|
11
|
+
@options = options
|
12
|
+
@cache = Refile.backends.fetch(@options[:cache].to_s)
|
13
|
+
@store = Refile.backends.fetch(@options[:store].to_s)
|
14
|
+
@errors = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def id
|
18
|
+
record.send(:"#{name}_id")
|
19
|
+
end
|
20
|
+
|
21
|
+
def id=(id)
|
22
|
+
record.send(:"#{name}_id=", id)
|
23
|
+
end
|
24
|
+
|
25
|
+
def file
|
26
|
+
if cache_id and not cache_id == ""
|
27
|
+
cache.get(cache_id)
|
28
|
+
elsif id and not id == ""
|
29
|
+
store.get(id)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def file=(uploadable)
|
34
|
+
@cache_file = cache.upload(uploadable)
|
35
|
+
@cache_id = @cache_file.id
|
36
|
+
@errors = []
|
37
|
+
rescue Refile::Invalid
|
38
|
+
@errors = [:too_large]
|
39
|
+
raise if @options[:raise_errors]
|
40
|
+
end
|
41
|
+
|
42
|
+
def cache_id=(id)
|
43
|
+
@cache_id = id unless @cache_file
|
44
|
+
end
|
45
|
+
|
46
|
+
def store!
|
47
|
+
if cache_id and not cache_id == ""
|
48
|
+
file = store.upload(cache.get(cache_id))
|
49
|
+
cache.delete(cache_id)
|
50
|
+
store.delete(id) if id
|
51
|
+
self.id = file.id
|
52
|
+
@cache_id = nil
|
53
|
+
@cache_file = nil
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def errors
|
58
|
+
@errors
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def attachment(name, cache: :cache, store: :store, raise_errors: true)
|
63
|
+
attachment = :"#{name}_attachment"
|
64
|
+
|
65
|
+
define_method attachment do
|
66
|
+
ivar = :"@#{attachment}"
|
67
|
+
instance_variable_get(ivar) or begin
|
68
|
+
instance_variable_set(ivar, Attachment.new(self, name, cache: cache, store: store, raise_errors: raise_errors))
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
define_method "#{name}=" do |uploadable|
|
73
|
+
send(attachment).file = uploadable
|
74
|
+
end
|
75
|
+
|
76
|
+
define_method name do
|
77
|
+
send(attachment).file
|
78
|
+
end
|
79
|
+
|
80
|
+
define_method "#{name}_cache_id=" do |cache_id|
|
81
|
+
send(attachment).cache_id = cache_id
|
82
|
+
end
|
83
|
+
|
84
|
+
define_method "#{name}_cache_id" do
|
85
|
+
send(attachment).cache_id
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Refile
|
2
|
+
module ActiveRecord
|
3
|
+
module Attachment
|
4
|
+
include Refile::Attachment
|
5
|
+
|
6
|
+
def attachment(name, cache: :cache, store: :store, raise_errors: false)
|
7
|
+
super
|
8
|
+
|
9
|
+
attachment = "#{name}_attachment"
|
10
|
+
|
11
|
+
validate do
|
12
|
+
errors = send(attachment).errors
|
13
|
+
self.errors.add(name, *errors) unless errors.empty?
|
14
|
+
end
|
15
|
+
|
16
|
+
before_save do
|
17
|
+
send(attachment).store!
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
::ActiveRecord::Base.extend(Refile::ActiveRecord::Attachment)
|
@@ -0,0 +1,70 @@
|
|
1
|
+
module Refile
|
2
|
+
module Backend
|
3
|
+
class FileSystem
|
4
|
+
attr_reader :directory
|
5
|
+
|
6
|
+
def initialize(directory, max_size: nil, hasher: Refile::RandomHasher.new)
|
7
|
+
@hasher = hasher
|
8
|
+
@directory = directory
|
9
|
+
@max_size = max_size
|
10
|
+
|
11
|
+
FileUtils.mkdir_p(@directory)
|
12
|
+
end
|
13
|
+
|
14
|
+
def upload(uploadable)
|
15
|
+
Refile.verify_uploadable(uploadable, @max_size)
|
16
|
+
|
17
|
+
id = @hasher.hash(uploadable)
|
18
|
+
|
19
|
+
if uploadable.respond_to?(:path) and ::File.exist?(uploadable.path)
|
20
|
+
FileUtils.cp(uploadable.path, path(id))
|
21
|
+
else
|
22
|
+
::File.open(path(id), "wb") do |file|
|
23
|
+
buffer = "" # reuse the same buffer
|
24
|
+
until uploadable.eof?
|
25
|
+
uploadable.read(Refile.read_chunk_size, buffer)
|
26
|
+
file.write(buffer)
|
27
|
+
end
|
28
|
+
uploadable.close
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
Refile::File.new(self, id)
|
33
|
+
end
|
34
|
+
|
35
|
+
def get(id)
|
36
|
+
Refile::File.new(self, id)
|
37
|
+
end
|
38
|
+
|
39
|
+
def delete(id)
|
40
|
+
FileUtils.rm(path(id)) if exists?(id)
|
41
|
+
end
|
42
|
+
|
43
|
+
def open(id)
|
44
|
+
::File.open(path(id), "rb")
|
45
|
+
end
|
46
|
+
|
47
|
+
def read(id)
|
48
|
+
::File.read(path(id)) if exists?(id)
|
49
|
+
end
|
50
|
+
|
51
|
+
def size(id)
|
52
|
+
::File.size(path(id)) if exists?(id)
|
53
|
+
end
|
54
|
+
|
55
|
+
def exists?(id)
|
56
|
+
::File.exists?(path(id))
|
57
|
+
end
|
58
|
+
|
59
|
+
def clear!(confirm = nil)
|
60
|
+
raise ArgumentError, "are you sure? this will remove all files in the backend, call as `clear!(:confirm)` if you're sure you want to do this" unless confirm == :confirm
|
61
|
+
FileUtils.rm_rf(@directory)
|
62
|
+
FileUtils.mkdir_p(@directory)
|
63
|
+
end
|
64
|
+
|
65
|
+
def path(id)
|
66
|
+
::File.join(@directory, id)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|