refile 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +27 -0
- data/.rspec +1 -0
- data/.travis.yml +8 -0
- data/Gemfile +3 -0
- data/LICENSE.txt +22 -0
- data/README.md +476 -0
- data/Rakefile +11 -0
- data/app/assets/javascripts/refile.js +50 -0
- data/app/helpers/attachment_helper.rb +52 -0
- data/config.ru +8 -0
- data/config/locales/en.yml +5 -0
- data/config/routes.rb +3 -0
- data/lib/refile.rb +72 -0
- data/lib/refile/app.rb +97 -0
- data/lib/refile/attachment.rb +89 -0
- data/lib/refile/attachment/active_record.rb +24 -0
- data/lib/refile/backend/file_system.rb +70 -0
- data/lib/refile/backend/s3.rb +129 -0
- data/lib/refile/file.rb +65 -0
- data/lib/refile/image_processing.rb +73 -0
- data/lib/refile/rails.rb +36 -0
- data/lib/refile/random_hasher.rb +5 -0
- data/lib/refile/version.rb +3 -0
- data/refile.gemspec +34 -0
- data/spec/refile/app_spec.rb +151 -0
- data/spec/refile/attachment_spec.rb +141 -0
- data/spec/refile/backend/file_system_spec.rb +30 -0
- data/spec/refile/backend/s3_spec.rb +11 -0
- data/spec/refile/backend_examples.rb +215 -0
- data/spec/refile/features/direct_upload_spec.rb +29 -0
- data/spec/refile/features/normal_upload_spec.rb +36 -0
- data/spec/refile/features/presigned_upload_spec.rb +29 -0
- data/spec/refile/fixtures/hello.txt +1 -0
- data/spec/refile/fixtures/large.txt +44 -0
- data/spec/refile/spec_helper.rb +58 -0
- data/spec/refile/test_app.rb +46 -0
- data/spec/refile/test_app/app/assets/javascripts/application.js +40 -0
- data/spec/refile/test_app/app/controllers/application_controller.rb +2 -0
- data/spec/refile/test_app/app/controllers/direct_posts_controller.rb +15 -0
- data/spec/refile/test_app/app/controllers/home_controller.rb +4 -0
- data/spec/refile/test_app/app/controllers/normal_posts_controller.rb +19 -0
- data/spec/refile/test_app/app/controllers/presigned_posts_controller.rb +30 -0
- data/spec/refile/test_app/app/models/post.rb +5 -0
- data/spec/refile/test_app/app/views/direct_posts/new.html.erb +16 -0
- data/spec/refile/test_app/app/views/home/index.html.erb +1 -0
- data/spec/refile/test_app/app/views/layouts/application.html.erb +14 -0
- data/spec/refile/test_app/app/views/normal_posts/new.html.erb +20 -0
- data/spec/refile/test_app/app/views/normal_posts/show.html.erb +9 -0
- data/spec/refile/test_app/app/views/presigned_posts/new.html.erb +16 -0
- data/spec/refile/test_app/config/database.yml +7 -0
- data/spec/refile/test_app/config/routes.rb +17 -0
- data/spec/refile/test_app/public/favicon.ico +0 -0
- data/spec/refile_spec.rb +35 -0
- metadata +294 -0
data/Rakefile
ADDED
@@ -0,0 +1,50 @@
|
|
1
|
+
"use strict";
|
2
|
+
|
3
|
+
document.addEventListener("change", function(e) {
|
4
|
+
if(e.target.tagName === "INPUT" && e.target.type === "file" && e.target.dataset.direct) {
|
5
|
+
var input = e.target;
|
6
|
+
var file = input.files[0];
|
7
|
+
if(file) {
|
8
|
+
var url = e.target.dataset.url;
|
9
|
+
if(e.target.dataset.fields) {
|
10
|
+
var fields = JSON.parse(e.target.dataset.fields);
|
11
|
+
}
|
12
|
+
|
13
|
+
var data = new FormData();
|
14
|
+
|
15
|
+
if(fields) {
|
16
|
+
Object.keys(fields).forEach(function(key) {
|
17
|
+
data.append(key, fields[key]);
|
18
|
+
});
|
19
|
+
}
|
20
|
+
data.append(input.dataset.as, file);
|
21
|
+
|
22
|
+
var xhr = new XMLHttpRequest();
|
23
|
+
xhr.addEventListener("load", function(e) {
|
24
|
+
input.classList.remove("uploading")
|
25
|
+
input.dispatchEvent(new CustomEvent("upload:complete", { detail: xhr.responseText, bubbles: true }));
|
26
|
+
if((xhr.status >= 200 && xhr.status < 300) || xhr.status === 304) {
|
27
|
+
var id = input.dataset.id || JSON.parse(xhr.responseText).id;
|
28
|
+
input.dispatchEvent(new CustomEvent("upload:success", { detail: xhr.responseText, bubbles: true }));
|
29
|
+
input.previousSibling.value = id;
|
30
|
+
input.removeAttribute("name");
|
31
|
+
} else {
|
32
|
+
input.dispatchEvent(new CustomEvent("upload:failure", { detail: xhr.responseText, bubbles: true }));
|
33
|
+
}
|
34
|
+
});
|
35
|
+
|
36
|
+
xhr.addEventListener("progress", function(e) {
|
37
|
+
if (e.lengthComputable) {
|
38
|
+
input.dispatchEvent(new CustomEvent("upload:progress", { detail: e, bubbles: true }));
|
39
|
+
}
|
40
|
+
});
|
41
|
+
|
42
|
+
xhr.open("POST", url, true);
|
43
|
+
xhr.send(data);
|
44
|
+
|
45
|
+
input.classList.add("uploading")
|
46
|
+
input.dispatchEvent(new CustomEvent("upload:start", { bubbles: true }));
|
47
|
+
}
|
48
|
+
}
|
49
|
+
});
|
50
|
+
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module AttachmentHelper
|
2
|
+
def attachment_url(record, name, *args, filename: nil, format: nil)
|
3
|
+
file = record.send(name)
|
4
|
+
|
5
|
+
filename ||= name.to_s
|
6
|
+
|
7
|
+
backend_name = Refile.backends.key(file.backend)
|
8
|
+
host = Refile.host || root_url
|
9
|
+
|
10
|
+
File.join(host, refile_app_path, backend_name, *args.map(&:to_s), file.id, filename.parameterize("_"))
|
11
|
+
end
|
12
|
+
|
13
|
+
def attachment_image_tag(record, name, *args, fallback: nil, format: nil, **options)
|
14
|
+
file = record.send(name)
|
15
|
+
classes = ["attachment", record.class.model_name.singular, name, *options[:class]]
|
16
|
+
|
17
|
+
if file
|
18
|
+
image_tag(attachment_url(record, name, *args, format: format), options.merge(class: classes))
|
19
|
+
elsif fallback
|
20
|
+
classes << "fallback"
|
21
|
+
image_tag(fallback, options.merge(class: classes))
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def attachment_field(object_name, method, options = {})
|
26
|
+
if options[:object]
|
27
|
+
cache = options[:object].send(:"#{method}_attachment").cache
|
28
|
+
|
29
|
+
if options[:direct]
|
30
|
+
host = Refile.host || root_url
|
31
|
+
backend_name = Refile.backends.key(cache)
|
32
|
+
|
33
|
+
options[:data] ||= {}
|
34
|
+
options[:data][:direct] = true
|
35
|
+
options[:data][:as] = "file"
|
36
|
+
options[:data][:url] = File.join(host, refile_app_path, backend_name)
|
37
|
+
end
|
38
|
+
|
39
|
+
if options[:presigned] and cache.respond_to?(:presign)
|
40
|
+
signature = cache.presign
|
41
|
+
options[:data] ||= {}
|
42
|
+
options[:data][:direct] = true
|
43
|
+
options[:data][:id] = signature.id
|
44
|
+
options[:data][:url] = signature.url
|
45
|
+
options[:data][:fields] = signature.fields
|
46
|
+
options[:data][:as] = signature.as
|
47
|
+
end
|
48
|
+
end
|
49
|
+
hidden_field(object_name, :"#{method}_cache_id", options.slice(:object)) +
|
50
|
+
file_field(object_name, method, options)
|
51
|
+
end
|
52
|
+
end
|
data/config.ru
ADDED
data/config/routes.rb
ADDED
data/lib/refile.rb
ADDED
@@ -0,0 +1,72 @@
|
|
1
|
+
require "uri"
|
2
|
+
require "fileutils"
|
3
|
+
require "tempfile"
|
4
|
+
|
5
|
+
module Refile
|
6
|
+
class Invalid < StandardError; end
|
7
|
+
|
8
|
+
class << self
|
9
|
+
attr_accessor :read_chunk_size, :app, :host, :direct_upload
|
10
|
+
attr_writer :store, :cache
|
11
|
+
|
12
|
+
def backends
|
13
|
+
@backends ||= {}
|
14
|
+
end
|
15
|
+
|
16
|
+
def processors
|
17
|
+
@processors ||= {}
|
18
|
+
end
|
19
|
+
|
20
|
+
def processor(name, processor = nil, &block)
|
21
|
+
processor ||= block
|
22
|
+
processors[name.to_s] = processor
|
23
|
+
end
|
24
|
+
|
25
|
+
def store
|
26
|
+
backends["store"]
|
27
|
+
end
|
28
|
+
|
29
|
+
def store=(backend)
|
30
|
+
backends["store"] = backend
|
31
|
+
end
|
32
|
+
|
33
|
+
def cache
|
34
|
+
backends["cache"]
|
35
|
+
end
|
36
|
+
|
37
|
+
def cache=(backend)
|
38
|
+
backends["cache"] = backend
|
39
|
+
end
|
40
|
+
|
41
|
+
def configure
|
42
|
+
yield self
|
43
|
+
end
|
44
|
+
|
45
|
+
def verify_uploadable(uploadable, max_size)
|
46
|
+
[:size, :read, :eof?, :close].each do |m|
|
47
|
+
unless uploadable.respond_to?(m)
|
48
|
+
raise ArgumentError, "does not respond to `#{m}`."
|
49
|
+
end
|
50
|
+
end
|
51
|
+
if max_size and uploadable.size > max_size
|
52
|
+
raise Refile::Invalid, "#{uploadable.inspect} is too large"
|
53
|
+
end
|
54
|
+
true
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
require "refile/version"
|
59
|
+
require "refile/attachment"
|
60
|
+
require "refile/random_hasher"
|
61
|
+
require "refile/file"
|
62
|
+
require "refile/app"
|
63
|
+
require "refile/backend/file_system"
|
64
|
+
end
|
65
|
+
|
66
|
+
Refile.configure do |config|
|
67
|
+
# FIXME: what is a sane default here? This is a little less than a
|
68
|
+
# memory page, which seemed like a good default, is there a better
|
69
|
+
# one?
|
70
|
+
config.read_chunk_size = 3000
|
71
|
+
config.direct_upload = ["cache"]
|
72
|
+
end
|
data/lib/refile/app.rb
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
require "logger"
|
2
|
+
require "json"
|
3
|
+
|
4
|
+
module Refile
|
5
|
+
class App
|
6
|
+
def initialize(logger: nil, allow_origin: nil)
|
7
|
+
@logger = logger
|
8
|
+
@logger ||= ::Logger.new(nil)
|
9
|
+
@allow_origin = allow_origin
|
10
|
+
end
|
11
|
+
|
12
|
+
class Proxy
|
13
|
+
def initialize(peek, file)
|
14
|
+
@peek = peek
|
15
|
+
@file = file
|
16
|
+
end
|
17
|
+
|
18
|
+
def close
|
19
|
+
@file.close
|
20
|
+
end
|
21
|
+
|
22
|
+
def each(&block)
|
23
|
+
block.call(@peek)
|
24
|
+
@file.each(&block)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def call(env)
|
29
|
+
@logger.info { "Refile: #{env["REQUEST_METHOD"]} #{env["PATH_INFO"]}" }
|
30
|
+
if env["REQUEST_METHOD"] == "GET"
|
31
|
+
backend_name, *process_args, id, filename = env["PATH_INFO"].sub(/^\//, "").split("/")
|
32
|
+
backend = Refile.backends[backend_name]
|
33
|
+
|
34
|
+
if backend and id
|
35
|
+
@logger.debug { "Refile: serving #{id.inspect} from #{backend_name} backend which is of type #{backend.class}" }
|
36
|
+
|
37
|
+
file = backend.get(id)
|
38
|
+
|
39
|
+
unless process_args.empty?
|
40
|
+
name = process_args.shift
|
41
|
+
unless Refile.processors[name]
|
42
|
+
@logger.debug { "Refile: no such processor #{name.inspect}" }
|
43
|
+
return not_found
|
44
|
+
end
|
45
|
+
file = Refile.processors[name].call(file, *process_args)
|
46
|
+
end
|
47
|
+
|
48
|
+
peek = begin
|
49
|
+
file.read(Refile.read_chunk_size)
|
50
|
+
rescue => e
|
51
|
+
log_error(e)
|
52
|
+
return not_found
|
53
|
+
end
|
54
|
+
|
55
|
+
headers = {}
|
56
|
+
headers["Access-Control-Allow-Origin"] = @allow_origin if @allow_origin
|
57
|
+
|
58
|
+
[200, headers, Proxy.new(peek, file)]
|
59
|
+
else
|
60
|
+
@logger.debug { "Refile: must specify backend and id" }
|
61
|
+
not_found
|
62
|
+
end
|
63
|
+
elsif env["REQUEST_METHOD"] == "POST"
|
64
|
+
backend_name, *rest = env["PATH_INFO"].sub(/^\//, "").split("/")
|
65
|
+
backend = Refile.backends[backend_name]
|
66
|
+
|
67
|
+
return not_found unless rest.empty?
|
68
|
+
return not_found unless backend and Refile.direct_upload.include?(backend_name)
|
69
|
+
|
70
|
+
file = backend.upload(Rack::Request.new(env).params.fetch("file").fetch(:tempfile))
|
71
|
+
[200, { "Content-Type" => "application/json" }, [{ id: file.id }.to_json]]
|
72
|
+
else
|
73
|
+
@logger.debug { "Refile: request methods other than GET and POST are not allowed" }
|
74
|
+
not_found
|
75
|
+
end
|
76
|
+
rescue => e
|
77
|
+
log_error(e)
|
78
|
+
[500, {}, ["error"]]
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def not_found
|
84
|
+
[404, {}, ["not found"]]
|
85
|
+
end
|
86
|
+
|
87
|
+
def log_error(e)
|
88
|
+
if @logger.debug?
|
89
|
+
@logger.debug "Refile: unable to read file"
|
90
|
+
@logger.debug "#{e.class}: #{e.message}"
|
91
|
+
e.backtrace.each do |line|
|
92
|
+
@logger.debug " #{line}"
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,89 @@
|
|
1
|
+
module Refile
|
2
|
+
module Attachment
|
3
|
+
IMAGE_TYPES = %w[jpg jpeg gif png]
|
4
|
+
|
5
|
+
class Attachment
|
6
|
+
attr_reader :record, :name, :cache, :store, :cache_id, :options
|
7
|
+
|
8
|
+
def initialize(record, name, **options)
|
9
|
+
@record = record
|
10
|
+
@name = name
|
11
|
+
@options = options
|
12
|
+
@cache = Refile.backends.fetch(@options[:cache].to_s)
|
13
|
+
@store = Refile.backends.fetch(@options[:store].to_s)
|
14
|
+
@errors = []
|
15
|
+
end
|
16
|
+
|
17
|
+
def id
|
18
|
+
record.send(:"#{name}_id")
|
19
|
+
end
|
20
|
+
|
21
|
+
def id=(id)
|
22
|
+
record.send(:"#{name}_id=", id)
|
23
|
+
end
|
24
|
+
|
25
|
+
def file
|
26
|
+
if cache_id and not cache_id == ""
|
27
|
+
cache.get(cache_id)
|
28
|
+
elsif id and not id == ""
|
29
|
+
store.get(id)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def file=(uploadable)
|
34
|
+
@cache_file = cache.upload(uploadable)
|
35
|
+
@cache_id = @cache_file.id
|
36
|
+
@errors = []
|
37
|
+
rescue Refile::Invalid
|
38
|
+
@errors = [:too_large]
|
39
|
+
raise if @options[:raise_errors]
|
40
|
+
end
|
41
|
+
|
42
|
+
def cache_id=(id)
|
43
|
+
@cache_id = id unless @cache_file
|
44
|
+
end
|
45
|
+
|
46
|
+
def store!
|
47
|
+
if cache_id and not cache_id == ""
|
48
|
+
file = store.upload(cache.get(cache_id))
|
49
|
+
cache.delete(cache_id)
|
50
|
+
store.delete(id) if id
|
51
|
+
self.id = file.id
|
52
|
+
@cache_id = nil
|
53
|
+
@cache_file = nil
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def errors
|
58
|
+
@errors
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def attachment(name, cache: :cache, store: :store, raise_errors: true)
|
63
|
+
attachment = :"#{name}_attachment"
|
64
|
+
|
65
|
+
define_method attachment do
|
66
|
+
ivar = :"@#{attachment}"
|
67
|
+
instance_variable_get(ivar) or begin
|
68
|
+
instance_variable_set(ivar, Attachment.new(self, name, cache: cache, store: store, raise_errors: raise_errors))
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
define_method "#{name}=" do |uploadable|
|
73
|
+
send(attachment).file = uploadable
|
74
|
+
end
|
75
|
+
|
76
|
+
define_method name do
|
77
|
+
send(attachment).file
|
78
|
+
end
|
79
|
+
|
80
|
+
define_method "#{name}_cache_id=" do |cache_id|
|
81
|
+
send(attachment).cache_id = cache_id
|
82
|
+
end
|
83
|
+
|
84
|
+
define_method "#{name}_cache_id" do
|
85
|
+
send(attachment).cache_id
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Refile
|
2
|
+
module ActiveRecord
|
3
|
+
module Attachment
|
4
|
+
include Refile::Attachment
|
5
|
+
|
6
|
+
def attachment(name, cache: :cache, store: :store, raise_errors: false)
|
7
|
+
super
|
8
|
+
|
9
|
+
attachment = "#{name}_attachment"
|
10
|
+
|
11
|
+
validate do
|
12
|
+
errors = send(attachment).errors
|
13
|
+
self.errors.add(name, *errors) unless errors.empty?
|
14
|
+
end
|
15
|
+
|
16
|
+
before_save do
|
17
|
+
send(attachment).store!
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
::ActiveRecord::Base.extend(Refile::ActiveRecord::Attachment)
|
@@ -0,0 +1,70 @@
|
|
1
|
+
module Refile
|
2
|
+
module Backend
|
3
|
+
class FileSystem
|
4
|
+
attr_reader :directory
|
5
|
+
|
6
|
+
def initialize(directory, max_size: nil, hasher: Refile::RandomHasher.new)
|
7
|
+
@hasher = hasher
|
8
|
+
@directory = directory
|
9
|
+
@max_size = max_size
|
10
|
+
|
11
|
+
FileUtils.mkdir_p(@directory)
|
12
|
+
end
|
13
|
+
|
14
|
+
def upload(uploadable)
|
15
|
+
Refile.verify_uploadable(uploadable, @max_size)
|
16
|
+
|
17
|
+
id = @hasher.hash(uploadable)
|
18
|
+
|
19
|
+
if uploadable.respond_to?(:path) and ::File.exist?(uploadable.path)
|
20
|
+
FileUtils.cp(uploadable.path, path(id))
|
21
|
+
else
|
22
|
+
::File.open(path(id), "wb") do |file|
|
23
|
+
buffer = "" # reuse the same buffer
|
24
|
+
until uploadable.eof?
|
25
|
+
uploadable.read(Refile.read_chunk_size, buffer)
|
26
|
+
file.write(buffer)
|
27
|
+
end
|
28
|
+
uploadable.close
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
Refile::File.new(self, id)
|
33
|
+
end
|
34
|
+
|
35
|
+
def get(id)
|
36
|
+
Refile::File.new(self, id)
|
37
|
+
end
|
38
|
+
|
39
|
+
def delete(id)
|
40
|
+
FileUtils.rm(path(id)) if exists?(id)
|
41
|
+
end
|
42
|
+
|
43
|
+
def open(id)
|
44
|
+
::File.open(path(id), "rb")
|
45
|
+
end
|
46
|
+
|
47
|
+
def read(id)
|
48
|
+
::File.read(path(id)) if exists?(id)
|
49
|
+
end
|
50
|
+
|
51
|
+
def size(id)
|
52
|
+
::File.size(path(id)) if exists?(id)
|
53
|
+
end
|
54
|
+
|
55
|
+
def exists?(id)
|
56
|
+
::File.exists?(path(id))
|
57
|
+
end
|
58
|
+
|
59
|
+
def clear!(confirm = nil)
|
60
|
+
raise ArgumentError, "are you sure? this will remove all files in the backend, call as `clear!(:confirm)` if you're sure you want to do this" unless confirm == :confirm
|
61
|
+
FileUtils.rm_rf(@directory)
|
62
|
+
FileUtils.mkdir_p(@directory)
|
63
|
+
end
|
64
|
+
|
65
|
+
def path(id)
|
66
|
+
::File.join(@directory, id)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|