remote_files 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +86 -0
- data/lib/remote_files/abstract_store.rb +21 -0
- data/lib/remote_files/file.rb +59 -0
- data/lib/remote_files/fog_store.rb +69 -0
- data/lib/remote_files/mock_store.rb +25 -0
- data/lib/remote_files/resque_job.rb +15 -0
- data/lib/remote_files/version.rb +3 -0
- data/lib/remote_files.rb +87 -0
- data/test/file_test.rb +84 -0
- data/test/fog_store_test.rb +108 -0
- data/test/remote_files_test.rb +184 -0
- data/test/resque_job_test.rb +46 -0
- data/test/test_helper.rb +26 -0
- metadata +168 -0
data/README.md
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
# RemoteFiles
|
2
|
+
|
3
|
+
A library for uploading files to multiple remote storage backends like Amazon S3 and Rackspace CloudFiles.
|
4
|
+
|
5
|
+
The purpose of the library is to implement a simple interface for uploading files to multiple backends
|
6
|
+
and to keep the backends in sync, so that your app will keep working when one backend is down.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your application's Gemfile:
|
11
|
+
|
12
|
+
gem 'remote_files'
|
13
|
+
|
14
|
+
And then execute:
|
15
|
+
|
16
|
+
$ bundle
|
17
|
+
|
18
|
+
Or install it yourself as:
|
19
|
+
|
20
|
+
$ gem install remote_files
|
21
|
+
|
22
|
+
## Configuration
|
23
|
+
|
24
|
+
First you configure the storage backends you want:
|
25
|
+
|
26
|
+
```ruby
|
27
|
+
RemoteFiles.add_store(:s3, :primary => true) do |s3|
|
28
|
+
s3[:provider] = 'AWS'
|
29
|
+
|
30
|
+
s3[:aws_access_key_id] = AWS_ACCESS_KEY_ID
|
31
|
+
s3[:aws_secret_access_key] = AWS_SECRET_ACCESS_KEY
|
32
|
+
|
33
|
+
s3[:directory] = 'my_s3_bucket'
|
34
|
+
end
|
35
|
+
|
36
|
+
RemoteFiles.add_store(:cf) do |cf|
|
37
|
+
cf[:provider] = 'Rackspace'
|
38
|
+
|
39
|
+
cf[:rackspace_username] = RACKSPACE_USERNAME
|
40
|
+
cf[:rackspace_api_key] = RACKSPACE_API_KEY
|
41
|
+
|
42
|
+
cf[:directory] = 'my_cf_container'
|
43
|
+
end
|
44
|
+
```
|
45
|
+
|
46
|
+
By default RemoteFiles will store your files to all stores synchronously. This is probably not what you want,
|
47
|
+
so you should tell RemoteFiles how to do it asynchronously:
|
48
|
+
|
49
|
+
```ruby
|
50
|
+
class RemoteFilesSyncJob
|
51
|
+
def initialize(identifier, stored_in)
|
52
|
+
@file = RemoteFiles::File.new(identifier, :stored_in => stored_in)
|
53
|
+
def
|
54
|
+
|
55
|
+
def work
|
56
|
+
@file.synchronize!
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
RemoteFiles.synchronize_stores do |file|
|
61
|
+
MyPreferredJobQueue.enqueue(RemoteFilesSyncJob, file.identifier, file.stored_in)
|
62
|
+
end
|
63
|
+
```
|
64
|
+
|
65
|
+
## Usage
|
66
|
+
|
67
|
+
Once everything is configured, you can store files like this:
|
68
|
+
|
69
|
+
```ruby
|
70
|
+
file = RemoteFiles::File.new(unique_file_name, :content => file_content, :content_type => content_type)
|
71
|
+
file.store!
|
72
|
+
```
|
73
|
+
|
74
|
+
This will store the file on one of the stores and then asynchronously copy the file to the remaining stores.
|
75
|
+
`RemoteFiles::File#store!` will raise a `RemoteFiles::Error` if all storage backends are down.
|
76
|
+
|
77
|
+
If you just need to store the file in a single store, the you can use `RemoteFiles::File#store_once!`. It will
|
78
|
+
behave exactly like `RemoteFiles::File#store!`, but will not asynchronously copy the file to the other stores.
|
79
|
+
|
80
|
+
## Contributing
|
81
|
+
|
82
|
+
1. Fork it
|
83
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
84
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
85
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
86
|
+
5. Create new Pull Request
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module RemoteFiles
|
2
|
+
class AbstractStore
|
3
|
+
attr_reader :identifier
|
4
|
+
|
5
|
+
def initialize(identifier)
|
6
|
+
@identifier = identifier
|
7
|
+
end
|
8
|
+
|
9
|
+
def store!(file)
|
10
|
+
raise "You need to implement #{self.class.name}#store!"
|
11
|
+
end
|
12
|
+
|
13
|
+
def retrieve!(identifier)
|
14
|
+
raise "You need to implement #{self.class.name}#retrieve!"
|
15
|
+
end
|
16
|
+
|
17
|
+
def url(identifier)
|
18
|
+
raise "You need to implement #{self.class.name}#url"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module RemoteFiles
|
2
|
+
class File
|
3
|
+
attr_reader :content, :content_type, :identifier, :stored_in
|
4
|
+
|
5
|
+
def initialize(identifier, options = {})
|
6
|
+
@identifier = identifier
|
7
|
+
@stored_in = options[:stored_in] || []
|
8
|
+
@content = options.delete(:content)
|
9
|
+
@content_type = options[:content_type]
|
10
|
+
@options = options
|
11
|
+
end
|
12
|
+
|
13
|
+
def options
|
14
|
+
@options.merge(
|
15
|
+
:identifier => identifier,
|
16
|
+
:stored_in => stored_in,
|
17
|
+
:content_type => content_type
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def stored?
|
22
|
+
!@stored_in.empty?
|
23
|
+
end
|
24
|
+
|
25
|
+
def stored_everywhere?
|
26
|
+
missing_stores.empty?
|
27
|
+
end
|
28
|
+
|
29
|
+
def missing_stores
|
30
|
+
RemoteFiles.stores.map(&:identifier) - @stored_in
|
31
|
+
end
|
32
|
+
|
33
|
+
def url(store_identifier = nil)
|
34
|
+
store = store_identifier ? RemoteFiles.lookup_store(store_identifier) : RemoteFiles.primary_store
|
35
|
+
return nil unless store
|
36
|
+
store.url(identifier)
|
37
|
+
end
|
38
|
+
|
39
|
+
def current_url
|
40
|
+
prioritized_stores = RemoteFiles.stores.map(&:identifier) & @stored_in
|
41
|
+
|
42
|
+
return nil if prioritized_stores.empty?
|
43
|
+
|
44
|
+
url(prioritized_stores[0])
|
45
|
+
end
|
46
|
+
|
47
|
+
def store!
|
48
|
+
RemoteFiles.store!(self)
|
49
|
+
end
|
50
|
+
|
51
|
+
def store_once!
|
52
|
+
RemoteFiles.store_once!(self)
|
53
|
+
end
|
54
|
+
|
55
|
+
def synchronize!
|
56
|
+
RemoteFiles.synchronize!(self)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'remote_files/abstract_store'
|
2
|
+
require 'fog'
|
3
|
+
|
4
|
+
module RemoteFiles
|
5
|
+
class FogStore < AbstractStore
|
6
|
+
def store!(file)
|
7
|
+
sucess = directory.files.create(
|
8
|
+
:body => file.content,
|
9
|
+
:content_type => file.content_type,
|
10
|
+
:key => file.identifier,
|
11
|
+
:public => options[:public]
|
12
|
+
)
|
13
|
+
|
14
|
+
raise RemoteFiles::Error unless sucess
|
15
|
+
|
16
|
+
true
|
17
|
+
rescue Fog::Errors::Error, Excon::Errors::Error
|
18
|
+
raise RemoteFiles::Error, $!.message, $!.backtrace
|
19
|
+
end
|
20
|
+
|
21
|
+
def retrieve!(identifier)
|
22
|
+
fog_file = directory.files.get(identifier)
|
23
|
+
|
24
|
+
raise NotFoundError, "#{identifier} not found in #{self.identifier} store" if fog_file.nil?
|
25
|
+
|
26
|
+
File.new(identifier,
|
27
|
+
:content => fog_file.body,
|
28
|
+
:content_type => fog_file.content_type,
|
29
|
+
:stored_in => [self.identifier]
|
30
|
+
)
|
31
|
+
rescue Fog::Errors::Error, Excon::Errors::Error
|
32
|
+
raise RemoteFiles::Error, $!.message, $!.backtrace
|
33
|
+
end
|
34
|
+
|
35
|
+
def url(identifier)
|
36
|
+
case options[:provider]
|
37
|
+
when 'AWS'
|
38
|
+
"https://s3.amazonaws.com/#{options[:directory]}/#{Fog::AWS.escape(identifier)}"
|
39
|
+
when 'Rackspace'
|
40
|
+
"https://storage.cloudfiles.com/#{options[:directory]}/#{Fog::Rackspace.escape(identifier, '/')}"
|
41
|
+
else
|
42
|
+
raise "#{self.class.name}#url was not implemented for the #{options[:provider]} provider"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def options
|
47
|
+
@options ||= {}
|
48
|
+
end
|
49
|
+
|
50
|
+
def []=(name, value)
|
51
|
+
options[name] = value
|
52
|
+
end
|
53
|
+
|
54
|
+
def connection
|
55
|
+
connection_options = options.dup
|
56
|
+
connection_options.delete(:directory)
|
57
|
+
connection_options.delete(:public)
|
58
|
+
@connection ||= Fog::Storage.new(connection_options)
|
59
|
+
end
|
60
|
+
|
61
|
+
def directory
|
62
|
+
@directory ||= connection.directories.new(
|
63
|
+
:key => options[:directory],
|
64
|
+
:public => options[:public]
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module RemoteFiles
|
2
|
+
class MockStore < AbstractStore
|
3
|
+
def data
|
4
|
+
@data ||= {}
|
5
|
+
end
|
6
|
+
|
7
|
+
def store!(file)
|
8
|
+
data[file.identifier] = { :content => file.content, :content_type => file.content_type}
|
9
|
+
end
|
10
|
+
|
11
|
+
def retrieve!(identifier)
|
12
|
+
raise NotFoundError, "#{identifier} not found in #{self.identifier} store" unless data.has_key?(identifier)
|
13
|
+
|
14
|
+
File.new(identifier,
|
15
|
+
:content => data[identifier][:content],
|
16
|
+
:content_type => data[identifier][:content_type],
|
17
|
+
:stored_in => [self.identifier]
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def url(identifier)
|
22
|
+
"mock://#{self.identifier}/#{identifier}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'remote_files'
|
2
|
+
require 'resque'
|
3
|
+
|
4
|
+
module RemoteFiles
|
5
|
+
class ResqueJob
|
6
|
+
def self.perform(options)
|
7
|
+
file = RemoteFiles::File.new(options.delete(:identifier), options)
|
8
|
+
file.synchronize!
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
synchronize_stores do |file|
|
13
|
+
Resque.enqueue(ResqueJob, file.options)
|
14
|
+
end
|
15
|
+
end
|
data/lib/remote_files.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'remote_files/version'
|
2
|
+
require 'remote_files/fog_store'
|
3
|
+
require 'remote_files/file'
|
4
|
+
|
5
|
+
module RemoteFiles
|
6
|
+
class Error < StandardError; end
|
7
|
+
class NotFoundError < Error; end
|
8
|
+
|
9
|
+
STORES_MAP = {}
|
10
|
+
STORES = []
|
11
|
+
|
12
|
+
def self.add_store(store_identifier, options = {}, &block)
|
13
|
+
store = (options[:class] || FogStore).new(store_identifier)
|
14
|
+
block.call(store) if block_given?
|
15
|
+
|
16
|
+
if options[:primary]
|
17
|
+
STORES.unshift(store)
|
18
|
+
else
|
19
|
+
STORES << store
|
20
|
+
end
|
21
|
+
|
22
|
+
STORES_MAP[store_identifier] = store
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.stores
|
26
|
+
raise "You need to configure add stores to RemoteFiles using 'RemoteFiles.add_store'" if STORES.empty?
|
27
|
+
STORES
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.lookup_store(store_identifier)
|
31
|
+
STORES_MAP[store_identifier]
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.primary_store
|
35
|
+
STORES.first
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.store_once!(file)
|
39
|
+
return file.stored_in.first if file.stored?
|
40
|
+
|
41
|
+
exception = nil
|
42
|
+
|
43
|
+
stores.each do |store|
|
44
|
+
begin
|
45
|
+
stored = store.store!(file)
|
46
|
+
file.stored_in << store.identifier
|
47
|
+
break
|
48
|
+
rescue ::RemoteFiles::Error => e
|
49
|
+
exception = e
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
raise exception unless file.stored?
|
54
|
+
|
55
|
+
file.stored_in.first
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.store!(file)
|
59
|
+
store_once!(file) unless file.stored?
|
60
|
+
|
61
|
+
synchronize_stores(file) unless file.stored_everywhere?
|
62
|
+
|
63
|
+
true
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.synchronize!(file)
|
67
|
+
file.missing_stores.each do |store_identifier|
|
68
|
+
store = lookup_store(store_identifier)
|
69
|
+
store.store!(file)
|
70
|
+
file.stored_in << store.identifier
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def self.synchronize_stores(file = nil, &block)
|
75
|
+
if file
|
76
|
+
if @synchronize_stores
|
77
|
+
@synchronize_stores.call(file)
|
78
|
+
else
|
79
|
+
synchronize!(file)
|
80
|
+
end
|
81
|
+
elsif block_given?
|
82
|
+
@synchronize_stores = block
|
83
|
+
else
|
84
|
+
raise "invalid call to RemoteFiles.synchronize_stores"
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
data/test/file_test.rb
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
describe RemoteFiles::File do
|
4
|
+
before do
|
5
|
+
@s3 = RemoteFiles.add_store(:s3, :class => RemoteFiles::MockStore, :primary => true)
|
6
|
+
@cf = RemoteFiles.add_store(:cf, :class => RemoteFiles::MockStore)
|
7
|
+
|
8
|
+
@file = RemoteFiles::File.new('identifier')
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#stored?' do
|
12
|
+
it 'should return true if the file is stored anywhere' do
|
13
|
+
@file.stored_in << :s3
|
14
|
+
@file.stored?.must_equal(true)
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should return false if the file is not stored anywhere' do
|
18
|
+
@file.stored_in.clear
|
19
|
+
@file.stored?.must_equal(false)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe '#stored_everywhere?' do
|
24
|
+
it 'should return false if the file is not stored anywhere' do
|
25
|
+
@file.stored_in.clear
|
26
|
+
@file.stored_everywhere?.must_equal(false)
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'should return false if the file only is stored in some of the stores' do
|
30
|
+
@file.stored_in.replace([:s3])
|
31
|
+
@file.stored_everywhere?.must_equal(false)
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'should return true if the file is stored in all stores' do
|
35
|
+
@file.stored_in.replace([:s3, :cf])
|
36
|
+
@file.stored_everywhere?.must_equal(true)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe '#missing_stores' do
|
41
|
+
it 'should give an array of store identifiers where the file is not stored' do
|
42
|
+
@file.stored_in.replace([:s3])
|
43
|
+
@file.missing_stores.must_equal([:cf])
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe '#url' do
|
48
|
+
before do
|
49
|
+
@s3.stubs(:url).returns('s3_url')
|
50
|
+
@cf.stubs(:url).returns('cf_url')
|
51
|
+
end
|
52
|
+
|
53
|
+
describe 'with no arguments' do
|
54
|
+
it 'should return the url on the primary store' do
|
55
|
+
@file.url.must_equal('s3_url')
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'with a store identifier' do
|
60
|
+
it 'should return the url from that store' do
|
61
|
+
@file.url(:cf).must_equal('cf_url')
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
describe 'current_url' do
|
67
|
+
it 'should return the url from the first store where the file is currently stored' do
|
68
|
+
@s3.stubs(:url).returns('s3_url')
|
69
|
+
@cf.stubs(:url).returns('cf_url')
|
70
|
+
|
71
|
+
@file.stored_in.replace([:s3])
|
72
|
+
@file.current_url.must_equal('s3_url')
|
73
|
+
|
74
|
+
@file.stored_in.replace([:cf])
|
75
|
+
@file.current_url.must_equal('cf_url')
|
76
|
+
|
77
|
+
@file.stored_in.replace([:cf, :s3])
|
78
|
+
@file.current_url.must_equal('s3_url')
|
79
|
+
|
80
|
+
@file.stored_in.replace([])
|
81
|
+
@file.current_url.must_be_nil
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
describe RemoteFiles::FogStore do
|
4
|
+
before do
|
5
|
+
@connection = Fog::Storage.new({
|
6
|
+
:provider => 'AWS',
|
7
|
+
:aws_access_key_id => 'access_key_id',
|
8
|
+
:aws_secret_access_key => 'secret_access_key'
|
9
|
+
})
|
10
|
+
|
11
|
+
@directory = @connection.directories.create(:key => 'directory')
|
12
|
+
|
13
|
+
@store = RemoteFiles::FogStore.new(:fog)
|
14
|
+
@store[:provider] = 'AWS'
|
15
|
+
@store[:aws_access_key_id] = 'access_key_id'
|
16
|
+
@store[:aws_secret_access_key] = 'secret_access_key'
|
17
|
+
@store[:directory] = 'directory'
|
18
|
+
@store[:public] = true
|
19
|
+
end
|
20
|
+
|
21
|
+
describe 'configuration' do
|
22
|
+
it 'should configure a fog connection' do
|
23
|
+
connection = @store.connection
|
24
|
+
|
25
|
+
connection.must_be_instance_of(Fog::Storage::AWS::Mock)
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should configure directory' do
|
29
|
+
directory = @store.directory
|
30
|
+
|
31
|
+
directory.must_be_instance_of(Fog::Storage::AWS::Directory)
|
32
|
+
|
33
|
+
directory.key.must_equal('directory')
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe '#store!' do
|
38
|
+
before do
|
39
|
+
@file = RemoteFiles::File.new('identifier', :content_type => 'text/plain', :content => 'content')
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'should store the file in the directory' do
|
43
|
+
@store.store!(@file)
|
44
|
+
|
45
|
+
fog_file = @directory.files.get('identifier')
|
46
|
+
|
47
|
+
fog_file.must_be_instance_of(Fog::Storage::AWS::File)
|
48
|
+
fog_file.content_type.must_equal('text/plain')
|
49
|
+
fog_file.body.must_equal('content')
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'should raise a RemoteFiles::Error when an error happens' do
|
53
|
+
@directory.destroy
|
54
|
+
proc { @store.store!(@file) }.must_raise(RemoteFiles::Error)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
describe '#retrieve!' do
|
59
|
+
it 'should return a RemoteFiles::File when found' do
|
60
|
+
@directory.files.create(
|
61
|
+
:body => 'content',
|
62
|
+
:content_type => 'text/plain',
|
63
|
+
:key => 'identifier'
|
64
|
+
)
|
65
|
+
|
66
|
+
file = @store.retrieve!('identifier')
|
67
|
+
|
68
|
+
file.must_be_instance_of(RemoteFiles::File)
|
69
|
+
file.content.must_equal('content')
|
70
|
+
file.content_type.must_equal('text/plain')
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'should raise a RemoteFiles::NotFoundError when not found' do
|
74
|
+
proc { @store.retrieve!('identifier') }.must_raise(RemoteFiles::NotFoundError)
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'should raise a RemoteFiles::Error when error' do
|
78
|
+
@directory.destroy
|
79
|
+
proc { @store.retrieve!('identifier') }.must_raise(RemoteFiles::Error)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
describe '#url' do
|
84
|
+
describe 'for S3 connections' do
|
85
|
+
before { @store[:provider] = 'AWS' }
|
86
|
+
|
87
|
+
it 'should return an S3 url' do
|
88
|
+
@store.url('identifier').must_equal('https://s3.amazonaws.com/directory/identifier')
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe 'for CloudFiles connections' do
|
93
|
+
before { @store[:provider] = 'Rackspace' }
|
94
|
+
|
95
|
+
it 'should return a CloudFiles url' do
|
96
|
+
@store.url('identifier').must_equal('https://storage.cloudfiles.com/directory/identifier')
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
describe 'for other connections' do
|
101
|
+
before { @store[:provider] = 'Google' }
|
102
|
+
|
103
|
+
it 'should raise' do
|
104
|
+
proc { @store.url('identifier') }.must_raise(RuntimeError)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
@@ -0,0 +1,184 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
require 'remote_files/mock_store'
|
3
|
+
|
4
|
+
describe RemoteFiles do
|
5
|
+
before do
|
6
|
+
@file = RemoteFiles::File.new('file', :content => 'content', :content_type => 'text/plain')
|
7
|
+
@mock_store1 = RemoteFiles.add_store(:mock1, :class => RemoteFiles::MockStore)
|
8
|
+
@mock_store2 = RemoteFiles.add_store(:mock2, :class => RemoteFiles::MockStore)
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '::add_store' do
|
12
|
+
describe 'when adding a non-primary store' do
|
13
|
+
before { @non_primary_store = RemoteFiles.add_store(:primary) }
|
14
|
+
|
15
|
+
it 'should add it to the tail of the list of stores' do
|
16
|
+
RemoteFiles.stores.must_equal([@mock_store1, @mock_store2, @non_primary_store])
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe 'when adding a promary store' do
|
21
|
+
before { @primary_store = RemoteFiles.add_store(:primary, :primary => true) }
|
22
|
+
|
23
|
+
it 'should add it to the head of the list of stores' do
|
24
|
+
RemoteFiles.stores.must_equal([@primary_store, @mock_store1, @mock_store2])
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe '::primary_store' do
|
30
|
+
before do
|
31
|
+
@primary_store1 = RemoteFiles.add_store(:primary1, :primary => true)
|
32
|
+
@primary_store2 = RemoteFiles.add_store(:primary2, :primary => true)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should return the head of the list of stores' do
|
36
|
+
RemoteFiles.primary_store.must_equal(@primary_store2)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe '::lookup_store' do
|
41
|
+
before do
|
42
|
+
@primary_store = RemoteFiles.add_store(:primary, :primary => true)
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'should find the store my identifier' do
|
46
|
+
RemoteFiles.lookup_store(:mock1).must_equal(@mock_store1)
|
47
|
+
RemoteFiles.lookup_store(:mock2).must_equal(@mock_store2)
|
48
|
+
RemoteFiles.lookup_store(:primary).must_equal(@primary_store)
|
49
|
+
RemoteFiles.lookup_store(:unknown).must_be_nil
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe '::store_once!' do
|
54
|
+
|
55
|
+
describe 'when the first store succeeds' do
|
56
|
+
before { RemoteFiles.store_once!(@file) }
|
57
|
+
|
58
|
+
it 'should only store the file in the first store' do
|
59
|
+
@mock_store1.data['file'].must_equal(:content => 'content', :content_type => 'text/plain')
|
60
|
+
@mock_store2.data['file'].must_be_nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe 'when the first store fails' do
|
65
|
+
before do
|
66
|
+
@mock_store1.expects(:store!).with(@file).raises(RemoteFiles::Error)
|
67
|
+
RemoteFiles.store_once!(@file)
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'should only store the file in the second store' do
|
71
|
+
@mock_store1.data['file'].must_be_nil
|
72
|
+
@mock_store2.data['file'].must_equal(:content => 'content', :content_type => 'text/plain')
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
describe 'when alls stores fail' do
|
77
|
+
before do
|
78
|
+
@mock_store1.expects(:store!).with(@file).raises(RemoteFiles::Error)
|
79
|
+
@mock_store2.expects(:store!).with(@file).raises(RemoteFiles::Error)
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'should raise a RemoteFiles::Error' do
|
83
|
+
proc { RemoteFiles.store_once!(@file) }.must_raise(RemoteFiles::Error)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
describe '::synchronize_stores' do
|
89
|
+
before do
|
90
|
+
@files = []
|
91
|
+
|
92
|
+
RemoteFiles.synchronize_stores do |file|
|
93
|
+
@files << file
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
it 'should use the block for store synchronizaation' do
|
98
|
+
file = RemoteFiles::File.new('file')
|
99
|
+
RemoteFiles.synchronize_stores(file)
|
100
|
+
@files.must_equal([file])
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
describe '::store!' do
|
105
|
+
describe 'when the file is already stored in some stores' do
|
106
|
+
before { @file.stored_in.replace([@mock_store1.identifier]) }
|
107
|
+
|
108
|
+
it 'should not store the file' do
|
109
|
+
RemoteFiles.expects(:store_once!).never
|
110
|
+
RemoteFiles.store!(@file)
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'should synchronize the stores' do
|
114
|
+
RemoteFiles.expects(:synchronize_stores).with(@file)
|
115
|
+
RemoteFiles.store!(@file)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
describe 'when the file is stored in all stores' do
|
120
|
+
before { @file.stored_in.replace([@mock_store1.identifier, @mock_store2.identifier]) }
|
121
|
+
|
122
|
+
it 'should not store the file' do
|
123
|
+
RemoteFiles.expects(:store_once!).never
|
124
|
+
RemoteFiles.store!(@file)
|
125
|
+
end
|
126
|
+
|
127
|
+
it 'should not synchronize the stores' do
|
128
|
+
RemoteFiles.expects(:synchronize_stores).never
|
129
|
+
RemoteFiles.store!(@file)
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
|
134
|
+
describe 'when the file is not stored anywhere' do
|
135
|
+
before { @file.stored_in.replace([]) }
|
136
|
+
|
137
|
+
it 'should store the file once' do
|
138
|
+
RemoteFiles.expects(:store_once!).with(@file)
|
139
|
+
RemoteFiles.store!(@file)
|
140
|
+
end
|
141
|
+
|
142
|
+
it 'should synchronize the stores' do
|
143
|
+
RemoteFiles.expects(:synchronize_stores).with(@file)
|
144
|
+
RemoteFiles.store!(@file)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
describe '::synchronize!' do
|
150
|
+
describe 'when the file is not stored anywhere' do
|
151
|
+
before { @file.stored_in.replace([]) }
|
152
|
+
|
153
|
+
it 'should store the file on all stores' do
|
154
|
+
@mock_store1.expects(:store!).returns(true)
|
155
|
+
@mock_store2.expects(:store!).returns(true)
|
156
|
+
|
157
|
+
RemoteFiles.synchronize!(@file)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
describe 'when the file is stored in some stores' do
|
162
|
+
before { @file.stored_in.replace([@mock_store1.identifier]) }
|
163
|
+
|
164
|
+
it 'should store the file in the remaining stores' do
|
165
|
+
@mock_store1.expects(:store!).never
|
166
|
+
@mock_store2.expects(:store!).with(@file).returns(true)
|
167
|
+
|
168
|
+
RemoteFiles.synchronize!(@file)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
describe 'when the file is stored everywhere' do
|
173
|
+
before { @file.stored_in.replace([@mock_store1.identifier, @mock_store2.identifier]) }
|
174
|
+
|
175
|
+
it 'should not do anything' do
|
176
|
+
@mock_store1.expects(:store!).never
|
177
|
+
@mock_store2.expects(:store!).never
|
178
|
+
|
179
|
+
RemoteFiles.synchronize!(@file)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
require 'remote_files/resque_job'
|
3
|
+
|
4
|
+
describe RemoteFiles::ResqueJob do
|
5
|
+
describe 'loading the implementation file' do
|
6
|
+
before { load 'remote_files/resque_job.rb' }
|
7
|
+
|
8
|
+
it 'should setup the right synchronize_stores hook' do
|
9
|
+
file = RemoteFiles::File.new('identifier',
|
10
|
+
:content_type => 'text/plain',
|
11
|
+
:content => 'content',
|
12
|
+
:stored_in => [:s3],
|
13
|
+
:foo => :bar
|
14
|
+
)
|
15
|
+
|
16
|
+
Resque.expects(:enqueue).with(RemoteFiles::ResqueJob,
|
17
|
+
:identifier => 'identifier',
|
18
|
+
:content_type => 'text/plain',
|
19
|
+
:stored_in => [:s3],
|
20
|
+
:foo => :bar
|
21
|
+
)
|
22
|
+
|
23
|
+
RemoteFiles.synchronize_stores(file)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should call #synchronize! on the reconstructed file' do
|
28
|
+
options = {
|
29
|
+
:identifier => 'identifier',
|
30
|
+
:content_type => 'text/plain',
|
31
|
+
:stored_in => [:s3],
|
32
|
+
:foo => :bar
|
33
|
+
}
|
34
|
+
|
35
|
+
file = stub
|
36
|
+
file.expects(:synchronize!)
|
37
|
+
|
38
|
+
RemoteFiles::File.expects(:new).with('identifier',
|
39
|
+
:content_type => 'text/plain',
|
40
|
+
:stored_in => [:s3],
|
41
|
+
:foo => :bar
|
42
|
+
).returns(file)
|
43
|
+
|
44
|
+
RemoteFiles::ResqueJob.perform(options)
|
45
|
+
end
|
46
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
|
3
|
+
require 'debugger'
|
4
|
+
|
5
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
6
|
+
require 'remote_files'
|
7
|
+
|
8
|
+
require 'minitest/autorun'
|
9
|
+
|
10
|
+
require 'mocha'
|
11
|
+
|
12
|
+
Fog.mock!
|
13
|
+
|
14
|
+
MiniTest::Spec.class_eval do
|
15
|
+
before do
|
16
|
+
Fog::Mock.reset
|
17
|
+
|
18
|
+
RemoteFiles::STORES.clear
|
19
|
+
RemoteFiles::STORES_MAP.clear
|
20
|
+
|
21
|
+
$syncs = []
|
22
|
+
RemoteFiles.synchronize_stores do |file|
|
23
|
+
$syncs << {:identifier => file.identifier, :missing_stores => file.missing_stores}
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,168 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: remote_files
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Mick Staugaard
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-09-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fog
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: minitest
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: debugger
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: mocha
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: resque
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
description: A library for uploading files to multiple remote storage backends like
|
111
|
+
Amazon S3 and Rackspace CloudFiles.
|
112
|
+
email:
|
113
|
+
- mick@staugaard.com
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files: []
|
117
|
+
files:
|
118
|
+
- lib/remote_files/abstract_store.rb
|
119
|
+
- lib/remote_files/file.rb
|
120
|
+
- lib/remote_files/fog_store.rb
|
121
|
+
- lib/remote_files/mock_store.rb
|
122
|
+
- lib/remote_files/resque_job.rb
|
123
|
+
- lib/remote_files/version.rb
|
124
|
+
- lib/remote_files.rb
|
125
|
+
- test/file_test.rb
|
126
|
+
- test/fog_store_test.rb
|
127
|
+
- test/remote_files_test.rb
|
128
|
+
- test/resque_job_test.rb
|
129
|
+
- test/test_helper.rb
|
130
|
+
- README.md
|
131
|
+
homepage: https://github.com/staugaard/remote_file
|
132
|
+
licenses: []
|
133
|
+
post_install_message:
|
134
|
+
rdoc_options: []
|
135
|
+
require_paths:
|
136
|
+
- lib
|
137
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
138
|
+
none: false
|
139
|
+
requirements:
|
140
|
+
- - ! '>='
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: '0'
|
143
|
+
segments:
|
144
|
+
- 0
|
145
|
+
hash: -3554087706226059699
|
146
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
147
|
+
none: false
|
148
|
+
requirements:
|
149
|
+
- - ! '>='
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '0'
|
152
|
+
segments:
|
153
|
+
- 0
|
154
|
+
hash: -3554087706226059699
|
155
|
+
requirements: []
|
156
|
+
rubyforge_project:
|
157
|
+
rubygems_version: 1.8.24
|
158
|
+
signing_key:
|
159
|
+
specification_version: 3
|
160
|
+
summary: The purpose of the library is to implement a simple interface for uploading
|
161
|
+
files to multiple backends and to keep the backends in sync, so that your app will
|
162
|
+
keep working when one backend is down.
|
163
|
+
test_files:
|
164
|
+
- test/file_test.rb
|
165
|
+
- test/fog_store_test.rb
|
166
|
+
- test/remote_files_test.rb
|
167
|
+
- test/resque_job_test.rb
|
168
|
+
- test/test_helper.rb
|