remote_files 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +86 -0
- data/lib/remote_files/abstract_store.rb +21 -0
- data/lib/remote_files/file.rb +59 -0
- data/lib/remote_files/fog_store.rb +69 -0
- data/lib/remote_files/mock_store.rb +25 -0
- data/lib/remote_files/resque_job.rb +15 -0
- data/lib/remote_files/version.rb +3 -0
- data/lib/remote_files.rb +87 -0
- data/test/file_test.rb +84 -0
- data/test/fog_store_test.rb +108 -0
- data/test/remote_files_test.rb +184 -0
- data/test/resque_job_test.rb +46 -0
- data/test/test_helper.rb +26 -0
- metadata +168 -0
data/README.md
ADDED
@@ -0,0 +1,86 @@
|
|
1
|
+
# RemoteFiles
|
2
|
+
|
3
|
+
A library for uploading files to multiple remote storage backends like Amazon S3 and Rackspace CloudFiles.
|
4
|
+
|
5
|
+
The purpose of the library is to implement a simple interface for uploading files to multiple backends
|
6
|
+
and to keep the backends in sync, so that your app will keep working when one backend is down.
|
7
|
+
|
8
|
+
## Installation
|
9
|
+
|
10
|
+
Add this line to your application's Gemfile:
|
11
|
+
|
12
|
+
gem 'remote_files'
|
13
|
+
|
14
|
+
And then execute:
|
15
|
+
|
16
|
+
$ bundle
|
17
|
+
|
18
|
+
Or install it yourself as:
|
19
|
+
|
20
|
+
$ gem install remote_files
|
21
|
+
|
22
|
+
## Configuration
|
23
|
+
|
24
|
+
First you configure the storage backends you want:
|
25
|
+
|
26
|
+
```ruby
|
27
|
+
RemoteFiles.add_store(:s3, :primary => true) do |s3|
|
28
|
+
s3[:provider] = 'AWS'
|
29
|
+
|
30
|
+
s3[:aws_access_key_id] = AWS_ACCESS_KEY_ID
|
31
|
+
s3[:aws_secret_access_key] = AWS_SECRET_ACCESS_KEY
|
32
|
+
|
33
|
+
s3[:directory] = 'my_s3_bucket'
|
34
|
+
end
|
35
|
+
|
36
|
+
RemoteFiles.add_store(:cf) do |cf|
|
37
|
+
cf[:provider] = 'Rackspace'
|
38
|
+
|
39
|
+
cf[:rackspace_username] = RACKSPACE_USERNAME
|
40
|
+
cf[:rackspace_api_key] = RACKSPACE_API_KEY
|
41
|
+
|
42
|
+
cf[:directory] = 'my_cf_container'
|
43
|
+
end
|
44
|
+
```
|
45
|
+
|
46
|
+
By default RemoteFiles will store your files to all stores synchronously. This is probably not what you want,
|
47
|
+
so you should tell RemoteFiles how to do it asynchronously:
|
48
|
+
|
49
|
+
```ruby
|
50
|
+
class RemoteFilesSyncJob
|
51
|
+
def initialize(identifier, stored_in)
|
52
|
+
@file = RemoteFiles::File.new(identifier, :stored_in => stored_in)
|
53
|
+
def
|
54
|
+
|
55
|
+
def work
|
56
|
+
@file.synchronize!
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
RemoteFiles.synchronize_stores do |file|
|
61
|
+
MyPreferredJobQueue.enqueue(RemoteFilesSyncJob, file.identifier, file.stored_in)
|
62
|
+
end
|
63
|
+
```
|
64
|
+
|
65
|
+
## Usage
|
66
|
+
|
67
|
+
Once everything is configured, you can store files like this:
|
68
|
+
|
69
|
+
```ruby
|
70
|
+
file = RemoteFiles::File.new(unique_file_name, :content => file_content, :content_type => content_type)
|
71
|
+
file.store!
|
72
|
+
```
|
73
|
+
|
74
|
+
This will store the file on one of the stores and then asynchronously copy the file to the remaining stores.
|
75
|
+
`RemoteFiles::File#store!` will raise a `RemoteFiles::Error` if all storage backends are down.
|
76
|
+
|
77
|
+
If you just need to store the file in a single store, the you can use `RemoteFiles::File#store_once!`. It will
|
78
|
+
behave exactly like `RemoteFiles::File#store!`, but will not asynchronously copy the file to the other stores.
|
79
|
+
|
80
|
+
## Contributing
|
81
|
+
|
82
|
+
1. Fork it
|
83
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
84
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
85
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
86
|
+
5. Create new Pull Request
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module RemoteFiles
|
2
|
+
class AbstractStore
|
3
|
+
attr_reader :identifier
|
4
|
+
|
5
|
+
def initialize(identifier)
|
6
|
+
@identifier = identifier
|
7
|
+
end
|
8
|
+
|
9
|
+
def store!(file)
|
10
|
+
raise "You need to implement #{self.class.name}#store!"
|
11
|
+
end
|
12
|
+
|
13
|
+
def retrieve!(identifier)
|
14
|
+
raise "You need to implement #{self.class.name}#retrieve!"
|
15
|
+
end
|
16
|
+
|
17
|
+
def url(identifier)
|
18
|
+
raise "You need to implement #{self.class.name}#url"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,59 @@
|
|
1
|
+
module RemoteFiles
|
2
|
+
class File
|
3
|
+
attr_reader :content, :content_type, :identifier, :stored_in
|
4
|
+
|
5
|
+
def initialize(identifier, options = {})
|
6
|
+
@identifier = identifier
|
7
|
+
@stored_in = options[:stored_in] || []
|
8
|
+
@content = options.delete(:content)
|
9
|
+
@content_type = options[:content_type]
|
10
|
+
@options = options
|
11
|
+
end
|
12
|
+
|
13
|
+
def options
|
14
|
+
@options.merge(
|
15
|
+
:identifier => identifier,
|
16
|
+
:stored_in => stored_in,
|
17
|
+
:content_type => content_type
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def stored?
|
22
|
+
!@stored_in.empty?
|
23
|
+
end
|
24
|
+
|
25
|
+
def stored_everywhere?
|
26
|
+
missing_stores.empty?
|
27
|
+
end
|
28
|
+
|
29
|
+
def missing_stores
|
30
|
+
RemoteFiles.stores.map(&:identifier) - @stored_in
|
31
|
+
end
|
32
|
+
|
33
|
+
def url(store_identifier = nil)
|
34
|
+
store = store_identifier ? RemoteFiles.lookup_store(store_identifier) : RemoteFiles.primary_store
|
35
|
+
return nil unless store
|
36
|
+
store.url(identifier)
|
37
|
+
end
|
38
|
+
|
39
|
+
def current_url
|
40
|
+
prioritized_stores = RemoteFiles.stores.map(&:identifier) & @stored_in
|
41
|
+
|
42
|
+
return nil if prioritized_stores.empty?
|
43
|
+
|
44
|
+
url(prioritized_stores[0])
|
45
|
+
end
|
46
|
+
|
47
|
+
def store!
|
48
|
+
RemoteFiles.store!(self)
|
49
|
+
end
|
50
|
+
|
51
|
+
def store_once!
|
52
|
+
RemoteFiles.store_once!(self)
|
53
|
+
end
|
54
|
+
|
55
|
+
def synchronize!
|
56
|
+
RemoteFiles.synchronize!(self)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'remote_files/abstract_store'
|
2
|
+
require 'fog'
|
3
|
+
|
4
|
+
module RemoteFiles
|
5
|
+
class FogStore < AbstractStore
|
6
|
+
def store!(file)
|
7
|
+
sucess = directory.files.create(
|
8
|
+
:body => file.content,
|
9
|
+
:content_type => file.content_type,
|
10
|
+
:key => file.identifier,
|
11
|
+
:public => options[:public]
|
12
|
+
)
|
13
|
+
|
14
|
+
raise RemoteFiles::Error unless sucess
|
15
|
+
|
16
|
+
true
|
17
|
+
rescue Fog::Errors::Error, Excon::Errors::Error
|
18
|
+
raise RemoteFiles::Error, $!.message, $!.backtrace
|
19
|
+
end
|
20
|
+
|
21
|
+
def retrieve!(identifier)
|
22
|
+
fog_file = directory.files.get(identifier)
|
23
|
+
|
24
|
+
raise NotFoundError, "#{identifier} not found in #{self.identifier} store" if fog_file.nil?
|
25
|
+
|
26
|
+
File.new(identifier,
|
27
|
+
:content => fog_file.body,
|
28
|
+
:content_type => fog_file.content_type,
|
29
|
+
:stored_in => [self.identifier]
|
30
|
+
)
|
31
|
+
rescue Fog::Errors::Error, Excon::Errors::Error
|
32
|
+
raise RemoteFiles::Error, $!.message, $!.backtrace
|
33
|
+
end
|
34
|
+
|
35
|
+
def url(identifier)
|
36
|
+
case options[:provider]
|
37
|
+
when 'AWS'
|
38
|
+
"https://s3.amazonaws.com/#{options[:directory]}/#{Fog::AWS.escape(identifier)}"
|
39
|
+
when 'Rackspace'
|
40
|
+
"https://storage.cloudfiles.com/#{options[:directory]}/#{Fog::Rackspace.escape(identifier, '/')}"
|
41
|
+
else
|
42
|
+
raise "#{self.class.name}#url was not implemented for the #{options[:provider]} provider"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def options
|
47
|
+
@options ||= {}
|
48
|
+
end
|
49
|
+
|
50
|
+
def []=(name, value)
|
51
|
+
options[name] = value
|
52
|
+
end
|
53
|
+
|
54
|
+
def connection
|
55
|
+
connection_options = options.dup
|
56
|
+
connection_options.delete(:directory)
|
57
|
+
connection_options.delete(:public)
|
58
|
+
@connection ||= Fog::Storage.new(connection_options)
|
59
|
+
end
|
60
|
+
|
61
|
+
def directory
|
62
|
+
@directory ||= connection.directories.new(
|
63
|
+
:key => options[:directory],
|
64
|
+
:public => options[:public]
|
65
|
+
)
|
66
|
+
end
|
67
|
+
|
68
|
+
end
|
69
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module RemoteFiles
|
2
|
+
class MockStore < AbstractStore
|
3
|
+
def data
|
4
|
+
@data ||= {}
|
5
|
+
end
|
6
|
+
|
7
|
+
def store!(file)
|
8
|
+
data[file.identifier] = { :content => file.content, :content_type => file.content_type}
|
9
|
+
end
|
10
|
+
|
11
|
+
def retrieve!(identifier)
|
12
|
+
raise NotFoundError, "#{identifier} not found in #{self.identifier} store" unless data.has_key?(identifier)
|
13
|
+
|
14
|
+
File.new(identifier,
|
15
|
+
:content => data[identifier][:content],
|
16
|
+
:content_type => data[identifier][:content_type],
|
17
|
+
:stored_in => [self.identifier]
|
18
|
+
)
|
19
|
+
end
|
20
|
+
|
21
|
+
def url(identifier)
|
22
|
+
"mock://#{self.identifier}/#{identifier}"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'remote_files'
|
2
|
+
require 'resque'
|
3
|
+
|
4
|
+
module RemoteFiles
|
5
|
+
class ResqueJob
|
6
|
+
def self.perform(options)
|
7
|
+
file = RemoteFiles::File.new(options.delete(:identifier), options)
|
8
|
+
file.synchronize!
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
synchronize_stores do |file|
|
13
|
+
Resque.enqueue(ResqueJob, file.options)
|
14
|
+
end
|
15
|
+
end
|
data/lib/remote_files.rb
ADDED
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'remote_files/version'
|
2
|
+
require 'remote_files/fog_store'
|
3
|
+
require 'remote_files/file'
|
4
|
+
|
5
|
+
module RemoteFiles
|
6
|
+
class Error < StandardError; end
|
7
|
+
class NotFoundError < Error; end
|
8
|
+
|
9
|
+
STORES_MAP = {}
|
10
|
+
STORES = []
|
11
|
+
|
12
|
+
def self.add_store(store_identifier, options = {}, &block)
|
13
|
+
store = (options[:class] || FogStore).new(store_identifier)
|
14
|
+
block.call(store) if block_given?
|
15
|
+
|
16
|
+
if options[:primary]
|
17
|
+
STORES.unshift(store)
|
18
|
+
else
|
19
|
+
STORES << store
|
20
|
+
end
|
21
|
+
|
22
|
+
STORES_MAP[store_identifier] = store
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.stores
|
26
|
+
raise "You need to configure add stores to RemoteFiles using 'RemoteFiles.add_store'" if STORES.empty?
|
27
|
+
STORES
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.lookup_store(store_identifier)
|
31
|
+
STORES_MAP[store_identifier]
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.primary_store
|
35
|
+
STORES.first
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.store_once!(file)
|
39
|
+
return file.stored_in.first if file.stored?
|
40
|
+
|
41
|
+
exception = nil
|
42
|
+
|
43
|
+
stores.each do |store|
|
44
|
+
begin
|
45
|
+
stored = store.store!(file)
|
46
|
+
file.stored_in << store.identifier
|
47
|
+
break
|
48
|
+
rescue ::RemoteFiles::Error => e
|
49
|
+
exception = e
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
raise exception unless file.stored?
|
54
|
+
|
55
|
+
file.stored_in.first
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.store!(file)
|
59
|
+
store_once!(file) unless file.stored?
|
60
|
+
|
61
|
+
synchronize_stores(file) unless file.stored_everywhere?
|
62
|
+
|
63
|
+
true
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.synchronize!(file)
|
67
|
+
file.missing_stores.each do |store_identifier|
|
68
|
+
store = lookup_store(store_identifier)
|
69
|
+
store.store!(file)
|
70
|
+
file.stored_in << store.identifier
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def self.synchronize_stores(file = nil, &block)
|
75
|
+
if file
|
76
|
+
if @synchronize_stores
|
77
|
+
@synchronize_stores.call(file)
|
78
|
+
else
|
79
|
+
synchronize!(file)
|
80
|
+
end
|
81
|
+
elsif block_given?
|
82
|
+
@synchronize_stores = block
|
83
|
+
else
|
84
|
+
raise "invalid call to RemoteFiles.synchronize_stores"
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
data/test/file_test.rb
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
describe RemoteFiles::File do
|
4
|
+
before do
|
5
|
+
@s3 = RemoteFiles.add_store(:s3, :class => RemoteFiles::MockStore, :primary => true)
|
6
|
+
@cf = RemoteFiles.add_store(:cf, :class => RemoteFiles::MockStore)
|
7
|
+
|
8
|
+
@file = RemoteFiles::File.new('identifier')
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '#stored?' do
|
12
|
+
it 'should return true if the file is stored anywhere' do
|
13
|
+
@file.stored_in << :s3
|
14
|
+
@file.stored?.must_equal(true)
|
15
|
+
end
|
16
|
+
|
17
|
+
it 'should return false if the file is not stored anywhere' do
|
18
|
+
@file.stored_in.clear
|
19
|
+
@file.stored?.must_equal(false)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
describe '#stored_everywhere?' do
|
24
|
+
it 'should return false if the file is not stored anywhere' do
|
25
|
+
@file.stored_in.clear
|
26
|
+
@file.stored_everywhere?.must_equal(false)
|
27
|
+
end
|
28
|
+
|
29
|
+
it 'should return false if the file only is stored in some of the stores' do
|
30
|
+
@file.stored_in.replace([:s3])
|
31
|
+
@file.stored_everywhere?.must_equal(false)
|
32
|
+
end
|
33
|
+
|
34
|
+
it 'should return true if the file is stored in all stores' do
|
35
|
+
@file.stored_in.replace([:s3, :cf])
|
36
|
+
@file.stored_everywhere?.must_equal(true)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe '#missing_stores' do
|
41
|
+
it 'should give an array of store identifiers where the file is not stored' do
|
42
|
+
@file.stored_in.replace([:s3])
|
43
|
+
@file.missing_stores.must_equal([:cf])
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe '#url' do
|
48
|
+
before do
|
49
|
+
@s3.stubs(:url).returns('s3_url')
|
50
|
+
@cf.stubs(:url).returns('cf_url')
|
51
|
+
end
|
52
|
+
|
53
|
+
describe 'with no arguments' do
|
54
|
+
it 'should return the url on the primary store' do
|
55
|
+
@file.url.must_equal('s3_url')
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
describe 'with a store identifier' do
|
60
|
+
it 'should return the url from that store' do
|
61
|
+
@file.url(:cf).must_equal('cf_url')
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
describe 'current_url' do
|
67
|
+
it 'should return the url from the first store where the file is currently stored' do
|
68
|
+
@s3.stubs(:url).returns('s3_url')
|
69
|
+
@cf.stubs(:url).returns('cf_url')
|
70
|
+
|
71
|
+
@file.stored_in.replace([:s3])
|
72
|
+
@file.current_url.must_equal('s3_url')
|
73
|
+
|
74
|
+
@file.stored_in.replace([:cf])
|
75
|
+
@file.current_url.must_equal('cf_url')
|
76
|
+
|
77
|
+
@file.stored_in.replace([:cf, :s3])
|
78
|
+
@file.current_url.must_equal('s3_url')
|
79
|
+
|
80
|
+
@file.stored_in.replace([])
|
81
|
+
@file.current_url.must_be_nil
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
|
3
|
+
describe RemoteFiles::FogStore do
|
4
|
+
before do
|
5
|
+
@connection = Fog::Storage.new({
|
6
|
+
:provider => 'AWS',
|
7
|
+
:aws_access_key_id => 'access_key_id',
|
8
|
+
:aws_secret_access_key => 'secret_access_key'
|
9
|
+
})
|
10
|
+
|
11
|
+
@directory = @connection.directories.create(:key => 'directory')
|
12
|
+
|
13
|
+
@store = RemoteFiles::FogStore.new(:fog)
|
14
|
+
@store[:provider] = 'AWS'
|
15
|
+
@store[:aws_access_key_id] = 'access_key_id'
|
16
|
+
@store[:aws_secret_access_key] = 'secret_access_key'
|
17
|
+
@store[:directory] = 'directory'
|
18
|
+
@store[:public] = true
|
19
|
+
end
|
20
|
+
|
21
|
+
describe 'configuration' do
|
22
|
+
it 'should configure a fog connection' do
|
23
|
+
connection = @store.connection
|
24
|
+
|
25
|
+
connection.must_be_instance_of(Fog::Storage::AWS::Mock)
|
26
|
+
end
|
27
|
+
|
28
|
+
it 'should configure directory' do
|
29
|
+
directory = @store.directory
|
30
|
+
|
31
|
+
directory.must_be_instance_of(Fog::Storage::AWS::Directory)
|
32
|
+
|
33
|
+
directory.key.must_equal('directory')
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe '#store!' do
|
38
|
+
before do
|
39
|
+
@file = RemoteFiles::File.new('identifier', :content_type => 'text/plain', :content => 'content')
|
40
|
+
end
|
41
|
+
|
42
|
+
it 'should store the file in the directory' do
|
43
|
+
@store.store!(@file)
|
44
|
+
|
45
|
+
fog_file = @directory.files.get('identifier')
|
46
|
+
|
47
|
+
fog_file.must_be_instance_of(Fog::Storage::AWS::File)
|
48
|
+
fog_file.content_type.must_equal('text/plain')
|
49
|
+
fog_file.body.must_equal('content')
|
50
|
+
end
|
51
|
+
|
52
|
+
it 'should raise a RemoteFiles::Error when an error happens' do
|
53
|
+
@directory.destroy
|
54
|
+
proc { @store.store!(@file) }.must_raise(RemoteFiles::Error)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
describe '#retrieve!' do
|
59
|
+
it 'should return a RemoteFiles::File when found' do
|
60
|
+
@directory.files.create(
|
61
|
+
:body => 'content',
|
62
|
+
:content_type => 'text/plain',
|
63
|
+
:key => 'identifier'
|
64
|
+
)
|
65
|
+
|
66
|
+
file = @store.retrieve!('identifier')
|
67
|
+
|
68
|
+
file.must_be_instance_of(RemoteFiles::File)
|
69
|
+
file.content.must_equal('content')
|
70
|
+
file.content_type.must_equal('text/plain')
|
71
|
+
end
|
72
|
+
|
73
|
+
it 'should raise a RemoteFiles::NotFoundError when not found' do
|
74
|
+
proc { @store.retrieve!('identifier') }.must_raise(RemoteFiles::NotFoundError)
|
75
|
+
end
|
76
|
+
|
77
|
+
it 'should raise a RemoteFiles::Error when error' do
|
78
|
+
@directory.destroy
|
79
|
+
proc { @store.retrieve!('identifier') }.must_raise(RemoteFiles::Error)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
describe '#url' do
|
84
|
+
describe 'for S3 connections' do
|
85
|
+
before { @store[:provider] = 'AWS' }
|
86
|
+
|
87
|
+
it 'should return an S3 url' do
|
88
|
+
@store.url('identifier').must_equal('https://s3.amazonaws.com/directory/identifier')
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
describe 'for CloudFiles connections' do
|
93
|
+
before { @store[:provider] = 'Rackspace' }
|
94
|
+
|
95
|
+
it 'should return a CloudFiles url' do
|
96
|
+
@store.url('identifier').must_equal('https://storage.cloudfiles.com/directory/identifier')
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
describe 'for other connections' do
|
101
|
+
before { @store[:provider] = 'Google' }
|
102
|
+
|
103
|
+
it 'should raise' do
|
104
|
+
proc { @store.url('identifier') }.must_raise(RuntimeError)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
@@ -0,0 +1,184 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
require 'remote_files/mock_store'
|
3
|
+
|
4
|
+
describe RemoteFiles do
|
5
|
+
before do
|
6
|
+
@file = RemoteFiles::File.new('file', :content => 'content', :content_type => 'text/plain')
|
7
|
+
@mock_store1 = RemoteFiles.add_store(:mock1, :class => RemoteFiles::MockStore)
|
8
|
+
@mock_store2 = RemoteFiles.add_store(:mock2, :class => RemoteFiles::MockStore)
|
9
|
+
end
|
10
|
+
|
11
|
+
describe '::add_store' do
|
12
|
+
describe 'when adding a non-primary store' do
|
13
|
+
before { @non_primary_store = RemoteFiles.add_store(:primary) }
|
14
|
+
|
15
|
+
it 'should add it to the tail of the list of stores' do
|
16
|
+
RemoteFiles.stores.must_equal([@mock_store1, @mock_store2, @non_primary_store])
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
describe 'when adding a promary store' do
|
21
|
+
before { @primary_store = RemoteFiles.add_store(:primary, :primary => true) }
|
22
|
+
|
23
|
+
it 'should add it to the head of the list of stores' do
|
24
|
+
RemoteFiles.stores.must_equal([@primary_store, @mock_store1, @mock_store2])
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
describe '::primary_store' do
|
30
|
+
before do
|
31
|
+
@primary_store1 = RemoteFiles.add_store(:primary1, :primary => true)
|
32
|
+
@primary_store2 = RemoteFiles.add_store(:primary2, :primary => true)
|
33
|
+
end
|
34
|
+
|
35
|
+
it 'should return the head of the list of stores' do
|
36
|
+
RemoteFiles.primary_store.must_equal(@primary_store2)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
describe '::lookup_store' do
|
41
|
+
before do
|
42
|
+
@primary_store = RemoteFiles.add_store(:primary, :primary => true)
|
43
|
+
end
|
44
|
+
|
45
|
+
it 'should find the store my identifier' do
|
46
|
+
RemoteFiles.lookup_store(:mock1).must_equal(@mock_store1)
|
47
|
+
RemoteFiles.lookup_store(:mock2).must_equal(@mock_store2)
|
48
|
+
RemoteFiles.lookup_store(:primary).must_equal(@primary_store)
|
49
|
+
RemoteFiles.lookup_store(:unknown).must_be_nil
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
describe '::store_once!' do
|
54
|
+
|
55
|
+
describe 'when the first store succeeds' do
|
56
|
+
before { RemoteFiles.store_once!(@file) }
|
57
|
+
|
58
|
+
it 'should only store the file in the first store' do
|
59
|
+
@mock_store1.data['file'].must_equal(:content => 'content', :content_type => 'text/plain')
|
60
|
+
@mock_store2.data['file'].must_be_nil
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
describe 'when the first store fails' do
|
65
|
+
before do
|
66
|
+
@mock_store1.expects(:store!).with(@file).raises(RemoteFiles::Error)
|
67
|
+
RemoteFiles.store_once!(@file)
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'should only store the file in the second store' do
|
71
|
+
@mock_store1.data['file'].must_be_nil
|
72
|
+
@mock_store2.data['file'].must_equal(:content => 'content', :content_type => 'text/plain')
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
describe 'when alls stores fail' do
|
77
|
+
before do
|
78
|
+
@mock_store1.expects(:store!).with(@file).raises(RemoteFiles::Error)
|
79
|
+
@mock_store2.expects(:store!).with(@file).raises(RemoteFiles::Error)
|
80
|
+
end
|
81
|
+
|
82
|
+
it 'should raise a RemoteFiles::Error' do
|
83
|
+
proc { RemoteFiles.store_once!(@file) }.must_raise(RemoteFiles::Error)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
describe '::synchronize_stores' do
|
89
|
+
before do
|
90
|
+
@files = []
|
91
|
+
|
92
|
+
RemoteFiles.synchronize_stores do |file|
|
93
|
+
@files << file
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
it 'should use the block for store synchronizaation' do
|
98
|
+
file = RemoteFiles::File.new('file')
|
99
|
+
RemoteFiles.synchronize_stores(file)
|
100
|
+
@files.must_equal([file])
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
describe '::store!' do
|
105
|
+
describe 'when the file is already stored in some stores' do
|
106
|
+
before { @file.stored_in.replace([@mock_store1.identifier]) }
|
107
|
+
|
108
|
+
it 'should not store the file' do
|
109
|
+
RemoteFiles.expects(:store_once!).never
|
110
|
+
RemoteFiles.store!(@file)
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'should synchronize the stores' do
|
114
|
+
RemoteFiles.expects(:synchronize_stores).with(@file)
|
115
|
+
RemoteFiles.store!(@file)
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
describe 'when the file is stored in all stores' do
|
120
|
+
before { @file.stored_in.replace([@mock_store1.identifier, @mock_store2.identifier]) }
|
121
|
+
|
122
|
+
it 'should not store the file' do
|
123
|
+
RemoteFiles.expects(:store_once!).never
|
124
|
+
RemoteFiles.store!(@file)
|
125
|
+
end
|
126
|
+
|
127
|
+
it 'should not synchronize the stores' do
|
128
|
+
RemoteFiles.expects(:synchronize_stores).never
|
129
|
+
RemoteFiles.store!(@file)
|
130
|
+
end
|
131
|
+
|
132
|
+
end
|
133
|
+
|
134
|
+
describe 'when the file is not stored anywhere' do
|
135
|
+
before { @file.stored_in.replace([]) }
|
136
|
+
|
137
|
+
it 'should store the file once' do
|
138
|
+
RemoteFiles.expects(:store_once!).with(@file)
|
139
|
+
RemoteFiles.store!(@file)
|
140
|
+
end
|
141
|
+
|
142
|
+
it 'should synchronize the stores' do
|
143
|
+
RemoteFiles.expects(:synchronize_stores).with(@file)
|
144
|
+
RemoteFiles.store!(@file)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
describe '::synchronize!' do
|
150
|
+
describe 'when the file is not stored anywhere' do
|
151
|
+
before { @file.stored_in.replace([]) }
|
152
|
+
|
153
|
+
it 'should store the file on all stores' do
|
154
|
+
@mock_store1.expects(:store!).returns(true)
|
155
|
+
@mock_store2.expects(:store!).returns(true)
|
156
|
+
|
157
|
+
RemoteFiles.synchronize!(@file)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
describe 'when the file is stored in some stores' do
|
162
|
+
before { @file.stored_in.replace([@mock_store1.identifier]) }
|
163
|
+
|
164
|
+
it 'should store the file in the remaining stores' do
|
165
|
+
@mock_store1.expects(:store!).never
|
166
|
+
@mock_store2.expects(:store!).with(@file).returns(true)
|
167
|
+
|
168
|
+
RemoteFiles.synchronize!(@file)
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
describe 'when the file is stored everywhere' do
|
173
|
+
before { @file.stored_in.replace([@mock_store1.identifier, @mock_store2.identifier]) }
|
174
|
+
|
175
|
+
it 'should not do anything' do
|
176
|
+
@mock_store1.expects(:store!).never
|
177
|
+
@mock_store2.expects(:store!).never
|
178
|
+
|
179
|
+
RemoteFiles.synchronize!(@file)
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
require_relative 'test_helper'
|
2
|
+
require 'remote_files/resque_job'
|
3
|
+
|
4
|
+
describe RemoteFiles::ResqueJob do
|
5
|
+
describe 'loading the implementation file' do
|
6
|
+
before { load 'remote_files/resque_job.rb' }
|
7
|
+
|
8
|
+
it 'should setup the right synchronize_stores hook' do
|
9
|
+
file = RemoteFiles::File.new('identifier',
|
10
|
+
:content_type => 'text/plain',
|
11
|
+
:content => 'content',
|
12
|
+
:stored_in => [:s3],
|
13
|
+
:foo => :bar
|
14
|
+
)
|
15
|
+
|
16
|
+
Resque.expects(:enqueue).with(RemoteFiles::ResqueJob,
|
17
|
+
:identifier => 'identifier',
|
18
|
+
:content_type => 'text/plain',
|
19
|
+
:stored_in => [:s3],
|
20
|
+
:foo => :bar
|
21
|
+
)
|
22
|
+
|
23
|
+
RemoteFiles.synchronize_stores(file)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'should call #synchronize! on the reconstructed file' do
|
28
|
+
options = {
|
29
|
+
:identifier => 'identifier',
|
30
|
+
:content_type => 'text/plain',
|
31
|
+
:stored_in => [:s3],
|
32
|
+
:foo => :bar
|
33
|
+
}
|
34
|
+
|
35
|
+
file = stub
|
36
|
+
file.expects(:synchronize!)
|
37
|
+
|
38
|
+
RemoteFiles::File.expects(:new).with('identifier',
|
39
|
+
:content_type => 'text/plain',
|
40
|
+
:stored_in => [:s3],
|
41
|
+
:foo => :bar
|
42
|
+
).returns(file)
|
43
|
+
|
44
|
+
RemoteFiles::ResqueJob.perform(options)
|
45
|
+
end
|
46
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
require 'bundler/setup'
|
2
|
+
|
3
|
+
require 'debugger'
|
4
|
+
|
5
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
6
|
+
require 'remote_files'
|
7
|
+
|
8
|
+
require 'minitest/autorun'
|
9
|
+
|
10
|
+
require 'mocha'
|
11
|
+
|
12
|
+
Fog.mock!
|
13
|
+
|
14
|
+
MiniTest::Spec.class_eval do
|
15
|
+
before do
|
16
|
+
Fog::Mock.reset
|
17
|
+
|
18
|
+
RemoteFiles::STORES.clear
|
19
|
+
RemoteFiles::STORES_MAP.clear
|
20
|
+
|
21
|
+
$syncs = []
|
22
|
+
RemoteFiles.synchronize_stores do |file|
|
23
|
+
$syncs << {:identifier => file.identifier, :missing_stores => file.missing_stores}
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
metadata
ADDED
@@ -0,0 +1,168 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: remote_files
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 1.0.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Mick Staugaard
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-09-06 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: fog
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: rake
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :development
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: minitest
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :development
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: debugger
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :development
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: mocha
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: resque
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
description: A library for uploading files to multiple remote storage backends like
|
111
|
+
Amazon S3 and Rackspace CloudFiles.
|
112
|
+
email:
|
113
|
+
- mick@staugaard.com
|
114
|
+
executables: []
|
115
|
+
extensions: []
|
116
|
+
extra_rdoc_files: []
|
117
|
+
files:
|
118
|
+
- lib/remote_files/abstract_store.rb
|
119
|
+
- lib/remote_files/file.rb
|
120
|
+
- lib/remote_files/fog_store.rb
|
121
|
+
- lib/remote_files/mock_store.rb
|
122
|
+
- lib/remote_files/resque_job.rb
|
123
|
+
- lib/remote_files/version.rb
|
124
|
+
- lib/remote_files.rb
|
125
|
+
- test/file_test.rb
|
126
|
+
- test/fog_store_test.rb
|
127
|
+
- test/remote_files_test.rb
|
128
|
+
- test/resque_job_test.rb
|
129
|
+
- test/test_helper.rb
|
130
|
+
- README.md
|
131
|
+
homepage: https://github.com/staugaard/remote_file
|
132
|
+
licenses: []
|
133
|
+
post_install_message:
|
134
|
+
rdoc_options: []
|
135
|
+
require_paths:
|
136
|
+
- lib
|
137
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
138
|
+
none: false
|
139
|
+
requirements:
|
140
|
+
- - ! '>='
|
141
|
+
- !ruby/object:Gem::Version
|
142
|
+
version: '0'
|
143
|
+
segments:
|
144
|
+
- 0
|
145
|
+
hash: -3554087706226059699
|
146
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
147
|
+
none: false
|
148
|
+
requirements:
|
149
|
+
- - ! '>='
|
150
|
+
- !ruby/object:Gem::Version
|
151
|
+
version: '0'
|
152
|
+
segments:
|
153
|
+
- 0
|
154
|
+
hash: -3554087706226059699
|
155
|
+
requirements: []
|
156
|
+
rubyforge_project:
|
157
|
+
rubygems_version: 1.8.24
|
158
|
+
signing_key:
|
159
|
+
specification_version: 3
|
160
|
+
summary: The purpose of the library is to implement a simple interface for uploading
|
161
|
+
files to multiple backends and to keep the backends in sync, so that your app will
|
162
|
+
keep working when one backend is down.
|
163
|
+
test_files:
|
164
|
+
- test/file_test.rb
|
165
|
+
- test/fog_store_test.rb
|
166
|
+
- test/remote_files_test.rb
|
167
|
+
- test/resque_job_test.rb
|
168
|
+
- test/test_helper.rb
|