awshucks 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. data/CHANGES +4 -0
  2. data/LICENSE +19 -0
  3. data/README +32 -0
  4. data/Rakefile +14 -0
  5. data/TODO +143 -0
  6. data/bin/awshucks +10 -0
  7. data/lib/awshucks.rb +29 -0
  8. data/lib/awshucks/cli.rb +76 -0
  9. data/lib/awshucks/command.rb +98 -0
  10. data/lib/awshucks/commands.rb +3 -0
  11. data/lib/awshucks/commands/backup.rb +59 -0
  12. data/lib/awshucks/commands/backups.rb +25 -0
  13. data/lib/awshucks/commands/help.rb +22 -0
  14. data/lib/awshucks/commands/list.rb +39 -0
  15. data/lib/awshucks/commands/new_config.rb +36 -0
  16. data/lib/awshucks/commands/reset_metadata_cache.rb +38 -0
  17. data/lib/awshucks/commands/restore.rb +54 -0
  18. data/lib/awshucks/config.rb +83 -0
  19. data/lib/awshucks/ext.rb +23 -0
  20. data/lib/awshucks/file_info.rb +23 -0
  21. data/lib/awshucks/file_store.rb +111 -0
  22. data/lib/awshucks/gemspec.rb +48 -0
  23. data/lib/awshucks/scanner.rb +58 -0
  24. data/lib/awshucks/specification.rb +128 -0
  25. data/lib/awshucks/version.rb +18 -0
  26. data/resources/awshucks.yml +21 -0
  27. data/spec/awshucks_spec.rb +7 -0
  28. data/spec/cli_spec.rb +130 -0
  29. data/spec/command_spec.rb +111 -0
  30. data/spec/commands/backup_spec.rb +164 -0
  31. data/spec/commands/backups_spec.rb +41 -0
  32. data/spec/commands/help_spec.rb +42 -0
  33. data/spec/commands/list_spec.rb +77 -0
  34. data/spec/commands/new_config_spec.rb +102 -0
  35. data/spec/commands/reset_metadata_cache_spec.rb +93 -0
  36. data/spec/commands/restore_spec.rb +219 -0
  37. data/spec/config_spec.rb +152 -0
  38. data/spec/ext_spec.rb +28 -0
  39. data/spec/file_info_spec.rb +45 -0
  40. data/spec/file_store_spec.rb +352 -0
  41. data/spec/scanner_spec.rb +106 -0
  42. data/spec/spec_helper.rb +36 -0
  43. data/spec/specification_spec.rb +41 -0
  44. metadata +121 -0
@@ -0,0 +1,219 @@
1
+ require File.join(File.dirname(__FILE__),"..","spec_helper.rb")
2
+
3
+ describe Awshucks::RestoreCommand do
4
+ it "has 'restore' as the command" do
5
+ Awshucks::RestoreCommand.command.should == 'restore'
6
+ end
7
+ end
8
+
9
+ describe Awshucks::RestoreCommand, '#execute, with no backup configuration specified' do
10
+
11
+ before(:each) do
12
+ @config = mock('config')
13
+ @restore = Awshucks::RestoreCommand.new
14
+ @old_stderr, $stderr = $stderr, StringIO.new
15
+ end
16
+
17
+ after(:each) do
18
+ $stderr = @old_stderr
19
+ end
20
+
21
+ it "prints out an error" do
22
+ @restore.execute([], @config)
23
+ $stderr.string.should =~ /specify a backup/i
24
+ end
25
+
26
+ end
27
+
28
+ describe Awshucks::RestoreCommand, '#execute, with just a backup configuration specified' do
29
+
30
+ before(:each) do
31
+ @connection = mock('connection')
32
+
33
+ @backup = mock('backup configuration')
34
+ @backup.stub!(:name).and_return('testbackup')
35
+ @backup.stub!(:location).and_return('/test/location')
36
+
37
+ @config = mock('config')
38
+ @config.stub!(:connection).and_return(@connection)
39
+ @config.stub!(:backup).and_return(@backup)
40
+
41
+ @config.stub!(:bucket).and_return('testbucket')
42
+ @restore = Awshucks::RestoreCommand.new
43
+ @old_stderr, $stderr = $stderr, StringIO.new
44
+ @old_stdout, $stdout = $stdout, StringIO.new
45
+
46
+ @file_store = mock(Awshucks::FileStore)
47
+ end
48
+
49
+ after(:each) do
50
+ $stderr = @old_stderr
51
+ $stdout = @old_stdout
52
+ end
53
+
54
+ it "instantiates a FileStore object for the backup config" do
55
+ Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
56
+ @file_store.stub!(:each_file)
57
+ @restore.execute(['testbackup'], @config)
58
+ end
59
+
60
+ it "iterates through each file in the file store and calls restore on each entry" do
61
+ Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
62
+ @file_store.should_receive(:each_file).and_yield('file')
63
+ @file_store.should_receive(:restore).with('file', '/test/location/file')
64
+
65
+ @restore.execute(['testbackup'], @config)
66
+ end
67
+
68
+ it "prints a warning if the backup config isn't found" do
69
+ @config.stub!(:backup).and_raise(Awshucks::UnknownBackupError.new('foo'))
70
+ @restore.execute(['foo'], @config)
71
+ end
72
+
73
+ end
74
+
75
+ describe Awshucks::RestoreCommand, '#execute, with a backup configuration and a specific file' do
76
+
77
+ before(:each) do
78
+ @connection = mock('connection')
79
+
80
+ @backup = mock('backup configuration')
81
+ @backup.stub!(:name).and_return('testbackup')
82
+ @backup.stub!(:location).and_return('/test/location')
83
+
84
+ @config = mock('config')
85
+ @config.stub!(:connection).and_return(@connection)
86
+ @config.stub!(:backup).and_return(@backup)
87
+
88
+ @config.stub!(:bucket).and_return('testbucket')
89
+ @restore = Awshucks::RestoreCommand.new
90
+
91
+ @old_stdout, $stdout = $stdout, StringIO.new
92
+
93
+ @file_store = mock(Awshucks::FileStore)
94
+ end
95
+
96
+ after(:each) do
97
+ $stdout = @old_stdout
98
+ end
99
+
100
+ it "iterates through each file in the file store and calls restore on each entry" do
101
+ Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
102
+ class << @file_store
103
+ def each_file
104
+ yield 'other_file'
105
+ yield 'specific_file'
106
+ yield 'another/file'
107
+ end
108
+ end
109
+
110
+ @file_store.should_receive(:restore).with('specific_file', '/test/location/specific_file')
111
+
112
+ @restore.execute(['testbackup', 'specific_file'], @config)
113
+ end
114
+
115
+ end
116
+
117
+ describe Awshucks::RestoreCommand, '#execute, with a backup configuration and a pattern' do
118
+
119
+ before(:each) do
120
+ @connection = mock('connection')
121
+
122
+ @backup = mock('backup configuration')
123
+ @backup.stub!(:name).and_return('testbackup')
124
+ @backup.stub!(:location).and_return('/test/location')
125
+
126
+ @config = mock('config')
127
+ @config.stub!(:connection).and_return(@connection)
128
+ @config.stub!(:backup).and_return(@backup)
129
+
130
+ @config.stub!(:bucket).and_return('testbucket')
131
+ @restore = Awshucks::RestoreCommand.new
132
+
133
+ @old_stdout, $stdout = $stdout, StringIO.new
134
+
135
+ @file_store = mock(Awshucks::FileStore)
136
+ end
137
+
138
+ after(:each) do
139
+ $stdout = @old_stdout
140
+ end
141
+
142
+ it "restores all files when the pattern is '*'" do
143
+ Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
144
+ class << @file_store
145
+ def each_file
146
+ yield 'some_file'
147
+ yield 'another/file'
148
+ end
149
+ end
150
+
151
+ @file_store.should_receive(:restore).with('some_file', '/test/location/some_file')
152
+ @file_store.should_receive(:restore).with('another/file', '/test/location/another/file')
153
+
154
+ @restore.execute(['testbackup', '*'], @config)
155
+ end
156
+
157
+ it "restores only the files matching the pattern" do
158
+ Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
159
+ class << @file_store
160
+ def each_file
161
+ yield 'some_file'
162
+ yield 'sub/file'
163
+ yield 'foo/bar'
164
+ end
165
+ end
166
+
167
+ @file_store.should_receive(:restore).with('some_file', '/test/location/some_file')
168
+ @file_store.should_receive(:restore).with('foo/bar', '/test/location/foo/bar')
169
+
170
+ @restore.execute(['testbackup', '*o*'], @config)
171
+ end
172
+
173
+ end
174
+
175
+ describe Awshucks::RestoreCommand, '#execute, with a backup, pattern, and a target directory' do
176
+
177
+ before(:each) do
178
+ @connection = mock('connection')
179
+
180
+ @backup = mock('backup configuration')
181
+ @backup.stub!(:name).and_return('testbackup')
182
+ @backup.stub!(:location).and_return('/test/location')
183
+
184
+ @config = mock('config')
185
+ @config.stub!(:connection).and_return(@connection)
186
+ @config.stub!(:backup).and_return(@backup)
187
+
188
+ @config.stub!(:bucket).and_return('testbucket')
189
+ @restore = Awshucks::RestoreCommand.new
190
+
191
+ @old_stdout, $stdout = $stdout, StringIO.new
192
+
193
+ @file_store = mock(Awshucks::FileStore)
194
+ end
195
+
196
+ after(:each) do
197
+ $stdout = @old_stdout
198
+ end
199
+
200
+ it "calls restore using the target directory as the base directory for the restore" do
201
+ Awshucks::FileStore.should_receive(:new).with(@connection, 'testbucket', 'testbackup').and_return(@file_store)
202
+ @file_store.should_receive(:each_file).and_yield('test/file')
203
+
204
+ @file_store.should_receive(:restore).with('test/file', '/another/place/test/file')
205
+
206
+ @restore.execute(['testbackup', '*', '/another/place'], @config)
207
+ end
208
+
209
+ it "calls restore with the target being a fully expanded path" do
210
+ Awshucks::FileStore.stub!(:new).and_return(@file_store)
211
+ @file_store.stub!(:each_file).and_yield('test/file')
212
+
213
+ @file_store.should_receive(:restore).with('test/file', '/full/path/another/place/test/file')
214
+
215
+ File.should_receive(:expand_path).with('another/place').and_return('/full/path/another/place')
216
+ @restore.execute(['testbackup', '*', 'another/place'], @config)
217
+ end
218
+
219
+ end
@@ -0,0 +1,152 @@
1
+ require File.join(File.dirname(__FILE__),"spec_helper.rb")
2
+
3
+ describe Awshucks::Config do
4
+
5
+ before(:each) do
6
+ @old_stderr, $stderr = $stderr, StringIO.new
7
+ end
8
+
9
+ after(:each) do
10
+ $stderr = @old_stderr
11
+ end
12
+
13
+ it "accepts a filename as a parameter to initialize" do
14
+ config = Awshucks::Config.new( fixture_file('awshucks.yml') )
15
+ end
16
+
17
+ it "lazily loads the yaml file" do
18
+ config = Awshucks::Config.new( fixture_file('awshucks.yml') )
19
+ # YAML.should_receive(:load).and_return({})
20
+ # config.send(:config) # force the lazy load
21
+ config.connection.should == {
22
+ :access_key_id => "access key",
23
+ :secret_access_key => 'secret access key',
24
+ :persistent => false,
25
+ :use_ssl => true
26
+ }
27
+ end
28
+
29
+ it "prints an error and exits if the config file wasn't found" do
30
+ config = Awshucks::Config.new( fixture_file('awshucks.yml') )
31
+ YAML.should_receive(:load).and_raise(Errno::ENOENT)
32
+ lambda { config.send(:config) }.should raise_error(SystemExit)
33
+ $stderr.string.should =~ /not found/
34
+ end
35
+
36
+ end
37
+
38
+ # describe Awshucks::Config, '.load' do
39
+ #
40
+ # before(:each) do
41
+ # @config = Awshucks::Config.load(fixture_file('awshucks.yml'))
42
+ # end
43
+ #
44
+ # it "should load the specified file as YAML" do
45
+ # @config.connection.should == {
46
+ # :access_key_id => "access key",
47
+ # :secret_access_key => 'secret access key',
48
+ # :persistent => false,
49
+ # :use_ssl => true
50
+ # }
51
+ # end
52
+ #
53
+ # it "should return a Config object" do
54
+ # @config.should be_instance_of(Awshucks::Config)
55
+ # end
56
+ # end
57
+
58
+ describe Awshucks::Config, '#connection' do
59
+
60
+ it "should return the connection info specified in the config file, appropriate for the AWS library" do
61
+ config = Awshucks::Config.new(fixture_file('awshucks.yml'))
62
+ config.connection.should == {
63
+ :access_key_id => "access key",
64
+ :secret_access_key => 'secret access key',
65
+ :persistent => false,
66
+ :use_ssl => true
67
+ }
68
+ end
69
+
70
+ Awshucks::Config::REQUIRED_CONNECTION_KEYS.each do |key|
71
+ it "should raise an exception if the config file is missing the #{key} connection option" do
72
+ data = YAML.load(fixture_file('awshucks.yml'))
73
+ data.delete(key)
74
+ File.stub!(:open)
75
+ YAML.should_receive(:load).and_return(data)
76
+ config = Awshucks::Config.new('lol')
77
+ lambda { config.connection }.should raise_error(Awshucks::RequiredConfigError)
78
+ end
79
+ end
80
+
81
+ end
82
+
83
+ describe Awshucks::Config, '#bucket' do
84
+ before(:each) do
85
+ @config = Awshucks::Config.new(fixture_file('awshucks.yml'))
86
+ end
87
+
88
+ it "should return the configured bucket name" do
89
+ @config.bucket.should == "testbucket"
90
+ end
91
+
92
+ end
93
+
94
+ describe Awshucks::Config, '#backup' do
95
+ before(:each) do
96
+ @config = Awshucks::Config.new(fixture_file('awshucks.yml'))
97
+ end
98
+
99
+ it "should return the information for a specific backup location" do
100
+ @config.backup('test_backup').should_not be_nil
101
+ end
102
+
103
+ it "should return an object with a location attribute" do
104
+ @config.backup('test_backup').location.should == '/primary/backup'
105
+ end
106
+
107
+ it "should return an object with a name attribute" do
108
+ @config.backup('test_backup').name.should == 'test_backup'
109
+ end
110
+
111
+ Awshucks::Config::REQUIRED_CONNECTION_KEYS.each do |word|
112
+ it "should raise an exception if the backup name is the reserved config name '#{word}'" do
113
+ lambda { @config.backup(word) }.should raise_error(Awshucks::UnknownBackupError)
114
+ end
115
+ end
116
+
117
+ Awshucks::Config::REQUIRED_BACKUP_CONFIG_KEYS.each do |key|
118
+ it "should raise an exception if the backup is missing a #{key}" do
119
+ lambda { @config.backup('invalid_backup') }.should raise_error(Awshucks::RequiredConfigError)
120
+ end
121
+ end
122
+
123
+ end
124
+
125
+ describe Awshucks::Config, '#backups' do
126
+ before(:each) do
127
+ @config = Awshucks::Config.new(fixture_file('awshucks.yml'))
128
+ end
129
+
130
+ it "should return the list of backup configurations in the config" do
131
+ @config.backups.should have_at_least(1).item
132
+ end
133
+
134
+ it "should return backup configuration items" do
135
+ @config.backups.first.name.should == 'test_backup'
136
+ end
137
+
138
+ Awshucks::Config::RESERVED_KEYS.each do |key|
139
+ it "should not include the reserved key #{key}" do
140
+ @config.backups.map {|b| b.name}.should_not include(key)
141
+ end
142
+ end
143
+
144
+ end
145
+
146
+ describe Awshucks::Config do
147
+ it "has a help_message attribute" do
148
+ @config = Awshucks::Config.new(fixture_file('awshucks.yml'))
149
+ @config.help_message = 'help message'
150
+ @config.help_message.should == 'help message'
151
+ end
152
+ end
@@ -0,0 +1,28 @@
1
+ require File.join(File.dirname(__FILE__),"spec_helper.rb")
2
+
3
+ describe Object, '#returning' do
4
+ it "yields the provided argument, then returns it" do
5
+ yielded = nil
6
+ returning(:foo) do |obj|
7
+ yielded = obj
8
+ end.should == :foo
9
+ yielded.should == :foo
10
+ end
11
+ end
12
+
13
+ describe Object, '#in?' do
14
+ it "calls include? on the provided enumerable" do
15
+ collection = mock('collection')
16
+ item = mock('item')
17
+ collection.should_receive(:include?).with(item).and_return(true)
18
+ item.in?(collection).should be_true
19
+ end
20
+ end
21
+
22
+ describe File, '.md5sum' do
23
+ it "calls read on the provided filename and computes the MD5 hex digest" do
24
+ sum = Digest::MD5.hexdigest('lol')
25
+ File.should_receive('read').with('pretend/file').and_return('lol')
26
+ File.md5sum('pretend/file').should == sum
27
+ end
28
+ end
@@ -0,0 +1,45 @@
1
+ require File.join(File.dirname(__FILE__),"spec_helper.rb")
2
+
3
+ describe Awshucks::FileInfo do
4
+
5
+ before(:each) do
6
+ @now = Time.now
7
+ @stat = mock('file stat instance')
8
+ @stat.stub!(:mtime).and_return(@now)
9
+ @stat.stub!(:size).and_return(12345)
10
+ @info = Awshucks::FileInfo.new('file', '/path/to/file', @stat)
11
+ end
12
+
13
+ it "is initialized with a relative filename, a full path, and a File::Stat object" do
14
+ @info.should be_instance_of(Awshucks::FileInfo)
15
+ end
16
+
17
+ it "has a filename" do
18
+ @info.filename.should == 'file'
19
+ end
20
+
21
+ it "has a full filename path" do
22
+ @info.full_path.should == '/path/to/file'
23
+ end
24
+
25
+ it "has an mtime in UTC" do
26
+ @info.mtime.should be_kind_of(Time)
27
+ @info.mtime.to_s.should == @now.gmtime.to_s
28
+ end
29
+
30
+ it "has a lazily-evaluated md5sum" do
31
+ File.should_receive(:md5sum).with('/path/to/file').and_return('placeholder sum')
32
+ @info.md5sum.should == 'placeholder sum'
33
+ end
34
+
35
+ it "returns an IO object when asked for the data" do
36
+ io = mock('pretend io')
37
+ File.should_receive(:open).and_return(io)
38
+ @info.data.should == io
39
+ end
40
+
41
+ it "has a size (in bytes)" do
42
+ @info.size.should == 12345
43
+ end
44
+
45
+ end
@@ -0,0 +1,352 @@
1
+ require File.join(File.dirname(__FILE__),"spec_helper.rb")
2
+
3
+ describe Awshucks::FileStore do
4
+
5
+ before(:each) do
6
+ @connection = mock('connection')
7
+ AWS::S3::Base.stub!(:establish_connection!)
8
+ end
9
+
10
+ it "accepts a connection info hash, bucket name, and a backup prefix as parameters to initialize" do
11
+ Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix').should be_instance_of(Awshucks::FileStore)
12
+ end
13
+
14
+ it "establishes a connection to S3 using the config when initialized" do
15
+ AWS::S3::Base.should_receive(:establish_connection!).with(@connection)
16
+ fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
17
+ end
18
+
19
+ end
20
+
21
+ describe Awshucks::FileStore, '#different_from? with a FileInfo object' do
22
+
23
+ before(:each) do
24
+
25
+ @bucket = mock('bucket')
26
+ @connection = mock('connection')
27
+ AWS::S3::Base.stub!(:establish_connection!)
28
+ AWS::S3::Bucket.stub!(:find).and_return(@bucket)
29
+
30
+ # for the metadata cache:
31
+ @now = Time.now.gmtime
32
+ @sum = "7589253b2c5f744411e43be22eb45b71" # "lol what"
33
+ @sum2 = "0d705691a3dddd72b11f5e9b7d61a306" # "no wai"
34
+ @meta = mock('metadata')
35
+ @meta.stub!(:value).and_return("---\n{}") # empty hash
36
+ @bucket.should_receive(:[]).any_number_of_times.with('testprefix_metadata.yml').and_return(@meta)
37
+
38
+ # for a pretend file stored on S3:
39
+ @file = mock('example stored s3 file')
40
+ @file.stub!(:key).and_return('testprefix/test/file')
41
+
42
+ @info = mock(Awshucks::FileInfo)
43
+ @info.stub!(:filename).and_return('test/file')
44
+ # @info.stub!(:full_path).and_return('/path/to/file')
45
+
46
+ @fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
47
+
48
+ end
49
+
50
+ it "returns true if the file isn't stored in the file store already" do
51
+ @bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([])
52
+ @fs.different_from?(@info).should be_true
53
+ end
54
+
55
+ it "returns false if the file's metadata mtime (in UTC!) matches" do
56
+ @bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
57
+ @file.stub!(:metadata).and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
58
+ @info.should_receive(:mtime).at_least(1).times.and_return(@now)
59
+ @info.stub!(:md5sum).and_return(@sum)
60
+ @fs.different_from?(@info).should be_false
61
+ end
62
+
63
+ it "returns false if the file's mtime is different but the md5sum still matches" do
64
+ @bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
65
+
66
+ @file.stub!(:metadata).and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
67
+ @info.should_receive(:mtime).any_number_of_times.and_return(@now+5)
68
+ @info.should_receive(:md5sum).and_return(@sum)
69
+ @fs.different_from?(@info).should be_false
70
+ end
71
+
72
+ it "updates the metadata cache with the new time if the file's mtime is different but md5sum is not" do
73
+ @meta.should_receive(:value).and_return(
74
+ {'test/file' => {'mtime' => @now, 'md5sum' => @sum}}.to_yaml
75
+ )
76
+ @info.should_receive(:mtime).twice.and_return(@now+5)
77
+ @info.should_receive(:md5sum).and_return(@sum)
78
+ @fs.different_from?(@info).should be_false
79
+ @fs.send(:metadata)['test/file']['mtime'].should == (@now+5).gmtime.to_s
80
+ end
81
+
82
+ it "returns true if the file's mtime != the provided mtime and the md5sum doesn't match" do
83
+ @bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
84
+
85
+ @file.stub!(:metadata).and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
86
+ @info.should_receive(:mtime).and_return(@now+5)
87
+ @info.should_receive(:md5sum).and_return(@sum2)
88
+
89
+ @fs.different_from?(@info).should be_true
90
+ end
91
+
92
+ it "preferentially uses the metadata cache to retrieve file information" do
93
+ @meta.should_receive(:value).and_return(
94
+ {'test/file' => {'mtime' => @now, 'md5sum' => @sum}}.to_yaml
95
+ )
96
+
97
+ @info.should_receive(:mtime).and_return(@now+5)
98
+ @info.should_receive(:md5sum).and_return(@sum2)
99
+
100
+ @fs.different_from?(@info).should be_true
101
+ end
102
+
103
+ it "falls back to the stored file on s3 for metadata if it's not in the cache" do
104
+ # the metadata cache is empty...
105
+ @file.should_receive(:metadata).at_least(1).times.and_return('mtime' => @now.gmtime.to_s, 'md5sum' => @sum)
106
+ @bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@file])
107
+
108
+ @info.should_receive(:mtime).and_return(@now+5)
109
+ @info.should_receive(:md5sum).and_return(@sum2)
110
+
111
+ @fs.different_from?(@info).should be_true
112
+ end
113
+
114
+ end
115
+
116
+ describe Awshucks::FileStore, '#each_file' do
117
+
118
+ before(:each) do
119
+ AWS::S3::Base.stub!(:establish_connection!)
120
+ @bucket = mock('bucket')
121
+ @bucket.stub!(:name).and_return('testbucket')
122
+ @bucket.stub!(:[])
123
+
124
+ AWS::S3::Bucket.stub!(:find).and_return(@bucket)
125
+
126
+ @fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
127
+ end
128
+
129
+ it "yields each filename stored in the bucket matching the configured prefix, with the prefix removed" do
130
+ @fileone = mock('file one')
131
+ @fileone.stub!(:key).and_return('testprefix/some/file')
132
+ @filetwo = mock('file two')
133
+ @filetwo.stub!(:key).and_return('testprefix/another')
134
+ @bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@fileone, @filetwo])
135
+
136
+ yielded = []
137
+ @fs.each_file { |filename| yielded << filename }
138
+ yielded.should == %w(some/file another)
139
+ end
140
+
141
+ it "does not yield the metadata file" do
142
+ @meta = mock('metadata')
143
+ @meta.stub!(:key).and_return('testprefix_metadata.yml')
144
+ @fileone = mock('file one')
145
+ @fileone.stub!(:key).and_return('testprefix/some/file')
146
+ @bucket.should_receive(:objects).with(:prefix => 'testprefix').and_return([@fileone, @meta])
147
+
148
+ yielded = []
149
+ @fs.each_file { |filename| yielded << filename }
150
+ yielded.should == %w(some/file)
151
+ end
152
+
153
+ end
154
+
155
+ describe Awshucks::FileStore, '#store' do
156
+
157
+ before(:each) do
158
+ AWS::S3::Base.stub!(:establish_connection!)
159
+
160
+ @bucket = mock('bucket')
161
+ @bucket.stub!(:name).and_return('testbucket')
162
+ @bucket.stub!(:[])
163
+
164
+ AWS::S3::Bucket.stub!(:find).and_return(@bucket)
165
+
166
+ @response = mock('s3 response object')
167
+ @response.stub!(:success?).and_return true
168
+
169
+ @gmtime = Time.now.gmtime
170
+ @sum = 'pretend md5 sum'
171
+
172
+ @info = mock('fileinfo')
173
+ @info.stub!(:filename).and_return('test/file')
174
+ @info.stub!(:mtime).and_return(@gmtime)
175
+ @info.stub!(:data).and_return('data')
176
+ @info.stub!(:md5sum).and_return(@sum)
177
+
178
+ ::AWS::S3::S3Object.should_receive(:store).with('testprefix/test/file', 'data', 'testbucket', {
179
+ 'x-amz-meta-mtime' => @gmtime.to_s, 'x-amz-meta-md5sum' => @sum
180
+ }).and_return(@response)
181
+
182
+ @fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
183
+ end
184
+
185
+ it "stores the provided file data, including the modified time as metadata" do
186
+ @fs.store(@info)
187
+ end
188
+
189
+ it "includes the stored file in the metadata cache" do
190
+ @fs.store(@info)
191
+ @fs.send(:metadata).should == {'test/file' => {'mtime' => @gmtime.to_s, 'md5sum' => @sum}}
192
+ end
193
+
194
+ end
195
+
196
+ describe Awshucks::FileStore, '#restore' do
197
+
198
+ before(:each) do
199
+ AWS::S3::Base.stub!(:establish_connection!)
200
+ @bucket = mock('bucket')
201
+ @bucket.stub!(:name).and_return('testbucket')
202
+ @bucket.stub!(:[])
203
+
204
+ @sum = "imaginary md5 sum"
205
+
206
+ @meta = mock('metadata object')
207
+ @meta.stub!(:key).and_return('testprefix_metadata.yml')
208
+ @meta.stub!(:value).and_return(
209
+ {'some/file' => {'mtime' => Time.now.gmtime.to_s, 'md5sum' => @sum}}.to_yaml
210
+ )
211
+
212
+ @bucket.should_receive(:[]).with('testprefix_metadata.yml').any_number_of_times.and_return(@meta)
213
+
214
+ AWS::S3::Bucket.stub!(:find).and_return(@bucket)
215
+
216
+ @file = mock('pretend s3 file')
217
+ @file.stub!(:key).and_return('testprefix/some/file')
218
+ @file.stub!(:value).and_yield('value')
219
+
220
+ # @bucket.stub!(:objects).and_return([@file, @meta])
221
+
222
+ @filehandle = mock('filehandle')
223
+
224
+ FileUtils.stub!(:mkdir_p)
225
+ File.stub!(:md5sum).and_return(@sum)
226
+
227
+ @fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
228
+ @old_stderr, $stderr = $stderr, StringIO.new
229
+ end
230
+
231
+ after(:each) do
232
+ $stderr = @old_stderr
233
+ end
234
+
235
+ it "raises an exception if the specified file doesn't exist in the bucket" do
236
+ @bucket.should_receive(:[]).with('testprefix/some/file').and_return(nil)
237
+ lambda {@fs.restore('some/file', 'foo/bar/some/file') }.should raise_error(Awshucks::UnknownFileError)
238
+ end
239
+
240
+ it "restores the specified file to the specified location" do
241
+ @bucket.should_receive(:[]).with('testprefix/some/file').and_return(@file)
242
+ File.should_receive(:open).with('/absolute/location/some/file', 'w').and_yield(@filehandle)
243
+ @filehandle.should_receive(:write).with('value')
244
+ @fs.restore('some/file', '/absolute/location/some/file')
245
+ end
246
+
247
+ it "creates the directory tree necessary to restore the file" do
248
+ @bucket.should_receive(:[]).with('testprefix/some/file').and_return(@file)
249
+ File.stub!(:open).and_yield(@filehandle)
250
+ @filehandle.stub!(:write)
251
+
252
+ FileUtils.should_receive(:mkdir_p).with('/absolute/location/some')
253
+ @fs.restore('some/file', '/absolute/location/some/file')
254
+ end
255
+
256
+ it "prints a warning if the md5sum of the written file doesn't match the stored metadata" do
257
+ @bucket.should_receive(:[]).with('testprefix/some/file').and_return(@file)
258
+ File.stub!(:open).and_yield(@filehandle)
259
+ @filehandle.stub!(:write)
260
+
261
+ File.should_receive(:md5sum).with('/absolute/location/some/file').and_return('different sum')
262
+ @fs.restore('some/file', '/absolute/location/some/file')
263
+
264
+ $stderr.string.should =~ /warning.*md5/i
265
+ end
266
+
267
+ end
268
+
269
+ describe Awshucks::FileStore, '#save_cache' do
270
+
271
+ before(:each) do
272
+ @bucket = mock('bucket')
273
+ @bucket.stub!(:name).and_return('testbucket')
274
+ @connection = mock('connection')
275
+ AWS::S3::Base.should_receive(:establish_connection!).with(@connection)
276
+ @fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
277
+ @fs.stub!(:bucket).and_return(@bucket)
278
+ @meta = mock('metadata')
279
+ @meta.stub!(:value).and_return("---\nfoo: bar")
280
+ @meta.stub!(:to_yaml).and_return("--- \nfoo: bar\n")
281
+ end
282
+
283
+ it "stores the metadata file on S3" do
284
+ @bucket.stub!(:[]).and_return(@meta)
285
+ AWS::S3::S3Object.should_receive(:store).with('testprefix_metadata.yml', "--- \nfoo: bar\n", 'testbucket')
286
+ @fs.save_cache
287
+ end
288
+ end
289
+
290
+ describe Awshucks::FileStore, '#reset_cache' do
291
+
292
+ before(:each) do
293
+ AWS::S3::Base.stub!(:establish_connection!)
294
+ @bucket = mock('bucket')
295
+ @bucket.stub!(:name).and_return('testbucket')
296
+ @bucket.stub!(:[])
297
+
298
+ AWS::S3::Bucket.stub!(:find).and_return(@bucket)
299
+
300
+ @fs = Awshucks::FileStore.new(nil, 'testbucket', 'testprefix')
301
+
302
+ @old_stdout, $stdout = $stdout, StringIO.new
303
+ end
304
+
305
+ after(:each) do
306
+ $stdout = @old_stdout
307
+ end
308
+
309
+ it "deletes the metadata cache file from the S3 service" do
310
+ AWS::S3::S3Object.should_receive(:delete).with('testprefix_metadata.yml', 'testbucket')
311
+ @fs.reset_cache
312
+ end
313
+
314
+ end
315
+
316
+ describe Awshucks::FileStore, '#bucket (private method)' do
317
+
318
+ before(:each) do
319
+ @connection = mock('connection')
320
+ AWS::S3::Base.should_receive(:establish_connection!).with(@connection)
321
+ @fs = Awshucks::FileStore.new(@connection, 'testbucket', 'testprefix')
322
+ @old_stdout, $stdout = $stdout, StringIO.new
323
+ end
324
+
325
+ after(:each) do
326
+ $stdout = @old_stdout
327
+ end
328
+
329
+ it "returns the S3 bucket matching the configured bucket name" do
330
+ bucket = mock('bucket')
331
+ AWS::S3::Bucket.should_receive(:find).with('testbucket').and_return(bucket)
332
+ @fs.send(:bucket).should == bucket
333
+ end
334
+
335
+ it "creates the bucket and returns it if the bucket doesn't already exist" do
336
+ bucket = mock('bucket')
337
+
338
+ raised = false
339
+ AWS::S3::Bucket.should_receive(:find).twice.with('testbucket').and_return do
340
+ if raised
341
+ bucket
342
+ else
343
+ raised = true
344
+ raise AWS::S3::ResponseError.new("The specified bucket does not exist", nil)
345
+ end
346
+ end
347
+ AWS::S3::Bucket.should_receive(:create).with('testbucket').and_return(true)
348
+
349
+ @fs.send(:bucket).should == bucket
350
+ end
351
+
352
+ end