s3aps 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ require "s3aps/runner"
2
+ require "s3aps/file_adapter"
3
+ require "s3aps/s3_adapter"
4
+ require "s3aps/sync"
5
+ require "s3aps/version"
@@ -0,0 +1,58 @@
1
+ module S3aps
2
+
3
+ # Reads, writes and lists files on the filesystem.
4
+ class FileAdapter
5
+
6
+ # Options:
7
+ #
8
+ # * :path - where the local files are (default is +tmp+)
9
+ def initialize(options = {})
10
+ @path = options[:path] || "tmp"
11
+ end
12
+
13
+ # List all the files in the path, recursively
14
+ def list
15
+ Dir.glob("#{@path}/**/*").select{|path| File.file?(path) }.map do |path|
16
+ path.sub Regexp.new("^#{@path}\/"), ''
17
+ end
18
+ end
19
+
20
+ # Write the file. It will always write the file whether it's there
21
+ # or not, unless you supply an +md5sum+ in which case it will check
22
+ # whether it's different first.
23
+ def write(path, value, md5sum = nil)
24
+ just_path = path.sub /\/[^\/]*$/, ''
25
+ local_md5 = md5sum(path)
26
+ if local_md5.nil? || local_md5 != md5sum
27
+ FileUtils.mkdir_p(File.join(@path, just_path))
28
+ File.open(File.join(@path, path), 'w') {|outfile|
29
+ outfile.print(value)
30
+ }
31
+ true
32
+ else
33
+ false
34
+ end
35
+ end
36
+
37
+ # Read the file.
38
+ def read(path)
39
+ File.open(File.join(@path, path),"rb") {|io| io.read}
40
+ end
41
+
42
+ def md5sum(path) #:nodoc:
43
+ md5 = nil
44
+ begin
45
+ md5 = File.open(File.join(@path, path), 'rb') do |io|
46
+ d = Digest::MD5.new
47
+ s = ""
48
+ d.update(s) while io.read(4096, s)
49
+ d
50
+ end
51
+ rescue
52
+ end
53
+ md5 ? md5.to_s : nil
54
+ end
55
+
56
+ end
57
+
58
+ end
@@ -0,0 +1,83 @@
1
+ require 'optparse'
2
+ require 'fileutils'
3
+ require 's3aps/sync'
4
+
5
+ module S3aps
6
+
7
+ # Allows you to run S3aps from the command line
8
+ class Runner
9
+
10
+ attr_accessor :argv
11
+
12
+ def initialize(argv)
13
+ @options = {}
14
+ @cmd = :help
15
+ @option_parser = OptionParser.new do |o|
16
+ o.banner = <<EOS
17
+ S3aps version #{S3aps::VERSION}
18
+ Usage: s3aps (list|pull|push|version) [OPTIONS]
19
+
20
+ list List files locally and remotely
21
+ pull Pull new files from remote
22
+ push Push new files to remote (be careful)
23
+
24
+ EOS
25
+ o.on("-c", "--config FILE", "YAML file containing S3 credentials") {|name| @options[:config] = name }
26
+ o.on("-e", "--env ENV", "Section of YAML to use") {|name| @options[:env] = name }
27
+ o.on("-b", "--bucket NAME", "S3 bucket name") {|name| @options[:bucket] = name }
28
+ o.on("-k", "--key KEY", "S3 access key") {|name| @options[:key] = name }
29
+ o.on("-s", "--secret KEY", "S3 secret key") {|name| @options[:secret] = name }
30
+ o.on("-p", "--path PATH", "Local path") {|name| @options[:path] = name }
31
+ o.parse!(argv)
32
+ end
33
+ if argv.size == 1 && %w(list push pull version).include?(argv.first)
34
+ @sync = S3aps::Sync.new @options
35
+ @cmd = argv.first
36
+ end
37
+ end
38
+
39
+ def run #:nodoc:
40
+ send(@cmd)
41
+ end
42
+
43
+ def help #:nodoc:
44
+ puts @option_parser.help
45
+ end
46
+
47
+ def sync(options) #:nodoc:
48
+ S3aps::Sync.new options
49
+ end
50
+
51
+ # Prints out a list of all remote and local files and shows which ones
52
+ # are remote, local or both.
53
+ def list
54
+ array = @sync.list
55
+ local_count = 0
56
+ remote_count = 0
57
+ array.each do |item|
58
+ line = item.join(' ')
59
+ local_count +=1 if line =~ /^L\s\s/
60
+ remote_count +=1 if line =~ /^\s\sR/
61
+ puts item.join(' ')
62
+ end
63
+ puts "Found #{array.size} file(s): #{local_count} local only, #{remote_count} remote only, #{array.size - local_count - remote_count} common."
64
+ end
65
+
66
+ # Pull files from S3
67
+ def pull
68
+ count, total = @sync.pull
69
+ puts "\nPulled #{count}/#{total} file#{'s' unless count == 1} from S3"
70
+ end
71
+
72
+ # Push files to S3
73
+ def push
74
+ count, total = @sync.push
75
+ puts "\nPushed #{count} file#{'s' unless count == 1} to S3"
76
+ end
77
+
78
+ def version #:nodoc:
79
+ puts S3aps::VERSION
80
+ end
81
+
82
+ end
83
+ end
@@ -0,0 +1,40 @@
1
+ require 'rubygems'
2
+ require 'right_aws'
3
+
4
+ module S3aps
5
+
6
+ # Reads, writes and lists files on S3.
7
+ class S3Adapter
8
+
9
+ def initialize(options)
10
+ @s3 = RightAws::S3Interface.new(options[:key], options[:secret], :protocol => 'http', :port => 80)
11
+ @bucket_name = options[:bucket]
12
+ end
13
+
14
+ # List the keys of all files in the bucket
15
+ def list
16
+ keys = []
17
+ @s3.incrementally_list_bucket(@bucket_name) {|key_set|
18
+ keys << key_set[:contents].map {|key|
19
+ key
20
+ }
21
+ }
22
+ keys.flatten
23
+ end
24
+
25
+ # Write a value, potentially overwriting what is already there.
26
+ def write(s3_name, value)
27
+ @s3.put(@bucket_name, s3_name, value)
28
+ end
29
+
30
+ def read(key)
31
+ begin
32
+ @s3.get_object(@bucket_name, key)
33
+ rescue
34
+ raise exc
35
+ end
36
+ end
37
+
38
+ end
39
+
40
+ end
@@ -0,0 +1,110 @@
1
+ require 's3aps/file_adapter'
2
+ require 's3aps/s3_adapter'
3
+ require 'yaml'
4
+
5
+ module S3aps
6
+
7
+ def self.symbolize_keys(hash) #:nodoc:
8
+ new_hash = {}
9
+ hash.each do |key, value|
10
+ if Hash === value
11
+ new_hash[key.to_sym] = symbolize_keys value
12
+ else
13
+ new_hash[key.to_sym] = value
14
+ end
15
+ end
16
+ new_hash
17
+ end
18
+
19
+ # The controller for everything else.
20
+ class Sync
21
+
22
+ def initialize(options = {})
23
+ options = S3aps::symbolize_keys(options)
24
+ s3_file = [options[:config], "s3.yml", "config/s3.yml"].detect {|f| f && File.file?(f) }
25
+ if s3_file
26
+ puts "Configuring S3 from #{s3_file}"
27
+ yaml_options = S3aps::symbolize_keys YAML.load_file(s3_file)
28
+ env = options[:env]
29
+ if env && yaml_options[env.to_sym]
30
+ yaml_options = S3aps::symbolize_keys yaml_options[env.to_sym]
31
+ puts "Using #{env} environment"
32
+ end
33
+ # Support for alias: access_key_id = key
34
+ if yaml_options[:access_key_id]
35
+ yaml_options[:key] ||= yaml_options[:access_key_id]
36
+ yaml_options.delete(:access_key_id)
37
+ end
38
+ # Support for alias: secret_access_key = secret
39
+ if yaml_options[:secret_access_key]
40
+ yaml_options[:secret] ||= yaml_options[:secret_access_key]
41
+ yaml_options.delete(:secret_access_key)
42
+ end
43
+ options.merge!(yaml_options) {|k,o,n| o }
44
+ end
45
+ @s3_adapter = S3aps::S3Adapter.new options
46
+ @file_adapter = S3aps::FileAdapter.new options
47
+ end
48
+
49
+ # Merge the local and remote lists into one and indicate whether
50
+ # each item is local, remote or both
51
+ def list
52
+ local = @file_adapter.list
53
+ remote = @s3_adapter.list.map{|i| i[:key]}
54
+ all = (local + remote).uniq.sort
55
+ all.map do |path|
56
+ [
57
+ local.include?(path) ? "L" : " ",
58
+ remote.include?(path) ? "R" : " ",
59
+ path
60
+ ]
61
+ end
62
+ end
63
+
64
+ # Get all the remote files and write them out locally, if they don't already
65
+ # exist. For simplicity, we assume that if a file of the same name already
66
+ # exists then it is the same as the remote one.
67
+ def pull
68
+ check_list = @file_adapter.list
69
+ count = 0
70
+ total = 0
71
+ @s3_adapter.list.each do |o|
72
+ if !check_list.include?(o[:key])
73
+ $stdout.print "*"
74
+ if @file_adapter.write(o[:key], @s3_adapter.read(o[:key]), o[:etag])
75
+ count += 1
76
+ end
77
+ else
78
+ $stdout.print "."
79
+ end
80
+ $stdout.flush
81
+ total += 1
82
+ end
83
+ [count, total]
84
+ end
85
+
86
+ # Get all the local files, recursively, and write them to S3, if they don't already
87
+ # exist. For simplicity, we assume that if a file of the same name already
88
+ # exists then it is the same as the local one.
89
+ def push
90
+ check_list = @s3_adapter.list.map{|i| i[:key]}
91
+ count = 0
92
+ total = 0
93
+ @file_adapter.list.each do |path|
94
+ if !check_list.include?(path)
95
+ $stdout.print "*"
96
+ if @s3_adapter.write(path, @file_adapter.read(path))
97
+ count += 1
98
+ end
99
+ else
100
+ $stdout.print "."
101
+ end
102
+ $stdout.flush
103
+ total += 1
104
+ end
105
+ [count, total]
106
+ end
107
+
108
+ end
109
+
110
+ end
@@ -0,0 +1,3 @@
1
+ module S3aps #:nodoc:
2
+ VERSION = "0.0.1" #:nodoc:
3
+ end
@@ -0,0 +1,23 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "s3aps/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "s3aps"
7
+ s.version = S3aps::VERSION
8
+ s.platform = Gem::Platform::RUBY
9
+ s.authors = ["Bill Horsman"]
10
+ s.email = ["bill@logicalcobwebs.com"]
11
+ s.homepage = "http://github.com/billhorsman/s3aps"
12
+ s.summary = "s3aps-#{S3aps::VERSION}"
13
+ s.description = "Synchronize between S3 bucket and filesystem"
14
+
15
+ s.required_rubygems_version = ">= 1.3.6"
16
+
17
+ s.add_development_dependency "bundler", ">= 1.0.0"
18
+ s.add_runtime_dependency "right_aws"
19
+
20
+ s.files = `git ls-files`.split("\n")
21
+ s.executables = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
22
+ s.require_path = 'lib'
23
+ end
metadata ADDED
@@ -0,0 +1,103 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: s3aps
3
+ version: !ruby/object:Gem::Version
4
+ prerelease:
5
+ version: 0.0.1
6
+ platform: ruby
7
+ authors:
8
+ - Bill Horsman
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+
13
+ date: 2011-05-17 00:00:00 -07:00
14
+ default_executable:
15
+ dependencies:
16
+ - !ruby/object:Gem::Dependency
17
+ name: bundler
18
+ prerelease: false
19
+ requirement: &id001 !ruby/object:Gem::Requirement
20
+ none: false
21
+ requirements:
22
+ - - ">="
23
+ - !ruby/object:Gem::Version
24
+ version: 1.0.0
25
+ type: :development
26
+ version_requirements: *id001
27
+ - !ruby/object:Gem::Dependency
28
+ name: right_aws
29
+ prerelease: false
30
+ requirement: &id002 !ruby/object:Gem::Requirement
31
+ none: false
32
+ requirements:
33
+ - - ">="
34
+ - !ruby/object:Gem::Version
35
+ version: "0"
36
+ type: :runtime
37
+ version_requirements: *id002
38
+ description: Synchronize between S3 bucket and filesystem
39
+ email:
40
+ - bill@logicalcobwebs.com
41
+ executables:
42
+ - s3aps
43
+ extensions: []
44
+
45
+ extra_rdoc_files: []
46
+
47
+ files:
48
+ - .gitignore
49
+ - Gemfile
50
+ - README.rdoc
51
+ - Rakefile
52
+ - bin/s3aps
53
+ - doc/S3aps.html
54
+ - doc/S3aps/FileAdapter.html
55
+ - doc/S3aps/Runner.html
56
+ - doc/S3aps/S3Adapter.html
57
+ - doc/S3aps/Sync.html
58
+ - doc/created.rid
59
+ - doc/index.html
60
+ - doc/lib/s3aps/file_adapter_rb.html
61
+ - doc/lib/s3aps/runner_rb.html
62
+ - doc/lib/s3aps/s3_adapter_rb.html
63
+ - doc/lib/s3aps/sync_rb.html
64
+ - doc/lib/s3aps/version_rb.html
65
+ - doc/lib/s3aps_rb.html
66
+ - doc/rdoc.css
67
+ - lib/s3aps.rb
68
+ - lib/s3aps/file_adapter.rb
69
+ - lib/s3aps/runner.rb
70
+ - lib/s3aps/s3_adapter.rb
71
+ - lib/s3aps/sync.rb
72
+ - lib/s3aps/version.rb
73
+ - s3aps.gemspec
74
+ has_rdoc: true
75
+ homepage: http://github.com/billhorsman/s3aps
76
+ licenses: []
77
+
78
+ post_install_message:
79
+ rdoc_options: []
80
+
81
+ require_paths:
82
+ - lib
83
+ required_ruby_version: !ruby/object:Gem::Requirement
84
+ none: false
85
+ requirements:
86
+ - - ">="
87
+ - !ruby/object:Gem::Version
88
+ version: "0"
89
+ required_rubygems_version: !ruby/object:Gem::Requirement
90
+ none: false
91
+ requirements:
92
+ - - ">="
93
+ - !ruby/object:Gem::Version
94
+ version: 1.3.6
95
+ requirements: []
96
+
97
+ rubyforge_project:
98
+ rubygems_version: 1.6.2
99
+ signing_key:
100
+ specification_version: 3
101
+ summary: s3aps-0.0.1
102
+ test_files: []
103
+