s3aps 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +6 -0
- data/Gemfile +3 -0
- data/README.rdoc +105 -0
- data/Rakefile +2 -0
- data/bin/s3aps +17 -0
- data/doc/S3aps.html +161 -0
- data/doc/S3aps/FileAdapter.html +335 -0
- data/doc/S3aps/Runner.html +363 -0
- data/doc/S3aps/S3Adapter.html +322 -0
- data/doc/S3aps/Sync.html +380 -0
- data/doc/created.rid +7 -0
- data/doc/index.html +87 -0
- data/doc/lib/s3aps/file_adapter_rb.html +52 -0
- data/doc/lib/s3aps/runner_rb.html +58 -0
- data/doc/lib/s3aps/s3_adapter_rb.html +56 -0
- data/doc/lib/s3aps/sync_rb.html +58 -0
- data/doc/lib/s3aps/version_rb.html +52 -0
- data/doc/lib/s3aps_rb.html +62 -0
- data/doc/rdoc.css +706 -0
- data/lib/s3aps.rb +5 -0
- data/lib/s3aps/file_adapter.rb +58 -0
- data/lib/s3aps/runner.rb +83 -0
- data/lib/s3aps/s3_adapter.rb +40 -0
- data/lib/s3aps/sync.rb +110 -0
- data/lib/s3aps/version.rb +3 -0
- data/s3aps.gemspec +23 -0
- metadata +103 -0
data/lib/s3aps.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
module S3aps
|
2
|
+
|
3
|
+
# Reads, writes and lists files on the filesystem.
|
4
|
+
class FileAdapter
|
5
|
+
|
6
|
+
# Options:
|
7
|
+
#
|
8
|
+
# * :path - where the local files are (default is +tmp+)
|
9
|
+
def initialize(options = {})
|
10
|
+
@path = options[:path] || "tmp"
|
11
|
+
end
|
12
|
+
|
13
|
+
# List all the files in the path, recursively
|
14
|
+
def list
|
15
|
+
Dir.glob("#{@path}/**/*").select{|path| File.file?(path) }.map do |path|
|
16
|
+
path.sub Regexp.new("^#{@path}\/"), ''
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# Write the file. It will always write the file whether it's there
|
21
|
+
# or not, unless you supply an +md5sum+ in which case it will check
|
22
|
+
# whether it's different first.
|
23
|
+
def write(path, value, md5sum = nil)
|
24
|
+
just_path = path.sub /\/[^\/]*$/, ''
|
25
|
+
local_md5 = md5sum(path)
|
26
|
+
if local_md5.nil? || local_md5 != md5sum
|
27
|
+
FileUtils.mkdir_p(File.join(@path, just_path))
|
28
|
+
File.open(File.join(@path, path), 'w') {|outfile|
|
29
|
+
outfile.print(value)
|
30
|
+
}
|
31
|
+
true
|
32
|
+
else
|
33
|
+
false
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Read the file.
|
38
|
+
def read(path)
|
39
|
+
File.open(File.join(@path, path),"rb") {|io| io.read}
|
40
|
+
end
|
41
|
+
|
42
|
+
def md5sum(path) #:nodoc:
|
43
|
+
md5 = nil
|
44
|
+
begin
|
45
|
+
md5 = File.open(File.join(@path, path), 'rb') do |io|
|
46
|
+
d = Digest::MD5.new
|
47
|
+
s = ""
|
48
|
+
d.update(s) while io.read(4096, s)
|
49
|
+
d
|
50
|
+
end
|
51
|
+
rescue
|
52
|
+
end
|
53
|
+
md5 ? md5.to_s : nil
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
data/lib/s3aps/runner.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'fileutils'
|
3
|
+
require 's3aps/sync'
|
4
|
+
|
5
|
+
module S3aps
|
6
|
+
|
7
|
+
# Allows you to run S3aps from the command line
|
8
|
+
class Runner
|
9
|
+
|
10
|
+
attr_accessor :argv
|
11
|
+
|
12
|
+
def initialize(argv)
|
13
|
+
@options = {}
|
14
|
+
@cmd = :help
|
15
|
+
@option_parser = OptionParser.new do |o|
|
16
|
+
o.banner = <<EOS
|
17
|
+
S3aps version #{S3aps::VERSION}
|
18
|
+
Usage: s3aps (list|pull|push|version) [OPTIONS]
|
19
|
+
|
20
|
+
list List files locally and remotely
|
21
|
+
pull Pull new files from remote
|
22
|
+
push Push new files to remote (be careful)
|
23
|
+
|
24
|
+
EOS
|
25
|
+
o.on("-c", "--config FILE", "YAML file containing S3 credentials") {|name| @options[:config] = name }
|
26
|
+
o.on("-e", "--env ENV", "Section of YAML to use") {|name| @options[:env] = name }
|
27
|
+
o.on("-b", "--bucket NAME", "S3 bucket name") {|name| @options[:bucket] = name }
|
28
|
+
o.on("-k", "--key KEY", "S3 access key") {|name| @options[:key] = name }
|
29
|
+
o.on("-s", "--secret KEY", "S3 secret key") {|name| @options[:secret] = name }
|
30
|
+
o.on("-p", "--path PATH", "Local path") {|name| @options[:path] = name }
|
31
|
+
o.parse!(argv)
|
32
|
+
end
|
33
|
+
if argv.size == 1 && %w(list push pull version).include?(argv.first)
|
34
|
+
@sync = S3aps::Sync.new @options
|
35
|
+
@cmd = argv.first
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def run #:nodoc:
|
40
|
+
send(@cmd)
|
41
|
+
end
|
42
|
+
|
43
|
+
def help #:nodoc:
|
44
|
+
puts @option_parser.help
|
45
|
+
end
|
46
|
+
|
47
|
+
def sync(options) #:nodoc:
|
48
|
+
S3aps::Sync.new options
|
49
|
+
end
|
50
|
+
|
51
|
+
# Prints out a list of all remote and local files and shows which ones
|
52
|
+
# are remote, local or both.
|
53
|
+
def list
|
54
|
+
array = @sync.list
|
55
|
+
local_count = 0
|
56
|
+
remote_count = 0
|
57
|
+
array.each do |item|
|
58
|
+
line = item.join(' ')
|
59
|
+
local_count +=1 if line =~ /^L\s\s/
|
60
|
+
remote_count +=1 if line =~ /^\s\sR/
|
61
|
+
puts item.join(' ')
|
62
|
+
end
|
63
|
+
puts "Found #{array.size} file(s): #{local_count} local only, #{remote_count} remote only, #{array.size - local_count - remote_count} common."
|
64
|
+
end
|
65
|
+
|
66
|
+
# Pull files from S3
|
67
|
+
def pull
|
68
|
+
count, total = @sync.pull
|
69
|
+
puts "\nPulled #{count}/#{total} file#{'s' unless count == 1} from S3"
|
70
|
+
end
|
71
|
+
|
72
|
+
# Push files to S3
|
73
|
+
def push
|
74
|
+
count, total = @sync.push
|
75
|
+
puts "\nPushed #{count} file#{'s' unless count == 1} to S3"
|
76
|
+
end
|
77
|
+
|
78
|
+
def version #:nodoc:
|
79
|
+
puts S3aps::VERSION
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'right_aws'
|
3
|
+
|
4
|
+
module S3aps
|
5
|
+
|
6
|
+
# Reads, writes and lists files on S3.
|
7
|
+
class S3Adapter
|
8
|
+
|
9
|
+
def initialize(options)
|
10
|
+
@s3 = RightAws::S3Interface.new(options[:key], options[:secret], :protocol => 'http', :port => 80)
|
11
|
+
@bucket_name = options[:bucket]
|
12
|
+
end
|
13
|
+
|
14
|
+
# List the keys of all files in the bucket
|
15
|
+
def list
|
16
|
+
keys = []
|
17
|
+
@s3.incrementally_list_bucket(@bucket_name) {|key_set|
|
18
|
+
keys << key_set[:contents].map {|key|
|
19
|
+
key
|
20
|
+
}
|
21
|
+
}
|
22
|
+
keys.flatten
|
23
|
+
end
|
24
|
+
|
25
|
+
# Write a value, potentially overwriting what is already there.
|
26
|
+
def write(s3_name, value)
|
27
|
+
@s3.put(@bucket_name, s3_name, value)
|
28
|
+
end
|
29
|
+
|
30
|
+
def read(key)
|
31
|
+
begin
|
32
|
+
@s3.get_object(@bucket_name, key)
|
33
|
+
rescue
|
34
|
+
raise exc
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
data/lib/s3aps/sync.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
require 's3aps/file_adapter'
|
2
|
+
require 's3aps/s3_adapter'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module S3aps
|
6
|
+
|
7
|
+
def self.symbolize_keys(hash) #:nodoc:
|
8
|
+
new_hash = {}
|
9
|
+
hash.each do |key, value|
|
10
|
+
if Hash === value
|
11
|
+
new_hash[key.to_sym] = symbolize_keys value
|
12
|
+
else
|
13
|
+
new_hash[key.to_sym] = value
|
14
|
+
end
|
15
|
+
end
|
16
|
+
new_hash
|
17
|
+
end
|
18
|
+
|
19
|
+
# The controller for everything else.
|
20
|
+
class Sync
|
21
|
+
|
22
|
+
def initialize(options = {})
|
23
|
+
options = S3aps::symbolize_keys(options)
|
24
|
+
s3_file = [options[:config], "s3.yml", "config/s3.yml"].detect {|f| f && File.file?(f) }
|
25
|
+
if s3_file
|
26
|
+
puts "Configuring S3 from #{s3_file}"
|
27
|
+
yaml_options = S3aps::symbolize_keys YAML.load_file(s3_file)
|
28
|
+
env = options[:env]
|
29
|
+
if env && yaml_options[env.to_sym]
|
30
|
+
yaml_options = S3aps::symbolize_keys yaml_options[env.to_sym]
|
31
|
+
puts "Using #{env} environment"
|
32
|
+
end
|
33
|
+
# Support for alias: access_key_id = key
|
34
|
+
if yaml_options[:access_key_id]
|
35
|
+
yaml_options[:key] ||= yaml_options[:access_key_id]
|
36
|
+
yaml_options.delete(:access_key_id)
|
37
|
+
end
|
38
|
+
# Support for alias: secret_access_key = secret
|
39
|
+
if yaml_options[:secret_access_key]
|
40
|
+
yaml_options[:secret] ||= yaml_options[:secret_access_key]
|
41
|
+
yaml_options.delete(:secret_access_key)
|
42
|
+
end
|
43
|
+
options.merge!(yaml_options) {|k,o,n| o }
|
44
|
+
end
|
45
|
+
@s3_adapter = S3aps::S3Adapter.new options
|
46
|
+
@file_adapter = S3aps::FileAdapter.new options
|
47
|
+
end
|
48
|
+
|
49
|
+
# Merge the local and remote lists into one and indicate whether
|
50
|
+
# each item is local, remote or both
|
51
|
+
def list
|
52
|
+
local = @file_adapter.list
|
53
|
+
remote = @s3_adapter.list.map{|i| i[:key]}
|
54
|
+
all = (local + remote).uniq.sort
|
55
|
+
all.map do |path|
|
56
|
+
[
|
57
|
+
local.include?(path) ? "L" : " ",
|
58
|
+
remote.include?(path) ? "R" : " ",
|
59
|
+
path
|
60
|
+
]
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Get all the remote files and write them out locally, if they don't already
|
65
|
+
# exist. For simplicity, we assume that if a file of the same name already
|
66
|
+
# exists then it is the same as the remote one.
|
67
|
+
def pull
|
68
|
+
check_list = @file_adapter.list
|
69
|
+
count = 0
|
70
|
+
total = 0
|
71
|
+
@s3_adapter.list.each do |o|
|
72
|
+
if !check_list.include?(o[:key])
|
73
|
+
$stdout.print "*"
|
74
|
+
if @file_adapter.write(o[:key], @s3_adapter.read(o[:key]), o[:etag])
|
75
|
+
count += 1
|
76
|
+
end
|
77
|
+
else
|
78
|
+
$stdout.print "."
|
79
|
+
end
|
80
|
+
$stdout.flush
|
81
|
+
total += 1
|
82
|
+
end
|
83
|
+
[count, total]
|
84
|
+
end
|
85
|
+
|
86
|
+
# Get all the local files, recursively, and write them to S3, if they don't already
|
87
|
+
# exist. For simplicity, we assume that if a file of the same name already
|
88
|
+
# exists then it is the same as the local one.
|
89
|
+
def push
|
90
|
+
check_list = @s3_adapter.list.map{|i| i[:key]}
|
91
|
+
count = 0
|
92
|
+
total = 0
|
93
|
+
@file_adapter.list.each do |path|
|
94
|
+
if !check_list.include?(path)
|
95
|
+
$stdout.print "*"
|
96
|
+
if @s3_adapter.write(path, @file_adapter.read(path))
|
97
|
+
count += 1
|
98
|
+
end
|
99
|
+
else
|
100
|
+
$stdout.print "."
|
101
|
+
end
|
102
|
+
$stdout.flush
|
103
|
+
total += 1
|
104
|
+
end
|
105
|
+
[count, total]
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
data/s3aps.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "s3aps/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "s3aps"
|
7
|
+
s.version = S3aps::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Bill Horsman"]
|
10
|
+
s.email = ["bill@logicalcobwebs.com"]
|
11
|
+
s.homepage = "http://github.com/billhorsman/s3aps"
|
12
|
+
s.summary = "s3aps-#{S3aps::VERSION}"
|
13
|
+
s.description = "Synchronize between S3 bucket and filesystem"
|
14
|
+
|
15
|
+
s.required_rubygems_version = ">= 1.3.6"
|
16
|
+
|
17
|
+
s.add_development_dependency "bundler", ">= 1.0.0"
|
18
|
+
s.add_runtime_dependency "right_aws"
|
19
|
+
|
20
|
+
s.files = `git ls-files`.split("\n")
|
21
|
+
s.executables = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
|
22
|
+
s.require_path = 'lib'
|
23
|
+
end
|
metadata
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: s3aps
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.1
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Bill Horsman
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-05-17 00:00:00 -07:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: bundler
|
18
|
+
prerelease: false
|
19
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
20
|
+
none: false
|
21
|
+
requirements:
|
22
|
+
- - ">="
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 1.0.0
|
25
|
+
type: :development
|
26
|
+
version_requirements: *id001
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: right_aws
|
29
|
+
prerelease: false
|
30
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
31
|
+
none: false
|
32
|
+
requirements:
|
33
|
+
- - ">="
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: "0"
|
36
|
+
type: :runtime
|
37
|
+
version_requirements: *id002
|
38
|
+
description: Synchronize between S3 bucket and filesystem
|
39
|
+
email:
|
40
|
+
- bill@logicalcobwebs.com
|
41
|
+
executables:
|
42
|
+
- s3aps
|
43
|
+
extensions: []
|
44
|
+
|
45
|
+
extra_rdoc_files: []
|
46
|
+
|
47
|
+
files:
|
48
|
+
- .gitignore
|
49
|
+
- Gemfile
|
50
|
+
- README.rdoc
|
51
|
+
- Rakefile
|
52
|
+
- bin/s3aps
|
53
|
+
- doc/S3aps.html
|
54
|
+
- doc/S3aps/FileAdapter.html
|
55
|
+
- doc/S3aps/Runner.html
|
56
|
+
- doc/S3aps/S3Adapter.html
|
57
|
+
- doc/S3aps/Sync.html
|
58
|
+
- doc/created.rid
|
59
|
+
- doc/index.html
|
60
|
+
- doc/lib/s3aps/file_adapter_rb.html
|
61
|
+
- doc/lib/s3aps/runner_rb.html
|
62
|
+
- doc/lib/s3aps/s3_adapter_rb.html
|
63
|
+
- doc/lib/s3aps/sync_rb.html
|
64
|
+
- doc/lib/s3aps/version_rb.html
|
65
|
+
- doc/lib/s3aps_rb.html
|
66
|
+
- doc/rdoc.css
|
67
|
+
- lib/s3aps.rb
|
68
|
+
- lib/s3aps/file_adapter.rb
|
69
|
+
- lib/s3aps/runner.rb
|
70
|
+
- lib/s3aps/s3_adapter.rb
|
71
|
+
- lib/s3aps/sync.rb
|
72
|
+
- lib/s3aps/version.rb
|
73
|
+
- s3aps.gemspec
|
74
|
+
has_rdoc: true
|
75
|
+
homepage: http://github.com/billhorsman/s3aps
|
76
|
+
licenses: []
|
77
|
+
|
78
|
+
post_install_message:
|
79
|
+
rdoc_options: []
|
80
|
+
|
81
|
+
require_paths:
|
82
|
+
- lib
|
83
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: "0"
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 1.3.6
|
95
|
+
requirements: []
|
96
|
+
|
97
|
+
rubyforge_project:
|
98
|
+
rubygems_version: 1.6.2
|
99
|
+
signing_key:
|
100
|
+
specification_version: 3
|
101
|
+
summary: s3aps-0.0.1
|
102
|
+
test_files: []
|
103
|
+
|