s3aps 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +6 -0
- data/Gemfile +3 -0
- data/README.rdoc +105 -0
- data/Rakefile +2 -0
- data/bin/s3aps +17 -0
- data/doc/S3aps.html +161 -0
- data/doc/S3aps/FileAdapter.html +335 -0
- data/doc/S3aps/Runner.html +363 -0
- data/doc/S3aps/S3Adapter.html +322 -0
- data/doc/S3aps/Sync.html +380 -0
- data/doc/created.rid +7 -0
- data/doc/index.html +87 -0
- data/doc/lib/s3aps/file_adapter_rb.html +52 -0
- data/doc/lib/s3aps/runner_rb.html +58 -0
- data/doc/lib/s3aps/s3_adapter_rb.html +56 -0
- data/doc/lib/s3aps/sync_rb.html +58 -0
- data/doc/lib/s3aps/version_rb.html +52 -0
- data/doc/lib/s3aps_rb.html +62 -0
- data/doc/rdoc.css +706 -0
- data/lib/s3aps.rb +5 -0
- data/lib/s3aps/file_adapter.rb +58 -0
- data/lib/s3aps/runner.rb +83 -0
- data/lib/s3aps/s3_adapter.rb +40 -0
- data/lib/s3aps/sync.rb +110 -0
- data/lib/s3aps/version.rb +3 -0
- data/s3aps.gemspec +23 -0
- metadata +103 -0
data/lib/s3aps.rb
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
module S3aps
|
2
|
+
|
3
|
+
# Reads, writes and lists files on the filesystem.
|
4
|
+
class FileAdapter
|
5
|
+
|
6
|
+
# Options:
|
7
|
+
#
|
8
|
+
# * :path - where the local files are (default is +tmp+)
|
9
|
+
def initialize(options = {})
|
10
|
+
@path = options[:path] || "tmp"
|
11
|
+
end
|
12
|
+
|
13
|
+
# List all the files in the path, recursively
|
14
|
+
def list
|
15
|
+
Dir.glob("#{@path}/**/*").select{|path| File.file?(path) }.map do |path|
|
16
|
+
path.sub Regexp.new("^#{@path}\/"), ''
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
# Write the file. It will always write the file whether it's there
|
21
|
+
# or not, unless you supply an +md5sum+ in which case it will check
|
22
|
+
# whether it's different first.
|
23
|
+
def write(path, value, md5sum = nil)
|
24
|
+
just_path = path.sub /\/[^\/]*$/, ''
|
25
|
+
local_md5 = md5sum(path)
|
26
|
+
if local_md5.nil? || local_md5 != md5sum
|
27
|
+
FileUtils.mkdir_p(File.join(@path, just_path))
|
28
|
+
File.open(File.join(@path, path), 'w') {|outfile|
|
29
|
+
outfile.print(value)
|
30
|
+
}
|
31
|
+
true
|
32
|
+
else
|
33
|
+
false
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# Read the file.
|
38
|
+
def read(path)
|
39
|
+
File.open(File.join(@path, path),"rb") {|io| io.read}
|
40
|
+
end
|
41
|
+
|
42
|
+
def md5sum(path) #:nodoc:
|
43
|
+
md5 = nil
|
44
|
+
begin
|
45
|
+
md5 = File.open(File.join(@path, path), 'rb') do |io|
|
46
|
+
d = Digest::MD5.new
|
47
|
+
s = ""
|
48
|
+
d.update(s) while io.read(4096, s)
|
49
|
+
d
|
50
|
+
end
|
51
|
+
rescue
|
52
|
+
end
|
53
|
+
md5 ? md5.to_s : nil
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
data/lib/s3aps/runner.rb
ADDED
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'optparse'
|
2
|
+
require 'fileutils'
|
3
|
+
require 's3aps/sync'
|
4
|
+
|
5
|
+
module S3aps
|
6
|
+
|
7
|
+
# Allows you to run S3aps from the command line
|
8
|
+
class Runner
|
9
|
+
|
10
|
+
attr_accessor :argv
|
11
|
+
|
12
|
+
def initialize(argv)
|
13
|
+
@options = {}
|
14
|
+
@cmd = :help
|
15
|
+
@option_parser = OptionParser.new do |o|
|
16
|
+
o.banner = <<EOS
|
17
|
+
S3aps version #{S3aps::VERSION}
|
18
|
+
Usage: s3aps (list|pull|push|version) [OPTIONS]
|
19
|
+
|
20
|
+
list List files locally and remotely
|
21
|
+
pull Pull new files from remote
|
22
|
+
push Push new files to remote (be careful)
|
23
|
+
|
24
|
+
EOS
|
25
|
+
o.on("-c", "--config FILE", "YAML file containing S3 credentials") {|name| @options[:config] = name }
|
26
|
+
o.on("-e", "--env ENV", "Section of YAML to use") {|name| @options[:env] = name }
|
27
|
+
o.on("-b", "--bucket NAME", "S3 bucket name") {|name| @options[:bucket] = name }
|
28
|
+
o.on("-k", "--key KEY", "S3 access key") {|name| @options[:key] = name }
|
29
|
+
o.on("-s", "--secret KEY", "S3 secret key") {|name| @options[:secret] = name }
|
30
|
+
o.on("-p", "--path PATH", "Local path") {|name| @options[:path] = name }
|
31
|
+
o.parse!(argv)
|
32
|
+
end
|
33
|
+
if argv.size == 1 && %w(list push pull version).include?(argv.first)
|
34
|
+
@sync = S3aps::Sync.new @options
|
35
|
+
@cmd = argv.first
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def run #:nodoc:
|
40
|
+
send(@cmd)
|
41
|
+
end
|
42
|
+
|
43
|
+
def help #:nodoc:
|
44
|
+
puts @option_parser.help
|
45
|
+
end
|
46
|
+
|
47
|
+
def sync(options) #:nodoc:
|
48
|
+
S3aps::Sync.new options
|
49
|
+
end
|
50
|
+
|
51
|
+
# Prints out a list of all remote and local files and shows which ones
|
52
|
+
# are remote, local or both.
|
53
|
+
def list
|
54
|
+
array = @sync.list
|
55
|
+
local_count = 0
|
56
|
+
remote_count = 0
|
57
|
+
array.each do |item|
|
58
|
+
line = item.join(' ')
|
59
|
+
local_count +=1 if line =~ /^L\s\s/
|
60
|
+
remote_count +=1 if line =~ /^\s\sR/
|
61
|
+
puts item.join(' ')
|
62
|
+
end
|
63
|
+
puts "Found #{array.size} file(s): #{local_count} local only, #{remote_count} remote only, #{array.size - local_count - remote_count} common."
|
64
|
+
end
|
65
|
+
|
66
|
+
# Pull files from S3
|
67
|
+
def pull
|
68
|
+
count, total = @sync.pull
|
69
|
+
puts "\nPulled #{count}/#{total} file#{'s' unless count == 1} from S3"
|
70
|
+
end
|
71
|
+
|
72
|
+
# Push files to S3
|
73
|
+
def push
|
74
|
+
count, total = @sync.push
|
75
|
+
puts "\nPushed #{count} file#{'s' unless count == 1} to S3"
|
76
|
+
end
|
77
|
+
|
78
|
+
def version #:nodoc:
|
79
|
+
puts S3aps::VERSION
|
80
|
+
end
|
81
|
+
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'right_aws'
|
3
|
+
|
4
|
+
module S3aps
|
5
|
+
|
6
|
+
# Reads, writes and lists files on S3.
|
7
|
+
class S3Adapter
|
8
|
+
|
9
|
+
def initialize(options)
|
10
|
+
@s3 = RightAws::S3Interface.new(options[:key], options[:secret], :protocol => 'http', :port => 80)
|
11
|
+
@bucket_name = options[:bucket]
|
12
|
+
end
|
13
|
+
|
14
|
+
# List the keys of all files in the bucket
|
15
|
+
def list
|
16
|
+
keys = []
|
17
|
+
@s3.incrementally_list_bucket(@bucket_name) {|key_set|
|
18
|
+
keys << key_set[:contents].map {|key|
|
19
|
+
key
|
20
|
+
}
|
21
|
+
}
|
22
|
+
keys.flatten
|
23
|
+
end
|
24
|
+
|
25
|
+
# Write a value, potentially overwriting what is already there.
|
26
|
+
def write(s3_name, value)
|
27
|
+
@s3.put(@bucket_name, s3_name, value)
|
28
|
+
end
|
29
|
+
|
30
|
+
def read(key)
|
31
|
+
begin
|
32
|
+
@s3.get_object(@bucket_name, key)
|
33
|
+
rescue
|
34
|
+
raise exc
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
data/lib/s3aps/sync.rb
ADDED
@@ -0,0 +1,110 @@
|
|
1
|
+
require 's3aps/file_adapter'
|
2
|
+
require 's3aps/s3_adapter'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module S3aps
|
6
|
+
|
7
|
+
def self.symbolize_keys(hash) #:nodoc:
|
8
|
+
new_hash = {}
|
9
|
+
hash.each do |key, value|
|
10
|
+
if Hash === value
|
11
|
+
new_hash[key.to_sym] = symbolize_keys value
|
12
|
+
else
|
13
|
+
new_hash[key.to_sym] = value
|
14
|
+
end
|
15
|
+
end
|
16
|
+
new_hash
|
17
|
+
end
|
18
|
+
|
19
|
+
# The controller for everything else.
|
20
|
+
class Sync
|
21
|
+
|
22
|
+
def initialize(options = {})
|
23
|
+
options = S3aps::symbolize_keys(options)
|
24
|
+
s3_file = [options[:config], "s3.yml", "config/s3.yml"].detect {|f| f && File.file?(f) }
|
25
|
+
if s3_file
|
26
|
+
puts "Configuring S3 from #{s3_file}"
|
27
|
+
yaml_options = S3aps::symbolize_keys YAML.load_file(s3_file)
|
28
|
+
env = options[:env]
|
29
|
+
if env && yaml_options[env.to_sym]
|
30
|
+
yaml_options = S3aps::symbolize_keys yaml_options[env.to_sym]
|
31
|
+
puts "Using #{env} environment"
|
32
|
+
end
|
33
|
+
# Support for alias: access_key_id = key
|
34
|
+
if yaml_options[:access_key_id]
|
35
|
+
yaml_options[:key] ||= yaml_options[:access_key_id]
|
36
|
+
yaml_options.delete(:access_key_id)
|
37
|
+
end
|
38
|
+
# Support for alias: secret_access_key = secret
|
39
|
+
if yaml_options[:secret_access_key]
|
40
|
+
yaml_options[:secret] ||= yaml_options[:secret_access_key]
|
41
|
+
yaml_options.delete(:secret_access_key)
|
42
|
+
end
|
43
|
+
options.merge!(yaml_options) {|k,o,n| o }
|
44
|
+
end
|
45
|
+
@s3_adapter = S3aps::S3Adapter.new options
|
46
|
+
@file_adapter = S3aps::FileAdapter.new options
|
47
|
+
end
|
48
|
+
|
49
|
+
# Merge the local and remote lists into one and indicate whether
|
50
|
+
# each item is local, remote or both
|
51
|
+
def list
|
52
|
+
local = @file_adapter.list
|
53
|
+
remote = @s3_adapter.list.map{|i| i[:key]}
|
54
|
+
all = (local + remote).uniq.sort
|
55
|
+
all.map do |path|
|
56
|
+
[
|
57
|
+
local.include?(path) ? "L" : " ",
|
58
|
+
remote.include?(path) ? "R" : " ",
|
59
|
+
path
|
60
|
+
]
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
# Get all the remote files and write them out locally, if they don't already
|
65
|
+
# exist. For simplicity, we assume that if a file of the same name already
|
66
|
+
# exists then it is the same as the remote one.
|
67
|
+
def pull
|
68
|
+
check_list = @file_adapter.list
|
69
|
+
count = 0
|
70
|
+
total = 0
|
71
|
+
@s3_adapter.list.each do |o|
|
72
|
+
if !check_list.include?(o[:key])
|
73
|
+
$stdout.print "*"
|
74
|
+
if @file_adapter.write(o[:key], @s3_adapter.read(o[:key]), o[:etag])
|
75
|
+
count += 1
|
76
|
+
end
|
77
|
+
else
|
78
|
+
$stdout.print "."
|
79
|
+
end
|
80
|
+
$stdout.flush
|
81
|
+
total += 1
|
82
|
+
end
|
83
|
+
[count, total]
|
84
|
+
end
|
85
|
+
|
86
|
+
# Get all the local files, recursively, and write them to S3, if they don't already
|
87
|
+
# exist. For simplicity, we assume that if a file of the same name already
|
88
|
+
# exists then it is the same as the local one.
|
89
|
+
def push
|
90
|
+
check_list = @s3_adapter.list.map{|i| i[:key]}
|
91
|
+
count = 0
|
92
|
+
total = 0
|
93
|
+
@file_adapter.list.each do |path|
|
94
|
+
if !check_list.include?(path)
|
95
|
+
$stdout.print "*"
|
96
|
+
if @s3_adapter.write(path, @file_adapter.read(path))
|
97
|
+
count += 1
|
98
|
+
end
|
99
|
+
else
|
100
|
+
$stdout.print "."
|
101
|
+
end
|
102
|
+
$stdout.flush
|
103
|
+
total += 1
|
104
|
+
end
|
105
|
+
[count, total]
|
106
|
+
end
|
107
|
+
|
108
|
+
end
|
109
|
+
|
110
|
+
end
|
data/s3aps.gemspec
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "s3aps/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "s3aps"
|
7
|
+
s.version = S3aps::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Bill Horsman"]
|
10
|
+
s.email = ["bill@logicalcobwebs.com"]
|
11
|
+
s.homepage = "http://github.com/billhorsman/s3aps"
|
12
|
+
s.summary = "s3aps-#{S3aps::VERSION}"
|
13
|
+
s.description = "Synchronize between S3 bucket and filesystem"
|
14
|
+
|
15
|
+
s.required_rubygems_version = ">= 1.3.6"
|
16
|
+
|
17
|
+
s.add_development_dependency "bundler", ">= 1.0.0"
|
18
|
+
s.add_runtime_dependency "right_aws"
|
19
|
+
|
20
|
+
s.files = `git ls-files`.split("\n")
|
21
|
+
s.executables = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
|
22
|
+
s.require_path = 'lib'
|
23
|
+
end
|
metadata
ADDED
@@ -0,0 +1,103 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: s3aps
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease:
|
5
|
+
version: 0.0.1
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Bill Horsman
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
|
13
|
+
date: 2011-05-17 00:00:00 -07:00
|
14
|
+
default_executable:
|
15
|
+
dependencies:
|
16
|
+
- !ruby/object:Gem::Dependency
|
17
|
+
name: bundler
|
18
|
+
prerelease: false
|
19
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
20
|
+
none: false
|
21
|
+
requirements:
|
22
|
+
- - ">="
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: 1.0.0
|
25
|
+
type: :development
|
26
|
+
version_requirements: *id001
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: right_aws
|
29
|
+
prerelease: false
|
30
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
31
|
+
none: false
|
32
|
+
requirements:
|
33
|
+
- - ">="
|
34
|
+
- !ruby/object:Gem::Version
|
35
|
+
version: "0"
|
36
|
+
type: :runtime
|
37
|
+
version_requirements: *id002
|
38
|
+
description: Synchronize between S3 bucket and filesystem
|
39
|
+
email:
|
40
|
+
- bill@logicalcobwebs.com
|
41
|
+
executables:
|
42
|
+
- s3aps
|
43
|
+
extensions: []
|
44
|
+
|
45
|
+
extra_rdoc_files: []
|
46
|
+
|
47
|
+
files:
|
48
|
+
- .gitignore
|
49
|
+
- Gemfile
|
50
|
+
- README.rdoc
|
51
|
+
- Rakefile
|
52
|
+
- bin/s3aps
|
53
|
+
- doc/S3aps.html
|
54
|
+
- doc/S3aps/FileAdapter.html
|
55
|
+
- doc/S3aps/Runner.html
|
56
|
+
- doc/S3aps/S3Adapter.html
|
57
|
+
- doc/S3aps/Sync.html
|
58
|
+
- doc/created.rid
|
59
|
+
- doc/index.html
|
60
|
+
- doc/lib/s3aps/file_adapter_rb.html
|
61
|
+
- doc/lib/s3aps/runner_rb.html
|
62
|
+
- doc/lib/s3aps/s3_adapter_rb.html
|
63
|
+
- doc/lib/s3aps/sync_rb.html
|
64
|
+
- doc/lib/s3aps/version_rb.html
|
65
|
+
- doc/lib/s3aps_rb.html
|
66
|
+
- doc/rdoc.css
|
67
|
+
- lib/s3aps.rb
|
68
|
+
- lib/s3aps/file_adapter.rb
|
69
|
+
- lib/s3aps/runner.rb
|
70
|
+
- lib/s3aps/s3_adapter.rb
|
71
|
+
- lib/s3aps/sync.rb
|
72
|
+
- lib/s3aps/version.rb
|
73
|
+
- s3aps.gemspec
|
74
|
+
has_rdoc: true
|
75
|
+
homepage: http://github.com/billhorsman/s3aps
|
76
|
+
licenses: []
|
77
|
+
|
78
|
+
post_install_message:
|
79
|
+
rdoc_options: []
|
80
|
+
|
81
|
+
require_paths:
|
82
|
+
- lib
|
83
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: "0"
|
89
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
90
|
+
none: false
|
91
|
+
requirements:
|
92
|
+
- - ">="
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: 1.3.6
|
95
|
+
requirements: []
|
96
|
+
|
97
|
+
rubyforge_project:
|
98
|
+
rubygems_version: 1.6.2
|
99
|
+
signing_key:
|
100
|
+
specification_version: 3
|
101
|
+
summary: s3aps-0.0.1
|
102
|
+
test_files: []
|
103
|
+
|