s3rsync 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/README.md +17 -4
- data/lib/s3rsync/base.rb +18 -0
- data/lib/s3rsync/sync.rb +37 -5
- data/lib/s3rsync/version.rb +1 -1
- data/s3rsync.gemspec +3 -0
- data/spec/spec_helper.rb +6 -0
- metadata +43 -1
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MzYzODU3N2U1OGFmNzEwNzFlNzRkZWRkMmQ2YTZhY2ZlZTNkM2QyMw==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
YjhlMjJjZmFmNThhNzRkYjYwNWNmNzdjODc2N2Q2OWQwODY2NGE4OQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NTAzOGRlMTc5NWVlMjg5ZGNkNWNhMDVhMzRlNmViYjNmMTIzZGQ3ZmM2MTRl
|
10
|
+
ZjA5YzMyYzBiNjI2Mzk2OTY1ZjVlYzcwOTZiNzdkYTY2MzYzYmZiMGQ2YjU0
|
11
|
+
ODdmZGFkNjY1NDM3NjhhOTU0NjFjODMzOTVhZTVjNzYxOTY0MzU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
N2U1OGI0OWEyNDIwOWNlMDkwOTljMDVmMTRlMTBlMzM4ZTY4ZGI0ZDk3MTdi
|
14
|
+
YWNkMzQ1ZmMwYmY5NGIxN2I0NmE3N2NjZmMyODIwZjE0MzQxMzVmMWIyZDY2
|
15
|
+
MmEyNzhlMGIxMmEwYmQ0ODQ2MjU5YjY5OWYyMGZiZjFkYzBkZmE=
|
data/README.md
CHANGED
@@ -1,13 +1,23 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
s3rsync
|
3
2
|
============================================
|
4
3
|
[![Gem Version](http://img.shields.io/gem/v/s3rsync.svg)][gem]
|
5
4
|
[![Build Status](http://img.shields.io/travis/idevops/s3rsync.svg)][travis]
|
5
|
+
[![Coverage Status](https://img.shields.io/coveralls/idevops/s3rsync.svg)][coveralls]
|
6
6
|
|
7
7
|
[gem]: https://rubygems.org/gems/s3rsync
|
8
8
|
[travis]: http://travis-ci.org/idevops/s3rsync
|
9
|
+
[coveralls]: https://coveralls.io/r/idevops/s3rsync
|
10
|
+
|
11
|
+
Wrapper gem to sync files with s3cmd tool.
|
12
|
+
Remote update protocol between local set of files and S3 storage.
|
9
13
|
|
10
|
-
|
14
|
+
Prerequisites
|
15
|
+
-------------
|
16
|
+
Omnibus is designed to run with a minimal set of prerequisites. You will need the following:
|
17
|
+
|
18
|
+
- Ruby 1.9+
|
19
|
+
- Bundler
|
20
|
+
- s3cmd
|
11
21
|
|
12
22
|
## Installation
|
13
23
|
|
@@ -25,7 +35,10 @@ Or install it yourself as:
|
|
25
35
|
|
26
36
|
## Usage
|
27
37
|
|
28
|
-
|
38
|
+
$ s3rsync help [COMMAND] # Describe available commands or one specific command
|
39
|
+
$ s3rsync upload -c, --config-path=CONFIG_PATH -d, --dir=DIR # Sync local dir to s3 location
|
40
|
+
$ s3rsync download -c, --config-path=CONFIG_PATH -d, --dir=DIR # Sync from s3 location to local dir
|
41
|
+
|
29
42
|
|
30
43
|
## Contributing
|
31
44
|
|
data/lib/s3rsync/base.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
require 'logger'
|
2
|
+
require 'statsd'
|
3
|
+
require 'socket'
|
2
4
|
|
3
5
|
module S3Rsync
|
4
6
|
class Base
|
@@ -9,6 +11,22 @@ module S3Rsync
|
|
9
11
|
logger
|
10
12
|
end
|
11
13
|
|
14
|
+
# Usage:
|
15
|
+
#@statsd.increment 'garets'
|
16
|
+
#@statsd.timing 'glork', 320
|
17
|
+
#@statsd.gauge 'bork', 100
|
18
|
+
def init_statsd(host = 'localhost', port = 9125)
|
19
|
+
hostname_prefix = Socket.gethostname.split('.').first
|
20
|
+
statsd = Statsd.new host, port
|
21
|
+
statsd.namespace = hostname_prefix
|
22
|
+
statsd
|
23
|
+
end
|
24
|
+
|
25
|
+
def error_exit(msg)
|
26
|
+
@log.error msg
|
27
|
+
return 1
|
28
|
+
end
|
29
|
+
|
12
30
|
def error_exit(msg)
|
13
31
|
@log.error msg
|
14
32
|
return 1
|
data/lib/s3rsync/sync.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
require "s3rsync"
|
2
2
|
require "json"
|
3
|
+
require "lockfile"
|
3
4
|
|
4
5
|
module S3Rsync
|
5
6
|
class Sync < Base
|
@@ -24,22 +25,53 @@ module S3Rsync
|
|
24
25
|
end
|
25
26
|
|
26
27
|
@log.debug "conf: #{conf}"
|
27
|
-
@params[:s3_bucket]
|
28
|
-
@params[:s3_access]
|
29
|
-
@params[:s3_secret]
|
28
|
+
@params[:s3_bucket] = conf['env.static.s3.bucket']
|
29
|
+
@params[:s3_access] = conf['env.static.s3.key.user']
|
30
|
+
@params[:s3_secret] = conf['env.static.s3.key.secret']
|
31
|
+
@params[:statsd_host] = conf['env.metrics.host'] || 'localhost'
|
32
|
+
@params[:statsd_port] = conf['env.metrics.port'] || 8125
|
30
33
|
@log.debug "@params: #{@params.inspect}"
|
34
|
+
@statsd = init_statsd @params[:statsd_host], @params[:statsd_port]
|
35
|
+
@log.debug "@statsd: #{@statsd.inspect}"
|
36
|
+
end
|
37
|
+
|
38
|
+
def get_lock(name = 'common')
|
39
|
+
lockfile = Lockfile.new(
|
40
|
+
"/tmp/s3rsync_#{name}.lock",
|
41
|
+
:timeout => 10,
|
42
|
+
:max_age => 3600,
|
43
|
+
:debug => !ENV['DEBUG'].nil?
|
44
|
+
)
|
31
45
|
end
|
32
46
|
|
33
47
|
def run(mode = :upload)
|
34
48
|
sync_locations = "#{@params[:dir]} s3://#{@params[:s3_bucket]}/#{@params[:s3_prefix]}/"
|
35
49
|
@log.debug "original sync_locations: #{sync_locations}"
|
50
|
+
lockfile = get_lock(mode)
|
51
|
+
@log.debug "lockfile: #{lockfile.inspect}"
|
52
|
+
if lockfile.locked?
|
53
|
+
@log.warn "lock file found, exiting: #{lockfile.inspect}"
|
54
|
+
return 5
|
55
|
+
end
|
56
|
+
|
36
57
|
if mode == :download
|
37
58
|
@log.debug "reversing sync_locations"
|
38
59
|
sync_locations = sync_locations.split(' ').reverse.join(' ')
|
39
60
|
@log.debug "download sync_locations: #{sync_locations}"
|
40
61
|
end
|
41
|
-
|
42
|
-
|
62
|
+
|
63
|
+
sync_output = 1
|
64
|
+
begin
|
65
|
+
lockfile.lock
|
66
|
+
sync_output = pass_cli "s3cmd sync #{sync_locations}"
|
67
|
+
@log.debug "sync_output: #{sync_output}"
|
68
|
+
rescue Exception => e
|
69
|
+
sync_output = 1
|
70
|
+
@log.error "something wrong while trying to run the sync: #{e.inspect}"
|
71
|
+
ensure
|
72
|
+
lockfile.unlock
|
73
|
+
end
|
74
|
+
|
43
75
|
if (sync_output == 1) || sync_output.downcase.include?('error')
|
44
76
|
@log.error "errors found in output: #{sync_output}"
|
45
77
|
return 1
|
data/lib/s3rsync/version.rb
CHANGED
data/s3rsync.gemspec
CHANGED
@@ -19,6 +19,8 @@ Gem::Specification.new do |spec|
|
|
19
19
|
spec.require_paths = ["lib"]
|
20
20
|
|
21
21
|
spec.add_dependency "thor", "0.19.1"
|
22
|
+
spec.add_dependency "lockfile", "2.1.3"
|
23
|
+
spec.add_dependency "statsd-ruby", "1.2.1"
|
22
24
|
|
23
25
|
spec.add_development_dependency "bundler", "~> 1.5"
|
24
26
|
spec.add_development_dependency "rake"
|
@@ -28,4 +30,5 @@ Gem::Specification.new do |spec|
|
|
28
30
|
spec.add_development_dependency "growl"
|
29
31
|
spec.add_development_dependency "guard"
|
30
32
|
spec.add_development_dependency "guard-rspec"
|
33
|
+
spec.add_development_dependency "coveralls"
|
31
34
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,7 +1,13 @@
|
|
1
1
|
require "simplecov"
|
2
|
+
require "coveralls"
|
2
3
|
require "s3rsync"
|
3
4
|
require "s3rsync/base"
|
4
5
|
|
6
|
+
SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
|
7
|
+
SimpleCov::Formatter::HTMLFormatter,
|
8
|
+
Coveralls::SimpleCov::Formatter
|
9
|
+
]
|
10
|
+
|
5
11
|
SimpleCov.start do
|
6
12
|
add_filter '/spec/'
|
7
13
|
add_filter '/bin/'
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3rsync
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- idevops
|
@@ -24,6 +24,34 @@ dependencies:
|
|
24
24
|
- - '='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 0.19.1
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: lockfile
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 2.1.3
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 2.1.3
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: statsd-ruby
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 1.2.1
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 1.2.1
|
27
55
|
- !ruby/object:Gem::Dependency
|
28
56
|
name: bundler
|
29
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -136,6 +164,20 @@ dependencies:
|
|
136
164
|
- - ! '>='
|
137
165
|
- !ruby/object:Gem::Version
|
138
166
|
version: '0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: coveralls
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ! '>='
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ! '>='
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
139
181
|
description: Remote update protocol allows sync files to transfer just the differences
|
140
182
|
between two sets of files across the network connection between local filesystem
|
141
183
|
and s3 storage
|