right_develop 1.2.2 → 2.0.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.ruby-version +1 -0
- data/CHANGELOG.rdoc +6 -1
- data/Rakefile +5 -3
- data/VERSION +1 -1
- data/lib/right_develop/ci/java_spec_formatter.rb +1 -1
- data/lib/right_develop/commands/git.rb +28 -2
- data/lib/right_develop/git/rake_task.rb +102 -0
- data/lib/right_develop/git.rb +27 -19
- data/lib/right_develop/parsers/sax_parser.rb +22 -5
- data/lib/right_develop/parsers/xml_post_parser.rb +11 -2
- data/lib/right_develop/parsers.rb +1 -1
- data/lib/right_develop/s3/interface.rb +298 -0
- data/lib/right_develop/s3/rake_task.rb +168 -0
- data/lib/right_develop/s3.rb +31 -0
- data/lib/right_develop/utility/git.rb +364 -0
- data/lib/right_develop/utility/shell.rb +131 -0
- data/lib/right_develop/utility/versioning.rb +183 -0
- data/lib/right_develop/utility.rb +32 -0
- data/lib/right_develop.rb +3 -1
- data/right_develop.gemspec +25 -298
- data/right_develop.rconf +3 -3
- metadata +184 -1729
- data/lib/right_develop/git/branch.rb +0 -74
- data/lib/right_develop/git/branch_collection.rb +0 -72
- data/lib/right_develop/git/commit.rb +0 -30
- data/lib/right_develop/git/repository.rb +0 -57
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.8.7-p371
|
data/CHANGELOG.rdoc
CHANGED
data/Rakefile
CHANGED
@@ -11,6 +11,8 @@ require 'rspec/core/rake_task'
|
|
11
11
|
require 'cucumber/rake/task'
|
12
12
|
|
13
13
|
# We use RightDevelop's CI harness in its own Rakefile. Hooray dogfood!
|
14
|
+
lib_dir = File.expand_path('../lib', __FILE__)
|
15
|
+
$: << lib_dir unless $:.include?(lib_dir)
|
14
16
|
require 'right_develop'
|
15
17
|
|
16
18
|
# But, we have a very special need, because OUR Cucumbers need to run with a pristine
|
@@ -62,10 +64,10 @@ Jeweler::Tasks.new do |gem|
|
|
62
64
|
gem.files.exclude "spec/**/*"
|
63
65
|
end
|
64
66
|
|
65
|
-
|
66
|
-
# push this gem to the public!
|
67
|
-
#Jeweler::RubygemsDotOrgTasks.new
|
67
|
+
Jeweler::RubygemsDotOrgTasks.new
|
68
68
|
|
69
69
|
CLEAN.include('pkg')
|
70
70
|
|
71
71
|
RightDevelop::CI::RakeTask.new
|
72
|
+
RightDevelop::Git::RakeTask.new
|
73
|
+
RightDevelop::S3::RakeTask.new
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
2.0.1
|
@@ -25,7 +25,7 @@ require 'time'
|
|
25
25
|
require 'builder'
|
26
26
|
|
27
27
|
# Try to load RSpec 2.x - 1.x formatters
|
28
|
-
['rspec/core', 'spec', 'rspec/core/formatters/
|
28
|
+
['rspec/core', 'spec', 'rspec/core/formatters/base_formatter', 'spec/runner/formatter/base_text_formatter'].each do |f|
|
29
29
|
begin
|
30
30
|
require f
|
31
31
|
rescue LoadError
|
@@ -1,3 +1,27 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'right_git'
|
24
|
+
require 'right_develop'
|
1
25
|
require "action_view"
|
2
26
|
|
3
27
|
module RightDevelop::Commands
|
@@ -46,8 +70,10 @@ EOS
|
|
46
70
|
|
47
71
|
case task
|
48
72
|
when "prune"
|
49
|
-
|
50
|
-
|
73
|
+
repo = ::RightGit::Repository.new(
|
74
|
+
::Dir.pwd,
|
75
|
+
::RightDevelop::Utility::Git::DEFALT_REPO_OPTIONS)
|
76
|
+
self.new(repo, :prune, options)
|
51
77
|
else
|
52
78
|
Trollop.die "unknown task #{task}"
|
53
79
|
end
|
@@ -0,0 +1,102 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
|
23
|
+
# Make sure the rest of RightDevelop & Gitis required, since this file can be
|
24
|
+
# required directly.
|
25
|
+
require 'right_develop/git'
|
26
|
+
|
27
|
+
# Once this file is required, the Rake DSL is loaded - don't do this except inside Rake!!
|
28
|
+
require 'rake/tasklib'
|
29
|
+
|
30
|
+
module RightDevelop::Git
|
31
|
+
|
32
|
+
class RakeTask < ::Rake::TaskLib
|
33
|
+
DEFAULT_OPTIONS = {
|
34
|
+
:git_namespace => :git,
|
35
|
+
:pre_checkout_step => nil,
|
36
|
+
:post_checkout_step => nil,
|
37
|
+
:pre_verify_step => nil,
|
38
|
+
:post_verify_step => nil,
|
39
|
+
}
|
40
|
+
|
41
|
+
include ::Rake::DSL if defined?(::Rake::DSL)
|
42
|
+
|
43
|
+
attr_accessor :git_namespace
|
44
|
+
attr_accessor :pre_checkout_step, :post_checkout_step
|
45
|
+
attr_accessor :pre_verify_step, :post_verify_step
|
46
|
+
|
47
|
+
def initialize(options = {})
|
48
|
+
# Let client provide options object-style, in our initializer
|
49
|
+
options = DEFAULT_OPTIONS.merge(options)
|
50
|
+
self.git_namespace = options[:git_namespace]
|
51
|
+
self.pre_checkout_step = options[:pre_checkout_step]
|
52
|
+
self.post_checkout_step = options[:post_checkout_step]
|
53
|
+
self.pre_verify_step = options[:pre_verify_step]
|
54
|
+
self.post_verify_step = options[:post_verify_step]
|
55
|
+
|
56
|
+
# Let client provide options DSL-style by calling our writers
|
57
|
+
yield(self) if block_given?
|
58
|
+
|
59
|
+
namespace self.git_namespace do
|
60
|
+
|
61
|
+
desc "Perform 'git submodule update --init --recursive'"
|
62
|
+
task :setup do
|
63
|
+
git.setup
|
64
|
+
end
|
65
|
+
|
66
|
+
desc "If HEAD is a branch or tag ref, ensure that all submodules are checked out to the same tag or branch or ensure consistency for SHA"
|
67
|
+
task :verify, [:revision, :base_dir] do |_, args|
|
68
|
+
revision = args[:revision].to_s.strip
|
69
|
+
base_dir = args[:base_dir].to_s.strip
|
70
|
+
revision = nil if revision.empty?
|
71
|
+
base_dir = '.' if base_dir.empty?
|
72
|
+
::Dir.chdir(base_dir) do
|
73
|
+
pre_verify_step.call(revision) if pre_verify_step
|
74
|
+
git.verify_revision(revision)
|
75
|
+
post_verify_step.call(revision) if post_verify_step
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
desc "Checkout supermodule and all submodules to given tag, branch or SHA"
|
80
|
+
task :checkout, [:revision, :base_dir] do |_, args|
|
81
|
+
revision = args[:revision].to_s.strip
|
82
|
+
base_dir = args[:base_dir].to_s.strip
|
83
|
+
raise ::ArgumentError, 'revision is required' if revision.empty?
|
84
|
+
base_dir = '.' if base_dir.empty?
|
85
|
+
::Dir.chdir(base_dir) do
|
86
|
+
pre_checkout_step.call(revision) if pre_checkout_step
|
87
|
+
git.checkout_revision(revision, :force => true, :recursive => true)
|
88
|
+
post_checkout_step.call(revision) if post_checkout_step
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
end # namespace
|
93
|
+
end # initialize
|
94
|
+
|
95
|
+
private
|
96
|
+
|
97
|
+
def git
|
98
|
+
::RightDevelop::Utility::Git
|
99
|
+
end
|
100
|
+
|
101
|
+
end # RakeTask
|
102
|
+
end # RightDevelop::Git
|
data/lib/right_develop/git.rb
CHANGED
@@ -1,22 +1,30 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
6
22
|
|
7
|
-
|
8
|
-
|
9
|
-
lines = message.split("\n").map { |l| l.strip }.reject { |l| l.empty? }
|
10
|
-
super(lines.last || @output)
|
11
|
-
end
|
12
|
-
end
|
23
|
+
# ancestor
|
24
|
+
require 'right_develop'
|
13
25
|
|
14
|
-
|
15
|
-
|
26
|
+
module RightDevelop
|
27
|
+
module Git
|
28
|
+
autoload :RakeTask, 'right_develop/git/rake_task'
|
16
29
|
end
|
17
|
-
|
18
|
-
require "right_develop/git/branch"
|
19
|
-
require "right_develop/git/branch_collection"
|
20
|
-
require "right_develop/git/commit"
|
21
|
-
require "right_develop/git/repository"
|
22
|
-
end
|
30
|
+
end
|
@@ -1,9 +1,18 @@
|
|
1
|
-
require 'xml/libxml'
|
2
|
-
require 'active_support/inflector'
|
3
|
-
require 'right_develop/parsers/xml_post_parser.rb'
|
4
|
-
|
5
1
|
module RightDevelop::Parsers
|
6
2
|
class SaxParser
|
3
|
+
begin
|
4
|
+
# libxml-ruby requires a C library and headers to be available before it will install; thus,
|
5
|
+
# we do not call it out as a gemspec dependency
|
6
|
+
require 'xml/libxml'
|
7
|
+
|
8
|
+
# ActiveSupport is not a runtime
|
9
|
+
require 'active_support/inflector'
|
10
|
+
|
11
|
+
AVAILABLE = true
|
12
|
+
rescue LoadError => e
|
13
|
+
AVAILABLE = false
|
14
|
+
end
|
15
|
+
|
7
16
|
extend XmlPostParser
|
8
17
|
|
9
18
|
# Parses XML into a ruby hash
|
@@ -14,6 +23,10 @@ module RightDevelop::Parsers
|
|
14
23
|
# the return content of the initial xml parser.
|
15
24
|
# @return [Array or Hash] returns rubified XML in Hash and Array format
|
16
25
|
def self.parse(text, opts = {})
|
26
|
+
unless AVAILABLE
|
27
|
+
raise NotImplementedError, "#{self.name} is unavailable on this system because libxml-ruby and/or active_support are not installed"
|
28
|
+
end
|
29
|
+
|
17
30
|
# Parse the xml text
|
18
31
|
# http://libxml.rubyforge.org/rdoc/
|
19
32
|
xml = ::XML::SaxParser::string(text)
|
@@ -32,6 +45,10 @@ module RightDevelop::Parsers
|
|
32
45
|
end
|
33
46
|
|
34
47
|
def initialize
|
48
|
+
unless AVAILABLE
|
49
|
+
raise NotImplementedError, "#{self.name} is unavailable on this system because libxml-ruby and/or active_support are not installed"
|
50
|
+
end
|
51
|
+
|
35
52
|
@tag = {}
|
36
53
|
@path = []
|
37
54
|
end
|
@@ -136,4 +153,4 @@ module RightDevelop::Parsers
|
|
136
153
|
def on_end_document
|
137
154
|
end
|
138
155
|
end
|
139
|
-
end
|
156
|
+
end
|
@@ -1,7 +1,12 @@
|
|
1
|
-
require 'active_support/inflector'
|
2
|
-
|
3
1
|
module RightDevelop::Parsers
|
4
2
|
module XmlPostParser
|
3
|
+
begin
|
4
|
+
require 'active_support/inflector'
|
5
|
+
|
6
|
+
AVAILABLE = true
|
7
|
+
rescue LoadError => e
|
8
|
+
AVAILABLE = false
|
9
|
+
end
|
5
10
|
|
6
11
|
# Parses a rubified XML hash/array, removing the top level xml tag, along with
|
7
12
|
# any arrays encoded with singular/plural for parent/child nodes.
|
@@ -40,6 +45,10 @@ module RightDevelop::Parsers
|
|
40
45
|
# @return [Array or Hash] returns a ruby Array or Hash with top level xml tags removed,
|
41
46
|
# as well as any extra XML encoded array tags.
|
42
47
|
def self.remove_nesting(xml_object)
|
48
|
+
unless AVAILABLE
|
49
|
+
raise NotImplementedError, "#{self.name} is unavailable on this system because libxml-ruby and/or active_support are not installed"
|
50
|
+
end
|
51
|
+
|
43
52
|
if xml_object.length != 1 || (!xml_object.is_a?(Hash) && !xml_object.is_a?(Array))
|
44
53
|
raise ArgumentError, "xml_object format doesn't have a single top level entry"
|
45
54
|
end
|
@@ -0,0 +1,298 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2013 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'right_aws'
|
24
|
+
|
25
|
+
module RightDevelop
|
26
|
+
module S3
|
27
|
+
|
28
|
+
# Provides a Ruby OOP interface to Amazon S3.
|
29
|
+
#
|
30
|
+
# Note: filters are used as options for multiple storage actions below and
|
31
|
+
# refers to an array of Regexp or wildcard-style filter strings
|
32
|
+
# (e.g. '*.txt'). they are used to match file paths relative to a given
|
33
|
+
# subdirectory or else from the root of the bucket or directory on disk).
|
34
|
+
class Interface
|
35
|
+
NO_SLASHES_REGEXP = /^[^\/]+$/
|
36
|
+
|
37
|
+
DEFAULT_OPTIONS = {
|
38
|
+
:filters => nil,
|
39
|
+
:subdirectory => nil,
|
40
|
+
:recursive => true,
|
41
|
+
:aws_access_key_id => nil,
|
42
|
+
:aws_secret_access_key => nil,
|
43
|
+
:logger => nil
|
44
|
+
}.freeze
|
45
|
+
|
46
|
+
# @option options [String] :aws_access_key_id defaults to using env var value
|
47
|
+
# @option options [String] :aws_secret_access_key defaults to using env var value
|
48
|
+
# @option options [Logger] :logger or nil to log to STDOUT
|
49
|
+
def initialize(options={})
|
50
|
+
options = DEFAULT_OPTIONS.merge(options)
|
51
|
+
|
52
|
+
aws_access_key_id = options[:aws_access_key_id]
|
53
|
+
aws_secret_access_key = options[:aws_secret_access_key]
|
54
|
+
unless aws_access_key_id && aws_secret_access_key
|
55
|
+
raise ::ArgumentError,
|
56
|
+
'Missing one or both mandatory options - :aws_access_key_id and :aws_secret_access_key'
|
57
|
+
end
|
58
|
+
|
59
|
+
@logger = options[:logger] || Logger.new(STDOUT)
|
60
|
+
@s3 = ::RightAws::S3Interface.new(aws_access_key_id, aws_secret_access_key, :logger => @logger)
|
61
|
+
end
|
62
|
+
|
63
|
+
attr_accessor :logger
|
64
|
+
|
65
|
+
# Lists the files in the given bucket.
|
66
|
+
#
|
67
|
+
# @param [String] bucket to query
|
68
|
+
# @option options [String] :subdirectory to start from or nil
|
69
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
70
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
71
|
+
# @return [Array] list of relative file paths or empty
|
72
|
+
def list_files(bucket, options={})
|
73
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
74
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
75
|
+
filters = normalize_filters(options)
|
76
|
+
files = []
|
77
|
+
trivial_filters = filters.select { |filter| filter.is_a?(String) }
|
78
|
+
if trivial_filters.empty?
|
79
|
+
@s3.incrementally_list_bucket(bucket, 'prefix' => prefix) do |response|
|
80
|
+
incremental_files = response[:contents].map do |details|
|
81
|
+
details[:key][(prefix.length)..-1]
|
82
|
+
end
|
83
|
+
files += filter_files(incremental_files, filters)
|
84
|
+
end
|
85
|
+
else
|
86
|
+
trivial_filters.each do |filename|
|
87
|
+
begin
|
88
|
+
# use head to query file existence.
|
89
|
+
@s3.head(bucket, "#{prefix}#{filename}")
|
90
|
+
files << filename
|
91
|
+
rescue RightAws::AwsError => e
|
92
|
+
# do nothing if file not found
|
93
|
+
raise unless '404' == e.http_code
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
return files
|
98
|
+
end
|
99
|
+
|
100
|
+
# Downloads all files from the given bucket to the given directory.
|
101
|
+
#
|
102
|
+
# @param [String] bucket for download
|
103
|
+
# @param [String] to_dir_path source directory to upload
|
104
|
+
# @option options [String] :subdirectory to start from or nil
|
105
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
106
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
107
|
+
# @return [Fixnum] count of uploaded files
|
108
|
+
def download_files(bucket, to_dir_path, options={})
|
109
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
110
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
111
|
+
files = list_files(bucket, options)
|
112
|
+
if files.empty?
|
113
|
+
logger.info("No files found in \"#{bucket}/#{prefix}\"")
|
114
|
+
else
|
115
|
+
logger.info("Downloading #{files.count} files...")
|
116
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
117
|
+
downloaded = 0
|
118
|
+
files.each do |path|
|
119
|
+
key = "#{prefix}#{path}"
|
120
|
+
to_file_path = File.join(to_dir_path, path)
|
121
|
+
parent_path = File.dirname(to_file_path)
|
122
|
+
FileUtils.mkdir_p(parent_path) unless File.directory?(parent_path)
|
123
|
+
|
124
|
+
disk_file = to_file_path
|
125
|
+
file_md5 = File.exist?(disk_file) && Digest::MD5.hexdigest(File.read(disk_file))
|
126
|
+
|
127
|
+
if file_md5
|
128
|
+
head = @s3.head(bucket, key) rescue nil
|
129
|
+
key_md5 = head && head['etag'].gsub(/[^0-9a-fA-F]/, '')
|
130
|
+
skip = (key_md5 == file_md5)
|
131
|
+
end
|
132
|
+
|
133
|
+
if skip
|
134
|
+
logger.info("Skipping #{bucket}/#{key} (identical contents)")
|
135
|
+
else
|
136
|
+
logger.info("Downloading #{bucket}/#{key}")
|
137
|
+
::File.open(to_file_path, 'wb') do |f|
|
138
|
+
@s3.get(bucket, key) { |chunk| f.write(chunk) }
|
139
|
+
end
|
140
|
+
downloaded += 1
|
141
|
+
end
|
142
|
+
|
143
|
+
logger.info("Downloaded to \"#{to_file_path}\"")
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
downloaded
|
148
|
+
end
|
149
|
+
|
150
|
+
# Uploads all files from the given directory (ignoring any empty
|
151
|
+
# directories) to the given bucket.
|
152
|
+
#
|
153
|
+
# @param [String] bucket for upload
|
154
|
+
# @param [String] from_dir_path source directory to upload
|
155
|
+
# @option options [String] :subdirectory to start from or nil
|
156
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
157
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
158
|
+
# @option options [String] :visibility for uploaded files, defaults to 'public-read'
|
159
|
+
# @return [Fixnum] count of downloaded files
|
160
|
+
def upload_files(bucket, from_dir_path, options={})
|
161
|
+
Dir.chdir(from_dir_path) do
|
162
|
+
logger.info("Working in #{Dir.pwd.inspect}")
|
163
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
164
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
165
|
+
filters = normalize_filters(options)
|
166
|
+
pattern = options[:recursive] ? '**/*' : '*'
|
167
|
+
files = Dir.glob(pattern).select { |path| File.file?(path) }
|
168
|
+
filter_files(files, filters)
|
169
|
+
access = normalize_access(options)
|
170
|
+
uploaded = 0
|
171
|
+
files.each do |path|
|
172
|
+
key = "#{prefix}#{path}"
|
173
|
+
file_md5 = Digest::MD5.hexdigest(File.read(path))
|
174
|
+
File.open(path, 'rb') do |f|
|
175
|
+
head = @s3.head(bucket, key) rescue nil
|
176
|
+
key_md5 = head && head['etag'].gsub(/[^0-9a-fA-F]/, '')
|
177
|
+
|
178
|
+
if file_md5 == key_md5
|
179
|
+
logger.info("Skipping #{bucket}/#{key} (identical contents)")
|
180
|
+
else
|
181
|
+
logger.info("Uploading to #{bucket}/#{key}")
|
182
|
+
@s3.put(bucket, key, f, 'x-amz-acl' => access)
|
183
|
+
uploaded += 1
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
|
188
|
+
uploaded
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
# Deletes all files from the given bucket.
|
193
|
+
#
|
194
|
+
# @param [String] bucket for delete
|
195
|
+
# @option options [String] :subdirectory to start from or nil
|
196
|
+
# @option options [TrueClass|FalseClass] :recursive true if recursive (default)
|
197
|
+
# @option options [Regexp] :filter for files to delete or nil
|
198
|
+
# @return [Fixnum] count of deleted files
|
199
|
+
def delete_files(bucket, options={})
|
200
|
+
options = DEFAULT_OPTIONS.dup.merge(options)
|
201
|
+
prefix = normalize_subdirectory_path(options[:subdirectory])
|
202
|
+
files = list_files(bucket, options)
|
203
|
+
if files.empty?
|
204
|
+
logger.info("No files found in \"#{bucket}/#{prefix}\"")
|
205
|
+
else
|
206
|
+
logger.info("Deleting #{files.count} files...")
|
207
|
+
files.each do |path|
|
208
|
+
@s3.delete(bucket, "#{prefix}#{path}")
|
209
|
+
logger.info("Deleted \"#{bucket}/#{prefix}#{path}\"")
|
210
|
+
end
|
211
|
+
end
|
212
|
+
return files.size
|
213
|
+
end
|
214
|
+
|
215
|
+
protected
|
216
|
+
|
217
|
+
# Normalizes a relative file path for use with S3.
|
218
|
+
#
|
219
|
+
# @param [String] subdirectory
|
220
|
+
def normalize_file_path(path)
|
221
|
+
# remove leading and trailing slashes and change any multiple slashes to single.
|
222
|
+
return (path || '').gsub("\\", '/').gsub(/^\/+/, '').gsub(/\/+$/, '').gsub(/\/+/, '/')
|
223
|
+
end
|
224
|
+
|
225
|
+
# Normalizes subdirectory path for use with S3.
|
226
|
+
#
|
227
|
+
# @param [String] path
|
228
|
+
# @return [String] normalized path
|
229
|
+
def normalize_subdirectory_path(path)
|
230
|
+
path = normalize_file_path(path)
|
231
|
+
path += '/' unless path.empty?
|
232
|
+
return path
|
233
|
+
end
|
234
|
+
|
235
|
+
# Normalizes storage filters from options.
|
236
|
+
#
|
237
|
+
# @option options [Array] :filters for returned paths or nil or empty
|
238
|
+
def normalize_filters(options)
|
239
|
+
initial_filters = Array(options[:filters])
|
240
|
+
normalized_filters = nil
|
241
|
+
|
242
|
+
# support trivial filters as simple string array for direct lookup of
|
243
|
+
# one or more S3 object (since listing entire buckets can be slow).
|
244
|
+
# recursion always requires a listing so that cannot be trivial.
|
245
|
+
if !options[:recursive] && initial_filters.size == 1
|
246
|
+
# filter is trivial unless it contains wildcards. more than one
|
247
|
+
# non-wildcard filenames delimited by semicolon can be trivial.
|
248
|
+
filter = initial_filters.first
|
249
|
+
if filter.kind_of?(String) && filter == filter.gsub('*', '').gsub('?', '')
|
250
|
+
normalized_filters = filter.split(';').uniq
|
251
|
+
end
|
252
|
+
end
|
253
|
+
unless normalized_filters
|
254
|
+
normalized_filters = []
|
255
|
+
normalized_filters << NO_SLASHES_REGEXP unless options[:recursive]
|
256
|
+
initial_filters.each do |filter|
|
257
|
+
if filter.kind_of?(String)
|
258
|
+
# split on semicolon (;) and OR the result into one regular expression.
|
259
|
+
# example: "*.tar;*.tgz;*.zip" -> /^.*\.tar|.*\.tgz|.*\.zip$/
|
260
|
+
#
|
261
|
+
# convert wildcard-style filter string (e.g. '*.txt') to Regexp.
|
262
|
+
escaped = Regexp.escape(filter).gsub("\\*", '.*').gsub("\\?", '.').gsub(';', '|')
|
263
|
+
regexp = Regexp.compile("^#{escaped}$")
|
264
|
+
filter = regexp
|
265
|
+
end
|
266
|
+
normalized_filters << filter unless normalized_filters.index(filter)
|
267
|
+
end
|
268
|
+
end
|
269
|
+
return normalized_filters
|
270
|
+
end
|
271
|
+
|
272
|
+
# Normalizes access from options (for uploading files).
|
273
|
+
#
|
274
|
+
# Note: access strings are AWS S3-style but can easily be mapped to any
|
275
|
+
# bucket storage implementation which supports ACLs.
|
276
|
+
#
|
277
|
+
# @option options [String] :access requested ACL or nil for public-read
|
278
|
+
# @return @return [String] normalized access
|
279
|
+
def normalize_access(options)
|
280
|
+
access = options[:access].to_s.empty? ? nil : options[:access]
|
281
|
+
return access || 'public-read'
|
282
|
+
end
|
283
|
+
|
284
|
+
# Filters the given list of file paths using the given filters, if any.
|
285
|
+
#
|
286
|
+
# @param [Array] files to filter
|
287
|
+
# @param [Array] filters for matching or empty
|
288
|
+
# @return [Array] filtered files
|
289
|
+
def filter_files(files, filters)
|
290
|
+
return files if filters.empty?
|
291
|
+
|
292
|
+
# select each path only if it matches all filters.
|
293
|
+
return files.select { |path| filters.all? { |filter| filter.match(path) } }
|
294
|
+
end
|
295
|
+
|
296
|
+
end # Interface
|
297
|
+
end # Buckets
|
298
|
+
end # RightDevelop
|