volley 0.1.0.alpha4 → 0.1.0.alpha5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/.gitignore +3 -1
  2. data/.rspec +1 -0
  3. data/Gemfile +4 -1
  4. data/Rakefile +34 -0
  5. data/bin/volley +88 -40
  6. data/conf/common.volleyfile +12 -29
  7. data/features/publisher.feature +45 -0
  8. data/features/step_definitions/common_steps.rb +9 -0
  9. data/features/step_definitions/publisher_steps.rb +92 -0
  10. data/features/support/env.rb +19 -0
  11. data/init/Volleyfile +1 -104
  12. data/lib/volley/descriptor.rb +28 -0
  13. data/lib/volley/dsl/action.rb +31 -0
  14. data/lib/volley/dsl/argument.rb +96 -0
  15. data/lib/volley/dsl/file.rb +70 -0
  16. data/lib/volley/dsl/plan.rb +110 -235
  17. data/lib/volley/dsl/project.rb +10 -2
  18. data/lib/volley/dsl/pull_action.rb +50 -0
  19. data/lib/volley/dsl/push_action.rb +101 -0
  20. data/lib/volley/dsl/stage.rb +40 -0
  21. data/lib/volley/dsl.rb +7 -0
  22. data/lib/volley/log.rb +22 -8
  23. data/lib/volley/meta.rb +24 -0
  24. data/lib/volley/publisher/amazons3.rb +67 -66
  25. data/lib/volley/publisher/base.rb +81 -42
  26. data/lib/volley/publisher/exceptions.rb +7 -0
  27. data/lib/volley/publisher/local.rb +41 -27
  28. data/lib/volley/scm/base.rb +10 -0
  29. data/lib/volley.rb +38 -12
  30. data/spec/descriptor_spec.rb +39 -0
  31. data/spec/dsl_plan_spec.rb +103 -0
  32. data/spec/dsl_project_spec.rb +36 -0
  33. data/spec/dsl_volleyfile_spec.rb +21 -0
  34. data/spec/meta_spec.rb +26 -0
  35. data/spec/publisher_spec.rb +92 -0
  36. data/test/dsl/amazons3_publisher.volleyfile +6 -0
  37. data/test/dsl/local_publisher.volleyfile +4 -0
  38. data/test/dsl/log_console.volleyfile +2 -0
  39. data/test/dsl/log_file.volleyfile +2 -0
  40. data/test/dsl/simple.volleyfile +17 -0
  41. data/test/meta.yml +3 -0
  42. data/test/project/Rakefile +13 -0
  43. data/test/project/Volleyfile +18 -0
  44. data/test/trunk-1.tgz +0 -0
  45. data/volley.gemspec +2 -1
  46. metadata +67 -5
  47. data/lib/volley/config.rb +0 -8
  48. data/lib/volley/volley_file.rb +0 -45
@@ -0,0 +1,101 @@
1
+
2
+ module Volley
3
+ module Dsl
4
+ class PushAction < Action
5
+ def initialize(name, options={}, &block)
6
+ @name = name.to_sym
7
+ @stage = options.delete(:stage)
8
+ @plan = options.delete(:plan)
9
+ @options = {
10
+ }.merge(options)
11
+ raise "stage instance must be set" unless @stage
12
+ raise "plan instance must be set" unless @plan
13
+
14
+ @plan.action :files, :post do
15
+ raise "branch(#{branch}) and version(#{version}) must be specified" unless branch && version
16
+
17
+ list = yield
18
+ list = [*list].flatten
19
+ # use #exists? so it can work for directories
20
+ notfound = list.reject { |f| File.exists?(f) }
21
+ raise "built files not found: #{notfound.join(",")}" unless notfound.count == 0
22
+ files list
23
+ file Volley.config.volleyfile if Volley.config.volleyfile
24
+ end
25
+
26
+ if attributes.pack
27
+ @plan.action :pack, :post do
28
+ raise "branch(#{branch}) and version(#{version}) must be specified" unless branch && version
29
+
30
+ path = attributes.pack_dir = "/var/tmp/volley-%d-%d-%05d" % [Time.now.to_i, $$, rand(99999)]
31
+ Dir.mkdir(path)
32
+ dir = Dir.pwd
33
+
34
+ files.each do |art|
35
+ Volley::Log.debug "art:#{art}"
36
+ next unless art
37
+ if art =~ /^\// && art !~ /^#{dir}/
38
+ # file is full path and not in current directory
39
+ source = art
40
+ dest = "#{path}/#{File.basename(art)}"
41
+ else
42
+ # file is relative path or in current directory
43
+ f = art.gsub(/^#{dir}/, "").gsub(/^\//, "")
44
+ source = "#{dir}/#{f}"
45
+ dest = "#{path}/#{f}"
46
+ end
47
+
48
+ begin
49
+ Volley::Log.debug "pack file: #{source} => #{dest}"
50
+ FileUtils.mkdir_p(File.dirname(dest))
51
+ if File.directory?(source)
52
+ FileUtils.cp_r(source, dest)
53
+ else
54
+ FileUtils.copy(source, dest)
55
+ end
56
+ rescue => e
57
+ raise "could not copy file #{source}: #{e.message}"
58
+ end
59
+ end
60
+
61
+ origpath = Dir.pwd
62
+ Dir.chdir(path)
63
+ case attributes.pack_type
64
+ when "tgz"
65
+ n = "#{branch}-#{version}.tgz"
66
+ c = "tar cvfz #{n} *"
67
+ Volley::Log.debug "command:#{c}"
68
+ command(c)
69
+
70
+ attributes.artifact = "#{path}/#{n}"
71
+ else
72
+ raise "unknown pack type '#{attributes.pack_type}'"
73
+ end
74
+
75
+ Dir.chdir(origpath)
76
+ end
77
+ end
78
+
79
+ if attributes.encrypt
80
+ @plan.action :encrypt, :post do
81
+ art = attributes.artifact
82
+ key = attributes.encrypt_key
83
+ cpt = "#{art}.cpt"
84
+
85
+ raise "in action encrypt: artifact file does not exist: #{art}" unless File.file?(art)
86
+ raise "in action encrypt: encrypted file #{cpt} already exists" if File.file?(cpt) && !attributes.encrypt_overwrite
87
+ shellout("ccrypt -e --key '#{key}' #{art}")
88
+
89
+ attributes.artifact_unencrypted = art
90
+ attributes.artifact = cpt
91
+ end
92
+ end
93
+
94
+ @plan.action :push, :post do
95
+ publisher = Volley::Dsl.publisher
96
+ publisher.push(project.name, branch, version, attributes.artifact)
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -0,0 +1,40 @@
1
+
2
+ module Volley
3
+ module Dsl
4
+ class Stage
5
+ attr_reader :actions
6
+ def initialize(name, options={}, &block)
7
+ @name = name.to_sym
8
+ @plan = options.delete(:plan)
9
+ @options = {
10
+ }.merge(options)
11
+ @actions = []
12
+ raise "plan instance must be set" unless @plan
13
+ instance_eval &block if block_given?
14
+ end
15
+
16
+ def call
17
+ Volley::Log.debug ".. #@name"
18
+ @actions.each do |action|
19
+ action.call
20
+ end
21
+ end
22
+
23
+ def add(action)
24
+ @actions << action
25
+ end
26
+
27
+ def count
28
+ @actions.count
29
+ end
30
+
31
+ def action(name, options={}, &block)
32
+ o = {
33
+ :stage => @name,
34
+ :plan => nil,
35
+ }.merge(options)
36
+ @actions << Volley::Dsl::Action.new(name, o, &block)
37
+ end
38
+ end
39
+ end
40
+ end
data/lib/volley/dsl.rb CHANGED
@@ -11,11 +11,18 @@ module Volley
11
11
  Volley::Dsl::Publisher.get
12
12
  end
13
13
 
14
+ def file(name)
15
+ return false unless name
16
+ Volley::Dsl::VolleyFile.load(name)
17
+ end
18
+
14
19
  def project(name)
20
+ return false unless name
15
21
  Volley::Dsl::Project.get(name)
16
22
  end
17
23
 
18
24
  def project?(name)
25
+ return false unless name
19
26
  Volley::Dsl::Project.exists?(name)
20
27
  end
21
28
 
data/lib/volley/log.rb CHANGED
@@ -4,24 +4,38 @@ module Volley
4
4
  class Log
5
5
  class << self
6
6
  def init
7
- @loggers = []
8
- @loggers << Yell.new(STDOUT, :level => [:info,:warn], :format => Yell::NoFormat)
9
- @loggers << Yell.new(STDERR, :level => [:error,:fatal], :format => Yell::NoFormat)
7
+ @loggers = { }
8
+ @loggers["STDOUT"] = Yell.new(STDOUT, :level => [:info, :warn], :format => Yell::NoFormat)
9
+ @loggers["STDERR"] = Yell.new(STDERR, :level => [:error, :fatal], :format => Yell::NoFormat)
10
10
  end
11
11
 
12
12
  def add(level, dest, format=Yell::DefaultFormat)
13
- if [STDOUT,STDERR].include?(dest)
14
- @loggers << Yell.new(dest, :level => level, :format => format)
13
+ if [STDOUT, STDERR].include?(dest)
14
+ console_disable
15
+ @loggers["STDOUT"] = Yell.new(dest, :level => level, :format => format)
15
16
  else
16
- @loggers << Yell.new(:datefile, dest, :level => level, :format => format,
17
- :keep => 7, :symlink_original_filename => true)
17
+ @loggers.delete(dest) if @loggers[dest]
18
+ FileUtils.mkdir_p(File.dirname(dest))
19
+ @loggers[dest] = Yell.new(:datefile, dest,
20
+ :level => level, :format => format,
21
+ :keep => 7, :symlink_original_filename => true)
18
22
  end
19
23
  end
20
24
 
25
+ def console_disable
26
+ %w{STDOUT STDERR}.each { |s| @loggers.delete(s) }
27
+ end
28
+
29
+ def console_debug
30
+ console_disable
31
+ @loggers["STDOUT"] = Yell.new(STDOUT, :level => [:debug, :info, :warn], :format => Yell::NoFormat)
32
+ @loggers["STDERR"] = Yell.new(STDERR, :level => [:error, :fatal], :format => Yell::NoFormat)
33
+ end
34
+
21
35
  %w{debug info warn error fatal}.each do |method_name|
22
36
  class_eval(<<-METHOD_DEFN, __FILE__, __LINE__)
23
37
  def #{method_name}(msg=nil, &block)
24
- @loggers.each {|l| l.#{method_name}(msg, &block) }
38
+ @loggers.each {|k, l| l.#{method_name}(msg, &block) }
25
39
  end
26
40
  METHOD_DEFN
27
41
  end
@@ -0,0 +1,24 @@
1
+
2
+ module Volley
3
+ class Meta
4
+ def initialize(file="#{Volley.config.directory}/meta.yaml")
5
+ Volley::Log.info "meta: #{file}"
6
+ @file = file
7
+ raise "file does not exist" unless File.file?(@file)
8
+ @data = YAML.load_file(@file)
9
+ Volley::Log.info "#{@data.inspect}"
10
+ end
11
+
12
+ def [](project)
13
+ @data[project.to_sym]
14
+ end
15
+
16
+ def []=(project, version)
17
+ @data[project.to_sym] = version
18
+ end
19
+
20
+ def save
21
+ File.open(@file, "w") {|f| f.write(@data.to_yaml)}
22
+ end
23
+ end
24
+ end
@@ -5,112 +5,91 @@ module Volley
5
5
  class Amazons3 < Base
6
6
  attr_accessor :key, :secret
7
7
 
8
- def files
9
- hash = {:desc => {}, :all => {}, :latest => {}}
10
- @connection.directories.get(@bucket).files.collect{|e| e.key}.each do |e|
11
- (pr, br, vr) = e.split(/\//)
12
- hash[:desc][pr] ||= {}
13
- hash[:desc][pr][br] ||= {}
14
- hash[:desc][pr][br][vr] ||= []
15
- hash[:desc][pr][br][vr] << e
16
- hash[:all] ||= {}
17
- hash[:latest] ||= {}
18
- v = "#{pr}/#{br}/#{vr}"
19
- hash[:latest]["#{pr}/#{br}"] ||= latest(pr, br)
20
- hash[:all][v] = hash["latest"] == v
21
- end
22
- #ap hash
23
- hash
24
- end
25
-
26
- def all
27
- files[:all]
28
- end
29
-
30
8
  def projects
31
- data = files
32
- data[:desc].keys
33
- #files.collect{|e| e.split(/\//).first }.uniq
9
+ files[:desc].keys
34
10
  rescue => e
35
11
  Volley::Log.warn "error getting project list from publisher: #{e.message} at #{e.backtrace.first}"
36
12
  []
37
13
  end
38
14
 
39
15
  def branches(pr)
40
- puts "branches for #{pr}"
41
16
  files[:desc][pr].keys
42
- #files.select{|e| e.split(/\//).first == pr}.collect{|e| e.split(/\//)[1]}
43
17
  rescue => e
44
18
  Volley::Log.warn "error getting branch list from publisher: #{e.message}"
45
19
  []
46
20
  end
47
21
 
48
- def versions(pr,br)
22
+ def versions(pr, br)
49
23
  files[:desc][pr][br].keys
50
- #raise "not implemented"
51
24
  rescue => e
52
25
  Volley::Log.warn "error getting version list from publisher: #{e.message}"
53
26
  []
54
27
  end
55
28
 
56
29
  def contents(pr, br, vr)
57
- files[:desc][pr][br][vr]
30
+ files[:desc][pr][br][vr].map {|e| e.gsub("#{pr}/#{br}/#{vr}/","")}
31
+ rescue => e
32
+ Volley::Log.warn "error getting contents list from publisher: #{e.message}"
33
+ []
58
34
  end
59
35
 
60
- def latest(pr, br)
61
- @project = pr
62
- @branch = br
63
- f = @connection.directories.get(@bucket).files.get("#{branch}/latest")
64
- f.body
36
+ def exists?(project, branch, version)
37
+ !files[:desc][project][branch][version].nil? rescue false
38
+ end
39
+
40
+ def delete_project(project)
41
+ Volley::Log.info "delete_project #{project}"
42
+ dir = @connection.directories.get(@bucket)
43
+ dir.files.select{|e| e.key =~ /^#{project}\//}.each do |f|
44
+ Volley::Log.info "- #{f.key}"
45
+ f.destroy
46
+ end
47
+ true
48
+ rescue => e
49
+ Volley::Log.error "error deleting project: #{e.message} at #{e.backtrace.first}"
50
+ Volley::Log.debug e
51
+ false
65
52
  end
66
53
 
67
- private
54
+ protected
68
55
 
69
56
  def load_configuration
70
57
  @key = requires(:aws_access_key_id)
71
58
  @secret = requires(:aws_secret_access_key)
72
59
  @bucket = requires(:bucket)
73
- @local = requires(:local)
74
- @debug = optional(:debug, false)
75
- @encrypted = optional(:encrypted, false)
76
60
  connect
77
61
  end
78
62
 
79
- def remote_file
80
- "#@branch-#@version.tgz#{".cpt" if @encrypted}"
63
+ def push_file(dir, file, contents)
64
+ file = File.basename(file)
65
+ dest = "#{dir}/#{file}"
66
+ #log "-> #@bucket/#{path}"
67
+ f = root.files.create(:key => dest, :body => contents, :public => true)
68
+ log "=> #{f.public_url.gsub("%2F", "/")}"
69
+ dest
81
70
  end
82
71
 
83
- def push_file(name, dir, contents)
84
- Volley::Log.info ".. #{name}"
85
- file = File.basename(name)
86
- path = "#{dir}/#{file}"
87
- Volley::Log.info ".. -> s3:#@bucket/#{path}"
88
- @dir ||= @connection.directories.create({ :key => @bucket })
89
- s3f = @dir.files.create(
90
- :key => "#{path}",
91
- :body => contents,
92
- :public => true
93
- )
94
- Volley::Log.info ".. => #{s3f.public_url.gsub("%2F", "/")}"
95
- "#{path}"
96
- end
72
+ def pull_file(dir, file, localdir=nil)
73
+ remote = "#{dir}/#{file}"
74
+ f = root.files.get(remote)
75
+ raise ArtifactMissing, "missing: #{remote}" unless f
97
76
 
98
- def pull_file(name, dir, ldir=nil)
99
- Volley::Log.info ".. <- s3:#@bucket/#{dir}/#{name}"
100
- if ldir
101
- FileUtils.mkdir_p(ldir)
102
- end
103
- f = @connection.directories.get(@bucket).files.get("#{dir}/#{name}")
104
- contents = f.body
105
- if ldir
106
- lfile = "#{ldir}/#{name}"
107
- File.open(lfile, "w") { |lf| lf.write(contents) }
108
- Volley::Log.info ".. <= #{lfile}"
77
+ contents = f.body
78
+
79
+ if localdir
80
+ FileUtils.mkdir_p(localdir)
81
+ local = "#{localdir}/#{file}"
82
+ log "<= #{local}"
83
+ File.open(local, "w") { |lf| lf.write(contents) }
109
84
  else
110
85
  contents
111
86
  end
112
87
  end
113
88
 
89
+ def root
90
+ @root ||= @connection.directories.create({ :key => @bucket })
91
+ end
92
+
114
93
  def connect
115
94
  @connection ||= Fog::Storage.new(
116
95
  :provider => "AWS",
@@ -118,6 +97,28 @@ module Volley
118
97
  :aws_secret_access_key => @secret,
119
98
  )
120
99
  end
100
+
101
+ def files
102
+ hash = {:desc => {}, :all => {}, :latest => {}}
103
+ @connection.directories.get(@bucket).files.collect{|e| e.key}.each do |e|
104
+ (pr, br, vr) = e.split(/\//)
105
+ hash[:desc][pr] ||= {}
106
+ hash[:desc][pr][br] ||= {}
107
+ hash[:desc][pr][br][vr] ||= []
108
+ hash[:desc][pr][br][vr] << e
109
+ hash[:all] ||= {}
110
+ hash[:latest] ||= {}
111
+ v = "#{pr}/#{br}/#{vr}"
112
+ hash[:latest]["#{pr}/#{br}"] ||= latest(pr, br)
113
+ hash[:all][v] = hash["latest"] == v
114
+ end
115
+ #ap hash
116
+ hash
117
+ end
118
+
119
+ def all
120
+ files[:all]
121
+ end
121
122
  end
122
123
  end
123
124
  end
@@ -5,6 +5,12 @@ module Volley
5
5
  @options = {
6
6
  :overwrite => false,
7
7
  }.merge(options)
8
+
9
+ @debug = optional(:debug, false)
10
+ @encrypted = optional(:encrypted, false)
11
+ @local = optional(:local, Volley.config.directory)
12
+ @loglevel = @debug ? :info : :debug
13
+
8
14
  load_configuration
9
15
  end
10
16
 
@@ -12,73 +18,81 @@ module Volley
12
18
  raise "not implemented"
13
19
  end
14
20
 
15
- def volleyfile(desc={ })
16
- @project = desc[:project]
17
- @branch = desc[:branch]
18
- @version = desc[:version] && desc[:version] != 'latest' ? desc[:version] : get_latest(@project, @branch)
19
- contents = pull_file("Volleyfile", version)
20
- dest = @options[:destination] || "/tmp/Volleyfile-#{Time.now.to_i}-#{$$}"
21
+ def branches(project)
22
+ raise "not implemented"
23
+ end
24
+
25
+ def versions(project, branch)
26
+ raise "not implemented"
27
+ end
28
+
29
+ def exists?(project, branch, version)
30
+ raise "not implemented"
31
+ end
32
+
33
+ def contents(project, branch, version)
34
+ raise "not implemented"
35
+ end
36
+
37
+ def delete_project(project)
38
+ raise "not implemented"
39
+ end
40
+
41
+ def latest(project, branch)
42
+ v = pull_file(dir(project,branch), "latest")
43
+ "#{project}/#{branch}/#{v}"
44
+ end
45
+
46
+ def volleyfile(project, branch, version="latest")
47
+ contents = pull_file(dir(project,branch,version), "Volleyfile")
48
+
49
+ dest = "#{@options[:local]}/Volleyfile-#{Time.now.to_i}-#{$$}"
21
50
  raise "File #{dest} already exists" if File.exists?(dest)
22
- Volley::Log.debug("saving Volleyfile: #{dest}")
51
+
52
+ log "saving Volleyfile: #{dest}"
23
53
  File.open(dest, "w") { |f| f.write(contents) }
24
54
  dest
25
55
  end
26
56
 
27
- def push(project, br, ver, localfiles)
28
- @project = project
29
- @branch = br
30
- @version = ver
57
+ def push(project, branch, version, localfiles)
58
+ raise ArtifactExists, "the artifact already exists" if exists?(project, branch, version)
31
59
 
32
60
  localfiles = [*localfiles].flatten
33
- Volley::Log.info ".. pushing:" if @debug
61
+ log "^^ #{me}#push"
34
62
 
63
+ dir = dir(project, branch, version)
35
64
  localfiles.each do |localfile|
36
- Volley::Log.info ".. .. #{localfile}" if @debug
37
- push_file(localfile, version, File.open(localfile))
65
+ push_file(dir, localfile, File.open(localfile))
38
66
  end
39
- push_file("latest", branch, version)
40
- push_file("Volleyfile", version, File.open(Volley.config.volleyfile))
41
- end
42
-
43
- def pull(project, branch, ver="latest")
44
- @project = project
45
- @branch = branch
46
- @version = ver
47
67
 
48
- if @version == "latest"
49
- @version = get_latest(@project, @branch)
68
+ if Volley.config.volleyfile && File.file?(Volley.config.volleyfile)
69
+ push_file(dir, "Volleyfile", File.open(Volley.config.volleyfile))
50
70
  end
51
71
 
52
- Volley::Log.info "remote: #{version}" if @debug
53
- Volley::Log.info "remote_file: #{remote_file}" if @debug
54
- pull_file(remote_file, version, "#@local/#{version}")
72
+ push_file(dir(project, branch), "latest", version)
55
73
 
56
- "#@local/#{version}/#{remote_file}"
74
+ true
57
75
  end
58
76
 
59
- def get_latest(project, branch)
60
- cur = pull_file("latest", "#{project}/#{branch}")
61
- (p, n, v) = cur.split(/\//)
62
- v
63
- end
77
+ def pull(project, branch, version="latest")
78
+ dir = dir(project, branch, version)
79
+ file = remote_file(branch, version)
64
80
 
65
- private
66
- def me
67
- self.class.name.split("::").last
68
- end
81
+ log "vv #{me}#pull"
82
+ pull_file(dir, file, "#@local/#{dir}")
69
83
 
70
- def branch
71
- "#@project/#@branch"
84
+ "#@local/#{dir}/#{file}"
72
85
  end
73
86
 
74
- def version
75
- "#{branch}/#@version"
87
+ protected
88
+
89
+ def me
90
+ self.class.name.split("::").last
76
91
  end
77
92
 
78
93
  def requires(name)
79
94
  v = get_option(name)
80
95
  if v.nil?
81
- #ap @options
82
96
  raise "Publisher #{me} requires option #{name}"
83
97
  end
84
98
  v
@@ -97,6 +111,31 @@ module Volley
97
111
  n = name.to_sym
98
112
  Volley.config.send(n)
99
113
  end
114
+
115
+
116
+
117
+ def push_file(dir, name, contents)
118
+ raise "not implemented"
119
+ end
120
+
121
+ def pull_file(dir, name, localdir=nil)
122
+ raise "not implemented"
123
+ end
124
+
125
+ def remote_file(branch, version)
126
+ version = version == 'latest' ? get_latest(project, branch) : version
127
+ "#{branch}-#{version}.tgz#{".cpt" if @encrypted}"
128
+ end
129
+
130
+ def dir(project, branch, version=nil)
131
+ version = version == 'latest' ? get_latest(project, branch) : version
132
+ [project, branch, version].compact.join("/")
133
+ end
134
+
135
+ def log(msg)
136
+ Volley::Log.send(@loglevel, msg)
137
+ end
138
+
100
139
  end
101
140
  end
102
141
  end
@@ -0,0 +1,7 @@
1
+
2
+ module Volley
3
+ module Publisher
4
+ class ArtifactExists < StandardError; end
5
+ class ArtifactMissing < StandardError; end
6
+ end
7
+ end