grably 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/LICENSE.txt +202 -0
- data/README.md +2 -0
- data/exe/grably +2 -0
- data/lib/ext/class.rb +28 -0
- data/lib/grably.rb +83 -0
- data/lib/grably/cli.rb +46 -0
- data/lib/grably/core.rb +15 -0
- data/lib/grably/core/app/enchancer.rb +36 -0
- data/lib/grably/core/application.rb +8 -0
- data/lib/grably/core/colors.rb +86 -0
- data/lib/grably/core/commands.rb +23 -0
- data/lib/grably/core/commands/cp.rb +103 -0
- data/lib/grably/core/commands/digest.rb +12 -0
- data/lib/grably/core/commands/log.rb +19 -0
- data/lib/grably/core/commands/run.rb +85 -0
- data/lib/grably/core/commands/serialize.rb +16 -0
- data/lib/grably/core/configuration.rb +39 -0
- data/lib/grably/core/configuration/pretty_print.rb +22 -0
- data/lib/grably/core/digest.rb +93 -0
- data/lib/grably/core/dsl.rb +15 -0
- data/lib/grably/core/essentials.rb +49 -0
- data/lib/grably/core/module.rb +64 -0
- data/lib/grably/core/product.rb +301 -0
- data/lib/grably/core/task.rb +30 -0
- data/lib/grably/core/task/bucket.rb +29 -0
- data/lib/grably/core/task/enchancer.rb +50 -0
- data/lib/grably/core/task/expand.rb +15 -0
- data/lib/grably/core/task/jobs.rb +58 -0
- data/lib/grably/job.rb +28 -0
- data/lib/grably/job/class.rb +93 -0
- data/lib/grably/job/exceptions.rb +0 -0
- data/lib/grably/job/instance.rb +159 -0
- data/lib/grably/job/manifest.rb +67 -0
- data/lib/grably/jobs.rb +4 -0
- data/lib/grably/jobs/sync.rb +91 -0
- data/lib/grably/jobs/text.rb +4 -0
- data/lib/grably/jobs/text/erb.rb +40 -0
- data/lib/grably/jobs/text/json.rb +12 -0
- data/lib/grably/jobs/text/text.rb +21 -0
- data/lib/grably/jobs/text/yaml.rb +12 -0
- data/lib/grably/jobs/unzip.rb +1 -0
- data/lib/grably/jobs/upload.rb +1 -0
- data/lib/grably/jobs/zip.rb +2 -0
- data/lib/grably/jobs/zip/unzip.rb +24 -0
- data/lib/grably/jobs/zip/zip.rb +46 -0
- data/lib/grably/runner.rb +31 -0
- data/lib/grably/server.rb +83 -0
- data/lib/grably/utils/pretty_printer.rb +63 -0
- data/lib/grably/version.rb +12 -0
- metadata +164 -0
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'digest/sha1'
|
3
|
+
|
4
|
+
require_relative 'commands/digest'
|
5
|
+
require_relative 'commands/cp'
|
6
|
+
require_relative 'commands/log'
|
7
|
+
require_relative 'commands/run'
|
8
|
+
require_relative 'commands/serialize'
|
9
|
+
|
10
|
+
module Grably # :nodoc:
|
11
|
+
# Make all methods as MODULE methods
|
12
|
+
def self.method_added(m)
|
13
|
+
module_function m
|
14
|
+
end
|
15
|
+
|
16
|
+
def relative_path(base, path)
|
17
|
+
# TODO: Reimplement
|
18
|
+
Dir.chdir(base) do
|
19
|
+
path = File.expand_path(path)
|
20
|
+
end
|
21
|
+
path
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
# Copy files or products to destination directory
|
3
|
+
# @param products [Array<Product>] list of products to copy
|
4
|
+
# @param dst_dir [String] destination directory to copy products
|
5
|
+
# @param base_dir [String] if provided all products will be copied relative to
|
6
|
+
# this path inside destination
|
7
|
+
# @param log [Boolean] if provided will log all actions to STDOUT. false by
|
8
|
+
# default
|
9
|
+
# @return [Array<Product>] list of resulting products.
|
10
|
+
def cp(products, dst_dir, base_dir: nil, log: false)
|
11
|
+
products = Product.expand(products)
|
12
|
+
dst_dir = File.expand_path(dst_dir)
|
13
|
+
dst_dir = File.join(dst_dir, base_dir) if base_dir
|
14
|
+
|
15
|
+
products.map { |product| copy_product(product, dst_dir, log: log) }
|
16
|
+
end
|
17
|
+
|
18
|
+
# Smart copy operates over product lists in the way like it directory
|
19
|
+
# structures. All chaned products will be replaced and missing products
|
20
|
+
# will be removed.
|
21
|
+
# @param products [Array<Product>] list of products to copy. Any expand
|
22
|
+
# expression will work as expected.
|
23
|
+
# @param dst_dir [String] target directory path.
|
24
|
+
# @param log [Boolean] if set to true log all actions to STDOUT. false by
|
25
|
+
# default
|
26
|
+
# @return [Array<Product>] list of resulting products
|
27
|
+
def cp_smart(products, dst_dir, log: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
28
|
+
# Ensure dst_dir is created
|
29
|
+
FileUtils.mkdir_p(dst_dir) unless File.exist?(dst_dir)
|
30
|
+
# Create Hash structures containing product.dst => product mappings
|
31
|
+
update = ->(acc, elem) { acc.update(elem.dst => elem) }
|
32
|
+
src_products = Product.expand(products).inject({}, &update)
|
33
|
+
dst_products = Product.expand(dst_dir).inject({}, &update)
|
34
|
+
|
35
|
+
# Looking for missing files
|
36
|
+
remove_files = (dst_products.keys - src_products.keys).each do |dst_key|
|
37
|
+
log "Remove #{dst_key} (#{dst_products[dst_key].src}" if log
|
38
|
+
end
|
39
|
+
FileUtils.rm(remove_files.map { |k| dst_products[k].src })
|
40
|
+
rm_empty_dirs(dst_dir)
|
41
|
+
# Update rest
|
42
|
+
src_products.map do |dst, product|
|
43
|
+
dst_product = dst_products[dst]
|
44
|
+
update_smart(product, dst_product, dst_dir, log: log)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Copy product to dst using FileUtils
|
49
|
+
def cp_sys(src, dst, log: false)
|
50
|
+
src = src.src if src.is_a?(Product)
|
51
|
+
dst = dst.src if dst.is_a?(Product)
|
52
|
+
log "Copy #{File.basename(src)} to #{dst}" if log
|
53
|
+
FileUtils.cp_r(src, dst)
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def rm_empty_dirs(dir)
|
59
|
+
Dir[File.join(dir, '**/*')]
|
60
|
+
.select { |p| File.directory?(p) }
|
61
|
+
.select { |d| (Dir.entries(d) - %w(. ..)).empty? }
|
62
|
+
.each { |d| Dir.rmdir(d) }
|
63
|
+
end
|
64
|
+
|
65
|
+
def copy_product(product, dst_dir, log: false)
|
66
|
+
copy = product.map do |_src, dst, _meta|
|
67
|
+
File.join(dst_dir, dst)
|
68
|
+
end
|
69
|
+
|
70
|
+
dir = File.dirname(copy.dst)
|
71
|
+
FileUtils.mkdir_p(dir) unless File.exist?(dir)
|
72
|
+
log "Copy #{File.basename(product.src)} to #{copy.src}" if log
|
73
|
+
FileUtils.cp(product.src, copy.src)
|
74
|
+
|
75
|
+
copy
|
76
|
+
end
|
77
|
+
|
78
|
+
def update_smart(src_product, dst_product, dst_dir, log: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/LineLength
|
79
|
+
if dst_product
|
80
|
+
# Check if file changed. If so updating it
|
81
|
+
# TODO: Should we or should not check hashsum
|
82
|
+
if product_changed?(src_product, dst_product)
|
83
|
+
log "Update #{src_product.basename} to #{dst_product.src}" if log
|
84
|
+
FileUtils.cp(src_product.src, dst_product.src)
|
85
|
+
end
|
86
|
+
else
|
87
|
+
dst_product = src_product.map do |_src, dst, _meta|
|
88
|
+
File.join(dst_dir, dst)
|
89
|
+
end
|
90
|
+
dir = File.dirname(dst_product.src)
|
91
|
+
FileUtils.mkdir_p(dir) unless File.exist?(dir)
|
92
|
+
log "Copy #{src_product.basename} to #{dst_product.src}" if log
|
93
|
+
FileUtils.cp(src_product.src, dst_product.src)
|
94
|
+
end
|
95
|
+
|
96
|
+
dst_product
|
97
|
+
end
|
98
|
+
|
99
|
+
def product_changed?(p1, p2)
|
100
|
+
# TODO: Should we or should not check hashsums of files?
|
101
|
+
digest(p1) != digest(p2) if File.mtime(p1.src) != File.mtime(p2.src)
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
def log(msg)
|
3
|
+
puts msg
|
4
|
+
# flush stdout - we need that to have submodules displaying OK
|
5
|
+
$stdout.flush
|
6
|
+
end
|
7
|
+
|
8
|
+
def err(msg)
|
9
|
+
puts 'error: '.red.bright + msg
|
10
|
+
end
|
11
|
+
|
12
|
+
def warn(msg)
|
13
|
+
puts 'warning: '.cyan.bright + msg.to_s.cyan
|
14
|
+
end
|
15
|
+
|
16
|
+
def trace(msg)
|
17
|
+
puts msg if ENV['TRACE']
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# Ruby array extensions
|
2
|
+
class Array
|
3
|
+
# Run array content as shell command
|
4
|
+
def run(&block)
|
5
|
+
Grably.run(self, &block)
|
6
|
+
end
|
7
|
+
|
8
|
+
# Run array content as shell command and handle exception automaticly
|
9
|
+
def run_safe
|
10
|
+
Grably.run_safe(self)
|
11
|
+
end
|
12
|
+
|
13
|
+
# Run array content as shell command and print it output to STDOUT
|
14
|
+
def run_log
|
15
|
+
Grably.run_log(self)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
module Grably # :nodoc:
|
20
|
+
# TODO: Rewimplement
|
21
|
+
# rubocop:disable all
|
22
|
+
class << self
|
23
|
+
def run_log(cmd)
|
24
|
+
run(cmd) { |l| log " #{l}" }
|
25
|
+
end
|
26
|
+
|
27
|
+
def run(cmd, &block)
|
28
|
+
env = {}
|
29
|
+
|
30
|
+
cmd = [cmd] if cmd.is_a?(Hash)
|
31
|
+
cmd = cmd.split(' ') unless cmd.is_a?(Array)
|
32
|
+
cmd = cmd.flatten.compact
|
33
|
+
|
34
|
+
cmd.map! do |c|
|
35
|
+
if c.is_a?(String) && c.empty?
|
36
|
+
c = nil
|
37
|
+
elsif c.is_a?(Hash)
|
38
|
+
env.merge!(c)
|
39
|
+
c = nil
|
40
|
+
end
|
41
|
+
c = c.to_s unless c.nil?
|
42
|
+
c
|
43
|
+
end
|
44
|
+
|
45
|
+
cmd = cmd.flatten.compact
|
46
|
+
|
47
|
+
# Not all windows apps "slash tolerant"
|
48
|
+
if windows?
|
49
|
+
cmd[0] = cmd[0].tr('/', '\\') unless cmd.empty?
|
50
|
+
end
|
51
|
+
|
52
|
+
pwd = nil
|
53
|
+
if env['__WORKING_DIR']
|
54
|
+
pwd = Dir.pwd
|
55
|
+
Dir.chdir(env.delete('__WORKING_DIR'))
|
56
|
+
end
|
57
|
+
|
58
|
+
r = []
|
59
|
+
IO.popen([env, cmd, { err: %i(child out) }].flatten) do |o|
|
60
|
+
o.sync = true
|
61
|
+
o.each do |l|
|
62
|
+
r << l
|
63
|
+
yield(l) unless block.nil?
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
Dir.chdir(pwd) if pwd
|
68
|
+
|
69
|
+
raise 'error: '.red.bright + cmd.red + "\nfail log: #{r}".green unless $?.exitstatus.zero?
|
70
|
+
|
71
|
+
r.join
|
72
|
+
end
|
73
|
+
|
74
|
+
def run_safe(cmd)
|
75
|
+
begin
|
76
|
+
run(cmd)
|
77
|
+
rescue
|
78
|
+
return false
|
79
|
+
end
|
80
|
+
|
81
|
+
true
|
82
|
+
end
|
83
|
+
# rubocop:enable all
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
def load_obj(filename)
|
3
|
+
return nil unless File.exist? filename
|
4
|
+
File.open(filename) do |f|
|
5
|
+
Marshal.load(f) # rubocop:disable Security/MarshalLoad
|
6
|
+
end
|
7
|
+
rescue StandardError => x
|
8
|
+
raise(x, "Can't deserialize #{filename}")
|
9
|
+
end
|
10
|
+
|
11
|
+
def save_obj(filename, obj)
|
12
|
+
dir = File.dirname(filename)
|
13
|
+
FileUtils.mkdir(dir) unless File.exist?(dir)
|
14
|
+
File.open(filename, 'w') { |f| Marshal.dump(obj, f) }
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'ostruct'
|
3
|
+
require 'jac'
|
4
|
+
|
5
|
+
require_relative 'configuration/pretty_print'
|
6
|
+
|
7
|
+
module Grably
|
8
|
+
module Core
|
9
|
+
# Grably configuration module.
|
10
|
+
# We use jac for configuration
|
11
|
+
# @see https://github.com/vizor-games/jac
|
12
|
+
module Configuration
|
13
|
+
# Default configuration file names
|
14
|
+
CONFIGURATION_FILES = %w(grably.yml grably.user.yml grably.override.yml).freeze
|
15
|
+
|
16
|
+
class << self
|
17
|
+
# Generates configuration object for given profile
|
18
|
+
# and list of streams with YAML document
|
19
|
+
# @param profile [Array] list of profile names to merge
|
20
|
+
# @param streams [Array] list of YAML documents and their
|
21
|
+
# names to read
|
22
|
+
# @return [OpenStruct] instance which contains all resolved profile fields
|
23
|
+
def read(profile, *streams)
|
24
|
+
Jac::Configuration.read(profile, *streams)
|
25
|
+
end
|
26
|
+
|
27
|
+
# Read configuration from configuration files.
|
28
|
+
# @praram [String or Array] profile which should be loaded
|
29
|
+
# @param [String] dir base directory path for provided files may be nil
|
30
|
+
# @return [OpenStruct] resolved profile values
|
31
|
+
def load(profile, dir: nil)
|
32
|
+
obj = Jac::Configuration.load(profile, files: CONFIGURATION_FILES, dir: dir)
|
33
|
+
obj.extend(self)
|
34
|
+
obj
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require_relative '../colors'
|
3
|
+
|
4
|
+
module Grably
|
5
|
+
module Core
|
6
|
+
module Configuration # :nodoc:
|
7
|
+
# rubocop:disable all
|
8
|
+
class ConfigurationVisitor < Psych::Visitors::YAMLTree
|
9
|
+
def visit_Symbol(o)
|
10
|
+
visit_String(o.to_s)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
# rubocop:enable all
|
14
|
+
|
15
|
+
def pretty_print
|
16
|
+
visitor = ConfigurationVisitor.create
|
17
|
+
visitor << to_h
|
18
|
+
visitor.tree.yaml STDOUT, {}
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
require 'digest'
|
2
|
+
require 'set'
|
3
|
+
|
4
|
+
require_relative 'product'
|
5
|
+
|
6
|
+
module Grably
|
7
|
+
# Set of utilities to digest products and stat product set changes
|
8
|
+
module Digest
|
9
|
+
# Describes product state. If two digests for same file differs assume file changed
|
10
|
+
class ProductDigest
|
11
|
+
attr_reader :mtime, :size, :md5, :product
|
12
|
+
|
13
|
+
def initialize(product, mtime:, size:, md5:)
|
14
|
+
@product = product
|
15
|
+
@mtime = mtime
|
16
|
+
@size = size
|
17
|
+
@md5 = md5
|
18
|
+
end
|
19
|
+
|
20
|
+
def eql?(other)
|
21
|
+
self == other
|
22
|
+
end
|
23
|
+
|
24
|
+
def ==(other)
|
25
|
+
[
|
26
|
+
product == other.product,
|
27
|
+
mtime == other.mtime,
|
28
|
+
size == other.size,
|
29
|
+
md5 == other.md5
|
30
|
+
].all?
|
31
|
+
end
|
32
|
+
|
33
|
+
def hash
|
34
|
+
md5.to_i
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.[](*products)
|
38
|
+
products.map { |p| of_product(p) }
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.of_product(product)
|
42
|
+
product = Product.new(product) if product.is_a?(String)
|
43
|
+
raise 'Expected string or Product got ' + product.inspect unless product.is_a? Product
|
44
|
+
src = product.src
|
45
|
+
raise 'File does not exist' unless File.exist? src
|
46
|
+
ProductDigest.new(
|
47
|
+
product,
|
48
|
+
mtime: File.mtime(src),
|
49
|
+
size: File.size(src),
|
50
|
+
md5: ::Digest::MD5.hexdigest(IO.binread(src))
|
51
|
+
)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class << self
|
56
|
+
def digest(*products)
|
57
|
+
products.map { |product| ProductDigest.of_product(product) }
|
58
|
+
end
|
59
|
+
|
60
|
+
# Given two lists of product digests
|
61
|
+
# find missing, changed, and added products
|
62
|
+
# @return [deleted, added, updated]
|
63
|
+
def diff_digests(old_products, new_products)
|
64
|
+
# create maps of product sets
|
65
|
+
old_map, new_map = [old_products, new_products].map do |products|
|
66
|
+
return [] unless products
|
67
|
+
Hash[*products.flat_map { |d| [d.product, d] }]
|
68
|
+
end
|
69
|
+
|
70
|
+
build_diff(new_map, old_map)
|
71
|
+
end
|
72
|
+
|
73
|
+
def build_diff(new_map, old_map)
|
74
|
+
old_keys = old_map.keys.to_set
|
75
|
+
new_keys = new_map.keys.to_set
|
76
|
+
|
77
|
+
missing = old_keys - new_keys
|
78
|
+
added = new_keys - old_keys
|
79
|
+
|
80
|
+
updated = (old_keys & new_keys).reject do |product|
|
81
|
+
old_map[product] == new_map[product]
|
82
|
+
end
|
83
|
+
|
84
|
+
[missing, added, updated]
|
85
|
+
end
|
86
|
+
|
87
|
+
# Tells if two product digest lists are differ
|
88
|
+
def differ?(old_products, new_products)
|
89
|
+
!diff_digests(old_products, new_products).flatten.empty?
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Grably
|
2
|
+
# Contains custom Grably DSL definitions.
|
3
|
+
module DSL
|
4
|
+
def grab(module_call, as:, &block)
|
5
|
+
executor = Grably.server.schedule(module_call)
|
6
|
+
|
7
|
+
last_desc = Rake.application.last_description
|
8
|
+
desc module_call.pretty_print unless last_desc
|
9
|
+
task(as) do |t|
|
10
|
+
products = executor.call(t.task_dir)
|
11
|
+
block ? yield(t, products) : (t << products)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
# Short OS family name
|
3
|
+
# :win - Windows
|
4
|
+
# :linux - Linux
|
5
|
+
# :mac - OS X
|
6
|
+
PLATFORM = case RUBY_PLATFORM
|
7
|
+
when /mingw/, /cygwin/
|
8
|
+
:windows
|
9
|
+
when /mac/, /darwin/
|
10
|
+
:mac
|
11
|
+
else
|
12
|
+
:linux
|
13
|
+
end
|
14
|
+
# Number of CPU cores
|
15
|
+
CORES_NUMER = case PLATFORM
|
16
|
+
when :windows
|
17
|
+
# this works for windows 2000 or greater
|
18
|
+
require 'win32ole'
|
19
|
+
wmi = WIN32OLE.connect('winmgmts://')
|
20
|
+
query = 'select * from Win32_ComputerSystem'
|
21
|
+
wmi.ExecQuery(query).each do |system|
|
22
|
+
begin
|
23
|
+
processors = system.NumberOfLogicalProcessors
|
24
|
+
rescue StandardError => x
|
25
|
+
puts 'Warn: ' + x.message
|
26
|
+
processors = 1
|
27
|
+
end
|
28
|
+
return [system.NumberOfProcessors, processors].max
|
29
|
+
end
|
30
|
+
when :mac
|
31
|
+
`sysctl -n hw.logicalcpu`.to_i
|
32
|
+
when :linux
|
33
|
+
`cat /proc/cpuinfo | grep processor | wc -l`.to_i
|
34
|
+
else
|
35
|
+
raise "can't determine 'number_of_processors' for '#{RUBY_PLATFORM}'"
|
36
|
+
end
|
37
|
+
|
38
|
+
class << self
|
39
|
+
%w(windows mac linux).each do |platform|
|
40
|
+
# rubocop:disable Security/Eval
|
41
|
+
eval("def #{platform}?; #{PLATFORM == platform} end")
|
42
|
+
# rubocop:enable Security/Eval
|
43
|
+
end
|
44
|
+
|
45
|
+
def cores_number
|
46
|
+
CORES_NUMBER
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|