grably 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +202 -0
- data/README.md +2 -0
- data/exe/grably +2 -0
- data/lib/ext/class.rb +28 -0
- data/lib/grably.rb +83 -0
- data/lib/grably/cli.rb +46 -0
- data/lib/grably/core.rb +15 -0
- data/lib/grably/core/app/enchancer.rb +36 -0
- data/lib/grably/core/application.rb +8 -0
- data/lib/grably/core/colors.rb +86 -0
- data/lib/grably/core/commands.rb +23 -0
- data/lib/grably/core/commands/cp.rb +103 -0
- data/lib/grably/core/commands/digest.rb +12 -0
- data/lib/grably/core/commands/log.rb +19 -0
- data/lib/grably/core/commands/run.rb +85 -0
- data/lib/grably/core/commands/serialize.rb +16 -0
- data/lib/grably/core/configuration.rb +39 -0
- data/lib/grably/core/configuration/pretty_print.rb +22 -0
- data/lib/grably/core/digest.rb +93 -0
- data/lib/grably/core/dsl.rb +15 -0
- data/lib/grably/core/essentials.rb +49 -0
- data/lib/grably/core/module.rb +64 -0
- data/lib/grably/core/product.rb +301 -0
- data/lib/grably/core/task.rb +30 -0
- data/lib/grably/core/task/bucket.rb +29 -0
- data/lib/grably/core/task/enchancer.rb +50 -0
- data/lib/grably/core/task/expand.rb +15 -0
- data/lib/grably/core/task/jobs.rb +58 -0
- data/lib/grably/job.rb +28 -0
- data/lib/grably/job/class.rb +93 -0
- data/lib/grably/job/exceptions.rb +0 -0
- data/lib/grably/job/instance.rb +159 -0
- data/lib/grably/job/manifest.rb +67 -0
- data/lib/grably/jobs.rb +4 -0
- data/lib/grably/jobs/sync.rb +91 -0
- data/lib/grably/jobs/text.rb +4 -0
- data/lib/grably/jobs/text/erb.rb +40 -0
- data/lib/grably/jobs/text/json.rb +12 -0
- data/lib/grably/jobs/text/text.rb +21 -0
- data/lib/grably/jobs/text/yaml.rb +12 -0
- data/lib/grably/jobs/unzip.rb +1 -0
- data/lib/grably/jobs/upload.rb +1 -0
- data/lib/grably/jobs/zip.rb +2 -0
- data/lib/grably/jobs/zip/unzip.rb +24 -0
- data/lib/grably/jobs/zip/zip.rb +46 -0
- data/lib/grably/runner.rb +31 -0
- data/lib/grably/server.rb +83 -0
- data/lib/grably/utils/pretty_printer.rb +63 -0
- data/lib/grably/version.rb +12 -0
- metadata +164 -0
@@ -0,0 +1,23 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'digest/sha1'
|
3
|
+
|
4
|
+
require_relative 'commands/digest'
|
5
|
+
require_relative 'commands/cp'
|
6
|
+
require_relative 'commands/log'
|
7
|
+
require_relative 'commands/run'
|
8
|
+
require_relative 'commands/serialize'
|
9
|
+
|
10
|
+
module Grably # :nodoc:
|
11
|
+
# Make all methods as MODULE methods
|
12
|
+
def self.method_added(m)
|
13
|
+
module_function m
|
14
|
+
end
|
15
|
+
|
16
|
+
def relative_path(base, path)
|
17
|
+
# TODO: Reimplement
|
18
|
+
Dir.chdir(base) do
|
19
|
+
path = File.expand_path(path)
|
20
|
+
end
|
21
|
+
path
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,103 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
# Copy files or products to destination directory
|
3
|
+
# @param products [Array<Product>] list of products to copy
|
4
|
+
# @param dst_dir [String] destination directory to copy products
|
5
|
+
# @param base_dir [String] if provided all products will be copied relative to
|
6
|
+
# this path inside destination
|
7
|
+
# @param log [Boolean] if provided will log all actions to STDOUT. false by
|
8
|
+
# default
|
9
|
+
# @return [Array<Product>] list of resulting products.
|
10
|
+
def cp(products, dst_dir, base_dir: nil, log: false)
|
11
|
+
products = Product.expand(products)
|
12
|
+
dst_dir = File.expand_path(dst_dir)
|
13
|
+
dst_dir = File.join(dst_dir, base_dir) if base_dir
|
14
|
+
|
15
|
+
products.map { |product| copy_product(product, dst_dir, log: log) }
|
16
|
+
end
|
17
|
+
|
18
|
+
# Smart copy operates over product lists in the way like it directory
|
19
|
+
# structures. All chaned products will be replaced and missing products
|
20
|
+
# will be removed.
|
21
|
+
# @param products [Array<Product>] list of products to copy. Any expand
|
22
|
+
# expression will work as expected.
|
23
|
+
# @param dst_dir [String] target directory path.
|
24
|
+
# @param log [Boolean] if set to true log all actions to STDOUT. false by
|
25
|
+
# default
|
26
|
+
# @return [Array<Product>] list of resulting products
|
27
|
+
def cp_smart(products, dst_dir, log: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
|
28
|
+
# Ensure dst_dir is created
|
29
|
+
FileUtils.mkdir_p(dst_dir) unless File.exist?(dst_dir)
|
30
|
+
# Create Hash structures containing product.dst => product mappings
|
31
|
+
update = ->(acc, elem) { acc.update(elem.dst => elem) }
|
32
|
+
src_products = Product.expand(products).inject({}, &update)
|
33
|
+
dst_products = Product.expand(dst_dir).inject({}, &update)
|
34
|
+
|
35
|
+
# Looking for missing files
|
36
|
+
remove_files = (dst_products.keys - src_products.keys).each do |dst_key|
|
37
|
+
log "Remove #{dst_key} (#{dst_products[dst_key].src}" if log
|
38
|
+
end
|
39
|
+
FileUtils.rm(remove_files.map { |k| dst_products[k].src })
|
40
|
+
rm_empty_dirs(dst_dir)
|
41
|
+
# Update rest
|
42
|
+
src_products.map do |dst, product|
|
43
|
+
dst_product = dst_products[dst]
|
44
|
+
update_smart(product, dst_product, dst_dir, log: log)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Copy product to dst using FileUtils
|
49
|
+
def cp_sys(src, dst, log: false)
|
50
|
+
src = src.src if src.is_a?(Product)
|
51
|
+
dst = dst.src if dst.is_a?(Product)
|
52
|
+
log "Copy #{File.basename(src)} to #{dst}" if log
|
53
|
+
FileUtils.cp_r(src, dst)
|
54
|
+
end
|
55
|
+
|
56
|
+
private
|
57
|
+
|
58
|
+
def rm_empty_dirs(dir)
|
59
|
+
Dir[File.join(dir, '**/*')]
|
60
|
+
.select { |p| File.directory?(p) }
|
61
|
+
.select { |d| (Dir.entries(d) - %w(. ..)).empty? }
|
62
|
+
.each { |d| Dir.rmdir(d) }
|
63
|
+
end
|
64
|
+
|
65
|
+
def copy_product(product, dst_dir, log: false)
|
66
|
+
copy = product.map do |_src, dst, _meta|
|
67
|
+
File.join(dst_dir, dst)
|
68
|
+
end
|
69
|
+
|
70
|
+
dir = File.dirname(copy.dst)
|
71
|
+
FileUtils.mkdir_p(dir) unless File.exist?(dir)
|
72
|
+
log "Copy #{File.basename(product.src)} to #{copy.src}" if log
|
73
|
+
FileUtils.cp(product.src, copy.src)
|
74
|
+
|
75
|
+
copy
|
76
|
+
end
|
77
|
+
|
78
|
+
def update_smart(src_product, dst_product, dst_dir, log: false) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/LineLength
|
79
|
+
if dst_product
|
80
|
+
# Check if file changed. If so updating it
|
81
|
+
# TODO: Should we or should not check hashsum
|
82
|
+
if product_changed?(src_product, dst_product)
|
83
|
+
log "Update #{src_product.basename} to #{dst_product.src}" if log
|
84
|
+
FileUtils.cp(src_product.src, dst_product.src)
|
85
|
+
end
|
86
|
+
else
|
87
|
+
dst_product = src_product.map do |_src, dst, _meta|
|
88
|
+
File.join(dst_dir, dst)
|
89
|
+
end
|
90
|
+
dir = File.dirname(dst_product.src)
|
91
|
+
FileUtils.mkdir_p(dir) unless File.exist?(dir)
|
92
|
+
log "Copy #{src_product.basename} to #{dst_product.src}" if log
|
93
|
+
FileUtils.cp(src_product.src, dst_product.src)
|
94
|
+
end
|
95
|
+
|
96
|
+
dst_product
|
97
|
+
end
|
98
|
+
|
99
|
+
def product_changed?(p1, p2)
|
100
|
+
# TODO: Should we or should not check hashsums of files?
|
101
|
+
digest(p1) != digest(p2) if File.mtime(p1.src) != File.mtime(p2.src)
|
102
|
+
end
|
103
|
+
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
def log(msg)
|
3
|
+
puts msg
|
4
|
+
# flush stdout - we need that to have submodules displaying OK
|
5
|
+
$stdout.flush
|
6
|
+
end
|
7
|
+
|
8
|
+
def err(msg)
|
9
|
+
puts 'error: '.red.bright + msg
|
10
|
+
end
|
11
|
+
|
12
|
+
def warn(msg)
|
13
|
+
puts 'warning: '.cyan.bright + msg.to_s.cyan
|
14
|
+
end
|
15
|
+
|
16
|
+
def trace(msg)
|
17
|
+
puts msg if ENV['TRACE']
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
# Ruby array extensions
|
2
|
+
class Array
|
3
|
+
# Run array content as shell command
|
4
|
+
def run(&block)
|
5
|
+
Grably.run(self, &block)
|
6
|
+
end
|
7
|
+
|
8
|
+
# Run array content as shell command and handle exception automaticly
|
9
|
+
def run_safe
|
10
|
+
Grably.run_safe(self)
|
11
|
+
end
|
12
|
+
|
13
|
+
# Run array content as shell command and print it output to STDOUT
|
14
|
+
def run_log
|
15
|
+
Grably.run_log(self)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
module Grably # :nodoc:
|
20
|
+
# TODO: Rewimplement
|
21
|
+
# rubocop:disable all
|
22
|
+
class << self
|
23
|
+
def run_log(cmd)
|
24
|
+
run(cmd) { |l| log " #{l}" }
|
25
|
+
end
|
26
|
+
|
27
|
+
def run(cmd, &block)
|
28
|
+
env = {}
|
29
|
+
|
30
|
+
cmd = [cmd] if cmd.is_a?(Hash)
|
31
|
+
cmd = cmd.split(' ') unless cmd.is_a?(Array)
|
32
|
+
cmd = cmd.flatten.compact
|
33
|
+
|
34
|
+
cmd.map! do |c|
|
35
|
+
if c.is_a?(String) && c.empty?
|
36
|
+
c = nil
|
37
|
+
elsif c.is_a?(Hash)
|
38
|
+
env.merge!(c)
|
39
|
+
c = nil
|
40
|
+
end
|
41
|
+
c = c.to_s unless c.nil?
|
42
|
+
c
|
43
|
+
end
|
44
|
+
|
45
|
+
cmd = cmd.flatten.compact
|
46
|
+
|
47
|
+
# Not all windows apps "slash tolerant"
|
48
|
+
if windows?
|
49
|
+
cmd[0] = cmd[0].tr('/', '\\') unless cmd.empty?
|
50
|
+
end
|
51
|
+
|
52
|
+
pwd = nil
|
53
|
+
if env['__WORKING_DIR']
|
54
|
+
pwd = Dir.pwd
|
55
|
+
Dir.chdir(env.delete('__WORKING_DIR'))
|
56
|
+
end
|
57
|
+
|
58
|
+
r = []
|
59
|
+
IO.popen([env, cmd, { err: %i(child out) }].flatten) do |o|
|
60
|
+
o.sync = true
|
61
|
+
o.each do |l|
|
62
|
+
r << l
|
63
|
+
yield(l) unless block.nil?
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
Dir.chdir(pwd) if pwd
|
68
|
+
|
69
|
+
raise 'error: '.red.bright + cmd.red + "\nfail log: #{r}".green unless $?.exitstatus.zero?
|
70
|
+
|
71
|
+
r.join
|
72
|
+
end
|
73
|
+
|
74
|
+
def run_safe(cmd)
|
75
|
+
begin
|
76
|
+
run(cmd)
|
77
|
+
rescue
|
78
|
+
return false
|
79
|
+
end
|
80
|
+
|
81
|
+
true
|
82
|
+
end
|
83
|
+
# rubocop:enable all
|
84
|
+
end
|
85
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
def load_obj(filename)
|
3
|
+
return nil unless File.exist? filename
|
4
|
+
File.open(filename) do |f|
|
5
|
+
Marshal.load(f) # rubocop:disable Security/MarshalLoad
|
6
|
+
end
|
7
|
+
rescue StandardError => x
|
8
|
+
raise(x, "Can't deserialize #{filename}")
|
9
|
+
end
|
10
|
+
|
11
|
+
def save_obj(filename, obj)
|
12
|
+
dir = File.dirname(filename)
|
13
|
+
FileUtils.mkdir(dir) unless File.exist?(dir)
|
14
|
+
File.open(filename, 'w') { |f| Marshal.dump(obj, f) }
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require 'ostruct'
|
3
|
+
require 'jac'
|
4
|
+
|
5
|
+
require_relative 'configuration/pretty_print'
|
6
|
+
|
7
|
+
module Grably
|
8
|
+
module Core
|
9
|
+
# Grably configuration module.
|
10
|
+
# We use jac for configuration
|
11
|
+
# @see https://github.com/vizor-games/jac
|
12
|
+
module Configuration
|
13
|
+
# Default configuration file names
|
14
|
+
CONFIGURATION_FILES = %w(grably.yml grably.user.yml grably.override.yml).freeze
|
15
|
+
|
16
|
+
class << self
|
17
|
+
# Generates configuration object for given profile
|
18
|
+
# and list of streams with YAML document
|
19
|
+
# @param profile [Array] list of profile names to merge
|
20
|
+
# @param streams [Array] list of YAML documents and their
|
21
|
+
# names to read
|
22
|
+
# @return [OpenStruct] instance which contains all resolved profile fields
|
23
|
+
def read(profile, *streams)
|
24
|
+
Jac::Configuration.read(profile, *streams)
|
25
|
+
end
|
26
|
+
|
27
|
+
# Read configuration from configuration files.
|
28
|
+
# @praram [String or Array] profile which should be loaded
|
29
|
+
# @param [String] dir base directory path for provided files may be nil
|
30
|
+
# @return [OpenStruct] resolved profile values
|
31
|
+
def load(profile, dir: nil)
|
32
|
+
obj = Jac::Configuration.load(profile, files: CONFIGURATION_FILES, dir: dir)
|
33
|
+
obj.extend(self)
|
34
|
+
obj
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
require_relative '../colors'
|
3
|
+
|
4
|
+
module Grably
|
5
|
+
module Core
|
6
|
+
module Configuration # :nodoc:
|
7
|
+
# rubocop:disable all
|
8
|
+
class ConfigurationVisitor < Psych::Visitors::YAMLTree
|
9
|
+
def visit_Symbol(o)
|
10
|
+
visit_String(o.to_s)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
# rubocop:enable all
|
14
|
+
|
15
|
+
def pretty_print
|
16
|
+
visitor = ConfigurationVisitor.create
|
17
|
+
visitor << to_h
|
18
|
+
visitor.tree.yaml STDOUT, {}
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,93 @@
|
|
1
|
+
require 'digest'
|
2
|
+
require 'set'
|
3
|
+
|
4
|
+
require_relative 'product'
|
5
|
+
|
6
|
+
module Grably
|
7
|
+
# Set of utilities to digest products and stat product set changes
|
8
|
+
module Digest
|
9
|
+
# Describes product state. If two digests for same file differs assume file changed
|
10
|
+
class ProductDigest
|
11
|
+
attr_reader :mtime, :size, :md5, :product
|
12
|
+
|
13
|
+
def initialize(product, mtime:, size:, md5:)
|
14
|
+
@product = product
|
15
|
+
@mtime = mtime
|
16
|
+
@size = size
|
17
|
+
@md5 = md5
|
18
|
+
end
|
19
|
+
|
20
|
+
def eql?(other)
|
21
|
+
self == other
|
22
|
+
end
|
23
|
+
|
24
|
+
def ==(other)
|
25
|
+
[
|
26
|
+
product == other.product,
|
27
|
+
mtime == other.mtime,
|
28
|
+
size == other.size,
|
29
|
+
md5 == other.md5
|
30
|
+
].all?
|
31
|
+
end
|
32
|
+
|
33
|
+
def hash
|
34
|
+
md5.to_i
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.[](*products)
|
38
|
+
products.map { |p| of_product(p) }
|
39
|
+
end
|
40
|
+
|
41
|
+
def self.of_product(product)
|
42
|
+
product = Product.new(product) if product.is_a?(String)
|
43
|
+
raise 'Expected string or Product got ' + product.inspect unless product.is_a? Product
|
44
|
+
src = product.src
|
45
|
+
raise 'File does not exist' unless File.exist? src
|
46
|
+
ProductDigest.new(
|
47
|
+
product,
|
48
|
+
mtime: File.mtime(src),
|
49
|
+
size: File.size(src),
|
50
|
+
md5: ::Digest::MD5.hexdigest(IO.binread(src))
|
51
|
+
)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
class << self
|
56
|
+
def digest(*products)
|
57
|
+
products.map { |product| ProductDigest.of_product(product) }
|
58
|
+
end
|
59
|
+
|
60
|
+
# Given two lists of product digests
|
61
|
+
# find missing, changed, and added products
|
62
|
+
# @return [deleted, added, updated]
|
63
|
+
def diff_digests(old_products, new_products)
|
64
|
+
# create maps of product sets
|
65
|
+
old_map, new_map = [old_products, new_products].map do |products|
|
66
|
+
return [] unless products
|
67
|
+
Hash[*products.flat_map { |d| [d.product, d] }]
|
68
|
+
end
|
69
|
+
|
70
|
+
build_diff(new_map, old_map)
|
71
|
+
end
|
72
|
+
|
73
|
+
def build_diff(new_map, old_map)
|
74
|
+
old_keys = old_map.keys.to_set
|
75
|
+
new_keys = new_map.keys.to_set
|
76
|
+
|
77
|
+
missing = old_keys - new_keys
|
78
|
+
added = new_keys - old_keys
|
79
|
+
|
80
|
+
updated = (old_keys & new_keys).reject do |product|
|
81
|
+
old_map[product] == new_map[product]
|
82
|
+
end
|
83
|
+
|
84
|
+
[missing, added, updated]
|
85
|
+
end
|
86
|
+
|
87
|
+
# Tells if two product digest lists are differ
|
88
|
+
def differ?(old_products, new_products)
|
89
|
+
!diff_digests(old_products, new_products).flatten.empty?
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Grably
|
2
|
+
# Contains custom Grably DSL definitions.
|
3
|
+
module DSL
|
4
|
+
def grab(module_call, as:, &block)
|
5
|
+
executor = Grably.server.schedule(module_call)
|
6
|
+
|
7
|
+
last_desc = Rake.application.last_description
|
8
|
+
desc module_call.pretty_print unless last_desc
|
9
|
+
task(as) do |t|
|
10
|
+
products = executor.call(t.task_dir)
|
11
|
+
block ? yield(t, products) : (t << products)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Grably # :nodoc:
|
2
|
+
# Short OS family name
|
3
|
+
# :win - Windows
|
4
|
+
# :linux - Linux
|
5
|
+
# :mac - OS X
|
6
|
+
PLATFORM = case RUBY_PLATFORM
|
7
|
+
when /mingw/, /cygwin/
|
8
|
+
:windows
|
9
|
+
when /mac/, /darwin/
|
10
|
+
:mac
|
11
|
+
else
|
12
|
+
:linux
|
13
|
+
end
|
14
|
+
# Number of CPU cores
|
15
|
+
CORES_NUMER = case PLATFORM
|
16
|
+
when :windows
|
17
|
+
# this works for windows 2000 or greater
|
18
|
+
require 'win32ole'
|
19
|
+
wmi = WIN32OLE.connect('winmgmts://')
|
20
|
+
query = 'select * from Win32_ComputerSystem'
|
21
|
+
wmi.ExecQuery(query).each do |system|
|
22
|
+
begin
|
23
|
+
processors = system.NumberOfLogicalProcessors
|
24
|
+
rescue StandardError => x
|
25
|
+
puts 'Warn: ' + x.message
|
26
|
+
processors = 1
|
27
|
+
end
|
28
|
+
return [system.NumberOfProcessors, processors].max
|
29
|
+
end
|
30
|
+
when :mac
|
31
|
+
`sysctl -n hw.logicalcpu`.to_i
|
32
|
+
when :linux
|
33
|
+
`cat /proc/cpuinfo | grep processor | wc -l`.to_i
|
34
|
+
else
|
35
|
+
raise "can't determine 'number_of_processors' for '#{RUBY_PLATFORM}'"
|
36
|
+
end
|
37
|
+
|
38
|
+
class << self
|
39
|
+
%w(windows mac linux).each do |platform|
|
40
|
+
# rubocop:disable Security/Eval
|
41
|
+
eval("def #{platform}?; #{PLATFORM == platform} end")
|
42
|
+
# rubocop:enable Security/Eval
|
43
|
+
end
|
44
|
+
|
45
|
+
def cores_number
|
46
|
+
CORES_NUMBER
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|