dake 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: 0bbdbd9a0326d99f69811c766b89258377e872f5fc520c99bdfdec9d9e24fcf0
4
+ data.tar.gz: 78e6a370fec08445f8e7c2b16d182b6d00d5a2dc1413be1617d5d5a90cf1467f
5
+ SHA512:
6
+ metadata.gz: 72eeb89e6d34de732cf390ccc81c45214f0139a76dbf63227dba93e1b805bc7cbc59fb3686ad309dee280f6b0338c931d5eef5359d19bc4441015faf934d8af7
7
+ data.tar.gz: de369174d69b02799c2c815014e446de2368f96d1a00a0ac1fc5c081d997f996c0b4eaf6b05a0b663b26e400013b346cc7278f9d2d240f1138644ba0584ccf74
data/.gitignore ADDED
@@ -0,0 +1,8 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /_yardoc/
4
+ /coverage/
5
+ /doc/
6
+ /pkg/
7
+ /spec/reports/
8
+ /tmp/
data/Gemfile ADDED
@@ -0,0 +1,6 @@
1
+ source "https://rubygems.org"
2
+
3
+ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
4
+
5
+ # Specify your gem's dependencies in dake.gemspec
6
+ gemspec
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2019 minor6th
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,36 @@
1
+ # Dake
2
+
3
+ Dake is a data workflow tool inspired by [Drake](https://github.com/Factual/drake).
4
+ The basic syntax is mostly the same as [Drake](https://docs.google.com/document/d/1bF-OKNLIG10v_lMes_m4yyaJtAaJKtdK0Jizvi_MNsg/edit).
5
+
6
+ ## Installation
7
+
8
+ Add this line to your application's Gemfile:
9
+
10
+ ```ruby
11
+ gem 'dake'
12
+ ```
13
+
14
+ And then execute:
15
+
16
+ $ bundle
17
+
18
+ Or install it yourself as:
19
+
20
+ $ gem install dake
21
+
22
+ ## Usage
23
+
24
+ To show the help message:
25
+
26
+ $ dake help
27
+
28
+ ## Development
29
+
30
+ After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
31
+
32
+ To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
33
+
34
+ ## Contributing
35
+
36
+ Bug reports and pull requests are welcome on GitHub at https://github.com/minor6th/dake.
data/Rakefile ADDED
@@ -0,0 +1,17 @@
1
+ require "bundler/gem_tasks"
2
+ require 'rake/clean'
3
+ require 'rubygems'
4
+ require 'rubygems/package_task'
5
+ require 'rdoc/task'
6
+
7
+ Rake::RDocTask.new do |rd|
8
+ rd.main = "README.rdoc"
9
+ rd.rdoc_files.include("README.rdoc","lib/**/*.rb","bin/**/*")
10
+ rd.title = 'Dake'
11
+ end
12
+
13
+ spec = eval(File.read('dake.gemspec'))
14
+ Gem::PackageTask.new(spec) do |pkg|
15
+ end
16
+
17
+ task :default => :spec
data/bin/console ADDED
@@ -0,0 +1,14 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require "bundler/setup"
4
+ require "dake"
5
+
6
+ # You can add fixtures and/or initialization code here to make experimenting
7
+ # with your gem easier. You can also use a different console, if you like.
8
+
9
+ # (If you use this, don't forget to add pry to your Gemfile!)
10
+ # require "pry"
11
+ # Pry.start
12
+
13
+ require "irb"
14
+ IRB.start(__FILE__)
data/bin/setup ADDED
@@ -0,0 +1,8 @@
1
+ #!/usr/bin/env bash
2
+ set -euo pipefail
3
+ IFS=$'\n\t'
4
+ set -vx
5
+
6
+ bundle install
7
+
8
+ # Do any other automated setup that you need to do here
data/dake.gemspec ADDED
@@ -0,0 +1,51 @@
1
+
2
+ lib = File.expand_path("../lib", __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require "dake/version"
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "dake"
8
+ spec.version = Dake::VERSION
9
+ spec.authors = ["minor6th"]
10
+ spec.email = ["minor6th@outlook.com"]
11
+
12
+ spec.summary = %q{Dake is a data workflow tool inspired by Drake.}
13
+ spec.description = %q{Dake is a data workflow tool inspired by Drake.}
14
+ spec.homepage = "https://github.com/minor6th/dake"
15
+
16
+ # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
17
+ # to allow pushing to a single host or delete this section to allow pushing to any host.
18
+ # if spec.respond_to?(:metadata)
19
+ # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
20
+ #
21
+ # spec.metadata["homepage_uri"] = spec.homepage
22
+ # spec.metadata["source_code_uri"] = "TODO: Put your gem's public repo URL here."
23
+ # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
24
+ # else
25
+ # raise "RubyGems 2.0 or newer is required to protect against " \
26
+ # "public gem pushes."
27
+ # end
28
+
29
+ # Specify which files should be added to the gem when it is released.
30
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
31
+ spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do
32
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
33
+ end
34
+ spec.bindir = "exe"
35
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
36
+ spec.require_paths = ["lib"]
37
+
38
+ spec.add_development_dependency "bundler", "~> 2.0"
39
+ spec.add_development_dependency "rake", "~> 10.0"
40
+ spec.add_development_dependency "rdoc"
41
+ spec.add_development_dependency "rspec"
42
+ spec.add_development_dependency "awesome_print"
43
+
44
+ spec.add_runtime_dependency "gli"
45
+ spec.add_runtime_dependency "git"
46
+ #spec.add_runtime_dependency "sqlite3"
47
+ spec.add_runtime_dependency "colorize"
48
+ spec.add_runtime_dependency "sinatra"
49
+ spec.add_runtime_dependency "parslet"
50
+ spec.add_runtime_dependency "concurrent-ruby"
51
+ end
data/exe/dake ADDED
@@ -0,0 +1,170 @@
1
+ #!/usr/bin/env ruby
2
+ require 'gli'
3
+ require 'colorize'
4
+ require 'pathname'
5
+ require 'dake'
6
+
7
+ DAKE_EXEC_PID = Process.pid
8
+ DAKE_EXEC_TIME = Time.now
9
+
10
+ include GLI::App
11
+
12
+ program_desc 'Dake is a data workflow tool inspired by Drake.'
13
+ version Dake::VERSION
14
+
15
+ subcommand_option_handling :normal
16
+ arguments :strict
17
+
18
+ accept(Hash) do |value|
19
+ result = {}
20
+ value.split(/,/).each do |pair|
21
+ k,v = pair.split(/:/)
22
+ result[k] = v
23
+ end
24
+ result
25
+ end
26
+
27
+ def target_opts(target)
28
+ opts = Dake::TargetOption.new(false, false, :check, :up_tree)
29
+ mdata = /(?<build_mode>[+-]?)(?<tree_mode>[=^]?)(?<tag>@?)(?<regex>%?)(?<target_name>.+)/.match(target)
30
+ opts.build_mode = (mdata[:build_mode] == '+' ? :forced : :exclusion) unless mdata[:build_mode].empty?
31
+ opts.tree_mode = (mdata[:tree_mode] == '=' ? 'target_only' : :down_tree) unless mdata[:tree_mode].empty?
32
+ opts.tag = true unless mdata[:tag].empty?
33
+ opts.regex = true unless mdata[:regex].empty?
34
+ [mdata[:target_name], opts]
35
+ end
36
+
37
+ switch [:c, :color],
38
+ desc: 'Enable colorization',
39
+ default_value: true
40
+
41
+ flag [:f, :workflow],
42
+ desc: 'Specify the workflow file',
43
+ arg_name:'filename',
44
+ default_value: 'Dakefile'
45
+
46
+ flag [:var],
47
+ multiple: true,
48
+ desc: 'Set workflow variables',
49
+ arg_name: 'key:val,...',
50
+ type: Hash
51
+
52
+ desc 'Run all steps needed to build the specified target'
53
+ arg_name 'Targets'
54
+ command :build do |c|
55
+ c.flag [:j, :jobs],
56
+ desc: 'Specifies the maximum number of steps to execute in parallel',
57
+ arg_name: 'max_num',
58
+ type: Integer
59
+
60
+ c.switch [:n, :'dry-run'],
61
+ desc: 'Do a dry run without executing actions',
62
+ default_value: false
63
+
64
+ c.switch [:'log'],
65
+ desc: 'Log the STDOUT and STDERR to separate files',
66
+ default_value: false
67
+
68
+ c.action do |global_options, options, args|
69
+ String.disable_colorization = true unless global_options[:color]
70
+ raise 'No targets specified.' if args.empty?
71
+ if Dir.exist? global_options[:workflow]
72
+ global_options[:workflow] = File.join(global_options[:workflow], 'Dakefile')
73
+ end
74
+ raise "Cannot open workflow file `#{global_options[:workflow]}'." unless File.exist? global_options[:workflow]
75
+ path = File.absolute_path global_options[:workflow]
76
+ workflow_file = File.open path
77
+ env = global_options[:var].reduce({}, :merge!)
78
+ env.merge! 'BASE' => File.dirname(path)
79
+ begin
80
+ target_dict = {}
81
+ args.each do |arg|
82
+ target, opts = target_opts(arg)
83
+ target_dict[target] = opts
84
+ end
85
+ tree = DakeParser.new.parse(workflow_file.read)
86
+ workflow = DakeTransform.new.apply(tree, src_file: workflow_file.to_path)
87
+ analyzer = DakeAnalyzer.new(workflow, [path], env).analyze
88
+ resolver = DakeResolver.new(analyzer)
89
+ dep_graph = resolver.resolve(target_dict)
90
+ dake_db = DakeDB.new(workflow_file.to_path)
91
+ DakeExecutor.new(analyzer, dake_db, dep_graph, options[:jobs]).execute(options[:'dry-run'], options[:log])
92
+ rescue Parslet::ParseFailed => failure
93
+ puts failure.cause
94
+ exit(1)
95
+ end
96
+ end
97
+ end
98
+
99
+ desc 'List all targets defined in the workflow'
100
+ command :list do |c|
101
+ c.switch [:'dep'],
102
+ desc: 'Show the full dependency tree for each target',
103
+ default_value: false
104
+ c.action do |global_options,options,args|
105
+ String.disable_colorization = false unless global_options[:color]
106
+ if Dir.exist? global_options[:workflow]
107
+ global_options[:workflow] = File.join(global_options[:workflow], 'Dakefile')
108
+ end
109
+ raise "Cannot open workflow file `#{global_options[:workflow]}'." unless File.exist? global_options[:workflow]
110
+ path = File.absolute_path global_options[:workflow]
111
+ workflow_file = File.open path
112
+ env = global_options[:var].reduce({}, :merge!)
113
+ env.merge! 'BASE' => File.dirname(path)
114
+ begin
115
+ tree = DakeParser.new.parse(workflow_file.read)
116
+ workflow = DakeTransform.new.apply(tree, src_file: workflow_file.to_path)
117
+ analyzer = DakeAnalyzer.new(workflow, [path], env).analyze
118
+ resolver = DakeResolver.new(analyzer)
119
+
120
+ tag_list = analyzer.tag_target_dict.keys.map { |tag| [tag, Dake::TargetOption.new(true)] }
121
+ file_list = analyzer.file_target_dict.keys.map { |file| [file, Dake::TargetOption.new(false)] }
122
+
123
+ dep_graph = (options['dep'] ? resolver.resolve((tag_list + file_list).to_h, true) : nil)
124
+ analyzer.tag_target_dict.each do |tag, steps|
125
+ steps.each do |step|
126
+ next if options['dep'] and not dep_graph.root_step.include? step
127
+ analyzer.print_target(tag, true, step, dep_graph)
128
+ end
129
+ end
130
+ analyzer.file_target_dict.each do |file, step|
131
+ next if options['dep'] and not dep_graph.root_step.include? step
132
+ analyzer.print_target(file, false, step, dep_graph)
133
+ end
134
+ rescue Parslet::ParseFailed => failure
135
+ STDERR.puts "Failed parsing #{global_options[:workflow]}."
136
+ raise failure.message
137
+ end
138
+ end
139
+ end
140
+
141
+ desc 'Start the Dake monitor server'
142
+ arg_name 'Describe arguments to server here'
143
+ command :server do |c|
144
+ c.action do |global_options, options, args|
145
+ raise 'Not implemented.'
146
+ end
147
+ end
148
+
149
+ pre do |global,command,options,args|
150
+ # Pre logic here
151
+ # Return true to proceed; false to abort and not call the
152
+ # chosen command
153
+ # Use skips_pre before a command to skip this block
154
+ # on that command only
155
+ true
156
+ end
157
+
158
+ post do |global,command,options,args|
159
+ # Post logic here
160
+ # Use skips_post before a command to skip this
161
+ # block on that command only
162
+ end
163
+
164
+ on_error do |exception|
165
+ # Error logic here
166
+ # return false to skip default error handling
167
+ true
168
+ end
169
+
170
+ exit run(ARGV)