dake 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0bbdbd9a0326d99f69811c766b89258377e872f5fc520c99bdfdec9d9e24fcf0
-  data.tar.gz: 78e6a370fec08445f8e7c2b16d182b6d00d5a2dc1413be1617d5d5a90cf1467f
+  metadata.gz: 480b83fa04055fcf5d393da94b870bafdb901c5a39788d52dbee7d89244f5f1f
+  data.tar.gz: a10e599b85562d3302997cd2500117357fbbacd2bae85cf7232452246afe5613
 SHA512:
-  metadata.gz: 72eeb89e6d34de732cf390ccc81c45214f0139a76dbf63227dba93e1b805bc7cbc59fb3686ad309dee280f6b0338c931d5eef5359d19bc4441015faf934d8af7
-  data.tar.gz: de369174d69b02799c2c815014e446de2368f96d1a00a0ac1fc5c081d997f996c0b4eaf6b05a0b663b26e400013b346cc7278f9d2d240f1138644ba0584ccf74
+  metadata.gz: 6c849058294a6c5daf084d19ba34d6cbecc53736863958cd064f1ab5f2f5d1cbe1a83c903fc137622ce02d2fa6547285a07882643b84d01c42b6a35b809bacb4
+  data.tar.gz: c665db3ce2bafab077723497299735e5c9d455654385b8c1845e944454ec0140c5410bbf5b4b67410224eb27b39ac97d7798a284b47c9d3e6d77ff2ce4394cd4

data/.gitignore

@@ -1,3 +1,4 @@
+/.idea/
 /.bundle/
 /.yardoc
 /_yardoc/
@@ -6,3 +7,4 @@
 /pkg/
 /spec/reports/
 /tmp/
+Gemfile.lock

data/dake.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |spec|
   spec.summary       = %q{Dake is a data workflow tool inspired by Drake.}
   spec.description   = %q{Dake is a data workflow tool inspired by Drake.}
   spec.homepage      = "https://github.com/minor6th/dake"
+  spec.license       = "MIT"
 
   # Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
   # to allow pushing to a single host or delete this section to allow pushing to any host.

data/examples/Dakefile

@@ -0,0 +1,40 @@
+# This is an example
+
+imdb/dataset_urls <- [ruby]
+    require 'net/http'
+    html = Net::HTTP.get(URI('https://datasets.imdbws.com/index.html'))
+    urls = html.each_line.grep(/href=.+tsv.gz/) { |line| line.sub(/.*<a href=(https:\/\/.+tsv.gz)>.*\n/, '\1') }
+    FileUtils.mkdir('imdb') unless Dir.exist? 'imdb'
+    file = File.open('$[OUTPUT]', 'w')
+    file.write(urls.join("\n"))
+    file.close
+
+^"imdb/.+\.tsv.gz" <- imdb/dataset_urls
+    FILE=`basename $[OUTPUT]`
+    URL=`grep $FILE $[INPUT]`
+    wget -O $[OUTPUT] $URL
+    touch $[OUTPUT]
+
+^"imdb/(?<file>.+\.tsv)" <- imdb/$[file].gz
+    gunzip -k $[INPUT]
+    touch $[OUTPUT]
+
+imdb/title_type_rating_dist.pdf <- imdb/title.ratings.tsv, imdb/title.basics.tsv [r]
+    require(tidyverse)
+    title_ratings <- read_tsv("$[INPUT0]")
+    title_basics <- read_tsv("$[INPUT1]")
+    pd <- title_basics %>% left_join(title_ratings, by='tconst') %>%
+                           group_by(titleType) %>%
+                           summarise(averageRating = mean(averageRating, na.rm = T))
+    pdf("$[OUTPUT]", width = 5, height = 4)
+    ggplot(pd, aes(x = reorder(titleType, averageRating), y = averageRating)) +
+        geom_col() +
+        coord_flip() +
+        xlab("titleType")
+
+imdb/rating_dist.pdf <- imdb/title.ratings.tsv [r]
+    require(tidyverse)
+    title_ratings <- read_tsv("$[INPUT0]")
+    pdf("$[OUTPUT]", width = 5, height = 4)
+    ggplot(title_ratings, aes(x = averageRating)) + geom_histogram()
+

data/exe/dake

@@ -28,7 +28,7 @@ def target_opts(target)
   opts = Dake::TargetOption.new(false, false, :check, :up_tree)
   mdata = /(?<build_mode>[+-]?)(?<tree_mode>[=^]?)(?<tag>@?)(?<regex>%?)(?<target_name>.+)/.match(target)
   opts.build_mode = (mdata[:build_mode] == '+' ? :forced : :exclusion) unless mdata[:build_mode].empty?
-  opts.tree_mode = (mdata[:tree_mode] == '=' ? 'target_only' : :down_tree) unless mdata[:tree_mode].empty?
+  opts.tree_mode = (mdata[:tree_mode] == '=' ? :target_only : :down_tree) unless mdata[:tree_mode].empty?
   opts.tag = true unless mdata[:tag].empty?
   opts.regex = true unless mdata[:regex].empty?
   [mdata[:target_name], opts]
@@ -77,18 +77,42 @@ command :build do |c|
     env = global_options[:var].reduce({}, :merge!)
     env.merge! 'BASE' => File.dirname(path)
     begin
-      target_dict = {}
-      args.each do |arg|
-        target, opts = target_opts(arg)
-        target_dict[target] = opts
-      end
       tree = DakeParser.new.parse(workflow_file.read)
       workflow = DakeTransform.new.apply(tree, src_file: workflow_file.to_path)
       analyzer = DakeAnalyzer.new(workflow, [path], env).analyze
+      target_pairs = []
+      args.each do |arg|
+        target, opts = target_opts(arg)
+        if target == '...'
+          tag_list = analyzer.tag_target_dict.keys.map { |tag| [tag, Dake::TargetOption.new(true, false, :check, :up_tree)] }
+          file_list = analyzer.file_target_dict.keys.map { |file| [file, Dake::TargetOption.new(false, false, :check, :up_tree)] }
+          target_pairs += (tag_list + file_list)
+        else
+          if opts.regex
+            matched_targets =
+                if opts.tag
+                  analyzer.tag_target_dict.keys.find_all { |tag_name| tag_name.match target }
+                else
+                  analyzer.file_target_dict.keys.find_all { |file_name| file_name.match target }
+                end
+            matched_targets.each { |matched_target| target_pairs << [matched_target, opts] }
+          else
+            target_pairs << [target, opts]
+          end
+        end
+      end
+      if target_pairs.any? { |_, opts| opts.tree_mode == :down_tree }
+        tag_list = analyzer.tag_target_dict.keys.map { |tag| [tag, Dake::TargetOption.new(true, false, :check, :up_tree)] }
+        file_list = analyzer.file_target_dict.keys.map { |file| [file, Dake::TargetOption.new(false, false, :check, :up_tree)] }
+        resolve_pairs = (tag_list + file_list)
+      else
+        resolve_pairs = target_pairs
+      end
       resolver = DakeResolver.new(analyzer)
-      dep_graph = resolver.resolve(target_dict)
+      dep_graph = resolver.resolve(resolve_pairs)
+      rebuild_set = resolver.target_rebuild_set(target_pairs, dep_graph)
       dake_db = DakeDB.new(workflow_file.to_path)
-      DakeExecutor.new(analyzer, dake_db, dep_graph, options[:jobs]).execute(options[:'dry-run'], options[:log])
+      DakeExecutor.new(analyzer, dake_db, dep_graph, options[:jobs]).execute(rebuild_set, options[:'dry-run'], options[:log])
     rescue Parslet::ParseFailed => failure
       puts failure.cause
       exit(1)
@@ -116,11 +140,13 @@ command :list do |c|
       workflow = DakeTransform.new.apply(tree, src_file: workflow_file.to_path)
       analyzer = DakeAnalyzer.new(workflow, [path], env).analyze
       resolver = DakeResolver.new(analyzer)
-
-      tag_list = analyzer.tag_target_dict.keys.map { |tag| [tag, Dake::TargetOption.new(true)] }
-      file_list = analyzer.file_target_dict.keys.map { |file| [file, Dake::TargetOption.new(false)] }
-
-      dep_graph = (options['dep'] ? resolver.resolve((tag_list + file_list).to_h, true) : nil)
+      dep_graph = nil
+      if options['dep']
+        tag_list = analyzer.tag_target_dict.keys.map { |tag| [tag, Dake::TargetOption.new(true, false, :check, :up_tree)] }
+        file_list = analyzer.file_target_dict.keys.map { |file| [file, Dake::TargetOption.new(false, false, :check, :up_tree)] }
+        target_pairs = (tag_list + file_list)
+        dep_graph = resolver.resolve(target_pairs)
+      end
       analyzer.tag_target_dict.each do |tag, steps|
         steps.each do |step|
           next if options['dep'] and not dep_graph.root_step.include? step

data/lib/dake/analyzer.rb

@@ -1,18 +1,6 @@
 require 'set'
 require 'open3'
 
-# the data struct needed by the executor
-# note that this is not nessesarily the complete graph,
-# the graph is only used to produce the given targets
-DepGraph = Struct.new(
-  :succ_step,    # a dict maps each step in the DepGraph to the steps depend on it
-  :dep_step,     # a dict maps each step in the DepGraph to the steps it depends on
-  :step_list,    # a list of steps represents one sequential execution order
-  :root_step,    # a set of steps that hos no dependant
-  :leaf_step,    # a set of steps that has no prerequisite
-  :need_rebuild  # a set of steps in step_list which should be executed to update their targets
-)
-
 class DakeAnalyzer
   attr_reader :workflow, :variable_dict, :method_dict, :included_files
   attr_reader :tag_target_dict, :file_target_dict
@@ -138,9 +126,9 @@ class DakeAnalyzer
           end
         else
           # Generated file list should not be used in targets
-          if type == :targets
-            raise "File list `#{file_name}' in #{step.src_file} at #{line}:#{column} cannot be used as targets."
-          end
+          # if type == :targets
+          #   raise "File list `#{file_name}' in #{step.src_file} at #{line}:#{column} cannot be used as targets."
+          # end
         end
         newfile = file.dup
         newfile.scheme = scheme

data/lib/dake/executor.rb

@@ -15,16 +15,17 @@ class DakeExecutor
     ) if @async
   end
 
-  def execute(dry_run=false, log=false)
-    if @dep_graph.need_rebuild.empty?
+  def execute(rebuild_set, dry_run=false, log=false)
+    if rebuild_set.empty?
       STDERR.puts "Nothing to be done.".colorize(:green)
       return
     end
     if @async
       dep_map = Hash.new
-      @dep_graph.dep_step.each do |step, dep_set|
+      rebuild_set.each do |step|
+        dep_set = @dep_graph.dep_step[step]
         next if dep_set.empty?
-        dep_map[step] = dep_set.dup
+        dep_map[step] = dep_set & rebuild_set
       end
 
       queue = Queue.new
@@ -42,7 +43,7 @@ class DakeExecutor
       end
 
       lock = Concurrent::ReadWriteLock.new
-      @dep_graph.leaf_step.each { |step| queue << step }
+      @dep_graph.leaf_step.each { |step| queue << step if rebuild_set.include? step }
 
       while next_step = queue.deq
         @pool.post(next_step) do |step|
@@ -55,7 +56,7 @@ class DakeExecutor
                   "skipped due to prerequisite step(s) error."
             error_queue << Exception.new(msg)
           else
-            execute_step(step, dry_run, log) if @dep_graph.need_rebuild.include? step
+            execute_step(step, dry_run, log)
           end
           lock.acquire_write_lock
           dep_map.delete step
@@ -63,6 +64,7 @@ class DakeExecutor
             queue.close
           else
             @dep_graph.succ_step[step].each do |succ|
+              next unless dep_map[succ]
               dep_map[succ].delete step
               if dep_map[succ].empty?
                 queue << succ
@@ -99,7 +101,7 @@ class DakeExecutor
       raise "Failed to execute some step(s)" unless error_steps.empty?
     else
       @dep_graph.step_list.each do |step|
-        execute_step(step, dry_run, log) if @dep_graph.need_rebuild.include? step
+        execute_step(step, dry_run, log) if rebuild_set.include? step
       end
     end
   end

data/lib/dake/resolver.rb

@@ -1,8 +1,115 @@
+# the data struct needed by the executor
+# note that this is not nessesarily the complete graph,
+# the graph is only used to produce the given targets
+DepGraph = Struct.new(
+    :succ_step,    # a dict maps each step in the DepGraph to the steps depend on it
+    :dep_step,     # a dict maps each step in the DepGraph to the steps it depends on
+    :step_list,    # a list of steps represents one sequential execution order
+    :root_step,    # a set of steps that hos no dependant
+    :leaf_step,    # a set of steps that has no prerequisite
+    :step_target   # a dict maps each step in the DepGraph to its output files used while resolving targets
+)
+
 class DakeResolver
   def initialize(analyzer)
     @analyzer = analyzer
   end
 
+  def target_rebuild_set(target_pairs, dep_graph)
+    rebuild_set = Set.new
+    target_pairs.each do |target_name, target_opts|
+      if target_opts.tag
+        dummy_step = Step.new([], [], [], {}, nil, nil, @analyzer.variable_dict, nil, nil)
+        scheme = DakeScheme::Tag.new('@', target_name, dummy_step)
+      else
+        scheme = @analyzer.analyze_scheme(target_name, nil, nil, nil)
+      end
+      target_steps = find_steps(scheme, target_opts.tag).to_set
+      if target_steps.empty? and not scheme.exist?
+        raise "No step found for building file `#{target_name}'."
+      end
+
+      visited = Set.new
+      path_visited = Set.new
+      down_tree_steps = Set.new
+      up_tree_steps = Set.new
+      need_rebuild = Set.new
+      up_tree = 0
+      dep_step_list = {}
+
+      init_steps = (target_opts.tree_mode == :down_tree ? dep_graph.root_step : target_steps)
+      init_steps.each do |init_step|
+        stack = [init_step]
+
+        until stack.empty?
+          step = stack.last
+          visited << step
+          path_visited << step
+          up_tree_steps << step if up_tree > 0 or target_steps.include? step
+
+          dep_step_list[step] ||= dep_graph.dep_step[step].to_a
+          while next_step = dep_step_list[step].pop
+            break unless visited.include? next_step
+          end
+          if next_step
+            stack.push next_step
+            up_tree += 1 if target_steps.include? step
+          else
+            stack.pop
+
+            if dep_graph.leaf_step.include? step
+              step.prerequisites.each do |prereq|
+                if prereq.flag != '?' and not prereq.scheme.exist?
+                  raise "No step found for building file `#{prereq.scheme.path}'."
+                end
+              end
+            end
+
+            if target_steps.include? step or dep_graph.dep_step[step].any? { |s| down_tree_steps.include? s }
+              down_tree_steps << step
+            end
+
+            if target_opts.build_mode == :check and (up_tree_steps.include? step or down_tree_steps.include? step)
+              if dep_graph.leaf_step.include? step
+                need_rebuild << step if need_execute?(dep_graph.step_target[step], step)
+              else
+                if dep_graph.dep_step[step].any? { |dep_step| need_rebuild.include? dep_step }
+                  need_rebuild << step
+                else
+                  need_rebuild << step if need_execute?(dep_graph.step_target[step], step)
+                end
+              end
+            end
+
+            up_tree -= 1 if target_steps.include? step
+            path_visited.delete step
+          end
+        end
+      end
+      case target_opts.build_mode
+      when :forced
+        case target_opts.tree_mode
+        when :up_tree then rebuild_set |= up_tree_steps
+        when :down_tree then rebuild_set |= down_tree_steps
+        when :target_only then rebuild_set |= target_steps
+        end
+      when :exclusion
+        case target_opts.tree_mode
+        when :up_tree then rebuild_set -= up_tree_steps
+        when :down_tree then rebuild_set -= down_tree_steps
+        when :target_only then rebuild_set -= target_steps
+        end
+      when :check
+        case target_opts.tree_mode
+        when :up_tree then rebuild_set |= (up_tree_steps & need_rebuild)
+        when :down_tree then rebuild_set |= (down_tree_steps & need_rebuild)
+        when :target_only then rebuild_set |= (target_steps & need_rebuild)
+        end
+      end
+    end
+    rebuild_set
+  end
+
   # check if a step needs to be executed to produce the given targets
   def need_execute?(targets, step)
     max_mtime = nil
@@ -43,7 +150,7 @@ class DakeResolver
     false
   end
 
-  def find_steps(target_scheme, tag, optional=false)
+  def find_steps(target_scheme, tag)
     target_name = target_scheme.path
     target_src = target_scheme.src
     if tag
@@ -58,8 +165,8 @@ class DakeResolver
           steps = []
           template_steps.each do |template_step|
             if @analyzer.step_template_dict[template_step] and
-               @analyzer.step_template_dict[template_step][mdata.named_captures]
-              step = @analyzer.step_template_dict[template_step][mdata.named_captures]
+               @analyzer.step_template_dict[template_step][mdata]
+              step = @analyzer.step_template_dict[template_step][mdata]
             else
               step = template_step.dup
               step.targets = template_step.targets.dup
@@ -67,7 +174,7 @@ class DakeResolver
               step.context = template_step.context.dup
               step.context.merge! mdata.named_captures
               @analyzer.step_template_dict[template_step] ||= {}
-              @analyzer.step_template_dict[template_step][mdata.named_captures] = step
+              @analyzer.step_template_dict[template_step][mdata] = step
             end
             step.targets.map! do |file|
               if file.scheme.is_a? DakeScheme::Regex and file.scheme.path.match target_name
@@ -96,8 +203,8 @@ class DakeResolver
         end
         if template_step
           if @analyzer.step_template_dict[template_step] and
-             @analyzer.step_template_dict[template_step][mdata.named_captures]
-            step = @step_template_dict[template_step][mdata.named_captures]
+             @analyzer.step_template_dict[template_step][mdata]
+            step = @analyzer.step_template_dict[template_step][mdata]
           else
             step = template_step.dup
             step.targets = template_step.targets.dup
@@ -105,7 +212,7 @@ class DakeResolver
             step.context = template_step.context.dup
             step.context.merge! mdata.named_captures
             @analyzer.step_template_dict[template_step] ||= {}
-            @analyzer.step_template_dict[template_step][mdata.named_captures] = step
+            @analyzer.step_template_dict[template_step][mdata] = step
           end
           step.targets.map! do |file|
             if file.scheme.is_a? DakeScheme::Regex and file.scheme.path.match target_src
@@ -120,12 +227,7 @@ class DakeResolver
           @analyzer.file_target_dict[target_name] = step
           steps = [step]
         else
-          scheme = @analyzer.analyze_scheme(target_name, nil, nil, nil)
-          if optional or scheme.exist?
-            steps = []
-          else
-            raise "No step found for building file `#{target_name}'."
-          end
+          steps = []
         end
       end
     end
@@ -133,26 +235,25 @@ class DakeResolver
   end
 
   # resolve the dependency graph and generate step list for sequential execution
-  def resolve(target_dict, no_check=false)
+  def resolve(target_pairs)
     step_list = []
     visited = Set.new
     path_visited = Set.new
     leaf_steps = Set.new
     target_steps = Set.new
-    need_rebuild = Set.new
     succ_step_dict = {}
     dep_step_dict = {}
     dep_step_list = {}
     succ_target_dict = {}
 
-    target_dict.each do |target_name, target_opts|
+    target_pairs.each do |target_name, target_opts|
       if target_opts.tag
         dummy_step = Step.new([], [], [], {}, nil, nil, @analyzer.variable_dict, nil, nil)
         scheme = DakeScheme::Tag.new('@', target_name, dummy_step)
       else
         scheme = @analyzer.analyze_scheme(target_name, nil, nil, nil)
       end
-      dep_steps = find_steps(scheme, target_opts.tag, no_check)
+      dep_steps = find_steps(scheme, target_opts.tag)
       dep_steps.each do |dep_step|
         target = dep_step.targets.find { |target| target.scheme.path == scheme.path }
         succ_target_dict[dep_step] ||= Set.new
@@ -173,7 +274,7 @@ class DakeResolver
           dep_step_dict[step] = Set.new
           step.prerequisites.map! { |file| @analyzer.analyze_file(file, :prerequisites, step) }.flatten!
           step.prerequisites.each do |dep|
-            dep_steps = find_steps(dep.scheme, dep.tag, (dep.flag == '?' or no_check))
+            dep_steps = find_steps(dep.scheme, dep.tag)
             dep_steps.each do |dep_step|
               dep_step_dict[step] << dep_step
               succ_step_dict[dep_step] ||= Set.new
@@ -205,18 +306,7 @@ class DakeResolver
         end
       end
     end
-    step_list.each do |step|
-      if leaf_steps.include? step
-        need_rebuild << step if no_check or need_execute?(succ_target_dict[step], step)
-      else
-        if dep_step_dict[step].any? { |dep_step| need_rebuild.include? dep_step }
-          need_rebuild << step
-        else
-          need_rebuild << step if no_check or need_execute?(succ_target_dict[step], step)
-        end
-      end
-    end
     root_steps = target_steps.select { |step| succ_step_dict[step].empty? }.to_set
-    DepGraph.new(succ_step_dict, dep_step_dict, step_list, root_steps, leaf_steps, need_rebuild)
+    DepGraph.new(succ_step_dict, dep_step_dict, step_list, root_steps, leaf_steps, succ_target_dict)
   end
 end

data/lib/dake/scheme.rb

@@ -34,11 +34,12 @@ module DakeScheme
   class Local < Scheme
     PATTERN = ['local:']
     def initialize(scheme_part, path_part, step)
-      @src = path_part
       if path_part.start_with? '/'
        @path = path_part
+       @src = Pathname.new(path_part).relative_path_from(step.context['BASE'])
       else
         @path = File.expand_path(path_part, step.context['BASE'])
+        @src = path_part
       end
       @step = step
     end

data/lib/dake/version.rb

@@ -1,3 +1,3 @@
 module Dake
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dake
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - minor6th
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-02 00:00:00.000000000 Z
+date: 2019-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -180,6 +180,7 @@ files:
 - bin/console
 - bin/setup
 - dake.gemspec
+- examples/Dakefile
 - exe/dake
 - lib/dake.rb
 - lib/dake/analyzer.rb
@@ -193,7 +194,8 @@ files:
 - vim/ftdetect/dake.vim
 - vim/syntax/dake.vim
 homepage: https://github.com/minor6th/dake
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []