dake 0.1.0

data/lib/dake/scheme.rb

@@ -0,0 +1,85 @@
+module DakeScheme
+  class Scheme
+    PATTERN = ['']
+    attr_reader :path, :step, :src
+    def initialize(scheme_part, path_part, step)
+      @src = scheme_part + path_part
+      @path = path_part
+      @step = step
+    end
+    def checksum; end
+    def mtime; end
+    def exist?; false end
+  end
+
+  class Tag < Scheme
+    PATTERN = ['@']
+    def initialize(scheme_part, path_part, step)
+      @src = scheme_part + path_part
+      @path = path_part
+      @step = step
+      @step = step
+    end
+  end
+
+  class Regex < Scheme
+    PATTERN = ['^']
+    def initialize(scheme_part, path_part, step)
+      @src = path_part
+      @path = Regexp.compile("^#{path_part}$")
+      @step = step
+    end
+  end
+
+  class Local < Scheme
+    PATTERN = ['local:']
+    def initialize(scheme_part, path_part, step)
+      @src = path_part
+      if path_part.start_with? '/'
+       @path = path_part
+      else
+        @path = File.expand_path(path_part, step.context['BASE'])
+      end
+      @step = step
+    end
+
+    def mtime
+      File.mtime(@path)
+    end
+
+    def exist?
+      File.exist?(@path)
+    end
+  end
+
+  class HDFS < Scheme
+    PATTERN = ['hdfs:', 'afs:']
+    def initialize(scheme_part, path_part, step)
+      @src = scheme_part + path_part
+      @path = scheme_part + path_part
+      @step = step
+      if @step.context['HADOOP_HOME']
+        @hadoop_bin = "#{@step.context['HADOOP_HOME']}/hadoop/bin/hadoop"
+      elsif ENV['HADOOP_HOME']
+        @hadoop_bin = "#{ENV['HADOOP_HOME']}/hadoop/bin/hadoop"
+      else
+        raise "HADOOP_HOME not set." if not hadoop_home or hadoop_home.empty?
+      end
+      raise "#{@hadoop_bin} not found." unless File.exist? @hadoop_bin
+    end
+
+    def mtime
+      Time.at(`#{@hadoop_bin} fs -stat %Y #{@path}`.chomp.to_i / 1000)
+    end
+
+    def exist?
+      system("#{@hadoop_bin} fs -test -e #{@path}")
+    end
+  end
+
+  SchemeDict = {
+      'local:' => Local,
+      'hdfs:'  => HDFS,
+      'afs:'   => HDFS
+  }
+end

data/lib/dake/version.rb

@@ -0,0 +1,3 @@
+module Dake
+  VERSION = "0.1.0"
+end

data/lib/dake.rb

@@ -0,0 +1,13 @@
+require "dake/version"
+require 'dake/parser'
+require 'dake/analyzer'
+require 'dake/executor'
+require 'dake/protocol'
+require 'dake/scheme'
+require 'dake/resolver'
+require 'dake/database'
+
+module Dake
+  class Error < StandardError; end
+  TargetOption = Struct.new(:tag, :regex, :build_mode, :tree_mode)
+end

data/vim/ftdetect/dake.vim

@@ -0,0 +1,9 @@
+au BufNewFile,BufRead *.dake,Dakefile call SetDakeOptions()
+if !exists("*SetDakeOptions")
+    function SetDakeOptions()
+        set autoindent
+        set filetype=dake syntax=dake
+        setlocal shiftwidth=4 softtabstop=4
+        setlocal indentkeys=!^F,o,O
+    endfunction
+endif

data/vim/syntax/dake.vim

@@ -0,0 +1,97 @@
+"
+"  Vim syntax file for dake
+"  Based on https://bitbucket.org/larsyencken/vim-drake-syntax.git
+"  Language: dake
+"  Latest Revision: 2019-03-31
+"
+
+if exists("b:current_syntax")
+  finish
+endif
+
+syn include @Shell syntax/sh.vim
+unlet b:current_syntax
+
+syn include @Python syntax/python.vim
+unlet b:current_syntax
+
+syn include @Ruby syntax/ruby.vim
+unlet b:current_syntax
+
+syn include @Awk syntax/awk.vim
+unlet b:current_syntax
+
+syn include @R syntax/r.vim
+let b:current_syntax = 'dake'
+
+" Comments Tag Pattern Directives and Protocols
+syn match dakeTag /@[^ ,@"'[\]]\+\|"[^"]*"\|'[^']*'/ contained contains=dakeVariableRef
+syn match dakeDirective "^%[a-zA-Z_][a-zA-Z_]*"
+syn match dakeComment "#.*$" contains=dakeTodo
+syn keyword dakeTodo contained TODO NOTE FIXME XXX
+syn keyword dakeProtocol contained shell python ruby awk R
+
+" Variable definitions
+syn match dakeSetVariable "^[a-zA-Z_][a-zA-Z0-9_]* *|\?= *" contains=dakeVariable nextgroup=dakeString
+syn match dakeVariable "^[a-zA-Z_][a-zA-Z0-9_]*" contained
+syn match dakeString /[^ ,@"'[\]]\+\|"[^"]*"\|'[^']*'/ contained contains=dakeVariableRef containedin=dakeSetValue nextgroup=dakeString
+syn match dakeEscChar /\\\(x[0-9a-fA-F]\{1,2}\|u[0-9a-fA-F]\{1,4}\|.\)/ contained containedin=dakeString
+
+" Method blocks
+syn region dakeMethodBlock start="^[a-zA-Z_][a-zA-Z0-9_]*()" end="^$" contains=dakeMethodSignature
+syn match dakeMethodSignature "^[a-zA-Z_][a-zA-Z0-9_]*" contained nextgroup=dakeMethodBraces
+syn match dakeMethodBraces "()" contained nextgroup=dakeOptionListPy,dakeOptionListRb,dakeOptionListAwk,dakeOptionListR,dakeOptionListSh,dakeOptionList,dakeDefaultShBlock skipwhite skipnl
+
+" Variable references in strings
+syn region dakeVariableRef start='\$\[' end='\]' contained containedin=dakeString,dakeRbBlock,rubyString,dakePyBlock,pythonString,dakeAwkBlock,awkString,dakeRBlock,rString,dakeDefaultShBlock,dakeShBLock,shCmdParenRegion,shPattern,shDeref,shDerefSimple,shDoubleQuote,shExDoubleQuote,shSingleQuote,shExSingleQuote,shHereDoc,shHereString,shEcho contains=dakeVariableName
+syn match dakeVariableName "[a-zA-Z_][a-zA-Z0-9_]*" contained containedin=dakeVariableRef
+
+" Rule blocks
+syn region dakeBlock start="[^<#, ][^<#, ]*\(, [^<#, ][^<#, ]*\)* <-" end="^$" contains=dakeRule
+syn match dakeRule "[^<#, ].* <-\( *[^[<# ][^<# ]*\)*" contains=dakeTargets nextgroup=dakeOptionListPy,dakeOptionListRb,dakeOptionListAwk,dakeOptionListR,dakeOptionListSh,dakeOptionList,dakeDefaultShBlock skipwhite skipnl
+syn match dakeTargets "[^<#, ][^<#, ]*\(, [^<#, ][^<#, ]*\)*" contained nextgroup=dakeRuleIdentifier contains=dakeTag,dakeString,dakeTargetSep
+syn match dakeRuleIdentifier " <-" contained nextgroup=dakeSources
+syn match dakeTargetSep ", " contained containedin=dakeTargets
+syn region dakeOptionList matchgroup=Snip start=/\v \[/ end="\]" contained contains=dakeOption nextgroup=dakeShBlock skipwhite skipnl
+syn region dakeOptionListSh matchgroup=Snip start=/\v \[(shell|.* shell)/ end="\]" contained contains=dakeOption,dakeProtocol nextgroup=dakeShBlock skipwhite skipnl
+syn region dakeOptionListPy matchgroup=Snip start=/\v \[(python|.* python)/ end="\]" contained contains=dakeOption,dakeProtocol nextgroup=dakePyBlock skipwhite skipnl
+syn region dakeOptionListRb matchgroup=Snip start=/\v \[(ruby|.* ruby)/ end="\]" contained contains=dakeOption,dakeProtocol nextgroup=dakeRbBlock skipwhite skipnl
+syn region dakeOptionListAwk matchgroup=Snip start=/\v \[(awk|.* awk)/ end="\]" contained contains=dakeOption,dakeProtocol nextgroup=dakeAwkBlock skipwhite skipnl
+syn region dakeOptionListR matchgroup=Snip start=/\v \[(R|.* R)/ end="\]" contained contains=dakeOption,dakeProtocol nextgroup=dakeRBlock skipwhite skipnl
+syn match dakeOption '[a-zA-Z_][a-zA-Z0-9_]*:\|[+-][a-zA-Z_][a-zA-Z0-9_]*' contained nextgroup=dakeString
+
+hi link dakeComment Comment
+hi link dakeTodo rubyConstant
+hi link dakeProtocol PreProc
+hi link dakeVariable shVariable
+hi link dakeVariableName shDerefSimple
+hi link dakeSetIdentifier Delimiter
+hi link dakeRuleIdentifier Delimiter
+hi link dakeTag rubyConstant
+hi link dakeString String
+hi link dakeEscChar shEscape
+hi link VarBraces SpecialComment
+hi link dakeTargetSep Delimiter
+hi link dakeDirective PreProc
+hi link dakeOption Constant
+
+" Embedded shell region in block
+syn region dakeDefaultShBlock start='^\(#.*$\n\)*[ \t][ \t]*' end='^$' contained containedin=dakeBlock,dakeMethodBlock contains=@Shell
+syn region dakeShBlock start=/^\(#.*$\n\)*[ \t]/ end='^$' contained contains=@Shell
+syn region dakePyBlock start=/^\(#.*$\n\)*[ \t]/ end='^$' contained contains=@Python
+syn region dakeRbBlock start=/^\(#.*$\n\)*[ \t]/ end='^$' contained contains=@Ruby
+syn region dakeAwkBlock start=/^\(#.*$\n\)*[ \t]/ end='^$' contained contains=@Awk
+syn region dakeRBlock start=/^\(#.*$\n\)*[ \t]/ end='^$' contained contains=@R
+
+" Embedded shell regions in strings
+syn region shellBrackets matchgroup=SnipBraces start='\$(' end=')' containedin=dakeString contains=@Shell
+
+hi link Snip SpecialComment
+hi link SnipBraces SpecialComment
+hi link dakeVariableRef shDerefSimple
+hi link dakeMethodSignature Function
+hi link dakeMethodBraces SpecialComment
+
+" Syncing
+syn sync minlines=20 maxlines=200
+

metadata

@@ -0,0 +1,217 @@
+--- !ruby/object:Gem::Specification
+name: dake
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- minor6th
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-04-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: git
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: parslet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Dake is a data workflow tool inspired by Drake.
+email:
+- minor6th@outlook.com
+executables:
+- dake
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- dake.gemspec
+- exe/dake
+- lib/dake.rb
+- lib/dake/analyzer.rb
+- lib/dake/database.rb
+- lib/dake/executor.rb
+- lib/dake/parser.rb
+- lib/dake/protocol.rb
+- lib/dake/resolver.rb
+- lib/dake/scheme.rb
+- lib/dake/version.rb
+- vim/ftdetect/dake.vim
+- vim/syntax/dake.vim
+homepage: https://github.com/minor6th/dake
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Dake is a data workflow tool inspired by Drake.
+test_files: []