rake_hdfs 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1310a15bf3d66d81d1b6b51ff81c72222369100d
+  data.tar.gz: b59b9a58c2baac58adf6a0476607c4252b5a3b71
+SHA512:
+  metadata.gz: 335720d67d9b386cf63945bb36819b745be82ab1c8d9e6f1c3e96c85f36c93d9b0b1772bbac70a6fadc500bb0487fe8df5d05860a6360a38dc904f00b6f8bd76
+  data.tar.gz: 75311d15ac8fe3d460f3644ff32623ccb8242ec46764ae61c98bd206f82f90f224a08068227aa79f8de05aa3b6be13f6de721180d8fc615cced466b2146d64b0

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 2.3.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rake_hadoop.gemspec
+gemspec

data/README.md

@@ -0,0 +1,71 @@
+# RakeHdfs
+
+this gem makes rake run on hdfs file system. it's based on webhdfs.
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'rake_hdfs'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rake_hdfs
+
+## Usage
+
+```ruby
+require "webhdfs/fileutils"
+require "rake_hdfs"
+$dir = "/user/chenkovsky.chen"
+
+WebHDFS::FileUtils.set_server("localhost", 50070, "chenkovsky", nil)
+
+hdirectory "#{$dir}/tmp_dir"
+desc "test hdfs rake"
+hfile "#{$dir}/tmp_dir/tmp.txt" => ["#{$dir}/tmp_dir"] do
+  raise "tmp file should not exist." if hexist? "#{$dir}/tmp_dir/tmp.txt"
+  files = hls $dir
+  puts files
+  dir_mtime = hmtime $dir
+  puts dir_mtime
+  hcopy_from_local "tmp.txt", "#{$dir}/tmp_dir/tmp.txt"
+  hcopy_from_local_via_stream "tmp.txt", "#{$dir}/tmp_dir/tmp2.txt"
+
+  hcopy_to_local "#{$dir}/tmp_dir/tmp.txt", "tmp3.txt"
+
+  happend("#{$dir}/tmp_dir/tmp2.txt", "hahaha")
+
+  hmkdir "#{$dir}/tmp2_dir"
+
+  raise "tmp2_dir should not exist." if not hexist? "#{$dir}/tmp2_dir"
+
+  hrm "#{$dir}/tmp2_dir"
+  hmkdir "#{$dir}/tmp3_dir"
+  hcopy_from_local "tmp.txt", "#{$dir}/tmp3_dir/tmp.txt"
+
+  hrmr "#{$dir}/tmp3_dir"
+
+  hrename "#{$dir}/tmp_dir/tmp2.txt", "#{$dir}/tmp_dir/tmp4.txt"
+
+  hchmod 0755, "#{$dir}/tmp_dir/tmp4.txt"
+
+  puts (hstat "#{$dir}/tmp_dir/tmp4.txt")
+  raise "not correct uptodate" unless huptodate? "#{$dir}/tmp_dir/tmp4.txt", ["#{$dir}/tmp_dir/tmp.txt"]
+end
+```
+
+## Contributing
+
+1. Fork it ( https://github.com/chenkovsky/rake_hdfs/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+gem "webhdfs"

data/lib/rake_hdfs/hdfs_dsl_definition.rb

@@ -0,0 +1,24 @@
+require 'rake/file_utils_ext'
+module Rake
+    module DSL
+        def hfile(*args, &block) # :doc:
+          Rake::HDFSFileTask.define_task(*args, &block)
+        end
+
+        def hfile_create(*args, &block)
+          Rake::HDFSFileCreationTask.define_task(*args, &block)
+        end
+
+        def hdirectory(*args, &block) # :doc:
+          result = hfile_create(*args, &block)
+          dir, _ = *Rake.application.resolve_args(args)
+          dir = Rake.from_pathname(dir)
+          Rake.each_dir_parent(dir) do |d|
+            hfile_create d do |t|
+              hmkdir_p t.name unless WebHDFS::FileUtils.exist?(t.name)
+            end
+          end
+          result
+        end
+    end
+end

data/lib/rake_hdfs/hdfs_file_creation_task.rb

@@ -0,0 +1,23 @@
+require 'rake/file_task'
+require 'rake/early_time'
+
+module Rake
+
+  # A FileCreationTask is a file task that when used as a dependency will be
+  # needed if and only if the file has not been created.  Once created, it is
+  # not re-triggered if any of its dependencies are newer, nor does trigger
+  # any rebuilds of tasks that depend on it whenever it is updated.
+  #
+  class HDFSFileCreationTask < FileTask
+    # Is this file task needed?  Yes if it doesn't exist.
+    def needed?
+      ! WebHDFS::FileUtils.exist?(name)
+    end
+
+    # Time stamp for file creation task.  This time stamp is earlier
+    # than any other time stamp.
+    def timestamp
+      Rake::EARLY
+    end
+  end
+end

data/lib/rake_hdfs/hdfs_file_task.rb

@@ -0,0 +1,46 @@
+require 'rake/task.rb'
+require 'rake/early_time'
+
+module Rake
+
+  # A FileTask is a task that includes time based dependencies.  If any of a
+  # FileTask's prerequisites have a timestamp that is later than the file
+  # represented by this task, then the file must be rebuilt (using the
+  # supplied actions).
+  #
+  class HDFSFileTask < Task
+
+    # Is this file task needed?  Yes if it doesn't exist, or if its time stamp
+    # is out of date.
+    def needed?
+      ! WebHDFS::FileUtils.exist?(name) || out_of_date?(timestamp) || @application.options.build_all
+    end
+
+    # Time stamp for file task.
+    def timestamp
+      if WebHDFS::FileUtils.exist?(name)
+        WebHDFS::FileUtils.mtime(name.to_s)
+      else
+        Rake::LATE
+      end
+    end
+
+    private
+
+    # Are there any prerequisites with a later time than the given time stamp?
+    def out_of_date?(stamp)
+      @prerequisites.any? { |n| application[n, @scope].timestamp > stamp }
+    end
+
+    # ----------------------------------------------------------------
+    # Task class methods.
+    #
+    class << self
+      # Apply the scope to the task name according to the rules for this kind
+      # of task.  File based tasks ignore the scope when creating the name.
+      def scope_name(scope, task_name)
+        Rake.from_pathname(task_name)
+      end
+    end
+  end
+end

data/lib/rake_hdfs/hdfs_fileutils.rb

@@ -0,0 +1,203 @@
+require "webhdfs/fileutils"
+module WebHDFS
+  module FileUtils
+    def exist?(file, options={})
+      begin
+        client.stat(file, options)
+        return true
+      rescue FileNotFoundError => e
+        return false
+      end
+    end
+    module_function :exist?
+    def ls(path, options={})
+      opts = options.dup
+      fu_log "ls #{path}" if opts.delete(:verbose)
+      client.list(path, options)
+    end
+    module_function :ls
+
+    def stat(path, options={})
+      opts = options.dup
+      fu_log "stat #{path}" if opts.delete(:verbose)
+      client.stat(path, options)
+    end
+    module_function :stat
+
+    def uptodate?(new, old_list, options = {})
+      return false unless exist?(new)
+      new_time = mtime(new)
+      old_list.each do |old|
+        if exist?(old)
+          return false unless new_time > mtime(old)
+        end
+      end
+      true
+    end
+    module_function :uptodate?
+
+    def mtime(path, options = {})
+      file_stat = stat(path, options)
+      file_stat["modificationTime"]
+    end
+    module_function :mtime
+  end
+end
+
+module FileUtils
+  def hdfs_copy_from_local(file, path, options={})
+    WebHDFS::FileUtils.copy_from_local(file, path, options)
+  end
+  module_function :hdfs_copy_from_local
+
+  alias :hcopy_from_local :hdfs_copy_from_local
+  module_function :hcopy_from_local
+
+  def hdfs_copy_from_local_via_stream(file, path, options={})
+    WebHDFS::FileUtils.copy_from_local_via_stream(file, path, options)
+  end
+  module_function :hdfs_copy_from_local_via_stream
+  alias :hcopy_from_local_via_stream :hdfs_copy_from_local_via_stream
+  module_function :hcopy_from_local_via_stream
+
+  def hdfs_copy_to_local(path, file, options={})
+    WebHDFS::FileUtils.copy_to_local(path, file, options)
+  end
+
+  module_function :hdfs_copy_to_local
+
+  alias :hcopy_to_local :hdfs_copy_to_local
+  module_function :hcopy_to_local
+
+  def hdfs_append(path, body, options={})
+    WebHDFS::FileUtils.append(path,body,options)
+  end
+  module_function :hdfs_append
+
+  alias :happend :hdfs_append
+  module_function :happend
+
+  def hdfs_mkdir(list, options={})
+    WebHDFS::FileUtils.mkdir(list, options)
+  end
+
+  module_function :hdfs_mkdir
+
+  alias :hmkdir :hdfs_mkdir
+  module_function :hmkdir
+
+  alias :hdfs_mkdir_p :hdfs_mkdir
+  module_function :hdfs_mkdir_p
+
+  alias :hmkdir_p :hdfs_mkdir
+  module_function :hmkdir_p
+
+  def hdfs_rm(list, options={})
+    WebHDFS::FileUtils.rm(list, options)
+  end
+  module_function :hdfs_rm
+  alias :hrm :hdfs_rm
+  module_function :hrm
+
+  def hdfs_rmr(list, options={})
+    WebHDFS::FileUtils.rmr(list, options)
+  end
+  module_function :hdfs_rmr
+
+  alias :hrmr :hdfs_rmr
+  module_function :hrmr
+
+  def hdfs_rename(src, dst, options={})
+    WebHDFS::FileUtils.rename(src, dst, options)
+  end
+
+  module_function :hdfs_rename
+
+  alias :hrename :hdfs_rename
+  module_function :hrename
+
+  def hdfs_chmod(mode, list, options={})
+    WebHDFS::FileUtils.chmod(mode, list, options)
+  end
+
+  module_function :hdfs_chmod
+
+  alias :hchmod :hdfs_chmod
+  module_function :hchmod
+
+  def hdfs_chown(user, group, list, options={})
+    WebHDFS::FileUtils.chown(user, group, list, options)
+  end
+
+  module_function :hdfs_chown
+
+  alias :hchown :hdfs_chown
+  module_function :hchown
+
+  def hdfs_set_repl_factor(list, num, options={})
+    WebHDFS::FileUtils.set_repl_factor(list, num, options)
+  end
+  module_function :hdfs_set_repl_factor
+
+  alias :hset_repl_factor :hdfs_set_repl_factor
+  module_function :hset_repl_factor
+
+  def hdfs_set_atime(list, time, options={})
+    WebHDFS::FileUtils.set_atime(list, time, options)
+  end
+
+  module_function :hdfs_set_atime
+
+  alias :hset_atime :hdfs_set_atime
+  module_function :hset_atime
+
+  def  hdfs_set_mtime(list, time, options={})
+    WebHDFS::FileUtils.set_mtime(list, time, options)
+  end
+
+  module_function :hdfs_set_mtime
+
+  alias :hset_mtime :hdfs_set_mtime
+  module_function :hset_mtime
+
+  def hdfs_ls(path, options={})
+    WebHDFS::FileUtils.ls(path, options)
+  end
+
+  module_function :hdfs_ls
+  alias :hls :hdfs_ls
+  module_function :hls
+
+  def hdfs_stat(path, options={})
+    WebHDFS::FileUtils.stat(path, options)
+  end
+
+  module_function :hdfs_stat
+  alias :hstat :hdfs_stat
+  module_function :hstat
+
+  def hdfs_exist?(path, options={})
+     WebHDFS::FileUtils.exist?(path, options)
+  end
+
+  module_function :hdfs_exist?
+  alias :hexist? :hdfs_exist?
+  module_function :hexist?
+
+  def hdfs_uptodate?(new, old_list, options={})
+     WebHDFS::FileUtils.uptodate?(new, old_list, options)
+  end
+
+  module_function :hdfs_uptodate?
+  alias :huptodate? :hdfs_uptodate?
+  module_function :huptodate?
+
+  def hdfs_mtime(path, options={})
+    WebHDFS::FileUtils.mtime(path, options)
+  end
+
+  module_function :hdfs_mtime
+  alias :hmtime :hdfs_mtime
+  module_function :hmtime
+
+end

data/lib/rake_hdfs/version.rb

@@ -0,0 +1,3 @@
+module RakeHDFS
+  VERSION = "0.1.1"
+end

data/lib/rake_hdfs.rb

@@ -0,0 +1,5 @@
+require "rake_hdfs/version"
+require "rake_hdfs/hdfs_dsl_definition"
+require "rake_hdfs/hdfs_file_creation_task"
+require "rake_hdfs/hdfs_file_task"
+require "rake_hdfs/hdfs_fileutils"

data/rake_hdfs.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rake_hdfs/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rake_hdfs"
+  spec.version       = RakeHDFS::VERSION
+  spec.authors       = ["chenkovsky.chen"]
+  spec.email         = ["chenkovsky.chen@cootek.cn"]
+
+  spec.summary       = %q{some rake dsl for hadoop}
+  spec.description   = %q{some rake dsl for hadoop}
+  spec.homepage      = "http://github.com/chenkovsky/rake_hdfs"
+
+  # Prevent pushing this gem to RubyGems.org by setting 'allowed_push_host', or
+  # delete this section to allow pushing this gem to any host.
+  if spec.respond_to?(:metadata)
+    spec.metadata['allowed_push_host'] = "https://rubygems.org"
+  else
+    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
+  end
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.9"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_dependency "webhdfs", "~> 0.8.0"
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: rake_hdfs
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- chenkovsky.chen
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-03-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.9'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: webhdfs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+description: some rake dsl for hadoop
+email:
+- chenkovsky.chen@cootek.cn
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- README.md
+- Rakefile
+- lib/rake_hdfs.rb
+- lib/rake_hdfs/hdfs_dsl_definition.rb
+- lib/rake_hdfs/hdfs_file_creation_task.rb
+- lib/rake_hdfs/hdfs_file_task.rb
+- lib/rake_hdfs/hdfs_fileutils.rb
+- lib/rake_hdfs/version.rb
+- rake_hdfs.gemspec
+homepage: http://github.com/chenkovsky/rake_hdfs
+licenses: []
+metadata:
+  allowed_push_host: https://rubygems.org
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: some rake dsl for hadoop
+test_files: []
+has_rdoc: