jrb-hdfs 0.1.1-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c5b78dd9b6ad1eb8250e1233b5e751b9843f512
+  data.tar.gz: bf97eed99c7b6fcac7ef38664b2828c2f40fb880
+SHA512:
+  metadata.gz: 96f8b9d89177cccaf72a4f41e12679936600ac0386a92d50a750f25f2a8da4f41a57f28c13966085d988a8e739a668acc38ebe972be21e7a6ea21b17fa510c71
+  data.tar.gz: 9fc2cce2c32ae33bf79203c437036f40a2d31105fe7c06cc715f559ec91a5084ccc4ad3ecd4b260d2fc76ec27d8a6ffc2e628e594198dd88fdf785e8ea2b6616

data/lib/jrb-hdfs_jars.rb

@@ -0,0 +1,71 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+require 'jar_dependencies'
+
+require_jar( 'com.google.protobuf', 'protobuf-java', '2.5.0' )
+require_jar( 'com.sun.jersey', 'jersey-json', '1.9' )
+require_jar( 'org.apache.hadoop', 'hadoop-annotations', '2.7.1' )
+require_jar( 'org.apache.directory.api', 'api-asn1-api', '1.0.0-M20' )
+require_jar( 'org.codehaus.jackson', 'jackson-xc', '1.8.3' )
+require_jar( 'org.apache.directory.api', 'api-util', '1.0.0-M20' )
+require_jar( 'org.tukaani', 'xz', '1.0' )
+require_jar( 'org.apache.httpcomponents', 'httpclient', '4.1.2' )
+require_jar( 'commons-lang', 'commons-lang', '2.6' )
+require_jar( 'xml-apis', 'xml-apis', '1.3.04' )
+require_jar( 'com.sun.jersey', 'jersey-core', '1.9' )
+require_jar( 'org.codehaus.jackson', 'jackson-mapper-asl', '1.9.13' )
+require_jar( 'org.fusesource.leveldbjni', 'leveldbjni-all', '1.8' )
+require_jar( 'commons-beanutils', 'commons-beanutils', '1.7.0' )
+require_jar( 'commons-daemon', 'commons-daemon', '1.0.13' )
+require_jar( 'org.mortbay.jetty', 'jetty-util', '6.1.26' )
+require_jar( 'log4j', 'log4j', '1.2.17' )
+require_jar( 'commons-httpclient', 'commons-httpclient', '3.1' )
+require_jar( 'org.apache.hadoop', 'hadoop-common', '2.7.1' )
+require_jar( 'com.google.guava', 'guava', '11.0.2' )
+require_jar( 'com.jcraft', 'jsch', '0.1.42' )
+require_jar( 'commons-digester', 'commons-digester', '1.8' )
+require_jar( 'javax.activation', 'activation', '1.1' )
+require_jar( 'com.thoughtworks.paranamer', 'paranamer', '2.3' )
+require_jar( 'io.netty', 'netty', '3.7.0.Final' )
+require_jar( 'commons-logging', 'commons-logging', '1.1.3' )
+require_jar( 'com.google.code.findbugs', 'jsr305', '3.0.0' )
+require_jar( 'xmlenc', 'xmlenc', '0.52' )
+require_jar( 'commons-beanutils', 'commons-beanutils-core', '1.8.0' )
+require_jar( 'org.apache.curator', 'curator-recipes', '2.7.1' )
+require_jar( 'org.slf4j', 'slf4j-api', '1.7.10' )
+require_jar( 'commons-cli', 'commons-cli', '1.2' )
+require_jar( 'org.apache.directory.server', 'apacheds-kerberos-codec', '2.0.0-M15' )
+require_jar( 'javax.xml.stream', 'stax-api', '1.0-2' )
+require( '/usr/lib/jvm/java-7-openjdk-amd64/jre/../lib/tools.jar' )
+require_jar( 'org.apache.zookeeper', 'zookeeper', '3.4.6' )
+require_jar( 'org.apache.avro', 'avro', '1.7.4' )
+require_jar( 'org.apache.directory.server', 'apacheds-i18n', '2.0.0-M15' )
+require_jar( 'javax.xml.bind', 'jaxb-api', '2.2.2' )
+require_jar( 'org.codehaus.jettison', 'jettison', '1.1' )
+require_jar( 'org.apache.hadoop', 'hadoop-hdfs', '2.7.1' )
+require_jar( 'org.codehaus.jackson', 'jackson-jaxrs', '1.8.3' )
+require_jar( 'org.apache.commons', 'commons-compress', '1.4.1' )
+require_jar( 'io.netty', 'netty-all', '4.0.23.Final' )
+require_jar( 'javax.servlet', 'servlet-api', '2.5' )
+require_jar( 'commons-codec', 'commons-codec', '1.4' )
+require_jar( 'commons-collections', 'commons-collections', '3.2.1' )
+require_jar( 'xerces', 'xercesImpl', '2.9.1' )
+require_jar( 'org.apache.hadoop', 'hadoop-auth', '2.7.1' )
+require_jar( 'org.apache.curator', 'curator-framework', '2.7.1' )
+require_jar( 'org.apache.commons', 'commons-math3', '3.1.1' )
+require_jar( 'commons-configuration', 'commons-configuration', '1.6' )
+require_jar( 'org.apache.curator', 'curator-client', '2.7.1' )
+require_jar( 'org.codehaus.jackson', 'jackson-core-asl', '1.9.13' )
+require_jar( 'commons-io', 'commons-io', '2.4' )
+require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.10' )
+require_jar( 'com.sun.xml.bind', 'jaxb-impl', '2.2.3-1' )
+require_jar( 'javax.servlet.jsp', 'jsp-api', '2.1' )
+require_jar( 'net.java.dev.jets3t', 'jets3t', '0.9.0' )
+require_jar( 'org.apache.httpcomponents', 'httpcore', '4.1.2' )
+require_jar( 'commons-net', 'commons-net', '3.1' )
+require_jar( 'com.google.code.gson', 'gson', '2.2.4' )
+require_jar( 'org.xerial.snappy', 'snappy-java', '1.0.4.1' )
+require_jar( 'com.jamesmurty.utils', 'java-xmlbuilder', '0.4' )
+require_jar( 'asm', 'asm', '3.1' )
+require_jar( 'org.mortbay.jetty', 'jetty', '6.1.26' )
+require_jar( 'com.sun.jersey', 'jersey-server', '1.9' )
+require_jar( 'org.apache.htrace', 'htrace-core', '3.1.0-incubating' )

data/lib/jrb/hdfs.rb

@@ -0,0 +1,2 @@
+require 'jrb-hdfs_jars.rb'
+require "jrb/hdfs/client"

data/lib/jrb/hdfs/client.rb

@@ -0,0 +1,114 @@
+require "jrb/hdfs/version"
+
+module Jrb
+  module Hdfs
+    class Client
+      java_import 'org.apache.hadoop.fs.Path'
+      java_import 'org.apache.hadoop.security.UserGroupInformation'
+      java_import 'org.apache.hadoop.conf.Configuration'
+      java_import 'java.net.URI'
+      java_import 'org.apache.hadoop.fs.FileUtil'
+      # Constructor
+      #
+      # @param [String]uri uri of hdfs namenode
+      # @param [String]conf_path configure dir of hadoop
+      # @param [Hash]opts
+      #   [Array[String]]   configure file that to be added into hdfs_conf resource
+      #   [boolean]use_kerberos  if use kerberos
+      #   [String]kerberos_username
+      #   [String]kerberos_keytab_path
+      def initialize(uri, conf_path, opts={})
+        @uri = uri
+        @hdfs_conf = Configuration.new
+        @hdfs_conf.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem")
+
+        conf_files = opts[:conf_files] || ['core-site.xml', 'hdfs-site.xml']
+        conf_files.each do |config_file|
+          @hdfs_conf.add_resource(Path.new(File.join(conf_path, config_file)))
+        end
+
+        if opts[:use_kerberos] == true
+          @hdfs_conf.set("hadoop.security.authentication", "kerberos")
+          UserGroupInformation.setConfiguration(@hdfs_conf)
+          UserGroupInformation.loginUserFromKeytab(opts[:kerberos_username], opts[:kerberos_keytab_path])
+        end
+
+        @hdfs = org.apache.hadoop.fs.FileSystem.get(URI.create(uri), @hdfs_conf)
+      end
+
+      # Get list of hdfs entry
+      #
+      # @param [String]path
+      # @param [Boolean]recursively
+      #
+      # @return [Array[org.apache.hadoop.fs.Path]]
+      def list(path, recursively=false)
+        if recursively
+          paths = []
+          dir_itr = @hdfs.listFiles(Path.new(path), true)
+
+          while dir_itr.hasNext
+            next_path = dir_itr.next.getPath
+            paths << next_path
+          end
+          paths
+        else
+          file_status = @hdfs.listStatus(Path.new(path))
+          FileUtil.stat2Paths(file_status)
+        end
+      end
+
+      # Copy a hdfs file to local fs
+      #
+      # @param [String]hdfs_src
+      # @param [String]local_dst
+      #
+      # @return [Boolean]
+      def copy_to_local(hdfs_src, local_dst)
+        return false unless exists?(hdfs_src)
+        begin
+          @hdfs.copy_to_local_file(false, Path.new(hdfs_src), Path.new(local_dst), true)
+          true
+        rescue
+          false
+        end
+      end
+
+      # Check if entry exists in hdfs
+      #
+      # @param [String]path
+      #
+      # @return [Boolean]
+      def exists?(path)
+        @hdfs.exists(Path.new(path))
+      end
+
+      # Remove a hdfs entry
+      #
+      # @param [String]path
+      # @param [Boolean]recursively
+      #
+      # @return [Boolean]
+      def rm(path, recursively=false)
+        return false unless exists?(path)
+        @hdfs.delete(Path.new(path), recursively)
+      end
+
+      # Put local entry into hdfs
+      #
+      # @param [String]local_src
+      # @param [String]hdfs_dst
+      #
+      # @return [Boolean]
+      def put(local_src, hdfs_dst)
+        return false unless File.exists?(local_src)
+        begin
+          @hdfs.copyFromLocalFile(false, true, Path.new(local_src), Path.new(hdfs_dst))
+          true
+        rescue
+          false
+        end
+      end
+    end
+  end
+end

data/lib/jrb/hdfs/version.rb

@@ -0,0 +1,5 @@
+module Jrb
+  module Hdfs
+    VERSION = "0.1.1"
+  end
+end

metadata

@@ -0,0 +1,174 @@
+--- !ruby/object:Gem::Specification
+name: jrb-hdfs
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: java
+authors:
+- Yuli Mo
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2016-04-27 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jar-dependencies
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.2
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: minitest
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  prerelease: false
+  type: :development
+description: hdfs client for jruby.
+email:
+- lizz@lizz.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/asm/asm/3.1/asm-3.1.jar
+- lib/com/google/code/findbugs/jsr305/3.0.0/jsr305-3.0.0.jar
+- lib/com/google/code/gson/gson/2.2.4/gson-2.2.4.jar
+- lib/com/google/guava/guava/11.0.2/guava-11.0.2.jar
+- lib/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar
+- lib/com/jamesmurty/utils/java-xmlbuilder/0.4/java-xmlbuilder-0.4.jar
+- lib/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar
+- lib/com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar
+- lib/com/sun/jersey/jersey-json/1.9/jersey-json-1.9.jar
+- lib/com/sun/jersey/jersey-server/1.9/jersey-server-1.9.jar
+- lib/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar
+- lib/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar
+- lib/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar
+- lib/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar
+- lib/commons-cli/commons-cli/1.2/commons-cli-1.2.jar
+- lib/commons-codec/commons-codec/1.4/commons-codec-1.4.jar
+- lib/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar
+- lib/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar
+- lib/commons-daemon/commons-daemon/1.0.13/commons-daemon-1.0.13.jar
+- lib/commons-digester/commons-digester/1.8/commons-digester-1.8.jar
+- lib/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar
+- lib/commons-io/commons-io/2.4/commons-io-2.4.jar
+- lib/commons-lang/commons-lang/2.6/commons-lang-2.6.jar
+- lib/commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar
+- lib/commons-net/commons-net/3.1/commons-net-3.1.jar
+- lib/io/netty/netty-all/4.0.23.Final/netty-all-4.0.23.Final.jar
+- lib/io/netty/netty/3.7.0.Final/netty-3.7.0.Final.jar
+- lib/javax/activation/activation/1.1/activation-1.1.jar
+- lib/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar
+- lib/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar
+- lib/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar
+- lib/javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar
+- lib/jrb-hdfs_jars.rb
+- lib/jrb/hdfs.rb
+- lib/jrb/hdfs/client.rb
+- lib/jrb/hdfs/version.rb
+- lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
+- lib/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar
+- lib/org/apache/avro/avro/1.7.4/avro-1.7.4.jar
+- lib/org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar
+- lib/org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar
+- lib/org/apache/curator/curator-client/2.7.1/curator-client-2.7.1.jar
+- lib/org/apache/curator/curator-framework/2.7.1/curator-framework-2.7.1.jar
+- lib/org/apache/curator/curator-recipes/2.7.1/curator-recipes-2.7.1.jar
+- lib/org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar
+- lib/org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar
+- lib/org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar
+- lib/org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar
+- lib/org/apache/hadoop/hadoop-annotations/2.7.1/hadoop-annotations-2.7.1.jar
+- lib/org/apache/hadoop/hadoop-auth/2.7.1/hadoop-auth-2.7.1.jar
+- lib/org/apache/hadoop/hadoop-common/2.7.1/hadoop-common-2.7.1.jar
+- lib/org/apache/hadoop/hadoop-hdfs/2.7.1/hadoop-hdfs-2.7.1.jar
+- lib/org/apache/htrace/htrace-core/3.1.0-incubating/htrace-core-3.1.0-incubating.jar
+- lib/org/apache/httpcomponents/httpclient/4.1.2/httpclient-4.1.2.jar
+- lib/org/apache/httpcomponents/httpcore/4.1.2/httpcore-4.1.2.jar
+- lib/org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar
+- lib/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar
+- lib/org/codehaus/jackson/jackson-jaxrs/1.8.3/jackson-jaxrs-1.8.3.jar
+- lib/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar
+- lib/org/codehaus/jackson/jackson-xc/1.8.3/jackson-xc-1.8.3.jar
+- lib/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar
+- lib/org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar
+- lib/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar
+- lib/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar
+- lib/org/slf4j/slf4j-api/1.7.10/slf4j-api-1.7.10.jar
+- lib/org/slf4j/slf4j-log4j12/1.7.10/slf4j-log4j12-1.7.10.jar
+- lib/org/tukaani/xz/1.0/xz-1.0.jar
+- lib/org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar
+- lib/xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar
+- lib/xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar
+- lib/xmlenc/xmlenc/0.52/xmlenc-0.52.jar
+homepage: https://github.com/myl2821/jrb-hdfs
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'org.apache.hadoop:hadoop-hdfs', '2.7.1'
+- jar 'org.apache.hadoop:hadoop-common', '2.7.1'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.10'
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: hdfs client for jruby.
+test_files: []