triglav-agent-hdfs 1.0.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: eb35b92839db390883b29d44368580291c010b41
+  data.tar.gz: cb68b718595ca529dd01dc59eed23b31233fc1ee
+SHA512:
+  metadata.gz: b98f2a7fe15dfca2bacd1a055a63e08033e14d1815b73b75feef605d842f255f3e933247ea420c05ccfa3e995c5411f779cd168c8c228d5c3844c24443f14ea7
+  data.tar.gz: fda052f21bbcd0eb0cc6060391c8f1f131d389cad086519fee5ca28ead0c2c50b530589903ffc0a126d64a632795ca509cd067e0100f9449dcce9e279055d602

data/.gitignore

@@ -0,0 +1,34 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.env
+/config.yml
+/status.yml
+/token.yml
+/.ruby-version
+/vendor/
+/lib/com/
+/lib/commons-beanutils/
+/lib/commons-cli/
+/lib/commons-codec/
+/lib/commons-collections/
+/lib/commons-configuration/
+/lib/commons-digester/
+/lib/commons-httpclient/
+/lib/commons-io/
+/lib/commons-lang/
+/lib/commons-logging/
+/lib/commons-net/
+/lib/io/
+/lib/javax/
+/lib/log4j/
+/lib/org/
+/lib/xerces/
+/lib/xml-apis/
+/lib/xmlenc/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,14 @@
+language: ruby
+cache: bundler
+rvm:
+  - jruby-9.1.5.0
+  - jruby-head
+jdk:
+  - oraclejdk8
+env:
+  - JAR_DEPENDENCIES_VERSION=0.3.5
+before_install:
+  - gem install bundler
+matrix:
+  allow_failures:
+    - rvm: jruby-head

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 1.0.0 (2017-04-14)
+
+initial release

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,49 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, and in the interest of
+fostering an open and welcoming community, we pledge to respect all people who
+contribute through reporting issues, posting feature requests, updating
+documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free
+experience for everyone, regardless of level of experience, gender, gender
+identity and expression, sexual orientation, disability, personal appearance,
+body size, race, ethnicity, age, religion, or nationality.
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery
+* Personal attacks
+* Trolling or insulting/derogatory comments
+* Public or private harassment
+* Publishing other's private information, such as physical or electronic
+  addresses, without explicit permission
+* Other unethical or unprofessional conduct
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+By adopting this Code of Conduct, project maintainers commit themselves to
+fairly and consistently applying these principles to every aspect of managing
+this project. Project maintainers who do not follow or enforce the Code of
+Conduct may be permanently removed from the project team.
+
+This code of conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting a project maintainer at sonots@gmail.com. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. Maintainers are
+obligated to maintain confidentiality with regard to the reporter of an
+incident.
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 1.3.0, available at
+[http://contributor-covenant.org/version/1/3/0/][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/3/0/

data/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+
+gemspec
+gem 'triglav_client', git: 'https://github.com/triglav-dataflow/triglav-client-ruby'
+gem 'triglav-agent', git: 'https://github.com/triglav-dataflow/triglav-agent-framework-ruby'
+gem 'pry-nav'

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Triglav Team
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,134 @@
+# Triglav::Agent::Hdfs
+
+Triglav Agent for Hdfs
+
+## Requirements
+
+* JRuby >= 9.1.5.0
+* Java >= 1.8.0_45
+
+
+## Prerequisites
+
+* HDFS path to be monitored must be created or modified atomically. To modify HDFS path atomically, use either of following strategies for example:
+  * Create a tmp directory and copy files into the directory, then move to the target path
+  * Create a marker file such as `_SUCCESS` after copying is done, and monitor the `_SUCESSES` file
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'triglav-agent-hdfs'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install triglav-agent-hdfs
+
+## CLI
+
+```
+Usage: triglav-agent-hdfs [options]
+    -c, --config VALUE               Config file (default: config.yml)
+    -s, --status VALUE               Status stroage file (default: status.yml)
+    -t, --token VALUE                Triglav access token storage file (default: token.yml)
+        --dotenv                     Load environment variables from .env file (default: false)
+    -h, --help                       help
+        --log VALUE                  Log path (default: STDOUT)
+        --log-level VALUE            Log level (default: info)
+```
+
+Run as:
+
+```
+TRIGLAV_ENV=development bundle exec triglav-agent-hdfs --dotenv -c config.yml
+```
+
+## Configuration
+
+Prepare config.yml as [example/config.yml](./example/config.yml).
+
+You can use erb template. You may load environment variables from .env file with `--dotenv` option.
+
+### serverengine section
+
+You can specify any [serverengine](https://github.com/fluent/serverengine) options at this section
+
+### triglav section
+
+Specify triglav api url, and a credential to authenticate.
+
+The access token obtained is stored into a token storage file (--token option).
+
+### hdfs section
+
+This section is the special section for triglav-agent-hdfs.
+
+* **monitor_interval**: The interval to watch tables (number, default: 60)
+* **connection_info**: key-value pairs of hdfs connection info where keys are resource URI pattern in regular expression, and values are connection information
+
+### Specification of Resource URI
+
+Resource URI must be a form of:
+
+```
+hdfs://{namespace}/#{path}
+```
+
+Path accepts `strftime` format such as `%Y-%m-%d`.
+
+## How it behaves
+
+1. Authenticate with triglav
+  * Store the access token into the token storage file
+  * Read the token from the token storage file next time
+  * Refresh the access token if it is expired
+2. Repeat followings in `monitor_interval` seconds:
+3. Obtain resource (table) lists of the specified prefix (keys of connection_info) from triglav.
+4. Connect to hdfs with an appropriate connection info for a resource uri, and find tables which are newer than last check.
+5. Store checking information into the status storage file for the next time check.
+
+## Development
+
+### Prepare
+
+```
+bundle
+bundle exec rake vendor_jars
+```
+
+```
+./prepare.sh
+```
+
+Edit `.env` file or `config.yml` file directly.
+
+### Start
+
+Start up triglav api on localhost.
+
+Run triglav-agent-hdfs as:
+
+```
+TRIGLAV_ENV=development bundle exec triglav-agent-hdfs --dotenv --debug -c example/config.yml
+```
+
+The debug mode with --debug option ignores the `last_modification_time` value in status file.
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/triglav-agent-hdfs/triglav-agent-hdfs. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+
+## ToDo
+
+* prepare mocks of both triglav and hdfs for tests

data/Rakefile

@@ -0,0 +1,16 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+desc 'Run test_unit based test'
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = Dir["test/**/test_*.rb"]
+  t.verbose = false
+  t.warning = false
+end
+task :default => :test
+
+task :clean_jars do
+  require 'fileutils'
+  Dir['lib/*'].reject {|_| _.include?('triglav') }.each {|_| FileUtils.rm_r(_) }
+end

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "triglav/agent/hdfs"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/example/config.yml

@@ -0,0 +1,41 @@
+defaults: &defaults
+  serverengine:
+    log: 'STDOUT'
+    log_level: 'debug'
+    log_rotate_age: 5
+    log_rotate_size: 10485760
+  triglav:
+    url: <%= ENV['TRIGLAV_URL'] || 'http://localhost:7800' %>
+    credential:
+      username: <%= ENV['TRIGLAV_USERNAME'] || 'triglav_test' %>
+      password: <%= ENV['TRIGLAV_PASSWORD'] || 'triglav_test' %>
+      authenticator: local
+    timeout: 60
+    debugging: false
+    retries: 3
+    retry_interval: 3 # sec
+  hdfs:
+    monitor_interval: 5
+    parallel:
+      size: 1 # default: 1
+      type: 'thread' # thread or process. default: thread
+    connection_pool:
+      size: 1 # default: same with parallel.size
+      timeout: 60 # sec. default: 60
+    connection_info:
+      "hdfs://":
+        config_files:
+          # - /etc/hadoop/conf/core-site.xml
+          # - /etc/hadoop/conf/hdfs-site.xml
+        config:
+          fs.defaultFS: <%= "hdfs://#{ENV['HDFS_HOST']}:#{ENV['HDFS_PORT']}" %>
+          dfs.replication: 1
+          fs.hdfs.impl: 'org.apache.hadoop.hdfs.DistributedFileSystem'
+          fs.file.impl: 'org.apache.hadoop.fs.LocalFileSystem'
+        doas: hadoop
+
+development:
+  <<: *defaults
+
+test:
+  <<: *defaults

data/example/example.env

@@ -0,0 +1,5 @@
+TRIGLAV_URL=http://localhost:7800
+TRIGLAV_USERNAME=triglav_test
+TRIGLAV_PASSWORD=triglav_test
+HDFS_HOST=xxx.xxx.xxx.xxx
+HDFS_PORT=8020

data/exe/triglav-agent-hdfs

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+
+require 'triglav/agent/hdfs'
+Triglav::Agent::Configuration.configure do |config|
+  config.name = :hdfs
+  # config.cli_class = Triglav::Agent::Hdfs::CLI
+  # config.setting_class = Triglav::Agent::Hdfs::Setting
+  # config.worker_module = Triglav::Agent::Hdfs::Worker
+  # config.processor_class = Triglav::Agent::Hdfs::Processor
+  config.monitor_class = Triglav::Agent::Hdfs::Monitor
+  config.connection_class = Triglav::Agent::Hdfs::Connection
+end
+Triglav::Agent::Configuration.cli_class.new.run

data/lib/triglav-agent-hdfs_jars.rb

@@ -0,0 +1,140 @@
+# this is a generated file, to avoid over-writing it just delete this comment
+begin
+  require 'jar_dependencies'
+rescue LoadError
+  require 'org/apache/httpcomponents/httpcore/4.2.4/httpcore-4.2.4.jar'
+  require 'log4j/log4j/1.2.17/log4j-1.2.17.jar'
+  require 'com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar'
+  require 'org/apache/commons/commons-compress/1.4.1/commons-compress-1.4.1.jar'
+  require 'org/apache/directory/server/apacheds-kerberos-codec/2.0.0-M15/apacheds-kerberos-codec-2.0.0-M15.jar'
+  require 'org/apache/curator/curator-recipes/2.6.0/curator-recipes-2.6.0.jar'
+  require 'org/apache/hadoop/hadoop-hdfs/2.6.5/hadoop-hdfs-2.6.5.jar'
+  require 'org/apache/commons/commons-math3/3.1.1/commons-math3-3.1.1.jar'
+  require 'org/apache/hadoop/hadoop-auth/2.6.5/hadoop-auth-2.6.5.jar'
+  require 'org/apache/hadoop/hadoop-mapreduce-client-shuffle/2.6.5/hadoop-mapreduce-client-shuffle-2.6.5.jar'
+  require 'org/fusesource/leveldbjni/leveldbjni-all/1.8/leveldbjni-all-1.8.jar'
+  require 'org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar'
+  require 'org/slf4j/slf4j-log4j12/1.7.5/slf4j-log4j12-1.7.5.jar'
+  require 'commons-io/commons-io/2.4/commons-io-2.4.jar'
+  require 'org/apache/hadoop/hadoop-mapreduce-client-core/2.6.5/hadoop-mapreduce-client-core-2.6.5.jar'
+  require 'org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar'
+  require 'org/slf4j/slf4j-api/1.7.5/slf4j-api-1.7.5.jar'
+  require 'com/sun/jersey/jersey-core/1.9/jersey-core-1.9.jar'
+  require 'org/codehaus/jackson/jackson-jaxrs/1.9.13/jackson-jaxrs-1.9.13.jar'
+  require 'xml-apis/xml-apis/1.3.04/xml-apis-1.3.04.jar'
+  require 'xmlenc/xmlenc/0.52/xmlenc-0.52.jar'
+  require 'com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar'
+  require 'org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar'
+  require 'org/apache/directory/api/api-util/1.0.0-M20/api-util-1.0.0-M20.jar'
+  require 'com/google/code/gson/gson/2.2.4/gson-2.2.4.jar'
+  require 'org/apache/directory/api/api-asn1-api/1.0.0-M20/api-asn1-api-1.0.0-M20.jar'
+  require 'com/sun/jersey/jersey-client/1.9/jersey-client-1.9.jar'
+  require 'org/apache/curator/curator-client/2.6.0/curator-client-2.6.0.jar'
+  require 'org/apache/hadoop/hadoop-mapreduce-client-common/2.6.5/hadoop-mapreduce-client-common-2.6.5.jar'
+  require 'commons-net/commons-net/3.1/commons-net-3.1.jar'
+  require 'commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar'
+  require 'com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar'
+  require 'commons-digester/commons-digester/1.8/commons-digester-1.8.jar'
+  require 'org/apache/hadoop/hadoop-mapreduce-client-jobclient/2.6.5/hadoop-mapreduce-client-jobclient-2.6.5.jar'
+  require 'org/apache/directory/server/apacheds-i18n/2.0.0-M15/apacheds-i18n-2.0.0-M15.jar'
+  require 'commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar'
+  require 'javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar'
+  require 'commons-lang/commons-lang/2.6/commons-lang-2.6.jar'
+  require 'org/htrace/htrace-core/3.0.4/htrace-core-3.0.4.jar'
+  require 'javax/activation/activation/1.1/activation-1.1.jar'
+  require 'org/apache/zookeeper/zookeeper/3.4.6/zookeeper-3.4.6.jar'
+  require 'org/tukaani/xz/1.0/xz-1.0.jar'
+  require 'org/apache/hadoop/hadoop-client/2.6.5/hadoop-client-2.6.5.jar'
+  require 'commons-cli/commons-cli/1.2/commons-cli-1.2.jar'
+  require 'xerces/xercesImpl/2.9.1/xercesImpl-2.9.1.jar'
+  require 'org/apache/httpcomponents/httpclient/4.2.5/httpclient-4.2.5.jar'
+  require 'commons-codec/commons-codec/1.4/commons-codec-1.4.jar'
+  require 'org/apache/hadoop/hadoop-yarn-api/2.6.5/hadoop-yarn-api-2.6.5.jar'
+  require 'org/apache/hadoop/hadoop-common/2.6.5/hadoop-common-2.6.5.jar'
+  require 'commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar'
+  require 'commons-collections/commons-collections/3.2.2/commons-collections-3.2.2.jar'
+  require 'javax/xml/stream/stax-api/1.0-2/stax-api-1.0-2.jar'
+  require 'org/apache/hadoop/hadoop-yarn-client/2.6.5/hadoop-yarn-client-2.6.5.jar'
+  require 'org/codehaus/jackson/jackson-xc/1.9.13/jackson-xc-1.9.13.jar'
+  require 'commons-logging/commons-logging/1.1.3/commons-logging-1.1.3.jar'
+  require 'org/apache/hadoop/hadoop-mapreduce-client-app/2.6.5/hadoop-mapreduce-client-app-2.6.5.jar'
+  require 'org/apache/hadoop/hadoop-yarn-server-common/2.6.5/hadoop-yarn-server-common-2.6.5.jar'
+  require 'javax/servlet/servlet-api/2.5/servlet-api-2.5.jar'
+  require 'com/google/guava/guava/11.0.2/guava-11.0.2.jar'
+  require 'org/apache/hadoop/hadoop-yarn-common/2.6.5/hadoop-yarn-common-2.6.5.jar'
+  require 'org/xerial/snappy/snappy-java/1.0.4.1/snappy-java-1.0.4.1.jar'
+  require 'org/apache/curator/curator-framework/2.6.0/curator-framework-2.6.0.jar'
+  require 'org/apache/hadoop/hadoop-annotations/2.6.5/hadoop-annotations-2.6.5.jar'
+  require 'org/apache/avro/avro/1.7.4/avro-1.7.4.jar'
+  require 'io/netty/netty/3.6.2.Final/netty-3.6.2.Final.jar'
+  require 'commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar'
+end
+
+if defined? Jars
+  require_jar( 'org.apache.httpcomponents', 'httpcore', '4.2.4' )
+  require_jar( 'log4j', 'log4j', '1.2.17' )
+  require_jar( 'com.google.protobuf', 'protobuf-java', '2.5.0' )
+  require_jar( 'org.apache.commons', 'commons-compress', '1.4.1' )
+  require_jar( 'org.apache.directory.server', 'apacheds-kerberos-codec', '2.0.0-M15' )
+  require_jar( 'org.apache.curator', 'curator-recipes', '2.6.0' )
+  require_jar( 'org.apache.hadoop', 'hadoop-hdfs', '2.6.5' )
+  require_jar( 'org.apache.commons', 'commons-math3', '3.1.1' )
+  require_jar( 'org.apache.hadoop', 'hadoop-auth', '2.6.5' )
+  require_jar( 'org.apache.hadoop', 'hadoop-mapreduce-client-shuffle', '2.6.5' )
+  require_jar( 'org.fusesource.leveldbjni', 'leveldbjni-all', '1.8' )
+  require_jar( 'org.mortbay.jetty', 'jetty-util', '6.1.26' )
+  require_jar( 'org.slf4j', 'slf4j-log4j12', '1.7.5' )
+  require_jar( 'commons-io', 'commons-io', '2.4' )
+  require_jar( 'org.apache.hadoop', 'hadoop-mapreduce-client-core', '2.6.5' )
+  require_jar( 'org.codehaus.jackson', 'jackson-mapper-asl', '1.9.13' )
+  require_jar( 'org.slf4j', 'slf4j-api', '1.7.5' )
+  require_jar( 'com.sun.jersey', 'jersey-core', '1.9' )
+  require_jar( 'org.codehaus.jackson', 'jackson-jaxrs', '1.9.13' )
+  require_jar( 'xml-apis', 'xml-apis', '1.3.04' )
+  require_jar( 'xmlenc', 'xmlenc', '0.52' )
+  require_jar( 'com.thoughtworks.paranamer', 'paranamer', '2.3' )
+  require_jar( 'org.codehaus.jackson', 'jackson-core-asl', '1.9.13' )
+  require_jar( 'org.apache.directory.api', 'api-util', '1.0.0-M20' )
+  require_jar( 'com.google.code.gson', 'gson', '2.2.4' )
+  require_jar( 'org.apache.directory.api', 'api-asn1-api', '1.0.0-M20' )
+  require_jar( 'com.sun.jersey', 'jersey-client', '1.9' )
+  require_jar( 'org.apache.curator', 'curator-client', '2.6.0' )
+  require_jar( 'org.apache.hadoop', 'hadoop-mapreduce-client-common', '2.6.5' )
+  require_jar( 'commons-net', 'commons-net', '3.1' )
+  require_jar( 'commons-configuration', 'commons-configuration', '1.6' )
+  require_jar( 'com.google.code.findbugs', 'jsr305', '1.3.9' )
+  require_jar( 'commons-digester', 'commons-digester', '1.8' )
+  require_jar( 'org.apache.hadoop', 'hadoop-mapreduce-client-jobclient', '2.6.5' )
+  require_jar( 'org.apache.directory.server', 'apacheds-i18n', '2.0.0-M15' )
+  require_jar( 'commons-beanutils', 'commons-beanutils', '1.7.0' )
+  require_jar( 'javax.xml.bind', 'jaxb-api', '2.2.2' )
+  require_jar( 'commons-lang', 'commons-lang', '2.6' )
+  require_jar( 'org.htrace', 'htrace-core', '3.0.4' )
+  require_jar( 'javax.activation', 'activation', '1.1' )
+  require_jar( 'org.apache.zookeeper', 'zookeeper', '3.4.6' )
+  require_jar( 'org.tukaani', 'xz', '1.0' )
+  require_jar( 'org.apache.hadoop', 'hadoop-client', '2.6.5' )
+  require_jar( 'commons-cli', 'commons-cli', '1.2' )
+  require_jar( 'xerces', 'xercesImpl', '2.9.1' )
+  require_jar( 'org.apache.httpcomponents', 'httpclient', '4.2.5' )
+  require_jar( 'commons-codec', 'commons-codec', '1.4' )
+  require_jar( 'org.apache.hadoop', 'hadoop-yarn-api', '2.6.5' )
+  require_jar( 'org.apache.hadoop', 'hadoop-common', '2.6.5' )
+  require_jar( 'commons-beanutils', 'commons-beanutils-core', '1.8.0' )
+  require_jar( 'commons-collections', 'commons-collections', '3.2.2' )
+  require_jar( 'javax.xml.stream', 'stax-api', '1.0-2' )
+  require_jar( 'org.apache.hadoop', 'hadoop-yarn-client', '2.6.5' )
+  require_jar( 'org.codehaus.jackson', 'jackson-xc', '1.9.13' )
+  require_jar( 'commons-logging', 'commons-logging', '1.1.3' )
+  require_jar( 'org.apache.hadoop', 'hadoop-mapreduce-client-app', '2.6.5' )
+  require_jar( 'org.apache.hadoop', 'hadoop-yarn-server-common', '2.6.5' )
+  require_jar( 'javax.servlet', 'servlet-api', '2.5' )
+  require_jar( 'com.google.guava', 'guava', '11.0.2' )
+  require_jar( 'org.apache.hadoop', 'hadoop-yarn-common', '2.6.5' )
+  require_jar( 'org.xerial.snappy', 'snappy-java', '1.0.4.1' )
+  require_jar( 'org.apache.curator', 'curator-framework', '2.6.0' )
+  require_jar( 'org.apache.hadoop', 'hadoop-annotations', '2.6.5' )
+  require_jar( 'org.apache.avro', 'avro', '1.7.4' )
+  require_jar( 'io.netty', 'netty', '3.6.2.Final' )
+  require_jar( 'commons-httpclient', 'commons-httpclient', '3.1' )
+end

data/lib/triglav/agent/hdfs.rb

@@ -0,0 +1,11 @@
+module Triglav
+  module Agent
+    module Hdfs
+    end
+  end
+end
+
+require 'triglav-agent'
+require 'triglav/agent/hdfs/connection'
+require 'triglav/agent/hdfs/version'
+require 'triglav/agent/hdfs/monitor'

data/lib/triglav/agent/hdfs/connection.rb

@@ -0,0 +1,144 @@
+require 'triglav/agent/base/connection'
+require 'triglav-agent-hdfs_jars'
+require 'uri'
+require 'tempfile'
+
+module Triglav::Agent
+  module Hdfs
+    class Connection < Base::Connection
+      Path = org.apache.hadoop.fs.Path
+      DistributedFileSystem = org.apache.hadoop.hdfs.DistributedFileSystem
+      PathFilter = org.apache.hadoop.fs.PathFilter
+      FileSystem = org.apache.hadoop.fs.FileSystem
+
+      attr_reader :connection_info
+
+      # @param [Hash] connection_info
+      # @option connection_info [Array] :config_files config files for org.apache.hadoop.conf.Configuration
+      # @option connection_info [Hash] :config config key value parameters for org.apache.hadoop.conf.Configuration
+      # @option connection_info [String] :doas
+      def initialize(connection_info)
+        @connection_info = connection_info
+        @configurations = {}
+        @filesystems = {}
+      end
+
+      def close
+        @filesystems.values.each {|fs| fs.close rescue nil }
+      end
+
+      # Get latest modification file under given path
+      #
+      # @param [Array of String, or String] hdfs path glob patterns
+      #
+      # @return [org.apache.hadoop.fs.FileStatus]
+      def get_latest_file_under(paths)
+        entries = []
+        Array(paths).each do |path|
+          entries.concat(glob_files_recursively(path))
+        end
+
+        latest_entry = nil
+        if entries.size > 0
+          latest_entry = entries.first
+          entries[1..entries.size].each do |entry|
+            latest_entry = entry.modification_time > latest_entry.modification_time ? entry : latest_entry
+          end
+        end
+        latest_entry
+      end
+
+      # for test
+      #
+      # @param [String] hdfs path
+      # @return [Boolean] true for success
+      def mkdir(path)
+        fs = get_fs(namespace = URI.parse(path).host)
+        fs.mkdirs(Path.new(path))
+      end
+
+      # for test
+      #
+      # @param [String] hdfs path
+      # @return [Boolean] true for success
+      def touch(path, overwrite = false)
+        fs = get_fs(namespace = URI.parse(path).host)
+        Tempfile.create('triglav-agent-hdfs') do |fp|
+          src = Path.new(fp.path)
+          dst = Path.new(path) # hdfs://
+          del_src = false
+          overwrite = overwrite
+          fs.copyFromLocalFile(del_src, overwrite, src, dst)
+        end
+      end
+
+      # for test
+      #
+      # @param [String] hdfs path
+      # @return [Boolean] true for success
+      def delete(path, recursive = false)
+        fs = get_fs(namespace = URI.parse(path).host)
+        fs.delete(Path.new(path), recursive)
+      end
+
+      private
+
+      # @param [String] path glob patterns
+      #
+      # @return [Array of org.apache.hadoop.fs.FileStatus] list of files
+      def glob_files_recursively(path, fs = nil)
+        fs ||= get_fs(namespace = URI.parse(path).host)
+
+        entries = []
+        glob_entries = fs.glob_status(Path.new(path))
+        glob_entries.each do |entry|
+          entries.concat(list_files_recursively(entry, fs))
+        end if glob_entries
+        entries
+      end
+
+      def list_files_recursively(entry, fs = nil)
+        return [entry] unless entry.is_directory
+        fs ||= get_fs(namespace = URI.parse(entry.get_path).host)
+
+        entries = []
+        list_entries = fs.list_status(entry.get_path)
+        list_entries.each do |entry|
+          entries.concat(list_files_recursively(entry, fs))
+        end
+        entries
+      end
+
+      def get_configuration(namespace)
+        return @configurations[namespace] if @configurations[namespace]
+
+        configuration = org.apache.hadoop.conf.Configuration.new
+
+        (connection_info[:config_files] || []).each do |config_file|
+          configuration.add_resource(config_file)
+        end
+        configuration.reload_configuration
+
+        (connection_info[:config] || {}).each do |key, value|
+          configuration.set(key.to_s, value.to_s)
+        end
+
+        configuration.set('fs.defaultFS', "hdfs://#{namespace}")
+
+        @configurations[namespace] = configuration
+      end
+
+      def get_fs(namespace)
+        return @filesystems[namespace] if @filesystems[namespace]
+        configuration = get_configuration(namespace)
+        if doas = connection_info[:doas]
+          uri = FileSystem.get_default_uri(configuration)
+          fs = FileSystem.get(uri, configuration, doas)
+        else
+          fs = FileSystem.get(configuration)
+        end
+        @filesystems[namespace] = fs
+      end
+    end
+  end
+end

data/lib/triglav/agent/hdfs/monitor.rb

@@ -0,0 +1,178 @@
+require 'triglav/agent/base/monitor'
+require 'triglav-agent-hdfs_jars'
+require 'triglav/agent/hdfs/connection'
+require 'uri'
+require 'securerandom'
+
+module Triglav::Agent
+  module Hdfs
+    class Monitor < Base::Monitor
+      attr_reader :connection, :resource_uri_prefix, :resource, :last_modification_times
+
+      # @param [Triglav::Agent::Hdfs::Connection] connection
+      # @param [TriglavClient::ResourceResponse] resource
+      # resource:
+      #   uri: hdfs://host/path
+      #   unit: 'daily', 'hourly', or 'singular'
+      #   timezone: '+09:00'
+      #   span_in_days: 32
+      def initialize(connection, resource_uri_prefix, resource)
+        @connection = connection
+        @resource_uri_prefix = resource_uri_prefix
+        @resource = resource
+        @status = Triglav::Agent::Status.new(resource_uri_prefix, resource.uri)
+        @last_modification_times = get_last_modification_times
+      end
+
+      def process
+        unless resource_valid?
+          $logger.warn { "Broken resource: #{resource.to_s}" }
+          return nil
+        end
+        $logger.debug { "Start process #{resource.uri}" }
+
+        events, new_last_modification_times = get_events
+
+        $logger.debug { "Finish process #{resource.uri}" }
+
+        return nil if events.nil? || events.empty?
+        yield(events) if block_given? # send_message
+        update_status_file(new_last_modification_times)
+        true
+      end
+
+      private
+
+      def get_events
+        new_last_modification_times = get_new_last_modification_times
+        latest_files = select_latest_files(new_last_modification_times)
+        events = build_events(latest_files)
+        [events, new_last_modification_times]
+      rescue => e
+        $logger.warn { "#{e.class} #{e.message} #{e.backtrace.join("\n  ")}" }
+        nil
+      end
+
+      def update_status_file(last_modification_times)
+        last_modification_times[:max] = last_modification_times.values.max
+        @status.set(last_modification_times)
+      end
+
+      def get_last_modification_times
+        last_modification_times = @status.get || {}
+        max_last_modification_time = last_modification_times[:max] || @status.getsetnx([:max], $setting.debug? ? 0 : get_current_time)
+        removes = last_modification_times.keys - paths.keys
+        appends = paths.keys - last_modification_times.keys
+        removes.each {|path| last_modification_times.delete(path) }
+        appends.each {|path| last_modification_times[path] = max_last_modification_time }
+        last_modification_times
+      end
+
+      def get_current_time
+        (Time.now.to_f * 1000).to_i # msec
+      end
+
+      def resource_valid?
+        self.class.resource_valid?(resource)
+      end
+
+      def self.resource_valid?(resource)
+        resource_unit_valid?(resource) && !resource.timezone.nil? && !resource.span_in_days.nil?
+      end
+
+      # Two or more combinations are not allowed for hdfs because
+      # * hourly should have %d, %H
+      # * daily should have %d, but not have %H
+      # * singualr should not have %d
+      # These conditions conflict.
+      def self.resource_unit_valid?(resource)
+        units = resource.unit.split(',').sort
+        return false if units.size >= 2
+        # if units.include?('hourly')
+        #   return false unless resource.uri.match(/%H/)
+        # end
+        # if units.include?('daily')
+        #   return false unless resource.uri.match(/%d/)
+        # end
+        if units.include?('singular')
+          return false if resource.uri.match(/%[YmdH]/)
+        end
+        true
+      end
+
+      def dates
+        return @dates if @dates
+        now = Time.now.localtime(resource.timezone)
+        @dates = resource.span_in_days.times.map do |i|
+          (now - (i * 86400)).to_date
+        end
+      end
+
+      def paths
+        return @paths if @paths
+        paths = {}
+        # If path becomes same, use newer date
+        case resource.unit
+        when 'hourly'
+          dates.each do |date|
+            date_time = date.to_time
+            (0..23).each do |hour|
+              path = (date_time + hour * 3600).strftime(resource.uri)
+              paths[path.to_sym] = [date, hour]
+            end
+          end
+        when 'daily'
+          hour = 0
+          dates.each do |date|
+            path = date.strftime(resource.uri)
+            paths[path.to_sym] = [date, hour]
+          end
+        when 'singular'
+          path = resource.uri
+          paths[path.to_sym] = [nil, nil]
+        end
+        @paths = paths
+      end
+
+      def get_new_last_modification_times
+        new_last_modification_times = {}
+        paths.each do |path, date_hour|
+          latest_file = connection.get_latest_file_under(path.to_s)
+          unless latest_file
+            $logger.debug { "get_latest_file_under(\"#{path.to_s}\") #=> does not exist" }
+            next
+          end
+          new_last_modification_times[path.to_sym] = latest_file.modification_time
+        end
+        new_last_modification_times
+      end
+
+      def select_latest_files(new_last_modification_times)
+        new_last_modification_times.select do |path, new_last_modification_time|
+          is_newer = new_last_modification_time > (last_modification_times[path] || 0)
+          $logger.debug { "#{path.to_s} #=> last_modification_time:#{new_last_modification_time}, is_newer:#{is_newer}" }
+          is_newer
+        end
+      end
+
+      def build_events(latest_files)
+        latest_files.map do |path, last_modification_time|
+          date, hour = date_hour = paths[path]
+          {
+            uuid: SecureRandom.uuid,
+            resource_uri: resource.uri,
+            resource_unit: resource.unit,
+            resource_time: date_hour_to_i(date, hour, resource.timezone),
+            resource_timezone: resource.timezone,
+            payload: {path: path.to_s, modification_time: last_modification_time}.to_json, # msec
+          }
+        end
+      end
+
+      def date_hour_to_i(date, hour, timezone)
+        return 0 if date.nil?
+        Time.strptime("#{date.to_s} #{hour.to_i} #{timezone}", '%Y-%m-%d %H %z').to_i
+      end
+    end
+  end
+end

data/lib/triglav/agent/hdfs/version.rb

@@ -0,0 +1,7 @@
+module Triglav
+  module Agent
+    module Hdfs
+      VERSION = "1.0.0"
+    end
+  end
+end

data/prepare.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+test -f config.yml || cp example/config.yml config.yml
+test -f .env || cp example/example.env .env

data/start.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+ABSPATH=$(cd $(dirname $0) && pwd)/$(basename $0)
+APP_ROOT=$(dirname $ABSPATH)
+if [ -z "${SHARED_ROOT}" ]; then SHARED_ROOT=.; fi
+
+CMD="bundle exec triglav-agent-hdfs --dotenv -c config.yml --status ${SHARED_ROOT}/status.yml --token ${SHARED_ROOT}/token.yml"
+echo $CMD
+$CMD

data/triglav-agent-hdfs.gemspec

@@ -0,0 +1,41 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'triglav/agent/hdfs/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "triglav-agent-hdfs"
+  spec.version       = Triglav::Agent::Hdfs::VERSION
+  spec.authors       = ["Triglav Team"]
+  spec.email         = ["triglav_admin_my@dena.jp"]
+
+  spec.summary       = %q{HDFS agent for triglav, data-driven workflow tool.}
+  spec.description   = %q{HDFS agent for triglav, data-driven workflow tool.}
+  spec.homepage      = "https://github.com/triglav-dataflow/triglav-agent-hdfs"
+  spec.license       = "MIT"
+
+  # important to get the jars installed
+  spec.platform      = 'java'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files        += Dir['lib/*.jar']
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # ENV is for travis
+  spec.add_dependency "jar-dependencies", ENV['JAR_DEPENDENCIES_VERSION'] ? "= #{ENV['JAR_DEPENDENCIES_VERSION']}" : "~> 0.3.5"
+  spec.add_dependency "triglav-agent"
+  spec.add_dependency "triglav_client"
+  spec.add_dependency "parallel"
+  spec.add_dependency "connection_pool"
+
+  spec.requirements << "jar org.apache.hadoop:hadoop-client, 2.6.5"
+
+  spec.add_development_dependency "bundler", "~> 1.11"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "test-unit"
+  spec.add_development_dependency "test-unit-rr"
+  spec.add_development_dependency "test-unit-power_assert"
+  spec.add_development_dependency "timecop"
+end

metadata

@@ -0,0 +1,222 @@
+--- !ruby/object:Gem::Specification
+name: triglav-agent-hdfs
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: java
+authors:
+- Triglav Team
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2017-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.5
+  name: jar-dependencies
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.5
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: triglav-agent
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: triglav_client
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: parallel
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: connection_pool
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.11'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-rr
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: test-unit-power_assert
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: timecop
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: HDFS agent for triglav, data-driven workflow tool.
+email:
+- triglav_admin_my@dena.jp
+executables:
+- triglav-agent-hdfs
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- example/config.yml
+- example/example.env
+- exe/triglav-agent-hdfs
+- lib/triglav-agent-hdfs_jars.rb
+- lib/triglav/agent/hdfs.rb
+- lib/triglav/agent/hdfs/connection.rb
+- lib/triglav/agent/hdfs/monitor.rb
+- lib/triglav/agent/hdfs/version.rb
+- prepare.sh
+- start.sh
+- triglav-agent-hdfs.gemspec
+homepage: https://github.com/triglav-dataflow/triglav-agent-hdfs
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar org.apache.hadoop:hadoop-client, 2.6.5
+rubyforge_project:
+rubygems_version: 2.6.6
+signing_key:
+specification_version: 4
+summary: HDFS agent for triglav, data-driven workflow tool.
+test_files: []