embulk-input-remote 0.3.3 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae62e6b2f381fbc643b9f8c5e9cc9bdd4246d944
-  data.tar.gz: 8b8b3e6a6f61a69a6aa1660dd78eb210d6a13dbc
+  metadata.gz: 24bb3008bbdbe51c44da96c8b15ff08cbeac29ff
+  data.tar.gz: 3295982e957968931d6c5803377ad41b0db4349e
 SHA512:
-  metadata.gz: 46e848501723870c0e8e0043c2014a1451815145c5c6bee4e3f4d0dc1f7f272c832caa27ed1172b058006bb72160d101b6f92510c3b116ba005c3913cb5ded59
-  data.tar.gz: b6448a41ec225d167ddc73ba0b4bd040b883a10a8c832b7a265ee283396e5bcde95da86f6bee1d72f570be9caa39b4df5fa3a327793bcd11846b9a01cc0c6bed
+  metadata.gz: 09f523ee72f8ff8a827741439390320aab48f948d68bd8a1aae5e45404335b060d172c5d6859143621439748c61e5d2e70b3e370d00a0fe391e53ce922444efa
+  data.tar.gz: 8edb2b6b79faac14ab323b9038388c55a222d9fa6e7ef7d7a21b88a3129e30c41861e10e963fb02a8389ed849833f4176f354dc9971843b8cf70988fa538eb13

data/.circleci/config.yml

@@ -7,18 +7,32 @@ jobs:
     steps:
       - checkout
 
-      - run: ssh-keygen -t ecdsa -f id_rsa_test -N ''
       - run: docker-compose up -d
       - run: docker-compose ps
-
+      - run: |
+          docker cp $(docker inspect --format="{{.Id}}" embulk-input-remote_host1):/home/ubuntu/.ssh/id_rsa_test . && \
+          chmod 400 id_rsa_test
       - restore_cache:
-          key: embulk-input-remote-{{ checksum "build.gradle" }}
+          key: v1-gradle-{{ checksum "build.gradle" }}
+          key: v1-gradle-
       - run: ./gradlew checkstyle
-      - run: ./gradlew check --full-stacktrace
+      - run: ./gradlew check --info
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/junit/
+            find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/junit/ \;
+          when: always
+      - store_test_results:
+          path: ~/junit
+      - store_artifacts:
+          path: ~/junit
+
       - save_cache:
           paths:
             - "~/.gradle"
-          key: embulk-input-remote-{{ checksum "build.gradle" }}
+            - "~/.m2"
+          key: v1-gradle-{{ checksum "build.gradle" }}
 
       - deploy:
           name: Push Gem to RubyGems.org and bump up

data/.gitignore

@@ -11,4 +11,3 @@ build/
 .classpath
 .project
 id_rsa_test
-id_rsa_test.pub

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/kamatama41/embulk-input-remote.svg?branch=master)](https://travis-ci.org/kamatama41/embulk-input-remote)
+[![CircleCI](https://circleci.com/gh/kamatama41/embulk-input-remote.svg?style=svg)](https://circleci.com/gh/kamatama41/embulk-input-remote)
 
 # Remote file input plugin for [Embulk](https://github.com/embulk/embulk)
 
@@ -12,21 +12,21 @@ This plugin load data from Remote hosts by SCP
 
 ## Configuration
 
-- **hosts**: Target hosts, format should be `host` or `host:port` (overrides `default_port`) (list, default: [])
-- **hosts_command**: Command for getting hosts(Windows not supported). If given the option, "hosts" is overwritten. (string, default: null)
-- **hosts_separator**: Separator for "hosts_command" result (string, default: " ")
-- **default_port**: Default port number for SSH (integer, default: 22)
-- **path**: Path of remote host (File or Directory) (string, default: "")
-- **path_command**: Command for getting path (Windows not supported). If given the option "path" is overwritten. (string, default: null)
-- **ignore_not_found_hosts**: If the option is true, Hosts which meet the following conditions are skipped. (Means they are not included into resume target.) (boolean, default: false)
-  - Target file (or directory) isn't found
-  - Occurred SSH error
+- **hosts**: Target hosts, its format should be `hostname` or `hostname:port` (its `port` overrides `default_port`) (list, default: [])
+- **hosts_command**: Command to get `hosts` (Windows not supported). If given the option, `hosts` is overwritten. (string, default: null)
+- **hosts_separator**: Separator for `hosts_command` result (string, default: " ")
+- **default_port**: Port number for SSH (integer, default: 22)
+- **path**: File or directory path of remote host (string, default: "")
+- **path_command**: Command to get `path` (Windows not supported). If given the option, `path` is overwritten. (string, default: null)
+- **ignore_not_found_hosts**: If true, hosts that meet the following conditions are skipped, which means they are not included into the the target of resuming.) (boolean, default: false)
+  - Target file (or directory) is not found
+  - An error that is related to SSH connectivity occurred 
 - **auth**: SSH authentication setting (hash, default: {})
-    - **user**: SSH username (string, default: execute user)
-    - **type**: public_key or password (string, default: public_key)
-    - **key_path**: Path of secret key (If you choose type "public_key") (string, default: "~/.ssh/id_rsa or id_dsa")
-    - **password**: SSH password (If you choose type "password") (string)
-    - **skip_host_key_verification**: If the option is true, HostKey verification will be skipped (boolean, default: false)
+    - **user**: SSH username (string, default: current user)
+    - **type**: `public_key` or `password` (string, default: public_key)
+    - **key_path**: Path of your secret key (required when you choose `public_key` type) (string, default: `~/.ssh/id_rsa` or `id_dsa`")
+    - **password**: Password of the `user` (required when you choose `password` type) (string)
+    - **skip_host_key_verification**: If true, verification of host key will be skipped (boolean, default: false)
 
 ## Example
 
@@ -50,8 +50,8 @@ in:
 ```
 
 ## Note
-When this plugin run on Linux, a task might be blocked.  
-The cause is java.security.SecureRandom. Please try one of the followings.
+When running Embulk with this plugin on Linux, it might be blocked due to SecureRandom. Please try one of followings to resolve it.
+(Note: it makes SecureRandom more insecure than default)
 
 ### set JVM_OPTION "-Djava.security.egd"
 
@@ -60,10 +60,9 @@ $ export JAVA_TOOL_OPTIONS="-Djava.security.egd=file:/dev/./urandom"
 $ embulk run config.yml
 ```
 
-### rewrite ${JAVA_HOME}/jre/lib/security/java.security
-
+### Update "securerandom.source" in ${JAVA_HOME}/jre/lib/security/java.security
 ```
-# securerandom.source=file:/dev/random # before
+# securerandom.source=file:/dev/random  # before
 securerandom.source=file:/dev/./urandom # after
 ```
 
@@ -72,15 +71,19 @@ securerandom.source=file:/dev/./urandom # after
 http://stackoverflow.com/questions/137212/how-to-solve-performance-problem-with-java-securerandom
 
 ## Development on local machine
-- Install Docker and then we can create SSH-able containers
+Install Docker and Docker compose so that you can start SSH-able containers and run tests.
+
 ```sh
-$ ssh-keygen -t ecdsa -f ./id_rsa_test -N ''
 $ docker-compose up -d
 $ docker-compose ps
-          Name                 Command       State           Ports         
+          Name                 Command       State           Ports        
 --------------------------------------------------------------------------
-embulkinputremote_host1_1   /entrypoint.sh   Up      0.0.0.0:10022->22/tcp 
-embulkinputremote_host2_1   /entrypoint.sh   Up      0.0.0.0:10023->22/tcp 
+embulk-input-remote_host1   /entrypoint.sh   Up      0.0.0.0:10022->22/tcp
+embulk-input-remote_host2   /entrypoint.sh   Up      0.0.0.0:10023->22/tcp
+
+$ docker cp $(docker inspect --format="{{.Id}}" embulk-input-remote_host1):/home/ubuntu/.ssh/id_rsa_test .
+$ chmod 400 id_rsa_test
+$ ./gradlew test
 ```
 
 ## Build

data/build.gradle

@@ -1,18 +1,17 @@
 buildscript {
-    ext.kotlinVersion = '1.1.1'
     repositories {
-        mavenCentral()
         jcenter()
         maven { url 'http://kamatama41.github.com/maven-repository/repository' }
     }
     dependencies {
-        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
         classpath "com.github.kamatama41:gradle-embulk-plugin:0.1.4"
         classpath "net.researchgate:gradle-release:2.5.0"
     }
 }
 
-apply plugin: "kotlin"
+plugins {
+    id "org.jetbrains.kotlin.jvm" version "1.2.31"
+}
 apply plugin: "com.github.kamatama41.embulk"
 apply plugin: "net.researchgate.release"
 
@@ -25,19 +24,19 @@ repositories {
     maven { url 'http://kamatama41.github.com/maven-repository/repository' }
 }
 
-sourceCompatibility = 1.7
-targetCompatibility = 1.7
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
 
 dependencies {
-    compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
-    compile  "com.hierynomus:sshj:0.19.1"
+    compile "org.jetbrains.kotlin:kotlin-stdlib"
+    compile  "com.hierynomus:sshj:0.24.0"
     compile  "com.jcraft:jzlib:1.1.3"
-    testCompile "com.github.kamatama41:embulk-test-helpers:0.3.2"
+    testCompile "com.github.kamatama41:embulk-test-helpers:0.4.0"
     testCompile "com.github.docker-java:docker-java:3.0.7"
 }
 
 embulk {
-    version = "0.8.18"
+    version = "0.9.7"
     category = "file-input"
     name = "remote"
     authors = ["Shinichi Ishimura"]
@@ -47,6 +46,10 @@ embulk {
     homepage = "https://github.com/kamatama41/embulk-input-remote"
 }
 
+test {
+    jvmArgs '-Djava.security.egd=file:/dev/./urandom'
+}
+
 release {
     git { requireBranch = 'master' }
 }

data/docker-compose.yml

@@ -1,16 +1,16 @@
-version: '2'
+version: '3'
 services:
   host1:
-    image: "sickp/alpine-sshd:7.2"
+    container_name: embulk-input-remote_host1
+    image: "kamatama41/ubuntu-sshd"
     ports:
      - "10022:22"
     volumes:
      - ./src/test/resources/input/host1/:/mount
-     - ./id_rsa_test.pub:/root/.ssh/authorized_keys
   host2:
-    image: "sickp/alpine-sshd:7.2"
+    container_name: embulk-input-remote_host2
+    image: "kamatama41/ubuntu-sshd"
     ports:
      - "10023:22"
     volumes:
      - ./src/test/resources/input/host2/:/mount
-     - ./id_rsa_test.pub:/root/.ssh/authorized_keys

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Thu Feb 02 08:54:57 JST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-all.zip

data/gradle.properties

@@ -1 +1 @@
-version=0.3.3
+version=0.4.0

data/gradlew

@@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -155,7 +155,7 @@ if $cygwin ; then
 fi
 
 # Escape application args
-save ( ) {
+save () {
     for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
     echo " "
 }

data/src/test/kotlin/org/embulk/input/remote/TestRemoteFileInputPlugin.kt

@@ -37,7 +37,6 @@ class TestRemoteFileInputPlugin : EmbulkPluginTest() {
         assertRecords(record(1, "user1"))
     }
 
-    @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
     @Test fun loadFromRemoteViaPublicKey() {
         val keyPath = System.getenv("KEY_PATH") ?: "./id_rsa_test"
 
@@ -159,8 +158,8 @@ class TestRemoteFileInputPlugin : EmbulkPluginTest() {
     }
 
     companion object DockerUtils {
-        private val CONTAINER_ID_HOST1 = "embulkinputremote_host1_1"
-        private val CONTAINER_ID_HOST2 = "embulkinputremote_host2_1"
+        private const val CONTAINER_ID_HOST1 = "embulk-input-remote_host1"
+        private const val CONTAINER_ID_HOST2 = "embulk-input-remote_host2"
         private val dockerClient = DockerClientBuilder.getInstance().build()
 
         private fun stopContainer(containerId: String) {

data/src/test/resources/yaml/base.yml

@@ -3,9 +3,9 @@ hosts:
   - localhost:10022
 path: /mount/test.csv
 auth:
-  user: root
+  user: ubuntu
   type: password
-  password: root
+  password: passw0rd
   skip_host_key_verification: true
 parser:
   type: csv

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-remote
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.4.0
 platform: ruby
 authors:
 - Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-20 00:00:00.000000000 Z
+date: 2018-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -47,7 +47,6 @@ extra_rdoc_files: []
 files:
 - .circleci/config.yml
 - .gitignore
-- Dockerfile
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -70,14 +69,14 @@ files:
 - src/test/resources/input/host2/test_command.csv
 - src/test/resources/script/hosts.sh
 - src/test/resources/yaml/base.yml
-- classpath/sshj-0.19.1.jar
-- classpath/bcprov-jdk15on-1.51.jar
+- classpath/bcpkix-jdk15on-1.57.jar
+- classpath/kotlin-stdlib-1.2.31.jar
+- classpath/sshj-0.24.0.jar
 - classpath/jzlib-1.1.3.jar
-- classpath/embulk-input-remote-0.3.3.jar
+- classpath/eddsa-0.2.0.jar
+- classpath/embulk-input-remote-0.4.0.jar
 - classpath/annotations-13.0.jar
-- classpath/bcpkix-jdk15on-1.51.jar
-- classpath/eddsa-0.1.0.jar
-- classpath/kotlin-stdlib-1.1.1.jar
+- classpath/bcprov-jdk15on-1.57.jar
 homepage: https://github.com/kamatama41/embulk-input-remote
 licenses:
 - MIT

data/Dockerfile

@@ -1,17 +0,0 @@
-# Deprecated
-## ref: https://docs.docker.com/engine/examples/running_ssh_service/
-#FROM ubuntu:16.04
-#
-#RUN apt-get update && apt-get install -y openssh-server
-#RUN mkdir /var/run/sshd
-#RUN echo 'root:screencast' | chpasswd
-#RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-#
-## SSH login fix. Otherwise user is kicked off after login
-#RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-#
-#ENV NOTVISIBLE "in users profile"
-#RUN echo "export VISIBLE=now" >> /etc/profile
-#
-#EXPOSE 22
-#CMD ["/usr/sbin/sshd", "-D"]