RubyGems - embulk-input-sftp - Versions diffs - 0.1.0 - Mend

embulk-input-sftp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

checksums.yaml +7 -0
data/.gitignore +14 -0
data/README.md +126 -0
data/build.gradle +95 -0
data/config/checkstyle/checkstyle.xml +128 -0
data/config/checkstyle/default.xml +108 -0
data/gradle/wrapper/gradle-wrapper.jar +0 -0
data/gradle/wrapper/gradle-wrapper.properties +6 -0
data/gradlew +160 -0
data/gradlew.bat +90 -0
data/lib/embulk/input/sftp.rb +3 -0
data/src/main/java/org/embulk/input/sftp/PluginTask.java +67 -0
data/src/main/java/org/embulk/input/sftp/ProxyTask.java +85 -0
data/src/main/java/org/embulk/input/sftp/SftpFileInput.java +217 -0
data/src/main/java/org/embulk/input/sftp/SftpFileInputPlugin.java +67 -0
data/src/main/java/org/embulk/input/sftp/SingleFileProvider.java +71 -0
data/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java +5 -0
data/src/test/resources/id_rsa +30 -0
data/src/test/resources/id_rsa.pub +1 -0
metadata +95 -0

checksums.yaml ADDED

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 72aaee2da1604f1a298e13d37ab74be174cddb5f
+  data.tar.gz: 3c062e178443efd8b44874eb9cb2d446385d010b
+SHA512:
+  metadata.gz: d21a1f92aae66016ea648dc94e2f757e7bc3d968bda4953f35623307f649a49bde0d8b0bbbc84b79cedfb25c35e069f7b6b30134a3959d9211baad7581cb3c59
+  data.tar.gz: 67c3b0a1c2b054093f61aa5ca0946c9fed2f74a0f223e0ee60c44737fa33cbbd5fd90469f3b5baafc62eb45d74c19e504cd87c498a3ebdf8d3bb1dc7a8e5967f

data/.gitignore ADDED

@@ -0,0 +1,14 @@
+target/
+build/
+pkg/
+*.iml
+*~
+._*
+.idea
+tmp/
+vendor/
+/classpath/
+/.bundle
+.yardoc
+/embulk-*.jar
+/.gradle

data/README.md ADDED

@@ -0,0 +1,126 @@
+# SFTP file input plugin for Embulk
+Reads files stored on remote server using SFTP
+## Overview
+* **Plugin type**: file input
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+## Configuration
+- **host**: (string, required)
+- **port**: (string, default: `22`)
+- **user**: (string, required)
+- **password**: (string, default: `null`)
+- **secret_key_file**: (string, default: `null`)
+- **secret_key_passphrase**: (string, default: `""`)
+- **user_directory_is_root**: (boolean, default: `true`)
+- **timeout**: sftp connection timeout seconds (integer, default: `600`)
+- **path_prefix**: Prefix of output paths (string, required)
+- **file_ext**: Extension of output files (string, required)
+- **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
+### Proxy configuration
+- **proxy**:
+    - **type**: (string(http | socks | stream), required, default: `null`)
+        - **http**: use HTTP Proxy
+        - **socks**: use SOCKS Proxy
+        - **stream**: Connects to the SFTP server through a remote host reached by SSH
+    - **host**: (string, required)
+    - **port**: (int, default: `22`)
+    - **user**: (string, optional)
+    - **password**: (string, optional, default: `null`)
+    - **command**: (string, optional)
+### Example
+```yaml
+in:
+  type: sftp
+  host: 127.0.0.1
+  port: 22
+  user: embulk
+  secret_key_file: /Users/embulk/.ssh/id_rsa
+  secret_key_passphrase: secret_pass
+  user_directory_is_root: false
+  timeout: 600
+  path_prefix: /data/sftp
+```
+With proxy
+```yaml
+in:
+  type: sftp
+  host: 127.0.0.1
+  port: 22
+  user: embulk
+  secret_key_file: /Users/embulk/.ssh/id_rsa
+  secret_key_passphrase: secret_pass
+  user_directory_is_root: false
+  timeout: 600
+  path_prefix: /data/sftp
+  proxy:
+    type: http
+    host: proxy_host
+    port: 8080
+    user: proxy_user
+    password: proxy_secret_pass
+    command:
+```
+## Proxy settings
+### Example
+```yaml
+in:
+  type: sftp
+  host: 127.0.0.1
+  port: 22
+  user: embulk
+  secret_key_file: /Users/embulk/.ssh/id_rsa
+  secret_key_passphrase: secret_pass
+  user_directory_is_root: false
+  timeout: 600
+  path_prefix: /data/sftp
+```
+### Secret Keyfile configuration
+Please set path of secret_key_file as follows.
+```yaml
+in:
+  type: sftp
+  ...
+  secret_key_file: /path/to/id_rsa
+  ...
+```
+You can also embed contents of secret_key_file at config.yml.
+```yaml
+in:
+  type: sftp
+  ...
+  secret_key_file:
+    content |
+      -----BEGIN RSA PRIVATE KEY-----
+      ABCDEFG...
+      HIJKLMN...
+      OPQRSTU...
+      -----END RSA PRIVATE KEY-----
+  ...
+```
+## Build
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```
+## Test
+```
+$ ./gradlew test  # -t to watch change of files and rebuild continuously
+```

data/build.gradle ADDED

@@ -0,0 +1,95 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "java"
+    id "checkstyle"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+version = "0.1.0"
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
+dependencies {
+    compile  "org.embulk:embulk-core:0.8.6"
+    provided "org.embulk:embulk-core:0.8.6"
+    compile "org.apache.commons:commons-vfs2:2.+"
+    compile "org.apache.commons:commons-io:1.3.2"
+    compile "com.jcraft:jsch:0.1.53"
+    testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.6:tests"
+    testCompile "org.embulk:embulk-standards:0.8.6"
+    testCompile "org.apache.sshd:apache-sshd:1.1.0"
+    testCompile "org.littleshoot:littleproxy:1.1.0-beta1"
+    testCompile "io.netty:netty-all:4.0.34.Final"
+}
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
+    script "build/gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+task "package"(dependsOn: ["gemspec", "classpath"]) << {
+    println "> Build succeeded."
+    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+}
+task gemspec << { file("build/gemspec").write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Satoshi Akama"]
+  spec.summary       = %[SFTP file input plugin for Embulk]
+  spec.description   = %[Reads files stored on remote server using SFTP.]
+  spec.email         = ["satoshiakama@gmail.com"]
+  spec.licenses      = ["Apache-2.0"]
+  spec.homepage      = "https://github.com/sakama/embulk-input-sftp"
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end
+/$)
+}
+clean { delete "${project.name}.gemspec" }

data/config/checkstyle/checkstyle.xml ADDED

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/config/checkstyle/default.xml ADDED

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>