embulk-input-bigquery_extract_files 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3ad9b4688bba442662baac2209f35077b14c669b
+  data.tar.gz: 4b184864e80181b615cfef548ea065e6c2140a0d
+SHA512:
+  metadata.gz: f6617b86ba4e1dff3cf2fb689c8b770b12a3c71598e5b2d68269037b456f18bb29a576f127bacdf3d088913912c6eb2f6f1305a19d17f66543ef5a769668f7bf
+  data.tar.gz: c5394ab788b193dfc1920b508a715453ae30c40c1a629b3582ea8066778ee2ff965872b1a5b3f1cd2230892be958bb8a22fe79551481f958591ebb09abee90f1

data/.gitignore

@@ -0,0 +1,16 @@
+*~
+/bin/
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project
+/setenv.bat
+/local_config.yml
+/local_setenv.bat

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,92 @@
+# Google Cloud Bigquery extract file input plugin for Embulk 
+
+development version. 
+
+## Overview
+
+* **Plugin type**: file input
+* **Resume supported**: no
+* **Cleanup supported**: yes
+
+### Detail
+
+Read files stored in Google Cloud Storage, that exported from Google Cloud Bigquery's table or query result.
+
+Maybe solution for very big data in bigquery.
+
+If you set  **table** config without **query** config, 
+then just extract table to Google Cloud Storage.
+
+If you set **query** config,
+then query result save to temp table and then extracted that temp table to Google Cloud Storage uri.
+see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.extract
+   
+## Usage
+
+### Install plugin
+
+```bash
+embulk gem install embulk-input-bigquery_extract_files
+```
+
+* rubygem url : https://rubygems.org/profiles/jo8937
+
+
+## Configuration
+
+- **project**: Google Cloud Platform (gcp) project id (string, required)
+- **json_keyfile**: gcp service account's private key with json (string, required)
+- **gcs_uri**: bigquery result saved uri. bucket and path names parsed from this uri.  (string, required)
+- **temp_local_path**: extract files download directory in local machine (string, required)
+
+- **dataset**: target datasource dataset (string, default: `null`)
+- **table**: target datasource table. either query or table are required. (string, default: `null`)
+- **query**: target datasource query. either query or table are required. (string, default: `null`)
+
+- **temp_dataset**: if you use **query** param, query result saved here  (string, default: `null`)
+- **temp_table**: if you use **query** param, query result saved here. if not set, plugin generate temp name (string, default: `null`)
+- **use_legacy_sql**: if you use **query** param, see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.useLegacySql (string, default: `false`)
+- **cache**: if you use **query** param, see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.useQueryCache (string, default: `true`)
+- **create_disposition**: if you use **query** param, see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.createDisposition (string, default: `CREATE_IF_NEEDED`)
+- **write_disposition**: if you use **query** param, see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.query.writeDisposition (string, default: `WRITE_APPEND`)
+
+- **file_format**: Table extract file format. see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.extract.destinationFormat (string, default: `CSV`)
+- **compression**: Table extract file compression setting. see : https://cloud.google.com/bigquery/docs/reference/rest/v2/jobs#configuration.extract.compression (string, default: `GZIP`)
+
+- **temp_schema_file_path**: bigquery result schema file for parser. (Optional) (string, default: `null`)
+
+- **temp_schema_file_type**: default is embulk's Schema object. (Optional) (string, default: `null`)  
+
+- **decoders**: embulk java-file-input plugin's default attribute. see : http://www.embulk.org/docs/built-in.html#gzip-decoder-plugin
+- **parser**: embulk java-file-input plugin's default .attribute see : http://www.embulk.org/docs/built-in.html#csv-parser-plugin
+
+
+## Example
+
+```yaml
+in:
+  type: bigquery_extract_files
+  project: googlecloudplatformproject
+  json_keyfile: gcp-service-account-private-key.json
+  dataset: target_dataset
+  #table: target_table
+  query: 'select a,b,c from target_table'
+  gcs_uri: gs://bucket/subdir
+  temp_dataset: temp_dataset
+  temp_local_path: C:\Temp
+  file_format: 'NEWLINE_DELIMITED_JSON'
+  compression: 'GZIP'
+  decoders:
+  - {type: gzip}  
+  parser:
+    type: json
+out: 
+  type: stdout
+```
+
+
+## Build
+
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```

data/build.gradle

@@ -0,0 +1,102 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "java"
+    id "checkstyle"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+
+version = "0.0.3"
+
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
+
+dependencies {
+    compile ("org.embulk:embulk-core:0.8.36")
+    provided("org.embulk:embulk-core:0.8.36")
+    
+    compile "com.google.http-client:google-http-client-jackson2:1.21.0"
+    compile ('com.google.apis:google-api-services-bigquery:v2-rev363-1.23.0') {exclude module: "guava-jdk5"}
+    compile ("com.google.apis:google-api-services-storage:v1-rev59-1.21.0") {exclude module: "guava-jdk5"}
+    
+    testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.36:tests"
+    testCompile "org.embulk:embulk-standards:0.8.36"
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
+    script "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
+}
+
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["jo8937"]
+  spec.summary       = %[Bigquery Exports GCS file input plugin for Embulk]
+  spec.description   = %[Reads files stored on Google Cloud Storage, that exportes from Bigquery query result or table]
+  spec.email         = ["jo8937@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/jo8937/embulk-input-bigquery_extract_files"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }

data/config.yml

@@ -0,0 +1,30 @@
+in:
+  type: bigquery_extract_files
+  project: project
+  json_keyfile: google-key.json
+  #dataset: testdataset
+  #table: testtable
+  query: 'select a,b,c from testdataset.testtable'
+  gcs_uri: gs://bucket/dir/csv-*
+  temp_dataset: tempdataset
+  temp_local_path: /tmp/test/data
+  file_format: 'CSV'
+  compression: 'GZIP'
+  temp_schema_file_path: /tmp/test/schema/csv_schema.json
+  decoders:
+  - {type: gzip}
+  parser:
+    type: csv_with_schema_file
+    default_timestamp_format: '%Y-%m-%d %H:%M:%S %z'
+    schema_path: /tmp/test/schema/csv_schema.json
+out: 
+  type: mysql
+  host: localhost
+  user: user
+  password: passwd
+  port: 10315
+  database: dbname
+  table: test_table
+  options: {connectTimeout: 200000}
+  mode: insert_direct
+  

data/config/checkstyle/checkstyle.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/config/checkstyle/default.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>