embulk-input-bigquery_extract_files 0.0.3

data/src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java

@@ -0,0 +1,56 @@
+package org.embulk.input.bigquery_export_gcs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assume.assumeNotNull;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestPluginFunctions extends UnitTestInitializer
+{
+	private static final Logger log = LoggerFactory.getLogger(TestPluginFunctions.class);
+		
+    @Test
+    public void regexTest(){
+    	testTableName("select * from aaa.bbb","bbb");
+    	testTableName("select * from aaa.bbb.ccc","ccc");
+    	testTableName("select * from [aaa.bbb]","bbb");
+    	testTableName("select * from aaa.bbb$20171123","bbb");
+    	testTableName("select * from aaa.t_b_b_b","t_b_b_b");
+    }
+    
+    public void testTableName(String query, String expect){
+    	String word = BigqueryExportUtils.parseQueryToBaseTableName(query);
+    	log.info("{}", word );
+    	assertEquals(word, expect);
+    }
+
+    @Test
+    public void testParseGcsUrl(){
+    	ConfigSource c = config.deepCopy();
+    	c.set("gcs_uri", "gs://aaa/bbb/ccc_*");
+    	
+    	BigqueryExportGcsFileInputPlugin.PluginTask task = c.loadConfig(BigqueryExportGcsFileInputPlugin.PluginTask.class );
+    	
+    	BigqueryExportUtils.parseGcsUri(task);
+    	
+    	assertEquals("", "aaa", task.getGcsBucket());
+    	assertEquals("", "bbb/ccc_", task.getGcsBlobNamePrefix());
+    }
+
+}

data/src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java

@@ -0,0 +1,86 @@
+package org.embulk.input.bigquery_export_gcs;
+
+import static org.junit.Assume.assumeNotNull;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.input.bigquery_export_gcs.BigqueryExportGcsFileInputPlugin.PluginTask;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class UnitTestInitializer
+{
+	private static final Logger log = LoggerFactory.getLogger(UnitTestInitializer.class);
+		
+    private static String TEST_EMBULK_CUSTOM_CONFIG_PATH;
+    
+    /*
+     * This test case requires environment variables
+     *   GCP_EMAIL
+     *   GCP_P12_KEYFILE
+     *   GCP_JSON_KEYFILE
+     *   GCP_BUCKET
+     */
+    @BeforeClass
+    public static void initializeConstantVariables()
+    {
+    	TEST_EMBULK_CUSTOM_CONFIG_PATH = "local_config.yml"; //System.getenv("TEST_EMBULK_CUSTOM_CONFIG_PATH");
+    }
+    
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    protected ConfigSource config;
+    protected BigqueryExportGcsFileInputPlugin plugin;
+    protected FileInputRunner runner;
+    protected MockPageOutput output;
+    
+    /**
+     * @throws IOException 
+     * 
+     */
+    @Before
+    public void createResources() throws IOException 
+    {
+    	File f =new File(TEST_EMBULK_CUSTOM_CONFIG_PATH);
+    	if(f.exists()){
+    		log.info("Test env load from file : {}",f);
+    		config = new ConfigLoader(Exec.session().getModelManager()).fromYamlFile(f).getNested("in");	
+    		log.info("values : {}",config.toString());
+    	}else{
+    		log.error("test config file NOT FOUND ! :: {}",f);
+    		log.info("load test env from system env ... ");
+    		config = Exec.newConfigSource();
+    		config.set("project", System.getenv("embulk_config_project"));
+    		config.set("json_keyfile", System.getenv("embulk_config_json_keyfile"));
+    		config.set("dataset", System.getenv("embulk_config_dataset"));
+    		config.set("query", System.getenv("embulk_config_query"));
+    		config.set("gcs_uri", System.getenv("embulk_config_gcs_uri"));
+    		config.set("temp_dataset", System.getenv("embulk_config_temp_dataset"));
+    		config.set("temp_local_path", System.getenv("embulk_config_temp_local_path"));
+    	}
+    	        
+        plugin = new BigqueryExportGcsFileInputPlugin();
+        runner = new FileInputRunner(runtime.getInstance(BigqueryExportGcsFileInputPlugin.class));
+        output = new MockPageOutput();
+
+        assumeNotNull(
+                      config.get(String.class, "project"),
+                      config.get(String.class, "json_keyfile"),
+                      config.get(String.class, "gcs_uri"),
+                      config.get(String.class, "temp_local_path")
+                      );
+    }
+
+
+}

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-bigquery_extract_files
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- jo8937
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2017-11-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Reads files stored on Google Cloud Storage, that exportes from Bigquery query result or table
+email:
+- jo8937@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE.txt
+- README.md
+- build.gradle
+- config.yml
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/input/bigquery_extract_files.rb
+- src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportGcsFileInputPlugin.java
+- src/main/java/org/embulk/input/bigquery_export_gcs/BigqueryExportUtils.java
+- src/test/java/org/embulk/input/bigquery_export_gcs/TestGoogleCloudAccessData.java
+- src/test/java/org/embulk/input/bigquery_export_gcs/TestPluginFunctions.java
+- src/test/java/org/embulk/input/bigquery_export_gcs/UnitTestInitializer.java
+- classpath/commons-codec-1.3.jar
+- classpath/commons-logging-1.1.1.jar
+- classpath/embulk-input-bigquery_extract_files-0.0.3.jar
+- classpath/google-api-client-1.23.0.jar
+- classpath/google-api-services-bigquery-v2-rev363-1.23.0.jar
+- classpath/google-api-services-storage-v1-rev59-1.21.0.jar
+- classpath/google-http-client-1.23.0.jar
+- classpath/google-http-client-jackson2-1.23.0.jar
+- classpath/google-oauth-client-1.23.0.jar
+- classpath/httpclient-4.0.1.jar
+- classpath/httpcore-4.0.1.jar
+- classpath/jsr305-1.3.9.jar
+homepage: https://github.com/jo8937/embulk-input-bigquery_extract_files
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Bigquery Exports GCS file input plugin for Embulk
+test_files: []