embulk-parser-jdbc-schema-csv 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 775c538cd41b7550efbedec4ec547c21a28808d1
+  data.tar.gz: d1d95b87d4be564d71e3dbaf626caab9b80eed77
+SHA512:
+  metadata.gz: bfb0d20a9f313288e5a66ac442797c450ee87ddaecabf13e23804677e035ba820b8c1f1c9c93bf6240dc9150ede9b78fc8a18a7f14493c0b2a8e8fff822f242f
+  data.tar.gz: 2b14eb314f62912fc7d97535f0634748250b88b7bd412f82f8b85e48cd8492fccf4b468960325751a4ae354b62ce375a5258b98f4fb178f2e1d36da264dd563c

data/README.md

@@ -0,0 +1,39 @@
+# Embulk CSV parser plugin using JDBC to define schema
+
+This Embulk plugin extends CSV parser to define columns based on database meta data.
+
+## Overview
+
+* **Plugin type**: parser
+
+## Configuration
+
+- **schema** database table to define columns. Same as embulk-input-plugin.
+
+Others are same as CSV parser plugin except columns (columns are ignored).
+
+### Example
+
+```yaml
+in:
+  type: file
+  path_prefix: 'data/test.csv'
+  parser:
+    type: jdbc-schema-csv
+    delimiter: ','
+    header_line: false
+    schema: &OUT
+      host: localhost
+      user: myuser
+      password: ""
+      database: my_database
+      table: my_table
+      mode: insert
+out: *OUT
+```
+
+### Build
+
+```
+$ ./gradle gem
+```

data/build.gradle

@@ -0,0 +1,71 @@
+plugins {
+    id 'com.jfrog.bintray' version '1.1'
+    id 'com.github.jruby-gradle.base' version '0.1.5'
+    id 'java'
+}
+import com.github.jrubygradle.JRubyExec
+
+apply plugin: 'java'
+apply plugin: 'com.github.jruby-gradle.base'
+
+[compileJava, compileTestJava]*.options*.encoding = 'UTF-8'
+
+project.version = '0.0.1'
+
+repositories {
+    mavenCentral()
+    jcenter()
+    flatDir {
+    	dirs 'depends'
+    }
+}
+
+configurations {
+    provided
+}
+
+dependencies {
+    compile     'org.embulk:embulk-core:0.6.5'
+    compile     'org.embulk:embulk-standards:0.6.5'
+    provided    'org.embulk:embulk-core:0.6.5'
+    provided    'org.embulk:embulk-standards:0.6.5'
+    testCompile 'junit:junit:4.+'
+    testCompile name:'embulk-input-jdbc-0.4.0'
+    testCompile name:'embulk-input-mysql-0.4.0'
+    testCompile name:'mysql-connector-java-5.1.34'
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file('classpath').deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into 'classpath'
+}
+clean { delete 'classpath' }
+
+task gem(type: JRubyExec, dependsOn: ['build', 'gemspec', 'classpath']) {
+    jrubyArgs '-rrubygems/gem_runner', "-eGem::GemRunner.new.run(ARGV)", 'build'
+    script "build/gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemspec << {
+    file('build').mkdirs();
+    file('build/gemspec').write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Hitoshi Tanaka"]
+  spec.homepage      = "https://github.com/hito4t/embulk-parser-jdbc-schema-csv"
+  spec.summary       = "This Embulk plugin extends CSV parser to define columns based on database meta data."
+  spec.licenses      = ["Apache 2.0"]
+  spec.files         = `git ls-files`.split("\n").grep(%r"^(?!\.)").grep(%r"^(?!depends/)") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+end
+/$)
+}
+
+
+task gempush << {
+    "gem push pkg/embulk-parser-jdbc-schema-csv-${project.version}.gem".execute().waitFor()
+}

data/build.xml

@@ -0,0 +1,7 @@
+<project name="embulk-parser-jdbc-schema-csv" default="jar" basedir="./">
+  <target name="jar" depends="" description="embulk-parser-jdbc-schema-csv.jarファイルを作成します。">
+    <mkdir dir="${basedir}/jar" />
+    <jar destfile="${basedir}/jar/embulk-parser-jdbc-schema-csv-x.jar" basedir="${basedir}/out/main">
+    </jar>
+  </target>
+</project>

data/lib/embulk/parser/jdbc-schema-csv.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "jdbc-schema-csv", "org.embulk.parser.JdbcSchemaCsvParser",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/parser/JdbcInputPluginHelper.java

@@ -0,0 +1,54 @@
+package org.embulk.parser;
+
+import java.lang.reflect.Method;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Schema;
+
+public class JdbcInputPluginHelper
+{
+    private final InputPlugin plugin;
+	private final Class<?> abstractJdbcInputPluginClass;
+
+	public JdbcInputPluginHelper(InputPlugin plugin) throws ClassNotFoundException
+	{
+		this.plugin = plugin;
+		abstractJdbcInputPluginClass = plugin.getClass().getClassLoader().loadClass("org.embulk.input.jdbc.AbstractJdbcInputPlugin");
+	}
+
+	public Schema getSchema(ConfigSource config) throws Exception
+	{
+		Class<?> taskClass = getTaskClass();
+		Object task = config.loadConfig(taskClass);
+
+		try (AutoCloseable connection = newConnection(task)) {
+			return setupTask(connection, task);
+		}
+	}
+
+	private Class<?> getTaskClass() throws Exception
+	{
+		Method method = abstractJdbcInputPluginClass.getDeclaredMethod("getTaskClass");
+		method.setAccessible(true);
+		return (Class<?>)method.invoke(plugin);
+	}
+
+	private AutoCloseable newConnection(Object task) throws Exception
+	{
+		Class<?> taskClass =  plugin.getClass().getClassLoader().loadClass("org.embulk.input.jdbc.AbstractJdbcInputPlugin$PluginTask");
+		Method method = abstractJdbcInputPluginClass.getDeclaredMethod("newConnection", taskClass);
+		method.setAccessible(true);
+		return (AutoCloseable)method.invoke(plugin, task);
+	}
+
+	public Schema setupTask(Object connection, Object task) throws Exception
+	{
+		Class<?> connectionClass =  plugin.getClass().getClassLoader().loadClass("org.embulk.input.jdbc.JdbcInputConnection");
+		Class<?> taskClass =  plugin.getClass().getClassLoader().loadClass("org.embulk.input.jdbc.AbstractJdbcInputPlugin$PluginTask");
+		Method method = abstractJdbcInputPluginClass.getDeclaredMethod("setupTask", connectionClass, taskClass);
+		method.setAccessible(true);
+		return (Schema)method.invoke(plugin, connection, task);
+	}
+
+}

data/src/main/java/org/embulk/parser/JdbcSchemaCsvParser.java

@@ -0,0 +1,59 @@
+package org.embulk.parser;
+
+
+import java.util.List;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.standards.CsvParserPlugin;
+import org.slf4j.Logger;
+
+public class JdbcSchemaCsvParser extends CsvParserPlugin
+{
+    private final Logger logger = Exec.getLogger(JdbcSchemaCsvParser.class);
+
+    public static interface PluginTask extends CsvParserPlugin.PluginTask
+    {
+    	@Override
+    	public SchemaConfig getSchemaConfig();
+
+    }
+
+	@Override
+	public void transaction(ConfigSource config, final Control control)
+	{
+		// to avoid config error
+		config.set("columns", java.util.Collections.emptyList());
+		ExecSession session = Exec.session();
+		ConfigSource child = config.getNested("schema");
+		String type = child.get(String.class, "type");
+		InputPlugin input = session.newPlugin(InputPlugin.class, new PluginType(type));
+
+		try {
+			JdbcInputPluginHelper helper = new JdbcInputPluginHelper(input);
+			final Schema schemaFromJdbc = helper.getSchema(child);
+			for (int i = 0; i < schemaFromJdbc.getColumnCount(); i++) {
+				Column column = schemaFromJdbc.getColumn(i);
+				logger.info(String.format("column %d : name = %s, type = %s", i + 1, column.getName(), column.getType()));
+			}
+
+			super.transaction(config, new Control() {
+				@Override
+				public void run(TaskSource taskSource, Schema schema) {
+					control.run(taskSource, schemaFromJdbc);
+				}
+			});
+
+		} catch (Exception e) {
+			throw new RuntimeException(e);
+		}
+	}
+
+}

data/src/test/java/org/embulk/parser/JdbcSchemaCsvParserTest.java

@@ -0,0 +1,62 @@
+package org.embulk.parser;
+
+import static org.junit.Assert.assertEquals;
+
+import java.nio.charset.Charset;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.embulk.input.MySQLInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.test.EmbulkPluginTester;
+import org.embulk.test.TestExtension;
+import org.junit.Test;
+
+public class JdbcSchemaCsvParserTest
+{
+
+/*
+Prepare MySQL table:
+
+create table embulk_test;
+grant all on embulk_test.* to embulk_user@"%" identified by 'embulk_pass';
+
+create table embulk_test.input_test (
+    id        bigint,
+    name      char(8),
+    value     double,
+    creation  timestamp
+);
+*/
+
+	private static EmbulkPluginTester tester = new EmbulkPluginTester(ParserPlugin.class, "jdbc-schema-csv", JdbcSchemaCsvParser.class);
+	static {
+		TestExtension.addPlugin(InputPlugin.class, "mysql", MySQLInputPlugin.class);
+	}
+
+	@Test
+	public void testCsv() throws Exception
+	{
+		test("yml/csv.yml");
+	}
+
+	@Test
+	public void testJdbcSchemaCsv() throws Exception
+	{
+		test("yml/jdbc-csv.yml");
+	}
+
+	private void test(String yml) throws Exception
+	{
+		tester.run(yml);
+
+		FileSystem fs = FileSystems.getDefault();
+		List<String> lines = Files.readAllLines(fs.getPath("result.000.00.tsv"), Charset.forName("UTF-8"));
+		assertEquals(2, lines.size());
+		assertEquals("1\ttest\t123.4\t2015-04-27 11:23:45", lines.get(0));
+		assertEquals("2\tsample\t-1.0\t2015-12-31 23:59:59", lines.get(1));
+	}
+}

data/src/test/java/org/embulk/test/EmbulkPluginTester.java

@@ -0,0 +1,52 @@
+package org.embulk.test;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.embulk.command.Runner;
+
+public class EmbulkPluginTester {
+
+	public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
+	{
+		TestExtension.addPlugin(iface, name, impl);
+	}
+
+	public void run(String ymlPath) throws Exception
+	{
+		Runner runner = new Runner("{}");
+		runner.run(convert(ymlPath));
+	}
+
+	private String convert(String yml) throws Exception
+	{
+		File rootPath = new File(EmbulkPluginTester.class.getResource("/resource.txt").toURI()).getParentFile();
+		File ymlPath = new File(EmbulkPluginTester.class.getResource("/" + yml).toURI());
+		File tempYmlPath = new File(ymlPath.getParentFile(), "temp-" + ymlPath.getName());
+		Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
+		try (BufferedReader reader = new BufferedReader(new FileReader(ymlPath))) {
+			try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempYmlPath))) {
+				String line;
+				while ((line = reader.readLine()) != null) {
+					Matcher matcher = pathPrefixPattern.matcher(line);
+					if (matcher.matches()) {
+						int group = 2;
+						writer.write(line.substring(0, matcher.start(group)));
+						writer.write(new File(rootPath, matcher.group(group)).getAbsolutePath());
+						writer.write(line.substring(matcher.end(group)));
+					} else {
+						writer.write(line);
+					}
+					writer.newLine();
+				}
+			}
+		}
+		return tempYmlPath.getAbsolutePath();
+	}
+
+}

data/src/test/java/org/embulk/test/TestExtension.java

@@ -0,0 +1,52 @@
+package org.embulk.test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.InjectedPluginSource;
+import org.embulk.spi.Extension;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+
+public class TestExtension implements Extension
+{
+    private static class PluginDefinition
+    {
+        public final Class<?> iface;
+        public final String name;
+        public final Class<?> impl;
+
+        public PluginDefinition(Class<?> iface, String name, Class<?> impl)
+        {
+            this.iface = iface;
+            this.name = name;
+            this.impl = impl;
+        }
+    }
+
+    private static List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
+
+    public static void addPlugin(Class<?> iface, String name, Class<?> impl)
+    {
+        plugins.add(new PluginDefinition(iface, name, impl));
+    }
+
+	@Override
+	public List<Module> getModules(ConfigSource configsource) {
+		Module module = new Module() {
+
+			@Override
+			public void configure(Binder binder) {
+                for (PluginDefinition plugin : plugins) {
+                    InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
+                }
+			}
+		};
+		return ImmutableList.of(module);
+	}
+
+}

data/src/test/resources/META-INF/services/org.embulk.spi.Extension

@@ -0,0 +1 @@
+org.embulk.test.TestExtension

data/src/test/resources/data/test.csv

@@ -0,0 +1,2 @@
+1,test,123.4,2015-4-27 11:23:45
+2,sample,-1,2015-12-31 23:59:59

data/src/test/resources/yml/csv.yml

@@ -0,0 +1,20 @@
+in:
+  type: file
+  path_prefix: 'data/test.csv'
+  parser:
+    type: csv
+    delimiter: ","
+    header_line: false
+    columns:
+    - {name: id, type: long}
+    - {name: name, type: string}
+    - {name: value, type: double}
+    - {name: creation, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+out:
+  type: file
+  path_prefix: result
+  file_ext: .tsv
+  formatter:
+    type: csv
+    delimiter: "\t"
+    header_line: false

data/src/test/resources/yml/jdbc-csv-jdbc-alias.yml

@@ -0,0 +1,16 @@
+in:
+  type: file
+  path_prefix: 'data/test.csv'
+  parser:
+    type: jdbc-schema-csv
+    delimiter: ","
+    header_line: false
+    schema: &OUT
+      type: mysql
+      host: localhost
+      database: embulk_test
+      user: embulk_user
+      password: embulk_pass
+      table: input_test
+      mode:  insert
+out: *OUT

data/src/test/resources/yml/jdbc-csv-jdbc.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: 'data/test.csv'
+  parser:
+    type: jdbc-schema-csv
+    delimiter: ","
+    header_line: false
+    schema:
+      type: mysql
+      host: localhost
+      database: embulk_test
+      user: embulk_user
+      password: embulk_pass
+      table: input_test
+out:
+  type: mysql
+  host: localhost
+  database: embulk_test
+  user: embulk_user
+  password: embulk_pass
+  table: input_test
+  mode: insert

data/src/test/resources/yml/jdbc-csv.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: 'data/test.csv'
+  parser:
+    type: jdbc-schema-csv
+    delimiter: ","
+    header_line: false
+    schema:
+      type: mysql
+      host: localhost
+      database: embulk_test
+      user: embulk_user
+      password: embulk_pass
+      table: input_test
+out:
+  type: file
+  path_prefix: result
+  file_ext: .tsv
+  formatter:
+    type: csv
+    delimiter: "\t"
+    header_line: false

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: embulk-parser-jdbc-schema-csv
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Hitoshi Tanaka
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-04-27 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- build.gradle
+- build.xml
+- lib/embulk/parser/jdbc-schema-csv.rb
+- src/main/java/org/embulk/parser/JdbcInputPluginHelper.java
+- src/main/java/org/embulk/parser/JdbcSchemaCsvParser.java
+- src/test/java/org/embulk/parser/JdbcSchemaCsvParserTest.java
+- src/test/java/org/embulk/test/EmbulkPluginTester.java
+- src/test/java/org/embulk/test/TestExtension.java
+- src/test/resources/META-INF/services/org.embulk.spi.Extension
+- src/test/resources/data/test.csv
+- src/test/resources/resource.txt
+- src/test/resources/yml/csv.yml
+- src/test/resources/yml/jdbc-csv-jdbc-alias.yml
+- src/test/resources/yml/jdbc-csv-jdbc.yml
+- src/test/resources/yml/jdbc-csv.yml
+- classpath/embulk-parser-jdbc-schema-csv-0.0.1.jar
+homepage: https://github.com/hito4t/embulk-parser-jdbc-schema-csv
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: This Embulk plugin extends CSV parser to define columns based on database meta data.
+test_files: []