embulk-parser-csv_with_schema_file 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9deedb78f712bd47f5c8f70d2ef2e9044551a50a
-  data.tar.gz: 9b6c9c9608f42ec7261f37d81954f7cea979627d
+  metadata.gz: 31db0959f10ea5f4ed7dd1cf502966899eb60e36
+  data.tar.gz: 9e6b08b3f90978ec99a74d2c1630ecef31b31a3f
 SHA512:
-  metadata.gz: 0540e1b7302979e29b6f77f10c5a18731197e3fa2f0e8a8a2edc761a3212de3298db205e7fb4a267fe9901cf7e965b17482b42f1cc92c3681ba24e5756774706
-  data.tar.gz: 7e59b5115f104eedcbf19fa40fd9923ab8df3f10cd76c4bcb3904d08c47f956851e6e2a82b279639b7fc9a24cf5409108dfa5d2b7b4f23cb4901708c6dd2a8b7
+  metadata.gz: 965a3fe66e997f53401325bda3f7fa8a1ec55722a5416ddd782f10e5dfb3ce1343267f2e9ff2584b76024da8a23cb24074dafd3bbc7c932a8bfd85df4edc9d62
+  data.tar.gz: 309e34839bb2572a5d800b7d00d7022d43b7d612ac36442d96911b0c9191af81d5336febb0c8bba1afabc3b1a3a3f25a4bf75c0b721b732112530a83508ceb5a

data/README.md

@@ -18,7 +18,7 @@ $ embulk gem install embulk-parser-csv_with_schema_file
 ## Configuration
 
 - **schema_path**: schema file path. json. (string, required)
-- **columns**: this config is ignored in this plugin.
+- **columns**: Optional. If exists, overwrite in schema file that same column name. (hash, default: `[]`)
 - other configs extends csv parser. see : http://www.embulk.org/docs/built-in.html#csv-parser-plugin
 
 ## Example

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.0.1"
+version = "0.0.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/parser/csv_with_schema_file/CsvParserWithSchemaFilePlugin.java

@@ -2,6 +2,7 @@ package org.embulk.parser.csv_with_schema_file;
 
 import java.io.File;
 import java.util.List;
+import java.util.Map;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -12,11 +13,14 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.type.Type;
 import org.embulk.standards.CsvParserPlugin;
+import org.jruby.org.objectweb.asm.TypeReference;
 import org.slf4j.Logger;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
 
 public class CsvParserWithSchemaFilePlugin
         extends CsvParserPlugin
@@ -29,6 +33,10 @@ public class CsvParserWithSchemaFilePlugin
 		@Config("schema_path")
 		String getSchemaPath();
 		
+		@Config("schema_class")
+		@ConfigDefault("\"Schema\"")
+		String getSchemaClass();
+		
         @Config("columns")
         @ConfigDefault("[]")
         @Override
@@ -42,29 +50,92 @@ public class CsvParserWithSchemaFilePlugin
 
     	log.info("default timestamp format : {}", schemaTask.getDefaultTimestampFormat() );
 
-    	config.set("columns", getSchemaConfig(schemaTask.getSchemaPath(), config));
+    	config.set("columns", getSchemaConfig(schemaTask, config));
     	
         super.transaction(config, control);
     }
     
-    public SchemaConfig getSchemaConfig(String path, ConfigSource config) {
+    public static class ColumnConfigTemp
+    {
+    	
+        public ColumnConfigTemp() {
+			super();
+		}
+        
+		private String name;
+        private Type type;
+        private String format;
+		public String getName() {
+			return name;
+		}
+		public void setName(String name) {
+			this.name = name;
+		}
+		public Type getType() {
+			return type;
+		}
+		public void setType(Type type) {
+			this.type = type;
+		}
+		public String getFormat() {
+			return format;
+		}
+		public void setFormat(String format) {
+			this.format = format;
+		}
+        
+    }
+    /**
+     * if "columns" in embulk config file, use that 
+     * @param schemaTask
+     * @param config
+     * @return
+     */
+    public SchemaConfig getSchemaConfig(BqPluginTask schemaTask, ConfigSource config) {
+    	Map<String, ColumnConfig> map = Maps.newHashMap();
+    	if(schemaTask.getSchemaConfig() != null && schemaTask.getSchemaConfig().isEmpty() == false) {
+    		// schemaTask.getSchemaConfig().getColumns().stream().collect(Collectors.toMap(x-> x.getName(), y -> y, (a,b) -> b));
+    		for(ColumnConfig c : schemaTask.getSchemaConfig().getColumns()) {
+    			map.put(c.getName(), c);
+    		}
+    	}
     	List<ColumnConfig> columns = Lists.newArrayList();
-    	Schema schema = getSchemaFromFile(path);
-    	for(Column c : schema.getColumns()){
-    		columns.add(new ColumnConfig(c.getName(), c.getType(), config));
+    	if("SchemaConfig".equals(schemaTask.getSchemaClass().trim())) {
+    		//SchemaConfig schema = getSchemaFromFile(schemaTask.getSchemaPath(), SchemaConfig.class);
+    		ColumnConfigTemp[] mapList = getSchemaFromFile(schemaTask.getSchemaPath(), ColumnConfigTemp[].class);
+    		for(ColumnConfigTemp c : mapList){
+    			if(map.containsKey(c.getName())) {
+    				columns.add(map.get(c.getName()));
+    			}else {
+    				columns.add(new ColumnConfig(c.getName(), c.getType(), c.getFormat()));	
+    			}
+        	}
+    	}else {
+        	Schema schema = getSchemaFromFile(schemaTask.getSchemaPath(), Schema.class);
+        	for(Column c : schema.getColumns()){
+    			if(map.containsKey(c.getName())) {
+    				columns.add(map.get(c.getName()));
+    			}else {
+    				columns.add(new ColumnConfig(c.getName(), c.getType(), config));	
+    			}
+        	}
     	}
     	SchemaConfig conf = new SchemaConfig(columns);
+
+    	log.info("Final Config : {}", conf.toSchema());
+    	
     	return conf;
     }
     
-    public Schema getSchemaFromFile(String path) {
+    
+    
+    public <T> T getSchemaFromFile(String path, Class<T> cls) {
     	ObjectMapper mapper = new ObjectMapper();
 		try {
-			Schema schema = mapper.readValue(new File(path), Schema.class);
+			T schema = mapper.readValue(new File(path), cls);
 			return schema;
 		} catch (Exception e) {
-			throw new RuntimeException("error when parse schema file : " + path,e);
-			
+			throw new RuntimeException("error when parse Schema : <"+cls+"> file : " + path,e);
 		}
     }
 }

data/src/test/java/org/embulk/parser/csv_with_schema_file/TestCsvParserWithSchemaFilePlugin.java

@@ -1,11 +1,13 @@
 package org.embulk.parser.csv_with_schema_file;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assume.assumeNotNull;
 
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
 
+import org.codehaus.plexus.util.FileUtils;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
@@ -30,7 +32,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.collect.Lists;
 
 public class TestCsvParserWithSchemaFilePlugin
@@ -57,9 +62,10 @@ public class TestCsvParserWithSchemaFilePlugin
     public void createResources() throws IOException 
     {
     	config = Exec.newConfigSource();
-    	config.set("schema_path", "D:\\Temp\\gcstemp2\\csv_schema.json");
     	config.set("default_timestamp_format", "%Y-%m-%d %H:%M:%S %z");
-    	//config.set("columns","");
+    	
+    	SchemaConfig con = createSchemaConfigForTest();
+    	config.set("schema_path", createSchemaFileForTest(con));
     	        
         plugin = new CsvParserWithSchemaFilePlugin();
         runner = new FileInputRunner(runtime.getInstance(LocalFileInputPlugin.class));
@@ -68,12 +74,14 @@ public class TestCsvParserWithSchemaFilePlugin
         assumeNotNull(
                       config.get(String.class, "schema_path")
                       );
+        
+        
     }
     
     @Test
     public void testParseSchema(){
     	BqPluginTask task = config.loadConfig(BqPluginTask.class);
-    	Schema schema = plugin.getSchemaFromFile(task.getSchemaPath());
+    	Schema schema = plugin.getSchemaFromFile(task.getSchemaPath(), Schema.class);
     	log.info("{}",schema.getColumns());
     	assumeNotNull(schema.getColumns());
     }
@@ -81,6 +89,7 @@ public class TestCsvParserWithSchemaFilePlugin
     public String toJson(Object o){
 		ObjectMapper mapper = new ObjectMapper();
 		try {
+			mapper.enable(SerializationFeature.INDENT_OUTPUT);
 			String jsonString = mapper.writeValueAsString(o);
 			return jsonString;
 		} catch (JsonProcessingException e) {
@@ -88,26 +97,97 @@ public class TestCsvParserWithSchemaFilePlugin
 			return null;
 		}
     }
-    @Test
-    public void testColumnConfig(){
+    
+    public SchemaConfig createSchemaConfigForTest() {
     	List<ColumnConfig> l = Lists.newArrayList();
+    	ConfigSource emptySource = Exec.newConfigSource();
+    	l.add( new ColumnConfig("idx",Types.LONG, emptySource));
+    	l.add( new ColumnConfig("title",Types.STRING, emptySource));
+    	l.add( new ColumnConfig("regdate",Types.TIMESTAMP, "%Y-%m-%d %H:%M:%S"));
+    	SchemaConfig con = new SchemaConfig(l);
+    	return con;
+    }
+    
+    
+    public File createSchemaConfigFileForTest(SchemaConfig con) throws IOException {
+    	//con.getColumns().stream().map(x -> x.getConfigSource().getObjectNode()).collect(Collectors.toList());
+    	List<ObjectNode> jsonList = Lists.newArrayList();
+    	for(ColumnConfig c : con.getColumns()) {
+    		jsonList.add( c.getConfigSource().getObjectNode() );
+    	}
+    	String configString = toJson(jsonList);
+    	log.info("SchemaConfig String : {}",configString);
+    	    	    	
+    	File f = File.createTempFile("embulk-test-schemaconfig", ".json");
+		FileUtils.fileWrite(f, configString);
+    	f.deleteOnExit();
+    	
+    	return f;
+    }
+
+    public File createSchemaFileForTest(SchemaConfig con) throws IOException {
+    	String schemaString = toJson(con.toSchema());
+    	log.debug("Schema String : {}",schemaString);
+    	File f = File.createTempFile("embulk-test-schema", ".json");
+		FileUtils.fileWrite(f, schemaString);
+    	f.deleteOnExit();
+    	return f;
+    }
+
+
+    @Test
+    public void testSchemaFile() throws IOException{
+    	BqPluginTask task = config.loadConfig(BqPluginTask.class);
+    	SchemaConfig finalconfig = plugin.getSchemaConfig(task,config);
+    	log.info("final config : {}",toJson(finalconfig.getColumns()));
+
+    	assertEquals(finalconfig.getColumn(0).getName(), "idx");
+    	assertEquals(finalconfig.getColumn(1).getName(), "title");
+    	assertEquals(finalconfig.getColumn(2).getName(), "regdate");
+    }
+    
+    @Test
+    public void testSchemaClass() throws IOException{
+    	SchemaConfig con = createSchemaConfigForTest();    	
+    	File f = createSchemaConfigFileForTest(con);
+    	
+    	ConfigSource c = config.deepCopy();
+    	c.set("schema_path",f);
+    	c.set("schema_class","SchemaConfig");
+    	
+    	BqPluginTask task = c.loadConfig(BqPluginTask.class);
+    	SchemaConfig finalconfig = plugin.getSchemaConfig(task,c);
+    	log.info("final config : {}",toJson(finalconfig.getColumns()));
+    }
+    
+    @Test
+    public void testMergeOriginalConfig() throws IOException{
+       	List<ColumnConfig> l = Lists.newArrayList();
     	//ColumnConfig c = new ColumnConfig(config);
     	ConfigSource emptySource = Exec.newConfigSource();
+    	l.add( new ColumnConfig("regdate",Types.TIMESTAMP, "%Y-%m-%d %H:%M:%S.%H %z"));
+    	SchemaConfig originalSchemaConfig = new SchemaConfig(l);
     	
-    	l.add( new ColumnConfig("aa",Types.LONG, emptySource));
-    	l.add( new ColumnConfig("bb",Types.STRING, emptySource));
-    	l.add( new ColumnConfig("cc",Types.TIMESTAMP, "%Y-%m-%d %H:%M:%S"));
+    	SchemaConfig con = createSchemaConfigForTest();    	
+    	File f = createSchemaFileForTest(con);
     	
-    	log.info("list : {}", toJson(l));
+    	ConfigSource c = config.deepCopy();
+    	c.set("columns", originalSchemaConfig );
+    	c.set("schema_path",f);
+    	BqPluginTask task = c.loadConfig(BqPluginTask.class);
+    	SchemaConfig finalconfig = plugin.getSchemaConfig(task,config);
     	
-    	SchemaConfig con = new SchemaConfig(l);
-    	log.info("SchemaConfig : {}", toJson(con));
+    	log.info("final config : {}",toJson(finalconfig));
+    	log.info("final config to schema : {}", toJson(finalconfig.toSchema()));
+    	
+    	assertEquals(finalconfig.getColumn(0).getName(), con.getColumn(0).getName());
+    	assertEquals(finalconfig.getColumn(1).getName(), con.getColumn(1).getName());
+    	assertEquals(finalconfig.getColumn(2).getName(), con.getColumn(2).getName());
     	
-    	log.info("{}", toJson(con.toSchema()) );
-    	log.info("{}", con.toSchema().getColumns());
+    	assertEquals(finalconfig.getColumn(2).getOption().get(String.class, "format"), "%Y-%m-%d %H:%M:%S.%H %z");
+    	assertEquals(con.getColumn(2).getOption().get(String.class, "format"), "%Y-%m-%d %H:%M:%S");
     }
     
-
     @Test
     public void testInit(){
     	BqPluginTask task = config.loadConfig(BqPluginTask.class);
@@ -121,9 +201,8 @@ public class TestCsvParserWithSchemaFilePlugin
     	CsvParserPlugin.PluginTask parents = config.loadConfig(CsvParserPlugin.PluginTask.class);
     	log.info("{}", parents);
     }
-    
 
-    @Test
+    // @Test
     public void testParserDefaultrConfig() throws IOException{
     	File f = new File("D:\\temp\\embulk_test.yml");
     	ConfigSource cpn = new ConfigLoader(Exec.session().getModelManager()).fromYamlFile(f).getNested("in").getNested("parser");

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-csv_with_schema_file
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - jo8937
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-20 00:00:00.000000000 Z
+date: 2017-11-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,7 +59,7 @@ files:
 - src/main/java/org/embulk/parser/csv_with_schema_file/CsvParserWithSchemaFilePlugin.java
 - src/test/java/org/embulk/parser/csv_with_schema_file/TestCsvParserWithSchemaFilePlugin.java
 - classpath/commons-compress-1.10.jar
-- classpath/embulk-parser-csv_with_schema_file-0.0.1.jar
+- classpath/embulk-parser-csv_with_schema_file-0.0.2.jar
 - classpath/embulk-standards-0.8.36.jar
 homepage: https://github.com/jo8937/embulk-parser-csv_with_schema_file
 licenses: