embulk-executor-mapreduce 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b2abda7db750f6c161ab8867474fdccfa67eb265
-  data.tar.gz: 8cfc89242d0a57368b5803db9b55e6494b6916e6
+  metadata.gz: 4461eebeecc53f99b9b9683d7553a585a87e1a1f
+  data.tar.gz: a019cd9224918ae2721482a9cf92c9c8148a05a6
 SHA512:
-  metadata.gz: 0e8b2f14207ec85d1ba60b531cc5876c90689a2b259f76a5edb57d8b50d025bd5e7151307be0381d82e813b5f263d5326e39cbccaa56238d8aa3f1171e8a21fd
-  data.tar.gz: 645417db32cc29813fee20f2175a2858485596159fb80c31e77c1f45d1c715f40b239f7417ef0caae927f75428b8f93175eb477281d620fe38edae694e76f66a
+  metadata.gz: 91e107ce10160fc097930b139f07b59dcb80b1201dde0723cc302fd1e142a283ad8a817d7518ac7684f0b066ed55537c20ae0af5446230eb0f63026d9bf7e21d
+  data.tar.gz: bc045316fedf83de62e34bbf9304152680d90e46ec6fa885fc054aca50d9a967b2d8f8eb3fa86e7c685badba7e3f15834f985fee923c5bf54c9f5197f43a68fb

data/src/main/java/org/embulk/executor/mapreduce/BufferedPagePartitioner.java

@@ -71,6 +71,15 @@ public class BufferedPagePartitioner
             }
         }
 
+        public void jsonColumn(Column column)
+        {
+            if (source.isNull(column)) {
+                destination.setNull(column);
+            } else {
+                destination.setJson(column, source.getJson(column));
+            }
+        }
+
         public void timestampColumn(Column column)
         {
             if (source.isNull(column)) {

data/src/main/java/org/embulk/executor/mapreduce/EmbulkPartitioningMapReduce.java

@@ -254,6 +254,7 @@ public class EmbulkPartitioningMapReduce
             this.output = output;
         }
 
+        @Override
         public ConfigDiff transaction(ConfigSource config,
                 Schema schema, int taskCount,
                 OutputPlugin.Control control)
@@ -262,6 +263,7 @@ public class EmbulkPartitioningMapReduce
             throw new RuntimeException("");
         }
 
+        @Override
         public ConfigDiff resume(TaskSource taskSource,
                 Schema schema, int taskCount,
                 OutputPlugin.Control control)
@@ -270,6 +272,7 @@ public class EmbulkPartitioningMapReduce
             throw new RuntimeException("");
         }
 
+        @Override
         public void cleanup(TaskSource taskSource,
                 Schema schema, int taskCount,
                 List<TaskReport> successTaskReports)
@@ -278,6 +281,7 @@ public class EmbulkPartitioningMapReduce
             throw new RuntimeException("");
         }
 
+        @Override
         public TransactionalPageOutput open(TaskSource taskSource, final Schema schema, int taskIndex)
         {
             return new TransactionalPageOutput() {

data/src/main/java/org/embulk/executor/mapreduce/PageWritable.java

@@ -7,6 +7,13 @@ import java.util.List;
 import java.util.ArrayList;
 import org.apache.hadoop.io.Writable;
 import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.DataOutputOutputStream;
+import org.msgpack.value.Value;
+import org.msgpack.value.ImmutableValue;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessageBufferPacker;
+import org.msgpack.core.MessageUnpacker;
+import org.msgpack.core.buffer.MessageBuffer;
 import org.embulk.spi.Buffer;
 import org.embulk.spi.Page;
 import static java.nio.charset.StandardCharsets.UTF_8;
@@ -40,6 +47,22 @@ public class PageWritable
         for (String s : stringReferences) {
             out.writeUTF(s);
         }
+
+        List<ImmutableValue> valueReferences = page.getValueReferences();
+        WritableUtils.writeVInt(out, valueReferences.size());
+        for (Value value : valueReferences) {
+            MessageBufferPacker packer = MessagePack.newDefaultBufferPacker();  // TODO reuse allocated buffer
+            value.writeTo(packer);
+            List<MessageBuffer> buffers = packer.toBufferList();
+            int size = 0;
+            for (MessageBuffer b : buffers) {
+                size += b.size();
+            }
+            WritableUtils.writeVInt(out, size);
+            for (MessageBuffer b : buffers) {
+                out.write(b.array(), b.arrayOffset(), b.size());
+            }
+        }
     }
 
     @Override
@@ -51,13 +74,31 @@ public class PageWritable
         Buffer buffer = Buffer.wrap(bytes);
 
         int stringCount = WritableUtils.readVInt(in);
-        List<String> strings = new ArrayList<String>(stringCount);
+        List<String> strings = new ArrayList<>(stringCount);
         for (int i=0; i < stringCount; i++) {
             strings.add(in.readUTF());
         }
 
+        int valueCount = WritableUtils.readVInt(in);
+        List<ImmutableValue> values = new ArrayList<>(valueCount);
+        byte[] b = new byte[32 * 1024];
+        for (int i=0; i < valueCount; i++) {
+            int size = WritableUtils.readVInt(in);
+            if (b.length < size) {
+                int ns = b.length;
+                while (ns < size) {
+                    ns *= 2;
+                }
+                b = new byte[ns];
+            }
+            in.readFully(b, 0, size);
+            MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(b, 0, size);
+            values.add(unpacker.unpackValue());
+        }
+
         Page newPage = Page.wrap(buffer);
         newPage.setStringReferences(strings);
+        newPage.setValueReferences(values);
         if (page != null) {
             page.release();
         }

data/src/test/java/org/embulk/executor/mapreduce/TestMapReduceExecutor.java

@@ -25,16 +25,19 @@ import org.slf4j.impl.Log4jLoggerFactory;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.file.Files;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Random;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -62,36 +65,42 @@ public class TestMapReduceExecutor
         bootstrap.setSystemConfig(systemConfig);
         bootstrap.overrideModules(getModuleOverrides(systemConfig));
         embulk = bootstrap.initialize();
+
+        new File("tmp").mkdirs();
     }
 
     @Test
     public void testEmbulkMapper()
             throws Exception
     {
+        new File("tmp/embulk_mapred_output.000.00.csv").delete();
+        new File("tmp/embulk_mapred_output.001.00.csv").delete();
         ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_config.yml");
         embulk.run(config);
         assertFileContent(
                 Lists.newArrayList(
-                        "fixtures/csv/sample1.csv",
-                        "fixtures/csv/sample1.csv"),
+                        "src/test/resources/fixtures/csv/sample1.csv",
+                        "src/test/resources/fixtures/csv/sample1.csv"),
                 Lists.newArrayList(
-                        "fixtures/csv/embulk_mapred_output.000.00.csv",
-                        "fixtures/csv/embulk_mapred_output.001.00.csv"));
+                        "tmp/embulk_mapred_output.000.00.csv",
+                        "tmp/embulk_mapred_output.001.00.csv"));
     }
 
     @Test
     public void testEmbulkPartitioningMapperReducer()
             throws Exception
     {
+        new File("tmp/embulk_mapred_partitioning_output.000.00.csv").delete();
+        new File("tmp/embulk_mapred_partitioning_output.001.00.csv").delete();
         ConfigSource config = loadConfigSource(embulk.newConfigLoader(), "config/embulk_mapred_partitioning_config.yml");
         embulk.run(config);
         assertFileContent(
                 Lists.newArrayList(
-                        "fixtures/csv/sample1.csv",
-                        "fixtures/csv/sample1.csv"),
+                        "src/test/resources/fixtures/csv/sample1.csv",
+                        "src/test/resources/fixtures/csv/sample1.csv"),
                 Lists.newArrayList(
-                        "fixtures/csv/embulk_mapred_partitioning_output.000.00.csv",
-                        "fixtures/csv/embulk_mapred_partitioning_output.001.00.csv"));
+                        "tmp/embulk_mapred_partitioning_output.000.00.csv",
+                        "tmp/embulk_mapred_partitioning_output.001.00.csv"));
     }
 
     @Test
@@ -104,7 +113,8 @@ public class TestMapReduceExecutor
             fail();
         }
         catch (Throwable t) {
-            assertTrue(t instanceof ConfigException);
+            assertTrue(t instanceof PartialExecutionException);
+            assertTrue(t.getCause() instanceof ConfigException);
         }
     }
 
@@ -118,7 +128,8 @@ public class TestMapReduceExecutor
             fail();
         }
         catch (Throwable t) {
-            assertTrue(t instanceof ConfigException);
+            assertTrue(t instanceof PartialExecutionException);
+            assertTrue(t.getCause() instanceof ConfigException);
         }
     }
 
@@ -132,7 +143,8 @@ public class TestMapReduceExecutor
             fail();
         }
         catch (Throwable t) {
-            assertTrue(t instanceof ConfigException);
+            assertTrue(t instanceof PartialExecutionException);
+            assertTrue(t.getCause() instanceof ConfigException);
         }
     }
 
@@ -146,7 +158,9 @@ public class TestMapReduceExecutor
             fail();
         }
         catch (Throwable t) {
-            assertTrue(t.getCause() instanceof FileNotFoundException);
+            assertTrue(t instanceof PartialExecutionException);
+            assertTrue(t.getCause() instanceof RuntimeException);
+            assertTrue(t.getCause().getCause() instanceof FileNotFoundException);
         }
     }
 
@@ -273,6 +287,7 @@ public class TestMapReduceExecutor
     }
 
     private static void assertFileContent(List<String> inputFiles, List<String> outputFiles)
+        throws IOException
     {
         List<List<String>> inputRecords = getRecords(inputFiles);
         Collections.sort(inputRecords, new RecordComparator());
@@ -294,6 +309,7 @@ public class TestMapReduceExecutor
     }
 
     private static List<List<String>> getRecords(List<String> files)
+        throws IOException
     {
         List<List<String>> records = new ArrayList<>();
 
@@ -327,8 +343,8 @@ public class TestMapReduceExecutor
     }
 
     private static BufferedReader newReader(String filePath)
+        throws IOException
     {
-        InputStream in = new BufferedInputStream(TestMapReduceExecutor.class.getClassLoader().getResourceAsStream(filePath));
-        return new BufferedReader(new InputStreamReader(in));
+        return Files.newBufferedReader(new File(filePath).toPath(), UTF_8);
     }
 }

data/src/test/resources/config/embulk_mapred_config.yml

@@ -19,8 +19,8 @@ in:
     newline: CRLF
     type: csv
     delimiter: ','
-    quote: ''
-    escape: ''
+    quote: '"'
+    escape: '"'
     skip_header_lines: 1
     columns:
     - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S"}
@@ -34,13 +34,16 @@ in:
     - {name: size, type: long}
     - {name: d, type: double}
     - {name: flag, type: boolean}
+    - {name: v_json, type: json}
 out:
   type: file
-  path_prefix: 'src/test/resources/fixtures/csv/embulk_mapred_output.'
+  path_prefix: 'tmp/embulk_mapred_output.'
   file_ext: 'csv'
   formatter:
     charset: UTF-8
     newline: CRLF
+    quote: '"'
+    escape: '"'
     type: csv
     column_options:
       timestamp: {format: '%Y-%m-%d %H:%M:%S'}

data/src/test/resources/config/embulk_mapred_partitioning_config.yml

@@ -25,8 +25,8 @@ in:
     newline: CRLF
     type: csv
     delimiter: ','
-    quote: ''
-    escape: ''
+    quote: '"'
+    escape: '"'
     skip_header_lines: 1
     columns:
     - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S"}
@@ -40,13 +40,16 @@ in:
     - {name: size, type: long}
     - {name: d, type: double}
     - {name: flag, type: boolean}
+    - {name: v_json, type: json}
 out:
   type: file
-  path_prefix: 'src/test/resources/fixtures/csv/embulk_mapred_partitioning_output.'
+  path_prefix: 'tmp/embulk_mapred_partitioning_output.'
   file_ext: 'csv'
   formatter:
     charset: UTF-8
     newline: CRLF
+    quote: '"'
+    escape: '"'
     type: csv
     column_options:
       timestamp: {format: '%Y-%m-%d %H:%M:%S'}

data/src/test/resources/fixtures/csv/sample1.csv

@@ -1,3 +1,3 @@
-timestamp,host,path,method,referer,code,agent,user,size,d,flag
-2014-10-02 22:15:39,84.186.29.187,/category/electronics,GET,/category/music,200,Mozilla/5.0,-,136,1.1,true
-2014-10-02 22:15:01,140.36.216.47,/category/music?from=10,GET,-,200,Mozilla/5.0,-,70,1.2,false
+timestamp,host,path,method,referer,code,agent,user,size,d,flag,v_json
+2014-10-02 22:15:39,84.186.29.187,/category/electronics,GET,/category/music,200,Mozilla/5.0,-,136,1.1,true,"{""k0"":""v0"",""k1"":""v1""}"
+2014-10-02 22:15:01,140.36.216.47,/category/music?from=10,GET,-,200,Mozilla/5.0,-,70,1.2,false,"[1,2,""3""]"

data/src/test/resources/fixtures/csv/sample2.csv

@@ -1,4 +1,3 @@
-timestamp,host,path,method,referer,code,agent,user,size,d,flag
-2014-10-02 22:15:39,84.186.29.187,/category/electronics,GET,/category/music,200,Mozilla/5.0,-,136,1.1,true
-2014-10-02 22:15:01,140.36.216.47,/category/music?from=10,GET,-,200,Mozilla/5.0,-,70,1.2,false
-
+timestamp,host,path,method,referer,code,agent,user,size,d,flag,v_json
+2014-10-02 22:15:39,84.186.29.187,/category/electronics,GET,/category/music,200,Mozilla/5.0,-,136,1.1,true,"{""k0"":""v0"",""k1"":""v1""}"
+2014-10-02 22:15:01,140.36.216.47,/category/music?from=10,GET,-,200,Mozilla/5.0,-,70,1.2,false,"[1,2,""3""]"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-executor-mapreduce
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-21 00:00:00.000000000 Z
+date: 2016-02-09 00:00:00.000000000 Z
 dependencies: []
 description: Executes tasks on Hadoop.
 email:
@@ -84,7 +84,7 @@ files:
 - classpath/curator-client-2.6.0.jar
 - classpath/curator-framework-2.6.0.jar
 - classpath/curator-recipes-2.6.0.jar
-- classpath/embulk-executor-mapreduce-0.2.4.jar
+- classpath/embulk-executor-mapreduce-0.2.5.jar
 - classpath/gson-2.2.4.jar
 - classpath/hadoop-annotations-2.6.0.jar
 - classpath/hadoop-auth-2.6.0.jar