embulk-filter-copy 0.0.2 → 0.1.0

data/src/main/java/org/embulk/filter/copy/forward/InForwardService.java

@@ -1,24 +1,25 @@
 package org.embulk.filter.copy.forward;
 
 import com.google.common.base.Optional;
+import com.google.common.util.concurrent.AbstractExecutionThreadService;
 import influent.EventStream;
 import influent.Tag;
 import influent.forward.ForwardCallback;
 import influent.forward.ForwardServer;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.filter.copy.util.ElapsedTime;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
+import org.embulk.filter.copy.spi.ElapsedTime;
 
 import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.Executor;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Consumer;
 
 public class InForwardService
+        extends AbstractExecutionThreadService
 {
     private final static Logger logger = Exec.getLogger(InForwardService.class);
 
@@ -55,7 +56,7 @@ public class InForwardService
     }
 
     public interface Task
-            extends ForwardBaseTask
+            extends ForwardParentTask
     {
         @Config("in_forward")
         @ConfigDefault("{}")
@@ -94,30 +95,19 @@ public class InForwardService
         return new Builder();
     }
 
-    // TODO: configure?
-    private static final String THREAD_NAME = "in_forward";
-    private static final int NUM_THREADS = 1;
-
     private final Task task;
     private final ForwardServer server;
-    private final ExecutorService callbackThread;
     private final AtomicBoolean shouldShutdown = new AtomicBoolean(false);
 
     private InForwardService(Task task, Consumer<EventStream> eventConsumer)
     {
         this.task = task;
-        this.callbackThread = Executors.newFixedThreadPool(
-                NUM_THREADS,
-                r -> new Thread(r, THREAD_NAME));
-        this.server = buildServer(task.getInForwardTask(), eventConsumer, callbackThread);
+        this.server = buildServer(task.getInForwardTask(), eventConsumer);
     }
 
-    private ForwardServer buildServer(InForwardTask t, Consumer<EventStream> eventConsumer, Executor callbackThread)
+    private ForwardServer buildServer(InForwardTask t, Consumer<EventStream> eventConsumer)
     {
-        ForwardServer.Builder builder = new ForwardServer.Builder(
-                ForwardCallback.ofSyncConsumer(
-                        wrapEventConsumer(eventConsumer),
-                        callbackThread));
+        ForwardServer.Builder builder = new ForwardServer.Builder(wrapEventConsumer(eventConsumer));
 
         builder.localAddress(t.getPort());
 
@@ -139,25 +129,27 @@ public class InForwardService
         if (t.getTcpNoDelayEnabled().isPresent()) {
             builder.tcpNoDelayEnabled(t.getTcpNoDelayEnabled().get());
         }
+        // TODO: builder.workerPoolSize(1);
 
         return builder.build();
     }
 
-    private Consumer<EventStream> wrapEventConsumer(Consumer<EventStream> eventConsumer)
+    private ForwardCallback wrapEventConsumer(Consumer<EventStream> eventConsumer)
     {
-        return eventStream ->
+        return ForwardCallback.of(es ->
         {
-            if (isShutdownTag(eventStream.getTag())) {
-                logger.info("Receive shutdown tag: {}", eventStream.getTag());
+            if (isShutdownTag(es.getTag())) {
+                logger.info("Receive shutdown tag: {}", es.getTag());
                 shouldShutdown.set(true);
             }
-            else if (isMessageTag(eventStream.getTag())) {
-                eventConsumer.accept(eventStream);
+            else if (isMessageTag(es.getTag())) {
+                eventConsumer.accept(es);
             }
             else {
-                throw new RuntimeException(String.format("Unknown Tag received: %s", eventStream.getTag().getName()));
+                throw new DataException(String.format("Unknown Tag received: %s", es.getTag().getName()));
             }
-        };
+            return CompletableFuture.completedFuture(null);
+        });
     }
 
     private boolean isShutdownTag(Tag tag)
@@ -170,18 +162,93 @@ public class InForwardService
         return tag.getName().contentEquals(task.getMessageTag());
     }
 
-    public void runUntilShouldShutdown()
+    @Override
+    protected void startUp()
+            throws Exception
     {
         server.start();
-        ElapsedTime.debugUntil(shouldShutdown::get, logger, "in_forward server", 1000L);
-        shutdown();
     }
 
-    private void shutdown()
+    @Override
+    protected void shutDown()
+            throws Exception
     {
-        ElapsedTime.debug(logger, "shutdown in_forward callback", callbackThread::shutdown);
-        CompletableFuture<Void> shutdown = server.shutdown();
-        ElapsedTime.debugUntil(() -> shutdown.isCancelled() || shutdown.isCompletedExceptionally() || shutdown.isDone(),
-                logger, "shutdown in_forward server", 1000L);
+        ElapsedTime.measureWithPolling(new ElapsedTime.Pollable<Void>() {
+            private CompletableFuture<Void> future;
+
+            @Override
+            public boolean poll()
+            {
+                return future.isCancelled() || future.isCompletedExceptionally() || future.isDone();
+            }
+
+            @Override
+            public Void getResult()
+            {
+                try {
+                    return future.get();
+                }
+                catch (InterruptedException | ExecutionException e) {
+                    logger.warn("InForwardService: Server Shutdown is failed.", e);
+                    return null;
+                }
+            }
+
+            @Override
+            public void onStart()
+            {
+                logger.info("InForwardService: Server Shutdown Start.");
+                this.future = server.shutdown();
+            }
+
+            @Override
+            public void onWaiting(long elapsedMillis)
+            {
+                logger.info("InForwardService: Server Shutdown Running. (Elapsed: {} ms)", elapsedMillis);
+            }
+
+            @Override
+            public void onFinished(long elapsedMillis)
+            {
+                logger.info("InForwardService: Server Shutdown Finish. (Elapsed: {} ms)", elapsedMillis);
+            }
+        });
+    }
+
+    @Override
+    protected void run()
+            throws Exception
+    {
+        ElapsedTime.measureWithPolling(new ElapsedTime.Pollable<Void>() {
+            @Override
+            public boolean poll()
+            {
+                return shouldShutdown.get();
+            }
+
+            @Override
+            public Void getResult()
+            {
+                return null;
+            }
+
+            @Override
+            public void onStart()
+            {
+                logger.info("InForwardService: Server Start.");
+            }
+
+            @Override
+            public void onWaiting(long elapsedMillis)
+            {
+                logger.info("InForwardService: Server Running. (Elapsed: {} ms)", elapsedMillis);
+            }
+
+            @Override
+            public void onFinished(long elapsedMillis)
+            {
+                logger.info("InForwardService: Server Shutdown. (Elapsed: {} ms)", elapsedMillis);
+            }
+        });
     }
 }

data/src/main/java/org/embulk/filter/copy/forward/OutForwardEventBuilder.java

@@ -1,135 +1,147 @@
 package org.embulk.filter.copy.forward;
 
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import org.embulk.spi.Column;
 import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
 import org.msgpack.value.Value;
+import org.embulk.filter.copy.spi.DataBuilder;
 
+import java.util.List;
 import java.util.Map;
 
 public class OutForwardEventBuilder
+        implements DataBuilder
 {
+    private final static String VALUES_KEY = "values";
+
     private final Schema schema;
-    private final TimestampFormatter timestampFormatter;
+    private final OutForwardService outForward;
 
-    private Map<String, Object> message;
+    private List<Object> values;
 
     public OutForwardEventBuilder(
             Schema schema,
-            TimestampFormatter timestampFormatter)
+            OutForwardService outForward)
     {
-
         this.schema = schema;
-        this.timestampFormatter = timestampFormatter;
+        this.outForward = outForward;
 
         setNewMessage();
     }
 
     private void setNewMessage()
     {
-        this.message = Maps.newHashMap();
-    }
-
-    public void emitMessage(OutForwardService outForward)
-    {
-        outForward.emit(message);
-        setNewMessage();
+        this.values = Lists.newArrayListWithCapacity(schema.getColumnCount());
     }
 
+    @Override
     public Schema getSchema()
     {
         return schema;
     }
 
-    private Column getColumn(int columnIndex)
-    {
-        return getSchema().getColumn(columnIndex);
-    }
-
-    private void setValue(String columnName, Object value)
-    {
-        message.put(columnName, value);
-    }
-
-    private void setValue(Column column, Object value)
+    @Override
+    public void addRecord()
     {
-        setValue(column.getName(), value);
+        Map<String, Object> message = Maps.newHashMap();
+        message.put(VALUES_KEY, values);
+        outForward.emit(message);
+        setNewMessage();
     }
 
-    public void setValue(int columnIndex, Object value)
+    private void setValue(int columnIndex, Object v)
     {
-        setValue(getColumn(columnIndex), value);
+        values.add(columnIndex, v);
     }
 
+    @Override
     public void setNull(Column column)
     {
-        setValue(column, null);
+        setNull(column.getIndex());
     }
 
-    private void setNull(int columnIndex)
+    @Override
+    public void setNull(int columnIndex)
     {
-        setNull(getColumn(columnIndex));
+        setValue(columnIndex, null);
     }
 
-    public void setBoolean(Column column, boolean value)
+    @Override
+    public void setBoolean(Column column, boolean v)
     {
-        setValue(column, value);
+        setBoolean(column.getIndex(), v);
     }
 
-    public void setBoolean(int columnIndex, boolean value)
+    @Override
+    public void setBoolean(int columnIndex, boolean v)
     {
-        setBoolean(getColumn(columnIndex), value);
+        setValue(columnIndex, v);
     }
 
-    public void setLong(Column column, long value)
+    @Override
+    public void setString(Column column, String v)
     {
-        setValue(column, value);
+        setString(column.getIndex(), v);
     }
 
-    public void setLong(int columnIndex, long value)
+    @Override
+    public void setString(int columnIndex, String v)
     {
-        setLong(getColumn(columnIndex), value);
+        setValue(columnIndex, v);
     }
 
-    public void setDouble(Column column, double value)
+    @Override
+    public void setLong(Column column, long v)
     {
-        setValue(column, value);
+        setLong(column.getIndex(), v);
     }
 
-    public void setDouble(int columnIndex, double value)
+    @Override
+    public void setLong(int columnIndex, long v)
     {
-        setDouble(getColumn(columnIndex), value);
+        setValue(columnIndex, v);
     }
 
-    public void setString(Column column, String value)
+    @Override
+    public void setDouble(Column column, double v)
     {
-        setValue(column, value);
+        setDouble(column.getIndex(), v);
     }
 
-    public void setString(int columnIndex, String value)
+    @Override
+    public void setDouble(int columnIndex, double v)
     {
-        setString(getColumn(columnIndex), value);
+        setValue(columnIndex, v);
     }
 
-    public void setTimestamp(Column column, Timestamp value)
+    @Override
+    public void setTimestamp(Column column, Timestamp v)
     {
-        setValue(column, timestampFormatter.format(value));
+
+        setTimestamp(column.getIndex(), v);
     }
 
-    public void setTimestamp(int columnIndex, Timestamp value)
+    @Override
+    public void setTimestamp(int columnIndex, Timestamp v)
     {
-        setTimestamp(getColumn(columnIndex), value);
+        // TODO: confirm correct value is stored
+        long epochSecond = v.getEpochSecond();
+        long nanoAdjustment = v.getNano();
+        setValue(columnIndex, new long[]{epochSecond, nanoAdjustment});
     }
 
-    public void setJson(Column column, Value value)
+    @Override
+    public void setJson(Column column, Value v)
     {
-        setValue(column, value.toJson());
+        setJson(column.getIndex(), v);
     }
 
-    public void setJson(int columnIndex, Value value)
+    @Override
+    public void setJson(int columnIndex, Value v)
     {
-        setJson(getColumn(columnIndex), value);
+        // TODO: confirm correct value is stored
+        setValue(columnIndex, v.toJson());
     }
 }

data/src/main/java/org/embulk/filter/copy/forward/OutForwardService.java

@@ -64,7 +64,7 @@ public class OutForwardService
     }
 
     public interface Task
-            extends ForwardBaseTask
+            extends ForwardParentTask
     {
         @Config("out_forward")
         @ConfigDefault("{}")
@@ -73,7 +73,7 @@ public class OutForwardService
 
     public static void sendShutdownMessage(Task task)
     {
-        logger.info("out_forward: send shutdown message.");
+        logger.info("OutForwardService: Send a Shutdown Message.");
         OutForwardService outForward = new OutForwardService(task);
         outForward.emit(task.getShutdownTag(), Maps.newHashMap());
         outForward.finish();

data/src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java

@@ -9,11 +9,12 @@ import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.filter.copy.forward.InForwardEventReader;
 import org.embulk.filter.copy.forward.InForwardService;
-import org.embulk.filter.copy.forward.InForwardVisitor;
+import org.embulk.filter.copy.spi.PageBuilder;
+import org.embulk.filter.copy.spi.StandardColumnVisitor;
 import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
 import org.embulk.spi.InputPlugin;
-import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
@@ -74,28 +75,25 @@ public class InternalForwardInputPlugin
         PluginTask task = taskSource.loadTask(PluginTask.class);
 
         try (PageBuilder pageBuilder = new PageBuilder(task.getBufferAllocator(), schema, output)) {
-            TimestampParser timestampParser = new TimestampParser(
-                    task.getJRuby(),
-                    task.getDefaultTimestampFormat(),
-                    task.getDefaultTimeZone());
-            InForwardEventReader eventReader = new InForwardEventReader(schema, timestampParser);
-            InForwardVisitor inForwardVisitor = new InForwardVisitor(eventReader, pageBuilder);
-
-            InForwardService.builder()
+            InForwardEventReader eventReader = new InForwardEventReader(schema);
+            ColumnVisitor inForwardVisitor = new StandardColumnVisitor(eventReader, pageBuilder);
+
+            InForwardService inForwardService = InForwardService.builder()
                     .task(task)
                     .forEachEventCallback(
                             event ->
                             {
                                 // TODO: here is not thread-safe
                                 eventReader.setEvent(event);
-                                while (eventReader.nextMessage()) {
+                                while (eventReader.nextRecord()) {
                                     schema.visitColumns(inForwardVisitor);
                                     pageBuilder.addRecord();
                                 }
                             }
                     )
-                    .build()
-                    .runUntilShouldShutdown();
+                    .build();
+
+            inForwardService.startAsync().awaitTerminated();
 
             pageBuilder.finish();
         }