embulk-filter-speedometer 0.2.1

data/src/main/java/org/embulk/filter/SpeedometerUtil.java

@@ -0,0 +1,63 @@
+package org.embulk.filter;
+
+public class SpeedometerUtil {
+
+    public static String toNumberText(long originalNum) {
+        long baseNum = 1000;
+        long remain = 0;
+        float num = originalNum;
+
+        String[] units = new String[]{ "b", "kb", "mb", "gb", "tb", "pb", "eb", "zb", "yb" };
+        for (String unit : units) {
+            if (num < baseNum) {
+                if (num < baseNum/10) {
+                    return String.format("%1.1f%s", num + ((float)remain) / baseNum, unit);
+                } else {
+                    return String.format("%2d%s", (long)num, unit);
+                }
+            } else {
+                num /= baseNum;
+            }
+        }
+
+        return String.valueOf(originalNum);
+    }
+
+    public static String toTimeText(long timeMillisecs) {
+        long num = timeMillisecs;
+
+        long mseconds = num % 1000;
+        num /= 1000;
+
+        if (num < 10) {
+            return String.format("%1.2f", num + mseconds / 1000.0);
+        }
+
+        if (num < 60) {
+            return String.format("%1.1f", num + mseconds / 1000.0);
+        }
+
+        long seconds = num % 60;
+        num /= 60;
+
+        if (num < 60) {
+            return String.format("%1d:%02d", num, seconds);
+        }
+
+        long mins = num % 60;
+        num /= 60;
+
+        if (num < 24) {
+            return String.format("%1d:%02d:%02d", num, mins, seconds);
+        }
+
+        long hours = num % 24;
+        num /= 24;
+
+        if (num < 10) {
+            return String.format("%1.1f days", num + hours / 24.0);
+        }
+
+        return String.format("%1d days", num);
+    }
+}

data/src/test/java/org/embulk/filter/TestSpeedometerFilterPlugin.java

@@ -0,0 +1,114 @@
+package org.embulk.filter;
+
+import mockit.Mocked;
+import mockit.NonStrictExpectations;
+import mockit.Verifications;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.filter.SpeedometerFilterPlugin.PluginTask;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.junit.Test;
+
+public class TestSpeedometerFilterPlugin
+{
+    @Mocked
+    Exec exec;
+
+    @Mocked
+    ConfigSource config;
+
+    @Mocked
+    PluginTask task;
+
+    @Mocked
+    TaskSource taskSource;
+
+    @Mocked
+    Schema schema;
+
+    @Mocked
+    PageOutput inPageOutput;
+
+    @Mocked
+    FilterPlugin.Control control;
+
+    @Test
+    public void testTransaction() {
+        new NonStrictExpectations() {{
+            config.loadConfig(PluginTask.class); result = task;
+        }};
+
+        SpeedometerFilterPlugin plugin = new SpeedometerFilterPlugin();
+        plugin.transaction(config, schema, control);
+
+        new Verifications() {{
+            config.loadConfig(PluginTask.class); times = 1;
+            control.run((TaskSource)any, schema); times = 1;
+        }};
+    }
+
+    @Test
+    public void testOpen(final @Mocked PageReader reader, final @Mocked PageBuilder builder, final @Mocked Page page) throws Exception {
+        new NonStrictExpectations() {{
+            taskSource.loadTask(PluginTask.class); result = task;
+            task.getDelimiter(); result = "";
+            reader.nextRecord(); result = true; result = false;
+        }};
+
+        SpeedometerFilterPlugin plugin = new SpeedometerFilterPlugin();
+        PageOutput output = plugin.open(taskSource, schema, schema, inPageOutput);
+        output.add(page);
+
+        new Verifications() {{
+            taskSource.loadTask(PluginTask.class); times = 1;
+            builder.finish(); times = 1;
+            builder.addRecord(); times = 1;
+            reader.nextRecord(); times = 2;
+            reader.setPage(page); times = 1;
+            schema.visitColumns(withInstanceOf(ColumnVisitor.class)); times = 2;
+        }};
+    }
+
+    @Test
+    public void testFinish(final @Mocked PageReader reader, final @Mocked PageBuilder builder, final @Mocked Page page) throws Exception {
+        new NonStrictExpectations() {{
+            taskSource.loadTask(PluginTask.class); result = task;
+            task.getDelimiter(); result = "";
+        }};
+
+        SpeedometerFilterPlugin plugin = new SpeedometerFilterPlugin();
+        PageOutput output = plugin.open(taskSource, schema, schema, inPageOutput);
+        output.finish();
+
+        new Verifications() {{
+            taskSource.loadTask(PluginTask.class); times = 1;
+            inPageOutput.finish(); times = 1;
+        }};
+    }
+
+    @Test
+    public void testClose(final @Mocked PageReader reader, final @Mocked PageBuilder builder, final @Mocked Page page) throws Exception {
+        new NonStrictExpectations() {{
+            taskSource.loadTask(PluginTask.class); result = task;
+            task.getDelimiter(); result = "";
+        }};
+
+        SpeedometerFilterPlugin plugin = new SpeedometerFilterPlugin();
+        PageOutput output = plugin.open(taskSource, schema, schema, inPageOutput);
+        output.close();
+
+        new Verifications() {{
+            taskSource.loadTask(PluginTask.class); times = 1;
+            reader.close(); times = 1;
+            inPageOutput.close(); times = 1;
+        }};
+    }
+}

data/src/test/java/org/embulk/filter/TestSpeedometerSpeedAggregator.java

@@ -0,0 +1,120 @@
+package org.embulk.filter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import mockit.Mocked;
+import mockit.NonStrictExpectations;
+import mockit.Verifications;
+
+import org.embulk.spi.Exec;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+public class TestSpeedometerSpeedAggregator {
+    @Mocked SpeedometerSpeedController controller;
+    @Mocked Exec exec;
+
+    @Test
+    public void testGetInstance() {
+        assertNotNull("Verify there is a singleton.", SpeedometerSpeedAggregator.getInstance());
+    }
+
+    @Test
+    public void testSpeedometerSpeedAggregator() {
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
+        assertEquals("Verify the default global start time is zero.", 0, aggregator.getGlobalStartTime());
+        assertEquals("Verify the default active count is zero.", 0, aggregator.getActiveControllerCount());
+    }
+
+    @Test
+    public void testStartController() {
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
+        long nowTime = System.currentTimeMillis();
+        aggregator.startController(controller, nowTime);
+        assertEquals("Verify global start time is set.", nowTime, aggregator.getGlobalStartTime());
+        assertEquals("Verify active controller count.", 1, aggregator.getActiveControllerCount());
+        assertTrue("Verify there is a registered controller.", aggregator.getControllerList().contains(controller));
+    }
+
+    @Test
+    public void testStopController() {
+        new NonStrictExpectations() {{
+            controller.getTotalBytes(); result = 11;
+        }};
+
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
+        long nowTime = System.currentTimeMillis();
+        aggregator.startController(controller, nowTime);
+        aggregator.stopController(controller);
+        assertEquals("Verify active controller count.", 0, aggregator.getActiveControllerCount());
+        assertFalse("Verify there is a registered controller.", aggregator.getControllerList().contains(controller));
+        assertEquals("Verify controller's total bytes are added to aggregator.", 11, aggregator.getGlobalTotalBytes());
+    }
+
+    @Test
+    public void testStopControllerShowOverallMessage(@Mocked final Logger logger) {
+        new NonStrictExpectations() {{
+            Exec.getLogger(SpeedometerFilterPlugin.class); result = logger;
+        }};
+
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
+        long nowTime = System.currentTimeMillis();
+        aggregator.startController(controller, nowTime);
+        aggregator.startController(controller, nowTime);
+        aggregator.stopController(controller);
+        aggregator.stopController(controller);
+
+        new Verifications() {{
+            logger.info(withAny("Overall message.")); times = 2;
+        }};
+   }
+
+    @Test
+    public void testGetSpeedLimitForController() {
+        new NonStrictExpectations() {{
+            controller.getSpeedLimit(); result = 10;
+        }};
+
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
+        long nowTime = System.currentTimeMillis();
+        aggregator.startController(controller, nowTime);
+        assertEquals("Verify speed limit for one controller.", 10, aggregator.getSpeedLimitForController(controller));
+        aggregator.startController(controller, nowTime);
+        assertEquals("Verify speed limit for two controllers.", 10 / 2, aggregator.getSpeedLimitForController(controller));
+    }
+
+    @Test
+    public void testCheckProgress() throws Exception {
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
+        long initPeriodTime = System.currentTimeMillis();
+        final long startPeriodTime = initPeriodTime + 2;
+        aggregator.startController(controller, initPeriodTime);
+        aggregator.checkProgress(initPeriodTime, 1);
+        aggregator.checkProgress(startPeriodTime, 1);
+        aggregator.checkProgress(startPeriodTime, 1); // This call should not affect anything.
+
+        new Verifications() {{
+            controller.renewPeriod(); times = 1;
+            controller.getTotalBytes(); times = 1;
+            controller.getPeriodBytesPerSec(startPeriodTime); times = 1;
+        }};
+    }
+
+    @Test
+    public void testCheckProgressIsDisabled() throws Exception {
+        SpeedometerSpeedAggregator aggregator = new SpeedometerSpeedAggregator();
+        long initPeriodTime = System.currentTimeMillis();
+        final long startPeriodTime = initPeriodTime + 2;
+        aggregator.startController(controller, initPeriodTime);
+        aggregator.checkProgress(initPeriodTime, 0);
+        aggregator.checkProgress(startPeriodTime, 0);
+
+        new Verifications() {{
+            controller.renewPeriod(); times = 0;
+            controller.getTotalBytes(); times = 0;
+            controller.getPeriodBytesPerSec(startPeriodTime); times = 0;
+        }};
+    }
+}

data/src/test/java/org/embulk/filter/TestSpeedometerSpeedController.java

@@ -0,0 +1,114 @@
+package org.embulk.filter;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import mockit.Mocked;
+import mockit.NonStrictExpectations;
+import mockit.Verifications;
+
+import org.embulk.filter.SpeedometerFilterPlugin.PluginTask;
+import org.junit.Test;
+
+public class TestSpeedometerSpeedController {
+    @Mocked
+    SpeedometerSpeedAggregator aggregator;
+
+    @Mocked
+    PluginTask task;
+
+    SpeedometerSpeedController controller;
+
+    @Test
+    public void testSpeedometerSpeedController() {
+        new NonStrictExpectations() {{
+            task.getSpeedLimit(); result = 1L;
+            task.getMaxSleepMillisec(); result = 2;
+            task.getLogIntervalSeconds(); result = 3;
+        }};
+
+        controller = new SpeedometerSpeedController(task, aggregator);
+        assertEquals("Verify aggregator is set.", aggregator, controller.getAggregator());
+        assertEquals("Verify sleepdLimit is set.", 1, controller.getSpeedLimit());
+        assertEquals("Verify maxSleepMillisec is set.", 2, controller.getMaxSleepMillisec());
+        assertEquals("Verify logIntervalSeconds is set.", 3 * 1000, controller.getLogIntervalMillisec());
+
+        new Verifications() {{
+            task.getSpeedLimit(); times = 1;
+            task.getMaxSleepMillisec(); times = 1;
+            task.getLogIntervalSeconds(); times = 1;
+        }};
+    }
+
+    @Test
+    public void testStop() {
+        controller = new SpeedometerSpeedController(task, aggregator);
+        controller.stop();
+
+        new Verifications() {{
+            aggregator.stopController(controller); times = 1;
+        }};
+    }
+
+    @Test
+    public void testGetTotalBytes() {
+        new NonStrictExpectations() {{
+            task.getSpeedLimit(); result = 1L;
+            task.getMaxSleepMillisec(); result = 2;
+            task.getLogIntervalSeconds(); result = 3;
+            aggregator.getSpeedLimitForController((SpeedometerSpeedController)any); result = 10000;
+        }};
+        long nowTime = System.currentTimeMillis();
+        int newDataSize = 3;
+
+        controller = new SpeedometerSpeedController(task, aggregator);
+        controller.checkSpeedLimit(nowTime, newDataSize);
+        assertEquals("Verify total bytes", newDataSize, controller.getTotalBytes());
+        controller.checkSpeedLimit(nowTime + 1, newDataSize);
+        assertEquals("Verify total bytes", newDataSize * 2, controller.getTotalBytes());
+    }
+
+    @Test
+    public void testGetPeriodBytesPerSec() {
+        new NonStrictExpectations() {{
+            task.getSpeedLimit(); result = 1L;
+            task.getMaxSleepMillisec(); result = 2;
+            task.getLogIntervalSeconds(); result = 3;
+            aggregator.getSpeedLimitForController((SpeedometerSpeedController)any); result = 10000;
+        }};
+        long nowTime = System.currentTimeMillis();
+        int newDataSize = 3;
+
+        controller = new SpeedometerSpeedController(task, aggregator);
+        controller.checkSpeedLimit(nowTime, newDataSize);
+        assertTrue("Verify total bytes", controller.getPeriodBytesPerSec(System.currentTimeMillis()) > 0);
+    }
+
+    @Test
+    public void testCheckSpeedLimit() {
+        new NonStrictExpectations() {{
+            task.getSpeedLimit(); result = 1L;
+            task.getMaxSleepMillisec(); result = 2;
+            task.getLogIntervalSeconds(); result = 3;
+            aggregator.getSpeedLimitForController((SpeedometerSpeedController)any); result = 10000;
+        }};
+        long nowTime = System.currentTimeMillis();
+        int newDataSize = 3;
+
+        controller = new SpeedometerSpeedController(task, aggregator);
+        controller.checkSpeedLimit(nowTime, newDataSize);
+
+        new Verifications() {{
+            aggregator.startController(controller, anyLong); times = 1;
+            aggregator.checkProgress(anyLong, 3 * 1000); times = 1;
+            aggregator.getSpeedLimitForController(controller); times = 1;
+        }};
+    }
+
+    @Test
+    public void testRenewPeriod() {
+        controller = new SpeedometerSpeedController(task, aggregator);
+        controller.renewPeriod();
+        assertTrue("Verify renewPeriod flag is set.", controller.isRenewPeriodSet());
+    }
+
+}

data/src/test/java/org/embulk/filter/TestSpeedometerUtil.java

@@ -0,0 +1,41 @@
+package org.embulk.filter;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+public class TestSpeedometerUtil {
+
+    @Test
+    public void testToNumberText() {
+        assertEquals("Verify 0 bytes",  "0.0b", SpeedometerUtil.toNumberText(0));
+        assertEquals("Verify 0 bytes",  "100b", SpeedometerUtil.toNumberText(100));
+        assertEquals("Verify 0 bytes", "1.0kb", SpeedometerUtil.toNumberText(1000L));
+        assertEquals("Verify 0 bytes", "100kb", SpeedometerUtil.toNumberText(1000L * 100));
+        assertEquals("Verify 0 bytes", "1.0mb", SpeedometerUtil.toNumberText(1000L * 1000));
+        assertEquals("Verify 0 bytes", "100mb", SpeedometerUtil.toNumberText(1000L * 1000 * 100));
+        assertEquals("Verify 0 bytes", "1.0gb", SpeedometerUtil.toNumberText(1000L * 1000 * 1000));
+        assertEquals("Verify 0 bytes", "100gb", SpeedometerUtil.toNumberText(1000L * 1000 * 1000 * 100));
+        assertEquals("Verify 0 bytes", "1.0tb", SpeedometerUtil.toNumberText(1000L * 1000 * 1000 * 1000));
+        assertEquals("Verify 0 bytes", "100tb", SpeedometerUtil.toNumberText(1000L * 1000 * 1000 * 1000 * 100));
+        assertEquals("Verify 0 bytes", "1.0pb", SpeedometerUtil.toNumberText(1000L * 1000 * 1000 * 1000 * 1000));
+        assertEquals("Verify 0 bytes", "100pb", SpeedometerUtil.toNumberText(1000L * 1000 * 1000 * 1000 * 1000 * 100));
+        assertEquals("Verify 0 bytes", "1.0eb", SpeedometerUtil.toNumberText(1000L * 1000 * 1000 * 1000 * 1000 * 1000));
+    }
+
+    @Test
+    public void testToTimeText() {
+        assertEquals("Verify 0 msec", "0.00", SpeedometerUtil.toTimeText(0));
+        assertEquals("Verify 1 sec", "1.00", SpeedometerUtil.toTimeText(1000));
+        assertEquals("Verify 9 secs", "9.00", SpeedometerUtil.toTimeText(9000));
+        assertEquals("Verify 10 secs", "10.0", SpeedometerUtil.toTimeText(10000));
+        assertEquals("Verify 1 min", "1:00", SpeedometerUtil.toTimeText(60000));
+        assertEquals("Verify 10 mins 10 secs", "10:01", SpeedometerUtil.toTimeText(60000 * 10 + 1000));
+        assertEquals("Verify 1 hour 10 mins 10 secs", "1:10:01", SpeedometerUtil.toTimeText(60000 * 10 + 1000 + 3600 * 1000));
+        assertEquals("Verify 10 hours 10 mins 10 secs", "10:10:01", SpeedometerUtil.toTimeText(60000 * 10 + 1000 + 10 * 3600 * 1000));
+        assertEquals("Verify 23 hours 10 mins 10 secs", "23:10:01", SpeedometerUtil.toTimeText(60000 * 10 + 1000 + 23 * 3600 * 1000));
+        assertEquals("Verify 1.0 days", "1.0 days", SpeedometerUtil.toTimeText(60L * 60 * 24 * 1000));
+        assertEquals("Verify 100 days", "100 days", SpeedometerUtil.toTimeText(60L * 60 * 24 * 1000 * 100));
+    }
+
+}