embulk-filter-timestamp_format 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5bbc918d5bf6ffd22de31d35b00909dcb7e29b8a
-  data.tar.gz: 23ac08f9781e5c0accdd06c58034801f738ab4b0
+  metadata.gz: 3a7dcaa4b130acfa1790e07f7c132d63eb677cc5
+  data.tar.gz: 412c0aa7676be1171a43e8a903cf428389d54bb6
 SHA512:
-  metadata.gz: c1db3ebc3893536c371f7765151d9ea673a21f05bd655a311a0a935b2f0da61cbb66bb18bd33dd59a7e63a12754be0f74991a431696ef091e48f68f2a2ff1b9d
-  data.tar.gz: e26a3e99225d16ed059f0201ecb7a1ecf009fb1544b0137ea32d2d7d49a6f89815bbc08e608b73f503fcbd21e353ea12f9a62aa5d5fee161af8ad37806b172f4
+  metadata.gz: 4af5ea7bba8fc5f6f67a342aa2170d77f5cce3d6c0ad92e0ee90bd48e587e0172b227736f5c4361251589475b398e3943d334c12dd442f3c762b3f235a712925
+  data.tar.gz: 3b5fdde8febd1afb7dc5d9242ef07f6f15d2a4f003f5a1a55716fa6a4e589e9a6dfaee847cf81176d18a7ded3ff9becbfd0e5bdbcbaa3f50976854a0dde3e3a9

data/CHANGELOG.md

@@ -0,0 +1,9 @@
+# 0.1.1 (2016-04-26)
+
+Enhancements:
+
+* Check whether specified columns exist
+
+# 0.1.0 (2016-04-26)
+
+initial version

data/README.md

@@ -8,11 +8,11 @@ A filter plugin for Embulk to change timesatmp format
 
 - **columns**: columns to retain (array of hash)
   - **name**: name of column, must be a string column (required)
-  - **from_format**: specify the format of the input timestamp (string or an array, default is default_from_format)
+  - **from_format**: specify the format of the input timestamp (array of strings, default is default_from_format)
   - **from_timezone**: specify the timezone of the input timestamp (string, default is default_from_timezone)
   - **to_format**: specify the format of the output timestamp (string, default is default_to_format)
   - **to_timezone**: specify the timezone of the output timestamp (string, default is default_to_timezone)
-- **default_from_format**: default timestamp format for the input timestamp columns (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
+- **default_from_format**: default timestamp format for the input timestamp columns (array of strings, default is `["%Y-%m-%d %H:%M:%S.%N %z"]`)
 - **default_from_timezone**: default timezone for the input timestamp columns (string, default is `UTC`)
 - **default_to_format**: default timestamp format for the output timestamp columns (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
 - **default_to_timezone**: default timezone for the output timestamp olumns (string, default is `UTC`)
@@ -54,6 +54,7 @@ Output will be as:
 
 ## ToDo
 
+* Currently, input must be a String column and output will be a String column. But, support Timestamp column (input / output)
 * Write test
 
 ## Development

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
@@ -70,12 +70,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")
   spec.require_paths = ["lib"]
 
-  if spec.respond_to?(:metadata)
-    spec.metadata['allowed_push_host'] = "https://rubygems.dena.jp"
-  else
-    raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
-  end
-
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
   spec.add_development_dependency 'embulk-parser-jsonl'

data/src/main/java/org/embulk/filter/TimestampFormatFilterPlugin.java

@@ -1,18 +1,17 @@
 package org.embulk.filter;
 
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 
 import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 
-import org.embulk.filter.timestamp_format.TimestampParser;
 import org.embulk.filter.timestamp_format.TimestampFormatter;
+import org.embulk.filter.timestamp_format.TimestampParser;
 
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
@@ -23,21 +22,20 @@ import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
-import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParseException;
 
-import org.jruby.embed.ScriptingContainer;
 import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
 import org.msgpack.value.ArrayValue;
 import org.msgpack.value.MapValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 import org.slf4j.Logger;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
-import java.util.ArrayList;
 import java.util.Map;
 import java.util.Objects;
 
@@ -84,6 +82,13 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
             throw new ConfigException("\"columns\" must be specified.");
         }
 
+        for (ColumnConfig columnConfig : columns) {
+            String name = columnConfig.getName();
+            if (!name.startsWith("$.")) {
+                inputSchema.lookupColumn(name); // throw Column 'name' is not found
+            }
+        }
+
         control.run(task.dump(), inputSchema);
     }
 
@@ -122,40 +127,13 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
         }
 
         return new PageOutput() {
-            private PageReader pageReader = new PageReader(inputSchema);
-            private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
-            private ColumnVisitorImpl visitor = new ColumnVisitorImpl(pageBuilder);
-
-            @Override
-            public void finish()
-            {
-                pageBuilder.finish();
-            }
-
-            @Override
-            public void close()
-            {
-                pageBuilder.close();
-            }
-
-            @Override
-            public void add(Page page)
-            {
-                pageReader.setPage(page);
-
-                while (pageReader.nextRecord()) {
-                    outputSchema.visitColumns(visitor);
-                    pageBuilder.addRecord();
-                }
-            }
-
             public Value formatTimestampStringRecursively(PluginTask task, String name, Value value)
                     throws TimestampParseException
             {
                 if (value.isArrayValue()) {
                     ArrayValue arrayValue = value.asArrayValue();
                     int size = arrayValue.size();
-                    Value newValue[] = new Value[size];
+                    Value[] newValue = new Value[size];
                     for (int i = 0; i < size; i++) {
                         String k = new StringBuilder(name).append("[").append(Integer.toString(i)).append("]").toString();
                         Value v = arrayValue.get(i);
@@ -166,7 +144,7 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                 else if (value.isMapValue()) {
                     MapValue mapValue = value.asMapValue();
                     int size = mapValue.size() * 2;
-                    Value newValue[] = new Value[size];
+                    Value[] newValue = new Value[size];
                     int i = 0;
                     for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
                         Value k = entry.getKey();
@@ -179,7 +157,7 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                     return ValueFactory.newMap(newValue, true);
                 }
                 else if (value.isStringValue()) {
-                    String stringValue = value.asStringValue().asString() ;
+                    String stringValue = value.asStringValue().asString();
                     String newValue = formatTimestampString(task, name, stringValue);
                     return (Objects.equals(newValue, stringValue)) ? value : ValueFactory.newString(newValue);
                 }
@@ -203,18 +181,47 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                 catch (TimestampParseException ex) {
                     if (task.getStopOnInvalidRecord()) {
                         throw Throwables.propagate(ex);
-                    } else {
+                    }
+                    else {
                         logger.warn("invalid value \"{}\":\"{}\"", name, value);
                         return value;
                     }
                 }
             }
 
+            private PageReader pageReader = new PageReader(inputSchema);
+            private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
+            private ColumnVisitorImpl visitor = new ColumnVisitorImpl(pageBuilder);
+
+            @Override
+            public void finish()
+            {
+                pageBuilder.finish();
+            }
+
+            @Override
+            public void close()
+            {
+                pageBuilder.close();
+            }
+
+            @Override
+            public void add(Page page)
+            {
+                pageReader.setPage(page);
+
+                while (pageReader.nextRecord()) {
+                    outputSchema.visitColumns(visitor);
+                    pageBuilder.addRecord();
+                }
+            }
+
             class ColumnVisitorImpl implements ColumnVisitor
             {
                 private final PageBuilder pageBuilder;
 
-                ColumnVisitorImpl(PageBuilder pageBuilder) {
+                ColumnVisitorImpl(PageBuilder pageBuilder)
+                {
                     this.pageBuilder = pageBuilder;
                 }
 
@@ -223,7 +230,8 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                 {
                     if (pageReader.isNull(column)) {
                         pageBuilder.setNull(column);
-                    } else {
+                    }
+                    else {
                         pageBuilder.setBoolean(column, pageReader.getBoolean(column));
                     }
                 }
@@ -233,7 +241,8 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                 {
                     if (pageReader.isNull(column)) {
                         pageBuilder.setNull(column);
-                    } else {
+                    }
+                    else {
                         pageBuilder.setLong(column, pageReader.getLong(column));
                     }
                 }
@@ -243,7 +252,8 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                 {
                     if (pageReader.isNull(column)) {
                         pageBuilder.setNull(column);
-                    } else {
+                    }
+                    else {
                         pageBuilder.setDouble(column, pageReader.getDouble(column));
                     }
                 }
@@ -279,7 +289,8 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                 {
                     if (pageReader.isNull(column)) {
                         pageBuilder.setNull(column);
-                    } else {
+                    }
+                    else {
                         pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
                     }
                 }

data/src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java

@@ -1,19 +1,22 @@
 package org.embulk.filter.timestamp_format;
 
-import java.util.Locale;
-import org.jruby.embed.ScriptingContainer;
-import org.joda.time.DateTime;
-import org.joda.time.DateTimeZone;
 import com.google.common.base.Optional;
-import org.jruby.embed.ScriptingContainer;
-import org.jruby.util.RubyDateFormat;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.spi.util.LineEncoder;
-import org.embulk.spi.time.Timestamp;
 
 import org.embulk.filter.TimestampFormatFilterPlugin.PluginTask;
 
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.util.LineEncoder;
+
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import org.jruby.util.RubyDateFormat;
+
+import java.util.Locale;
+
 public class TimestampFormatter
 {
     public interface Task
@@ -72,7 +75,7 @@ public class TimestampFormatter
     public String format(Timestamp value)
     {
         // TODO optimize by using reused StringBuilder
-        toDateFormat.setDateTime(new DateTime(value.getEpochSecond()*1000, toTimeZone));
+        toDateFormat.setDateTime(new DateTime(value.getEpochSecond() * 1000, toTimeZone));
         toDateFormat.setNSec(value.getNano());
         return toDateFormat.format(null);
     }

data/src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java

@@ -1,19 +1,24 @@
 package org.embulk.filter.timestamp_format;
 
-import org.jruby.embed.ScriptingContainer;
-import org.joda.time.DateTimeZone;
 import com.google.common.base.Optional;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampParseException;
+
+import org.embulk.filter.TimestampFormatFilterPlugin.PluginTask;
+
 import org.embulk.spi.time.JRubyTimeParserHelper;
 import org.embulk.spi.time.JRubyTimeParserHelperFactory;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParseException;
+
+import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
+
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
 
-import org.embulk.filter.TimestampFormatFilterPlugin.PluginTask;
-import java.util.List;
 import java.util.ArrayList;
+import java.util.List;
 
 public class TimestampParser
 {
@@ -81,7 +86,8 @@ public class TimestampParser
             helper = h;
             try {
                 localUsec = helper.strptimeUsec(text);
-            } catch (TimestampParseException ex) {
+            }
+            catch (TimestampParseException ex) {
                 exception = ex;
             }
         }
@@ -101,7 +107,7 @@ public class TimestampParser
 
         long localSec = localUsec / 1000000;
         long usec = localUsec % 1000000;
-        long sec = timeZone.convertLocalToUTC(localSec*1000, false) / 1000;
+        long sec = timeZone.convertLocalToUTC(localSec * 1000, false) / 1000;
 
         return Timestamp.ofEpochSecond(sec, usec * 1000);
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-timestamp_format
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -77,12 +77,11 @@ files:
 - src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java
 - src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java
 - src/test/java/org/embulk/filter/TestTimestampFormatFilterPlugin.java
-- classpath/embulk-filter-timestamp_format-0.1.0.jar
+- classpath/embulk-filter-timestamp_format-0.1.1.jar
 homepage: https://github.com/sonots/embulk-filter-timestamp_format
 licenses:
 - MIT
-metadata:
-  allowed_push_host: https://rubygems.dena.jp
+metadata: {}
 post_install_message:
 rdoc_options: []
 require_paths: