embulk-filter-timestamp_format 0.1.5 → 0.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7d7569b8adc1db79b292e271214f852fb080151b
-  data.tar.gz: df0c01a5893dc4a4bbb1f1228e3d72b031e59f93
+  metadata.gz: cb84426fbf7dfbaac21925ece7475f322c7c2010
+  data.tar.gz: 2c5accec8470864588c20d62c42626a89a584bab
 SHA512:
-  metadata.gz: d81f4f2df4775444b5608432a2451158453f768cf6687188afae1169ea5eb15c699141d670987e73139c14c5ae8bbeb2122fbfcb6f73c89f5e98a425db8f2519
-  data.tar.gz: 5ecdc2f30763b7768fd1e9176c2c6b01fdafbd214f9191885e79de7f333303a9a7b86054a7c10acbf87ed458db5cab5d3b2c4871115198b829f65b8c36d855cc
+  metadata.gz: d77f18cde0d2a3626198d1e94aa1c7e1b6c47d3c8572c9397cf1c5ec8a4ec808fcb9dea636339c0e838e00f418cefd290d92d77b44669a6df31c1c3e2fd8b9bb
+  data.tar.gz: c3464272e96b244c782b65e8e536a1f64fc488646958c58d04a068550c85d718f137081662ce5fd7fe1370aa6788bda4b8598a7e0e606b7a00e6811e096177a5

data/.gitignore

@@ -6,7 +6,7 @@
 /classpath/
 build/
 .idea
-*.csv
 .tags
 .ruby-version
 *.iml
+dummy.csv

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.1.6 (2016-05-01)
+
+Enhancements:
+
+* Support unixtimestamp unit such as milli sec, micro sec, nano sec
+* Support Java timestamp parser/formatter (SimpleDateFormat)
+
 # 0.1.5 (2016-04-29)
 
 Enhancements:

data/README.md

@@ -13,11 +13,15 @@ A filter plugin for Embulk to change timestamp format
   - **from_timezone**: specify the timezone of the input string (string, default is default_from_timezone)
   - **to_format**: specify the format of the output string (string, default is default_to_timestamp_format)
   - **to_timezone**: specify the timezone of the output string (string, default is default_to_timezone)
+  - **from_unit**: specify the time unit of the input unixtimestamp (string, default is default_from_timestamp_unit)
+  - **to_unit**: specify the time unit of the output unixtimestamp (string, default is default_to_timestamp_unit)
 - **default_from_timestamp_format**: default timestamp format for the input string (array of strings, default is `["%Y-%m-%d %H:%M:%S.%N %z"]`)
 - **default_from_timezone**: default timezone for the input string (string, default is `UTC`)
 - **default_to_timestamp_format**: default timestamp format for the output string (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
 - **default_to_timezone**: default timezone for the output string (string, default is `UTC`)
-* **stop_on_invalid_record**: stop bulk load transaction if a invalid record is found (boolean, default is `false)
+- **default_from_timetamp_unit**: default time unit such as second, ms, us, ns for the input unixtimestamp (string, default is `second`)
+- **default_to_timetamp_unit**: default time unit such as second, ms, us, ns for the output unixtimestamp (string, default is `second`)
+- **stop_on_invalid_record**: stop bulk load transaction if a invalid record is found (boolean, default is `false`)
 
 ## Example
 
@@ -56,6 +60,47 @@ Output will be as:
 
 See [./example](./example) for more examples.
 
+## Timestamp Parser/Formatter Performance Issue
+
+Embulk's timestamp parser/formatter originally uses jruby implementation, but it is slow.
+To improve performance, this plugin also supports Java's [SimpleDateFormat](https://docs.oracle.com/javase/jp/6/api/java/text/SimpleDateFormat.html) format as:
+
+```yaml
+in:
+  type: file
+  path_prefix: example/example.jsonl
+  parser:
+    type: jsonl
+    columns:
+    - {name: timestamp, type: string}
+    - {name: nested, type: json}
+filters:
+  - type: timestamp_format
+    default_from_timezone: "Asia/Taipei"
+    default_from_timestamp_format: ["yyyy-MM-dd HH:mm:ss.SSS z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"]
+    default_to_timezone: "Asia/Taipei"
+    default_to_timestamp_format: "yyyy-MM-dd HH:mm:ss.SSS Z"
+    columns:
+      - {name: timestamp}
+      - {name: $.nested.timestamp}
+out:
+  type: stdout
+```
+
+If format strings contain `%`, jruby parser/formatter is used. Otherwirse, java parser/formatter is used
+
+**COMPARISON:**
+
+Benchmark test sets are available at [./bench](./bench).  In my environment (Mac Book Pro), for 1000000 timestamps:
+
+* jruby parser/formatter: 65.06s
+* java parser/formatter: 1.3s
+
+**NOTICE:**
+
+* JRuby parser has micro second resolution, but Java parser (SimpleDateFormat) has only milli second resolution
+* `S` requires three digits always. For example, `yyyy-MM-dd HH:mm::ss.S` for `2015-12-17 01:02:03.1` gives 001 milli seconds wrongly, but it is the specification of SimpleDateFormat.
+
 ## ToDo
 
 * Write test

data/bench/config_java.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: bench/dummy
+  parser:
+    type: csv
+    columns:
+      - {name: timestamp, type: string}
+filters:
+  - type: timestamp_format
+    columns:
+      - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.SSS"], to_format: "yyyy-MM-dd"}
+
+out:
+  type: "null"

data/bench/config_jruby.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: bench/dummy
+  parser:
+    type: csv
+    columns:
+      - {name: timestamp, type: string}
+filters:
+  - type: timestamp_format
+    columns:
+      - {name: timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N"], to_format: "%Y-%m-%d"}
+
+out:
+  type: "null"

data/bench/gen_dummy.rb

@@ -0,0 +1,5 @@
+File::open('bench/dummy.csv', 'w') { |f|
+  (1..1000000).each {
+    f.puts(Time.now.strftime('%Y-%m-%d %H:%M:%S.%L'))
+  }
+}

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.5"
+version = "0.1.6"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/double.csv

@@ -0,0 +1,2 @@
+1436713200100.2,1436713200100.2,1436713200100.2,1436713200100.2
+

data/example/double.yml

@@ -0,0 +1,20 @@
+in:
+  type: file
+  path_prefix: example/double.csv
+  parser:
+    type: csv
+    columns:
+      - {name: double1, type: double}
+      - {name: double2, type: double}
+      - {name: double3, type: double}
+      - {name: double4, type: double}
+filters:
+  - type: timestamp_format
+    default_from_timestamp_unit: ms
+    columns:
+      - {name: double1, to_timezone: "Asia/Tokyo", to_format: "%Y-%m-%d %H:%M:%S.%N"}
+      - {name: double2, type: timestamp}
+      - {name: double3, type: long}
+      - {name: double4, type: double}
+out:
+  type: "null"

data/example/example.yml

@@ -1,22 +1,14 @@
 in:
   type: file
-  path_prefix: example/string_example.csv
+  path_prefix: example/example.jsonl
   parser:
-    type: csv
-    columns:
-      - {name: string1, type: string}
-      - {name: string2, type: string}
-      - {name: string3, type: string}
-      - {name: string4, type: string}
+    type: json
 filters:
   - type: timestamp_format
     default_to_timezone: "Asia/Tokyo"
     default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
-    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]
     columns:
-      - {name: string1}
-      - {name: string2, type: timestamp}
-      - {name: string3, type: long}
-      - {name: string4, type: double}
+      - {name: "$.record.timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
+      - {name: "$.record.nested.nested[0].timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
 out:
   type: "null"

data/example/example2.csv

@@ -0,0 +1,2 @@
+2015-07-12 15:00:00 UTC,2015-07-12 15:00:00 UTC
+2015-07-12 15:00:00.1 UTC,2015-07-12 15:00:00.1 UTC

data/example/example2.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: example/example2.csv
+  parser:
+    type: csv
+    columns:
+    - {name: string1, type: string}
+    - {name: string2, type: string}
+filters:
+  - type: timestamp_format
+    columns:
+      - {name: string1, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"], to_format: "%Y-%m-%m", to_timezone: "Asia/Tokyo"}
+out:
+  type: "null"

data/example/json_double.jsonl

@@ -0,0 +1 @@
+{"double1":1436713200100.2,"double2":1436713200100.2,"double3":1436713200100.2}

data/example/json_double.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: example/json_double.jsonl
+  parser:
+    type: json
+filters:
+  - type: timestamp_format
+    default_from_timestamp_unit: ms
+    columns:
+      - {name: $.record.double1, to_timezone: "Asia/Tokyo", to_format: "%Y-%m-%d %H:%M:%S.%N"}
+      - {name: $.record.double2, type: long}
+      - {name: $.record.double3, type: double}
+out:
+  type: "null"

data/example/json_long.jsonl

@@ -0,0 +1 @@
+{"long1":1436713200100,"long2":1436713200100,"long3":1436713200100}

data/example/json_long.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: example/json_long.jsonl
+  parser:
+    type: json
+filters:
+  - type: timestamp_format
+    default_from_timestamp_unit: ms
+    columns:
+      - {name: $.record.long1, to_timezone: "Asia/Tokyo", to_format: "%Y-%m-%d %H:%M:%S.%N"}
+      - {name: $.record.long2, type: long}
+      - {name: $.record.long3, type: double}
+out:
+  type: "null"

data/example/json_string.jsonl

@@ -0,0 +1,2 @@
+{"string1":"2015-07-12 15:00:00 UTC","string2":"2015-07-12 15:00:00 UTC","string3":"2015-07-12 15:00:00 UTC"}
+{"string1":"2015-07-12 15:00:00.1 UTC","string2":"2015-07-12 15:00:00.1 UTC","string3":"2015-07-12 15:00:00.1 UTC"}

data/example/json_string.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: example/json_string.jsonl
+  parser:
+    type: json
+filters:
+  - type: timestamp_format
+    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d %H:%M:%S.%N %z"]
+    columns:
+      - {name: $.record.string1, to_timezone: "Asia/Tokyo", to_format: "%Y-%m-%d %H:%M:%S.%N"}
+      - {name: $.record.string2, type: long, to_unit: ms}
+      - {name: $.record.string3, type: double, to_unit: ms}
+out:
+  type: "null"

data/example/long.csv

@@ -0,0 +1 @@
+1436713200100,1436713200100,1436713200100,1436713200100

data/example/long.yml

@@ -0,0 +1,20 @@
+in:
+  type: file
+  path_prefix: example/long.csv
+  parser:
+    type: csv
+    columns:
+      - {name: long1, type: long}
+      - {name: long2, type: long}
+      - {name: long3, type: long}
+      - {name: long4, type: long}
+filters:
+  - type: timestamp_format
+    default_from_timestamp_unit: ms
+    columns:
+      - {name: long1, to_timezone: "Asia/Tokyo", to_format: "%Y-%m-%d %H:%M:%S.%N"}
+      - {name: long2, type: timestamp}
+      - {name: long3, type: long}
+      - {name: long4, type: double}
+out:
+  type: "null"

data/example/string.csv

@@ -0,0 +1,4 @@
+2015-07-13 00:00:00,2015-07-13 00:00:00,2015-07-13 00:00:00,2015-07-13 00:00:00
+2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC
+2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC
+

data/example/{string_example.yml → string.yml}

@@ -1,6 +1,6 @@
 in:
   type: file
-  path_prefix: example/string_example.csv
+  path_prefix: example/string.csv
   parser:
     type: csv
     columns:
@@ -10,13 +10,14 @@ in:
       - {name: string4, type: string}
 filters:
   - type: timestamp_format
-    default_to_timezone: "Asia/Tokyo"
+    default_from_timezone: "Asia/Taipei"
+    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d %H:%M:%S"]
+    default_to_timezone: "Asia/Taipei"
     default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
-    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]
     columns:
       - {name: string1}
       - {name: string2, type: timestamp}
-      - {name: string3, type: long}
-      - {name: string4, type: double}
+      - {name: string3, type: long, to_unit: ms}
+      - {name: string4, type: double, to_unit: ms}
 out:
   type: "null"

data/example/string_java.yml

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: example/string.csv
+  parser:
+    type: csv
+    columns:
+      - {name: string1, type: string}
+      - {name: string2, type: string}
+      - {name: string3, type: string}
+      - {name: string4, type: string}
+filters:
+  - type: timestamp_format
+    default_from_timezone: "Asia/Taipei"
+    default_from_timestamp_format: ["yyyy-MM-dd HH:mm:ss.S z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"] # SSS must be three digit ...
+    default_to_timezone: "Asia/Taipei"
+    default_to_timestamp_format: "yyyy-MM-dd HH:mm:ss.SSS Z"
+    columns:
+      - {name: string1}
+      - {name: string2, type: timestamp}
+      - {name: string3, type: long, to_unit: ms}
+      - {name: string4, type: double, to_unit: ms}
+out:
+  type: "null"

data/example/timestamp.csv

@@ -0,0 +1,2 @@
+2015-07-12 15:00:00.1 UTC,2015-07-12 15:00:00.1 UTC,2015-07-12 15:00:00.1 UTC,2015-07-12 15:00:00.1 UTC
+

data/example/{timestamp_example.yml → timestamp.yml}

@@ -1,6 +1,6 @@
 in:
   type: file
-  path_prefix: example/timestamp_example.csv
+  path_prefix: example/timestamp.csv
   parser:
     type: csv
     default_timestamp_format: "%Y-%m-%d %H:%M:%S.%N %z"
@@ -14,9 +14,9 @@ filters:
     default_to_timezone: "Asia/Tokyo"
     default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
     columns:
-      - {name: timestamp1}
+      - {name: timestamp1, to_format: "%Y-%m-%d %H:%M:%S.%N"}
       - {name: timestamp2, type: timestamp}
-      - {name: timestamp3, type: long}
-      - {name: timestamp4, type: double}
+      - {name: timestamp3, type: long, to_unit: ms}
+      - {name: timestamp4, type: double, to_unit: ms}
 out:
   type: "null"

data/src/main/java/org/embulk/filter/timestamp_format/ColumnCaster.java

@@ -1,5 +1,7 @@
 package org.embulk.filter.timestamp_format;
 
+import org.embulk.filter.timestamp_format.cast.DoubleCast;
+import org.embulk.filter.timestamp_format.cast.LongCast;
 import org.embulk.filter.timestamp_format.cast.StringCast;
 import org.embulk.filter.timestamp_format.cast.TimestampCast;
 import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.ColumnConfig;
@@ -32,6 +34,8 @@ public class ColumnCaster
     private final PageBuilder pageBuilder;
     private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<>();
     private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<>();
+    private final HashMap<String, TimestampUnit> fromTimestampUnitMap = new HashMap<>();
+    private final HashMap<String, TimestampUnit> toTimestampUnitMap = new HashMap<>();
     private final JsonVisitor jsonVisitor;
 
     ColumnCaster(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
@@ -44,29 +48,23 @@ public class ColumnCaster
 
         buildTimestampParserMap();
         buildTimestampFormatterMap();
+        buildFromTimestampUnitMap();
+        buildToTimestampUnitMap();
 
-        JsonCaster jsonCaster = new JsonCaster(task, timestampParserMap, timestampFormatterMap);
+        JsonCaster jsonCaster = new JsonCaster(task, timestampParserMap, timestampFormatterMap, fromTimestampUnitMap, toTimestampUnitMap);
         this.jsonVisitor = new JsonVisitor(task, jsonCaster);
     }
 
     private void buildTimestampParserMap()
     {
         // columnName or jsonPath => TimestampParser
+        // we do not know input type of json here, so creates anyway
         for (ColumnConfig columnConfig : task.getColumns()) {
             TimestampParser parser = getTimestampParser(columnConfig, task);
             this.timestampParserMap.put(columnConfig.getName(), parser);
         }
     }
 
-    private void buildTimestampFormatterMap()
-    {
-        // columnName or jsonPath => TimestampFormatter
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
-            this.timestampFormatterMap.put(columnConfig.getName(), parser);
-        }
-    }
-
     private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
     {
         DateTimeZone timezone = columnConfig.getFromTimeZone().or(task.getDefaultFromTimeZone());
@@ -74,6 +72,17 @@ public class ColumnCaster
         return new TimestampParser(task.getJRuby(), formatList, timezone);
     }
 
+    private void buildTimestampFormatterMap()
+    {
+        // columnName or jsonPath => TimestampFormatter
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            if (columnConfig.getType() instanceof StringType) {
+                TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
+                this.timestampFormatterMap.put(columnConfig.getName(), parser);
+            }
+        }
+    }
+
     private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
     {
         String format = columnConfig.getToFormat().or(task.getDefaultToTimestampFormat());
@@ -81,6 +90,86 @@ public class ColumnCaster
         return new TimestampFormatter(task.getJRuby(), format, timezone);
     }
 
+    private void buildFromTimestampUnitMap()
+    {
+        // columnName or jsonPath => TimestampUnit
+        // we do not know input type of json here, so creates anyway
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            TimestampUnit unit = getFromTimestampUnit(columnConfig, task);
+            this.fromTimestampUnitMap.put(columnConfig.getName(), unit);
+        }
+    }
+
+    private TimestampUnit getFromTimestampUnit(ColumnConfig columnConfig, PluginTask task)
+    {
+        return columnConfig.getFromUnit().or(task.getDefaultFromTimestampUnit());
+    }
+
+    private void buildToTimestampUnitMap()
+    {
+        // columnName or jsonPath => TimestampUnit
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            Type type = columnConfig.getType();
+            if (type instanceof LongType || type instanceof DoubleType) {
+                TimestampUnit unit = getToTimestampUnit(columnConfig, task);
+                this.toTimestampUnitMap.put(columnConfig.getName(), unit);
+            }
+        }
+    }
+
+    private TimestampUnit getToTimestampUnit(ColumnConfig columnConfig, PluginTask task)
+    {
+        return columnConfig.getToUnit().or(task.getDefaultToTimestampUnit());
+    }
+
+    public void setFromLong(Column outputColumn, long value)
+    {
+        Type outputType = outputColumn.getType();
+        TimestampUnit fromUnit = fromTimestampUnitMap.get(outputColumn.getName());
+        if (outputType instanceof StringType) {
+            TimestampFormatter timestampFormatter = timestampFormatterMap.get(outputColumn.getName());
+            pageBuilder.setString(outputColumn, LongCast.asString(value, fromUnit, timestampFormatter));
+        }
+        else if (outputType instanceof TimestampType) {
+            pageBuilder.setTimestamp(outputColumn, LongCast.asTimestamp(value, fromUnit));
+        }
+        else if (outputType instanceof LongType) {
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setLong(outputColumn, LongCast.asLong(value, fromUnit, toUnit));
+        }
+        else if (outputType instanceof DoubleType) {
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setDouble(outputColumn, LongCast.asDouble(value, fromUnit, toUnit));
+        }
+        else {
+            assert false;
+        }
+    }
+
+    public void setFromDouble(Column outputColumn, double value)
+    {
+        Type outputType = outputColumn.getType();
+        TimestampUnit fromUnit = fromTimestampUnitMap.get(outputColumn.getName());
+        if (outputType instanceof StringType) {
+            TimestampFormatter timestampFormatter = timestampFormatterMap.get(outputColumn.getName());
+            pageBuilder.setString(outputColumn, DoubleCast.asString(value, fromUnit, timestampFormatter));
+        }
+        else if (outputType instanceof TimestampType) {
+            pageBuilder.setTimestamp(outputColumn, DoubleCast.asTimestamp(value, fromUnit));
+        }
+        else if (outputType instanceof LongType) {
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setLong(outputColumn, DoubleCast.asLong(value, fromUnit, toUnit));
+        }
+        else if (outputType instanceof DoubleType) {
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setDouble(outputColumn, DoubleCast.asDouble(value, fromUnit, toUnit));
+        }
+        else {
+            assert false;
+        }
+    }
+
     public void setFromString(Column outputColumn, String value)
     {
         Type outputType = outputColumn.getType();
@@ -93,10 +182,12 @@ public class ColumnCaster
             pageBuilder.setTimestamp(outputColumn, StringCast.asTimestamp(value, timestampParser));
         }
         else if (outputType instanceof LongType) {
-            pageBuilder.setLong(outputColumn, StringCast.asLong(value, timestampParser));
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setLong(outputColumn, StringCast.asLong(value, timestampParser, toUnit));
         }
         else if (outputType instanceof DoubleType) {
-            pageBuilder.setDouble(outputColumn, StringCast.asDouble(value, timestampParser));
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setDouble(outputColumn, StringCast.asDouble(value, timestampParser, toUnit));
         }
         else {
             assert false;
@@ -114,10 +205,12 @@ public class ColumnCaster
             pageBuilder.setTimestamp(outputColumn, value);
         }
         else if (outputType instanceof LongType) {
-            pageBuilder.setLong(outputColumn, TimestampCast.asLong(value));
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setLong(outputColumn, TimestampCast.asLong(value, toUnit));
         }
         else if (outputType instanceof DoubleType) {
-            pageBuilder.setDouble(outputColumn, TimestampCast.asDouble(value));
+            TimestampUnit toUnit = toTimestampUnitMap.get(outputColumn.getName());
+            pageBuilder.setDouble(outputColumn, TimestampCast.asDouble(value, toUnit));
         }
         else {
             assert false;