embulk-input-marketo 0.5.7.alpha.2 → 0.5.7.alpha.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 904e8e63e2beb5f890d0692d8d881d6acadabf05
-  data.tar.gz: 02b5ed1e5c54f7d39607987115abbb11bab34dcc
+  metadata.gz: d74b33cde7c0649fe4f4cac9de7180e1ae1facce
+  data.tar.gz: 9c55a46eb01f9ff2c7d304529d90a21528ae7a6f
 SHA512:
-  metadata.gz: ce86f08d72e0ff762fe3fae49aa412dd05c82184a8db70e28c12cb557c2eb64660be93014a6f30eb6bb5f8718e49860b13cc7f91f7738cbd60f8645d2a369f5d
-  data.tar.gz: e0b44bd7eb5841db29376ae933f8e7dbe08dd36e19617f0246ae9477980a439109d134e833983675f3dba41203884c5402266ebd89879797db60172a75577344
+  metadata.gz: e3625b3fe800f9d005811df70a7f881f2183ad0a82dff1d5a5917955119c0594cc0848d73a5ef913175de6fb9f4449fc96fabed9f03ba5fd08ec0e45ad0ead85
+  data.tar.gz: db721e1064761acaf2d95f0bccc877985a2279b9ef5bf8b0a88ff1985dabe35d6cf8fbf146b7b050c9bfcf3421675fff5c937c9b547df9a1a885fc72a33d7f63

data/build.gradle

@@ -16,18 +16,18 @@ repositories {
 configurations {
     provided
 }
-version = "0.5.7.alpha.2"
+version = "0.5.7.alpha.3"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.33"
-    provided "org.embulk:embulk-core:0.8.33"
+    compile  "org.embulk:embulk-core:0.8.+"
+    provided "org.embulk:embulk-core:0.8.+"
     compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"
     compile  "org.embulk.base.restclient:embulk-util-retryhelper-jetty92:0.5.3"
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.16:tests"
-    testCompile "org.embulk:embulk-test:0.8.16"
+    testCompile "org.embulk:embulk-core:0.8.+:tests"
+    testCompile "org.embulk:embulk-test:0.8.+"
     testCompile "org.mockito:mockito-core:2.+"
 }
 

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -18,7 +18,7 @@ public interface MarketoService
 
     List<MarketoField> describeLeadByLists();
 
-    File extractLead(Date startTime, Date endTime, List<String> extractFields, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+    File extractLead(Date startTime, Date endTime, List<String> extractedFields, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
     File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -2,18 +2,13 @@ package org.embulk.input.marketo;
 
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
-import com.google.common.base.Predicate;
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Iterables;
 import com.google.common.io.ByteStreams;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.input.marketo.rest.RecordPagingIterable;
-import org.embulk.spi.Column;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
-import org.embulk.spi.type.Types;
 import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
@@ -36,10 +31,6 @@ public class MarketoServiceImpl implements MarketoService
 
     private static final String DEFAULT_FILE_FORMAT = "csv";
 
-    private static final String LIST_ID_COLUMN_NAME = "listId";
-
-    private static final String PROGRAM_ID_COLUMN_NAME = "programId";
-
     private MarketoRestClient marketoRestClient;
 
     public MarketoServiceImpl(MarketoRestClient marketoRestClient)
@@ -48,9 +39,9 @@ public class MarketoServiceImpl implements MarketoService
     }
 
     @Override
-    public File extractLead(Date startTime, Date endTime, List<String> extractFields, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    public File extractLead(Date startTime, Date endTime, List<String> extractedFields, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
     {
-        String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractFields);
+        String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields);
         marketoRestClient.startLeadBulkExtract(exportID);
         try {
             marketoRestClient.waitLeadExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
@@ -66,7 +57,7 @@ public class MarketoServiceImpl implements MarketoService
     private File saveExtractedFile(String exportID, InputStream leadBulkExtractResult)
     {
         File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
-        try (OutputStream fileOuputStream = new FileOutputStream(tempFile)){
+        try (OutputStream fileOuputStream = new FileOutputStream(tempFile)) {
             ByteStreams.copy(leadBulkExtractResult, fileOuputStream);
         }
         catch (IOException e) {
@@ -99,13 +90,13 @@ public class MarketoServiceImpl implements MarketoService
         List<Iterable<ObjectNode>> iterables = new ArrayList<>();
         for (ObjectNode node : lists) {
             final String id = node.get("id").asText();
-            iterables.add(Iterables.transform(marketoRestClient.getLeadsByList(id, filterFieldName(fieldNames, LIST_ID_COLUMN_NAME)), new Function<ObjectNode, ObjectNode>()
+            iterables.add(Iterables.transform(marketoRestClient.getLeadsByList(id, fieldNames), new Function<ObjectNode, ObjectNode>()
             {
                 @Nullable
                 @Override
                 public ObjectNode apply(@Nullable ObjectNode input)
                 {
-                    input.put(LIST_ID_COLUMN_NAME, id);
+                    input.put(MarketoUtils.LIST_ID_COLUMN_NAME, id);
                     return input;
                 }
             }));
@@ -113,18 +104,6 @@ public class MarketoServiceImpl implements MarketoService
         return Iterables.concat(iterables);
     }
 
-    public static ImmutableList<String> filterFieldName(List<String> fieldNames, final String columName)
-    {
-        return FluentIterable.from(fieldNames).filter(new Predicate<String>()
-        {
-            @Override
-            public boolean apply(@Nullable String input)
-            {
-                return !columName.equals(input);
-            }
-        }).toList();
-    }
-
     @Override
     public Iterable<ObjectNode> getAllProgramLead(List<String> fieldNames)
     {
@@ -132,13 +111,13 @@ public class MarketoServiceImpl implements MarketoService
         List<Iterable<ObjectNode>> iterables = new ArrayList<>();
         for (ObjectNode node : lists) {
             final String id = node.get("id").asText();
-            iterables.add(Iterables.transform(marketoRestClient.getLeadsByProgram(id, filterFieldName(fieldNames, PROGRAM_ID_COLUMN_NAME)), new Function<ObjectNode, ObjectNode>()
+            iterables.add(Iterables.transform(marketoRestClient.getLeadsByProgram(id, fieldNames), new Function<ObjectNode, ObjectNode>()
             {
                 @Nullable
                 @Override
                 public ObjectNode apply(@Nullable ObjectNode input)
                 {
-                    input.put(PROGRAM_ID_COLUMN_NAME, id);
+                    input.put(MarketoUtils.PROGRAM_ID_COLUMN_NAME, id);
                     return input;
                 }
             }));
@@ -162,7 +141,7 @@ public class MarketoServiceImpl implements MarketoService
     public List<MarketoField> describeLeadByProgram()
     {
         List<MarketoField> columns = marketoRestClient.describeLead();
-        columns.add(new MarketoField(PROGRAM_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
+        columns.add(new MarketoField(MarketoUtils.PROGRAM_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
         return columns;
     }
 
@@ -170,7 +149,7 @@ public class MarketoServiceImpl implements MarketoService
     public List<MarketoField> describeLeadByLists()
     {
         List<MarketoField> columns = marketoRestClient.describeLead();
-        columns.add(new MarketoField(LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
+        columns.add(new MarketoField(MarketoUtils.LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
         return columns;
     }
 }

data/src/main/java/org/embulk/input/marketo/MarketoUtils.java

@@ -3,8 +3,7 @@ package org.embulk.input.marketo;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
-import com.google.common.base.Strings;
-import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Sets;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.jackson.JacksonServiceRecord;
 import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
@@ -12,31 +11,27 @@ import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.input.marketo.model.MarketoField;
-import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
-import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
 
-import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Calendar;
 import java.util.Date;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 /**
  * Created by tai.khuu on 9/18/17.
  */
 public class MarketoUtils
 {
-    private static final Logger LOGGER = Exec.getLogger(MarketoUtils.class);
     public static final String MARKETO_DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z";
     public static final String MARKETO_DATE_FORMAT = "%Y-%m-%d";
-    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+    public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
     public static final Function<ObjectNode, ServiceRecord> TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION = new Function<ObjectNode, ServiceRecord>()
     {
         @Nullable
@@ -49,108 +44,41 @@ public class MarketoUtils
 
     public static final String MARKETO_DATE_SIMPLE_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
 
+    public static final String LIST_ID_COLUMN_NAME = "listId";
+
+    public static final String PROGRAM_ID_COLUMN_NAME = "programId";
+
     private MarketoUtils()
     {
     }
 
-    public static ServiceResponseMapper<? extends ValueLocator> buildDynamicResponseMapper(List<MarketoField> columns)
+    public static ServiceResponseMapper<? extends ValueLocator> buildDynamicResponseMapper(String prefix, List<MarketoField> columns)
     {
         JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
         for (MarketoField column : columns) {
+            String columName = buildColumnName(prefix, column.getName());
             MarketoField.MarketoDataType marketoDataType = column.getMarketoDataType();
             if (marketoDataType.getFormat().isPresent()) {
-                builder.add(new JacksonTopLevelValueLocator(column.getName()), column.getName(), marketoDataType.getType(), marketoDataType.getFormat().get());
+                builder.add(new JacksonTopLevelValueLocator(column.getName()), columName, marketoDataType.getType(), marketoDataType.getFormat().get());
             }
             else {
-                builder.add(new JacksonTopLevelValueLocator(column.getName()), column.getName(), marketoDataType.getType());
+                builder.add(new JacksonTopLevelValueLocator(column.getName()), columName, marketoDataType.getType());
             }
         }
         return builder.build();
     }
 
-    public static List<String> getFieldNameFromSchema(Schema schema)
-    {
-        return FluentIterable.from(schema.getColumns()).transform(new Function<Column, String>()
-        {
-            @Override
-            public String apply(Column input)
-            {
-                return input.getName();
-            }
-        }).toList();
-    }
-    public static  ObjectNode transformToObjectNode(final Map<String, String> kvMap, Schema schema)
+    public static List<String> getFieldNameFromMarketoFields(List<MarketoField> columns, String... excludedFields)
     {
-        final ObjectNode objectNode = OBJECT_MAPPER.createObjectNode();
-        schema.visitColumns(new ColumnVisitor()
-        {
-            @Override
-            public void booleanColumn(Column column)
-            {
-                String s = kvMap.get(column.getName());
-                objectNode.put(column.getName(), s == null ? null : Boolean.parseBoolean(s));
-            }
-
-            @Override
-            public void longColumn(Column column)
-            {
-                String s = kvMap.get(column.getName());
-                long l = 0;
-                try {
-                    l = Long.parseLong(s);
-                }
-                catch (NumberFormatException ex) {
-                    if (!Strings.isNullOrEmpty(s)) {
-                        LOGGER.error("Can't convert value : [{}] to long", s, ex);
-                        throw new DataException("Can't parse value to long");
-                    }
-                }
-                objectNode.put(column.getName(), l);
-            }
-
-            @Override
-            public void doubleColumn(Column column)
-            {
-                String s = kvMap.get(column.getName());
-                try {
-                    objectNode.put(column.getName(), s == null ? null : Double.parseDouble(s));
-                }
-                catch (NumberFormatException ex) {
-                    if (!Strings.isNullOrEmpty(s)) {
-                        LOGGER.error("Can't convert value : [{}] to double", s, ex);
-                        throw new DataException("Can't parse value to double");
-                    }
-                }
-            }
-
-            @Override
-            public void stringColumn(Column column)
-            {
-                String s = kvMap.get(column.getName());
-                objectNode.put(column.getName(), s);
-            }
-
-            @Override
-            public void timestampColumn(Column column)
-            {
-                String s = kvMap.get(column.getName());
-                objectNode.put(column.getName(), s);
-            }
-
-            @Override
-            public void jsonColumn(Column column)
-            {
-                String s = kvMap.get(column.getName());
-                try {
-                    objectNode.set(column.getName(), s != null ? OBJECT_MAPPER.readTree(s) : null);
-                }
-                catch (IOException e) {
-                    LOGGER.error("Can't convert column [{}] value [{}] to json", column.getName(), s, e);
-                    throw new DataException("Can't convert column " + column.getName() + " value to json");
-                }
+        Set<String> excludeFields= Sets.newHashSet(excludedFields);
+        List<String> extractedFields = new ArrayList<>();
+        for (MarketoField column : columns) {
+            if (excludeFields.contains(column.getName())) {
+                continue;
             }
-        });
-        return objectNode;
+            extractedFields.add(column.getName());
+        }
+        return extractedFields;
     }
 
     public static  <K, V> Map<K, V> zip(List<K> keys, List<V> values)
@@ -169,4 +97,9 @@ public class MarketoUtils
         c.add(Calendar.DATE, addDate);
         return c.getTime();
     }
+
+    public static String buildColumnName(String prefix, String columnName)
+    {
+        return prefix + "_" + columnName;
+    }
 }

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -1,10 +1,12 @@
 package org.embulk.input.marketo.delegate;
 
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
@@ -39,10 +41,10 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            List<String> fieldNames = MarketoUtils.getFieldNameFromSchema(schema);
+            List<String> fieldNames = task.getExtractedFields();
             Date fromDate = task.getFromDate().orNull();
             File file = marketoService.extractLead(fromDate, MarketoUtils.addDate(fromDate, task.getFetchDays()), fieldNames, task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond());
-                return new FileInputStream(file);
+            return new FileInputStream(file);
         }
         catch (FileNotFoundException e) {
             LOGGER.error("File not found", e);
@@ -55,7 +57,9 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            return MarketoUtils.buildDynamicResponseMapper(marketoService.describeLead());
+            List<MarketoField> columns = marketoService.describeLead();
+            task.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(columns));
+            return MarketoUtils.buildDynamicResponseMapper(task.getSchemaColumnPrefix(), columns);
         }
     }
 }

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java

@@ -9,6 +9,7 @@ import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
@@ -33,8 +34,7 @@ public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<Lead
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            List<String> fieldNames = MarketoUtils.getFieldNameFromSchema(pageBuilder.getSchema());
-            FluentIterable<ServiceRecord> serviceRecords = FluentIterable.from(marketoService.getAllListLead(fieldNames)).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION);
+            FluentIterable<ServiceRecord> serviceRecords = FluentIterable.from(marketoService.getAllListLead(task.getExtractedFields())).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION);
             int imported = 0;
             for (ServiceRecord serviceRecord : serviceRecords) {
                 if (imported >= PREVIEW_RECORD_LIMIT && Exec.isPreview()) {
@@ -52,7 +52,9 @@ public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<Lead
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            return MarketoUtils.buildDynamicResponseMapper(marketoService.describeLeadByLists());
+            List<MarketoField> columns = marketoService.describeLeadByLists();
+            task.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(columns, MarketoUtils.LIST_ID_COLUMN_NAME));
+            return MarketoUtils.buildDynamicResponseMapper(task.getSchemaColumnPrefix(), columns);
         }
     }
 }

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java

@@ -9,6 +9,7 @@ import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
@@ -29,8 +30,7 @@ public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<L
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            List<String> fieldNames = MarketoUtils.getFieldNameFromSchema(pageBuilder.getSchema());
-
+            List<String> fieldNames = task.getExtractedFields();
             FluentIterable<ServiceRecord> serviceRecords = FluentIterable.from(marketoService.getAllProgramLead(fieldNames)).
                     transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION);
             int imported = 0;
@@ -50,7 +50,9 @@ public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<L
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            return MarketoUtils.buildDynamicResponseMapper(marketoService.describeLeadByProgram());
+            List<MarketoField> columns = marketoService.describeLeadByProgram();
+            task.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(columns, MarketoUtils.PROGRAM_ID_COLUMN_NAME));
+            return MarketoUtils.buildDynamicResponseMapper(task.getSchemaColumnPrefix(), columns);
         }
     }
 }

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -4,7 +4,9 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Optional;
 import org.embulk.base.restclient.jackson.JacksonServiceRecord;
+import org.embulk.base.restclient.jackson.JacksonServiceValue;
 import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -25,8 +27,10 @@ import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.util.InputStreamFileInput;
 import org.embulk.spi.util.LineDecoder;
-import org.joda.time.DateTimeZone;
-import org.slf4j.Logger;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormatter;
+import org.joda.time.format.ISODateTimeFormat;
+import org.msgpack.value.Value;
 
 import java.io.InputStream;
 import java.text.DateFormat;
@@ -42,12 +46,12 @@ import java.util.Set;
  */
 public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBulkExtractInputPlugin.PluginTask> extends MarketoBaseInputPluginDelegate<T>
 {
-    private static final Logger LOGGER = Exec.getLogger(MarketoBaseBulkExtractInputPlugin.class);
-
     private static final String LATEST_FETCH_TIME = "latest_fetch_time";
 
     private static final String LATEST_UID_LIST = "latest_uids";
 
+    private static final DateTimeFormatter ISO_DATETIME_FORMAT = ISODateTimeFormat.dateTimeParser();
+
     public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CsvTokenizer.PluginTask
     {
         @Config("from_date")
@@ -197,11 +201,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         Set<String> latestUids = task.getPreviousUids();
         TaskReport taskReport = Exec.newTaskReport();
         int imported = 0;
-        Timestamp currentTimestamp = null;
+        DateTime currentTimestamp = null;
         if (task.getLatestFetchTime().isPresent()) {
-            currentTimestamp = Timestamp.ofEpochMilli(task.getLatestFetchTime().get());
+            currentTimestamp = new DateTime(task.getLatestFetchTime().get());
         }
-        TimestampParser timestampParser = new TimestampParser(MarketoUtils.MARKETO_DATE_TIME_FORMAT, DateTimeZone.UTC);
         try (LineDecoder lineDecoder = new LineDecoder(new InputStreamFileInput(task.getBufferAllocator(), inputStream), task)) {
             CsvTokenizer csvTokenizer = new CsvTokenizer(lineDecoder, task);
             if (!csvTokenizer.nextFile()) {
@@ -218,21 +221,20 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                     values.add(csvTokenizer.nextColumnOrNull());
                 }
                 final Map<String, String> kvMap = MarketoUtils.zip(headers, values);
-                ObjectNode objectNode = MarketoUtils.transformToObjectNode(kvMap, pageBuilder.getSchema());
-                if (!objectNode.has(incrementalColumn)) {
+                ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(kvMap);
+
+                if (!kvMap.containsKey(incrementalColumn)) {
                     throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
                 }
-                JsonNode incrementalJsonNode = objectNode.get(incrementalColumn);
                 if (uidColumn != null) {
-                    JsonNode uidField = objectNode.get(uidColumn);
-                    String uid = uidField.asText();
+                    String uid = kvMap.get(uidColumn);
                     if (latestUids.contains(uid)) {
                         //Duplicate value
                         continue;
                     }
                 }
-                String incrementalTimeStamp = incrementalJsonNode.asText();
-                Timestamp timestamp = timestampParser.parse(incrementalTimeStamp);
+                String incrementalTimeStamp = kvMap.get(incrementalColumn);
+                DateTime timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp);
                 if (currentTimestamp == null) {
                     currentTimestamp = timestamp;
                 }
@@ -251,13 +253,83 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                         }
                     }
                 }
-                recordImporter.importRecord(new JacksonServiceRecord(objectNode), pageBuilder);
+                recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
                 imported++;
             }
         }
-        taskReport.set(LATEST_FETCH_TIME, currentTimestamp == null ? null : currentTimestamp.toEpochMilli());
+        taskReport.set(LATEST_FETCH_TIME, currentTimestamp == null ? null : currentTimestamp.getMillis());
         taskReport.set(LATEST_UID_LIST, latestUids);
         return taskReport;
     }
+
     protected abstract InputStream getExtractedStream(T task, Schema schema);
+
+    private static class AllStringJacksonServiceRecord extends JacksonServiceRecord
+    {
+        public AllStringJacksonServiceRecord(ObjectNode record)
+        {
+            super(record);
+        }
+
+        @Override
+        public JacksonServiceValue getValue(ValueLocator locator)
+        {
+            // We know that this thing only contain text.
+            JacksonServiceValue value = super.getValue(locator);
+            return new StringConverterJacksonServiceRecord(value.stringValue());
+        }
+    }
+
+    private static class StringConverterJacksonServiceRecord extends JacksonServiceValue
+    {
+        private String textValue;
+
+        public StringConverterJacksonServiceRecord(String textValue)
+        {
+            super(null);
+            this.textValue = textValue;
+        }
+
+        @Override
+        public boolean isNull()
+        {
+            return textValue == null || textValue.equals("null");
+        }
+
+        @Override
+        public boolean booleanValue()
+        {
+            return Boolean.parseBoolean(textValue);
+        }
+
+        @Override
+        public double doubleValue()
+        {
+            return Double.parseDouble(textValue);
+        }
+
+        @Override
+        public Value jsonValue(JsonParser jsonParser)
+        {
+            return jsonParser.parse(textValue);
+        }
+
+        @Override
+        public long longValue()
+        {
+            return Long.parseLong(textValue);
+        }
+
+        @Override
+        public String stringValue()
+        {
+            return textValue;
+        }
+
+        @Override
+        public Timestamp timestampValue(TimestampParser timestampParser)
+        {
+            return timestampParser.parse(textValue);
+        }
+    }
 }

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java

@@ -39,6 +39,17 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
         @Config("maximum_retries_interval_milis")
         @ConfigDefault("120000")
         Integer getMaximumRetriesIntervalMilis();
+
+        @Config("schema_column_prefix")
+        @ConfigDefault("\"mk\"")
+        String getSchemaColumnPrefix();
+
+        @Config("extracted_fields")
+        @ConfigDefault("[]")
+        List<String> getExtractedFields();
+
+        void setExtractedFields(List<String> extractedFields);
+
     }
 
     @Override

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -68,7 +68,6 @@ public class MarketoRestClient extends MarketoBaseRestClient
 
     public interface PluginTask extends Task
     {
-
         @Config("account_id")
         String getAccountId();
 
@@ -85,7 +84,6 @@ public class MarketoRestClient extends MarketoBaseRestClient
         @Config("batch_size")
         @ConfigDefault("300")
         Integer getBatchSize();
-
         void setBatchSize(Integer batchSize);
     }
 
@@ -225,6 +223,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
     private void waitExportJobComplete(MarketoRESTEndpoint marketoRESTEndpoint, String exportId, int pollingInterval, int waitTimeout) throws InterruptedException
     {
         long waitTime = 0;
+        long waitTimeoutMs = waitTimeout * 1000;
         long now = System.currentTimeMillis();
         while (true) {
             MarketoResponse<ObjectNode> marketoResponse = doGet(this.endPoint + marketoRESTEndpoint.getEndpoint(
@@ -238,6 +237,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
                 LOGGER.info("Jobs [{}] status is [{}]", exportId, status);
                 switch (status) {
                     case "Completed":
+                        LOGGER.info("Total wait time ms is [{}]", waitTime);
                         return;
                     case "Failed":
                         throw new DataException("Bulk extract job failed exportId: " + exportId + " errorMessage: " + objectNode.get("errorMsg").asText());
@@ -247,7 +247,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
             }
             Thread.sleep(pollingInterval * 1000);
             waitTime = waitTime + (System.currentTimeMillis() - now);
-            if (waitTime >= (waitTimeout * 1000)) {
+            if (waitTime >= waitTimeoutMs) {
                 throw new DataException("Job timeout exception, exportJob: " + exportId + ", run longer than " + waitTimeout + " seconds");
             }
         }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.5.7.alpha.2
+  version: 0.5.7.alpha.3
 platform: ruby
 authors:
 - Tai Khuu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-22 00:00:00.000000000 Z
+date: 2017-09-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -90,9 +90,9 @@ files:
 - src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java
 - src/test/java/org/embulk/input/marketo/TestMarketoInputPlugin.java
 - src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java
+- classpath/embulk-input-marketo-0.5.7.alpha.3.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
-- classpath/embulk-input-marketo-0.5.7.alpha.2.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar