embulk-input-marketo 0.5.7.alpha.5 → 0.5.7.alpha.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eee177691c3a0ab66a962ff6a4d3fb6773e55217
-  data.tar.gz: 73c4017cbe372d9e5bac35c085750b7edf690030
+  metadata.gz: 3c729920ccd16968b191ec5c12cf8f982f925f80
+  data.tar.gz: 1cf0acc73b4c5e90b8a11fd78770ee56791dc565
 SHA512:
-  metadata.gz: 103f84a50be5589e023d2d7ed353c20e2509deeece64257fcc31393d3d3e89c33208b3ea5779f2f69126abf0ac18ca1227bb1fb4859955d64a2d49d8041f9f59
-  data.tar.gz: f425bda7a48ad4aa94b50a039b7825ac8480ea4e568a2e8cd1ebbddfd0fcede9572226de5f3f17d1a63ab98d8112169b33631b8a751ffcb0aaff6225f165d170
+  metadata.gz: 3f14afd21ae9c968c6c0e6f1ebaf324421ae4ff10c1ceb74020ea458e970f039c2b81e9cf026c40e73089986155ea116ee1d3d5cdb47b1455b529f7de5c51506
+  data.tar.gz: f54d8dff5fc370b8b6f4b2f793c3f90f9e9f57a510383edc83b4c996dc255951d8d6ee882b65d9146e609180a861cea793a1c2c8167d2905f44c8c479c0458c1

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.5.7.alpha.5"
+version = "0.5.7.alpha.6"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java

@@ -42,8 +42,10 @@ public class MarketoInputPluginDelegate
         @ConfigDefault("120000")
         Integer getMaximumRetriesIntervalMilis();
 
+        //We don't need to let the internal plugin know that it being dispatched and force it to set require field optional
+        //We will hide the real from_date, and set it when validating task
         @Config("hidden_from_date")
-        @ConfigDefault("\"2017-09-01\"")
+        @ConfigDefault("\"1970-01-01\"")
         @Override
         Date getFromDate();
 

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -38,7 +38,7 @@ public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputP
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
             Date fromDate = task.getFromDate();
-            return new FileInputStream(marketoService.extractAllActivity(fromDate, task.getToDate().get(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+            return new FileInputStream(marketoService.extractAllActivity(fromDate, task.getToDate().orNull(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
         }
         catch (FileNotFoundException e) {
             LOGGER.error("Exception when trying to extract activity", e);

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.input.marketo.delegate;
 
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.input.marketo.MarketoService;
@@ -43,7 +42,7 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
             List<String> fieldNames = task.getExtractedFields();
             Date fromDate = task.getFromDate();
-            File file = marketoService.extractLead(fromDate, task.getToDate().get(), fieldNames, task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond());
+            File file = marketoService.extractLead(fromDate, task.getToDate().orNull(), fieldNames, task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond());
             return new FileInputStream(file);
         }
         catch (FileNotFoundException e) {

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -80,6 +80,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @ConfigDefault("3600")
         Integer getBulkJobTimeoutSecond();
 
+        @Config("incremental")
+        @ConfigDefault("true")
+        Boolean getIncremental();
+
         @Config("latest_uids")
         @ConfigDefault("[]")
         Set<String> getPreviousUids();
@@ -134,7 +138,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
         Long currentLatestFetchTime = 0L;
         Set latestUIds = null;
-        if (incrementalColumn != null) {
+        if (incrementalColumn != null && task.getIncremental()) {
             int imported = 0;
             DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
             for (TaskReport taskReport : taskReports) {
@@ -149,7 +153,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                 imported = imported + taskReport.get(Integer.class, IMPORTED_RECORD_COUNT);
             }
             // in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
-            configDiff.set(FROM_DATE, df.format(task.getToDate().get()));
+            configDiff.set(FROM_DATE, df.format(task.getToDate().orNull()));
             configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
             configDiff.set(LATEST_UID_LIST, latestUIds);
         }
@@ -178,46 +182,46 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
     {
         final JsonParser jsonParser = new JsonParser();
         Schema schema = pageBuilder.getSchema();
-        ColumnVisitor visitor = new ColumnVisitor()
-        {
-            @Override
-            public void booleanColumn(Column column)
+        for (int i = 1; i <= PREVIEW_RECORD_LIMIT; i++) {
+            final int rowNum = i;
+            schema.visitColumns(new ColumnVisitor()
             {
-                pageBuilder.setBoolean(column, false);
-            }
+                @Override
+                public void booleanColumn(Column column)
+                {
+                    pageBuilder.setBoolean(column, false);
+                }
 
-            @Override
-            public void longColumn(Column column)
-            {
-                pageBuilder.setLong(column, 12345L);
-            }
+                @Override
+                public void longColumn(Column column)
+                {
+                    pageBuilder.setLong(column, 12345L);
+                }
 
-            @Override
-            public void doubleColumn(Column column)
-            {
-                pageBuilder.setDouble(column, 12345.123);
-            }
+                @Override
+                public void doubleColumn(Column column)
+                {
+                    pageBuilder.setDouble(column, 12345.123);
+                }
 
-            @Override
-            public void stringColumn(Column column)
-            {
-                pageBuilder.setString(column, "Mock Value");
-            }
+                @Override
+                public void stringColumn(Column column)
+                {
+                    pageBuilder.setString(column, column.getName() + "_" + rowNum);
+                }
 
-            @Override
-            public void timestampColumn(Column column)
-            {
-                pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(System.currentTimeMillis()));
-            }
+                @Override
+                public void timestampColumn(Column column)
+                {
+                    pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(System.currentTimeMillis()));
+                }
 
-            @Override
-            public void jsonColumn(Column column)
-            {
-                pageBuilder.setJson(column, jsonParser.parse("{\"mockKey\":\"mockValue\"}"));
-            }
-        };
-        for (int i = 0; i < PREVIEW_RECORD_LIMIT; i++) {
-            schema.visitColumns(visitor);
+                @Override
+                public void jsonColumn(Column column)
+                {
+                    pageBuilder.setJson(column, jsonParser.parse("{\"mockKey\":\"mockValue\"}"));
+                }
+            });
             pageBuilder.addRecord();
         }
         return Exec.newTaskReport();
@@ -250,33 +254,36 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                     }
                 }
                 catch (CsvTokenizer.InvalidValueException ex) {
-                    throw new DataException("Encounter exception when parse csv file. Please check the quote or escape character", ex);
+                    throw new DataException("Encounter exception when parse csv file. Please check to see if you are using the correct" +
+                            "quote or escape character.", ex);
                 }
                 final Map<String, String> kvMap = MarketoUtils.zip(headers, values);
                 ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(kvMap);
 
-                if (!kvMap.containsKey(incrementalColumn)) {
-                    throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
-                }
-                if (uidColumn != null) {
-                    String uid = kvMap.get(uidColumn);
-                    if (latestUids.contains(uid)) {
-                        //Duplicate value
-                        continue;
+                if (task.getIncremental()) {
+                    if (!kvMap.containsKey(incrementalColumn)) {
+                        throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
                     }
-                }
-                String incrementalTimeStamp = kvMap.get(incrementalColumn);
-                long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
-                if (currentTimestamp < timestamp) {
-                    currentTimestamp = timestamp;
-                    //switch timestamp
-                    latestUids.clear();
-                }
-                else if (currentTimestamp == timestamp) {
-                    //timestamp is equal
                     if (uidColumn != null) {
-                        JsonNode uidField = objectNode.get(uidColumn);
-                        latestUids.add(uidField.asText());
+                        String uid = kvMap.get(uidColumn);
+                        if (latestUids.contains(uid)) {
+                            //Duplicate value
+                            continue;
+                        }
+                    }
+                    String incrementalTimeStamp = kvMap.get(incrementalColumn);
+                    long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
+                    if (currentTimestamp < timestamp) {
+                        currentTimestamp = timestamp;
+                        //switch timestamp
+                        latestUids.clear();
+                    }
+                    else if (currentTimestamp == timestamp) {
+                        //timestamp is equal
+                        if (uidColumn != null) {
+                            JsonNode uidField = objectNode.get(uidColumn);
+                            latestUids.add(uidField.asText());
+                        }
                     }
                 }
                 recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -2,10 +2,7 @@ package org.embulk.input.marketo.rest;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Joiner;
 import com.google.common.collect.ArrayListMultimap;
-import com.google.common.collect.ImmutableListMultimap;
-import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
 import org.eclipse.jetty.client.HttpClient;
 import org.eclipse.jetty.client.api.ContentProvider;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.5.7.alpha.5
+  version: 0.5.7.alpha.6
 platform: ruby
 authors:
 - Tai Khuu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-28 00:00:00.000000000 Z
+date: 2017-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,11 +92,11 @@ files:
 - src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
-- classpath/embulk-input-marketo-0.5.7.alpha.5.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.5.7.alpha.6.jar
 homepage:
 licenses:
 - MIT