embulk-input-marketo 0.6.3 → 0.6.4.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd1171d528fb826e6f5a55aa69e3006fea9e08e3
-  data.tar.gz: 800a0f43636d84a5bdf503d8ce7d10b17007f285
+  metadata.gz: 47ff0275d371bcd43598a2ab817a3bc4c78812e7
+  data.tar.gz: 0b78b699fa476b68f513a325739dea9dd02b359d
 SHA512:
-  metadata.gz: 28b96cceda90b61e429e9c76a01d1034852ec0348ccb33e45c4d1bba8e8cf0b5e47e1d0eb89d732519cd2d55b797629481a11e237b2c0da7eb11718533bb794b
-  data.tar.gz: 36e22aa23c8f08fb8c3731fce02fe9ef329f9db9c8fbb8702e0a4053f622b82286ff4959a025ee06c58a0eca9d0988264f11f15c7bf601bc5038874f287d0eaf
+  metadata.gz: 7e893d80d51a2dc629ff237e00459587b72b39ae09e95c967e7f6ebf01eed1e9706ce6afd18470d160ac4c76188a5c1bb1e194c7684b443672af8281b72a8d45
+  data.tar.gz: f4bf3162dc79643f725a722e0ab853a2ff7b709c30cd648b581bf62b4d11339fbfd59bcdee578608e77c5e3d21e51cfd1a6053650b9de0722ca04a689ef1b2fa

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.6.4 - 2017-12-13
+* [fixed] Fix incorrect job timeout calculation  [#78](https://github.com/treasure-data/embulk-input-marketo/pull/78)
+* [enhance] Disable incremental import by updatedAt  [#77](https://github.com/treasure-data/embulk-input-marketo/pull/77)
+* [enhance] Add log for exported file size  [#76](https://github.com/treasure-data/embulk-input-marketo/pull/76)
+
+
 ## 0.6.3 - 2017-11-13
 * [enhance] Ignore records with timestamp smaller or equal to latest_fetch_time  [#74](https://github.com/treasure-data/embulk-input-marketo/pull/74)
 

data/README.md

@@ -73,7 +73,7 @@ Configuration:
 
 | name               | required | default value | description                                                                                                                                                      |
 |--------------------|----------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| **use_updated_at** | false    | false         | Lead data are not immutable so it better to do incremental ingesting with `updateAt` column, but not all Marketo Account have the feature to filter by updatedAt |
+| **use_updated_at** | false    | false       | Support filter with `updateAt` column, but not all Marketo Account have the feature to filter by updatedAt, updatedAt don't support incremental ingestion        |
 
 Schema type: Dynamic via describe lead endpoint.
 

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.3"
+version = "0.6.4.alpha.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -127,6 +127,11 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         if (task.getFromDate().getTime() >= task.getJobStartTime().getMillis()) {
             throw new ConfigException("From date can't not be in future");
         }
+        if (task.getIncremental()
+                && task.getIncrementalColumn().isPresent()
+                && task.getIncrementalColumn().get().equals("updatedAt")) {
+            throw new ConfigException("Column 'updatedAt' cannot be incremental imported");
+        }
         //Calculate to date
         DateTime toDate = getToDate(task);
         task.setToDate(Optional.of(toDate.toDate()));

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -256,6 +256,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
                 switch (status) {
                     case "Completed":
                         LOGGER.info("Total wait time ms is [{}]", waitTime);
+                        LOGGER.info("File size is [{}] bytes", objectNode.get("fileSize"));
                         return;
                     case "Failed":
                         throw new DataException("Bulk extract job failed exportId: " + exportId + " errorMessage: " + objectNode.get("errorMsg").asText());
@@ -264,7 +265,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
                 }
             }
             Thread.sleep(pollingInterval * 1000);
-            waitTime = waitTime + (System.currentTimeMillis() - now);
+            waitTime = System.currentTimeMillis() - now;
             if (waitTime >= waitTimeoutMs) {
                 throw new DataException("Job timeout exception, exportJob: " + exportId + ", run longer than " + waitTimeout + " seconds");
             }

data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java

@@ -81,7 +81,7 @@ public class LeadBulkExtractInputPluginTest
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitLeadExportJobComplete(eq(exportId1), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startLeadBulkExtract(eq(exportId2));
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitLeadExportJobComplete(eq(exportId2), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
-        String filterField = "updatedAt";
+        String filterField = "createdAt";
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createLeadBulkExtract(startDate.toDate(), startDate.plusDays(30).toDate(), fieldNameFromMarketoFields, filterField);
         DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createLeadBulkExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate(), fieldNameFromMarketoFields, filterField);
@@ -89,7 +89,7 @@ public class LeadBulkExtractInputPluginTest
         Assert.assertEquals(19, leadIds.size());
         long latestFetchTime = taskReport.get(Long.class, "latest_fetch_time");
         Assert.assertTrue(taskReport.get(Set.class, "latest_uids").isEmpty());
-        Assert.assertEquals(1504888753000L, latestFetchTime);
+        Assert.assertEquals(1504888754000L, latestFetchTime);
         Assert.assertArrayEquals(new Long[]{102488L, 102456L, 102445L, 102439L, 102471L, 102503L, 102424L, 102473L, 102505L, 102492L, 102495L, 102452L, 102435L, 102467L, 102420L, 102496L, 102448L, 102499L, 102431L}, leadIds.toArray());
     }
 }

data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java

@@ -5,6 +5,8 @@ import com.google.common.collect.Sets;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoInputPluginDelegate;
 import org.embulk.input.marketo.MarketoUtils;
@@ -16,6 +18,7 @@ import org.junit.Test;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
+import java.io.IOException;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
@@ -24,6 +27,9 @@ import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
+import static org.mockito.AdditionalAnswers.delegatesTo;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 /**
  * Created by khuutantaitai on 10/3/17.
@@ -34,11 +40,16 @@ public class MarketoBaseBulkExtractInputPluginTest
     public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
 
     private MarketoBaseBulkExtractInputPlugin<MarketoBaseBulkExtractInputPlugin.PluginTask> baseBulkExtractInputPlugin;
+    private MarketoBaseBulkExtractInputPlugin.PluginTask validBaseTask;
 
     @Before
-    public void prepare()
+    public void prepare() throws IOException
     {
         baseBulkExtractInputPlugin = Mockito.mock(MarketoBaseBulkExtractInputPlugin.class, Mockito.CALLS_REAL_METHODS);
+        ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
+        ConfigSource configSource = configLoader.fromYaml(
+                this.getClass().getResourceAsStream("/config/activity_bulk_extract_config.yaml"));
+        validBaseTask = configSource.loadConfig(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
     }
 
     @Test(expected = ConfigException.class)
@@ -49,6 +60,39 @@ public class MarketoBaseBulkExtractInputPluginTest
         baseBulkExtractInputPlugin.validateInputTask(pluginTask);
     }
 
+    @Test(expected = ConfigException.class)
+    public void invalidInputTaskWhenIncrementalByUpdatedAt()
+    {
+        MarketoBaseBulkExtractInputPlugin.PluginTask task = mock(
+                MarketoBaseBulkExtractInputPlugin.PluginTask.class,
+                delegatesTo(validBaseTask));
+        when(task.getIncrementalColumn()).thenReturn(Optional.of("updatedAt"));
+        when(task.getIncremental()).thenReturn(true);
+        baseBulkExtractInputPlugin.validateInputTask(task);
+    }
+
+    @Test
+    public void validInputTaskWhenIncrementalOtherThanUpdatedAt()
+    {
+        MarketoBaseBulkExtractInputPlugin.PluginTask task = mock(
+                MarketoBaseBulkExtractInputPlugin.PluginTask.class,
+                delegatesTo(validBaseTask));
+        when(task.getIncremental()).thenReturn(true);
+        when(task.getIncrementalColumn()).thenReturn(Optional.of("anythingButUpdatedAt"));
+        baseBulkExtractInputPlugin.validateInputTask(task);  // should not throw
+    }
+
+    @Test
+    public void validInputTaskWhenNonIncrementalWhileSetUpdatedAt()
+    {
+        MarketoBaseBulkExtractInputPlugin.PluginTask task = mock(
+                MarketoBaseBulkExtractInputPlugin.PluginTask.class,
+                delegatesTo(validBaseTask));
+        when(task.getIncremental()).thenReturn(false);
+        when(task.getIncrementalColumn()).thenReturn(Optional.of("updatedAt"));
+        baseBulkExtractInputPlugin.validateInputTask(task);  // should not throw
+    }
+
     @Test()
     public void validateInputTaskToDateLessThanJobStartTime()
     {

data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java

@@ -1,6 +1,5 @@
 package org.embulk.input.marketo.rest;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JavaType;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -237,11 +236,11 @@ public class MarketoRestClientTest
         Mockito.when(result.get("status")).thenReturn(new TextNode("Queued")).thenReturn(new TextNode("Processing"));
         Mockito.doReturn(marketoResponse).when(marketoRestClient).doGet(Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_LEAD_EXPORT_STATUS.getEndpoint(pathParams)), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.any(MarketoResponseJetty92EntityReader.class));
         try {
-            marketoRestClient.waitLeadExportJobComplete(bulkExportId, 1, 4);
+            marketoRestClient.waitLeadExportJobComplete(bulkExportId, 2, 4);
         }
         catch (DataException e) {
             Assert.assertTrue(e.getMessage().contains("Job timeout exception"));
-            Mockito.verify(marketoRestClient, Mockito.times(3)).doGet(Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_LEAD_EXPORT_STATUS.getEndpoint(pathParams)), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.any(MarketoResponseJetty92EntityReader.class));
+            Mockito.verify(marketoRestClient, Mockito.times(2)).doGet(Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_LEAD_EXPORT_STATUS.getEndpoint(pathParams)), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.any(MarketoResponseJetty92EntityReader.class));
             return;
         }
         Assert.fail();

data/src/test/resources/config/lead_bulk_extract_config.yaml

@@ -5,4 +5,4 @@ client_secret: client_secret
 fetch_days: "60"
 escape: "\""
 incremental: true
-use_updated_at: true
+use_updated_at: false

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.4.alpha.1
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-13 00:00:00.000000000 Z
+date: 2017-12-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -125,8 +125,8 @@ files:
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-input-marketo-0.6.4.alpha.1.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.3.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
@@ -145,9 +145,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9