embulk-input-marketo 0.6.1 → 0.6.2.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cea99bba894ff8f3b8bd6abd2b5d089906dc1bbc
-  data.tar.gz: 4b62273710bd3e6729eceeb73e300bb8490254c3
+  metadata.gz: 11dcb4f3b03a1a8f44ab526a8904844c66d87939
+  data.tar.gz: 19d86812f3f1d0b129a2b5212b0331455f4b64c9
 SHA512:
-  metadata.gz: 4d8f1c4b38e78f8785c22e788ed1b920f0d948401e17c5d9c666cf9c8833f6cdf652782f4486d4d74a4a5d711ddca6379bd0534c0d3a461470d3d25dcdf9b8d3
-  data.tar.gz: 8976a9a930280e271704729a0b9b85ff26bba1276a6a6504a49605e5f88565bf96a6b35d10de92bade553558a5df5fa1df0554a61430375f11484882a5cace37
+  metadata.gz: f6797483eff01213b29cce8b2b23b659419c35a2fba75d499ef55eae2950dcb5d9fa99d6996a5747646b133531bdf3b37755516a1a83c8d9c7d6f80252e0bfc8
+  data.tar.gz: 69f31537e1ca83a97f18a7b2539e2de193c1c31ca90b1637655fb0ac9101b3057ce3f52091f31982efb7c055dde962aa7b97b8e730e9e90dc3c50ee1b4b0a722

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.6.1 - 2017-10-12
+* [fixed] OutOfMemeory when run Lead by list, Lead by program   [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
+* [fixed] SOAP only field cause NullPointerException [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
+* [enhancement] Implement file download resume in Bulk extract [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
+
 ## 0.6.0 - 2017-10-10
 * [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
 * [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
@@ -5,7 +10,6 @@
 * [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
 * [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
 
-
 ## 0.5.6 - 2016-12-14
 * [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
 

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.1"
+version = "0.6.2.alpha.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -43,6 +43,7 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -123,6 +124,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         if (task.getFromDate() == null) {
             throw new ConfigException("From date is required for Bulk Extract");
         }
+        if (task.getFromDate().getTime() >= task.getJobStartTime().getMillis()) {
+            throw new ConfigException("From date can't not be in future");
+        }
         //Calculate to date
         DateTime toDate = getToDate(task);
         task.setToDate(Optional.of(toDate.toDate()));
@@ -134,8 +138,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         DateTime dateTime = new DateTime(fromDate);
         DateTime toDate = dateTime.plusDays(task.getFetchDays());
         if (toDate.isAfter(task.getJobStartTime())) {
-            //Minus 1 hour and lock down toDate
-            toDate = task.getJobStartTime().minusHours(1);
+            //Lock down to date
+            toDate = task.getJobStartTime();
         }
         return toDate;
     }
@@ -145,7 +149,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
     {
         ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
         Long currentLatestFetchTime = 0L;
-        Set latestUIds = null;
+        Set latestUIds = new HashSet();
         String incrementalColumn = task.getIncrementalColumn().orNull();
         int imported = 0;
         if (incrementalColumn != null && task.getIncremental()) {
@@ -168,11 +172,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
             }
             // in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
             if (imported == 0) {
-                configDiff.set(FROM_DATE, df.format(task.getToDate().orNull()));
+                Date toDate = task.getToDate().orNull();
+                configDiff.set(FROM_DATE, df.format(toDate));
             }
             else {
-            // Otherwise it's should start from the currentLastFetchTime. If Marketo could guaranteed that they always return data of full range we could just set it to_date
-                configDiff.set(FROM_DATE, df.format(new Date(currentLatestFetchTime)));
+            // Otherwise it's should start from the currentLastFetchTime plus 1 second.
+                configDiff.set(FROM_DATE, df.format(new DateTime(currentLatestFetchTime).plusSeconds(1).toDate()));
             }
             configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
             configDiff.set(LATEST_UID_LIST, latestUIds);
@@ -199,7 +204,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                         return new CsvRecordIterator(input, task);
                     }
                 }));
-                long currentTimestamp = 0L;
+                long currentTimestamp = task.getLatestFetchTime().or(0L);
                 Set<String> latestUids = task.getPreviousUids();
                 //Keep the preview code here when we can enable real preview
                 if (Exec.isPreview()) {

data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java

@@ -23,6 +23,7 @@ import java.util.Date;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 
 /**
  * Created by khuutantaitai on 10/3/17.
@@ -52,8 +53,10 @@ public class MarketoBaseBulkExtractInputPluginTest
     public void validateInputTaskToDateLessThanJobStartTime()
     {
         Date fromDate = new Date(1504224000000L);
+        DateTime jobStartTime = new DateTime(1506842144000L);
         MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
         Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
+        Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
         Mockito.when(pluginTask.getFetchDays()).thenReturn(7);
         baseBulkExtractInputPlugin.validateInputTask(pluginTask);
         ArgumentCaptor<Optional<Date>> argumentCaptor = ArgumentCaptor.forClass(Optional.class);
@@ -61,6 +64,24 @@ public class MarketoBaseBulkExtractInputPluginTest
         assertEquals(1504828800000L, argumentCaptor.getValue().get().getTime());
     }
 
+    @Test()
+    public void validateInputTaskFromDateMoreThanJobStartTime()
+    {
+        Date fromDate = new Date(1507619744000L);
+        DateTime jobStartTime = new DateTime(1506842144000L);
+        MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
+        Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
+        Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
+
+        try {
+            baseBulkExtractInputPlugin.validateInputTask(pluginTask);
+        }
+        catch (ConfigException ex) {
+            return;
+        }
+        fail();
+    }
+
     @Test()
     public void validateInputTaskToDateMoreThanJobStartTime()
     {
@@ -73,7 +94,7 @@ public class MarketoBaseBulkExtractInputPluginTest
         baseBulkExtractInputPlugin.validateInputTask(pluginTask);
         ArgumentCaptor<Optional<Date>> toDateArgumentCaptor = ArgumentCaptor.forClass(Optional.class);
         Mockito.verify(pluginTask, Mockito.times(1)).setToDate(toDateArgumentCaptor.capture());
-        assertEquals(jobStartTime.minusHours(1).getMillis(), toDateArgumentCaptor.getValue().get().getTime());
+        assertEquals(jobStartTime.getMillis(), toDateArgumentCaptor.getValue().get().getTime());
     }
 
     @Test

data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java

@@ -139,20 +139,6 @@ public class MarketoRestClientTest
         Assert.fail();
     }
 
-    @Test()
-    public void createLeadBulkExtractWithParseError() throws Exception
-    {
-        Mockito.doThrow(JsonProcessingException.class).when(marketoRestClient).doPost(Mockito.eq(END_POINT + MarketoRESTEndpoint.CREATE_LEAD_EXTRACT.getEndpoint()), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.anyString(), Mockito.any(MarketoResponseJetty92EntityReader.class));
-        String filterField = "filterField";
-        try {
-            marketoRestClient.createLeadBulkExtract(new Date(), new Date(), Arrays.asList("extract_field1", "extract_field2"), filterField);
-        }
-        catch (DataException ex) {
-            return;
-        }
-        Assert.fail();
-    }
-
     @Test
     public void createActitvityExtract() throws Exception
     {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2.alpha.1
 platform: ruby
 authors:
 - uu59
@@ -10,36 +10,36 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-12 00:00:00.000000000 Z
+date: 2017-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: Loads records from Marketo.
 email:
 - k@uu59.org
@@ -125,8 +125,8 @@ files:
 - src/test/resources/fixtures/program_response.json
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
-- classpath/embulk-input-marketo-0.6.1.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.6.2.alpha.1.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
@@ -145,9 +145,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9