embulk-input-marketo 0.6.6 → 0.6.7.test.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f1e72b282ed342b95e3930ef55e016638987e11e
-  data.tar.gz: fdcf297f81251c20bbaa96ffa00234bb0713f4cf
+  metadata.gz: fa96517b794a4be81d0c96ee7d02965aeeefff06
+  data.tar.gz: 2f7b3bc43e55c8009c0a7177504db1fbf53406e4
 SHA512:
-  metadata.gz: 82a613c895244d77442f9154799cd57e729ef1d41a2287a83db5faee2def0afbd6ec36f9580560b43fadf4d2b643f1280f4e44d6304af4b0cabb43744a79cf80
-  data.tar.gz: e1e5a7918318f15fbbdf83d70d39f285bcb2df3e996d3b177c9f909478821cc5b7735800f4b323446bd61c91b5485e6480671d6775e257432d605a2f9a591803
+  metadata.gz: 88f03e659566b8dd879fa52e02102cb1de879cd56264474a106eedbceb7bdc4071b93fb8975f501728568ba01d3e65be9b316e7b6ff37c14056732b14d1d117a
+  data.tar.gz: fd9ca5fd78bd3b164796a25b8322390020d96a5edacd8469f230b4da2a111f8fd3b95a0eeb1b8fed9c5654687b56e37d73332e288b037e05bc90895acd69c7c4

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.6"
+version = "0.6.7.test.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -33,8 +33,6 @@ import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.util.InputStreamFileInput;
 import org.embulk.spi.util.LineDecoder;
 import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormatter;
-import org.joda.time.format.ISODateTimeFormat;
 import org.msgpack.value.Value;
 
 import java.io.InputStream;
@@ -43,7 +41,6 @@ import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -55,18 +52,10 @@ import java.util.Set;
  */
 public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBulkExtractInputPlugin.PluginTask> extends MarketoBaseInputPluginDelegate<T>
 {
-    private static final String LATEST_FETCH_TIME = "latest_fetch_time";
-
-    private static final String LATEST_UID_LIST = "latest_uids";
-
-    private static final DateTimeFormatter ISO_DATETIME_FORMAT = ISODateTimeFormat.dateTimeParser();
-
     private static final String FROM_DATE = "from_date";
 
     private static final int MARKETO_MAX_RANGE_EXTRACT = 30;
 
-    private static final String IMPORTED = "imported";
-
     public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CsvTokenizer.PluginTask
     {
         @Config("from_date")
@@ -95,10 +84,6 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @ConfigDefault("true")
         Boolean getIncremental();
 
-        @Config("latest_uids")
-        @ConfigDefault("[]")
-        Set<String> getPreviousUids();
-
         @Config("to_date")
         @ConfigDefault("null")
         Optional<Date> getToDate();
@@ -107,6 +92,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
 
         @Config("incremental_column")
         @ConfigDefault("\"createdAt\"")
+        //Incremental column are only keep here since we don't want to introduce too much change to plugin
+        //Consider remove it in next release
         Optional<String> getIncrementalColumn();
 
         void setIncrementalColumn(Optional<String> incrementalColumn);
@@ -153,39 +140,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
     public ConfigDiff buildConfigDiff(T task, Schema schema, int taskCount, List<TaskReport> taskReports)
     {
         ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
-        Long currentLatestFetchTime = 0L;
-        Set latestUIds = new HashSet();
         String incrementalColumn = task.getIncrementalColumn().orNull();
-        int imported = 0;
         if (incrementalColumn != null && task.getIncremental()) {
             DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
-            for (TaskReport taskReport : taskReports) {
-                Long latestFetchTime = taskReport.get(Long.class, LATEST_FETCH_TIME);
-                if (latestFetchTime == null) {
-                    continue;
-                }
-                if (currentLatestFetchTime < latestFetchTime) {
-                    currentLatestFetchTime = latestFetchTime;
-                    latestUIds = taskReport.get(Set.class, LATEST_UID_LIST);
-                }
-                else if (currentLatestFetchTime.equals(latestFetchTime)) {
-                    latestUIds.addAll(taskReport.get(Set.class, LATEST_UID_LIST));
-                }
-                if (taskReport.has(IMPORTED)) {
-                    imported = imported + taskReport.get(Integer.class, IMPORTED);
-                }
-            }
-            // in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
-            if (imported == 0) {
-                Date toDate = task.getToDate().orNull();
-                configDiff.set(FROM_DATE, df.format(toDate));
-            }
-            else {
-            // Otherwise it's should start from the currentLastFetchTime plus 1 second.
-                configDiff.set(FROM_DATE, df.format(new DateTime(currentLatestFetchTime).plusSeconds(1).toDate()));
-            }
-            configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
-            configDiff.set(LATEST_UID_LIST, latestUIds);
+            // We will always move the range forward.
+            Date toDate = task.getToDate().orNull();
+            configDiff.set(FROM_DATE, df.format(toDate));
         }
         return configDiff;
     }
@@ -194,8 +154,6 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
     public TaskReport ingestServiceData(final T task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
     {
         TaskReport taskReport = Exec.newTaskReport();
-        String incrementalColumn = task.getIncrementalColumn().orNull();
-        String uidColumn = task.getUidColumn().orNull();
         if (Exec.isPreview()) {
             return importMockPreviewData(pageBuilder);
         }
@@ -209,9 +167,6 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                         return new CsvRecordIterator(input, task);
                     }
                 }));
-                Long latestFetchTime = task.getLatestFetchTime().or(0L);
-                long currentTimestamp = latestFetchTime;
-                Set<String> latestUids = task.getPreviousUids();
                 //Keep the preview code here when we can enable real preview
                 if (Exec.isPreview()) {
                     csvRecords = Iterators.limit(csvRecords, PREVIEW_RECORD_LIMIT);
@@ -219,43 +174,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                 int imported = 0;
                 while (csvRecords.hasNext()) {
                     Map<String, String> csvRecord = csvRecords.next();
-                    if (task.getIncremental()) {
-                        String incrementalTimeStamp = csvRecord.get(incrementalColumn);
-                        long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
-                        //Ignore records that have timestamp smaller or equal with latestFetchTime
-                        if (latestFetchTime >= timestamp) {
-                            continue;
-                        }
-                        if (!csvRecord.containsKey(incrementalColumn)) {
-                            throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
-                        }
-                        if (uidColumn != null) {
-                            String uid = csvRecord.get(uidColumn);
-                            if (latestUids.contains(uid)) {
-                                //Duplicate value
-                                continue;
-                            }
-                        }
-                        if (currentTimestamp < timestamp) {
-                            currentTimestamp = timestamp;
-                            //switch timestamp
-                            latestUids.clear();
-                        }
-                        else if (currentTimestamp == timestamp) {
-                            //timestamp is equal
-                            if (uidColumn != null) {
-                                String uid = csvRecord.get(uidColumn);
-                                latestUids.add(uid);
-                            }
-                        }
-                    }
                     ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(csvRecord);
                     recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
                     imported = imported + 1;
                 }
-                taskReport.set(LATEST_FETCH_TIME, currentTimestamp);
-                taskReport.set(LATEST_UID_LIST, latestUids);
-                taskReport.set(IMPORTED, imported);
                 return taskReport;
             }
         }

data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java

@@ -3,14 +3,18 @@ package org.embulk.input.marketo.delegate;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.BulkExtractRangeHeader;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
 import org.joda.time.DateTime;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -18,6 +22,8 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
@@ -74,12 +80,8 @@ public class ActivityBulkExtractInputPluginTest
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate.toDate(), startDate.plusDays(30).toDate());
         DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
-        List<String> marketoUids = argumentCaptor.getAllValues();
-        assertEquals(55, marketoUids.size());
-        long latestFetchTime = taskReport.get(Long.class, "latest_fetch_time");
-        Set latestUids = taskReport.get(Set.class, "latest_uids");
-        assertEquals(1504888754000L, latestFetchTime);
-        assertEquals(Arrays.asList("558681", "558682", "558683", "558684", "558685", "558686", "558687", "558688", "558689", "558690", "558691", "558692", "558693", "558694", "558695", "558696", "558697", "558698", "558699", "558700", "558701", "558702", "558703", "558704", "558705", "558706", "558707", "558708", "558709", "558710", "558711", "558712", "558713", "558714", "558716", "558717", "558718", "558719", "558720", "558721", "558722", "558723", "558724", "558725", "558726", "558727", "558728", "558729", "558730", "558731", "558732", "558733", "558734", "558735", "558736"), marketoUids);
-        assertEquals(36, latestUids.size());
+        ConfigDiff configDiff = activityBulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport));
+        DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+        Assert.assertEquals(df.format(startDate.plusDays(task.getFetchDays()).toDate()), configDiff.get(String.class, "from_date"));
     }
 }

data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java

@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
@@ -14,6 +15,7 @@ import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
 import org.joda.time.DateTime;
 import org.junit.Assert;
 import org.junit.Before;
@@ -23,9 +25,11 @@ import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
-import java.util.Set;
 
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
@@ -87,9 +91,9 @@ public class LeadBulkExtractInputPluginTest
         Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createLeadBulkExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate(), fieldNameFromMarketoFields, filterField);
         List<Long> leadIds = argumentCaptor.getAllValues();
         Assert.assertEquals(19, leadIds.size());
-        long latestFetchTime = taskReport.get(Long.class, "latest_fetch_time");
-        Assert.assertTrue(taskReport.get(Set.class, "latest_uids").isEmpty());
-        Assert.assertEquals(1504888754000L, latestFetchTime);
+        ConfigDiff configDiff = bulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport));
+        DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+        Assert.assertEquals(df.format(startDate.plusDays(task.getFetchDays()).toDate()), configDiff.get(String.class, "from_date"));
         Assert.assertArrayEquals(new Long[]{102488L, 102456L, 102445L, 102439L, 102471L, 102503L, 102424L, 102473L, 102505L, 102492L, 102495L, 102452L, 102435L, 102467L, 102420L, 102496L, 102448L, 102499L, 102431L}, leadIds.toArray());
     }
 }

data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java

@@ -158,20 +158,12 @@ public class MarketoBaseBulkExtractInputPluginTest
     public void buildConfigDiff() throws Exception
     {
         TaskReport taskReport1 = Mockito.mock(TaskReport.class);
-        TaskReport taskReport2 = Mockito.mock(TaskReport.class);
-        Mockito.when(taskReport1.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id1", "id2"));
-        Mockito.when(taskReport2.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id3", "id4"));
-        Mockito.when(taskReport1.get(Long.class, "latest_fetch_time")).thenReturn(1507539328000L);
-        Mockito.when(taskReport2.get(Long.class, "latest_fetch_time")).thenReturn(1507625728000L);
         MarketoInputPluginDelegate.PluginTask task = Mockito.mock(MarketoInputPluginDelegate.PluginTask.class);
         Mockito.when(task.getIncremental()).thenReturn(true);
         Mockito.when(task.getIncrementalColumn()).thenReturn(Optional.of("createdAt"));
         Date toDate = new Date(1507625728000L);
         Mockito.when(task.getToDate()).thenReturn(Optional.of(toDate));
-        ConfigDiff configDiff = baseBulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport1, taskReport2));
-        long latestFetchTime = configDiff.get(Long.class, "latest_fetch_time");
-        assertEquals(1507625728000L, latestFetchTime);
-        assertEquals(Sets.newHashSet("id3", "id4"), configDiff.get(Set.class, "latest_uids"));
+        ConfigDiff configDiff = baseBulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport1));
         DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
         assertEquals(df.format(toDate), configDiff.get(String.class, "from_date"));
     }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.6
+  version: 0.6.7.test.1
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-30 00:00:00.000000000 Z
+date: 2018-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -127,9 +127,9 @@ files:
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo-0.6.7.test.1.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.6.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo
 licenses:
 - Apache2
@@ -145,9 +145,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9