embulk-input-marketo 0.6.0 → 0.6.1.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +11 -0
  3. data/README.md +132 -12
  4. data/build.gradle +1 -1
  5. data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +115 -44
  6. data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +83 -0
  7. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +17 -3
  8. data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
  9. data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +8 -3
  10. data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +1 -1
  11. data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +2 -4
  12. data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +47 -20
  13. data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +60 -57
  14. data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +16 -15
  15. data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +10 -10
  16. data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +22 -21
  17. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +22 -20
  18. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +22 -20
  19. data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +27 -28
  20. data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +56 -57
  21. data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +136 -125
  22. metadata +6 -5
data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java CHANGED
@@ -6,6 +6,7 @@ import org.embulk.base.restclient.record.ValueLocator;
6
6
  import org.embulk.config.ConfigLoader;
7
7
  import org.embulk.config.ConfigSource;
8
8
  import org.embulk.config.TaskReport;
9
+ import org.embulk.input.marketo.model.BulkExtractRangeHeader;
9
10
  import org.embulk.input.marketo.rest.MarketoRestClient;
10
11
  import org.embulk.spi.Column;
11
12
  import org.embulk.spi.PageBuilder;
@@ -14,6 +15,7 @@ import org.junit.Before;
14
15
  import org.junit.Rule;
15
16
  import org.junit.Test;
16
17
  import org.mockito.ArgumentCaptor;
18
+ import org.mockito.Mockito;
17
19
 
18
20
  import java.io.IOException;
19
21
  import java.util.Arrays;
@@ -23,7 +25,6 @@ import java.util.Set;
23
25
 
24
26
  import static org.junit.Assert.assertEquals;
25
27
  import static org.mockito.ArgumentMatchers.any;
26
- import static org.mockito.Mockito.*;
27
28
 
28
29
  /**
29
30
  * Created by khuutantaitai on 10/3/17.
@@ -42,11 +43,11 @@ public class ActivityBulkExtractInputPluginTest
42
43
  @Before
43
44
  public void prepare() throws IOException
44
45
  {
45
- activityBulkExtractInputPlugin = spy(new ActivityBulkExtractInputPlugin());
46
+ activityBulkExtractInputPlugin = Mockito.spy(new ActivityBulkExtractInputPlugin());
46
47
  ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
47
48
  configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/activity_bulk_extract_config.yaml"));
48
- mockMarketoRestclient = mock(MarketoRestClient.class);
49
- doReturn(mockMarketoRestclient).when(activityBulkExtractInputPlugin).createMarketoRestClient(any(ActivityBulkExtractInputPlugin.PluginTask.class));
49
+ mockMarketoRestclient = Mockito.mock(MarketoRestClient.class);
50
+ Mockito.doReturn(mockMarketoRestclient).when(activityBulkExtractInputPlugin).createMarketoRestClient(any(ActivityBulkExtractInputPlugin.PluginTask.class));
50
51
  }
51
52
 
52
53
  @Test
@@ -54,25 +55,25 @@ public class ActivityBulkExtractInputPluginTest
54
55
  {
55
56
  ActivityBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(ActivityBulkExtractInputPlugin.PluginTask.class);
56
57
  DateTime startDate = new DateTime(task.getFromDate());
57
- PageBuilder pageBuilder = mock(PageBuilder.class);
58
+ PageBuilder pageBuilder = Mockito.mock(PageBuilder.class);
58
59
  String exportId1 = "exportId1";
59
60
  String exportId2 = "exportId2";
60
- when(mockMarketoRestclient.createActivityExtract(any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
61
- when(mockMarketoRestclient.getActivitiesBulkExtractResult(eq(exportId1))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract1.csv"));
62
- when(mockMarketoRestclient.getActivitiesBulkExtractResult(eq(exportId2))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract2.csv"));
61
+ Mockito.when(mockMarketoRestclient.createActivityExtract(any(Date.class), any(Date.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
62
+ Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId1), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract1.csv"));
63
+ Mockito.when(mockMarketoRestclient.getActivitiesBulkExtractResult(Mockito.eq(exportId2), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/activity_extract2.csv"));
63
64
  ServiceResponseMapper<? extends ValueLocator> mapper = activityBulkExtractInputPlugin.buildServiceResponseMapper(task);
64
65
  activityBulkExtractInputPlugin.validateInputTask(task);
65
66
  TaskReport taskReport = activityBulkExtractInputPlugin.ingestServiceData(task, mapper.createRecordImporter(), 1, pageBuilder);
66
67
  ArgumentCaptor<String> argumentCaptor = ArgumentCaptor.forClass(String.class);
67
68
  Column marketoGUID = mapper.getEmbulkSchema().lookupColumn("marketoGUID");
68
- verify(pageBuilder, times(55)).setString(eq(marketoGUID), argumentCaptor.capture());
69
- verify(mockMarketoRestclient, times(1)).startActitvityBulkExtract(eq(exportId1));
70
- verify(mockMarketoRestclient, times(1)).waitActitvityExportJobComplete(eq(exportId1), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
71
- verify(mockMarketoRestclient, times(1)).startActitvityBulkExtract(eq(exportId2));
72
- verify(mockMarketoRestclient, times(1)).waitActitvityExportJobComplete(eq(exportId2), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
73
- verify(mockMarketoRestclient, times(1)).createActivityExtract(startDate.toDate(), startDate.plusDays(30).toDate());
69
+ Mockito.verify(pageBuilder, Mockito.times(55)).setString(Mockito.eq(marketoGUID), argumentCaptor.capture());
70
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId1));
71
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId1), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
72
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startActitvityBulkExtract(Mockito.eq(exportId2));
73
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitActitvityExportJobComplete(Mockito.eq(exportId2), Mockito.eq(task.getPollingIntervalSecond()), Mockito.eq(task.getBulkJobTimeoutSecond()));
74
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate.toDate(), startDate.plusDays(30).toDate());
74
75
  DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
75
- verify(mockMarketoRestclient, times(1)).createActivityExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
76
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createActivityExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate());
76
77
  List<String> marketoUids = argumentCaptor.getAllValues();
77
78
  assertEquals(55, marketoUids.size());
78
79
  long latestFetchTime = taskReport.get(Long.class, "latest_fetch_time");
data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java CHANGED
@@ -17,12 +17,12 @@ import org.junit.Before;
17
17
  import org.junit.Rule;
18
18
  import org.junit.Test;
19
19
  import org.mockito.ArgumentCaptor;
20
+ import org.mockito.Mockito;
20
21
 
21
22
  import java.io.IOException;
22
23
  import java.util.List;
23
24
 
24
25
  import static org.junit.Assert.assertArrayEquals;
25
- import static org.mockito.Mockito.*;
26
26
 
27
27
  /**
28
28
  * Created by tai.khuu on 10/10/17.
@@ -43,30 +43,30 @@ public class CampaignInputPluginTest
43
43
  @Before
44
44
  public void setUp() throws Exception
45
45
  {
46
- campaignInputPlugin = spy(new CampaignInputPlugin());
46
+ campaignInputPlugin = Mockito.spy(new CampaignInputPlugin());
47
47
  ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
48
48
  configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
49
- mockMarketoRestClient = mock(MarketoRestClient.class);
50
- doReturn(mockMarketoRestClient).when(campaignInputPlugin).createMarketoRestClient(any(CampaignInputPlugin.PluginTask.class));
49
+ mockMarketoRestClient = Mockito.mock(MarketoRestClient.class);
50
+ Mockito.doReturn(mockMarketoRestClient).when(campaignInputPlugin).createMarketoRestClient(Mockito.any(CampaignInputPlugin.PluginTask.class));
51
51
  }
52
52
 
53
53
  @Test
54
54
  public void testRun() throws IOException
55
55
  {
56
- RecordPagingIterable<ObjectNode> mockRecordPagingIterable = mock(RecordPagingIterable.class);
56
+ RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
57
57
  JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
58
58
  List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/campaign_response_full.json"), javaType);
59
- when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
60
- when(mockMarketoRestClient.getCampaign()).thenReturn(mockRecordPagingIterable);
59
+ Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
60
+ Mockito.when(mockMarketoRestClient.getCampaign()).thenReturn(mockRecordPagingIterable);
61
61
  CampaignInputPlugin.PluginTask task = configSource.loadConfig(CampaignInputPlugin.PluginTask.class);
62
62
  ServiceResponseMapper<? extends ValueLocator> mapper = campaignInputPlugin.buildServiceResponseMapper(task);
63
63
  RecordImporter recordImporter = mapper.createRecordImporter();
64
- PageBuilder mockPageBuilder = mock(PageBuilder.class);
64
+ PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
65
65
  campaignInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
66
- verify(mockMarketoRestClient, times(1)).getCampaign();
66
+ Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getCampaign();
67
67
  Schema embulkSchema = mapper.getEmbulkSchema();
68
68
  ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
69
- verify(mockPageBuilder, times(10)).setLong(eq(embulkSchema.lookupColumn("id")), longArgumentCaptor.capture());
69
+ Mockito.verify(mockPageBuilder, Mockito.times(10)).setLong(Mockito.eq(embulkSchema.lookupColumn("id")), longArgumentCaptor.capture());
70
70
  List<Long> allValues = longArgumentCaptor.getAllValues();
71
71
  assertArrayEquals(new Long[]{1003L, 1004L, 1005L, 1006L, 1007L, 1008L, 1029L, 1048L, 1051L, 1065L}, allValues.toArray());
72
72
  }
data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java CHANGED
@@ -9,25 +9,26 @@ import org.embulk.config.ConfigLoader;
9
9
  import org.embulk.config.ConfigSource;
10
10
  import org.embulk.config.TaskReport;
11
11
  import org.embulk.input.marketo.MarketoUtils;
12
+ import org.embulk.input.marketo.model.BulkExtractRangeHeader;
12
13
  import org.embulk.input.marketo.model.MarketoField;
13
14
  import org.embulk.input.marketo.rest.MarketoRestClient;
14
15
  import org.embulk.spi.Column;
15
16
  import org.embulk.spi.PageBuilder;
16
17
  import org.joda.time.DateTime;
18
+ import org.junit.Assert;
17
19
  import org.junit.Before;
18
20
  import org.junit.Rule;
19
21
  import org.junit.Test;
20
22
  import org.mockito.ArgumentCaptor;
23
+ import org.mockito.Mockito;
21
24
 
22
25
  import java.io.IOException;
23
26
  import java.util.Date;
24
27
  import java.util.List;
25
28
  import java.util.Set;
26
29
 
27
- import static org.junit.Assert.*;
28
30
  import static org.mockito.ArgumentMatchers.any;
29
31
  import static org.mockito.ArgumentMatchers.eq;
30
- import static org.mockito.Mockito.*;
31
32
 
32
33
  /**
33
34
  * Created by khuutantaitai on 10/3/17.
@@ -48,11 +49,11 @@ public class LeadBulkExtractInputPluginTest
48
49
  @Before
49
50
  public void prepare() throws IOException
50
51
  {
51
- bulkExtractInputPlugin = spy(new LeadBulkExtractInputPlugin());
52
+ bulkExtractInputPlugin = Mockito.spy(new LeadBulkExtractInputPlugin());
52
53
  ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
53
54
  configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/lead_bulk_extract_config.yaml"));
54
- mockMarketoRestclient = mock(MarketoRestClient.class);
55
- doReturn(mockMarketoRestclient).when(bulkExtractInputPlugin).createMarketoRestClient(any(LeadBulkExtractInputPlugin.PluginTask.class));
55
+ mockMarketoRestclient = Mockito.mock(MarketoRestClient.class);
56
+ Mockito.doReturn(mockMarketoRestclient).when(bulkExtractInputPlugin).createMarketoRestClient(any(LeadBulkExtractInputPlugin.PluginTask.class));
56
57
  }
57
58
 
58
59
  @Test
@@ -60,35 +61,35 @@ public class LeadBulkExtractInputPluginTest
60
61
  {
61
62
  LeadBulkExtractInputPlugin.PluginTask task = configSource.loadConfig(LeadBulkExtractInputPlugin.PluginTask.class);
62
63
  DateTime startDate = new DateTime(task.getFromDate());
63
- PageBuilder pageBuilder = mock(PageBuilder.class);
64
+ PageBuilder pageBuilder = Mockito.mock(PageBuilder.class);
64
65
  String exportId1 = "exportId1";
65
66
  String exportId2 = "exportId2";
66
67
  JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
67
68
  List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), javaType);
68
69
  List<String> fieldNameFromMarketoFields = MarketoUtils.getFieldNameFromMarketoFields(marketoFields);
69
- when(mockMarketoRestclient.describeLead()).thenReturn(marketoFields);
70
- when(mockMarketoRestclient.createLeadBulkExtract(any(Date.class), any(Date.class), any(List.class), any(String.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
71
- when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId1))).thenReturn(this.getClass().getResourceAsStream("/fixtures/lead_extract1.csv"));
72
- when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId2))).thenReturn(this.getClass().getResourceAsStream("/fixtures/leads_extract2.csv"));
70
+ Mockito.when(mockMarketoRestclient.describeLead()).thenReturn(marketoFields);
71
+ Mockito.when(mockMarketoRestclient.createLeadBulkExtract(any(Date.class), any(Date.class), any(List.class), any(String.class))).thenReturn(exportId1).thenReturn(exportId2).thenReturn(null);
72
+ Mockito.when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId1), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/lead_extract1.csv"));
73
+ Mockito.when(mockMarketoRestclient.getLeadBulkExtractResult(eq(exportId2), any(BulkExtractRangeHeader.class))).thenReturn(this.getClass().getResourceAsStream("/fixtures/leads_extract2.csv"));
73
74
  ServiceResponseMapper<? extends ValueLocator> mapper = bulkExtractInputPlugin.buildServiceResponseMapper(task);
74
75
  bulkExtractInputPlugin.validateInputTask(task);
75
76
  TaskReport taskReport = bulkExtractInputPlugin.ingestServiceData(task, mapper.createRecordImporter(), 1, pageBuilder);
76
77
  ArgumentCaptor<Long> argumentCaptor = ArgumentCaptor.forClass(Long.class);
77
78
  Column idColumn = mapper.getEmbulkSchema().lookupColumn("mk_id");
78
- verify(pageBuilder, times(19)).setLong(eq(idColumn), argumentCaptor.capture());
79
- verify(mockMarketoRestclient, times(1)).startLeadBulkExtract(eq(exportId1));
80
- verify(mockMarketoRestclient, times(1)).waitLeadExportJobComplete(eq(exportId1), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
81
- verify(mockMarketoRestclient, times(1)).startLeadBulkExtract(eq(exportId2));
82
- verify(mockMarketoRestclient, times(1)).waitLeadExportJobComplete(eq(exportId2), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
79
+ Mockito.verify(pageBuilder, Mockito.times(19)).setLong(eq(idColumn), argumentCaptor.capture());
80
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startLeadBulkExtract(eq(exportId1));
81
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitLeadExportJobComplete(eq(exportId1), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
82
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).startLeadBulkExtract(eq(exportId2));
83
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).waitLeadExportJobComplete(eq(exportId2), eq(task.getPollingIntervalSecond()), eq(task.getBulkJobTimeoutSecond()));
83
84
  String filterField = "updatedAt";
84
- verify(mockMarketoRestclient, times(1)).createLeadBulkExtract(startDate.toDate(), startDate.plusDays(30).toDate(), fieldNameFromMarketoFields, filterField);
85
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createLeadBulkExtract(startDate.toDate(), startDate.plusDays(30).toDate(), fieldNameFromMarketoFields, filterField);
85
86
  DateTime startDate2 = startDate.plusDays(30).plusSeconds(1);
86
- verify(mockMarketoRestclient, times(1)).createLeadBulkExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate(), fieldNameFromMarketoFields, filterField);
87
+ Mockito.verify(mockMarketoRestclient, Mockito.times(1)).createLeadBulkExtract(startDate2.toDate(), startDate.plusDays(task.getFetchDays()).toDate(), fieldNameFromMarketoFields, filterField);
87
88
  List<Long> leadIds = argumentCaptor.getAllValues();
88
- assertEquals(19, leadIds.size());
89
+ Assert.assertEquals(19, leadIds.size());
89
90
  long latestFetchTime = taskReport.get(Long.class, "latest_fetch_time");
90
- assertTrue(taskReport.get(Set.class, "latest_uids").isEmpty());
91
- assertEquals(1504888753000L, latestFetchTime);
92
- assertArrayEquals(new Long[]{102488L, 102456L, 102445L, 102439L, 102471L, 102503L, 102424L, 102473L, 102505L, 102492L, 102495L, 102452L, 102435L, 102467L, 102420L, 102496L, 102448L, 102499L, 102431L}, leadIds.toArray());
91
+ Assert.assertTrue(taskReport.get(Set.class, "latest_uids").isEmpty());
92
+ Assert.assertEquals(1504888753000L, latestFetchTime);
93
+ Assert.assertArrayEquals(new Long[]{102488L, 102456L, 102445L, 102439L, 102471L, 102503L, 102424L, 102473L, 102505L, 102492L, 102495L, 102452L, 102435L, 102467L, 102420L, 102496L, 102448L, 102499L, 102431L}, leadIds.toArray());
93
94
  }
94
95
  }
data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java CHANGED
@@ -3,6 +3,7 @@ package org.embulk.input.marketo.delegate;
3
3
  import com.fasterxml.jackson.databind.JavaType;
4
4
  import com.fasterxml.jackson.databind.ObjectMapper;
5
5
  import com.fasterxml.jackson.databind.node.ObjectNode;
6
+ import org.apache.commons.lang3.StringUtils;
6
7
  import org.embulk.EmbulkTestRuntime;
7
8
  import org.embulk.base.restclient.ServiceResponseMapper;
8
9
  import org.embulk.base.restclient.record.RecordImporter;
@@ -19,6 +20,7 @@ import org.junit.Before;
19
20
  import org.junit.Rule;
20
21
  import org.junit.Test;
21
22
  import org.mockito.ArgumentCaptor;
23
+ import org.mockito.Mockito;
22
24
 
23
25
  import java.io.IOException;
24
26
  import java.util.ArrayList;
@@ -27,7 +29,6 @@ import java.util.List;
27
29
  import static org.junit.Assert.assertEquals;
28
30
  import static org.mockito.ArgumentMatchers.any;
29
31
  import static org.mockito.ArgumentMatchers.eq;
30
- import static org.mockito.Mockito.*;
31
32
 
32
33
  /**
33
34
  * Created by tai.khuu on 10/10/17.
@@ -48,49 +49,50 @@ public class LeadWithListInputPluginTest
48
49
  @Before
49
50
  public void setUp() throws Exception
50
51
  {
51
- leadWithListInputPlugin = spy(new LeadWithListInputPlugin());
52
+ leadWithListInputPlugin = Mockito.spy(new LeadWithListInputPlugin());
52
53
  ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
53
54
  configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
54
- mockMarketoRestClient = mock(MarketoRestClient.class);
55
- doReturn(mockMarketoRestClient).when(leadWithListInputPlugin).createMarketoRestClient(any(LeadWithListInputPlugin.PluginTask.class));
55
+ mockMarketoRestClient = Mockito.mock(MarketoRestClient.class);
56
+ Mockito.doReturn(mockMarketoRestClient).when(leadWithListInputPlugin).createMarketoRestClient(any(LeadWithListInputPlugin.PluginTask.class));
56
57
  }
57
58
 
58
59
  @Test
59
60
  public void testRun() throws IOException
60
61
  {
61
- RecordPagingIterable<ObjectNode> mockLeadRecordPagingIterable = mock(RecordPagingIterable.class);
62
- RecordPagingIterable<ObjectNode> mockLeadEmptyRecordPagingIterable = mock(RecordPagingIterable.class);
63
- RecordPagingIterable<ObjectNode> mocklistRecords = mock(RecordPagingIterable.class);
62
+ RecordPagingIterable<ObjectNode> mockLeadRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
63
+ RecordPagingIterable<ObjectNode> mockLeadEmptyRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
64
+ RecordPagingIterable<ObjectNode> mocklistRecords = Mockito.mock(RecordPagingIterable.class);
64
65
 
65
- when(mockLeadEmptyRecordPagingIterable.iterator()).thenReturn(new ArrayList<ObjectNode>().iterator());
66
+ Mockito.when(mockLeadEmptyRecordPagingIterable.iterator()).thenReturn(new ArrayList<ObjectNode>().iterator());
66
67
  JavaType objectNodeListType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
67
68
  JavaType marketoFieldsType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
68
69
  List<ObjectNode> leads = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_response_full.json"), objectNodeListType);
69
70
  List<ObjectNode> lists = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/list_reponse_full.json"), objectNodeListType);
70
- when(mocklistRecords.iterator()).thenReturn(lists.iterator());
71
+ Mockito.when(mocklistRecords.iterator()).thenReturn(lists.iterator());
71
72
  List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), marketoFieldsType);
72
- when(mockLeadRecordPagingIterable.iterator()).thenReturn(leads.iterator());
73
- when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
74
- when(mockMarketoRestClient.getLists()).thenReturn(mocklistRecords);
73
+ Mockito.when(mockLeadRecordPagingIterable.iterator()).thenReturn(leads.iterator());
74
+ Mockito.when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
75
+ Mockito.when(mockMarketoRestClient.getLists()).thenReturn(mocklistRecords);
75
76
  List<String> fieldNameFromMarketoFields = MarketoUtils.getFieldNameFromMarketoFields(marketoFields);
76
- when(mockMarketoRestClient.getLeadsByList(anyString(), eq(fieldNameFromMarketoFields))).thenReturn(mockLeadEmptyRecordPagingIterable);
77
- when(mockMarketoRestClient.getLeadsByList("1009", fieldNameFromMarketoFields)).thenReturn(mockLeadRecordPagingIterable);
77
+ String fieldNameString = StringUtils.join(fieldNameFromMarketoFields, ",");
78
+ Mockito.when(mockMarketoRestClient.getLeadsByList(Mockito.anyString(), eq(fieldNameString))).thenReturn(mockLeadEmptyRecordPagingIterable);
79
+ Mockito.when(mockMarketoRestClient.getLeadsByList("1009", fieldNameString)).thenReturn(mockLeadRecordPagingIterable);
78
80
 
79
81
  LeadWithListInputPlugin.PluginTask task = configSource.loadConfig(LeadWithListInputPlugin.PluginTask.class);
80
82
  ServiceResponseMapper<? extends ValueLocator> mapper = leadWithListInputPlugin.buildServiceResponseMapper(task);
81
83
 
82
84
  RecordImporter recordImporter = mapper.createRecordImporter();
83
- PageBuilder mockPageBuilder = mock(PageBuilder.class);
85
+ PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
84
86
  leadWithListInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
85
- verify(mockMarketoRestClient, times(1)).getLists();
86
- verify(mockMarketoRestClient, times(24)).getLeadsByList(anyString(), eq(fieldNameFromMarketoFields));
87
- verify(mockMarketoRestClient, times(1)).describeLead();
87
+ Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getLists();
88
+ Mockito.verify(mockMarketoRestClient, Mockito.times(24)).getLeadsByList(Mockito.anyString(), eq(fieldNameString));
89
+ Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
88
90
 
89
91
  Schema embulkSchema = mapper.getEmbulkSchema();
90
92
  ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
91
93
 
92
- verify(mockPageBuilder, times(300)).setLong(eq(embulkSchema.lookupColumn("mk_id")), longArgumentCaptor.capture());
93
- verify(mockPageBuilder, times(300)).setString(eq(embulkSchema.lookupColumn("mk_listId")), eq("1009"));
94
+ Mockito.verify(mockPageBuilder, Mockito.times(300)).setLong(eq(embulkSchema.lookupColumn("mk_id")), longArgumentCaptor.capture());
95
+ Mockito.verify(mockPageBuilder, Mockito.times(300)).setString(eq(embulkSchema.lookupColumn("mk_listId")), eq("1009"));
94
96
 
95
97
  List<Long> allValues = longArgumentCaptor.getAllValues();
96
98
  long actualValue = allValues.get(0);
data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java CHANGED
@@ -3,6 +3,7 @@ package org.embulk.input.marketo.delegate;
3
3
  import com.fasterxml.jackson.databind.JavaType;
4
4
  import com.fasterxml.jackson.databind.ObjectMapper;
5
5
  import com.fasterxml.jackson.databind.node.ObjectNode;
6
+ import org.apache.commons.lang3.StringUtils;
6
7
  import org.embulk.EmbulkTestRuntime;
7
8
  import org.embulk.base.restclient.ServiceResponseMapper;
8
9
  import org.embulk.base.restclient.record.RecordImporter;
@@ -19,6 +20,7 @@ import org.junit.Before;
19
20
  import org.junit.Rule;
20
21
  import org.junit.Test;
21
22
  import org.mockito.ArgumentCaptor;
23
+ import org.mockito.Mockito;
22
24
 
23
25
  import java.io.IOException;
24
26
  import java.util.ArrayList;
@@ -28,7 +30,6 @@ import static org.junit.Assert.assertEquals;
28
30
  import static org.mockito.ArgumentMatchers.any;
29
31
  import static org.mockito.ArgumentMatchers.anyString;
30
32
  import static org.mockito.ArgumentMatchers.eq;
31
- import static org.mockito.Mockito.*;
32
33
 
33
34
  /**
34
35
  * Created by tai.khuu on 10/10/17.
@@ -49,50 +50,51 @@ public class LeadWithProgramInputPluginTest
49
50
  @Before
50
51
  public void setUp() throws Exception
51
52
  {
52
- leadWithProgramInputPlugin = spy(new LeadWithProgramInputPlugin());
53
+ leadWithProgramInputPlugin = Mockito.spy(new LeadWithProgramInputPlugin());
53
54
  ConfigLoader configLoader = embulkTestRuntime.getInjector().getInstance(ConfigLoader.class);
54
55
  configSource = configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
55
- mockMarketoRestClient = mock(MarketoRestClient.class);
56
- doReturn(mockMarketoRestClient).when(leadWithProgramInputPlugin).createMarketoRestClient(any(LeadWithProgramInputPlugin.PluginTask.class));
56
+ mockMarketoRestClient = Mockito.mock(MarketoRestClient.class);
57
+ Mockito.doReturn(mockMarketoRestClient).when(leadWithProgramInputPlugin).createMarketoRestClient(any(LeadWithProgramInputPlugin.PluginTask.class));
57
58
  }
58
59
 
59
60
  @Test
60
61
  public void testRun() throws IOException
61
62
  {
62
- RecordPagingIterable<ObjectNode> mockLeadRecordPagingIterable = mock(RecordPagingIterable.class);
63
- RecordPagingIterable<ObjectNode> mockLeadEmptyRecordPagingIterable = mock(RecordPagingIterable.class);
64
- RecordPagingIterable<ObjectNode> mockProgramRecords = mock(RecordPagingIterable.class);
63
+ RecordPagingIterable<ObjectNode> mockLeadRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
64
+ RecordPagingIterable<ObjectNode> mockLeadEmptyRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
65
+ RecordPagingIterable<ObjectNode> mockProgramRecords = Mockito.mock(RecordPagingIterable.class);
65
66
 
66
- when(mockLeadEmptyRecordPagingIterable.iterator()).thenReturn(new ArrayList<ObjectNode>().iterator());
67
+ Mockito.when(mockLeadEmptyRecordPagingIterable.iterator()).thenReturn(new ArrayList<ObjectNode>().iterator());
67
68
  JavaType objectNodeListType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
68
69
  JavaType marketoFieldsType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, MarketoField.class);
69
70
  List<ObjectNode> leads = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_with_program_full.json"), objectNodeListType);
70
71
  List<ObjectNode> programs = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/all_program_full.json"), objectNodeListType);
71
- when(mockProgramRecords.iterator()).thenReturn(programs.iterator());
72
+ Mockito.when(mockProgramRecords.iterator()).thenReturn(programs.iterator());
72
73
  List<MarketoField> marketoFields = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/lead_describe_marketo_fields_full.json"), marketoFieldsType);
73
- when(mockLeadRecordPagingIterable.iterator()).thenReturn(leads.iterator());
74
- when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
75
- when(mockMarketoRestClient.getPrograms()).thenReturn(mockProgramRecords);
74
+ Mockito.when(mockLeadRecordPagingIterable.iterator()).thenReturn(leads.iterator());
75
+ Mockito.when(mockMarketoRestClient.describeLead()).thenReturn(marketoFields);
76
+ Mockito.when(mockMarketoRestClient.getPrograms()).thenReturn(mockProgramRecords);
76
77
  List<String> fieldNameFromMarketoFields = MarketoUtils.getFieldNameFromMarketoFields(marketoFields);
77
- when(mockMarketoRestClient.getLeadsByProgram(anyString(), eq(fieldNameFromMarketoFields))).thenReturn(mockLeadEmptyRecordPagingIterable);
78
- when(mockMarketoRestClient.getLeadsByProgram("1003", fieldNameFromMarketoFields)).thenReturn(mockLeadRecordPagingIterable);
78
+ String fieldNameString = StringUtils.join(fieldNameFromMarketoFields, ",");
79
+ Mockito.when(mockMarketoRestClient.getLeadsByProgram(anyString(), eq(fieldNameString))).thenReturn(mockLeadEmptyRecordPagingIterable);
80
+ Mockito.when(mockMarketoRestClient.getLeadsByProgram("1003", fieldNameString)).thenReturn(mockLeadRecordPagingIterable);
79
81
 
80
82
  LeadWithProgramInputPlugin.PluginTask task = configSource.loadConfig(LeadWithProgramInputPlugin.PluginTask.class);
81
83
  ServiceResponseMapper<? extends ValueLocator> mapper = leadWithProgramInputPlugin.buildServiceResponseMapper(task);
82
84
 
83
85
  RecordImporter recordImporter = mapper.createRecordImporter();
84
- PageBuilder mockPageBuilder = mock(PageBuilder.class);
86
+ PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
85
87
 
86
88
  leadWithProgramInputPlugin.ingestServiceData(task, recordImporter, 1, mockPageBuilder);
87
- verify(mockMarketoRestClient, times(1)).getPrograms();
88
- verify(mockMarketoRestClient, times(3)).getLeadsByProgram(anyString(), eq(fieldNameFromMarketoFields));
89
- verify(mockMarketoRestClient, times(1)).describeLead();
89
+ Mockito.verify(mockMarketoRestClient, Mockito.times(1)).getPrograms();
90
+ Mockito.verify(mockMarketoRestClient, Mockito.times(3)).getLeadsByProgram(anyString(), eq(fieldNameString));
91
+ Mockito.verify(mockMarketoRestClient, Mockito.times(1)).describeLead();
90
92
 
91
93
  Schema embulkSchema = mapper.getEmbulkSchema();
92
94
  ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
93
95
 
94
- verify(mockPageBuilder, times(1)).setLong(eq(embulkSchema.lookupColumn("mk_id")), longArgumentCaptor.capture());
95
- verify(mockPageBuilder, times(1)).setString(eq(embulkSchema.lookupColumn("mk_programId")), eq("1003"));
96
+ Mockito.verify(mockPageBuilder, Mockito.times(1)).setLong(eq(embulkSchema.lookupColumn("mk_id")), longArgumentCaptor.capture());
97
+ Mockito.verify(mockPageBuilder, Mockito.times(1)).setString(eq(embulkSchema.lookupColumn("mk_programId")), eq("1003"));
96
98
 
97
99
  List<Long> allValues = longArgumentCaptor.getAllValues();
98
100
  long actualValue = allValues.get(0);
data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java CHANGED
@@ -23,7 +23,6 @@ import java.util.Date;
23
23
  import java.util.Set;
24
24
 
25
25
  import static org.junit.Assert.assertEquals;
26
- import static org.mockito.Mockito.*;
27
26
 
28
27
  /**
29
28
  * Created by khuutantaitai on 10/3/17.
@@ -38,14 +37,14 @@ public class MarketoBaseBulkExtractInputPluginTest
38
37
  @Before
39
38
  public void prepare()
40
39
  {
41
- baseBulkExtractInputPlugin = mock(MarketoBaseBulkExtractInputPlugin.class, Mockito.CALLS_REAL_METHODS);
40
+ baseBulkExtractInputPlugin = Mockito.mock(MarketoBaseBulkExtractInputPlugin.class, Mockito.CALLS_REAL_METHODS);
42
41
  }
43
42
 
44
43
  @Test(expected = ConfigException.class)
45
44
  public void validateInputTaskError()
46
45
  {
47
- MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
48
- when(pluginTask.getFromDate()).thenReturn(null);
46
+ MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
47
+ Mockito.when(pluginTask.getFromDate()).thenReturn(null);
49
48
  baseBulkExtractInputPlugin.validateInputTask(pluginTask);
50
49
  }
51
50
 
@@ -53,12 +52,12 @@ public class MarketoBaseBulkExtractInputPluginTest
53
52
  public void validateInputTaskToDateLessThanJobStartTime()
54
53
  {
55
54
  Date fromDate = new Date(1504224000000L);
56
- MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
57
- when(pluginTask.getFromDate()).thenReturn(fromDate);
58
- when(pluginTask.getFetchDays()).thenReturn(7);
55
+ MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
56
+ Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
57
+ Mockito.when(pluginTask.getFetchDays()).thenReturn(7);
59
58
  baseBulkExtractInputPlugin.validateInputTask(pluginTask);
60
59
  ArgumentCaptor<Optional<Date>> argumentCaptor = ArgumentCaptor.forClass(Optional.class);
61
- verify(pluginTask, times(1)).setToDate(argumentCaptor.capture());
60
+ Mockito.verify(pluginTask, Mockito.times(1)).setToDate(argumentCaptor.capture());
62
61
  assertEquals(1504828800000L, argumentCaptor.getValue().get().getTime());
63
62
  }
64
63
 
@@ -67,13 +66,13 @@ public class MarketoBaseBulkExtractInputPluginTest
67
66
  {
68
67
  Date fromDate = new Date(1504224000000L);
69
68
  DateTime jobStartTime = new DateTime(1504396800000L);
70
- MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
71
- when(pluginTask.getFromDate()).thenReturn(fromDate);
72
- when(pluginTask.getFetchDays()).thenReturn(7);
73
- when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
69
+ MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
70
+ Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
71
+ Mockito.when(pluginTask.getFetchDays()).thenReturn(7);
72
+ Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
74
73
  baseBulkExtractInputPlugin.validateInputTask(pluginTask);
75
74
  ArgumentCaptor<Optional<Date>> toDateArgumentCaptor = ArgumentCaptor.forClass(Optional.class);
76
- verify(pluginTask, times(1)).setToDate(toDateArgumentCaptor.capture());
75
+ Mockito.verify(pluginTask, Mockito.times(1)).setToDate(toDateArgumentCaptor.capture());
77
76
  assertEquals(jobStartTime.minusHours(1).getMillis(), toDateArgumentCaptor.getValue().get().getTime());
78
77
  }
79
78
 
@@ -82,10 +81,10 @@ public class MarketoBaseBulkExtractInputPluginTest
82
81
  {
83
82
  DateTime date = new DateTime(1505033728000L);
84
83
  DateTime jobStartTime = new DateTime(1507625728000L);
85
- MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = mock(MarketoInputPluginDelegate.PluginTask.class);
86
- when(pluginTask.getFromDate()).thenReturn(date.toDate());
87
- when(pluginTask.getFetchDays()).thenReturn(30);
88
- when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
84
+ MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoInputPluginDelegate.PluginTask.class);
85
+ Mockito.when(pluginTask.getFromDate()).thenReturn(date.toDate());
86
+ Mockito.when(pluginTask.getFetchDays()).thenReturn(30);
87
+ Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
89
88
  DateTime toDate = baseBulkExtractInputPlugin.getToDate(pluginTask);
90
89
  assertEquals(toDate, jobStartTime);
91
90
  }
@@ -93,18 +92,18 @@ public class MarketoBaseBulkExtractInputPluginTest
93
92
  @Test
94
93
  public void buildConfigDiff() throws Exception
95
94
  {
96
- TaskReport taskReport1 = mock(TaskReport.class);
97
- TaskReport taskReport2 = mock(TaskReport.class);
98
- when(taskReport1.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id1", "id2"));
99
- when(taskReport2.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id3", "id4"));
100
- when(taskReport1.get(Long.class, "latest_fetch_time")).thenReturn(1507539328000L);
101
- when(taskReport2.get(Long.class, "latest_fetch_time")).thenReturn(1507625728000L);
102
- MarketoInputPluginDelegate.PluginTask task = mock(MarketoInputPluginDelegate.PluginTask.class);
103
- when(task.getIncremental()).thenReturn(true);
104
- when(task.getIncrementalColumn()).thenReturn(Optional.of("createdAt"));
95
+ TaskReport taskReport1 = Mockito.mock(TaskReport.class);
96
+ TaskReport taskReport2 = Mockito.mock(TaskReport.class);
97
+ Mockito.when(taskReport1.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id1", "id2"));
98
+ Mockito.when(taskReport2.get(Set.class, "latest_uids")).thenReturn(Sets.newHashSet("id3", "id4"));
99
+ Mockito.when(taskReport1.get(Long.class, "latest_fetch_time")).thenReturn(1507539328000L);
100
+ Mockito.when(taskReport2.get(Long.class, "latest_fetch_time")).thenReturn(1507625728000L);
101
+ MarketoInputPluginDelegate.PluginTask task = Mockito.mock(MarketoInputPluginDelegate.PluginTask.class);
102
+ Mockito.when(task.getIncremental()).thenReturn(true);
103
+ Mockito.when(task.getIncrementalColumn()).thenReturn(Optional.of("createdAt"));
105
104
  Date toDate = new Date(1507625728000L);
106
- when(task.getToDate()).thenReturn(Optional.of(toDate));
107
- ConfigDiff configDiff = baseBulkExtractInputPlugin.buildConfigDiff(task, mock(Schema.class), 1, Arrays.asList(taskReport1, taskReport2));
105
+ Mockito.when(task.getToDate()).thenReturn(Optional.of(toDate));
106
+ ConfigDiff configDiff = baseBulkExtractInputPlugin.buildConfigDiff(task, Mockito.mock(Schema.class), 1, Arrays.asList(taskReport1, taskReport2));
108
107
  long latestFetchTime = configDiff.get(Long.class, "latest_fetch_time");
109
108
  assertEquals(1507625728000L, latestFetchTime);
110
109
  assertEquals(Sets.newHashSet("id3", "id4"), configDiff.get(Set.class, "latest_uids"));