embulk-input-marketo 0.6.0 → 0.6.1.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 64d9351af259f18fec636911fe392269ac1419be
-  data.tar.gz: 1d0cc1b8dc270a739cac47e7e1623b7e29b1f50b
+  metadata.gz: ad07d4e0f62200bd2f12b4cdd7ed1dfc17542a7d
+  data.tar.gz: 1a52eb039bc6f9a1ed2f847681d1775e7d7f7d09
 SHA512:
-  metadata.gz: ef07a774a24d7f1ca97c16204ee1e564dc26a7fcb70f2d1fc4f16edbb5446c1aefbc676ff3109eb1ec65eac028042fb16ffffde6455d00e93f05fa08001e4ab2
-  data.tar.gz: a124d8beba158de3813190e26c4b9ad2db30b16a480364472cd3b9127282ba2a4a7b32b6be74676a927c417f159f5a891a4d4a675926f0b01ab1e02f3fd3bc04
+  metadata.gz: 8801d26caac30fab6ce0fbcb5f1e7df7dfaf58712e8273b5d11ae00c23e7cc70edf171efcd6d44e222412a54355b576b64dcdd0187dd94aee02494ada3df57df
+  data.tar.gz: a22b92dd76b7de41b386cf9a8a0763294a3d43d8daf5850797075da57fc95b46b4b1bb8acd3a90179314099d60d203d6cd59279450842978d598f1cde2b97ed0

data/CHANGELOG.md

@@ -1,3 +1,14 @@
+## 0.6.0 - 2017-10-10
+* [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
+* [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
+* [major] Add 3 more target, Campaign, Lead by list and Lead by program [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
+* [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
+* [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
+
+
+## 0.5.6 - 2016-12-14
+* [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
+
 ## 0.5.6 - 2016-12-14
 * [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
 

data/README.md

@@ -1,32 +1,152 @@
+[![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
+[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
+[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
+[![Gem Version](https://badge.fury.io/rb/embulk-input-marketo.svg)](http://badge.fury.io/rb/embulk-input-marketo)
+
 # Marketo input plugin for Embulk
 
-TODO: Write short description here and build.gradle file.
+embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](http://www.marketo.com/).
+
+- Lead(lead)
+- Activity log(activity)
+- Lead by list(all_lead_with_list_id)
+- Lead by program(all_lead_with_program_id)
+- Campaign(campaign)
+
+This plugin uses Marketo REST API.
 
 ## Overview
 
+Required Embulk version >= 0.8.33 (since 0.6.0).
+
 * **Plugin type**: input
-* **Resume supported**: yes
-* **Cleanup supported**: yes
+* **Resume supported**: no
+* **Cleanup supported**: no
 * **Guess supported**: no
 
+## Install
+
+```
+$ embulk gem install embulk-input-marketo
+```
+
 ## Configuration
 
-- **option1**: description (integer, required)
-- **option2**: description (string, default: `"myvalue"`)
-- **option3**: description (string, default: `null`)
+### API
+
+Below parameters are shown in "Admin" > "Web Services" page in Marketo.
+
+### Base configuration parameter
+
+All target have this configuration parameters
+
+| name                             | required | default value | description                                                                                                                      |
+|----------------------------------|----------|---------------|----------------------------------------------------------------------------------------------------------------------------------|
+| **target**                       | true     |               | Marketo targets                                                                                                                  |
+| **account_id**                   | true     |               | Marketo Muchkin id                                                                                                               |
+| **client_id**                    | true     |               | Marketo REST client id                                                                                                           |
+| **client_secret**                | true     |               | Marketo REST client secret                                                                                                       |
+| **marketo_limit_interval_milis** | false    | 20            | Marketo have limitation of 100 calls per 20 second. If REST API calls are failed they will wait this amount of time before retry |
+| **batch_size**                   | false    | 300           | Token paging batch size. Some REST API support batch                                                                             |
+| **max_return**                   | false    | 200           | Max return for Endpoint that use offset paging                                                                                   |
+
+### Bulk extract target configuration parameter (Lead and Activity)
+
+All bulk extract target use this configuration parameter
+
+| name                        | required | default value | description                                                                                                                   |
+|-----------------------------|----------|---------------|-------------------------------------------------------------------------------------------------------------------------------|
+| **from_date**               | true     |               | Import data since this date. Example: 2017-10-11T06:43:24+00:00                                                               |
+| **fetch_days**              | false    | 1             | Ammount of days to fetch since from_date                                                                                      |
+| **polling_interval_second** | false    | 60            | Amount of time to wait between pooling job status in second                                                                   |
+| **bulk_job_timeout_second** | false    | 3600          | Amount of time to wait for bulk job to complete in second                                                                     |
+| **incremental**             | false    | true          | If incremental is set to true, next run will have from_date set to the previous to_date(calculated by from_date + fetch_days) |
+| **incremental_column**      | false    | createdAt     | Column use to filter from_date and to_date                                                                                    |
+
+### Lead
+
+Lead target extract all Marketo leads, it use Marketo bulk extract feature. Configuration include bulk extract configuration. 
+
+`target: lead` 
+
+Configuration:
+
+| name               | required | default value | description                                                                                                                                                      |
+|--------------------|----------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **use_updated_at** | false    | false         | Lead data are not immutable so it better to do incremental ingesting with `updateAt` column, but not all Marketo Account have the feature to filter by updatedAt |
+
+Schema type: Dynamic via describe lead endpoint.
+
+Incremental support: yes
+
+Range ingestion: yes
+
+### Activity
+
+Activity target extract all Marketo  actvity log. Configuration include all bulk extract configuraiont
+
+`target: activity`
+
+Schema type: Static schema
+
+Incremental support: yes
+
+Range ingestion: yes
+
+### Campaign
+
+Campaign extract all campaign data from Marketo
+
+`target: campaign`
+
+Schema type: Static schema
+
+Incremental support: no
+
+Range ingestion: no
+
+### Lead by list
+
+Extract all Lead data including lead's list id
+
+`target: all_lead_with_list_id`
+
+Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
+
+Incremental support: no
+
+Range ingestion: no
+
+### Laed by program
+
+Extract all Lead data including lead's program id
+
+`target: all_lead_with_program_id`
+
+Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
+
+Incremental support: no
+
+Range ingestion: no
 
 ## Example
 
+For lead, you have `partial-config.yml` like below:
+
 ```yaml
 in:
   type: marketo
-  option1: example1
-  option2: example2
+  target: lead
+  account_id: ACCOUNT_ID
+  client_id: CLIENT_ID
+  client_secret: CLIENT_SECRET
+  from_date: 2017-09-01
+  fetch_days: 1
+out:
+  type: stdout
 ```
 
+You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-config.yml`. `lead-config.yml` includes a schema for Lead.
 
-## Build
+Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
 
-```
-$ ./gradlew gem  # -t to watch change of files and rebuild continuously
-```

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.0"
+version = "0.6.1.alpha.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -3,7 +3,8 @@ package org.embulk.input.marketo;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Function;
 import com.google.common.collect.Iterables;
-import com.google.common.io.ByteStreams;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.input.marketo.model.BulkExtractRangeHeader;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.input.marketo.rest.RecordPagingIterable;
@@ -11,14 +12,11 @@ import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
-import javax.annotation.Nullable;
-
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
 
@@ -31,6 +29,10 @@ public class MarketoServiceImpl implements MarketoService
 
     private static final String DEFAULT_FILE_FORMAT = "csv";
 
+    private static final int BUF_SIZE = 0x1000;
+
+    private static final int MAX_RESUME_TIME = 50;
+
     private MarketoRestClient marketoRestClient;
 
     public MarketoServiceImpl(MarketoRestClient marketoRestClient)
@@ -39,9 +41,9 @@ public class MarketoServiceImpl implements MarketoService
     }
 
     @Override
-    public File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    public File extractLead(final Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, final int bulkJobTimeoutSecond)
     {
-        String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
+        final String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
         marketoRestClient.startLeadBulkExtract(exportID);
         try {
             marketoRestClient.waitLeadExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
@@ -50,27 +52,41 @@ public class MarketoServiceImpl implements MarketoService
             LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
             throw new DataException("Error when wait for bulk extract");
         }
-        InputStream leadBulkExtractResult = marketoRestClient.getLeadBulkExtractResult(exportID);
-        return saveExtractedFile(exportID, leadBulkExtractResult);
+        return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
+        {
+            @Override
+            public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
+            {
+                return marketoRestClient.getLeadBulkExtractResult(exportID, bulkExtractRangeHeader);
+            }
+        });
     }
 
-    private File saveExtractedFile(String exportID, InputStream leadBulkExtractResult)
+    private long saveExtractedFile(InputStream extractResult, File tempFile) throws DownloadBulkExtractException
     {
-        File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
-        try (OutputStream fileOuputStream = new FileOutputStream(tempFile)) {
-            ByteStreams.copy(leadBulkExtractResult, fileOuputStream);
+        long total = 0;
+        try (OutputStream fileOuputStream = new FileOutputStream(tempFile, true)) {
+            byte[] buf = new byte[BUF_SIZE];
+            while (true) {
+                int r = extractResult.read(buf);
+                if (r == -1) {
+                    break;
+                }
+                fileOuputStream.write(buf, 0, r);
+                total += r;
+            }
         }
         catch (IOException e) {
-            LOGGER.error("Encounter exception when download bulk extract file, job id [{}]", exportID, e);
-            throw new DataException("Can't download bulk extract file");
+            LOGGER.error("Encounter exception when download bulk extract file", e);
+            throw new DownloadBulkExtractException("Encounter exception when download bulk extract file", e, total);
         }
-        return tempFile;
+        return total;
     }
 
     @Override
     public File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
     {
-        String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
+        final String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
         marketoRestClient.startActitvityBulkExtract(exportID);
         try {
             marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
@@ -79,49 +95,82 @@ public class MarketoServiceImpl implements MarketoService
             LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
             throw new DataException("Error when wait for bulk extract");
         }
-        InputStream activitiesBulkExtractResult = marketoRestClient.getActivitiesBulkExtractResult(exportID);
-        return saveExtractedFile(exportID, activitiesBulkExtractResult);
+        return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
+        {
+            @Override
+            public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
+            {
+                return marketoRestClient.getActivitiesBulkExtractResult(exportID, bulkExtractRangeHeader);
+            }
+        });
+    }
+    private File downloadBulkExtract(Function<BulkExtractRangeHeader, InputStream> getBulkExtractfunction)
+    {
+        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
+        long startByte = 0;
+        int resumeTime = 0;
+        while (resumeTime < MAX_RESUME_TIME) {
+            BulkExtractRangeHeader bulkExtractRangeHeader = new BulkExtractRangeHeader(startByte);
+            InputStream bulkExtractResult = getBulkExtractfunction.apply(bulkExtractRangeHeader);
+            try {
+                saveExtractedFile(bulkExtractResult, tempFile);
+                return tempFile;
+            }
+            catch (DownloadBulkExtractException e) {
+                startByte = startByte + e.getByteWritten();
+                LOGGER.warn("will resume activity bulk extract at byte [{}]", startByte);
+            }
+            resumeTime = resumeTime + 1;
+        }
+        //Too many resume we still can't get the file
+        throw new DataException("Can't down load bulk extract");
     }
-
     @Override
     public Iterable<ObjectNode> getAllListLead(List<String> fieldNames)
     {
         RecordPagingIterable<ObjectNode> lists = marketoRestClient.getLists();
-        List<Iterable<ObjectNode>> iterables = new ArrayList<>();
-        for (ObjectNode node : lists) {
-            final String id = node.get("id").asText();
-            iterables.add(Iterables.transform(marketoRestClient.getLeadsByList(id, fieldNames), new Function<ObjectNode, ObjectNode>()
+        final String fieldNameString = StringUtils.join(fieldNames, ",");
+        return MarketoUtils.flatMap(lists, new Function<ObjectNode, Iterable<ObjectNode>>()
+        {
+            @Override
+            public Iterable<ObjectNode> apply(ObjectNode input)
             {
-                @Override
-                public ObjectNode apply(ObjectNode input)
+                final String id = input.get("id").asText();
+                return Iterables.transform(marketoRestClient.getLeadsByList(id, fieldNameString), new Function<ObjectNode, ObjectNode>()
                 {
-                    input.put(MarketoUtils.LIST_ID_COLUMN_NAME, id);
-                    return input;
-                }
-            }));
-        }
-        return Iterables.concat(iterables);
+                    @Override
+                    public ObjectNode apply(ObjectNode input)
+                    {
+                        input.put(MarketoUtils.LIST_ID_COLUMN_NAME, id);
+                        return input;
+                    }
+                });
+            }
+        });
     }
 
     @Override
     public Iterable<ObjectNode> getAllProgramLead(List<String> fieldNames)
     {
         RecordPagingIterable<ObjectNode> lists = marketoRestClient.getPrograms();
-        List<Iterable<ObjectNode>> iterables = new ArrayList<>();
-        for (ObjectNode node : lists) {
-            final String id = node.get("id").asText();
-            iterables.add(Iterables.transform(marketoRestClient.getLeadsByProgram(id, fieldNames), new Function<ObjectNode, ObjectNode>()
+        final String fieldNameString = StringUtils.join(fieldNames, ",");
+        return MarketoUtils.flatMap(lists, new Function<ObjectNode, Iterable<ObjectNode>>()
+        {
+            @Override
+            public Iterable<ObjectNode> apply(ObjectNode input)
             {
-                @Nullable
-                @Override
-                public ObjectNode apply(@Nullable ObjectNode input)
+                final String id = input.get("id").asText();
+                return Iterables.transform(marketoRestClient.getLeadsByProgram(id, fieldNameString), new Function<ObjectNode, ObjectNode>()
                 {
-                    input.put(MarketoUtils.PROGRAM_ID_COLUMN_NAME, id);
-                    return input;
-                }
-            }));
-        }
-        return Iterables.concat(iterables);
+                    @Override
+                    public ObjectNode apply(ObjectNode input)
+                    {
+                        input.put(MarketoUtils.PROGRAM_ID_COLUMN_NAME, id);
+                        return input;
+                    }
+                });
+            }
+        });
     }
 
     @Override
@@ -151,4 +200,26 @@ public class MarketoServiceImpl implements MarketoService
         columns.add(new MarketoField(MarketoUtils.LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
         return columns;
     }
+
+    private static class DownloadBulkExtractException extends Exception
+    {
+        private final long byteWritten;
+
+        public DownloadBulkExtractException(String message, Throwable cause, long byteWritten)
+        {
+            super(message, cause);
+            this.byteWritten = byteWritten;
+        }
+
+        public DownloadBulkExtractException(Throwable cause, long byteWritten)
+        {
+            super(cause);
+            this.byteWritten = byteWritten;
+        }
+
+        public long getByteWritten()
+        {
+            return byteWritten;
+        }
+    }
 }

data/src/main/java/org/embulk/input/marketo/MarketoUtils.java

@@ -11,12 +11,17 @@ import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
 import org.joda.time.DateTime;
+import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
 
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
 /**
@@ -126,4 +131,82 @@ public class MarketoUtils
                     '}';
         }
     }
+
+    public static <T> T executeWithRetry(int maximumRetries, int initialRetryIntervalMillis, int maximumRetryIntervalMillis, AlwaysRetryRetryable<T> alwaysRetryRetryable) throws RetryExecutor.RetryGiveupException, InterruptedException
+    {
+        return RetryExecutor
+                .retryExecutor()
+                .withRetryLimit(maximumRetries)
+                .withInitialRetryWait(initialRetryIntervalMillis)
+                .withMaxRetryWait(maximumRetryIntervalMillis)
+                .runInterruptible(alwaysRetryRetryable);
+    }
+
+    public abstract static class AlwaysRetryRetryable<T> implements  RetryExecutor.Retryable<T>
+    {
+        private static final Logger LOGGER = Exec.getLogger(AlwaysRetryRetryable.class);
+
+        @Override
+        public abstract T call() throws Exception;
+
+        @Override
+        public boolean isRetryableException(Exception exception)
+        {
+            return true;
+        }
+
+        @Override
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException
+        {
+            LOGGER.info("Retry [{}]/[{}] with retryWait [{}] on exception {}", retryCount, retryLimit, retryWait, exception.getMessage());
+        }
+
+        @Override
+        public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException
+        {
+            LOGGER.info("Giving up execution on exception", lastException);
+        }
+    }
+    public static <T, R> Iterable<R> flatMap(final Iterable<T> iterable, final Function<T, Iterable<R>> function)
+    {
+        final Iterator<T> iterator = iterable.iterator();
+        return new Iterable<R>()
+        {
+            @Override
+            public Iterator<R> iterator()
+            {
+                return new Iterator<R>()
+                {
+                    Iterator<R> currentIterator;
+                    @Override
+                    public boolean hasNext()
+                    {
+                        if (currentIterator != null && currentIterator.hasNext()) {
+                            return true;
+                        }
+                        while (iterator.hasNext()) {
+                            currentIterator = function.apply(iterator.next()).iterator();
+                            if (currentIterator.hasNext()) {
+                                return true;
+                            }
+                        }
+                        return false;
+                    }
+                    @Override
+                    public R next()
+                    {
+                        if (hasNext()) {
+                            return currentIterator.next();
+                        }
+                        throw new NoSuchElementException();
+                    }
+                    @Override
+                    public void remove()
+                    {
+                        throw new UnsupportedOperationException();
+                    }
+                };
+            }
+        };
+    }
 }