embulk-input-marketo-through-proxy 0.6.20

data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java

@@ -0,0 +1,85 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.base.Optional;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.ServiceResponseMapperBuildable;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Created by tai.khuu on 5/21/18.
+ */
+public class LeadServiceResponseMapperBuilder<T extends LeadServiceResponseMapperBuilder.PluginTask> implements ServiceResponseMapperBuildable<T>
+{
+    private static final Logger LOGGER = Exec.getLogger(LeadServiceResponseMapperBuilder.class);
+    private MarketoService marketoService;
+
+    private T pluginTask;
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    {
+        @Config("included_fields")
+        @ConfigDefault("null")
+        Optional<List<String>> getIncludedLeadFields();
+
+        @Config("extracted_fields")
+        @ConfigDefault("[]")
+        List<String> getExtractedFields();
+
+        void setExtractedFields(List<String> extractedFields);
+    }
+
+    public LeadServiceResponseMapperBuilder(T task, MarketoService marketoService)
+    {
+        this.pluginTask = task;
+        this.marketoService = marketoService;
+    }
+
+    protected List<MarketoField> getLeadColumns()
+    {
+        List<MarketoField> columns = marketoService.describeLead();
+        if (pluginTask.getIncludedLeadFields().isPresent() && !pluginTask.getIncludedLeadFields().get().isEmpty()) {
+            List<MarketoField> filteredColumns = new ArrayList<>();
+            List<String> includedFields = pluginTask.getIncludedLeadFields().get();
+            for (String fieldName : includedFields) {
+                Optional<MarketoField> includedField = lookupFieldIgnoreCase(columns, fieldName);
+                if (includedField.isPresent()) {
+                    filteredColumns.add(includedField.get());
+                }
+                else {
+                    LOGGER.warn("Included field [{}] not found in Marketo lead field", fieldName);
+                }
+            }
+            columns = filteredColumns;
+            LOGGER.info("Included Fields option is set, included columns: [{}]", columns);
+        }
+        return columns;
+    }
+
+    private static Optional<MarketoField> lookupFieldIgnoreCase(List<MarketoField> inputList, String lookupFieldName)
+    {
+        for (MarketoField marketoField : inputList) {
+            if (marketoField.getName().equalsIgnoreCase(lookupFieldName)) {
+                return Optional.of(marketoField);
+            }
+        }
+        return Optional.absent();
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(T task)
+    {
+        List<MarketoField> leadColumns = getLeadColumns();
+        pluginTask.setExtractedFields(MarketoUtils.getFieldNameFromMarketoFields(leadColumns));
+        return MarketoUtils.buildDynamicResponseMapper(pluginTask.getSchemaColumnPrefix(), leadColumns);
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java

@@ -0,0 +1,89 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * Created by tai.khuu on 9/18/17.
+ */
+public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<LeadWithListInputPlugin.PluginTask>
+{
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
+    {
+        @Config("list_ids")
+        @ConfigDefault("null")
+        Optional<String> getListIds();
+    }
+
+    public LeadWithListInputPlugin()
+    {
+    }
+
+    @Override
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
+    {
+        List<String> extractedFields = task.getExtractedFields();
+
+        Iterable<ObjectNode> listsToRequest;
+        if (isUserInputLists(task)) {
+            final String[] idsStr = StringUtils.split(task.getListIds().get(), ID_LIST_SEPARATOR_CHAR);
+            Function<Set<String>, Iterable<ObjectNode>> getListIds = (ids) -> marketoService.getListsByIds(ids);
+            listsToRequest = super.getObjectsByIds(idsStr, getListIds);
+        }
+        else {
+            listsToRequest = marketoService.getLists();
+        }
+
+        // Remove LIST_ID_COLUMN_NAME when sent fields to Marketo since LIST_ID_COLUMN_NAME are added by plugin code
+        extractedFields.remove(MarketoUtils.LIST_ID_COLUMN_NAME);
+        return FluentIterable.from(marketoService.getAllListLead(extractedFields, listsToRequest)).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+    }
+
+    private boolean isUserInputLists(PluginTask task)
+    {
+        return task.getListIds().isPresent() && StringUtils.isNotBlank(task.getListIds().get());
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
+            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
+            LeadWithListServiceResponseMapper serviceResponseMapper = new LeadWithListServiceResponseMapper(task, marketoService);
+            return serviceResponseMapper.buildServiceResponseMapper(task);
+        }
+    }
+
+    private static class LeadWithListServiceResponseMapper extends LeadServiceResponseMapperBuilder<PluginTask>
+    {
+        public LeadWithListServiceResponseMapper(LeadWithListInputPlugin.PluginTask task, MarketoService marketoService)
+        {
+            super(task, marketoService);
+        }
+
+        @Override
+        protected List<MarketoField> getLeadColumns()
+        {
+            List<MarketoField> leadColumns = super.getLeadColumns();
+            leadColumns.add(new MarketoField(MarketoUtils.LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
+            return leadColumns;
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java

@@ -0,0 +1,85 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * Created by tai.khuu on 9/18/17.
+ */
+public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<LeadWithProgramInputPlugin.PluginTask>
+{
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
+    {
+        @Config("program_ids")
+        @ConfigDefault("null")
+        Optional<String> getProgramIds();
+    }
+
+    @Override
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
+    {
+        List<String> fieldNames = task.getExtractedFields();
+
+        Iterable<ObjectNode> programsToRequest;
+        if (isUserInputProgs(task)) {
+            final String[] idsStr = StringUtils.split(task.getProgramIds().get(), ID_LIST_SEPARATOR_CHAR);
+            Function<Set<String>, Iterable<ObjectNode>> getListIds = (ids) -> marketoService.getProgramsByIds(ids);
+            programsToRequest = super.getObjectsByIds(idsStr, getListIds);
+        }
+        else {
+            programsToRequest = marketoService.getPrograms();
+        }
+
+        // Remove PROGRAM_ID_COLUMN_NAME when sent fields to Marketo since PROGRAM_ID_COLUMN_NAME are added by plugin code
+        fieldNames.remove(MarketoUtils.PROGRAM_ID_COLUMN_NAME);
+        return FluentIterable.from(marketoService.getAllProgramLead(fieldNames, programsToRequest)).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+    }
+
+    private boolean isUserInputProgs(LeadWithProgramInputPlugin.PluginTask task)
+    {
+        return task.getProgramIds().isPresent() && StringUtils.isNotBlank(task.getProgramIds().get());
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
+            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
+            LeadWithProgramServiceResponseMapper serviceResponseMapper = new LeadWithProgramServiceResponseMapper(task, marketoService);
+            return serviceResponseMapper.buildServiceResponseMapper(task);
+        }
+    }
+
+    private static class LeadWithProgramServiceResponseMapper extends LeadServiceResponseMapperBuilder<PluginTask>
+    {
+        public LeadWithProgramServiceResponseMapper(LeadWithProgramInputPlugin.PluginTask task, MarketoService marketoService)
+        {
+            super(task, marketoService);
+        }
+
+        @Override
+        protected List<MarketoField> getLeadColumns()
+        {
+            List<MarketoField> leadColumns = super.getLeadColumns();
+            leadColumns.add(new MarketoField(MarketoUtils.PROGRAM_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
+            return leadColumns;
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -0,0 +1,448 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.collect.Iterators;
+import org.embulk.base.restclient.jackson.JacksonServiceRecord;
+import org.embulk.base.restclient.jackson.JacksonServiceValue;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.CsvTokenizer;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.LineDecoder;
+import org.msgpack.value.Value;
+
+import java.io.InputStream;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+/**
+ * Created by tai.khuu on 9/18/17.
+ */
+public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBulkExtractInputPlugin.PluginTask> extends MarketoBaseInputPluginDelegate<T>
+{
+    private static final String FROM_DATE = "from_date";
+
+    private static final int MARKETO_MAX_RANGE_EXTRACT = 30;
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CsvTokenizer.PluginTask
+    {
+        @Config("from_date")
+        Date getFromDate();
+
+        @Config("fetch_days")
+        @ConfigDefault("1")
+        Integer getFetchDays();
+
+        @Config("latest_fetch_time")
+        @ConfigDefault("null")
+        Optional<Long> getLatestFetchTime();
+
+        @ConfigInject
+        BufferAllocator getBufferAllocator();
+
+        @Config("polling_interval_second")
+        @ConfigDefault("60")
+        Integer getPollingIntervalSecond();
+
+        @Config("bulk_job_timeout_second")
+        @ConfigDefault("3600")
+        Integer getBulkJobTimeoutSecond();
+
+        @Config("to_date")
+        @ConfigDefault("null")
+        Optional<Date> getToDate();
+
+        void setToDate(Optional<Date> toDate);
+
+        @Config("incremental_column")
+        @ConfigDefault("\"createdAt\"")
+        //Incremental column are only keep here since we don't want to introduce too much change to plugin
+        //Consider remove it in next release
+        Optional<String> getIncrementalColumn();
+
+        void setIncrementalColumn(Optional<String> incrementalColumn);
+
+        @Config("uid_column")
+        @ConfigDefault("null")
+        Optional<String> getUidColumn();
+        void setUidColumn(Optional<String> uidColumn);
+    }
+
+    @Override
+    public void validateInputTask(T task)
+    {
+        super.validateInputTask(task);
+        if (task.getFromDate() == null) {
+            throw new ConfigException("From date is required for Bulk Extract");
+        }
+        if (task.getFromDate().getTime() >= OffsetDateTime.parse(task.getJobStartTime(), DateTimeFormatter.ISO_OFFSET_DATE_TIME).toInstant().toEpochMilli()) {
+            throw new ConfigException("From date can't not be in future");
+        }
+        if (task.getIncremental()
+                && task.getIncrementalColumn().isPresent()
+                && task.getIncrementalColumn().get().equals("updatedAt")) {
+            throw new ConfigException("Column 'updatedAt' cannot be incremental imported");
+        }
+        //Calculate to date
+        OffsetDateTime toDate = getToDate(task);
+        task.setToDate(Optional.of(Date.from(toDate.toInstant())));
+    }
+
+    public OffsetDateTime getToDate(T task)
+    {
+        Date fromDate = task.getFromDate();
+        final OffsetDateTime jobStartTime = OffsetDateTime.parse(task.getJobStartTime(), DateTimeFormatter.ISO_OFFSET_DATE_TIME);
+        OffsetDateTime dateTime = OffsetDateTime.ofInstant(fromDate.toInstant(), ZoneOffset.UTC);
+        OffsetDateTime toDate = dateTime.plusDays(task.getFetchDays());
+        if (toDate.isAfter(jobStartTime)) {
+            //Lock down to date
+            toDate = jobStartTime;
+        }
+        return toDate;
+    }
+
+    @Override
+    public ConfigDiff buildConfigDiff(T task, Schema schema, int taskCount, List<TaskReport> taskReports)
+    {
+        ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
+        String incrementalColumn = task.getIncrementalColumn().orNull();
+        if (incrementalColumn != null && task.getIncremental()) {
+            DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+            // We will always move the range forward.
+            Date toDate = task.getToDate().orNull();
+            configDiff.set(FROM_DATE, df.format(toDate));
+        }
+        return configDiff;
+    }
+
+    @Override
+    public TaskReport ingestServiceData(final T task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    {
+        TaskReport taskReport = Exec.newTaskReport();
+        if (Exec.isPreview()) {
+            return importMockPreviewData(pageBuilder);
+        }
+        else {
+            try (LineDecoderIterator decoderIterator = getLineDecoderIterator(task)) {
+                Iterator<Map<String, String>> csvRecords = Iterators.concat(Iterators.transform(decoderIterator, new Function<LineDecoder, Iterator<Map<String, String>>>()
+                {
+                    @Override
+                    public Iterator<Map<String, String>> apply(LineDecoder input)
+                    {
+                        return new CsvRecordIterator(input, task);
+                    }
+                }));
+                //Keep the preview code here when we can enable real preview
+                if (Exec.isPreview()) {
+                    csvRecords = Iterators.limit(csvRecords, PREVIEW_RECORD_LIMIT);
+                }
+                int imported = 0;
+                while (csvRecords.hasNext()) {
+                    Map<String, String> csvRecord = csvRecords.next();
+                    ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(csvRecord);
+                    recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
+                    imported = imported + 1;
+                }
+                return taskReport;
+            }
+        }
+    }
+
+    /**
+     * This method should be removed when we allow skip preview phase
+     * @param pageBuilder
+     * @return TaskReport
+     */
+    private TaskReport importMockPreviewData(final PageBuilder pageBuilder)
+    {
+        final JsonParser jsonParser = new JsonParser();
+        Schema schema = pageBuilder.getSchema();
+        for (int i = 1; i <= PREVIEW_RECORD_LIMIT; i++) {
+            final int rowNum = i;
+            schema.visitColumns(new ColumnVisitor()
+            {
+                @Override
+                public void booleanColumn(Column column)
+                {
+                    pageBuilder.setBoolean(column, false);
+                }
+
+                @Override
+                public void longColumn(Column column)
+                {
+                    pageBuilder.setLong(column, 12345L);
+                }
+
+                @Override
+                public void doubleColumn(Column column)
+                {
+                    pageBuilder.setDouble(column, 12345.123);
+                }
+
+                @Override
+                public void stringColumn(Column column)
+                {
+                    pageBuilder.setString(column, column.getName() + "_" + rowNum);
+                }
+
+                @Override
+                public void timestampColumn(Column column)
+                {
+                    pageBuilder.setTimestamp(column, Timestamp.ofEpochMilli(System.currentTimeMillis()));
+                }
+
+                @Override
+                public void jsonColumn(Column column)
+                {
+                    pageBuilder.setJson(column, jsonParser.parse("{\"mockKey\":\"mockValue\"}"));
+                }
+            });
+            pageBuilder.addRecord();
+        }
+        return Exec.newTaskReport();
+    }
+
+    private LineDecoderIterator getLineDecoderIterator(T task)
+    {
+        final OffsetDateTime fromDate = OffsetDateTime.ofInstant(task.getFromDate().toInstant(), ZoneOffset.UTC);
+        final OffsetDateTime toDate = task.getToDate().isPresent() ?
+                OffsetDateTime.ofInstant(task.getToDate().get().toInstant(), ZoneOffset.UTC) :
+                OffsetDateTime.now(ZoneOffset.UTC);
+        List<MarketoUtils.DateRange> dateRanges = MarketoUtils.sliceRange(fromDate, toDate, MARKETO_MAX_RANGE_EXTRACT);
+        final Iterator<MarketoUtils.DateRange> iterator = dateRanges.iterator();
+        return new LineDecoderIterator(iterator, task);
+    }
+
+    @Override
+    protected final Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, T task)
+    {
+        throw new UnsupportedOperationException();
+    }
+
+    protected abstract InputStream getExtractedStream(MarketoService service, T task, OffsetDateTime fromDate, OffsetDateTime toDate);
+
+    private static class AllStringJacksonServiceRecord extends JacksonServiceRecord
+    {
+        public AllStringJacksonServiceRecord(ObjectNode record)
+        {
+            super(record);
+        }
+
+        @Override
+        public JacksonServiceValue getValue(ValueLocator locator)
+        {
+            // We know that this thing only contain text.
+            JacksonServiceValue value = super.getValue(locator);
+            return new StringConverterJacksonServiceRecord(value.stringValue());
+        }
+    }
+
+    private static class StringConverterJacksonServiceRecord extends JacksonServiceValue
+    {
+        private String textValue;
+
+        public StringConverterJacksonServiceRecord(String textValue)
+        {
+            super(null);
+            this.textValue = textValue;
+        }
+
+        @Override
+        public boolean isNull()
+        {
+            return textValue == null || textValue.equals("null");
+        }
+
+        @Override
+        public boolean booleanValue()
+        {
+            return Boolean.parseBoolean(textValue);
+        }
+
+        @Override
+        public double doubleValue()
+        {
+            return Double.parseDouble(textValue);
+        }
+
+        @Override
+        public Value jsonValue(JsonParser jsonParser)
+        {
+            return jsonParser.parse(textValue);
+        }
+
+        @Override
+        public long longValue()
+        {
+            return Long.parseLong(textValue);
+        }
+
+        @Override
+        public String stringValue()
+        {
+            return textValue;
+        }
+
+        @Override
+        public Timestamp timestampValue(TimestampParser timestampParser)
+        {
+            return timestampParser.parse(textValue);
+        }
+    }
+
+    private final class LineDecoderIterator implements Iterator<LineDecoder>, AutoCloseable
+    {
+        private LineDecoder currentLineDecoder;
+
+        private Iterator<MarketoUtils.DateRange> dateRangeIterator;
+
+        private MarketoService marketoService;
+
+        private MarketoRestClient marketoRestClient;
+        private T task;
+        public LineDecoderIterator(Iterator<MarketoUtils.DateRange> dateRangeIterator, T task)
+        {
+            marketoRestClient = createMarketoRestClient(task);
+            marketoService = new MarketoServiceImpl(marketoRestClient);
+            this.dateRangeIterator = dateRangeIterator;
+            this.task = task;
+        }
+
+        @Override
+        public void close()
+        {
+            if (currentLineDecoder != null) {
+                currentLineDecoder.close();
+            }
+            if (marketoRestClient != null) {
+                marketoRestClient.close();
+            }
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return dateRangeIterator.hasNext();
+        }
+
+        @Override
+        public LineDecoder next()
+        {
+            if (hasNext()) {
+                MarketoUtils.DateRange next = dateRangeIterator.next();
+                InputStream extractedStream = getExtractedStream(marketoService, task, next.fromDate, next.toDate);
+                currentLineDecoder = new LineDecoder(new InputStreamFileInput(task.getBufferAllocator(), extractedStream), task);
+                return currentLineDecoder;
+            }
+            throw new NoSuchElementException();
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException("Removed are not supported");
+        }
+    }
+
+    private class CsvRecordIterator implements Iterator<Map<String, String>>
+    {
+        private CsvTokenizer tokenizer;
+
+        private List<String> headers;
+
+        private Map<String, String> currentCsvRecord;
+        public CsvRecordIterator(LineDecoder lineDecoder, T task)
+        {
+            tokenizer = new CsvTokenizer(lineDecoder, task);
+            if (!tokenizer.nextFile()) {
+                throw new DataException("Can't read extract input stream");
+            }
+            headers = new ArrayList<>();
+            tokenizer.nextRecord();
+            while (tokenizer.hasNextColumn()) {
+                headers.add(tokenizer.nextColumn());
+            }
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            if (currentCsvRecord == null) {
+                currentCsvRecord = getNextCSVRecord();
+            }
+            return currentCsvRecord != null;
+        }
+
+        @Override
+        public Map<String, String> next()
+        {
+            try {
+                if (hasNext()) {
+                    return currentCsvRecord;
+                }
+            }
+            finally {
+                currentCsvRecord = null;
+            }
+            throw new NoSuchElementException();
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException();
+        }
+        private Map<String, String> getNextCSVRecord()
+        {
+            if (!tokenizer.nextRecord()) {
+                return null;
+            }
+            Map<String, String> kvMap = new HashMap<>();
+            try {
+                int i = 0;
+                while (tokenizer.hasNextColumn()) {
+                    kvMap.put(headers.get(i), tokenizer.nextColumnOrNull());
+                    i++;
+                }
+            }
+            catch (CsvTokenizer.InvalidValueException ex) {
+                throw new DataException("Encounter exception when parse csv file. Please check to see if you are using the correct" +
+                        "quote or escape character.", ex);
+            }
+            return kvMap;
+        }
+    }
+}