embulk-input-adebis 0.0.1.pre1

data/src/main/java/org/embulk/input/adebis/AdebisInputPlugin.java

@@ -0,0 +1,157 @@
+package org.embulk.input.adebis;
+
+import java.util.List;
+
+
+import com.google.common.base.Optional;
+import com.google.common.base.Strings;
+import org.embulk.config.*;
+import org.embulk.input.adebis.operation.AdConversionAttribute;
+import org.embulk.input.adebis.operation.Operation;
+import org.embulk.spi.*;
+
+
+public class AdebisInputPlugin
+        implements InputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("log_id")
+        public String getLogId();
+
+        @Config("log_argument")
+        public String getLogArgument();
+
+        @Config("api_key")
+        public String getApiKey();
+
+        @Config("operation")
+        public String getOperation();
+
+        @Config("condition")
+        @ConfigDefault("null")
+        public Optional<ConfigSource> getCondition();
+
+        @Config("retry_limit")
+        @ConfigDefault("5")
+        public Optional<Integer> getRetryLimit();
+
+        @Config("retry_limit_wait_sec")
+        @ConfigDefault("2")
+        public Optional<Integer> getRetryLimitWaitSec();
+
+        @Config("incremental")
+        @ConfigDefault("true")
+        public Boolean getIncremental();
+
+        @Config("last_record_time")
+        @ConfigDefault("null")
+        public Optional<String> getLastRecordTime();
+
+        @ConfigInject
+        BufferAllocator getBufferAllocator();
+    }
+
+    private Operation createOperation(String operationName) throws ConfigException {
+        switch (operationName) {
+            case "AdConversionAttribute":
+                return new AdConversionAttribute();
+            case "AdEffect":
+            case "AdEffectConversion":
+            case "AdEffectsIndirect":
+            default:
+                String msg = String.format("AdebisInputPlugin is not supported '%s'.", operationName);
+                throw new ConfigException(msg);
+        }
+    }
+
+    private void validateInputConfig(PluginTask task) throws ConfigException {
+        if (Strings.isNullOrEmpty(task.getLogId())) {
+            throw new ConfigException("log_id is empty.");
+        }
+        if (Strings.isNullOrEmpty(task.getLogArgument())) {
+            throw new ConfigException("log_argument is empty.");
+        }
+        if (Strings.isNullOrEmpty(task.getApiKey())) {
+            throw new ConfigException("api_key is empty.");
+        }
+        if (Strings.isNullOrEmpty(task.getOperation())) {
+            throw new ConfigException("operation is empty.");
+        }
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+            InputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        validateInputConfig(task);
+
+        Schema schema = createOperation(task.getOperation()).buildSchema();
+
+        return resume(task.dump(), schema, 1, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            InputPlugin.Control control)
+    {
+        List<TaskReport> report = control.run(taskSource, schema, taskCount);
+
+        ConfigDiff configDiff = Exec.newConfigDiff();
+        if (report.size() > 0 && report.get(0).has("last_record_time")) {
+            configDiff.set("last_record_time", report.get(0).get(String.class, "last_record_time"));
+        }
+
+        return configDiff;
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+        // do nothing
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource,
+            Schema schema, int taskIndex,
+            PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        BufferAllocator allocator = task.getBufferAllocator();
+        PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
+
+        Operation operation = createOperation(task.getOperation());
+        operation.setLastRecordTime(task.getLastRecordTime());
+        operation.setCredentials(
+                task.getLogId(),
+                task.getLogArgument(),
+                task.getApiKey()
+        );
+        Optional<ConfigSource> condition = task.getCondition();
+        if (condition.isPresent()) {
+            operation.loadCondition(condition.get());
+        }
+
+        operation.run(pageBuilder);
+
+        pageBuilder.finish();
+        Optional<String> lastRecordTime = operation.getLastRecordTime();
+
+        TaskReport report = Exec.newTaskReport();
+        report.set("last_record_time", lastRecordTime.or("") );
+
+        return report;
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        return Exec.newConfigDiff();
+    }
+}

data/src/main/java/org/embulk/input/adebis/operation/AbstractOperation.java

@@ -0,0 +1,100 @@
+package org.embulk.input.adebis.operation;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.spi.time.Timestamp;
+
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.time.format.ResolverStyle;
+
+public abstract class AbstractOperation implements Operation {
+
+    static protected int FETCH_LIMIT_AT_ONCE = 1000;
+    static protected DateTimeFormatter adebisDateFormat = DateTimeFormatter.ofPattern("uuuu/MM/dd").withResolverStyle(ResolverStyle.STRICT);
+    static protected DateTimeFormatter adebisDateTimeFormat = DateTimeFormatter.ofPattern("uuuu/MM/dd HH:mm:ss").withResolverStyle(ResolverStyle.STRICT);
+    static protected DateTimeFormatter embulkDateTimeFormat = DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm:ss").withResolverStyle(ResolverStyle.STRICT);
+
+    public interface Condition extends Task {
+        @Config("start_date")
+        @ConfigDefault("null")
+        public Optional<String> getStartDate();
+
+        @Config("end_date")
+        @ConfigDefault("null")
+        public Optional<String> getEndDate();
+    }
+
+    protected String logId;
+    protected String logArgument;
+    protected String apiKey;
+
+    protected Condition condition;
+    protected Optional<String> last_record_time;
+
+    public Timestamp toTimestamp(String t) {
+        try {
+            Instant instant = LocalDateTime.parse(t, adebisDateTimeFormat).toInstant(ZoneOffset.UTC);
+            return Timestamp.ofInstant(instant);
+        } catch (DateTimeParseException e) {
+            e.printStackTrace();
+            return null;
+        }
+    }
+
+    public void loadCondition(ConfigSource config) {
+        setCondition(config.loadConfig(Condition.class));
+    }
+
+    public void setCredentials(String logId, String logArgument, String apiKey){
+        setLogId(logId);
+        setLogArgument(logArgument);
+        setApiKey(apiKey);
+    }
+
+    public String getLogId() {
+        return logId;
+    }
+
+    public void setLogId(String logId) {
+        this.logId = logId;
+    }
+
+    public String getLogArgument() {
+        return logArgument;
+    }
+
+    public void setLogArgument(String logArgument) {
+        this.logArgument = logArgument;
+    }
+
+    public String getApiKey() {
+        return apiKey;
+    }
+
+    public void setApiKey(String apiKey) {
+        this.apiKey = apiKey;
+    }
+
+    public Condition getCondition() {
+        return condition;
+    }
+
+    public void setCondition(Condition condition) {
+        this.condition = condition;
+    }
+
+    public Optional<String> getLastRecordTime() {
+        return last_record_time;
+    }
+
+    public void setLastRecordTime(Optional<String> last_record_time) {
+        this.last_record_time = last_record_time;
+    }
+}

data/src/main/java/org/embulk/input/adebis/operation/AdConversionAttribute.java

@@ -0,0 +1,273 @@
+package org.embulk.input.adebis.operation;
+
+import com.google.common.base.Optional;
+import jp.ne.ebis.extreme.ws.AdServiceStub;
+import org.apache.axis2.AxisFault;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
+
+import java.rmi.RemoteException;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.List;
+
+public class AdConversionAttribute extends AbstractOperation {
+    public interface Condition extends Task {
+        @Config("start_date")
+        @ConfigDefault("null")
+        public Optional<String> getStartDate();
+
+        @Config("end_date")
+        @ConfigDefault("null")
+        public Optional<String> getEndDate();
+
+        @Config("ad_group1_ids")
+        @ConfigDefault("null")
+        public Optional<List<Integer>> getAdGroup1Ids();
+
+        @Config("ad_group2_ids")
+        @ConfigDefault("null")
+        public Optional<List<Integer>> getAdGroup2Ids();
+
+        @Config("media_ids")
+        @ConfigDefault("null")
+        public Optional<List<Integer>> getMediaIds();
+
+        @Config("ad_ids")
+        @ConfigDefault("null")
+        public Optional<List<String>> getAdIds();
+
+        @Config("terminal_types")
+        @ConfigDefault("null")
+        public Optional<List<Integer>> getTerminalTypes();
+
+        @Config("keyword")
+        @ConfigDefault("null")
+        public Optional<String> getKeyword();
+
+        @Config("conversion_ids")
+        @ConfigDefault("null")
+        public Optional<List<Integer>> getConversionIds();
+    }
+
+
+    public void run(PageBuilder pageBuilder) {
+        Schema schema = pageBuilder.getSchema();
+        AdServiceStub stub;
+        try {
+            stub = new AdServiceStub();
+        } catch (AxisFault axisFault) {
+            axisFault.printStackTrace();
+            return;
+        }
+
+        LocalDate startDate = LocalDate.parse(condition.getStartDate().get(), adebisDateFormat);
+        LocalDate endDate = LocalDate.parse(condition.getEndDate().get(), adebisDateFormat);
+        LocalDateTime lastRecordTime, newestRecordTime;
+
+        if(getLastRecordTime().isPresent())
+            lastRecordTime = LocalDateTime.parse(getLastRecordTime().get(), embulkDateTimeFormat);
+        else
+            lastRecordTime = startDate.atStartOfDay();
+
+        if(lastRecordTime.isAfter(startDate.atStartOfDay())){
+            startDate = lastRecordTime.toLocalDate();
+        }
+
+        AdServiceStub.GetAdConversionAttributes params = new AdServiceStub.GetAdConversionAttributes();
+        AdServiceStub.AdConversionAttributeCondition cond = new AdServiceStub.AdConversionAttributeCondition();
+
+        // Build Header
+        AdServiceStub.HoteiAuthHeader header = new AdServiceStub.HoteiAuthHeader();
+        AdServiceStub.HoteiAuthHeaderE headerE = new AdServiceStub.HoteiAuthHeaderE();
+        header.setLogId(getLogId());
+        header.setLogArgument(getLogArgument());
+        header.setApiKey(getApiKey());
+        headerE.setHoteiAuthHeader(header);
+
+        // Build condition
+        cond.setStartDate(startDate.format(adebisDateFormat));
+        cond.setEndDate(endDate.format(adebisDateFormat));
+
+        // Build parameter
+        params.setAdConversionAttributeCondition(cond);
+
+        int offset = 0, limit = FETCH_LIMIT_AT_ONCE;
+        params.setLimit(limit);
+        AdServiceStub.GetAdConversionAttributesResponse response;
+
+        newestRecordTime = lastRecordTime;
+        while(true) {
+            params.setOffset(offset);
+            try {
+                response = stub.getAdConversionAttributes(params, headerE);
+            } catch (RemoteException e) {
+                e.printStackTrace();
+                return;
+            } catch (NullPointerException e) {
+                return;
+            }
+
+            AdServiceStub.AdConversionAttribute[] results = response.get_return();
+            if (results != null) offset += limit;
+            else break;
+            for (AdServiceStub.AdConversionAttribute record : results) {
+                Boolean isExcluded = false;
+                for (Column c : schema.getColumns()) {
+                    switch (c.getName()) {
+                        case "amount":
+                            if (record.isAmountSpecified()) pageBuilder.setLong(c, record.getAmount());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "conversionDate":
+                            if (record.isConversionDateSpecified()) {
+                                String conversionDate = record.getConversionDate();
+                                LocalDateTime recordTime = LocalDateTime.parse(conversionDate, adebisDateTimeFormat);
+                                if (recordTime.isAfter(lastRecordTime)) {
+                                    pageBuilder.setTimestamp(c, toTimestamp(conversionDate));
+                                    if (recordTime.isAfter(newestRecordTime))
+                                        newestRecordTime = recordTime;
+                                } else {
+                                    isExcluded = true;
+                                }
+                            } else pageBuilder.setNull(c);
+                            break;
+                        case "conversionId":
+                            if (record.isConversionIdSpecified()) pageBuilder.setLong(c, record.getConversionId());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "directAd":
+                            if (record.isDirectAdSpecified()) pageBuilder.setString(c, record.getDirectAd());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "firstAd":
+                            if (record.isFirstAdSpecified()) pageBuilder.setString(c, record.getFirstAd());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "firstAdDate":
+                            if (record.isFirstAdDateSpecified()) {
+                                String fistAdDateSpecified = record.getFirstAdDate();
+                                pageBuilder.setTimestamp(c, toTimestamp(fistAdDateSpecified));
+                            } else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd2":
+                            if (record.isIndirectAd2Specified()) pageBuilder.setString(c, record.getIndirectAd2());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd3":
+                            if (record.isIndirectAd3Specified()) pageBuilder.setString(c, record.getIndirectAd3());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd4":
+                            if (record.isIndirectAd4Specified()) pageBuilder.setString(c, record.getIndirectAd4());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd5":
+                            if (record.isIndirectAd5Specified()) pageBuilder.setString(c, record.getIndirectAd5());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd6":
+                            if (record.isIndirectAd6Specified()) pageBuilder.setString(c, record.getIndirectAd6());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd7":
+                            if (record.isIndirectAd7Specified()) pageBuilder.setString(c, record.getIndirectAd7());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd8":
+                            if (record.isIndirectAd8Specified()) pageBuilder.setString(c, record.getIndirectAd8());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd9":
+                            if (record.isIndirectAd9Specified()) pageBuilder.setString(c, record.getIndirectAd9());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "indirectAd10":
+                            if (record.isIndirectAd10Specified()) pageBuilder.setString(c, record.getIndirectAd10());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "latencyTime":
+                            if (record.isLatencyTimeSpecified()) pageBuilder.setLong(c, record.getLatencyTime());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "memberName":
+                            if (record.isMemberNameSpecified()) pageBuilder.setString(c, record.getMemberName());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "other1":
+                            if (record.isOther1Specified()) pageBuilder.setString(c, record.getOther1());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "other2":
+                            if (record.isOther2Specified()) pageBuilder.setString(c, record.getOther2());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "other3":
+                            if (record.isOther3Specified()) pageBuilder.setString(c, record.getOther3());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "other4":
+                            if (record.isOther4Specified()) pageBuilder.setString(c, record.getOther4());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "other5":
+                            if (record.isOther5Specified()) pageBuilder.setString(c, record.getOther5());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "terminalType":
+                            if (record.isTerminalTypeSpecified()) pageBuilder.setLong(c, record.getTerminalType());
+                            else pageBuilder.setNull(c);
+                            break;
+                        case "userId":
+                            if (record.isUserIdSpecified()) pageBuilder.setString(c, record.getUserId());
+                            else pageBuilder.setNull(c);
+                            break;
+                        default:
+                            break;
+                    }
+                    if (isExcluded) break;
+                }
+                if (isExcluded)
+                    pageBuilder.flush();
+                else
+                    pageBuilder.addRecord();
+            }
+        }
+
+        setLastRecordTime(Optional.of(newestRecordTime.format(embulkDateTimeFormat)));
+    }
+
+    public Schema buildSchema() {
+        return Schema.builder()
+                .add("amount", Types.LONG)
+                .add("conversionDate", Types.TIMESTAMP)
+                .add("conversionId", Types.LONG)
+                .add("directAd", Types.STRING)
+                .add("firstAd", Types.STRING)
+                .add("firstAdDate", Types.TIMESTAMP)
+                .add("indirectAd2", Types.STRING)
+                .add("indirectAd3", Types.STRING)
+                .add("indirectAd4", Types.STRING)
+                .add("indirectAd5", Types.STRING)
+                .add("indirectAd6", Types.STRING)
+                .add("indirectAd7", Types.STRING)
+                .add("indirectAd8", Types.STRING)
+                .add("indirectAd9", Types.STRING)
+                .add("indirectAd10", Types.STRING)
+                .add("latencyTime", Types.LONG)
+                .add("memberName", Types.STRING)
+                .add("other1", Types.STRING)
+                .add("other2", Types.STRING)
+                .add("other3", Types.STRING)
+                .add("other4", Types.STRING)
+                .add("other5", Types.STRING)
+                .add("terminalType", Types.LONG)
+                .add("userId", Types.STRING)
+                .build();
+    }
+}

data/src/main/java/org/embulk/input/adebis/operation/Operation.java

@@ -0,0 +1,15 @@
+package org.embulk.input.adebis.operation;
+
+import com.google.common.base.Optional;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+
+public interface Operation {
+    void setCredentials(String logId, String logArgment, String apiKey);
+    Optional<String> getLastRecordTime();
+    void setLastRecordTime(Optional<String> lastRecordTime);
+    void loadCondition(ConfigSource config);
+    Schema buildSchema();
+    void run(PageBuilder pageBuilder);
+}