embulk-input-jira 0.2.5 → 0.2.6

data/src/main/java/org/embulk/input/jira/AuthenticateMethod.java

@@ -0,0 +1,27 @@
+package org.embulk.input.jira;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+
+import org.embulk.config.ConfigException;
+
+public enum AuthenticateMethod {
+    BASIC;
+    @JsonValue
+    @Override
+    public String toString()
+    {
+        return this.name().toLowerCase();
+    }
+
+    @JsonCreator
+    public static AuthenticateMethod fromString(String value)
+    {
+        switch(value) {
+        case "basic":
+            return BASIC;
+        default:
+            throw new ConfigException(String.format("Unknown AuthenticateMethod value '%s'. Supported values is basic.", value));
+        }
+    }
+}

data/src/main/java/org/embulk/input/jira/Constant.java

@@ -0,0 +1,17 @@
+package org.embulk.input.jira;
+
+public final class Constant
+{
+    public static final int MAX_RESULTS = 50;
+    public static final int MIN_RESULTS = 1;
+    public static final int GUESS_RECORDS_COUNT = 50;
+    public static final int PREVIEW_RECORDS_COUNT = 10;
+    public static final int GUESS_BUFFER_SIZE = 5 * 1024 * 1024;
+
+    public static final String DEFAULT_TIMESTAMP_PATTERN = "%Y-%m-%dT%H:%M:%S.%L%z";
+
+    public static final String CREDENTIAL_URI_PATH = "rest/api/latest/myself";
+    public static final String SEARCH_URI_PATH = "rest/api/latest/search";
+
+    private Constant(){}
+}

data/src/main/java/org/embulk/input/jira/Issue.java

@@ -0,0 +1,150 @@
+package org.embulk.input.jira;
+
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.stream.StreamSupport;
+
+public class Issue
+{
+    private JsonObject flatten;
+    private JsonObject json;
+
+    public Issue(JsonObject original)
+    {
+        this.json = original;
+    }
+
+    public JsonElement getValue(String path)
+    {
+        List<String> keys = new ArrayList<>(Arrays.asList(path.split("\\.")));
+        return get(json, keys);
+    }
+
+    private JsonElement get(JsonElement json, List<String> keys)
+    {
+        if (json == null || json.isJsonNull()) {
+            return JsonNull.INSTANCE;
+        }
+        else if (keys.isEmpty() || (json.isJsonArray() && json.getAsJsonArray().size() == 0)) {
+            return json;
+        }
+        String key = keys.get(0);
+        keys.remove(0);
+        if (json.isJsonArray()) {
+            JsonArray arrays = new JsonArray();
+            for (JsonElement elem : json.getAsJsonArray()) {
+                if (elem.isJsonObject()) {
+                    arrays.add(elem.getAsJsonObject().get(key));
+                }
+                else {
+                    arrays.add(elem);
+                }
+            }
+            return get(arrays, keys);
+        }
+        else {
+            return get(json.getAsJsonObject().get(key), keys);
+        }
+    }
+
+    public synchronized JsonObject getFlatten()
+    {
+        if (flatten == null) {
+            flatten = new JsonObject();
+            manipulatingFlattenJson(json, "");
+        }
+        return flatten;
+    }
+
+    private void manipulatingFlattenJson(JsonElement in, String prefix)
+    {
+        if (in.isJsonObject()) {
+            JsonObject obj = in.getAsJsonObject();
+            // NOTE: If you want to flatten JSON completely, please remove this if and addHeuristicValue
+            if (StringUtils.countMatches(prefix, ".") > 1) {
+                addHeuristicValue(obj, prefix);
+                return;
+            }
+            if (obj.entrySet().isEmpty()) {
+                flatten.add(prefix, obj);
+            }
+            else {
+                for (Entry<String, JsonElement> entry : obj.entrySet()) {
+                    String key = entry.getKey();
+                    JsonElement value = entry.getValue();
+                    manipulatingFlattenJson(value, appendPrefix(prefix, key));
+                }
+            }
+        }
+        else if (in.isJsonArray()) {
+            JsonArray arrayObj = in.getAsJsonArray();
+            boolean isAllJsonObject = arrayObj.size() > 0 && StreamSupport.stream(arrayObj.spliterator(), false).allMatch(JsonElement::isJsonObject);
+            if (isAllJsonObject) {
+                Map<String, Integer> occurents = new HashMap<>();
+                for (JsonElement element : arrayObj) {
+                    JsonObject obj = element.getAsJsonObject();
+                    for (Entry<String, JsonElement> entry : obj.entrySet()) {
+                        String key = entry.getKey();
+                        occurents.merge(key, 1, Integer::sum);
+                    }
+                }
+                JsonObject newObj = new JsonObject();
+                for (String key : occurents.keySet()) {
+                    newObj.add(key, new JsonArray());
+                    for (JsonElement elem : arrayObj) {
+                        newObj.get(key).getAsJsonArray().add(elem.getAsJsonObject().get(key));
+                    }
+                }
+                manipulatingFlattenJson(newObj, prefix);
+            }
+            else {
+                flatten.add(prefix,
+                        new JsonPrimitive("String value"));
+            }
+        }
+        else if (in.isJsonPrimitive()) {
+            flatten.add(prefix, in.getAsJsonPrimitive());
+        }
+        else {
+            flatten.add(prefix, JsonNull.INSTANCE);
+        }
+    }
+
+    private void addHeuristicValue(JsonObject json, String prefix)
+    {
+        List<String> keys = Arrays.asList("name", "key", "id");
+        List<String> heuristic = new ArrayList<>();
+        for (Entry<String, JsonElement> entry : json.entrySet()) {
+            String key = entry.getKey();
+            JsonElement value = entry.getValue();
+            if (keys.contains(key) && !value.isJsonNull()) {
+                heuristic.add(key);
+            }
+        }
+        if (heuristic.isEmpty()) {
+            flatten.add(prefix, new JsonPrimitive(json.toString()));
+        }
+        else {
+            for (String key : heuristic) {
+                JsonElement value = json.get(key);
+                flatten.add(appendPrefix(prefix, key), value);
+            }
+        }
+    }
+
+    private String appendPrefix(String prefix, String key)
+    {
+        return prefix.isEmpty() ? key : prefix + "." + key;
+    }
+}

data/src/main/java/org/embulk/input/jira/JiraInputPlugin.java

@@ -0,0 +1,226 @@
+package org.embulk.input.jira;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonNull;
+import com.google.gson.JsonObject;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.exec.GuessExecutor;
+import org.embulk.input.jira.client.JiraClient;
+import org.embulk.input.jira.util.JiraUtil;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import static org.embulk.input.jira.Constant.GUESS_BUFFER_SIZE;
+import static org.embulk.input.jira.Constant.GUESS_RECORDS_COUNT;
+import static org.embulk.input.jira.Constant.MAX_RESULTS;
+import static org.embulk.input.jira.Constant.PREVIEW_RECORDS_COUNT;
+
+public class JiraInputPlugin
+        implements InputPlugin
+{
+    private static final Logger LOGGER = Exec.getLogger(JiraInputPlugin.class);
+
+    public interface PluginTask
+            extends Task
+    {
+        @Config("username")
+        public String getUsername();
+
+        @Config("password")
+        public String getPassword();
+
+        @Config("uri")
+        public String getUri();
+
+        @Config("initial_retry_interval_millis")
+        @ConfigDefault("1000")
+        int getInitialRetryIntervalMillis();
+
+        @Config("maximum_retry_interval_millis")
+        @ConfigDefault("120000")
+        int getMaximumRetryIntervalMillis();
+
+        @Config("timeout_millis")
+        @ConfigDefault("300000")
+        int getTimeoutMillis();
+
+        @Config("retry_limit")
+        @ConfigDefault("5")
+        public int getRetryLimit();
+
+        @Config("jql")
+        public String getJQL();
+
+        @Config("columns")
+        public SchemaConfig getColumns();
+
+        // For future support of other authentication methods
+        @Config("auth_method")
+        @ConfigDefault("\"basic\"")
+        public AuthenticateMethod getAuthMethod();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+            InputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        Schema schema = task.getColumns().toSchema();
+        int taskCount = 1;
+
+        return resume(task.dump(), schema, taskCount, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            InputPlugin.Control control)
+    {
+        control.run(taskSource, schema, taskCount);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource,
+            Schema schema, int taskIndex,
+            PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        JiraUtil.validateTaskConfig(task);
+        JiraClient jiraClient = getJiraClient();
+        jiraClient.checkUserCredentials(task);
+        try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) {
+            if (isPreview()) {
+                List<Issue> issues = jiraClient.searchIssues(task, 0, PREVIEW_RECORDS_COUNT);
+                issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
+            }
+            else {
+                int currentPage = 0;
+                int totalCount = jiraClient.getTotalCount(task);
+                int totalPage = JiraUtil.calculateTotalPage(totalCount, MAX_RESULTS);
+                LOGGER.info(String.format("Total pages (%d)", totalPage));
+                while (currentPage < totalPage) {
+                    LOGGER.info(String.format("Fetching page %d/%d", (currentPage + 1), totalPage));
+                    List<Issue> issues = jiraClient.searchIssues(task, (currentPage * MAX_RESULTS), MAX_RESULTS);
+                    issues.forEach(issue -> JiraUtil.addRecord(issue, schema, task, pageBuilder));
+                    currentPage++;
+                }
+            }
+            pageBuilder.finish();
+        }
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        // Reset columns in case already have or missing on configuration
+        config.set("columns", new ObjectMapper().createArrayNode());
+        PluginTask task = config.loadConfig(PluginTask.class);
+        JiraUtil.validateTaskConfig(task);
+        JiraClient jiraClient = getJiraClient();
+        jiraClient.checkUserCredentials(task);
+        List<Issue> issues = jiraClient.searchIssues(task, 0, GUESS_RECORDS_COUNT);
+        if (issues.isEmpty()) {
+            throw new ConfigException("Could not guess schema due to empty data set");
+        }
+        Buffer sample = Buffer.copyOf(createSamples(issues, getUniqueAttributes(issues)).toString().getBytes());
+        JsonNode columns = Exec.getInjector().getInstance(GuessExecutor.class)
+                                .guessParserConfig(sample, Exec.newConfigSource(), createGuessConfig())
+                                .getObjectNode().get("columns");
+        return Exec.newConfigDiff().set("columns", columns);
+    }
+
+    private ConfigSource createGuessConfig()
+    {
+        return Exec.newConfigSource()
+                    .set("guess_plugins", ImmutableList.of("jira"))
+                    .set("guess_sample_buffer_bytes", GUESS_BUFFER_SIZE);
+    }
+
+    private SortedSet<String> getUniqueAttributes(List<Issue> issues)
+    {
+        SortedSet<String> uniqueAttributes = new TreeSet<>();
+        for (Issue issue : issues) {
+            for (Entry<String, JsonElement> entry : issue.getFlatten().entrySet()) {
+                uniqueAttributes.add(entry.getKey());
+            }
+        }
+        return uniqueAttributes;
+    }
+
+    private JsonArray createSamples(List<Issue> issues, Set<String> uniqueAttributes)
+    {
+        JsonArray samples = new JsonArray();
+        for (Issue issue : issues) {
+            JsonObject flatten = issue.getFlatten();
+            JsonObject unified = new JsonObject();
+            for (String key : uniqueAttributes) {
+                JsonElement value = flatten.get(key);
+                if (value == null) {
+                    value = JsonNull.INSTANCE;
+                }
+                unified.add(key, value);
+            }
+            samples.add(unified);
+        }
+        return samples;
+    }
+
+    @VisibleForTesting
+    public GuessExecutor getGuessExecutor()
+    {
+        return Exec.getInjector().getInstance(GuessExecutor.class);
+    }
+
+    @VisibleForTesting
+    public PageBuilder getPageBuilder(Schema schema, PageOutput output)
+    {
+        return new PageBuilder(Exec.getBufferAllocator(), schema, output);
+    }
+
+    @VisibleForTesting
+    public boolean isPreview()
+    {
+        return Exec.isPreview();
+    }
+
+    @VisibleForTesting
+    public JiraClient getJiraClient()
+    {
+        return new JiraClient();
+    }
+}

data/src/main/java/org/embulk/input/jira/client/JiraClient.java

@@ -0,0 +1,254 @@
+package org.embulk.input.jira.client;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.JsonArray;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParser;
+import com.google.gson.JsonPrimitive;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.entity.StringEntity;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.embulk.config.ConfigException;
+import org.embulk.input.jira.Issue;
+import org.embulk.input.jira.JiraInputPlugin.PluginTask;
+import org.embulk.input.jira.util.JiraException;
+import org.embulk.input.jira.util.JiraUtil;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+import static java.util.Base64.getEncoder;
+import static org.apache.http.HttpHeaders.ACCEPT;
+import static org.apache.http.HttpHeaders.AUTHORIZATION;
+import static org.apache.http.HttpHeaders.CONTENT_TYPE;
+import static org.embulk.input.jira.Constant.MIN_RESULTS;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
+public class JiraClient
+{
+    public JiraClient() {}
+
+    private static final int CONNECTION_TIME_OUT = 300000;
+
+    private static final Logger LOGGER = Exec.getLogger(JiraClient.class);
+
+    public void checkUserCredentials(final PluginTask task)
+    {
+        try {
+            authorizeAndRequest(task, JiraUtil.buildPermissionUrl(task.getUri()), null);
+        }
+        catch (JiraException e) {
+            LOGGER.error(String.format("JIRA return status (%s), reason (%s)", e.getStatusCode(), e.getMessage()));
+            if (e.getStatusCode() == HttpStatus.SC_UNAUTHORIZED) {
+                throw new ConfigException("Could not authorize with your credential.");
+            }
+            else {
+                throw new ConfigException("Could not authorize with your credential due to problems when contacting JIRA API.");
+            }
+        }
+    }
+
+    public List<Issue> searchIssues(final PluginTask task, int startAt, int maxResults)
+    {
+        String response = searchJiraAPI(task, startAt, maxResults);
+        JsonObject result = new JsonParser().parse(response).getAsJsonObject();
+        return StreamSupport.stream(result.get("issues").getAsJsonArray().spliterator(), false)
+                            .map(jsonElement -> {
+                                JsonObject json = jsonElement.getAsJsonObject();
+                                JsonObject fields = json.get("fields").getAsJsonObject();
+                                Set<Entry<String, JsonElement>> entries = fields.entrySet();
+                                json.remove("fields");
+                                // Merged all properties in fields to the object
+                                for (Entry<String, JsonElement> entry : entries) {
+                                    json.add(entry.getKey(), entry.getValue());
+                                }
+                                return new Issue(json);
+                            })
+                            .collect(Collectors.toList());
+    }
+
+    public int getTotalCount(final PluginTask task)
+    {
+        return new JsonParser().parse(searchJiraAPI(task, 0, MIN_RESULTS)).getAsJsonObject().get("total").getAsInt();
+    }
+
+    private String searchJiraAPI(final PluginTask task, int startAt, int maxResults)
+    {
+        try {
+            return retryExecutor().withRetryLimit(task.getRetryLimit())
+            .withInitialRetryWait(task.getInitialRetryIntervalMillis())
+            .withMaxRetryWait(task.getMaximumRetryIntervalMillis())
+            .runInterruptible(new Retryable<String>()
+            {
+                @Override
+                public String call() throws Exception
+                {
+                    return authorizeAndRequest(task, JiraUtil.buildSearchUrl(task.getUri()), createSearchIssuesBody(task, startAt, maxResults));
+                }
+
+                @Override
+                public boolean isRetryableException(Exception exception)
+                {
+                    if (exception instanceof JiraException) {
+                        int statusCode = ((JiraException) exception).getStatusCode();
+                        // When overloading JIRA APIs (i.e 100 requests per second) the API will return 401 although the credential is correct. So add retry for this
+                        // 429 is stand for "Too many requests"
+                        // Other 4xx considered errors
+                        return statusCode / 100 != 4 || statusCode == HttpStatus.SC_UNAUTHORIZED || statusCode == 429;
+                    }
+                    return false;
+                }
+
+                @Override
+                public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                        throws RetryGiveupException
+                {
+                    if (exception instanceof JiraException) {
+                        String message = String
+                                .format("Retrying %d/%d after %d seconds. HTTP status code: %s",
+                                        retryCount, retryLimit,
+                                        retryWait / 1000,
+                                        ((JiraException) exception).getStatusCode());
+                        LOGGER.warn(message);
+                    }
+                    else {
+                        String message = String
+                                .format("Retrying %d/%d after %d seconds. Message: %s",
+                                        retryCount, retryLimit,
+                                        retryWait / 1000,
+                                        exception.getMessage());
+                        LOGGER.warn(message, exception);
+                    }
+                }
+
+                @Override
+                public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
+                {
+                    LOGGER.warn("Retry Limit Exceeded");
+                }
+            });
+        }
+        catch (RetryGiveupException | InterruptedException e) {
+            if (e instanceof RetryGiveupException && e.getCause() != null && e.getCause() instanceof JiraException) {
+                throw new ConfigException(e.getCause().getMessage());
+            }
+            throw new ConfigException(e);
+        }
+    }
+
+    private String authorizeAndRequest(final PluginTask task, String url, String body) throws JiraException
+    {
+        try {
+            HttpClient client = createHttpClient();
+            HttpRequestBase request;
+            if (body == null) {
+                request = createGetRequest(task, url);
+            }
+            else {
+                request = createPostRequest(task, url, body);
+            }
+            HttpResponse response = client.execute(request);
+            // Check for HTTP response code : 200 : SUCCESS
+            int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode != HttpStatus.SC_OK) {
+                throw new JiraException(statusCode, extractErrorMessages(EntityUtils.toString(response.getEntity())));
+            }
+            return EntityUtils.toString(response.getEntity());
+        }
+        catch (IOException e) {
+            throw new JiraException(-1, e.getMessage());
+        }
+    }
+
+    private String extractErrorMessages(String errorResponse)
+    {
+        List<String> messages = new ArrayList<>();
+        try {
+            JsonObject errorObject = new JsonParser().parse(errorResponse).getAsJsonObject();
+            for (JsonElement element : errorObject.get("errorMessages").getAsJsonArray()) {
+                messages.add(element.getAsString());
+            }
+        }
+        catch (Exception e) {
+            messages.add(errorResponse);
+        }
+        return String.join(" , ", messages);
+    }
+
+    @VisibleForTesting
+    public HttpClient createHttpClient()
+    {
+        RequestConfig config = RequestConfig.custom()
+                .setConnectTimeout(CONNECTION_TIME_OUT)
+                .setConnectionRequestTimeout(CONNECTION_TIME_OUT)
+                .build();
+        return HttpClientBuilder.create().setDefaultRequestConfig(config).build();
+    }
+
+    private HttpRequestBase createPostRequest(PluginTask task, String url, String body) throws IOException
+    {
+        HttpPost request = new HttpPost(url);
+        switch (task.getAuthMethod()) {
+        default:
+            request.setHeader(
+                    AUTHORIZATION,
+                    String.format("Basic %s",
+                                getEncoder().encodeToString(String.format("%s:%s",
+                                task.getUsername(),
+                                task.getPassword()).getBytes())));
+            request.setHeader(ACCEPT, "application/json");
+            request.setHeader(CONTENT_TYPE, "application/json");
+            break;
+        }
+        request.setEntity(new StringEntity(body));
+        return request;
+    }
+
+    private HttpRequestBase createGetRequest(PluginTask task, String url)
+    {
+        HttpGet request = new HttpGet(url);
+        switch (task.getAuthMethod()) {
+        default:
+            request.setHeader(
+                    AUTHORIZATION,
+                    String.format("Basic %s",
+                                getEncoder().encodeToString(String.format("%s:%s",
+                                task.getUsername(),
+                                task.getPassword()).getBytes())));
+            request.setHeader(ACCEPT, "application/json");
+            request.setHeader(CONTENT_TYPE, "application/json");
+            break;
+        }
+        return request;
+    }
+
+    private String createSearchIssuesBody(PluginTask task, int startAt, int maxResults)
+    {
+        JsonObject body = new JsonObject();
+        body.add("jql", new JsonPrimitive(task.getJQL()));
+        body.add("startAt", new JsonPrimitive(startAt));
+        body.add("maxResults", new JsonPrimitive(maxResults));
+        JsonArray fields = new JsonArray();
+        fields.add("*all");
+        body.add("fields", fields);
+        return body.toString();
+    }
+}