embulk-input-cloudwatch_logs 0.1.0

data/src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java

@@ -0,0 +1,57 @@
+package org.embulk.input.cloudwatch_logs;
+
+import java.util.List;
+
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
+import com.amazonaws.services.logs.AWSLogsClientBuilder;
+import com.amazonaws.services.logs.AWSLogs;
+import com.google.common.base.Optional;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+
+public class CloudwatchLogsInputPlugin
+        extends AbstractCloudwatchLogsInputPlugin
+{
+    public interface CloudWatchLogsPluginTask
+            extends PluginTask
+    {
+        @Config("region")
+        @ConfigDefault("null")
+        Optional<String> getRegion();
+    }
+
+    @Override
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return CloudWatchLogsPluginTask.class;
+    }
+
+    @Override
+    protected AWSLogs newLogsClient(PluginTask task)
+    {
+        CloudWatchLogsPluginTask t = (CloudWatchLogsPluginTask) task;
+        Optional<String> region = t.getRegion();
+        AWSLogsClientBuilder builder = super.defaultLogsClientBuilder(t);
+
+        if (region.isPresent()) {
+            builder.setRegion(region.get());
+        }
+        else {
+            throw new ConfigException("region is required");
+        }
+
+        return builder.build();
+    }
+}

data/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentials.java

@@ -0,0 +1,183 @@
+package org.embulk.input.cloudwatch_logs.aws;
+
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSSessionCredentials;
+import com.amazonaws.auth.AWSSessionCredentialsProvider;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
+import com.amazonaws.auth.DefaultAWSCredentialsProviderChain;
+import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.auth.profile.ProfilesConfigFile;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.util.Optional;
+
+public abstract class AwsCredentials
+{
+    private AwsCredentials()
+    {
+    }
+
+    public static AWSCredentialsProvider getAWSCredentialsProvider(AwsCredentialsConfig task)
+    {
+        String authenticationMethodOption = "authentication_method";
+        String awsSessionTokenOption = "aws_session_token";
+        String awsAccessKeyIdOption = "aws_access_key_id";
+        String awsSecretAccessKeyOption = "aws_secret_access_key";
+        String awsProfileFileOption = "aws_profile_file";
+        String awsProfileNameOption = "aws_profile_name";
+
+        switch (task.getAuthenticationMethod()) {
+        case "basic": {
+            String accessKeyId = required(task.getAwsAccessKeyId(), "'" + awsAccessKeyIdOption + "', '" + awsSecretAccessKeyOption + "'");
+            String secretAccessKey = required(task.getAwsSecretAccessKey(), "'" + awsSecretAccessKeyOption + "'");
+            invalid(task.getAwsProfileFile(), awsProfileFileOption);
+            invalid(task.getAwsProfileName(), awsProfileNameOption);
+
+            final BasicAWSCredentials creds = new BasicAWSCredentials(accessKeyId, secretAccessKey);
+                return new AWSCredentialsProvider() {
+                    public AWSCredentials getCredentials()
+                    {
+                        return creds;
+                    }
+
+                    public void refresh()
+                    {
+                    }
+                };
+        }
+
+        case "env":
+            invalid(task.getAwsAccessKeyId(), awsAccessKeyIdOption);
+            invalid(task.getAwsSecretAccessKey(), awsSecretAccessKeyOption);
+            invalid(task.getAwsProfileFile(), awsProfileFileOption);
+            invalid(task.getAwsProfileName(), awsProfileNameOption);
+
+            return overwriteAwsCredentials(task, new EnvironmentVariableCredentialsProvider().getCredentials());
+
+        case "instance":
+            invalid(task.getAwsAccessKeyId(), awsAccessKeyIdOption);
+            invalid(task.getAwsSecretAccessKey(), awsSecretAccessKeyOption);
+            invalid(task.getAwsProfileFile(), awsProfileFileOption);
+            invalid(task.getAwsProfileName(), awsProfileNameOption);
+
+            return new InstanceProfileCredentialsProvider();
+
+        case "profile":
+            {
+                invalid(task.getAwsAccessKeyId(), awsAccessKeyIdOption);
+                invalid(task.getAwsSecretAccessKey(), awsSecretAccessKeyOption);
+
+                String profileName = task.getAwsProfileName().orElse("default");
+                ProfileCredentialsProvider provider;
+                if (task.getAwsProfileFile().isPresent()) {
+                    ProfilesConfigFile file = new ProfilesConfigFile(task.getAwsProfileFile().get().getFile());
+                    provider = new ProfileCredentialsProvider(file, profileName);
+                }
+                else {
+                    provider = new ProfileCredentialsProvider(profileName);
+                }
+
+                return overwriteAwsCredentials(task, provider.getCredentials());
+            }
+
+        case "properties":
+            invalid(task.getAwsAccessKeyId(), awsAccessKeyIdOption);
+            invalid(task.getAwsSecretAccessKey(), awsSecretAccessKeyOption);
+            invalid(task.getAwsProfileFile(), awsProfileFileOption);
+            invalid(task.getAwsProfileName(), awsProfileNameOption);
+
+            return overwriteAwsCredentials(task, new SystemPropertiesCredentialsProvider().getCredentials());
+
+        case "anonymous":
+            invalid(task.getAwsAccessKeyId(), awsAccessKeyIdOption);
+            invalid(task.getAwsSecretAccessKey(), awsSecretAccessKeyOption);
+            invalid(task.getAwsProfileFile(), awsProfileFileOption);
+            invalid(task.getAwsProfileName(), awsProfileNameOption);
+            return new AWSCredentialsProvider() {
+                public AWSCredentials getCredentials()
+                {
+                    return new AnonymousAWSCredentials();
+                }
+
+                public void refresh()
+                {
+                }
+            };
+
+        case "session":
+            {
+                String accessKeyId = required(task.getAwsAccessKeyId(),
+                                              "'" + awsAccessKeyIdOption + "', '" + awsSecretAccessKeyOption + "', '" + awsSessionTokenOption + "'");
+                String secretAccessKey = required(task.getAwsSecretAccessKey(), "'" + awsSecretAccessKeyOption + "', '" + awsSessionTokenOption + "'");
+                String sessionToken = required(task.getAwsSessionToken(),
+                                               "'" + awsSessionTokenOption + "'");
+                invalid(task.getAwsProfileFile(), awsProfileFileOption);
+                invalid(task.getAwsProfileName(), awsProfileNameOption);
+                final AWSSessionCredentials creds = new BasicSessionCredentials(accessKeyId, secretAccessKey, sessionToken);
+                return new AWSSessionCredentialsProvider() {
+                    public AWSSessionCredentials getCredentials()
+                    {
+                        return creds;
+                    }
+
+                    public void refresh()
+                    {
+                    }
+                };
+            }
+
+        case "default":
+            {
+                invalid(task.getAwsAccessKeyId(), awsAccessKeyIdOption);
+                invalid(task.getAwsSecretAccessKey(), awsSecretAccessKeyOption);
+                invalid(task.getAwsProfileFile(), awsProfileFileOption);
+                invalid(task.getAwsProfileName(), awsProfileNameOption);
+
+                return new DefaultAWSCredentialsProviderChain();
+            }
+
+        default:
+            throw new ConfigException(String.format("Unknown authentication_method '%s'. Supported methods are basic, env, instance, profile, properties, anonymous, and default.",
+                        task.getAuthenticationMethod()));
+        }
+    }
+
+    private static AWSCredentialsProvider overwriteAwsCredentials(AwsCredentialsConfig task, final AWSCredentials creds)
+    {
+        return new AWSCredentialsProvider() {
+            public AWSCredentials getCredentials()
+            {
+                return creds;
+            }
+
+            public void refresh()
+            {
+            }
+        };
+    }
+
+    private static <T> T required(Optional<T> value, String message)
+    {
+        if (value.isPresent()) {
+            return value.get();
+        }
+        else {
+            throw new ConfigException("Required option is not set: " + message);
+        }
+    }
+
+    private static <T> void invalid(Optional<T> value, String message)
+    {
+        if (value.isPresent()) {
+            throw new ConfigException("Invalid option is set: " + message);
+        }
+    }
+}

data/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsConfig.java

@@ -0,0 +1,32 @@
+package org.embulk.input.cloudwatch_logs.aws;
+
+import org.embulk.spi.unit.LocalFile;
+
+import java.util.Optional;
+
+public interface AwsCredentialsConfig
+{
+    String getAuthenticationMethod();
+
+    void setAuthenticationMethod(String method);
+
+    Optional<String> getAwsAccessKeyId();
+
+    void setAwsAccessKeyId(Optional<String> value);
+
+    Optional<String> getAwsSecretAccessKey();
+
+    void setAwsSecretAccessKey(Optional<String> value);
+
+    Optional<String> getAwsSessionToken();
+
+    void setAwsSessionToken(Optional<String> value);
+
+    Optional<LocalFile> getAwsProfileFile();
+
+    void setAwsProfileFile(Optional<String> value);
+
+    Optional<String> getAwsProfileName();
+
+    void setAwsProfileName(Optional<String> value);
+}

data/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsTask.java

@@ -0,0 +1,41 @@
+package org.embulk.input.cloudwatch_logs.aws;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.unit.LocalFile;
+
+import java.util.Optional;
+
+public interface AwsCredentialsTask
+    extends AwsCredentialsConfig
+{
+    @Override
+    @Config("authentication_method")
+    @ConfigDefault("\"basic\"")
+    String getAuthenticationMethod();
+
+    @Override
+    @Config("aws_access_key_id")
+    @ConfigDefault("null")
+    Optional<String> getAwsAccessKeyId();
+
+    @Override
+    @Config("aws_secret_access_key")
+    @ConfigDefault("null")
+    Optional<String> getAwsSecretAccessKey();
+
+    @Override
+    @Config("aws_session_token")
+    @ConfigDefault("null")
+    Optional<String> getAwsSessionToken();
+
+    @Override
+    @Config("aws_profile_file")
+    @ConfigDefault("null")
+    Optional<LocalFile> getAwsProfileFile();
+
+    @Override
+    @Config("aws_profile_name")
+    @ConfigDefault("null")
+    Optional<String> getAwsProfileName();
+}

data/src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java

@@ -0,0 +1,188 @@
+package org.embulk.input.cloudwatch_logs;
+
+import com.amazonaws.services.logs.AWSLogsClient;
+import com.amazonaws.services.logs.AWSLogsClientBuilder;
+import com.amazonaws.services.logs.AWSLogs;
+import com.amazonaws.services.logs.model.DescribeLogStreamsRequest;
+import com.amazonaws.services.logs.model.DescribeLogStreamsResult;
+import com.amazonaws.services.logs.model.LogStream;
+import com.amazonaws.services.logs.model.GetLogEventsRequest;
+import com.amazonaws.services.logs.model.GetLogEventsResult;
+import com.amazonaws.services.logs.model.OutputLogEvent;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.util.Pages;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin;
+
+import static org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin.CloudWatchLogsPluginTask;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assume.assumeNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestAwsCredentials
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "cloudwatch_logs", CloudwatchLogsInputPlugin.class)
+            .build();
+
+    private CloudwatchLogsInputPlugin plugin;
+
+    private ConfigSource config;
+    private MockPageOutput output = new MockPageOutput();
+    private PageBuilder pageBuilder;
+
+    private static String EMBULK_LOGS_TEST_GROUP_NAME;
+    private static String EMBULK_LOGS_TEST_STREAM_NAME;
+    private static String EMBULK_LOGS_TEST_REGION;
+    private static String EMBULK_LOGS_TEST_ACCESS_KEY_ID;
+    private static String EMBULK_LOGS_TEST_SECRET_ACCESS_KEY;
+
+    /*
+     * This test case requires environment variables:
+     *   EMBULK_LOGS_TEST_GROUP_NAME
+     *   EMBULK_LOGS_TEST_STREAM_NAME
+     *   EMBULK_LOGS_TEST_REGION
+     *   EMBULK_LOGS_TEST_ACCESS_KEY_ID
+     *   EMBULK_LOGS_TEST_SECRET_ACCESS_KEY
+     * If the variables not set, the test case is skipped.
+     */
+    @BeforeClass
+    public static void initializeConstantVariables()
+    {
+        EMBULK_LOGS_TEST_GROUP_NAME = System.getenv("EMBULK_LOGS_TEST_GROUP_NAME");
+        EMBULK_LOGS_TEST_STREAM_NAME = System.getenv("EMBULK_LOGS_TEST_STREAM_NAME");
+        EMBULK_LOGS_TEST_REGION = System.getenv("EMBULK_LOGS_TEST_REGION");
+        EMBULK_LOGS_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_LOGS_TEST_ACCESS_KEY_ID");
+        EMBULK_LOGS_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_LOGS_TEST_SECRET_ACCESS_KEY");
+        assumeNotNull(EMBULK_LOGS_TEST_GROUP_NAME, EMBULK_LOGS_TEST_STREAM_NAME, EMBULK_LOGS_TEST_REGION, EMBULK_LOGS_TEST_ACCESS_KEY_ID, EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
+    }
+
+    @Before
+    public void setUp() throws IOException
+    {
+        if (plugin == null) {
+            plugin = Mockito.spy(new CloudwatchLogsInputPlugin());
+            config = runtime.getExec().newConfigSource()
+                    .set("type", "cloudwatch_logs")
+                    .set("log_group_name", EMBULK_LOGS_TEST_GROUP_NAME)
+                    .set("log_stream_name", EMBULK_LOGS_TEST_STREAM_NAME)
+                    .set("use_log_stream_name_prefix", "true")
+                    .set("region", EMBULK_LOGS_TEST_REGION);
+            pageBuilder = Mockito.mock(PageBuilder.class);
+        doReturn(pageBuilder).when(plugin).getPageBuilder(Mockito.any(), Mockito.any());
+        }
+    }
+
+    private void doTest(ConfigSource config) throws IOException
+    {
+        plugin.transaction(config, new Control());
+        verify(pageBuilder, times(1)).finish();
+    }
+
+    @Test
+    public void useBasic() throws IOException
+    {
+        ConfigSource config = this.config.deepCopy()
+                .set("authentication_method", "basic")
+                .set("aws_access_key_id", EMBULK_LOGS_TEST_ACCESS_KEY_ID)
+                .set("aws_secret_access_key", EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
+        doTest(config);
+    }
+
+    @Test
+    public void useEnv()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useInstance()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useProfile()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useProperties() throws IOException
+    {
+        String prevAccessKeyId = System.getProperty("aws.accessKeyId");
+        String prevSecretKey = System.getProperty("aws.secretKey");
+        try {
+            ConfigSource config = this.config.deepCopy().set("authentication_method", "properties");
+            System.setProperty("aws.accessKeyId", EMBULK_LOGS_TEST_ACCESS_KEY_ID);
+            System.setProperty("aws.secretKey", EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
+            doTest(config);
+        }
+        finally {
+            if (prevAccessKeyId != null) {
+                System.setProperty("aws.accessKeyId", prevAccessKeyId);
+            }
+            if (prevSecretKey != null) {
+                System.setProperty("aws.secretKey", prevSecretKey);
+            }
+        }
+    }
+
+    @Test
+    public void useAnonymous()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useSession()
+    {
+        //TODO
+    }
+
+    private class Control implements InputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
+        {
+            List<TaskReport> reports = new ArrayList<>();
+            for (int i = 0; i < taskCount; i++) {
+                reports.add(plugin.run(taskSource, schema, i, output));
+            }
+            return reports;
+        }
+    }
+}