embulk-input-cloudwatch_logs 0.1.0

data/src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java

@@ -0,0 +1,144 @@
+package org.embulk.input.cloudwatch_logs;
+
+import com.amazonaws.services.logs.AWSLogsClient;
+import com.amazonaws.services.logs.AWSLogsClientBuilder;
+import com.amazonaws.services.logs.AWSLogs;
+import com.amazonaws.services.logs.model.DescribeLogStreamsRequest;
+import com.amazonaws.services.logs.model.DescribeLogStreamsResult;
+import com.amazonaws.services.logs.model.LogStream;
+import com.amazonaws.services.logs.model.GetLogEventsRequest;
+import com.amazonaws.services.logs.model.GetLogEventsResult;
+import com.amazonaws.services.logs.model.OutputLogEvent;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.util.Pages;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.Spy;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin;
+
+import static org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin.CloudWatchLogsPluginTask;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assume.assumeNotNull;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestCloudwatchLogsInputPlugin
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "cloudwatch_logs", CloudwatchLogsInputPlugin.class)
+            .build();
+
+    private CloudwatchLogsInputPlugin plugin;
+
+    private ConfigSource config;
+    private MockPageOutput output = new MockPageOutput();
+    private PageBuilder pageBuilder;
+
+    private static String EMBULK_LOGS_TEST_GROUP_NAME;
+    private static String EMBULK_LOGS_TEST_STREAM_NAME;
+    private static String EMBULK_LOGS_TEST_REGION;
+    private static String EMBULK_LOGS_TEST_ACCESS_KEY_ID;
+    private static String EMBULK_LOGS_TEST_SECRET_ACCESS_KEY;
+
+    /*
+     * This test case requires environment variables:
+     *   EMBULK_LOGS_TEST_GROUP_NAME
+     *   EMBULK_LOGS_TEST_STREAM_NAME
+     *   EMBULK_LOGS_TEST_REGION
+     *   EMBULK_LOGS_TEST_ACCESS_KEY_ID
+     *   EMBULK_LOGS_TEST_SECRET_ACCESS_KEY
+     * If the variables not set, the test case is skipped.
+     */
+    @BeforeClass
+    public static void initializeConstantVariables()
+    {
+        EMBULK_LOGS_TEST_GROUP_NAME = System.getenv("EMBULK_LOGS_TEST_GROUP_NAME");
+        EMBULK_LOGS_TEST_STREAM_NAME = System.getenv("EMBULK_LOGS_TEST_STREAM_NAME");
+        EMBULK_LOGS_TEST_REGION = System.getenv("EMBULK_LOGS_TEST_REGION");
+        EMBULK_LOGS_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_LOGS_TEST_ACCESS_KEY_ID");
+        EMBULK_LOGS_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_LOGS_TEST_SECRET_ACCESS_KEY");
+        assumeNotNull(EMBULK_LOGS_TEST_GROUP_NAME, EMBULK_LOGS_TEST_STREAM_NAME, EMBULK_LOGS_TEST_REGION, EMBULK_LOGS_TEST_ACCESS_KEY_ID, EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
+    }
+
+    @Before
+    public void setUp() throws IOException
+    {
+        if (plugin == null) {
+            plugin = Mockito.spy(new CloudwatchLogsInputPlugin());
+            config = runtime.getExec().newConfigSource()
+                    .set("type", "cloudwatch_logs")
+                    .set("log_group_name", EMBULK_LOGS_TEST_GROUP_NAME)
+                    .set("log_stream_name", EMBULK_LOGS_TEST_STREAM_NAME)
+                    .set("use_log_stream_name_prefix", "true")
+                    .set("region", EMBULK_LOGS_TEST_REGION)
+                    .set("aws_access_key_id", EMBULK_LOGS_TEST_ACCESS_KEY_ID)
+                    .set("aws_secret_access_key", EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
+            pageBuilder = Mockito.mock(PageBuilder.class);
+        }
+        doReturn(pageBuilder).when(plugin).getPageBuilder(Mockito.any(), Mockito.any());
+    }
+
+    @Test
+    public void test_simple() throws IOException
+    {
+        plugin.transaction(config, new Control());
+        verify(pageBuilder, times(1)).finish();
+    }
+
+    @Test
+    public void configuredRegion()
+    {
+        CloudWatchLogsPluginTask task = this.config.deepCopy()
+                .set("region", "ap-southeast-2")
+                .remove("endpoint")
+                .loadConfig(CloudWatchLogsPluginTask.class);
+        CloudwatchLogsInputPlugin plugin = runtime.getInstance(CloudwatchLogsInputPlugin.class);
+        AWSLogs logsClient = plugin.newLogsClient(task);
+
+        // Should not be null
+        assumeNotNull(logsClient);
+    }
+
+    private class Control implements InputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
+        {
+            List<TaskReport> reports = new ArrayList<>();
+            for (int i = 0; i < taskCount; i++) {
+                reports.add(plugin.run(taskSource, schema, i, output));
+            }
+            return reports;
+        }
+    }
+}

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-cloudwatch_logs
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hiroshi Hatake
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2020-03-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+description: Loads records from Cloudwatch Logs.
+email:
+- cosmo0920.wp@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- LICENSE.txt
+- README.md
+- build.gradle
+- classpath/aws-java-sdk-core-1.11.749.jar
+- classpath/aws-java-sdk-logs-1.11.749.jar
+- classpath/aws-java-sdk-sts-1.11.749.jar
+- classpath/commons-codec-1.11.jar
+- classpath/commons-logging-1.2.jar
+- classpath/embulk-input-cloudwatch_logs-0.1.0.jar
+- classpath/httpclient-4.5.9.jar
+- classpath/httpcore-4.4.11.jar
+- classpath/ion-java-1.0.2.jar
+- classpath/jackson-databind-2.6.7.3.jar
+- classpath/jackson-dataformat-cbor-2.6.7.jar
+- classpath/jmespath-java-1.11.749.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/input/cloudwatch_logs.rb
+- src/main/java/org/embulk/input/cloudwatch_logs/AbstractCloudwatchLogsInputPlugin.java
+- src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java
+- src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentials.java
+- src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsConfig.java
+- src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsTask.java
+- src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java
+- src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java
+homepage: https://github.com/cosmo0920/embulk-input-cloudwatch_logs
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Cloudwatch Logs input plugin for Embulk
+test_files: []