embulk-input-cloudwatch_logs 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (12) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/build.yml +25 -0
  3. data/.gitignore +1 -0
  4. data/build.gradle +10 -6
  5. data/src/main/java/org/embulk/input/cloudwatch_logs/AbstractCloudwatchLogsInputPlugin.java +6 -9
  6. data/src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java +0 -13
  7. data/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentials.java +0 -2
  8. data/src/main/java/org/embulk/input/cloudwatch_logs/utils/DateUtils.java +29 -0
  9. data/src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java +46 -31
  10. data/src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java +43 -30
  11. data/src/test/java/org/embulk/input/cloudwatch_logs/TestHelpers.java +78 -0
  12. metadata +6 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 87992b58676fdeaddad2c841f96e3bbd53a44329
4
- data.tar.gz: 35deb6b6a8741deb4044e565442827980b93844b
3
+ metadata.gz: 0c8514b9710d424e207be6f8d9ba1eaba9722756
4
+ data.tar.gz: e429b3906d0ca115b6d0fa9e827904cd823cf534
5
5
  SHA512:
6
- metadata.gz: eabd966215d768006705c8921b0cd90982dff5fb93811cde636ba20b98033f466554ec4164687bc282eec7a5902eaa61c9edff18e2548d57b54666b0e68e9739
7
- data.tar.gz: 13902ef7a5c355490a25cd653a370c04d927e8f72037680a5b38cfa2e57a538481c4788e7b4d0f197135ff9593a8b3a9ae98a041b37d59a77c3c25c5f1ded2b6
6
+ metadata.gz: aaa51712b7cbc2255a9988b950e8485e3a9e41c8047edb45972e055e9ed931edc0c15534a9a949ff3a0876971eb9538a7b4513a4627ad92cd44d86e76ed3dabf
7
+ data.tar.gz: cee8e80247f51920c0e57cdedf668a32936c762b2bd7e0ef59086fb078462f6549bf3032005d72c8b16322556b71f964adecb4c47c8528e0d733a9031f7eeaa6
data/.github/workflows/build.yml ADDED
@@ -0,0 +1,25 @@
1
+ name: Build and test
2
+ on:
3
+ - push
4
+ - pull_request
5
+ jobs:
6
+ build:
7
+ runs-on: ${{ matrix.os }}
8
+ strategy:
9
+ fail-fast: false
10
+ matrix:
11
+ os:
12
+ - ubuntu-latest
13
+ # - macOS-latest
14
+ # - windows-latest
15
+ gradle_task:
16
+ - "gem"
17
+ - "assemble"
18
+ steps:
19
+ - uses: actions/checkout@v1
20
+ - name: Set up JDK 1.8
21
+ uses: actions/setup-java@v1
22
+ with:
23
+ java-version: 1.8
24
+ - name: Build and test
25
+ run: ./gradlew ${{ matrix.gradle_task }}
data/.gitignore CHANGED
@@ -3,6 +3,7 @@
3
3
  /tmp/
4
4
  .DS_Store
5
5
  config.yaml
6
+ *.sh
6
7
  *.gemspec
7
8
  .gradle/
8
9
  /classpath/
data/build.gradle CHANGED
@@ -4,6 +4,7 @@ plugins {
4
4
  id "java"
5
5
  id "checkstyle"
6
6
  }
7
+
7
8
  import com.github.jrubygradle.JRubyExec
8
9
  repositories {
9
10
  mavenCentral()
@@ -13,22 +14,25 @@ configurations {
13
14
  provided
14
15
  }
15
16
 
16
- version = "0.1.0"
17
+ version = "0.1.1"
17
18
 
18
19
  sourceCompatibility = 1.8
19
20
  targetCompatibility = 1.8
20
21
 
21
22
  dependencies {
22
- compile "org.embulk:embulk-core:0.9.21"
23
- provided "org.embulk:embulk-core:0.9.21"
23
+ compile "org.embulk:embulk-core:0.10.1"
24
+ provided "org.embulk:embulk-core:0.10.1"
25
+ implementation "com.google.guava:guava:28.2-jre"
24
26
  compile group: 'com.amazonaws', name: 'aws-java-sdk-logs', version: '1.11.749'
25
27
  compile group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: '1.11.749'
26
28
  // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
27
29
  testCompile "junit:junit:4.+"
28
30
  testCompile "org.mockito:mockito-core:1.+"
29
- testCompile "org.embulk:embulk-core:0.9.21:tests"
30
- testCompile "org.embulk:embulk-standards:0.9.21"
31
- testCompile "org.embulk:embulk-test:0.9.21"
31
+ testCompile "org.embulk:embulk-core:0.10.1:tests"
32
+ testCompile "org.embulk:embulk-standards:0.10.1"
33
+ testCompile "org.embulk:embulk-junit4:0.10.1"
34
+ testCompile "org.embulk:embulk-deps-buffer:0.10.1"
35
+ testCompile "org.embulk:embulk-deps-config:0.10.1"
32
36
  }
33
37
 
34
38
  // add tests/javadoc/source jar tasks as artifacts to be released
data/src/main/java/org/embulk/input/cloudwatch_logs/AbstractCloudwatchLogsInputPlugin.java CHANGED
@@ -1,15 +1,16 @@
1
1
  package org.embulk.input.cloudwatch_logs;
2
2
 
3
+ import com.google.common.annotations.VisibleForTesting;
4
+ import com.google.common.base.Optional;
5
+
3
6
  import java.util.ArrayList;
4
7
  import java.util.Collections;
5
8
  import java.util.Date;
6
9
  import java.util.List;
7
10
 
8
- import com.google.common.base.Optional;
9
- import com.google.common.annotations.VisibleForTesting;
10
-
11
11
  import org.embulk.config.Config;
12
12
  import org.embulk.config.ConfigDefault;
13
+
13
14
  import org.embulk.config.ConfigDiff;
14
15
  import org.embulk.config.ConfigException;
15
16
  import org.embulk.config.ConfigSource;
@@ -21,25 +22,21 @@ import org.embulk.spi.InputPlugin;
21
22
  import org.embulk.spi.PageBuilder;
22
23
  import org.embulk.spi.PageOutput;
23
24
  import org.embulk.spi.Schema;
24
- import org.embulk.spi.SchemaConfig;
25
25
  import org.embulk.spi.time.Timestamp;
26
26
  import org.embulk.spi.type.Types;
27
- import org.embulk.spi.util.RetryExecutor;
28
27
 
29
28
  import com.amazonaws.AmazonServiceException;
30
29
  import com.amazonaws.ClientConfiguration;
31
- import com.amazonaws.Protocol;
32
30
  import com.amazonaws.auth.AWSCredentialsProvider;
33
31
  import com.amazonaws.retry.PredefinedRetryPolicies;
34
32
 
35
- import com.amazonaws.services.logs.AWSLogsClient;
36
- import com.amazonaws.services.logs.AWSLogsClientBuilder;
37
33
  import com.amazonaws.services.logs.AWSLogs;
34
+ import com.amazonaws.services.logs.AWSLogsClientBuilder;
38
35
  import com.amazonaws.services.logs.model.DescribeLogStreamsRequest;
39
36
  import com.amazonaws.services.logs.model.DescribeLogStreamsResult;
40
- import com.amazonaws.services.logs.model.LogStream;
41
37
  import com.amazonaws.services.logs.model.GetLogEventsRequest;
42
38
  import com.amazonaws.services.logs.model.GetLogEventsResult;
39
+ import com.amazonaws.services.logs.model.LogStream;
43
40
  import com.amazonaws.services.logs.model.OutputLogEvent;
44
41
 
45
42
  import org.embulk.input.cloudwatch_logs.aws.AwsCredentials;
data/src/main/java/org/embulk/input/cloudwatch_logs/CloudwatchLogsInputPlugin.java CHANGED
@@ -1,25 +1,12 @@
1
1
  package org.embulk.input.cloudwatch_logs;
2
2
 
3
- import java.util.List;
4
-
5
- import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
6
3
  import com.amazonaws.services.logs.AWSLogsClientBuilder;
7
4
  import com.amazonaws.services.logs.AWSLogs;
8
5
  import com.google.common.base.Optional;
9
6
 
10
7
  import org.embulk.config.Config;
11
8
  import org.embulk.config.ConfigDefault;
12
- import org.embulk.config.ConfigDiff;
13
9
  import org.embulk.config.ConfigException;
14
- import org.embulk.config.ConfigSource;
15
- import org.embulk.config.Task;
16
- import org.embulk.config.TaskReport;
17
- import org.embulk.config.TaskSource;
18
- import org.embulk.spi.Exec;
19
- import org.embulk.spi.InputPlugin;
20
- import org.embulk.spi.PageOutput;
21
- import org.embulk.spi.Schema;
22
- import org.embulk.spi.SchemaConfig;
23
10
 
24
11
  public class CloudwatchLogsInputPlugin
25
12
  extends AbstractCloudwatchLogsInputPlugin
data/src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentials.java CHANGED
@@ -14,8 +14,6 @@ import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
14
14
  import com.amazonaws.auth.profile.ProfileCredentialsProvider;
15
15
  import com.amazonaws.auth.profile.ProfilesConfigFile;
16
16
  import org.embulk.config.ConfigException;
17
- import org.embulk.spi.Exec;
18
- import org.slf4j.Logger;
19
17
 
20
18
  import java.util.Optional;
21
19
 
data/src/main/java/org/embulk/input/cloudwatch_logs/utils/DateUtils.java ADDED
@@ -0,0 +1,29 @@
1
+ package org.embulk.input.cloudwatch_logs.utils;
2
+
3
+ import com.google.common.base.Joiner;
4
+ import org.embulk.config.ConfigException;
5
+ import org.joda.time.format.DateTimeFormat;
6
+
7
+ import java.util.Date;
8
+ import java.util.List;
9
+
10
+ public class DateUtils
11
+ {
12
+ public static Date parseDateStr(final String value, final List<String> supportedDateFormats)
13
+ throws ConfigException
14
+ {
15
+ for (final String format : supportedDateFormats) {
16
+ try {
17
+ return DateTimeFormat.forPattern(format).parseDateTime(value).toDate();
18
+ }
19
+ catch (final IllegalArgumentException e) {
20
+ // ignorable exception
21
+ }
22
+ }
23
+ throw new ConfigException("Unsupported DateTime value: '" + value + "', supported formats: [" + Joiner.on(",").join(supportedDateFormats) + "]");
24
+ }
25
+
26
+ private DateUtils()
27
+ {
28
+ }
29
+ }
data/src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java CHANGED
@@ -1,52 +1,38 @@
1
1
  package org.embulk.input.cloudwatch_logs;
2
2
 
3
- import com.amazonaws.services.logs.AWSLogsClient;
4
- import com.amazonaws.services.logs.AWSLogsClientBuilder;
5
3
  import com.amazonaws.services.logs.AWSLogs;
6
- import com.amazonaws.services.logs.model.DescribeLogStreamsRequest;
7
- import com.amazonaws.services.logs.model.DescribeLogStreamsResult;
8
- import com.amazonaws.services.logs.model.LogStream;
9
- import com.amazonaws.services.logs.model.GetLogEventsRequest;
10
- import com.amazonaws.services.logs.model.GetLogEventsResult;
11
- import com.amazonaws.services.logs.model.OutputLogEvent;
4
+ import com.amazonaws.services.logs.model.InputLogEvent;
12
5
 
13
6
  import org.embulk.EmbulkTestRuntime;
14
- import org.embulk.config.ConfigDiff;
15
7
  import org.embulk.config.ConfigSource;
16
8
  import org.embulk.config.TaskReport;
17
9
  import org.embulk.config.TaskSource;
18
10
  import org.embulk.spi.InputPlugin;
19
11
  import org.embulk.spi.PageBuilder;
20
- import org.embulk.spi.PageOutput;
21
12
  import org.embulk.spi.Schema;
22
13
  import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
23
- import org.embulk.spi.util.Pages;
24
- import org.embulk.test.EmbulkTests;
25
14
  import org.embulk.test.TestingEmbulk;
26
15
 
16
+ import org.junit.After;
27
17
  import org.junit.Before;
28
18
  import org.junit.BeforeClass;
29
19
  import org.junit.Rule;
30
20
  import org.junit.Test;
31
21
  import org.mockito.Mockito;
32
- import org.mockito.Spy;
33
22
 
34
23
  import java.io.IOException;
35
24
  import java.util.ArrayList;
25
+ import java.util.Date;
36
26
  import java.util.List;
37
- import org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin;
27
+
28
+ import org.embulk.input.cloudwatch_logs.TestHelpers;
29
+ import org.embulk.input.cloudwatch_logs.TestHelpers.CloudWatchLogsTestUtils;
38
30
 
39
31
  import static org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin.CloudWatchLogsPluginTask;
40
- import static org.junit.Assert.assertEquals;
41
- import static org.junit.Assert.assertFalse;
42
- import static org.junit.Assert.assertNull;
43
32
  import static org.junit.Assume.assumeNotNull;
44
- import static org.junit.Assert.assertEquals;
45
- import static org.junit.Assert.assertTrue;
46
33
  import static org.mockito.Mockito.doReturn;
47
34
  import static org.mockito.Mockito.times;
48
35
  import static org.mockito.Mockito.verify;
49
- import static org.mockito.Mockito.when;
50
36
 
51
37
  public class TestAwsCredentials
52
38
  {
@@ -63,17 +49,17 @@ public class TestAwsCredentials
63
49
  private ConfigSource config;
64
50
  private MockPageOutput output = new MockPageOutput();
65
51
  private PageBuilder pageBuilder;
52
+ private String logGroupName;
53
+ private String logStreamName;
54
+ private AWSLogs logsClient;
55
+ private CloudWatchLogsTestUtils testUtils;
66
56
 
67
- private static String EMBULK_LOGS_TEST_GROUP_NAME;
68
- private static String EMBULK_LOGS_TEST_STREAM_NAME;
69
57
  private static String EMBULK_LOGS_TEST_REGION;
70
58
  private static String EMBULK_LOGS_TEST_ACCESS_KEY_ID;
71
59
  private static String EMBULK_LOGS_TEST_SECRET_ACCESS_KEY;
72
60
 
73
61
  /*
74
62
  * This test case requires environment variables:
75
- * EMBULK_LOGS_TEST_GROUP_NAME
76
- * EMBULK_LOGS_TEST_STREAM_NAME
77
63
  * EMBULK_LOGS_TEST_REGION
78
64
  * EMBULK_LOGS_TEST_ACCESS_KEY_ID
79
65
  * EMBULK_LOGS_TEST_SECRET_ACCESS_KEY
@@ -82,32 +68,61 @@ public class TestAwsCredentials
82
68
  @BeforeClass
83
69
  public static void initializeConstantVariables()
84
70
  {
85
- EMBULK_LOGS_TEST_GROUP_NAME = System.getenv("EMBULK_LOGS_TEST_GROUP_NAME");
86
- EMBULK_LOGS_TEST_STREAM_NAME = System.getenv("EMBULK_LOGS_TEST_STREAM_NAME");
87
71
  EMBULK_LOGS_TEST_REGION = System.getenv("EMBULK_LOGS_TEST_REGION");
88
72
  EMBULK_LOGS_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_LOGS_TEST_ACCESS_KEY_ID");
89
73
  EMBULK_LOGS_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_LOGS_TEST_SECRET_ACCESS_KEY");
90
- assumeNotNull(EMBULK_LOGS_TEST_GROUP_NAME, EMBULK_LOGS_TEST_STREAM_NAME, EMBULK_LOGS_TEST_REGION, EMBULK_LOGS_TEST_ACCESS_KEY_ID, EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
74
+ assumeNotNull(EMBULK_LOGS_TEST_REGION, EMBULK_LOGS_TEST_ACCESS_KEY_ID, EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
91
75
  }
92
76
 
93
77
  @Before
94
78
  public void setUp() throws IOException
95
79
  {
80
+ logGroupName = TestHelpers.getLogGroupName();
81
+ logStreamName = TestHelpers.getLogStreamName();
82
+
96
83
  if (plugin == null) {
97
84
  plugin = Mockito.spy(new CloudwatchLogsInputPlugin());
98
85
  config = runtime.getExec().newConfigSource()
99
86
  .set("type", "cloudwatch_logs")
100
- .set("log_group_name", EMBULK_LOGS_TEST_GROUP_NAME)
101
- .set("log_stream_name", EMBULK_LOGS_TEST_STREAM_NAME)
102
- .set("use_log_stream_name_prefix", "true")
87
+ .set("log_group_name", logGroupName)
88
+ .set("log_stream_name", logStreamName)
103
89
  .set("region", EMBULK_LOGS_TEST_REGION);
104
90
  pageBuilder = Mockito.mock(PageBuilder.class);
105
- doReturn(pageBuilder).when(plugin).getPageBuilder(Mockito.any(), Mockito.any());
91
+ doReturn(pageBuilder).when(plugin).getPageBuilder(Mockito.any(), Mockito.any());
92
+ }
93
+ }
94
+
95
+ @After
96
+ public void tearDown() throws IOException
97
+ {
98
+ if (testUtils != null) {
99
+ testUtils.clearLogGroup();
106
100
  }
107
101
  }
108
102
 
109
103
  private void doTest(ConfigSource config) throws IOException
110
104
  {
105
+ CloudWatchLogsPluginTask task = config.loadConfig(CloudWatchLogsPluginTask.class);
106
+ CloudwatchLogsInputPlugin plugin = runtime.getInstance(CloudwatchLogsInputPlugin.class);
107
+ logsClient = plugin.newLogsClient(task);
108
+ testUtils = new CloudWatchLogsTestUtils(logsClient, logGroupName, logStreamName);
109
+
110
+ testUtils.createLogStream();
111
+
112
+ List<InputLogEvent> events = new ArrayList<>();
113
+ Date d = new Date();
114
+ for (int i = 0; i < 3; i++) {
115
+ InputLogEvent event = new InputLogEvent();
116
+ event.setTimestamp(d.getTime());
117
+ event.setMessage(String.format("CloudWatchLogs from Embulk take %d", i));
118
+ events.add(event);
119
+ }
120
+ testUtils.putLogEvents(events);
121
+ try {
122
+ Thread.sleep(10000);
123
+ } catch (InterruptedException e) {
124
+ }
125
+
111
126
  plugin.transaction(config, new Control());
112
127
  verify(pageBuilder, times(1)).finish();
113
128
  }
data/src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java CHANGED
@@ -1,52 +1,38 @@
1
1
  package org.embulk.input.cloudwatch_logs;
2
2
 
3
- import com.amazonaws.services.logs.AWSLogsClient;
4
- import com.amazonaws.services.logs.AWSLogsClientBuilder;
5
3
  import com.amazonaws.services.logs.AWSLogs;
6
- import com.amazonaws.services.logs.model.DescribeLogStreamsRequest;
7
- import com.amazonaws.services.logs.model.DescribeLogStreamsResult;
8
- import com.amazonaws.services.logs.model.LogStream;
9
- import com.amazonaws.services.logs.model.GetLogEventsRequest;
10
- import com.amazonaws.services.logs.model.GetLogEventsResult;
11
- import com.amazonaws.services.logs.model.OutputLogEvent;
4
+ import com.amazonaws.services.logs.model.InputLogEvent;
12
5
 
13
6
  import org.embulk.EmbulkTestRuntime;
14
- import org.embulk.config.ConfigDiff;
15
7
  import org.embulk.config.ConfigSource;
16
8
  import org.embulk.config.TaskReport;
17
9
  import org.embulk.config.TaskSource;
18
10
  import org.embulk.spi.InputPlugin;
19
11
  import org.embulk.spi.PageBuilder;
20
- import org.embulk.spi.PageOutput;
21
12
  import org.embulk.spi.Schema;
22
13
  import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
23
- import org.embulk.spi.util.Pages;
24
- import org.embulk.test.EmbulkTests;
25
14
  import org.embulk.test.TestingEmbulk;
26
15
 
16
+ import org.junit.After;
27
17
  import org.junit.Before;
28
18
  import org.junit.BeforeClass;
29
19
  import org.junit.Rule;
30
20
  import org.junit.Test;
31
21
  import org.mockito.Mockito;
32
- import org.mockito.Spy;
33
22
 
34
23
  import java.io.IOException;
35
24
  import java.util.ArrayList;
25
+ import java.util.Date;
36
26
  import java.util.List;
37
- import org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin;
27
+
28
+ import org.embulk.input.cloudwatch_logs.TestHelpers;
29
+ import org.embulk.input.cloudwatch_logs.TestHelpers.CloudWatchLogsTestUtils;
38
30
 
39
31
  import static org.embulk.input.cloudwatch_logs.CloudwatchLogsInputPlugin.CloudWatchLogsPluginTask;
40
- import static org.junit.Assert.assertEquals;
41
- import static org.junit.Assert.assertFalse;
42
- import static org.junit.Assert.assertNull;
43
32
  import static org.junit.Assume.assumeNotNull;
44
- import static org.junit.Assert.assertEquals;
45
- import static org.junit.Assert.assertTrue;
46
33
  import static org.mockito.Mockito.doReturn;
47
34
  import static org.mockito.Mockito.times;
48
35
  import static org.mockito.Mockito.verify;
49
- import static org.mockito.Mockito.when;
50
36
 
51
37
  public class TestCloudwatchLogsInputPlugin
52
38
  {
@@ -63,17 +49,17 @@ public class TestCloudwatchLogsInputPlugin
63
49
  private ConfigSource config;
64
50
  private MockPageOutput output = new MockPageOutput();
65
51
  private PageBuilder pageBuilder;
52
+ private String logGroupName;
53
+ private String logStreamName;
54
+ private AWSLogs logsClient;
55
+ private CloudWatchLogsTestUtils testUtils;
66
56
 
67
- private static String EMBULK_LOGS_TEST_GROUP_NAME;
68
- private static String EMBULK_LOGS_TEST_STREAM_NAME;
69
57
  private static String EMBULK_LOGS_TEST_REGION;
70
58
  private static String EMBULK_LOGS_TEST_ACCESS_KEY_ID;
71
59
  private static String EMBULK_LOGS_TEST_SECRET_ACCESS_KEY;
72
60
 
73
61
  /*
74
62
  * This test case requires environment variables:
75
- * EMBULK_LOGS_TEST_GROUP_NAME
76
- * EMBULK_LOGS_TEST_STREAM_NAME
77
63
  * EMBULK_LOGS_TEST_REGION
78
64
  * EMBULK_LOGS_TEST_ACCESS_KEY_ID
79
65
  * EMBULK_LOGS_TEST_SECRET_ACCESS_KEY
@@ -82,36 +68,63 @@ public class TestCloudwatchLogsInputPlugin
82
68
  @BeforeClass
83
69
  public static void initializeConstantVariables()
84
70
  {
85
- EMBULK_LOGS_TEST_GROUP_NAME = System.getenv("EMBULK_LOGS_TEST_GROUP_NAME");
86
- EMBULK_LOGS_TEST_STREAM_NAME = System.getenv("EMBULK_LOGS_TEST_STREAM_NAME");
87
71
  EMBULK_LOGS_TEST_REGION = System.getenv("EMBULK_LOGS_TEST_REGION");
88
72
  EMBULK_LOGS_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_LOGS_TEST_ACCESS_KEY_ID");
89
73
  EMBULK_LOGS_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_LOGS_TEST_SECRET_ACCESS_KEY");
90
- assumeNotNull(EMBULK_LOGS_TEST_GROUP_NAME, EMBULK_LOGS_TEST_STREAM_NAME, EMBULK_LOGS_TEST_REGION, EMBULK_LOGS_TEST_ACCESS_KEY_ID, EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
74
+ assumeNotNull(EMBULK_LOGS_TEST_REGION, EMBULK_LOGS_TEST_ACCESS_KEY_ID, EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
91
75
  }
92
76
 
93
77
  @Before
94
78
  public void setUp() throws IOException
95
79
  {
80
+ logGroupName = TestHelpers.getLogGroupName();
81
+ logStreamName = TestHelpers.getLogStreamName();
82
+
96
83
  if (plugin == null) {
97
84
  plugin = Mockito.spy(new CloudwatchLogsInputPlugin());
98
85
  config = runtime.getExec().newConfigSource()
99
86
  .set("type", "cloudwatch_logs")
100
- .set("log_group_name", EMBULK_LOGS_TEST_GROUP_NAME)
101
- .set("log_stream_name", EMBULK_LOGS_TEST_STREAM_NAME)
102
- .set("use_log_stream_name_prefix", "true")
87
+ .set("log_group_name", logGroupName)
88
+ .set("log_stream_name", logStreamName)
103
89
  .set("region", EMBULK_LOGS_TEST_REGION)
104
90
  .set("aws_access_key_id", EMBULK_LOGS_TEST_ACCESS_KEY_ID)
105
91
  .set("aws_secret_access_key", EMBULK_LOGS_TEST_SECRET_ACCESS_KEY);
106
92
  pageBuilder = Mockito.mock(PageBuilder.class);
107
93
  }
108
94
  doReturn(pageBuilder).when(plugin).getPageBuilder(Mockito.any(), Mockito.any());
95
+ CloudWatchLogsPluginTask task = config.loadConfig(CloudWatchLogsPluginTask.class);
96
+ CloudwatchLogsInputPlugin plugin = runtime.getInstance(CloudwatchLogsInputPlugin.class);
97
+ logsClient = plugin.newLogsClient(task);
98
+ testUtils = new CloudWatchLogsTestUtils(logsClient, logGroupName, logStreamName);
99
+ }
100
+
101
+ @After
102
+ public void tearDown() throws IOException
103
+ {
104
+ testUtils.clearLogGroup();
109
105
  }
110
106
 
111
107
  @Test
112
108
  public void test_simple() throws IOException
113
109
  {
110
+ testUtils.createLogStream();
111
+
112
+ List<InputLogEvent> events = new ArrayList<>();
113
+ Date d = new Date();
114
+ for (int i = 0; i < 3; i++) {
115
+ InputLogEvent event = new InputLogEvent();
116
+ event.setTimestamp(d.getTime());
117
+ event.setMessage(String.format("CloudWatchLogs from Embulk take %d", i));
118
+ events.add(event);
119
+ }
120
+ testUtils.putLogEvents(events);
121
+ try {
122
+ Thread.sleep(10000);
123
+ } catch (InterruptedException e) {
124
+ }
125
+
114
126
  plugin.transaction(config, new Control());
127
+ verify(pageBuilder, times(3)).addRecord();
115
128
  verify(pageBuilder, times(1)).finish();
116
129
  }
117
130
 
data/src/test/java/org/embulk/input/cloudwatch_logs/TestHelpers.java ADDED
@@ -0,0 +1,78 @@
1
+ package org.embulk.input.cloudwatch_logs;
2
+
3
+ import java.io.File;
4
+ import java.io.FileReader;
5
+ import java.io.IOException;
6
+ import java.util.Date;
7
+ import java.util.List;
8
+ import java.util.UUID;
9
+
10
+ import com.amazonaws.services.logs.AWSLogs;
11
+ import com.amazonaws.services.logs.model.CreateLogGroupRequest;
12
+ import com.amazonaws.services.logs.model.CreateLogStreamRequest;
13
+ import com.amazonaws.services.logs.model.DeleteLogGroupRequest;
14
+ import com.amazonaws.services.logs.model.InputLogEvent;
15
+ import com.amazonaws.services.logs.model.PutLogEventsRequest;
16
+ import com.amazonaws.services.logs.model.ResourceNotFoundException;
17
+
18
+ public final class TestHelpers
19
+ {
20
+ private TestHelpers() {}
21
+
22
+ public static String getLogGroupName()
23
+ {
24
+ Date d = new Date();
25
+ return String.format("embulk-input-cloudwatch-test-%d", d.getTime());
26
+ }
27
+
28
+ public static String getLogStreamName()
29
+ {
30
+ UUID uuid = UUID.randomUUID();
31
+ return uuid.toString();
32
+ }
33
+
34
+ static class CloudWatchLogsTestUtils
35
+ {
36
+ private final AWSLogs logs;
37
+ private final String logGroupName;
38
+ private final String logStreamName;
39
+
40
+ public CloudWatchLogsTestUtils(AWSLogs logs, String logGroupName, String logStreamName)
41
+ {
42
+ this.logs = logs;
43
+ this.logGroupName = logGroupName;
44
+ this.logStreamName = logStreamName;
45
+ }
46
+
47
+ public void clearLogGroup()
48
+ {
49
+ DeleteLogGroupRequest request = new DeleteLogGroupRequest();
50
+ request.setLogGroupName(logGroupName);
51
+ try {
52
+ logs.deleteLogGroup(request);
53
+ } catch (ResourceNotFoundException ex) {
54
+ // Just ignored.
55
+ }
56
+ }
57
+
58
+ public void createLogStream()
59
+ {
60
+ CreateLogGroupRequest groupRequest = new CreateLogGroupRequest();
61
+ groupRequest.setLogGroupName(logGroupName);
62
+ logs.createLogGroup(groupRequest);
63
+
64
+ CreateLogStreamRequest streamRequest = new CreateLogStreamRequest();
65
+ streamRequest.setLogGroupName(logGroupName);
66
+ streamRequest.setLogStreamName(logStreamName);
67
+ logs.createLogStream(streamRequest);
68
+ }
69
+
70
+ public void putLogEvents(List<InputLogEvent> events) {
71
+ PutLogEventsRequest request = new PutLogEventsRequest()
72
+ .withLogGroupName(logGroupName)
73
+ .withLogStreamName(logStreamName)
74
+ .withLogEvents(events);
75
+ logs.putLogEvents(request);
76
+ }
77
+ }
78
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-cloudwatch_logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Hiroshi Hatake
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-03-30 00:00:00.000000000 Z
11
+ date: 2020-04-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -45,6 +45,7 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
+ - ".github/workflows/build.yml"
48
49
  - ".gitignore"
49
50
  - LICENSE.txt
50
51
  - README.md
@@ -54,7 +55,7 @@ files:
54
55
  - classpath/aws-java-sdk-sts-1.11.749.jar
55
56
  - classpath/commons-codec-1.11.jar
56
57
  - classpath/commons-logging-1.2.jar
57
- - classpath/embulk-input-cloudwatch_logs-0.1.0.jar
58
+ - classpath/embulk-input-cloudwatch_logs-0.1.1.jar
58
59
  - classpath/httpclient-4.5.9.jar
59
60
  - classpath/httpcore-4.4.11.jar
60
61
  - classpath/ion-java-1.0.2.jar
@@ -73,8 +74,10 @@ files:
73
74
  - src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentials.java
74
75
  - src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsConfig.java
75
76
  - src/main/java/org/embulk/input/cloudwatch_logs/aws/AwsCredentialsTask.java
77
+ - src/main/java/org/embulk/input/cloudwatch_logs/utils/DateUtils.java
76
78
  - src/test/java/org/embulk/input/cloudwatch_logs/TestAwsCredentials.java
77
79
  - src/test/java/org/embulk/input/cloudwatch_logs/TestCloudwatchLogsInputPlugin.java
80
+ - src/test/java/org/embulk/input/cloudwatch_logs/TestHelpers.java
78
81
  homepage: https://github.com/cosmo0920/embulk-input-cloudwatch_logs
79
82
  licenses:
80
83
  - MIT