embulk-parser-apache-custom-log 0.2.0

data/src/test/java/org/embulk/parser/apache/log/StringLogElementFactoryTest.java

@@ -0,0 +1,91 @@
+package org.embulk.parser.apache.log;
+
+import junit.framework.TestCase;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+@RunWith(Enclosed.class)
+public class StringLogElementFactoryTest extends TestCase {
+
+    public static class WithName{
+
+        @Test
+        public void testCreateWithNull() throws Exception {
+
+            StringLogElementFactory factory = new StringLogElementFactory("test-name");
+
+            StringLogElement logElement = factory.create(null);
+
+            assertThat(logElement.getName(), is("test-name"));
+            assertThat(logElement.getRegexp(), is("(.*)"));
+
+        }
+
+        @Test
+        public void testCreateWithEmptyString() throws Exception {
+
+            StringLogElementFactory factory = new StringLogElementFactory("test-name");
+
+            StringLogElement logElement = factory.create(null);
+
+            assertThat(logElement.getName(), is("test-name"));
+            assertThat(logElement.getRegexp(), is("(.*)"));
+
+        }
+
+        @Test
+        public void testCreateWithParameter() throws Exception {
+
+            StringLogElementFactory factory = new StringLogElementFactory("test-name");
+
+            StringLogElement logElement = factory.create("param");
+
+            assertThat(logElement.getName(), is("test-name-param"));
+            assertThat(logElement.getRegexp(), is("(.*)"));
+
+        }
+    }
+
+    public static class WithNameAndRegexp{
+
+        @Test
+        public void testCreateWithNull() throws Exception {
+
+            StringLogElementFactory factory = new StringLogElementFactory("test-name", "(.+)");
+
+            StringLogElement logElement = factory.create(null);
+
+            assertThat(logElement.getName(), is("test-name"));
+            assertThat(logElement.getRegexp(), is("(.+)"));
+
+        }
+
+        @Test
+        public void testCreateWithEmptyString() throws Exception {
+
+            StringLogElementFactory factory = new StringLogElementFactory("test-name", "(.+)");
+
+            StringLogElement logElement = factory.create(null);
+
+            assertThat(logElement.getName(), is("test-name"));
+            assertThat(logElement.getRegexp(), is("(.+)"));
+
+        }
+
+        @Test
+        public void testCreateWithParameter() throws Exception {
+
+            StringLogElementFactory factory = new StringLogElementFactory("test-name", "(.+)");
+
+            StringLogElement logElement = factory.create("param");
+
+            assertThat(logElement.getName(), is("test-name-param"));
+            assertThat(logElement.getRegexp(), is("(.+)"));
+
+        }
+    }
+}

data/src/test/java/org/embulk/parser/apache/log/StringLogElementTest.java

@@ -0,0 +1,51 @@
+package org.embulk.parser.apache.log;
+
+import junit.framework.TestCase;
+import org.embulk.spi.PageBuilder;
+import org.junit.Test;
+import org.junit.experimental.runners.Enclosed;
+import org.junit.runner.RunWith;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
+@RunWith(Enclosed.class)
+public class StringLogElementTest extends TestCase {
+
+    public static class TestParse {
+
+        StringLogElement elem = new StringLogElement("test-elem", "(.*)");
+
+        @Test
+        public void testParseWithNull() throws Exception {
+            assertThat(elem.parse(null), is((String)null));
+        }
+
+        @Test
+        public void testParseWithEmpty() throws Exception {
+            assertThat(elem.parse(""), is(""));
+        }
+
+        @Test
+        public void testParseWithNonEmptyString() throws Exception {
+            assertThat(elem.parse("str"), is("str"));
+        }
+
+        @Test
+        public void testParseWithCLFEmptyString() throws Exception {
+            assertThat(elem.parse("-"), is((String)null));
+        }
+
+    }
+
+    public static class TestSetToPageBuilder{
+        @Test
+        public void testSetToPageBuilder() throws Exception {
+            //TODO implement
+        }
+    }
+
+
+
+
+}

data/src/test/java/org/embulk/tester/DummyConfigSource.java

@@ -0,0 +1,86 @@
+package org.embulk.tester;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSource;
+
+import java.util.List;
+import java.util.Map;
+
+public class DummyConfigSource implements ConfigSource {
+    @Override
+    public <T> T loadConfig(Class<T> taskType) {
+        return null;
+    }
+
+    @Override
+    public List<String> getAttributeNames() {
+        return null;
+    }
+
+    @Override
+    public Iterable<Map.Entry<String, JsonNode>> getAttributes() {
+        return null;
+    }
+
+    @Override
+    public boolean isEmpty() {
+        return false;
+    }
+
+    @Override
+    public <E> E get(Class<E> type, String attrName) {
+        return null;
+    }
+
+    @Override
+    public <E> E get(Class<E> type, String attrName, E defaultValue) {
+        return null;
+    }
+
+    @Override
+    public ConfigSource getNested(String attrName) {
+        return null;
+    }
+
+    @Override
+    public ConfigSource getNestedOrSetEmpty(String attrName) {
+        return null;
+    }
+
+    @Override
+    public ConfigSource set(String attrName, Object v) {
+        return null;
+    }
+
+    @Override
+    public ConfigSource setNested(String attrName, DataSource v) {
+        return null;
+    }
+
+    @Override
+    public ConfigSource setAll(DataSource other) {
+        return null;
+    }
+
+    @Override
+    public ConfigSource remove(String attrName) {
+        return null;
+    }
+
+    @Override
+    public ConfigSource deepCopy() {
+        return null;
+    }
+
+    @Override
+    public ConfigSource merge(DataSource other) {
+        return null;
+    }
+
+    @Override
+    public ObjectNode getObjectNode() {
+        return null;
+    }
+}

data/src/test/java/org/embulk/tester/EmbulkPluginTester.java

@@ -0,0 +1,52 @@
+package org.embulk.tester;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.embulk.command.Runner;
+
+public class EmbulkPluginTester {
+
+	public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
+	{
+		TestExtension.addPlugin(iface, name, impl);
+	}
+
+	public void run(String ymlPath) throws Exception
+	{
+		Runner runner = new Runner("{}");
+		runner.run(convert(ymlPath));
+	}
+
+	private String convert(String yml) throws Exception
+	{
+		File rootPath = new File(EmbulkPluginTester.class.getResource("/resource.txt").toURI()).getParentFile();
+		File ymlPath = new File(EmbulkPluginTester.class.getResource(yml).toURI());
+		File tempYmlPath = new File(ymlPath.getParentFile(), "temp-" + ymlPath.getName());
+		Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
+		try (BufferedReader reader = new BufferedReader(new FileReader(ymlPath))) {
+			try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempYmlPath))) {
+				String line;
+				while ((line = reader.readLine()) != null) {
+					Matcher matcher = pathPrefixPattern.matcher(line);
+					if (matcher.matches()) {
+						int group = 2;
+						writer.write(line.substring(0, matcher.start(group)));
+						writer.write(new File(rootPath, matcher.group(group)).getAbsolutePath());
+						writer.write(line.substring(matcher.end(group)));
+					} else {
+						writer.write(line);
+					}
+					writer.newLine();
+				}
+			}
+		}
+		return tempYmlPath.getAbsolutePath();
+	}
+
+}

data/src/test/java/org/embulk/tester/TestExtension.java

@@ -0,0 +1,52 @@
+package org.embulk.tester;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.InjectedPluginSource;
+import org.embulk.spi.Extension;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Binder;
+import com.google.inject.Module;
+
+
+public class TestExtension implements Extension
+{
+    private static class PluginDefinition
+    {
+        public final Class<?> iface;
+        public final String name;
+        public final Class<?> impl;
+
+        public PluginDefinition(Class<?> iface, String name, Class<?> impl)
+        {
+            this.iface = iface;
+            this.name = name;
+            this.impl = impl;
+        }
+    }
+
+    private static List<PluginDefinition> plugins = new ArrayList<>();
+
+    public static void addPlugin(Class<?> iface, String name, Class<?> impl)
+    {
+        plugins.add(new PluginDefinition(iface, name, impl));
+    }
+
+	@Override
+	public List<Module> getModules(ConfigSource configsource) {
+		Module module = new Module() {
+
+			@Override
+			public void configure(Binder binder) {
+                for (PluginDefinition plugin : plugins) {
+                    InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
+                }
+			}
+		};
+		return ImmutableList.of(module);
+	}
+
+}

data/src/test/resources/META-INF/services/org.embulk.spi.Extension

@@ -0,0 +1 @@
+org.embulk.tester.TestExtension

data/src/test/resources/data/access_log_2_combined

@@ -0,0 +1 @@
+24.93.39.209 - - [25/Jul/2015:15:31:32 +0900] "POST /search/?c=Computers HTTP/1.1" 200 88 "/category/health" "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; WOW64; Trident/4.0; GTB6; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30618; .NET4.0C)"

data/src/test/resources/data/access_log_combined

@@ -0,0 +1,2 @@
+127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326 "http://www.example.com/start.html" "Mozilla/4.08 [en] (Win98; I ;Nav)"
+

data/src/test/resources/data/access_log_common

@@ -0,0 +1 @@
+127.0.0.1 - frank [10/Oct/2000:13:55:36 -0700] "GET /apache_pb.gif HTTP/1.0" 200 2326

data/src/test/resources/yml/test_combined.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: 'data/access_log_combined'
+  parser:
+    type: apache-log
+    format: '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"'
+out:
+  type: file
+  path_prefix: '/temp/result_combined.'
+  file_ext: tsv
+  formatter:
+    type: csv
+    delimiter: "\t"

data/src/test/resources/yml/test_combined2.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: 'data/access_log_2_combined'
+  parser:
+    type: apache-log
+    format: '%h %l %u %t \"%m %U%q %H\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"'
+out:
+  type: file
+  path_prefix: '/temp/result_2_combined.'
+  file_ext: tsv
+  formatter:
+    type: csv
+    delimiter: "\t"

data/src/test/resources/yml/test_common.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: 'data/access_log_common'
+  parser:
+    type: apache-log
+    format: '%h %l %u %t \"%r\" %>s %b'
+out:
+  type: file
+  path_prefix: '/temp/result_common.'
+  file_ext: tsv
+  formatter:
+    type: csv
+    delimiter: "\t"

metadata

@@ -0,0 +1,115 @@
+--- !ruby/object:Gem::Specification
+name: embulk-parser-apache-custom-log
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Hiroyuki Sato
+- Osamu Ishikawa
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-10-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Parses Apache Custom Log files read by other file input plugins.
+email:
+- hiroysato@gmail.com
+- bass.duo@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- CHANGES.md
+- LICENSE.txt
+- README.md
+- build.gradle
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/guess/apache-custom-log.rb
+- lib/embulk/parser/apache-custom-log.rb
+- src/main/java/org/embulk/parser/ApacheCustomLogParserPlugin.java
+- src/main/java/org/embulk/parser/apache/log/LogElement.java
+- src/main/java/org/embulk/parser/apache/log/LogElementFactory.java
+- src/main/java/org/embulk/parser/apache/log/LogFormats.java
+- src/main/java/org/embulk/parser/apache/log/LongLogElement.java
+- src/main/java/org/embulk/parser/apache/log/LongLogElementFactory.java
+- src/main/java/org/embulk/parser/apache/log/Patterns.java
+- src/main/java/org/embulk/parser/apache/log/Replacement.java
+- src/main/java/org/embulk/parser/apache/log/StringLogElement.java
+- src/main/java/org/embulk/parser/apache/log/StringLogElementFactory.java
+- src/main/java/org/embulk/parser/apache/log/TimestampLogElement.java
+- src/main/java/org/embulk/parser/apache/log/TimestampLogElementFactory.java
+- src/test/java/org/embulk/parser/TestApacheLogParserPlugin.java
+- src/test/java/org/embulk/parser/apache/log/LogFormatsTest.java
+- src/test/java/org/embulk/parser/apache/log/PatternsTest.java
+- src/test/java/org/embulk/parser/apache/log/StringLogElementFactoryTest.java
+- src/test/java/org/embulk/parser/apache/log/StringLogElementTest.java
+- src/test/java/org/embulk/tester/DummyConfigSource.java
+- src/test/java/org/embulk/tester/EmbulkPluginTester.java
+- src/test/java/org/embulk/tester/TestExtension.java
+- src/test/resources/META-INF/services/org.embulk.spi.Extension
+- src/test/resources/data/access_log_2_combined
+- src/test/resources/data/access_log_combined
+- src/test/resources/data/access_log_common
+- src/test/resources/resource.txt
+- src/test/resources/temp/dummy
+- src/test/resources/yml/test_combined.yml
+- src/test/resources/yml/test_combined2.yml
+- src/test/resources/yml/test_common.yml
+- classpath/embulk-parser-apache-custom-log-0.2.0.jar
+homepage: https://github.com/jami-i/embulk-parser-apache-custom-log
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Apache Custom Log parser plugin for Embulk
+test_files: []