embulk-filter-gsub 0.1.0

data/src/main/kotlin/org/embulk/filter/gsub/replacer/UpperCaseReplacer.kt

@@ -0,0 +1,18 @@
+package org.embulk.filter.gsub.replacer
+
+class UpperCaseReplacer(private val pattern: Regex?) : TextReplacer {
+    constructor() : this(null)
+
+    override fun execute(text: String): String {
+        return pattern?.let { replaceWithPattern(it, text)} ?: replaceWholeText(text)
+    }
+
+    private fun replaceWithPattern(pattern: Regex, text: String): String {
+        return pattern.replace(text, { matchResult -> matchResult.value.toUpperCase() })
+    }
+
+    private fun replaceWholeText(text: String): String {
+        return text.toUpperCase()
+    }
+}
+

data/src/test/kotlin/org/embulk/filter/gsub/TestGsubFilterPlugin.kt

@@ -0,0 +1,221 @@
+package org.embulk.filter.gsub
+
+import org.embulk.EmbulkTestRuntime
+import org.embulk.config.ConfigLoader
+import org.embulk.config.ConfigSource
+import org.embulk.config.TaskSource
+import org.embulk.spi.*
+import org.embulk.spi.type.Types
+import org.embulk.spi.util.Pages
+import org.junit.Assert
+import org.junit.Rule
+import org.junit.Test
+
+import org.hamcrest.Matchers.*
+
+class TestGsubFilterPlugin {
+    @get:Rule
+    val runtime = EmbulkTestRuntime()
+
+    @Test
+    fun testConfig() {
+        val configYaml = """
+        |type: gsub
+        |target_columns:
+        |  foo:
+        |    - type: to_lower_case
+        |      pattern: "[A-Z]*"
+        |  bar:
+        |    - type: "regexp_replace"
+        |      pattern: "<br\\s*/?>"
+        |      to: "\\n"
+        |    - pattern: "(\\d+):(.*)"
+        |      to: "\\1 [\\2]"
+        """.trimMargin()
+
+        val config = getConfigFromYaml(configYaml)
+        val task = config.loadConfig(GsubFilterPlugin.PluginTask::class.java)
+
+        val fooRules = task.targetColumns["foo"]
+        Assert.assertThat(fooRules, hasSize(1))
+
+        val fooRule = fooRules!![0]
+        Assert.assertEquals("to_lower_case", fooRule.type)
+        Assert.assertEquals("[A-Z]*", fooRule.pattern.get())
+
+        val barRules = task.targetColumns["bar"]
+        Assert.assertThat(barRules, hasSize(2))
+
+        val barRule1 = barRules!![0]
+        Assert.assertEquals("regexp_replace", barRule1.type)
+        Assert.assertEquals("<br\\s*/?>", barRule1.pattern.get())
+        Assert.assertEquals("\\n", barRule1.to.get())
+
+        val barRule2 = barRules[1]
+        Assert.assertEquals("regexp_replace", barRule2.type)
+        Assert.assertEquals("(\\d+):(.*)", barRule2.pattern.get())
+        Assert.assertEquals("\\1 [\\2]", barRule2.to.get())
+    }
+
+    @Test
+    fun testEmptyFilter() {
+        val configYaml = """
+        |type: gsub
+        """.trimMargin()
+
+        val config = getConfigFromYaml(configYaml)
+
+        val inputSchema = Schema.builder()
+                .add("bool", Types.BOOLEAN)
+                .add("long", Types.LONG)
+                .add("string", Types.STRING)
+                .build()
+
+        val plugin = GsubFilterPlugin()
+        plugin.transaction(config, inputSchema, object: FilterPlugin.Control {
+            override fun run(taskSource: TaskSource, outputSchema: Schema) {
+                val mockPageOutput = TestPageBuilderReader.MockPageOutput()
+                val pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)
+
+                val inputPage = PageTestUtils.buildPage(runtime.bufferAllocator, inputSchema, true, 1234L, "test for echo")
+                for (page in inputPage) {
+                    pageOutput.add(page)
+                }
+                pageOutput.finish()
+                pageOutput.close()
+
+                val records = Pages.toObjects(outputSchema, mockPageOutput.pages)
+                val record = records.get(0)
+                Assert.assertEquals(true, record[0])
+                Assert.assertEquals(1234L, record[1])
+                Assert.assertEquals("test for echo", record[2])
+            }
+        })
+    }
+
+    @Test
+    fun testRegexReplaceFilter() {
+        val configYaml = """
+        |type: gsub
+        |target_columns:
+        |  string:
+        |    - type: regexp_replace
+        |      pattern: "test"
+        |      to: "[replaced]"
+        """.trimMargin()
+
+        val config = getConfigFromYaml(configYaml)
+
+        val inputSchema = Schema.builder()
+                .add("bool", Types.BOOLEAN)
+                .add("long", Types.LONG)
+                .add("string", Types.STRING)
+                .build()
+
+        val plugin = GsubFilterPlugin()
+        plugin.transaction(config, inputSchema, object: FilterPlugin.Control {
+            override fun run(taskSource: TaskSource, outputSchema: Schema) {
+                val mockPageOutput = TestPageBuilderReader.MockPageOutput()
+                val pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)
+
+                val inputPage = PageTestUtils.buildPage(runtime.bufferAllocator, inputSchema, true, 1234L, "test for echo")
+                for (page in inputPage) {
+                    pageOutput.add(page)
+                }
+                pageOutput.finish()
+                pageOutput.close()
+
+                val records = Pages.toObjects(outputSchema, mockPageOutput.pages)
+                val record = records.get(0)
+                Assert.assertEquals(true, record[0])
+                Assert.assertEquals(1234L, record[1])
+                Assert.assertEquals("[replaced] for echo", record[2])
+            }
+        })
+    }
+
+    @Test
+    fun testToUpperCaseFilter() {
+        val configYaml = """
+        |type: gsub
+        |target_columns:
+        |  string:
+        |    - type: to_upper_case
+        |      pattern: "test"
+        """.trimMargin()
+
+        val config = getConfigFromYaml(configYaml)
+
+        val inputSchema = Schema.builder()
+                .add("bool", Types.BOOLEAN)
+                .add("long", Types.LONG)
+                .add("string", Types.STRING)
+                .build()
+
+        val plugin = GsubFilterPlugin()
+        plugin.transaction(config, inputSchema, object: FilterPlugin.Control {
+            override fun run(taskSource: TaskSource, outputSchema: Schema) {
+                val mockPageOutput = TestPageBuilderReader.MockPageOutput()
+                val pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)
+
+                val inputPage = PageTestUtils.buildPage(runtime.bufferAllocator, inputSchema, true, 1234L, "test for echo")
+                for (page in inputPage) {
+                    pageOutput.add(page)
+                }
+                pageOutput.finish()
+                pageOutput.close()
+
+                val records = Pages.toObjects(outputSchema, mockPageOutput.pages)
+                val record = records.get(0)
+                Assert.assertEquals(true, record[0])
+                Assert.assertEquals(1234L, record[1])
+                Assert.assertEquals("TEST for echo", record[2])
+            }
+        })
+    }
+
+    @Test
+    fun testToLowerCaseFilter() {
+        val configYaml = """
+        |type: gsub
+        |target_columns:
+        |  string:
+        |    - type: to_lower_case
+        |      pattern: "TEST"
+        """.trimMargin()
+
+        val config = getConfigFromYaml(configYaml)
+
+        val inputSchema = Schema.builder()
+                .add("bool", Types.BOOLEAN)
+                .add("long", Types.LONG)
+                .add("string", Types.STRING)
+                .build()
+
+        val plugin = GsubFilterPlugin()
+        plugin.transaction(config, inputSchema, object: FilterPlugin.Control {
+            override fun run(taskSource: TaskSource, outputSchema: Schema) {
+                val mockPageOutput = TestPageBuilderReader.MockPageOutput()
+                val pageOutput = plugin.open(taskSource, inputSchema, outputSchema, mockPageOutput)
+
+                val inputPage = PageTestUtils.buildPage(runtime.bufferAllocator, inputSchema, true, 1234L, "TEST FOR ECHO")
+                for (page in inputPage) {
+                    pageOutput.add(page)
+                }
+                pageOutput.finish()
+                pageOutput.close()
+
+                val records = Pages.toObjects(outputSchema, mockPageOutput.pages)
+                val record = records.get(0)
+                Assert.assertEquals(true, record[0])
+                Assert.assertEquals(1234L, record[1])
+                Assert.assertEquals("test FOR ECHO", record[2])
+            }
+        })
+    }
+
+    private fun getConfigFromYaml(yaml: String): ConfigSource {
+        val loader = ConfigLoader(Exec.getModelManager())
+        return loader.fromYamlString(yaml)
+    }
+}

data/src/test/kotlin/org/embulk/filter/gsub/replacer/CombinedReplacerTest.kt

@@ -0,0 +1,37 @@
+package org.embulk.filter.gsub.replacer
+
+import org.junit.Assert
+import org.junit.Test
+
+class CombinedReplacerTest {
+    @Test
+    fun testExecuteWithEmptyReplacer() {
+        val replacer = CombinedReplacer(ArrayList<TextReplacer>())
+        val result = replacer.execute("foo bar baz")
+        Assert.assertEquals("foo bar baz", result)
+    }
+
+    @Test
+    fun testExecuteWithSingleReplacer() {
+        val lowerCaseReplacer = LowerCaseReplacer()
+        val replacers = ArrayList<TextReplacer>()
+        replacers.add(lowerCaseReplacer)
+
+        val combinedReplacer = CombinedReplacer(replacers)
+        val result = combinedReplacer.execute("FOO BAR BAZ")
+        Assert.assertEquals("foo bar baz", result)
+    }
+
+    @Test
+    fun testExecuteWithMultipleReplacer() {
+        val lowerCaseReplacer = LowerCaseReplacer()
+        val regexReplacer = RegexReplacer("\\s+".toRegex(), "-")
+        val replacers = ArrayList<TextReplacer>()
+        replacers.add(lowerCaseReplacer)
+        replacers.add(regexReplacer)
+
+        val combinedReplacer = CombinedReplacer(replacers)
+        val result = combinedReplacer.execute("FOO BAR BAZ")
+        Assert.assertEquals("foo-bar-baz", result)
+    }
+}

data/src/test/kotlin/org/embulk/filter/gsub/replacer/LowerCaseReplacerTest.kt

@@ -0,0 +1,20 @@
+package org.embulk.filter.gsub.replacer
+
+import org.junit.Assert
+import org.junit.Test
+
+class LowerCaseReplacerTest {
+    @Test
+    fun testExecuteWithoutPattern() {
+        val replacer = LowerCaseReplacer()
+        val result = replacer.execute("Foo Bar BAZ")
+        Assert.assertEquals("foo bar baz", result)
+    }
+
+    @Test
+    fun testExecuteWithPattern() {
+        val replacer = LowerCaseReplacer("[A-Z]{3}".toRegex())
+        val result = replacer.execute("Foo Bar BAZ")
+        Assert.assertEquals("Foo Bar baz", result)
+    }
+}

data/src/test/kotlin/org/embulk/filter/gsub/replacer/RegexReplacerTest.kt

@@ -0,0 +1,19 @@
+package org.embulk.filter.gsub.replacer
+
+import org.junit.Assert
+import org.junit.Test
+
+class RegexReplacerTest {
+    @Test
+    fun testExecute() {
+        val optionConfig = RegexOptionConfig()
+        optionConfig.ignoreCase = true
+
+        val factory = RegexFactory()
+        val pattern = factory.create("(\\w*):\\s*(.*)", optionConfig)
+
+        val replacer = RegexReplacer(pattern, "$1 [$2]")
+        val result = replacer.execute("test: foo bar baz")
+        Assert.assertEquals("test [foo bar baz]", result)
+    }
+}

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: embulk-filter-gsub
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Sawada Tadashi
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-01-21 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: gsub
+email:
+- cesare@mayverse.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .circleci/config.yml
+- .gitignore
+- LICENSE.txt
+- README.md
+- build.gradle
+- gradle.properties
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/filter/gsub.rb
+- src/main/kotlin/org/embulk/filter/gsub/ColumnReplacerFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/ColumnVisitorImpl.kt
+- src/main/kotlin/org/embulk/filter/gsub/GsubFilterPlugin.kt
+- src/main/kotlin/org/embulk/filter/gsub/LowerCaseReplacerFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/RegexReplacerFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/SubstitutionRule.kt
+- src/main/kotlin/org/embulk/filter/gsub/TextReplacerFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/UpperCaseReplacerFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/replacer/CombinedReplacer.kt
+- src/main/kotlin/org/embulk/filter/gsub/replacer/LowerCaseReplacer.kt
+- src/main/kotlin/org/embulk/filter/gsub/replacer/RegexFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/replacer/RegexOptionConfig.kt
+- src/main/kotlin/org/embulk/filter/gsub/replacer/RegexReplacer.kt
+- src/main/kotlin/org/embulk/filter/gsub/replacer/TextReplacer.kt
+- src/main/kotlin/org/embulk/filter/gsub/replacer/UpperCaseReplacer.kt
+- src/test/kotlin/org/embulk/filter/gsub/TestGsubFilterPlugin.kt
+- src/test/kotlin/org/embulk/filter/gsub/replacer/CombinedReplacerTest.kt
+- src/test/kotlin/org/embulk/filter/gsub/replacer/LowerCaseReplacerTest.kt
+- src/test/kotlin/org/embulk/filter/gsub/replacer/RegexReplacerTest.kt
+- classpath/annotations-13.0.jar
+- classpath/embulk-filter-gsub-0.1.0.jar
+- classpath/kotlin-stdlib-1.2.10.jar
+homepage: https://github.com/cesare/embulk-filter-gsub
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Gsub filter plugin for Embulk
+test_files: []