embulk-filter-gsub 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fffdfbb5f26bf7c3ee0abf13c355b29080a09336
-  data.tar.gz: 96e001cd2b5162bd6bf50c7ec239e2134496ca9d
+  metadata.gz: f2225445bc1080f16f10ccf3b514f4103f70cf39
+  data.tar.gz: 6cb469f097ee14aa27ced7847a9dc787eb0dc211
 SHA512:
-  metadata.gz: 7929b95e1cb3aa18b800305a835d385ccd6245fc6eb0421c667a40468936a587d4a8523e889d9c36eead27fa94109556b9ff7e361476016a9f68ab805d654d54
-  data.tar.gz: e0a1dce998d24320a9c188dd4cb0ea4189bbad9a148be600a2f09f0c084f9251c435208946bc28f6ae12fb244857ebb168da89cba829676eb1fe31a62ebdfb92
+  metadata.gz: 7e33ff204e96d633f242a975966b547df3353230a40c292131d2eb609e45494a63b2e3213c7874c2c1f7a389ab8adf26e2a88ca2d6de65e435cbde98d2c3830a
+  data.tar.gz: f9818862b57c4f7609ff3f153de35488de1098ea00b6a2a569a9acbdcf5f966ca89c8f7b6ecf6cfa0696b60e69c82e694b38879ab2800d323a44865c3fa84ee9

data/README.md

@@ -133,6 +133,26 @@ target_columns:
       to: "$1 = [$2]"
 ```
 
+### Regular expression options
+
+You can specify some regular expression options.
+
+```yaml
+target_columns:
+  foo:
+    - type: regexp_replace
+      pattern: 'foo'
+      to: "***"
+      regexp_options:
+        ignore_case: true
+```
+
+Supported options are:
+
+* **ignore_case** (boolean, default: false)
+* **multiline** (boolean, default: true)
+* **dot_matches_all** (boolean, default: false)
+* **enable_comments** (boolean, default: false)
 
 ## Build
 

data/build.gradle

@@ -1,5 +1,5 @@
 buildscript {
-    ext.kotlin_version = '1.2.10'
+    ext.kotlin_version = '1.2.21'
     repositories {
         mavenCentral()
         jcenter()
@@ -18,6 +18,7 @@ embulk {
     version = "0.8.38"
     category = "filter"
     name = "gsub"
+    description = "Embulk filter plugin to convert text column values with regular expressions"
     authors = ["Sawada Tadashi"]
     email = "cesare@mayverse.jp"
     homepage = "https://github.com/cesare/embulk-filter-gsub"

data/gradle.properties

@@ -1 +1 @@
-version=0.1.0
+version=0.2.0

data/src/main/kotlin/org/embulk/filter/gsub/LowerCaseReplacerFactory.kt

@@ -1,7 +1,6 @@
 package org.embulk.filter.gsub
 
 import org.embulk.filter.gsub.replacer.LowerCaseReplacer
-import org.embulk.filter.gsub.replacer.RegexFactory
 import org.embulk.filter.gsub.replacer.RegexOptionConfig
 import org.embulk.filter.gsub.replacer.TextReplacer
 
@@ -9,10 +8,8 @@ class LowerCaseReplacerFactory : TextReplacerFactory() {
     override fun create(rule: SubstitutionRule): TextReplacer {
         val pattern = rule.pattern.orNull()
         if (pattern != null) {
-            // TODO set regex options
-            val regexOptionConfig = RegexOptionConfig()
             val factory = RegexFactory()
-            val regex = factory.create(pattern, regexOptionConfig)
+            val regex = factory.create(pattern, rule.regexOptions)
 
             return LowerCaseReplacer(regex)
         }

data/src/main/kotlin/org/embulk/filter/gsub/RegexFactory.kt

@@ -0,0 +1,32 @@
+package org.embulk.filter.gsub
+
+import org.embulk.filter.gsub.RegexOptions
+
+class RegexFactory {
+    fun create(patternString: String, regexOptions: RegexOptions): Regex {
+        val options = buildOptions(regexOptions)
+        return Regex(patternString, options)
+    }
+
+    private fun buildOptions(regexOptions: RegexOptions): Set<RegexOption> {
+        val options = HashSet<RegexOption>()
+
+        if (regexOptions.ignoreCase) {
+            options.add(RegexOption.IGNORE_CASE)
+        }
+
+        if (regexOptions.multiline) {
+            options.add(RegexOption.MULTILINE)
+        }
+
+        if (regexOptions.dotMatchesAll) {
+            options.add(RegexOption.DOT_MATCHES_ALL)
+        }
+
+        if (regexOptions.enableComments) {
+            options.add(RegexOption.COMMENTS)
+        }
+
+        return options
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/RegexOptions.kt

@@ -0,0 +1,23 @@
+package org.embulk.filter.gsub
+
+import org.embulk.config.Config
+import org.embulk.config.ConfigDefault
+import org.embulk.config.Task
+
+interface RegexOptions : Task {
+    @get:Config("ignore_case")
+    @get:ConfigDefault("false")
+    val ignoreCase: Boolean
+
+    @get:Config("multiline")
+    @get:ConfigDefault("true")
+    val multiline: Boolean
+
+    @get:Config("dot_matches_all")
+    @get:ConfigDefault("false")
+    val dotMatchesAll: Boolean
+
+    @get:Config("enable_comments")
+    @get:ConfigDefault("false")
+    val enableComments: Boolean
+}

data/src/main/kotlin/org/embulk/filter/gsub/RegexReplacerFactory.kt

@@ -1,6 +1,5 @@
 package org.embulk.filter.gsub
 
-import org.embulk.filter.gsub.replacer.RegexFactory
 import org.embulk.filter.gsub.replacer.RegexOptionConfig
 import org.embulk.filter.gsub.replacer.RegexReplacer
 import org.embulk.filter.gsub.replacer.TextReplacer
@@ -10,10 +9,8 @@ class RegexReplacerFactory : TextReplacerFactory() {
         val pattern = rule.pattern.get()
         val to = rule.to.get()
 
-        val regexOptionConfig = RegexOptionConfig()
-
         val factory = RegexFactory()
-        val regex = factory.create(pattern, regexOptionConfig)
+        val regex = factory.create(pattern, rule.regexOptions)
 
         return RegexReplacer(regex, to)
     }

data/src/main/kotlin/org/embulk/filter/gsub/SubstitutionRule.kt

@@ -23,4 +23,8 @@ interface SubstitutionRule : Task {
     @get:Config("to")
     @get:ConfigDefault("null")
     val to: Optional<String>
+
+    @get:Config("regexp_options")
+    @get:ConfigDefault("{}")
+    val regexOptions: RegexOptions
 }

data/src/main/kotlin/org/embulk/filter/gsub/UpperCaseReplacerFactory.kt

@@ -1,6 +1,5 @@
 package org.embulk.filter.gsub
 
-import org.embulk.filter.gsub.replacer.RegexFactory
 import org.embulk.filter.gsub.replacer.RegexOptionConfig
 import org.embulk.filter.gsub.replacer.TextReplacer
 import org.embulk.filter.gsub.replacer.UpperCaseReplacer
@@ -9,10 +8,8 @@ class UpperCaseReplacerFactory : TextReplacerFactory() {
     override fun create(rule: SubstitutionRule): TextReplacer {
         val pattern = rule.pattern.orNull()
         if (pattern != null) {
-            // TODO set regex options
-            val regexOptionConfig = RegexOptionConfig()
             val factory = RegexFactory()
-            val regex = factory.create(pattern, regexOptionConfig)
+            val regex = factory.create(pattern, rule.regexOptions)
 
             return UpperCaseReplacer(regex)
         }

data/src/test/kotlin/org/embulk/filter/gsub/TestGsubFilterPlugin.kt

@@ -57,6 +57,54 @@ class TestGsubFilterPlugin {
         Assert.assertEquals("\\1 [\\2]", barRule2.to.get())
     }
 
+    @Test
+    fun testDefaultRegexOptions() {
+        val configYaml = """
+        |type: gsub
+        |target_columns:
+        |  foo:
+        |    - type: regexp_replace
+        |      pattern: "test"
+        """.trimMargin()
+
+        val config = getConfigFromYaml(configYaml)
+        val task = config.loadConfig(GsubFilterPlugin.PluginTask::class.java)
+        val fooRules = task.targetColumns["foo"]!!
+        val fooRule = fooRules[0]
+        val regexOptions = fooRule.regexOptions
+        Assert.assertFalse(regexOptions.ignoreCase)
+        Assert.assertTrue(regexOptions.multiline)
+        Assert.assertFalse(regexOptions.dotMatchesAll)
+        Assert.assertFalse(regexOptions.enableComments)
+    }
+
+    @Test
+    fun testRegexOptions() {
+        val configYaml = """
+        |type: gsub
+        |target_columns:
+        |  foo:
+        |    - type: regexp_replace
+        |      pattern: "test"
+        |      regexp_options:
+        |        ignore_case: true
+        |        multiline: true
+        |        dot_matches_all: true
+        |        enable_comments: true
+        """.trimMargin()
+
+        val config = getConfigFromYaml(configYaml)
+        val task = config.loadConfig(GsubFilterPlugin.PluginTask::class.java)
+        val fooRules = task.targetColumns["foo"]!!
+        val fooRule = fooRules[0]
+        val regexOptions = fooRule.regexOptions
+
+        Assert.assertTrue(regexOptions.ignoreCase)
+        Assert.assertTrue(regexOptions.multiline)
+        Assert.assertTrue(regexOptions.dotMatchesAll)
+        Assert.assertTrue(regexOptions.enableComments)
+    }
+
     @Test
     fun testEmptyFilter() {
         val configYaml = """

data/src/test/kotlin/org/embulk/filter/gsub/replacer/RegexReplacerTest.kt

@@ -1,19 +1,134 @@
 package org.embulk.filter.gsub.replacer
 
+import org.embulk.config.TaskSource
+import org.embulk.filter.gsub.RegexFactory
+import org.embulk.filter.gsub.RegexOptions
 import org.junit.Assert
 import org.junit.Test
 
 class RegexReplacerTest {
     @Test
     fun testExecute() {
-        val optionConfig = RegexOptionConfig()
-        optionConfig.ignoreCase = true
+        val regexOptions = createRegexpOption()
 
         val factory = RegexFactory()
-        val pattern = factory.create("(\\w*):\\s*(.*)", optionConfig)
+        val pattern = factory.create("(\\w*):\\s*(.*)", regexOptions)
 
         val replacer = RegexReplacer(pattern, "$1 [$2]")
         val result = replacer.execute("test: foo bar baz")
         Assert.assertEquals("test [foo bar baz]", result)
     }
+
+    @Test
+    fun testExecuteWithoutIgnoreCaseOption() {
+        val regexOptions = createRegexpOption(ignoreCase = false)
+
+        val factory = RegexFactory()
+        val pattern = factory.create("foo", regexOptions)
+
+        val replacer = RegexReplacer(pattern, "*test-foo*")
+
+        Assert.assertEquals("*test-foo* bar baz", replacer.execute("foo bar baz"))
+        Assert.assertEquals("Foo bar baz", replacer.execute("Foo bar baz"))
+        Assert.assertEquals("FOO bar baz", replacer.execute("FOO bar baz"))
+    }
+
+    @Test
+    fun testExecuteWithIgnoreCaseOption() {
+        val regexOptions = createRegexpOption(ignoreCase = true)
+        val factory = RegexFactory()
+        val pattern = factory.create("foo", regexOptions)
+
+        val replacer = RegexReplacer(pattern, "*test-foo*")
+
+        Assert.assertEquals("*test-foo* bar baz", replacer.execute("foo bar baz"))
+        Assert.assertEquals("*test-foo* bar baz", replacer.execute("Foo bar baz"))
+        Assert.assertEquals("*test-foo* bar baz", replacer.execute("FOO bar baz"))
+    }
+
+    @Test
+    fun testExecuteWithoutMultilineOption() {
+        val regexOptions = createRegexpOption(multiline = false)
+        val factory = RegexFactory()
+        val pattern = factory.create("^bar", regexOptions)
+
+        val replacer = RegexReplacer(pattern, "*BAR*")
+
+        Assert.assertEquals("foo\nbar\nbaz", replacer.execute("foo\nbar\nbaz"))
+    }
+
+    @Test
+    fun testExecuteWithMultilineOption() {
+        val regexOptions = createRegexpOption(multiline = true)
+        val factory = RegexFactory()
+        val pattern = factory.create("^bar", regexOptions)
+
+        val replacer = RegexReplacer(pattern, "*BAR*")
+
+        Assert.assertEquals("foo\n*BAR*\nbaz", replacer.execute("foo\nbar\nbaz"))
+    }
+
+    @Test
+    fun testExecuteWithoutDotMatchesAllOption() {
+        val regexOptions = createRegexpOption(dotMatchesAll = false)
+        val factory = RegexFactory()
+        val pattern = factory.create("foo.bar.baz", regexOptions)
+
+        val replacer = RegexReplacer(pattern, "[foo-bar-baz]")
+
+        Assert.assertEquals("[foo-bar-baz]", replacer.execute("foo/bar/baz"))
+        Assert.assertEquals("foo\nbar/baz", replacer.execute("foo\nbar/baz"))
+    }
+
+    @Test
+    fun testExecuteWithDotMatchesAllOption() {
+        val regexOptions = createRegexpOption(dotMatchesAll = true)
+        val factory = RegexFactory()
+        val pattern = factory.create("foo.bar.baz", regexOptions)
+
+        val replacer = RegexReplacer(pattern, "[foo-bar-baz]")
+
+        Assert.assertEquals("[foo-bar-baz]", replacer.execute("foo/bar/baz"))
+        Assert.assertEquals("[foo-bar-baz]", replacer.execute("foo\nbar/baz"))
+    }
+
+    @Test
+    fun testExecuteWithEnableCommentsOption() {
+        val regexOptions = createRegexpOption(enableComments = true)
+        val factory = RegexFactory()
+
+        val patternString = """
+            |(ba\w)    # matches bar and baz
+            """.trimMargin()
+        val pattern = factory.create(patternString, regexOptions)
+
+        val replacer = RegexReplacer(pattern, "*$1*")
+
+        Assert.assertEquals("foo *bar* *baz*", replacer.execute("foo bar baz"))
+    }
+
+    private fun createRegexpOption(
+            ignoreCase:     Boolean = false,
+            multiline:      Boolean = true,
+            dotMatchesAll:  Boolean = false,
+            enableComments: Boolean = false
+    ): RegexOptions {
+        return object: RegexOptions {
+            override val ignoreCase: Boolean
+                get() = ignoreCase
+            override val multiline: Boolean
+                get() = multiline
+            override val dotMatchesAll: Boolean
+                get() = dotMatchesAll
+            override val enableComments: Boolean
+                get() = enableComments
+
+            override fun validate() {
+            }
+
+            override fun dump(): TaskSource {
+                throw NotImplementedError()
+            }
+        }
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-gsub
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sawada Tadashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-21 00:00:00.000000000 Z
+date: 2018-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-description: gsub
+description: Embulk filter plugin to convert text column values with regular expressions
 email:
 - cesare@mayverse.jp
 executables: []
@@ -60,13 +60,14 @@ files:
 - src/main/kotlin/org/embulk/filter/gsub/ColumnVisitorImpl.kt
 - src/main/kotlin/org/embulk/filter/gsub/GsubFilterPlugin.kt
 - src/main/kotlin/org/embulk/filter/gsub/LowerCaseReplacerFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/RegexFactory.kt
+- src/main/kotlin/org/embulk/filter/gsub/RegexOptions.kt
 - src/main/kotlin/org/embulk/filter/gsub/RegexReplacerFactory.kt
 - src/main/kotlin/org/embulk/filter/gsub/SubstitutionRule.kt
 - src/main/kotlin/org/embulk/filter/gsub/TextReplacerFactory.kt
 - src/main/kotlin/org/embulk/filter/gsub/UpperCaseReplacerFactory.kt
 - src/main/kotlin/org/embulk/filter/gsub/replacer/CombinedReplacer.kt
 - src/main/kotlin/org/embulk/filter/gsub/replacer/LowerCaseReplacer.kt
-- src/main/kotlin/org/embulk/filter/gsub/replacer/RegexFactory.kt
 - src/main/kotlin/org/embulk/filter/gsub/replacer/RegexOptionConfig.kt
 - src/main/kotlin/org/embulk/filter/gsub/replacer/RegexReplacer.kt
 - src/main/kotlin/org/embulk/filter/gsub/replacer/TextReplacer.kt
@@ -76,8 +77,8 @@ files:
 - src/test/kotlin/org/embulk/filter/gsub/replacer/LowerCaseReplacerTest.kt
 - src/test/kotlin/org/embulk/filter/gsub/replacer/RegexReplacerTest.kt
 - classpath/annotations-13.0.jar
-- classpath/embulk-filter-gsub-0.1.0.jar
-- classpath/kotlin-stdlib-1.2.10.jar
+- classpath/embulk-filter-gsub-0.2.0.jar
+- classpath/kotlin-stdlib-1.2.21.jar
 homepage: https://github.com/cesare/embulk-filter-gsub
 licenses:
 - MIT

data/src/main/kotlin/org/embulk/filter/gsub/replacer/RegexFactory.kt

@@ -1,22 +0,0 @@
-package org.embulk.filter.gsub.replacer
-
-class RegexFactory {
-    fun create(patternString: String, regexOptionConfig: RegexOptionConfig): Regex {
-        val options = buildOptions(regexOptionConfig)
-        return Regex(patternString, options)
-    }
-
-    private fun buildOptions(optionConfig: RegexOptionConfig): Set<RegexOption> {
-        val options = HashSet<RegexOption>()
-
-        if (optionConfig.ignoreCase) {
-            options.add(RegexOption.IGNORE_CASE)
-        }
-
-        if (optionConfig.multiline) {
-            options.add(RegexOption.MULTILINE)
-        }
-
-        return options
-    }
-}