embulk-filter-gsub 0.1.0

data/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/filter/gsub.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_filter(
+  "gsub", "org.embulk.filter.gsub.GsubFilterPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/kotlin/org/embulk/filter/gsub/ColumnReplacerFactory.kt

@@ -0,0 +1,20 @@
+package org.embulk.filter.gsub
+
+import org.embulk.filter.gsub.replacer.*
+
+class ColumnReplacerFactory {
+    fun create(task: GsubFilterPlugin.PluginTask): Map<String, TextReplacer> {
+        return task.targetColumns.mapValues { column ->
+            createReplacerForRules(column.value)
+        }
+    }
+
+    private fun createReplacerForRules(rules: List<SubstitutionRule>): TextReplacer {
+        val replacers = rules.map { createReplacer(it) }
+        return CombinedReplacer(replacers)
+    }
+
+    private fun createReplacer(rule: SubstitutionRule): TextReplacer {
+        return TextReplacerFactory.create(rule)
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/ColumnVisitorImpl.kt

@@ -0,0 +1,72 @@
+package org.embulk.filter.gsub
+
+import org.embulk.filter.gsub.GsubFilterPlugin.PluginTask
+import org.embulk.filter.gsub.replacer.TextReplacer
+import org.embulk.spi.*
+
+class ColumnVisitorImpl
+    constructor(
+        task: PluginTask,
+        private val pageReader: PageReader,
+        private val pageBuilder: PageBuilder
+    ) : ColumnVisitor {
+
+    private val columnReplacers: Map<String, TextReplacer> = ColumnReplacerFactory().create(task)
+
+    override fun booleanColumn(column: Column) {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column)
+        }
+        else {
+            pageBuilder.setBoolean(column, pageReader.getBoolean(column))
+        }
+    }
+
+    override fun longColumn(column: Column) {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column)
+        }
+        else {
+            pageBuilder.setLong(column, pageReader.getLong(column))
+        }
+    }
+
+    override fun doubleColumn(column: Column) {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column)
+        }
+        else {
+            pageBuilder.setDouble(column, pageReader.getDouble(column))
+        }
+    }
+
+    override fun stringColumn(column: Column) {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column)
+        }
+        else {
+            val text = pageReader.getString(column)
+            val replacer = columnReplacers[column.name]
+            val replacedText = replacer?.let { it.execute(text) } ?: text
+            pageBuilder.setString(column, replacedText)
+        }
+    }
+
+    override fun timestampColumn(column: Column) {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column)
+        }
+        else {
+            pageBuilder.setTimestamp(column, pageReader.getTimestamp(column))
+        }
+    }
+
+    override fun jsonColumn(column: Column) {
+        if (pageReader.isNull(column)) {
+            pageBuilder.setNull(column)
+        }
+        else {
+            pageBuilder.setJson(column, pageReader.getJson(column))
+        }
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/GsubFilterPlugin.kt

@@ -0,0 +1,51 @@
+package org.embulk.filter.gsub
+
+import org.embulk.config.Config
+import org.embulk.config.ConfigDefault
+import org.embulk.config.ConfigSource
+import org.embulk.config.Task
+import org.embulk.config.TaskSource
+import org.embulk.spi.*
+
+class GsubFilterPlugin : FilterPlugin {
+    interface PluginTask : Task {
+        @get:Config("target_columns")
+        @get:ConfigDefault("{}")
+        val targetColumns: Map<String, List<SubstitutionRule>>
+    }
+
+    override fun transaction(config: ConfigSource, inputSchema: Schema,
+                             control: FilterPlugin.Control) {
+        val task = config.loadConfig<PluginTask>(PluginTask::class.java)
+
+        control.run(task.dump(), inputSchema)
+    }
+
+    override fun open(taskSource: TaskSource, inputSchema: Schema,
+                      outputSchema: Schema, output: PageOutput): PageOutput {
+        val task = taskSource.loadTask<PluginTask>(PluginTask::class.java)
+
+        return object: PageOutput {
+            val pageReader = PageReader(inputSchema)
+            val pageBuilder = PageBuilder(Exec.getBufferAllocator(), outputSchema, output)
+            val columnVisitor = ColumnVisitorImpl(task, pageReader, pageBuilder)
+
+            override fun add(page: Page) {
+                pageReader.setPage(page)
+
+                while (pageReader.nextRecord()) {
+                    inputSchema.visitColumns(columnVisitor)
+                    pageBuilder.addRecord()
+                }
+            }
+
+            override fun finish() {
+                pageBuilder.finish()
+            }
+
+            override fun close() {
+                pageBuilder.close()
+            }
+        }
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/LowerCaseReplacerFactory.kt

@@ -0,0 +1,23 @@
+package org.embulk.filter.gsub
+
+import org.embulk.filter.gsub.replacer.LowerCaseReplacer
+import org.embulk.filter.gsub.replacer.RegexFactory
+import org.embulk.filter.gsub.replacer.RegexOptionConfig
+import org.embulk.filter.gsub.replacer.TextReplacer
+
+class LowerCaseReplacerFactory : TextReplacerFactory() {
+    override fun create(rule: SubstitutionRule): TextReplacer {
+        val pattern = rule.pattern.orNull()
+        if (pattern != null) {
+            // TODO set regex options
+            val regexOptionConfig = RegexOptionConfig()
+            val factory = RegexFactory()
+            val regex = factory.create(pattern, regexOptionConfig)
+
+            return LowerCaseReplacer(regex)
+        }
+        else {
+            return LowerCaseReplacer()
+        }
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/RegexReplacerFactory.kt

@@ -0,0 +1,20 @@
+package org.embulk.filter.gsub
+
+import org.embulk.filter.gsub.replacer.RegexFactory
+import org.embulk.filter.gsub.replacer.RegexOptionConfig
+import org.embulk.filter.gsub.replacer.RegexReplacer
+import org.embulk.filter.gsub.replacer.TextReplacer
+
+class RegexReplacerFactory : TextReplacerFactory() {
+    override fun create(rule: SubstitutionRule): TextReplacer {
+        val pattern = rule.pattern.get()
+        val to = rule.to.get()
+
+        val regexOptionConfig = RegexOptionConfig()
+
+        val factory = RegexFactory()
+        val regex = factory.create(pattern, regexOptionConfig)
+
+        return RegexReplacer(regex, to)
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/SubstitutionRule.kt

@@ -0,0 +1,26 @@
+package org.embulk.filter.gsub
+
+import com.google.common.base.Optional
+import org.embulk.config.Config
+import org.embulk.config.ConfigDefault
+import org.embulk.config.Task
+
+interface SubstitutionRule : Task {
+    enum class SubstitutionType(val label: String) {
+        REGEXP_REPLACE("regexp_replace"),
+        TO_UPPER_CASE("to_upper_case"),
+        TO_LOWER_CASE("to_lower_case"),
+    }
+
+    @get:Config("type")
+    @get:ConfigDefault("\"regexp_replace\"")
+    val type: String
+
+    @get:Config("pattern")
+    @get:ConfigDefault("null")
+    val pattern: Optional<String>
+
+    @get:Config("to")
+    @get:ConfigDefault("null")
+    val to: Optional<String>
+}

data/src/main/kotlin/org/embulk/filter/gsub/TextReplacerFactory.kt

@@ -0,0 +1,33 @@
+package org.embulk.filter.gsub
+
+import org.embulk.filter.gsub.replacer.TextReplacer
+import org.embulk.filter.gsub.SubstitutionRule.SubstitutionType
+
+abstract class TextReplacerFactory {
+    abstract fun create(rule: SubstitutionRule): TextReplacer
+
+    companion object Factory {
+        fun create(rule: SubstitutionRule): TextReplacer {
+            val factory = createFactory(rule)
+            return factory.create(rule)
+        }
+
+        private fun createFactory(rule: SubstitutionRule): TextReplacerFactory {
+            val type = findSubstitutionType(rule.type)
+            return when (type) {
+                SubstitutionType.REGEXP_REPLACE -> RegexReplacerFactory()
+                SubstitutionType.TO_UPPER_CASE -> UpperCaseReplacerFactory()
+                SubstitutionType.TO_LOWER_CASE -> LowerCaseReplacerFactory()
+            }
+        }
+
+        private fun findSubstitutionType(typeName: String): SubstitutionType {
+            try {
+                return SubstitutionType.valueOf(typeName.toUpperCase())
+            }
+            catch (e: IllegalArgumentException) {
+                throw RuntimeException("Unknown substitution type: ${typeName}", e)
+            }
+        }
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/UpperCaseReplacerFactory.kt

@@ -0,0 +1,23 @@
+package org.embulk.filter.gsub
+
+import org.embulk.filter.gsub.replacer.RegexFactory
+import org.embulk.filter.gsub.replacer.RegexOptionConfig
+import org.embulk.filter.gsub.replacer.TextReplacer
+import org.embulk.filter.gsub.replacer.UpperCaseReplacer
+
+class UpperCaseReplacerFactory : TextReplacerFactory() {
+    override fun create(rule: SubstitutionRule): TextReplacer {
+        val pattern = rule.pattern.orNull()
+        if (pattern != null) {
+            // TODO set regex options
+            val regexOptionConfig = RegexOptionConfig()
+            val factory = RegexFactory()
+            val regex = factory.create(pattern, regexOptionConfig)
+
+            return UpperCaseReplacer(regex)
+        }
+        else {
+            return UpperCaseReplacer()
+        }
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/replacer/CombinedReplacer.kt

@@ -0,0 +1,7 @@
+package org.embulk.filter.gsub.replacer
+
+class CombinedReplacer(private val replacers: List<TextReplacer>) : TextReplacer {
+    override fun execute(text: String): String {
+        return replacers.fold(text, { str, replacer -> replacer.execute(str) })
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/replacer/LowerCaseReplacer.kt

@@ -0,0 +1,17 @@
+package org.embulk.filter.gsub.replacer
+
+class LowerCaseReplacer(private val pattern: Regex?) : TextReplacer {
+    constructor() : this(null)
+
+    override fun execute(text: String): String {
+        return pattern?.let { replaceWithPattern(it, text)} ?: replaceWholeText(text)
+    }
+
+    private fun replaceWithPattern(pattern: Regex, text: String): String {
+        return pattern.replace(text, { matchResult -> matchResult.value.toLowerCase() })
+    }
+
+    private fun replaceWholeText(text: String): String {
+        return text.toLowerCase()
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/replacer/RegexFactory.kt

@@ -0,0 +1,22 @@
+package org.embulk.filter.gsub.replacer
+
+class RegexFactory {
+    fun create(patternString: String, regexOptionConfig: RegexOptionConfig): Regex {
+        val options = buildOptions(regexOptionConfig)
+        return Regex(patternString, options)
+    }
+
+    private fun buildOptions(optionConfig: RegexOptionConfig): Set<RegexOption> {
+        val options = HashSet<RegexOption>()
+
+        if (optionConfig.ignoreCase) {
+            options.add(RegexOption.IGNORE_CASE)
+        }
+
+        if (optionConfig.multiline) {
+            options.add(RegexOption.MULTILINE)
+        }
+
+        return options
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/replacer/RegexOptionConfig.kt

@@ -0,0 +1,6 @@
+package org.embulk.filter.gsub.replacer
+
+class RegexOptionConfig {
+    var ignoreCase: Boolean = false
+    var multiline: Boolean = false
+}

data/src/main/kotlin/org/embulk/filter/gsub/replacer/RegexReplacer.kt

@@ -0,0 +1,7 @@
+package org.embulk.filter.gsub.replacer
+
+class RegexReplacer(private val pattern: Regex, private val to: String) : TextReplacer {
+    override fun execute(text: String): String {
+        return pattern.replace(text, to)
+    }
+}

data/src/main/kotlin/org/embulk/filter/gsub/replacer/TextReplacer.kt

@@ -0,0 +1,5 @@
+package org.embulk.filter.gsub.replacer
+
+interface TextReplacer {
+    fun execute(text: String): String
+}