embulk-filter-gsub 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fffdfbb5f26bf7c3ee0abf13c355b29080a09336
+  data.tar.gz: 96e001cd2b5162bd6bf50c7ec239e2134496ca9d
+SHA512:
+  metadata.gz: 7929b95e1cb3aa18b800305a835d385ccd6245fc6eb0421c667a40468936a587d4a8523e889d9c36eead27fa94109556b9ff7e361476016a9f68ab805d654d54
+  data.tar.gz: e0a1dce998d24320a9c188dd4cb0ea4189bbad9a148be600a2f09f0c084f9251c435208946bc28f6ae12fb244857ebb168da89cba829676eb1fe31a62ebdfb92

data/.circleci/config.yml

@@ -0,0 +1,42 @@
+# Java Gradle CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-java/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+      - image: circleci/openjdk:8-jdk
+      
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    environment:
+      # Customize the JVM maximum heap limit
+      JVM_OPTS: -Xmx3200m
+      TERM: dumb
+    
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "build.gradle" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run: gradle dependencies
+
+      - save_cache:
+          paths:
+            - ~/.gradle
+          key: v1-dependencies-{{ checksum "build.gradle" }}
+        
+      # run tests!
+      - run: gradle test

data/.gitignore

@@ -0,0 +1,14 @@
+*~
+/pkg/
+/tmp/
+/out/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project
+*.iml

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,141 @@
+# Gsub filter plugin for Embulk
+
+[![CircleCI](https://circleci.com/gh/cesare/embulk-filter-gsub.svg?style=svg)](https://circleci.com/gh/cesare/embulk-filter-gsub)
+
+Embulk filter plugin to convert text column values with regular expressions.
+
+## Overview
+
+* **Plugin type**: filter
+
+## Configuration
+
+- **target_columns**: columns to convert text value (array, default: `[]`)
+
+### Example
+
+```yaml
+filters:
+  - type: gsub
+    target_columns:
+      foo:
+        - type: regexp_replace
+          pattern: '(\w*):\s*(\w*)'
+          to: "$1 = [$2]"
+      bar:
+        - type: to_lower_case
+      baz:
+        - type: to_upper_case
+          pattern: "test"
+```
+
+### target column configuration
+
+- **type**: type of text substitution (string, default: `regexp_replace`)
+
+Supported Types are:
+
+* regexp_replace
+* to_lower_case
+* to_upper_case
+
+#### regexp_replace
+
+- **pattern**: regular expression pattern to be substituted (string, required)
+- **to**: replacement string (string, required)
+
+##### Example
+
+```yaml
+target_columns:
+  foo:
+    - type: regexp_replace
+      pattern: '(\w*):\s*(\w*)'
+      to: "$1 = [$2]"
+```
+
+it converts input like this
+
+foo | bar
+-----|-----
+example-foo: 1234 | example-bar: 9876
+
+into the output
+
+foo | bar
+-----|-----
+example-foo = [1234] | example-bar: 9876
+
+#### to_lower_case
+
+- **pattern**: regular expression pattern to be substituted (string, optional)
+
+If `pattern` is omitted, whole text is converted into lower case letters.
+
+##### Example
+
+```yaml
+target_columns:
+  foo:
+    - type: to_lower_case
+```
+
+it converts input like this
+
+foo | bar
+-----|-----
+ABC | XYZ
+
+into the output
+
+foo | bar
+-----|-----
+abc | XYZ
+
+#### to_upper_case
+
+- **pattern**: regular expression pattern to be substituted (string, optional)
+
+If `pattern` is omitted, whole text is converted into upper case letters.
+
+##### Example
+
+```yaml
+target_columns:
+  foo:
+    - type: to_upper_case
+```
+
+it converts input like this
+
+foo | bar
+-----|-----
+abc | xyz
+
+into the output
+
+foo | bar
+-----|-----
+ABC | xyz
+
+### Multiple conversion
+
+You can apply multiple conversion on a column value.
+
+```yaml
+target_columns:
+  foo:
+    - type: regexp_replace
+      pattern: '</?\w*\s*/?>'
+      to: ""
+    - type: regexp_replace
+      pattern: '(\w*):\s*(\w*)'
+      to: "$1 = [$2]"
+```
+
+
+## Build
+
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```

data/build.gradle

@@ -0,0 +1,49 @@
+buildscript {
+    ext.kotlin_version = '1.2.10'
+    repositories {
+        mavenCentral()
+        jcenter()
+        maven { url 'http://kamatama41.github.com/maven-repository/repository' }
+    }
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+        classpath "com.github.kamatama41:gradle-embulk-plugin:0.1.4"
+    }
+}
+
+apply plugin: "kotlin"
+apply plugin: "com.github.kamatama41.embulk"
+
+embulk {
+    version = "0.8.38"
+    category = "filter"
+    name = "gsub"
+    authors = ["Sawada Tadashi"]
+    email = "cesare@mayverse.jp"
+    homepage = "https://github.com/cesare/embulk-filter-gsub"
+}
+
+repositories {
+    mavenCentral()
+}
+
+dependencies {
+    compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    testCompile "junit:junit:4.12", {
+        transitive = false
+    }
+    testCompile "org.hamcrest:hamcrest-all:1.3"
+    testCompile "org.embulk:embulk-core:0.8.38:tests"
+}
+
+compileKotlin {
+    kotlinOptions {
+        jvmTarget = "1.6"  // only 1.6 and 1.8 are supported, we can't choose 1.7
+    }
+}
+
+compileTestKotlin {
+    kotlinOptions {
+        jvmTarget = "1.6"  // only 1.6 and 1.8 are supported, we can't choose 1.7
+    }
+}

data/gradle.properties

@@ -0,0 +1 @@
+version=0.1.0

data/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,6 @@
+#Sat Jan 06 13:38:53 JST 2018
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.4.1-all.zip

data/gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"