embulk-filter-postgres_lookup 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: d6741ec605cc98b03bd51993021bf8a0d9dcecc828ce90f0af88617c7d2ffd80
+  data.tar.gz: b8e5cabee3f41285f0f112e5b22fadd600cf0945482c1069648e02ca3eb6ba51
+SHA512:
+  metadata.gz: 7a7ab485b7c312547d89f3fb92b4fdd9ef1d6e1c631fe4a2f31f65373f8974bf3312e2e033630014ed5ccaf8563348b18c1075cc16b8891c321f72ee58d7b986
+  data.tar.gz: 206c090fc9e79c89c287cda83a284c02b65edcabb8a75c4c032252b2638a386d78932e26a7daa5c549906a3de488b371615b4704b028dbb3adb294f641f519f5

data/.gitignore

@@ -0,0 +1,12 @@
+*~
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2022 InfoObjects
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,134 @@
+# Postgres lookup filter plugin for Embulk
+​
+An Embulk filter plugin for Lookup Transformation with Postgres database
+​
+## Configuration
+​
+- **postgres_lookup**: Required attributes for the LookUp Filter Plugin -
+    - **host**: database host (example `localhost`) (required)
+    - **port**: database port (example port for postgres `1433`) (required)
+    - **database**: database name (required)
+    - **tablename**: table name of your database (required)
+    - **username**: username for your database (required)
+    - **password**: password for database (required)
+    - **mapping_from**: (Name of columns to be matched with table 2 columns) (required)
+        - **Name of column-1**: column name-1 from input file
+        - **Name of column-2**: column name-2 from input file etc ...
+    - **mapping_to**:   (Name of columns to be matched with table 1 columns) (required)
+        - **Name of column-1**: column name-1 from input file
+        - **Name of column-2**: column name-2 from input file
+    - **new_columns**:   (New generated column names) (required)
+        - **Name-1,Type-1**: Any Name, Type of the name (name: country_name, type: string)
+        - **Name-2,Type-2**: Any Name, Type of the name (name: country_address, type: string) etc ...
+## Example - columns
+​
+Input1 for table 1 is as follows :-
+​
+```
+    year               country_code                 country_name            literacy_rate
+    
+    1990                    1                          India                       80%
+    1993                    2                           USA                        83%
+    1997                    3                          JAPAN                        
+    1999                    4                          China                       72%
+    2000                    5                         Ukraine                      68%
+    2002                    6                          Italy                       79%
+    2004                    7                            UK                        75%
+    2011                    8                           NULL                       42%
+```
+​
+Input2 for table 2 is as follows :-
+​
+```
+    id               country_population                        country_address               country_GDP
+    
+    1                       11.3                                    India                       1.67
+    2                       18.2                                     USA                        16.72
+    3                       30                                      JAPAN                       5.00
+    4                       4                                       China                       9.33
+    5                       57                                     Ukraine                      1.08
+    6                       63                                      Italy                       2.068
+    7                       17                                       UK                         2.49
+    8                       28                                       UAE                        1.18                            
+    
+    
+    Note: country_population is calculated in Billion and country_GDP is calculated in $USD Trillion
+```
+​
+As shown in yaml below, columns mentioned in mapping_from will be mapped with columns mentioned in mapping_to      
+ie:
+
+​
+country_code : id                       
+country_name : country_address
+
+After successful mapping an Output.csv file containing the columns mentioned in new_columns will be generated              
+​
+​
+
+Output File generated :-
+​
+```
+    year               country_code                 country_name              literacy_rate                 country_GDP                   country_population
+    
+    1990                    1                          India                       80%                         1.67                                11.3
+    1993                    2                           USA                        83%                         16.72                               18.2
+    1997                    3                          JAPAN                                                   5.00                                30
+    1999                    4                          China                       72%                         9.33                                4
+    2000                    5                         Ukraine                      68%                         1.08                                57
+    2002                    6                          Italy                       79%                         2.068                               63
+    2004                    7                            UK                        75%                         2.49                                17
+    2011                    8                           NULL                       42%                                                         
+```
+​
+​
+​
+```yaml
+- type: postgress_lookup
+    lookup_destination: mysql
+    host: localhost
+    port: 5432
+    database: Abhishek
+    tablename: country
+    username: postgres
+    password: root
+    mapping_from:
+      - quarter_number
+      - attr_1
+    mapping_to:
+      - id
+      - country_address
+    new_columns:
+      - { name: country_name, type: string }
+      - { name: country_address, type: string }
+```
+​
+Notes:
+1. mapping_from attribute should be in same order as mentioned in input file.
+   ​
+## Development
+​
+Run example:
+​
+```
+$ ./gradlew package
+$ embulk run -I ./lib seed.yml
+```
+​
+Deployment Steps:
+​
+```
+Install ruby in your machine
+$ gem install gemcutter (For windows OS)
+​
+$ ./gradlew gemPush
+$ gem build NameOfYourPlugins (example: embulk-filter-postgress_lookup)
+$ gem push embulk-filter-postgress_lookup-0.1.0.gem (You will get this name after running above command)
+```
+​
+​
+Release gem:
+​
+```
+$ ./gradlew gemPush
+```

data/build.gradle

@@ -0,0 +1,98 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "1.5.0"
+    id "java"
+    id "checkstyle"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+
+version = "0.1.0"
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+dependencies {
+    compile  "org.embulk:embulk-core:0.9.8"
+    provided "org.embulk:embulk-core:0.9.8"
+    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+    testCompile "junit:junit:4.+"
+    compile group: 'mysql', name: 'mysql-connector-java', version: '8.0.19'
+    compile group: 'postgresql', name: 'postgresql', version: '9.0-801.jdbc4'
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "build", "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
+}
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Infoobjects Inc."]
+  spec.summary       = %[Postgress Lookup filter plugin for Embulk]
+  spec.description   = %[Postgress Lookup]
+  spec.email         = ["utkarsh@infoobjects.com"]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/InfoObjects/embulk-filter-postgres_lookup"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['~> 12.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }

data/calendar.csv

@@ -0,0 +1,10 @@
+dim_calendar_key,year_number,quarter_number,attr_1
+-1,0,0
+19900101,1990,1,FirstIndia
+19900102,1990,2,SecondUSA
+19900103,1990,,1000.56
+19900104,1990,4,40_China
+19900105,1990,5,50_Ukraine
+19900106,1990,2,10_USA
+19900107,1990,3,30_UK
+19900109,1990,6,

data/config/checkstyle/checkstyle.xml

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/config/checkstyle/default.xml

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/example/config.yml

@@ -0,0 +1,63 @@
+exec:
+  max_threads: 2
+  min_output_tasks: 1
+in:
+  type: file
+  path_prefix: C:\Users\Abhishek Gupta\Desktop\new one\embulk-filter-postgress_lookup\calendar.csv
+  parser:
+    type: csv
+    columns:
+      - { name: dim_calendar_key, type: long }
+      - { name: year_number, type: long }
+      - { name: quarter_number, type: long }
+      - { name: attr_1, type: string }
+filters:
+  - type: postgress_lookup
+    lookup_destination: mysql
+    host: localhost
+    port: 5432
+    database: Abhishek
+    tablename: country
+    username: postgres
+    password: root
+    mapping_from:
+      - quarter_number
+      - attr_1
+    mapping_to:
+      - id
+      - country_address
+    new_columns:
+      - { name: country_name, type: string }
+      - { name: country_address, type: string }
+  - type: postgress_lookup
+    lookup_destination: mysql
+    host: localhost
+    port: 5432
+    database: Abhishek
+    tablename: country
+    username: postgres
+    password: root
+    mapping_from:
+      - quarter_number
+      - attr_1
+    mapping_to:
+      - id
+      - country_address
+    new_columns:
+      - { name: country_name, type: string }
+      - { name: country_address, type: string }
+out:
+  type: file
+  path_prefix: C:\Users\Abhishek Gupta\Desktop\B\output.csv
+  file_ext: csv
+  formatter:
+      type: csv
+      delimiter: "\t"
+      newline: CRLF
+      newline_in_field: LF
+      charset: UTF-8
+      quote_policy: MINIMAL
+      quote: '"'
+      escape: "\\"
+      null_string: "\\N"
+      default_timezone: 'UTC'

data/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip