embulk-filter-insert 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 09934c336dd5ea3b9af1e9e52a7ba4f5647bac8d
+  data.tar.gz: afa263b7fbe96133f5149cf271443a0efbcbc60a
+SHA512:
+  metadata.gz: 69433687684b90978a8363801d9aded46144f805f7b88ca15bf4f0214408ce452892e9a3269e83e67d3b7b00c1bc22a37f3a0068ceb53b8d385a6fa2731b1d17
+  data.tar.gz: 4af757d250752ac14cf85cbcf563ccb245d308a5b57ab82f3b3f2495509e8e9d4d9df4d3abc58dce06d79392b196398a01cd9813d4df0e51cdfa3482eab239f6

data/.gitignore

@@ -0,0 +1,5 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,101 @@
+# Insert filter plugin for Embulk
+
+Embulk filter plugin that inserts column(s) at any position (e.g., the top/bottom of the columns, before/after the specified column name)
+
+## Overview
+
+* **Plugin type**: filter
+
+## Configuration
+
+### Column(s)
+
+Either "column" or "columns" is required for specifying inserted column(s).
+
+- **column**: associative array that contains only one key-value pair (key means a column name, value means a constant value in the column)
+- **columns**: List of the associative arrays
+
+The "column" associate array can contain following optional configuration.
+
+- **as**: type of the constant vaule in the column, i.e. boolean, long, double, string or timestamp (string, default: string)
+
+### Position
+
+Any of the following configurations is required for specifying a position where new columns are inserted.
+
+- **at**: "top", "head", "bottom", "tail" or index number where the new column(s) is/are inserted (string)
+- **before**: column name that comes before the new column(s) (string)
+- **after**: column name that comes after the new column(s) (string)
+
+If none of the configurations is specified, the new columns are inserted at the bottom of the existing columns.
+
+## Example
+
+Example 1: Insert "host_name" column at the top of the columns
+
+```yaml
+filters:
+  - { type: insert, column: { host_name: host01 }, at: top }
+```
+
+Example 2: Insert "host_name" column at the bottom of the columns
+
+```yaml
+filters:
+  - { type: insert, column: { host_name: host01 }, at: bottom }
+```
+
+Example 3: Insert "host_name" column after second column
+
+```yaml
+filters:
+  - { type: insert, column: { host_name: host01 }, at: 2 }
+```
+
+Example 4: Insert "service_name" column before "host_name" column
+
+```yaml
+filters:
+  - { type: insert, column: { service_name: service01 }, before: host_name }
+```
+
+Example 5: Insert "service_name" column after "host_name" column
+
+```yaml
+filters:
+  - { type: insert, column: { service_name: service01 }, after: host_name }
+```
+
+Example 6: Insert "user_id" column as integer at the bottom of the columns
+
+```yaml
+filters:
+  - { type: insert, column: { user_id: 1234567, as: long } }
+```
+
+Example 7: Insert multiple columns in a row at the bottom of the columns
+
+```yaml
+filters:
+  - type: insert
+    columns:
+      - host_name: host01
+      - service_name: service01
+```
+
+Example 8: Combination of the above examples
+
+```yaml
+filters:
+  - type: insert
+    columns:
+      - service_name: service01
+      - { user_id: 1234567, as: long }
+    after: host_name
+```
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-filter-insert.gemspec

@@ -0,0 +1,19 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-filter-insert"
+  spec.version       = "1.0.0"
+  spec.authors       = ["muziyoshiz"]
+  spec.summary       = "Embulk filter plugin that inserts column(s) at any position"
+  spec.description   = "Embulk filter plugin that inserts column(s) at any position (e.g., the top/bottom of the columns, before/after the specified column name)"
+  spec.email         = ["muziyoshiz@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/muziyoshiz/embulk-filter-insert"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/lib/embulk/filter/insert.rb

@@ -0,0 +1,144 @@
+module Embulk
+  module Filter
+
+    class InsertFilterPlugin < FilterPlugin
+      Plugin.register_filter("insert", self)
+
+      def self.transaction(config, in_schema, &control)
+        task = {}
+
+        column = config.param("column", :hash, default: nil)
+        columns = config.param("columns", :array, default: nil)
+        # ^ = XOR
+        unless (column.nil? ^ columns.nil?)
+          raise ArgumentError, "Either \"column\" or \"columns\" is needed"
+        end
+
+        if column
+          columns = [ InsertFilterPlugin.get_column(column) ]
+        else
+          columns = InsertFilterPlugin.get_columns(columns)
+        end
+
+        task["values"] = columns.map{|c| c[:value] }
+
+        at = config.param("at", :string, default: nil)
+        before = config.param("before", :string, default: nil)
+        after = config.param("after", :string, default: nil)
+
+        if at.nil? and before.nil? and after.nil?
+          at = "bottom"
+        end
+
+        no_of_position_param = 0
+        no_of_position_param += 1 unless at.nil?
+        no_of_position_param += 1 unless before.nil?
+        no_of_position_param += 1 unless after.nil?
+
+        unless no_of_position_param == 1
+          raise ArgumentError, "Either \"at\", \"before\" or \"after\" is needed"
+        end
+
+        if at
+          case at
+          when "top", "head"
+            task["position"] = 0
+          when "bottom", "tail"
+            task["position"] = in_schema.size
+          else
+            task["position"] = at.to_i
+          end
+        elsif before
+          schema_cols = in_schema.select{|c| c.name == before }
+          if schema_cols.empty?
+            raise ArgumentError, "Column #{before} is not found"
+          end
+          task["position"] = schema_cols[0].index
+        else
+          schema_cols = in_schema.select{|c| c.name == after }
+          if schema_cols.empty?
+            raise ArgumentError, "Column #{after} is not found"
+          end
+          task["position"] = schema_cols[0].index + 1
+        end
+
+        # modify column definition
+        inserted_schema = []
+        columns.each{|c| inserted_schema.push(Column.new(0, c[:name], c[:type])) }
+        out_columns = in_schema.map{|c| c }
+        out_columns.insert(task["position"], *inserted_schema)
+
+        # renumber index
+        out_columns.each_with_index{|c, idx| c.index = idx }
+
+        yield(task, out_columns)
+      end
+
+      # return { :name => name1, :value => value1, :type => type1 }
+      def self.get_column(column_hash)
+        if column_hash.size > 2
+          raise ArgumentError, "Invalid column parameter: #{column_hash.to_s}"
+        end
+
+        # default type is string
+        type = :string
+
+        if column_hash.size == 2
+          unless column_hash.keys.include?("as")
+            raise ArgumentError, "Invalid column parameter: #{column_hash.to_s}"
+          end
+          type = column_hash["as"].to_sym
+          column_hash = column_hash.select{|k, v| k != "as" }
+        end
+
+        column = {
+          :name => column_hash.keys.first,
+          :value => column_hash.values.first,
+          :type => type
+        }
+
+        case type
+        when :boolean
+          column[:value] = (column[:value] != "false")
+        when :long
+          column[:value] = column[:value].to_i
+        when :double
+          column[:value] = column[:value].to_f
+        when :string
+          # do nothing
+        when :timestamp
+          column[:value] = Date.parse(column[:value])
+        else
+          raise ArgumentError, "Unknown type #{type}: supported types are boolean, long, double, string and timestamp"
+        end
+
+        column
+      end
+
+      # return array of column
+      def self.get_columns(columns_array)
+        columns_array.map{|column_hash| InsertFilterPlugin.get_column(column_hash) }
+      end
+
+      def init
+        @values = task["values"]
+        @position = task["position"]
+      end
+
+      def close
+      end
+
+      def add(page)
+        page.each do |record|
+          record.insert(@position, *@values)
+          page_builder.add(record)
+        end
+      end
+
+      def finish
+        page_builder.finish
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: embulk-filter-insert
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- muziyoshiz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Embulk filter plugin that inserts column(s) at any position (e.g., the
+  top/bottom of the columns, before/after the specified column name)
+email:
+- muziyoshiz@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-filter-insert.gemspec
+- lib/embulk/filter/insert.rb
+homepage: https://github.com/muziyoshiz/embulk-filter-insert
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Embulk filter plugin that inserts column(s) at any position
+test_files: []