embulk-output-influxdb08 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 05de8c09ef40e9b5d7b4d55fea895865ec094d6d
+  data.tar.gz: 52bb9d1d246c5438ecc06ec0435c1b0c44ef7f21
+SHA512:
+  metadata.gz: 03a925fca8c2e5bed91e18596ece84cc0465ff8582568df19cfc520a4038510bda1f3fdeba17458b09adc33b56988a3b1edb315c99a0e60fce344606efac1b72
+  data.tar.gz: 9ff0731225d3eda7bf9aaa5fae8567c2e6a95decefe2bd894fb6b5f692bac5cb6050b37f13bde4a0d3b481f3d646dfdb02646522b1b194d5b42d887452887bbc

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock
+/jruby

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.1.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,58 @@
+# InfluxDB v0.8 output plugin for Embulk
+
+**This plug in is the fork of [joker1007/embulk-output-influxdb](https://github.com/joker1007/embulk-output-influxdb) and supports only influxdb v0.8.**
+**It is strongly recommended to use newer influxdb and [joker1007/embulk-output-influxdb](https://github.com/joker1007/embulk-output-influxdb).**
+
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: yes
+* **Dynamic Database creating**: yes
+* **Dynamic Series creating**: yes
+
+## Configuration
+
+- **host**: hostname (string, default: localhost)
+- **port**: port number (integer, default: 8086)
+- **username**: username (string, default: 'root')
+- **password**: password (string, default: 'root')
+- **database**: database name (string, required)
+- **series**:    series name (string, required) (can use column value placeholder. see example)
+- **mode**:     "insert", or "replace". See bellow. (string, default: insert)
+- **timestamp_column**: timestamp column (string, default: nil)
+- **ignore_columns**: ignore column names (array[string], default: [])
+- **default_timezone**: default timezone for column (string, default: 'UTC')
+
+### Modes
+
+* **insert**:
+  * Behavior: This mode inserts rows simplly.
+* **replace**:
+  * Behavior: Same with insert mode excepting that it drops the target series first.
+
+## Example
+
+```yaml
+out:
+  type: influxdb
+  username: root
+  password: root
+  database: dbname
+  series: ${key_name}_series
+  timestamp_column: day
+  mode: replace
+  ignore_columns:
+    - key_name
+```
+
+## ToDo
+- column_options support
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-output-influxdb08.gemspec

@@ -0,0 +1,23 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-influxdb08"
+  spec.version       = "0.1.1"
+  spec.authors       = ["atsaki"]
+  spec.summary       = "InfluxDB v0.8 output plugin for Embulk"
+  spec.description   = "Dumps records to InfluxDB v0.8."
+  spec.email         = ["atsaki01@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/atsaki/embulk-output-influxdb08.git"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'embulk', ['~> 0.7.4']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'tapp'
+
+  spec.add_runtime_dependency 'influxdb', ['0.1.9']
+  spec.add_runtime_dependency 'timezone'
+end

data/example.csv

@@ -0,0 +1,13 @@
+key_name,day,value,name
+new_clients,2015-08-22,1,a
+new_clients,2015-08-25,2,b
+new_clients,2015-08-26,3,c
+new_clients,2015-08-27,4,a
+new_clients,2015-08-28,10,b
+new_clients,2015-08-29,11,c
+new_clients,2015-09-01,12,a
+new_clients,2015-09-03,13,b
+new_clients,2015-09-05,21,c
+new_clients,2015-09-07,22,a
+new_clients,2015-09-11,23,b
+new_clients,2015-09-12,24,c

data/example.yml

@@ -0,0 +1,24 @@
+in:
+  type: file
+  path_prefix: example.csv
+  parser:
+    type: csv
+    skip_header_lines: 1
+    columns:
+      - {name: key_name, type: string}
+      - {name: day, type: timestamp, format: '%Y-%m-%d'}
+      - {name: new_clients, type: long}
+      - {name: name, type: string}
+
+out:
+  type: influxdb
+  username: root
+  password: root
+  database: dbname
+  series: ${key_name}_series
+  # series_per_column: true
+  tag_columns: [name]
+  timestamp_column: day
+  mode: replace
+  ignore_columns:
+    - key_name

data/lib/embulk/output/influxdb08.rb

@@ -0,0 +1,178 @@
+require 'influxdb'
+require 'timezone'
+
+module Embulk
+  module Output
+
+    class Influxdb08 < OutputPlugin
+      Plugin.register_output("influxdb08", self)
+
+      def self.transaction(config, schema, count, &control)
+        # configuration code:
+        task = {
+          "host" => config.param("host", :string, default: nil),
+          "hosts" => config.param("hosts", :array, default: nil),
+          "port" => config.param("port", :integer, default: 8086),
+          "username" => config.param("username", :string, default: "root"),
+          "password" => config.param("password", :string, default: "root"),
+          "database" => config.param("database", :string),
+          "series" => config.param("series", :string, default: nil),
+          "series_per_column" => config.param("series_per_column", :bool, default: false),
+          "timestamp_column" => config.param("timestamp_column", :string, default: nil),
+          "ignore_columns" => config.param("ignore_columns", :array, default: []),
+          "default_timezone" => config.param("default_timezone", :string, default: "UTC"),
+          "mode" => config.param("mode", :string, default: "insert"),
+          "use_ssl" => config.param("use_ssl", :bool, default: false),
+          "verify_ssl" => config.param("verify_ssl", :bool, default: true),
+          "ssl_ca_cert" => config.param("ssl_ca_cert", :string, default: nil),
+          "time_precision" => config.param("time_precision", :string, default: "s"),
+          "initial_delay" => config.param("initial_delay", :float, default: 0.01),
+          "max_delay" => config.param("max_delay", :float, default: 30),
+          "open_timeout" => config.param("open_timeout", :integer, default: 5),
+          "read_timeout" => config.param("read_timeout", :integer, default: 300),
+          "async" => config.param("async", :bool, default: false),
+          "udp" => config.param("udp", :bool, default: false),
+          "retry" => config.param("retry", :integer, default: nil),
+          "denormalize" => config.param("denormalize", :bool, default: true),
+        }
+
+        # resumable output:
+        # resume(task, schema, count, &control)
+
+        # non-resumable output:
+        task_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  task_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
+      def self.replaced_series
+        @replaced_series ||= {}
+      end
+
+      def init
+        # initialization code:
+        task["hosts"] ||= Array(task["host"] || "localhost")
+        @database = task["database"]
+        @series = task["series"]
+        @series_per_column = task["series_per_column"]
+        unless @series
+          raise "Need series or series_per_column parameter" unless @series_per_column
+        end
+        if task["timestamp_column"]
+          @timestamp_column = schema.find { |col| col.name == task["timestamp_column"] }
+        end
+        @ignore_columns = task["ignore_columns"]
+        @time_precision = task["time_precision"]
+        @replace = task["mode"].downcase == "replace"
+        @default_timezone = task["default_timezone"]
+
+        @connection = InfluxDB::Client.new(@database,
+          task.map { |k, v| [k.to_sym, v] }.to_h
+        )
+        create_database_if_not_exist
+      end
+
+      def close
+      end
+
+      def add(page)
+        data = @series ? build_payload(page) : build_payload_per_column(page)
+
+        Embulk.logger.info { "embulk-output-influxdb08: Writing to #{@database}" }
+        Embulk.logger.debug { "embulk-output-influxdb08: #{data}" }
+
+        @connection.write_points(data, @async, @time_precision)
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        task_report = {}
+        return task_report
+      end
+
+      private
+
+      def build_payload(page)
+        data = page.map do |record|
+          series = resolve_placeholder(record, @series)
+          delete_series_if_exist(series)
+          payload = {
+            name: series,
+            data: Hash[
+              target_columns.map { |col| [col.name, convert_timezone(record[col.index])] }
+            ],
+          }
+          payload[:data][:time] = convert_timezone(record[@timestamp_column.index]).to_i if @timestamp_column
+          payload
+        end
+      end
+
+      def build_payload_per_column(page)
+        page.flat_map do |record|
+          target_columns.map do |col|
+            series = col.name
+            delete_series_if_exist(series)
+            payload = {
+              name: series,
+              data: {value: record[col.index]},
+            }
+            payload[:data][:time] = convert_timezone(record[@timestamp_column.index]).to_i if @timestamp_column
+            payload
+          end
+        end
+      end
+
+      def delete_series_if_exist(series)
+        if @replace && self.class.replaced_series[series].nil? && find_series(series)
+          Embulk.logger.info { "embulk-output-influxdb08: Delete series #{series} from #{@database}" }
+          self.class.replaced_series[series] = true
+          @connection.delete_series(series)
+        end
+      end
+
+      def find_series(series)
+        @connection.query('LIST SERIES')["list_series_result"].find { |v|
+          v["name"] == series
+        }
+      end
+
+      def create_database_if_not_exist
+        unless @connection.get_database_list.any? { |db| db["name"] == @database }
+          @connection.create_database(@database)
+        end
+      end
+
+      def resolve_placeholder(record, series)
+        series.gsub(/\$\{(.*?)\}/) do |name|
+          index = schema.index { |col| col.name == $1 }
+          record[index]
+        end
+      end
+
+      def target_columns
+        schema.reject do |col|
+          col.name == @timestamp_column.name || @ignore_columns.include?(col.name)
+        end
+      end
+
+      def convert_timezone(value)
+        return value unless value.is_a?(Time)
+
+        timezone = Timezone::Zone.new(zone: @default_timezone)
+        timezone.time(value)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-influxdb08
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- atsaki
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-08-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.4
+  name: embulk
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.4
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: tapp
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.9
+  name: influxdb
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.1.9
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: timezone
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Dumps records to InfluxDB v0.8.
+email:
+- atsaki01@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-output-influxdb08.gemspec
+- example.csv
+- example.yml
+- lib/embulk/output/influxdb08.rb
+homepage: https://github.com/atsaki/embulk-output-influxdb08.git
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: InfluxDB v0.8 output plugin for Embulk
+test_files: []