embulk-output-influxdb 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: bd1ac581900456f6a43fcfb3ddd7276526fc2764
+  data.tar.gz: 7b3bc69eb29a6bac858236853ae4e09382c6ad57
+SHA512:
+  metadata.gz: d0f949ab9a638167771aed41f4c4a8ff338f3b60597ac225562105bdde59926dff07fcc4b71c7b6decd7f1e9fd161c03317c4b4c4be16f0568327aac1f283ca8
+  data.tar.gz: 21984a22409343d5c896850fc1f52e0febcd2e766b481df7f494d08edfe0246d9a6305c4f721a19c718d574ccb8f0709c4fa02c3d40529fcaa0d0df78975653f

data/.gitignore

@@ -0,0 +1,6 @@
+*~
+/pkg/
+/tmp/
+/.bundle/
+/Gemfile.lock
+/jruby

data/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.1.0

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,52 @@
+# InfluxDB output plugin for Embulk
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: yes
+* **Dynamic Database creating**: yes
+* **Dynamic Series creating**: yes
+
+## Configuration
+
+- **host**: hostname (string, default: localhost)
+- **port**: port number (integer, default: 8086)
+- **username**: username (string, default: 'root')
+- **password**: password (string, default: 'root')
+- **database**: database name (string, required)
+- **series**:    series name (string, required) (can use column value placeholder. see example)
+- **mode**:     "insert", or "replace". See bellow. (string, default: insert)
+- **timestamp_column**: timestamp column (string, default: nil)
+- **ignore_columns**: ignore column names (array[string], default: nil)
+- **default_timezone**: default timezone for column (string, default: 'UTC')
+
+### Modes
+
+* **insert**:
+  * Behavior: This mode inserts rows simplly.
+* **replace**:
+  * Behavior: Same with insert mode excepting that it drops the target series first.
+
+## Example
+
+```yaml
+out:
+  type: influxdb
+  username: root
+  password: root
+  database: dbname
+  series: ${key_name}_series
+  timestamp_column: day
+  mode: replace
+  ignore_columns:
+    - key_name
+```
+
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-output-influxdb.gemspec

@@ -0,0 +1,23 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-influxdb"
+  spec.version       = "0.1.0"
+  spec.authors       = ["joker1007"]
+  spec.summary       = "InfluxDB output plugin for Embulk"
+  spec.description   = "Dumps records to InfluxDB."
+  spec.email         = ["kakyoin.hierophant@gmail.com"]
+  spec.licenses      = ["MIT"]
+  spec.homepage      = "https://github.com/joker1007/embulk-output-influxdb"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency 'embulk', ['~> 0.7.4']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+  spec.add_development_dependency 'tapp'
+
+  spec.add_runtime_dependency 'influxdb', ['~> 0.2']
+  spec.add_runtime_dependency 'timezone'
+end

data/example.csv

@@ -0,0 +1,13 @@
+key_name,day,value
+new_clients,2015-08-22,1
+new_clients,2015-08-25,2
+new_clients,2015-08-26,3
+new_clients,2015-08-27,4
+new_clients,2015-08-28,10
+new_clients,2015-08-29,11
+new_clients,2015-09-01,12
+new_clients,2015-09-03,13
+new_clients,2015-09-05,21
+new_clients,2015-09-07,22
+new_clients,2015-09-11,23
+new_clients,2015-09-12,24

data/example.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: example.csv
+  parser:
+    type: csv
+    skip_header_lines: 1
+    columns:
+      - {name: key_name, type: string}
+      - {name: day, type: timestamp, format: '%Y-%m-%d'}
+      - {name: new_clients, type: long}
+
+out:
+  type: influxdb
+  username: root
+  password: root
+  database: dbname
+  # series: ${key_name}_series
+  series_per_column: true
+  timestamp_column: day
+  mode: replace
+  ignore_columns:
+    - key_name

data/lib/embulk/output/influxdb.rb

@@ -0,0 +1,175 @@
+require 'influxdb'
+require 'timezone'
+
+module Embulk
+  module Output
+
+    class Influxdb < OutputPlugin
+      Plugin.register_output("influxdb", self)
+
+      def self.transaction(config, schema, count, &control)
+        # configuration code:
+        task = {
+          "host" => config.param("host", :string, default: "localhost"),
+          "hosts" => config.param("hosts", :array, default: "localhost"),
+          "port" => config.param("port", :integer, default: 8086),
+          "username" => config.param("username", :string, default: "root"),
+          "password" => config.param("password", :string, default: "root"),
+          "database" => config.param("database", :string),
+          "series" => config.param("series", :string, default: nil),
+          "series_per_column" => config.param("series_per_column", :bool, default: false),
+          "timestamp_column" => config.param("timestamp_column", :string, default: nil),
+          "ignore_columns" => config.param("ignore_columns", :array, default: []),
+          "default_timezone" => config.param("default_timezone", :string, default: "UTC"),
+          "mode" => config.param("mode", :string, default: "insert"),
+          "use_ssl" => config.param("use_ssl", :bool, default: false),
+          "verify" => config.param("verify_ssl", :bool, default: true),
+          "ssl_ca_cert" => config.param("ssl_ca_cert", :string, default: nil),
+          "time_precision" => config.param("time_precision", :string, default: "s"),
+          "initial_delay" => config.param("initial_delay", :float, default: 0.01),
+          "max_delay" => config.param("max_delay", :float, default: 30),
+          "open_timeout" => config.param("open_timeout", :integer, default: 5),
+          "read_timeout" => config.param("read_timeout", :integer, default: 300),
+          "async" => config.param("async", :bool, default: false),
+          "udp" => config.param("udp", :bool, default: false),
+          "retry" => config.param("retry", :integer, default: nil),
+          "denormalize" => config.param("denormalize", :bool, default: true),
+        }
+
+        # resumable output:
+        # resume(task, schema, count, &control)
+
+        # non-resumable output:
+        task_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  task_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
+      def self.replaced_measurements
+        @replaced_measurements ||= {}
+      end
+
+      def init
+        # initialization code:
+        @database = task["database"]
+        @series = task["series"]
+        @series_per_column = task["series_per_column"]
+        unless @series || @series_per_column
+          raise "Need series or series_per_column parameter"
+        end
+        if task["timestamp_column"]
+          @timestamp_column = schema.find { |col| col.name == task["timestamp_column"] }
+        end
+        @ignore_columns = task["ignore_columns"]
+        @time_precision = task["time_precision"]
+        @replace = task["mode"].downcase == "replace"
+        @default_timezone = task["default_timezone"]
+
+        @connection = InfluxDB::Client.new(@database,
+          task.map { |k, v| [k.to_sym, v] }.to_h
+        )
+        create_database_if_not_exist
+      end
+
+      def close
+      end
+
+      def add(page)
+        data = @series ? build_payload(page) : build_payload_per_column(page)
+
+        Embulk.logger.info { "embulk-output-influxdb: Writing to #{@database}" }
+        Embulk.logger.debug { "embulk-output-influxdb: #{data}" }
+
+        @connection.write_points(data, @time_precision)
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        task_report = {}
+        return task_report
+      end
+
+      private
+
+      def build_payload(page)
+        data = page.map do |record|
+          series = resolve_placeholder(record, @series)
+          drop_measurement_if_exist(series)
+          payload = {
+            series: series,
+            values: Hash[
+              target_columns.map { |col| [col.name, convert_timezone(record[col.index])] }
+            ],
+          }
+          payload[:timestamp] = convert_timezone(record[@timestamp_column.index]).to_i if @timestamp_column
+          payload
+        end
+      end
+
+      def build_payload_per_column(page)
+        page.flat_map do |record|
+          target_columns.map do |col|
+            series = col.name
+            drop_measurement_if_exist(series)
+            payload = {
+              series: series,
+              values: {value: record[col.index]},
+            }
+            payload[:timestamp] = convert_timezone(record[@timestamp_column.index]).to_i if @timestamp_column
+            payload
+          end
+        end
+      end
+
+      def drop_measurement_if_exist(series)
+        if @replace && self.class.replaced_measurements[series].nil? && find_measurement(series)
+          Embulk.logger.info { "embulk-output-influxdb: Drop measurement #{series} from #{@database}" }
+          self.class.replaced_measurements[series] = true
+          @connection.query("DROP MEASUREMENT #{series}")
+        end
+      end
+
+      def find_measurement(series)
+        @connection.query("SHOW MEASUREMENTS")[0]["values"].find { |v| v["name"] == series }
+      end
+
+      def create_database_if_not_exist
+        unless @connection.list_databases.any? { |db| db["name"] == @database }
+          @connection.create_database(@database)
+        end
+      end
+
+      def resolve_placeholder(record, series)
+        series.gsub(/\$\{(.*?)\}/) do |name|
+          index = schema.index { |col| col.name == $1 }
+          record[index]
+        end
+      end
+
+      def target_columns
+        schema.reject do |col|
+          col.name == @timestamp_column.name || @ignore_columns.include?(col.name)
+        end
+      end
+
+      def convert_timezone(value)
+        return value unless value.is_a?(Time)
+
+        timezone = Timezone::Zone.new(zone: @default_timezone)
+        timezone.time(value)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-influxdb
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- joker1007
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-09-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: embulk
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.4
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.4
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: tapp
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: influxdb
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: timezone
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+description: Dumps records to InfluxDB.
+email:
+- kakyoin.hierophant@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-version"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-output-influxdb.gemspec
+- example.csv
+- example.yml
+- lib/embulk/output/influxdb.rb
+homepage: https://github.com/joker1007/embulk-output-influxdb
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: InfluxDB output plugin for Embulk
+test_files: []
+has_rdoc: