embulk-output-wendelin 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a3945e56edb675247ac6b31f37d43b9f2c867ea8
+  data.tar.gz: 419b936e9107857563c010697be4c6625f15d270
+SHA512:
+  metadata.gz: 1f7452537e2afe252d78df8c773aafb072a1217c676c763d4d7e1911c88061920cf90a50ca0aa204a1e23dd263533a79d9592df5f763cc5352326c8bd2eb487b
+  data.tar.gz: 17acfdfb768fb6a5d914cd708db697f990462dcc77eb6d22f06ea939fa57ebffc7e46104eac4eaf0b3f08abc89cbd88d64ac85e0a11593f3a4df95ee0a929e04

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,24 @@
+PATH
+  remote: .
+  specs:
+    embulk-output-wendelin (0.1.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    embulk (0.8.18)
+      jruby-jars (= 9.1.5.0)
+    jruby-jars (9.1.5.0)
+    rake (12.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.10.6)
+  embulk (>= 0.8.13)
+  embulk-output-wendelin!
+  rake (>= 10.0)
+
+BUNDLED WITH
+   1.14.6

data/LICENSE.txt

@@ -0,0 +1,14 @@
+Copyright (C) 2016 Nexedi SA and Contributors
+                   Klaus Wölfel <klaus@nexedi.com>
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,39 @@
+# Wendelin output plugin for Embulk
+
+TODO: Write short description here and embulk-output-wendelin.gemspec file.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: yes
+
+## Configuration
+
+- **option1**: description (integer, required)
+- **option2**: description (string, default: `"myvalue"`)
+- **option3**: description (string, default: `null`)
+
+## Example
+
+```yaml
+in:
+  type: file
+  path_prefix: /sample_
+  parser:
+    type: none
+out:
+  type: wendelin
+  tag: app.tag1.txt
+  streamtool_uri: https://url_to_erp5_instance/erp5/portal_ingestion_policies/my_policy
+  user: my_username
+  password: may_password
+```
+
+
+## Build
+
+```
+$ rake
+```

data/Rakefile

@@ -0,0 +1,3 @@
+require "bundler/gem_tasks"
+
+task default: :build

data/embulk-output-wendelin.gemspec

@@ -0,0 +1,20 @@
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-output-wendelin"
+  spec.version       = "0.1.1"
+  spec.authors       = ["Klaus Woelfel"]
+  spec.summary       = "Wendelin output plugin for Embulk"
+  spec.description   = "Dumps records to Wendelin."
+  spec.email         = ["YOUR_NAME"]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/YOUR_NAME/embulk-output-wendelin"
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r{^(test|spec)/})
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'embulk', ['>= 0.8.13']
+  spec.add_development_dependency 'bundler', ['>= 1.10.6']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end

data/lib/embulk/output/wendelin.rb

@@ -0,0 +1,96 @@
+require 'base64'
+require_relative 'wendelin_client'
+
+module Embulk
+  module Output
+
+    class Wendelin < OutputPlugin
+      Plugin.register_output("wendelin", self)
+
+      def self.transaction(config, schema, count, &control)
+        # configuration code:
+        task = {
+           "tag" => config.param("tag", :string, default: nil),
+          # where Wendelin's Input Stream Tool is located,
+          # ex http://example.com/erp5/portal_ingestion_policies/example_ingestion
+          "streamtool_uri" => config.param("streamtool_uri", :string),
+
+          # credentials to authenticate this fluentd to wendelin
+          # by default credentials are not used
+          # TODO user/password -> certificate
+          "user" => config.param("user", :string, defualt: nil),
+          "password" => config.param("password", :string, default: nil),
+
+          "path_prefix" => config.param("path_prefix", :string,  :default => nil),
+        }
+
+        # resumable output:
+        # resume(task, schema, count, &control)
+
+        #Embulk.logger.info { config.to_s }
+        #Embulk.logger.info { schema.to_s }
+        #Embulk.logger.info { count.to_s }
+
+
+        # non-resumable output:
+        task_reports = yield(task)
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      #def self.resume(task, schema, count, &control)
+      #  task_reports = yield(task)
+      #
+      #  next_config_diff = {}
+      #  return next_config_diff
+      #end
+
+      def init
+        # initialization code:
+        credentials = {}
+        #Embulk.logger.info { "Test" }
+        #Embulk.logger.info { schema.to_s }
+        #Embulk.logger.info "Test"
+        @tag = task["tag"]
+        streamtool_uri = task["streamtool_uri"]
+        unless task["user"].nil?
+          credentials["user"] = task["user"]
+          credentials["password"] = task["password"]
+        end
+        @wendelin = WendelinClient.new(streamtool_uri, credentials, Embulk.logger)
+      end
+
+      def close
+      end
+
+      def add(page)
+        # output code:
+        #page.each do |record|
+          #hash = Hash[schema.names.zip(record)]
+        #end
+        #Embulk.logger.info { page.to_s }
+        page.each do |record|
+          #Embulk.logger.info { record.to_s }
+          tag = record[1].gsub(File::ALT_SEPARATOR || File::SEPARATOR, '.')[1..-1]
+          unless @tag.nil?
+            tag = @tag + "." + tag
+          end
+          Embulk.logger.info{ tag }
+          @wendelin.ingest(tag, Base64.decode64(record[0]))
+        end
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        task_report = {}
+        return task_report
+      end
+    end
+
+  end
+end

data/lib/embulk/output/wendelin_client.rb

@@ -0,0 +1,105 @@
+# module to ingest data to a Wendelin through HTTP endpoint
+# Copyright (C) 2015  Nexedi SA and Contributors.
+#                     Kirill Smelkov <kirr@nexedi.com>
+#
+#   This program is free software: you can Use, Study, Modify and Redistribute
+#   it under the terms of the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+
+require 'net/http'
+require 'openssl'
+
+
+# class representing a Wendelin client
+class WendelinClient
+
+  # `streamtool_uri` - URI pointing to portal_input_data_stream "mountpoint"
+  # `credentials`    # {'user' => _, 'password' => _}     TODO change to certificate
+  # `log` - logger to use
+  def initialize(streamtool_uri, credentials, log)
+    @streamtool_uri = streamtool_uri
+    @credentials    = credentials
+    @log            = log
+  end
+
+
+  # ingest `data_chunk` to a stream referenced as `reference`
+  def ingest(reference, data_chunk)
+      uri = URI("#{@streamtool_uri}/ingest?reference=#{reference}")
+      req = Net::HTTP::Post.new(uri)
+      if @credentials.has_key?('user')
+        req.basic_auth @credentials['user'], @credentials['password']
+      end
+
+      # TODO ensure content-type is 'raw', e.g. this way
+      # (but then querystring ?reference=... is lost)
+      # req.body = data_chunk
+      # req.content_type = 'application/octet-stream'
+      req.set_form_data('data_chunk' => data_chunk)
+
+      #@log.on_trace do
+      #    @log.trace '>>> REQUEST'
+      #    @log.trace "method\t=> #{req.method}"
+      #    @log.trace "path\t=> #{req.path}"
+      #    @log.trace "uri\t=> #{req.uri}"
+      #    @log.trace "body\t=> #{req.body}"
+      #    @log.trace "body_stream\t=> #{req.body_stream}"
+      #    req.each {|h| @log.trace "#{h}:\t#{req[h]}"}
+      #    @log.trace
+      #end
+
+      begin
+          # TODO keep connection open (so that every new ingest does not do
+          # full connect again)
+          res = Net::HTTP.start(uri.hostname, uri.port,
+                      :use_ssl      => (uri.scheme == 'https'),
+                      # NOTE = "do not check server cert"
+                      # TODO move this out to conf parameters
+                      :verify_mode  => OpenSSL::SSL::VERIFY_NONE,
+
+                      # Net::HTTP default open timeout is infinity, which results
+                      # in thread hang forever if other side does not fully
+                      # establish connection. Default read_timeout is 60 seconds.
+                      # We go safe way and make sure all timeouts are defined.
+                      :ssl_timeout  => 60,
+                      :open_timeout => 60,
+                      :read_timeout => 60,
+                ) do |http|
+              http.request(req)
+          end
+
+      rescue
+          # some http/ssl/other connection error
+          #@log.warn "HTTP ERROR:"
+          raise
+
+      else
+          #@log.on_trace do
+          #    @log.trace '>>> RESPONSE'
+          #    res.each {|h| @log.trace "#{h}:\t#{res[h]}"}
+          #    @log.trace "code\t=> #{res.code}"
+          #    @log.trace "msg\t=> #{res.message}"
+          #    @log.trace "class\t=> #{res.class}"
+          #    @log.trace "body:", res.body
+          #end
+
+          if res.kind_of?(Net::HTTPSuccess)       # res.code is 2XX
+              #@log.info "ingested ok"
+          else
+              #@log.warn "FAIL:"
+              res.value
+          end
+      end
+  end
+
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-wendelin
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Klaus Woelfel
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-03-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: embulk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.13
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.8.13
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.10.6
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Dumps records to Wendelin.
+email:
+- YOUR_NAME
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-output-wendelin.gemspec
+- lib/embulk/output/wendelin.rb
+- lib/embulk/output/wendelin_client.rb
+- vendor/cache/embulk-0.8.18.gem
+- vendor/cache/jruby-jars-9.1.5.0.gem
+- vendor/cache/rake-12.0.0.gem
+homepage: 
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Wendelin output plugin for Embulk
+test_files: []