fluent-plugin-sql 0.2.2 → 0.3.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9ca23328ec98228710390c94c0f3f3a1d762e3c4
+  data.tar.gz: 0ff6ef9512db112b7938b2af612a2ffe59ddd59e
+SHA512:
+  metadata.gz: 302054c46157cc5438085b75e612a8353d6859b5f997ca66984b2064231e25587cdaee438050e645c345ad33c3a2377a21a85f3317f6d25c1fdceaf181e8f29f
+  data.tar.gz: d567b461ca80f21adbd7c07e33a4deb801a20ea54ccc955e6091e84230ce51c7925b41520aa8640788079921d197bf1f33727a9b4f736dad2158a921cefc8537

data/README.md

@@ -1,4 +1,4 @@
-# SQL input plugin for Fluentd event collector
+# SQL input plugin for [Fluentd](http://fluentd.org) event collector
 
 ## Overview
 
@@ -23,7 +23,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
       host rdb_host
       database rdb_database
       adapter mysql2_or_postgresql_etc
-      user myusername
+      username myusername
       password mypassword
 
       tag_prefix my.rdb  # optional, but recommended

data/VERSION

@@ -1 +1 @@
-0.2.2
+0.3.0

data/fluent-plugin-sql.gemspec

@@ -18,7 +18,8 @@ Gem::Specification.new do |gem|
   gem.license = "Apache 2.0"
 
   gem.add_dependency "fluentd", "~> 0.10.0"
-  gem.add_dependency 'activerecord', ['3.2.12']
+  gem.add_dependency 'activerecord', "~> 4.0.2"
+  gem.add_dependency 'activerecord-import', "~> 0.4.1"
   gem.add_dependency 'mysql2', ['~> 0.3.12']
   gem.add_dependency 'pg', ['~> 0.16.0']
   gem.add_development_dependency "rake", ">= 0.9.2"

data/lib/fluent/plugin/in_sql.rb

@@ -28,12 +28,17 @@ module Fluent
     config_param :database, :string
     config_param :username, :string, :default => nil
     config_param :password, :string, :default => nil
+    config_param :socket, :string, :default => nil
 
     config_param :state_file, :string, :default => nil
     config_param :tag_prefix, :string, :default => nil
     config_param :select_interval, :time, :default => 60
     config_param :select_limit, :time, :default => 500
 
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
     class TableElement
       include Configurable
 
@@ -44,11 +49,6 @@ module Fluent
 
       def configure(conf)
         super
-
-        unless @state_file
-          $log.warn "'state_file PATH' parameter is not set to a 'sql' source."
-          $log.warn "this parameter is highly recommended to save the last rows to resume tailing."
-        end
       end
 
       def init(tag_prefix, base_model)
@@ -129,6 +129,11 @@ module Fluent
     def configure(conf)
       super
 
+      unless @state_file
+        $log.warn "'state_file PATH' parameter is not set to a 'sql' source."
+        $log.warn "this parameter is highly recommended to save the last rows to resume tailing."
+      end
+
       @tables = conf.elements.select {|e|
         e.name == 'table'
       }.map {|e|
@@ -154,6 +159,7 @@ module Fluent
         :database => @database,
         :username => @username,
         :password => @password,
+        :socket => @socket,
       }
 
       # creates subclass of ActiveRecord::Base so that it can have different
@@ -166,7 +172,7 @@ module Fluent
       # ActiveRecord requires the base_model to have a name. Here sets name
       # of an anonymous class by assigning it to a constant. In Ruby, class has
       # a name of a constant assigned first
-      SQLInput.const_set("BaseModel_#{rand(1<<31)}", @base_model)
+      SQLInput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
 
       # Now base_model can have independent configuration from ActiveRecord::Base
       @base_model.establish_connection(config)
@@ -193,11 +199,11 @@ module Fluent
       @tables.reject! do |te|
         begin
           te.init(@tag_prefix, @base_model)
-          $log.info "Selecting '#{te.table}' table"
+          log.info "Selecting '#{te.table}' table"
           false
-        rescue
-          $log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => $!
-          $log.warn_backtrace $!.backtrace
+        rescue => e
+          log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+          log.warn_backtrace e.backtrace
           true
         end
       end
@@ -219,9 +225,9 @@ module Fluent
             last_record = @state_store.last_records[t.table]
             @state_store.last_records[t.table] = t.emit_next_records(last_record, @select_limit)
             @state_store.update!
-          rescue
-            $log.error "unexpected error", :error=>$!.to_s
-            $log.error_backtrace
+          rescue => e
+            log.error "unexpected error", :error => e.message, :error_class => e.class
+            log.error_backtrace e.backtrace
           end
         end
       end

data/lib/fluent/plugin/out_sql.rb

@@ -0,0 +1,232 @@
+module Fluent
+  class SQLOutput < BufferedOutput
+    Plugin.register_output('sql', self)
+
+    include SetTimeKeyMixin
+    include SetTagKeyMixin
+
+    config_param :host, :string
+    config_param :port, :integer, :default => nil
+    config_param :adapter, :string
+    config_param :username, :string, :default => nil
+    config_param :password, :string, :default => nil
+    config_param :database, :string
+    config_param :socket, :string, :default => nil
+    config_param :remove_tag_prefix, :string, :default => nil
+
+    attr_accessor :tables
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
+    # TODO: Merge SQLInput's TableElement
+    class TableElement
+      include Configurable
+
+      config_param :table, :string
+      config_param :column_mapping, :string
+      config_param :num_retries, :integer, :default => 5
+
+      attr_reader :model
+      attr_reader :pattern
+
+      def initialize(pattern, log)
+        super()
+        @pattern = MatchPattern.create(pattern)
+        @log = log
+      end
+
+      def configure(conf)
+        super
+
+        @mapping = parse_column_mapping(@column_mapping)
+        @format_proc = Proc.new { |record|
+          new_record = {}
+          @mapping.each { |k, c|
+            new_record[c] = record[k]
+          }
+          new_record
+        }
+      end
+
+      def init(base_model)
+        # See SQLInput for more details of following code
+        table_name = @table
+        @model = Class.new(base_model) do
+          self.table_name = table_name
+          self.inheritance_column = '_never_use_output_'
+        end
+
+        class_name = table_name.singularize.camelize
+        base_model.const_set(class_name, @model)
+        model_name = ActiveModel::Name.new(@model, nil, class_name)
+        @model.define_singleton_method(:model_name) { model_name }
+
+        # TODO: check column_names and table schema
+        columns = @model.columns.map { |column| column.name }.sort
+      end
+
+      def import(chunk)
+        records = []
+        chunk.msgpack_each { |tag, time, data|
+          begin
+            # format process should be moved to emit / format after supports error stream.
+            records << @model.new(@format_proc.call(data))
+          rescue => e
+            args = {:error => e.message, :error_class => e.class, :table => @table, :record => Yajl.dump(data)}
+            @log.warn "Failed to create the model. Ignore a record:", args
+          end
+        }
+        begin
+          @model.import(records)
+        rescue ActiveRecord::StatementInvalid, ActiveRecord::ThrowResult, ActiveRecord::Import::MissingColumnError => e
+          # ignore other exceptions to use Fluentd retry mechanizm
+          @log.warn "Got deterministic error. Fallback to one-by-one import", :error => e.message, :error_class => e.class
+          one_by_one_import(records)
+        end
+      end
+
+      def one_by_one_import(records)
+        records.each { |record|
+          retries = 0
+          begin
+            @model.import([record])
+          rescue ActiveRecord::StatementInvalid, ActiveRecord::ThrowResult, ActiveRecord::Import::MissingColumnError => e
+            @log.error "Got deterministic error again. Dump a record", :error => e.message, :error_class => e.class, :record => record
+          rescue => e
+            retries += 1
+            if retries > @num_retries
+              @log.error "Can't recover undeterministic error. Dump a record", :error => e.message, :error_class => e.class, :record => record
+              next
+            end
+
+            @log.warn "Failed to import a record: retry number = #{retries}", :error  => e.message, :error_class => e.class
+            sleep 0.5
+            retry
+          end
+        }
+      end
+
+      private
+
+      def parse_column_mapping(column_mapping_conf)
+        mapping = {}
+        column_mapping_conf.split(',').each { |column_map|
+          key, column = column_map.strip.split(':', 2)
+          column = key if column.nil?
+          mapping[key] = column
+        }
+        mapping
+      end
+    end
+
+    def initialize
+      super
+      require 'active_record'
+      require 'activerecord-import'
+    end
+
+    def configure(conf)
+      super
+
+      if remove_tag_prefix = conf['remove_tag_prefix']
+        @remove_tag_prefix = Regexp.new('^' + Regexp.escape(remove_tag_prefix))
+      end
+
+      @tables = []
+      @default_table = nil
+      conf.elements.select { |e|
+        e.name == 'table'
+      }.each { |e|
+        te = TableElement.new(e.arg, log)
+        te.configure(e)
+        if e.arg.empty?
+          $log.warn "Detect duplicate default table definition" if @default_table
+          @default_table = te
+        else
+          @tables << te
+        end
+      }
+      @only_default = @tables.empty?
+
+      if @default_table.nil?
+        raise ConfigError, "There is no default table. <table> is required in sql output"
+      end
+    end
+
+    def start
+      super
+
+      config = {
+        :adapter => @adapter,
+        :host => @host,
+        :port => @port,
+        :database => @database,
+        :username => @username,
+        :password => @password,
+        :socket => @socket,
+      }
+
+      @base_model = Class.new(ActiveRecord::Base) do
+        self.abstract_class = true
+      end
+
+      SQLOutput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
+      @base_model.establish_connection(config)
+
+      # ignore tables if TableElement#init failed
+      @tables.reject! do |te|
+        init_table(te, @base_model)
+      end
+      init_table(@default_table, @base_model)
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit(tag, es, chain)
+      if @only_default
+        super(tag, es, chain)
+      else
+        super(tag, es, chain, format_tag(tag))
+      end
+    end
+
+    def format(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def write(chunk)
+      @tables.each { |table|
+        if table.pattern.match(chunk.key)
+          return table.import(chunk)
+        end
+      }
+      @default_table.import(chunk)
+    end
+
+    private
+
+    def init_table(te, base_model)
+      begin
+        te.init(base_model)
+        log.info "Selecting '#{te.table}' table"
+        false
+      rescue => e
+        log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+        log.warn_backtrace e.backtrace
+        true
+      end
+    end
+
+    def format_tag(tag)
+      if @remove_tag_prefix
+        tag.gsub(@remove_tag_prefix, '')
+      else
+        tag
+      end
+    end
+  end
+end

metadata

@@ -1,94 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 0.2.2
-  prerelease: 
+  version: 0.3.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-09 00:00:00.000000000 Z
+date: 2014-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.10.0
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.2.12
+        version: 4.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.2.12
+        version: 4.0.2
+- !ruby/object:Gem::Dependency
+  name: activerecord-import
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.12
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.3.12
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.16.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.16.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.9.2
 description: SQL input/output plugin for Fluentd event collector
@@ -103,30 +106,29 @@ files:
 - VERSION
 - fluent-plugin-sql.gemspec
 - lib/fluent/plugin/in_sql.rb
+- lib/fluent/plugin/out_sql.rb
 homepage: https://github.com/frsyuki/fluent-plugin-sql
 licenses:
 - Apache 2.0
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: SQL input/output plugin for Fluentd event collector
 test_files: []
-has_rdoc: false