fluent-plugin-sql 0.2.1

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/README.md

@@ -0,0 +1,72 @@
+# SQL input plugin for Fluentd event collector
+
+## Overview
+
+This sql input plugin reads records from a RDBMS periodically. Thus you can replicate tables to other storages through Fluentd.
+
+## How does it work?
+
+This plugin runs following SQL repeatedly every 60 seconds to *tail* a table like `tail` command of UNIX.
+
+SELECT * FROM *table* WHERE *update\_column* > *last\_update\_column\_value* ORDER BY *update_column* ASC LIMIT 500
+
+What you need to configure is *update\_column*. The column needs to be updated every time when you update the row so that this plugin detects newly updated rows. Generally, the column is a timestamp such as `updated_at`.
+If you omit to set the column, it uses primary key. And this plugin can't detect updated but it only reads newly inserted rows.
+
+It stores last selected rows to a file named state\_file to not forget the last row when fluentd restarted.
+
+## Configuration
+
+    <source>
+      type sql
+
+      host rdb_host
+      database rdb_database
+      adapter mysql2_or_postgresql_etc
+      user myusername
+      password mypassword
+
+      tag_prefix my.rdb
+
+      select_interval 60s
+      select_limit 500
+
+      state_file /var/run/fluentd/sql_state
+
+      <table>
+        tag table1
+        table table1
+        update_column update_col1
+        time_column time_col2
+      </table>
+
+      <table>
+        tag table2
+        table table2
+        update_column updated_at
+        time_column updated_at
+      </table>
+
+      # detects all tables instead of <table> sections
+      #all_tables
+    </source>
+
+* **host** RDBMS host
+* **port** RDBMS port
+* **database** RDBMS database name
+* **adapter** RDBMS driver name (mysql2 for MySQL, postgresql for PostgreSQL, etc.)
+* **user** RDBMS login user name
+* **password** RDBMS login password
+* **tag_prefix** prefix of tags of events. actual tag will be this\_tag\_prefix.tables\_tag (optional)
+* **select_interval** interval to run SQLs (optional)
+* **select_limit** LIMIT of number of rows for each SQL (optional)
+* **state_file** path to a file to store last rows
+* **all_tables** reads all tables instead of configuring each tables in \<table\> sections
+
+\<table\> sections:
+
+* **tag** tag name of events (optional; default value is table name)
+* **table** RDBM table name
+* **update_column**
+* **time_column** (optional)
+

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+#require 'rake/testtask'
+#
+#Rake::TestTask.new(:test) do |test|
+#  test.libs << 'lib' << 'test'
+#  test.test_files = FileList['test/*.rb']
+#  test.verbose = true
+#end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.2.1

data/fluent-plugin-sql.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-sql"
+  gem.description = "SQL input/output plugin for Fluentd event collector"
+  gem.homepage    = "https://github.com/frsyuki/fluent-plugin-sql"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Sadayuki Furuhashi"]
+  gem.email       = "frsyuki@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency 'activerecord', ['3.2.12']
+  gem.add_dependency 'mysql2', ['~> 0.3.12']
+  gem.add_dependency 'pg', ['~> 0.16.0']
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/in_sql.rb

@@ -0,0 +1,218 @@
+#
+# Fluent
+#
+# Copyright (C) 2013 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+  require 'active_record'
+
+  class SQLInput < Input
+    Plugin.register_input('sql', self)
+
+    config_param :host, :string
+    config_param :port, :integer, :default => nil
+    config_param :adapter, :string
+    config_param :database, :string
+    config_param :username, :string, :default => nil
+    config_param :password, :string, :default => nil
+
+    config_param :state_file, :string, :default => nil
+    config_param :tag_prefix, :string, :default => nil
+    config_param :select_interval, :time, :default => 60
+    config_param :select_limit, :time, :default => 500
+
+    class TableElement
+      include Configurable
+
+      config_param :table, :string
+      config_param :tag, :string, :default => nil
+      config_param :update_column, :string, :default => nil
+      config_param :time_column, :string, :default => nil
+
+      def configure(conf)
+        super
+
+        unless @state_file
+          $log.warn "'state_file PATH' parameter is not set to a 'sql' source."
+          $log.warn "this parameter is highly recommended to save the last rows to resume tailing."
+        end
+      end
+
+      def init(tag_prefix, base_model)
+        @tag = "#{tag_prefix}.#{@tag}" if tag_prefix
+
+        table_name = @table
+        @model = Class.new(base_model) do
+          self.table_name = table_name
+          self.inheritance_column = '_never_use_'
+        end
+        class_name = table_name.singularize.camelize
+        base_model.const_set(class_name, @model)
+        model_name = ActiveModel::Name.new(@model, nil, class_name)
+        @model.define_singleton_method(:model_name) { model_name }
+
+        unless @update_column
+          columns = Hash[@model.columns.map {|c| [c.name, c] }]
+          pk = columns[@model.primary_key]
+          unless pk
+            raise "Composite primary key is not supported. Set update_column parameter to <table> section."
+          end
+          @update_column = pk.name
+        end
+      end
+
+      def emit_next_records(last_record, limit)
+        relation = @model
+        if last_record && last_update_value = last_record[@update_column]
+          relation = relation.where("#{@update_column} > ?", last_update_value)
+        end
+        relation = relation.order("#{@update_column} ASC").limit(limit)
+
+        now = Engine.now
+        entry_name = @model.table_name.singularize
+
+        me = MultiEventStream.new
+        relation.each do |obj|
+          record = obj.as_json[entry_name] rescue nil
+          if record
+            if tv = record[@time_column]
+              time = Time.parse(tv.to_s) rescue now
+            else
+              time = now
+            end
+            me.add(time, record)
+            last_record = record
+          end
+        end
+
+        Engine.emit_stream(@tag, me)
+
+        return last_record
+      end
+    end
+
+    def configure(conf)
+      super
+
+      @tables = conf.elements.select {|e|
+        e.name == 'table'
+      }.map {|e|
+        te = TableElement.new
+        te.configure(e)
+        te
+      }
+
+      if config['all_tables']
+        @all_tables = true
+      end
+    end
+
+    SKIP_TABLE_REGEXP = /\Aschema_migrations\Z/i
+
+    def start
+      @state_store = StateStore.new(@state_file)
+
+      config = {
+        :adapter => @adapter,
+        :host => @host,
+        :port => @port,
+        :database => @database,
+        :username => @username,
+        :password => @password,
+      }
+
+      @base_model = Class.new(ActiveRecord::Base) do
+        self.abstract_class = true
+      end
+      SQLInput.const_set("BaseModel_#{rand(1<<31)}", @base_model)
+      @base_model.establish_connection(config)
+
+      if @all_tables
+        @tables = @base_model.connection.tables.map do |table_name|
+          if table_name.match(SKIP_TABLE_REGEXP)
+            nil
+          else
+            te = TableElement.new
+            te.configure({
+              'table' => table_name,
+              'tag' => table_name,
+              'update_column' => nil,
+            })
+            te
+          end
+        end.compact
+      end
+
+      @tables.reject! do |te|
+        begin
+          te.init(@tag_prefix, @base_model)
+          $log.info "Selecting '#{te.table}' table"
+          false
+        rescue
+          $log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => $!
+          $log.warn_backtrace $!.backtrace
+          true
+        end
+      end
+
+      @stop_flag = false
+      @thread = Thread.new(&method(:thread_main))
+    end
+
+    def shutdown
+      @stop_flag = true
+    end
+
+    def thread_main
+      until @stop_flag
+        sleep @select_interval
+
+        @tables.each do |t|
+          begin
+            last_record = @state_store.last_records[t.table]
+            @state_store.last_records[t.table] = t.emit_next_records(last_record, @select_limit)
+            @state_store.update!
+          rescue
+            $log.error "unexpected error", :error=>$!.to_s
+            $log.error_backtrace
+          end
+        end
+      end
+    end
+
+    class StateStore
+      def initialize(path)
+        @path = path
+        if File.exists?(@path)
+          @data = YAML.load_file(@path)
+        else
+          @data = {}
+        end
+      end
+
+      def last_records
+        @data['last_records'] ||= {}
+      end
+
+      def update!
+        File.open(@path, 'w') {|f|
+          f.write YAML.dump(@data)
+        }
+      end
+    end
+  end
+
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-sql
+version: !ruby/object:Gem::Version
+  version: 0.2.1
+  prerelease: 
+platform: ruby
+authors:
+- Sadayuki Furuhashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.0
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.12
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.12
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.12
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.3.12
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.16.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: SQL input/output plugin for Fluentd event collector
+email: frsyuki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-sql.gemspec
+- lib/fluent/plugin/in_sql.rb
+homepage: https://github.com/frsyuki/fluent-plugin-sql
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: SQL input/output plugin for Fluentd event collector
+test_files: []
+has_rdoc: false