fluent-plugin-sql-enchanced 0.5.3 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ddedbc4c3f5babcc5b2eaf9f3a41b730f2d370d1
-  data.tar.gz: 7a15b6a37feac936116e4c06933c135a3166eb73
+  metadata.gz: 8b75bc87abe0cbb08f1daf17600a67674cb67c6f
+  data.tar.gz: f3b4b9627884ce8d777a0be24dbe592f9d965193
 SHA512:
-  metadata.gz: bf80c88deee559b0b425613401f6c7ac517edf604f1a142473ca69824cd2b4d63cef08a10f6c3ef7c74430d3ccdd03bf0d21b92aa80c1e37593d33eb42adc00a
-  data.tar.gz: 3f18833a4a466ffa103b75c347df05773387394f06a00523f58d3a5a89e7c357a0e26eb1be6b85b19408678629b703148489c411ca14793b239f97ae59b648d2
+  metadata.gz: ac85a5965b87fa1a280837decf528147f4e2156fc6a1966fc9d184cf75675ecb827ac9e3104cb4ad44e2c15432ce4c8e6449c4411f2f55b00e3e3a83a4b062c1
+  data.tar.gz: 0048de749270caaf8311900d194ce726376b6c52146e1e9c84b798c66d184254477c985906836dabe83be9ff31fa3e7754ddb01870a6d5188057777d1d276405

data/VERSION

@@ -1 +1 @@
-0.5.3
+0.5.5

data/lib/fluent/plugin/in_sql.rb

@@ -1,194 +1,356 @@
-class Fluentd
-  module Setting
-    class InSql
-      include ActiveModel::Model
-      include Common
-
-      KEYS = [
-          :host,
-          :port,
-          :database,
-          :adapter,
-          :username,
-          :password,
-          :tag_prefix,
-          :select_interval,
-          :select_limit,
-          :state_file,
-          :table,
-          :all_tables
-      ].freeze
-
-      attr_accessor(*KEYS)
-      attr_accessor(:fields_descriptions)
-
-      validates :host, presence: true
-      validates :database, presence: true
-      validates :adapter, presence: true
-      validates :username, presence: true
-      validates :password, presence: true
-      validates :state_file, presence: true
-
-      def self.initial_params
-        {
-            host: 'localhost',
-            database: 'rdb_database',
-            adapter: 'mysql2',
-            username: 'myusername',
-            password: 'mypassword',
-            tag_prefix: 'my.rdb',
-            select_interval: '60s',
-            select_limit: '500',
-            state_file: "/tmp/data_enchilada-sql-#{Fluentd.instance.id}-#{Time.now.to_i}.pos",
-            table: [
-                {
-                    table: 'rdb_table',
-                    tag: 'rdb_table_tag',
-                    update_column: 'updated_at',
-                    time_column: 'updated_at',
-                    primary_key: ''
-                }
-            ]
-        }
+#
+# Fluent
+#
+# Copyright (C) 2013 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+  require 'active_record'
+  require 'redis'
+  require 'pry'
+
+  SERVER_PREFIX = "ip#{Socket.ip_address_list.detect(&:ipv4_private?).try(:ip_address).delete('.')}"
+  # contents = IO.read("/etc/data_enchilada/data_enchilada.properties").strip
+
+  # KAFKA_SERVER = contents.lines.first.split("=")[1].strip
+  KAFKA_SERVER = '10.1.0.57'
+
+  class SQLInput < Input
+    Plugin.register_input('sql', self)
+
+    config_param :host, :string
+    config_param :port, :integer, :default => nil
+    config_param :adapter, :string
+    config_param :database, :string
+    config_param :username, :string, :default => nil
+    config_param :password, :string, :default => nil, :secret => true
+    config_param :socket, :string, :default => nil
+
+    config_param :state_file, :string, :default => nil
+    config_param :tag_prefix, :string, :default => nil
+    config_param :select_interval, :time, :default => 60
+    config_param :select_limit, :time, :default => 500
+
+    unless method_defined?(:log)
+      define_method(:log) { $log }
+    end
+
+    class TableElement
+      include Configurable
+
+      config_param :table, :string
+      config_param :tag, :string, :default => nil
+      config_param :update_column, :string, :default => nil
+      config_param :time_column, :string, :default => nil
+      config_param :primary_key, :string, :default => nil
+
+      def configure(conf)
+        super
       end
 
-      def fields_descriptions
-        {
-            host: '* RDBMS host (required)',
-            port: 'RDBMS port (optional)',
-            database: '* RDBMS database name (required)',
-            adapter: '* RDBMS driver name. You should install corresponding gem before start (mysql2 gem for mysql2 adapter, pg gem for postgresql adapter, etc. (required)',
-            username: '* RDBMS login user name (required)',
-            password: '* RDBMS login password (required)',
-            tag_prefix: 'prefix of tags of events. actual tag will be this_tag_prefix.tables_tag (optional, but recommended)',
-            select_interval: 'interval to run SQLs (optional)',
-            select_limit: 'LIMIT of number of rows for each SQL (optional)',
-            state_file: '* path to a file to store last rows (required)',
-            all_tables: 'reads all tables instead of configuring each tables in &lt;table&gt; sections (optional)',
-            table: {
-                tag: 'tag name of events (optional; default value is table name)',
-                table: '* RDBM table name',
-                update_column: '* see above description',
-                time_column: "(optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.",
-                primary_key: "(optional): if you want to get data from the table which doesn't have primary key like PostgreSQL's View, set this parameter."
-            }
-        }
+      def init(tag_prefix, base_model, router)
+        @router = router
+        @tag = "#{tag_prefix}.#{@tag}" if tag_prefix
+
+        # creates a model for this table
+        table_name = @table
+        primary_key = @primary_key
+        @model = Class.new(base_model) do
+          self.table_name = table_name
+          self.inheritance_column = '_never_use_'
+          self.primary_key = primary_key if primary_key
+
+          #self.include_root_in_json = false
+
+          def read_attribute_for_serialization(n)
+            v = send(n)
+            if v.respond_to?(:to_msgpack)
+              v
+            else
+              v.to_s
+            end
+          end
+        end
+
+        # ActiveRecord requires model class to have a name.
+        class_name = table_name.singularize.camelize
+        base_model.const_set(class_name, @model)
+
+        # Sets model_name otherwise ActiveRecord causes errors
+        model_name = ActiveModel::Name.new(@model, nil, class_name)
+        @model.define_singleton_method(:model_name) { model_name }
+
+        # if update_column is not set, here uses primary key
+        unless @update_column
+          columns = Hash[@model.columns.map {|c| [c.name, c] }]
+          pk = columns[@model.primary_key]
+          unless pk
+            raise "Composite primary key is not supported. Set update_column parameter to <table> section."
+          end
+          @update_column = pk.name
+        end
       end
 
-      def common_options
-        [
-            :host,
-            :port,
-            :database,
-            :adapter,
-            :username,
-            :password,
-            :tag_prefix
-        ]
+      # emits next records and returns the last record of emitted records
+      def emit_next_records(last_record, limit)
+        relation = @model
+        if last_record && last_update_value = last_record[@update_column]
+          relation = relation.where("#{@update_column} > ?", last_update_value)
+        end
+        relation = relation.order("#{@update_column} ASC")
+        relation = relation.limit(limit) if limit > 0
+
+        now = Engine.now
+        entry_name = @model.table_name.singularize
+
+        me = MultiEventStream.new
+
+        relation.each do |obj|
+          record = obj.serializable_hash rescue nil
+          if record
+            if @time_column && tv = obj.read_attribute(@time_column)
+              if tv.is_a?(Time)
+                time = tv.to_i
+              else
+                time = Time.parse(tv.to_s).to_i rescue now
+              end
+            else
+              time = now
+            end
+            me.add(time, record)
+            last_record = record
+          end
+        end
+
+        last_record = last_record.dup if last_record  # some plugin rewrites record :(
+        @router.emit_stream(@tag, me)
+
+        return last_record
       end
+    end
+
+    def configure(conf)
+      super
 
-      def advanced_options
-        [
-            :select_interval,
-            :select_limit,
-            :state_file
-        ]
+      unless @state_file
+        $log.warn "'state_file PATH' parameter is not set to a 'sql' source."
+        $log.warn "this parameter is highly recommended to save the last rows to resume tailing."
       end
 
-      def table=(value)
-        @tables = value.map do |t|
-          t.map{|field, value| ([field.to_sym, value] if table_fields.include?(field.to_sym))}.to_h
-        end
+      @tables = conf.elements.select {|e|
+        e.name == 'table'
+      }.map {|e|
+        te = TableElement.new
+        te.configure(e)
+        te
+      }
+
+      if config['all_tables']
+        @all_tables = true
       end
+    end
+
+    SKIP_TABLE_REGEXP = /\Aschema_migrations\Z/i
+
+    def start
+      @state_store = @state_file.nil? ? MemoryStateStore.new : StateStore.new(@state_file)
+
+      config = {
+        :adapter => @adapter,
+        :host => @host,
+        :port => @port,
+        :database => @database,
+        :username => @username,
+        :password => @password,
+        :socket => @socket,
+      }
 
-      def table
-        @tables
+      # creates subclass of ActiveRecord::Base so that it can have different
+      # database configuration from ActiveRecord::Base.
+      @base_model = Class.new(ActiveRecord::Base) do
+        # base model doesn't have corresponding phisical table
+        self.abstract_class = true
       end
 
-      def table_fields
-        [
-            :table,
-            :tag,
-            :update_column,
-            :time_column,
-            :primary_key
-        ]
+      # ActiveRecord requires the base_model to have a name. Here sets name
+      # of an anonymous class by assigning it to a constant. In Ruby, class has
+      # a name of a constant assigned first
+      SQLInput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
+
+      # Now base_model can have independent configuration from ActiveRecord::Base
+      @base_model.establish_connection(config)
+
+      if @all_tables
+        # get list of tables from the database
+        @tables = @base_model.connection.tables.map do |table_name|
+          if table_name.match(SKIP_TABLE_REGEXP)
+            # some tables such as "schema_migrations" should be ignored
+            nil
+          else
+            te = TableElement.new
+            te.configure({
+              'table' => table_name,
+              'tag' => table_name,
+              'update_column' => nil,
+            })
+            te
+          end
+        end.compact
       end
 
-      def to_config
-        indent = "  "
-        config = "<source>\n"
-        config << "#{indent}type #{plugin_type_name}\n"
-        self.class.const_get(:KEYS).each do |key|
-          next if key == :table
-          next if key == :all_tables
-          config << indent
-          config << conf(key)
-          config << "\n"
-        end
-        tables = send(:table).reject{|t| t.values.join('') == ''} rescue []
-        if tables.present? && all_tables != '1'
-          tables.each do |tab|
-            config << "\n"
-            config << indent
-            config << "<table>\n"
-            tab.each do |key, value|
-              config << indent
-              config << indent
-              config << "#{key} #{value}"
-              config << "\n"
-            end
-            config << indent
-            config << "</table>\n"
+      init_redis
+
+      # ignore tables if TableElement#init failed
+      @tables.reject! do |te|
+        begin
+          schema_name = "#{SERVER_PREFIX}_#{@tag_prefix}_#{te.tag.presence || te.table}_#{Digest::MD5.new.hexdigest(@base_model.connection.columns(te.table).map{|c| c.name}.to_s)[0..5]}"
+          unless get_schema_from_redis_by_name schema_name
+            generate_schema te, schema_name
           end
-        else
-          config << indent
-          config << 'all_tables'
-          config << "\n"
+
+          te.init(@tag_prefix, @base_model, router)
+          log.info "Selecting '#{te.table}' table"
+          false
+        rescue => e
+          log.warn "Can't handle '#{te.table}' table. Ignoring.", :error => e.message, :error_class => e.class
+          log.warn_backtrace e.backtrace
+          true
         end
+      end
 
+      @stop_flag = false
+      @thread = Thread.new(&method(:thread_main))
+    end
 
-        config << "</source>\n"
-        config.gsub(/^[ ]*\n/m, "")
-      end
-
-      def self.create_from_config config
-        @setting = {}
-        @tables = []
-        table = {}
-        to_table = false
-        str_params = config.split("\r\n").map{|str| str.squish}
-        str_params.each do |str|
-          case str
-            when '<source>', '</source>'
-              next
-            when '<table>'
-              to_table = true
-            when '</table>'
-              to_table = false
-              @tables << table
-              table = {}
-            when 'all_tables'
-              @setting[:all_tables] = true
-            else
-              param = str.split(' ')
-              if to_table
-                table[param.first] = param.second
-              else
-                @setting[param.first] = param.second unless param.first == 'type'
-              end
+    def shutdown
+      @stop_flag = true
+    end
+
+    def thread_main
+      until @stop_flag
+        sleep @select_interval
+
+        @tables.each do |t|
+          begin
+            last_record = @state_store.last_records[t.table]
+            @state_store.last_records[t.table] = t.emit_next_records(last_record, @select_limit)
+            @state_store.update!
+          rescue => e
+            log.error "unexpected error", :error => e.message, :error_class => e.class
+            log.error_backtrace e.backtrace
           end
-          if @tables.present?
-            @setting[:table] = @tables
+        end
+      end
+    end
+
+    def init_redis
+      $redis = Redis.new
+    end
+
+    def generate_schema table, schema_name
+      require "avro_turf"
+      require 'avro_turf/messaging'
+      require "avro/builder"
+      avro = AvroTurf::Messaging.new(registry_url: "http://#{KAFKA_SERVER}:8081")
+      fields = @base_model.connection.columns(table.table).map do |col|
+        col_type = if col.sql_type.include? 'bigint'
+                     'long'
+                   elsif ['int', 'bool'].any? {|needle| col.sql_type.include?(needle)}
+                     'int'
+                   elsif ['float', 'double', 'real'].any? {|needle| col.sql_type.include?(needle)}
+                     'float'
+                   else
+                     'string'
+                   end
+        {
+            'name' => col.name,
+            'type' => ['null', col_type]
+        }
+      end
+      field_types = fields.map{|field| [field['name'], (field['type'] - ['null']).first]}.to_h
+      fields << {"name" => "enchilada_timestamp", "type" => "long"}
+      fields << {"name" => "enchilada_time_with_format", "type" => "string"}
+      schema_json = {
+          "type": "record",
+          "name": schema_name,
+          "fields": fields
+      }.to_json
+      registry = avro.instance_variable_get('@registry')
+      schema = Avro::Schema.parse(schema_json)
+      schema_id = registry.register("#{schema_name}-value", schema)
+
+      stored_schema = {
+          'schema_json' => schema_json,
+          'schema_id' => schema_id,
+          'field_types' => field_types,
+          'schema' => schema
+      }
+
+      set_schema_to_redis(schema_name, stored_schema)
+    end
+
+    def set_schema_to_redis schema_name, schema
+      $redis.set(schema_name, schema.to_json)
+    end
+
+    def get_schema_from_redis_by_name schema_name
+      stored_schema = $redis.get(schema_name)
+    end
+
+    class StateStore
+      def initialize(path)
+        require 'yaml'
+
+        @path = path
+        if File.exists?(@path)
+          @data = YAML.load_file(@path)
+          if @data == false || @data == []
+            # this happens if an users created an empty file accidentally
+            @data = {}
+          elsif !@data.is_a?(Hash)
+            raise "state_file on #{@path.inspect} is invalid"
           end
+        else
+          @data = {}
         end
-        self.new @setting
       end
 
-      def plugin_name
-        "sql"
+      def last_records
+        @data['last_records'] ||= {}
+      end
+
+      def update!
+        File.open(@path, 'w') {|f|
+          f.write YAML.dump(@data)
+        }
+      end
+    end
+
+    class MemoryStateStore
+      def initialize
+        @data = {}
+      end
+
+      def last_records
+        @data['last_records'] ||= {}
+      end
+
+      def update!
       end
     end
   end
+
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql-enchanced
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-31 00:00:00.000000000 Z
+date: 2017-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd