fluent-plugin-sql 0.2.1 → 0.2.2

data/README.md

@@ -2,18 +2,18 @@
 
 ## Overview
 
-This sql input plugin reads records from a RDBMS periodically. Thus you can replicate tables to other storages through Fluentd.
+This sql input plugin reads records from a RDBMS periodically. Thus you can copy tables to other storages through Fluentd.
 
 ## How does it work?
 
-This plugin runs following SQL repeatedly every 60 seconds to *tail* a table like `tail` command of UNIX.
+This plugin runs following SQL periodically:
 
 SELECT * FROM *table* WHERE *update\_column* > *last\_update\_column\_value* ORDER BY *update_column* ASC LIMIT 500
 
-What you need to configure is *update\_column*. The column needs to be updated every time when you update the row so that this plugin detects newly updated rows. Generally, the column is a timestamp such as `updated_at`.
-If you omit to set the column, it uses primary key. And this plugin can't detect updated but it only reads newly inserted rows.
+What you need to configure is *update\_column*. The column should be an incremental column (such as AUTO\_ INCREMENT primary key) so that this plugin reads newly INSERTed rows. Alternatively, you can use a column incremented every time when you update the row (such as `last_updated_at` column) so that this plugin reads the UPDATEd rows as well.
+If you omit to set *update\_column* parameter, it uses primary key.
 
-It stores last selected rows to a file named state\_file to not forget the last row when fluentd restarted.
+It stores last selected rows to a file (named *state\_file*) to not forget the last row when Fluentd restarts.
 
 ## Configuration
 
@@ -26,25 +26,25 @@ It stores last selected rows to a file named state\_file to not forget the last
       user myusername
       password mypassword
 
-      tag_prefix my.rdb
+      tag_prefix my.rdb  # optional, but recommended
 
-      select_interval 60s
-      select_limit 500
+      select_interval 60s  # optional
+      select_limit 500     # optional
 
       state_file /var/run/fluentd/sql_state
 
       <table>
-        tag table1
         table table1
+        tag table1  # optional
         update_column update_col1
-        time_column time_col2
+        time_column time_col2  # optional
       </table>
 
       <table>
-        tag table2
         table table2
+        tag table2  # optional
         update_column updated_at
-        time_column updated_at
+        time_column updated_at  # optional
       </table>
 
       # detects all tables instead of <table> sections
@@ -67,6 +67,11 @@ It stores last selected rows to a file named state\_file to not forget the last
 
 * **tag** tag name of events (optional; default value is table name)
 * **table** RDBM table name
-* **update_column**
-* **time_column** (optional)
+* **update_column**: see above description
+* **time_column** (optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.
 
+## Limitation
+
+You should make sure target tables have index (and/or partitions) on the *update\_column*. Otherwise SELECT causes full table scan and serious performance problem.
+
+You can't replicate DELETEd rows.

data/VERSION

@@ -1 +1 @@
-0.2.1
+0.2.2

data/fluent-plugin-sql.gemspec

@@ -15,6 +15,7 @@ Gem::Specification.new do |gem|
   gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
+  gem.license = "Apache 2.0"
 
   gem.add_dependency "fluentd", "~> 0.10.0"
   gem.add_dependency 'activerecord', ['3.2.12']

data/lib/fluent/plugin/in_sql.rb

@@ -54,16 +54,32 @@ module Fluent
       def init(tag_prefix, base_model)
         @tag = "#{tag_prefix}.#{@tag}" if tag_prefix
 
+        # creates a model for this table
         table_name = @table
         @model = Class.new(base_model) do
           self.table_name = table_name
           self.inheritance_column = '_never_use_'
+          #self.include_root_in_json = false
+
+          def read_attribute_for_serialization(n)
+            v = send(n)
+            if v.respond_to?(:to_msgpack)
+              v
+            else
+              v.to_s
+            end
+          end
         end
+
+        # ActiveRecord requires model class to have a name.
         class_name = table_name.singularize.camelize
         base_model.const_set(class_name, @model)
+
+        # Sets model_name otherwise ActiveRecord causes errors
         model_name = ActiveModel::Name.new(@model, nil, class_name)
         @model.define_singleton_method(:model_name) { model_name }
 
+        # if update_column is not set, here uses primary key
         unless @update_column
           columns = Hash[@model.columns.map {|c| [c.name, c] }]
           pk = columns[@model.primary_key]
@@ -74,6 +90,7 @@ module Fluent
         end
       end
 
+      # emits next records and returns the last record of emitted records
       def emit_next_records(last_record, limit)
         relation = @model
         if last_record && last_update_value = last_record[@update_column]
@@ -86,10 +103,14 @@ module Fluent
 
         me = MultiEventStream.new
         relation.each do |obj|
-          record = obj.as_json[entry_name] rescue nil
+          record = obj.serializable_hash rescue nil
           if record
-            if tv = record[@time_column]
-              time = Time.parse(tv.to_s) rescue now
+            if @time_column && tv = obj.read_attribute(@time_column)
+              if tv.is_a?(Time)
+                time = tv.to_i
+              else
+                time = Time.parse(tv.to_s).to_i rescue now
+              end
             else
               time = now
             end
@@ -98,6 +119,7 @@ module Fluent
           end
         end
 
+        last_record = last_record.dup  # some plugin rewrites record :(
         Engine.emit_stream(@tag, me)
 
         return last_record
@@ -134,15 +156,26 @@ module Fluent
         :password => @password,
       }
 
+      # creates subclass of ActiveRecord::Base so that it can have different
+      # database configuration from ActiveRecord::Base.
       @base_model = Class.new(ActiveRecord::Base) do
+        # base model doesn't have corresponding phisical table
         self.abstract_class = true
       end
+
+      # ActiveRecord requires the base_model to have a name. Here sets name
+      # of an anonymous class by assigning it to a constant. In Ruby, class has
+      # a name of a constant assigned first
       SQLInput.const_set("BaseModel_#{rand(1<<31)}", @base_model)
+
+      # Now base_model can have independent configuration from ActiveRecord::Base
       @base_model.establish_connection(config)
 
       if @all_tables
+        # get list of tables from the database
         @tables = @base_model.connection.tables.map do |table_name|
           if table_name.match(SKIP_TABLE_REGEXP)
+            # some tables such as "schema_migrations" should be ignored
             nil
           else
             te = TableElement.new
@@ -156,6 +189,7 @@ module Fluent
         end.compact
       end
 
+      # ignore tables if TableElement#init failed
       @tables.reject! do |te|
         begin
           te.init(@tag_prefix, @base_model)
@@ -198,6 +232,12 @@ module Fluent
         @path = path
         if File.exists?(@path)
           @data = YAML.load_file(@path)
+          if @data == false || @data == []
+            # this happens if an users created an empty file accidentally
+            @data = {}
+          elsif !@data.is_a?(Hash)
+            raise "state_file on #{@path.inspect} is invalid"
+          end
         else
           @data = {}
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-04 00:00:00.000000000 Z
+date: 2013-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -104,7 +104,8 @@ files:
 - fluent-plugin-sql.gemspec
 - lib/fluent/plugin/in_sql.rb
 homepage: https://github.com/frsyuki/fluent-plugin-sql
-licenses: []
+licenses:
+- Apache 2.0
 post_install_message: 
 rdoc_options: []
 require_paths: