fluent-plugin-sqlite3 0.0.0 → 0.1.0

data/lib/fluent/plugin/out_sqlite3.rb

@@ -1,2 +1,85 @@
-class Sqlite3Output < Fluent::BufferedOutput
+require "sqlite3"
+
+class Fluent::Sqlite3Output < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('sqlite3', self)
+
+  config_param :path,     :string
+  config_param :table,    :string, :default => nil
+  config_param :columns,  :string, :default => nil
+  config_param :includes, :string, :default => nil
+  config_param :excludes, :string, :default => nil
+
+  def initialize
+    super
+  end
+
+  def configure(conf)
+    super
+    @type = conf["type"]
+    if (@table and not(@columns)) or (not(@table) and @columns)
+      raise "strict mode requires table and columns parameters"
+    end
+  end
+
+  DELIMITER = / *, */
+  
+  def start
+    super
+    @db = ::SQLite3::Database.new @path
+    @stmts = {}
+    if @table
+      cols = @columns.split(DELIMITER).map {|e| ":#{e}"}.join(",")
+      @stmts[@table] = @db.prepare "INSERT INTO #{@table}(#{@columns}) VALUES(#{cols})"
+    end
+  end
+
+  def to_insert(table, columns)
+    cols = columns.split(DELIMITER).map {|e| ":#{e}"}.join(",")
+    "INSERT INTO #{table}(#{columns}) VALUES(#{cols})"
+  end
+
+  def shutdown
+    super
+    $log.debug "shutdown"
+    @stmts.each {|k,v| v.close}
+    @db.close
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def write(chunk)
+    @db.transaction
+    begin
+      write1(chunk)
+      @db.commit
+    rescue => ex
+      @db.rollback
+      $log.error "rollback: ", ex
+    end
+  end
+
+  def write1(chunk)
+    chunk.msgpack_each do |tag, time, record|
+      if record.keys.length == 0
+        $log.warn "no any keys for #{tag}"
+        return
+      end
+      table = (@table or tag.slice(@type.length + 1, tag.length))
+      if @includes
+        (record.keys - @includes.split(DELIMITER)).each {|e| record.delete e}
+      end
+      if @excludes
+        @excludes.split(DELIMITER).each {|e| record.delete e}
+      end
+      unless @stmts[table]
+        cols = record.keys.join ","
+        @db.execute "CREATE TABLE IF NOT EXISTS #{table} (id INTEGER PRIMARY KEY AUTOINCREMENT,#{cols})"
+        @stmts[table] = @db.prepare (a = to_insert(table, cols))
+        $log.debug "create a new table, #{table.upcase} (it may have been already created)"
+      end
+      @stmts[table].execute record
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sqlite3
 version: !ruby/object:Gem::Version
-  version: 0.0.0
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,8 +9,40 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-26 00:00:00.000000000 Z
-dependencies: []
+date: 2013-02-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: fluentd output to sqlite3
 email: ktmtmks@gmail.com
 executables: []
@@ -18,7 +50,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - lib/fluent/plugin/out_sqlite3.rb
-homepage: http://rubygems.org/gems/fluent-plugin-sqlite3
+homepage: https://github.com/tmtk75/fluent-plugin-sqlite3.git
 licenses: []
 post_install_message: 
 rdoc_options: []