logstash-input-jdbc 0.1.3 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cec484e2d7cd55a6842131e8f5918c85e3352c35
-  data.tar.gz: fc85af02081d8e3a94ba080c685935b35f6ff548
+  metadata.gz: 9026f853d22bd2451bfe72e0b045318ec605d77b
+  data.tar.gz: df99f72e2ba6ddffc1f9d40c193189a1941c2dad
 SHA512:
-  metadata.gz: bbd57390af1d2bc22a066c640dbcb19b773a9b5288070f9348f03685042f2c30cf600d74fa234f52956ea4de171c8272e27305b0813447111687cb3e71b21afd
-  data.tar.gz: aad311585c5b5774b0f5b602528d8a693bc78dd08d0e04701ab91bf262462e684fa5491cc1a0ed1d8a985caf40bcb528bc1cefee2aee88ba451dfe3925ba5456
+  metadata.gz: 948d8d265d5b6158daef86e4032958b344f49d0116ca30f6f7d3c697e875b19f69eb12ef63627c6a94c4e8bbcb5c389c9e6f935853ce5552fd2967216fe1cf92
+  data.tar.gz: e77a3ee5bac991ef164db17776c5e35713c97e4d4667a3f5703d4b09d2a15f7b67d8e6a5732e42ce450cbaa71a1223a9f5f52282d78e7915eb8c234abfef12d5

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+* 1.0.0
+  - Initial release

data/NOTICE.TXT

@@ -0,0 +1,5 @@
+Elasticsearch
+Copyright 2012-2015 Elasticsearch
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).

data/README.md

@@ -1,7 +1,5 @@
 # Logstash JDBC Input Plugin
 
-## WIP: Under Development, NOT FOR PRODUCTION
-
 This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
@@ -15,7 +13,7 @@ Logstash provides infrastructure to automatically generate documentation for thi
 
 ## Need Help?
 
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
 
 ## Developing
 
@@ -86,6 +84,37 @@ bin/plugin install /your/local/plugin/logstash-input-jdbc.gem
 ```
 - Start Logstash and proceed to test the plugin
 
+## Example configuration
+
+Reading data from MySQL:  
+
+	input {
+	  jdbc {
+	    jdbc_driver_library => "/path/to/mysql-connector-java-5.1.33-bin.jar"
+	    jdbc_driver_class => "com.mysql.jdbc.Driver"
+	    jdbc_connection_string => "jdbc:mysql://host:port/database"
+	    jdbc_user => "user"
+	    jdbc_password => "password"
+	    statement => "SELECT ..."
+	    jdbc_paging_enabled => "true"
+	    jdbc_page_size => "50000"
+	  }
+	}
+
+	filter {
+	  [some filters here]
+	}
+
+	output {
+	  stdout {
+	    codec => rubydebug
+	  }
+	  elasticsearch_http {
+	    host => "host"
+	    index => "myindex"
+	  }
+	}
+
 ## Contributing
 
 All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.

data/lib/logstash/inputs/jdbc.rb

@@ -2,41 +2,99 @@
 require "logstash/inputs/base"
 require "logstash/namespace"
 require "logstash/plugin_mixins/jdbc"
+require "yaml" # persistence
 
-# INFORMATION
+# This plugin was created as a way to ingest data in any database
+# with a JDBC interface into Logstash. You can periodically schedule ingestion
+# using a cron syntax (see `schedule` setting) or run the query one time to load
+# data into Logstash. Each row in the resultset becomes a single event.
+# Columns in the resultset are converted into fields in the event.
 #
-# This plugin was created as a way to iteratively ingest any database
-# with a JDBC interface into Logstash.
+# ==== Drivers
 #
-# #### JDBC Mixin
+# This plugin does not come packaged with JDBC driver libraries. The desired 
+# jdbc driver library must be explicitly passed in to the plugin using the
+# `jdbc_driver_library` configuration option.
+# 
+# ==== Scheduling
 #
-# This plugin utilizes a mixin that helps Logstash plugins manage JDBC connections.
-# The mixin provides its own set of configurations (some are required) to properly 
-# set up the connection to the appropriate database.
+# Input from this plugin can be scheduled to run periodically according to a specific 
+# schedule. This scheduling syntax is powered by [rufus-scheduler](https://github.com/jmettraux/rufus-scheduler).
+# The syntax is cron-like with some extensions specific to Rufus (e.g. timezone support ).
 #
-# #### Predefined Parameters
-#
-# Some parameters are built-in and can be used from within your queries.
-# Here is the list:
+# Examples:
 #
 # |==========================================================
-# |sql_last_start |The time the last query executed in plugin
+# | * 5 * 1-3 *               | will execute every minute of 5am every day of January through March.
+# | 0 * * * *                 | will execute on the 0th minute of every hour every day.
+# | 0 6 * * * America/Chicago | will execute at 6:00am (UTC/GMT -5) every day.
 # |==========================================================
+#   
+#
+# Further documentation describing this syntax can be found [here](https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings)
+#
+# ==== State
+#
+# The plugin will persist the `sql_last_start` parameter in the form of a 
+# metadata file stored in the configured `last_run_metadata_path`. Upon shutting down, 
+# this file will be updated with the current value of `sql_last_start`. Next time
+# the pipeline starts up, this value will be updated by reading from the file. If 
+# `clean_run` is set to true, this value will be ignored and `sql_last_start` will be
+# set to Jan 1, 1970, as if no query has ever been executed.
+#
+# ==== Dealing With Large Result-sets
+#
+# Many JDBC drivers use the `fetch_size` parameter to limit how many
+# results are pre-fetched at a time from the cursor into the client's cache
+# before retrieving more results from the result-set. This is configured in
+# this plugin using the `jdbc_fetch_size` configuration option. No fetch size
+# is set by default in this plugin, so the specific driver's default size will 
+# be used.
+#
+# ==== Usage:
+#
+# Here is an example of setting up the plugin to fetch data from a MySQL database.
+# First, we place the appropriate JDBC driver library in our current
+# path (this can be placed anywhere on your filesystem). In this example, we connect to 
+# the 'mydb' database using the user: 'mysql' and wish to input all rows in the 'songs'
+# table that match a specific artist. The following examples demonstrates a possible 
+# Logstash configuration for this. The `schedule` option in this example will 
+# instruct the plugin to execute this input statement on the minute, every minute.
 #
-# #### Usage:
-# This is an example logstash config
 # [source,ruby]
+# ----------------------------------
 # input {
 #   jdbc {
-#     jdbc_driver_class => "org.apache.derby.jdbc.EmbeddedDriver" (required; from mixin)
-#     jdbc_connection_string => "jdbc:derby:memory:testdb;create=true" (required; from mixin)
-#     jdbc_user => "username" (from mixin)
-#     jdbc_password => "mypass" (from mixin)
-#     statement => "SELECT * from table where created_at > :sql_last_start and id = :my_id" (required)
-#     parameters => { "my_id" => "231" }
+#     jdbc_driver_library => "mysql-connector-java-5.1.36-bin.jar"
+#     jdbc_driver_class => "com.mysql.jdbc.Driver"
+#     jdbc_connection_string => "jdbc:mysql://localhost:3306/mydb"
+#     jdbc_user => "mysql"
+#     parameters => { "favorite_artist" => "Beethoven" }
 #     schedule => "* * * * *"
+#     statement => "SELECT * from songs where artist = :favorite_artist"
 #   }
 # }
+# ----------------------------------
+#
+# ==== Configuring SQL statement
+# 
+# A sql statement is required for this input. This can be passed-in via a 
+# statement option in the form of a string, or read from a file (`statement_filepath`). File 
+# option is typically used when the SQL statement is large or cumbersome to supply in the config.
+# The file option only supports one SQL statement. The plugin will only accept one of the options.
+# It cannot read a statement from a file as well as from the `statement` configuration parameter.
+#
+# ==== Predefined Parameters
+#
+# Some parameters are built-in and can be used from within your queries.
+# Here is the list:
+#
+# |==========================================================
+# |sql_last_start | The last time a statement was executed. This is set to
+# |               | Thursday, 1 January 1970 before any query is run, and updated 
+# |               | accordingly after first query is run.
+# |==========================================================
+#
 class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   include LogStash::PluginMixins::Jdbc
   config_name "jdbc"
@@ -45,25 +103,59 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   default :codec, "plain" 
 
   # Statement to execute
+  #
   # To use parameters, use named parameter syntax.
   # For example:
+  #
+  # [source, ruby]
+  # ----------------------------------
   # "SELECT * FROM MYTABLE WHERE id = :target_id"
-  # here ":target_id" is a named parameter
+  # ----------------------------------
   #
-  config :statement, :validate => :string, :required => true
+  # here, ":target_id" is a named parameter. You can configure named parameters
+  # with the `parameters` setting.
+  config :statement, :validate => :string
+
+  # Path of file containing statement to execute
+  config :statement_filepath, :validate => :path
 
   # Hash of query parameter, for example `{ "target_id" => "321" }`
   config :parameters, :validate => :hash, :default => {}
 
   # Schedule of when to periodically run statement, in Cron format
   # for example: "* * * * *" (execute query every minute, on the minute)
+  #
+  # There is no schedule by default. If no schedule is given, then the statement is run
+  # exactly once.
   config :schedule, :validate => :string
 
+  # Path to file with last run time
+  config :last_run_metadata_path, :validate => :string, :default => "#{ENV['HOME']}/.logstash_jdbc_last_run"
+
+  # Whether the previous run state should be preserved
+  config :clean_run, :validate => :boolean, :default => false
+
+  # Whether to save state or not in last_run_metadata_path
+  config :record_last_run, :validate => :boolean, :default => true
+
   public
 
   def register
     require "rufus/scheduler"
-    prepare_jdbc_connection()
+    prepare_jdbc_connection
+
+    # load sql_last_start from file if exists
+    if @clean_run && File.exist?(@last_run_metadata_path)
+      File.delete(@last_run_metadata_path)
+    elsif File.exist?(@last_run_metadata_path)
+      @sql_last_start = YAML.load(File.read(@last_run_metadata_path))
+    end
+
+    unless @statement.nil? ^ @statement_filepath.nil?
+      raise(LogStash::ConfigurationError, "Must set either :statement or :statement_filepath. Only one may be set at a time.")
+    end
+
+    @statement = File.read(@statement_filepath) if @statement_filepath
   end # def register
 
   def run(queue)
@@ -79,13 +171,18 @@ class LogStash::Inputs::Jdbc < LogStash::Inputs::Base
   end # def run
 
   def teardown
-    if @scheduler
-      @scheduler.stop
+    @scheduler.stop if @scheduler
+
+    # update state file for next run
+    if @record_last_run
+      File.write(@last_run_metadata_path, YAML.dump(@sql_last_start))
     end
-    close_jdbc_connection()
+
+    close_jdbc_connection
   end # def teardown
 
   private
+
   def execute_query(queue)
     # update default parameters
     @parameters['sql_last_start'] = @sql_last_start

data/lib/logstash/plugin_mixins/jdbc.rb

@@ -13,14 +13,14 @@ module LogStash::PluginMixins::Jdbc
     # Add these methods to the 'base' given.
     base.extend(self)
     base.setup_jdbc_config
-
-    @sql_last_start = Time.at(0).utc
   end
 
 
   public
   def setup_jdbc_config
     # JDBC driver library path to third party driver library.
+    #
+    # If not provided, Plugin will look for the driver class in the Logstash Java classpath.
     config :jdbc_driver_library, :validate => :path
 
     # JDBC driver class to load, for example "oracle.jdbc.OracleDriver" or "org.apache.derby.jdbc.ClientDriver"
@@ -30,18 +30,33 @@ module LogStash::PluginMixins::Jdbc
     config :jdbc_connection_string, :validate => :string, :required => true
 
     # JDBC user
-    config :jdbc_user, :validate => :string, :default => "tal"
+    config :jdbc_user, :validate => :string, :required => true
 
     # JDBC password
     config :jdbc_password, :validate => :password
 
+    # JDBC enable paging
+    #
+    # This will cause a sql statement to be broken up into multiple queries.
+    # Each query will use limits and offsets to collectively retrieve the full
+    # result-set. The limit size is set with `jdbc_page_size`.
+    #
+    # Be aware that ordering is not guaranteed between queries.
+    config :jdbc_paging_enabled, :validate => :boolean, :default => false 
+
+    # JDBC page size
+    config :jdbc_page_size, :validate => :number, :default => 100000
+
+    # JDBC fetch size. if not provided, respective driver's default will be used
+    config :jdbc_fetch_size, :validate => :number
+
     # Connection pool configuration.
     # Validate connection before use.
     config :jdbc_validate_connection, :validate => :boolean, :default => false
 
     # Connection pool configuration.
     # How often to validate a connection (in seconds)
-    config :jdcb_validation_timeout, :validate => :number, :default => 3600
+    config :jdbc_validation_timeout, :validate => :number, :default => 3600
   end
 
   public
@@ -50,18 +65,32 @@ module LogStash::PluginMixins::Jdbc
     require "sequel"
     require "sequel/adapters/jdbc"
     require @jdbc_driver_library if @jdbc_driver_library
-    Sequel::JDBC.load_driver(@jdbc_driver_class)
+    begin
+      Sequel::JDBC.load_driver(@jdbc_driver_class)
+    rescue Sequel::AdapterNotFound => e
+      message = if @jdbc_driver_library.nil?
+                  ":jdbc_driver_library is not set, are you sure you included 
+                  the proper driver client libraries in your classpath?"
+                else
+                  "Are you sure you've included the correct jdbc driver in :jdbc_driver_library?"
+                end
+      raise LogStash::ConfigurationError, "#{e}. #{message}"
+    end
     @database = Sequel.connect(@jdbc_connection_string, :user=> @jdbc_user, :password=>  @jdbc_password.nil? ? nil : @jdbc_password.value)
+    @database.extension(:pagination)
     if @jdbc_validate_connection
       @database.extension(:connection_validator)
-      @database.pool.connection_validation_timeout = @jdcb_validation_timeout
+      @database.pool.connection_validation_timeout = @jdbc_validation_timeout
     end
+    @database.fetch_size = @jdbc_fetch_size unless @jdbc_fetch_size.nil?
     begin
       @database.test_connection
     rescue Sequel::DatabaseConnectionError => e
       #TODO return false and let the plugin raise a LogStash::ConfigurationError
       raise e
     end
+
+    @sql_last_start = Time.at(0).utc
   end # def prepare_jdbc_connection
 
   public
@@ -77,9 +106,19 @@ module LogStash::PluginMixins::Jdbc
       query = @database[statement, parameters]
       @logger.debug? and @logger.debug("Executing JDBC query", :statement => statement, :parameters => parameters)
       @sql_last_start = Time.now.utc
-      query.all do |row|
-        #Stringify row keys
-        yield Hash[row.map { |k, v| [k.to_s, v] }]
+
+      if @jdbc_paging_enabled
+        query.each_page(@jdbc_page_size) do |paged_dataset|
+          paged_dataset.each do |row|
+            #Stringify row keys
+            yield Hash[row.map { |k, v| [k.to_s, v] }]
+          end
+        end
+      else
+        query.each do |row|
+          #Stringify row keys
+          yield Hash[row.map { |k, v| [k.to_s, v] }]
+        end
       end
       success = true
     rescue Sequel::DatabaseConnectionError, Sequel::DatabaseError => e

data/logstash-input-jdbc.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-jdbc'
-  s.version         = '0.1.3'
+  s.version         = '1.0.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This example input streams a string at a definable interval."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/jdbc_spec.rb

@@ -1,84 +1,304 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/jdbc"
 require "jdbc/derby"
+require "sequel"
+require "sequel/adapters/jdbc"
 require "timecop"
+require "stud/temporary"
 
 
 describe "jdbc" do
-  let(:mixin_settings) { {"jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver", "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"} }
-
-  before :each do
-    Jdbc::Derby.load_driver
+  let(:mixin_settings) { {"jdbc_user" => ENV['USER'], "jdbc_driver_class" => "org.apache.derby.jdbc.EmbeddedDriver", "jdbc_connection_string" => "jdbc:derby:memory:testdb;create=true"} }
+  let(:settings) { {} }
+  let(:plugin) { LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings)) }
+  let(:queue) { Queue.new }
+  let (:db) do
+    Sequel.connect(mixin_settings['jdbc_connection_string'], :user=> nil, :password=> nil)
   end
 
-  it "should register and tear down" do
-    settings = {"statement" => "SELECT 1 as col1 FROM SYSIBM.SYSDUMMY1"}
-    plugin = LogStash::Plugin.lookup("input", "jdbc").new(mixin_settings.merge(settings))
-    expect { plugin.register }.to_not raise_error
-    expect { plugin.teardown }.to_not raise_error
-  end
-
-  it "should retrieve params correctly from Event" do
-    settings = {"statement" => "SELECT :num_param as num_param FROM SYSIBM.SYSDUMMY1", "parameters" => {"num_param" => 10} }
-    plugin = LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings))
-    plugin.register
-    q = Queue.new
-    plugin.run(q)
-    insist { q.size } == 1
-    insist { q.pop['num_param'] } == settings['parameters']['num_param']
-    plugin.teardown
-  end
-
-  it "should properly schedule" do
-    settings = {"statement" => "SELECT 1 as num_param FROM SYSIBM.SYSDUMMY1", "schedule" => "* * * * *"}
-    plugin = LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings))
-    plugin.register
-    q = Queue.new
-    Timecop.travel(Time.new(2000))
-    Timecop.scale(60)
-    runner = Thread.new do
-      plugin.run(q)
-    end
-    sleep 3
-    plugin.teardown
-    runner.kill
-    runner.join
-    insist { q.size } == 2
-    Timecop.return
-  end
-
-  it "should successfully iterate table with respect to field values" do
-    require "sequel"
-    require "sequel/adapters/jdbc"
+  before :each do
     Jdbc::Derby.load_driver
-    @database = Sequel.connect(mixin_settings['jdbc_connection_string'], :user=> nil, :password=> nil)
-    @database.create_table :test_table do
+    db.create_table :test_table do
       DateTime :created_at
       Integer :num
     end
-    test_table = @database[:test_table]
-    settings = {"statement" => "SELECT num, created_at FROM test_table WHERE created_at > :sql_last_start"}
-    plugin = LogStash::Inputs::Jdbc.new(mixin_settings.merge(settings))
-    plugin.register
-    q = Queue.new
+  end
+
+  after :each do
+    db.drop_table(:test_table)
+  end
+
+  context "when registering and tearing down" do
+    let(:settings) { {"statement" => "SELECT 1 as col1 FROM test_table"} }
+
+    it "should register without raising exception" do
+      expect { plugin.register }.to_not raise_error
+      plugin.teardown
+    end
+
+    it "should teardown without raising exception" do
+      plugin.register
+      expect { plugin.teardown }.to_not raise_error
+    end
+  end
+
+  context "when neither statement and statement_filepath arguments are passed" do
+    it "should fail to register" do
+      expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "when both statement and statement_filepath arguments are passed" do
+    let(:statement) { "SELECT * from test_table" }
+    let(:statement_file_path) { Stud::Temporary.pathname }
+    let(:settings) { { "statement_filepath" => statement_file_path, "statement" => statement } }
+
+    it "should fail to register" do
+      expect{ plugin.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "when statement is passed in from a file" do
+    let(:statement) { "SELECT * from test_table" }
+    let(:statement_file_path) { Stud::Temporary.pathname }
+    let(:settings) { { "statement_filepath" => statement_file_path } }
+
+    before do
+      File.write(statement_file_path, statement)
+      plugin.register
+    end
+
+    after do
+      plugin.teardown
+    end
+
+    it "should read in statement from file" do
+      expect(plugin.statement).to eq(statement)
+    end
+  end
+
+  context "when passing parameters" do
+    let(:settings) do
+      {
+        "statement" => "SELECT :num_param as num_param FROM SYSIBM.SYSDUMMY1",
+        "parameters" => {"num_param" => 10}
+      }
+    end
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.teardown
+    end
+
+    it "should retrieve params correctly from Event" do
+      plugin.run(queue)
+      expect(queue.pop['num_param']).to eq(settings['parameters']['num_param'])
+    end
+  end
+
+  context "when scheduling" do
+    let(:settings) { {"statement" => "SELECT 1 as num_param FROM SYSIBM.SYSDUMMY1", "schedule" => "* * * * * UTC"} }
+
+    before do
+      plugin.register
+    end
+
+    it "should properly schedule" do
+      Timecop.travel(Time.new(2000))
+      Timecop.scale(60)
+      runner = Thread.new do
+        plugin.run(queue)
+      end
+      sleep 3
+      plugin.teardown
+      runner.kill
+      runner.join
+      expect(queue.size).to eq(2)
+      Timecop.return
+    end
+
+  end
+
+  context "when iterating result-set via paging" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from test_table",
+        "jdbc_paging_enabled" => true,
+        "jdbc_page_size" => 20
+      }
+    end
+
+    let(:num_rows) { 1000 }
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.teardown
+    end
+
+    it "should fetch all rows" do
+      num_rows.times do
+        db[:test_table].insert(:num => 1, :created_at => Time.now.utc)
+      end
+
+      plugin.run(queue)
+
+      expect(queue.size).to eq(num_rows)
+    end
+
+  end
+
+  context "when iteratively running plugin#run" do
+    let(:settings) do
+      {"statement" => "SELECT num, created_at FROM test_table WHERE created_at > :sql_last_start"}
+    end
+
+    let(:nums) { [10, 20, 30, 40, 50] }
 
-    nums = [10, 20, 30, 40, 50]
-    plugin.run(q)
-    test_table.insert(:num => nums[0], :created_at => Time.now.utc)
-    test_table.insert(:num => nums[1], :created_at => Time.now.utc)
-    plugin.run(q)
-    test_table.insert(:num => nums[2], :created_at => Time.now.utc)
-    test_table.insert(:num => nums[3], :created_at => Time.now.utc)
-    test_table.insert(:num => nums[4], :created_at => Time.now.utc)
-    plugin.run(q)
+    before do
+      plugin.register
+    end
 
-    actual_sum = 0
-    until q.empty? do
-      actual_sum += q.pop['num']
+    after do
+      plugin.teardown
     end
 
-    plugin.teardown
+    it "should successfully iterate table with respect to field values" do
+      test_table = db[:test_table]
+
+      plugin.run(queue)
+      test_table.insert(:num => nums[0], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[1], :created_at => Time.now.utc)
+      plugin.run(queue)
+      test_table.insert(:num => nums[2], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[3], :created_at => Time.now.utc)
+      test_table.insert(:num => nums[4], :created_at => Time.now.utc)
+      plugin.run(queue)
+
+      actual_sum = 0
+      until queue.empty? do
+        actual_sum += queue.pop['num']
+      end
+
+      expect(actual_sum).to eq(nums.inject{|sum,x| sum + x })
+    end
+  end
+
+  context "when previous runs are to be respected" do
+
+    let(:settings) do
+      { "statement" => "SELECT * FROM test_table",
+        "last_run_metadata_path" => Stud::Temporary.pathname }
+    end
+
+    let(:last_run_time) { Time.at(1).utc }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_time))
+      plugin.register
+    end
+
+    after do
+      plugin.teardown
+    end
+
+    it "should respect last run metadata" do
+      expect(plugin.instance_variable_get("@sql_last_start")).to eq(last_run_time)
+    end
+  end
+
+  context "when doing a clean run" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * FROM test_table",
+        "last_run_metadata_path" => Stud::Temporary.pathname,
+        "clean_run" => true
+      }
+    end
+
+    let(:last_run_time) { Time.at(1).utc }
+
+    before do
+      File.write(settings["last_run_metadata_path"], YAML.dump(last_run_time))
+      plugin.register
+    end
+
+    after do
+      plugin.teardown
+    end
 
-    insist { actual_sum } == nums.inject{|sum,x| sum + x }
+    it "should ignore last run metadata if :clean_run set to true" do
+      expect(plugin.instance_variable_get("@sql_last_start")).to eq(Time.at(0).utc)
+    end
+  end
+
+  context "when state is not to be persisted" do
+    let(:settings) do
+      {
+        "statement" => "SELECT * FROM test_table",
+        "last_run_metadata_path" => Stud::Temporary.pathname,
+        "record_last_run" => false
+      }
+    end
+
+    before do
+      plugin.register
+    end
+
+    after do
+      plugin.teardown
+    end
+
+    it "should not save state if :record_last_run is false" do
+      expect(File).not_to exist(settings["last_run_metadata_path"])
+    end
+  end
+
+  context "when setting fetch size" do
+
+    let(:settings) do
+      {
+        "statement" => "SELECT * from test_table",
+        "jdbc_fetch_size" => 1
+      }
+    end
+
+    let(:num_rows) { 10 }
+
+    before do
+      num_rows.times do
+        db[:test_table].insert(:num => 1, :created_at => Time.now.utc)
+      end
+
+      plugin.register
+    end
+
+    after do
+      plugin.teardown
+    end
+
+    it "should fetch all rows" do
+      plugin.run(queue)
+      expect(queue.size).to eq(num_rows)
+    end
+  end
+
+  context "when driver is not found" do
+    let(:settings) { { "statement" => "SELECT * FROM test_table" } }
+
+    before do
+      mixin_settings['jdbc_driver_class'] = "org.not.ExistsDriver"
+    end
+
+    it "should fail" do
+      expect { plugin.register }.to raise_error(LogStash::ConfigurationError)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 1.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-06-04 00:00:00.000000000 Z
+date: 2015-07-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
@@ -138,6 +138,7 @@ files:
 - CHANGELOG.md
 - Gemfile
 - LICENSE
+- NOTICE.TXT
 - README.md
 - Rakefile
 - lib/logstash/inputs/jdbc.rb