logstash-input-mongodb 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 069aa95b8c9675b2079f871d66a496da135741f8
-  data.tar.gz: ee102f0da33e5a562bcc4facfa28a63f743a466a
+  metadata.gz: 797d63879b842bd324be17f4afafeb142cbc16d4
+  data.tar.gz: 8becb42077309b41d88a2b1808d00980433af012
 SHA512:
-  metadata.gz: 32c33f58cd391152b7c0ce788007cff3428c3b675a9a2734039d00d88b989ca921623b7b398b60c250ff38eb6c42187b4576552b2ccd40a1649e2e914c9db1a5
-  data.tar.gz: f5e8da5a9250404735a6d5c0d92c2870c14e73c8bab1768a0da522805ef5c2527ce7c011d197d154d4803e1224bd9e99e77d43b722dfbe379e6ea80e295ab9fb
+  metadata.gz: 9b0f0be58b4a1b57cd24d8560e5bb71649bf93775e33bc570c94ac30a39c2fecc2baa5e88f00e80058af69dc89594835ec90747fbe9cc196043337fd89b3043b
+  data.tar.gz: e97852fee59bba3b14739fef7a89f6fcde0e9f29a3eede187d006688c68f178ebe668b6c144cff1e4fa72ff4a82ab213835d7e19446bf557006b51eeda41219d

data/lib/logstash/inputs/mongodb.rb

@@ -79,12 +79,24 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     end
   end
 
+  public
+  def init_placeholder(sqlitedb, since_table, mongodb, mongo_collection_name)
+    @logger.debug("init placeholder for #{since_table}_#{mongo_collection_name}")
+    since = sqlitedb[SINCE_TABLE]
+    mongo_collection = mongodb.collection(mongo_collection_name)
+    first_entry = mongo_collection.find({}, :sort => ['_id', Mongo::ASCENDING], :limit => 1).first
+    first_entry_id = first_entry['_id'].to_s
+    since.insert(:table => "#{since_table}_#{mongo_collection_name}", :place => first_entry_id)
+    return first_entry_id
+  end
+
   public
   def get_placeholder(sqlitedb, since_table, mongodb, mongo_collection_name)
     since = sqlitedb[SINCE_TABLE]
     x = since.where(:table => "#{since_table}_#{mongo_collection_name}")
     if x[:place].nil? || x[:place] == 0
       first_entry_id = init_placeholder(sqlitedb, since_table, mongodb, mongo_collection_name)
+      @logger.debug("FIRST ENTRY ID for #{mongo_collection_name} is #{first_entry_id}")
       return first_entry_id
     else
       @logger.debug("placeholder already exists, it is #{x[:place]}")
@@ -92,20 +104,9 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     end
   end
 
-  public
-  def init_placeholder(sqlitedb, since_table, mongodb, mongo_collection_name)
-    @logger.debug("init placeholder for #{since_table}_#{mongo_collection_name}")
-    since = sqlitedb[SINCE_TABLE]
-    mongo_collection = mongodb.collection(mongo_collection_name)
-    first_entry = mongo_collection.find_one({})
-    first_entry_id = first_entry['_id'].to_s
-    since.insert(:table => "#{since_table}_#{mongo_collection_name}", :place => first_entry_id)
-    return first_entry_id
-  end
-
   public
   def update_placeholder(sqlitedb, since_table, mongo_collection_name, place)
-    @logger.debug("updating placeholder for #{since_table}_#{mongo_collection_name} to #{place}")
+    #@logger.debug("updating placeholder for #{since_table}_#{mongo_collection_name} to #{place}")
     since = sqlitedb[SINCE_TABLE]
     since.where(:table => "#{since_table}_#{mongo_collection_name}").update(:place => place)
   end
@@ -130,7 +131,9 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
   public
   def get_cursor_for_collection(mongodb, mongo_collection_name, last_id_object, batch_size)
     collection = mongodb.collection(mongo_collection_name)
-    return collection.find({:_id => {:$gt => last_id_object}}).limit(batch_size)
+    # Need to make this sort by date in object id then get the first of the series
+    # db.events_20150320.find().limit(1).sort({ts:1})
+    return collection.find({:_id => {:$gte => last_id_object}}).limit(batch_size)
   end
 
   public
@@ -222,24 +225,23 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
           collection_name = collection[:name]
           @logger.debug("collection_data is: #{@collection_data}")
           last_id = @collection_data[index][:last_id]
-          @logger.debug("last_id is #{last_id}", :index => index, :collection => collection_name)
+          #@logger.debug("last_id is #{last_id}", :index => index, :collection => collection_name)
           # get batch of events starting at the last_place if it is set
           last_id_object = BSON::ObjectId(last_id)
           cursor = get_cursor_for_collection(@mongodb, collection_name, last_id_object, batch_size)
           cursor.each do |doc|
-            @logger.debug("Date from mongo: #{doc['_id'].generation_time.to_s}")
             logdate = DateTime.parse(doc['_id'].generation_time.to_s)
-            @logger.debug("logdate.iso8601: #{logdate.iso8601}")
             event = LogStash::Event.new("host" => @host)
             decorate(event)
             event["logdate"] = logdate.iso8601
-            @logger.debug("type of doc is: "+doc.class.to_s)
             log_entry = doc.to_h.to_s
             log_entry['_id'] = log_entry['_id'].to_s
             event["log_entry"] = log_entry
-            @logger.debug("EVENT looks like: "+event.to_s)
-            @logger.debug("Sent message: "+doc.to_h.to_s)
-            @logger.debug("EVENT looks like: "+event.to_s)
+            event["mongo_id"] = doc['_id'].to_s
+            @logger.debug("mongo_id: "+doc['_id'].to_s)
+            #@logger.debug("EVENT looks like: "+event.to_s)
+            #@logger.debug("Sent message: "+doc.to_h.to_s)
+            #@logger.debug("EVENT looks like: "+event.to_s)
             # Extract the HOST_ID and PID from the MongoDB BSON::ObjectID
             if @unpack_mongo_id
               doc_obj_bin = doc['_id'].to_a.pack("C*").unpack("a4 a3 a2 a3")
@@ -253,6 +255,14 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
               # Flatten the JSON so that the data is usable in Kibana
               flat_doc = flatten(doc)
               # Check for different types of expected values and add them to the event
+              if flat_doc['info_message'] && (flat_doc['info_message']  =~ /collection stats: .+/)
+                # Some custom stuff I'm having to do to fix formatting in past logs...
+                sub_value = flat_doc['info_message'].sub("collection stats: ", "")
+                JSON.parse(sub_value).each do |k1,v1|
+                  flat_doc["collection_stats_#{k1.to_s}"] = v1
+                end
+              end
+
               flat_doc.each do |k,v|
                 # Check for an integer
                 if /\A[-+]?\d+[.][\d]+\z/ === v
@@ -303,11 +313,12 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
 
             queue << event
             @collection_data[index][:last_id] = doc['_id'].to_s
-            @collection_data = update_watched_collections(@mongodb, @collection, @sqlitedb)
           end
           # Store the last-seen doc in the database
           update_placeholder(@sqlitedb, since_table, collection_name, @collection_data[index][:last_id])
         end
+        @logger.debug("Updating watch collections")
+        @collection_data = update_watched_collections(@mongodb, @collection, @sqlitedb)
 
         # nothing found in that iteration
         # sleep a bit

data/logstash-input-mongodb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-mongodb'
-  s.version = '0.1.0'
+  s.version = '0.1.1'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This takes entries from mongodb as an input to logstash."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Philip Hutchins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-18 00:00:00.000000000 Z
+date: 2015-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash