logstash-input-mongoprofile 0.1.15 → 0.1.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d4b22c8ef7d441e7d6d7130e51f18990fe34f272
-  data.tar.gz: 3534f85ef64368f518f18ac27cb259a64c862a46
+  metadata.gz: 91179ba0a1df249cae35aa208dbe2439272eb878
+  data.tar.gz: d5df9210fd7cf7f60331f7d0c0a229ff670a4e69
 SHA512:
-  metadata.gz: 40e4941008a948ab16952549645d78b6a4194d566024aba8bb1b38b66c2c9d83643228d141e42a76e2399acf9927917ab632adb20499608ca0aba456ddb847b6
-  data.tar.gz: f560de9428f0d222f5810218cc1782efe77bd74f9143d257621b7a2191f48d7b4b3db9e41c8ae54f39f5d963bb652364b5b8c88c683f74aa025d9c8e1b5e5307
+  metadata.gz: b18b777e1f8d2862b563ce5bf698415ab6983de2712e35753657c838c577573031236b4e9ceca02c4f58e08ce63516ecdf04cdb8dd50b5f5856d69862422c3a2
+  data.tar.gz: cf27b8bae36e7aad52b6f1a03ec117cecc9644a51350369314f773533b97f5bf14ab56f8036aea7ff737f79b376379b0841032b226ad196546fa4f378f99bb2e

data/lib/logstash/inputs/mongoprofile.rb

@@ -75,11 +75,11 @@ class MongoAccessor
   end
 
   def get_documents_by_ts(date, limit)
-    @collection.find({:ts => {:$gt => DateTime.parse(date)}, :client => {:$ne => @client_host}}).limit(limit).sort(:ts => -1)
+    @collection.find({:ts => {:$gt => DateTime.parse(date) + 0.00002}, :client => {:$ne => @client_host}}).limit(limit).sort(:ts => 1)
   end
 
   def get_documents(limit)
-    @collection.find({:client => {:$ne => @client_host}}).limit(limit).sort(:ts => -1)
+    @collection.find({:client => {:$ne => @client_host}}).limit(limit).sort(:ts => 1)
   end
 end
 
@@ -103,10 +103,12 @@ class ProfileCollection
 
       yield @parser.parse(document)
     end
+
+    @documents = []
   end
 
   def get_last_document_date
-    if @documents and @documents[-1] != nil
+    if @documents != nil and @documents[-1] != nil
       @documents[-1]['ts']
     else
       nil
@@ -183,9 +185,9 @@ class Controller
 
     if  profile_collection.get_last_document_date != nil
       @last_value_store.save_last_value(profile_collection.get_last_document_date)
-    end
-      @logger.info('Nothing to get...')
     else
+      @logger.info('Nothing to get...')
+    end
 
     profile_collection
   end

data/lib/logstash/test.rb

@@ -1,3 +1,176 @@
-require('date')
-a = DateTime.parse("2017-06-22 08:14:12 UTC\n")
-puts a
+require 'mongo'
+
+class Test
+  def each
+    arr = [1, 2, 3]
+
+    arr.each do |i|
+      yield i
+    end
+
+    puts 'end each'
+  end
+end
+
+test = Test.new
+
+test.each do |i|
+  puts i
+end
+
+class EventMock
+  def set(arg)
+    puts arg
+  end
+end
+
+class LoggerMock
+  def info(arg)
+    puts arg
+  end
+
+  def debug(arg)
+    puts arg
+  end
+end
+
+
+class MongoAccessor
+  def initialize(url, collection, client_host)
+    connection = Mongo::Client.new(url)
+
+    @mongodb = connection.database
+    @collection = @mongodb.collection(collection)
+    @client_host = client_host
+  end
+
+  def get_documents_by_ts(date, limit)
+    @collection.find({:ts => {:$gt => DateTime.parse(date) + 0.00002}, :client => {:$ne => @client_host}}).limit(limit).sort(:ts => 1)
+  end
+
+  def get_documents(limit)
+    @collection.find({:client => {:$ne => @client_host}}).limit(limit).sort(:ts => 1)
+  end
+end
+
+class ProfileCollection
+  def initialize(documents, parser, generate_id)
+    @generate_id = generate_id
+    @documents = []
+
+    documents.each do |document|
+      @documents.push(document)
+      puts document['ts']
+    end
+
+    @parser = parser
+  end
+
+  def each
+    @documents.each do |document|
+      if @generate_id
+        document['_id'] = generate_id.to_s
+      end
+
+      yield @parser.parse(document)
+    end
+
+    @documents = []
+  end
+
+  def get_last_document_date
+    if @documents != nil and @documents[-1] != nil
+      @documents[-1]['ts']
+    else
+      nil
+    end
+  end
+
+  def length
+    @documents.length
+  end
+
+  private
+  def generate_id
+    # noinspection RubyArgCount
+    BSON::ObjectId.new
+  end
+end
+
+class DocumentParser
+  def initialize(host, logger)
+    @host = host
+    @logger = logger
+  end
+
+  def parse(document)
+    @logger.info('Start documents parsing')
+    event = LogStash::Event.new('host' => @host)
+
+    document.each do |key, value|
+      @logger.debug("Try set event field key: #{key} value: #{value}")
+      event.set(key, value)
+    end
+
+    event
+  end
+end
+
+class LastValueStore
+  def initialize(path, name)
+    @file_full_name = "#{path}/#{name}"
+  end
+
+  def save_last_value(value)
+    file = File.open(@file_full_name, 'a+')
+
+    file.truncate(0)
+    file.puts(value)
+
+    file.close
+  end
+
+  def get_last_value
+    File.read(@file_full_name)
+  end
+end
+
+class Controller
+  def initialize(event, url, collection, limit, path, client_host, logger, generate_id)
+    @mongo_accessor = MongoAccessor.new(url, collection, client_host)
+    @last_value_store = LastValueStore.new(path, collection)
+    @document_parser = DocumentParser.new(event, logger)
+    @generate_id = generate_id
+    @limit = limit
+    @logger = logger
+  end
+
+  def get_next_events
+    last_date_value = @last_value_store.get_last_value
+
+    if last_date_value == ''
+      @logger.info('Getting documents from mongo first time')
+      documents = @mongo_accessor.get_documents(@limit)
+    else
+      @logger.info("Getting documents from mongo start at #{last_date_value}")
+      documents = @mongo_accessor.get_documents_by_ts(last_date_value, @limit)
+    end
+
+    profile_collection = ProfileCollection.new(documents, @document_parser, @generate_id)
+
+    if profile_collection.get_last_document_date != nil
+      @last_value_store.save_last_value(profile_collection.get_last_document_date)
+    else
+      @logger.info('Nothing to get...')
+    end
+
+    profile_collection
+  end
+end
+
+controller = Controller.new(EventMock.new, 'mongodb://192.168.1.37/eleet-v2-dev', 'system.profile', 10, '/home/artem/', '192.168.1.35', LoggerMock.new, false)
+
+while true
+  events = controller.get_next_events
+  puts events.length
+end

data/logstash-input-mongoprofile.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-input-mongoprofile'
-  s.version       = '0.1.15'
+  s.version       = '0.1.16'
   s.licenses      = ['Apache License (2.0)']
   s.summary       = 'MongoDB system.profile input plugin'
   s.description   = 'MongoDB system.profile input plugin'

data/spec/inputs/mongoprofile_spec.rb

@@ -5,7 +5,7 @@ require "logstash/inputs/mongoprofile"
 describe LogStash::Inputs::Mongoprofile do
 
   it_behaves_like "an interruptible input plugin" do
-    let(:config) { { "interval" => 100 } }
+    let(:config) { { "interval" => 100, "url" => "mongodb://192.168.1.37/eleet-v2-dev", "path" => "/home/artem"} }
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-mongoprofile
 version: !ruby/object:Gem::Version
-  version: 0.1.15
+  version: 0.1.16
 platform: ruby
 authors:
 - Artem Antonov