sms-logparser 0.12.1 → 0.12.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8392eba52e67bfc44639dc8b2e916ca9497335e2
-  data.tar.gz: 3409ba5f83db6a33334d58bbd2ccb1f00e1165ae
+  metadata.gz: f0b45a865df500739a1bea384895fbee8fbad15c
+  data.tar.gz: 41b69f59f7d1d691a66e7c0b2cced2ad16e4ec66
 SHA512:
-  metadata.gz: b3350474625d5cc675cfb18b5e4243ce81e6912ef123df9a86d9d7bef67a59b831926678288684e683b58043b29314ee7ff9a8794566319ece8f2f6045f12148
-  data.tar.gz: ac650bc8109cede2eb992035c94e0ae6936633e21cb3d408ece3c1a9a00962204bcf0e0a18a41208c1aa759bff075344a42a3bdf59e5d28b060a904531215dd5
+  metadata.gz: 39d0e9abc5cb3777f2b222196b94387ede340817fcf52b58999bc9c93175ae20f156b384940ada9946703799883ed20fb4c3a2f6eb792a7f830cb2f04571eb5f
+  data.tar.gz: 8c3efb728d6c670cdcac62bf9cecbf022d65a2f1dc9e12528dde9aa89e77a0cbcfa6e3ee4e8d416606edf00f5bbcecedab0aad51059bea24d41be265420f369c

data/lib/sms-logparser/api.rb

@@ -13,7 +13,6 @@ module SmsLogparser
       @data_cache = {}
     end
 
-
     def send(data)
       requests = build_urls(data)
       return requests if @options[:simulate]
@@ -36,11 +35,11 @@ module SmsLogparser
       requests
     end
 
-    def send_from_queue(data_sets)
+    def send_sets(data_sets, concurrency=4)
       queue     = Queue.new
       semaphore = Mutex.new
       data_sets.each {|set| queue << set }
-      threads = 4.times.map do
+      threads = concurrency.times.map do
         Thread.new do
           while !queue.empty?
             begin

data/lib/sms-logparser/cli.rb

@@ -33,10 +33,14 @@ module SmsLogparser
     option :limit, type: :numeric, aliases: %w(-L), desc: "Limit the number of entries to query"
     option :accepted_api_responses, type: :array, aliases: %w(-r),
       desc: "API HTTP responses which are accepted (Default: only accept 200)."
+    option :cache, type: :boolean, desc: "Accumulate and cache results and send totals"
+    option :concurrency, type: :numeric, default: 4,
+      desc: "How many threads to use in parallel when sending cached results"
     def parse
       start_message = "Parser started"
       start_message += options[:simulate] ? " in simulation mode." : "."
       logger.info(start_message)
+      cache = DataCache.new if options[:cache]
       mysql = Mysql.new(options)
       if !options[:simulate] && mysql.parser_running?
         logger.warn("Exit. Another instance of the parser is already running.")
@@ -55,83 +59,23 @@ module SmsLogparser
         entries.each do |entry| 
           Parser.extract_data_from_msg(entry['Message']) do |data|
             if data
-              requests = api.send(data)
-              state[:match_count] += 1
-              verbose_parser_output(entry['ID'], data, requests)
+              if options[:cache]
+                cache.add(data)
+                logger.debug {"Cached data: #{data}"}
+              else
+                requests = api.send(data)
+                verbose_parser_output(entry['ID'], data, requests)
+              end
               state[:last_event_id] = entry['ID']
-            end
-          end
-        end
-      end
-    rescue SystemExit, Interrupt
-      logger.error("Received an interrupt. Stopping the parser run.")
-      state[:status] = STATUS[:interrupted] if state
-    rescue => e
-      logger.error "Aborting the parser run."
-      logger.error e
-      state[:status] = STATUS[:api_error] if state
-    else
-      state[:status] = STATUS[:ok]
-    ensure
-      begin
-        if mysql && state
-          state[:run_time] = (Time.now - state[:started_at]).round(2)
-          if state[:id]
-            mysql.write_parse_result(state) unless options[:simulate]
-          end
-          log_parse_results(state)
-          SmsLogparser::Loggster.instance.close
-        end
-      rescue => e
-        logger.fatal e
-      end
-    end
-
-    desc "cached_pase", "Check the database for pcache logs and put them into the cache"
-    option :api_base_url, aliases: %w(-a),
-      desc: "Base path of the SMS API (Default: http://localhost:8080/creator/rest/)"
-    option :api_key, aliases: %w(-k), desc: "SMS API Key"
-    option :simulate, type: :boolean, aliases: %w(-s),
-      desc: "Dry run without submitting any data"
-    option :verbose, type: :boolean, aliases: %w(-v), desc: "Verbose output"
-    option :limit, type: :numeric, aliases: %w(-L), desc: "Limit the number of entries to query"
-    option :accepted_api_responses, type: :array, aliases: %w(-r),
-      desc: "API HTTP responses which are accepted (Default: only accept 200)."
-    def cached_parse
-      start_message = "Parser started"
-      start_message += options[:simulate] ? " in simulation mode." : "."
-      logger.info(start_message)
-      mysql = Mysql.new(options)
-      if !options[:simulate] && mysql.parser_running?
-        logger.warn("Exit. Another instance of the parser is already running.")
-        exit!
-      end
-      state = {
-        last_event_id: mysql.get_last_parse_id, 
-        match_count: 0,
-        status: STATUS[:running],
-        started_at: Time.now,
-        run_time: 0.0
-      }
-      state = mysql.start_run(state) unless options[:simulate]
-      cache = DataCache.new
-      mysql = Mysql.new(options)
-      say "Getting entries from database..."
-      mysql.get_entries(last_id: state[:last_event_id], limit: options[:limit]) do |entries|
-        entries.each do |entry| 
-          Parser.extract_data_from_msg(entry['Message']) do |data|
-            if data
-              cache.add(data)
-              logger.debug {"Cached data: #{data}"}
               state[:match_count] += 1
-              state[:last_event_id] = entry['ID']
             end
           end
         end
       end
-      api = Api.new(options)
-      api.send_from_queue(cache.data_sets) do |url, response|
-        say "#{url} (#{response})"
+      if options[:cache]
+        api.send_sets(cache.data_sets, options[:concurrency]) do |url, response|
+          logger.info { "POST #{url} (#{response})" }
+        end
       end
     rescue SystemExit, Interrupt
       logger.error("Received an interrupt. Stopping the parser run.")
@@ -155,7 +99,6 @@ module SmsLogparser
       rescue => e
         logger.fatal e
       end
-
     end
 
     desc "history", "List the last paser runs"

data/lib/sms-logparser/mysql.rb

@@ -108,7 +108,7 @@ module SmsLogparser
     def select_entries(last_id, max_id)
       query = %Q{
         SELECT ID, Message FROM SystemEvents
-        WHERE ID BETWEEN #{last_id} AND #{max_id}
+        WHERE ID BETWEEN #{last_id + 1} AND #{max_id}
         ORDER BY ID ASC
         LIMIT #{@query_limit};
       }.gsub(/\s+/, " ").strip

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.12.1"
+  VERSION = "0.12.2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.12.2
 platform: ruby
 authors:
 - niwo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-15 00:00:00.000000000 Z
+date: 2014-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler