sms-logparser 0.12.0 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be3b90122d60da5a8342511ee4a4aa3879c549a7
-  data.tar.gz: 9928653741ef481378f0761ba5f5d5f25c83bc74
+  metadata.gz: 8392eba52e67bfc44639dc8b2e916ca9497335e2
+  data.tar.gz: 3409ba5f83db6a33334d58bbd2ccb1f00e1165ae
 SHA512:
-  metadata.gz: bc1cb0028bd7ac7392a7e504aebd6f471b8f636f0983872f7e2f7e23b9dba18eb11359f34714efaf2d49b33aefa849583821a8b003af0bfe82db52961a23830f
-  data.tar.gz: 139425fa7c82772ab61af0c2aefe88acec215177f559f770da307b8df00c4e848e4c834704b73d5c8ea80b47fbcc8fc1280b6d19f0498f519f80b9715151ea98
+  metadata.gz: b3350474625d5cc675cfb18b5e4243ce81e6912ef123df9a86d9d7bef67a59b831926678288684e683b58043b29314ee7ff9a8794566319ece8f2f6045f12148
+  data.tar.gz: ac650bc8109cede2eb992035c94e0ae6936633e21cb3d408ece3c1a9a00962204bcf0e0a18a41208c1aa759bff075344a42a3bdf59e5d28b060a904531215dd5

data/lib/sms-logparser/api.rb

@@ -14,7 +14,6 @@ module SmsLogparser
     end
 
 
-
     def send(data)
       requests = build_urls(data)
       return requests if @options[:simulate]
@@ -37,6 +36,45 @@ module SmsLogparser
       requests
     end
 
+    def send_from_queue(data_sets)
+      queue     = Queue.new
+      semaphore = Mutex.new
+      data_sets.each {|set| queue << set }
+      threads = 4.times.map do
+        Thread.new do
+          while !queue.empty?
+            begin
+              data = queue.pop
+              url = @base_path + [
+                data[:customer_id],
+                data[:author_id],
+                data[:project_id],
+                data[:type],
+                data[:value]
+              ].join('/')
+              if @options[:simulate]
+                semaphore.synchronize {
+                  yield url, 0
+                }
+                break
+              end
+              response = @connection.post(url)
+            rescue => e
+              raise RuntimeError, "Can't send request to #{url}. #{e.message}", caller
+            end
+            unless @accepted_responses.include?(response.status)
+              msg = "Received HTTP status #{response.status} from API. Only accepting #{@accepted_responses.join(', ')}."
+              raise RuntimeError, msg, caller
+            end
+            semaphore.synchronize {
+              yield url, response.status
+            }
+          end
+        end
+      end
+      threads.each {|thread| thread.join }
+    end
+
     def build_urls(data)
       requests = []
       path = @base_path + [data[:customer_id], data[:author_id], data[:project_id]].join('/')

data/lib/sms-logparser/cli.rb

@@ -88,24 +88,74 @@ module SmsLogparser
     end
 
     desc "cached_pase", "Check the database for pcache logs and put them into the cache"
+    option :api_base_url, aliases: %w(-a),
+      desc: "Base path of the SMS API (Default: http://localhost:8080/creator/rest/)"
+    option :api_key, aliases: %w(-k), desc: "SMS API Key"
+    option :simulate, type: :boolean, aliases: %w(-s),
+      desc: "Dry run without submitting any data"
+    option :verbose, type: :boolean, aliases: %w(-v), desc: "Verbose output"
     option :limit, type: :numeric, aliases: %w(-L), desc: "Limit the number of entries to query"
+    option :accepted_api_responses, type: :array, aliases: %w(-r),
+      desc: "API HTTP responses which are accepted (Default: only accept 200)."
     def cached_parse
+      start_message = "Parser started"
+      start_message += options[:simulate] ? " in simulation mode." : "."
+      logger.info(start_message)
+      mysql = Mysql.new(options)
+      if !options[:simulate] && mysql.parser_running?
+        logger.warn("Exit. Another instance of the parser is already running.")
+        exit!
+      end
+      state = {
+        last_event_id: mysql.get_last_parse_id, 
+        match_count: 0,
+        status: STATUS[:running],
+        started_at: Time.now,
+        run_time: 0.0
+      }
+      state = mysql.start_run(state) unless options[:simulate]
       cache = DataCache.new
       mysql = Mysql.new(options)
       say "Getting entries from database..."
-      mysql.get_entries(last_id: mysql.get_last_parse_id, limit: options[:limit]) do |entries|
+      mysql.get_entries(last_id: state[:last_event_id], limit: options[:limit]) do |entries|
         entries.each do |entry| 
           Parser.extract_data_from_msg(entry['Message']) do |data|
             if data
               cache.add(data)
-              say "Cached data ", :magenta
-              say data
+              logger.debug {"Cached data: #{data}"}
+              state[:match_count] += 1
+              state[:last_event_id] = entry['ID']
             end
           end
         end
       end
-      puts
-      puts cache.cache
+      api = Api.new(options)
+      api.send_from_queue(cache.data_sets) do |url, response|
+        say "#{url} (#{response})"
+      end
+    rescue SystemExit, Interrupt
+      logger.error("Received an interrupt. Stopping the parser run.")
+      state[:status] = STATUS[:interrupted] if state
+    rescue => e
+      logger.error "Aborting the parser run."
+      logger.error e
+      state[:status] = STATUS[:api_error] if state
+    else
+      state[:status] = STATUS[:ok]
+    ensure
+      begin
+        if mysql && state
+          state[:run_time] = (Time.now - state[:started_at]).round(2)
+          if state[:id]
+            mysql.write_parse_result(state) unless options[:simulate]
+          end
+          log_parse_results(state)
+          SmsLogparser::Loggster.instance.close
+        end
+      rescue => e
+        logger.fatal e
+      end
+
     end
 
     desc "history", "List the last paser runs"

data/lib/sms-logparser/data_cache.rb

@@ -20,6 +20,23 @@ module SmsLogparser
       @cache
     end
 
+    def data_sets
+      sets = []
+      types = %w(TRAFFIC_PODCAST TRAFFIC_MOBILE TRAFFIC_WEBCAST VISITORS_PODCAST VISITORS_MOBILE VISITORS_WEBCAST)
+      @cache.each do |key, values|
+        types.each do |type|
+          sets << {
+            customer_id: values[:customer_id],
+            author_id: values[:author_id],
+            project_id: values[:project_id],
+            type: type,
+            value: values[type]
+          } if values[type]
+        end
+      end
+      sets
+    end
+
     private
 
     def initialize_value(data)

data/lib/sms-logparser/version.rb

@@ -1,3 +1,3 @@
 module SmsLogparser
-  VERSION = "0.12.0"
+  VERSION = "0.12.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sms-logparser
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.1
 platform: ruby
 authors:
 - niwo