tac_scribe 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 485c9e87c957fd0e66db7cc762a36283baa2c4852546008b2d4dbd5e2372a53a
-  data.tar.gz: 95576f042c7455ac1e5b87bcab21d6a5ff555d8f67a83b4c7cd2531b56f696c1
+  metadata.gz: 3f357b6fb0a4a45accdc07e20ff29fe6a93c66d7b88665ea51a652ca28a2b1d4
+  data.tar.gz: b01e4759b8e523fee4541b71b43102fcb62d228a15b3c87bb6b969fc33e6e206
 SHA512:
-  metadata.gz: 1aefe11c62a16ac70377bc48f262b699f2aebb4c38480bbe677b1490e5d14320c0e213d1976e0b4a477e1687e24b8bccae1b9a7f111bdaaeca1c0ebba63f9d62
-  data.tar.gz: 2d54201ba482399aafd29a4633b3b7361220457e85e8aae0447be8920191fc708e3582617affe5176d6bacd6682688c73e315772e094a71ada60af1101b6a2f5
+  metadata.gz: fc97730d7e4b521a46987f2ea01d2680715933571ce33218e8ad69033a35df0b9722fcc123af082d71cd70487d7caab60765d171c5b29301445d0ec6c3178ea8
+  data.tar.gz: 3e27a7a3957fab068f168dad158a6ae555faab3a40fa8535c0ee65629fe8d4915255c41890ff0d0ea79e3a36b24e8642d4df4f3747a891fd90b175e4883a9a59

data/.rubocop.yml

@@ -14,7 +14,10 @@ Metrics/BlockLength:
     # There isn't much sense breaking up the OptionParser block since splitting
     # into db and tacview option methods will just break the method length cop
     # and splitting further doesn't aid readability
-    - exe/start_scribe
+    - exe/tac_scribe
+    # Cannot really avoid this
+    - tac_scribe.gemspec
+    
 Metrics/MethodLength:
   Exclude:
     # Breaking up the initializer doesn't really do much for readability
@@ -29,3 +32,13 @@ Style/GlobalVars:
   Exclude:
     # Using a global variable makes it available to the Pry console
     - bin/console
+AllCops:
+  Exclude:
+    # These files will need a MAJOR refactoring to get rid of these warnings
+    - lib/tac_scribe/cache.rb
+    - lib/tac_scribe/datastore.rb
+    - lib/tac_scribe/daemon.rb
+ClassVars:
+  Exclude:
+    # We are fine with this and understand the risks. We are not inheriting.
+    - lib/tac_scribe/event_processor.rb

data/CHANGELOG.md

@@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased]
+## [0.6.0]
+### Added
+- Calculates and stores speed of units
+
+### Fixed
+- Made more robust against failures and more logging messages
 
 ## [0.5.0]
 ### Changed

data/db/001_create_unit_table.rb

@@ -12,6 +12,7 @@ Sequel.migration do
       String :group, null: true
       Integer :coalition
       Integer :heading
+      Integer :speed
       Time :updated_at
       Boolean :deleted
       # TODO: GIST Index on the position

data/lib/tac_scribe/cache.rb

@@ -5,8 +5,11 @@ require 'sequel'
 require 'sequel-postgis-georuby'
 require 'singleton'
 require 'concurrent-ruby'
+require 'haversine'
 
 module TacScribe
+  # The in-memory cache that is updated by events in real-time before the data
+  # is synced to the DB
   class Cache
     include Singleton
     include GeoRuby::SimpleFeatures
@@ -48,9 +51,11 @@ module TacScribe
 
       if cache_object
         object[:heading] = calculate_heading(cache_object, object)
+        object[:speed] = calculate_speed(cache_object, object)
         cache_object.merge!(object)
       else
         object[:heading] = -1
+        object[:speed] = 0
         cache_object = object
       end
 
@@ -75,7 +80,7 @@ module TacScribe
     def delete_object(id)
       @@cache[id][:deleted] = true if @@cache[id]
     end
-    
+
     def clear
       @@cache.clear
     end
@@ -104,11 +109,16 @@ module TacScribe
                    cache_object[:position].x
                  end
 
+      # This "Point" class is not lat/long aware which is why we are
+      # flipping things up here because when it converts to SQL we need
+      # the long to be first since that is what postgresql expects.
       Point.from_x_y(longitude, latitude)
     end
 
     def calculate_heading(cache_object, object)
-      return cache_object[:heading] if cache_object[:position] == object[:position]
+      if cache_object[:position] == object[:position]
+        return cache_object[:heading]
+      end
 
       begin
         cache_object[:position].bearing_to(object[:position]).to_i
@@ -118,5 +128,21 @@ module TacScribe
         puts 'New Position: ' + object[:position].inspect
       end
     end
+
+    def calculate_speed(cache_object, object)
+      time = object[:game_time] - cache_object[:game_time]
+
+      start_point = cache_object[:position]
+      end_point = object[:position]
+
+      # Because of the above issues with Point and Lat/Lon
+      # the values are reverse here :(
+      distance = Haversine.distance(start_point.y, start_point.x,
+                                    end_point.y, end_point.x).to_meters
+
+      speed =  distance / time
+
+      speed.to_i
+    end
   end
 end

data/lib/tac_scribe/daemon.rb

@@ -31,7 +31,7 @@ module TacScribe
 
       @populate_airfields = populate_airfields
       @thread_count = thread_count
-      @threads = []
+      @threads = {}
       @whitelist = Set.new(IO.read(whitelist).split) if whitelist
 
       @client = TacviewClient::Client.new(
@@ -50,7 +50,7 @@ module TacScribe
     # for example
     def start_processing
       loop do
-        kill_threads
+        puts 'Starting processing loop'
         @event_queue.clear
         Datastore.instance.truncate_table
         Cache.instance.clear
@@ -58,23 +58,34 @@ module TacScribe
         start_db_sync_thread
         start_reporting_thread
         populate_airfields if @populate_airfields
+        @threads.each_pair do |key, _value|
+          puts "#{key} thread started"
+        end
         @client.connect
+        # If this code is executed it means we have been disconnected without
+        # exceptions. We will still sleep to stop a retry storm. Just not as
+        # long.
+        sleep 10
       # Rescuing reliably from Net::HTTP is a complete bear so rescue
       # StandardError. It ain't pretty but it is reliable. We will puts
       # the exception just in case
       # https://stackoverflow.com/questions/5370697/what-s-the-best-way-to-handle-exceptions-from-nethttp
       rescue StandardError => e
+        puts 'Exception in processing loop'
         puts e.message
         puts e.backtrace
+        kill_threads
         sleep 30
         next
       end
     end
 
     def kill_threads
-      @threads.each do |thr|
-        thr.kill
-        thr.join
+      puts 'Killing Threads'
+      @threads.each_pair do |key, thread|
+        puts "Killing #{key} thread"
+        thread.kill
+        thread.join
       end
     end
 
@@ -87,7 +98,7 @@ module TacScribe
         @event_processor.start
       end
       event_processor_thread.name = 'Event Processor'
-      @threads << event_processor_thread
+      @threads[:processing] = event_processor_thread
     end
 
     def start_db_sync_thread
@@ -96,16 +107,19 @@ module TacScribe
           sleep 1
           deleted = Datastore.instance.write_objects(Cache.instance.data.values)
           deleted.each { |id| Cache.instance.data.delete(id) }
+        rescue StandardError
+          next
         end
       end
       db_write_thread.name = 'Database Writing'
-      @threads << db_write_thread
+      @threads[:database] = db_write_thread
     end
 
     def start_reporting_thread
       return unless @verbose_logging
 
       reporting_thread = Thread.new do
+        sleep 5
         loop do
           puts "#{Time.now.strftime('%FT%T')}\t" \
                "Events Incoming: #{@event_queue.events_written}\t" \
@@ -123,11 +137,12 @@ module TacScribe
         end
       end
       reporting_thread.name = 'Reporting'
-      @threads << reporting_thread
+      @threads[:reporting] = reporting_thread
     end
 
     def populate_airfields
-      json = File.read(File.join(File.dirname(__FILE__), '../../data/airfields.json'))
+      json = File.read(File.join(File.dirname(__FILE__),
+                                 '../../data/airfields.json'))
       airfields = JSON.parse(json)
       airfields.each_with_index do |airfield, i|
         @event_queue.update_object(

data/lib/tac_scribe/datastore.rb

@@ -53,11 +53,18 @@ module TacScribe
     end
 
     def write_objects(objects)
+      objects = objects.map do |object|
+        obj = object.clone
+        obj.delete(:game_time)
+        obj
+      end
+
       @db[:units].insert_conflict(
         constraint: :units_pkey,
         update: { position: Sequel[:excluded][:position],
                   altitude: Sequel[:excluded][:altitude],
                   heading: Sequel[:excluded][:heading],
+                  speed: Sequel[:excluded][:speed],
                   updated_at: Sequel[:excluded][:updated_at],
                   deleted: Sequel[:excluded][:deleted] }
       )

data/lib/tac_scribe/event_processor.rb

@@ -23,7 +23,7 @@ module TacScribe
 
     def start
       loop do
-        wrapped_event = @event_queue.get_event
+        wrapped_event = @event_queue.shift
         process_event(wrapped_event)
       rescue StandardError => e
         puts wrapped_event
@@ -38,6 +38,10 @@ module TacScribe
         update_object(wrapped_event[:event], wrapped_event[:time])
       when :delete_object
         delete_object(wrapped_event[:object_id])
+      when :set_latitude
+        update_latitude wrapped_event[:value]
+      when :set_longitude
+        update_longitude wrapped_event[:value]
       end
     end
 
@@ -60,20 +64,28 @@ module TacScribe
     #   format.
     # @option event [BigDecimal] :altitude The object altitude above sea level
     #   in meters to 1 decimal place.
-    def update_object(event, _time)
+    def update_object(event, time)
+      return if ignore_unit?(even)
+
+      event[:game_time] = time
+
+      self.events_processed += 1
+      @cache.write_object(event)
+    end
+
+    def ignore_unit?(event)
       if @@ignored_units.include?(event[:object_id])
         self.events_ignored += 1
-        return
+        return true
       end
 
       if @whitelist && event[:type] && !@whitelist.include?(event[:type])
         @@ignored_units << event[:object_id]
         self.events_ignored += 1
-        return
+        return true
       end
 
-      self.events_processed += 1
-      @cache.write_object(event)
+      false
     end
 
     # Process a delete event for an object
@@ -81,12 +93,19 @@ module TacScribe
     # @param id [String] A hexadecimal number representing the object ID
     def delete_object(id)
       if @@ignored_units.delete?(id)
-        self.events_processed += 1
         nil
       else
-        self.events_processed += 1
         @cache.delete_object(id)
       end
+      self.events_processed += 1
+    end
+
+    def update_latitude(value)
+      Cache.instance.reference_latitude = value
+    end
+
+    def update_longitude(value)
+      Cache.instance.reference_longitude = value
     end
   end
 end

data/lib/tac_scribe/event_queue.rb

@@ -21,7 +21,7 @@ module TacScribe
       @events.size
     end
 
-    def get_event
+    def shift
       @events.shift
     end
 
@@ -71,9 +71,9 @@ module TacScribe
     def set_property(property:, value:)
       case property
       when 'ReferenceLatitude'
-        Cache.instance.reference_latitude = BigDecimal(value)
+        @events << { type: :set_latitude, value: BigDecimal(value) }
       when 'ReferenceLongitude'
-        Cache.instance.reference_longitude = BigDecimal(value)
+        @events << { type: :set_longitude, value: BigDecimal(value) }
       when 'ReferenceTime'
         @reference_time = @time = Time.parse(value)
       end

data/lib/tac_scribe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TacScribe
-  VERSION = '0.5.0'
+  VERSION = '0.6.0'
 end

data/tac_scribe.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   # of an API breaking change are higher then normal. Therefore lock the
   # version
   spec.add_dependency 'concurrent-ruby', '~>1.1.5'
+  spec.add_dependency 'haversine', '~>0.3'
   spec.add_dependency 'sequel-postgis-georuby', '0.1.2'
   spec.add_dependency 'tacview_client', '~>0.1'
 

metadata

@@ -1,178 +1,192 @@
 --- !ruby/object:Gem::Specification
 name: tac_scribe
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Jeffrey Jones
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-17 00:00:00.000000000 Z
+date: 2020-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
-  name: activerecord-jdbcpostgresql-adapter
-  prerelease: false
+        version: '1.1'
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
+  name: georuby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
-  name: georuby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
 - !ruby/object:Gem::Dependency
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.22'
-  name: sequel
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.22'
 - !ruby/object:Gem::Dependency
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.5
-  name: concurrent-ruby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.5
 - !ruby/object:Gem::Dependency
+  name: haversine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+- !ruby/object:Gem::Dependency
+  name: sequel-postgis-georuby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.1.2
-  name: sequel-postgis-georuby
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.1.2
 - !ruby/object:Gem::Dependency
+  name: tacview_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-  name: tacview_client
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
+  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-  name: bundler
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.8'
-  name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.8'
 - !ruby/object:Gem::Dependency
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.73'
-  name: rubocop
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.73'
 - !ruby/object:Gem::Dependency
+  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
-  name: simplecov
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
 - !ruby/object:Gem::Dependency
+  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-  name: yard
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -219,7 +233,7 @@ licenses:
 - AGPL-3.0-or-later
 metadata:
   yard.run: yri
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -234,9 +248,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
-signing_key:
+rubygems_version: 3.0.3
+signing_key: 
 specification_version: 4
 summary: Write Tacview data to PostGIS database
 test_files: []