tac_scribe 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddf21892437d67f745cc10328223aae85ade3459fb8dfdc5f758658cc5af4676
-  data.tar.gz: d71160d27dbc0a52b1b606c5b52ede5714a7ef1a7897bb18a93a3933003b393c
+  metadata.gz: f6540479b0949db23d8dcbfc7965a716d4909b7e03b314f9b1687b7b7b044e1b
+  data.tar.gz: 9c6ee7a6e020b81eea6cd809c043eb31f2353256a41463da5349ee78699d3f1b
 SHA512:
-  metadata.gz: bd25300946c4fb0d98749097c3e115af921b407802ad88fa8d6d3fea8d98eb3bf59874137d4ec560497cf9c477872a7af900e449ac568400109d65dd25c2c6e6
-  data.tar.gz: 56e62c543996a89203d85633094e745cc43193f4232d46790f8253df1fb3c407209e2d3d736f063be50625c29af319c1ad64f849c7761b1a268895447444edec
+  metadata.gz: 5236421e641674570e47e65c309ed12c0beff782cfd7fddea75d09687a4aa7660a445cc5249e2d8489f4a56df2cba8ec9fd76df1390b916053cd35ab9dc19f84
+  data.tar.gz: 3a500080c5d44f78c1bfd0914face93c40d318eb8721d7a2e0876a7e5cfdc9cfdddded214fe0f99da2fbcb356420d1319159f085cbcbdd2d61eac3064afaf4c3

data/CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.4.0]
+### Changed
+- Switch to periodic writing to the database
+
 ## [0.3.0] - 2019-10-27
 ### Changed
 - Changed the command-line name to match gem

data/lib/tac_scribe/cache.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'georuby'
+require 'sequel'
+require 'sequel-postgis-georuby'
+require 'singleton'
+require 'concurrent-ruby'
+
+module TacScribe
+  class Cache
+    include Singleton
+    include GeoRuby::SimpleFeatures
+
+    @@cache = Concurrent::Hash.new
+
+    attr_accessor :reference_latitude, :reference_longitude
+
+    def data
+      @@cache
+    end
+
+    def write_object(object)
+      if reference_latitude != 0 || reference_longitude != 0
+        localize_position(object)
+      end
+
+      id = object[:object_id]
+      object[:id] = id
+
+      cache_object = @@cache[id]
+
+      object[:position] = set_position(object, cache_object)
+
+      %i[object_id latitude longitude color country].each do |key|
+        object.delete(key)
+      end
+
+      if object[:coalition]
+        object[:coalition] = case object[:coalition]
+                             when 'Allies'
+                               0
+                             when 'Enemies'
+                               1
+                             else
+                               2
+                             end
+      end
+
+      if cache_object
+        object[:heading] = calculate_heading(cache_object, object)
+        cache_object.merge!(object)
+      else
+        object[:heading] = -1
+        cache_object = object
+      end
+
+      if !cache_object.key?(:altitude) || !cache_object[:altitude]
+        cache_object[:altitude] = 0
+      end
+      if !cache_object.key?(:coalition) || !cache_object[:coalition]
+        cache_object[:coalition] = 2
+      end
+
+      cache_object[:pilot] = nil unless cache_object.key?(:pilot)
+
+      cache_object[:group] = nil unless cache_object.key?(:group)
+
+      cache_object[:deleted] = false unless cache_object.key?(:deleted)
+
+      cache_object[:updated_at] = Time.now
+
+      @@cache[id] = cache_object
+    end
+
+    def delete_object(id)
+      @@cache[id][:deleted] = true if @@cache[id]
+    end
+
+    private
+
+    def localize_position(object)
+      if reference_latitude && object.key?(:latitude)
+        object[:latitude] = reference_latitude + object[:latitude]
+      end
+      if reference_longitude && object.key?(:longitude)
+        object[:longitude] = reference_longitude + object[:longitude]
+      end
+    end
+
+    def set_position(object, cache_object)
+      longitude = if object.key?(:longitude)
+                    object[:longitude]
+                  else
+                    cache_object[:position].y
+                  end
+
+      latitude = if object.key?(:latitude)
+                   object[:latitude]
+                 else
+                   cache_object[:position].x
+                 end
+
+      Point.from_x_y(longitude, latitude)
+    end
+
+    def calculate_heading(cache_object, object)
+      return cache_object[:heading] if cache_object[:position] == object[:position]
+
+      begin
+        cache_object[:position].bearing_to(object[:position]).to_i
+      rescue Math::DomainError => e
+        puts 'Could not calculate heading: ' + e.message
+        puts 'Old Position: ' + cache_object[:position].inspect
+        puts 'New Position: ' + object[:position].inspect
+      end
+    end
+  end
+end

data/lib/tac_scribe/daemon.rb

@@ -4,6 +4,7 @@ require 'json'
 require 'tacview_client'
 require 'set'
 require_relative 'datastore'
+require_relative 'cache'
 require_relative 'event_queue'
 require_relative 'event_processor'
 
@@ -24,11 +25,13 @@ module TacScribe
       end
       Datastore.instance.connect
 
-      @event_queue = EventQueue.new verbose_logging: verbose_logging
+      @event_queue = EventQueue.new
+
+      @verbose_logging = verbose_logging
 
       @populate_airfields = populate_airfields
       @thread_count = thread_count
-      @processing_threads = []
+      @threads = []
       @whitelist = Set.new(IO.read(whitelist).split) if whitelist
 
       @client = TacviewClient::Client.new(
@@ -47,10 +50,12 @@ module TacScribe
     # for example
     def start_processing
       loop do
-        kill_processing_threads
+        kill_threads
         @event_queue.clear
         Datastore.instance.truncate_table
         start_processing_threads
+        start_db_sync_thread
+        start_reporting_thread
         populate_airfields if @populate_airfields
         @client.connect
       # Rescuing reliably from Net::HTTP is a complete bear so rescue
@@ -65,21 +70,55 @@ module TacScribe
       end
     end
 
-    def kill_processing_threads
-      @processing_threads.each do |thr|
+    def kill_threads
+      @threads.each do |thr|
         thr.kill
         thr.join
       end
     end
 
     def start_processing_threads
-      @thread_count.times do
-        @processing_threads << Thread.new do
-          EventProcessor.new(datastore: Datastore.instance,
+      @event_processor = EventProcessor.new(cache: Cache.instance,
+                             datastore: Datastore.instance,
                              event_queue: @event_queue,
-                             whitelist: @whitelist).start
+                             whitelist: @whitelist)
+      event_processor_thread = Thread.new do
+        @event_processor.start
+      end
+      event_processor_thread.name = "Event Processor"
+      @threads << event_processor_thread
+    end
+
+    def start_db_sync_thread
+      db_write_thread = Thread.new do
+        loop do
+          sleep 1
+          deleted = Datastore.instance.write_objects(Cache.instance.data.values)
+          deleted.each { |id| Cache.instance.data.delete(id) }
+        end
+      end
+      db_write_thread.name = "Database Writing"
+      @threads << db_write_thread
+    end
+
+    def start_reporting_thread
+      return unless @verbose_logging
+
+      reporting_thread = Thread.new do
+        loop do
+          puts "#{Time.now.strftime('%FT%T')}\t" + 
+               "Events Incoming: #{@event_queue.events_written}\t" +
+               "Processed: #{@event_processor.events_processed}\t" +
+               "Ignored: #{@event_processor.events_ignored}\t" +
+               "Queue Size: #{@event_queue.size}"
+          @event_queue.events_written = 0
+          @event_processor.events_processed = 0
+          @event_processor.events_ignored = 0
+          sleep 1
         end
       end
+      reporting_thread.name = "Reporting"
+      @threads << reporting_thread
     end
 
     def populate_airfields

data/lib/tac_scribe/datastore.rb

@@ -52,24 +52,19 @@ module TacScribe
       @db[:units].truncate
     end
 
-    def write_object(event, timestamp)
-      unit = get_unit(event[:object_id])
-
-      # Tacview omits values that don't change to save
-      # data bandwidth and storage. If something has not changed then
-      # use the old value
-      current_position = get_position(event, unit)
-
-      if unit
-        update_unit(event, unit, current_position, timestamp)
-      else
-        insert_unit(event, current_position, timestamp)
-      end
-    end
-
-    def delete_object(object_id)
-      count = @db[:units].where(id: object_id).delete
-      "Deleted #{object_id} #{object_id.class} - #{count}"
+    def write_objects(objects)
+      @db[:units].insert_conflict(
+        constraint: :units_pkey,
+        update: { position: Sequel[:excluded][:position],
+                  altitude: Sequel[:excluded][:altitude],
+                  heading: Sequel[:excluded][:heading],
+                  updated_at: Sequel[:excluded][:updated_at],
+                  deleted: Sequel[:excluded][:deleted] }
+      )
+                 .multi_insert(objects)
+      deleted_ids = @db[:units].where(deleted: true).select_map(:id)
+      @db[:units].where(deleted: true).delete
+      deleted_ids
     end
 
     private
@@ -85,56 +80,5 @@ module TacScribe
         "/#{@configuration.database}"
       end
     end
-
-    def get_unit(id)
-      @db[:units].where(id: id).first
-    end
-
-    def update_unit(event, unit, current_position, timestamp)
-      heading = calculate_heading(unit[:position],
-                                  current_position,
-                                  unit[:heading])
-
-      @db[:units].where(id: event[:object_id]).update(
-        position: current_position[:lat_lon],
-        altitude: current_position[:altitude],
-        heading: heading,
-        updated_at: timestamp
-      )
-    end
-
-    def insert_unit(event, current_position, timestamp)
-      @db[:units].insert(id: event[:object_id],
-                         position: current_position[:lat_lon],
-                         altitude: current_position[:altitude],
-                         type: event[:type],
-                         name: event[:name],
-                         group: event[:group],
-                         pilot: event[:pilot],
-                         coalition: event[:coalition] == 'Allies' ? 0 : 1,
-                         updated_at: timestamp)
-    end
-
-    def get_position(event, unit)
-      {
-        lat_lon: Point.from_x_y(
-          event.fetch(:longitude) { unit ? unit[:position].y : nil },
-          event.fetch(:latitude) { unit ? unit[:position].x : nil }
-        ),
-        altitude: event.fetch(:altitude) { unit ? unit[:altitude] : nil }
-      }
-    end
-
-    def calculate_heading(old_position, current_position, current_heading)
-      return current_heading if old_position == current_position[:lat_lon]
-
-      begin
-        old_position.bearing_to(current_position[:lat_lon]).to_i
-      rescue Math::DomainError => e
-        puts 'Could not calculate heading: ' + e.message
-        puts 'Old Position: ' + old_position.inspect
-        puts 'New Position: ' + current_position.inspect
-      end
-    end
   end
 end

data/lib/tac_scribe/event_processor.rb

@@ -3,17 +3,22 @@
 require 'tacview_client/base_processor'
 require 'time'
 require 'concurrent-ruby'
-require_relative 'datastore'
+require_relative 'cache'
 
 module TacScribe
   # Processes the events emitted by the Ruby Tacview Client
   class EventProcessor
     @@ignored_units = Concurrent::Set.new
 
-    def initialize(datastore:, event_queue:, whitelist: nil)
+    attr_accessor :events_processed, :events_ignored
+
+    def initialize(cache:, datastore:, event_queue:, whitelist: nil)
+      @cache = cache
       @datastore = datastore
       @event_queue = event_queue
       @whitelist = whitelist
+      self.events_processed = 0
+      self.events_ignored = 0
     end
 
     def start
@@ -55,45 +60,33 @@ module TacScribe
     #   format.
     # @option event [BigDecimal] :altitude The object altitude above sea level
     #   in meters to 1 decimal place.
-    def update_object(event, time)
-      return if @@ignored_units.include?(event[:object_id])
+    def update_object(event, _time)
+      if @@ignored_units.include?(event[:object_id])
+        self.events_ignored += 1
+        return
+      end
 
       if @whitelist && event[:type] && !@whitelist.include?(event[:type])
         @@ignored_units << event[:object_id]
+        self.events_ignored += 1
         return
       end
 
-      if @event_queue.reference_latitude || @event_queue.reference_longitude
-        localize_position(event)
-      end
-
-      @datastore.write_object(event, time)
+      self.events_processed += 1
+      @cache.write_object(event)
     end
 
     # Process a delete event for an object
     #
-    # @param object_id [String] A hexadecimal number representing the object
-    #   ID
-    def delete_object(object_id)
-      return if @@ignored_units.delete?(object_id)
-
-      @datastore.delete_object(object_id)
-    end
-
-    # If we have reference lat/long then use that as a base and update the event
-    # position
-    def localize_position(event)
-      # There is no combination of layouts for these lines that doesn' trip
-      # at least one rubocop cop so just ignore the guard clause. The best
-      # one is using an inline if but that breaks the line length.
-      # rubocop:disable Style/GuardClause
-      if event[:latitude]
-        event[:latitude] = @event_queue.reference_latitude + event[:latitude]
-      end
-      if event[:longitude]
-        event[:longitude] = @event_queue.reference_longitude + event[:longitude]
+    # @param id [String] A hexadecimal number representing the object ID
+    def delete_object(id)
+      if @@ignored_units.delete?(id)
+        self.events_processed += 1
+        return
+      else
+        self.events_processed += 1
+        @cache.delete_object(id)
       end
-      # rubocop:enable Style/GuardClause
     end
   end
 end

data/lib/tac_scribe/event_queue.rb

@@ -6,32 +6,23 @@ require 'time'
 module TacScribe
   # Processes the events emitted by the Ruby Tacview Client
   class EventQueue < TacviewClient::BaseProcessor
-    attr_accessor :reference_latitude, :reference_longitude
 
-    def initialize(verbose_logging:)
-      @verbose_logging = verbose_logging
-      @events = Queue.new
-      @event_write = 0
-      @event_read = 0
-
-      return unless verbose_logging == true
+    attr_accessor :events_written
 
-      Thread.new do
-        loop do
-          puts "#{Time.now.strftime('%FT%T')} - Queue Size: #{@events.size} \t Events Added: #{@event_write} \t Events Processed: #{@event_read}"
-          @event_write = 0
-          @event_read = 0
-          sleep 1
-        end
-      end
+    def initialize
+      @events = Queue.new
+      self.events_written = 0
     end
 
     def clear
       @events.clear
     end
 
+    def size
+      @events.size
+    end
+
     def get_event
-      @event_read += 1
       @events.shift
     end
 
@@ -53,7 +44,7 @@ module TacScribe
     # @option event [BigDecimal] :altitude The object altitude above sea level
     #   in meters to 1 decimal place.
     def update_object(event)
-      @event_write += 1
+      self.events_written +=1
       @events << { type: :update_object, event: event, time: @time }
     end
 
@@ -62,7 +53,7 @@ module TacScribe
     # @param object_id [String] A hexadecimal number representing the object
     #   ID
     def delete_object(object_id)
-      @event_write += 1
+      self.events_written +=1
       @events << { type: :delete_object, object_id: object_id }
     end
 
@@ -81,9 +72,9 @@ module TacScribe
     def set_property(property:, value:)
       case property
       when 'ReferenceLatitude'
-        self.reference_latitude = BigDecimal(value)
+        Cache.instance.reference_latitude = BigDecimal(value)
       when 'ReferenceLongitude'
-        self.reference_longitude = BigDecimal(value)
+        Cache.instance.reference_longitude = BigDecimal(value)
       when 'ReferenceTime'
         @reference_time = @time = Time.parse(value)
       end

data/lib/tac_scribe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module TacScribe
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end

data/tac_scribe.gemspec

@@ -25,8 +25,6 @@ Gem::Specification.new do |spec|
   spec.metadata['yard.run'] = 'yri'
 
   if RUBY_PLATFORM == 'java'
-    # TODO: Specifying a verison chokes bundler for some reason
-    spec.platform = 'java'
     spec.add_dependency 'activerecord-jdbcpostgresql-adapter'
   else
     spec.add_dependency 'pg', '~>1.1'

metadata

@@ -1,178 +1,178 @@
 --- !ruby/object:Gem::Specification
 name: tac_scribe
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Jeffrey Jones
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2019-10-26 00:00:00.000000000 Z
+date: 2020-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
-  type: :runtime
+        version: '0'
+  name: activerecord-jdbcpostgresql-adapter
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: georuby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
-  type: :runtime
+  name: georuby
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
 - !ruby/object:Gem::Dependency
-  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.22'
-  type: :runtime
+  name: sequel
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '5.22'
 - !ruby/object:Gem::Dependency
-  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.5
-  type: :runtime
+  name: concurrent-ruby
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.5
 - !ruby/object:Gem::Dependency
-  name: sequel-postgis-georuby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.1.2
-  type: :runtime
+  name: sequel-postgis-georuby
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.1.2
 - !ruby/object:Gem::Dependency
-  name: tacview_client
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
-  type: :runtime
+  name: tacview_client
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.1'
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
-  type: :development
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
-  type: :development
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.0'
 - !ruby/object:Gem::Dependency
-  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.8'
-  type: :development
+  name: rspec
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.8'
 - !ruby/object:Gem::Dependency
-  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.73'
-  type: :development
+  name: rubocop
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.73'
 - !ruby/object:Gem::Dependency
-  name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
-  type: :development
+  name: simplecov
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.17'
 - !ruby/object:Gem::Dependency
-  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-  type: :development
+  name: yard
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -205,6 +205,7 @@ files:
 - db/README.md
 - exe/tac_scribe
 - lib/tac_scribe.rb
+- lib/tac_scribe/cache.rb
 - lib/tac_scribe/daemon.rb
 - lib/tac_scribe/datastore.rb
 - lib/tac_scribe/event_processor.rb
@@ -217,7 +218,7 @@ licenses:
 - AGPL-3.0-or-later
 metadata:
   yard.run: yri
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -232,8 +233,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.7.6
+signing_key:
 specification_version: 4
 summary: Write Tacview data to PostGIS database
 test_files: []