openra 1.6.0 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 345905d788dbaf8a6ffad6e70be070553de601d3e0b5a56b3c2e3840ba6ff490
-  data.tar.gz: c1ac04f3de64aba8bb52206308c8b9cf92730e4ef00c236729449f131a3b8a64
+  metadata.gz: 0f5589223dd3591d9db257c50c14843f99ff9d6cb25e54f667fc2b43a8dbcfaf
+  data.tar.gz: 83d7082b488eb31f84e0f891e93acf1514f3b1548826190385190d6234a31ea6
 SHA512:
-  metadata.gz: ea61c392895a2cec57e9e64e085219752f6dbca7c1eafada4b2a6d78f7f5706f6838c0d3a1d4d91796e726f1b5cd82d61bf69a09c2daa1d4f08399aea4f8234b
-  data.tar.gz: bbb8382e3c7d991c9e68e5c3e31e84b81cddd1480d671a700bf1f3295b9c56772802969dc93de510e3ac5ac575b41ff0bba4899e1625f6108892f3be8611dc63
+  metadata.gz: 56da77ac510188b14c9c2497809f65989ce92751706026350b860c111d8c94a00ad414a47e8d158a03130ff5602b2615266a3e19229f786d3dc4f4d07389aa4c
+  data.tar.gz: c0ebd68879e4d35bc3084052034b4068105346458665c7a475fb36a550fcb9de57269188e7fc3e5da16df871e53b87b00434fbea6576a03927c9de8397c0ef5b

data/CHANGELOG.md

@@ -1,6 +1,14 @@
 ## Unreleased
 
-[Compare v1.6.0...HEAD](https://github.com/AMHOL/openra-ruby/compare/v1.6.0...HEAD)
+[Compare v1.7.0...HEAD](https://github.com/AMHOL/openra-ruby/compare/v1.7.0...HEAD)
+
+## v1.7.0
+
+### Improvements
+
+* Decreased memory usage by streaming orders with iterator ([AMHOL](https://github.com/AMHOL))
+
+[Compare v1.6.0...v1.7.0](https://github.com/AMHOL/openra-ruby/compare/v1.6.0...v1.7.0)
 
 ## v1.6.0
 
@@ -8,7 +16,7 @@
 
 * Updated to work with release-20200202 (current release) ([AMHOL](https://github.com/AMHOL))
 
-[Compare v1.5.0...HEAD](https://github.com/AMHOL/openra-ruby/compare/v1.5.0...v1.6.0)
+[Compare v1.5.0...v1.6.0](https://github.com/AMHOL/openra-ruby/compare/v1.5.0...v1.6.0)
 
 ## v1.5.0
 

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 gemspec

data/Rakefile

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'bundler/gem_tasks'

data/lib/bindata/big_integer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class BigInteger < BinData::BasePrimitive
   def value_to_binary_string(value)
     value = value.abs

data/lib/bindata/pascal_string.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class PascalString < BinData::Primitive
   big_integer :len, value: -> { data.length }
   string :data, read_length: :len

data/lib/openra-cli.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'openra/cli'

data/lib/openra-replays.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'openra/replays'

data/lib/openra-struct.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'openra/struct'

data/lib/openra-types.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'openra/types'

data/lib/openra-yaml.rb

@@ -1 +1,3 @@
+# frozen_string_literal: true
+
 require 'openra/yaml

data/lib/openra.rb

@@ -1,2 +1,4 @@
+# frozen_string_literal: true
+
 require 'openra/version'
 require 'openra/replays' # Replays requires everything else

data/lib/openra/cli.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 require 'yaml'
 require 'json'
@@ -12,8 +14,8 @@ require 'openra/cli/command_registry'
 
 module Openra
   class CLI
-    def call(*args)
-      Dry::CLI.new(CommandRegistry).call(*args)
+    def call(*args, **kwargs, &block)
+      Dry::CLI.new(CommandRegistry).call(*args, **kwargs, &block)
     end
   end
 end

data/lib/openra/cli/command_registry.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class CLI
     class CommandRegistry

data/lib/openra/cli/commands/detect_production_macros.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class CLI
     module Commands
@@ -12,13 +14,20 @@ module Openra
         def call(replay:, **options)
           replay = Openra::Replays::Replay.new(replay)
           commands = Hash.new(&ARRAY_HASH)
+          sync_info = nil
 
-          replay.orders.each do |order|
+          replay.each_order do |order|
             case order.command
+            when 'StartGame'
+              game_started = true
+            when 'SyncInfo'
+              sync_info = Openra::Struct::SyncInfo.new(
+                Openra::YAML.load(order.target)
+              ) unless game_started
             when 'StartProduction'
               commands[order.client_index.to_s] << {
                 target: order.target,
-                msec: order.frame * replay.frametime_multiplier
+                msec: order.frame * sync_info.global_settings.frametime_multiplier
               }
             end
           end
@@ -58,7 +67,9 @@ module Openra
 
         def detect_sequences(player_commands)
           sequences = []
-          groups = player_commands.each_with_object(Hash.new(&ARRAY_HASH)).with_index do |(command, hash), index|
+          groups = player_commands.each_with_object(
+            Hash.new(&ARRAY_HASH)
+          ).with_index do |(command, hash), index|
             hash[command[:delay]] << command.merge(index: index)
           end
 

data/lib/openra/cli/commands/formatters.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class CLI
     module Commands

data/lib/openra/cli/commands/replay_data.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class CLI
     module Commands
@@ -13,7 +15,7 @@ module Openra
           data = {
             mod: replay.metadata.mod,
             version: replay.metadata.version,
-            server_name: replay.global_settings.server_name,
+            server_name: nil,
             map: {
               name: replay.metadata.map_name,
               hash: replay.metadata.map_hash
@@ -29,58 +31,70 @@ module Openra
               start_time: replay.metadata.start_time.iso8601,
               end_time: replay.metadata.end_time.iso8601,
               duration: time((replay.metadata.end_time - replay.metadata.start_time) * 1000),
-              options: {
-                explored_map: replay.game_options.explored_map_enabled.value,
-                speed: replay.game_options.game_speed.value,
-                starting_cash: replay.game_options.starting_cash.value,
-                starting_units: replay.game_options.starting_units.value,
-                fog_enabled: replay.game_options.fog_enabled.value,
-                cheats_enabled: replay.game_options.cheats_enabled.value,
-                kill_bounty_enabled: replay.game_options.bounties_enabled.value,
-                allow_undeploy: replay.game_options.conyard_undeploy_enabled.value,
-                crates_enabled: replay.game_options.crates_enabled.value,
-                build_off_allies: replay.game_options.build_off_allies_enabled.value,
-                restrict_build_radius: replay.game_options.restricted_build_radius_enabled.value,
-                short_game: replay.game_options.short_game_enabled.value,
-                techlevel: replay.game_options.tech_level.value
-              }
-            },
-            clients: replay.clients.map { |client|
-              player = replay.player(client.index)
-
-              {
-                index: client.index,
-                name: client.name,
-                preferred_color: client.preferred_color,
-                color: client.preferred_color,
-                spawn: {
-                  random: player&.is_random_spawn,
-                  point: client.spawn_point
-                },
-                faction: {
-                  chosen: client.faction_name.downcase,
-                  actual: player&.faction_id
-                },
-                ip: client.ip,
-                team: player&.team,
-                is_bot: player&.is_bot || false,
-                is_admin: client.is_admin,
-                is_player: !player.nil?,
-                is_winner: player&.outcome == 'Won',
-                build: []
-              }
+              options: nil
             },
+            clients: [],
             chat: []
           }
 
+          game_started = false
           current_sync_clients = []
+          sync_info = nil
 
-          replay.orders.each do |order|
+          replay.each_order do |order|
             client = current_sync_clients.find do |candidate|
               candidate.index == order.client_index.to_s
             end
 
             case order.command
+            when 'StartGame'
+              game_started = true
+
+              data[:clients] = sync_info.clients.map do |client|
+                player = replay.player(client.index)
+
+                {
+                  index: client.index,
+                  name: client.name,
+                  preferred_color: client.preferred_color,
+                  color: client.color,
+                  spawn: {
+                    random: player&.is_random_spawn,
+                    point: client.spawn_point
+                  },
+                  faction: {
+                    chosen: client.faction_name.downcase,
+                    actual: player&.faction_id
+                  },
+                  ip: client.ip,
+                  team: player&.team,
+                  is_bot: player&.is_bot || false,
+                  is_admin: client.is_admin,
+                  is_player: !player.nil?,
+                  is_winner: player&.outcome == 'Won',
+                  build: []
+                }
+              end
+
+              data[:game][:options] = {
+                explored_map: sync_info.global_settings.game_options.explored_map_enabled.value,
+                speed: sync_info.global_settings.game_options.game_speed.value,
+                starting_cash: sync_info.global_settings.game_options.starting_cash.value,
+                starting_units: sync_info.global_settings.game_options.starting_units.value,
+                fog_enabled: sync_info.global_settings.game_options.fog_enabled.value,
+                cheats_enabled: sync_info.global_settings.game_options.cheats_enabled.value,
+                kill_bounty_enabled: sync_info.global_settings.game_options.bounties_enabled.value,
+                allow_undeploy: sync_info.global_settings.game_options.conyard_undeploy_enabled.value,
+                crates_enabled: sync_info.global_settings.game_options.crates_enabled.value,
+                build_off_allies: sync_info.global_settings.game_options.build_off_allies_enabled.value,
+                restrict_build_radius: sync_info.global_settings.game_options.restricted_build_radius_enabled.value,
+                short_game: sync_info.global_settings.game_options.short_game_enabled.value,
+                techlevel: sync_info.global_settings.game_options.tech_level.value
+              }
+            when 'SyncInfo'
+              sync_info = Openra::Struct::SyncInfo.new(
+                Openra::YAML.load(order.target)
+              ) unless game_started
             when 'SyncLobbyClients'
               current_sync_clients = Openra::Struct::SyncLobbyClients.new(
                 Openra::YAML.load(order.target)
@@ -91,40 +105,38 @@ module Openra
               end
 
               client_hash[:build] << {
-                structure: utf8(order.target),
-                game_time: time(order.frame * replay.frametime_multiplier),
+                structure: order.target,
+                game_time: time(order.frame * sync_info.global_settings.frametime_multiplier),
                 placement: order.target_pos.to_i
               }
             when 'Message'
               data[:chat] << {
                 channel: 'server',
                 name: nil,
-                message: utf8(order.target)
+                message: order.target
               }
             when 'Chat'
               data[:chat] << {
                 channel: 'global',
                 name: client.name,
-                message: utf8(order.target)
+                message: order.target
               }
             when 'TeamChat'
               data[:chat] << {
                 channel: client.team,
                 name: client.name,
-                message: utf8(order.target)
+                message: order.target
               }
             end
           end
 
+          data[:server_name] = sync_info.global_settings.server_name
+
           puts FORMATTERS.fetch(options[:format]).call(data)
         end
 
         private
 
-        def utf8(string)
-          string.force_encoding('UTF-8').to_s
-        end
-
         def time(msec)
           sec = msec / 1000
           mm, ss = sec.divmod(60)

data/lib/openra/cli/commands/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class CLI
     module Commands

data/lib/openra/replays.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'date'
 require 'bindata'
 require 'bindata/big_integer'
@@ -8,7 +10,6 @@ require 'openra/replays/order_decorator'
 require 'openra/replays/order'
 require 'openra/replays/order_list'
 require 'openra/replays/packet'
-require 'openra/replays/packet_list'
 require 'openra/replays/metadata_marker'
 require 'openra/replays/metadata'
 require 'openra/replays/file'

data/lib/openra/replays/file.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class File
@@ -5,20 +7,28 @@ module Openra
         @filename = filename
       end
 
-      def packets
-        @packets ||= PacketList.read(fs)
-      end
+      def each_order(&block)
+        return enum_for(:each_order) unless block_given?
+
+        file.rewind
 
-      def orders
-        @orders ||= packets.orders
+        io = BinData::IO::Read.new(file)
+        template = Packet.new(fields: Packet.fields)
+
+        loop do
+          template.new.read(file).orders.each(&block)
+        rescue EOFError, IOError
+          break
+        end
       end
 
       def metadata
-        metadata_fs = fs
-        offset = -(metadata_marker.data_length + 4)
-        metadata_fs.seek(offset, IO::SEEK_END)
-
-        @metadata ||= Metadata.read(metadata_fs)
+        @metadata ||= begin
+          metadata_fs = file.tap(&:rewind)
+          offset = -(metadata_marker.data_length + 4)
+          metadata_fs.seek(offset, IO::SEEK_END)
+          Metadata.read(metadata_fs)
+        end
       end
 
 
@@ -26,13 +36,12 @@ module Openra
 
       attr_reader :filename
 
-      def fs
-        @fs ||= ::File.open(filename, 'rb')
-        @fs.tap(&:rewind)
+      def file
+        @file ||= ::File.open(filename, 'rb')
       end
 
       def metadata_marker
-        @metadata_marker ||= MetadataMarker.read(fs)
+        @metadata_marker ||= MetadataMarker.read(file)
       end
     end
   end

data/lib/openra/replays/metadata.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class Metadata < BinData::Record

data/lib/openra/replays/metadata_marker.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class MetadataMarker < BinData::Record

data/lib/openra/replays/order.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class Order < BinData::Record
-      HEX_FE = ?\xFE.force_encoding('ASCII-8BIT').freeze
-      HEX_FF = ?\xFF.force_encoding('ASCII-8BIT').freeze
+      HEX_FE = ?\xFE.dup.force_encoding('ASCII-8BIT').freeze
+      HEX_FF = ?\xFF.dup.force_encoding('ASCII-8BIT').freeze
       # NEED TO LEARN HOW TO READ FLAGS AND TARGET TYPE
       IS_STANDARD_ORDER = -> { order_type == HEX_FF }
       IS_IMMEDIATE_ORDER = -> { order_type == HEX_FE }

data/lib/openra/replays/order_decorator.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class OrderDecorator < SimpleDelegator

data/lib/openra/replays/order_list.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class OrderList < BinData::Record

data/lib/openra/replays/packet.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class Packet < BinData::Record
@@ -7,9 +9,7 @@ module Openra
       string :data, read_length: :data_length
 
       def orders
-        return unless valid_order_list?
-
-        order_list = OrderList.read(data)
+        return [] unless valid_order_list?
 
         @orders ||= order_list.orders.map do |order|
           OrderDecorator.new(order, client_index, order_list.frame)

data/lib/openra/replays/replay.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   module Replays
     class Replay
@@ -13,8 +15,8 @@ module Openra
         )
       end
 
-      def orders
-        @orders ||= file.orders
+      def each_order(&block)
+        file.each_order(&block)
       end
 
       def players
@@ -26,45 +28,6 @@ module Openra
           candidate.index == index
         end
       end
-
-      def clients
-        sync_info.clients
-      end
-
-      def client(index)
-        clients.find do |candidate|
-          candidate.index == index
-        end
-      end
-
-      def global_settings
-        sync_info.global_settings
-      end
-
-      def frametime_multiplier
-        global_settings.frametime_multiplier
-      end
-
-      def game_options
-        global_settings.game_options
-      end
-
-      private
-
-      def sync_info
-        return @sync_info if @sync_info
-
-        start_game_order = orders.find { |order| order[:command] == 'StartGame' }
-        start_game_index = orders.index(start_game_order)
-        pre_game_orders = orders[0..start_game_index]
-        start_game_sync_order = pre_game_orders.reverse.find do |order|
-          order[:command] == 'SyncInfo'
-        end
-
-        @sync_info = Openra::Struct::SyncInfo.new(
-          Openra::YAML.load(start_game_sync_order.target)
-        )
-      end
     end
   end
 end

data/lib/openra/struct.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'dry/transformer/all'
 require 'dry-struct'
 require 'openra/types'

data/lib/openra/struct/client.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class Client < Openra::Struct

data/lib/openra/struct/functions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     module Functions

data/lib/openra/struct/game_options.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class GameOptions < Openra::Struct

data/lib/openra/struct/game_options/boolean_option.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class GameOptions < Openra::Struct

data/lib/openra/struct/game_options/integer_option.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class GameOptions < Openra::Struct

data/lib/openra/struct/game_options/string_option.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class GameOptions < Openra::Struct

data/lib/openra/struct/global_settings.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class GlobalSettings < Openra::Struct

data/lib/openra/struct/metadata.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class Metadata < Openra::Struct

data/lib/openra/struct/player.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class Player < Openra::Struct

data/lib/openra/struct/pre_processor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class PreProcessor

data/lib/openra/struct/sync_info.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class SyncInfo < Openra::Struct

data/lib/openra/struct/sync_lobby_clients.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class Struct < Dry::Struct
     class SyncLobbyClients < Openra::Struct

data/lib/openra/types.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'dry-types'
 
 module Openra

data/lib/openra/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
-  VERSION = '1.6.0'.freeze
+  VERSION = '1.7.0'.freeze
 end

data/lib/openra/yaml.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Openra
   class YAML
     MATCHER = /(?<indentation>\t+)?(?<key>[^\:]+)?\:?\s?(?<value>.+)?/.freeze

data/openra.gemspec

@@ -10,7 +10,11 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/AMHOL/openra-ruby'
   spec.license       = 'MIT'
 
-  spec.files         = `git ls-files -z`.split("\x0") - ['bin/console']
+  spec.files         = `git ls-files -z`.split("\x0") - %w(
+    bin/console
+    bin/profile
+    bin/trace_alloc
+  )
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
@@ -24,4 +28,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'memory_profiler'
+  spec.add_development_dependency 'stackprof'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: openra
 version: !ruby/object:Gem::Version
-  version: 1.6.0
+  version: 1.7.0
 platform: ruby
 authors:
 - Andy Holland
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-10 00:00:00.000000000 Z
+date: 2020-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,6 +122,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: memory_profiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stackprof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - andyholland1991@aol.com
@@ -159,7 +187,6 @@ files:
 - lib/openra/replays/order_decorator.rb
 - lib/openra/replays/order_list.rb
 - lib/openra/replays/packet.rb
-- lib/openra/replays/packet_list.rb
 - lib/openra/replays/replay.rb
 - lib/openra/struct.rb
 - lib/openra/struct/client.rb
@@ -197,7 +224,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Openra Rubygem

data/lib/openra/replays/packet_list.rb

@@ -1,12 +0,0 @@
-module Openra
-  module Replays
-    class PacketList < BinData::Record
-      endian :little
-      array :packets, type: :packet, read_until: :eof
-
-      def orders
-        @orders ||= packets.select(&:valid_order_list?).flat_map(&:orders)
-      end
-    end
-  end
-end