eve_app 0.1.0

data/lib/eve_app/xml_api/client.rb

@@ -0,0 +1,86 @@
+require 'fileutils'
+require 'net/http'
+require 'rest-client'
+
+module EveApp
+  module XmlApi
+    class Client
+      API_HOST = 'https://api.eveonline.com'
+      MAX_TRIES = 4
+      REQUEST_TIMEOUT = 60
+
+      attr_reader :key_id, :vcode
+      attr_accessor :save_responses, :character_id
+
+      def initialize(key_id, vcode)
+        @key_id = key_id
+        @vcode = vcode
+        @save_responses = false
+        @character_id = nil
+      end
+
+      def method_missing(name, params={})
+        call = Calls.const_get(name.to_s.camelize)
+        params[:last_id] ? walk(call, params) : request(call, params)
+      # rescue NameError
+      #   super
+      end
+
+      private
+
+      def walk(call, params={})
+        last_id = params.delete(:last_id)
+        from_id = nil
+        collection = call.new
+
+        50.times do
+          page = request(call, params.merge(rowCount: 1000, fromID: from_id))
+          page.results.reject! { |r| r.id <= last_id }
+
+          if collection.cached_until.nil? || collection.cached_until > page.cached_until
+            collection.cached_until = page.cached_until
+          end
+          break if page.blank?
+
+          collection.merge(page)
+          from_id = page.ids.min
+          break if from_id <= last_id
+        end
+
+        collection
+      end
+
+      def request(call, params={})
+        params = params.merge({ rowCount: 500, keyID: key_id, vCode: vcode })
+        params[:characterID] = character_id if character_id
+        response = nil
+
+        1.upto(MAX_TRIES) do |try|
+          begin
+            response = RestClient::Request.execute(
+              method:  :get,
+              url:     "#{API_HOST}#{call.endpoint}",
+              timeout: REQUEST_TIMEOUT,
+              headers: { params:  params }
+            )
+          # rescue RestClient::Exception => ex
+          #   error = Calls::ErrorResponse.new(ex, call, params)
+          #   raise error
+          end
+          break if response
+        end
+
+        if save_responses
+          folder = Rails.root.join('tmp', call.to_s.underscore)
+          FileUtils.mkdir_p(folder)
+          File.write(folder.join("#{Time.now.to_i.to_s}.xml"), response.body)
+        end
+
+        xml = Nokogiri::XML(response.body)
+        call.new(xml)
+      rescue Errno::ECONNRESET, Net::ReadTimeout, RestClient::Exceptions::ReadTimeout
+        raise ConnectionError, $!.message
+      end
+    end
+  end
+end

data/lib/eve_app/xml_api.rb

@@ -0,0 +1,10 @@
+module EveApp
+  module XmlApi
+    autoload :Calls,   'eve_app/xml_api/calls'
+    autoload :Classes, 'eve_app/xml_api/classes'
+    autoload :Client,  'eve_app/xml_api/client'
+
+    class ApiError < StandardError; end
+    class ConnectionError < ApiError; end
+  end
+end

data/lib/eve_app.rb

@@ -0,0 +1,35 @@
+require "pathname"
+require "active_model_serializers"
+require "eve_app/engine"
+
+module EveApp
+  autoload :EveCentral, 'eve_app/eve_central'
+  autoload :ItemParser, 'eve_app/item_parser'
+  autoload :SDE,        'eve_app/sde'
+  autoload :XmlApi,     'eve_app/xml_api'
+
+  class << self
+    def root
+      @root ||= Pathname.new(File.expand_path("../../", __FILE__))
+    end
+
+    def table_name_prefix
+      @_table_name_prefix ||= begin
+        prefix = EveApp::SDE.config.table_prefix.presence
+        prefix ? "#{prefix}_" : super
+      end
+    end
+  end
+end
+
+if defined?(::Rails)
+  module ::Rails
+    class Application
+      rake_tasks do
+        Dir[File.join(EveApp.root, "/lib/tasks/", "**/*.rake")].each do |file|
+          load file
+        end
+      end
+    end
+  end
+end

data/lib/table-list.yml

@@ -0,0 +1,88 @@
+- agtAgentTypes
+- agtAgents
+- agtResearchAgents
+- certCerts
+- certMasteries
+- certSkills
+- chrAncestries
+- chrAttributes
+- chrBloodlines
+- chrFactions
+- chrRaces
+- crpActivities
+- crpNPCCorporationDivisions
+- crpNPCCorporationResearchFields
+- crpNPCCorporationTrades
+- crpNPCCorporations
+- crpNPCDivisions
+- dgmAttributeCategories
+- dgmAttributeTypes
+- dgmEffects
+- dgmExpressions
+- dgmTypeAttributes
+- dgmTypeEffects
+- eveGraphics
+- eveIcons
+- eveUnits
+- industryActivity
+- industryActivityMaterials
+- industryActivityProbabilities
+- industryActivityProducts
+- industryActivityRaces
+- industryActivitySkills
+- industryBlueprints
+- invCategories
+- invContrabandTypes
+- invControlTowerResourcePurposes
+- invControlTowerResources
+- invFlags
+- invGroups
+- invItems
+- invMarketGroups
+- invMetaGroups
+- invMetaTypes
+- invNames
+- invPositions
+- invTraits
+- invTypeMaterials
+- invTypeReactions
+- invTypes
+- invUniqueNames
+- invVolumes
+- mapCelestialStatistics
+- mapConstellationJumps
+- mapConstellations
+- mapDenormalize
+- mapJumps
+- mapLandmarks
+- mapLocationScenes
+- mapLocationWormholeClasses
+- mapRegionJumps
+- mapRegions
+- mapSolarSystemJumps
+- mapSolarSystems
+- mapUniverse
+- planetSchematics
+- planetSchematicsPinMap
+- planetSchematicsTypeMap
+- ramActivities
+- ramAssemblyLineStations
+- ramAssemblyLineTypeDetailPerCategory
+- ramAssemblyLineTypeDetailPerGroup
+- ramAssemblyLineTypes
+- ramInstallationTypeContents
+- skinLicense
+- skinMaterials
+- skinShip
+- skins
+- staOperationServices
+- staOperations
+- staServices
+- staStationTypes
+- staStations
+- translationTables
+- trnTranslationColumns
+- trnTranslationLanguages
+- trnTranslations
+- warCombatZoneSystems
+- warCombatZones

data/lib/table-map.yml

@@ -0,0 +1,177 @@
+---
+agtAgentTypes:
+  destination: agt_agent_types
+agtAgents:
+  destination: agt_agents
+agtResearchAgents:
+  destination: agt_research_agents
+certCerts:
+  destination: cert_certs
+certMasteries:
+  destination: cert_masteries
+certSkills:
+  destination: cert_skills
+chrAncestries:
+  destination: chr_ancestries
+chrAttributes:
+  destination: chr_attributes
+chrBloodlines:
+  destination: chr_bloodlines
+chrFactions:
+  destination: chr_factions
+chrRaces:
+  destination: chr_races
+crpActivities:
+  destination: crp_activities
+crpNPCCorporationDivisions:
+  destination: crp_npccorporation_divisions
+crpNPCCorporationResearchFields:
+  destination: crp_npccorporation_research_fields
+crpNPCCorporationTrades:
+  destination: crp_npccorporation_trades
+crpNPCCorporations:
+  destination: crp_npccorporations
+crpNPCDivisions:
+  destination: crp_npcdivisions
+dgmAttributeCategories:
+  destination: dgm_attribute_categories
+dgmAttributeTypes:
+  destination: dgm_attribute_types
+dgmEffects:
+  destination: dgm_effects
+dgmExpressions:
+  destination: dgm_expressions
+dgmTypeAttributes:
+  destination: dgm_type_attributes
+dgmTypeEffects:
+  destination: dgm_type_effects
+eveGraphics:
+  destination: eve_graphics
+eveIcons:
+  destination: eve_icons
+eveUnits:
+  destination: eve_units
+industryActivity:
+  destination: industry_activity
+industryActivityMaterials:
+  destination: industry_activity_materials
+industryActivityProbabilities:
+  destination: industry_activity_probabilities
+industryActivityProducts:
+  destination: industry_activity_products
+industryActivityRaces:
+  destination: industry_activity_races
+industryActivitySkills:
+  destination: industry_activity_skills
+industryBlueprints:
+  destination: industry_blueprints
+invCategories:
+  destination: inv_categories
+invContrabandTypes:
+  destination: inv_contraband_types
+invControlTowerResourcePurposes:
+  destination: inv_control_tower_resource_purposes
+invControlTowerResources:
+  destination: inv_control_tower_resources
+invFlags:
+  destination: inv_flags
+invGroups:
+  destination: inv_groups
+invItems:
+  destination: inv_items
+invMarketGroups:
+  destination: inv_market_groups
+invMetaGroups:
+  destination: inv_meta_groups
+invMetaTypes:
+  destination: inv_meta_types
+invNames:
+  destination: inv_names
+invPositions:
+  destination: inv_positions
+invTraits:
+  destination: inv_traits
+invTypeMaterials:
+  destination: inv_type_materials
+invTypeReactions:
+  destination: inv_type_reactions
+invTypes:
+  destination: inv_types
+invUniqueNames:
+  destination: inv_unique_names
+invVolumes:
+  destination: inv_volumes
+mapCelestialStatistics:
+  destination: map_celestial_statistics
+mapConstellationJumps:
+  destination: map_constellation_jumps
+mapConstellations:
+  destination: map_constellations
+mapDenormalize:
+  destination: map_denormalize
+mapJumps:
+  destination: map_jumps
+mapLandmarks:
+  destination: map_landmarks
+mapLocationScenes:
+  destination: map_location_scenes
+mapLocationWormholeClasses:
+  destination: map_location_wormhole_classes
+mapRegionJumps:
+  destination: map_region_jumps
+mapRegions:
+  destination: map_regions
+mapSolarSystemJumps:
+  destination: map_solar_system_jumps
+mapSolarSystems:
+  destination: map_solar_systems
+mapUniverse:
+  destination: map_universe
+planetSchematics:
+  destination: planet_schematics
+planetSchematicsPinMap:
+  destination: planet_schematics_pin_map
+planetSchematicsTypeMap:
+  destination: planet_schematics_type_map
+ramActivities:
+  destination: ram_activities
+ramAssemblyLineStations:
+  destination: ram_assembly_line_stations
+ramAssemblyLineTypeDetailPerCategory:
+  destination: ram_assembly_line_type_detail_per_category
+ramAssemblyLineTypeDetailPerGroup:
+  destination: ram_assembly_line_type_detail_per_group
+ramAssemblyLineTypes:
+  destination: ram_assembly_line_types
+ramInstallationTypeContents:
+  destination: ram_installation_type_contents
+skinLicense:
+  destination: skin_license
+skinMaterials:
+  destination: skin_materials
+skinShip:
+  destination: skin_ship
+skins:
+  destination: skins
+staOperationServices:
+  destination: sta_operation_services
+staOperations:
+  destination: sta_operations
+staServices:
+  destination: sta_services
+staStationTypes:
+  destination: sta_station_types
+staStations:
+  destination: sta_stations
+translationTables:
+  destination: translation_tables
+trnTranslationColumns:
+  destination: trn_translation_columns
+trnTranslationLanguages:
+  destination: trn_translation_languages
+trnTranslations:
+  destination: trn_translations
+warCombatZoneSystems:
+  destination: war_combat_zone_systems
+warCombatZones:
+  destination: war_combat_zones

data/lib/tasks/eve_app.rake

@@ -0,0 +1,30 @@
+require 'eve_app'
+
+namespace :eve_app do
+  namespace :sde do
+    def importer
+      @importer ||= EveApp::SDE::DataImporter.new
+    end
+
+    desc "Download latest postgresql datadump"
+    task :download do
+      importer.download
+    end
+
+    desc "Import and normalize EVE database"
+    task import: :environment do
+      importer.restore
+      importer.normalize
+    end
+
+    desc "Import raw data into the database"
+    task restore: :environment do
+      importer.restore
+    end
+
+    desc "Import raw data into the database"
+    task normalize: :environment do
+      importer.normalize
+    end
+  end
+end

metadata

@@ -0,0 +1,186 @@
+--- !ruby/object:Gem::Specification
+name: eve_app
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Danny Hiemstra
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.1
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sshkit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: kaminari
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: active_model_serializers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Basic models for the EveSDE data
+email:
+- dannyhiemstra@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/controllers/eve_app/application_controller.rb
+- app/controllers/eve_app/regions_controller.rb
+- app/controllers/eve_app/simple_resource_controller.rb
+- app/controllers/eve_app/solar_systems_controller.rb
+- app/controllers/eve_app/types_controller.rb
+- app/helpers/eve_app/entity_helper.rb
+- app/helpers/eve_app/output_helper.rb
+- app/models/eve_app/activity.rb
+- app/models/eve_app/activity_material.rb
+- app/models/eve_app/activity_product.rb
+- app/models/eve_app/activity_skill.rb
+- app/models/eve_app/application_record.rb
+- app/models/eve_app/category.rb
+- app/models/eve_app/concerns/activity_relation.rb
+- app/models/eve_app/group.rb
+- app/models/eve_app/market_group.rb
+- app/models/eve_app/region.rb
+- app/models/eve_app/solar_system.rb
+- app/models/eve_app/station.rb
+- app/models/eve_app/type.rb
+- app/serializers/eve_app/application_serializer.rb
+- app/serializers/eve_app/region_serializer.rb
+- app/serializers/eve_app/solar_system_serializer.rb
+- app/serializers/eve_app/type_serializer.rb
+- config/routes.rb
+- lib/eve_app.rb
+- lib/eve_app/engine.rb
+- lib/eve_app/eve_central.rb
+- lib/eve_app/item_parser.rb
+- lib/eve_app/sde.rb
+- lib/eve_app/sde/data_importer.rb
+- lib/eve_app/sde/downloader.rb
+- lib/eve_app/sde/normalizer.rb
+- lib/eve_app/version.rb
+- lib/eve_app/xml_api.rb
+- lib/eve_app/xml_api/calls.rb
+- lib/eve_app/xml_api/classes.rb
+- lib/eve_app/xml_api/client.rb
+- lib/table-list.yml
+- lib/table-map.yml
+- lib/tasks/eve_app.rake
+homepage: http://www.evebuddy.net
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: EveApp API
+test_files: []