eve_app 0.1.0

data/lib/eve_app/sde/data_importer.rb

@@ -0,0 +1,43 @@
+require "sshkit"
+
+module EveApp
+  module SDE
+    class DataImporter < SSHKit::Backend::Local
+      include Downloader
+      include Normalizer
+
+      def initialize
+        super
+        execute :mkdir, '-p', SDE.config.tmp_path
+      end
+
+      private
+
+      def db_config
+        ActiveRecord::Base.connection_config
+      end
+
+      def sql(sql)
+        log "[SQL] #{sql}"
+        db.execute(sql)
+      end
+
+      def db
+        ActiveRecord::Base.connection
+      end
+
+      def table_list
+        @_table_list ||= begin
+          whitelist = Array[SDE.config.table_whitelist].flatten.compact.map(&:to_s)
+          tables = SDE.table_list
+          tables = whitelist.any? ? tables & whitelist : tables
+          Hash[tables.map { |name| [name, normalize_table_name(name)] }]
+        end
+      end
+
+      def normalize_table_name(name)
+        (SDE.config.table_prefix.to_s + name.gsub(/^#{SDE::PREFIXES.join('|')}/, '')).pluralize.underscore
+      end
+    end
+  end
+end

data/lib/eve_app/sde/downloader.rb

@@ -0,0 +1,53 @@
+module EveApp
+  module SDE
+    module Downloader
+      def download
+        within SDE.config.tmp_path do
+          execute :wget, '-q', download_uri + '{,.md5}'
+          verify!
+          execute :bunzip2, SDE.config.archive
+        end
+      end
+
+      def restore
+        table_list.each do |table_name,normalized_name|
+          sql %Q(DROP TABLE IF EXISTS "#{table_name}")
+          sql %Q(DROP TABLE IF EXISTS "#{normalized_name}")
+        end
+
+        options = ['-x -O']
+        options << "-h #{db_config[:host]}" if db_config[:host]
+        options << "-U #{db_config[:username]}" if db_config[:username]
+        options << "-d #{db_config[:database]}"
+        options += table_list.keys.map { |name| "-t #{name}" }
+        options << local_archive.gsub('.bz2', '')
+
+        execute :pg_restore, *options
+      end
+
+      private
+
+      def local_archive
+        @_local_archive ||= [SDE.config.tmp_path, SDE.config.archive].join('/')
+      end
+
+      def download_uri
+        @_download_url ||= begin
+          uri = URI(SDE.config.download_host)
+          uri.path = [uri.path, SDE.config.archive].join('/')
+          uri.to_s
+        end
+      end
+
+      def verify!
+        md5 = capture :md5, '-q', local_archive
+        md5_file = local_archive + '.md5'
+        if md5 == open(md5_file).read.split(' ').first
+          execute :rm, '-f', md5_file
+        else
+          raise "Downloaded data dump is invalid (MD5 hash verification failed)"
+        end
+      end
+    end
+  end
+end

data/lib/eve_app/sde/normalizer.rb

@@ -0,0 +1,106 @@
+module EveApp
+  module SDE
+    module Normalizer
+      def normalize
+        table_names
+        column_names
+        missing_relations
+        indexes
+      end
+
+      def table_names
+        table_list.each do |table_name, normalized_name|
+          sql %Q(ALTER TABLE IF EXISTS "#{table_name}" RENAME TO "#{normalized_name}")
+        end
+      end
+
+      def column_names
+        columns.each do |row|
+          resource_name = row[:table_name].singularize.gsub('eve_', '')
+          row[:target_column_name] = row[:column_name].underscore
+          if row[:target_column_name].starts_with?(resource_name)
+            row[:target_column_name] = row[:target_column_name].gsub("#{resource_name}_", "")
+          end
+
+          if row[:column_name] != row[:target_column_name]
+            sql %Q(ALTER TABLE "#{row[:table_name]}" RENAME COLUMN "#{row[:column_name]}" TO "#{row[:target_column_name]}")
+          end
+        end
+      end
+
+      def indexes
+        columns.each do |row|
+          if (row[:column_name].ends_with?('id') || row[:column_name] == 'name') && !has_index?(db, row[:table_name], row[:column_name])
+            sql %Q(CREATE INDEX IF NOT EXISTS idx_#{row[:table_name]}_#{row[:column_name]} ON #{row[:table_name]} (#{row[:column_name]});)
+          end
+          if row[:column_name] == 'id'
+            sql %Q(ALTER TABLE #{row[:table_name]} DROP CONSTRAINT IF EXISTS #{row[:table_name]}_pkey)
+
+            if complex_id_index?(row[:table_name])
+              sql %Q(ALTER TABLE #{row[:table_name]} ADD PRIMARY KEY (id, type_id))
+            else
+              sql %Q(ALTER TABLE #{row[:table_name]} ADD PRIMARY KEY (id))
+            end
+          end
+        end
+      end
+
+      def missing_relations
+        sql %Q(ALTER TABLE #{table_list['invTypes']} ADD IF NOT EXISTS category_id integer)
+        sql %Q(ALTER TABLE #{table_list['invMarketGroups']} ADD root_group_id INTEGER DEFAULT NULL)
+        sql %Q(ALTER TABLE #{table_list['invTypes']} ADD market_group_root_id integer)
+        sql %Q(UPDATE #{table_list['invTypes']} SET category_id = (SELECT category_id FROM #{table_list['invGroups']} WHERE id = #{table_list['invTypes']}.group_id))
+        sql %Q(
+          WITH RECURSIVE mg_roots(id, root_id) AS (
+            SELECT mg.id, mg.id AS root_id FROM #{table_list['invMarketGroups']} AS mg WHERE mg.parent_group_id IS NULL
+            UNION ALL
+              SELECT c.id, p.root_id FROM mg_roots AS p, #{table_list['invMarketGroups']} AS c WHERE c.parent_group_id = p.id
+          )
+          UPDATE #{table_list['invMarketGroups']} SET root_group_id = mg_roots.root_id FROM (
+            SELECT id, root_id FROM mg_roots WHERE root_id != id
+          ) AS mg_roots WHERE #{table_list['invMarketGroups']}.id = mg_roots.id;
+        )
+        sql %Q(UPDATE #{table_list['invTypes']} SET market_group_root_id = (SELECT root_group_id FROM #{table_list['invMarketGroups']} WHERE id = #{table_list['invTypes']}.market_group_id))
+      end
+
+      private
+
+      def complex_id_index?(table_name)
+        SDE::ID_TYPE_INDEX.include?(table_name.gsub("#{SDE.config.table_prefix.to_s}_", ''))
+      end
+
+      def columns
+        query = %Q(
+          SELECT table_name, column_name, ordinal_position
+          FROM information_schema.columns
+          WHERE
+            table_catalog = '#{db_config[:database]}' AND
+            table_schema = 'public' AND
+            table_name IN(#{table_list.values.map { |n| "'#{n}'" }.join(', ')})
+        )
+        db.select_all(query).map(&:symbolize_keys)
+      end
+
+      def has_index?(db, table, column)
+        count = db.select_value %Q(
+          select
+            COUNT(*)
+          from
+            pg_class t,
+            pg_class i,
+            pg_index ix,
+            pg_attribute a
+          where
+            t.oid = ix.indrelid
+            and i.oid = ix.indexrelid
+            and a.attrelid = t.oid
+            and a.attnum = ANY(ix.indkey)
+            and t.relkind = 'r'
+            and t.relname = '#{table}'
+            and a.attname = '#{column}';
+        )
+        count.to_i > 0
+      end
+    end
+  end
+end

data/lib/eve_app/sde.rb

@@ -0,0 +1,27 @@
+module EveApp
+  module SDE
+    autoload :DataImporter, 'eve_app/sde/data_importer'
+    autoload :Downloader,   'eve_app/sde/downloader'
+    autoload :Normalizer,   'eve_app/sde/normalizer'
+
+    DEFAULT_CONFIG = {
+      table_prefix:    :eve,
+      download_host:   'https://www.fuzzwork.co.uk/dump',
+      archive:         'postgres-latest.dmp.bz2',
+      tmp_path:        Rails.root.join('tmp', 'eve-sde'),
+      table_list_file: EveApp.root.join('lib', 'table-list.yml')
+    }
+    PREFIXES = %w(agt dgm map trn inv sta industry)
+    ID_TYPE_INDEX = %w(activities)
+
+    class << self
+      def config
+        @_config ||= OpenStruct.new(DEFAULT_CONFIG)
+      end
+
+      def table_list
+        @_table_list ||= YAML::load_file(config.table_list_file)
+      end
+    end
+  end
+end

data/lib/eve_app/version.rb

@@ -0,0 +1,3 @@
+module EveApp
+  VERSION = '0.1.0'
+end

data/lib/eve_app/xml_api/calls.rb

@@ -0,0 +1,197 @@
+module EveApp
+  module XmlApi
+    class Calls
+      class Base
+        class_attribute :endpoint
+        class_attribute :selector
+        class_attribute :class_name
+
+        self.selector = '//rowset/row'
+
+        attr_reader :xml, :results
+        attr_accessor :cached_until
+
+        delegate :blank?, :present?, :any?, to: :results
+
+        def initialize(xml=nil)
+          if xml
+            @xml = xml
+            @results = xml.search(selector).map { |row| class_name.new(row) }
+            @cached_until = (xml.search('cachedUntil').text + 'Z').to_time
+          else
+            @results = []
+          end
+        end
+
+        def merge(call)
+          @results += call.results
+        end
+
+        def ids
+          @results.map(&:id)
+        end
+
+        def error?
+          false
+        end
+      end
+
+      class Characters < Base
+        self.class_name = Classes::Characters
+        self.endpoint = '/account/Characters.xml.aspx'
+      end
+
+      class Sheet < Base
+        attr_reader :sheet
+
+        def initialize(xml)
+          @xml = xml
+          @sheet = class_name.new(xml)
+        end
+      end
+      class CharacterSheet < Sheet
+        self.class_name = Classes::CharacterSheet
+        self.endpoint = '/char/CharacterSheet.xml.aspx'
+      end
+      class CorporationSheet < Sheet
+        self.class_name = Classes::CorporationSheet
+        self.endpoint = '/corp/CorporationSheet.xml.aspx'
+      end
+
+      class AccountBalance < Base
+        self.class_name = Classes::AccountBalance
+      end
+      class PersonalAccountBalance < AccountBalance
+        self.endpoint = '/char/AccountBalance.xml.aspx'
+      end
+      class CorporateAccountBalance < AccountBalance
+        self.endpoint = '/corp/AccountBalance.xml.aspx'
+      end
+
+      class WalletTransactions < Base
+        self.class_name = Classes::WalletTransaction
+      end
+      class PersonalWalletTransactions < WalletTransactions
+        self.endpoint = '/char/WalletTransactions.xml.aspx'
+      end
+      class CorporateWalletTransactions < WalletTransactions
+        self.endpoint = '/corp/WalletTransactions.xml.aspx'
+      end
+
+      class WalletJournals < Base
+        self.class_name = Classes::WalletJournal
+      end
+      class PersonalWalletJournals < WalletJournals
+        self.endpoint = '/char/WalletJournal.xml.aspx'
+      end
+      class CorporateWalletJournals < WalletJournals
+        self.endpoint = '/corp/WalletJournal.xml.aspx'
+      end
+
+      class Blueprints < Base
+        self.class_name = Classes::Blueprint
+      end
+      class PersonalBlueprints < Blueprints
+        self.endpoint = '/char/Blueprints.xml.aspx'
+      end
+      class CorporateBlueprints < Blueprints
+        self.endpoint = '/corp/Blueprints.xml.aspx'
+      end
+
+      class MarketOrders < Base
+        self.class_name = Classes::MarketOrder
+      end
+      class PersonalMarketOrders < MarketOrders
+        self.endpoint = '/char/MarketOrders.xml.aspx'
+      end
+      class CorporateMarketOrders < MarketOrders
+        self.endpoint = '/corp/MarketOrders.xml.aspx'
+      end
+
+      class IndustryJobs < Base
+        self.class_name = Classes::IndustryJob
+      end
+      class PersonalIndustryJobs < IndustryJobs
+        self.endpoint = '/char/IndustryJobs.xml.aspx'
+      end
+      class CorporateIndustryJobs < IndustryJobs
+        self.endpoint = '/corp/IndustryJobs.xml.aspx'
+      end
+      class PersonalIndustryJobHistory < IndustryJobs
+        self.endpoint = '/char/IndustryJobsHistory.xml.aspx'
+      end
+      class CorporateIndustryJobHistory < IndustryJobs
+        self.endpoint = '/corp/IndustryJobsHistory.xml.aspx'
+      end
+
+      class AssetList < Base
+        attr_reader :assets
+
+        def initialize(xml)
+          @assets = []
+          parse_rows(xml.search("//rowset[@name='assets']/row"))
+          @cached_until = (xml.search('cachedUntil').text + 'Z').to_time
+        end
+
+        def parse_rows(rows, parent=nil)
+          rows.each do |container|
+            asset = Classes::Asset.new(container)
+            asset.container = container.children.any?
+            asset.parent = parent if parent
+            parse_rows(container.search("rowset/row"), asset) if asset.container?
+            @assets << asset
+          end
+        end
+      end
+      class PersonalAssetList < AssetList
+        self.endpoint = '/char/AssetList.xml.aspx'
+      end
+      class CorporateAssetList < AssetList
+        self.endpoint = '/corp/AssetList.xml.aspx'
+      end
+
+      class CorporateMemberTracking < Base
+        self.class_name = Classes::CorporateMemberTracking
+        self.endpoint = '/corp/MemberTracking.xml.aspx'
+      end
+
+      class CorporateMemberSecurity < Base
+        self.class_name = Classes::CorporateMemberSecurity
+        self.endpoint = '/corp/MemberSecurity.xml.aspx'
+      end
+
+      class CorporateTitles < Base
+        self.class_name = Classes::CorporateTitle
+        self.endpoint = '/corp/Titles.xml.aspx'
+      end
+
+      class ErrorResponse < Base
+        attr_reader :exception, :call, :params
+
+        def initialize(exception, call=nil, params={})
+          @exception = exception
+          @call = call
+          @params = params
+        end
+
+        def error?
+          true
+        end
+
+        def title
+          exception.http_body.scan(/<title>(.+?)<\/title>/).flatten[0]
+        end
+
+        def to_s
+          puts "================================"
+          puts "API Error: #{exception.class}"
+          puts "title: #{title}"
+          puts "body: #{exception.http_body}"
+          puts "call: #{call}"
+          puts "params: #{params.inspect}"
+          puts "================================"
+        end
+      end
+    end
+  end
+end

data/lib/eve_app/xml_api/classes.rb

@@ -0,0 +1,260 @@
+module EveApp
+  module XmlApi
+    class Classes
+      class Base
+        protected
+
+        def parse_time(value)
+          !value || value.starts_with?('0001') ? nil : "#{value}Z".to_time
+        end
+      end
+
+      class Characters < Base
+        attr_reader :id, :name, :corporation_id, :corporation_name, :alliance_id, :alliance_name, :faction_id, :faction_name
+
+        def initialize(elem)
+          @id = elem['characterID'].to_i
+          @name = elem['name']
+          @corporation_id = elem['corporationID'].to_i
+          @corporation_name = elem['corporationName']
+          @alliance_id = elem['allianceID'].to_i
+          @alliance_name = elem['allianceName']
+          @faction_id = elem['factionID'].to_i
+          @faction_name = elem['factionName']
+        end
+      end
+
+      class AccountBalance < Base
+        attr_reader :id, :key, :balance
+
+        def initialize(elem)
+          @id      = elem['accountID'].to_i
+          @key     = elem['accountKey'].to_i
+          @balance = elem['balance'].to_f
+        end
+      end
+
+      class CharacterSheet < Base
+        attr_reader :id, :name, :corporation_id
+
+        def initialize(xml)
+          @id             = xml.search('//result/characterID[1]').text.to_i
+          @name           = xml.search('//result/name[1]').text
+          @corporation_id = xml.search('//result/corporationID[1]').text.to_i
+        end
+      end
+
+      class CorporationSheet < Base
+        attr_reader :id, :name, :alliance_id
+
+        def initialize(xml)
+          @id          = xml.search('//result/corporationID[1]').text.to_i
+          @name        = xml.search('//result/corporationName[1]').text
+          @alliance_id = xml.search('//result/allianceID[1]').text.to_i
+        end
+      end
+
+      class WalletTransaction < Base
+        attr_reader :created_at, :id, :quantity, :type_name, :type_id, :price,
+                    :client_id, :client_name, :character_id, :station_id, :station_name, :type,
+                    :transaction_for, :character_name
+
+        def initialize(elem)
+          @created_at       = parse_time(elem['transactionDateTime'])
+          @id               = elem['transactionID'].to_i
+          @quantity         = elem['quantity'].to_i
+          @type_name        = elem['typeName']
+          @type_id          = elem['typeID'].to_i
+          @price            = elem['price'].to_f
+          @client_id        = elem['clientID'].to_i if elem['clientID']
+          @client_name      = elem['clientName']
+          @character_name   = elem['characterName']
+          @station_id       = elem['stationID'].to_i
+          @station_name     = elem['stationName']
+          @character_id     = elem['characterID'].to_i if elem['characterID'] && elem['characterID'] != '0'
+          @type             = elem['transactionType']
+          @transaction_for  = elem['transactionFor']
+        end
+      end
+
+      class WalletJournal < Base
+        attr_reader :created_at, :id, :ref_type_id, :owner1_name, :owner1_id, :owner1_type_id,
+                    :owner2_name, :owner2_id, :owner2_type_id, :arg_name, :arg_id, :amount, :balance, :reason
+
+        def initialize(elem)
+          @created_at  = parse_time(elem['date'])
+          @id          = elem['refID'].to_i
+          @ref_type_id = elem['refTypeID'].to_i
+          @owner1_id   = elem['ownerID1'].to_i if elem['ownerID1']
+          @owner1_name = elem['ownerName1']
+          @owner1_type = elem['owner1TypeID']
+          @owner2_id   = elem['ownerID2'].to_i if elem['ownerID2']
+          @owner2_name = elem['ownerName2']
+          @owner2_type = elem['owner2TypeID']
+          @arg_name    = elem['argName1']
+          @arg_id      = elem['argID1'].to_i if elem['argID1']
+          @amount      = elem['amount'].to_f
+          @balance     = elem['balance'].to_f
+          @reason      = elem['reason']
+        end
+      end
+
+      class Blueprint < Base
+        attr_reader :item_id, :location_id, :type_id, :type_name, :flag, :quantity, :te, :me, :runs
+
+        def initialize(elem)
+          @item_id     = elem['itemID'].to_i
+          @location_id = elem['locationID']
+          @type_id     = elem['typeID'].to_i
+          @type_name   = elem['typeName']
+          @flag        = elem['flagID'].to_i
+          @quantity    = elem['quantity'].to_i
+          @te          = elem['timeEfficiency'].to_i
+          @me          = elem['materialEfficiency'].to_i
+          @runs        = elem['runs'].to_i
+        end
+      end
+
+      class Asset < Base
+        attr_accessor :container, :parent
+        attr_reader :item_id, :type_id, :location_id, :quantity, :flag, :repacked, :raw_quantity
+
+        def initialize(elem)
+          @container = false
+          @parent = nil
+          @item_id = elem['itemID'].to_i
+          @type_id = elem['typeID'].to_i
+          @location_id = elem['locationID'].to_i
+          @quantity = elem['quantity'].to_i
+          @flag = elem['flag'].to_i
+          @repacked = elem['singleton'].to_i
+          @raw_quantity = elem['rawQuantity'].to_i
+        end
+
+        def container?
+          !!container
+        end
+
+        def parent?
+          !!parent
+        end
+      end
+
+      class MarketOrder < Base
+        attr_reader :order_id, :character_id, :station_id, :volume_entered, :volume_remaining, :minimum_volume,
+                    :state, :type_id, :range, :account_key, :duration, :escrow, :price, :bid, :issued_at
+
+        def initialize(elem) #:nodoc:
+          @order_id = elem['orderID'].to_i
+          @character_id = elem['charID'].to_i
+          @station_id = elem['stationID'].to_i
+          @volume_entered = elem['volEntered'].to_i
+          @volume_remaining = elem['volRemaining'].to_i
+          @minimum_volume = elem['minVolume'].to_i
+          @state = case elem['orderState'].to_i
+          when 0
+            'Active'
+          when 1
+            'Closed'
+          when 2
+            'Expired'
+          when 3
+            'Cancelled'
+          when 4
+            'Pending'
+          when 5
+            'Character Deleted'
+          end
+          @type_id = elem['typeID'].to_i
+          @range = elem['range'].to_i
+          @account_key = elem['accountKey'].to_i
+          @escrow = elem['escrow'].to_f
+          @price = elem['price'].to_f
+          @bid = elem['bid'] == '1'
+          @duration = elem['duration'].to_i
+          @issued_at = parse_time(elem['issued'])
+        end
+      end
+
+      class IndustryJob < Base
+        attr_reader :job_id, :installer_id, :installer_name, :facility_id, :system_id, :system_name,
+                    :station_id, :activity_id, :blueprint_id, :blueprint_type_id, :blueprint_type_name,
+                    :blueprint_location_id, :output_location_id, :runs, :cost, :team_id, :licensed_runs,
+                    :probability, :product_type_id, :product_type_name, :status, :duration, :start_date,
+                    :end_date, :pause_date, :completed_date, :completed_character_id, :successful_runs
+
+        def initialize(elem)
+          @job_id = elem['jobID'].to_i    # "306488326"
+          @installer_id = elem['installerID'].to_i    # "1663516939"
+          @installer_name = elem['installerName']    # "GaIIente Slave"
+          @facility_id = elem['facilityID'].to_i    # "1022215707239"
+          @system_id = elem['solarSystemID'].to_i    # "30001363"
+          @system_name = elem['solarSystemName']    # "Sobaseki"
+          @station_id = elem['stationID'].to_i    # "1022182744039"
+          @activity_id = elem['activityID'].to_i    # "1"
+          @blueprint_id = elem['blueprintID'].to_i    # "1022215694764"
+          @blueprint_type_id = elem['blueprintTypeID'].to_i    # "22545"
+          @blueprint_type_name = elem['blueprintTypeName']    # "Hulk Blueprint"
+          @blueprint_location_id = elem['blueprintLocationID'].to_i    # "1022215707239"
+          @output_location_id = elem['outputLocationID'].to_i    # "1022215707239"
+          @runs = elem['runs'].to_i    # "1"
+          @cost = elem['cost'].to_f    # "8920512.00"
+          @team_id = elem['teamID'].to_i    # "0"
+          @licensed_runs = elem['licensedRuns'].to_i    # "1"
+          @probability = elem['probability'].to_f    # "1"
+          @product_type_id = elem['productTypeID'].to_i
+          @product_type_name = elem['productTypeName']
+          @status = elem['status'].to_i
+          @duration = elem['timeInSeconds'].to_i
+          @start_date = parse_time(elem['startDate'])
+          @end_date = parse_time(elem['endDate'])
+          @pause_date = parse_time(elem['pauseDate'])
+          @completed_date = parse_time(elem['completedDate'])
+          @completed_character_id = elem['completedCharacterID'].to_i
+          @successful_runs = elem['successfulRuns'].to_i
+        end
+      end
+
+      class CorporateMemberTracking < Base
+        attr_reader :character_id, :name, :start_date, :base_id, :base, :title, :logon_date, :logoff_date, :location_id, :location, :ship_type_id, :ship_type, :roles, :grantable_roles
+
+        def initialize(elem)
+          @character_id = elem['characterID'].to_i
+          @name = elem['name']
+          @start_date = parse_time(elem['startDateTime'])
+          @base_id = elem['baseID'].to_i
+          @base = elem['base']
+          @title = elem['title']
+          @logon_date = parse_time(elem['logonDateTime'])
+          @logoff_date = parse_time(elem['logoffDateTime'])
+          @location_id = elem['locationID'].to_i > 0 ? elem['locationID'].to_i : nil
+          @location = elem['location'].presence
+          @ship_type_id = elem['shipTypeID'].to_i
+          @ship_type = elem['shipType']
+          @roles = elem['roles']
+          @grantable_roles = elem['grantableRoles']
+        end
+      end
+
+      class CorporateMemberSecurity < Base
+        attr_reader :character_id, :name, :titles
+
+        def initialize(elem)
+          @character_id = elem['characterID'].to_i
+          @name = elem['name']
+          @titles = elem.search('rowset[@name="titles"]/row').map { |t| t['titleName'] }
+        end
+      end
+
+      class CorporateTitle < Base
+        attr_reader :id, :name, :description
+
+        def initialize(elem)
+          @id = elem['roleID'].to_i
+          @name = elem['roleName']
+          @description = elem['roleDescription']
+        end
+      end
+    end
+  end
+end