caddie 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ae68e873fe5e7e634110966042264a149da5f8b
-  data.tar.gz: 18100d92938dc47517cbcb0b058c4b309e2cbe96
+  metadata.gz: dec5dc1e0bf8562abd8ce5afcbefcb44ef4d0f4c
+  data.tar.gz: b27051154d8eb7d652ad9dc79b0f91397ca6fc75
 SHA512:
-  metadata.gz: f7a9babedd01608b5b19751525e57fe7127e056b7410df5c130d783c41cd3e6c9f1620e7b5c21e13c046f437b4730d0ca0779288b4d5c3186e55502c23f1ddf6
-  data.tar.gz: f9ff63e670a1bd2bc334c405b4cb9ad47a944bd23ee7950a2fbfe3925bc637111eeec735400ef091743ab0c8a98d4c730b27f78e44557112f42f2aa4bd71cf31
+  metadata.gz: 0d0b8305fd526f1f677d77554b466b6264f95cb6fe1ad2b586a5cc4e82e8963cae7ce45dd2cd10b6d475ef4545175d3e2c5818a4985416a54158c69b0e8d6543
+  data.tar.gz: 29b46515c9bfd855198f712468ea687d7441dc90635c004e7d86c9313a51735a296e1cc84b94864f3d043e7e2b328dc7b9a386c352860852b77162e3d825c921

data/app/models/caddie/crest_data_retriever.rb

@@ -7,7 +7,7 @@ module Caddie
 
     def get_markets( region_id, type_id )
 
-      debug = ENV[ 'CADDIE_DEBUG_MODE' ] && ENV[ 'CADDIE_DEBUG_MODE' ].downcase == 'true'
+      debug = ENV[ 'EBS_DEBUG_MODE' ] && ENV[ 'EBS_DEBUG_MODE' ].downcase == 'true'
 
       type_url = "https://crest-tq.eveonline.com/inventory/types/#{type_id}"
       items, connections_count = get_multipage_data( "market/#{region_id}/history/?type=#{type_url}", debug )

data/app/models/caddie/crest_price_history_update.rb

@@ -8,18 +8,26 @@ module Caddie
 
     extend Caddie::CrestDataRetriever
 
+    NB_THREADS=4
+
     def self.update
       current_path = File.dirname( __FILE__ )
       request = File.open( "#{current_path}/update_table.sql" ).read
       ActiveRecord::Base.connection.execute( request )
     end
 
-    def self.feed_price_histories
+    def self.feed_price_histories( thread_id = nil )
       total_connections_counts = 0
       total_inserts = 0
       date_deb = Time.now
 
-      daily_operations_list.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id ).each do |row|
+      dol = daily_operations_list
+
+      # pp dol.to_a
+      dol = dol.where( thread_slice_id: thread_id ) if thread_id
+
+      # puts dol.reload.count
+      dol.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id ).each do |row|
 
         # puts "Processing row = #{row}"
 
@@ -30,6 +38,7 @@ module Caddie
 
         ActiveRecord::Base.transaction do
 
+          # TODO : This operation is extremely costly. Keep the last updated timestamp and use it to shorten insert.
           # puts 'About to reject already used lines'
           timestamps = CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id ).pluck( :day_timestamp ).to_set
           items.reject! do |item|
@@ -39,7 +48,6 @@ module Caddie
           end
           # puts 'Lines rejected'
 
-
           items.each do |item_data|
 
             date_info = DateTime.parse( item_data['date'] )

data/app/models/caddie/m_threaded_updater.rb

@@ -1,6 +1,43 @@
-class MThreadedUpdater
+require 'thwait'
+require 'matrix'
+
+class Caddie::MThreadedUpdater
 
   # TODO : - split the work in database using thread_slice_id (set numbers : 1 .. MAX_THREADS)
   # TODO : Then run N threads each threads get the records associated to it's number
 
+  def initialize( max_threads, daily_operations_list )
+    @max_threads = max_threads
+    @daily_operations_list = daily_operations_list
+  end
+
+  def feed_price_histories_threaded
+    Thread::abort_on_exception = true
+    threads = []
+    0.upto( @max_threads-1 ).each do |thread_id|
+      threads << Thread.new {
+        puts Thread.inspect
+        Thread.current[:timings] = Caddie::CrestPriceHistoryUpdate.feed_price_histories( thread_id )
+      }
+    end
+    result = []
+    ThreadsWait.all_waits( *threads ) do |t|
+      thread_result = t[:timings]
+      result << thread_result
+    end
+    tmp = result.map { |a| Vector[*a] }.inject(:+)
+    tmp.to_a
+  end
+
+  def split_work_for_threads
+    ids = @daily_operations_list.pluck( :id )
+    ActiveRecord::Base.transaction do
+      slice_size = ids.count/@max_threads + 1
+      ( 0 ... @max_threads ).each do |thread_id|
+        ids_slice = ids.shift( slice_size )
+        @daily_operations_list.where( id: ids_slice ).update_all( thread_slice_id: thread_id )
+      end
+    end
+  end
+
 end

data/app/models/crest_price_history.rb

@@ -0,0 +1,4 @@
+class CrestPriceHistory < ActiveRecord::Base
+  belongs_to :region
+  belongs_to :eve_item
+end

data/db/migrate/20160725091214_add_indexes_to_crest_price_history_update.rb

@@ -0,0 +1,8 @@
+class AddIndexesToCrestPriceHistoryUpdate < ActiveRecord::Migration
+  def change
+    add_index :caddie_crest_price_history_updates, :nb_days
+    add_index :caddie_crest_price_history_updates, :process_queue
+    add_index :caddie_crest_price_history_updates, :next_process_date
+    add_index :caddie_crest_price_history_updates, :thread_slice_id
+  end
+end

data/lib/caddie/version.rb

@@ -1,3 +1,3 @@
 module Caddie
-  VERSION = '0.1.2'
+  VERSION = '0.2.0'
 end

data/lib/tasks/caddie_tasks.rake

@@ -6,14 +6,19 @@ namespace :caddie do
     start_time = Time.now
     feed_date = start_time.to_date
 
-    puts 'About to compute crest_price_history_updates'
-    Caddie::CrestPriceHistoryUpdate.update
+    unless ENV[ 'EBS_NO_HIST_UPDATE' ] && ENV[ 'EBS_NO_HIST_UPDATE' ].downcase == 'true'
+      puts 'About to compute crest_price_history_updates'
+      Caddie::CrestPriceHistoryUpdate.update
+    end
 
     end_update_time = Time.now
     update_planning_time = end_update_time - start_time
 
     puts 'About to feed crest_price_histories'
-    total_inserts, total_connections, total_time = Caddie::CrestPriceHistoryUpdate.feed_price_histories
+    # total_inserts, total_connections, total_time = Caddie::CrestPriceHistoryUpdate.feed_price_histories
+    th = Caddie::MThreadedUpdater.new( Caddie::CrestPriceHistoryUpdate::NB_THREADS, Caddie::CrestPriceHistoryUpdate.daily_operations_list )
+    th.split_work_for_threads
+    total_inserts, total_connections, total_time = th.feed_price_histories_threaded
     puts "#{total_inserts} insertions, #{total_connections} connections in #{total_time.round( 2 )} seconds. #{(total_connections/total_time).round( 2 )} co/sec"
 
     end_feeding_time = Time.now

data/lib/tasks/m_threaded_updater_test.rake

@@ -0,0 +1,53 @@
+require 'mocha/api'
+require 'minitest'
+
+include Mocha::API
+include FactoryGirl::Syntax::Methods
+
+require_relative '../../test/dummy/app/models/eve_item'
+require_relative '../../test/dummy/app/models/region'
+require_relative '../../test/factories/caddie/eve_items'
+require_relative '../../test/factories/caddie/regions'
+require_relative '../../app/models/caddie/crest_data_retriever'
+require_relative '../../app/models/caddie/crest_price_history_update'
+require_relative '../../test/factories/caddie/crest_price_history_updates'
+
+# Regular test unit does not commit between insertions, this algo rely on database to split work between threads
+# So we have to fake a test outside the test unit
+namespace :test do
+
+  desc "M threaded uptader test"
+  task :m_threaded_uptader_test=> :environment do
+    if (ENV['RAILS_ENV'] == "test") # protect against execution in dev mode
+
+      Caddie::CrestPriceHistoryUpdate.stubs( :get_multipage_data ).returns( [ [], 5 ] )
+
+      Region.delete_all
+      EveItem.delete_all
+      Caddie::CrestPriceHistoryUpdate.delete_all
+
+      next_process_date = Time.now.to_date
+      region = create( :caddie_heimatar )
+      1.upto( 53 ).each do
+        item = create( :caddie_eve_item )
+        create( :crest_price_history_update, region: region, eve_item: item, next_process_date: next_process_date )
+      end
+
+      th = Caddie::MThreadedUpdater.new( 4, Caddie::CrestPriceHistoryUpdate.daily_operations_list )
+      th.split_work_for_threads
+      timings = th.feed_price_histories_threaded
+      # pp timings
+      unless 53 == timings[1]/5
+        "Test failed : #{timings[1]/5} != 53"
+      else
+        puts 'Test passed'
+      end
+    else
+      puts 'Can be only executed in test env'
+    end
+
+    Region.delete_all
+    EveItem.delete_all
+    Caddie::CrestPriceHistoryUpdate.delete_all
+  end
+end

data/test/dummy/app/models/eve_item.rb

@@ -0,0 +1,53 @@
+# require 'open-uri'
+# # require 'open-uri/cached'
+require 'pp'
+
+class EveItem < ActiveRecord::Base
+
+  # include Assert
+  # include ItemsInit::ItemSetupAndComp
+  # extend ItemsInit::ItemSetupAndCompSelf
+  # extend MultiplePriceRetriever
+  # extend Setup::UpdateEveItems
+
+  has_and_belongs_to_many :users
+  has_one :blueprint, dependent: :destroy
+  has_many :materials, through: :blueprint
+  has_many :components, through: :materials
+  belongs_to :market_group
+  has_many :crest_price_histories, dependent: :destroy
+  has_many :crest_prices_last_month_averages, dependent: :destroy
+
+  # Itemps containing non ascii characters
+  UNPROCESSABLE_ITEMS=[34457,34458,34459,34460,34461,34462,34463,34464,34465,34466,34467,34468,34469,34470,34471,34472,
+                       34473,34474,34475,34476,34477,34478,34479,34480,30952,32371,32372]
+
+  AVG_INDUSTRY_TAX = 0.11 # 10 % base + 1 % system costs (assuming players are smart enough to go in low cost systems)
+
+  def self.to_eve_item_id(cpp_eve_item_id)
+    eve_item=EveItem.where( 'cpp_eve_item_id=?', cpp_eve_item_id).first
+    eve_item ? eve_item.id : nil
+  end
+
+  def self.used_items
+    used_items, _ = User.get_used_items_and_trade_hubs
+    used_items
+  end
+
+  def single_unit_cost
+    (cost*(1+AVG_INDUSTRY_TAX))/blueprint.prod_qtt if cost && blueprint
+  end
+
+  def pcent_margin( price )
+    (price / single_unit_cost)-1 if price && single_unit_cost
+  end
+
+  def margin( price )
+    price - single_unit_cost if price && single_unit_cost
+  end
+
+  def full_batch_size
+    blueprint.nb_runs*blueprint.prod_qtt if blueprint
+  end
+
+end

data/test/dummy/app/models/region.rb

@@ -0,0 +1,4 @@
+class Region < ActiveRecord::Base
+  has_many :trade_hubs
+  has_many :crest_prices_last_month_averages
+end

data/test/dummy/config/database.yml

@@ -9,11 +9,11 @@ default: &default
   timeout: 5000
   adapter: postgresql
   encoding: unicode
-  username: eve_business_server
+  username: caddie
 
 development:
   <<: *default
-  database: eve_business_server_dev
+  database: caddie_dev
 
 
 # Warning: The database defined as "test" will be erased and
@@ -21,4 +21,4 @@ development:
 # Do not set this db to the same as development or production.
 test:
   <<: *default
-  database: eve_business_server_test
+  database: caddie_test

data/test/dummy/db/migrate/20150410082132_create_eve_items.rb

@@ -0,0 +1,11 @@
+class CreateEveItems < ActiveRecord::Migration
+  def change
+    create_table :eve_items do |t|
+      t.integer :eve_item_id
+      t.string :name
+
+      t.timestamps
+    end
+    add_index :eve_items, :eve_item_id
+  end
+end

data/test/dummy/db/migrate/20150414095303_add_name_lowcase_to_eve_item.rb

@@ -0,0 +1,5 @@
+class AddNameLowcaseToEveItem < ActiveRecord::Migration
+  def change
+    add_column :eve_items, :name_lowcase, :string
+  end
+end

data/test/dummy/db/migrate/20150416154256_rename_eve_item_id.rb

@@ -0,0 +1,5 @@
+class RenameEveItemId < ActiveRecord::Migration
+  def change
+    rename_column :eve_items, :eve_item_id, :cpp_eve_item_id
+  end
+end

data/test/dummy/db/migrate/20150417135716_add_cost_to_eve_item.rb

@@ -0,0 +1,5 @@
+class AddCostToEveItem < ActiveRecord::Migration
+  def change
+    add_column :eve_items, :cost, :float
+  end
+end

data/test/dummy/db/migrate/20150906171550_add_cpp_market_group_id_to_eve_item.rb

@@ -0,0 +1,6 @@
+class AddCppMarketGroupIdToEveItem < ActiveRecord::Migration
+  def change
+    add_column :eve_items, :cpp_market_group_id, :integer
+    add_index :eve_items, :cpp_market_group_id
+  end
+end

data/test/dummy/db/migrate/20150909162142_add_involved_in_blueprint_to_eve_item.rb

@@ -0,0 +1,5 @@
+class AddInvolvedInBlueprintToEveItem < ActiveRecord::Migration
+  def change
+    add_column :eve_items, :involved_in_blueprint, :bool, default: false, index: true
+  end
+end

data/test/dummy/db/migrate/20150910074544_create_regions.rb

@@ -0,0 +1,11 @@
+class CreateRegions < ActiveRecord::Migration
+  def change
+    create_table :regions do |t|
+      t.string :cpp_region_id, null: false
+      t.string :name, null: false
+
+      t.timestamps null: false
+    end
+    add_index :regions, :cpp_region_id, unique: true
+  end
+end

data/test/dummy/db/migrate/20150910080646_create_crest_price_histories.rb

@@ -0,0 +1,18 @@
+class CreateCrestPriceHistories < ActiveRecord::Migration
+  def change
+    create_table :crest_price_histories do |t|
+      t.references :region, index: true, foreign_key: true, null: false
+      t.references :eve_item, index: true, foreign_key: true, null: false
+      t.string :day_timestamp, index: true, null: false
+      t.timestamp :history_date, null: false
+      t.column :order_count, :bigint
+      t.column :volume, :bigint
+      t.float :low_price
+      t.float :avg_price
+      t.float :high_price
+
+      t.timestamps null: false
+    end
+    add_index :crest_price_histories, [:region_id, :eve_item_id, :day_timestamp], unique: true, name: 'price_histories_all_keys_index'
+  end
+end