caddie 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c989d34f34beda7cd0080ba8727797560fd3b564
-  data.tar.gz: 966a09a7fdf2044efa9cecb9dd62a5a72c29fa0e
+  metadata.gz: d606561f09dd648d0014e7e792072fd3a1a3f04a
+  data.tar.gz: d9bb7137dd695d8bf4e3e1a071b7d1e984329e05
 SHA512:
-  metadata.gz: 33abff89f2ae029ddf6691027b51525bf262de8881b13b8351277e3abdc71eecd355c2c0760fdf12b9cfa2ea5980045bc25f49c8753169e94e51460ce7c295e8
-  data.tar.gz: 2675616080c8e7755b88d3e30cf7513434869cd8f2031a1869fabbe1ec7c664a683431367fa31ab98e053f96d24463da685df2fac0b2290fefc2585eb52eee16
+  metadata.gz: baeec0fd6bb87e6704ad84a7241adac21b5086f1a073df8f0429b38c05f32b97e9284fcfc56fbaa3119bda697b8bdbd93abb07d0dd4970e20dcecc36ce285f6f
+  data.tar.gz: c941f62d954c7f843dd04326debc5778eb3f6e103586a976c4138fc4189e05d6dd2734ceaba19a33a5040abe17804ee392b1bae77f51f0e7edff2b8ead58713c

data/app/models/caddie/crest_data_retriever.rb

@@ -5,12 +5,13 @@ module Caddie
 
     CREST_BASE_URL='https://crest-tq.eveonline.com/'
 
-    def get_markets( region_id, type_id )
+    def get_markets( region_id, type_id, thread_log_file: nil )
 
       debug = ENV[ 'EBS_DEBUG_MODE' ] && ENV[ 'EBS_DEBUG_MODE' ].downcase == 'true'
+      output = thread_log_file ? thread_log_file : STDERR
 
       type_url = "https://crest-tq.eveonline.com/inventory/types/#{type_id}"
-      items, connections_count = get_multipage_data( "market/#{region_id}/history/?type=#{type_url}", debug )
+      items, connections_count = get_multipage_data( "market/#{region_id}/history/?type=#{type_url}", debug, thread_log_file: thread_log_file )
 
       [ items, connections_count ]
     end
@@ -21,17 +22,19 @@ module Caddie
       "#{CREST_BASE_URL}/#{rest}/"
     end
 
-    def get_multipage_data( rest, debug_request = false )
+    def get_multipage_data( rest, debug_request = false, thread_log_file: nil )
+
+      output = thread_log_file ? thread_log_file : STDOUT
 
       next_url = get_crest_url( rest )
       items = []
       connections_count = 0
       begin
 
-        puts "Fetching : #{next_url}" if debug_request
+        output.puts "Fetching : #{next_url}" if debug_request
 
-        @start_time = Time.now unless @start_time
-        @hit_count = 0 unless @hit_count
+        @start_time ||= Time.now
+        @hit_count ||= 0
         @hit_count += 1
 
         json_result = open( next_url ).read
@@ -46,7 +49,7 @@ module Caddie
 
       end while next_url
 
-      puts "items.count = #{items.count}" if debug_request
+      output.puts "Retrieved items.count = #{items.count}" if debug_request
 
       [ items, connections_count ]
     end

data/app/models/caddie/crest_price_history_last_day_timestamp.rb

@@ -0,0 +1,30 @@
+module Caddie
+  class CrestPriceHistoryLastDayTimestamp < ActiveRecord::Base
+    belongs_to :eve_item
+    belongs_to :region
+
+    def self.find_or_create_last_day_timestamp( region_id, eve_item_id )
+
+      last_update_record = CrestPriceHistoryLastDayTimestamp.find_by_region_id_and_eve_item_id( region_id, eve_item_id )
+
+      if last_update_record
+        last_update_date = last_update_record.day_timestamp
+      else
+        last_update_date = CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id ).maximum( :history_date )
+      end
+
+      last_update_date = Time.new( 0 ) unless last_update_date
+
+      [ last_update_date, last_update_record ]
+    end
+
+    def self.create_or_update_last_day_timestamp( last_update_record, max_date_info, region_id, eve_item_id )
+      if last_update_record
+        last_update_record.update!( day_timestamp: max_date_info )
+      else
+        last_update_record = CrestPriceHistoryLastDayTimestamp.create!( region_id: region_id, eve_item_id: eve_item_id, day_timestamp: max_date_info )
+      end
+      last_update_record
+    end
+  end
+end

data/app/models/caddie/crest_price_history_update.rb

@@ -1,4 +1,5 @@
 require 'set'
+require 'open-uri'
 
 module Caddie
 
@@ -16,59 +17,83 @@ module Caddie
       ActiveRecord::Base.connection.execute( request )
     end
 
-    def self.feed_price_histories( thread_id = nil )
+    def self.feed_price_histories( updates_ids: nil, thread_log_file: nil )
       total_connections_counts = 0
       total_inserts = 0
       date_deb = Time.now
 
-      dol = daily_operations_list
+      debug = ENV[ 'EBS_DEBUG_MODE' ] && ENV[ 'EBS_DEBUG_MODE' ].downcase == 'true' && Rails.env != 'test'
+      output = thread_log_file ? thread_log_file : STDOUT
+      outerr = thread_log_file ? thread_log_file : STDERR
 
-      # pp dol.to_a
-      dol = dol.where( thread_slice_id: thread_id ) if thread_id
+      if updates_ids
+        dol = self.where( id: updates_ids ).order( :process_queue_priority ) if updates_ids
+      else
+        dol = daily_operations_list
+      end
 
-      # puts dol.reload.count
-      dol.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id ).each do |row|
+      output.puts "dol.count = #{dol.count}" if debug
 
-        # puts "Processing row = #{row}"
+      global_errors_count = 0
+      dol.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id, :id ).each do |row|
 
         eve_item_id, region_id, cpp_eve_item_id, cpp_region_id = row
-        # puts "Requesting : #{cpp_region_id}, #{cpp_eve_item_id}"
+        if debug
+          output.puts
+          output.puts '*'*50
+          output.puts 'Requesting:'
+          output.puts "r = Region.find_by_cpp_region_id( #{cpp_region_id} )"
+          output.puts "i = EveItem.find_by_cpp_eve_item_id( #{cpp_eve_item_id} )"
+        end
 
         items = []
         connections_count = 0
+        http_errors = 0
+        # We retry 2 times
         begin
-          items, connections_count = get_markets( cpp_region_id, cpp_eve_item_id )
-        rescue OpenURI::HTTPError => e
-          puts e.inspect
-        end
+          begin
+            items, connections_count = get_markets( cpp_region_id, cpp_eve_item_id, thread_log_file: thread_log_file )
+          rescue => e
+            http_errors += 1
+            global_errors_count += 1
+            outerr.puts "#{e.message} for cpp_region_id = #{cpp_region_id} and cpp_eve_item_id = #{cpp_eve_item_id}" unless Rails.env == 'test'
+            sleep( 5 ) # in case of an error, we don't retry immediately.
+          end
+        end while http_errors >= 1 && http_errors < 2
 
         total_connections_counts += connections_count
 
         ActiveRecord::Base.transaction do
 
-          # TODO : This operation is extremely costly. Keep the last updated timestamp and use it to shorten insert.
-          # puts 'About to reject already used lines'
-          timestamps = CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id ).pluck( :day_timestamp ).to_set
-          items.reject! do |item|
-            date_info = DateTime.parse( item['date'] )
-            date_info_ts = date_info.strftime( '%Y%m%d' )
-            timestamps.include?( date_info_ts )
-          end
-          # puts 'Lines rejected'
+          last_update_date, last_update_record = Caddie::CrestPriceHistoryLastDayTimestamp.
+            find_or_create_last_day_timestamp( region_id, eve_item_id )
 
-          items.each do |item_data|
+          output.puts "Last update record date = #{last_update_date.inspect}" if debug
+          
+          items.reject!{ |item| DateTime.parse( item['date'] ) <= last_update_date }
 
+          output.puts "Kept items.count = #{items.count}" if debug
+
+          max_date_info = last_update_date
+          items.each do |item_data|
             date_info = DateTime.parse( item_data['date'] )
-            date_info_ts = date_info.strftime( '%Y%m%d' )
+            max_date_info = [ max_date_info, date_info ].max
 
-            CrestPriceHistory.create!( region_id: region_id, eve_item_id: eve_item_id, day_timestamp: date_info_ts,
+            CrestPriceHistory.create!( region_id: region_id, eve_item_id: eve_item_id,
               history_date: date_info,  order_count: item_data['orderCount'], volume: item_data['volume'],
               low_price: item_data['lowPrice'], avg_price: item_data['avgPrice'], high_price: item_data['highPrice'] )
               total_inserts += 1
           end
+
+          output.puts "Computed max date info = #{max_date_info}" if debug
+
+          Caddie::CrestPriceHistoryLastDayTimestamp.create_or_update_last_day_timestamp(
+            last_update_record, max_date_info, region_id, eve_item_id )
         end
+        output.flush
+
       end
-      [ total_inserts, total_connections_counts, Time.now - date_deb ]
+      [ total_inserts, total_connections_counts, Time.now - date_deb, global_errors_count ]
     end
 
     private

data/app/models/caddie/crest_price_history_update_log.rb

@@ -1,4 +1,14 @@
 module Caddie
   class CrestPriceHistoryUpdateLog < ActiveRecord::Base
+
+    def self.store_log_data( feed_date, update_planning_time, feeding_time, total_inserts, total_connections, total_time )
+      log = Caddie::CrestPriceHistoryUpdateLog.where( feed_date: feed_date ).first_or_initialize
+      log.update_planning_time = update_planning_time
+      log.feeding_time = feeding_time
+      log.total_inserts = total_inserts
+      log.co_seconds = total_connections.to_f / total_time
+      log.save!
+      log
+    end
   end
 end

data/app/models/caddie/m_threaded_updater.rb

@@ -12,12 +12,14 @@ class Caddie::MThreadedUpdater
   end
 
   def feed_price_histories_threaded
+    split_work_for_threads
     Thread::abort_on_exception = true
     threads = []
     0.upto( @max_threads-1 ).each do |thread_id|
       threads << Thread.new {
-        puts Thread.inspect
-        Thread.current[:timings] = Caddie::CrestPriceHistoryUpdate.feed_price_histories( thread_id )
+        thread_log = File.open( "log/feed_price_histories_threaded_#{thread_id}.log", 'w')
+        Thread.current[:timings] = Caddie::CrestPriceHistoryUpdate.
+          feed_price_histories( updates_ids: @threads_split[ thread_id ], thread_log_file: thread_log )
       }
     end
     result = []
@@ -25,19 +27,19 @@ class Caddie::MThreadedUpdater
       thread_result = t[:timings]
       result << thread_result
     end
-    tmp = result.map { |a| Vector[*a] }.inject(:+)
-    tmp.to_a
+    results = result.map { |a| Vector[*a] }.inject(:+).to_a
+    results[ 2 ] = result.map{ |e| e[2] }.max # For timings we are using the max, not the sum.
+    results
   end
 
   def split_work_for_threads
     puts 'Start splitting work for threads'
     ids = @daily_operations_list.pluck( :id )
-    ActiveRecord::Base.transaction do
-      slice_size = ids.count/@max_threads + 1
-      ( 0 ... @max_threads ).each do |thread_id|
-        ids_slice = ids.shift( slice_size )
-        @daily_operations_list.where( id: ids_slice ).update_all( thread_slice_id: thread_id )
-      end
+    @threads_split = []
+
+    slice_size = ids.count/@max_threads + 1
+    ( 0 ... @max_threads ).each do |thread_id|
+      @threads_split[ thread_id ] = ids.shift( slice_size )
     end
     puts 'Finished splitting work for threads'
   end

data/db/migrate/20160803094530_create_caddie_crest_price_history_last_day_timestamps.rb

@@ -0,0 +1,12 @@
+class CreateCaddieCrestPriceHistoryLastDayTimestamps < ActiveRecord::Migration
+  def change
+    create_table :caddie_crest_price_history_last_day_timestamps do |t|
+      t.references :eve_item, index: false, foreign_key: true
+      t.references :region, index: false, foreign_key: true
+      t.datetime :day_timestamp
+
+      t.timestamps null: false
+    end
+    add_index :caddie_crest_price_history_last_day_timestamps, [ :region_id, :eve_item_id ], unique: true, name: :index_caddie_crest_price_history_last_day_timestamps
+  end
+end

data/lib/caddie/version.rb

@@ -1,3 +1,3 @@
 module Caddie
-  VERSION = '0.2.3'
+  VERSION = '0.2.4'
 end

data/lib/tasks/caddie_tasks.rake

@@ -15,17 +15,15 @@ namespace :caddie do
     update_planning_time = end_update_time - start_time
 
     puts 'About to feed crest_price_histories'
-    # total_inserts, total_connections, total_time = Caddie::CrestPriceHistoryUpdate.feed_price_histories
-    th = Caddie::MThreadedUpdater.new( Caddie::CrestPriceHistoryUpdate::NB_THREADS, Caddie::CrestPriceHistoryUpdate.daily_operations_list )
-    th.split_work_for_threads
-    total_inserts, total_connections, total_time = th.feed_price_histories_threaded
+    total_inserts, total_connections, total_time = Caddie::CrestPriceHistoryUpdate.feed_price_histories
+    # th = Caddie::MThreadedUpdater.new( Caddie::CrestPriceHistoryUpdate::NB_THREADS, Caddie::CrestPriceHistoryUpdate.daily_operations_list )
+    # total_inserts, total_connections, total_time = th.feed_price_histories_threaded
     puts "#{total_inserts} insertions, #{total_connections} connections in #{total_time.round( 2 )} seconds. #{(total_connections/total_time).round( 2 )} co/sec"
 
     end_feeding_time = Time.now
     feeding_time = end_feeding_time - end_update_time
 
-    Caddie::CrestPriceHistoryUpdateLog.create!( feed_date: feed_date, update_planning_time: update_planning_time,
-       feeding_time: feeding_time, total_inserts: total_inserts, co_seconds: total_connections.to_f / total_time )
+    Caddie::CrestPriceHistoryUpdateLog.store_log_data( feed_date, update_planning_time, feeding_time, total_inserts, total_connections, total_time )
 
   end
 end

data/test/dummy/app/models/eve_item.rb

@@ -1,15 +1,7 @@
-# require 'open-uri'
-# # require 'open-uri/cached'
 require 'pp'
 
 class EveItem < ActiveRecord::Base
 
-  # include Assert
-  # include ItemsInit::ItemSetupAndComp
-  # extend ItemsInit::ItemSetupAndCompSelf
-  # extend MultiplePriceRetriever
-  # extend Setup::UpdateEveItems
-
   has_and_belongs_to_many :users
   has_one :blueprint, dependent: :destroy
   has_many :materials, through: :blueprint
@@ -47,7 +39,11 @@ class EveItem < ActiveRecord::Base
   end
 
   def full_batch_size
-    blueprint.nb_runs*blueprint.prod_qtt if blueprint
+    unless blueprint
+      puts "EveItem#full_batch_size : #{self.inspect} has no blueprint"
+      return -Float::INFINITY
+    end
+    blueprint.nb_runs*blueprint.prod_qtt
   end
 
 end

data/test/dummy/db/migrate/20150815033941_add_epic_blueprint_to_eve_item.rb

@@ -0,0 +1,5 @@
+class AddEpicBlueprintToEveItem < ActiveRecord::Migration
+  def change
+    add_column :eve_items, :epic_blueprint, :boolean, default: false
+  end
+end

data/test/dummy/db/migrate/20150916052729_add_market_group_ref_to_eve_item.rb

@@ -0,0 +1,6 @@
+class AddMarketGroupRefToEveItem < ActiveRecord::Migration
+  def change
+    add_reference :eve_items, :market_group, index: true
+    remove_column :eve_items, :cpp_market_group_id
+  end
+end

data/test/dummy/db/migrate/20160803143148_crest_price_history_index_rework_and_others.rb

@@ -0,0 +1,17 @@
+class CrestPriceHistoryIndexReworkAndOthers < ActiveRecord::Migration
+  def change
+
+    remove_column :crest_price_histories, :day_timestamp
+
+    remove_index :crest_price_histories, :eve_item_id
+    remove_index :crest_price_histories, :region_id
+    # remove_index :crest_price_histories, :history_date
+    # remove_index :crest_price_histories, name: :price_histories_all_keys_index
+
+    add_index :crest_price_histories, [ :region_id, :eve_item_id ], name: :index_crest_price_histories_on_region_and_item
+    add_index :crest_price_histories, :history_date
+
+    # remove_index :crest_prices_last_month_averages, name: :prices_lmavg_all_keys_index
+
+  end
+end

data/test/dummy/db/schema.rb

@@ -11,11 +11,21 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160725091214) do
+ActiveRecord::Schema.define(version: 20160803143148) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "caddie_crest_price_history_last_day_timestamps", force: :cascade do |t|
+    t.integer  "eve_item_id"
+    t.integer  "region_id"
+    t.datetime "day_timestamp"
+    t.datetime "created_at",    null: false
+    t.datetime "updated_at",    null: false
+  end
+
+  add_index "caddie_crest_price_history_last_day_timestamps", ["region_id", "eve_item_id"], name: "index_caddie_crest_price_history_last_day_timestamps", unique: true, using: :btree
+
   create_table "caddie_crest_price_history_update_logs", force: :cascade do |t|
     t.date     "feed_date"
     t.integer  "update_planning_time"
@@ -51,23 +61,20 @@ ActiveRecord::Schema.define(version: 20160725091214) do
   add_index "caddie_crest_price_history_updates", ["thread_slice_id"], name: "index_caddie_crest_price_history_updates_on_thread_slice_id", using: :btree
 
   create_table "crest_price_histories", force: :cascade do |t|
-    t.integer  "region_id",               null: false
-    t.integer  "eve_item_id",             null: false
-    t.string   "day_timestamp",           null: false
-    t.datetime "history_date",            null: false
-    t.integer  "order_count",   limit: 8
-    t.integer  "volume",        limit: 8
+    t.integer  "region_id",              null: false
+    t.integer  "eve_item_id",            null: false
+    t.datetime "history_date",           null: false
+    t.integer  "order_count",  limit: 8
+    t.integer  "volume",       limit: 8
     t.float    "low_price"
     t.float    "avg_price"
     t.float    "high_price"
-    t.datetime "created_at",              null: false
-    t.datetime "updated_at",              null: false
+    t.datetime "created_at",             null: false
+    t.datetime "updated_at",             null: false
   end
 
-  add_index "crest_price_histories", ["day_timestamp"], name: "index_crest_price_histories_on_day_timestamp", using: :btree
-  add_index "crest_price_histories", ["eve_item_id"], name: "index_crest_price_histories_on_eve_item_id", using: :btree
-  add_index "crest_price_histories", ["region_id", "eve_item_id", "day_timestamp"], name: "price_histories_all_keys_index", unique: true, using: :btree
-  add_index "crest_price_histories", ["region_id"], name: "index_crest_price_histories_on_region_id", using: :btree
+  add_index "crest_price_histories", ["history_date"], name: "index_crest_price_histories_on_history_date", using: :btree
+  add_index "crest_price_histories", ["region_id", "eve_item_id"], name: "index_crest_price_histories_on_region_and_item", using: :btree
 
   create_table "eve_items", force: :cascade do |t|
     t.integer  "cpp_eve_item_id"
@@ -76,12 +83,13 @@ ActiveRecord::Schema.define(version: 20160725091214) do
     t.datetime "updated_at"
     t.string   "name_lowcase"
     t.float    "cost"
-    t.integer  "cpp_market_group_id"
+    t.boolean  "epic_blueprint",        default: false
     t.boolean  "involved_in_blueprint", default: false
+    t.integer  "market_group_id"
   end
 
   add_index "eve_items", ["cpp_eve_item_id"], name: "index_eve_items_on_cpp_eve_item_id", using: :btree
-  add_index "eve_items", ["cpp_market_group_id"], name: "index_eve_items_on_cpp_market_group_id", using: :btree
+  add_index "eve_items", ["market_group_id"], name: "index_eve_items_on_market_group_id", using: :btree
 
   create_table "regions", force: :cascade do |t|
     t.string   "cpp_region_id", null: false
@@ -92,6 +100,8 @@ ActiveRecord::Schema.define(version: 20160725091214) do
 
   add_index "regions", ["cpp_region_id"], name: "index_regions_on_cpp_region_id", unique: true, using: :btree
 
+  add_foreign_key "caddie_crest_price_history_last_day_timestamps", "eve_items"
+  add_foreign_key "caddie_crest_price_history_last_day_timestamps", "regions"
   add_foreign_key "caddie_crest_price_history_updates", "eve_items"
   add_foreign_key "caddie_crest_price_history_updates", "regions"
   add_foreign_key "crest_price_histories", "eve_items"

data/test/factories/caddie/crest_price_history.rb

@@ -0,0 +1,7 @@
+FactoryGirl.define do
+  factory :crest_price_history, class: CrestPriceHistory do
+    history_date { Time.now }
+    region
+    eve_item
+  end
+end

data/test/factories/caddie/crest_price_history_last_day_timestamp.rb

@@ -0,0 +1,7 @@
+FactoryGirl.define do
+  factory :crest_price_history_last_day_timestamp, class: Caddie::CrestPriceHistoryLastDayTimestamp do
+    eve_item
+    region
+    day_timestamp { Time.now }
+  end
+end

data/test/factories/caddie/crest_price_history_updates.rb

@@ -1,4 +1,7 @@
 FactoryGirl.define do
   factory :crest_price_history_update, class: Caddie::CrestPriceHistoryUpdate do
+    eve_item
+    region
+    next_process_date { Time.now }
   end
 end

data/test/factories/caddie/eve_items.rb

@@ -1,7 +1,5 @@
 FactoryGirl.define do
-
-  factory :caddie_eve_item, class: EveItem do
-
+  factory :eve_item, class: EveItem do
     cost 5
     involved_in_blueprint true
 
@@ -14,41 +12,5 @@ FactoryGirl.define do
     sequence :name_lowcase do |n|
       "name #{n}"
     end
-
-    # An example of item with blueprint and market group
-    factory :caddie_inferno_fury_cruise_missile do
-      cpp_eve_item_id 2621
-      name "Inferno Fury Cruise Missile"
-      name_lowcase "inferno fury cruise missile"
-      cost 1815252.83
-      # market_group { FactoryGirl.create( :advanced_high_damage_cruise_missiles_market_group ) }
-
-      after(:create) do |eve_item|
-        # create( :inferno_fury_cruise_blueprint, eve_item: eve_item )
-      end
-
-    end
-
-    # An example of item with blueprint but no market group
-    factory :caddie_mjolnir_fury_cruise_missile do
-      cpp_eve_item_id 24535
-      name "Mjolnir Fury Cruise Missile"
-      name_lowcase "mjolnir fury cruise missile"
-      cost 1815252.83
-
-      after(:create) do |eve_item|
-        # create( :inferno_fury_cruise_blueprint, eve_item: eve_item )
-      end
-
-    end
-
-    # An example with no blueprint and no market group
-    factory :caddie_inferno_precision_cruise_missile do
-      cpp_eve_item_id 2637
-      name "Inferno Precision Cruise Missile"
-      name_lowcase "inferno precision cruise missile"
-      cost 1815252.83
-    end
-
   end
 end

data/test/factories/caddie/regions.rb

@@ -1,33 +1,10 @@
 FactoryGirl.define do
-
-  factory :caddie_region, class: Region do
-
-    factory :caddie_heimatar do
+  factory :region, class: Region do
+    cpp_region_id '123456'
+    name 'Region test'
+    factory :heimatar do
       cpp_region_id '10000030'
       name 'Heimatar'
-
-      after(:create) do |region|
-
-        # pator = create( :rens, region: region )
-        # rens = create( :pator, region: region )
-
-        blueprint_and_market_group = create( :caddie_inferno_fury_cruise_missile )
-        blueprint_but_no_market_group = create( :caddie_mjolnir_fury_cruise_missile )
-        no_market_group_and_no_blueprint = create( :caddie_inferno_precision_cruise_missile )
-
-        [ blueprint_and_market_group, blueprint_but_no_market_group, no_market_group_and_no_blueprint ].each do |item|
-
-          # [ pator, rens ].each do |trade_hub|
-          #   create( :min_price, trade_hub: trade_hub, eve_item: item )
-          # end
-
-          # create( :crest_prices_last_month_average, eve_item: item, region: region )
-
-        end
-      end
-
     end
-
   end
-
 end

data/test/models/caddie/crest_data_retriever_test.rb

@@ -0,0 +1,36 @@
+require 'test_helper'
+
+module Caddie
+  class CrestDataRetrieverTest < ActiveSupport::TestCase
+
+    def setup
+      create( :crest_price_history_update )
+
+      Object.stubs( :read ).returns( { items: [ 'item_test' ] }.to_json )
+      Caddie::CrestPriceHistoryUpdate.stubs( :open ).returns( Object )
+    end
+
+    test 'get_multipage_data' do
+      result = Caddie::CrestPriceHistoryUpdate.get_markets( 1, 1 )
+      assert_equal [['item_test'], 1], result
+    end
+
+    test 'Shoud properly handle OpenURI::HTTPError' do
+      Caddie::CrestPriceHistoryUpdate.unstub( :open )
+      Caddie::CrestPriceHistoryUpdate.stubs(:open).raises( OpenURI::HTTPError.new( 'No connection', STDERR ) )
+
+      Caddie::CrestPriceHistoryUpdate.stubs(:sleep)
+
+      result = Caddie::CrestPriceHistoryUpdate.feed_price_histories
+      assert_equal 2, result[3]
+
+      Caddie::CrestPriceHistoryUpdate.unstub(:sleep)
+    end
+
+    teardown do
+      Object.unstub(:read)
+      Caddie::CrestPriceHistoryUpdate.unstub(:open)
+    end
+
+  end
+end