caddie 0.2.3 → 0.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/models/caddie/crest_data_retriever.rb +10 -7
- data/app/models/caddie/crest_price_history_last_day_timestamp.rb +30 -0
- data/app/models/caddie/crest_price_history_update.rb +50 -25
- data/app/models/caddie/crest_price_history_update_log.rb +10 -0
- data/app/models/caddie/m_threaded_updater.rb +12 -10
- data/db/migrate/20160803094530_create_caddie_crest_price_history_last_day_timestamps.rb +12 -0
- data/lib/caddie/version.rb +1 -1
- data/lib/tasks/caddie_tasks.rake +4 -6
- data/{app → test/dummy/app}/models/crest_price_history.rb +0 -0
- data/test/dummy/app/models/eve_item.rb +5 -9
- data/test/dummy/db/migrate/20150815033941_add_epic_blueprint_to_eve_item.rb +5 -0
- data/test/dummy/db/migrate/20150916052729_add_market_group_ref_to_eve_item.rb +6 -0
- data/test/dummy/db/migrate/20160803143148_crest_price_history_index_rework_and_others.rb +17 -0
- data/test/dummy/db/schema.rb +25 -15
- data/test/factories/caddie/crest_price_history.rb +7 -0
- data/test/factories/caddie/crest_price_history_last_day_timestamp.rb +7 -0
- data/test/factories/caddie/crest_price_history_updates.rb +3 -0
- data/test/factories/caddie/eve_items.rb +1 -39
- data/test/factories/caddie/regions.rb +4 -27
- data/test/models/caddie/crest_data_retriever_test.rb +36 -0
- data/test/models/caddie/crest_price_history_last_day_timestamp_test.rb +102 -0
- data/test/models/caddie/crest_price_history_update_log_test.rb +15 -3
- data/test/models/caddie/crest_price_history_update_test.rb +31 -9
- data/test/models/caddie/m_threaded_updater_test.rb +13 -25
- data/test/test_helper.rb +7 -2
- metadata +64 -66
- data/lib/tasks/m_threaded_updater_test.no_rake +0 -53
- data/test/dummy/log/development.log +0 -637
- data/test/dummy/log/test.log +0 -46048
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d606561f09dd648d0014e7e792072fd3a1a3f04a
|
4
|
+
data.tar.gz: d9bb7137dd695d8bf4e3e1a071b7d1e984329e05
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: baeec0fd6bb87e6704ad84a7241adac21b5086f1a073df8f0429b38c05f32b97e9284fcfc56fbaa3119bda697b8bdbd93abb07d0dd4970e20dcecc36ce285f6f
|
7
|
+
data.tar.gz: c941f62d954c7f843dd04326debc5778eb3f6e103586a976c4138fc4189e05d6dd2734ceaba19a33a5040abe17804ee392b1bae77f51f0e7edff2b8ead58713c
|
@@ -5,12 +5,13 @@ module Caddie
|
|
5
5
|
|
6
6
|
CREST_BASE_URL='https://crest-tq.eveonline.com/'
|
7
7
|
|
8
|
-
def get_markets( region_id, type_id )
|
8
|
+
def get_markets( region_id, type_id, thread_log_file: nil )
|
9
9
|
|
10
10
|
debug = ENV[ 'EBS_DEBUG_MODE' ] && ENV[ 'EBS_DEBUG_MODE' ].downcase == 'true'
|
11
|
+
output = thread_log_file ? thread_log_file : STDERR
|
11
12
|
|
12
13
|
type_url = "https://crest-tq.eveonline.com/inventory/types/#{type_id}"
|
13
|
-
items, connections_count = get_multipage_data( "market/#{region_id}/history/?type=#{type_url}", debug )
|
14
|
+
items, connections_count = get_multipage_data( "market/#{region_id}/history/?type=#{type_url}", debug, thread_log_file: thread_log_file )
|
14
15
|
|
15
16
|
[ items, connections_count ]
|
16
17
|
end
|
@@ -21,17 +22,19 @@ module Caddie
|
|
21
22
|
"#{CREST_BASE_URL}/#{rest}/"
|
22
23
|
end
|
23
24
|
|
24
|
-
def get_multipage_data( rest, debug_request = false )
|
25
|
+
def get_multipage_data( rest, debug_request = false, thread_log_file: nil )
|
26
|
+
|
27
|
+
output = thread_log_file ? thread_log_file : STDOUT
|
25
28
|
|
26
29
|
next_url = get_crest_url( rest )
|
27
30
|
items = []
|
28
31
|
connections_count = 0
|
29
32
|
begin
|
30
33
|
|
31
|
-
puts "Fetching : #{next_url}" if debug_request
|
34
|
+
output.puts "Fetching : #{next_url}" if debug_request
|
32
35
|
|
33
|
-
@start_time
|
34
|
-
@hit_count
|
36
|
+
@start_time ||= Time.now
|
37
|
+
@hit_count ||= 0
|
35
38
|
@hit_count += 1
|
36
39
|
|
37
40
|
json_result = open( next_url ).read
|
@@ -46,7 +49,7 @@ module Caddie
|
|
46
49
|
|
47
50
|
end while next_url
|
48
51
|
|
49
|
-
puts "items.count = #{items.count}" if debug_request
|
52
|
+
output.puts "Retrieved items.count = #{items.count}" if debug_request
|
50
53
|
|
51
54
|
[ items, connections_count ]
|
52
55
|
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
module Caddie
|
2
|
+
class CrestPriceHistoryLastDayTimestamp < ActiveRecord::Base
|
3
|
+
belongs_to :eve_item
|
4
|
+
belongs_to :region
|
5
|
+
|
6
|
+
def self.find_or_create_last_day_timestamp( region_id, eve_item_id )
|
7
|
+
|
8
|
+
last_update_record = CrestPriceHistoryLastDayTimestamp.find_by_region_id_and_eve_item_id( region_id, eve_item_id )
|
9
|
+
|
10
|
+
if last_update_record
|
11
|
+
last_update_date = last_update_record.day_timestamp
|
12
|
+
else
|
13
|
+
last_update_date = CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id ).maximum( :history_date )
|
14
|
+
end
|
15
|
+
|
16
|
+
last_update_date = Time.new( 0 ) unless last_update_date
|
17
|
+
|
18
|
+
[ last_update_date, last_update_record ]
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.create_or_update_last_day_timestamp( last_update_record, max_date_info, region_id, eve_item_id )
|
22
|
+
if last_update_record
|
23
|
+
last_update_record.update!( day_timestamp: max_date_info )
|
24
|
+
else
|
25
|
+
last_update_record = CrestPriceHistoryLastDayTimestamp.create!( region_id: region_id, eve_item_id: eve_item_id, day_timestamp: max_date_info )
|
26
|
+
end
|
27
|
+
last_update_record
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'set'
|
2
|
+
require 'open-uri'
|
2
3
|
|
3
4
|
module Caddie
|
4
5
|
|
@@ -16,59 +17,83 @@ module Caddie
|
|
16
17
|
ActiveRecord::Base.connection.execute( request )
|
17
18
|
end
|
18
19
|
|
19
|
-
def self.feed_price_histories(
|
20
|
+
def self.feed_price_histories( updates_ids: nil, thread_log_file: nil )
|
20
21
|
total_connections_counts = 0
|
21
22
|
total_inserts = 0
|
22
23
|
date_deb = Time.now
|
23
24
|
|
24
|
-
|
25
|
+
debug = ENV[ 'EBS_DEBUG_MODE' ] && ENV[ 'EBS_DEBUG_MODE' ].downcase == 'true' && Rails.env != 'test'
|
26
|
+
output = thread_log_file ? thread_log_file : STDOUT
|
27
|
+
outerr = thread_log_file ? thread_log_file : STDERR
|
25
28
|
|
26
|
-
|
27
|
-
|
29
|
+
if updates_ids
|
30
|
+
dol = self.where( id: updates_ids ).order( :process_queue_priority ) if updates_ids
|
31
|
+
else
|
32
|
+
dol = daily_operations_list
|
33
|
+
end
|
28
34
|
|
29
|
-
|
30
|
-
dol.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id ).each do |row|
|
35
|
+
output.puts "dol.count = #{dol.count}" if debug
|
31
36
|
|
32
|
-
|
37
|
+
global_errors_count = 0
|
38
|
+
dol.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id, :id ).each do |row|
|
33
39
|
|
34
40
|
eve_item_id, region_id, cpp_eve_item_id, cpp_region_id = row
|
35
|
-
|
41
|
+
if debug
|
42
|
+
output.puts
|
43
|
+
output.puts '*'*50
|
44
|
+
output.puts 'Requesting:'
|
45
|
+
output.puts "r = Region.find_by_cpp_region_id( #{cpp_region_id} )"
|
46
|
+
output.puts "i = EveItem.find_by_cpp_eve_item_id( #{cpp_eve_item_id} )"
|
47
|
+
end
|
36
48
|
|
37
49
|
items = []
|
38
50
|
connections_count = 0
|
51
|
+
http_errors = 0
|
52
|
+
# We retry 2 times
|
39
53
|
begin
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
54
|
+
begin
|
55
|
+
items, connections_count = get_markets( cpp_region_id, cpp_eve_item_id, thread_log_file: thread_log_file )
|
56
|
+
rescue => e
|
57
|
+
http_errors += 1
|
58
|
+
global_errors_count += 1
|
59
|
+
outerr.puts "#{e.message} for cpp_region_id = #{cpp_region_id} and cpp_eve_item_id = #{cpp_eve_item_id}" unless Rails.env == 'test'
|
60
|
+
sleep( 5 ) # in case of an error, we don't retry immediately.
|
61
|
+
end
|
62
|
+
end while http_errors >= 1 && http_errors < 2
|
44
63
|
|
45
64
|
total_connections_counts += connections_count
|
46
65
|
|
47
66
|
ActiveRecord::Base.transaction do
|
48
67
|
|
49
|
-
|
50
|
-
|
51
|
-
timestamps = CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id ).pluck( :day_timestamp ).to_set
|
52
|
-
items.reject! do |item|
|
53
|
-
date_info = DateTime.parse( item['date'] )
|
54
|
-
date_info_ts = date_info.strftime( '%Y%m%d' )
|
55
|
-
timestamps.include?( date_info_ts )
|
56
|
-
end
|
57
|
-
# puts 'Lines rejected'
|
68
|
+
last_update_date, last_update_record = Caddie::CrestPriceHistoryLastDayTimestamp.
|
69
|
+
find_or_create_last_day_timestamp( region_id, eve_item_id )
|
58
70
|
|
59
|
-
|
71
|
+
output.puts "Last update record date = #{last_update_date.inspect}" if debug
|
72
|
+
|
73
|
+
items.reject!{ |item| DateTime.parse( item['date'] ) <= last_update_date }
|
60
74
|
|
75
|
+
output.puts "Kept items.count = #{items.count}" if debug
|
76
|
+
|
77
|
+
max_date_info = last_update_date
|
78
|
+
items.each do |item_data|
|
61
79
|
date_info = DateTime.parse( item_data['date'] )
|
62
|
-
|
80
|
+
max_date_info = [ max_date_info, date_info ].max
|
63
81
|
|
64
|
-
CrestPriceHistory.create!( region_id: region_id, eve_item_id: eve_item_id,
|
82
|
+
CrestPriceHistory.create!( region_id: region_id, eve_item_id: eve_item_id,
|
65
83
|
history_date: date_info, order_count: item_data['orderCount'], volume: item_data['volume'],
|
66
84
|
low_price: item_data['lowPrice'], avg_price: item_data['avgPrice'], high_price: item_data['highPrice'] )
|
67
85
|
total_inserts += 1
|
68
86
|
end
|
87
|
+
|
88
|
+
output.puts "Computed max date info = #{max_date_info}" if debug
|
89
|
+
|
90
|
+
Caddie::CrestPriceHistoryLastDayTimestamp.create_or_update_last_day_timestamp(
|
91
|
+
last_update_record, max_date_info, region_id, eve_item_id )
|
69
92
|
end
|
93
|
+
output.flush
|
94
|
+
|
70
95
|
end
|
71
|
-
[ total_inserts, total_connections_counts, Time.now - date_deb ]
|
96
|
+
[ total_inserts, total_connections_counts, Time.now - date_deb, global_errors_count ]
|
72
97
|
end
|
73
98
|
|
74
99
|
private
|
@@ -1,4 +1,14 @@
|
|
1
1
|
module Caddie
|
2
2
|
class CrestPriceHistoryUpdateLog < ActiveRecord::Base
|
3
|
+
|
4
|
+
def self.store_log_data( feed_date, update_planning_time, feeding_time, total_inserts, total_connections, total_time )
|
5
|
+
log = Caddie::CrestPriceHistoryUpdateLog.where( feed_date: feed_date ).first_or_initialize
|
6
|
+
log.update_planning_time = update_planning_time
|
7
|
+
log.feeding_time = feeding_time
|
8
|
+
log.total_inserts = total_inserts
|
9
|
+
log.co_seconds = total_connections.to_f / total_time
|
10
|
+
log.save!
|
11
|
+
log
|
12
|
+
end
|
3
13
|
end
|
4
14
|
end
|
@@ -12,12 +12,14 @@ class Caddie::MThreadedUpdater
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def feed_price_histories_threaded
|
15
|
+
split_work_for_threads
|
15
16
|
Thread::abort_on_exception = true
|
16
17
|
threads = []
|
17
18
|
0.upto( @max_threads-1 ).each do |thread_id|
|
18
19
|
threads << Thread.new {
|
19
|
-
|
20
|
-
Thread.current[:timings] = Caddie::CrestPriceHistoryUpdate.
|
20
|
+
thread_log = File.open( "log/feed_price_histories_threaded_#{thread_id}.log", 'w')
|
21
|
+
Thread.current[:timings] = Caddie::CrestPriceHistoryUpdate.
|
22
|
+
feed_price_histories( updates_ids: @threads_split[ thread_id ], thread_log_file: thread_log )
|
21
23
|
}
|
22
24
|
end
|
23
25
|
result = []
|
@@ -25,19 +27,19 @@ class Caddie::MThreadedUpdater
|
|
25
27
|
thread_result = t[:timings]
|
26
28
|
result << thread_result
|
27
29
|
end
|
28
|
-
|
29
|
-
|
30
|
+
results = result.map { |a| Vector[*a] }.inject(:+).to_a
|
31
|
+
results[ 2 ] = result.map{ |e| e[2] }.max # For timings we are using the max, not the sum.
|
32
|
+
results
|
30
33
|
end
|
31
34
|
|
32
35
|
def split_work_for_threads
|
33
36
|
puts 'Start splitting work for threads'
|
34
37
|
ids = @daily_operations_list.pluck( :id )
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
end
|
38
|
+
@threads_split = []
|
39
|
+
|
40
|
+
slice_size = ids.count/@max_threads + 1
|
41
|
+
( 0 ... @max_threads ).each do |thread_id|
|
42
|
+
@threads_split[ thread_id ] = ids.shift( slice_size )
|
41
43
|
end
|
42
44
|
puts 'Finished splitting work for threads'
|
43
45
|
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
class CreateCaddieCrestPriceHistoryLastDayTimestamps < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
create_table :caddie_crest_price_history_last_day_timestamps do |t|
|
4
|
+
t.references :eve_item, index: false, foreign_key: true
|
5
|
+
t.references :region, index: false, foreign_key: true
|
6
|
+
t.datetime :day_timestamp
|
7
|
+
|
8
|
+
t.timestamps null: false
|
9
|
+
end
|
10
|
+
add_index :caddie_crest_price_history_last_day_timestamps, [ :region_id, :eve_item_id ], unique: true, name: :index_caddie_crest_price_history_last_day_timestamps
|
11
|
+
end
|
12
|
+
end
|
data/lib/caddie/version.rb
CHANGED
data/lib/tasks/caddie_tasks.rake
CHANGED
@@ -15,17 +15,15 @@ namespace :caddie do
|
|
15
15
|
update_planning_time = end_update_time - start_time
|
16
16
|
|
17
17
|
puts 'About to feed crest_price_histories'
|
18
|
-
|
19
|
-
th = Caddie::MThreadedUpdater.new( Caddie::CrestPriceHistoryUpdate::NB_THREADS, Caddie::CrestPriceHistoryUpdate.daily_operations_list )
|
20
|
-
th.
|
21
|
-
total_inserts, total_connections, total_time = th.feed_price_histories_threaded
|
18
|
+
total_inserts, total_connections, total_time = Caddie::CrestPriceHistoryUpdate.feed_price_histories
|
19
|
+
# th = Caddie::MThreadedUpdater.new( Caddie::CrestPriceHistoryUpdate::NB_THREADS, Caddie::CrestPriceHistoryUpdate.daily_operations_list )
|
20
|
+
# total_inserts, total_connections, total_time = th.feed_price_histories_threaded
|
22
21
|
puts "#{total_inserts} insertions, #{total_connections} connections in #{total_time.round( 2 )} seconds. #{(total_connections/total_time).round( 2 )} co/sec"
|
23
22
|
|
24
23
|
end_feeding_time = Time.now
|
25
24
|
feeding_time = end_feeding_time - end_update_time
|
26
25
|
|
27
|
-
Caddie::CrestPriceHistoryUpdateLog.
|
28
|
-
feeding_time: feeding_time, total_inserts: total_inserts, co_seconds: total_connections.to_f / total_time )
|
26
|
+
Caddie::CrestPriceHistoryUpdateLog.store_log_data( feed_date, update_planning_time, feeding_time, total_inserts, total_connections, total_time )
|
29
27
|
|
30
28
|
end
|
31
29
|
end
|
File without changes
|
@@ -1,15 +1,7 @@
|
|
1
|
-
# require 'open-uri'
|
2
|
-
# # require 'open-uri/cached'
|
3
1
|
require 'pp'
|
4
2
|
|
5
3
|
class EveItem < ActiveRecord::Base
|
6
4
|
|
7
|
-
# include Assert
|
8
|
-
# include ItemsInit::ItemSetupAndComp
|
9
|
-
# extend ItemsInit::ItemSetupAndCompSelf
|
10
|
-
# extend MultiplePriceRetriever
|
11
|
-
# extend Setup::UpdateEveItems
|
12
|
-
|
13
5
|
has_and_belongs_to_many :users
|
14
6
|
has_one :blueprint, dependent: :destroy
|
15
7
|
has_many :materials, through: :blueprint
|
@@ -47,7 +39,11 @@ class EveItem < ActiveRecord::Base
|
|
47
39
|
end
|
48
40
|
|
49
41
|
def full_batch_size
|
50
|
-
|
42
|
+
unless blueprint
|
43
|
+
puts "EveItem#full_batch_size : #{self.inspect} has no blueprint"
|
44
|
+
return -Float::INFINITY
|
45
|
+
end
|
46
|
+
blueprint.nb_runs*blueprint.prod_qtt
|
51
47
|
end
|
52
48
|
|
53
49
|
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class CrestPriceHistoryIndexReworkAndOthers < ActiveRecord::Migration
|
2
|
+
def change
|
3
|
+
|
4
|
+
remove_column :crest_price_histories, :day_timestamp
|
5
|
+
|
6
|
+
remove_index :crest_price_histories, :eve_item_id
|
7
|
+
remove_index :crest_price_histories, :region_id
|
8
|
+
# remove_index :crest_price_histories, :history_date
|
9
|
+
# remove_index :crest_price_histories, name: :price_histories_all_keys_index
|
10
|
+
|
11
|
+
add_index :crest_price_histories, [ :region_id, :eve_item_id ], name: :index_crest_price_histories_on_region_and_item
|
12
|
+
add_index :crest_price_histories, :history_date
|
13
|
+
|
14
|
+
# remove_index :crest_prices_last_month_averages, name: :prices_lmavg_all_keys_index
|
15
|
+
|
16
|
+
end
|
17
|
+
end
|
data/test/dummy/db/schema.rb
CHANGED
@@ -11,11 +11,21 @@
|
|
11
11
|
#
|
12
12
|
# It's strongly recommended that you check this file into your version control system.
|
13
13
|
|
14
|
-
ActiveRecord::Schema.define(version:
|
14
|
+
ActiveRecord::Schema.define(version: 20160803143148) do
|
15
15
|
|
16
16
|
# These are extensions that must be enabled in order to support this database
|
17
17
|
enable_extension "plpgsql"
|
18
18
|
|
19
|
+
create_table "caddie_crest_price_history_last_day_timestamps", force: :cascade do |t|
|
20
|
+
t.integer "eve_item_id"
|
21
|
+
t.integer "region_id"
|
22
|
+
t.datetime "day_timestamp"
|
23
|
+
t.datetime "created_at", null: false
|
24
|
+
t.datetime "updated_at", null: false
|
25
|
+
end
|
26
|
+
|
27
|
+
add_index "caddie_crest_price_history_last_day_timestamps", ["region_id", "eve_item_id"], name: "index_caddie_crest_price_history_last_day_timestamps", unique: true, using: :btree
|
28
|
+
|
19
29
|
create_table "caddie_crest_price_history_update_logs", force: :cascade do |t|
|
20
30
|
t.date "feed_date"
|
21
31
|
t.integer "update_planning_time"
|
@@ -51,23 +61,20 @@ ActiveRecord::Schema.define(version: 20160725091214) do
|
|
51
61
|
add_index "caddie_crest_price_history_updates", ["thread_slice_id"], name: "index_caddie_crest_price_history_updates_on_thread_slice_id", using: :btree
|
52
62
|
|
53
63
|
create_table "crest_price_histories", force: :cascade do |t|
|
54
|
-
t.integer "region_id",
|
55
|
-
t.integer "eve_item_id",
|
56
|
-
t.
|
57
|
-
t.
|
58
|
-
t.integer "
|
59
|
-
t.integer "volume", limit: 8
|
64
|
+
t.integer "region_id", null: false
|
65
|
+
t.integer "eve_item_id", null: false
|
66
|
+
t.datetime "history_date", null: false
|
67
|
+
t.integer "order_count", limit: 8
|
68
|
+
t.integer "volume", limit: 8
|
60
69
|
t.float "low_price"
|
61
70
|
t.float "avg_price"
|
62
71
|
t.float "high_price"
|
63
|
-
t.datetime "created_at",
|
64
|
-
t.datetime "updated_at",
|
72
|
+
t.datetime "created_at", null: false
|
73
|
+
t.datetime "updated_at", null: false
|
65
74
|
end
|
66
75
|
|
67
|
-
add_index "crest_price_histories", ["
|
68
|
-
add_index "crest_price_histories", ["eve_item_id"], name: "
|
69
|
-
add_index "crest_price_histories", ["region_id", "eve_item_id", "day_timestamp"], name: "price_histories_all_keys_index", unique: true, using: :btree
|
70
|
-
add_index "crest_price_histories", ["region_id"], name: "index_crest_price_histories_on_region_id", using: :btree
|
76
|
+
add_index "crest_price_histories", ["history_date"], name: "index_crest_price_histories_on_history_date", using: :btree
|
77
|
+
add_index "crest_price_histories", ["region_id", "eve_item_id"], name: "index_crest_price_histories_on_region_and_item", using: :btree
|
71
78
|
|
72
79
|
create_table "eve_items", force: :cascade do |t|
|
73
80
|
t.integer "cpp_eve_item_id"
|
@@ -76,12 +83,13 @@ ActiveRecord::Schema.define(version: 20160725091214) do
|
|
76
83
|
t.datetime "updated_at"
|
77
84
|
t.string "name_lowcase"
|
78
85
|
t.float "cost"
|
79
|
-
t.
|
86
|
+
t.boolean "epic_blueprint", default: false
|
80
87
|
t.boolean "involved_in_blueprint", default: false
|
88
|
+
t.integer "market_group_id"
|
81
89
|
end
|
82
90
|
|
83
91
|
add_index "eve_items", ["cpp_eve_item_id"], name: "index_eve_items_on_cpp_eve_item_id", using: :btree
|
84
|
-
add_index "eve_items", ["
|
92
|
+
add_index "eve_items", ["market_group_id"], name: "index_eve_items_on_market_group_id", using: :btree
|
85
93
|
|
86
94
|
create_table "regions", force: :cascade do |t|
|
87
95
|
t.string "cpp_region_id", null: false
|
@@ -92,6 +100,8 @@ ActiveRecord::Schema.define(version: 20160725091214) do
|
|
92
100
|
|
93
101
|
add_index "regions", ["cpp_region_id"], name: "index_regions_on_cpp_region_id", unique: true, using: :btree
|
94
102
|
|
103
|
+
add_foreign_key "caddie_crest_price_history_last_day_timestamps", "eve_items"
|
104
|
+
add_foreign_key "caddie_crest_price_history_last_day_timestamps", "regions"
|
95
105
|
add_foreign_key "caddie_crest_price_history_updates", "eve_items"
|
96
106
|
add_foreign_key "caddie_crest_price_history_updates", "regions"
|
97
107
|
add_foreign_key "crest_price_histories", "eve_items"
|
@@ -1,7 +1,5 @@
|
|
1
1
|
FactoryGirl.define do
|
2
|
-
|
3
|
-
factory :caddie_eve_item, class: EveItem do
|
4
|
-
|
2
|
+
factory :eve_item, class: EveItem do
|
5
3
|
cost 5
|
6
4
|
involved_in_blueprint true
|
7
5
|
|
@@ -14,41 +12,5 @@ FactoryGirl.define do
|
|
14
12
|
sequence :name_lowcase do |n|
|
15
13
|
"name #{n}"
|
16
14
|
end
|
17
|
-
|
18
|
-
# An example of item with blueprint and market group
|
19
|
-
factory :caddie_inferno_fury_cruise_missile do
|
20
|
-
cpp_eve_item_id 2621
|
21
|
-
name "Inferno Fury Cruise Missile"
|
22
|
-
name_lowcase "inferno fury cruise missile"
|
23
|
-
cost 1815252.83
|
24
|
-
# market_group { FactoryGirl.create( :advanced_high_damage_cruise_missiles_market_group ) }
|
25
|
-
|
26
|
-
after(:create) do |eve_item|
|
27
|
-
# create( :inferno_fury_cruise_blueprint, eve_item: eve_item )
|
28
|
-
end
|
29
|
-
|
30
|
-
end
|
31
|
-
|
32
|
-
# An example of item with blueprint but no market group
|
33
|
-
factory :caddie_mjolnir_fury_cruise_missile do
|
34
|
-
cpp_eve_item_id 24535
|
35
|
-
name "Mjolnir Fury Cruise Missile"
|
36
|
-
name_lowcase "mjolnir fury cruise missile"
|
37
|
-
cost 1815252.83
|
38
|
-
|
39
|
-
after(:create) do |eve_item|
|
40
|
-
# create( :inferno_fury_cruise_blueprint, eve_item: eve_item )
|
41
|
-
end
|
42
|
-
|
43
|
-
end
|
44
|
-
|
45
|
-
# An example with no blueprint and no market group
|
46
|
-
factory :caddie_inferno_precision_cruise_missile do
|
47
|
-
cpp_eve_item_id 2637
|
48
|
-
name "Inferno Precision Cruise Missile"
|
49
|
-
name_lowcase "inferno precision cruise missile"
|
50
|
-
cost 1815252.83
|
51
|
-
end
|
52
|
-
|
53
15
|
end
|
54
16
|
end
|
@@ -1,33 +1,10 @@
|
|
1
1
|
FactoryGirl.define do
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
factory :
|
2
|
+
factory :region, class: Region do
|
3
|
+
cpp_region_id '123456'
|
4
|
+
name 'Region test'
|
5
|
+
factory :heimatar do
|
6
6
|
cpp_region_id '10000030'
|
7
7
|
name 'Heimatar'
|
8
|
-
|
9
|
-
after(:create) do |region|
|
10
|
-
|
11
|
-
# pator = create( :rens, region: region )
|
12
|
-
# rens = create( :pator, region: region )
|
13
|
-
|
14
|
-
blueprint_and_market_group = create( :caddie_inferno_fury_cruise_missile )
|
15
|
-
blueprint_but_no_market_group = create( :caddie_mjolnir_fury_cruise_missile )
|
16
|
-
no_market_group_and_no_blueprint = create( :caddie_inferno_precision_cruise_missile )
|
17
|
-
|
18
|
-
[ blueprint_and_market_group, blueprint_but_no_market_group, no_market_group_and_no_blueprint ].each do |item|
|
19
|
-
|
20
|
-
# [ pator, rens ].each do |trade_hub|
|
21
|
-
# create( :min_price, trade_hub: trade_hub, eve_item: item )
|
22
|
-
# end
|
23
|
-
|
24
|
-
# create( :crest_prices_last_month_average, eve_item: item, region: region )
|
25
|
-
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
8
|
end
|
30
|
-
|
31
9
|
end
|
32
|
-
|
33
10
|
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
require 'test_helper'
|
2
|
+
|
3
|
+
module Caddie
|
4
|
+
class CrestDataRetrieverTest < ActiveSupport::TestCase
|
5
|
+
|
6
|
+
def setup
|
7
|
+
create( :crest_price_history_update )
|
8
|
+
|
9
|
+
Object.stubs( :read ).returns( { items: [ 'item_test' ] }.to_json )
|
10
|
+
Caddie::CrestPriceHistoryUpdate.stubs( :open ).returns( Object )
|
11
|
+
end
|
12
|
+
|
13
|
+
test 'get_multipage_data' do
|
14
|
+
result = Caddie::CrestPriceHistoryUpdate.get_markets( 1, 1 )
|
15
|
+
assert_equal [['item_test'], 1], result
|
16
|
+
end
|
17
|
+
|
18
|
+
test 'Shoud properly handle OpenURI::HTTPError' do
|
19
|
+
Caddie::CrestPriceHistoryUpdate.unstub( :open )
|
20
|
+
Caddie::CrestPriceHistoryUpdate.stubs(:open).raises( OpenURI::HTTPError.new( 'No connection', STDERR ) )
|
21
|
+
|
22
|
+
Caddie::CrestPriceHistoryUpdate.stubs(:sleep)
|
23
|
+
|
24
|
+
result = Caddie::CrestPriceHistoryUpdate.feed_price_histories
|
25
|
+
assert_equal 2, result[3]
|
26
|
+
|
27
|
+
Caddie::CrestPriceHistoryUpdate.unstub(:sleep)
|
28
|
+
end
|
29
|
+
|
30
|
+
teardown do
|
31
|
+
Object.unstub(:read)
|
32
|
+
Caddie::CrestPriceHistoryUpdate.unstub(:open)
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|