caddie 0.0.2 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3bdc2a5e9c25ef5e2dcd4c493cfc3dcbb1d8e7ba
-  data.tar.gz: 4f3e2dfe992c6372efd9ac22748dbf6c8d8ab1ee
+  metadata.gz: 8cde684aa557176c35056c250c5c83aa6d94f9ff
+  data.tar.gz: 5eed164762cd4574a8f6334eed84d2494c4cc87d
 SHA512:
-  metadata.gz: 79bf09bff7a35246301f19eb7ad4ea4cafc0d09802956bae39848584db6d2e88649bb58917899ee164521700fe32dca847ccb4a3a82e4275d26839adb992a30f
-  data.tar.gz: 7c3130658d924ea1b7fa68d168ee5634bea345b341c0bed9bfc7ddd7e448cb91d076ac25360fc025cec8b8ce6651f4ecb040f52c222b6ae0ce0d1b5248e994d6
+  metadata.gz: 6e0b29ddfcd8346b4c381a0b6bf0707a1c370c708f34524f3077ca3b5eca5645ed5a9687fb06973749e604daa830111c8f54ea59ef6c6719ea8890fc9388d40c
+  data.tar.gz: 6b89a77002858571d2a73faa33e27a91b167c5ce31d6c79dd7244b79aeaf84d2efb2ae7f648db8e093a18410b02d4f94af4e1231e9af2be55c0102def1301c6d

data/app/models/caddie/crest_price_history_update.rb

@@ -1,3 +1,5 @@
+require 'set'
+
 module Caddie
 
   class CrestPriceHistoryUpdate < ActiveRecord::Base
@@ -19,26 +21,34 @@ module Caddie
 
       daily_operations_list.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id ).each do |row|
 
+        # puts "Processing row = #{row}"
+
         eve_item_id, region_id, cpp_eve_item_id, cpp_region_id = row
         # puts "Requesting : #{cpp_region_id}, #{cpp_eve_item_id}"
         items, connections_count = get_markets( cpp_region_id, cpp_eve_item_id )
         total_connections_counts += connections_count
 
         ActiveRecord::Base.transaction do
+
+          # puts 'About to reject already used lines'
+          timestamps = CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id ).pluck( :day_timestamp ).to_set
+          items.reject! do |item|
+            date_info = DateTime.parse( item['date'] )
+            date_info_ts = date_info.strftime( '%Y%m%d' )
+            timestamps.include?( date_info_ts )
+          end
+          # puts 'Lines rejected'
+
+
           items.each do |item_data|
 
             date_info = DateTime.parse( item_data['date'] )
             date_info_ts = date_info.strftime( '%Y%m%d' )
 
-            CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id, day_timestamp: date_info_ts ).first_or_create! do |h|
-              h.history_date = date_info
-              h.order_count = item_data['orderCount']
-              h.volume = item_data['volume']
-              h.low_price = item_data['lowPrice']
-              h.avg_price = item_data['avgPrice']
-              h.high_price = item_data['highPrice']
+            CrestPriceHistory.create!( region_id: region_id, eve_item_id: eve_item_id, day_timestamp: date_info_ts,
+              history_date: date_info,  order_count: item_data['orderCount'], volume: item_data['volume'],
+              low_price: item_data['lowPrice'], avg_price: item_data['avgPrice'], high_price: item_data['highPrice'] )
               total_inserts += 1
-            end
           end
         end
       end
@@ -48,7 +58,7 @@ module Caddie
     private
 
     def self.daily_operations_list
-      self.where( next_process_date: Time.now.to_date ).order( :process_queue_priority )
+      self.where( next_process_date: Time.now.to_date ).order( :process_queue_priority ).limit( 100 )
     end
 
   end

data/app/models/caddie/m_threaded_updater.rb

@@ -0,0 +1,6 @@
+class MThreadedUpdater
+
+  # TODO : - split the work in database using thread_slice_id (set numbers : 1 .. MAX_THREADS)
+  # TODO : Then run N threads each threads get the records associated to it's number
+
+end

data/app/models/caddie/update_table.sql

@@ -36,4 +36,5 @@ WHERE nb_days <= 30 AND nb_days > 7;
 
 UPDATE caddie_crest_price_history_updates cphu
 SET process_queue = 'MONTHLY', next_process_date = date_trunc( 'month', ( current_date + ( interval '1 month' ) ) ), process_queue_priority = 3
-WHERE nb_days > 30 OR nb_days IS NULL;
+WHERE nb_days > 30 OR nb_days IS NULL;
+

data/db/migrate/20160527152027_add_thread_slice_id_to_caddie_crest_price_history_updates.rb

@@ -0,0 +1,5 @@
+class AddThreadSliceIdToCaddieCrestPriceHistoryUpdates < ActiveRecord::Migration
+  def change
+    add_column :caddie_crest_price_history_updates, :thread_slice_id, :integer, index: true
+  end
+end

data/lib/caddie/version.rb

@@ -1,3 +1,3 @@
 module Caddie
-  VERSION = "0.0.2"
+  VERSION = '0.0.4'
 end

data/lib/tasks/caddie_tasks.rake

@@ -4,7 +4,7 @@ namespace :caddie do
   task :feed_price_histories => :environment do
 
     puts 'About to compute crest_price_history_updates'
-    Caddie::CrestPriceHistoryUpdate.update
+    # Caddie::CrestPriceHistoryUpdate.update
 
     puts 'About to feed crest_price_histories'
     total_inserts, total_connections, total_time = Caddie::CrestPriceHistoryUpdate.feed_price_histories

data/test/dummy/db/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20160405090537) do
+ActiveRecord::Schema.define(version: 20160527152027) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -38,6 +38,24 @@ ActiveRecord::Schema.define(version: 20160405090537) do
   add_index "blueprints", ["cpp_blueprint_id"], name: "index_blueprints_on_cpp_blueprint_id", using: :btree
   add_index "blueprints", ["eve_item_id"], name: "index_blueprints_on_eve_item_id", using: :btree
 
+  create_table "caddie_crest_price_history_updates", force: :cascade do |t|
+    t.integer  "eve_item_id"
+    t.integer  "region_id"
+    t.date     "max_update"
+    t.date     "max_eve_item_create"
+    t.date     "max_region_create"
+    t.date     "max_date"
+    t.integer  "nb_days"
+    t.string   "process_queue"
+    t.integer  "process_queue_priority"
+    t.date     "next_process_date"
+    t.datetime "created_at",             null: false
+    t.datetime "updated_at",             null: false
+    t.integer  "thread_slice_id"
+  end
+
+  add_index "caddie_crest_price_history_updates", ["eve_item_id", "region_id"], name: "index_caddie_cphu_on_eve_item_id_and_region_id", unique: true, using: :btree
+
   create_table "components", force: :cascade do |t|
     t.integer  "cpp_eve_item_id"
     t.string   "name",            limit: 255
@@ -336,6 +354,8 @@ ActiveRecord::Schema.define(version: 20160405090537) do
 
   add_foreign_key "api_key_errors", "users"
   add_foreign_key "blueprints", "eve_items"
+  add_foreign_key "caddie_crest_price_history_updates", "eve_items"
+  add_foreign_key "caddie_crest_price_history_updates", "regions"
   add_foreign_key "crest_costs", "eve_items"
   add_foreign_key "crest_price_histories", "eve_items"
   add_foreign_key "crest_price_histories", "regions"