caddie 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0a900bac7034bb9bb55a04958828c13fd101cac4
+  data.tar.gz: e48bd88a06693f128d63d2f4fd28b7493d9a4b3c
+SHA512:
+  metadata.gz: 2e28b839fdfaab5e58fe377a59b2d0d1a9074b1a049272197b8647d9a568453153fe8cb7efec9da3f18edc43f9c9efeea09d78dc2980b16daa1deecdd15a1ac9
+  data.tar.gz: a95b096686c5206da4f6aab2369af6772b169b49af92d05ad49853b33bd5cec92d252408fd13c30f0b4d86b88cd81350a200a36fcce9a026726d6deb51abbbb2

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2016 Zuger Cédric
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,3 @@
+= Caddie
+
+This project rocks and uses MIT-LICENSE.

data/Rakefile

@@ -0,0 +1,37 @@
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+
+require 'rdoc/task'
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Caddie'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+APP_RAKEFILE = File.expand_path("../test/dummy/Rakefile", __FILE__)
+load 'rails/tasks/engine.rake'
+
+
+load 'rails/tasks/statistics.rake'
+
+
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+
+task default: :test

data/app/assets/javascripts/caddie/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/app/assets/stylesheets/caddie/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/app/controllers/caddie/application_controller.rb

@@ -0,0 +1,5 @@
+module Caddie
+  class ApplicationController < ActionController::Base
+    protect_from_forgery with: :exception
+  end
+end

data/app/helpers/caddie/application_helper.rb

@@ -0,0 +1,4 @@
+module Caddie
+  module ApplicationHelper
+  end
+end

data/app/models/caddie/crest_data_retriever.rb

@@ -0,0 +1,50 @@
+require 'pp'
+
+module Caddie
+  module CrestDataRetriever
+
+    CREST_BASE_URL='https://crest-tq.eveonline.com/'
+
+    def get_markets( region_id, type_id )
+      items, connections_count = get_multipage_data( "market/#{region_id}/types/#{type_id}/history", false )
+      [ items, connections_count ]
+    end
+
+    private
+
+    def get_crest_url( rest )
+      "#{CREST_BASE_URL}/#{rest}/"
+    end
+
+    def get_multipage_data( rest, debug_request = false )
+
+      next_url = get_crest_url( rest )
+      items = []
+      connections_count = 0
+      begin
+
+        puts "Fetching : #{next_url}" if debug_request
+
+        @start_time = Time.now unless @start_time
+        @hit_count = 0 unless @hit_count
+        @hit_count += 1
+
+        json_result = open( next_url ).read
+        connections_count += 1
+        parsed_json = JSON.parse( json_result )
+
+        next_url = nil
+
+        items += parsed_json['items']
+        next_url_hash = parsed_json['next']
+        next_url = next_url_hash['href'] if next_url_hash
+
+      end while next_url
+
+      puts "items.count = #{items.count}" if debug_request
+
+      [ items, connections_count ]
+    end
+
+  end
+end

data/app/models/caddie/crest_price_history_update.rb

@@ -0,0 +1,56 @@
+module Caddie
+
+  class CrestPriceHistoryUpdate < ActiveRecord::Base
+    belongs_to :eve_item
+    belongs_to :region
+
+    extend Caddie::CrestDataRetriever
+
+    def self.update
+      current_path = File.dirname( __FILE__ )
+      request = File.open( "#{current_path}/update_table.sql" ).read
+      ActiveRecord::Base.connection.execute( request )
+    end
+
+    def self.feed_price_histories
+      total_connections_counts = 0
+      total_inserts = 0
+      date_deb = Time.now
+
+      daily_operations_list.joins( :eve_item, :region ).pluck( :eve_item_id, :region_id, :cpp_eve_item_id, :cpp_region_id ).each do |row|
+
+        eve_item_id, region_id, cpp_eve_item_id, cpp_region_id = row
+        # puts "Requesting : #{cpp_region_id}, #{cpp_eve_item_id}"
+        items, connections_count = get_markets( cpp_region_id, cpp_eve_item_id )
+        total_connections_counts += connections_count
+
+        ActiveRecord::Base.transaction do
+          items.each do |item_data|
+
+            date_info = DateTime.parse( item_data['date'] )
+            date_info_ts = date_info.strftime( '%Y%m%d' )
+
+            CrestPriceHistory.where( region_id: region_id, eve_item_id: eve_item_id, day_timestamp: date_info_ts ).first_or_create! do |h|
+              h.history_date = date_info
+              h.order_count = item_data['orderCount']
+              h.volume = item_data['volume']
+              h.low_price = item_data['lowPrice']
+              h.avg_price = item_data['avgPrice']
+              h.high_price = item_data['highPrice']
+              total_inserts += 1
+            end
+          end
+        end
+      end
+      [ total_inserts, total_connections_counts, Time.now - date_deb ]
+    end
+
+    private
+
+    def self.daily_operations_list
+      self.where( next_process_date: Time.now.to_date ).order( :process_queue_priority ).limit( 100 )
+    end
+
+  end
+
+end

data/app/models/caddie/update_table.sql

@@ -0,0 +1,39 @@
+/* Don't forget to set the postgres sequence to cycle on production */
+/* ALTER SEQUENCE caddie_crest_price_history_updates_id_seq CYCLE; */
+
+/* TRUNCATE TABLE caddie_crest_price_history_updates; */
+
+/* Insert new regions / items */
+INSERT INTO caddie_crest_price_history_updates( eve_item_id, region_id, created_at, updated_at )
+  SELECT eve_items.id, regions.id, now(), now()
+  FROM eve_items, regions
+  WHERE NOT EXISTS ( SELECT NULL FROM caddie_crest_price_history_updates WHERE eve_item_id = eve_items.id and region_id = regions.id );
+
+/* Update all datas */
+UPDATE caddie_crest_price_history_updates cphu
+SET max_update = sub.mua, max_eve_item_create = sub.max_eve_item_create, max_region_create = sub.max_region_create
+FROM (
+SELECT DISTINCT eve_item_id, region_id, MAX( crest_price_histories.updated_at ) mua, MAX( eve_items.created_at ) max_eve_item_create, MAX( regions.created_at ) max_region_create
+	FROM crest_price_histories, eve_items, regions
+	WHERE eve_item_id = eve_items.id 
+	AND region_id = regions.id
+GROUP BY eve_item_id, region_id
+) sub
+WHERE cphu.eve_item_id = sub.eve_item_id
+AND cphu.region_id = sub.region_id;
+
+UPDATE caddie_crest_price_history_updates cphu SET max_date = GREATEST( max_update, max_eve_item_create, max_region_create );
+
+UPDATE caddie_crest_price_history_updates cphu SET nb_days = ( CURRENT_DATE - max_date );
+
+UPDATE caddie_crest_price_history_updates cphu
+SET process_queue = 'DAILY', next_process_date = current_date, process_queue_priority = 1
+WHERE nb_days <= 7;
+
+UPDATE caddie_crest_price_history_updates cphu
+SET process_queue = 'WEEKLY', next_process_date = date_trunc( 'week', ( current_date + ( interval '1 week' ) ) ), process_queue_priority = 2
+WHERE nb_days <= 30 AND nb_days > 7;
+
+UPDATE caddie_crest_price_history_updates cphu
+SET process_queue = 'MONTHLY', next_process_date = date_trunc( 'month', ( current_date + ( interval '1 month' ) ) ), process_queue_priority = 3
+WHERE nb_days > 30 OR nb_days IS NULL;

data/app/models/crest_price_history.rb

@@ -0,0 +1 @@
+../../../eve_business_server/app/models/crest_price_history.rb

data/app/models/eve_item.rb

@@ -0,0 +1 @@
+../../../eve_business_server/app/models/eve_item.rb

data/app/models/region.rb

@@ -0,0 +1 @@
+../../../eve_business_server/app/models/region.rb

data/app/views/layouts/caddie/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Caddie</title>
+  <%= stylesheet_link_tag    "caddie/application", media: "all" %>
+  <%= javascript_include_tag "caddie/application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/config/routes.rb

@@ -0,0 +1,2 @@
+Caddie::Engine.routes.draw do
+end

data/db/migrate/20160524122651_create_caddie_crest_price_history_updates.rb

@@ -0,0 +1,21 @@
+class CreateCaddieCrestPriceHistoryUpdates < ActiveRecord::Migration
+
+  def change
+    create_table :caddie_crest_price_history_updates do |t|
+      t.references :eve_item, foreign_key: true
+      t.references :region, foreign_key: true
+      t.date :max_update
+      t.date :max_eve_item_create
+      t.date :max_region_create
+      t.date :max_date
+      t.integer :nb_days
+      t.string :process_queue
+      t.integer :process_queue_priority
+      t.date :next_process_date
+
+      t.timestamps null: false
+    end
+    add_index :caddie_crest_price_history_updates, [ :eve_item_id, :region_id ], unique: true, name: :index_caddie_cphu_on_eve_item_id_and_region_id
+  end
+
+end

data/lib/caddie/engine.rb

@@ -0,0 +1,5 @@
+module Caddie
+  class Engine < ::Rails::Engine
+    isolate_namespace Caddie
+  end
+end

data/lib/caddie/version.rb

@@ -0,0 +1,3 @@
+module Caddie
+  VERSION = "0.0.1"
+end

data/lib/caddie.rb

@@ -0,0 +1,4 @@
+require "caddie/engine"
+
+module Caddie
+end

data/lib/tasks/caddie_tasks.rake

@@ -0,0 +1,16 @@
+namespace :caddie do
+
+  desc "Feed the price histories table"
+  task :feed_price_histories => :environment do
+
+    puts 'About to compute crest_price_history_updates'
+    Caddie::CrestPriceHistoryUpdate.update
+
+    puts 'About to feed crest_price_histories'
+    total_inserts, total_connections, total_time = Caddie::CrestPriceHistoryUpdate.feed_price_histories
+    puts "#{total_inserts} insertions, #{total_connections} connections in #{total_time.round( 2 )} seconds. #{(total_connections/total_time).round( 2 )} co/sec"
+
+  end
+
+end
+

data/test/caddie_test.rb

@@ -0,0 +1,7 @@
+require 'test_helper'
+
+class CaddieTest < ActiveSupport::TestCase
+  test "truth" do
+    assert_kind_of Module, Caddie
+  end
+end

data/test/dummy/README.rdoc

@@ -0,0 +1,28 @@
+== README
+
+This README would normally document whatever steps are necessary to get the
+application up and running.
+
+Things you may want to cover:
+
+* Ruby version
+
+* System dependencies
+
+* Configuration
+
+* Database creation
+
+* Database initialization
+
+* How to run the test suite
+
+* Services (job queues, cache servers, search engines, etc.)
+
+* Deployment instructions
+
+* ...
+
+
+Please feel free to use a different markup language if you do not plan to run
+<tt>rake doc:app</tt>.

data/test/dummy/Rakefile

@@ -0,0 +1,6 @@
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Rails.application.load_tasks

data/test/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,13 @@
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or any plugin's vendor/assets/javascripts directory can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// compiled file.
+//
+// Read Sprockets README (https://github.com/rails/sprockets#sprockets-directives) for details
+// about supported directives.
+//
+//= require_tree .

data/test/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,15 @@
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or any plugin's vendor/assets/stylesheets directory can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the bottom of the
+ * compiled file so the styles you add here take precedence over styles defined in any styles
+ * defined in the other CSS/SCSS files in this directory. It is generally better to create a new
+ * file per style scope.
+ *
+ *= require_tree .
+ *= require_self
+ */

data/test/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,5 @@
+class ApplicationController < ActionController::Base
+  # Prevent CSRF attacks by raising an exception.
+  # For APIs, you may want to use :null_session instead.
+  protect_from_forgery with: :exception
+end

data/test/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,2 @@
+module ApplicationHelper
+end

data/test/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    'application', media: 'all', 'data-turbolinks-track' => true %>
+  <%= javascript_include_tag 'application', 'data-turbolinks-track' => true %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/test/dummy/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
+load Gem.bin_path('bundler', 'bundle')

data/test/dummy/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path('../../config/application', __FILE__)
+require_relative '../config/boot'
+require 'rails/commands'

data/test/dummy/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative '../config/boot'
+require 'rake'
+Rake.application.run

data/test/dummy/bin/setup

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+require 'pathname'
+
+# path to your application root.
+APP_ROOT = Pathname.new File.expand_path('../../',  __FILE__)
+
+Dir.chdir APP_ROOT do
+  # This script is a starting point to setup your application.
+  # Add necessary setup steps to this file:
+
+  puts "== Installing dependencies =="
+  system "gem install bundler --conservative"
+  system "bundle check || bundle install"
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?("config/database.yml")
+  #   system "cp config/database.yml.sample config/database.yml"
+  # end
+
+  puts "\n== Preparing database =="
+  system "bin/rake db:setup"
+
+  puts "\n== Removing old logs and tempfiles =="
+  system "rm -f log/*"
+  system "rm -rf tmp/cache"
+
+  puts "\n== Restarting application server =="
+  system "touch tmp/restart.txt"
+end

data/test/dummy/config/application.rb

@@ -0,0 +1,26 @@
+require File.expand_path('../boot', __FILE__)
+
+require 'rails/all'
+
+Bundler.require(*Rails.groups)
+require "caddie"
+
+module Dummy
+  class Application < Rails::Application
+    # Settings in config/environments/* take precedence over those specified here.
+    # Application configuration should go into files in config/initializers
+    # -- all .rb files in that directory are automatically loaded.
+
+    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
+    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
+    # config.time_zone = 'Central Time (US & Canada)'
+
+    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
+    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
+    # config.i18n.default_locale = :de
+
+    # Do not swallow errors in after_commit/after_rollback callbacks.
+    config.active_record.raise_in_transactional_callbacks = true
+  end
+end
+

data/test/dummy/config/boot.rb

@@ -0,0 +1,5 @@
+# Set up gems listed in the Gemfile.
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../../../Gemfile', __FILE__)
+
+require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
+$LOAD_PATH.unshift File.expand_path('../../../../lib', __FILE__)

data/test/dummy/config/database.yml

@@ -0,0 +1,24 @@
+# SQLite version 3.x
+#   gem install sqlite3
+#
+#   Ensure the SQLite 3 gem is defined in your Gemfile
+#   gem 'sqlite3'
+#
+default: &default
+  pool: 5
+  timeout: 5000
+  adapter: postgresql
+  encoding: unicode
+  username: eve_business_server
+
+development:
+  <<: *default
+  database: eve_business_server_dev
+
+
+# Warning: The database defined as "test" will be erased and
+# re-generated from your development database when you run "rake".
+# Do not set this db to the same as development or production.
+test:
+  <<: *default
+  database: eve_business_server_test

data/test/dummy/config/environment.rb

@@ -0,0 +1,5 @@
+# Load the Rails application.
+require File.expand_path('../application', __FILE__)
+
+# Initialize the Rails application.
+Rails.application.initialize!

data/test/dummy/config/environments/development.rb

@@ -0,0 +1,41 @@
+Rails.application.configure do
+  # Settings specified here will take precedence over those in config/application.rb.
+
+  # In the development environment your application's code is reloaded on
+  # every request. This slows down response time but is perfect for development
+  # since you don't have to restart the web server when you make code changes.
+  config.cache_classes = false
+
+  # Do not eager load code on boot.
+  config.eager_load = false
+
+  # Show full error reports and disable caching.
+  config.consider_all_requests_local       = true
+  config.action_controller.perform_caching = false
+
+  # Don't care if the mailer can't send.
+  config.action_mailer.raise_delivery_errors = false
+
+  # Print deprecation notices to the Rails logger.
+  config.active_support.deprecation = :log
+
+  # Raise an error on page load if there are pending migrations.
+  config.active_record.migration_error = :page_load
+
+  # Debug mode disables concatenation and preprocessing of assets.
+  # This option may cause significant delays in view rendering with a large
+  # number of complex assets.
+  config.assets.debug = true
+
+  # Asset digests allow you to set far-future HTTP expiration dates on all assets,
+  # yet still be able to expire them through the digest params.
+  config.assets.digest = true
+
+  # Adds additional error checking when serving assets at runtime.
+  # Checks for improperly declared sprockets dependencies.
+  # Raises helpful error messages.
+  config.assets.raise_runtime_errors = true
+
+  # Raises error for missing translations
+  # config.action_view.raise_on_missing_translations = true
+end