mram-evedata 0.1.inferno11.3 → 0.1.inferno11.4

data/README.md

@@ -0,0 +1,31 @@
+Eve Online
+==========
+
+(the following is shamelessly stolen from [Wikipedia (30.08.2012)](http://en.wikipedia.org/wiki/Eve_online):
+
+**Eve Online** (stylised **EVE Online**) is a video game by [CCP Games](http://en.wikipedia.org/wiki/CCP_Games). It is a player-driven, persistent-world MMORPG set in a science fiction space setting. Characters pilot customizable ships through a galaxy of over 7,500 star systems. Most star systems are connected to one or more other star systems by means of stargates. The star systems can contain moons, planets, stations, wormholes, asteroid belts and complexes.
+
+Players of Eve Online can participate in a number of in-game professions and activities, including mining, piracy, manufacturing, trading, exploration, and combat (both player versus environment and player versus player). The character advancement system is based upon training skills in real time, even while not logged into the game.
+
+mram-evedata, aka this gem
+==========================
+
+CCP Games releases a dump of the static database objects of the EVE Online Universe whenever there is a major expansion (additional game content). Information about this can be found [here](http://wiki.eve-id.net/CCP_Static_Data_Dump). Unfortunately, these data dumps come originally as a Microsoft SQL Server Backup file, which not many people consider to be optimal for using in their own portals/community websites/tools etc. Fortunately however, there are people in the community who convert these dumps to other SQL formats, like MySQL, PostgreSQL, and SQLite3.
+
+This gem aims to lessen the burden of manually importing new dumps, and also provides an ActiveRecord "abstraction layer" of the data presented.
+
+Disclaimer
+==========
+
+* i'm not a professional software developer, let alone a ruby/rails experienced programmer. i do this because i actually play EVE Online and because scripting and hacking together tools is a hobby of mine.
+* all data/information/code is presented to you as is. it might work for you, it might not, it might kill your dog. use at your own risk, you have been warned.
+* initially i planned to diversify this gem to support at least MySQL/PostgreSQL/SQLite, but since i myself only use MySQL for my current projects, i kind of just stuck with that for now.
+* i also had plans to allow for the simultaneous use of different EVE-Db-Dump versions, but scrapped those due to time constraints. so only the latest dump is supported atm.
+* i am **in no way affiliated with CCP Games**, all of the brands and names and whatnot are theirs and theirs only. insert the rest of the legal yadda yadda. i'm not doing this for profit.
+
+Data to be imported
+===================
+
+The data this gem is downloading and importing is **not** assembled by me. i am merely providing a "glue layer" to use it easily and (somewhat) efficiently in a Rails application.
+Currently, i am using [this guy's dump files](http://www.fuzzwork.co.uk/dump/) for the import, but this might change at any time.
+

data/lib/mram-evedata/version.rb

@@ -1,4 +1,5 @@
 module EveData
-  VERSION = "0.1.inferno11.3"
+  VERSION = "0.1.inferno11.4"
+  DATAURL = "http://cloud.github.com/downloads/mrambossek/mram-evedata/mysql55-inferno11-extended.sql.bz2"
   DATAFILE = "mysql55-inferno11-extended.sql.bz2"
 end

data/lib/tasks/mram-evedata_tasks.rake

@@ -1,10 +1,71 @@
+# vim: set foldmethod=marker smarttab sw=2: #
+
+require 'mram-evedata/version'
+require 'tempfile'
+require 'net/http'
+require 'uri'
+require 'digest/md5'
+require 'fileutils'
+require 'ruby-progressbar'
+
 namespace :eve_data do
+  desc "Download EDD to data/#{EveData::DATAFILE}"
+  task :download => :environment do # {{{
+    data_dir = Rails.root.join('data')
+    file_complete = data_dir.join(EveData::DATAFILE)
+    md5_complete = "#{file_complete}.md5"
+
+    if File.readable?(file_complete) then
+      puts "#{file_complete} already exists"
+    else
+      uri = URI(URI.join(EveData::DATAURL, EveData::DATAFILE))
+      puts "trying to download #{uri.to_s} to #{file_complete}"
+      ret = download_file(uri, file_complete, true)
+      puts (ret ? "success." : "failed.")
+    end
+
+    if File.readable?(md5_complete) then
+      puts "#{md5_complete} already exists"
+    else
+      uri = URI(URI.join(EveData::DATAURL, EveData::DATAFILE+".md5"))
+      puts "trying to download #{uri.to_s} to #{md5_complete}"
+      ret = download_file(uri, md5_complete, false)
+      puts (ret ? "success." : "failed.")
+    end
+
+    local_file_is_ok = false
+
+    File.open(md5_complete, "r") do |f|
+      while (line = f.gets) do
+        if matches = line.match(/^(?<md5>[a-zA-Z0-9]{32})\ (?<binary>.)(?<filename>.*)$/) then
+          if matches[:filename] == EveData::DATAFILE then
+            myhash = Digest::MD5.hexdigest(File.read(file_complete))
+            printf("local file  : %s\n", myhash)
+            printf("correct hash: %s\n", matches[:md5])
+            local_file_is_ok = (myhash == matches[:md5])
+          end
+        end
+      end
+    end
+
+    if local_file_is_ok then
+      puts "local file is ok"
+    else
+      uri = URI(URI.join(EveData::DATAURL, EveData::DATAFILE))
+      puts "local file hash MISMATCH, trying to redownload"
+      ret = download_file(uri, file_complete, true)
+      puts (ret ? "success." : "failed.")
+    end
+  end # }}}
+
   desc "Import Eve Online Static Data Dump to EDD database"
-  task :importdb => :environment do
-    require 'mram-evedata/version'
-    require 'tempfile'
+  task :import => :environment do # {{{
 
-    data_absolute = EveData::Engine.root.join('data', EveData::DATAFILE)
+    file_complete = Rails.root.join('data', EveData::DATAFILE)
+    unless File.readable?(file_complete) then
+      puts "data dump could not be found in #{file_complete}, trying to download"
+      Rake::Task["eve_data:download"].invoke
+    end
 
     unless cfg = ActiveRecord::Base.configurations['eve_data_dump'] then
       raise IOError, "missing 'eve_data_dump' block in config/database.yml"
@@ -26,11 +87,44 @@ namespace :eve_data do
         return false
       end
 
-      puts "now trying to import dump from '#{data_absolute}':"
-      ret = system("bzcat #{data_absolute} | mysql --defaults-extra-file=#{f.path} #{cfg["database"]}")
+      puts "now trying to import dump from '#{file_complete}':"
+      ret = system("bzcat #{file_complete} | mysql --defaults-extra-file=#{f.path} #{cfg["database"]}")
       puts (ret ? "success!" : "failed!")
     end
 
     true
-  end
+  end # }}}
 end
+
+def download_file(uri, fname, use_pbar = false) # {{{
+  dname = File.dirname(fname)
+
+  Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == 'https') do |http|
+    res = http.request_head(uri.path)
+
+    case res
+    when Net::HTTPSuccess
+    when Net::HTTPRedirection then
+      uri = URI(res['location'])
+      res = http.request_head(uri.path)
+    else
+      return false
+    end
+
+    if use_pbar then
+      pbar = ProgressBar.create(:title => EveData::DATAFILE, :total => res['Content-Length'].to_i, :format => '%a %e |%b>>%i| %p%% %t')
+    end
+
+    http.request_get(uri.path) do |res|
+      FileUtils.mkdir_p(dname) unless Dir.exists?(dname)
+      open fname, "wb" do |io|
+        res.read_body do |chunk|
+          io.write chunk
+          pbar.progress += chunk.length if pbar
+        end
+      end
+    end
+    pbar.finish if pbar
+  end
+  true
+end # }}}

data/mram-evedata.gemspec

@@ -22,4 +22,5 @@ Gem::Specification.new do |s|
 
   s.add_dependency 'rails', "~> 3.2.0"
   s.add_dependency 'composite_primary_keys', '~> 5.0'
+  s.add_dependency 'ruby-progressbar'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mram-evedata
 version: !ruby/object:Gem::Version
-  version: 0.1.inferno11.3
+  version: 0.1.inferno11.4
   prerelease: 4
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-29 00:00:00.000000000 Z
+date: 2012-08-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
-  requirement: &13670160 !ruby/object:Gem::Requirement
+  requirement: &6133420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,10 +21,10 @@ dependencies:
         version: 3.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *13670160
+  version_requirements: *6133420
 - !ruby/object:Gem::Dependency
   name: composite_primary_keys
-  requirement: &13669700 !ruby/object:Gem::Requirement
+  requirement: &6132420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,7 +32,18 @@ dependencies:
         version: '5.0'
   type: :runtime
   prerelease: false
-  version_requirements: *13669700
+  version_requirements: *6132420
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: &6131420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *6131420
 description: this gem will import the EVE Online Static Data Dump into your database
   and provide a framework for accessing it
 email:
@@ -44,7 +55,7 @@ files:
 - .gitignore
 - Gemfile
 - MIT-LICENSE
-- README.rdoc
+- README.md
 - Rakefile
 - app/assets/images/mram-evedata/.gitkeep
 - app/assets/javascripts/mram-evedata/application.js
@@ -72,7 +83,6 @@ files:
 - app/models/eve_data/type_requirement.rb
 - app/views/layouts/mram-evedata/application.html.erb
 - config/routes.rb
-- data/mysql55-inferno11-extended.sql.bz2
 - lib/generators/eve_data/sample_db_config/sample_db_config_generator.rb
 - lib/mram-evedata.rb
 - lib/mram-evedata/engine.rb

data/README.rdoc

@@ -1,3 +0,0 @@
-= Mram::EveData
-
-This project rocks and uses MIT-LICENSE.