d3_mpq 0.0.6

data/MIT-LICENSE

@@ -0,0 +1,21 @@
+Copyright 2012 Patrick Helm
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.rdoc

@@ -0,0 +1,33 @@
+= D3MPQ
+
+== "Diablo 3" data mining
+Parsing data out of Diablo 3.
+
+== Requirements
+  If you want to extract MPQs, you need:
+  * https://launchpad.net/smpq
+
+== NOTE
+  * No data is provided in this repo.
+  * You need to read the MPQs by yourself for parsing!
+
+== Setup:
+
+Copy the MPQ-Directory to your working directory:
+
+  cd /foo/bar
+  cp -r <path-to-mpqs>/* /foo/bar/mpqs/*
+
+== CommandLine-Usage:
+
+  cd /foo/bar
+
+  # Help
+  d3_mpq --help
+
+  # Extract MPQs
+  d3_mpq extract enUS_Text
+
+  # Create a CSV
+  d3_mpq --csv-input extracted_mpqs/GameBalance/ItemTypes.gam csv ItemTypes
+

data/Rakefile

@@ -0,0 +1,42 @@
+##!/usr/bin/env rake
+##include Rake::DSL
+
+require "bundler/gem_tasks"
+
+#begin
+#  require 'bundler/setup'
+#rescue LoadError
+#  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+#end
+#begin
+#  require 'rdoc/task'
+#rescue LoadError
+#  require 'rdoc/rdoc'
+#  require 'rake/rdoctask'
+#  RDoc::Task = Rake::RDocTask
+#end
+
+#RDoc::Task.new(:rdoc) do |rdoc|
+#  rdoc.rdoc_dir = 'rdoc'
+#  rdoc.title    = 'RubyRescuetime'
+#  rdoc.options << '--line-numbers'
+#  rdoc.rdoc_files.include('README.rdoc')
+#  rdoc.rdoc_files.include('lib/**/*.rb')
+#end
+
+
+
+#Bundler::GemHelper.install_tasks
+
+## require 'rake/testtask'
+
+##Rake::TestTask.new(:test) do |t|
+##  t.libs << 'lib'
+##  t.libs << 'test'
+##  t.pattern = 'test/**/*_test.rb'
+##  t.verbose = true
+##end
+
+
+##task :default => :test
+

data/bin/d3_mpq

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'd3_mpq/commands'
+

data/lib/bindata_ext/dsl.rb

@@ -0,0 +1,27 @@
+class BinData::DSLMixin::DSLParser
+  # Example: zeroes :length => 0x4
+  def zeroes(options = {:length => 0x1})
+    check_value 0x00000000, options
+  end
+
+  # Example: ffs :length => 0x4
+  def ffs(options = {:length => 0x1})
+    check_value 0xFFFFFFFF, options
+  end
+
+  # Example: ffs :length => 0x4
+  def unknown(options = {:length => 0x1})
+#    array :initial_length => options[:length] do
+#      uint32 #:check_value => lambda { value == v }
+#    end
+    string :length => options[:length]*4, :truncate => true
+  end
+
+  # Example: check_value 0x00, :length => 0x4
+  def check_value(v, options = {:length => 0x1})
+    array :initial_length => options[:length] do
+      uint32 :check_value => lambda { value == v }
+    end
+  end
+end
+

data/lib/bindata_ext/mod_code.rb

@@ -0,0 +1,192 @@
+class ModCode < BinData::Record
+  endian  :little
+
+  uint32  :mod_code
+
+  uint32  :mod_param1
+  uint32  :mod_param2
+  uint32  :mod_param3
+
+  uint32  :mod_offset
+  uint32  :mod_length
+
+  # Returns: nil if mod_code is 0xFFFFFFFF, mod_code otherwise
+  def id
+    self.mod_code == 0xFFFFFFFF ? nil : self.mod_code.to_i
+  end
+
+  # HACK
+  # Returns: Instance of BinData::Record that respond to :variable_content
+  def base
+    parent.parent.parent.parent
+  end
+
+  def index
+    (mod_offset - base.data_num_bytes - base.data_offset) / 4
+  end
+
+  def count
+    mod_length / 4
+  end
+
+  def data
+    base.variable_content[index, count]
+  end
+
+  def empty?
+    mod_length == 0
+  end
+
+  def min
+    stack.min || 0.0
+  end
+
+  def max
+    stack.max || 0.0
+  end
+
+  def trace
+    stack.trace
+  end
+
+  def stack
+    @stack ||= Stack.new(data)
+  end
+
+  private
+
+  # Pseudo-Stack to compute min/max of current ModCode
+  class Stack
+    attr_reader :value, :trace
+
+    def initialize(data)
+      @stack  = []
+      @trace  = []
+      @data   = *data
+      @value  = nil
+      parse
+    end
+
+    def min
+      @value.min
+    end
+
+    def max
+      @value.max
+    end
+
+    def push(item)
+      @stack.push item
+    end
+
+    def pop
+      @stack.pop
+    end
+
+    def peek
+      @stack.last
+    end
+
+    def count
+      @stack.length
+    end
+
+    def empty?
+      count == 0
+    end
+
+    # Lets go
+    def parse
+      return @value if @value
+
+      next_op = true
+      @data.each_with_index do |dword, index|
+        next_op = true and next unless next_op
+
+        case dword.bytes.first
+        when 0x00
+          # return the value at the top of the stack
+          raise "Return found, but Stack not empty!" if @data[index+1]
+          return @value = pop
+        when 0x01
+          # call the function of the next op-code (just pass here)
+        when 0x03
+          # rand1(A,B); pop 2 numbers and push a random value between A and A+B
+          op { |a, b| ["rand", a, a + b] }
+        when 0x04
+          # rand2(A,B); pop 2 numbers and push a random value between A and B
+          op { |a, b| ["rand2", a, b] }
+        when 0x06
+          # push the next DWord onto the stack
+          v = @data[index+1].unpack('F').first
+          push(StackValue.new(v))
+          @trace << "#{v}"
+          next_op = false
+        when 0x0B
+          # add(A,B); pop 2 numbers and push the sum
+          op { |a, b| ["add", (a + b)] }
+        when 0x0C
+          # sub(A,B); pop 2 numbers and push the difference
+          op { |a, b| ["sub", (a - b)] }
+        when 0x0D
+          # mul(A,B); pop 2 numbers and push the product
+          op { |a, b| ["mul", (a * b)] }
+        when 0x0E
+          # div(A,B); pop 2 numbers and push the quotient
+          op { |a, b| ["div", (a / b)] }
+        else
+          raise "OP-Code not found: #{op} | #{@data.inspect}"
+        end
+      end
+
+      raise "No return found: #{@data.inspect}"
+    end
+
+    # Perform OP-Code
+    def op(&blk)
+      b = pop
+      a = pop
+
+      formula, min, max = blk.call(a, b)
+
+      push(StackValue.random(min, min || max))
+      @trace << formula
+    end
+
+    def inspect
+      { :value => @value, :trace => @trace}
+    end
+
+    # Wrapper for a Float that handles: current_value, min_value, max_value
+    class StackValue
+      attr_accessor :value, :min_max
+
+      def self.random(min, max)
+        StackValue.new(MinMax.new(min.min, max.max).random, min.min, max.max)
+      end
+
+      def initialize(value, min = nil, max = nil)
+        @value   = value
+        min    ||= @value
+        @min_max = MinMax.new(min, max)
+      end
+
+      [:min, :max].each { |m| define_method(m) { min_max.send(m) } }
+      ["-", "+", "*", "/"].each do |m|
+        define_method(m) { |sv| with_other(sv){ |a, b| a.send(m, b) } }
+      end
+
+      def to_s
+        (min == max) ? "#{min}" : "#{min}..#{max}"
+      end
+
+      private
+      def with_other(oth, &block)
+        mm = block.call(min_max, oth.min_max)
+        v  = block.call(value, oth.value)
+        StackValue.new(v, mm.min, mm.max)
+      end
+    end
+  end
+end
+

data/lib/bindata_ext/mod_codes.rb

@@ -0,0 +1,4 @@
+class ModCodes < BinData::Array
+  default_parameters :type => :mod_code, :initial_length => 1
+end
+

data/lib/core_ext/min_max.rb

@@ -0,0 +1,47 @@
+class MinMax
+  attr_reader :min, :max
+
+  def initialize(min = nil, max = nil)
+    min ||= 0.0
+    max ||= min
+    @min, @max = (min <= max) ? [min, max] : [max, min]
+  end
+
+  def +(oth)
+    MinMax.new(self.min + oth.min, self.max + oth.max)
+  end
+
+  def -(oth)
+    MinMax.new(self.min - oth.max, self.max - oth.min)
+  end
+
+  # HACK: use a more clever solution
+  def *(oth)
+    v = possible_values(self, oth){ |a, b| a * b }
+    MinMax.new(v.min, v.max)
+  end
+
+  # HACK: use a more clever solution
+  def /(oth)
+    v = possible_values(self, oth){ |a, b| a / b }
+    MinMax.new(v.min, v.max)
+  end
+
+  # Returns: A random value between min and max
+  def random
+    (min == max) ? min : rand * (max - min) + min
+  end
+
+  private
+  # Returns: Array
+  def possible_values(a, b, &block)
+    values = []
+    values << block.call(a.min, b.min)
+    values << block.call(a.min, b.max)
+    values << block.call(a.max, b.min)
+    values << block.call(a.max, b.max)
+
+    return values
+  end
+end
+

data/lib/core_ext/string.rb

@@ -0,0 +1,6 @@
+class String
+  def to_d3_hash
+    self.each_char.inject(0) { |h, c| ((h * 0x21) + c.downcase.ord) & 0xFFFFFFFF }
+  end
+end
+

data/lib/d3_mpq/analyzer.rb

@@ -0,0 +1,144 @@
+require 'fileutils'
+
+module D3MPQ
+  class Analyzer
+    attr_reader :attributes
+
+    SEPERATOR = ","
+
+    def initialize(parser, files, fields = :content)
+      @parser     = parser.new
+      @files      = [*files]
+      @field      = fields
+#      @fields     = [*fields]
+    end
+
+    # Just write analyzed data to disc. (within /analyze)
+    def write
+      case parser_name
+      when "d3mpq_stringlist"
+        write_stringlist
+      when "d3mpq_attributes"
+        write_single_file("analyze")
+      when "d3mpq_coredata_actor"
+        write_single_file("analyze")
+      else
+        write_game_balance
+      end
+    end
+
+    # Writing if StringList
+    def write_stringlist
+      dir = File.join("analyze", "StringList")
+      write_single_file(dir, @files.first.split("/").last)
+    end
+
+    # Writing if GameBalance
+    def write_game_balance
+      write_single_file("analyze")
+
+      dir = File.join("analyze", parser_name)
+      dir = File.join(dir, @field.to_s) if @field
+      write_analyzed(dir)
+    end
+
+    # Example: d3mpq_stringlist
+    def parser_name
+      @parser_name ||= @parser.class.name.gsub("::", "_").downcase
+    end
+
+    # Write output to a single file. (dir/filename)
+    # Using parser as filename if none is given.
+    def write_single_file(dir, filename = nil)
+      FileUtils.mkdir_p File.join(dir)
+
+      # HACKY: NEED TO FIX
+      keys = snapshots.first.keys
+      keys = snapshots.first[@field].first.keys if @field
+
+      s = []
+      s << keys.join(SEPERATOR)
+      snapshots.each do |snapshot|
+        # HACKY: NEED TO FIX
+        if @field
+          snapshot[@field].each_with_index do |e, i|
+            s << e.values.map do |v|
+              if v.is_a?(String)
+                "\"#{v}\""
+              else
+                "#{v}"
+              end
+            end.join(SEPERATOR)
+          end
+        else
+          s << [*snapshot.values].map { |e|
+            e.is_a?(String) ? "\"#{e}\""  : "#{e}"
+          } .join(SEPERATOR)
+        end
+      end
+
+      filename ||= @parser.class.name.split("::").last
+      path = File.join(dir, filename)
+      File.open("#{path}.csv", 'w') { |f| f.write(s.join("\n")) }
+    end
+
+    # Writing multiple files to given dir.
+    def write_analyzed(dir)
+      FileUtils.mkdir_p(dir)
+
+      attributes.each do |a, v|
+        path = File.join(dir, a.to_s)
+        s = "Count|Value\n" + v.map { |e| "#{e[:count]}|#{e[:value]}" }.join("\n")
+        File.open("#{path}.csv", 'w') { |f| f.write(s) }
+      end
+    end
+
+    # Return analyzed attributes
+    def attributes
+      return @attributes if @attributes
+
+      unsorted = Hash.new { |h,k| h[k] = Hash.new(0) }
+      snapshots.each do |attributes|
+        attributes = attributes[@field] if @field
+
+        attributes.each do |h|
+          h.each { |attribute, value| unsorted[attribute][value] += 1 }
+        end
+      end
+
+      @attributes = Hash.new { |h,k| h[k] = [] }
+      unsorted.each do |name, h|
+        h.each do |value, count|
+          @attributes[name] << { :value => value, :count => count }
+        end
+        @attributes[name].sort! { |x,y| y[:count] <=> x[:count] }
+      end
+
+      return @attributes
+    end
+
+
+
+    private
+
+    # Return snapshots of parsed files
+    def snapshots
+      return @snapshots if @snapshots
+
+      @snapshots = []
+      @files.each do |f|
+        io = File.open(f)
+
+        begin
+          @parser.read(io)
+          @snapshots << @parser.snapshot
+        rescue EOFError => e
+          puts "#{e.inspect}\n#{f}"
+        end
+      end
+
+      return @snapshots
+    end
+  end
+end
+

data/lib/d3_mpq/attributes.rb

@@ -0,0 +1,56 @@
+require 'nokogiri'
+
+module D3MPQ
+  class Attributes
+    class Attribute < Struct.new( :id, :u2, :u3, :u4, :u5, :script_a, :script_b,
+                                  :name, :encoding_type, :u10, :min, :max, :bit_count )
+      def keys; self.members; end # HACK: to get the analyzer working
+    end
+
+
+    def initialize(path = nil)
+      read(path) if path
+    end
+
+    def read(path)
+      @doc = Nokogiri::XML(File.open(path))
+    end
+
+    # HACK: to get the analyzer working
+    def snapshot
+      {:content => content}
+    end
+
+    def content
+      return @content if @content
+
+      @content = []
+      entries.each do |e|
+        a = Attribute.new
+        a.id            = e["Id"]
+        a.u2            = e["U2"]
+        a.u3            = e["U3"]
+        a.u4            = e["U4"]
+        a.u5            = e["U5"]
+        a.script_a      = e["ScriptA"]
+        a.script_b      = e["ScriptB"]
+        a.name          = e["Name"]
+        a.encoding_type = e["EncodingType"]
+        a.u10           = e["U10"]
+        a.min           = e["Min"]
+        a.max           = e["Max"]
+        a.bit_count     = e["BitCount"]
+
+        @content << a
+      end
+
+      return @content
+    end
+
+    private
+    def entries
+      @doc.xpath("//Entry")
+    end
+  end
+end
+

data/lib/d3_mpq/commands/csv.rb

@@ -0,0 +1,47 @@
+require 'fileutils'
+
+module D3MPQ::Command
+  class CSV
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = options
+      @klass = @options[:klass]
+
+      FileUtils.mkdir_p csv_path
+
+      klass = eval("D3MPQ::CSVGenerator::#{@klass}")
+      generator = klass.new(generator_params)
+      generator.write
+    end
+
+    def generator_params
+      { :files => input_path, :output_path => output_path }
+    end
+
+    def output_path
+      "#{csv_path}/#{csv_name}#{patch}#{locale}.csv"
+    end
+
+    def csv_name
+      @options[:csv_name] || @klass.to_s
+    end
+
+    def locale
+      ".#{options[:locale]}" if options[:locale] && !options[:locale].empty?
+    end
+
+    def patch
+      ".#{options[:patch]}" if options[:patch] && !options[:patch].empty?
+    end
+
+    def input_path
+      options[:csv_input]
+    end
+
+    def csv_path
+      options[:csv_path]
+    end
+  end
+end
+

data/lib/d3_mpq/commands/extract.rb

@@ -0,0 +1,54 @@
+module D3MPQ::Command
+  class Extract
+    attr_reader :options
+
+    def initialize(options = {})
+      @options = options
+      @options[:mpq_path] += "/" unless @options[:mpq_path].end_with?("/")
+      logger "Extracting MPQ"
+      cmd = "#{cd} && smpq -ef #{mpq} #{patch}"
+      logger "Performing: #{cmd}"
+      logger `#{cmd}`
+    end
+
+    class << self
+      def all
+        known_mpqs.each { |mpq| self.new(mpq) }
+      end
+
+      def known_mpqs
+        [
+          "ClientData",
+          "GameBalance",
+          "enUS_Audio",
+          "enUS_Cutscene",
+          "enUS_Text",
+          "HLSLShaders",
+          "Sound",
+          "Texture"
+        ]
+      end
+    end
+
+    def cd
+      "cd #{options[:extract_path]}"
+    end
+
+    def patch
+      "-p #{mpq_path}base/* #{mpq_path}#{options[:locale]}/* #{mpq_path}Win/*"
+    end
+
+    def mpq
+      "#{mpq_path}#{options[:mpq]}.mpq"
+    end
+
+    def mpq_path
+      "../#{options[:mpq_path]}"
+    end
+
+    def logger(msg)
+      puts msg
+    end
+  end
+end
+