nwn-lib 0.4.9 → 0.4.10

data/CHANGELOG

@@ -269,3 +269,33 @@ Bernhard Stoeckner <elven@swordcoast.net> (10):
       Some medium rare bugfixes, and a overloaded operator for GFF structs,
       which can be used to comfortable walk on trees.
 
+=== 0.4.10
+Bernhard Stoeckner <elven@swordcoast.net> (23):
+      bin/nwn-erf: exit(1) on errors
+      bin/nwn-erf: support extracting parts of archive with -x
+      Erf::Erf: set correct internal year on initialize
+      bin/nwn-gff: fix reading gff data from stdin
+      Gff::Field: transform type to symbols on extend
+      Gff::Struct#path: return full data type by walking the tree on the fly
+      Gff: add :pretty format that prettyprints
+      Gff::Struct, Gff::Field: add generic box/unbox methods
+      Gff: JSON support
+      new ENV NWN_LIB_IN_ENCODING specifies nwn-gff encoding
+      :pretty-print boxes before printing
+      Gff: JSON support adheres NWN_LIB_PRETTY_JSON env var
+      Kivinen-format: refactor into Marshal load/dump interface
+      Res/ContentObject: minor internal cleanups
+      Gff::Reader: do not compact exolocstrs on read-in
+      YAML/JSON support: fix spec, add dep, fix files to not clash with gem names
+      Rakefile: spec support for >= 1.1.9
+      NWN::Key: .key, .bif reading support
+      Resources::DirectoryContainer: do not fail on invalid files in directory
+      documentation updates
+      bin/nwn-erf: do not set description str_ref to 0x0
+      NWN::Gff: refactor API to properly partition different file format handlers
+
+      A swath of bugfixes, JSON read/write, and finally .key and .bif reading support.
+      Also, Resources::Manager got some documentation and a few touches here and there,
+      as did bin/nwn-erf, which supports partial extraction now.
+      For our non-ASCII users, there is a ENV variable that can be used to specify the
+      encoding in which local GFF data is supposed to be in.

data/HOWTO

@@ -168,3 +168,43 @@ Now read-only access with a TlkSet:
 
 You cannot use TlkSet to write out .tlk files or modify
 existing entries - it is merely a wrapper.
+
+=== Accessing .key index files
+
+A key file is an index into all shipped game resources.
+
+ key = Key::Key.new(File.new("/path/to/chitin.key", "r"), "/path/to/")
+
+This will lookup all indexed bif files and the resources
+contained within.
+
+=== Resource Manager
+
+The resource manager can be used to simulate a NWN-style
+resource manager to read files from .key/bifs, override,
+haks, and similar sources. Files are located in the reverse
+order that their containers are added to the Manager.
+
+Example detailing the usual NWN lookup procedure:
+
+ nwn_path = "/path/to/nwn/"
+ mgr = Resources::Manager.new
+
+ # First, all the base data files.
+ for key in %w{chitin.key xp1.key xp1patch.key xp2.key xp2patch.key xp3.key}
+   mgr.add_container(Key::Key.new(File.new(nwn_path + key, "r"), nwn_path))
+ end
+
+ # Override
+ mgr.add_container(Resources::DirectoryContainer.new(nwn_path + "override"))
+
+ # All custom haks
+ for hak in %w{a.hak b.hak c.hak}
+   mgr.add_container(Erf::Erf.new(File.new(nwn_path + "hak/" + hak, "r")))
+ end
+
+ # Now you can retrieve any indexed file:
+ puts mgr.get("actions.2da")
+
+Note that initialising the whole Resource Manager this way takes
+a few seconds depending on IO and CPU speed.

data/README

@@ -11,15 +11,22 @@ They should work with NWN2 just as well, since the file format specifications di
 
 * GFF 3.2 (are, git, gic, dlg, itp, ifo, jrl, fac, ssf, ut*, among others)
 * ERF (mod, hak, erf, among others)
+* KEY, BIF (key, data/*.bif)
 * 2DA V2.0
 * TLK
 
-==== nwn-lib can read and write the following representations of GFF data:
+==== nwn-lib can handle the following representations of GFF data:
 
-* kivinen-style ("gffprint.pl") presentation of data
+===== read & write:
+* native gff
 * yaml presentation of data
+* json presentation of data (with proper UTF-8 conversion)
 * ruby-native marshalling of gff data
 
+===== just write, for now:
+* kivinen-style ("gffprint.pl") presentation of data
+* prettyprint (ruby-specific)
+
 ==== Also in the box:
 
 * shell scripts and tools to simplify your life (see BINARIES)

data/Rakefile

@@ -9,7 +9,7 @@ include FileUtils
 # Configuration
 ##############################################################################
 NAME = "nwn-lib"
-VERS = "0.4.9"
+VERS = "0.4.10"
 CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
 RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
   'nwn-lib: a ruby library for accessing NWN resource files', \
@@ -44,6 +44,7 @@ spec = Gem::Specification.new do |s|
   s.files = %w(COPYING CHANGELOG README Rakefile) + Dir.glob("{bin,doc,spec,lib,tools,scripts,data}/**/*")
   s.require_path = "lib"
   s.bindir = "bin"
+  s.add_dependency('json', '>= 1.1.9')
 end
 
 Rake::GemPackageTask.new(spec) do |p|
@@ -70,7 +71,7 @@ end
 
 desc "Upload nwn-lib gem to rubyforge"
 task :release => [:package] do
-  sh %{rubyforge login}
+  #sh %{rubyforge login}
   sh %{rubyforge add_release nwn-lib #{NAME} #{VERS} pkg/#{NAME}-#{VERS}.tgz}
   sh %{rubyforge add_file nwn-lib #{NAME} #{VERS} pkg/#{NAME}-#{VERS}.gem}
 end
@@ -80,8 +81,6 @@ require "spec/rake/spectask"
 desc "Run specs with coverage"
 Spec::Rake::SpecTask.new("spec") do |t|
   t.spec_files = FileList["spec/*_spec.rb"]
-  t.spec_opts  = File.read("spec/spec.opts").split("\n")
-  t.rcov_opts  = File.read("spec/rcov.opts").split("\n")
   t.rcov = true
 end
 
@@ -89,13 +88,10 @@ desc "Run specs without coverage"
 task :default => [:spec_no_cov]
 Spec::Rake::SpecTask.new("spec_no_cov") do |t|
   t.spec_files = FileList["spec/*_spec.rb"]
-  t.spec_opts  = File.read("spec/spec.opts").split("\n")
 end
 
 desc "Run rcov only"
 Spec::Rake::SpecTask.new("rcov") do |t|
-  t.rcov_opts  = File.read("spec/rcov.opts").split("\n")
-  t.spec_opts  = File.read("spec/spec.opts").split("\n")
   t.spec_files = FileList["spec/*_spec.rb"]
   t.rcov = true
 end

data/SETTINGS

@@ -6,6 +6,14 @@ Under linux, just add them to your shell environment (usually .bashrc), like so:
 
   export NWN_LIB_DEBUG=0
 
+== NWN_LIB_IN_ENCODING
+
+The character encoding your local data files are in. Defaults to ISO-8859-1.
+
+== NWN_LIB_PRETTY_JSON
+
+Set to non-nil to make :json print pretty output (with whitespace) instead of a spaghetti
+bowl. nwn-lib returns compact json output, by default. Does not affect reading json.
 
 == NWN_LIB_DEBUG
 
@@ -24,7 +32,7 @@ time tracking things down.
 
 Note that nwn-lib takes a parameter that can force a specific NWN version.
 
-=== NWN_LIB_RESREF32
+== NWN_LIB_RESREF32
 
 Set this to "1" if you are working with NWN2 files and do not want to see the warnings
 generated by too long resref values.

data/bin/nwn-erf

@@ -9,6 +9,8 @@ $type = 'ERF'
 $allow_duplicates = false
 $descriptions = {}
 $version = "V1.0"
+$file = nil
+$error = 0
 
 # tar-compat mode: first argument is options if no dash is specified.
 ARGV[0] = "-" + ARGV[0] if ARGV.size > 0 && ARGV[0][0] != ?-
@@ -28,7 +30,7 @@ begin OptionParser.new do |o|
   o.on "-c", "--create", "Create a new archive with the given files as contents." do
     $action = :c
   end
-  o.on "-x", "--extract", "Extract all files to current directory." do
+  o.on "-x", "--extract", "Extract FILEs (or all) to current directory." do
     $action = :x
   end
   o.on "-a", "--add", "Add files to the given archive.",
@@ -131,7 +133,17 @@ case $action
   when :x
     input {|f|
       erf = NWN::Erf::Erf.new(f)
+      ARGV.each {|x|
+        wot = erf.content.select {|cc| cc.filename == x }
+        if wot.size == 0
+          $stderr.puts "nwn-erf: #{x}: not found in erf"
+          $error = 1
+        end
+      }
+      what = ARGV.map {|x| x.downcase }
       erf.content.each {|c|
+        next if what.size > 0 && !what.index(c.filename.downcase)
+
         puts "%s" % [c.filename] if $verbose
         output(c.filename) {|ff|
           ff.write(c.get)
@@ -146,7 +158,6 @@ case $action
 
     if $descriptions
       erf.localized_strings.merge! $descriptions
-      erf.description_str_ref = 0
     end
 
     ARGV.each {|a|
@@ -163,7 +174,6 @@ case $action
 
       if $descriptions
         erf.localized_strings.merge! $descriptions
-        erf.description_str_ref = 0
       end
 
       ARGV.each {|arg|
@@ -192,3 +202,8 @@ case $action
   else
     raise ArgumentError, "You need to specify a mode of operation (try -h)."
 end
+
+if $error == 1
+  $stderr.puts "nwn-erf: Exiting with failure status due to previous errors"
+  exit(1)
+end

data/bin/nwn-gff

@@ -7,6 +7,7 @@ Thread.abort_on_exception = true
 
 $options = {
   :backup => nil,
+  :encoding => nil,
   :force => false,
   :infile => '-',
   :outfile => '-',
@@ -25,8 +26,8 @@ begin OptionParser.new do |o|
   o.on "-i", "--infile FILE", "Input file (default: stdin)" do |f|
     $options[:infile] = f
   end
-  o.on "-l, ""--infile-format FORMAT", [:auto] + NWN::Gff::FileFormats,
-      "Input format (#{([:auto] + NWN::Gff::FileFormats).join(', ')})",
+  o.on "-l, ""--infile-format FORMAT", [:auto] + NWN::Gff::InputFormats.keys,
+      "Input format (#{([:auto] + NWN::Gff::InputFormats.keys).join(', ')})",
       "(default: auto - try to guess based on extension)" do |f|
     $options[:informat] = f
   end
@@ -34,8 +35,8 @@ begin OptionParser.new do |o|
   o.on "-o", "--outfile FILE", "Output file (default: stdout)" do |f|
     $options[:outfile] = f
   end
-  o.on "-k", "--outfile-format FORMAT", [:in, :none] + NWN::Gff::FileFormats,
-      "Output format (#{([:none, :in] + NWN::Gff::FileFormats).join(', ')})",
+  o.on "-k", "--outfile-format FORMAT", [:in, :none] + NWN::Gff::OutputFormats.keys,
+      "Output format (#{([:none, :in] + NWN::Gff::OutputFormats.keys).join(', ')})",
       "(default: in when stdout, try to guess based on extension otherwise)" do |f|
     $options[:outformat] = f
   end
@@ -48,6 +49,10 @@ begin OptionParser.new do |o|
       "(see `man cp' for a description)" do |b|
     $options[:backup] = b.nil? ? true : b
   end
+  o.on "--encoding ENCODING", "sets the used input encoding in which your NWN",
+      "files are encoded in" do |e|
+    $options[:encoding] = e
+  end
 
   o.on "-1", "--nwn1", "Allow 16 byte resrefs." do
     ENV['NWN_LIB_RESREF32'] = nil
@@ -110,6 +115,10 @@ end
 $options[:informat] or fail "No input format specified."
 $options[:outformat] or fail "No output format specified."
 
+if $options[:encoding]
+  NWN.setting(:in_encoding, $options[:encoding])
+end
+
 if :auto == $options[:informat]
   $options[:informat] = NWN::Gff.guess_file_format($options[:infile].downcase)
   fail "Cannot guess infile format from filename, specify with -l." unless
@@ -125,7 +134,7 @@ elsif :in == $options[:outformat]
 end
 
 vputs "Reading: #{$options[:infile]}"
-data_in = $options[:infile] == '-' ? $stdin : File.open($options[:infile], "rb")
+data_in = $options[:infile] == '-' ? StringIO.new($stdin.read) : File.open($options[:infile], "rb")
 data_in = NWN::Gff.read(data_in, $options[:informat])
 
 # verify that we read a GFF struct

data/lib/nwn/all.rb

@@ -1,11 +1,13 @@
 require 'stringio'
 require 'nwn/io'
 require 'nwn/twoda'
+require 'nwn/settings'
 require 'nwn/res'
 require 'nwn/gff'
 require 'nwn/tlk'
+require 'nwn/key'
 require 'nwn/erf'
-require 'nwn/yaml'
-require 'nwn/kivinen'
+require 'nwn/yaml_support'
+require 'nwn/json_support'
+require 'nwn/kivinen_support'
 require 'nwn/scripting'
-require 'nwn/settings'

data/lib/nwn/erf.rb

@@ -22,7 +22,7 @@ module NWN
         @localized_strings = {}
         @io = io
         @file_type, @file_version = "ERF", "V1.0"
-        @year = Time.now.year
+        @year = Time.now.year - 1900
         @description_str_ref = 0xffffffff
         @day_of_year = Time.now.yday
         read_from io if io

data/lib/nwn/gff/field.rb

@@ -16,10 +16,15 @@ module NWN::Gff::Field
   # This is set internally by Gff::Reader on load.
   attr_accessor :parent
 
+  # Transform type to symbol on extend.
+  def self.extended p1 #:nodoc:
+    p1['type'] = p1['type'].to_sym if p1['type']
+  end
+
   # Create a new NWN::Gff::Field
   def self.new label, type, value
     s = {}.extend(self)
-    s['label'], s['type'], s['value'] = label, type, value
+    s['label'], s['type'], s['value'] = label, type.to_sym, value
     s.extend_meta_classes
     s.validate
     s
@@ -190,4 +195,48 @@ module NWN::Gff::Field
   end
 #:startdoc:
 
+  # Deep-unboxes a Hash, e.g. iterating down, converting all strings
+  # from the native charset.
+  def self.unbox! element, parent_label, parent = nil
+    element.extend(NWN::Gff::Field)
+    element.field_label = parent_label
+    element.parent = parent
+    element.str_ref ||= NWN::Gff::Field::DEFAULT_STR_REF if element.respond_to?('str_ref=')
+
+    element.extend_meta_classes
+    case element.field_type
+      when :cexolocstr
+        element.field_value.each {|x,y|
+          element.field_value[x.to_i] = NWN::IconvNativeToGff.call.iconv(element.field_value.delete(x))
+        }
+      when :cexostr
+        element.field_value = NWN::IconvNativeToGff.call.iconv(element.field_value)
+
+      when :list
+        element.field_value.each_with_index {|x,idx|
+          element.field_value[idx] = NWN::Gff::Struct.unbox!(x, element)
+        }
+      when :struct
+        element.field_value = NWN::Gff::Struct.unbox!(element.field_value, element)
+    end
+    element.validate
+    element
+  end
+
+  # Returns a hash of this Field without the API calls mixed in,
+  # all language-strings transformed by str_handler.
+  def box
+    t = Hash[self]
+    t.delete('label')
+    case field_type
+      when :cexolocstr
+        t['value'].each {|x,y|
+          t['value'][x] = NWN::IconvGffToNative.call.iconv(y)
+        }
+      when :cexostr
+        t['value'] = NWN::IconvGffToNative.call.iconv(t['value'])
+    end
+    t
+  end
+
 end

data/lib/nwn/gff/reader.rb

@@ -180,8 +180,6 @@ class NWN::Gff::Reader
           exostr[id] = str
         }
         len = total_size + 4
-        # Filter out empty strings.
-        exostr.reject! {|k,v| v.nil? || v.empty?}
         exostr
 
       when :void
@@ -189,7 +187,7 @@ class NWN::Gff::Reader
         @field_data[data_or_offset + 4, len].unpack("H*")[0]
 
       when :struct
-        read_struct data_or_offset, field.path, field.parent.data_version
+        read_struct data_or_offset, nil, field.parent.data_version
 
       when :list
         list = []
@@ -210,7 +208,7 @@ class NWN::Gff::Reader
         data_or_offset += 1
 
         for i in data_or_offset...(data_or_offset + count)
-          list << read_struct(@list_indices[i], field.path, field.parent.data_version)
+          list << read_struct(@list_indices[i], nil, field.parent.data_version)
         end
 
         list

data/lib/nwn/gff/struct.rb

@@ -30,7 +30,11 @@ module NWN::Gff::Struct
 
   # Returns the path to this struct (which is usually __data_type)
   def path
-    @data_type.to_s
+    if @element
+      @element.path
+    else
+      @data_type.to_s
+    end
   end
 
   def element= e #:nodoc:
@@ -200,4 +204,39 @@ module NWN::Gff::Struct
     by_path(path)
   end
 
+  # Deep-unboxes a Hash, e.g. iterating down, converting it to
+  # the native charset.
+  def self.unbox! o, parent = nil
+    o.extend(NWN::Gff::Struct)
+    o.struct_id = o.delete('__struct_id')
+    o.data_type = if o['__data_type']
+      o.delete('__data_type')
+    else
+      o.path
+    end
+    o.data_version = o.delete('__data_version')
+
+    o.element = parent if parent
+
+    o.each {|label,element|
+      o[label] = NWN::Gff::Field.unbox!(element, label, o)
+    }
+
+    o
+  end
+
+  # Returns a hash of this Struct without the API calls mixed in,
+  # converting it from the native charset.
+  def box
+    t = Hash[self]
+    t.merge!({
+      '__struct_id' => self.struct_id,
+      '__data_version' => self.data_version,
+    })
+    t.merge!({
+      '__data_type' => self.data_type
+    }) if self.element == nil
+    t
+  end
+
 end

data/lib/nwn/gff.rb

@@ -1,3 +1,5 @@
+require 'pp'
+
 module NWN
   module Gff
     # This error gets thrown if reading or writing fails.
@@ -56,20 +58,35 @@ module NWN
       :double => 'd',
     }.freeze
 
-    # These field types can never be inlined in YAML.
-    YAMLNonInlineableFields = [:struct, :list, :cexolocstr]
+    # Registers a new format handler that can deal with file formats for nwn-lib gff handling.
+    def self.register_format_handler name, fileFormatRegexp, klass, reads = true, writes = true
+      InputFormats[name.to_sym] = klass if reads
+      OutputFormats[name.to_sym] = klass if writes
+      FileFormatGuesses[fileFormatRegexp] = name.to_sym
+    end
+
+    class Handler
+      def self.load io
+        NWN::Gff::Reader.read(io)
+      end
+      def self.dump data, io
+        NWN::Gff::Writer.dump(data, io)
+      end
+    end
+
+    class Pretty
+      def self.dump data, io
+        old = $> ; $> = io ; pp data.box ; $> = old
+      end
+    end
 
-    FileFormats = [:gff, :yaml, :kivinen, :marshal]
+    InputFormats = {}
+    OutputFormats = {}
+    FileFormatGuesses = {}
 
-    FileFormatGuesses = {
-      /^ut[cdeimpstw]$/ => :gff,
-      /^(git|are|gic)$/ => :gff,
-      /^(mod|ifo|fac|ssf|dlg|itp)$/ => :gff,
-      /^(bic)$/ => :gff,
-      /^ya?ml$/ => :yaml,
-      /^marshal$/ => :marshal,
-      /^k(ivinen)?$/ => :kivinen,
-    }
+    register_format_handler :gff, /^(ut[cdeimpstw]|git|are|gic|mod|ifo|fac|ssf|dlg|itp|bic)$/, NWN::Gff::Handler
+    register_format_handler :marshal, /^marshal$/, Marshal
+    register_format_handler :pretty, /^$/, Pretty, false, true
 
     def self.guess_file_format(filename)
       extension = File.extname(filename.downcase)[1..-1]
@@ -77,34 +94,18 @@ module NWN
     end
 
     def self.read(io, format)
-      return case format
-        when :gff
-          NWN::Gff::Reader.read(io)
-        when :yaml
-          YAML.load(io)
-        when :marshal
-          Marshal.load(io)
-        when :kivinen
-          raise NotImplementedError, "Reading kivinen-style data is not supported."
-        else
-          raise NotImplementedError, "Don't know how to read #{format}."
+      if InputFormats[format]
+        InputFormats[format].load(io)
+      else
+        raise NotImplementedError, "Don't know how to read #{format}."
       end
     end
 
     def self.write(io, format, data)
-      case format
-        when :gff
-          NWN::Gff::Writer.dump(data, io)
-        when :yaml
-          io.puts data.to_yaml
-        when :marshal
-          io.print Marshal.dump(data)
-        when :kivinen
-          data.kivinen_format $options[:types], nil, nil do |l,v|
-            io.puts "%s:\t%s" % [l, v]
-          end
-        else
-          raise NotImplementedError, "Don't know how to write data-format #{format.inspect}"
+      if OutputFormats[format]
+        OutputFormats[format].dump(data, io)
+      else
+        raise NotImplementedError, "Don't know how to write #{format}."
       end
     end
   end

data/lib/nwn/json_support.rb

@@ -0,0 +1,37 @@
+require 'json'
+
+module NWN::Gff::Struct
+  def to_json(*a)
+    box.to_json(*a)
+  end
+end
+
+module NWN::Gff::Field
+  def to_json(*a)
+    box.to_json(*a)
+  end
+end
+
+module NWN::Gff::JSON
+  def self.load io
+    json = if io.respond_to?(:to_str)
+      io.to_str
+    elsif io.respond_to?(:to_io)
+      io.to_io.read
+    else
+      io.read
+    end
+
+    NWN::Gff::Struct.unbox!(JSON.parse(json), nil)
+  end
+
+  def self.dump struct, io
+    if NWN.setting(:pretty_json)
+      io.puts JSON.pretty_generate(struct)
+    else
+      io.print JSON.generate(struct)
+    end
+  end
+end
+
+NWN::Gff.register_format_handler :json, /^json$/, NWN::Gff::JSON

data/lib/nwn/key.rb

@@ -0,0 +1,144 @@
+module NWN
+  module Key
+
+    # A Bif object encapsulates an open file handle pointing
+    # to a .bif file. It's contents are indexed on first access,
+    # not on creation by NWN::Key::Key (to speed up things).
+    class Bif
+
+      # The Key object this Bif belongs to.
+      attr_reader :key
+
+      # The IO object pointing to the .bif file.
+      attr_reader :io
+
+      # A hash containing the resources contained. Usually not needed,
+      # accessed by the encapsulating Key object.
+      attr_reader :contained
+
+      def initialize key, io
+        @key = key
+        @io = io
+
+        @contained = {}
+
+        @file_type, @file_version,
+          @var_res_count, @fix_res_count,
+          @var_table_offset =
+          io.e_read(4 + 4 + 3 * 4, "header").unpack("a4 a4 V V V")
+
+        @io.seek(@var_table_offset)
+        data = @io.e_read(@var_res_count * 16, "var res table")
+        i = 0
+        while (x = data[i, 16]) && x.size == 16
+          i += 16
+          id, offset, size, restype = x.unpack("V V V V")
+          id &= 0xfffff
+          @contained[id] = [offset, size, restype]
+        end
+      end
+
+      def has_res_id? id
+        @contained[id] != nil
+      end
+
+      def get_res_id id
+        offset, size, restype = @contained[id]
+        @io.seek(offset)
+        @io.e_read(size, "resource #{id} of type #{restype}")
+      end
+    end
+
+    class Key < NWN::Resources::Container
+
+      # An array of Bif objects contained in this key index.
+      # Not needed to access individual files, use Container#content instead.
+      attr_reader :bif
+
+      attr_reader :file_type
+      attr_reader :file_version
+      attr_reader :day_of_year
+      attr_reader :year
+
+      # Creates a new Key wrapper. The parameters exepected are an
+      # IO object pointing to the .key-file, and the base path in
+      # which your data/.bif files can be found. (This is usually your
+      # NWN directory, NOT the data/ directory).
+      def initialize io, data_path
+        super()
+
+        @root = data_path
+        @bif = []
+
+        @file_type, @file_version,
+          bif_count, key_count,
+          offset_to_file_table, offset_to_key_table,
+          @year, @day_of_year, reserved =
+          io.e_read(8 + (4 * 6) + 32, "header").unpack("A4 A4 VVVVVV a32")
+
+        io.seek(offset_to_file_table)
+        data = io.e_read(12 * bif_count, "bif data")
+
+        # Contains all bifs linked in this key
+        i = 0
+        @file_table = []
+        while (x = data[i, 12]) && x.size == 12
+          i += 12
+          size, name_offset, name_size, drives = x.unpack("VVvv")
+          io.seek(name_offset)
+          name = io.e_read(name_size, "name table").unpack("A*")[0]
+          name.gsub!("\\", "/")
+          name = File.expand_path(@root + "/" + name)
+
+          _io = File.new(name, "r")
+          @bif << Bif.new(self, _io)
+
+          @file_table << [size, name, drives]
+        end
+
+        @key_table = {}
+        io.seek(offset_to_key_table)
+        data = io.e_read(22 * key_count, "key table")
+        i = 0
+        while (x = data[i, 22]) && x.size == 22
+          i += 22
+          resref, res_type, res_id = x.unpack("A16 v V")
+          @key_table[res_id] = [resref, res_type]
+        end
+
+        @fn_to_co = {}
+        @key_table.each {|res_id, (resref, res_type)|
+          bif_index = res_id >> 20
+          bif = @bif[bif_index]
+          id = res_id & 0xfffff
+          bif.contained[id] or fail "#{bif} does not have #{id}"
+          ofs, sz, _rt = bif.contained[id]
+          o = NWN::Resources::ContentObject.new(resref, res_type, bif.io, ofs, sz)
+          if @fn_to_co[o.filename] && @fn_to_co[o.filename][2] < bif_index
+            oo, biff = @fn_to_co[o.filename]
+            NWN.log_debug "#{o.filename} in #{biff.io.inspect} shadowed by file of same name in #{bif.io.inspect}"
+            @content.delete(oo)
+          end
+          @fn_to_co[o.filename] = [o, bif, bif_index]
+          @content << o
+        }
+      end
+    end
+
+    # Get the ContentObject pointing to the given filename.
+    # Raises ENOENT if not mapped.
+    def get_content_object filename
+      filename = filename.downcase
+      ret, bif = @fn_to_co[filename]
+      raise Errno::ENOENT,
+        "No ContentObject with the given filename #{filename.inspect} found." unless
+          ret
+      ret
+    end
+
+    def filenames
+      @fn_to_co.indices
+    end
+
+  end
+end

data/lib/nwn/kivinen_support.rb

@@ -0,0 +1,67 @@
+module NWN::Gff::Kivinen
+  def self.load io
+    raise NotImplementedError, "Reading kivinen not supported"
+  end
+
+  def self.dump struct, io
+    ret = ""
+    kivinen_format struct, $options[:types], nil, nil do |l,v|
+      ret += "%s:\t%s\n" % [l, v]
+    end
+    io.puts ret
+  end
+
+  # Parses +s+ as an arbitary GFF object and yields for each field found,
+  # with the proper prefix.
+  #
+  # [+struct+]     The root-struct to dump
+  # [+prefix+]     Supply a prefix to add to the output.
+  # [+types_too+]  Yield type definitions as well (gffprint.pl -t).
+  # [+add_prefix+] Add a prefix <tt>(unknown type)</tt> of no type information can be derived from the input.
+  # [+file_type+]  File type override. If non-null, add a global struct header with the given file type (useful for passing to gffencode.pl)
+  # [+struct_id+]  Provide a struct_id override (if printing a struct).
+  def self.kivinen_format struct, types_too = false, add_prefix = true, file_type = nil, struct_id = nil, &block
+
+    if types_too
+      yield("/", "")
+
+      ftype = file_type ? file_type : struct.data_type
+      yield("/ ____file_type", ftype) if ftype
+      yield("/ ____file_version", struct.data_version) if struct.data_version
+
+      yield("/ ____struct_type", struct.struct_id)
+    end
+
+    struct.each_by_flat_path {|path, field|
+      case field
+        when String
+          yield(path, field)
+
+        when NWN::Gff::Struct
+          yield(path + "/", path)
+          yield(path + "/ ____struct_type", field.struct_id)
+
+        when NWN::Gff::Field
+
+          case field.field_type
+            when :list
+            when :struct
+              yield(path + "/", path)
+              yield(path + "/ ____struct_type", field.field_value.struct_id)
+            when :cexolocstr
+            else
+              yield(path, field.field_value)
+          end
+
+          yield(path + ". ____string_ref",field.str_ref) if
+            field.has_str_ref? || field.field_type == :cexolocstr
+
+          yield(path + ". ____type", NWN::Gff::Types.index(field.field_type)) if
+            types_too
+
+      end
+    }
+  end
+end
+
+NWN::Gff.register_format_handler :kivinen, /^k(ivinen)?$/, NWN::Gff::Kivinen, false, true

data/lib/nwn/res.rb

@@ -61,18 +61,18 @@ module NWN
 
     # Wraps n ContentObjects; a baseclass for erf/key encapsulation.
     class Container
+
+      # An array of all ContentObjects indexed by this Container.
       attr_reader :content
 
       def initialize
         @content = []
       end
 
+      # Returns true if the given filename is contained herein.
+      # Case-insensitive.
       def has?(filename)
-        base = File.basename(filename)
-        @content.each {|f|
-          return true if f.filename.downcase == base.downcase
-        }
-        return false
+        filenames.index(filename.downcase) != nil
       end
 
       # Add a content object giving a +filename+ and a optional
@@ -86,17 +86,18 @@ module NWN
         @content << o
       end
 
-      # Returns a list of filenames
+      # Returns a list of filenames, all lowercase.
       def filenames
-        @content.map {|x| x.filename }
+        @content.map {|x| x.filename.downcase }
       end
 
       # Get the ContentObject pointing to the given filename.
       # Raises ENOENT if not mapped.
       def get_content_object filename
-        ret = @content.select {|x| filename.downcase == x.filename }
+        filename = filename.downcase
+        ret = @content.select {|x| filename == x.filename }
         raise Errno::ENOENT,
-          "No ContentObject with the given filename found." if
+          "No ContentObject with the given filename #{filename.inspect} found." if
             ret.size == 0
         ret[0]
       end
@@ -104,7 +105,7 @@ module NWN
       # Get the contents of the given filename.
       # Raises ENOENT if not mapped.
       def get filename
-        get_content_object(filename.downcase).get
+        get_content_object(filename).get
       end
     end
 
@@ -115,7 +116,10 @@ module NWN
         super()
         @path = path
         Dir[path + "/*.*"].each {|x|
-          add_file x
+          begin add_file x
+          rescue ArgumentError => e
+            NWN.log_debug e.to_s
+          end
         }
       end
     end
@@ -138,7 +142,7 @@ module NWN
           con.has?(filename) or next
           return con.get_content_object(filename)
         }
-        raise Errno::ENOENT, "No ContentObject with the given filename found."
+        raise Errno::ENOENT, "No ContentObject with the given filename #{filename.inspect} found."
       end
 
       # Get the contents of the given filename.

data/lib/nwn/settings.rb

@@ -1,4 +1,9 @@
+require 'iconv'
+
 module NWN
+  SETTING_DEFAULT_VALUES = {
+    'NWN_LIB_IN_ENCODING' => 'ISO-8859-1'
+  }
 
   # This writes a internal warnings and debug messages to stderr.
   #
@@ -28,9 +33,12 @@ module NWN
       ENV[name] = value.to_s if value != :_invalid_
       ret
     else
-      ENV[name] == "0" ? false : ENV[name]
+      ENV[name] == "0" ? false : (ENV[name] || SETTING_DEFAULT_VALUES[name])
     end
   end
+
+  IconvGffToNative = proc { Iconv.new('utf-8', NWN.setting(:in_encoding)) }
+  IconvNativeToGff = proc { Iconv.new(NWN.setting(:in_encoding), 'utf-8') }
 end
 
 NWN::TwoDA::Cache.setup NWN.setting("2da_location") if

data/lib/nwn/{yaml.rb → yaml_support.rb}

@@ -1,8 +1,20 @@
 # This file contains all YAML-specific loading and dumping code.
 require 'yaml'
 
-# See http://www.taguri.org/ for the exact meaning of this.
-NWN::YAML_DOMAIN = "nwn-lib.elv.es,2008-12"
+class NWN::Gff::YAML
+  # These field types can never be inlined in YAML.
+  NonInlineableFields = [:struct, :list, :cexolocstr]
+
+  # See http://www.taguri.org/ for the exact meaning of this.
+  Domain = "nwn-lib.elv.es,2008-12"
+
+  def self.load io
+    YAML.load(io)
+  end
+  def self.dump data, io
+    io.puts data.to_yaml
+  end
+end
 
 #:stopdoc:
 class Array
@@ -34,7 +46,7 @@ end
 
 module NWN::Gff::Struct
   def to_yaml_type
-    "!#{NWN::YAML_DOMAIN}/struct"
+    "!#{NWN::Gff::YAML::Domain}/struct"
   end
 
   def to_yaml(opts = {})
@@ -43,7 +55,7 @@ module NWN::Gff::Struct
         # Inline certain structs that are small enough.
         map.style = :inline if self.size <= 1 &&
           self.values.select {|x|
-            NWN::Gff::YAMLNonInlineableFields.index(x['type'])
+            NWN::Gff::YAML::NonInlineableFields.index(x['type'])
           }.size == 0
 
         map.add('__' + 'data_type', @data_type) if @data_type
@@ -63,7 +75,7 @@ module NWN::Gff::Field
   def to_yaml(opts = {})
     YAML::quick_emit(nil, opts) do |out|
       out.map(taguri, to_yaml_style) do |map|
-        map.style = :inline unless NWN::Gff::YAMLNonInlineableFields.index(self['type'])
+        map.style = :inline unless NWN::Gff::YAML::NonInlineableFields.index(self['type'])
         map.add('type', self['type'])
         map.add('str_ref', self['str_ref']) if has_str_ref?
         map.add('value', self['value'])
@@ -73,7 +85,7 @@ module NWN::Gff::Field
 end
 
 # This parses the struct and extends all fields with their proper type.
-YAML.add_domain_type(NWN::YAML_DOMAIN,'struct') {|t,hash|
+YAML.add_domain_type(NWN::Gff::YAML::Domain,'struct') {|t,hash|
   struct = {}
   struct.extend(NWN::Gff::Struct)
 
@@ -101,3 +113,5 @@ YAML.add_domain_type(NWN::YAML_DOMAIN,'struct') {|t,hash|
 
   struct
 }
+
+NWN::Gff.register_format_handler :yaml, /^(y|yml|yaml)$/, NWN::Gff::YAML

data/spec/bin_gff_spec.rb

@@ -3,10 +3,10 @@ require File.join(File.dirname(__FILE__), 'spec_helper')
 describe "nwn-gff" do
   it_should_behave_like "bin helper"
 
-  NWN::Gff::FileFormats.each do |in_format|
+  NWN::Gff::InputFormats.each do |in_format, handler|
     inf = in_format.to_s
 
-    NWN::Gff::FileFormats.each do |out_format|
+    NWN::Gff::OutputFormats.each do |out_format, handler|
       otf = out_format.to_s
 
       it "converts #{inf} to #{otf}" do

data/spec/erf_spec.rb

@@ -30,6 +30,13 @@ describe "Erf::Erf", :shared => true do
     wellformed_verify @erf
   end
 
+  it "sets the correct default parameters" do
+    t = Erf::Erf.new
+    t.year.should == Time.now.year - 1900
+    t.day_of_year.should == Time.now.yday
+    t.content.size.should == 0
+  end
+
   it "reproduces correct ERF binary data" do
     t = Erf::Erf.new(StringIO.new @erf)
     io = StringIO.new

data/spec/key_spec.rb

@@ -0,0 +1,38 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+describe "Key::Key", :shared => true do
+  def wellformed_verify binary
+    t = Key::Key.new(StringIO.new(binary), Dir.tmpdir)
+
+    t.file_type.should == "KEY"
+    t.file_version.should == "V1"
+    t.content.size.should == 2
+    t.content[0].resref.should == "abcdef"
+    t.content[1].resref.should == "123456"
+    t.content[0].get.should == "abcdefghij"
+    t.content[1].get.should == "0123456789"
+  end
+
+  it "should work" do
+    wellformed_verify @key
+  end
+
+  before do
+    @bif0 = File.join(Dir.tmpdir, "bif0.bif")
+    File.open(@bif0, "w") do |f|
+      f.write(WELLFORMED_BIF_0)
+    end
+  end
+
+  after do
+    File.unlink(@bif0)
+  end
+end
+
+describe "Key V1.0" do
+  before do
+    @key = WELLFORMED_KEY.dup
+  end
+
+  it_should_behave_like "Key::Key"
+end

data/spec/spec_helper.rb

@@ -92,6 +92,36 @@ WELLFORMED_TLK = ([
   "1", "22", "333", "4444"
 ].join("")).freeze
 
+WELLFORMED_BIF_0 = ([
+  "BIFF", "V1",
+  var_res_count = 2,
+  fix_res_count = 0,
+  var_table_offset = 20
+].pack("a4 a4 V V V") + [
+  id0 = 124, var_table_offset + (16 * var_res_count),         size0 = 10, type0 = 0,
+  id1 = 125, var_table_offset + (16 * var_res_count) + size0, size1 = 10, type1 = 0,
+].pack("VVVV VVVV") + [
+  "abcdefghij", "0123456789"
+].pack("a* a*")
+).freeze
+
+WELLFORMED_KEY = ([
+  "KEY", "V1",
+  bif_count = 1, key_count = 2,
+  offset_to_file_table = 8 + (4 * 6) + 32,
+  offset_to_key_table = offset_to_file_table + bif_count * 12 + 8,
+  100, 126, ""
+].pack("a4 a4 VVVVVV a32") + [ # file table containing bifs
+  bifsize = WELLFORMED_BIF_0.size, bifname0offset = offset_to_file_table + 12, fnsize = 8, drives = 0,
+].pack("VVvv") + [ #filename table
+  "bif0.bif"
+].pack("a*") + [ # key table
+  "abcdef", 0, 124,
+  "123456", 0, 125
+].pack("a16 v V a16 v V")
+).freeze
+
+
 TWODA_WELLFORMED = <<-EOT
 2DA V2.0
 
@@ -172,7 +202,7 @@ describe "bin helper", :shared => true do
     begin
     Dir.chdir(@tmp)
     Open3.popen3(
-      "ruby", "-I#{incl}",
+      "ruby", "-rubygems", "-I#{incl}",
       binary,
       *va
     ) do |i,o,e|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nwn-lib
 version: !ruby/object:Gem::Version 
-  version: 0.4.9
+  version: 0.4.10
 platform: ruby
 authors: 
 - Bernhard Stoeckner
@@ -9,10 +9,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-07-19 00:00:00 +02:00
+date: 2009-11-15 00:00:00 +01:00
 default_executable: 
-dependencies: []
-
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: json
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.1.9
+    version: 
 description: a ruby library for accessing Neverwinter Nights resource files
 email: elven@swordcoast.net
 executables: 
@@ -36,49 +45,50 @@ files:
 - CHANGELOG
 - README
 - Rakefile
-- bin/nwn-dsl
 - bin/nwn-gff
-- bin/nwn-irb
+- bin/nwn-dsl
 - bin/nwn-erf
-- spec/wellformed_gff.bic
+- bin/nwn-irb
+- spec/bin_dsl_spec.rb
 - spec/gff_spec.rb
-- spec/res_spec.rb
-- spec/spec.opts
-- spec/erf_spec.rb
+- spec/key_spec.rb
+- spec/bin_gff_spec.rb
+- spec/bin_erf_spec.rb
 - spec/field_spec.rb
 - spec/cexolocstr_spec.rb
+- spec/erf_spec.rb
 - spec/struct_spec.rb
-- spec/spec_helper.rb
+- spec/res_spec.rb
 - spec/tlk_spec.rb
-- spec/bin_dsl_spec.rb
-- spec/bin_gff_spec.rb
-- spec/rcov.opts
-- spec/bin_erf_spec.rb
 - spec/twoda_spec.rb
-- lib/nwn/scripting.rb
-- lib/nwn/yaml.rb
-- lib/nwn/twoda.rb
+- spec/spec_helper.rb
+- spec/wellformed_gff.bic
 - lib/nwn/erf.rb
-- lib/nwn/settings.rb
-- lib/nwn/gff.rb
-- lib/nwn/tlk.rb
+- lib/nwn/json_support.rb
 - lib/nwn/io.rb
-- lib/nwn/res.rb
-- lib/nwn/kivinen.rb
+- lib/nwn/settings.rb
+- lib/nwn/key.rb
 - lib/nwn/all.rb
-- lib/nwn/gff/field.rb
-- lib/nwn/gff/reader.rb
-- lib/nwn/gff/cexolocstr.rb
+- lib/nwn/res.rb
+- lib/nwn/twoda.rb
+- lib/nwn/tlk.rb
+- lib/nwn/gff.rb
+- lib/nwn/yaml_support.rb
+- lib/nwn/scripting.rb
+- lib/nwn/kivinen_support.rb
+- lib/nwn/gff/writer.rb
 - lib/nwn/gff/struct.rb
 - lib/nwn/gff/list.rb
-- lib/nwn/gff/writer.rb
+- lib/nwn/gff/reader.rb
+- lib/nwn/gff/cexolocstr.rb
+- lib/nwn/gff/field.rb
 - tools/verify.sh
 - tools/migrate_03x_to_04x.sh
 - scripts/truncate_floats.rb
-- scripts/reformat_2da
 - scripts/clean_locstrs.rb
-- scripts/extract_all_items.rb
 - scripts/debug_check_objid.rb
+- scripts/extract_all_items.rb
+- scripts/reformat_2da
 - BINARIES
 - HOWTO
 - SCRIPTING
@@ -116,7 +126,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: nwn-lib
-rubygems_version: 1.3.4
+rubygems_version: 1.3.5
 signing_key: 
 specification_version: 3
 summary: a ruby library for accessing Neverwinter Nights resource files

data/lib/nwn/kivinen.rb

@@ -1,55 +0,0 @@
-module NWN::Gff::Struct
-
-    # yield (key, value) for each element, recursing into substructs.
-
-    # Parses +s+ as an arbitary GFF object and yields for each field found,
-    # with the proper prefix.
-    #
-    # [+prefix+]     Supply a prefix to add to the output.
-    # [+types_too+]  Yield type definitions as well (gffprint.pl -t).
-    # [+add_prefix+] Add a prefix <tt>(unknown type)</tt> of no type information can be derived from the input.
-    # [+file_type+]  File type override. If non-null, add a global struct header with the given file type (useful for passing to gffencode.pl)
-    # [+struct_id+]  Provide a struct_id override (if printing a struct).
-    def kivinen_format types_too = false, add_prefix = true, file_type = nil, struct_id = nil, &block
-
-      if types_too
-        yield("/", "")
-
-        ftype = file_type ? file_type : self.data_type
-        yield("/ ____file_type", ftype) if ftype
-        yield("/ ____file_version", self.data_version) if self.data_version
-
-        yield("/ ____struct_type", self.struct_id)
-      end
-
-      self.each_by_flat_path {|path, field|
-        case field
-          when String
-            yield(path, field)
-
-          when NWN::Gff::Struct
-            yield(path + "/", path)
-            yield(path + "/ ____struct_type", field.struct_id)
-
-          when NWN::Gff::Field
-
-            case field.field_type
-              when :list
-              when :struct
-                yield(path + "/", path)
-                yield(path + "/ ____struct_type", field.field_value.struct_id)
-              when :cexolocstr
-              else
-                yield(path, field.field_value)
-            end
-
-            yield(path + ". ____string_ref",field.str_ref) if
-              field.has_str_ref? || field.field_type == :cexolocstr
-
-            yield(path + ". ____type", NWN::Gff::Types.index(field.field_type)) if
-              types_too
-
-        end
-      }
-    end
-end