nwn-lib 0.4.11 → 0.4.12

data/CHANGELOG

@@ -323,3 +323,32 @@ Bernhard Stoeckner <elven@swordcoast.net> (7):
       A mostly-maintenance release with some API changes. The most invasive changes
       are some method renames. New are some sane default values for Struct#add_*,
       and a few more specs. YAML now sets correct .element references.
+
+=== 0.4.12
+Bernhard Stoeckner (18):
+      Gff read/write API: documentation, Handler#dump always returns bytes written
+      Settings: iconv: refactor into methods
+      data_type_spec: refactor to be more concise
+      Gff::Struct#data_type=: only emit debug message when new type is non-nil
+      Gff::Handler::JSON: emit terminating newline regardless of :pretty_json
+      JSON: infer data_version like yaml, omit if DEFAULT
+      Gff: error if registering format of same name twice, reverse FileFormatRX hash
+      Gff::Struct: reparenting struct emits debug message only if element != nil
+      Gff::Reader: allow reading of arbitary Gff version files
+      Gff::Struct: data_version allow more freedom in inferring, update spec
+      NWN::Gff: custom xml data format, nwntools.sf.net modpacker read/write support
+      Rakefile: spec output in spec-style format
+      Rakefile: remove obsolete rubyforge target
+      bin/nwn-gff: --out-encoding specified output encoding
+      NWN.setting: return default values for setting new values too
+      spec_helper: set $options to unbreak kivinen spec
+      NWN.log_debug: non-lib frame if available, :debug_trace prints full traces
+      Gff::Field: fix data ranges for int64, dword64
+	  0.4.12-rel
+
+      A few new features and some bugfixes this release:
+      - more concise yaml dumps (but backwards-compatible)
+      - some iconv fixes
+      - dword64 and int64 data ranges now correct
+      - support for parsing/dumping modpacker XML
+      - support for parsing/dumping a custom, cleaner XML format

data/README

@@ -22,6 +22,8 @@ They should work with NWN2 just as well, since the file format specifications di
 * yaml presentation of data
 * json presentation of data (with proper UTF-8 conversion)
 * ruby-native marshalling of gff data
+* a compact native xml format
+* nwntools.sf.net-style xml files ("modpacker")
 
 ===== just write, for now:
 * kivinen-style ("gffprint.pl") presentation of data

data/Rakefile

@@ -9,7 +9,7 @@ include FileUtils
 # Configuration
 ##############################################################################
 NAME = "nwn-lib"
-VERS = "0.4.11"
+VERS = "0.4.12"
 CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
 RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
   'nwn-lib: a ruby library for accessing NWN resource files', \
@@ -68,17 +68,12 @@ task :uninstall => [:clean] do
   sh %{sudo gem1.8 uninstall #{NAME}}
 end
 
-desc "Upload nwn-lib gem to rubyforge"
-task :release => [:package] do
-  sh %{rubyforge add_release nwn-lib #{NAME} #{VERS} pkg/#{NAME}-#{VERS}.tgz}
-  sh %{rubyforge add_file nwn-lib #{NAME} #{VERS} pkg/#{NAME}-#{VERS}.gem}
-end
-
 require "spec/rake/spectask"
 
 desc "Run specs with coverage"
 Spec::Rake::SpecTask.new("spec") do |t|
   t.spec_files = FileList["spec/*_spec.rb"]
+  t.spec_opts = ["--format s"]
   t.rcov = true
 end
 
@@ -86,11 +81,13 @@ desc "Run specs without coverage"
 task :default => [:spec_no_cov]
 Spec::Rake::SpecTask.new("spec_no_cov") do |t|
   t.spec_files = FileList["spec/*_spec.rb"]
+  t.spec_opts = ["--format s"]
 end
 
 desc "Run rcov only"
 Spec::Rake::SpecTask.new("rcov") do |t|
   t.spec_files = FileList["spec/*_spec.rb"]
+  t.spec_opts = ["--format s"]
   t.rcov = true
 end
 

data/SETTINGS

@@ -24,6 +24,11 @@ resource files you are working with.
 If you want to suppress them for some reason, set NWN_LIB_DEBUG to "0" or "off". Any
 other value will indicate that you want to see the messages.
 
+== NWN_LIB_DEBUG_TRACES
+
+Set to non-nil to print full stack traces for each debug message emitted by NWN_LIB_DEBUG.
+Default is not to print them, just the first non-library-frame.
+
 == NWN_LIB_RESREF16
 
 Set this to "1" if you are sure that you are only working with NWN1 files. This will

data/bin/nwn-gff

@@ -8,6 +8,7 @@ Thread.abort_on_exception = true
 $options = {
   :backup => nil,
   :encoding => nil,
+  :out_encoding => nil,
   :force => false,
   :infile => '-',
   :outfile => '-',
@@ -53,6 +54,10 @@ begin OptionParser.new do |o|
       "files are encoded in" do |e|
     $options[:encoding] = e
   end
+  o.on "--out-encoding ENCODING", "sets the used output encoding",
+      "(defaults to UTF-8)" do |e|
+    $options[:out_encoding] = e
+  end
 
   o.on "-1", "--nwn1", "Allow 16 byte resrefs." do
     ENV['NWN_LIB_RESREF32'] = nil
@@ -115,9 +120,8 @@ end
 $options[:informat] or fail "No input format specified."
 $options[:outformat] or fail "No output format specified."
 
-if $options[:encoding]
-  NWN.setting(:in_encoding, $options[:encoding])
-end
+NWN.setting(:in_encoding, $options[:encoding]) if $options[:encoding]
+NWN.setting(:out_encoding, $options[:out_encoding]) if $options[:out_encoding]
 
 if :auto == $options[:informat]
   $options[:informat] = NWN::Gff.guess_file_format($options[:infile].downcase)

data/lib/nwn/all.rb

@@ -15,4 +15,11 @@ rescue LoadError => e
   NWN.log_debug "json support not available, install json or json_pure to enable"
 end
 require 'nwn/kivinen_support'
+begin
+  require 'xml'
+  require 'nwn/xml_support'
+rescue LoadError => e
+  NWN.log_debug "nxml and modpacker support not available, install libxml-ruby to enable"
+end
+
 require 'nwn/scripting'

data/lib/nwn/gff.rb

@@ -17,6 +17,60 @@ module NWN
     # not exist.
     class GffPathInvalidError < RuntimeError; end
 
+    # A namesoace for all Gff file format handlers.
+    module Handler
+      # Registers a new format handler that can deal with file formats for nwn-lib gff handling.
+      #
+      # [+name+]   The name of this format as a symbol. Must be unique.
+      # [+fileFormatRegexp+] A regular expression matching file extensions for auto-detection.
+      # [+klass+]  A object that responds to load(io) and dump(gff,io). load(io) reads from io
+      #            and always returns a NWN::Gff::Struct describing a root struct, dump(gff, io)
+      #            dumps the gff root struct in the handlers format to io and returns the number
+      #            of bytes written.
+      # [+reads+]  Boolean, indicates if this handler can read it's format and return gff data.
+      # [+writes+] Boolean, indicates if this handler can emit gff data in it's format.
+      def self.register name, fileFormatRegexp, klass, reads = true, writes = true
+        raise ArgumentError, "Handler for #{name.inspect} already registered." if
+          NWN::Gff::InputFormats[name.to_sym] || NWN::Gff::OutputFormats[name.to_sym]
+        NWN::Gff::InputFormats[name.to_sym] = klass if reads
+        NWN::Gff::OutputFormats[name.to_sym] = klass if writes
+        NWN::Gff::FileFormatGuesses[name.to_sym] = fileFormatRegexp
+      end
+
+      module Gff
+        def self.load io
+          NWN::Gff::Reader.read(io)
+        end
+        def self.dump data, io
+          NWN::Gff::Writer.dump(data, io)
+        end
+      end
+
+      module Pretty
+        def self.dump data, io
+          old = $> ; $> = StringIO.new
+          pp data.box
+          sz = $>.pos
+          $>.seek(0)
+          io.write $>.read
+          $> = old
+          sz
+        end
+      end
+
+      module Marshal
+        def self.dump data, io
+          d = ::Marshal.dump(data)
+          io.write(d)
+          d.size
+        end
+
+        def self.load io
+          ::Marshal.load(io)
+        end
+      end
+    end
+
     # This hash lists all possible NWN::Gff::Field types.
     Types = {
       0 => :byte,
@@ -58,39 +112,22 @@ module NWN
       :double => 'd',
     }.freeze
 
-    # Registers a new format handler that can deal with file formats for nwn-lib gff handling.
-    def self.register_format_handler name, fileFormatRegexp, klass, reads = true, writes = true
-      InputFormats[name.to_sym] = klass if reads
-      OutputFormats[name.to_sym] = klass if writes
-      FileFormatGuesses[fileFormatRegexp] = name.to_sym
-    end
-
-    class Handler
-      def self.load io
-        NWN::Gff::Reader.read(io)
-      end
-      def self.dump data, io
-        NWN::Gff::Writer.dump(data, io)
-      end
-    end
-
-    class Pretty
-      def self.dump data, io
-        old = $> ; $> = io ; pp data.box ; $> = old
-      end
-    end
-
     InputFormats = {}
     OutputFormats = {}
     FileFormatGuesses = {}
 
-    register_format_handler :gff, /^(ut[cdeimpstw]|git|are|gic|mod|ifo|fac|ssf|dlg|itp|bic)$/, NWN::Gff::Handler
-    register_format_handler :marshal, /^marshal$/, Marshal
-    register_format_handler :pretty, /^$/, Pretty, false, true
+    Handler.register :gff, /^(ut[cdeimpstw]|git|are|gic|mod|ifo|fac|ssf|dlg|itp|bic)$/, NWN::Gff::Handler::Gff
+    Handler.register :marshal, /^marshal$/, NWN::Gff::Handler::Marshal
+    Handler.register :pretty, /^$/, NWN::Gff::Handler::Pretty, false, true
 
     def self.guess_file_format(filename)
       extension = File.extname(filename.downcase)[1..-1]
-      FileFormatGuesses[FileFormatGuesses.keys.select {|key| extension =~ key}[0]]
+      matches = FileFormatGuesses.select {|fmt,rx| extension =~ rx }
+      if matches.size == 1
+        matches[0][0]
+      else
+        nil
+      end
     end
 
     def self.read(io, format)

data/lib/nwn/gff/field.rb

@@ -144,9 +144,9 @@ module NWN::Gff::Field
         value.is_a?(Integer) && value >= 0 && value <= 0xffffffff
 
       when :int64
-        value.is_a?(Integer) && value >= -0x800000000000 && value <= 0x7fffffffffff
+        value.is_a?(Integer) && value >= -0x8000000000000000 && value <= 0x7fffffffffffffff
       when :dword64
-        value.is_a?(Integer) && value >= 0 && value <= 0xffffffffffff
+        value.is_a?(Integer) && value >= 0 && value <= 0xffffffffffffffff
 
       when :float, :double
         value.is_a?(Float)
@@ -228,10 +228,10 @@ module NWN::Gff::Field
     case element.field_type
       when :cexolocstr
         element.field_value.each {|x,y|
-          element.field_value[x.to_i] = NWN::IconvNativeToGff.call.iconv(element.field_value.delete(x))
+          element.field_value[x.to_i] = NWN.iconv_native_to_gff(element.field_value.delete(x))
         }
       when :cexostr
-        element.field_value = NWN::IconvNativeToGff.call.iconv(element.field_value)
+        element.field_value = NWN.iconv_native_to_gff(element.field_value)
 
       when :list
         element.field_value.each_with_index {|x,idx|
@@ -252,10 +252,10 @@ module NWN::Gff::Field
     case field_type
       when :cexolocstr
         t['value'].each {|x,y|
-          t['value'][x] = NWN::IconvGffToNative.call.iconv(y)
+          t['value'][x] = NWN.iconv_gff_to_native(y)
         }
       when :cexostr
-        t['value'] = NWN::IconvGffToNative.call.iconv(t['value'])
+        t['value'] = NWN.iconv_gff_to_native(t['value'])
     end
     t
   end

data/lib/nwn/gff/reader.rb

@@ -29,9 +29,6 @@ class NWN::Gff::Reader
     list_indices_offset, list_indices_count =
       @io.e_read(160, "header").unpack("a4a4 VV VV VV VV VV VV")
 
-    raise GffError, "Unknown version #{version}; not a gff?" unless
-      version == "V3.2"
-
     raise GffError, "struct offset at wrong place, not a gff?" unless
       struct_offset == 56
 

data/lib/nwn/gff/struct.rb

@@ -3,8 +3,8 @@
 module NWN::Gff::Struct
   DEFAULT_DATA_VERSION = "V3.2"
 
-  # The file version. Usually "V3.2" for root structs,
-  # and nil for sub-structs.
+  # The file version. Usually "V3.2". If not given in a source
+  # format, DEFAULT_DATA_VERSION is inferred and set for all structs.
   attr_accessor :data_version
 
   # GFF struct type. The default is 0xffffffff.
@@ -40,13 +40,14 @@ module NWN::Gff::Struct
 
   # Overrides the data type (used by the built-in file format readers).
   def data_type= k
-    NWN.log_debug("Setting explicit data_type for parented element") if @element
+    k = nil if k == ""
+    NWN.log_debug("Setting explicit data_type for parented element") if k && @element
     @data_type = k
   end
 
   def element= e #:nodoc:
     @element = e
-    NWN.log_debug("Re-parenting a struct with explicit data_type #{@data_type.inspect}") if @data_type
+    NWN.log_debug("Re-parenting a struct with explicit data_type #{@data_type.inspect}") if e && @data_type
   end
 
   # Dump this struct as GFF binary data.
@@ -62,7 +63,7 @@ module NWN::Gff::Struct
   #
   # You can pass a block to this method, which will receive the newly-created
   # Struct as the only argument.
-  def self.new struct_id = 0xffffffff, data_type = nil, data_version = nil
+  def self.new struct_id = 0xffffffff, data_type = nil, data_version = DEFAULT_DATA_VERSION
     s = {}.extend(self)
     s.struct_id = struct_id
     s.data_type = data_type
@@ -228,6 +229,7 @@ module NWN::Gff::Struct
     o.struct_id = o.delete('__struct_id')
     o.data_type = o.delete('__data_type')
     o.data_version = o.delete('__data_version')
+    o.data_version ||= NWN::Gff::Struct::DEFAULT_DATA_VERSION
 
     NWN.log_debug("Unboxed without a root data type") if
       !parent && !o.data_type
@@ -250,7 +252,8 @@ module NWN::Gff::Struct
     })
     t.merge!({
       '__data_version' => self.data_version,
-    }) if self.data_version && self.data_version != DEFAULT_DATA_VERSION
+    }) if self.data_version && self.data_version !=
+      NWN::Gff::Struct::DEFAULT_DATA_VERSION
     t.merge!({
       '__data_type' => self.data_type
     }) if @data_type

data/lib/nwn/json_support.rb

@@ -12,7 +12,7 @@ module NWN::Gff::Field
   end
 end
 
-module NWN::Gff::JSON
+module NWN::Gff::Handler::JSON
   def self.load io
     json = if io.respond_to?(:to_str)
       io.to_str
@@ -26,12 +26,14 @@ module NWN::Gff::JSON
   end
 
   def self.dump struct, io
-    if NWN.setting(:pretty_json)
-      io.puts JSON.pretty_generate(struct)
+    d = if NWN.setting(:pretty_json)
+      ::JSON.pretty_generate(struct)
     else
-      io.print JSON.generate(struct)
+      ::JSON.generate(struct)
     end
+    io.puts d
+    d.size + 1
   end
 end
 
-NWN::Gff.register_format_handler :json, /^json$/, NWN::Gff::JSON
+NWN::Gff::Handler.register :json, /^json$/, NWN::Gff::Handler::JSON

data/lib/nwn/kivinen_support.rb

@@ -1,4 +1,4 @@
-module NWN::Gff::Kivinen
+module NWN::Gff::Handler::Kivinen
   def self.load io
     raise NotImplementedError, "Reading kivinen not supported"
   end
@@ -9,6 +9,7 @@ module NWN::Gff::Kivinen
       ret += "%s:\t%s\n" % [l, v]
     end
     io.puts ret
+    ret.size
   end
 
   # Parses +s+ as an arbitary GFF object and yields for each field found,
@@ -64,4 +65,4 @@ module NWN::Gff::Kivinen
   end
 end
 
-NWN::Gff.register_format_handler :kivinen, /^k(ivinen)?$/, NWN::Gff::Kivinen, false, true
+NWN::Gff::Handler.register :kivinen, /^k(ivinen)?$/, NWN::Gff::Handler::Kivinen, false, true

data/lib/nwn/settings.rb

@@ -2,7 +2,8 @@ require 'iconv'
 
 module NWN
   SETTING_DEFAULT_VALUES = {
-    'NWN_LIB_IN_ENCODING' => 'ISO-8859-1'
+    'NWN_LIB_IN_ENCODING' => 'ISO-8859-1',
+    'NWN_LIB_OUT_ENCODING' => 'UTF-8'
   }
 
   # This writes a internal warnings and debug messages to stderr.
@@ -17,9 +18,18 @@ module NWN
     # Do not print debug messages if explicitly turned off
     return false if [false, "off"].index(setting(:debug))
 
-    pa = caller[0].to_s
-    pa = pa[(pa.size - 36) .. -1] if pa.size > 36
-    $stderr.puts "(nwn-lib) %s: %s" % [pa, msg]
+    if NWN.setting(:debug_traces)
+      $stderr.puts "(nwn-lib): %s" % [msg]
+      $stderr.puts "  " + caller.join("\n  ") + "\n"
+    else
+      dir = File.expand_path(File.dirname(File.expand_path(__FILE__)) + "/../../")
+      pa = caller.reject {|x| x.index(dir) }[0]
+      pa ||= caller[0]
+      pa ||= "(no frames)"
+      pa = pa[(pa.size - 36) .. -1] if pa.size > 36
+      $stderr.puts "(nwn-lib) %s: %s" % [pa, msg]
+    end
+
     true
   end
 
@@ -29,7 +39,7 @@ module NWN
   def self.setting sym, value = :_invalid_
     name = "NWN_LIB_#{sym.to_s.upcase}"
     if value != :_invalid_
-      ret = ENV[name] == "0" ? false : ENV[name]
+      ret = setting(sym)
       ENV[name] = value.to_s if value != :_invalid_
       ret
     else
@@ -37,8 +47,25 @@ module NWN
     end
   end
 
-  IconvGffToNative = proc { Iconv.new('utf-8', NWN.setting(:in_encoding)) }
-  IconvNativeToGff = proc { Iconv.new(NWN.setting(:in_encoding), 'utf-8') }
+  IconvState = {} #:nodoc:
+
+  # Converts text from native format (such as json) to Gff (required by NWN).
+  def self.iconv_native_to_gff text
+    if IconvState[:in] != NWN.setting(:in_encoding) ||
+        IconvState[:out] != NWN.setting(:out_encoding)
+      IconvState[:in_i] = Iconv.new(NWN.setting(:in_encoding), NWN.setting(:out_encoding))
+    end
+    IconvState[:in_i].iconv(text)
+  end
+
+  # Converts text from Gff format to native/external, such as json (usually UTF-8).
+  def self.iconv_gff_to_native text
+    if IconvState[:in] != NWN.setting(:in_encoding) ||
+        IconvState[:out] != NWN.setting(:out_encoding)
+      IconvState[:out_i] = Iconv.new(NWN.setting(:out_encoding), NWN.setting(:in_encoding))
+    end
+    IconvState[:out_i].iconv(text)
+  end
 end
 
 NWN::TwoDA::Cache.setup NWN.setting("2da_location") if

data/lib/nwn/xml_support.rb

@@ -0,0 +1,202 @@
+class NWN::Gff::Handler::XML
+
+private
+
+  def struct_to_xml struct
+    s = XML::Node.new('struct')
+    case @format
+      when :nxml
+        s['id'] = struct.struct_id.to_s
+        s['dataType'] = struct.data_type if struct.data_type
+        s['dataVersion'] = struct.data_version if
+          struct.data_version != NWN::Gff::Struct::DEFAULT_DATA_VERSION
+      when :modpacker
+        s['id'] = [struct.struct_id].pack("L").unpack("l")[0].to_s
+        s['nwnLibDataType'] = struct.data_type if struct.data_type
+        s['nwnLibDataVersion'] = struct.data_version if
+          struct.data_version != NWN::Gff::Struct::DEFAULT_DATA_VERSION
+    end
+
+    struct.sort.each {|(k,v)|
+      s << field_to_xml(v)
+    }
+    s
+  end
+
+  def field_to_xml field
+    e = case @format
+      when :nxml
+        XML::Node.new('field')
+      when :modpacker
+        XML::Node.new('element')
+    end
+    e['name'] = field.field_label
+    e['type'] = case @format
+      when :modpacker
+        NWN::Gff::Types.index(field.field_type).to_s
+      when :nxml
+        field.field_type.to_s
+    end
+
+    vv = case field.field_type
+      when :cexolocstr
+        case @format
+          when :modpacker
+            e['value'] = [field.str_ref].pack("L").unpack("l")[0].to_s
+          when :nxml
+            e['strRef'] = field.str_ref.to_s if
+              field.str_ref != NWN::Gff::Cexolocstr::DEFAULT_STR_REF
+        end
+
+        field.field_value.each {|lid, tx|
+          e << se = XML::Node.new("localString")
+          se['languageId'] = lid.to_s
+          se['value'] = NWN.iconv_gff_to_native(tx)
+        }
+
+      when :cexostr
+        e['value'] = NWN.iconv_gff_to_native(field.field_value)
+
+      when :struct
+        e << struct_to_xml(field.field_value)
+
+      when :list
+        field.field_value.each {|ee|
+          e << struct_to_xml(ee)
+        }
+
+      else
+        e['value'] = field.field_value.to_s
+    end
+
+    e
+  end
+
+  def xml_to_struct e, parent_data_version = nil
+    case @format
+      when :nxml
+        struct_id    = e['id'] || raise("No struct id for: #{e.path}")
+        struct_id    = struct_id.to_i
+        data_type    = e['dataType']
+        parent_data_version ||= NWN::Gff::Struct::DEFAULT_DATA_VERSION
+        data_version = e['dataVersion'] || parent_data_version
+      when :modpacker
+        struct_id    = [e['id'].to_i].pack("l").unpack("L")[0]
+        data_type    = e['nwnLibDataType']
+        data_version = e['nwnLibDataVersion'] || parent_data_version
+    end
+
+    st = NWN::Gff::Struct.new(struct_id, data_type, data_version)
+    e.each_element {|f|
+      xml_to_field(f, st, parent_data_version) if
+        f.name == case @format
+          when :modpacker ; 'element'
+          when :nxml ; 'field'
+        end
+    }
+    st
+  end
+
+  def xml_to_field field, struct, parent_data_version
+    name = field['name'] || raise("No name for field: #{field.path}")
+    type = case @format
+      when :nxml
+        field['type']
+      when :modpacker
+        NWN::Gff::Types[field['type'].to_i]
+    end || raise("No type for field: #{field.path}")
+    v    = field['value']
+
+    f = struct.add_field(name, type,
+      case type.to_sym
+        when :cexostr
+          NWN.iconv_native_to_gff(v)
+        when :cexolocstr
+          Hash[field.children.reject {|x| x.node_type != XML::Node::ELEMENT_NODE }.map {|ee|
+            [ee['languageId'].to_i, NWN.iconv_native_to_gff(ee['value'] || '')]
+          }]
+        when :list
+          field.children.reject {|x| x.node_type != XML::Node::ELEMENT_NODE }.map {|ee|
+            xml_to_struct(ee, parent_data_version)
+          }
+        when :struct
+          xml_to_struct(field.children.select {|x|
+              x.node_type == XML::Node::ELEMENT_NODE
+            }[0], parent_data_version)
+        when :byte, :char, :word, :short, :dword, :int,
+            :dword64, :int64
+          v.to_i
+        when :float, :double
+          v.to_f
+        when :void, :resref
+          v
+        else
+          raise ArgumentError, "Invalid field type #{type.inspect}. Bug."
+      end
+    )
+
+    f.str_ref = case @format
+      when :nxml
+        field['strRef'] || NWN::Gff::Cexolocstr::DEFAULT_STR_REF
+      when :modpacker
+        [v.to_i].pack("l").unpack("L")[0]
+    end if f.is_a?(NWN::Gff::Cexolocstr)
+
+    f
+  end
+
+public
+
+  def initialize fmt
+    @format = fmt
+  end
+
+  def load io
+    old_encoding = NWN.setting(:out_encoding, 'UTF-8')
+    NWN.log_debug("Ignoring custom out_encoding for xml output, always UTF-8") if
+      old_encoding != 'UTF-8'
+
+    doc = XML::Parser.io(io)
+    root = doc.parse.root
+    ret = case @format
+      when :nxml
+        xml_to_struct(root)
+      when :modpacker
+        struct = root.children.select {|x| x.node_type == XML::Node::ELEMENT_NODE && x.name == 'struct' }[0]
+        xml_to_struct(struct, root['version'])
+      else
+        raise ArgumentError, "Unsupported XML format registered: #{@format.inspect}"
+    end
+
+    NWN.setting(:out_encoding, old_encoding)
+    ret
+  end
+
+  def dump data, io
+    old_encoding = NWN.setting(:out_encoding, 'UTF-8')
+    NWN.log_debug("Ignoring custom out_encoding for xml output, always UTF-8") if
+      old_encoding != 'UTF-8'
+
+    doc = XML::Document.new
+    doc.root = case @format
+      when :nxml
+        struct_to_xml(data)
+      when :modpacker
+        nd = XML::Node.new('gff')
+        nd['type'] = [data.data_type].pack("A4")
+        nd['version'] = [data.data_version].pack("A4")
+        nd << struct_to_xml(data)
+        nd
+      else
+        raise ArgumentError, "Unsupported XML format registered: #{@format.inspect}"
+    end
+    t = doc.to_s
+    io.write(t)
+
+    NWN.setting(:out_encoding, old_encoding)
+    t.size
+  end
+end
+
+NWN::Gff::Handler.register :nxml, /^nxml$/, NWN::Gff::Handler::XML.new(:nxml)
+NWN::Gff::Handler.register :modpacker, nil, NWN::Gff::Handler::XML.new(:modpacker)

data/lib/nwn/yaml_support.rb

@@ -1,7 +1,7 @@
 # This file contains all YAML-specific loading and dumping code.
 require 'yaml'
 
-class NWN::Gff::YAML
+module NWN::Gff::Handler::YAML
   # These field types can never be inlined in YAML.
   NonInlineableFields = [:struct, :list, :cexolocstr]
 
@@ -12,7 +12,9 @@ class NWN::Gff::YAML
     YAML.load(io)
   end
   def self.dump data, io
-    io.puts data.to_yaml
+    d = data.to_yaml
+    io.puts d
+    d.size
   end
 end
 
@@ -46,7 +48,7 @@ end
 
 module NWN::Gff::Struct
   def to_yaml_type
-    "!#{NWN::Gff::YAML::Domain}/struct"
+    "!#{NWN::Gff::Handler::YAML::Domain}/struct"
   end
 
   def to_yaml(opts = {})
@@ -55,7 +57,7 @@ module NWN::Gff::Struct
         # Inline certain structs that are small enough.
         map.style = :inline if self.size <= 1 &&
           self.values.select {|x|
-            NWN::Gff::YAML::NonInlineableFields.index(x['type'])
+            NWN::Gff::Handler::YAML::NonInlineableFields.index(x['type'])
           }.size == 0
 
         map.add('__' + 'data_type', @data_type) if @data_type
@@ -75,7 +77,7 @@ module NWN::Gff::Field
   def to_yaml(opts = {})
     YAML::quick_emit(nil, opts) do |out|
       out.map(taguri, to_yaml_style) do |map|
-        map.style = :inline unless NWN::Gff::YAML::NonInlineableFields.index(self['type'])
+        map.style = :inline unless NWN::Gff::Handler::YAML::NonInlineableFields.index(self['type'])
         map.add('type', self['type'])
         map.add('str_ref', self['str_ref']) if has_str_ref?
         map.add('value', self['value'])
@@ -85,7 +87,7 @@ module NWN::Gff::Field
 end
 
 # This parses the struct and extends all fields with their proper type.
-YAML.add_domain_type(NWN::Gff::YAML::Domain,'struct') {|t,hash|
+YAML.add_domain_type(NWN::Gff::Handler::YAML::Domain,'struct') {|t,hash|
   struct = {}
   struct.extend(NWN::Gff::Struct)
 
@@ -116,4 +118,4 @@ YAML.add_domain_type(NWN::Gff::YAML::Domain,'struct') {|t,hash|
   struct
 }
 
-NWN::Gff.register_format_handler :yaml, /^(y|yml|yaml)$/, NWN::Gff::YAML
+NWN::Gff::Handler.register :yaml, /^(y|yml|yaml)$/, NWN::Gff::Handler::YAML

data/spec/gff_spec.rb

@@ -18,6 +18,8 @@ describe "Gff.read/write API" do
   {
     :gff => %w{utc utd ute uti utm utp uts utt utw git are gic mod ifo fac ssf dlg itp bic},
     :yaml => %w{yml yaml},
+    :json => %w{json},
+    :nxml => %w{nxml},
     :kivinen => %w{k kivinen},
     :marshal => %w{marshal}
   }.each {|expect, arr|
@@ -46,21 +48,28 @@ describe "Gff::*" do
     t2.should == t
   end
 
-  it "writes to io and returns the number of written bytes" do
-    t = Gff::Reader.read(StringIO.new WELLFORMED_GFF)
-    out = StringIO.new
-    v = Gff::Writer.dump(t, out)
-    v.should == out.size
-    out.seek(0)
-    v = out.read(v)
-    t2 = wellformed_verify v
-    t2.should == t
+  NWN::Gff::OutputFormats.keys.each do |fmt|
+    it "#{fmt} writes to io and returns the number of written bytes" do
+      t = Gff::Reader.read(StringIO.new WELLFORMED_GFF)
+      out = StringIO.new
+      v = Gff.write(out, fmt, t)
+      v.should == out.pos
+    end
   end
 
-  it "fails on not enough data" do
-    proc {
-      wellformed_verify WELLFORMED_GFF[0 .. -2]
-    }.should raise_error IOError
+  (NWN::Gff::OutputFormats.keys & NWN::Gff::InputFormats.keys).each do |fmt|
+    it "#{fmt} fails on not enough data" do
+      proc {
+        gff = Gff::Reader.read(StringIO.new WELLFORMED_GFF)
+        out = StringIO.new
+        Gff.write(out, fmt, gff)
+        size = out.pos
+        out.seek(0)
+        out.truncate(size - 20)
+        Gff.read(out, fmt)
+      }.should raise_error
+    end
+
   end
 
 end

data/spec/kivinen_spec.rb

@@ -8,7 +8,7 @@ describe "Kivinen Support" do
 
     expected = KIVINEN_EXPECT.dup
 
-    NWN::Gff::Kivinen.format(g, true) do |label, entry|
+    NWN::Gff::Handler::Kivinen.format(g, true) do |label, entry|
       w = expected.shift
       label.should == w[0]
       case entry

data/spec/spec_helper.rb

@@ -10,6 +10,8 @@ unless Object.const_defined?('NWN')
   include NWN
 end
 
+$options = {}
+
 NWN.setting(:debug, 0)
 
 GffFieldValidations = {

data/spec/struct_data_type_version_spec.rb

@@ -0,0 +1,82 @@
+require File.join(File.dirname(__FILE__), 'spec_helper')
+
+describe "Gff::Struct" do
+  ReadWriteFormats = (NWN::Gff::OutputFormats.keys & NWN::Gff::InputFormats.keys)
+  NoRootNoMetadata = [:gff]
+
+  before(:each) do
+    @manual = Gff::Struct.new(0x0, 'root', 'V3.2') do |s|
+      s.add_struct 'a', Gff::Struct.new(0x1) do |a|
+        a.v.add_struct 'b', Gff::Struct.new(0x02) do |b|
+          b.v.add_field 'hi', :int, 1
+        end
+      end
+    end
+  end
+
+  def read_write what, format
+    out = StringIO.new
+    Gff.write(out, format, what)
+    out.seek(0)
+    n = Gff.read(out, format)
+  end
+
+  it "has the proper data structure when building inline" do
+    @manual.data_type.should == 'root'
+    @manual.data_version.should == 'V3.2'
+    (@manual / 'a').path.should == '/a'
+    (@manual / 'a').v.path.should == '/a'
+    (@manual / 'a$').path.should == '/a'
+    (@manual / 'a/b$').path.should == '/a/b'
+    (@manual / 'a/b/hi').path.should == '/a/b/hi'
+    (@manual / 'a/b/hi$').should == 1
+  end
+
+  describe "#data_type" do
+    ReadWriteFormats.each do |format|
+      it "#{format} keeps explicit root data types" do
+        @manual.data_type = 'EXPL'
+        n = read_write(@manual, format)
+        n.data_type.should == 'EXPL'
+      end
+    end
+
+    (ReadWriteFormats - NoRootNoMetadata).each do |format|
+      it "#{format} keeps explicit non-root data types" do
+        (@manual / 'a/b$').data_type = 'EXPL'
+        n = read_write(@manual, format)
+        (n / 'a/b$').data_type.should == 'EXPL'
+      end
+    end
+  end
+
+  describe "#data_version" do
+    ReadWriteFormats.each do |format|
+      it "#{format} keeps explicit root data version" do
+        @manual.data_version = 'EXPL'
+        n = read_write(@manual, format)
+        n.data_version.should == 'EXPL'
+      end
+    end
+
+    (ReadWriteFormats - NoRootNoMetadata).each do |format|
+      it "#{format} keeps explicit non-root data version" do
+        (@manual / 'a/b$').data_version = 'EXPL'
+        n = read_write(@manual, format)
+        (n / 'a/b$').data_version.should == 'EXPL'
+      end
+    end
+
+    ReadWriteFormats.each do |format|
+      it "#{format} reads implicit data_version as DEFAULT_DATA_VERSION" do
+        n = read_write(@manual, format)
+
+        (@manual / 'a/b$').data_version.should == NWN::Gff::Struct::DEFAULT_DATA_VERSION
+        @manual.data_version.should == NWN::Gff::Struct::DEFAULT_DATA_VERSION
+        (n / 'a/b$').data_version.should == NWN::Gff::Struct::DEFAULT_DATA_VERSION
+        n.data_version.should == NWN::Gff::Struct::DEFAULT_DATA_VERSION
+      end
+    end
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nwn-lib
 version: !ruby/object:Gem::Version 
-  version: 0.4.11
+  version: 0.4.12
 platform: ruby
 authors: 
 - Bernhard Stoeckner
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-11-20 00:00:00 +01:00
+date: 2010-01-15 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -36,54 +36,55 @@ files:
 - CHANGELOG
 - README
 - Rakefile
-- bin/nwn-gff
 - bin/nwn-dsl
-- bin/nwn-erf
+- bin/nwn-gff
 - bin/nwn-irb
-- spec/bin_dsl_spec.rb
+- bin/nwn-erf
+- spec/kivinen_expect.rb
+- spec/wellformed_gff.bic
 - spec/gff_spec.rb
+- spec/struct_data_type_version_spec.rb
+- spec/res_spec.rb
+- spec/erf_spec.rb
 - spec/key_spec.rb
-- spec/bin_gff_spec.rb
-- spec/bin_erf_spec.rb
-- spec/kivinen_expect.rb
+- spec/kivinen_spec.rb
 - spec/field_spec.rb
 - spec/cexolocstr_spec.rb
-- spec/kivinen_spec.rb
-- spec/erf_spec.rb
 - spec/struct_spec.rb
-- spec/res_spec.rb
+- spec/spec_helper.rb
 - spec/tlk_spec.rb
 - spec/json_spec.rb
+- spec/bin_dsl_spec.rb
+- spec/bin_gff_spec.rb
+- spec/bin_erf_spec.rb
 - spec/twoda_spec.rb
-- spec/spec_helper.rb
-- spec/data_type_spec.rb
-- spec/wellformed_gff.bic
-- lib/nwn/erf.rb
-- lib/nwn/json_support.rb
-- lib/nwn/io.rb
-- lib/nwn/settings.rb
+- lib/nwn/scripting.rb
 - lib/nwn/key.rb
-- lib/nwn/all.rb
-- lib/nwn/res.rb
 - lib/nwn/twoda.rb
-- lib/nwn/tlk.rb
+- lib/nwn/json_support.rb
+- lib/nwn/erf.rb
+- lib/nwn/kivinen_support.rb
+- lib/nwn/settings.rb
 - lib/nwn/gff.rb
 - lib/nwn/yaml_support.rb
-- lib/nwn/scripting.rb
-- lib/nwn/kivinen_support.rb
-- lib/nwn/gff/writer.rb
-- lib/nwn/gff/struct.rb
-- lib/nwn/gff/list.rb
+- lib/nwn/xml_support.rb
+- lib/nwn/tlk.rb
+- lib/nwn/io.rb
+- lib/nwn/res.rb
+- lib/nwn/all.rb
+- lib/nwn/gff/field.rb
 - lib/nwn/gff/reader.rb
 - lib/nwn/gff/cexolocstr.rb
-- lib/nwn/gff/field.rb
+- lib/nwn/gff/struct.rb
+- lib/nwn/gff/list.rb
+- lib/nwn/gff/writer.rb
 - tools/verify.sh
 - tools/migrate_03x_to_04x.sh
 - scripts/truncate_floats.rb
+- scripts/reformat_2da
 - scripts/clean_locstrs.rb
-- scripts/debug_check_objid.rb
 - scripts/extract_all_items.rb
-- scripts/reformat_2da
+- scripts/debug_check_objid.rb
 - BINARIES
 - HOWTO
 - SCRIPTING

data/spec/data_type_spec.rb

@@ -1,128 +0,0 @@
-require File.join(File.dirname(__FILE__), 'spec_helper')
-
-describe "Gff::Struct#data_type" do
-
-  before(:each) do
-    @manual = Gff::Struct.new(0x0, 'root', 'V3.1') do |s|
-      s.add_struct 'a', Gff::Struct.new(0x1) do |a|
-        a.v.add_struct 'b', Gff::Struct.new(0x02) do |b|
-          b.v.add_field 'hi', :int, 1
-        end
-      end
-    end
-  end
-
-  it "has the proper data types when building inline" do
-    verify @manual
-  end
-
-unless Gff::InputFormats[:json] && Gff::OutputFormats[:json]
-  $stderr.puts "Partial or no json support, not running json specs!"
-else
-
-  it "keeps explicit data types for json" do
-    @manual['a'].v['b'].v.data_type = 'EXPLICIT'
-    json = StringIO.new
-    Gff.write(json, :json, @manual)
-    json.seek(0)
-    n = Gff.read(json, :json)
-    n['a'].v.path.should == '/a'
-    n['a'].v.data_type.should == nil
-    n['a'].v['b'].v.data_type.should == 'EXPLICIT'
-  end
-
-  it "has the proper data type when reading from json" do
-    struct = <<EOS
-{
-  "a": {
-    "type": "struct",
-    "value": {
-      "b": {
-        "type": "struct",
-        "value": {
-          "hi": {
-            "type": "int",
-            "value": 1
-          },
-          "__struct_id": 2
-        }
-      },
-      "__struct_id": 1
-    }
-  },
-  "__data_version": "V3.1",
-  "__data_type": "root",
-  "__struct_id": 0
-}
-EOS
-
-    struct = Gff.read(StringIO.new(struct), :json)
-
-    verify struct
-  end
-
-end
-
-  it "keeps explicit data types for yaml" do
-    (@manual / 'a/b').v.data_type = 'EXPLICIT'
-    json = StringIO.new
-    Gff.write(json, :yaml, @manual)
-    json.seek(0)
-    n = Gff.read(json, :yaml)
-    (n / 'a/b/hi').path.should == "/a/b/hi"
-    (n / 'a$').data_type.should == nil
-    (n / 'a/b$').data_type.should == "EXPLICIT"
-  end
-
-  it "has the proper data type when reading from yaml" do
-    struct = <<EOS
---- !nwn-lib.elv.es,2008-12/struct
-__data_type: root
-__data_version: V3.1
-__struct_id: 0
-a:
-  type: :struct
-  value: !nwn-lib.elv.es,2008-12/struct
-    __data_type: ATYPE
-    __struct_id: 1
-    b:
-      type: :struct
-      value: !nwn-lib.elv.es,2008-12/struct {__data_type: BTYPE, __struct_id: 2, hi: {type: :int, value: 1}}
-EOS
-    struct = Gff.read(StringIO.new(struct), :yaml)
-
-    verify struct
-  end
-
-  it "has the proper data type when reading from yaml with overriden data_type" do
-    struct = <<EOS
---- !nwn-lib.elv.es,2008-12/struct
-__data_type: root
-__data_version: V3.1
-__struct_id: 0
-a:
-  type: :struct
-  value: !nwn-lib.elv.es,2008-12/struct
-    __data_type: DTYPE
-    __struct_id: 1
-    b:
-      type: :struct
-      value: !nwn-lib.elv.es,2008-12/struct {__data_type: BTYPE, __struct_id: 2, hi: {type: :int, value: 1}}
-EOS
-    struct = Gff.read(StringIO.new(struct), :yaml)
-
-    (struct / 'a$').data_type.should == 'DTYPE'
-    (struct / 'a$').path.should == '/a'
-  end
-
-  def verify struct
-    struct.data_type.should == 'root'
-    struct.data_version.should == 'V3.1'
-    (struct / 'a').path.should == '/a'
-    (struct / 'a').v.path.should == '/a'
-    (struct / 'a$').path.should == '/a'
-    (struct / 'a/b$').path.should == '/a/b'
-    (struct / 'a/b/hi').path.should == '/a/b/hi'
-    (struct / 'a/b/hi$').should == 1
-  end
-end