nwn-lib 0.4.6 → 0.4.7

data/CHANGELOG

@@ -203,3 +203,38 @@ Bernhard Stoeckner <elven@swordcoast.net> (26):
       to the newly-added specs.
 
       Please browse the commit log for a full list of changes.
+
+Bernhard Stoeckner <elven@swordcoast.net> (25):
+      bin/*: do not require rubygems explicitly
+      remove obsolete Gff::Helpers module
+      Erf: add support for ERF V1.1
+      env var NWN_LIB_DEBUG sends debug messages to stderr when set
+      add support for Microsoft Windows
+      R::ContentObject: do not fail on missing io, seek properly
+      R::ContentObject#filename: "unknown-N" instead of nil for unknown res_types
+      R::ContentObject: properly raise ENOENT on invalid io and file
+      G::Reader, Writer: work on io objects instead of strings
+      G::Writer: fix struct verify exception on reading substructs
+      specs: move all test stubs into _helper
+      Erf: improve broken locstr header handling
+      Erf: raise errors for invalid filenames on add, add_file
+      Erf: fix misaligned offsets in reslist
+      nwn-erf: fix parsing erf files where day_of_year = 0
+      nwn-erf: output non-errors to stdout instead of stderr
+      nwn-erf: fix packing up files
+      nwn-erf: add specs
+      Erf: reproduction spec checks for binary equality with sample
+      Gff: add specs for each_by_flat_path, to_gff
+      nwn-gff: fix backup strategy handler missing FileUtils
+      nwn-gff: add specs
+      rename spec/wellformed_gff.binary to .bic
+      nwn-dsl: add specs
+      0.4.7-rel
+
+      A real bugfix-release, this takes care of a whole swad of fixes
+      in erf and gff handling, and also improves Windows compatibility.
+
+      Particularily, erf writing was kind of broken in the previous release,
+      putting wrong offsets for files created with nwn-erf.
+
+      Please browse the commit log for a full list of changes.

data/Rakefile

@@ -9,7 +9,7 @@ include FileUtils
 # Configuration
 ##############################################################################
 NAME = "nwn-lib"
-VERS = "0.4.6"
+VERS = "0.4.7"
 CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
 RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
   'nwn-lib: a ruby library for accessing NWN resource files', \

data/bin/nwn-erf

@@ -1,5 +1,4 @@
 #!/usr/bin/ruby
-require 'rubygems'
 require 'optparse'
 require 'nwn/all'
 require 'tempfile'
@@ -9,6 +8,7 @@ $verbose = false
 $type = 'ERF'
 $allow_duplicates = false
 $descriptions = {}
+$version = "V1.0"
 
 # tar-compat mode: first argument is options if no dash is specified.
 ARGV[0] = "-" + ARGV[0] if ARGV.size > 0 && ARGV[0][0] != ?-
@@ -63,6 +63,13 @@ begin OptionParser.new do |o|
     $type = 'MOD'
   end
 
+  o.on "-0", "Create (only -c) V1.0 ERF, 16 byte resrefs. (NWN1, default)" do
+    $version = "V1.0"
+  end
+  o.on "-1", "Create (only -c) V1.0 ERF, 32 byte resrefs. (NWN2)." do
+    $version = "V1.1"
+  end
+
   o.separator " "
   o.separator "Hacks:"
 
@@ -91,15 +98,15 @@ def input filename = nil
   if $file.nil?
     yield $stdin
   else
-    File.open(filename || $file, "r") {|f| yield f}
+    File.open(filename || $file, "rb") {|f| yield f}
   end
 end
 
-def output mode = "w", filename = nil
+def output filename = nil
   if $file.nil?
     yield $stdout
   else
-    File.open(filename || $file, mode) {|f| yield f}
+    File.open(filename || $file, "wb") {|f| yield f}
   end
 end
 
@@ -107,14 +114,14 @@ case $action
   when :t
     input {|f|
       erf = NWN::Erf::Erf.new(f)
-      $stderr.puts "# %14s %16s %-10s %-4s %s" % %w{offset size date type filename} if $verbose
+      puts "# %14s %16s %-10s %-4s %s" % %w{offset size date type filename} if $verbose
       erf.content.each {|c|
         if !$verbose
-          $stderr.puts "%s" % [c.filename]
+          puts "%s" % [c.filename]
         else
-          $stderr.puts "%16d %16d %10s %4d %s" % [
+          puts "%16d %16d %10s %4d %s" % [
             c.offset, c.size,
-            Date.ordinal(1900 + erf.year, erf.day_of_year).strftime("%Y-%m-%d"),
+            Date.ordinal(1900 + erf.year, 1 + erf.day_of_year).strftime("%Y-%m-%d"),
             c.res_type, c.filename
           ]
         end
@@ -125,8 +132,8 @@ case $action
     input {|f|
       erf = NWN::Erf::Erf.new(f)
       erf.content.each {|c|
-        $stderr.puts "%s" % [c.filename] if $verbose
-        File.open(c.filename, "w") {|ff|
+        puts "%s" % [c.filename] if $verbose
+        output(c.filename) {|ff|
           ff.write(c.get)
         }
       }
@@ -135,6 +142,7 @@ case $action
   when :c
     erf = NWN::Erf::Erf.new
     erf.file_type = $type if $type
+    erf.file_version = $version
 
     if $descriptions
       erf.localized_strings.merge! $descriptions
@@ -142,10 +150,10 @@ case $action
     end
 
     ARGV.each {|a|
-      $stderr.puts File.basename(a) if $verbose
+      puts File.basename(a) if $verbose
       raise ArgumentError, "#{File.basename(a)} already present in erf." if
         !$allow_duplicates && erf.has?(a)
-      erf.add a
+      erf.add_file a
     }
     output {|f| erf.write_to(f) }
 
@@ -163,7 +171,7 @@ case $action
           when :a
             raise ArgumentError, "#{File.basename(arg)} already present in erf." if
               !$allow_duplicates && erf.has?(arg)
-            erf.add arg
+            erf.add_file arg
 
           when :r
             erf.content.reject! {|con|
@@ -173,7 +181,7 @@ case $action
       }
 
       tempfile = Tempfile.new("nwn-erf", File.dirname($file || "."))
-      output("w", tempfile.path) do |f|
+      output(tempfile.path) do |f|
         erf.write_to(f)
       end
 

data/bin/nwn-gff

@@ -1,7 +1,7 @@
 #!/usr/bin/ruby
-require 'rubygems'
 require 'optparse'
 require 'nwn/all'
+require 'fileutils'
 
 Thread.abort_on_exception = true
 
@@ -116,7 +116,7 @@ elsif :in == $options[:outformat]
 end
 
 vputs "Reading: #{$options[:infile]}"
-data_in = $options[:infile] == '-' ? $stdin.read : IO.read($options[:infile])
+data_in = $options[:infile] == '-' ? $stdin.read : File.open($options[:infile], "rb")
 data_in = NWN::Gff.read(data_in, $options[:informat])
 
 # verify that we read a GFF struct
@@ -193,6 +193,6 @@ if $options[:backup] && FileTest.exists?($options[:outfile])
   FileUtils.cp($options[:outfile], bdest) if bdest
 end
 
-write_to = $options[:outfile] == '-' ? $stdout : File.open($options[:outfile], 'w')
+write_to = $options[:outfile] == '-' ? $stdout : File.open($options[:outfile], 'wb')
 
 NWN::Gff.write(write_to, $options[:outformat], data_in)

data/bin/nwn-irb

@@ -1,5 +1,4 @@
 #!/usr/bin/ruby
-require 'rubygems'
 require 'optparse'
 require 'nwn/all'
 require 'irb'
@@ -19,7 +18,7 @@ def save destination = nil
 
   File.expand_path(destination ||= $file)
   $stderr.puts "Saving to `#{destination}' .."
-  File.open(destination, "w") {|d|
+  File.open(destination, "wb") {|d|
     NWN::Gff.write(d, NWN::Gff.guess_file_format(destination), $gff)
   }
   $stderr.puts "saved."

data/lib/nwn/all.rb

@@ -6,5 +6,4 @@ require 'nwn/erf'
 require 'nwn/yaml'
 require 'nwn/kivinen'
 require 'nwn/scripting'
-require 'nwn/helpers'
 require 'nwn/settings'

data/lib/nwn/erf.rb

@@ -28,8 +28,34 @@ module NWN
         read_from io if io
       end
 
+      def add_file filename, io = nil
+        fnlen = filename_length @file_version
+        raise ArgumentError, "Invalid filename: #{filename.inspect}" if
+          filename.size == 0 || filename.size > (fnlen + 4)
+        super(filename, io)
+      end
+
+      def add o
+        fnlen = filename_length @file_version
+        raise ArgumentError, "Invalid filename: #{o.filename.inspect}" if
+          o.resref.size == 0 || o.resref.size > fnlen
+        super(o)
+      end
+
     private
 
+      def filename_length version
+        case version
+          when "V1.0"
+            16
+          when "V1.1"
+            32
+        else
+          raise IOError, "Invalid ERF version: #{version}"
+        end
+      end
+
+
       def read_from io
         header = @io.read(160)
         raise IOError, "Cannot read header: Not a erf file?" unless
@@ -46,34 +72,44 @@ module NWN
         raise IOError, "Cannot read erf stream: invalid type #{@file_type.inspect}" unless
           NWN::Erf::ValidTypes.index(@file_type)
 
-        if @file_version == "V1.0"
-          @filename_length = 16
-        else
-          raise IOError, "Invalid erf version: #{@file_version}"
-        end
+        fnlen = filename_length @file_version
 
         @io.seek(offset_to_locstr)
         locstr = @io.read(locstr_size)
 
         for lstr in 0...locstr_count do
-          raise IOError, "locstr table does not contain enough entries or locstr_size is too small" if
-            locstr.nil? || locstr.size < 8
+          if locstr.nil? || locstr.size == 0
+            NWN.log_debug "locstr table: not enough entries (expected: #{locstr_count}, got: #{lstr})"
+            break
+          end
+
+          if locstr.size < 8
+            NWN.log_debug "locstr table: not enough entries (expected: #{locstr_count}, got: #{lstr})" +
+              " partial data: #{locstr.inspect}"
+            break
+          end
 
           lid, strsz = locstr.unpack("V V")
+          if strsz > locstr.size - 8
+            NWN.log_debug "locstr table: given strsz is bigger than available data, truncating"
+            strsz = locstr.size - 8
+          end
           str = locstr.unpack("x8 a#{strsz}")[0]
-          raise IOError,
-            "Expected locstr size does not match actual string size (want: #{strsz}, got #{str.size} of #{str.inspect})" if
-              strsz != str.size
+
+          # This just means that the given locstr size was encoded wrongly -
+          # the old erf.exe is known to do that.
+          NWN.log_debug "Expected locstr size does not match actual " +
+            "string size (want: #{strsz}, got #{str.size} of #{str.inspect})" if strsz != str.size
 
           @localized_strings[lid] = str
           locstr = locstr[8 + str.size .. -1]
         end
 
-        keylist_entry_size = @filename_length + 4 + 2 + 2
+        keylist_entry_size = fnlen + 4 + 2 + 2
         @io.seek(offset_to_keys)
         keylist = @io.read(keylist_entry_size * entry_count)
         raise IOError, "keylist too short" if keylist.size != keylist_entry_size * entry_count
-        keylist = keylist.unpack("A16 V v v" * entry_count)
+        keylist = keylist.unpack("A#{fnlen} V v v" * entry_count)
         keylist.each_slice(4) {|resref, res_id, res_type, unused|
           @content << NWN::Resources::ContentObject.new(resref, res_type, @io)
         }
@@ -95,9 +131,21 @@ module NWN
 
       # Writes this Erf to a io stream.
       def write_to io
+        fnlen = filename_length @file_version
+
         locstr = @localized_strings.map {|x| [x[0], x[1].size, x[1]].pack("V V a*") }.join("")
-        keylist = @content.map {|c| [c.resref, @content.index(c), c.res_type, 0].pack("a16 V v v") }.join("")
-        reslist = @content.map {|c| [c.offset, c.size].pack("V V") }.join("")
+        keylist = @content.map {|c|
+          NWN.log_debug "truncating filename #{c.resref}, longer than #{fnlen}" if c.resref.size > fnlen
+          [c.resref, @content.index(c), c.res_type, 0].pack("a#{fnlen} V v v")
+        }.join("")
+
+        pre_offset = 160 + locstr.size + keylist.size + 8 * @content.size
+
+        reslist = @content.map {|c|
+          offset = pre_offset +
+            @content[0, @content.index(c)].inject(0) {|sum,x| sum + x.size }
+          [offset, c.size].pack("V V")
+        }.join("")
 
         offset_to_locstr = 160
         offset_to_keylist = offset_to_locstr + locstr.size

data/lib/nwn/gff.rb

@@ -94,7 +94,7 @@ module NWN
     def self.write(io, format, data)
       case format
         when :gff
-          io.print NWN::Gff::Writer.dump(data)
+          NWN::Gff::Writer.dump(data, io)
         when :yaml
           io.puts data.to_yaml
         when :marshal

data/lib/nwn/gff/reader.rb

@@ -4,22 +4,26 @@ class NWN::Gff::Reader
 
   attr_reader :root_struct
 
-  # Create a new Reader with the given +bytes+ and immediately parse it.
-  # This is not needed usually; use Reader.read instead.
-  def initialize bytes
-    @bytes = bytes
+  private_class_method :new
 
-    read_all
+  # Create a new Reader with the given +io+ and immediately parse it.
+  def self.read io
+    t = new(io)
+    t.root_struct
   end
 
-  # Reads +bytes+ as gff data and returns a NWN::Gff:Gff object.
-  def self.read bytes
-    self.new(bytes).root_struct
+  def initialize io #:nodoc:
+    @io = io
+    read_all
   end
 
   private
 
   def read_all
+    header = @io.read(160)
+    raise IOError, "Cannot read header" unless header &&
+      header.size == 160
+
     type, version,
     struct_offset, struct_count,
     field_offset, field_count,
@@ -27,9 +31,9 @@ class NWN::Gff::Reader
     field_data_offset, field_data_count,
     field_indices_offset, field_indices_count,
     list_indices_offset, list_indices_count =
-      @bytes.unpack("a4a4 VV VV VV VV VV VV")
+      header.unpack("a4a4 VV VV VV VV VV VV")
 
-    raise GffError, "Unknown version #{@version}; not a gff?" unless
+    raise GffError, "Unknown version #{version}; not a gff?" unless
       version == "V3.2"
 
     raise GffError, "struct offset at wrong place, not a gff?" unless
@@ -39,12 +43,35 @@ class NWN::Gff::Reader
     field_len  = field_count * 16
     label_len  = label_count * 16
 
-    @structs = @bytes[struct_offset, struct_len].unpack("V*")
-    @fields  = @bytes[field_offset, field_len].unpack("V*")
-    @labels  = @bytes[label_offset, label_len].unpack("A16" * label_count)
-    @field_data = @bytes[field_data_offset, field_data_count]
-    @field_indices = @bytes[field_indices_offset, field_indices_count].unpack("V*")
-    @list_indices = @bytes[list_indices_offset, list_indices_count].unpack("V*")
+    @io.seek(struct_offset)
+    @structs = @io.read(struct_len)
+    raise IOError, "cannot read structs" unless @structs && @structs.size == struct_len
+    @structs = @structs.unpack("V*")
+
+    @io.seek(field_offset)
+    @fields  = @io.read(field_len)
+    raise IOError, "cannot read fields" unless @fields && @fields.size == field_len
+    @fields  = @fields.unpack("V*")
+
+    @io.seek(label_offset)
+    @labels  = @io.read(label_len)
+    raise IOError, "cannot read labels" unless @labels && @labels.size == label_len
+    @labels = @labels.unpack("A16" * label_count)
+
+    @io.seek(field_data_offset)
+    @field_data = @io.read(field_data_count)
+    raise IOError, "cannot read field_data" unless @field_data && @field_data.size == field_data_count
+
+    @io.seek(field_indices_offset)
+    @field_indices = @io.read(field_indices_count)
+    raise IOError, "cannot read field_indices" unless @field_indices && @field_indices.size == field_indices_count
+    @field_indices = @field_indices.unpack("V*")
+
+    @io.seek(list_indices_offset)
+    @list_indices = @io.read(list_indices_count)
+    raise IOError, "cannot read list_indices" unless @list_indices && @list_indices.size == list_indices_count
+    @list_indices = @list_indices.unpack("V*")
+
     @root_struct = read_struct 0, type.strip, version
   end
 

data/lib/nwn/gff/writer.rb

@@ -1,10 +1,25 @@
 class NWN::Gff::Writer
-  include NWN
   include NWN::Gff
 
+  private_class_method :new
+
   attr_reader :bytes
 
-  def initialize(gff, data_type = nil)
+  # Takes a NWN::Gff::Gff object and dumps it to +io+,
+  # including the header.
+  # If +io+ is nil, return the raw bytes, otherwise
+  # the number of bytes written.
+  def self.dump(gff, io = nil, data_type = nil)
+    ret = new(gff, data_type).bytes
+    if io
+      io.write(ret)
+      ret.size
+    else
+      ret
+    end
+  end
+
+  def initialize(gff, data_type = nil) #:nodoc:
     @gff = gff
     @data_type = data_type
 
@@ -18,12 +33,6 @@ class NWN::Gff::Writer
     write_all
   end
 
-  # Takes a NWN::Gff::Gff object and dumps it as raw bytes,
-  # including the header.
-  def self.dump(gff, data_type = nil)
-    self.new(gff, data_type).bytes
-  end
-
 private
 
   def get_label_id_for_label str
@@ -127,7 +136,7 @@ private
 
         when :struct
           raise GffError, "type = struct, but value not a hash" unless
-            v.field_value.is_a?(Gff::Struct)
+            v.field_value.is_a?(Struct)
 
           fields_of_this_struct << add_data_field(v.field_type, k, write_struct(v.field_value))