nwn-lib 0.4.1 → 0.4.2

data/CHANGELOG

@@ -106,13 +106,27 @@ Bernhard Stoeckner <elven@swordcoast.net> (5):
 Stuart Coyle <stuart.coyle@gmail.com> (1):
       TwoDA: use four spaces for field separation instead of tabs, indent columns
 
-==== 0.4.0
+=== 0.4.0
 Bernhard Stoeckner <elven@swordcoast.net>:
       too many to sanely list, see README for migration information
 
-==== 0.4.1
+=== 0.4.1
 Bernhard Stoeckner <elven@swordcoast.net> (3):
       Field#field_value=: unbreak setting a new value
       Scripting: add rudimentary ask method
       scripts/clean_locstrs.rb: optimise a bit
       0.4.1-rel
+
+=== 0.4.2
+Bernhard Stoeckner <elven@swordcoast.net> (10):
+      scripts/fix_facings.rb: removed, results in corruption
+      bin/nwn-dsl: cleanup exception handling
+      TwoDA::Table: add support for non-windows newlines
+      TwoDA::Table: quote cells with whitespaces
+      TwoDA::Table: a more robust parser handling slightly malformed 2da files
+      scripts: add reformat_2da for validating and prettyprinting 2da files
+      Tlk: basic talktable reading
+      scripts: add a debug script
+      add *.itp to GuessFormats
+      Tlk: basic talktable editing and writing
+      0.4.2-rel

data/Rakefile

@@ -9,7 +9,7 @@ include FileUtils
 # Configuration
 ##############################################################################
 NAME = "nwn-lib"
-VERS = "0.4.1"
+VERS = "0.4.2"
 CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
 RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
   'nwn-lib: a ruby library for accessing NWN resource files', \

data/SETTINGS

@@ -78,3 +78,9 @@ The default is to keep them as-is.
 Set to a path containing all 2da files to initialize the 2da cache. This is needed for
 most interactive helpers and a few type infer gizmos.
 
+== NWN_LIB_TWODA_NEWLINE
+
+Specify the type of newline that Table#to_2da uses to join rows.
+ 0 for windows newlines: \r\n (default)
+ 1 for unix newlines: \n
+ 2 for caret return only: \r

data/bin/nwn-dsl

@@ -18,8 +18,7 @@ $base_script = ARGV.shift
 begin
   NWN::Gff::Scripting.run_script(IO.read($base_script), nil, ARGV)
 rescue => e
-  message = e.message.split(':', 4)[3].strip
-  $stderr.puts message
+  $stderr.puts e.message
   if $backtrace
     $stderr.puts ""
     $stderr.puts "  " + e.backtrace.join("\n")

data/lib/nwn/gff.rb

@@ -64,7 +64,7 @@ module NWN
     FileFormatGuesses = {
       /^ut[cdeimpstw]$/ => :gff,
       /^(git|are|gic)$/ => :gff,
-      /^(mod|ifo|fac|ssf|dlg)$/ => :gff,
+      /^(mod|ifo|fac|ssf|dlg|itp)$/ => :gff,
       /^(bic)$/ => :gff,
       /^ya?ml$/ => :yaml,
       /^marshal$/ => :marshal,

data/lib/nwn/tlk.rb

@@ -0,0 +1,175 @@
+module NWN
+  module Tlk
+    Languages = {
+      0 => :english,
+      1 => :french,
+      2 => :german,
+      3 => :italian,
+      4 => :spanish,
+      5 => :polish,
+      128 => :korean,
+      129 => :chinese_traditional,
+      130 => :chinese_simplified,
+      131 => :japanese,
+    }.freeze
+
+    ValidGender = [:male, :female].freeze
+
+    # Tlk wraps a File object that points to a .tlk file.
+    class Tlk
+      HEADER_SIZE = 20
+      DATA_ELEMENT_SIZE = 4 + 16 + 4 + 4 + 4 + 4 + 4
+
+      # The number of strings this Tlk holds.
+      # attr_reader :size
+
+      # The language_id of this Tlk.
+      attr_reader :language
+
+      attr_reader :cache
+
+      # Cereate
+      def initialize io
+        @io = io
+
+        # Read the header
+        @file_type, @file_version, language_id,
+          string_count, string_entries_offset =
+            @io.read(HEADER_SIZE).unpack("A4 A4 I I I")
+
+        raise IOError, "The given IO does not describe a valid tlk table" unless
+          @file_type == "TLK" && @file_version == "V3.0"
+
+        @size = string_count
+        @language = language_id
+        @entries_offset = string_entries_offset
+
+        @cache = {}
+      end
+
+      # Returns a TLK entry as a hash with the following keys:
+      #  :text          string: The text
+      #  :sound         string: A sound resref, or "" if no sound is specified.
+      #  :sound_length  float: Length of the given resref (or 0.0 if no sound is given).
+      #
+      # id is the numeric offset within the Tlk, starting at 0.
+      # The maximum is Tlk#size - 1.
+      def [](id)
+        return { :text => "", :sound => "", :sound_length => 0.0, :volume_variance => 0, :pitch_variance => 0} if id == 0xffffffff
+
+        return @cache[id] if @cache[id]
+
+        raise ArgumentError, "No such string ID: #{id.inspect}" if id >= self.highest_id || id < 0
+        seek_to = HEADER_SIZE + (id) * DATA_ELEMENT_SIZE
+        @io.seek(seek_to)
+        data = @io.read(DATA_ELEMENT_SIZE)
+
+        raise IOError, "Cannot read TLK file, missing string header data." if !data || data.size != 40
+
+        flags, sound_resref, v_variance, p_variance, offset,
+          size, sound_length = data.unpack("I A16 I I I I f")
+        flags = flags.to_i
+
+        @io.seek(@entries_offset + offset)
+        text = @io.read(size)
+
+        raise IOError, "Cannot read TLK file, missing string text data." if !text || text.size != size
+
+        text = flags & 0x1 > 0 ? text : ""
+        sound = flags & 0x2 > 0 ? sound_resref : ""
+        sound_length = flags & 0x4 > 0 ? sound_length.to_f : 0.0
+
+        @cache[id] = {
+          :text => text, :sound => sound, :sound_length => sound_length,
+          :volume_variance => v_variance, :pitch_variance => p_variance
+        }
+      end
+
+      # Add a new entry to this Tlk and return the strref given to it.
+      # To override existing entries, use tlk[][:text] = ".."
+      def add text, sound = "", sound_length = 0.0, v_variance = 0, p_variance = 0
+        next_id = self.highest_id + 1
+        $stderr.puts "put in cache: #{next_id}"
+        @cache[next_id] = {:text => text, :sound => sound, :sound_length => 0.0, :volume_variance => v_variance, :pitch_variance => p_variance}
+        next_id
+      end
+
+      # Return the highest ID in this Tlk table.
+      def highest_id
+        highest_cached = @cache.keys.sort[-1] || 0
+        @size - 1 > highest_cached ? @size - 1 : highest_cached
+      end
+
+      # Write this Tlk to +io+.
+      # Take care not to write it to the same IO object you are reading from.
+      def write_to io
+        text_block = []
+        offsets = []
+        offset = 0
+        for i in 0..self.highest_id do
+          entry = self[i]
+          offsets[i] = offset
+          text_block << entry[:text]
+          offset += entry[:text].size
+        end
+        text_block = text_block.join("")
+
+        header = [
+          @file_type, @file_version,
+          @language,
+          self.highest_id + 1, HEADER_SIZE + (self.highest_id + 1) * DATA_ELEMENT_SIZE
+        ].pack("A4 A4 I I I")
+
+        entries = []
+        for i in 0..self.highest_id do
+          entry = self[i]
+          text, sound, sound_length = entry[:text], entry[:sound], entry[:sound_length]
+          flags = 0
+          flags |= 0x01 if text.size > 0
+          flags |= 0x02 if sound.size > 0
+          flags |= 0x04 if sound_length > 0.0
+          entries << [
+            flags, sound, entry[:volume_variance], entry[:pitch_variance], offsets[i], text.size, sound_length
+          ].pack("I a16 I I I I f")
+        end
+        entries = entries.join("")
+
+        io.write(header)
+        io.write(entries)
+        io.write(text_block)
+      end
+    end
+
+    # A TlkSet wraps a set of File objects, each pointing to the respective tlk file, making
+    # retrieval easier.
+    class TlkSet
+      # The default male Tlk.
+      attr_reader :dm
+      # The default female Tlk, (or the default male).
+      attr_reader :df
+      # The custom male Tlk, or nil.
+      attr_reader :cm
+      # The custom female Tlk, if specified (cm if no female custom tlk has been specified, nil if none).
+      attr_reader :cf
+
+      def initialize tlk, tlkf = nil, custom = nil, customf = nil
+        @dm = Tlk.new(tlk)
+        @df = tlkf ? Tlk.new(tlkf) : @dm
+        @cm = custom ? Tlk.new(custom) : nil
+        @cf = customf ? Tlk.new(customf) : @cm
+      end
+
+      def [](id, gender = :male)
+        raise ArgumentError, "Invalid Tlk ID: #{id.inspect}" if id > 0xffffffff
+        (if id < 0x01000000
+          gender == :female && @df ? @df : @dm
+        else
+          raise ArgumentError, "Wanted a custom ID, but no custom talk table has been specified." unless @cm
+          id -= 0x01000000
+          gender == :female && @cf ? @cf : @cm
+        end)[id]
+      end
+    end
+  end
+end
+

data/lib/nwn/twoda.rb

@@ -45,10 +45,20 @@ module NWN
       # An array of row arrays, without headers.
       attr_accessor :rows
 
+      # What to use to set up newlines.
+      # Alternatively, specify the environ variable NWN_LIB_2DA_NEWLINE
+      # with one of the following:
+      #  0 for windows newlines: \r\n
+      #  1 for unix newlines: \n
+      #  2 for caret return only: \r
+      # defaults to \r\n.
+      attr_accessor :newline
+
       # Create a new, empty 2da table.
       def initialize
         @columns = []
         @rows = []
+        @newline = "\r\n"
       end
 
       # Creates a new Table object from a given IO source.
@@ -74,52 +84,67 @@ module NWN
       # Parses a string that represents a valid 2da definition.
       # Replaces any content this table may already have.
       def parse bytes
-        magic, empty, header, *data = *bytes.split(/\r?\n/).map {|v| v.strip }
+        magic, *data = *bytes.split(/\r?\n/).map {|v| v.strip }
 
         raise ArgumentError, "Not valid 2da: No valid header found" if
           magic != "2DA V2.0"
 
+        # strip all empty lines; they are regarded as comments
+        data.reject! {|ln| ln.strip == ""}
 
-        if empty != ""
-          $stderr.puts "Warning: second line of 2da not empty, continuing anyways."
-          data = [header].concat(data)
-          header = empty
-        end
+        header = data.shift
 
         header = Shellwords.shellwords(header.strip)
         data.map! {|line|
-          Shellwords.shellwords(line.strip)
+          r = Shellwords.shellwords(line.strip)
+          # The last cell can be without double quotes even if it contains whitespace
+          r[header.size..-1] = r[header.size..-1].join(" ") if r.size > header.size
+          r
         }
 
-        data.reject! {|line|
-          line.size == 0
-        }
+        new_row_data = []
 
-        offset = 0
+        id_offset = 0
+        idx_offset = 0
         data.each_with_index {|row, idx|
-          if (idx + offset) != row[0].to_i
-            $stderr.puts "Warning: row #{idx} has a non-matching ID #{row[0]} (while parsing #{row[0,3].join(' ')})."
-            offset += (row[0].to_i - idx)
+          id = row.shift.to_i + id_offset
+
+          raise ArgumentError, "Invalid ID in row #{idx}" unless id >= 0
+
+          # Its an empty row - this is actually valid; we'll fill it up and dump it later.
+          while id > idx + idx_offset
+            new_row_data << Row.new([""] * header.size)
+            idx_offset += 1
+          end
+
+          # NWN automatically increments duplicate IDs - so do we.
+          while id + id_offset < idx
+            $stderr.puts "Warning: duplicate ID found at row #{idx} (id: #{id}); fixing that for you."
+            id_offset += 1
+            id += 1
           end
 
-          # [1..-1]: Strip off the ID
-          data[row[0].to_i] = row = Row.new(row[1..-1])
-          row.table = self
+          # NWN fills in missing columns with an empty value - so do we.
+          row << "" while row.size < header.size
+
+          new_row_data << k_row = Row.new(row)
+          k_row.table = self
 
-          row.map! {|cell|
+          k_row.map! {|cell|
             cell = case cell
               when nil; nil
               when "****"; ""
               else cell
             end
           }
+
           raise ArgumentError,
-            "Row #{idx} does not have the appropriate amount of cells (has: #{row.size}, want: #{header.size}) (while parsing #{row[0,3].join(' ')})." if
-              row.size != header.size
+            "Row #{idx} has too many cells for the given header (has #{k_row.size}, want <= #{header.size})" if
+              k_row.size != header.size
         }
 
         @columns = header
-        @rows = data
+        @rows = new_row_data
       end
 
 
@@ -170,6 +195,7 @@ module NWN
         id_cell_size = @rows.size.to_s.size + CELL_PAD_SPACES
         max_cell_size_by_column = @columns.map {|col|
           ([col] + by_col(col)).inject(0) {|max, cell|
+            cell = '"%s"' % cell if cell =~ /\s/
             cell.to_s.size > max ? cell.to_s.size : max
           } + CELL_PAD_SPACES
         }
@@ -189,11 +215,25 @@ module NWN
           rv << row_idx.to_s + " " * (id_cell_size - row_idx.to_s.size)
           row.each_with_index {|cell, column_idx|
             cell = "****" if cell == ""
+            cell = '"%s"' % cell if cell =~ /\s/
             rv << cell + " " * (max_cell_size_by_column[column_idx] - cell.size)
           }
           ret << rv.join("").rstrip
         }
-        ret.join("\r\n")
+
+        # Append an empty newline.
+        ret << ""
+
+        ret.join(case ENV['NWN_LIB_2DA_NEWLINE']
+          when "0"
+            "\r\n"
+          when "1"
+            "\n"
+          when "2"
+            "\r"
+          when nil
+            @newlines
+        end)
       end
     end
 

data/scripts/debug_check_objid.rb

@@ -0,0 +1,29 @@
+# This is a debug script used to verify anchored YAML dump loads.
+# There was/is a bug in some earlier ruby YAML libs that would
+# incorrectly anchor objects within large dumps; this prints out
+# those duplicate object ids.
+
+$obj_ids = {}
+
+def chk path, obj
+  if $obj_ids[obj.object_id] && $obj_ids[obj.object_id][1] == obj
+    log "Duplicate object ID:"
+    log "  #{path}"
+    log "  #{$obj_ids[obj.object_id]}"
+  else
+    $obj_ids[obj.object_id] = [path, obj]
+  end
+end
+
+self.each_by_flat_path do |path, obj|
+  case obj
+    when Gff::Field
+      chk path + ".v", obj.v unless obj.v.is_a?(Numeric) || obj.v.is_a?(String)
+      chk path + ".l", obj.l unless obj.l.is_a?(String)
+      chk path, obj
+    else
+      chk path, obj
+  end
+end
+
+log "#{$obj_ids.size} object ids verified."

data/scripts/reformat_2da

@@ -0,0 +1,9 @@
+#!/usr/bin/env nwn-dsl
+# vim: ft=ruby
+
+ARGV.each {|f|
+  log "Working on #{f} .."
+  t = TwoDA::Table.parse(IO.read(f))
+  File.open(f, "w") {|n| n.puts t.to_2da }
+  log "done."
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: nwn-lib
 version: !ruby/object:Gem::Version 
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors: 
 - Bernhard Stoeckner
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-04 00:00:00 +01:00
+date: 2009-02-05 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -47,6 +47,7 @@ files:
 - lib/nwn/twoda.rb
 - lib/nwn/settings.rb
 - lib/nwn/gff.rb
+- lib/nwn/tlk.rb
 - lib/nwn/infer.rb
 - lib/nwn/kivinen.rb
 - lib/nwn/all.rb
@@ -62,9 +63,10 @@ files:
 - tools/verify.sh
 - tools/migrate_03x_to_04x.sh
 - scripts/truncate_floats.rb
-- scripts/fix_facings.rb
+- scripts/reformat_2da
 - scripts/clean_locstrs.rb
 - scripts/extract_all_items.rb
+- scripts/debug_check_objid.rb
 - data/gff-common-nwn1.yaml
 - BINARIES
 - DATA_STRUCTURES

data/scripts/fix_facings.rb

@@ -1,22 +0,0 @@
-#!/usr/bin/env nwn-dsl
-
-# The nwn toolset sometimes does weird things with facings;
-# they flip signedness for no apparent reason.
-
-# This script fixes that by forcing all facings to be unsigned.
-
-want Gff::Struct
-
-count = 0
-
-self.each_by_flat_path do |label, field|
-	next unless field.is_a?(Gff::Field)
-	next unless field.field_type == :float
-	next unless label =~ %r{\[\d+\]/(Facing|Bearing)$}
-	if field.field_value < 0
-		field.field_value = field.field_value.abs
-		count += 1
-	end
-end
-
-log "#{count} bearings modified."