nwn-lib 0.5.0 → 0.5.1

data/README.rdoc

@@ -42,10 +42,9 @@ With the release of 0.4.0, the API changed significantly. Previous yaml dumps ma
 I can't help you with your API bindings, but for your YAML dumps, a converter script has been provided (see {file:BINARIES}).
 
 
-=== Attention Unicode/UTF-users
+=== Upgrade from 0.4.x to 0.5.x
 
-ruby 1.8 does not support character sets natively, and as such nwn-gff-irb will <b>FAIL</b> to encode non-standard characters properly on non-latin1 shells.
-This will be worked around in a future release until the release of ruby 1.9, which will provide native charset support.
+All with version 0.4.x produced files should be compatible with 0.5.0. Your application might require some porting to ruby 1.9.
 
 
 === Quickstart

data/SETTINGS.rdoc

@@ -8,7 +8,8 @@ Under linux, just add them to your shell environment (usually .bashrc), like so:
 
 == NWN_LIB_IN_ENCODING
 
-The character encoding your local data files are in. Defaults to ISO-8859-1.
+The character encoding your local data (gff) files are in. Defaults to ISO-8859-1.
+Extended dump formats like yaml, xml are always format-specific, usually UTF-8.
 
 == NWN_LIB_PRETTY_JSON
 

data/bin/nwn-gff

@@ -8,7 +8,6 @@ Thread.abort_on_exception = true
 $options = {
   :backup => nil,
   :encoding => nil,
-  :out_encoding => nil,
   :force => false,
   :infile => '-',
   :outfile => '-',
@@ -50,14 +49,10 @@ begin OptionParser.new do |o|
       "(see `man cp' for a description)" do |b|
     $options[:backup] = b.nil? ? true : b
   end
-  o.on "--encoding ENCODING", "sets the used input encoding in which your NWN",
+  o.on "--encoding ENCODING", "sets the used encoding in which your gff",
       "files are encoded in" do |e|
     $options[:encoding] = e
   end
-  o.on "--out-encoding ENCODING", "sets the used output encoding",
-      "(defaults to UTF-8)" do |e|
-    $options[:out_encoding] = e
-  end
 
   o.on "-1", "--nwn1", "Allow 16 byte resrefs." do
     ENV['NWN_LIB_RESREF32'] = nil
@@ -121,7 +116,6 @@ $options[:informat] or fail "No input format specified."
 $options[:outformat] or fail "No output format specified."
 
 NWN.setting(:in_encoding, $options[:encoding]) if $options[:encoding]
-NWN.setting(:out_encoding, $options[:out_encoding]) if $options[:out_encoding]
 
 if :auto == $options[:informat]
   $options[:informat] = NWN::Gff.guess_file_format($options[:infile].downcase)

data/lib/nwn/gff.rb

@@ -49,7 +49,7 @@ module NWN
       module Pretty
         def self.dump data, io
           old = $> ; $> = StringIO.new
-          pp data.box
+          pp data
           sz = $>.pos
           $>.seek(0)
           io.write $>.read

data/lib/nwn/gff/field.rb

@@ -216,58 +216,4 @@ module NWN::Gff::Field
     end
   end
 #:startdoc:
-
-  # Deep-unboxes a Hash, e.g. iterating down, converting all strings
-  # from the native charset.
-  def self.unbox! element, parent_label, parent
-    element.extend(NWN::Gff::Field)
-    element.field_label = parent_label
-    element.parent = parent
-    element.str_ref ||= NWN::Gff::Field::DEFAULT_STR_REF if element.respond_to?('str_ref=')
-
-    element.extend_meta_classes
-    case element.field_type
-      when :cexolocstr
-        mod = {}
-        element.field_value.each {|x,y|
-          mod[x] = NWN.iconv_native_to_gff(y)
-        }
-        mod.each {|x,y|
-          element.field_value.delete(x)
-          element.field_value[x.to_i] = y
-        }
-      when :cexostr
-        element.field_value = NWN.iconv_native_to_gff(element.field_value)
-
-      when :list
-        mod = {}
-        element.field_value.each_with_index {|x,idx|
-          mod[idx] = NWN::Gff::Struct.unbox!(x, element)
-        }
-        mod.each {|x,y|
-          element.field_value[x] = y
-        }
-      when :struct
-        element.field_value = NWN::Gff::Struct.unbox!(element.field_value, element)
-    end
-    element.validate
-    element
-  end
-
-  # Returns a hash of this Field without the API calls mixed in,
-  # all language-strings transformed by str_handler.
-  def box
-    t = Hash[self]
-    t.delete('label')
-    case field_type
-      when :cexolocstr
-        t['value'].each {|x,y|
-          t['value'][x] = NWN.iconv_gff_to_native(y)
-        }
-      when :cexostr
-        t['value'] = NWN.iconv_gff_to_native(t['value'])
-    end
-    t
-  end
-
 end

data/lib/nwn/gff/reader.rb

@@ -14,6 +14,7 @@ class NWN::Gff::Reader
 
   def initialize io #:nodoc:
     @io = io
+    io.internal_encoding == nil or raise "passed io needs to be binary, is #{io.internal_encoding.inspect}"
     read_all
   end
 
@@ -47,7 +48,7 @@ class NWN::Gff::Reader
     @io.seek(label_offset)
     @labels  = @io.e_read(label_len, "labels")
     @labels = @labels.unpack("A16" * label_count)
-    @labels.map! {|l| l.encode("ASCII") }
+    @labels.map! {|l| l.force_encoding("ASCII") }
 
     @io.seek(field_data_offset)
     @field_data = @io.e_read(field_data_count, "field_data")
@@ -75,8 +76,8 @@ class NWN::Gff::Reader
     raise GffError, "struct index #{index} outside of struct_array" if
       index * 3 + 3 > @structs.size + 1
 
-    file_type = file_type.encode('ASCII') if file_type
-    file_version = file_version.encode('ASCII') if file_version
+    file_type = file_type.force_encoding('ASCII') if file_type
+    file_version = file_version.force_encoding('ASCII') if file_version
 
     struct.struct_id = type
     struct.data_type = file_type
@@ -159,11 +160,15 @@ class NWN::Gff::Reader
 
       when :cexostr
         len = @field_data[data_or_offset, 4].unpack("V")[0]
-        @field_data[data_or_offset + 4, len].encode(NWN.setting :in_encoding)
+        str = @field_data[data_or_offset + 4, len].force_encoding(NWN.setting :in_encoding)
+        str.valid_encoding? or raise "Invalid encoding bytes in cexostr: #{str.inspect}"
+        str
 
       when :resref
         len = @field_data[data_or_offset, 1].unpack("C")[0]
-        @field_data[data_or_offset + 1, len].encode(NWN.setting :in_encoding)
+        str = @field_data[data_or_offset + 1, len].force_encoding(NWN.setting :in_encoding)
+        str.valid_encoding? or raise "Invalid encoding bytes in resref: #{str.inspect}"
+        str
 
       when :cexolocstr
         exostr = {}
@@ -176,7 +181,8 @@ class NWN::Gff::Reader
 
         str_count.times {
           id, len = all.unpack("VV")
-          str = all[8, len].unpack("a*")[0].encode(NWN.setting :in_encoding)
+          str = all[8, len].unpack("a*")[0].force_encoding(NWN.setting :in_encoding)
+          str.valid_encoding? or raise "Invalid encoding bytes in cexolocstr: #{str.inspect}"
           all = all[(8 + len)..-1]
           exostr[id] = str
         }
@@ -185,7 +191,7 @@ class NWN::Gff::Reader
 
       when :void
         len = @field_data[data_or_offset, 4].unpack("V")[0]
-        @field_data[data_or_offset + 4, len].unpack("a*")[0]
+        @field_data[data_or_offset + 4, len].unpack("a*")[0].force_encoding("BINARY")
 
       when :struct
         read_struct data_or_offset, nil, field.parent.data_version

data/lib/nwn/gff/struct.rb

@@ -222,44 +222,4 @@ module NWN::Gff::Struct
   def / path
     by_path(path)
   end
-
-  # Deep-unboxes a Hash, e.g. iterating down, converting it to
-  # the native charset.
-  def self.unbox! o, parent = nil
-    o.extend(NWN::Gff::Struct)
-    o.element = parent if parent
-    o.struct_id = o.delete('__struct_id')
-    o.data_type = o.delete('__data_type')
-    o.data_version = o.delete('__data_version')
-    o.data_version ||= NWN::Gff::Struct::DEFAULT_DATA_VERSION
-
-    NWN.log_debug("Unboxed without a root data type") if
-      !parent && !o.data_type
-    NWN.log_debug("Unboxed with explicit data type #{o.data_type.inspect}") if
-      parent && o.data_type
-
-    o.each {|label,element|
-      o[label] = NWN::Gff::Field.unbox!(element, label, o)
-    }
-
-    o
-  end
-
-  # Returns a hash of this Struct without the API calls mixed in,
-  # converting it from the native charset.
-  def box
-    t = Hash[self]
-    t.merge!({
-      '__struct_id' => self.struct_id
-    })
-    t.merge!({
-      '__data_version' => self.data_version,
-    }) if self.data_version && self.data_version !=
-      NWN::Gff::Struct::DEFAULT_DATA_VERSION
-    t.merge!({
-      '__data_type' => self.data_type
-    }) if @data_type
-    t
-  end
-
 end

data/lib/nwn/gff/writer.rb

@@ -160,11 +160,13 @@ private
 
         when :resref
           fields_of_this_struct << add_data_field(v.field_type, k, @field_data.size)
-          @field_data << [v.field_value.size, v.field_value].pack("Ca*")
+          fv = v.field_value.encode(NWN.setting :in_encoding)
+          @field_data << [fv.size, fv].pack("Ca*")
 
         when :cexostr
           fields_of_this_struct << add_data_field(v.field_type, k, @field_data.size)
-          @field_data << [v.field_value.size, v.field_value].pack("Va*")
+          fv = v.field_value.encode(NWN.setting :in_encoding)
+          @field_data << [fv.size, fv].pack("Va*")
 
         when :cexolocstr
           raise GffError, "type = cexolocstr, but value not a hash (#{v.field_value.class})" unless
@@ -184,7 +186,8 @@ private
           ].pack("VVV")
 
           v.field_value.each {|k,v|
-            @field_data << [k, v.size, v].pack("VVa*")
+            vn = v.encode(NWN.setting :in_encoding)
+            @field_data << [k, vn.size, vn].pack("VVa*")
           }
 
         else

data/lib/nwn/json_support.rb

@@ -1,18 +1,98 @@
 require 'json'
+require 'base64'
 
 module NWN::Gff::Struct
+  def json_box
+    t = Hash[self]
+    t.merge!({
+      '__struct_id' => self.struct_id
+    })
+    t.merge!({
+      '__data_version' => self.data_version,
+    }) if self.data_version && self.data_version !=
+      NWN::Gff::Struct::DEFAULT_DATA_VERSION
+    t.merge!({
+      '__data_type' => self.data_type
+    }) if @data_type
+    t
+  end
+  private :json_box
+
   def to_json(*a)
-    box.to_json(*a)
+    json_box.to_json(*a)
   end
 end
 
 module NWN::Gff::Field
+  def json_box
+    t = Hash[self]
+    t.delete('label')
+    case field_type
+      when :void
+        t['value'] = Base64::strict_encode64(t['value'])
+    end
+    t
+  end
+  private :json_box
+
   def to_json(*a)
-    box.to_json(*a)
+    json_box.to_json(*a)
   end
 end
 
 module NWN::Gff::Handler::JSON
+  def self.json_unbox_field element, parent_label, parent
+    element.extend(NWN::Gff::Field)
+    element.field_label = parent_label
+    element.parent = parent
+    element.str_ref ||= NWN::Gff::Field::DEFAULT_STR_REF if element.respond_to?('str_ref=')
+
+    element.extend_meta_classes
+    case element.field_type
+      when :cexolocstr
+        element.field_value.keys.each {|key|
+          val = element.field_value.delete(key)
+          element.field_value[key.to_i] = val
+        }
+
+      when :void
+        element.field_value = Base64::strict_decode64(element.field_value)
+
+      when :list
+        mod = {}
+        element.field_value.each_with_index {|x,idx|
+          mod[idx] = self.json_unbox_struct(x, element)
+        }
+        mod.each {|x,y|
+          element.field_value[x] = y
+        }
+      when :struct
+        element.field_value = self.json_unbox_struct(element.field_value, element)
+    end
+    element.validate
+    element
+  end
+
+  def self.json_unbox_struct o, parent = nil
+    o.extend(NWN::Gff::Struct)
+    o.element = parent if parent
+    o.struct_id = o.delete('__struct_id')
+    o.data_type = o.delete('__data_type')
+    o.data_version = o.delete('__data_version')
+    o.data_version ||= NWN::Gff::Struct::DEFAULT_DATA_VERSION
+
+    NWN.log_debug("Unboxed without a root data type") if
+      !parent && !o.data_type
+    NWN.log_debug("Unboxed with explicit data type #{o.data_type.inspect}") if
+      parent && o.data_type
+
+    o.each {|label,element|
+      o[label] = self.json_unbox_field(element, label, o)
+    }
+
+    o
+  end
+
   def self.load io
     json = if io.respond_to?(:to_str)
       io.to_str
@@ -22,7 +102,7 @@ module NWN::Gff::Handler::JSON
       io.read
     end
 
-    NWN::Gff::Struct.unbox!(JSON.parse(json), nil)
+    self.json_unbox_struct(JSON.parse(json), nil)
   end
 
   def self.dump struct, io

data/lib/nwn/settings.rb

@@ -1,7 +1,6 @@
 module NWN
   SETTING_DEFAULT_VALUES = {
-    'NWN_LIB_IN_ENCODING' => 'ISO-8859-1',
-    'NWN_LIB_OUT_ENCODING' => 'UTF-8'
+    'NWN_LIB_IN_ENCODING' => 'ISO-8859-1'
   }
 
   # This writes a internal warnings and debug messages to stderr.
@@ -49,7 +48,7 @@ module NWN
 
   # Converts text from native format (such as json) to Gff (required by NWN).
   def self.iconv_native_to_gff text
-    text.encode(NWN.setting(:out_encoding))
+    text.encode(NWN.setting(:in_encoding))
   end
 
   # Converts text from Gff format to native/external, such as json (usually UTF-8).

data/lib/nwn/version.rb

@@ -1,3 +1,3 @@
 module NWN
-  VERSION = "0.5.0"
+  VERSION = "0.5.1"
 end

data/lib/nwn/xml_support.rb

@@ -152,10 +152,6 @@ public
   end
 
   def load io
-    old_encoding = NWN.setting(:out_encoding, 'UTF-8')
-    NWN.log_debug("Ignoring custom out_encoding for xml output, always UTF-8") if
-      old_encoding != 'UTF-8'
-
     doc = XML::Parser.io(io)
     root = doc.parse.root
     ret = case @format
@@ -168,15 +164,10 @@ public
         raise ArgumentError, "Unsupported XML format registered: #{@format.inspect}"
     end
 
-    NWN.setting(:out_encoding, old_encoding)
     ret
   end
 
   def dump data, io
-    old_encoding = NWN.setting(:out_encoding, 'UTF-8')
-    NWN.log_debug("Ignoring custom out_encoding for xml output, always UTF-8") if
-      old_encoding != 'UTF-8'
-
     doc = XML::Document.new
     doc.root = case @format
       when :nxml
@@ -193,7 +184,6 @@ public
     t = doc.to_s
     io.write(t)
 
-    NWN.setting(:out_encoding, old_encoding)
     t.size
   end
 end

data/lib/nwn/yaml_support.rb

@@ -81,7 +81,16 @@ module NWN::Gff::Field
         map.style = :inline unless NWN::Gff::Handler::YAML::NonInlineableFields.index(self['type'])
         map.add('type', self['type'])
         map.add('str_ref', self['str_ref']) if has_str_ref?
-        map.add('value', self['value'])
+        map.add('value', case self['type']
+          when :resref, :cexostr
+            self['value'].encode('utf-8')
+          when :cexolocstr
+            Hash[self['value'].map {|lang,str|
+              [lang, str.encode('utf-8')]
+            }]
+          else
+            self['value']
+        end)
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: nwn-lib
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-30 00:00:00.000000000 Z
+date: 2012-12-28 00:00:00.000000000 Z
 dependencies: []
 description: Neverwinter Nights 1/2 file formats ruby library
 email: