nwn-lib 0.3.6 → 0.4.0

data/BINARIES

@@ -1,45 +1,40 @@
-== nwn-gff-print
-
- Usage: nwn-gff-print [options] file/- [file, file, ..]
-     -y, --yaml                       Dump as yaml
-     -k, --kivinen                    Dump as kivinens dump format (like the perl tools)
-         --kivinen-full-path          Print the full path (implies -k)
-     -b, --print-basename             Prefix the file basename to the output
-     -f, --print-filename             Prefix the full filename to the output
-     -t, --print-types                Print types as well
-     -m, --marshal                    Native ruby marshalling. Warning: raw bytes
-         --postfix P                  Write output to file postfixed with P instead of stdout
-         --float_rounding F           Round floating point numbers to the specified number of righthand digits
-         --type filetype              Override the file type (implies --struct 0xffffffff)
-         --struct id                  Override struct id (as hex, please)
-     -p, --path path                  Only print the given path
-     -v, --verbose                    Be verbose
-
-This prints out the given file(s) (or stdin, if -) as the specified format (-y, -k, -m)
-[-k] is the same as gffprint.pl
-[-kt] is the same as gffprint.pl -t
-[-m] is the native ruby marshal format (see Marshal.dump(obj))
-[-y] is standard YAML
-[--postfix] will write out each processed file to the same directory as the infile, with the given postfix appended
-[--float_rounding] can be used to truncate FPs to the given precision.
-[-p] will print out a given path within the GFF data (for example <tt>/ItemList[2]</tt>)
-
-== nwn-gff-import
-
- Usage: nwn-gff-import [options] file/- [file, file, file]
-     -y, --yaml                       Import as yaml
-     -m, --marshal                    Import as native ruby marshal data
-     -o, --outfile F                  Write to outfile instead of stdout
-         --postfix P                  Strip the given postfix from file and write to that instead of stdout (overrides -o)
-
-This is the equivalent to gffencode.pl. This can be used to transform marshalled gff data back into gff binary data.
-
-Example (save to run on a shell, prints out hex data):
-
- nwn-gff-print -y my_item.uti | nwn-gff-import.rb -y - - | xxd
-
-== nwn-gff-irb
-
- Usage: nwn-gff-irb file
-
-nwn-gff-irb allows interactive editing of gff files. There are some examples on the CHEATSHEET.
+== nwn-gff
+
+A generic converter that can be used for converting gff files to and fro from various
+file formats and presentations, and transform them with custom script filters.
+
+Type
+  nwn-gff -h
+for help, it should be self-explanatory.
+
+There are some usage examples on the CHEATSHEET.
+
+== nwn-irb
+
+nwn-irb is a interactive shell preloading all relevant libs, and optionally
+loading a gff file.
+
+There are some usage examples on the CHEATSHEET.
+
+== nwn-dsl
+
+A standalone script interpreter. See the example scripts in the gem distribution
+under scripts/.
+
+== Things under tools/
+
+These are not added to PATH, you'll have to specify their path explicitly; you can
+find them in your gem repository (/usr/lib/ruby/1.8/gems/ or similar).
+
+=== migrate_03x_to_04x.sh
+
+This can be used to migrate old YAML dumps made with version 0.3.x to the new,
+compacter format of 0.4.x.
+
+Usage is simple: just pass all old .yml files to the script, it will convert them
+in-place (read: make a backup!).
+
+This uses nwn-gff-convert and nwn-gff, so all environment variables are taken into
+consideration. You will want to point NWN_LIB_INFER_DATA_FILE to something useful.
+
+Backup and testing is ADVISED!

data/CHANGELOG

@@ -105,3 +105,7 @@ Bernhard Stoeckner <elven@swordcoast.net> (5):
 
 Stuart Coyle <stuart.coyle@gmail.com> (1):
       TwoDA: use four spaces for field separation instead of tabs, indent columns
+
+==== 0.4.0
+Bernhard Stoeckner <elven@swordcoast.net>:
+      too many to sanely list, see README for migration information

data/CHEATSHEET

@@ -1,75 +1,9 @@
-==== Extract item from creature inventory
+==== Convert all item files in the current directory to yaml and back
 
- nwn-gff-print --type UTI -kt -p 'ItemList[12]' creature.bic | gffencode.pl -o item.uti
+ for x in *.uti; do
+     nwn-gff-convert -i"$x" -o"$x.yml" -lg -ky
+ done
 
-==== Convert all item files in the current directory to yaml
-
- nwn-gff-print -v -y --postfix .yml *.uti
-
-==== Convert all yaml item files back to their gff counterpart
-
- nwn-gff-import -y --postfix .yml *.uti.yml
-
-==== Makefile rules for building resources from yml files
-
- postfix := uti
-
- objects := $(basename $(wildcard *.$(postfix).yml))
-
- all: $(objects)
-
- $(objects) : %.$(postfix) : %.$(postfix).yml
-     nwn-gff-import -y -o $@ $<
-
- clean:
-     -@rm $(objects)
-
-This ruleset will automatically rebuild changed ymls to their respective .uti counterparts.
-This only serves as an example on how to do it; it does not constitute a complete build environment.
-
-==== Quick-edit a item
- 
- your/prompt$ nwn-gff-irb my_item.uti
- Your GFF file is in `GFF' (type: "UTI ").
- Type `save' to save to the filename it came from (make a backup!), `exit' (or Ctrl+D) to exit (without saving).
- To save to a different location, type `save "path/to/new/location.ext"'.
-
- irb(main):001:0> GFF.root_struct.keys
- => ["ModelPart2", "Cursed", "ModelPart3", "Cost", "StackSize", "Charges", "Comment", "PaletteID", "BaseItem", "Tag", "DescIdentified", "Identified", "Plot", "TemplateResRef", "PropertiesList", "AddCost", "Stolen", "Description", "LocalizedName", "ModelPart1"]
-
- irb(main):002:0> GFF['LocalizedName/4']
- => "Mundane Feuerpfeile (aus Eibe)"
-
- irb(main):003:0> GFF['LocalizedName/4'] = 'New Name'
- => "New Name"
-
- irb(main):004:0> GFF['LocalizedName']
- => #<NWN::Gff::Element:0xb790641c @type=:cexolocstr, @value=[#<struct NWN::Gff::CExoLocString language=4, text="New Name">, #<struct NWN::Gff::CExoLocString language=0, text="Pfeil">], @_str_ref=1517, @label="LocalizedName">
-
- irb(main):005:0> save
- Saving to `/home/elven/code/nwn/nwn-lib.git/my_item.uti' ..
- saved.
- => nil
-
- irb(main):006:0> exit
-
-==== Add a item property, the semi-human way
-
- irb(main):001:0> Helpers.item_property('Cast_Spell')
- ArgumentError: Property Cast_Spell needs subtype of type IPRP_SPELLS, but none given.
- from /var/lib/gems/1.8/gems/nwn-lib-0.2.3/lib/nwn/helpers.rb:108:in `item_property'
- from (irb):1
-
- irb(main):002:0> Helpers.item_property('Cast_Spell', 'Invisib')
- ArgumentError: Cannot resolve invisib. Partial matches: ["Improved_Invisibility", "Invisibility", "Invisibility_Purge", "Invisibility_Sphere", "See_Invisibility"].
- from /var/lib/gems/1.8/gems/nwn-lib-0.2.3/lib/nwn/helpers.rb:79:in `resolve_or_match_partial'
- from /var/lib/gems/1.8/gems/nwn-lib-0.2.3/lib/nwn/helpers.rb:116:in `item_property'
- from (irb):2
-
- irb(main):003:0> Helpers.item_property('Cast_Spell', 'Invisibility')
- ArgumentError: Property Cast_Spell requires a cost value of type IPRP_CHARGECOST, but none given
- from /var/lib/gems/1.8/gems/nwn-lib-0.2.3/lib/nwn/helpers.rb:122:in `item_property'
- from (irb):3
-
- irb(main):004:0> Helpers.item_property('Cast_Spell', 'Invisibility', 'Unlimi')
- => {"PropertyName"=>#<NWN::Gff::Element:0xb75f4e68 ..
+ for x in *.uti.yml; do
+     nwn-gff-convert -i"$x" -o"$(basename $x .yml)" -ly -kg
+ done

data/COPYING

@@ -1,4 +1,4 @@
-Copyright (C) 2008 Bernhard Stoeckner <elven@swordcoast.net>
+Copyright (C) 2008 Bernhard Stoeckner <elven@swordcoast.net> and contributors
 
 This program is free software; you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

data/DATA_STRUCTURES

@@ -0,0 +1,50 @@
+== Internal data representation
+
+All data is representend internally as simple hashes or arrays. On load,
+each structure will be extended by a representing module containing
+the API and some application logic.
+
+* Structs are normal hashes extended by NWN::Gff::Struct
+* Lists are arrays extended by NWN::Gff::List
+* Fields are normal hashes extended by NWN::Gff::Field
+* CExoLocStrings are normal hashes extended by NWN::Gff::CExoLocString
+
+
+=== Struct
+
+Example:
+ "/AddCost" => {"value" => 5, "label" => "AddCost", "type" => :dword}
+
+Also, a NWN::Gff::Struct also has these accessors:
+
+* data_type: A string describing the data type of this struct. (see Data Type)
+* data_version: A string describing the version (usually "V3.2")
+* struct_id: the struct Id of this struct.
+* element: The NWN::Gff::Field which this struct contains. This is nil
+  for root structs.
+
+
+=== Field
+
+Each field is a hash containing two keys:
+ "value": The data contained in this field, depending on
+ "type": The data type. For known data types, see NWN::Gff::Gff.
+
+Additionally, "label" is a key set by the Readers saying its' label name.
+This key, however, will not be written out to YAML (since it would be
+redundant).
+
+Also, each field has a .parent, which is a simple accessor set by
+the various readers pointing to the struct it is member of.
+
+
+=== Data Type
+
+Each Struct has a data_type, which describes the type of data the struct contains.
+For top-level structs, this equals the data type written to the GFF file ("UTI",
+for example); for sub structures, this is usually the top-level data type + the
+field label ("UTI/PropertiesList", for example). This is used by various loaders/writers
+to infer field types and default values.
+
+This also means that inventory items in /ItemList/ structs have a type of "UTI" and
+thus could be exported to item files, or re-attached somewhere else with minimum scripting.

data/README

@@ -4,55 +4,45 @@ This package provides a library for reading, changing, and writing common file f
 
 They should work with NWN2 just as well, since the file format specifications did not change.
 
-There are various things included in this distribution:
-* some binaries under bin/ (see BINARIES)
-* the actual library under lib/nwn
+=== Upgrade from 0.3.6 to 0.4.x
 
-== Attention Unicode/UTF-users
+With the release of 0.4.0, the API changed significantly. Previous yaml dumps made with 0.3.x are INCOMPATIBLE, and so are all scripts.
+I can't help you with your API bindings, but for your YAML dumps, a converter script has been provided (see BINARIES).
 
-ruby 1.8 does not support character sets natively, and as such nwn-gff-irb will <b>FAIL</b> to encode non-standard characters properly on non-latin1 shells.
-This will be worked around in a future release until the release of ruby 1.9, which will provide native charset support.
-
-== Quickstart
+=== A word on version numbers
 
-It is easiest to just use the gem, which is available through rubyforge (<tt>sudo gem install nwn-lib</tt>).
+Please consider all version releases below 1.0.0 to be unstable, even though the code is fully functional - further API changes may not be avoidable between major revisions - such as is the case with 0.3 -> 0.4.
 
-As an alternative, fetch the latest development files with git[http://git.swordcoast.net/?p=nwn/nwn-lib.git;a=summary].
-
-  require 'rubygems'
-  require 'nwn/gff'
+=== Features of nwn-lib
 
-  # Read a GFF file
-  o = NWN::Gff::Reader.read(IO.read('/tmp/test-creature.gff'))
+* a feature-complete parser and generator of valid GFF V3.2 data
+* shell scripts and tools to simplify your life (see BINARIES)
+* kivinen-style ("gffprint.pl") presentation of data
+* yaml presentation of data
+* ruby-native marshalling of gff data
+* extensive developer API
+* a powerful get-out-of-my-way scripting system for data transformation (see SCRIPTING)
+* guessing of field-types and -values for shorter plaintext data dumps (see TYPE_AND_VALUE_INFERRING)
 
-  # Do some modifications to it
-  o['/Tag'] = 'testtag'
+Also in the box:
 
-  # And write it somewhere else
-  bytes = NWN::Gff::Writer.dump(o)
+* a CHEATSHEET showing off some cool tricks
+* some gadgetry for working with 2da files
 
-  File.open('/tmp/test-creature-2.gff', 'w') {|f| f.write(bytes) }
+=== Attention Unicode/UTF-users
 
-Modification API is fairly limited, for now, but will improve soon.
-
-== GFF data structures: Quick Intro
+ruby 1.8 does not support character sets natively, and as such nwn-gff-irb will <b>FAIL</b> to encode non-standard characters properly on non-latin1 shells.
+This will be worked around in a future release until the release of ruby 1.9, which will provide native charset support.
 
-Ruby GFF data structures are nearly a 1:1 wrapper around the known format.
+=== Quickstart
 
-Each GFF object has a root structure with a +struct_id+ of 0.
+To use it, simply install the gem (available on rubyforge):
 
-=== Label
-A Label is a string of up to 16 bytes, and occurs as keys in Structs.
+ gem1.8 install nwn-lib
 
-=== Struct
-A struct is a hash. It is a unordered collection of key => value pairs.
-Keys are Labels, values are either:
-* Structs
-* Lists
-* Elements
+And do the following in a script of your own devising:
 
-=== List
-A list is an *ordered* array of Structs.
+  require 'rubygems'
+  require 'nwn/all'
 
-=== Element
-A Element is a distinct value of a given type. For all possible types, see the +Types+ hash in the NWN::Gff module.
+For nwn-lib scripts, see SCRIPTING.

data/Rakefile

@@ -9,16 +9,18 @@ include FileUtils
 # Configuration
 ##############################################################################
 NAME = "nwn-lib"
-VERS = "0.3.6"
+VERS = "0.4.0"
 CLEAN.include ["**/.*.sw?", "pkg", ".config", "rdoc", "coverage"]
 RDOC_OPTS = ["--quiet", "--line-numbers", "--inline-source", '--title', \
   'nwn-lib: a ruby library for accessing NWN resource files', \
   '--main', 'README']
 
+DOCS = ["README", "BINARIES", "DATA_STRUCTURES", "SCRIPTING", "SETTINGS", "TYPE_VALUE_INFERRING", "CHEATSHEET", "CHANGELOG", "COPYING"]
+
 Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_dir = "rdoc"
   rdoc.options += RDOC_OPTS
-  rdoc.rdoc_files.add ["README", "BINARIES", "CHEATSHEET", "CHANGELOG", "COPYING", "doc/*.rdoc", "lib/**/*.rb"]
+  rdoc.rdoc_files.add DOCS + ["doc/*.rdoc", "lib/**/*.rb"]
 end
 
 desc "Packages up nwn-lib"
@@ -30,16 +32,16 @@ spec = Gem::Specification.new do |s|
   s.version = VERS
   s.platform = Gem::Platform::RUBY
   s.has_rdoc = true
-  s.extra_rdoc_files = ["README", "BINARIES", "CHEATSHEET", "CHANGELOG", "COPYING"] + Dir["doc/*.rdoc"]
+  s.extra_rdoc_files = DOCS + Dir["doc/*.rdoc"]
   s.rdoc_options += RDOC_OPTS + ["--exclude", "^(examples|extras)\/"]
   s.summary = "a ruby library for accessing Neverwinter Nights resource files"
   s.description = s.summary
   s.author = "Bernhard Stoeckner"
   s.email = "elven@swordcoast.net"
   s.homepage = "http://nwn-lib.elv.es"
-  s.executables = ["nwn-gff-print", "nwn-gff-irb", "nwn-gff-import"]
+  s.executables = ["nwn-gff", "nwn-dsl", "nwn-irb"]
   s.required_ruby_version = ">= 1.8.4"
-  s.files = %w(COPYING CHANGELOG README Rakefile) + Dir.glob("{bin,doc,spec,lib}/**/*")
+  s.files = %w(COPYING CHANGELOG README Rakefile) + Dir.glob("{bin,doc,spec,lib,tools,scripts,data}/**/*")
   s.require_path = "lib"
   s.bindir = "bin"
 end

data/SCRIPTING

@@ -0,0 +1,44 @@
+To help you manage your NWN data more efficiently, nwn-lib supports
+user-written, small (or large) scripts that follow the unix principle
+of piping.
+
+=== Filter Scripts
+
+Filter Scripts act as an argument to nwn-gff, and will be invoked
+as a pass-through for data filtering and transformation.
+Filters always depend on self being a Gff element.
+
+An example of a filter script would be a snippet, that truncates
+all floating points to a fixed size before outputting them to yaml.
+
+Filter scripts are usually not executable by themself.
+
+=== Standalone Scripts
+
+Standalone scripts are shell scripts invokable from command line,
+and do not operate on specific files; instead, they get invoked
+with their own (optional) parameters and act independently of
+data sources.
+
+An example for a standalone script would be a script, that asks
+the user for a name and then generates a random item from that.
+
+You can create standalone scripts by prefixing them with the proper
+shebang:
+
+  #!/usr/bin/env nwn-dsl
+
+If you are on systems which do not support executable scripts this
+way, simply call them with nwn-dsl.
+
+DSL stands for domain-specific-language by the way, and this is quite
+a stretch, considering your scripts will be written in plain old ruby.
+
+=== API
+
+There are various helpers available to scripts. See NWN::Gff::Scripting
+for a list of helper methods; additionally you can use all other API
+functions, of course.
+
+There are some examples packaged together with nwn-lib, check the scripts/
+directory with the distribution.

data/SETTINGS

@@ -0,0 +1,80 @@
+There are various environment variables you can configure to adjust various parts of nwn-lib.
+
+<b>All of them are optional!</b>
+
+Under linux, just add them to your shell environment (usually .bashrc), like so:
+
+  export NWN_LIB_INFER_DATA_FILE=/usr/lib/ruby/gems/1.8/gems/nwn-lib-0.4.0/data/gff-common-nwn1.yaml
+
+
+== NWN_LIB_INFER_DATA_FILE
+
+The path to the type/value inferring data file. See TYPE_VALUE_INFERRING for a definition.
+
+Please be advised that inferring support for YAML is <b>EXPERIMENTAL</b> and may result in
+<b>module and gff file corruption</b>.
+
+== NWN_LIB_DONT_COMPACT_FIELDS
+
+Set to non-nil ("1" will do) to prevent nwn-lib from compacting scalar fields into a more-readable
+(but still fully parseable) format, if type inferring data is available.
+
+=== Without compacting
+  ItemList:
+    value:
+    - !nwn-lib.elv.es,2008-12/struct
+      __data_type: UTC/ItemList
+      __struct_id: 0
+      InventoryRes: {value: herb053}
+  PaletteID: {value: 6}
+  SkillList: [{value: 0}, {value: 0}, {value: 0}, {value: 0}, ...
+
+=== With compacting
+  ItemList:
+  - !nwn-lib.elv.es,2008-12/struct
+    __data_type: UTC/ItemList
+    InventoryRes: herb053
+  PaletteID: 6
+  SkillList: [0, 0, 0, 0, 0, 0, 2, 0, 0, 0 ...
+
+
+== NWN_LIB_DONT_COMPACT_LIST_STRUCTS
+
+Setting this to non-nil ("1" will do) will prevent nwn-lib from compacting lists that contain
+structs, if type inferring data is available.
+
+=== Without compacting
+  SkillList:
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 0}
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 0}
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 0}
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 0}
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 0}
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 0}
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 2}
+  - !nwn-lib.elv.es,2008-12/struct {__data_type: UTC/SkillList, Rank: 0}
+  ...
+
+=== With compacting
+  SkillList: [0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, ...
+
+
+== NWN_LIB_CLEAR_KNOWN_VALUES
+
+Set to non-nil ("1" will do) to make nwn-lib omit fields that resolve to default values
+as configured in $NWN_LIB_INFER_DATA_FILE.
+
+The default is to not omit known-value data fields.
+
+== NWN_LIB_FILTER_EMPTY_EXOLOCSTR
+
+Set to non-nil ("1" will do) to make nwn-lib filter out empty exolocstr fields from
+input files on reading.
+
+The default is to keep them as-is.
+
+== NWN_LIB_2DA_LOCATION
+
+Set to a path containing all 2da files to initialize the 2da cache. This is needed for
+most interactive helpers and a few type infer gizmos.
+