stats_package_syntax_file_generator 1.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 0f9197c10adf1e85ad5b72ddf53aa414b9e38c08
+  data.tar.gz: c77ce671f48ee5eaa923ce8ea5f21c875f456aa8
+SHA512:
+  metadata.gz: c092c5f3ef02f13d9819a78831fd56817a4d753e0bfba16646732845a70488fed496204647fa834e0c02a094b47f5558a31d7c6529f381092bc74e34271df86b
+  data.tar.gz: 0836c59716068a5f5e87e62d6eb6665a69e9152c00b2ed1a2c1dd52c2d0131f04459ed349a495cf659f0933f06b98b26ff9a8e730ceb737c5939868242e05b1a

data/README

@@ -0,0 +1,128 @@
+This gem produces statistical package syntax files for fixed-column data files.
+
+    SAS
+    SPSS
+    Stata
+    Stat/Transfer STS metadata files
+
+
+Metadata can be supplied to the Controller in two general ways:
+
+    - Programmatically, using the API provided by the Controller class.
+      
+    - Via one or more YAML files -- typically just one YAML file, but the 
+      Controller will accept multiple files and merge their content.
+
+
+Basic usage:
+
+    require 'stats_package_syntax_file_generator'
+    
+    # Supply metadata via one YAML file.
+    sfc = StatsPackageSyntaxFileGenerator::Controller.new(:yaml_files => 'metadata.yaml')
+
+    # Or via multiple YAML files.
+    sfc = StatsPackageSyntaxFileGenerator::Controller.new(:yaml_files => ['md1.yaml', 'md2.yaml'])
+
+    # Or programmatically.
+    # For a working example, see devel/api_example.rb.
+    
+    # Generate all syntax files.
+    sfc.generate_syntax_files
+    
+    # Generate a syntax file of a specific TYPE (spss, sas, stata).
+    sfc.generate_syntax_file('TYPE')
+    
+    # Ditto, but get the syntax as a list of strings rather than writing a file.
+    syntax_lines = sfc.syntax('TYPE')
+
+
+Running tests:
+
+	devel/run_all_checks.sh
+
+
+Project structure:
+
+	devel/
+		# Developer area.
+		# Various scripts, plus the following:
+
+		input/
+			# YAML metadata used during development
+			# and in end-to-end acceptance testing.
+
+		output_expected/
+			# Acceptance testing: expected output.
+
+		output_result/
+			# Acceptance testing: directory where new output
+			# is written. This output is not included in Git.
+
+	lib/      # The syntax file utility itself.
+	tests/    # Unit test scripts and their YAML metadata.
+
+
+Class overview:
+
+    Controller
+
+        - The API for the gem. Users of the gem should not need to interact with
+          other classes.
+
+        - Serves as a container for the metadata needed to generate syntax files.
+
+        - Holds various attributes specifying the type of syntax files to
+          generate, the structure of the data files, and so forth.
+
+    Variable
+    Value
+
+        - Metadata classes holding information about variables in the data file.
+
+        - A Controller contains 1+ Variable objects.
+
+        - A Variable contains 0+ Value objects.
+
+    Maker
+    Maker_SAS
+    Maker_SPSS
+    Maker_STATA
+    Maker_STS
+
+        - Classes responsible for creating syntax.
+
+        - Maker provides methods that all classes have in common.
+
+        - The other classes inherit from Maker, overriding behavior as needed.
+
+
+
+General behavior of the specific Maker classes:
+
+    - The classes all define a primary method: syntax().
+
+    - The syntax() method consists of calls to various other methods responsible
+      for generating sections of the syntax.
+
+    - Methods that generate sub-sections of syntax have names beginning with 'syn_'.
+
+    - All of the syntax-generating methods return a list of strings.
+
+
+Notes on STS files:
+
+    1.  Hierchical data.
+        Stat/Transfer does not directly support hierarchical data structures. The
+        workaround is to perform multiple invocations of Stat/Transfer, once per record 
+        type. Each invocation must instruct Stat/Transfer to perform case selection to
+        filter the records such that a single record type is in scope, and must request
+        the variables appropriate for that record type.
+
+    2.  Variable and value labels with quotes.  
+        Stat/Transfer does not support escaping quotes within value labels.  Therefore,
+        double quotes are represented by a pair of single quotes.
+
+    3.  Only alphanumeric values are permitted for value labels.
+        Nonconforming values are skipped, with a comment added to the STS file that
+        documents this action.

data/lib/stats_package_syntax_file_generator.rb

@@ -0,0 +1,19 @@
+# This file is part of the Minnesota Population Center's stats_package_syntax_file_generator project.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/mnpopcenter/stats_package_syntax_file_generator
+
+require 'yaml'
+
+%w(
+    controller
+    variable
+    value
+    maker
+    maker_sas
+    maker_spss
+    maker_stata
+    maker_sts
+).each do |f|
+    require File.expand_path(File.join(File.dirname(__FILE__), 'stats_package_syntax_file_generator', f))
+end

data/lib/stats_package_syntax_file_generator/controller.rb

@@ -0,0 +1,323 @@
+# This file is part of the Minnesota Population Center's stats_package_syntax_file_generator project.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/mnpopcenter/stats_package_syntax_file_generator
+
+module StatsPackageSyntaxFileGenerator
+class Controller
+
+VERSION = "1.0.1"
+
+ATTR = {
+    :project                    => { :req => false, :rw => 'rw', :def => '',          :yaml => true  },
+    :caller                     => { :req => false, :rw => 'rw', :def => '',          :yaml => true  },
+    :data_dir_name              => { :req => false, :rw => 'rw', :def => '.',         :yaml => true  },
+    :data_file_name             => { :req => false, :rw => 'rw', :def => 'DATA_FILE', :yaml => true  },
+    :output_formats             => { :req => false, :rw => 'rw', :def => nil,         :yaml => true  },
+    :output_dir_name            => { :req => false, :rw => 'rw', :def => '.',         :yaml => true  },
+    :output_file_stem           => { :req => false, :rw => 'rw', :def => '%s',        :yaml => true  },
+    :output_file_ext            => { :req => false, :rw => 'rw', :def => nil,         :yaml => true  },
+    :output_overwrite           => { :req => false, :rw => 'rw', :def => false,       :yaml => true  },
+    :data_structure             => { :req => false, :rw => 'rw', :def => 'rect',      :yaml => true  },
+    :record_types               => { :req => false, :rw => 'rw', :def => nil,         :yaml => true  },
+    :record_type_var_name       => { :req => false, :rw => 'rw', :def => '',          :yaml => true  },
+    :rectangularize             => { :req => false, :rw => 'rw', :def => false,       :yaml => true  },
+    :all_vars_as_string         => { :req => false, :rw => 'rw', :def => false,       :yaml => true  },
+    :select_vars_by_record_type => { :req => false, :rw => 'rw', :def => false,       :yaml => true  },
+    :variables                  => { :req => false, :rw => 'r',  :def => nil,         :yaml => false },
+    :yaml_files                 => { :req => false, :rw => 'r',  :def => nil,         :yaml => false },
+}
+
+ATTR.each_key do |k|
+    attr_reader k if ATTR[k][:rw].include? 'r'
+    attr_writer k if ATTR[k][:rw].include? 'w'
+end
+
+def initialize (args = {})
+    ATTR.each_key { |k|
+        raise(ArgumentError, "Missing required parameter: '#{k}'.") if
+            ATTR[k][:req] and not args.has_key?(k)
+        v = args.has_key?(k) ? args[k] : ATTR[k][:def]
+        instance_variable_set("@#{k}".to_sym, v)
+    }
+
+    @output_file_ext = {
+        'sas'   => '.sas',
+        'spss'  => '.sps',
+        'stata' => '.do',
+        'sts'   => '.sts'
+    } if @output_file_ext.nil?
+    @output_formats  = [] if @output_formats.nil?
+    @record_types    = [] if @record_types.nil?
+    @variables       = [] if @variables.nil?
+    @yaml_files      = [] if @yaml_files.nil?
+    read_metadata_from_yaml
+end
+
+
+# Methods to import metadata from YAML files into the Controller object.
+
+def yaml_files= (file_names)
+    # Caller can supply a file name or an array of file names.
+    @yaml_files = file_names.to_a
+    read_metadata_from_yaml
+end
+
+def read_metadata_from_yaml
+    return if @yaml_files.empty?
+    md = {}
+    @yaml_files.each { |f| md.merge! YAML.load_file(f) }
+    md = symbolize_keys(md)
+    load_yaml_md(md)
+end
+
+def load_yaml_md (md)
+    # Uses metadata from yaml to set metadata-related instance variables.
+    ATTR.each_key do |k|
+        next unless md.has_key?(k) and ATTR[k][:yaml]
+        instance_variable_set("@#{k}".to_sym, md[k])
+    end
+    return unless md.has_key?(:variables)
+    @variables = []
+    return unless md[:variables].size > 0
+    md[:variables].each do |md_var|
+        vals = md_var.delete(:values)
+        var = add_variable(md_var)
+        vals.each { |v| var.add_value(v) } unless vals.nil?
+    end
+end
+
+def symbolize_keys (h)
+    # Recursively converts hash keys from strings to symbols.
+    if h.instance_of? Hash
+        h.inject({}) { |return_hash,(k,v)| return_hash[k.to_sym] = symbolize_keys(v); return_hash }
+    elsif h.instance_of? Array
+        h.map { |v| symbolize_keys(v) }
+    else
+        h
+    end
+end
+
+# Methods to add or get variables.
+
+def add_variable (args)
+    @variables.push Variable.new(args)
+    @variables[-1]
+end
+
+def clear_variables
+    @variables = []
+end
+
+def get_var_by_name (n)
+    @variables.find { |v| v.name == n }
+end
+
+def get_vars_by_record_type (rt)
+    @variables.find_all { |v| v.record_type == rt or v.is_common_var }
+end
+
+def get_vars_with_var_labels
+    @variables.find_all { |v| v.label.length > 0 }
+end
+
+def get_vars_with_values
+    @variables.find_all { |var|
+        var.values.size > 0 and
+        not var.suppress_labels
+    }
+end
+
+def get_big_nums
+    @variables.find_all { |var|
+        var.width > 8 and
+        not var.is_string_var
+    }
+end
+
+
+def record_type_var
+    get_var_by_name(@record_type_var_name)
+end
+
+
+# Methods for adding values to variables.
+
+def add_value (args)
+    @variables[-1].values.push Value.new(args)
+    @variables[-1].values[-1]
+end
+
+def new_values (*vals)
+    vals.flatten!
+    vals.map { |v| Value.new(v) }
+end
+
+
+# Methods for record types.
+
+def is_last_record_type (rt)
+    return true if @record_types.size > 0 and @record_types[-1] == rt
+    return false
+end
+
+def rec_types_except_last
+    r = Array.new(@record_types)
+    r.pop
+    r
+end
+
+
+# Helper methods.
+
+def max_var_name_length
+    return 0 if @variables.empty?
+    @variables.map { |v| v.name.length }.max
+end
+
+def max_col_loc_width
+    return 0 if @variables.empty?
+    @variables.map { |v| v.end_column.to_s.length }.max
+end
+
+def data_file_name_stem
+    File.basename(@data_file_name, '.*')
+end
+
+def rec_type_lookup_hash
+    Hash[ * @record_types.map { |rt| [rt, 0] }.flatten ]
+end
+
+def last_column_used
+    return 0 if @variables.empty?
+    @variables.map { |v| v.end_column }.max
+end
+
+# Output methods.
+
+def to_s
+    YAML.dump(self)
+end
+
+def generate_syntax_files
+    bad_metadata('no output formats')if @output_formats.empty?
+    @output_formats.each { |t| generate_syntax_file(t) }
+end
+
+def generate_syntax_file (syntax_type)
+    msg = "output directory does not exist => #{@output_dir_name}"
+    bad_metadata(msg) unless File.directory?(@output_dir_name)
+    file_name = File.join(
+        @output_dir_name,
+        sprintf(@output_file_stem, data_file_name_stem) + @output_file_ext[syntax_type]
+    )
+    if File.file?(file_name) and not @output_overwrite
+        $stderr.puts "Skipping file that aready exists => #{file_name}."
+    else
+        if RUBY_VERSION.start_with? "1.8"
+          File.open(file_name, 'w') { |f| f.puts syntax(syntax_type) }
+        else
+          File.open(file_name, 'w:iso-8859-1') { |f| 
+
+          lines =  syntax(syntax_type)
+          lines.each do |line|
+              begin
+                  f.puts line.rstrip.encode('iso-8859-1', line.encoding.to_s,{:invalid=>:replace, :undef=>:replace,:replace => '?'})
+              rescue Exception=>msg
+                  puts "Failed encoding on line #{line} #{msg}"
+              end
+          end
+          }
+        end
+
+    end
+end
+
+def syntax (syntax_type)
+    validate_metadata(:minimal => true)
+    modify_metadata
+    validate_metadata
+
+    maker_class = 'Maker' + syntax_type.upcase
+    syntax_maker = eval(maker_class).new(self, syntax_type)
+    syntax_maker.syntax
+end
+
+
+# Before generating syntax, we need to handle some controller-level
+# options that require global modification of the metadata.
+
+def modify_metadata
+    # Force all variables to be strings.
+    if @all_vars_as_string
+        @variables.each do |var|
+            var.is_string_var    = true
+            var.is_double_var = false
+            var.implied_decimals = 0
+        end
+    end
+
+    # If the user wants to rectangularize hierarchical data, the
+    # select_vars_by_record_type option is required.
+    @select_vars_by_record_type = true if @rectangularize
+
+    # Remove any variables not belonging to the declared record types.
+    if @select_vars_by_record_type
+        rt_lookup = rec_type_lookup_hash()
+        @variables = @variables.find_all { |var| var.is_common_var or rt_lookup[var.record_type] }
+    end
+end
+
+
+# Before generating syntax, run a sanity check on the metadata.
+
+def validate_metadata (check = {})
+    bad_metadata('no variables') if @variables.empty?
+
+    if @rectangularize
+        msg = 'the rectangularize option requires data_structure=hier'
+        bad_metadata(msg) unless @data_structure == 'hier'
+    end
+
+    if @data_structure == 'hier' or @select_vars_by_record_type
+        bad_metadata('no record types') if @record_types.empty?
+
+        msg = 'record types must be unique'
+        bad_metadata(msg) unless rec_type_lookup_hash.keys.size == @record_types.size
+
+        msg = 'all variables must have a record type'
+        bad_metadata(msg) unless @variables.find { |var| var.record_type.length == 0 }.nil?
+
+        msg = 'with no common variables, every record type needs at least one variable ('
+        if @variables.find { |var| var.is_common_var }.nil?
+            @record_types.each do |rt|
+                next if get_vars_by_record_type(rt).size > 0
+                bad_metadata(msg + rt + ')')
+            end
+        end
+    end
+
+    if @data_structure == 'hier'
+        bad_metadata('no record type variable') if record_type_var.nil?
+    end
+
+    return if check[:minimal]
+
+    @variables.each do |v|
+        v.start_column     = v.start_column.to_i
+        v.width            = v.width.to_i
+        v.implied_decimals = v.implied_decimals.to_i
+        bad_metadata("#{v.name}, start_column"    ) unless v.start_column     >  0
+        bad_metadata("#{v.name}, width"           ) unless v.width            >  0
+        bad_metadata("#{v.name}, implied_decimals") unless v.implied_decimals >= 0
+    end
+end
+
+def bad_metadata (msg)
+    msg = 'Invalid metadata: ' + msg
+    abort(msg) if @caller == 'vb' or @caller == 'dcp'
+    raise(RuntimeError, msg)
+end
+
+end
+end