stats_package_syntax_file_generator 1.0.1

data/lib/stats_package_syntax_file_generator/maker.rb

@@ -0,0 +1,126 @@
+# This file is part of the Minnesota Population Center's stats_package_syntax_file_generator project.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/mnpopcenter/stats_package_syntax_file_generator
+
+module StatsPackageSyntaxFileGenerator
+class Maker
+
+attr_reader   :sfc, :syntax_type
+attr_accessor :cmd_end
+
+def initialize (sfc, syntax_type)
+    @sfc         = sfc
+    @syntax_type = syntax_type
+    @cmd_end     = ''
+end
+
+
+# Syntax terminator.
+
+def syntax_end
+    [ @cmd_end, blank ]
+end
+
+def blank
+    ''
+end
+
+
+# Quoting methods.
+
+def q (s)
+    '"' + s.to_s.gsub('"', '""') + '"'
+end
+
+def val_q (var, v)
+    var.is_string_var ? q(v) : v.to_s
+end
+
+def val_as_s (var, val_orig)
+    # Write a value in a syntax file varies by variable type:
+    #   - Numeric variable: simply return the value as a string.
+    #   - String  variable: zero-pad the value if it looks like an integer.
+    v = val_orig.to_s
+    return v unless var.is_string_var
+    return v unless v =~ /^\-?\d+$/
+    sprintf('%0' + var.width.to_s + 'i', v.to_i)
+end
+
+
+# Methods to deal with long labels.
+
+def label_trunc (label, limit)
+    label.to_s[0,limit]
+end
+
+def label_segments (label, max_length)
+    # Takes a string and a max length.
+    # Returns the array of strings that results from chopping the
+    # original string into segments no longer than max length.
+    # This is needed because some stats packages have max line lengths.
+    label = label.to_s
+    return [label] if label.length <= max_length
+    label = String.new(label)
+    r = []
+    r.push( label.slice!(0,max_length) ) while label.length > 0
+    r
+end
+
+def weave_label_segments (fmt, a, b, op_a, op_c)
+    # The function takes a sprintf format, two lists (a, b), and
+    # two strings (the assignment and concatenation operators used
+    # by the stats package). The purpose of the function is to handle
+    # long values and labels for stats packages that have a max syntax
+    # line length. See unit tests for an illustration.
+    r = []
+    r.push(sprintf(fmt, a.shift, op_c, ''     )) while a.size > 1
+    r.push(sprintf(fmt, a.shift, op_a, b.shift))
+    r.push(sprintf(fmt, '',      op_c, b.shift)) while b.size > 0
+    r
+end
+
+
+# Helper methods for values and their labels.
+
+def labelable_values (var)
+    # For non-string variables, only values that look
+    # like integers can be labeled.
+    return var.values if var.is_string_var
+    var.values.find_all { |val| val.value.to_s =~ /^\-?\d+$/ }
+end
+
+def max_value_length (var, val_list)
+    return 0 if val_list.empty?
+    val_list.map { |val| val_as_s(var, val.value).length }.max
+end
+
+
+# Methods for comments at the start or end of the syntax file.
+
+def comments_start
+    # Comments not needed unless syntax file is for the web app.
+    return [] unless @sfc.caller == 'web_app'
+
+    return [
+        'NOTE: You need to set the Stata working directory to the path',
+        'where the data file is located.',
+    ] if @syntax_type == 'stata'
+
+    cmd = (@syntax_type == 'sas') ? 'libname' : 'cd'
+    result = [
+        "NOTE: You need to edit the `#{cmd}` command to specify the path to the directory",
+        'where the data file is located. For example: "C:\ipums_directory".'
+    ]
+    if @syntax_type == 'sas'
+      result << "Edit the `filename` command similarly to include the full path (the directory and the data file name)."
+    end
+    result
+end
+
+def comments_end
+    []
+end
+
+end
+end

data/lib/stats_package_syntax_file_generator/maker_sas.rb

@@ -0,0 +1,306 @@
+# This file is part of the Minnesota Population Center's stats_package_syntax_file_generator project.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/mnpopcenter/stats_package_syntax_file_generator
+
+module StatsPackageSyntaxFileGenerator
+class MakerSAS < Maker
+
+def initialize (sfc, syntax_type)
+    super
+    m = @sfc.max_var_name_length
+    @var_loc_format     = "  %-#{m}s "
+    @var_lab_format     = "  %-#{m}s %s %s"
+    @fmt_link_format    = "  %-#{m}s  %s."
+    @bignum_int_format  = "  %-#{m}s  %d."
+    @bignum_dec_format  = "  %-#{m}s  %d.%d"
+    @cmd_end            = ';'
+    @label_max_leng     = 256
+    @segment_max_leng   = 100
+    @sas_library_handle = 'IPUMS'
+    @sas_file_handle    = 'ASCIIDAT'
+    @sas_fmt_suffix     = '_f'
+    @sas_data_file_name = @sas_library_handle + '.' + @sfc.data_file_name_stem
+end
+
+def syntax
+    r = [
+        comments_start,
+        syn_libname,
+        syn_filename,
+        blank,
+        syn_val_labs,
+        syn_df,
+        syn_var_labs,
+        syn_fmt_link,
+        syn_fmt_big_nums,
+        syn_run,
+        comments_end,
+    ]
+    r.flatten!
+end
+
+def comments_start
+    convert_to_comments(super)
+end
+
+def comments_end
+    convert_to_comments(super)
+end
+
+def convert_to_comments (lines)
+    return [] if lines.empty?
+    [
+        '/*',
+        lines.map { |ln| '   ' + ln },
+        '*/',
+        blank,
+    ].flatten
+end
+
+def syn_libname
+    'libname ' + @sas_library_handle + ' ' + q(@sfc.data_dir_name) + @cmd_end
+end
+
+def syn_filename
+    'filename ' + @sas_file_handle + ' ' + q(@sfc.data_file_name) + @cmd_end
+end
+
+def syn_val_labs
+    var_list = @sfc.get_vars_with_values
+    return [] if var_list.empty?
+    r = [
+        syn_proc_format,
+        blank,
+        syn_val_labs_for_var_list(var_list),
+        syn_run,
+    ]
+    r.flatten!
+end
+
+def syn_proc_format
+    'proc format cntlout = ' + @sas_data_file_name + @sas_fmt_suffix + @cmd_end
+end
+
+def syn_val_labs_for_var_list (var_list)
+    r = []
+    var_list.each do |var|
+        r.push syn_val_labs_for_var_start(var)
+        r.push syn_val_labs_for_var(var)
+        r.push syntax_end
+    end
+    r.flatten!
+end
+
+def syn_val_labs_for_var_start (var)
+    'value'                           +
+    (var.is_string_var ? ' $ ' : ' ') +
+    var.name                          +
+    @sas_fmt_suffix
+end
+
+def syn_val_labs_for_var (var)
+    val_list = labelable_values(var)
+    return [] if val_list.empty?
+    m = max_value_length(var, val_list)
+    m = m + 2 if var.is_string_var
+    m = @segment_max_leng + 2 if m > @segment_max_leng + 2
+    fmt = "  %-#{m}s %s %s"
+    r = val_list.collect { |val| syn_val_lab_for_val(var, val, fmt) }
+    r.flatten!
+end
+
+def syn_val_lab_for_val (var, val, fmt)
+    lab = label_trunc(val.label, @label_max_leng)
+    lab = val.value if lab.nil? || (lab.strip.length == 0)
+    vs = val_as_s(var, val.value)
+    val_segments = label_segments(vs,  @segment_max_leng).map { |s| val_q(var, s) }
+    lab_segments = label_segments(lab, @segment_max_leng).map { |s| q(s) }
+    weave_label_segments(fmt, val_segments, lab_segments, '=', ' ')
+end
+
+def syn_run
+    [ 'run' + @cmd_end, blank ]
+end
+
+def syn_df
+    r = [
+        syn_df_start,
+        syn_df_infile,
+        blank,
+        @sfc.data_structure == 'hier' ? syn_dfh : syn_dfr,
+    ]
+    r.flatten!
+end
+
+def syn_df_start
+    'data ' + @sas_data_file_name + @cmd_end
+end
+
+def syn_df_infile
+    # The LRECL specification is needed because the default behavior on some
+    # operating systems is to truncate records to 256 columns.
+    c = @sfc.last_column_used
+    'infile ' + @sas_file_handle + ' pad missover lrecl=' + c.to_s + @cmd_end
+end
+
+def syn_dfr
+    r = syn_input(@sfc.variables)
+    r.push blank
+    r
+end
+
+def syn_input (var_list)
+    r = [
+        'input',
+        syn_var_locations(var_list),
+        @cmd_end,
+    ]
+    r.flatten!
+end
+
+def syn_var_locations (var_list)
+    var_list.collect { |v|
+        sprintf( @var_loc_format, v.name ) +
+        (v.is_string_var ? '$ ' : '  ')    +
+        v.column_locations_as_s            +
+        implied_decimal_fmt(v)
+    }
+end
+
+def syn_dfh
+    r = [
+        syn_dfh_retain,
+        syn_dfh_rec_type_block,
+        blank,
+        syn_dfh_if_blocks,
+    ]
+    r.flatten!
+end
+
+def syn_dfh_retain
+    return [] unless @sfc.rectangularize
+    var_list = non_last_non_common_vars
+    return [] if var_list.size == 0
+    r = [
+        'retain',
+        var_list.map { |var| '  ' + var.name },
+        syntax_end,
+    ]
+    r.flatten!
+end
+
+def syn_dfh_rec_type_block
+    r = syn_input( [@sfc.record_type_var])
+    r[1] = r[1] + ' @'
+    r
+end
+
+def syn_dfh_if_blocks
+    if_cmd = 'if'
+    r = []
+    @sfc.record_types.each do |rt|
+        r.push(
+            syn_dfh_if_start(if_cmd, rt),
+            syn_input( @sfc.get_vars_by_record_type(rt) ),
+            syn_dfh_if_end(rt)
+        )
+        if_cmd = 'else if'
+    end
+    r.flatten!
+end
+
+def syn_dfh_if_start (if_cmd, rt)
+    rt_var = @sfc.record_type_var
+    r = [
+        if_cmd,
+        @sfc.record_type_var.name,
+        '=',
+        val_q(rt_var, val_as_s(rt_var, rt)),
+        'then do' + @cmd_end,
+    ]
+    r.join(' ')
+end
+
+def syn_dfh_if_end (rt)
+    r = []
+    r.push 'output' + @cmd_end if (not @sfc.rectangularize) or @sfc.is_last_record_type(rt)
+    r.push 'end'    + @cmd_end
+    r.push blank
+    r
+end
+
+def syn_var_labs
+    var_list = @sfc.get_vars_with_var_labels
+    return [] if var_list.empty?
+    r = [
+        'label',
+        var_list.map { |var| syn_var_lab_for_var(var) },
+        syntax_end,
+    ]
+    r.flatten!
+end
+
+def syn_var_lab_for_var (var)
+    lab = label_trunc(var.label, @label_max_leng)
+    lab_segments = label_segments(lab, @segment_max_leng).map { |s| q(s) }
+    weave_label_segments(@var_lab_format, [var.name], lab_segments, '=', ' ')
+end
+
+def syn_fmt_big_nums
+    big_num_vars = @sfc.get_big_nums
+    return [] if big_num_vars.empty?
+    r = [
+        'format',
+        syn_fmt_big_nums_for_var_list(big_num_vars),
+        syntax_end,
+    ]
+    r.flatten!
+end
+
+def syn_fmt_big_nums_for_var_list (var_list)
+  var_list.map do |v|
+    if v.implied_decimals > 0
+      sprintf @bignum_dec_format, v.name, v.width + 1, v.implied_decimals
+    else
+      sprintf @bignum_int_format, v.name, v.width
+    end
+  end
+end
+
+
+def syn_fmt_link
+    var_list = @sfc.get_vars_with_values
+    return [] if var_list.empty?
+    r = [
+        'format',
+        syn_fmt_link_for_var_list(var_list),
+        syntax_end,
+    ]
+    r.flatten!
+end
+
+def syn_fmt_link_for_var_list (var_list)
+    var_list.map { |v|
+        sprintf @fmt_link_format, v.name, v.name + @sas_fmt_suffix
+    }
+end
+
+def implied_decimal_fmt (var)
+    return '' if var.is_string_var or var.implied_decimals == 0
+    return ' .' + var.implied_decimals.to_s
+end
+
+def non_last_non_common_vars
+    # Returns a list of variables, excluding:
+    #    - variables from the last record type
+    #    - common variables
+    var_list = @sfc.rec_types_except_last.map do |rt|
+        vars = @sfc.get_vars_by_record_type(rt)
+        vars.find_all { |var| not var.is_common_var }
+    end
+    var_list.flatten!
+end
+
+end
+end

data/lib/stats_package_syntax_file_generator/maker_spss.rb

@@ -0,0 +1,194 @@
+# This file is part of the Minnesota Population Center's stats_package_syntax_file_generator project.
+# For copyright and licensing information, see the NOTICE and LICENSE files
+# in this project's top-level directory, and also on-line at:
+#   https://github.com/mnpopcenter/stats_package_syntax_file_generator
+
+module StatsPackageSyntaxFileGenerator
+class MakerSPSS < Maker
+
+def initialize (sfc, syntax_type)
+    super
+    m = @sfc.max_var_name_length
+    @var_loc_format     = "  %-#{m}s  %s"
+    @var_lab_format     = "  %-#{m}s  %s %s"
+    @cmd_end            = '.'
+    @var_label_max_leng = 120
+    @val_label_max_leng = 120
+    @segment_max_leng   = 100
+end
+
+def syntax
+    r = [
+        comments_start,
+        syn_cd,
+        blank,
+        syn_df,
+        syn_var_labs,
+        syn_val_labs,
+        syn_execute,
+        blank,
+        comments_end,
+    ]
+    r.flatten
+end
+
+def comments_start
+    convert_to_comments(super)
+end
+
+def comments_end
+    convert_to_comments(super)
+end
+
+def convert_to_comments (lines)
+    return [] if lines.empty?
+    lines.push @cmd_end
+    [
+        lines.map { |ln| '* ' + ln },
+        blank,
+    ].flatten
+end
+
+def syn_cd
+    'cd ' + q(@sfc.data_dir_name) + @cmd_end
+end
+
+def syn_df
+    @sfc.data_structure == 'hier' ? syn_dfh : syn_dfr
+end
+
+def syn_dfr
+    r = [
+        syn_dfr_start,
+        syn_var_locations(@sfc.variables),
+        syntax_end,
+    ]
+    r.flatten
+end
+
+def syn_dfr_start
+    'data list file = ' + q(@sfc.data_file_name) + ' /'
+end
+
+def syn_dfh
+    r = [
+        syn_dfh_file_type,
+        syn_dfh_data_blocks,
+        'end file type' + @cmd_end,
+        blank,
+    ]
+    r.flatten
+end
+
+def syn_dfh_file_type
+    r = [
+        'file type ' + nested_or_mixed(),
+        '  /file = ' + q(@sfc.data_file_name),
+        '  /record = ' + var_loc_with_fmt(@sfc.record_type_var).to_s,
+        syntax_end,
+    ]
+    r.flatten
+end
+
+def syn_dfh_data_blocks
+    r = @sfc.record_types.map { |rt|
+        [
+            syn_dfh_data_block_start(rt),
+            syn_var_locations(@sfc.get_vars_by_record_type(rt)),
+            syntax_end,
+        ]
+    }
+    r.flatten
+end
+
+def syn_dfh_data_block_start (rt)
+    rt_var = @sfc.record_type_var
+    [
+        'record type ' + val_q(rt_var, val_as_s(rt_var, rt)) + @cmd_end,
+        'data list /',
+    ]
+end
+
+def syn_var_locations (var_list)
+    var_list.map { |v| sprintf @var_loc_format, v.name, var_loc_with_fmt(v) }
+end
+
+def syn_var_labs (var_list = [])
+    var_list = @sfc.get_vars_with_var_labels if var_list.empty?
+    return [] if var_list.empty?
+    r = [
+        'variable labels',
+        var_list.map { |var| syn_var_lab_for_var(var) },
+        syntax_end,
+    ]
+    r.flatten
+end
+
+def syn_var_lab_for_var (var)
+    lab = label_trunc(var.label, @var_label_max_leng)
+    lab_segments = label_segments(lab, @segment_max_leng).map { |s| q(s) }
+    weave_label_segments(@var_lab_format, [var.name], lab_segments, ' ', '+')
+end
+
+def syn_val_labs
+    var_list = @sfc.get_vars_with_values
+    return [] if var_list.empty?
+    r = [
+        'value labels',
+        syn_val_labs_for_var_list(var_list),
+        syntax_end,
+    ]
+    r.flatten
+end
+
+def syn_val_labs_for_var_list (var_list)
+    var_list.map { |var| syn_val_labs_for_var(var) }
+end
+
+def syn_val_labs_for_var (var)
+    val_list = labelable_values(var)
+    return [] if val_list.empty?
+    m = max_value_length(var, val_list)
+    m = m + 2 if var.is_string_var
+    m = @segment_max_leng + 2 if m > @segment_max_leng + 2
+    value_format = "    %-#{m}s %s %s"
+    r = [
+        syn_val_labs_for_var_start(var),
+        val_list.map { |val| syn_val_lab_for_val(var, val, value_format) },
+    ]
+    r.flatten
+end
+
+def syn_val_labs_for_var_start (var)
+    '  /' + var.name
+end
+
+def syn_val_lab_for_val (var, val, fmt)
+    lab = label_trunc(val.label, @val_label_max_leng)
+    lab = val.value if lab.nil? || (lab.strip.length == 0)
+    vs = val_as_s(var, val.value)
+    val_segments = label_segments(vs,  @segment_max_leng).map { |s| val_q(var, s) }
+    lab_segments = label_segments(lab, @segment_max_leng).map { |s| q(s) }
+    weave_label_segments(fmt, val_segments, lab_segments, ' ', '+')
+end
+
+def syn_execute
+    'execute' + @cmd_end
+end
+
+def nested_or_mixed
+    @sfc.rectangularize ? 'nested' : 'mixed'
+end
+
+def var_loc_with_fmt (var)
+    var.column_locations_as_s + var_fmt(var)
+end
+
+def var_fmt (var)
+    return ' (a)' if var.is_string_var
+    return '' unless var.implied_decimals > 0
+    return ' (' + var.implied_decimals.to_s + ')'
+end
+
+end
+end