stats_package_syntax_file_generator 1.1.7 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2006302f8dde73e249ee6175ef65b7c41439a14911f515f82f32f6f7dbf5403b
-  data.tar.gz: 4e3eaf39e8537c7e49ac3ea01a37c2e4440d4a07e80d5d8810b103e02257ec8d
+  metadata.gz: 8e54f08bcc60310213618ae309728257a75fde34af6dd0c9958a04279d47e111
+  data.tar.gz: 2c09c27ee28137762017fe52845ae61496310a2175adba9b2656390576b66e14
 SHA512:
-  metadata.gz: 68dd339e0624e374c374d7c006a7747dcf3800b9661e976594a6f005acdb46a00d972ab1cea25becc3ddb4c945672b3cf15bd3ede23487f21bdd5e0f3685cb5e
-  data.tar.gz: 553b2e56987e63490a7684c00e2bd3a12b4b18027a7fd60343e0d7089efc00f75f6385a49c94cd8b4301fb489b04e469819897e9c3cb815c9baafa1255ac1170
+  metadata.gz: 853aa7c596cf1964227877e6f562ad1b7e274262bd161e947e9138afff0d3d570d831efaaa96ec4694c1b6d794ba9798762b9311a41a35063127e1cde2416fc0
+  data.tar.gz: c4cfec89e1b2cf998f5aaf9b9edecd9cb2aec820bb9b04701300b378a41ed9dc0738458526255b5bf446d8aa803fce345bbb0e27c6f25c4f05a75c002f7280da

data/lib/syntax_file/controller.rb

@@ -6,7 +6,7 @@
 module SyntaxFile
   class Controller
 
-    VERSION = "1.1.7"
+    VERSION = "1.2.0"
 
     ATTR = {
       :project                    => { :req => false, :rw => 'rw', :def => '',          :yaml => true  },
@@ -52,9 +52,17 @@ module SyntaxFile
       @record_types = [] if @record_types.nil?
       @variables = [] if @variables.nil?
       @yaml_files = [] if @yaml_files.nil?
+
+      if @data_structure == 'hier' && is_csv?
+        raise(ArgumentError, 'Hierarchical data_structure is not supported for CSV data')
+      end
       read_metadata_from_yaml
     end
 
+    def is_csv?
+      @data_file_name.end_with?('.csv')
+    end
+
     # Methods to import metadata from YAML files into the Controller object.
 
     def yaml_files=(file_names)
@@ -66,11 +74,31 @@ module SyntaxFile
     def read_metadata_from_yaml
       return if @yaml_files.empty?
       md = {}
-      @yaml_files.each { |f| md.merge! YAML.load_file(f) }
+      @yaml_files.each { |f| 
+        md.merge! parse_yaml(f)
+    }
       md = symbolize_keys(md)
       load_yaml_md(md)
     end
 
+    def parse_yaml(yaml_file)      
+      if RUBY_ENGINE == 'jruby'
+        # This code exists to handle the new size limit in the Snakeyml Java library
+        # of three million code points.. The issue is discussed here:
+        # https://github.com/jruby/jruby/issues/7543
+        tree_builder = Psych::TreeBuilder.new
+        parser = Psych::Parser.new(tree_builder)
+        parser.code_point_limit = 20_000_000
+
+        yaml_data = File.read(yaml_file)
+        parser.parse(yaml_data)
+        # Convert to Ruby and get the  hash out of the document array
+        tree_builder.root.to_ruby.first
+      else
+        YAML.load_file(yaml_file)      
+      end  
+    end
+
     def load_yaml_md(md)
       # Uses metadata from yaml to set metadata-related instance variables.
       ATTR.each_key do |k|

data/lib/syntax_file/maker.rb

@@ -13,6 +13,14 @@ module SyntaxFile
       @sfc = sfc
       @syntax_type = syntax_type
       @cmd_end = ''
+
+      if @sfc.is_csv? && !supports_csv?
+        raise "CSV data not supported for #{@syntax_type.upcase} syntax files"
+      end
+    end
+
+    def supports_csv?
+      false
     end
 
     # Syntax terminator.

data/lib/syntax_file/maker_sas.rb

@@ -18,11 +18,15 @@ module SyntaxFile
       @label_max_leng = 256
       @segment_max_leng = 100
       @sas_library_handle = 'IPUMS'
-      @sas_file_handle = 'ASCIIDAT'
+      @sas_file_handle = @sfc.is_csv? ? 'CSV' : 'ASCIIDAT'
       @sas_fmt_suffix = '_f'
       @sas_data_file_name = @sas_library_handle + '.' + @sfc.data_file_name_stem
     end
 
+    def supports_csv?
+      true
+    end
+
     def syntax
       r = [
         comments_start,
@@ -138,10 +142,14 @@ module SyntaxFile
     end
 
     def syn_df_infile
-      # The LRECL specification is needed because the default behavior on some
-      # operating systems is to truncate records to 256 columns.
-      c = @sfc.last_column_used
-      'infile ' + @sas_file_handle + ' pad missover lrecl=' + c.to_s + @cmd_end
+      if @sfc.is_csv?
+        'infile ' + @sas_file_handle + " missover dsd delimiter=" + q(',') + " firstobs=2" + @cmd_end
+      else
+        # The LRECL specification is needed because the default behavior on some
+        # operating systems is to truncate records to 256 columns.
+        c = @sfc.last_column_used
+        'infile ' + @sas_file_handle + ' pad missover lrecl=' + c.to_s + @cmd_end
+      end
     end
 
     def syn_dfr
@@ -163,7 +171,7 @@ module SyntaxFile
       var_list.collect { |v|
         sprintf(@var_loc_format, v.name) +
           (v.is_string_var ? '$ ' : '  ') +
-          v.column_locations_as_s +
+          (@sfc.is_csv? ? '' : v.column_locations_as_s) +
           implied_decimal_fmt(v)
       }
     end
@@ -249,7 +257,7 @@ module SyntaxFile
 
     def syn_fmt_big_nums
       big_num_vars = @sfc.get_big_nums
-      return [] if big_num_vars.empty?
+      return [] if big_num_vars.empty? || @sfc.is_csv?
       r = [
         'format',
         syn_fmt_big_nums_for_var_list(big_num_vars),
@@ -270,7 +278,7 @@ module SyntaxFile
 
     def syn_fmt_link
       var_list = @sfc.get_vars_with_values
-      return [] if var_list.empty?
+      return [] if var_list.empty? || @sfc.is_csv?
       r = [
         'format',
         syn_fmt_link_for_var_list(var_list),
@@ -286,7 +294,7 @@ module SyntaxFile
     end
 
     def implied_decimal_fmt(var)
-      return '' if var.is_string_var or var.implied_decimals == 0
+      return '' if var.is_string_var or var.implied_decimals == 0 or @sfc.is_csv?
       return ' .' + var.implied_decimals.to_s
     end
 

data/lib/syntax_file/maker_spss.rb

@@ -17,6 +17,10 @@ module SyntaxFile
       @segment_max_leng = 100
     end
 
+    def supports_csv?
+      true
+    end
+
     def syntax
       r = [
         comments_start,
@@ -54,7 +58,33 @@ module SyntaxFile
     end
 
     def syn_df
-      @sfc.data_structure == 'hier' ? syn_dfh : syn_dfr
+      if @sfc.is_csv?
+        syn_dfr_csv
+      else
+        @sfc.data_structure == 'hier' ? syn_dfh : syn_dfr
+      end
+    end
+
+    def syn_dfr_csv
+      r = [
+        'GET DATA  /TYPE=TXT',
+        '  /FILE=' + q(@sfc.data_file_name),
+        '  /ENCODING=\'UTF8\'',
+        '  /DELIMITERS=","',
+        '  /QUALIFIER=\'"\'',
+        '  /ARRANGEMENT=DELIMITED',
+        '  /FIRSTCASE=2',
+        '  /DATATYPEMIN PERCENTAGE=100.0',
+        '  /VARIABLES=',
+        syn_vars_csv(@sfc.variables),
+        '  /MAP.',
+        'execute.'
+      ]
+      r.flatten
+    end
+
+    def syn_vars_csv(var_list)
+      var_list.map { |v| sprintf @var_loc_format, v.name, v.is_string_var ? 'A' : 'AUTO' }
     end
 
     def syn_dfr

data/lib/syntax_file/maker_stata.rb

@@ -16,7 +16,8 @@ module SyntaxFile
       @infix_format = "%#{mx_col + mx_var + 4}s"
       @replace_format = "replace %-#{mx_var}s = %-#{mx_var}s / %d"
       @fixed_point_display_format = "format %-#{mx_var}s %%%d.%df"
-      @general_display_format = "format %-#{mx_var}s %%%d.%dg"
+      @general_display_format = "format %-#{mx_var}s %%%d.0g"
+      @integer_display_format = "format %-#{mx_var}s %%%d.0f"
 
       @cmd_end = ''
       @cmd_continue = ' ///'
@@ -25,6 +26,10 @@ module SyntaxFile
       @sort_var_stem = '_line_num'
     end
 
+    def supports_csv?
+      true
+    end
+
     def syntax
       r = [
         comments_start,
@@ -79,17 +84,24 @@ module SyntaxFile
 
     def syn_infix(var_list)
       r = [
-        syn_infix_start,
-        syn_infix_var_locs(var_list),
-        syn_infix_end,
+        syn_infix_start
       ]
+      if !@sfc.is_csv?
+        r.push syn_infix_var_locs(var_list)
+        r.push syn_infix_end
+      end
       r.flatten
     end
 
     def syn_infix_start
+      if @sfc.is_csv?
+        infix_cmd = "quietly import delimited #{q(@sfc.data_file_name)}, stringcols(#{list_stringcols(@sfc.variables)})"
+      else
+        infix_cmd = 'quietly infix'
+      end
       [
         'clear',
-        'quietly infix' + sprintf(@infix_format, @cmd_continue),
+        infix_cmd + sprintf(@infix_format, @cmd_continue),
       ]
     end
 
@@ -189,7 +201,7 @@ module SyntaxFile
 
     def syn_convert_implied_decim
       var_list = @sfc.variables.find_all { |var| var.implied_decimals > 0 }
-      return [] if var_list.empty?
+      return [] if var_list.empty? || @sfc.is_csv?
       var_list.map { |var|
         v = var.name.downcase
         sprintf @replace_format, v, v, 10 ** var.implied_decimals
@@ -201,17 +213,25 @@ module SyntaxFile
         vf = var_fmt(var)
         vf == 'double' or vf == 'float'
       }
-      return [] if var_list.empty?
+      return [] if var_list.empty? || @sfc.is_csv?
       var_list.map { |var|
         v = var.name.downcase
 
-        # If implied decimals set, it means we know exactly how much precision
-        # to show. Otherwise we go with the underlying value and the 'g' (general) formatting rules in Stata
-        formatting = var.implied_decimals > 0 ?
-          sprintf(@fixed_point_display_format, v, var.width, var.implied_decimals) :
-          sprintf(@general_display_format, v, var.width, var.implied_decimals)
-
-        formatting
+        # When implied decimals are set, it means we know exactly how much precision to show: '.nf' where n is the
+        # number of implied decimals.
+        # When the number is a double float in our metadata, we go with the underlying value and the '.0g' (general)
+        # formatting rules in Stata.
+        # Otherwise the variable is a integer in our metadata, and we therefore use '.0f' for formatting
+        #
+        # In Stata you store large integer type data in 'double' or 'long double' type variables. It's confusing.
+        case
+        when var.implied_decimals > 0
+          sprintf(@fixed_point_display_format, v, var.width, var.implied_decimals)
+        when var.is_double_var
+          sprintf(@general_display_format, v, var.width)
+        else # default is integer
+          sprintf(@integer_display_format, v, var.width)
+        end
       }
     end
 
@@ -314,5 +334,17 @@ module SyntaxFile
       rt_var.name.downcase + ' != ' + val_q(rt_var, val_as_s(rt_var, rt))
     end
 
+    def list_stringcols(vars)
+      positions = []
+      index = 1
+      vars.each do |v|
+        if v.is_string_var
+          positions << index
+        end
+        index += 1
+      end
+      positions.join(' ')
+    end
+
   end
 end

data/tests/setup.rb

@@ -23,6 +23,13 @@ def new_controller
     SyntaxFile::Controller.new(:yaml_files => YAML_FILES)
 end
 
+def new_controller_csv
+    controller = new_controller
+    controller.data_file_name = 'data.csv'
+    controller.data_structure = 'rect'
+    controller
+end
+
 def new_variable
     SyntaxFile::Variable.new params_variable()
 end
@@ -31,9 +38,10 @@ def new_value
     SyntaxFile::Value.new params_value()
 end
 
-def new_maker(syntax_type = '')
+def new_maker(syntax_type = '', csv: false)
     maker_class = 'SyntaxFile::Maker' + syntax_type.upcase
-    eval(maker_class).new(new_controller, syntax_type)
+    controller = csv ? new_controller_csv : new_controller
+    eval(maker_class).new(controller, syntax_type)
 end
 
 # Parameters used when creating objects with known values.

data/tests/tc_controller.rb

@@ -374,5 +374,12 @@ def test_rec_type_lookup_hash
     assert_equal( {}, sfc.rec_type_lookup_hash, msg )
 end
 
+def test_hier_csv
+    error = assert_raises ArgumentError do
+        SyntaxFile::Controller.new(data_structure: 'hier', data_file_name: 'data.csv')
+    end
+    assert_equal('Hierarchical data_structure is not supported for CSV data', error.message)
+end
+
 end
 end

data/tests/tc_maker_sas.rb

@@ -247,5 +247,43 @@ def test_non_last_non_common_vars
     assert_equal [], vars_to_names(var_list), msg
 end
 
+def test_csv_import
+    msg = 'Compare against hardcoded result.'
+    mk = new_maker('sas', csv: true)
+    expected = [
+      'data IPUMS.data;',
+      'infile CSV missover dsd delimiter="," firstobs=2;',
+      '',
+      'input',
+      "  RECTYPE  $ ",
+      "  DWNUM      ",
+      "  HHNUM      ",
+      "  HDFIRSTD   ",
+      "  FBIG_ND    ",
+      "  BADDW      ",
+      "  CANTON     ",
+      "  URBAN      ",
+      "  DWTYPE     ",
+      "  OWNERSHP   ",
+      "  RENT     $ ",
+      "  RELATE     ",
+      "  SEX        ",
+      "  AGE        ",
+      "  RESPREV2   ",
+      "  SOCSEC     ",
+      "  EDLEVEL    ",
+      "  LIT        ",
+      "  BIGDEC     ",
+      "  BIGINT     ",
+      "  BIGSTR   $ ",
+      ";",
+      ''
+    ]
+    assert_equal expected, mk.syn_df, msg
+
+    expected2 = []
+    assert_equal expected2, mk.syn_fmt_link, msg
+end
+
 end
 end

data/tests/tc_maker_spss.rb

@@ -116,6 +116,17 @@ def test_var_fmt
     assert_equal expected, actual, msg
 end
 
+def test_csv_import
+    msg = 'Compare against hardcoded result.'
+    mk = new_maker('spss', csv: true)
+    syn_df = mk.syn_df
+    assert_equal 'GET DATA  /TYPE=TXT', syn_df[0], msg
+    assert_equal '  /FILE="data.csv"', syn_df[1], msg
+    assert_equal '  RECTYPE   A', syn_df[9], msg
+    assert_equal '  /MAP.', syn_df[-2], msg
+    assert_equal 'execute.', syn_df[-1], msg
+end
+
 
 end
 end

data/tests/tc_maker_stata.rb

@@ -145,7 +145,7 @@ def test_syn_display_format
         "format canton   %3.1f",
         "format resprev2 %4.3f",
         "format bigdec   %10.5f",
-        "format bigint   %19.0g"
+        "format bigint   %19.0f"
     ]
     actual = mk.syn_display_format
     assert_equal expected, actual, msg
@@ -220,5 +220,12 @@ def test_rt_ne_statement
     assert_equal expected, mk.rt_ne_statement('H'), msg
 end
 
+def test_csv_import
+    msg = 'Compare against hardcoded result.'
+    mk = new_maker('stata', csv: true)
+    expected = ['clear', 'quietly import delimited `"data.csv"\', stringcols(1 11 21)                ///']
+    assert_equal expected, mk.syn_df, msg
+end
+
 end
 end

data/tests/tc_maker_sts.rb

@@ -185,5 +185,12 @@ def test_syn_var_locations
     assert_equal expected, actual, msg
 end
 
+def test_csv_import
+    error = assert_raises RuntimeError do
+        new_maker('sts', csv: true)
+    end
+    assert_equal('CSV data not supported for STS syntax files', error.message)
+end
+
 end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stats_package_syntax_file_generator
 version: !ruby/object:Gem::Version
-  version: 1.1.7
+  version: 1.2.0
 platform: ruby
 authors:
 - Monty Hindman, Marcus Peterson, Colin Davis, Dan Elbert, Jayandra Pokharel
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-03-29 00:00:00.000000000 Z
+date: 2026-02-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -88,7 +88,7 @@ homepage: https://github.com/mnpopcenter/stats_package_syntax_file_generator
 licenses:
 - MPL-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -103,8 +103,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key: 
+rubygems_version: 3.5.14
+signing_key:
 specification_version: 4
 summary: Produces statistical package syntax files for fixed-column data.
 test_files: