csvreader 1.1.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b7c45afa79ca462aef1a8a9f1f6da26b7b08e125
-  data.tar.gz: b1a20a1acf5e88331900169f5d3542fb4f866f58
+  metadata.gz: c522e332ef3c1fead487b99d5fe147ba43ad2090
+  data.tar.gz: 51dd6d88ef8dc35615513961bab7e0e1c3b3512b
 SHA512:
-  metadata.gz: 7cf456f8e886e687ade020c70830332c1b2c3bdb1d14a148ed9a01ce2d7d1e9acffab2a3435907641fad220830f3eee39c20d8ca645e096bae5fe0bb2184a075
-  data.tar.gz: a887567bd67a3080d99adb2a609d8e79a5385b83abe1109f1f4d24a1ebd5fb9536e38701e8fefe5020c16753f674d45b673d00b6300b3422d713a630c3dbf0d6
+  metadata.gz: 7e563f75e916829e8de1b0a3b1208dd089de9a7907d010e3ba2cd23f1a70fedcb8d98c95e65c15ab7d3ad8705ae41a4ad6cd543ba20d6a72dc67f27b0060286b
+  data.tar.gz: 57036e2457b4dc1837748538062150650b47abef3d2493f4c4f42db4291fdd3001cb6fb218eca38c7c11816c67360cfea74030e137ff7edf8de1fb9e47f991ec

data/Manifest.txt

@@ -22,6 +22,7 @@ test/data/beer11.csv
 test/data/cars11.csv
 test/data/cities11.csv
 test/data/customers11.csv
+test/data/iris11.csv
 test/data/shakespeare.csv
 test/helper.rb
 test/test_buffer.rb

data/README.md

@@ -422,7 +422,7 @@ Csv.strict.read( ..., sep: "\t" )
 
 
 
-### Q: How can I read records with fixed width (and no separator)?
+### Q: How can I read records with fixed width fields (and no separator)?
 
 Pass in the `width` keyword option with the field widths / lengths
 to the "fixed" parser. Example:
@@ -432,7 +432,7 @@ txt = <<TXT
 12345678123456781234567890123456789012345678901212345678901234
 TXT
 
-Csv.fixed.parse( txt, width: [8,8,32,14] )
+Csv.fixed.parse( txt, width: [8,8,32,14] )  # or Csv.fix or Csv.f
 # => [["12345678","12345678", "12345678901234567890123456789012", "12345678901234"]]
 
 
@@ -441,15 +441,68 @@ John    Smith   john@example.com                1-888-555-6666
 Michele O'Reileymichele@example.com             1-333-321-8765
 TXT
 
-Csv.fixed.parse( txt, width: [8,8,32,14] )
+Csv.fixed.parse( txt, width: [8,8,32,14] )   # or Csv.fix or Csv.f
 # => [["John",    "Smith",    "john@example.com",    "1-888-555-6666"],
 #     ["Michele", "O'Reiley", "michele@example.com", "1-333-321-8765"]]
 
 # and so on
 ```
 
+<!--
 Note: You can use for your convenience the built-in
 `Csv.fix` or `Csv.f` aliases / shortcuts.
+-->
+
+
+Note: You can use negative widths (e.g. `-2`, `-3`, and so on)
+to "skip" filler fields (e.g. `--`, `---`, and so on).
+Example:
+
+``` ruby
+txt = <<TXT
+12345678--12345678---12345678901234567890123456789012--12345678901234XXX
+TXT
+
+Csv.fixed.parse( txt, width: [8,-2,8,-3,32,-2,14] )  # or Csv.fix or Csv.f
+# => [["12345678","12345678", "12345678901234567890123456789012", "12345678901234"]]
+```
+
+
+Bonus: If the width is a string (not an array)
+(e.g. `'a8 a8 a32 Z*'` or `'A8 A8 A32 Z*'` and so on)
+than the fixed width field parser
+will use  `String#unpack` and the value of width as its format string spec.
+Example:
+
+``` ruby
+txt = <<TXT
+12345678123456781234567890123456789012345678901212345678901234
+TXT
+
+Csv.fixed.parse( txt, width: 'a8 a8 a32 Z*' )  # or Csv.fix or Csv.f
+# => [["12345678","12345678", "12345678901234567890123456789012", "12345678901234"]]
+
+txt = <<TXT
+John    Smith   john@example.com                1-888-555-6666
+Michele O'Reileymichele@example.com             1-333-321-8765
+TXT
+
+Csv.fixed.parse( txt, width: 'A8 A8 A32 Z*' )   # or Csv.fix or Csv.f
+# => [["John",    "Smith",    "john@example.com",    "1-888-555-6666"],
+#     ["Michele", "O'Reiley", "michele@example.com", "1-333-321-8765"]]
+
+# and so on
+```
+
+| String Directive | Returns | Meaning                 |
+|------------------|---------|-------------------------|
+| `A`              | String  | Arbitrary binary string (remove trailing nulls and ASCII spaces) |
+| `a`              | String  | Arbitrary binary string |
+| `Z`              | String  | Null-terminated string  |
+
+
+and many more. See the `String#unpack` documentation
+for the complete format spec and directives.
 
 
 

data/lib/csvreader/parser_fixed.rb

@@ -69,15 +69,31 @@ def parse_lines( input, width:, &block )
     end
 
 
-    values = []
-    offset = 0  # start position / offset
-    width.each_with_index do |w,i|
-       logger.debug "[#{i}] start: #{offset}, width: #{w}"   if logger.debug?
-
-       value = line[offset, w]
-       value = value.strip   if value    ## note: if not nil strip; only use rstrip (for trailing only) - why? why not?
-       values << value
-       offset += w
+    if width.is_a?( String )
+      ## assume it's String#unpack format e.g.
+      ##   "209231-231992395    MoreData".unpack('aa5A1A9a4Z*')
+      ##     returns an array as follows :
+      ##   ["2", "09231", "-", "231992395", "    ", "MoreData"]
+      ##  see String#unpack
+
+      values = line.unpack( width )
+    else  ## assume array with integers
+      values = []
+      offset = 0  # start position / offset
+      width.each_with_index do |w,i|
+         logger.debug "[#{i}] start: #{offset}, width: #{w}"   if logger.debug?
+
+         if w < 0   ## convention - if width negative, skip column
+            # note: minus (-) and minus (-) equal plus (+)
+            ##   e.g. 2 - -2 = 4
+           offset -= w
+         else
+           value = line[offset, w]
+           value = value.strip   if value    ## note: if not nil strip; only use rstrip (for trailing only) - why? why not?
+           values << value
+           offset += w
+         end
+      end
     end
 
     ## note: requires block - enforce? how? why? why not?

data/lib/csvreader/version.rb

@@ -5,7 +5,7 @@ class CsvReader   ## note: uses a class for now - change to module - why? why no
 
   MAJOR = 1    ## todo: namespace inside version or something - why? why not??
   MINOR = 1
-  PATCH = 0
+  PATCH = 1
   VERSION = [MAJOR,MINOR,PATCH].join('.')
 
 

data/test/data/iris11.csv

@@ -0,0 +1,163 @@
+###
+# The Iris flower data set or Fisher's Iris data set is a multivariate data set
+#  introduced by the British statistician and biologist Ronald Fisher in his 1936 paper
+#  The use of multiple measurements in taxonomic problems as an example of
+#  linear discriminant analysis.
+#   See https://en.wikipedia.org/wiki/Iris_flower_data_set
+
+
+# The dataset contains a set of 150 records under five attributes
+# - petal length, petal width, sepal length, sepal width and species.
+
+
+Sepal length, Sepal width, Petal length, Petal width, Species
+5.1, 3.5, 1.4, 0.2, I. setosa
+4.9, 3.0, 1.4, 0.2, I. setosa
+4.7, 3.2, 1.3, 0.2, I. setosa
+4.6, 3.1, 1.5, 0.2, I. setosa
+5.0, 3.6, 1.4, 0.3, I. setosa
+5.4, 3.9, 1.7, 0.4, I. setosa
+4.6, 3.4, 1.4, 0.3, I. setosa
+5.0, 3.4, 1.5, 0.2, I. setosa
+4.4, 2.9, 1.4, 0.2, I. setosa
+4.9, 3.1, 1.5, 0.1, I. setosa
+5.4, 3.7, 1.5, 0.2, I. setosa
+4.8, 3.4, 1.6, 0.2, I. setosa
+4.8, 3.0, 1.4, 0.1, I. setosa
+4.3, 3.0, 1.1, 0.1, I. setosa
+5.8, 4.0, 1.2, 0.2, I. setosa
+5.7, 4.4, 1.5, 0.4, I. setosa
+5.4, 3.9, 1.3, 0.4, I. setosa
+5.1, 3.5, 1.4, 0.3, I. setosa
+5.7, 3.8, 1.7, 0.3, I. setosa
+5.1, 3.8, 1.5, 0.3, I. setosa
+5.4, 3.4, 1.7, 0.2, I. setosa
+5.1, 3.7, 1.5, 0.4, I. setosa
+4.6, 3.6, 1.0, 0.2, I. setosa
+5.1, 3.3, 1.7, 0.5, I. setosa
+4.8, 3.4, 1.9, 0.2, I. setosa
+5.0, 3.0, 1.6, 0.2, I. setosa
+5.0, 3.4, 1.6, 0.4, I. setosa
+5.2, 3.5, 1.5, 0.2, I. setosa
+5.2, 3.4, 1.4, 0.2, I. setosa
+4.7, 3.2, 1.6, 0.2, I. setosa
+4.8, 3.1, 1.6, 0.2, I. setosa
+5.4, 3.4, 1.5, 0.4, I. setosa
+5.2, 4.1, 1.5, 0.1, I. setosa
+5.5, 4.2, 1.4, 0.2, I. setosa
+4.9, 3.1, 1.5, 0.2, I. setosa
+5.0, 3.2, 1.2, 0.2, I. setosa
+5.5, 3.5, 1.3, 0.2, I. setosa
+4.9, 3.6, 1.4, 0.1, I. setosa
+4.4, 3.0, 1.3, 0.2, I. setosa
+5.1, 3.4, 1.5, 0.2, I. setosa
+5.0, 3.5, 1.3, 0.3, I. setosa
+4.5, 2.3, 1.3, 0.3, I. setosa
+4.4, 3.2, 1.3, 0.2, I. setosa
+5.0, 3.5, 1.6, 0.6, I. setosa
+5.1, 3.8, 1.9, 0.4, I. setosa
+4.8, 3.0, 1.4, 0.3, I. setosa
+5.1, 3.8, 1.6, 0.2, I. setosa
+4.6, 3.2, 1.4, 0.2, I. setosa
+5.3, 3.7, 1.5, 0.2, I. setosa
+5.0, 3.3, 1.4, 0.2, I. setosa
+7.0, 3.2, 4.7, 1.4, I. versicolor
+6.4, 3.2, 4.5, 1.5, I. versicolor
+6.9, 3.1, 4.9, 1.5, I. versicolor
+5.5, 2.3, 4.0, 1.3, I. versicolor
+6.5, 2.8, 4.6, 1.5, I. versicolor
+5.7, 2.8, 4.5, 1.3, I. versicolor
+6.3, 3.3, 4.7, 1.6, I. versicolor
+4.9, 2.4, 3.3, 1.0, I. versicolor
+6.6, 2.9, 4.6, 1.3, I. versicolor
+5.2, 2.7, 3.9, 1.4, I. versicolor
+5.0, 2.0, 3.5, 1.0, I. versicolor
+5.9, 3.0, 4.2, 1.5, I. versicolor
+6.0, 2.2, 4.0, 1.0, I. versicolor
+6.1, 2.9, 4.7, 1.4, I. versicolor
+5.6, 2.9, 3.6, 1.3, I. versicolor
+6.7, 3.1, 4.4, 1.4, I. versicolor
+5.6, 3.0, 4.5, 1.5, I. versicolor
+5.8, 2.7, 4.1, 1.0, I. versicolor
+6.2, 2.2, 4.5, 1.5, I. versicolor
+5.6, 2.5, 3.9, 1.1, I. versicolor
+5.9, 3.2, 4.8, 1.8, I. versicolor
+6.1, 2.8, 4.0, 1.3, I. versicolor
+6.3, 2.5, 4.9, 1.5, I. versicolor
+6.1, 2.8, 4.7, 1.2, I. versicolor
+6.4, 2.9, 4.3, 1.3, I. versicolor
+6.6, 3.0, 4.4, 1.4, I. versicolor
+6.8, 2.8, 4.8, 1.4, I. versicolor
+6.7, 3.0, 5.0, 1.7, I. versicolor
+6.0, 2.9, 4.5, 1.5, I. versicolor
+5.7, 2.6, 3.5, 1.0, I. versicolor
+5.5, 2.4, 3.8, 1.1, I. versicolor
+5.5, 2.4, 3.7, 1.0, I. versicolor
+5.8, 2.7, 3.9, 1.2, I. versicolor
+6.0, 2.7, 5.1, 1.6, I. versicolor
+5.4, 3.0, 4.5, 1.5, I. versicolor
+6.0, 3.4, 4.5, 1.6, I. versicolor
+6.7, 3.1, 4.7, 1.5, I. versicolor
+6.3, 2.3, 4.4, 1.3, I. versicolor
+5.6, 3.0, 4.1, 1.3, I. versicolor
+5.5, 2.5, 4.0, 1.3, I. versicolor
+5.5, 2.6, 4.4, 1.2, I. versicolor
+6.1, 3.0, 4.6, 1.4, I. versicolor
+5.8, 2.6, 4.0, 1.2, I. versicolor
+5.0, 2.3, 3.3, 1.0, I. versicolor
+5.6, 2.7, 4.2, 1.3, I. versicolor
+5.7, 3.0, 4.2, 1.2, I. versicolor
+5.7, 2.9, 4.2, 1.3, I. versicolor
+6.2, 2.9, 4.3, 1.3, I. versicolor
+5.1, 2.5, 3.0, 1.1, I. versicolor
+5.7, 2.8, 4.1, 1.3, I. versicolor
+6.3, 3.3, 6.0, 2.5, I. virginica
+5.8, 2.7, 5.1, 1.9, I. virginica
+7.1, 3.0, 5.9, 2.1, I. virginica
+6.3, 2.9, 5.6, 1.8, I. virginica
+6.5, 3.0, 5.8, 2.2, I. virginica
+7.6, 3.0, 6.6, 2.1, I. virginica
+4.9, 2.5, 4.5, 1.7, I. virginica
+7.3, 2.9, 6.3, 1.8, I. virginica
+6.7, 2.5, 5.8, 1.8, I. virginica
+7.2, 3.6, 6.1, 2.5, I. virginica
+6.5, 3.2, 5.1, 2.0, I. virginica
+6.4, 2.7, 5.3, 1.9, I. virginica
+6.8, 3.0, 5.5, 2.1, I. virginica
+5.7, 2.5, 5.0, 2.0, I. virginica
+5.8, 2.8, 5.1, 2.4, I. virginica
+6.4, 3.2, 5.3, 2.3, I. virginica
+6.5, 3.0, 5.5, 1.8, I. virginica
+7.7, 3.8, 6.7, 2.2, I. virginica
+7.7, 2.6, 6.9, 2.3, I. virginica
+6.0, 2.2, 5.0, 1.5, I. virginica
+6.9, 3.2, 5.7, 2.3, I. virginica
+5.6, 2.8, 4.9, 2.0, I. virginica
+7.7, 2.8, 6.7, 2.0, I. virginica
+6.3, 2.7, 4.9, 1.8, I. virginica
+6.7, 3.3, 5.7, 2.1, I. virginica
+7.2, 3.2, 6.0, 1.8, I. virginica
+6.2, 2.8, 4.8, 1.8, I. virginica
+6.1, 3.0, 4.9, 1.8, I. virginica
+6.4, 2.8, 5.6, 2.1, I. virginica
+7.2, 3.0, 5.8, 1.6, I. virginica
+7.4, 2.8, 6.1, 1.9, I. virginica
+7.9, 3.8, 6.4, 2.0, I. virginica
+6.4, 2.8, 5.6, 2.2, I. virginica
+6.3, 2.8, 5.1, 1.5, I. virginica
+6.1, 2.6, 5.6, 1.4, I. virginica
+7.7, 3.0, 6.1, 2.3, I. virginica
+6.3, 3.4, 5.6, 2.4, I. virginica
+6.4, 3.1, 5.5, 1.8, I. virginica
+6.0, 3.0, 4.8, 1.8, I. virginica
+6.9, 3.1, 5.4, 2.1, I. virginica
+6.7, 3.1, 5.6, 2.4, I. virginica
+6.9, 3.1, 5.1, 2.3, I. virginica
+5.8, 2.7, 5.1, 1.9, I. virginica
+6.8, 3.2, 5.9, 2.3, I. virginica
+6.7, 3.3, 5.7, 2.5, I. virginica
+6.7, 3.0, 5.2, 2.3, I. virginica
+6.3, 2.5, 5.0, 1.9, I. virginica
+6.5, 3.0, 5.2, 2.0, I. virginica
+6.2, 3.4, 5.4, 2.3, I. virginica
+5.9, 3.0, 5.1, 1.8, I. virginica

data/test/test_parser_fixed.rb

@@ -14,9 +14,9 @@ def parser() CsvReader::Parser::FIXED;  end
 def reader() CsvReader.fixed;           end
 
 
-def test_width
-   width( parser )
-   width( reader )
+def test_numbers
+   numbers( parser )
+   numbers( reader )
 end
 
 def test_contacts
@@ -25,7 +25,7 @@ def test_contacts
 end
 
 
-def width( parser )
+def numbers( parser )
   records = [["12345678","12345678", "12345678901234567890123456789012", "12345678901234"]]
 
   assert_equal records, parser.parse( <<TXT, width: [8,8,32,14] )
@@ -37,6 +37,11 @@ TXT
 
   assert_equal records, parser.parse( <<TXT, width: [8,8,32,14] )
 12345678123456781234567890123456789012345678901212345678901234
+TXT
+
+  ## note: negative width fields gets skipped
+  assert_equal records, parser.parse( <<TXT, width: [8,-2,8,-3,32,-2,14] )
+12345678XX12345678XXX12345678901234567890123456789012XX12345678901234XXX
 TXT
 end
 
@@ -61,20 +66,20 @@ TXT
 end
 
 
-def test_contacts
-  records = [["John",    "Smith",    "john@example.com",    "1-888-555-6666"],
-             ["Michele", "O'Reiley", "michele@example.com", "1-333-321-8765"]]
-
-  assert_equal records, parser.parse( <<TXT, width: [8,8,32,14] )
-# fixed width with comments and blank lines
 
-John    Smith   john@example.com                1-888-555-6666
-Michele O'Reileymichele@example.com             1-333-321-8765
+def test_unpack_numbers
+  records = [["12345678","12345678", "12345678901234567890123456789012", "12345678901234"]]
 
+  assert_equal records, parser.parse( <<TXT, width: 'a8 a8 a32 Z*' )
+12345678123456781234567890123456789012345678901212345678901234
 TXT
+end
 
+def test_unpack_contacts
+  records = [["John",    "Smith",    "john@example.com",    "1-888-555-6666"],
+             ["Michele", "O'Reiley", "michele@example.com", "1-333-321-8765"]]
 
-   assert_equal records, parser.parse( <<TXT, width: [8,8,32,14] )
+  assert_equal records, parser.parse( <<TXT, width: 'A8 A8 A32 Z*' )
 John    Smith   john@example.com                1-888-555-6666
 Michele O'Reileymichele@example.com             1-333-321-8765
 TXT

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: csvreader
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.1.1
 platform: ruby
 authors:
 - Gerald Bauer
@@ -73,6 +73,7 @@ files:
 - test/data/cars11.csv
 - test/data/cities11.csv
 - test/data/customers11.csv
+- test/data/iris11.csv
 - test/data/shakespeare.csv
 - test/helper.rb
 - test/test_buffer.rb