csvhuman 1.0.1 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d8038f1824b8772b53f57f7dc00c67ca902fd872
-  data.tar.gz: a95430def44f77428a32b4ceb80f12c96d078199
+  metadata.gz: eac10294bb4add6317df3fd1eb2dcab35f75315d
+  data.tar.gz: 2072e628e65873a95584fa70fec0bf11a66e2322
 SHA512:
-  metadata.gz: a2ee778e98288478930b92cb066b8d8ee65da61dca3d911e3f3de12c5ee3a0d0f3451db5ea1426c1c59a5a8cd6fc9ecb2bb509d8982e79b1ff0ce8d8c353765e
-  data.tar.gz: e02189132d596e2b5abc654f55c9dcf925ed96d0d5008b8b2eba956cdfc0044827c374409e2e22e45c95791ed5db256c441210a36d81172c53f3178007f8cc40
+  metadata.gz: 931ada0926ae767b7b2348c237694ec35222ff06dfdbac7298527744c0ab99444778d5031355b106ed148551646e301c12d2cc32df8b0dd7001e2b91003c259f
+  data.tar.gz: 0ba952f578bfafb466a99073792d9c0173b24bf689571ec140ed8e61a7ef00f7addea40979bb8ce495d1a8a1266ce01845f5e551653f02afb15c0ea516021bf3

data/Manifest.txt

@@ -2,20 +2,38 @@ HISTORY.md
 Manifest.txt
 README.md
 Rakefile
+config/attributes.csv
+config/langs.csv
+config/tags.csv
+config/types.csv
+config/versions.csv
 lib/csvhuman.rb
 lib/csvhuman/base.rb
 lib/csvhuman/column.rb
 lib/csvhuman/converter.rb
+lib/csvhuman/doc/helper.rb
+lib/csvhuman/doc/schema.rb
 lib/csvhuman/reader.rb
 lib/csvhuman/tag.rb
 lib/csvhuman/version.rb
+test/data/airports.csv
+test/data/ebola.csv
+test/data/hdx/ebola_treatment_centres.csv
+test/data/hdx/phl_haima_houses_damaged.csv
+test/data/hdx/zika_cases.csv
 test/data/sample1.csv
 test/data/sample2.csv
 test/data/sample3.csv
 test/data/sample4.csv
 test/data/test.csv
+test/data/unhcr.csv
 test/helper.rb
+test/test_doc.rb
+test/test_hdx.rb
 test/test_header_converter.rb
+test/test_misc.rb
 test/test_reader.rb
 test/test_samples.rb
 test/test_tags.rb
+test/test_type_converters.rb
+test/test_type_mappings.rb

data/config/attributes.csv

@@ -0,0 +1,54 @@
+attribute,since,category,tags,description
+adolescents,1.0,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,"Adolescents, loosely defined (precise age range varies); may overlap +children and +adult. You can optionally create custom attributes in addition to this to add precise age ranges, e.g. ""+adolescents +age12_17""."
+adults,1.0,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,"Adults, loosely defined (precise age range varies); may overlap +adolescents and +elderly. You can optionally create custom attributes in addition to this to add precise age ranges, e.g. ""+adults +age18_64""."
+children,1.0,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,"The associated hashtag applies to non-adults, loosely defined (precise age range varies; may overlap +infants and +adolescents). You can optionally create custom attributes in addition to this to add precise age ranges, e.g. ""+children +age3_11""."
+elderly,1.0,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,"Elderly people, loosely defined (precise age range varies). May overlap +adults. You can optionally create custom attributes in addition to this to add precise age ranges, e.g. ""+elderly +age65plus""."
+f,1.0,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,Female people. See also +m and +i.
+hh,1.1,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,"Households (vs +ind for individual people). The exact definition of ""household"" may vary among aid organisations."
+i,1.0,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,Intersex or non-gender-binary people. Use this attribute for any groups who do not identify as male (+m) or female (+f).
+ind,1.1,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,Individual people (vs +hh for households).
+infants,1.0,(1) Sex- and-age disaggregation (SADD) attributes,#affected #inneed #population #reached #targeted,"Infant children, loosely defined (precise age range varies; may overlap +children). You can optionally create custom attributes in addition to this to add precise age ranges, e.g. ""+infants +age0_2""."
+m,1.0,(2) Organisation and activity attributes,#affected #inneed #population #reached #targeted,Male people. See also +f and +i.
+activity,1.1,(2) Organisation and activity attributes,#activity,"The implementers classify this activity as an ""activity"" proper (may imply different hierarchical levels in different contexts)."
+cluster,1.1,(2) Organisation and activity attributes,#sector,Identifies a sector as a formal IASC humanitarian cluster.
+funder,1.0,(2) Organisation and activity attributes,#org,Funding org/agency (e.g. donor).
+impl,1.0,(2) Organisation and activity attributes,#org,Implementing partner.
+prog,1.0,(2) Organisation and activity attributes,#org,Programming org/agency.
+programme,1.1,(2) Organisation and activity attributes,#activity,"The implementers classify this activity as a ""programme"" (may imply different hierarchical levels in different contexts)."
+project,1.1,(2) Organisation and activity attributes,#activity,"The implementers classify this activity as a ""project"" (may imply different hierarchical levels in different contexts)."
+provided,1.1,(2) Organisation and activity attributes,#item #service,"Refers to a #service, #item, etc. that has been provided to people in need."
+used,1.1,(3) Classification attributes,#item #service,"Refers to a #service, #item, etc. that affected people have actually consumed or otherwise taken advantage of."
+code,1.0,(3) Classification attributes,#activity #adm1 #adm2 #adm3 #adm4 #adm5 #beneficiary #cause #channel #country #crisis #currency #event #group #impact #indicator #item #loc #modality #need #org #output #region #sector #service #severity #status #subsector,"A unique, machine-readable code."
+type,1.0,(4) Geographical attributes,#access #activity #beneficiary #capacity #cause #contact #crisis #description #event #frequency #group #impact #indicator #item #loc #need #operations #org #output #sector #service #severity #subsector,"Types or categories. Use with #org, #loc, #indicator, etc to provide classification information."
+bounds,1.0,(4) Geographical attributes,#geo,Boundary data (e.g. inline GeoJSON).
+coord,1.0,(4) Geographical attributes,#geo,Geodetic coordinates (lat+lon together).
+dest,1.1,(4) Geographical attributes,#adm1 #adm2 #adm3 #adm4 #adm5 #country #loc #region,Place of destination (intended or actual).
+elevation,1.0,(4) Geographical attributes,#geo,Elevation above sea level (usually metres).
+lat,1.0,(4) Geographical attributes,#geo,Latitude (decimal degrees preferred).
+lon,1.0,(4) Geographical attributes,#geo,Longitude (decimal degrees preferred).
+origin,1.1,(5) Date attributes,#adm1 #adm2 #adm3 #adm4 #adm5 #country #loc #region,"The data describes places of origin (intended or actual), e.g. the country of origin for displaced people."
+approved,1.0,(5) Date attributes,#date,Date or time when something was approved.
+canceled,1.0,(5) Date attributes,#date,Date or time when something (e.g. an #activity) was canceled.
+converted,1.1,(5) Date attributes,#date,Date or time used for converting a monetary value to another currency.
+end,1.0,(5) Date attributes,#date,Date or time when something finished or will finish.
+occurred,1.0,(5) Date attributes,#date,Date or time when something took place.
+reported,1.0,(5) Date attributes,#date,Date or time when the information was reported.
+start,1.0,(6) Impact attributes,#date,Date or time when something started or will start.
+abducted,1.1,(6) Impact attributes,#affected #inneed #reached #targeted,Hashtag refers to people who have been abducted.
+displaced,1.0,(6) Impact attributes,#affected #inneed #population #reached #targeted,Displaced people or households. Refers to all types of displacement: use +idps or +refugees to be more specific.
+idps,1.0,(6) Impact attributes,#affected #inneed #population #reached #targeted,Internally-displaced people or households. More specific than +displaced.
+incamp,1.0,(6) Impact attributes,#affected #inneed #population #reached #targeted,Located in camps.
+infected,1.0,(6) Impact attributes,#affected #inneed #reached #targeted,People infected with a disease.
+injured,1.0,(6) Impact attributes,#affected #inneed #reached #targeted,People injured.
+killed,1.0,(6) Impact attributes,#affected,People killed.
+noncamp,1.0,(6) Impact attributes,#affected #inneed #population #reached #targeted,Not located in camps.
+refugees,1.0,(7) General attributes,#affected #inneed #population #reached #targeted,Refugee people or households. More specific than +displaced.
+email,1.0,(7) General attributes,#contact,Email address.
+id,1.1,(7) General attributes,#meta,Use with #meta to provide internal identifiers for data records.
+label,1.0,(7) General attributes,,Text labels (for a table or chart).
+name,1.0,(7) General attributes,#activity #adm1 #adm2 #adm3 #adm4 #adm5 #beneficiary #cause #channel #contact #country #crisis #event #group #impact #indicator #item #loc #modality #need #org #output #region #sector #service #severity #status #subsector,"Human-readable name, title, or label."
+num,1.0,(7) General attributes,#capacity #indicator #output,"The data consists of quantitative, numeric information."
+phone,1.0,(7) General attributes,#contact,The data consists of #contact phone numbers.
+source,1.0,(7) General attributes,#meta,Information source for the data in the row or record.
+text,1.0,(7) General attributes,#indicator,"The data consists of qualitative, narrative textual information."
+url,1.0,(7) General attributes,#activity #contact #meta #org #service,"The data consists of web links related to the main hashtag (e.g. for an #org, #service, #activity, #loc, etc)."

data/config/langs.csv

@@ -0,0 +1,14 @@
+code, name+en, name
+ar,	Arabic,
+en, English, English
+es, Spanish, Español
+de, German, Deutsch
+fa, Dari / Farsi / Persian,
+fr, French,
+ms,	Malay,
+ps,	Pashto,
+ru, Russian,
+sw, Swahili,
+tl, Tagalog,
+uk, Ukrainian,
+ur, Urdu,

data/config/tags.csv

@@ -0,0 +1,45 @@
+tag,type,since,category,attributes,description
+adm1,,1.0,(1) Places,+code +dest +name +origin,Top-level subnational administrative area (e.g. a governorate in Syria).
+adm2,,1.0,(1) Places,+code +dest +name +origin,Second-level subnational administrative area (e.g. a subdivision in Bangladesh).
+adm3,,1.0,(1) Places,+code +dest +name +origin,Third-level subnational administrative area (e.g. a subdistrict in Afghanistan).
+adm4,,1.0,(1) Places,+code +dest +name +origin,Fourth-level subnational administrative area (e.g. a barangay in the Philippines).
+adm5,,1.0,(1) Places,+code +dest +name +origin,Fifth-level subnational administrative area (e.g. a ward of a city).
+country,,1.0,(1) Places,+code +dest +name +origin,Country (often left implied in a dataset). Also sometimes known as admin level 0.
+geo,,1.0,(1) Places,+bounds +coord +elevation +lat +lon,"Geodetic geometry information (points, lines, shapes). Use for latitude and longitude, as well as bounds information."
+loc,,1.0,(1) Places,+code +dest +name +origin +type,"Any general location, such as a village, camp, or clinic."
+region,,1.0,(2) People and households,+code +dest +name +origin,"A broad, supra- or cross-national geographical region (e.g. Sahel, Horn of Africa, Central Asia, Caribbean). Not to be confused with ""region"" used as the name of a subnational area (#adm1) in some countries."
+affected,number,1.0,(2) People and households,+abducted +adolescents +adults +children +displaced +elderly +f +hh +i +idps +incamp +ind +infants +infected +injured +killed +m +noncamp +refugees,Number of people or households affected by an emergency. Subset of #population; superset of #inneed.
+beneficiary,,1.0,(2) People and households,+code +name +type,"General (non-numeric) information about a person or group meant to benefit from aid activities, e.g. ""lactating women""."
+inneed,number,1.0,(2) People and households,+abducted +adolescents +adults +children +displaced +elderly +f +hh +i +idps +incamp +ind +infants +infected +injured +m +noncamp +refugees,Number of people or households in need of humanitarian assistance. Subset of #affected; superset of #targeted.
+population,number,1.0,(2) People and households,+adolescents +adults +children +displaced +elderly +f +hh +i +idps +incamp +ind +infants +m +noncamp +refugees,"General population number for an area or location, regardless of their specific humanitarian needs."
+reached,number,1.0,(2) People and households,+abducted +adolescents +adults +children +displaced +elderly +f +hh +i +idps +incamp +ind +infants +infected +injured +m +noncamp +refugees,Number of people or households reached with humanitarian assistance. Subset of #targeted.
+respondee,,1.1,(2) People and households,,"Descriptive information, such as name, identifier, or traits, for a single respondee (person, household, etc.) in survey-style data."
+targeted,number,1.0,(3) Responses and other operations,+abducted +adolescents +adults +children +displaced +elderly +f +hh +i +idps +incamp +ind +infants +infected +injured +m +noncamp +refugees,Number of people or households targeted for humanitarian assistance. Subset of #inneed; superset of #reached.
+access,,1.1,(3) Responses and other operations,+type,"Accessiblity and constraints on access to a market, distribution point, facility, etc."
+activity,,1.0,(3) Responses and other operations,+activity +code +name +programme +project +type +url,"A programme, project, or other activity. This hashtag applies to all levels; use the attributes +activity, +project, or +programme to distinguish different hierarchical levels."
+capacity,,1.0,(3) Responses and other operations,+num +type,"The response capacity of the entity being described (e.g. ""25 beds"")."
+contact,,1.0,(3) Responses and other operations,+email +name +phone +type +url,Contact information for the subject of a data record (e.g. an activity).
+frequency,,1.1,(3) Responses and other operations,+type,The frequency with which something occurs.
+indicator,,1.0,(3) Responses and other operations,+code +name +num +text +type,"A general hashtag for an indicator being tracked. See also #output, #capacity, #need, #impact, #severity, #affected, #inneed, #targeted, and #reached for more-specific indicator-related hashtags."
+item,,1.1,(3) Responses and other operations,+code +name +provided +type +used,"Physical things provided, stored, shipped, available, used, etc."
+need,,1.1,(3) Responses and other operations,+code +name +type,"A(n) (unfulfilled) need for an affected person, household, group, or population."
+operations,,1.0,(3) Responses and other operations,+type,"Information that affects humanitarian operations, such as a restriction on movement or road closure."
+org,,1.0,(3) Responses and other operations,+code +funder +impl +name +prog +type +url,"An organisation contributing to a humanitarian emergency response, e.g. a local government, community-based organisation, NGO, agency, donor, or law-enforcement or military unit. Use #group for organisations that are not part of the emergency response (e.g. a paramilitary group)."
+output,,1.0,(3) Responses and other operations,+code +name +num +type,"An output indicator (e.g. ""number of water-purification kits distributed""). A more-specific alternative to #indicator, especially for 3W-style activity reports."
+sector,,1.0,(3) Responses and other operations,+cluster +code +name +type,A humanitarian cluster or sector.
+service,,1.1,(3) Responses and other operations,+code +name +provided +type +url +used,"A service used or needed by an affected person, household, group, or population."
+subsector,,1.0,(4) Cash and finance,+code +name +type,A humanitarian subsector.
+channel,,1.1,(4) Cash and finance,+code +name,The detailed method of delivering aid (e.g. smartcard vs mobile transfer). More specific than #modality.
+currency,,1.1,(4) Cash and finance,+code,"Name or ISO 4217 currency code for all financial #value cells in the row (e.g. ""EUR""). Typically used together with #value in financial or cash data."
+modality,,1.1,(4) Cash and finance,+code +name,"The means by which an aid activity is accomplished. For cash transfers, values might include ""cash"", ""vouchers"", ""in-kind"", etc. May also be used for other types of modalities in other contexts."
+value,number,1.1,"(5) Crises, incidents, and events",,"A monetary value, such as the price of goods in a market, a project budget, or the amount of cash transferred to beneficiaries. May be used together with #currency in financial or cash data."
+cause,,1.0,"(5) Crises, incidents, and events",+code +name +type,"The cause of an event, crisis, etc."
+crisis,,1.0,"(5) Crises, incidents, and events",+code +name +type,A humanitarian emergency.
+event,,1.0,"(5) Crises, incidents, and events",+code +name +type,"An individual event or incident within a crisis/emergency, such as a (localised) flood, bridge collapse, or conflict."
+group,,1.0,"(5) Crises, incidents, and events",+code +name +type,"A non-humanitarian group (of any type) related to humanitarian crisis (e.g., a paramilitary group) Use #org instead for a humanitarian organisation such as an NGO, contributing to the humanitarian response."
+impact,,1.0,"(5) Crises, incidents, and events",+code +name +type,The impact of a crisis on a group or other entity.
+severity,,1.0,(6) Metadata,+code +name +type,Severity of the crisis or event.
+date,date,1.0,(6) Metadata,+approved +canceled +converted +end +occurred +reported +start,"Date related to the data in the record applies. Preferred format is ISO 8610 (e.g. ""2015-06-01"", ""2015-Q1"", etc.)"
+description,,1.0,(6) Metadata,+type,Long description for a data record.
+meta,,1.0,(6) Metadata,+id +source +url,Metadata about a row.
+status,,1.0,(6) Metadata,+code +name,"Project/activity status description (such as ""planned"", ""active"", ""canceled"", or ""complete"")."

data/config/types.csv

@@ -0,0 +1,7 @@
+type,description
+text,
+number,
+url,
+email,
+phone,
+date,

data/config/versions.csv

@@ -0,0 +1,4 @@
+version, date
+1.0, 2016-03-18
+1.1, 2018-04-30
+1.2,

data/lib/csvhuman/base.rb

@@ -10,7 +10,8 @@ require 'csvhuman/column'
 require 'csvhuman/converter'
 require 'csvhuman/reader'
 
-
+require 'csvhuman/doc/helper.rb'
+require 'csvhuman/doc/schema.rb'
 
 
 # say hello

data/lib/csvhuman/converter.rb

@@ -18,4 +18,113 @@ HEADER_CONVERTERS = {
                                        gsub( /[^\w]+/, '' ).to_sym }
 }
 
+
+
+
+  def self.guess_type( name, attributes )
+    if name == 'date'
+       if attributes.include?( 'year' )
+         Integer    ##  just the year (e.g. 2011); use an integer number
+       else
+         Date
+       end
+    ## todo/fix: add more well-known names with num required!!!
+    elsif ['affected', 'inneed', 'targeted', 'reached', 'population'].include?( name )
+       Integer
+    else
+      ## check attributes
+      if attributes.nil? || attributes.empty?
+        String  ## assume (default to) string
+      elsif attributes.include?( 'num' ) ||
+            attributes.include?( 'id')   ## assume id is (always) a rowid - why? why not?
+        Integer
+      elsif attributes.include?( 'date' )   ### todo/check: exists +date?
+        Date
+      elsif name == 'geo' && (attributes.include?('lat') ||
+                              attributes.include?('lon') ||
+                              attributes.include?('elevation'))
+        Float
+      elsif attributes.include?( 'killed' ) ||
+            attributes.include?( 'injured' ) ||
+            attributes.include?( 'infected' ) ||
+            attributes.include?( 'displaced' ) ||
+            attributes.include?( 'idps' ) ||
+            attributes.include?( 'refugees' ) ||
+            attributes.include?( 'abducted' ) ||
+            attributes.include?( 'threatened' ) ||
+            attributes.include?( 'affected' ) ||
+            attributes.include?( 'inneed' ) ||
+            attributes.include?( 'targeted' ) ||
+            attributes.include?( 'reached' )
+        Integer
+      else
+        String   ## assume (default to) string
+      end
+    end
+  end
+
+
+## convert guess_type to proc (is there a better/idomatic way)?
+#   ->(name, attributes) { guess_type( name, attributes ) }
+## TYPE_MAPPING_GUESS = Kernel.method( :guess_type )
+
+TYPE_MAPPINGS = {
+  ##  always returns string (that is, keep as is (assumes always string values))
+  none:      ->(name, attributes) { String },
+  guess:     ->(name, attributes) { guess_type( name, attributes ) },
+}
+
+## add aliases  (check - is there a better/idomatic way?)
+TYPE_MAPPINGS[ :default] = TYPE_MAPPINGS[:guess]  ## alias for guess
+TYPE_MAPPINGS[ :all ]    = TYPE_MAPPINGS[:guess]  ## alias for guess (yes, another one - why? why not?)
+
+
+
+
+
+def self.convert_to_i( value )
+  if value.nil? || value.empty?
+    nil   ## return nil - why? why not?
+  else
+    Integer( value )
+  end
+end
+
+def self.convert_to_f( value )
+  if value.nil? || value.empty?
+    nil   ## return nil - why? why not?
+  else
+    ## todo/fix: add support for NaN, Inf, -Inf etc.
+    ##    how to deal with conversion errors (throw exception? ignore? NaN? why? why not?)
+    Float( value )
+  end
+end
+
+def self.convert_to_date( value )
+  if value.nil? || value.empty?
+    nil   ## return nil - why? why not?
+  else
+    ## todo/fix: add support for more formats
+    ##    how to deal with conversion errors (throw exception? ignore? why? why not?)
+    if value =~ /\d{4}-\d{1,2}-\d{1,2}/    ### todo: check if 2014-1-9 works for strptime too (leading zero rquired)?
+      Date.strptime( value, "%Y-%m-%d" )    # 2014-11-09
+    elsif value =~ /\d{1,2}\/\d{1,2}\/\d{4}/
+      Date.strptime( value, "%d/%m/%Y" )    # 09/11/2014
+    else
+      ## todo/fix: throw argument/value error - why? why not
+      nil
+    end
+  end
+end
+
+
+
+TYPE_CONVERTERS = {
+  Integer => ->(value) { convert_to_i(value) },
+  Float   => ->(value) { convert_to_f(value) },
+  Date    => ->(value) { convert_to_date(value) },
+}
+
+
+
 end # class CsvHuman

data/lib/csvhuman/doc/helper.rb

@@ -0,0 +1,108 @@
+# encoding: utf-8
+
+class CsvHuman
+module DocHelper
+
+
+HASHTAG_LINE_RX= /^
+                   \s*
+                   \#
+                   (?<name>[a-z][a-z0-9]+)
+                   \s*
+                   $/x
+
+def match_hashtag( line )
+   if (m=HASHTAG_LINE_RX.match(line))
+     puts "hashtag >#{m[:name]}<"
+     m
+   else
+     nil
+   end
+end
+
+
+
+## note: attrib might be one letter only (e.g.) +m,+f, etc.
+ATTRIBUTE_LINE_RX= /^
+                   \s*
+                   \+
+                   (?<name>[a-z][a-z0-9]*)
+                   \s*
+                   $/x
+
+def match_attribute( line )
+   if (m=ATTRIBUTE_LINE_RX.match(line))
+     puts "attrib >#{m[:name]}<"
+     m
+   else
+     false
+   end
+end
+
+
+
+##
+## e.g. 1.1. Places
+##      2.1. Sex- and-age disaggregation (SADD) attributes
+
+HEADING_LINE_RX=/^
+                   \s*
+                   (?<level1>[1-9])
+                     \.
+                   (?<level2>[1-9])
+                     \.
+                     \s+
+                    (?<title>.+?)
+                     \s*
+                   $/x
+
+def match_heading( line )
+  if (m=HEADING_LINE_RX.match(line))
+    puts "heading #{m[:level1]}.#{m[:level2]}. (#{m[:level2]}) >#{m[:title]}<"
+    m
+  else
+    false
+  end
+end
+
+
+
+TYPE_RX = /Every value must be a (?<type>[a-z]+)./
+def match_type( line )
+  if (m=TYPE_RX.match(line))
+    puts "type: >#{m[:type]}<"
+    m
+  else
+    false
+  end
+end
+
+
+
+SINCE_HXL_RX = /Since HXL (?<version>[1]\.[0-9])\.?/
+def match_since_hxl( line )
+  if (m=SINCE_HXL_RX.match(line))
+    puts "version: >#{m[:version]}<"
+    m
+  else
+    false
+  end
+end
+
+
+
+def split_descr( line )
+  if( m=match_since_hxl( line ))
+    version = m[:version]
+    ## remove "Since HXL 1.0" from text
+    text    = line.gsub( SINCE_HXL_RX, '' ).strip
+  else
+    version = '?'
+    text = line
+  end
+  [text,version]
+end
+
+
+end # module DocHelper
+end # class CsvHuman

data/lib/csvhuman/doc/schema.rb

@@ -0,0 +1,151 @@
+# encoding: utf-8
+
+class CsvHuman
+class Doc    ## tags and attributes (schema) reader / converter (txt to csv)
+  include DocHelper
+
+
+
+def self.read_attributes( path )
+  self.open( path ) { |doc| doc.parse_attributes }
+end
+
+def self.read_tags( path )
+  self.open( path ) { |doc| doc.parse_tags }
+end
+
+def self.open( path, mode=nil, &block )   ## rename path to filename or name - why? why not?
+
+   ## note: default mode (if nil/not passed in) to 'r:bom|utf-8'
+   f  = File.open( path, mode ? mode : 'r:bom|utf-8' )
+   doc = self.new( f )
+
+   # handle blocks like Ruby's open(), not like the (old old) CSV library
+   if block_given?
+     begin
+       block.call( doc )
+     ensure
+       f.close
+     end
+   else
+     doc    ## note: caller responsible for closing (todo/fix: add close,closed? to doc!!!)
+   end
+end # method self.open
+
+
+
+
+def initialize( str_or_readable )
+  # note: must (only) support/respond_to read_line
+  @input = str_or_readable
+end
+
+
+def parse_attributes
+
+  attrib   = nil
+  category = nil
+  descr    = nil
+  version  = nil
+  tags     = []
+
+  next_line   =  nil   ## e.g. set to :descr
+
+  attribs = []
+
+  @input.each_line do |line|
+    line = line.chomp( '' )
+
+    line = line.strip   ## remove leading and trailing spaces
+
+
+    next if line.empty? || line.start_with?( '%' )   ## skip blank lines and comment lines
+
+    if next_line == :descr
+      ## auto-capture next line (if descr reset to nil)
+      descr, version = split_descr( line )
+      puts "descr >#{descr}<, version >#{version}<"
+
+      next_line = nil
+    elsif (m=match_heading( line ))
+      category = "(#{m[:level2]}) #{m[:title]}"
+    elsif (m=match_attribute( line ))
+      if attrib
+        attribs << [attrib, version, category, tags.join( ' ' ), descr]
+      end
+
+      attrib    = m[:name]
+      tags      = []
+      next_line = :descr  ## reset descr to nil - will auto-capture next line
+    elsif (m=match_hashtag( line ))
+      tags << "##{m[:name]}"
+    end
+  end
+
+  if attrib
+    attribs << [attrib, version, category, tags.join( ' ' ), descr]
+  end
+
+  attribs
+end  # method parse_attributes
+
+
+
+def parse_tags
+
+  tag     = nil
+  type    = nil
+  category = nil
+  descr    = nil
+  version  = nil
+  attribs  = []
+
+  next_line   =  nil   ## e.g. set to :descr
+
+
+  tags = []
+
+  @input.each_line do |line|
+    line = line.chomp( '' )
+
+    line = line.strip   ## remove leading and trailing spaces
+
+
+    next if line.empty? || line.start_with?( '%' )   ## skip blank lines and comment lines
+
+    if next_line == :descr
+      ## auto-capture next line (if descr reset to nil)
+      descr, version = split_descr( line )
+
+      ## descr = "(2) People and households"   if descr == "(2) Surveys and assessments"
+
+      puts "descr >#{descr}<, version >#{version}<"
+
+      next_line = nil
+    elsif (m=match_heading( line ))
+      category = "(#{m[:level2]}) #{m[:title]}"
+    elsif (m=match_type( line ))
+      type = m[:type]
+    elsif (m=match_hashtag( line ))
+      if tag
+        tags << [tag, type, version, category, attribs.join( ' ' ), descr]
+      end
+
+      tag      = m[:name]
+      attribs  = []
+      type     = nil
+      next_line = :descr  ## reset descr to nil - will auto-capture next line
+    elsif (m=match_attribute( line ))
+      attribs << "+#{m[:name]}"
+    end
+  end
+
+  if tag
+    tags << [tag, type, version, category, attribs.join( ' ' ), descr]
+  end
+
+  tags
+end  # method  parse_tags
+
+end  # class Doc
+end  # class CsvHuman