ny_frb_as_dataframe 0.1.1 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 90ab84e7883a2a7fd86e6349c210e7934b427ffaab4c31e4113eb0977ea0b2f5
-  data.tar.gz: dad8b0082e9e7b9c5bc67e0c29862f100d8824031ec2ed4029f1a1d9f4001f28
+  metadata.gz: 162dee7d9d8406fc00799a4e7fac2cc521266d19f5654c170a53357c69498b58
+  data.tar.gz: 78b8e0adec44302459f0d8a5431d31d997802828465131a7c82f21d9af8bfff3
 SHA512:
-  metadata.gz: 2e244d59b1beab7ff6342528e3604a216b313d58c9f5835febf17268e6301c064f342480c7f16d958eb08b645217d9ecc3ad8b8bbbd1661a76d6cc0788eef284
-  data.tar.gz: ba9513f960bd4fd72e58e9ccc8e1b40eafcb1b0d20c11f003bf669f20efd89564259ec0d1a2715b799a127919b6beb5524d56175c18982c8bee8f2b3ad3170cf
+  metadata.gz: 3c5343ba7ecfa4a5cc9ba66e0144b2554f30160dd3613a5e7d70376b7339477f10a31ec35859b0d9cc96fbf4d64b1f0862cc1e4984be47476428452db7ed7c20
+  data.tar.gz: 46a9beb59c45eb41907e63fb268ab0441e528e41d649ca7bbcb982f6c1fa6c49bbec0af98bea859f131956e5a46d46d04341157a7d8005a869cf02f1cfaa19b1

data/.ruby-version

@@ -0,0 +1 @@
+3.3.0

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 ## [Unreleased]
 
+## [0.2.1] - 2026-09-09
+
+- Parse HHDC workbooks with Roo instead of SimpleXlsxReader. This drops `auto_slurp` (absent on 1.0.5) and skips chart-only sheets that have no dimension node.
+- Match sheet titles and column headers after stripping HTML, asterisks, and extra whitespace, case-insensitively (`To 90+ days late` matches `To 90+ days Late`).
+- Merge continuation rows that carry values for the preceding quarter (Credit Limit / HE Revolving layout).
+
+## [0.2.0] - 2026-08-28
+
+- Required Ruby version >= 3.3
+- Updated polars-df dependency to 0.27.1
+
 ## [0.1.0] - 2024-08-12
 
 - Initial release

data/Gemfile

@@ -2,6 +2,8 @@
 
 source "https://rubygems.org"
 
+ruby '3.3.0'
+
 # Specify your gem's dependencies in ny_frb_as_dataframe.gemspec
 gemspec
 

data/README.md

@@ -2,7 +2,9 @@
 
 Up to date remote economic data access for ruby, using Polars dataframes. 
 
-This package will fetch economic and financial information from the Federal Reserve Bank of NY (specifically  several time series from the FRB NY's Household Credit and Debt report), and return the results as a Polars dataframe. 
+This package will fetch economic and financial information from the Federal Reserve Bank of NY (specifically  several time series from the FRB NY's Household Credit and Debt report), and return the results as a Polars dataframe.
+
+`NyFrbAsDataframe::Hhdc#fetch` downloads the latest `hhd_c_report_YYYYqN.xlsx` and parses data sheets with Roo. Sheet titles and headers are matched after stripping HTML, asterisks, and extra whitespace (case-insensitive). Chart-only sheets are skipped. The column names listed below are the names this gem emits; they are not always identical to the labels printed in the workbook. 
 
 ## Installation
 
@@ -22,8 +24,8 @@ Or install it yourself as:
 
 ## Usage
 
-```{ruby}
-3.1.2 :001 > NyFrbAsDataframe::Hhdc.new.fetch  # Fetches all data
+```ruby
+3.3.0 :001 > NyFrbAsDataframe::Hhdc.new.fetch  # Fetches all data
  => 
 shape: (102, 103)                                                                                       
 ┌────────────┬───────────────────┬────────────────────┬────────────────────────┬───┬────────────────────────┬────────────────────────┬────────────────────────┬───────────────────────┐
@@ -135,7 +137,7 @@ shape: (102, 103)
 │ 2024-01-01 ┆ 12.442            ┆ 0.376              ┆ 1.616                  ┆ … ┆ 30.2                   ┆ 28.66                  ┆ 18.3                   ┆ 9.88                  │
 │ 2024-04-01 ┆ 12.519            ┆ 0.38               ┆ 1.626                  ┆ … ┆ 34.26                  ┆ 31.0                   ┆ 18.92                  ┆ 11.42                 │
 └────────────┴───────────────────┴────────────────────┴────────────────────────┴───┴────────────────────────┴────────────────────────┴────────────────────────┴───────────────────────┘ 
-3.1.2 :002 > NyFrbAsDataframe::Hhdc.new('Mort Debt Balance ($T)').fetch
+3.3.0 :002 > NyFrbAsDataframe::Hhdc.new('Mort Debt Balance ($T)').fetch
  => 
 shape: (102, 2)                                                                      
 ┌────────────┬────────────────────────┐                                              
@@ -248,6 +250,12 @@ shape: (102, 2)
 └────────────┴────────────────────────┘ 
 ```
 
+## Requirements
+
+This gem requires **Ruby 3.3 or later** and uses `polars-df` 0.27.1 (exact) for dataframe functionality.
+
+## Attributes
+
 The attributes that can be queried at this time are
 
 * "Mort Debt Balance ($T)"                                       
@@ -282,12 +290,45 @@ The attributes that can be queried at this time are
 * "Median Riskscore for New Car Loan"
 * "Bottom 25%ile Riskscore for New Car Loan"
 * "Bottom 10%ile Riskscore for New Car Loan"
+* "CC Balance ($T)"
+* "CC Avail Credit ($T)"
+* "CC Limit ($T)"
+* "HELOC Balance ($T)"
+* "HELOC Avail Credit ($T)"
+* "HELOC Limit ($T)"
 * "% Balances Current"
 * "% Balances 30d late"
 * "% Balances 60d late"
 * "% Balances 90d late"
 * "% Balances 120+d late"
 * "% Balances Severely Impair"
+* "% Mort Bal 90d+ late"
+* "% HELOC Bal 90d+ late"
+* "% Car Loan Bal 90d+ late"
+* "% CC Bal 90d+ late"
+* "% Student Loan Bal 90d+ late"
+* "% Other Bal 90d+ late"
+* "% All Bal 90d+ late"
+* "% Car Loan Bal 0d->30d+ Late"
+* "% CC Bal 0d->30d+ Late"
+* "% Mort Bal 0d->30d+ Late"
+* "% HELOC Bal 0d->30d+ Late"
+* "% Student Loan Bal 0d->30d+ Late"
+* "% Other Bal 0d->30d+ Late"
+* "% All Bal 0d->30d+ Late"
+* "% Car Loan Bal ->90d+ Late"
+* "% CC Bal ->90d+ Late"
+* "% Mort Bal ->90d+ Late"
+* "% HELOC Bal ->90d+ Late"
+* "% Student Loan Bal ->90d+ Late"
+* "% Other Bal ->90d+ Late"
+* "% All Bal ->90d+ Late"
+* "Qtrly % Mort 0->30-60d late"
+* "Qtrly % Mort ->90d+ Late"
+* "Qtrly % Mort % 30-60d->0d Late"
+* "Qtrly % Mort 30-60d->90d+ Late"
+* "Num Consumers w New Foreclosure"
+* "Num Consumers w New Bankruptcy"
 * "% Consumers w Collection"
 * "Avg Collection Amt"
 * "Avg Total Debt Bal for 18-29 yo ($T)"
@@ -355,9 +396,13 @@ The attributes that can be queried at this time are
 
 Refer to https://www.newyorkfed.org/microeconomics/hhdc/background.html and https://www.newyorkfed.org/microeconomics/hhdc for additional information about available data.
 
+## Testing
+
+This gem uses local smoke tests for validation. No GitHub Actions or CI workflows are configured.
+
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/ny_frb_as_dataframe. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/[USERNAME]/ny_frb_as_dataframe/blob/main/CODE_OF_CONDUCT.md).
+Bug reports and pull requests are welcome on GitHub at https://github.com/bmck/ny_frb_as_dataframe. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [code of conduct](https://github.com/bmck/ny_frb_as_dataframe/blob/main/CODE_OF_CONDUCT.md).
 
 ## License
 
@@ -365,4 +410,4 @@ The gem is available as open source under the terms of the [MIT License](https:/
 
 ## Code of Conduct
 
-Everyone interacting in the NyFrbAsDataframe project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/ny_frb_as_dataframe/blob/main/CODE_OF_CONDUCT.md).
+Everyone interacting in the NyFrbAsDataframe project's codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/bmck/ny_frb_as_dataframe/blob/main/CODE_OF_CONDUCT.md).

data/lib/ny_frb_as_dataframe/hhdc.rb

@@ -1,8 +1,12 @@
+require 'set'
 require 'polars-df'
 require 'httparty'
-require 'simple_xlsx_reader'
+require 'roo'
 require 'csv'
+require 'tempfile'
 require 'active_support/core_ext/date/calculations'
+require 'active_support/core_ext/object/blank'
+require 'active_support/core_ext/object/try'
 require 'active_support/core_ext/string/conversions'
 
 
@@ -12,6 +16,36 @@ module NyFrbAsDataframe
     include ::HTTParty
     base_uri 'https://www.newyorkfed.org/medialibrary/interactives/householdcredit/data/xls/'
 
+    SHEET_COLUMNS =
+      {
+        'Total Debt Balance and Its Composition' => {'Mortgage' => 'Mort Debt Balance ($T)', 'HE Revolving' => 'HELOC Debt Balance ($T)',  'Auto Loan' => 'Car Loan Debt Balance ($T)', 'Credit Card' => 'CC Debt Balance ($T)', 'Student Loan' => 'Student Debt Balance ($T)',  'Other' => 'Other Debt Balance ($T)', 'Total' => 'Total Debt Balance ($T)'},
+        'Number of Accounts by Loan Type' => {'Auto Loan' => 'Num Car Loan Accts (M)', 'Credit Card' => 'Num CC Accts (M)', 'Mortgage' => 'Num Mort Accts (M)',  'HE Revolving' => 'Num HELOC Accts (M)'},
+        'Total Number of New and Closed Accounts and Consumer Credit Inquiries' => {'inquiry within 6 mo' => 'Num Credit Inquiries and New Accts', 'closed within 12 mo' => 'Num New Credit Accts Closed in 12 mo', 'open within 12 mo' => 'Num New Credit Accts Open after 12 mo'},
+        'Mortgage Origination Volume by Riskscore' => {'<620' => 'New Mort w Riskscore <620 ($B)','620-659' => 'New Mort w Riskscore 620-659 ($B)','660-719' => 'New Mort w Riskscore 660-719 ($B)','720-759' => 'New Mort w Riskscore 720-759 ($B)','760+' => 'New Mort w Riskscore 760+ ($B)','TOTAL' => 'New Mort ($B)'},
+        'Distribution of Riskscore of Mortgage Originations' => {'Median' => 'Median Riskscore for New Mort', '25th percentile' => 'Bottom 25%ile Riskscore for New Mort', '10th percentile' => 'Bottom 10%ile Riskscore for New Mort'},
+        'Auto Loan Origination Volume by Riskscore' => {'<620' => 'New Car Loans w Riskscore <620 ($B)','620-659' => 'New Car Loans w Riskscore 620-659 ($B)','660-719' => 'New Car Loans w Riskscore 660-719 ($B)','720-759' => 'New Car Loans w Riskscore 720-759 ($B)','760+' => 'New Car Loans w Riskscore 760+ ($B)','TOTAL' => 'All New Car Loans ($B)'},
+        'Distribution of Riskscore of Auto Loan Originations' => {'Median' => 'Median Riskscore for New Car Loan', '25th percentile' => 'Bottom 25%ile Riskscore for New Car Loan', '10th percentile' => 'Bottom 10%ile Riskscore for New Car Loan'},
+        'Credit Limit and Balance for Credit Cards and HE Revolving' => {'Credit Card Balance' => 'CC Balance ($T)', 'Credit Card Available Credit' => 'CC Avail Credit ($T)',  'Credit Card Limit' => 'CC Limit ($T)', 'HE Revolving Balance' => 'HELOC Balance ($T)',  'HE Revolving Available Credit' => 'HELOC Avail Credit ($T)', 'HE Revolving Limit' => 'HELOC Limit ($T)'},
+        'Total Balance by Delinquency Status' => {'Current' => '% Balances Current', '30 days late' => '% Balances 30d late',  '60 days late' => '% Balances 60d late',  '90 days late' => '% Balances 90d late',  '120+ days late' => '% Balances 120+d late',  'Severely Derogatory' => '% Balances Severely Impair'},
+        'Percent of Balance 90+ Days Delinquent by Loan Type' => {'MORTGAGE' => '% Mort Bal 90d+ late',  'HELOC' => '% HELOC Bal 90d+ late', 'AUTO' => '% Car Loan Bal 90d+ late',  'CC' => '% CC Bal 90d+ late',  'STUDENT LOAN' => '% Student Loan Bal 90d+ late',  'OTHER' => '% Other Bal 90d+ late', 'ALL' => '% All Bal 90d+ late'},
+        'New Delinquent* Balances by Loan Type' => {'AUTO' => '% Car Loan Bal 0d->30d+ Late',  'CC' => '% CC Bal 0d->30d+ Late',  'MORTGAGE' => '% Mort Bal 0d->30d+ Late',  'HELOC' => '% HELOC Bal 0d->30d+ Late', 'STUDENT LOAN' => '% Student Loan Bal 0d->30d+ Late',  'OTHER' => '% Other Bal 0d->30d+ Late', 'Total' => '% All Bal 0d->30d+ Late'},
+        'New Seriously Delinquent* Balances by Loan Type' => {'AUTO' => '% Car Loan Bal ->90d+ Late',  'CC' => '% CC Bal ->90d+ Late',  'MORTGAGE' => '% Mort Bal ->90d+ Late',  'HELOC' => '% HELOC Bal ->90d+ Late', 'STUDENT LOAN' => '% Student Loan Bal ->90d+ Late',  'OTHER' => '% Other Bal ->90d+ Late', 'ALL' => '% All Bal ->90d+ Late'},
+        'Quarterly Transition Rates for Current Mortgage Accounts' => {'To 30-60 days late' => 'Qtrly % Mort 0->30-60d late', 'To 90+ days Late' => 'Qtrly % Mort ->90d+ Late'},
+        'Quarterly Transition Rates for 30-60 Day Late Mortgage Accounts' => {'To current'  => 'Qtrly % Mort % 30-60d->0d Late', 'To 90+ days Late' => 'Qtrly % Mort 30-60d->90d+ Late'},
+        'Number of Consumers with New Foreclosures and Bankruptcies' => {'foreclosure' => 'Num Consumers w New Foreclosure',  'bankruptcy' => 'Num Consumers w New Bankruptcy'},
+        'Third Party Collections' => {'proportion of consumers with collection' => '% Consumers w Collection', 'average collection amount per person with item' => 'Avg Collection Amt'},
+        'Total Debt Balance by Age' => {'18-29' => 'Avg Total Debt Bal for 18-29 yo ($T)', '30-39' => 'Avg Total Debt Bal for 30-39 yo ($T)', '40-49' => 'Avg Total Debt Bal for 40-49 yo ($T)', '50-59' => 'Avg Total Debt Bal for 50-59 yo ($T)', '60-69' => 'Avg Total Debt Bal for 60-69 yo ($T)', '70+' => 'Avg Total Debt Bal for 70+ yo ($T)'},
+        'Auto Loan Originations By Age' => {'18-29' => 'New Car Loans for 18-29 yo ($B)', '30-39' => 'New Car Loans for 30-39 yo ($B)', '40-49' => 'New Car Loans for 40-49 yo ($B)', '50-59' => 'New Car Loans for 50-59 yo ($B)', '60-69' => 'New Car Loans for 60-69 yo ($B)', '70+' => 'New Car Loans for 70+ yo ($B)'},
+        'Total Mortgage Orginations by Age' => {'18-29' => 'New Mort for 18-29 yo ($B)', '30-39' => 'New Mort for 30-39 yo ($B)', '40-49' => 'New Mort for 40-49 yo ($B)', '50-59' => 'New Mort for 50-59 yo ($B)', '60-69' => 'New Mort for 60-69 yo ($B)', '70+' => 'New Mort for 70+ yo ($B)'},
+        'Transition into Serious Delinquency (90+) by Age' => {'18-29' => 'Qtrly % ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % ->90d+ late for 60-69 yo', '70+' => 'Qtrly % ->90d+ late for 70+ yo', 'all' => 'Qtrly % ->90d+ late'},
+        'Transition into Serious Delinquency (90+) for Mortgages by Age' => {'18-29' => 'Qtrly % Mort ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % Mort ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % Mort ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % Mort ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % Mort ->90d+ late for 60-69 yo', '70+' => 'Qtrly % Mort ->90d+ late for 70+ yo', 'all' => 'Qtrly % Mort ->90d+ late'},
+        'Transition into Serious Delinquency (90+) for Auto Loans by Age' => {'18-29' => 'Qtrly % Car Loans ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % Car Loans ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % Car Loans ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % Car Loans ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % Car Loans ->90d+ late for 60-69 yo', '70+' => 'Qtrly % Car Loans ->90d+ late for 70+ yo', 'all' => 'Qtrly % Car Loans ->90d+ late'},
+        'Transition into Serious Delinquency (90+) for Credit Cards by Age' => {'18-29' => 'Qtrly % CC ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % CC ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % CC ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % CC ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % CC ->90d+ late for 60-69 yo', '70+' => 'Qtrly % CC ->90d+ late for 70+ yo', 'all' => 'Qtrly % CC ->90d+ late'},
+        'Transition into Serious Delinquency (90+) for Student Loans by Age' => {'18-29' => 'Qtrly % Student Loan ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % Student Loan ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % Student Loan ->90d+ late for 40-49 yo', 'all' => 'Qtrly % Student Loan ->90d+ late'},
+        'New Foreclosures By Age' => {'18-29' => 'New Foreclosures for 18-29 yo (000)', '30-39' => 'New Foreclosures for 30-39 yo (000)', '40-49' => 'New Foreclosures for 40-49 yo (000)', '50-59' => 'New Foreclosures for 50-59 yo (000)', '60-69' => 'New Foreclosures for 60-69 yo (000)', '70+' => 'New Foreclosures for 70+ yo (000)'},
+        'New Bankruptcy By Age' => {'18-29' => 'New Bankruptcies for 18-29 yo (000)', '30-39' => 'New Bankruptcies for 30-39 yo (000)', '40-49' => 'New Bankruptcies for 40-49 yo (000)', '50-59' => 'New Bankruptcies for 50-59 yo (000)', '60-69' => 'New Bankruptcies for 60-69 yo (000)', '70+' => 'New Bankruptcies for 70+ yo (000)'}
+      }.freeze
+
     attr_reader :tag
 
     def initialize(series = nil)
@@ -21,8 +55,6 @@ module NyFrbAsDataframe
     def fetch(start: nil, fin: nil)
       df = nil
 
-      SimpleXlsxReader.configuration.auto_slurp = true
-
       Tempfile.open(['foo', '.xlsx'], '/tmp', binmode: true) do |f|
         qtr = Date.today.year * 4 + (Date.today.month.div(3)+1)
         dta = nil
@@ -38,139 +70,153 @@ module NyFrbAsDataframe
         f.write(resp)
         f.rewind
 
-        xlsx_file = SimpleXlsxReader.open(f.path)
-
-        col_nm_hsh = 
-          {
-            'Total Debt Balance and Its Composition' => {'Mortgage' => 'Mort Debt Balance ($T)', 'HE Revolving' => 'HELOC Debt Balance ($T)',  'Auto Loan' => 'Car Loan Debt Balance ($T)', 'Credit Card' => 'CC Debt Balance ($T)', 'Student Loan' => 'Student Debt Balance ($T)',  'Other' => 'Other Debt Balance ($T)', 'Total' => 'Total Debt Balance ($T)'},
-            'Number of Accounts by Loan Type' => {'Auto Loan' => 'Num Car Loan Accts (M)', 'Credit Card' => 'Num CC Accts (M)', 'Mortgage' => 'Num Mort Accts (M)',  'HE Revolving' => 'Num HELOC Accts (M)'},
-            'Total Number of New and Closed Accounts and Consumer Credit Inquiries' => {'inquiry within 6 mo' => 'Num Credit Inquiries and New Accts', 'closed within 12 mo' => 'Num New Credit Accts Closed in 12 mo', 'open within 12 mo' => 'Num New Credit Accts Open after 12 mo'},
-            'Mortgage Origination Volume by Riskscore' => {'<620' => 'New Mort w Riskscore <620 ($B)','620-659' => 'New Mort w Riskscore 620-659 ($B)','660-719' => 'New Mort w Riskscore 660-719 ($B)','720-759' => 'New Mort w Riskscore 720-759 ($B)','760+' => 'New Mort w Riskscore 760+ ($B)','TOTAL' => 'New Mort ($B)'},
-            'Distribution of Riskscore of Mortgage Originations' => {'Median' => 'Median Riskscore for New Mort', '25th percentile' => 'Bottom 25%ile Riskscore for New Mort', '10th percentile' => 'Bottom 10%ile Riskscore for New Mort'},
-            'Auto Loan Origination Volume by Riskscore' => {'<620' => 'New Car Loans w Riskscore <620 ($B)','620-659' => 'New Car Loans w Riskscore 620-659 ($B)','660-719' => 'New Car Loans w Riskscore 660-719 ($B)','720-759' => 'New Car Loans w Riskscore 720-759 ($B)','760+' => 'New Car Loans w Riskscore 760+ ($B)','TOTAL' => 'All New Car Loans ($B)'},
-            'Distribution of Riskscore of Auto Loan Originations' => {'Median' => 'Median Riskscore for New Car Loan', '25th percentile' => 'Bottom 25%ile Riskscore for New Car Loan', '10th percentile' => 'Bottom 10%ile Riskscore for New Car Loan'},
-            'Credit Limit and Balance for Credit Cards and HE Revolving' => {'Credit Card Balance' => 'CC Balance ($T)', 'Credit Card Available Credit' => 'CC Avail Credit ($T)',  'Credit Card Limit' => 'CC Limit ($T)', 'HE Revolving Balance' => 'HELOC Balance ($T)',  'HE Revolving Available Credit' => 'HELOC Avail Credit ($T)', 'HE Revolving Limit' => 'HELOC Limit ($T)'},
-            'Total Balance by Delinquency Status' => {'Current' => '% Balances Current', '30 days late' => '% Balances 30d late',  '60 days late' => '% Balances 60d late',  '90 days late' => '% Balances 90d late',  '120+ days late' => '% Balances 120+d late',  'Severely Derogatory' => '% Balances Severely Impair'},
-            'Percent of Balance 90+ Days Delinquent by Loan Type' => {'MORTGAGE' => '% Mort Bal 90d+ late',  'HELOC' => '% HELOC Bal 90d+ late', 'AUTO' => '% Car Loan Bal 90d+ late',  'CC' => '% CC Bal 90d+ late',  'STUDENT LOAN' => '% Student Loan Bal 90d+ late',  'OTHER' => '% Other Bal 90d+ late', 'ALL' => '% All Bal 90d+ late'},
-            'New Delinquent* Balances by Loan Type' => {'AUTO' => '% Car Loan Bal 0d->30d+ Late',  'CC' => '% CC Bal 0d->30d+ Late',  'MORTGAGE' => '% Mort Bal 0d->30d+ Late',  'HELOC' => '% HELOC Bal 0d->30d+ Late', 'STUDENT LOAN' => '% Student Loan Bal 0d->30d+ Late',  'OTHER' => '% Other Bal 0d->30d+ Late', 'Total' => '% All Bal 0d->30d+ Late'},
-            'New Seriously Delinquent* Balances by Loan Type' => {'AUTO' => '% Car Loan Bal ->90d+ Late',  'CC' => '% CC Bal ->90d+ Late',  'MORTGAGE' => '% Mort Bal ->90d+ Late',  'HELOC' => '% HELOC Bal ->90d+ Late', 'STUDENT LOAN' => '% Student Loan Bal ->90d+ Late',  'OTHER' => '% Other Bal ->90d+ Late', 'ALL' => '% All Bal ->90d+ Late'},
-            'Quarterly Transition Rates for Current Mortgage Accounts' => {'To 30-60 days late' => 'Qtrly % Mort 0->30-60d late', 'To 90+ days Late' => 'Qtrly % Mort ->90d+ Late'},
-            'Quarterly Transition Rates for 30-60 Day Late Mortgage Accounts' => {'To current'  => 'Qtrly % Mort % 30-60d->0d Late', 'To 90+ days Late' => 'Qtrly % Mort 30-60d->90d+ Late'},
-            'Number of Consumers with New Foreclosures and Bankruptcies' => {'foreclosure' => 'Num Consumers w New Foreclosure',  'bankruptcy' => 'Num Consumers w New Bankruptcy'},
-            'Third Party Collections' => {'proportion of consumers with collection' => '% Consumers w Collection', 'average collection amount per person with item' => 'Avg Collection Amt'},
-            'Total Debt Balance by Age' => {'18-29' => 'Avg Total Debt Bal for 18-29 yo ($T)', '30-39' => 'Avg Total Debt Bal for 30-39 yo ($T)', '40-49' => 'Avg Total Debt Bal for 40-49 yo ($T)', '50-59' => 'Avg Total Debt Bal for 50-59 yo ($T)', '60-69' => 'Avg Total Debt Bal for 60-69 yo ($T)', '70+' => 'Avg Total Debt Bal for 70+ yo ($T)'},
-            'Auto Loan Originations By Age' => {'18-29' => 'New Car Loans for 18-29 yo ($B)', '30-39' => 'New Car Loans for 30-39 yo ($B)', '40-49' => 'New Car Loans for 40-49 yo ($B)', '50-59' => 'New Car Loans for 50-59 yo ($B)', '60-69' => 'New Car Loans for 60-69 yo ($B)', '70+' => 'New Car Loans for 70+ yo ($B)'},
-            'Total Mortgage Orginations by Age' => {'18-29' => 'New Mort for 18-29 yo ($B)', '30-39' => 'New Mort for 30-39 yo ($B)', '40-49' => 'New Mort for 40-49 yo ($B)', '50-59' => 'New Mort for 50-59 yo ($B)', '60-69' => 'New Mort for 60-69 yo ($B)', '70+' => 'New Mort for 70+ yo ($B)'},
-            'Transition into Serious Delinquency (90+) by Age' => {'18-29' => 'Qtrly % ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % ->90d+ late for 60-69 yo', '70+' => 'Qtrly % ->90d+ late for 70+ yo', 'all' => 'Qtrly % ->90d+ late'},
-            'Transition into Serious Delinquency (90+) for Mortgages by Age' => {'18-29' => 'Qtrly % Mort ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % Mort ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % Mort ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % Mort ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % Mort ->90d+ late for 60-69 yo', '70+' => 'Qtrly % Mort ->90d+ late for 70+ yo', 'all' => 'Qtrly % Mort ->90d+ late'},
-            'Transition into Serious Delinquency (90+) for Auto Loans by Age' => {'18-29' => 'Qtrly % Car Loans ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % Car Loans ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % Car Loans ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % Car Loans ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % Car Loans ->90d+ late for 60-69 yo', '70+' => 'Qtrly % Car Loans ->90d+ late for 70+ yo', 'all' => 'Qtrly % Car Loans ->90d+ late'},
-            'Transition into Serious Delinquency (90+) for Credit Cards by Age' => {'18-29' => 'Qtrly % CC ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % CC ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % CC ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % CC ->90d+ late for 50-59 yo', '60-69' => 'Qtrly % CC ->90d+ late for 60-69 yo', '70+' => 'Qtrly % CC ->90d+ late for 70+ yo', 'all' => 'Qtrly % CC ->90d+ late'},
-            'Transition into Serious Delinquency (90+) for Student Loans by Age' => {'18-29' => 'Qtrly % Student Loan ->90d+ late for 18-29 yo', '30-39' => 'Qtrly % Student Loan ->90d+ late for 30-39 yo', '40-49' => 'Qtrly % Student Loan ->90d+ late for 40-49 yo', '50-59' => 'Qtrly % Student Loan ->90d+ late for 50-59 yo', 'all' => 'Qtrly % Student Loan ->90d+ late'},
-            'New Foreclosures By Age' => {'18-29' => 'New Foreclosures for 18-29 yo (000)', '30-39' => 'New Foreclosures for 30-39 yo (000)', '40-49' => 'New Foreclosures for 40-49 yo (000)', '50-59' => 'New Foreclosures for 50-59 yo (000)', '60-69' => 'New Foreclosures for 60-69 yo (000)', '70+' => 'New Foreclosures for 70+ yo (000)'},
-            'New Bankruptcy By Age' => {'18-29' => 'New Bankruptcies for 18-29 yo (000)', '30-39' => 'New Bankruptcies for 30-39 yo (000)', '40-49' => 'New Bankruptcies for 40-49 yo (000)', '50-59' => 'New Bankruptcies for 50-59 yo (000)', '60-69' => 'New Bankruptcies for 60-69 yo (000)', '70+' => 'New Bankruptcies for 70+ yo (000)'}
-          }
-
-        # Polars::Config.set_tbl_cols(-1)
+        xlsx_file = Roo::Excelx.new(f.path)
+        title_index = _index_by_normalized_key(SHEET_COLUMNS)
+        remaining = title_index.keys.to_set
+
         Polars::Config.set_tbl_rows(-1)
 
-        done = false
-        xlsx_file.sheets.each do |sht|  # Begin per-sheet analysis
-          nm = sht.name
-
-          begin 
-            _ = sht.rows
-            pg_nm_text = sht.rows
-            pg_nm_text = pg_nm_text.first 
-          rescue 
-            begin
-              pg_nm_text = sht.rows.first
-            rescue 
-              done = true
-            end
-          end
+        xlsx_file.sheets.each do |sheet_name|
+          break if remaining.empty?
 
-          break if done == true
+          rows = _sheet_rows(xlsx_file, sheet_name)
+          next if rows.nil? || rows.empty?
 
-          pg_nm_text = pg_nm_text.first
-          next unless col_nm_hsh.keys.include?(pg_nm_text)
+          pg_nm_text = title_index[_normalize_key(rows.first&.first)]
+          next if pg_nm_text.nil?
+
+          remaining.delete(_normalize_key(pg_nm_text))
+          header_norm_to_out = _index_by_normalized_key(SHEET_COLUMNS[pg_nm_text])
 
           header_found = false
           csv_txt = []
+          pending = nil
 
-          sht.rows.each do |row|   # Begin per-line analysis
-            r = row
-
-            if header_found == false 
-              if r.length > (r - col_nm_hsh[pg_nm_text].keys).length
-                header_found = true 
-                0.upto(r.length-1).to_a.each do |ndex|
-                  if ndex.zero?
-                    r[ndex] = 'Timestamps_'
-                  else
-                    col_nm_hsh[pg_nm_text].keys.each do |k|
-                      r[ndex] = col_nm_hsh[pg_nm_text][k] if r[ndex] == k
-                    end
-                  end
-                end
-              end
+          rows.each do |row|
+            r = row.dup
 
-              next if header_found == false
-            end 
+            unless header_found
+              if _header_row?(r, header_norm_to_out)
+                header_found = true
+                csv_txt << _csv_line(_rename_header_row(r, header_norm_to_out))
+              end
+              next
+            end
 
             break if r.compact.length.zero?
 
-            # puts "#{__FILE__}:#{__LINE__} r = >#{r.inspect}<"
-            if /(?<yr>\d{2}):Q(?<qtr>\d)/ =~ r[0].to_s
-              yr = yr.to_i + 2000
-              yr -= 100 if yr > Date.today.year
-              dte = Date.new(yr, qtr.to_i * 3 - 2, 1)
-              r[0] = dte
-            elsif (r[0].to_date rescue false)
-              r[0] = r[0].to_date.beginning_of_quarter
+            date = _parse_quarter_date(r[0])
+            if date
+              csv_txt << _csv_line(_filled_row(pending)) if pending
+              r[0] = date
+              pending = r
+            elsif pending
+              r.each_index { |i| pending[i] = r[i] if pending[i].nil? && !r[i].nil? }
             end
+          end
 
-            next if r.map{|rr| (rr.nil? ? '' : rr.to_s).length }.sum.zero?
-            rr = r.map{|c| c.nil? ? '' : c}
-            rr = rr.reverse.drop_while(&:nil?).reverse
-            next if rr.compact.length.zero?
-            csv_txt << rr.join(',')
-
-          end    # End per-line analysis
+          csv_txt << _csv_line(_filled_row(pending)) if pending
+          next if csv_txt.length < 2
 
           csv_dta = CSV.parse(csv_txt.join("\n"), headers: true, converters: [:numeric])
           csv_dta.delete(nil)
 
-          keys = csv_dta.headers.compact.intersection(['Timestamps_'] + col_nm_hsh[pg_nm_text].values)
+          keys = csv_dta.headers.compact.intersection(['Timestamps_'] + SHEET_COLUMNS[pg_nm_text].values)
           vals = csv_dta.to_a[1..-1].map{|r| (r + [''] * keys.length)[0..keys.length-1] }.transpose
 
           tmp_df = {}; (0..(keys.length-1)).to_a.each{|i| tmp_df[keys[i]] = vals[i]}
 
           tmp_df = Polars::DataFrame.new(tmp_df)
-          tmp_df = tmp_df.filter(Polars.col('Timestamps_') <= fin.try(:to_date)) unless fin.nil?
-          tmp_df = tmp_df.filter(Polars.col('Timestamps_') >= start.try(:to_date)) unless start.nil?
+          fin_date = fin.try(:to_date)
+          start_date = start.try(:to_date)
+          tmp_df = tmp_df.filter(Polars.col('Timestamps_') <= fin_date.to_s) unless fin_date.nil?
+          tmp_df = tmp_df.filter(Polars.col('Timestamps_') >= start_date.to_s) unless start_date.nil?
           tmp_df.sort!('Timestamps_')
 
           if df.nil?
-            df = tmp_df.dup 
-          else 
+            df = tmp_df.dup
+          else
             df = df.join(tmp_df, on: 'Timestamps_', how: 'full')
-
-            tmp_timestamps = df['Timestamps_'].to_a 
-            tmp_timestamps__ = df['Timestamps__right'].to_a
-            tmp_timestamps.length.times.each {|ndex| tmp_timestamps[ndex] ||= tmp_timestamps__[ndex] }
-            df.replace('Timestamps_', Polars::Series.new(tmp_timestamps))
-            df = df.drop(['Timestamps__right'])
+            df = df.with_columns(
+              Polars.coalesce(Polars.col('Timestamps_'), Polars.col('Timestamps__right')).alias('Timestamps_')
+            ).drop('Timestamps__right')
           end
           df = df.sort('Timestamps_')
- 
-        end    # End per-sheet analysis
+        end
+
+        raise 'HHDC workbook contained no recognized data sheets' if df.nil?
 
         s = Polars::Series.new('Timestamps', df['Timestamps_'].to_a)
-        df = df.insert_column(0,s).drop(['Timestamps_'])
+        df = df.insert_column(0, s).drop('Timestamps_')
       end
 
-      df = df.drop(df.columns - ['Timestamps', @tag]) unless @tag.nil? || !df.columns.include?(@tag)
+      df = df.drop(*(df.columns - ['Timestamps', @tag])) unless @tag.nil? || !df.columns.include?(@tag)
       df
     end
+
+    private
+
+    def _normalize_label(text)
+      return '' if text.nil?
+
+      text.to_s
+          .gsub(/<[^>]+>/, '')
+          .gsub(/[[:space:]]+/, ' ')
+          .delete('*')
+          .strip
+    end
+
+    def _normalize_key(text)
+      _normalize_label(text).downcase
+    end
+
+    def _index_by_normalized_key(hsh)
+      hsh.each_with_object({}) { |(key, value), acc| acc[_normalize_key(key)] = value.is_a?(Hash) ? key : value }
+    end
+
+    def _sheet_rows(xlsx_file, sheet_name)
+      sheet = xlsx_file.sheet(sheet_name)
+      last_row = sheet.last_row
+      return nil if last_row.nil?
+
+      (1..last_row).map { |idx| sheet.row(idx) }
+    rescue StandardError
+      nil
+    end
+
+    def _header_row?(row, header_norm_to_out)
+      row.any? { |cell| header_norm_to_out.key?(_normalize_key(cell)) }
+    end
+
+    def _rename_header_row(row, header_norm_to_out)
+      row.each_index do |ndex|
+        if ndex.zero?
+          row[ndex] = 'Timestamps_'
+        else
+          mapped = header_norm_to_out[_normalize_key(row[ndex])]
+          row[ndex] = mapped if mapped
+        end
+      end
+      row
+    end
+
+    def _parse_quarter_date(value)
+      if /(?<yr>\d{2}):Q(?<qtr>\d)/ =~ value.to_s
+        yr = yr.to_i + 2000
+        yr -= 100 if yr > Date.today.year
+        Date.new(yr, qtr.to_i * 3 - 2, 1)
+      elsif (parsed = (value.to_date rescue nil))
+        parsed.beginning_of_quarter
+      end
+    end
+
+    def _filled_row(row)
+      row.map { |cell| cell.nil? ? '' : cell }
+    end
+
+    def _csv_line(row)
+      trimmed = row.reverse.drop_while { |cell| cell.nil? || cell == '' }.reverse
+      trimmed.join(',')
+    end
   end
 end

data/lib/ny_frb_as_dataframe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module NyFrbAsDataframe
-  VERSION = "0.1.1"
+  VERSION = "0.2.1"
 end

data/ny_frb_as_dataframe.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.description   = "Alternative economic and financial data from the NY FRB"
   spec.homepage      = "https://github.com/bmck/ny_frb_as_dataframe"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3")
 
 
   spec.metadata["homepage_uri"] = spec.homepage
@@ -32,8 +32,8 @@ Gem::Specification.new do |spec|
 
   # For more information and examples about making a new gem, checkout our
   # guide at: https://bundler.io/guides/creating_gem.html
-  spec.add_dependency 'polars-df'
-  spec.add_dependency 'httparty'  
-  spec.add_dependency 'simple_xlsx_reader'
+  spec.add_dependency 'polars-df', '0.27.1'
+  spec.add_dependency 'httparty'
+  spec.add_dependency 'roo'
   spec.add_dependency 'activesupport'
 end

data/reqs/executed_plans.md

@@ -0,0 +1,3 @@
+# Executed plans
+
+No executed-plan notes have been recorded yet.

data/reqs/explicitly_stated_requirements.md

@@ -0,0 +1,5 @@
+# Explicitly stated requirements
+
+| ID | Title | File |
+| --- | --- | --- |
+| REQ-001 | Parse the current NY Fed HHDC workbook | [requirements/REQ-001.md](requirements/REQ-001.md) |

data/reqs/requirements/REQ-001.md

@@ -0,0 +1,15 @@
+# REQ-001 — Parse the current NY Fed HHDC workbook
+
+**Source:** user report of consumer-app failures against `NyFrbAsDataframe::Hhdc#fetch` (2026-09-09).
+
+## Statement
+
+`NyFrbAsDataframe::Hhdc#fetch` must download the current NY Fed `hhd_c_report_YYYYqN.xlsx` and return a Polars frame. The published workbook is chart-heavy and its printed titles/headers do not match the gem’s historical exact strings.
+
+## Required behavior
+
+- Do not set `SimpleXlsxReader.configuration.auto_slurp`. That attribute does not exist on `simple_xlsx_reader` 1.0.5 (`NoMethodError: undefined method 'auto_slurp='`).
+- Do not use SimpleXlsxReader on this workbook. Chart sheets have no dimension node; 1.0.5 assumes every sheet has one and raises `NoMethodError: undefined method 'at_xpath' for nil` in `Mapper#last_cell_label`. Parse data sheets with Roo instead.
+- Match sheet titles after normalizing HTML, trailing `*`, and extra spaces. Current titles include markup such as `<html>New Delinquent<sup>*</sup> Balances by Loan Type </html>`.
+- Match headers after the same normalize, case-insensitively. The workbook uses `To 90+ days late`; the gem’s map has `To 90+ days Late`.
+- Keep the column names this gem emits. Do not assume README names still match the xlsx labels (example: gem `Qtrly % Mort 0->30-60d late` vs a consumer’s `Qtrly % Mort Current->30-60d`).

data/test_basic.rb

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "bundler/setup"
+require_relative "lib/ny_frb_as_dataframe"
+
+puts "Testing ny_frb_as_dataframe gem..."
+puts "Ruby version: #{RUBY_VERSION}"
+puts "Required Ruby version: >= 3.3"
+
+# Verify Ruby version
+if RUBY_VERSION.split('.').first.to_i < 3 || 
+   (RUBY_VERSION.split('.').first.to_i == 3 && RUBY_VERSION.split('.')[1].to_i < 3)
+  puts "ERROR: Ruby version #{RUBY_VERSION} is below required 3.3"
+  exit 1
+end
+
+# Test basic gem loading
+begin
+  puts "\n✓ Gem loaded successfully"
+  
+  # Test that we can instantiate the Hhdc class
+  hhdc = NyFrbAsDataframe::Hhdc.new
+  puts "✓ NyFrbAsDataframe::Hhdc class instantiated"
+  
+  # Test that polars-df is available
+  require 'polars'
+  puts "✓ Polars library loaded successfully"
+  
+  puts "\nAll basic tests passed!"
+  exit 0
+rescue => e
+  puts "\nERROR: #{e.message}"
+  puts e.backtrace.first(5)
+  exit 1
+end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: ny_frb_as_dataframe
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.1
 platform: ruby
 authors:
 - Bill McKinnon
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-30 00:00:00.000000000 Z
+date: 2026-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: polars-df
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.27.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.27.1
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simple_xlsx_reader
+  name: roo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -75,6 +75,7 @@ extra_rdoc_files: []
 files:
 - ".DS_Store"
 - ".gitignore"
+- ".ruby-version"
 - ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -88,6 +89,10 @@ files:
 - lib/ny_frb_as_dataframe/hhdc.rb
 - lib/ny_frb_as_dataframe/version.rb
 - ny_frb_as_dataframe.gemspec
+- reqs/executed_plans.md
+- reqs/explicitly_stated_requirements.md
+- reqs/requirements/REQ-001.md
+- test_basic.rb
 homepage: https://github.com/bmck/ny_frb_as_dataframe
 licenses:
 - MIT
@@ -103,14 +108,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 3.0.0
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.5.10
 signing_key:
 specification_version: 4
 summary: Alternative economic and financial data from the NY FRB