worldbank_as_dataframe 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1141c280f36ed7062973307d4020dce4dc9800a942461b9f9333d68599fd1a4
-  data.tar.gz: f506f93a82cc9d85e00ec518e6a1cf88f6ce5dba88fd9c409e28db7ddfbe5c79
+  metadata.gz: dcb1ea76123deedc99ddce0e174ccd7930eba3a27ed76a617c7c8a7b28eb967a
+  data.tar.gz: daa92655da05f0df4014a67f4c3d35c14eff14c319c4da619747312c79a6b6d9
 SHA512:
-  metadata.gz: de67152e06b713eff2488cead10307c8b19fb19da27912bae2236849a7c6a43ab8fb1ed729d57a45b578589c9b4f7cb7f7d5aff07bf5acebebed6a245df98073
-  data.tar.gz: b5fc9be7e5419a206b9cfd99202eb0f96b8ea6661285a35ca47aec7c552e98e0080dd586fdb76ba2d45668ca9c22b86bebf3025383ee7815c8b61e5cce720080
+  metadata.gz: c08b5c1fda2faaef793185a0c8b3e1c56509d98ca38380faf892952476fb42a75da6b8c0d1be99cfeab4192c95f3803762cc048c4e5d040987672b9ce36e348c
+  data.tar.gz: f73a4ccd0590c351e8fde7e932b2744ec818034e7340eae868f3b79e8a89b1aae82bb746c9698c8ce542abfe6422607b3c9acd5f5755f8e4f16e267224a479a1

data/Changelog

@@ -1,3 +1,7 @@
+0.2.2
+	Parse CMO commodity workbooks with Roo instead of SimpleXlsxReader
+	Match the Monthly Prices sheet after normalizing title whitespace and case
+	Skip chart-only sheets such as AFOSHEET
 0.0.1
 	Initial release
 0.9.0

data/README.md

@@ -96,6 +96,9 @@ include WorldbankAsDataframe
   # modifiers (like #dates above) called as class methods or chained in a query.
   #
 
+  # Commodities#fetch downloads the Pink Sheet xlsx and parses the Monthly Prices
+  # sheet with Roo (chart-only sheets such as AFOSHEET are skipped).
+
   WorldbankAsDataframe::Commodities.new.fetch
   #    => 
   # shape: (777, 72)

data/lib/worldbank_as_dataframe/commodities.rb

@@ -1,39 +1,108 @@
 require 'open-uri'
-require 'simple_xlsx_reader'
+require 'roo'
 require 'polars-df'
+require 'tempfile'
+require 'date'
 
 module WorldbankAsDataframe
   class Commodities
+    CMO_URL = 'https://thedocs.worldbank.org/en/doc/5d903e848db1d1b83e0ec8f744e55570-0350012021/related/CMO-Historical-Data-Monthly.xlsx'
+    MONTHLY_PRICES_SHEET = 'Monthly Prices'
+    MISSING_MARKERS = ["…", "..."].freeze
+
     attr_reader :tag
 
     def initialize(series = nil, options={})
       @tag = series
     end
 
-    def fetch(start: nil, fin: nil)
-      doc = SimpleXlsxReader.parse(URI.open('https://thedocs.worldbank.org/en/doc/5d903e848db1d1b83e0ec8f744e55570-0350012021/related/CMO-Historical-Data-Monthly.xlsx'))
-      ary = doc.sheets.detect{|sht| sht.name == 'Monthly Prices'}.rows.to_a[4..-1]
+    def fetch(start: nil, fin: nil, path: nil)
+      if path
+        _frame_from_path(path, start: start, fin: fin)
+      else
+        Tempfile.open(['cmo', '.xlsx'], binmode: true) do |f|
+          f.write(URI.open(CMO_URL, 'User-Agent' => 'worldbank_as_dataframe').read)
+          f.flush
+          _frame_from_path(f.path, start: start, fin: fin)
+        end
+      end
+    end
+
+    def _frame_from_path(path, start: nil, fin: nil)
+      xlsx = Roo::Excelx.new(path)
+      rows = _monthly_price_rows(xlsx)
+      _build_frame(rows, start: start, fin: fin)
+    end
+
+    def _monthly_price_rows(xlsx)
+      name = xlsx.sheets.find { |sheet_name| _normalize_key(sheet_name) == _normalize_key(MONTHLY_PRICES_SHEET) }
+      raise 'CMO workbook contained no Monthly Prices sheet' if name.nil?
+
+      sheet = xlsx.sheet(name)
+      last_row = sheet.last_row
+      raise 'CMO Monthly Prices sheet is empty' if last_row.nil?
+
+      (1..last_row).map { |idx| sheet.row(idx) }
+    end
+
+    def _build_frame(rows, start: nil, fin: nil)
+      ary = rows[4..].map { |row| row.dup }
+      raise 'CMO Monthly Prices sheet is missing header rows' if ary.nil? || ary.length < 3
+
       ary[0][0] = 'Timestamps'
-      ary.map!{|a| a.reverse.drop_while(&:nil?).reverse }
-      ary[1].length.times {|i| ary[1][i] = [ary[0][i], ary[1][i]].compact.join(' ') }
-      ary = ary[1..-1]
+      ary.map! { |a| a.reverse.drop_while(&:nil?).reverse }
+      ary[1].length.times { |i| ary[1][i] = [ary[0][i], ary[1][i]].compact.join(' ') }
+      ary = ary[1..]
 
-      dat = ary[1..-1]
       cols = ary[0]
+      dat = ary[1..].map { |row| _coerce_row(row) }
+      dat.reject! { |row| row.compact.empty? }
 
-      dat.each{|a| dt = a[0].split('M'); a[0] = Date.new(dt[0].to_i, dt[1].to_i, 1).to_date }
-      dat.each{|row| row.map!{|c| c.in?(["…","..."]) ? nil : c }}
+      frame_hsh = {}
+      cols.each_with_index { |col, i| frame_hsh[col] = dat.map { |row| row[i] } }
+      df = Polars::DataFrame.new(frame_hsh)
 
-      df = Polars::DataFrame.new(dat.transpose, columns: cols)
-      df = df.filter(Polars.col('Timestamps') >= start.to_date) unless start.nil?
-      df = df.filter(Polars.col('Timestamps') <= fin.to_date) unless fin.nil?
+      start_date = _to_date(start)
+      fin_date = _to_date(fin)
+      df = df.filter(Polars.col('Timestamps') >= start_date) unless start_date.nil?
+      df = df.filter(Polars.col('Timestamps') <= fin_date) unless fin_date.nil?
 
       unless @tag.nil?
-        cols = cols.map{|c| (/#{tag.downcase}/ =~ c.downcase).nil? ? nil : c }.compact
-        df = df.select(['Timestamps', cols].flatten) if (cols.length>0)
+        matched = cols.select { |c| c.to_s.downcase.include?(@tag.to_s.downcase) }
+        df = df.select(['Timestamps', *matched]) if matched.length > 0
       end
 
       df
     end
+
+    def _normalize_label(text)
+      return '' if text.nil?
+
+      text.to_s
+          .gsub(/<[^>]+>/, '')
+          .gsub(/[[:space:]]+/, ' ')
+          .delete('*')
+          .strip
+    end
+
+    def _normalize_key(text)
+      _normalize_label(text).downcase
+    end
+
+    def _coerce_row(row)
+      row = row.dup
+      if row[0].to_s =~ /\A(\d{4})M(\d{2})\z/
+        row[0] = Date.new($1.to_i, $2.to_i, 1)
+      end
+      row.map { |cell| MISSING_MARKERS.include?(cell) ? nil : cell }
+    end
+
+    def _to_date(value)
+      return nil if value.nil?
+      return value if value.is_a?(Date)
+      return value.to_date if value.respond_to?(:to_date)
+
+      Date.parse(value.to_s)
+    end
   end
-end
+end

data/lib/worldbank_as_dataframe/version.rb

@@ -1,3 +1,3 @@
 module WorldbankAsDataframe
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/reqs/executed_plans.md

@@ -0,0 +1,3 @@
+# Executed plans
+
+No executed-plan notes have been recorded yet.

data/reqs/explicitly_stated_requirements.md

@@ -0,0 +1,5 @@
+# Explicitly stated requirements
+
+| ID | Title | File |
+| --- | --- | --- |
+| REQ-001 | Parse the current World Bank CMO workbook | [requirements/REQ-001.md](requirements/REQ-001.md) |

data/reqs/requirements/REQ-001.md

@@ -0,0 +1,13 @@
+# REQ-001 — Parse the current World Bank CMO workbook
+
+**Source:** user directive to fix the same SimpleXlsxReader failures found in `ny_frb_as_dataframe` (2026-09-09).
+
+## Statement
+
+`WorldbankAsDataframe::Commodities#fetch` must load the current Pink Sheet `CMO-Historical-Data-Monthly.xlsx` and return a Polars frame.
+
+## Required behavior
+
+- Do not use SimpleXlsxReader. The published workbook includes a chart sheet (`AFOSHEET`) and `simple_xlsx_reader` 1.0.5 fails on sheets without a dimension node. Parse data sheets with Roo.
+- Do not set `SimpleXlsxReader.configuration.auto_slurp`.
+- Match the `Monthly Prices` sheet after normalizing whitespace and case.

data/spec/worldbank_as_dataframe/commodities_spec.rb

@@ -0,0 +1,29 @@
+require 'helper'
+
+describe WorldbankAsDataframe::Commodities do
+  let(:fixture) { File.expand_path('../../fixtures/cmo_monthly_prices.xlsx', __FILE__) }
+
+  it 'is instantiable' do
+    WorldbankAsDataframe::Commodities.new.should be_a WorldbankAsDataframe::Commodities
+  end
+
+  it 'parses Monthly Prices from a chart-heavy workbook via Roo' do
+    df = WorldbankAsDataframe::Commodities.new.fetch(path: fixture)
+    df.columns.should include('Timestamps')
+    df.columns.should include('Crude oil, average ($/bbl)')
+    df['Timestamps'].to_a.should include(Date.new(1960, 1, 1))
+    df['Crude oil, average ($/bbl)'].to_a.first.should == 1.63
+  end
+
+  it 'treats ellipsis cells as null and matches series tags case-insensitively' do
+    df = WorldbankAsDataframe::Commodities.new('crude oil').fetch(path: fixture)
+    df.columns.should == ['Timestamps', 'Crude oil, average ($/bbl)']
+    df.filter(Polars.col('Timestamps') == Date.new(1960, 2, 1))['Crude oil, average ($/bbl)'].to_a.first.should be_nil
+  end
+
+  it 'filters by start and fin' do
+    df = WorldbankAsDataframe::Commodities.new.fetch(path: fixture, start: '2020-01-01', fin: '2020-12-31')
+    df.height.should == 1
+    df['Timestamps'].to_a.should == [Date.new(2020, 1, 1)]
+  end
+end

data/test/test_commodities.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'date'
+require_relative '../lib/worldbank_as_dataframe'
+
+FIXTURE = File.expand_path('../spec/fixtures/cmo_monthly_prices.xlsx', __dir__)
+failures = []
+
+failures << 'version' if WorldbankAsDataframe::VERSION.to_s.empty?
+
+df = WorldbankAsDataframe::Commodities.new.fetch(path: FIXTURE)
+failures << 'timestamps column' unless df.columns.include?('Timestamps')
+failures << 'crude column' unless df.columns.include?('Crude oil, average ($/bbl)')
+failures << '1960-01-01 row' unless df['Timestamps'].to_a.include?(Date.new(1960, 1, 1))
+failures << 'crude value' unless df['Crude oil, average ($/bbl)'].to_a.first == 1.63
+
+tagged = WorldbankAsDataframe::Commodities.new('CRUDE OIL').fetch(path: FIXTURE)
+failures << 'tag columns' unless tagged.columns == ['Timestamps', 'Crude oil, average ($/bbl)']
+
+windowed = WorldbankAsDataframe::Commodities.new.fetch(path: FIXTURE, start: '2020-01-01', fin: '2020-12-31')
+failures << 'date window' unless windowed.height == 1 && windowed['Timestamps'].to_a == [Date.new(2020, 1, 1)]
+
+c = WorldbankAsDataframe::Commodities.new
+failures << 'normalize' unless c._normalize_key('<html>Monthly Prices*</html>') == 'monthly prices'
+
+if failures.empty?
+  puts 'test_commodities: ok'
+else
+  abort "test_commodities failed: #{failures.join(', ')}"
+end

data/worldbank_as_dataframe.gemspec

@@ -26,5 +26,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency 'httparty'
   gem.add_runtime_dependency 'polars-df'
   gem.add_runtime_dependency 'multi_json'
+  gem.add_runtime_dependency 'roo'
   # gem.add_runtime_dependency 'rash', '~> 0.3.0'  
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: worldbank_as_dataframe
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Bill McKinnon
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-10-10 00:00:00.000000000 Z
+date: 2026-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ZenTest
@@ -150,6 +150,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: roo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: A Ruby wrapper around the World Bank's Development Indicators API, with
   credit to Code for America and Justin Stoller
 email:
@@ -191,7 +205,11 @@ files:
 - lib/worldbank_as_dataframe/source.rb
 - lib/worldbank_as_dataframe/topic.rb
 - lib/worldbank_as_dataframe/version.rb
+- reqs/executed_plans.md
+- reqs/explicitly_stated_requirements.md
+- reqs/requirements/REQ-001.md
 - spec/fixtures/brazil.json
+- spec/fixtures/cmo_monthly_prices.xlsx
 - spec/fixtures/countries.json
 - spec/fixtures/countries_india.json
 - spec/fixtures/income_level_lmc.json
@@ -208,6 +226,7 @@ files:
 - spec/fixtures/topics.json
 - spec/helper.rb
 - spec/worldbank_as_dataframe/client_spec.rb
+- spec/worldbank_as_dataframe/commodities_spec.rb
 - spec/worldbank_as_dataframe/country_spec.rb
 - spec/worldbank_as_dataframe/data_query_spec.rb
 - spec/worldbank_as_dataframe/data_spec.rb
@@ -220,6 +239,7 @@ files:
 - spec/worldbank_as_dataframe/source_spec.rb
 - spec/worldbank_as_dataframe/topic_spec.rb
 - spec/worldbank_as_dataframe_spec.rb
+- test/test_commodities.rb
 - worldbank_as_dataframe.gemspec
 homepage: https://github.com/bmck/worldbank_as_dataframe
 licenses: []
@@ -245,6 +265,7 @@ specification_version: 4
 summary: A Ruby wrapper around the World Bank's Development Indicators API
 test_files:
 - spec/fixtures/brazil.json
+- spec/fixtures/cmo_monthly_prices.xlsx
 - spec/fixtures/countries.json
 - spec/fixtures/countries_india.json
 - spec/fixtures/income_level_lmc.json
@@ -261,6 +282,7 @@ test_files:
 - spec/fixtures/topics.json
 - spec/helper.rb
 - spec/worldbank_as_dataframe/client_spec.rb
+- spec/worldbank_as_dataframe/commodities_spec.rb
 - spec/worldbank_as_dataframe/country_spec.rb
 - spec/worldbank_as_dataframe/data_query_spec.rb
 - spec/worldbank_as_dataframe/data_spec.rb
@@ -273,3 +295,4 @@ test_files:
 - spec/worldbank_as_dataframe/source_spec.rb
 - spec/worldbank_as_dataframe/topic_spec.rb
 - spec/worldbank_as_dataframe_spec.rb
+- test/test_commodities.rb