redfin_as_dataframe 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ff5109eae1acf4c4b239aba97337e3400865ea5abf175e7434cce92ce06c8d07
-  data.tar.gz: dd003b5db37739d3b5c5db2eb9674da57a79a92974b142774bd567e27c08bc65
+  metadata.gz: de325bc8ad975242070ac2c49f4f5c9ba95ba58e9b2bf05f8a243aee3e073363
+  data.tar.gz: a4a3879f60b2fc0379ebb367d6709c47ba50b3e3b12ea7fec23f2632a9c5d62c
 SHA512:
-  metadata.gz: fd8516a3ea881655a30c1be76b4718184a60fd46340678eba22af0261053fe5c1e56ae219b883412337012621428aca11f5d0087e282d7d197746011eb6d1092
-  data.tar.gz: 5ae220fcecf9d9e5d31675f8103fb8422c0f70bfcc5d969acb05e5ddc2cc2264096bccf41e9d12964a873b1ff591ffea462e9651e405a2948ca3915e2bded9e7
+  metadata.gz: ee8235b783a269d7f844a11dfe5e130dc75e964b05c26f3b85a7382cc09fd4beb2f7766735ad39b36030779e2b42fc1e2aab71ee2ad91e4cb8eb6a29e4253890
+  data.tar.gz: 0fbeeef954715f259e631437e23d6f3a9af16a24a944725521e49ee4acb4aefcfbf10ffaa7e27e0991635a80c9ca41c19fcd468e48a489fce5d8d2704a969679

data/.gitignore

@@ -6,3 +6,5 @@
 /pkg/
 /spec/reports/
 /tmp/
+/vendor/bundle/
+*.gem

data/.ruby-version

@@ -0,0 +1 @@
+3.3.6

data/Gemfile

@@ -1,7 +1,10 @@
 source "https://rubygems.org"
 
+ruby '3.3.6'
+
 # Specify your gem's dependencies in redfin_as_dataframe.gemspec
 gemspec
 
 gem "rake", "~> 12.0"
-gem "none", "~> "
+gem "rspec", "~> 3.0"
+gem "webmock", "~> 3.0"

data/Gemfile.lock

@@ -0,0 +1,72 @@
+PATH
+  remote: .
+  specs:
+    redfin_as_dataframe (0.2.0)
+      httparty
+      polars-df (= 0.27.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.9.0)
+      public_suffix (>= 2.0.2, < 8.0)
+    bigdecimal (4.1.2)
+    crack (1.0.1)
+      bigdecimal
+      rexml
+    csv (3.3.6)
+    diff-lcs (1.6.2)
+    hashdiff (1.2.1)
+    httparty (0.24.2)
+      csv
+      mini_mime (>= 1.0.0)
+      multi_xml (>= 0.5.2)
+    mini_mime (1.1.5)
+    multi_xml (0.9.1)
+      bigdecimal (>= 3.1, < 5)
+    polars-df (0.27.1-aarch64-linux)
+    polars-df (0.27.1-aarch64-linux-musl)
+    polars-df (0.27.1-arm64-darwin)
+    polars-df (0.27.1-x86_64-darwin)
+    polars-df (0.27.1-x86_64-linux)
+    polars-df (0.27.1-x86_64-linux-musl)
+    public_suffix (7.0.5)
+    rake (12.3.3)
+    rexml (3.4.4)
+    rspec (3.13.2)
+      rspec-core (~> 3.13.0)
+      rspec-expectations (~> 3.13.0)
+      rspec-mocks (~> 3.13.0)
+    rspec-core (3.13.6)
+      rspec-support (~> 3.13.0)
+    rspec-expectations (3.13.5)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-mocks (3.13.8)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.13.0)
+    rspec-support (3.13.7)
+    webmock (3.26.3)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  aarch64-linux
+  aarch64-linux-musl
+  arm64-darwin
+  x86_64-darwin
+  x86_64-linux
+  x86_64-linux-musl
+
+DEPENDENCIES
+  rake (~> 12.0)
+  redfin_as_dataframe!
+  rspec (~> 3.0)
+  webmock (~> 3.0)
+
+RUBY VERSION
+   ruby 3.3.6p108
+
+BUNDLED WITH
+   2.5.22

data/README.md

@@ -1,8 +1,13 @@
 # RedfinAsDataframe
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/redfin_as_dataframe`. To experiment with that code, run `bin/console` for an interactive prompt.
+Up to date housing market data access for Ruby, using Polars dataframes. 
 
-TODO: Delete this and the text above, and describe your gem
+This gem fetches housing market information from Redfin's public data API and returns the results as a Polars dataframe. For details regarding the data available from Redfin, see https://www.redfin.com/news/data-center/ .
+
+## Requirements
+
+- Ruby >= 3.3
+- polars-df 0.27.1
 
 ## Installation
 
@@ -22,17 +27,82 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
+### Basic Usage
 
-## Development
+Fetch all available national housing market data:
+
+```ruby
+require 'redfin_as_dataframe'
+
+national = RedfinAsDataframe::National.new
+df = national.fetch
+```
+
+This returns a Polars DataFrame with housing market metrics including median sale prices, inventory, months of supply, and more.
+
+### Filtering by Date Range
+
+You can filter data by specifying start and/or end dates:
+
+```ruby
+# Fetch data from a specific start date
+national = RedfinAsDataframe::National.new
+df = national.fetch(start: '2020-01-01')
+
+# Fetch data up to a specific end date
+df = national.fetch(fin: '2023-12-31')
+
+# Fetch data within a date range
+df = national.fetch(start: '2020-01-01', fin: '2023-12-31')
+```
+
+The `start` parameter filters to include only data where `period_begin` is on or after the specified date.
+The `fin` parameter filters to include only data where `period_end` is on or before the specified date.
 
-After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake none` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+### Selecting Specific Data Series
+
+You can fetch a specific data series by passing the column name to the initializer:
+
+```ruby
+# Fetch only median sale price data
+national = RedfinAsDataframe::National.new('median_sale_price')
+df = national.fetch
+
+# The returned DataFrame will contain only 'Timestamps' and 'median_sale_price' columns
+```
+
+Available series include:
+- `median_sale_price`, `median_list_price`
+- `median_ppsf` (price per square foot), `median_list_ppsf`
+- `homes_sold`, `pending_sales`, `new_listings`, `inventory`
+- `months_of_supply`, `median_dom` (days on market)
+- `avg_sale_to_list`, `sold_above_list`, `price_drops`
+- And many more (see the full TSV data for all available columns)
+
+## Testing
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+The test suite uses RSpec and includes:
+- Unit tests for the `National` class
+- Mocked HTTP requests (no live API calls during testing)
+- Tests for date filtering and data series selection
+
+To run the tests:
+
+    $ bundle exec rake spec
+
+Or run tests directly with RSpec:
+
+    $ bundle exec rspec
+
+## Development
 
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/redfin_as_dataframe.
+Bug reports and pull requests are welcome on GitHub at https://github.com/bmck/redfin_as_dataframe.
 
 
 ## License

data/Rakefile

@@ -1,2 +1,6 @@
 require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
 task :default => :spec

data/lib/redfin_as_dataframe/national.rb

@@ -0,0 +1,50 @@
+require 'polars-df'
+require 'httparty'
+require 'zlib'
+require 'stringio'
+require 'csv'
+require 'date'
+
+module RedfinAsDataframe
+  class National
+    # Get data for Redfin national housing market metrics.
+
+    include ::HTTParty
+
+    attr_reader :tag, :ary
+
+    def initialize(series = nil, options={})
+      @tag = series
+    end
+
+    def fetch(start: nil, fin: nil)
+      dat = _fetch_data.dup
+      hsh = dat.map!{|r| r.to_hash }
+      keys = hsh.first.keys
+      vals = hsh.map(&:values).transpose
+
+      df = {}; (0..(keys.length-1)).to_a.each{|i| df[keys[i]] = vals[i]}
+
+      df = Polars::DataFrame.new(df).drop(['period_duration', 'region_type', 'region_type_id','table_id','is_seasonally_adjusted','region','city','state','state_code', 'parent_metro_region', 'parent_metro_region_metro_code'])
+      df = df.filter(Polars.col('period_begin') >= (start.is_a?(Date) ? start : Date.parse(start.to_s))) unless start.nil?
+      df = df.filter(Polars.col('period_end') <= (fin.is_a?(Date) ? fin : Date.parse(fin.to_s))) unless fin.nil?
+      s = Polars::Series.new('Timestamps', df['period_begin'].to_a)
+      df = df.insert_column(0,s)
+
+      df = (tag.nil? ? df : df.drop(df.columns - ['Timestamps', tag])).sort('Timestamps')
+    end
+
+    protected
+
+    private
+
+    def _fetch_data #(options={})
+      if @ary.nil?
+        tsv_gz = self.class.get('https://redfin-public-data.s3.us-west-2.amazonaws.com/redfin_market_tracker/us_national_market_tracker.tsv000.gz').parsed_response
+        tsv = Zlib::GzipReader.new(StringIO.new(tsv_gz)).read    
+        @ary = CSV.parse(tsv, headers: true, col_sep: "\t", converters: :numeric).select{|a| a['is_seasonally_adjusted'] == 'f'}.each{|r| r['period_begin'] = Date.parse(r['period_begin']); r['period_end'] = Date.parse(r['period_end']) }
+      end
+      @ary
+    end
+  end
+end

data/lib/redfin_as_dataframe/version.rb

@@ -1,3 +1,3 @@
 module RedfinAsDataframe
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/redfin_as_dataframe.gemspec

@@ -10,13 +10,12 @@ Gem::Specification.new do |spec|
   spec.description   = 'Homeownership and purchase metrics published by Redfin'
   spec.homepage      = "https://github.com/bmck/redfin_as_dataframe"
   spec.license       = "MIT"
-  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+  spec.required_ruby_version = Gem::Requirement.new(">= 3.3")
 
   # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
 
   spec.metadata["homepage_uri"] = spec.homepage
   spec.metadata["source_code_uri"] = "https://github.com/bmck/redfin_as_dataframe"
-  spec.metadata["changelog_uri"] = "https://github.com/bmck/redfin_as_dataframe/CHANGELOG.md"
 
   # Specify which files should be added to the gem when it is released.
   # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
@@ -29,6 +28,6 @@ Gem::Specification.new do |spec|
   
   # Uncomment to register a new dependency of your gem
   # spec.add_dependency "typhoeus-gem", "~> 0.6.9"
-  spec.add_dependency 'polars-df'
+  spec.add_dependency 'polars-df', '0.27.1'
   spec.add_dependency 'httparty'  
 end

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: redfin_as_dataframe
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Bill McKinnon
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-05 00:00:00.000000000 Z
+date: 2026-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: polars-df
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.27.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.27.1
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
@@ -46,14 +46,17 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".ruby-version"
 - ".travis.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
 - lib/redfin_as_dataframe.rb
+- lib/redfin_as_dataframe/national.rb
 - lib/redfin_as_dataframe/version.rb
 - redfin_as_dataframe.gemspec
 homepage: https://github.com/bmck/redfin_as_dataframe
@@ -62,7 +65,6 @@ licenses:
 metadata:
   homepage_uri: https://github.com/bmck/redfin_as_dataframe
   source_code_uri: https://github.com/bmck/redfin_as_dataframe
-  changelog_uri: https://github.com/bmck/redfin_as_dataframe/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths:
@@ -71,14 +73,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.3.0
+      version: '3.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.6
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Home purchase metrics published by Redfin