econ_data_reader 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7e3c9921d1d0a0274bb8be2485b205f80e11c2d32e0a68c59bcad3bb11f8a733
+  data.tar.gz: 663998d1614036c876ad0a48c261f952fadbf543d61b8f26d8a3ed4b4641ef6d
+SHA512:
+  metadata.gz: d11d8049c00d08a528455d05783e395055c967f28bb7e89f8951e71d1bdad32846b502c670be3e0944ffc51ee5de1f22c3537edf421e960b6f21e006a2ba033d
+  data.tar.gz: 1f477e4b035be34b3c89ec87cdfae82e3362a8e78c0a93dd1609bcd14ca88944737fef07a5a672ed5889030da66c329ca2413261bb740c806825720f3cc799dc

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+.DS_Store

data/.ruby-gemset

@@ -0,0 +1 @@
+econ_data_reader

data/.ruby-version

@@ -0,0 +1 @@
+3.1.2

data/.travis.yml

@@ -0,0 +1,6 @@
+---
+language: ruby
+cache: bundler
+rvm:
+  - 2.7.7
+before_install: gem install bundler -v 2.1.4

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Bill McKinnon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,76 @@
+# EconDataReader
+
+Up to date remote economic data access for ruby, using Polars dataframes. 
+
+This package will fetch economic and financial information from several different sources.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'econ_data_reader'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install econ_data_reader
+
+
+### Configuration
+
+Some data sources will require the specification of an API key.  These keys should be provided as part of a configuration file, e.g., config/econ_data_reader.rb
+
+```ruby
+EconDataReader::Fred.configure do |config|
+  config.fred_api_key = '1234567890ABCDEF'
+    OR
+  config.fred_api_key = File.read(File.join('','home', 'user', '.fred_api_key.txt'), 16)
+end
+
+EconDataReader::Bls.configure do |config|
+  config.bls_api_key = '8675309-1111-1111-ABCD'
+end
+
+EconDataReader::Nasdaq.configure do |config|
+  config.nasdaq_api_key = 'YOUR_API_KEY_HERE'
+end
+```    
+
+## Usage
+
+``` ruby
+EconDataReader::BankOfCanada.new('IEXE0102').fetch
+
+EconDataReader::Fred.new('GS10').fetch 
+
+EconDataReader::Bls.new('LNS14000006')
+
+EconDataReader::Nasdaq.new('WIKI/AAPL').fetch
+```
+
+## Documentation
+
+TBD
+
+## Contributing
+
+Others are welcome to contribute to the project.
+
+The following conventions are intended for this project.
+ * Different sources are intended to reside in different classes.  
+ * API keys (if needed) should be able to be set in the single configuration file.  
+ * Series should be able to be identified via a single unique string, provided in the constructor.
+ * When fetched, the dataset may be filtered based on optional (hash) arguments.
+ * Output should be provided in a consistent DataFrame format (currently Polars::DataFrame).
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/bmck/econ_data_reader.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "econ_data_reader"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/econ_data_reader.gemspec

@@ -0,0 +1,34 @@
+require_relative 'lib/econ_data_reader/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "econ_data_reader"
+  spec.version       = EconDataReader::VERSION
+  spec.authors       = ["Bill McKinnon"]
+  spec.email         = ["bill.mckinnon@bmck.org"]
+
+  spec.summary       = "Integrated economic and financial data"
+  spec.description   = "Integrated economic and financial data"
+  spec.homepage      = "https://github.com/bmck/econ_data_reader"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  # spec.metadata["allowed_push_host"] = "TODO: Set to 'http://mygemserver.com'"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/bmck/econ_data_reader"
+  spec.metadata["changelog_uri"] = "https://github.com/bmck/econ_data_reader/CHANGELOG.md"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  # Uncomment to register a new dependency of your gem
+  # spec.add_dependency "typhoeus-gem", "~> 0.6.9"
+  spec.add_dependency 'polars-df'
+  spec.add_dependency 'httparty'
+end

data/lib/econ_data_reader/bank_of_canada.rb

@@ -0,0 +1,50 @@
+require 'httparty'
+
+module EconDataReader
+  class BankOfCanada
+    # Get data for the given name from the Bank of Canada.
+
+    include ::HTTParty
+    base_uri "https://www.bankofcanada.ca/valet"
+    format :json
+
+    attr_reader :tag
+
+    def initialize(series, options={})
+      @tag = series
+    end
+
+    def fetch(start: nil, fin: nil)
+      dta = observations({}).parsed_response['observations']
+      dta = dta.select{|d| start.nil? ? true : d['d'].to_date >= start.to_date }.select{|d| fin.nil? ? true : d['d'].to_date <= fin.to_date }
+
+      dates = dta.map{|d| d['d'].to_date }
+      vals = dta.map{|d| d[tag]['v'].to_f }
+
+      Polars::DataFrame.new({Timestamps: dates, Values: vals})
+    end
+
+    def self.list_series
+      dta = (get('/lists/series/json').parsed_response)['series']
+
+      series = dta.keys
+      desc = series.map{|s| [dta[s]['label'], dta[s]['description']].join('; ') }
+
+      Polars::Config.set_tbl_rows(-1)
+      Polars::DataFrame.new({Series: series, Description: desc})
+    end
+
+    protected
+
+    def default_options
+      { }
+    end
+
+    private
+
+    def observations(options={})
+      self.class.get("/observations/#{tag}/json", :query => options.merge(self.default_options))
+    end
+
+  end
+end

data/lib/econ_data_reader/bls.rb

@@ -0,0 +1,70 @@
+require 'httparty'
+
+module EconDataReader
+  class Bls
+    # Get data for the given name from the Bank of Canada.
+
+    include ::HTTParty
+    base_uri "https://api.bls.gov/publicAPI/v2/timeseries/data"
+    format :json
+    # debug_output $stdout
+
+    attr_reader :tag
+
+    def self.configure
+      yield self
+      true
+    end
+
+    class << self
+      attr_accessor :bls_api_key
+    end
+
+    def initialize(series, options={})
+      @api_key = options[:bls_api_key] || EconDataReader::Bls.bls_api_key
+      @tag = series
+    end
+
+    def fetch(start: nil, fin: nil)
+      dta = observations({ seriesid: ["\"#{tag}\""] })
+      # Rails.logger.info { "#{__FILE__}:#{__LINE__} dta = #{dta.inspect}" }
+      dta = dta.parsed_response['Results']['series'].first['data']
+      Rails.logger.info { "#{__FILE__}:#{__LINE__} dta = #{dta.inspect}" }
+
+      dates = []
+      dta.each do |d|
+        if d['period'][0] == 'M'
+          dates << "#{d['year']}-#{d['period'][1..-1]}-01".to_date.end_of_month
+        elsif d['period'][0] == 'Q'
+          dates << "#{d['year']}-#{d['period'][1..-1]*3}-01".to_date.end_of_month
+        end
+      end
+      dates = dates.map{|d| d.to_date }
+      vals = dta.map{|d| d['value'].to_f }
+      (0..dates.length-1).to_a.reverse.each do |d|
+        if (start.present? && dates[d] <= start.to_date) || (fin.present? && dates[d] >= fin.to_date)
+          dates.delete_at(d)
+          vals.delete_at(d)
+        end
+      end
+
+      Polars::DataFrame.new({Timestamps: dates.reverse, Values: vals.reverse})
+    end
+
+    protected
+
+    def self.default_options
+      { registrationkey: bls_api_key }
+    end
+
+    delegate :default_options, to: :class
+
+    private
+
+    def observations(options={})
+      # self.class.debug_output $stdout 
+      self.class.post("https://api.bls.gov/publicAPI/v2/timeseries/data/#{tag}", body: self.default_options.merge(options).merge({ 'Content-Type' => 'application/json' }))
+    end
+
+  end
+end

data/lib/econ_data_reader/fred.rb

@@ -0,0 +1,64 @@
+require 'httparty'
+require 'csv'
+
+module EconDataReader
+  class Fred
+    # Get data for the given name from the St. Louis FED (FRED).
+    extend ActiveSupport::Concern
+
+    include ::HTTParty
+    attr_reader :tag
+
+    def self.configure
+      yield self
+      true
+    end
+
+    class << self
+      attr_accessor :fred_api_key
+    end
+
+    def initialize(series, options={})
+      @api_key = options[:fred_api_key] || EconDataReader::Fred.fred_api_key
+      @tag = series
+    end
+
+    def fetch(start: nil, fin: nil)
+      data = _read(tag)
+
+      begin
+        data = data.select{ |date, _| (start.nil? || date >= start) && (fin.nil? || date <= fin) }
+      rescue KeyError => e
+        if data.keys[3].to_s[7, 5] == "Error"
+          raise IOError, "Failed to get the data. Check that '#{nm}' is a valid FRED series."
+        else
+          raise e
+        end
+      end
+
+      Polars::DataFrame.new({Timestamps: data.keys, tag.to_sym => data.values})
+    end
+
+    private
+
+    def _url; "https://fred.stlouisfed.org/graph/fredgraph.csv"; end
+
+    def _read(symbols)
+      names = Array(symbols)
+      urls = names.map { |n| "#{_url}?id=#{n}" }
+
+      data_frames = urls.zip(names).map { |url, nm| _fetch_data(url, nm) }
+      df = data_frames.reduce({}) { |acc, df| acc.merge(df) { |_, old_val, new_val| old_val || new_val } }
+      df
+    end
+
+    def _fetch_data(url, nm)
+      # Utility to fetch data
+      resp = self.class.get(url).parsed_response.map{|a| a.join(',')}.join("\n")
+      data = CSV.parse(resp, headers: true, header_converters: :symbol, converters: [:date, :float])
+      data = data.map { |row| [Date.parse(row[:date].to_s), row[data.headers.last].to_f] }.to_h
+
+      data
+    end
+  end
+end

data/lib/econ_data_reader/nasdaq.rb

@@ -0,0 +1,67 @@
+require 'httparty'
+
+module EconDataReader
+  class Nasdaq
+    # Get data for the given name from the Bank of Canada.
+
+    include HTTParty
+    base_uri 'https://data.nasdaq.com/api/v3'
+    format :json
+    # debug_output $stdout
+
+    attr_reader :tag
+    attr_reader :api_key
+
+    def self.configure
+      yield self
+      true
+    end
+
+    class << self
+      attr_accessor :nasdaq_api_key
+    end
+
+    def initialize(series, options={})
+      @api_key = options[:nasdaq_api_key] || EconDataReader::Nasdaq.nasdaq_api_key
+      @tag = series
+    end
+
+    def fetch(start: nil, fin: nil)
+      dta = observations({  }).parsed_response
+      trans_dta = dta.transpose
+      df = Polars::DataFrame.new({})
+
+      s = trans_dta[0][1..-1].dup 
+      s.map!{|t| t.to_date }
+      df['Timestamps'] = Polars::Series.new(s)
+      
+      (1..trans_dta.length-1).to_a.each do |i|
+        s = trans_dta[i][1..-1].dup
+        s.map!{|t| t.to_f }
+        df[dta[0][i]] = Polars::Series.new(s)
+      end
+
+      Polars::Config.set_tbl_cols(-1)
+      Polars::Config.set_tbl_rows(-1)
+      df
+    end
+
+
+
+    protected
+
+    def self.default_options
+      { api_key: @api_key }
+    end
+
+    delegate :default_options, to: :class
+
+    private
+
+    def observations(options={})
+      # self.class.debug_output $stdout 
+      self.class.get("https://data.nasdaq.com/api/v3/datasets/#{tag}/data.csv", body: self.default_options.merge(options).merge({ 'Content-Type' => 'application/json' }))
+    end
+
+  end
+end

data/lib/econ_data_reader/version.rb

@@ -0,0 +1,3 @@
+module EconDataReader
+  VERSION = "0.1.3"
+end

data/lib/econ_data_reader.rb

@@ -0,0 +1,32 @@
+
+require "econ_data_reader/version"
+require 'econ_data_reader/bank_of_canada'
+require 'econ_data_reader/bls'
+require 'econ_data_reader/fred'
+require 'econ_data_reader/nasdaq'
+
+directory = File.expand_path(File.dirname(__FILE__))
+
+
+# create config/initializers/econ_data_reader.rb
+# 
+# EconDataReader::Fred.configure do |config|
+#   config.fred_api_key = '1234567890ABCDEF'
+#     OR
+#   config.fred_api_key = File.read(File.join('','home', 'user', '.fred_api_key.txt'), 16)
+# end
+
+# EconDataReader::Bls.configure do |config|
+#   config.bls_api_key = '8675309-1111-1111-ABCD'
+# end
+
+# EconDataReader::Nasdaq.configure do |config|
+#   config.nasdaq_api_key = 'YOUR_API_KEY_HERE'
+# end
+
+
+module EconDataReader
+  class Error < StandardError; end
+  # Your code goes here...
+  
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: econ_data_reader
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- Bill McKinnon
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-07-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: polars-df
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Integrated economic and financial data
+email:
+- bill.mckinnon@bmck.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- ".travis.yml"
+- CHANGELOG.rst
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- econ_data_reader.gemspec
+- lib/econ_data_reader.rb
+- lib/econ_data_reader/bank_of_canada.rb
+- lib/econ_data_reader/bls.rb
+- lib/econ_data_reader/fred.rb
+- lib/econ_data_reader/nasdaq.rb
+- lib/econ_data_reader/version.rb
+homepage: https://github.com/bmck/econ_data_reader
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/bmck/econ_data_reader
+  source_code_uri: https://github.com/bmck/econ_data_reader
+  changelog_uri: https://github.com/bmck/econ_data_reader/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Integrated economic and financial data
+test_files: []