cotcube-cftcsource 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 75a8631a91ea90a9fdf4505c9265508cfac8b06d385ab1b41f2e5683876e8617
+  data.tar.gz: db8caed963d6dcd7dc13d29faac945965bf1b55dfbeb36ce51a89ace235c149a
+SHA512:
+  metadata.gz: 38aa716749ec46151657f0e503019ba05de75b3f26e461a7df850ba3b3580f7ece53b00ed9bc9fde3f667200dd74972060fdf7f71f8a1d13403b6238943fa060
+  data.tar.gz: d53d40b0b11e515b8cc405e915aca658e1fbafeb5a8b9cd6dc495bfb338bcd76daff3934a9c1e4674e3b147c28e74652cc733dd11914984a8534862e4a361d75

data/.irbrc.rb

@@ -0,0 +1,12 @@
+def verbose_toggle
+  irb_context.echo ? irb_context.echo = false : irb_context.echo = true
+end
+
+alias vt verbose_toggle
+
+$debug = true
+IRB.conf[:USE_MULTILINE] = false
+# require 'bundler'
+# Bundler.require
+
+require_relative 'lib/cotcube-cftcsource'

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.1 (December 12, 2020)
+
+

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in bitangent.gemspec
+gemspec
+

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2020 Benjamin L. Tischendorf
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,86 @@
+# cotcube-cftcsource
+
+This gem provides a module containing a set of methods to download, maintain, provide and 
+analyze the COT data issued by the CFTC. It is part of the Cotcube suite. 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'cotcube-cftcsource'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install cotcube-cftcsource
+
+## Usage
+
+### Configuration
+
+The gem expects a configfile 'cftcsource.yml', located in '/etc/cotcube/' on linux platform and '/usr/local/etc/cotcube/' on FreeBSD. The location of the configfile can be overwritten by passing the according parameter to `init`.
+
+Following parameters are currently supported (please let me know if something is missing): 
+
+```yaml
+data_path: 'path where the raw downloaded data is saved and other data is cached'
+symbols_file: 'A file containing a CSV with supported symbols--if ommited, an example set from constants.rb is used'
+headers_path: 'Where the CSV headers for cot data and signal sets are cached.'
+default_lookback: 'the amount of weeks to be used for indicators in signal processing'
+default_set: 'the name of the indicator set to be used (as saved in etc/cotcube/indicators-<default_set>.rb)
+```
+
+### constants.rb
+
+Basically sets up 3 constants:
+
+1. CFTC\_LINKS contains all URLs to download all zipped COT reports from cftc.gov
+2. CFTC\_HEADERS provides lists of symbols serving as headers for the CSV-saved COT data.
+3. CFTC\_SYMBOL\_EXAMPLES contains 2 sets of configuration data. Each set has the following structure:
+  * id: The CFTC id of the future.
+  * symbol: Some verbal, unambiguous short identifier for this future.
+  * ticksize:
+  * power: The leverage per tick.
+  * months: The set of trading months.
+  * bcf: A legacy constant, it is '1.0' in almost most cases.
+  * reports: uppercase letters, signaling which reports are available for the futures (Legacy, Financial, Disaggregated and Cit)
+  * name: 
+
+### init.rb
+### fetch and distribute
+
+`CftcSource.fetch` downloads and unzips all COT-packages from CFTC.gov matching the :year parameter, defaulting to the current year.
+Please see https://cftc.gov/MarketReports/CommitmentsofTraders/HistoricalCompressed/index.htm for which year parameters are available. 
+
+`CftcSource.distribute` breaks down the packages into the separated ID based directory tree. 
+
+NOTE: When building from scratch you should fetch and distribute all packages in proper annual order, as each run just updates data after the last available record.
+
+### provide.rb
+
+Returns the requested series of raw COT data for given either :symbol or :id. The parameter :subset currently filters for 'std\_count', a further implementation is pending. If :subset is set to false, the complete series is returned. 
+
+### series.rb
+
+Based on the data returned by CftcSource.provide and the given indicator set, a series is created, cached and returned. 
+
+The default indicator simply calculates the net positions for pairs of `_long` and `_short`. Other indicator sets can be created. A simple example is in `etc/cotcube/indicators-example.rb`.  
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/donkeybridge/bitangent.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/bin/console

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require __dir__ + '/../lib/cotcube-cftcsource.rb'
+
+require 'irb'
+require 'irb/completion'
+require 'irb/ext/save-history'
+
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+$debug = true
+
+IRB.conf[:SAVE_HISTORY] = 1500
+IRB.conf[:HISTORY_FILE] = '~/.irb-history'
+IRB.conf[:ECHO] = false
+
+
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/signals.rb

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/cotcube-cftcsource.rb'
+
+s = Cotcube::CftcSource.signals symbol:  ARGV[0], report: ARGV[1].downcase.to_sym, human: false, count: 800, quiet: true
+s.each {|x| puts x} 
+

data/cotcube-cftcsource.gemspec

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.name          = 'cotcube-cftcsource'
+  spec.version       = File.read("#{__dir__}/VERSION")
+  spec.authors       = ['Benjamin L. Tischendorf']
+  spec.email         = ['donkeybridge@jtown.eu']
+
+  spec.summary       = 'Functions to download and process COT data, aggregated into a module'
+  spec.description   = 'Functions to download and process COT data, aggregated into a module'
+
+  spec.homepage      = 'https://github.com/donkeybridge/'+ spec.name
+  spec.license       = 'BSD-4-Clause'
+  spec.required_ruby_version = Gem::Requirement.new('~> 2.7')
+
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = spec.homepage
+  spec.metadata['changelog_uri'] = spec.homepage + '/CHANGELOG.md'
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = 'bin'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_dependency 'cotcube-indicators'
+  spec.add_dependency 'yaml'
+  spec.add_dependency 'activesupport'
+  spec.add_dependency 'colorize'
+  spec.add_dependency 'httparty'
+  spec.add_dependency 'rubyzip'
+
+
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec', '~>3.6'
+  spec.add_development_dependency 'yard', '~>0.9'
+end

data/lib/cotcube-cftcsource.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# require 'bundler/setup'
+# require 'rubygems/package'
+require 'active_support'
+require '../ci-dev/lib/cotcube-indicators'
+require 'colorize'
+require 'date' unless defined?(DateTime)
+require 'csv'  unless defined?(CSV)
+require 'yaml' unless defined?(YAML)
+require 'httparty'
+require 'zip'
+
+
+require_relative 'cotcube-cftcsource/init'
+require_relative 'cotcube-cftcsource/constants'
+require_relative 'cotcube-cftcsource/dates'
+require_relative 'cotcube-cftcsource/fetch'
+require_relative 'cotcube-cftcsource/distribute'
+require_relative 'cotcube-cftcsource/provide'
+require_relative 'cotcube-cftcsource/series'
+
+private_files = Dir[__dir__ + '/cotcube-cftcsource/private/*.rb']
+private_files.each do |file| 
+  # puts 'Loading private module extension ' + file.chomp
+  require file.chomp
+end
+
+module Cotcube
+  module CftcSource
+    module_function :config_path, # provides the path of configuration directory
+      :config_prefix,             # provides the prefix of the configuration directory according to OS-specific FSH
+      :init,                      # checks whether environment is prepared and returns the config hash
+      :symbols,                   # reads and provides the symbols file
+      :date_of_last_report,       # returns the date of the currently last locally available report
+      :current_cot_date_on_website, # returns the date of the current report officially available on cftc.gov
+      :fetch,                     # downloads all (annual) reports, based on given year
+      :distribute,                # distributes currently downloaded ('fetch'ed) reports to id_based directories
+      :available_cftc_ids,        # returns a list of locally available cftc ids 
+      :provide,                   # returns a series based on id (or symbol) and report / combined
+      :series                     # returns a series based on provide, after application of indicator set
+
+    # please not that module_functions of source provided in private files must be published there
+  end
+end

data/lib/cotcube-cftcsource/constants.rb

@@ -0,0 +1,147 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+
+    CFTC_LINKS = {
+      disagg: {
+        fut: {
+          current: 'https://www.cftc.gov/dea/newcot/f_disagg.txt',
+          hist: 'https://www.cftc.gov/files/dea/history/fut_disagg_txt_'
+        },
+        com: {
+          current: 'https://www.cftc.gov/dea/newcot/c_disagg.txt',
+          hist: 'https://www.cftc.gov/files/dea/history/com_disagg_txt_'
+        }
+      },
+      legacy: {
+        fut: {
+          current: 'https://www.cftc.gov/dea/newcot/deafut.txt',
+          hist: 'https://www.cftc.gov/files/dea/history/deacot'
+        },
+        com: {
+          current: 'https://www.cftc.gov/dea/newcot/deacom.txt',
+          hist: 'https://www.cftc.gov/files/dea/history/deahistfo'
+        }
+      },
+      financial: {
+        fut: {
+          current: 'https://www.cftc.gov/dea/newcot/FinFutWk.txt',
+          hist: 'https://www.cftc.gov/files/dea/history/fut_fin_txt_'
+        },
+        com: {
+          current: 'https://www.cftc.gov/dea/newcot/FinComWk.txt',
+          hist: 'https://www.cftc.gov/files/dea/history/com_fin_txt_'
+        }
+      },
+      cit: {
+        com: {
+          current: 'https://www.cftc.gov/dea/newcot/deacit.txt',
+          hist: 'https://www.cftc.gov/files/dea/history/dea_cit_txt_'
+        }
+      }
+    }.freeze
+
+    CFTC_HEADERS = { 
+      legacy: %i[ name date date2 cftcid cftcid2 cftcid3 cftcid3] +
+      %i[ std_count_oi_all std_count_ncom_long std_count_ncom_short std_count_ncom_spread std_count_com_long std_count_com_short] + 
+      %i[ std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short] +
+      %i[ old_count_oi_all old_count_ncom_long old_count_ncom_short old_count_ncom_spread old_count_com_long old_count_com_short] +
+      %i[ old_count_rept_long old_count_rept_short old_count_nrept_long old_count_nrept_short] +
+      %i[ other_count_oi_all other_count_ncom_long other_count_ncom_short other_count_ncom_spread other_count_com_long other_count_com_short] +
+      %i[ other_count_rept_long other_count_rept_short other_count_nrept_long other_count_nrept_short] +
+      %i[ std_change_oi_all std_change_ncom_long std_change_ncom_short std_change_ncom_spread std_change_com_long std_change_com_short] +
+      %i[ std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short] +
+      %i[ std_pct_oi_all std_pct_ncom_long std_pct_ncom_short std_pct_ncom_spread std_pct_com_long std_pct_com_short] +
+      %i[ std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short] +
+      %i[ old_pct_oi_all old_pct_ncom_long old_pct_ncom_short old_pct_ncom_spread old_pct_com_long old_pct_com_short] +
+      %i[ old_pct_rept_long old_pct_rept_short old_pct_nrept_long old_pct_nrept_short] +
+      %i[ other_pct_oi_all other_pct_ncom_long other_pct_ncom_short other_pct_ncom_spread other_pct_com_long other_pct_com_short] +
+      %i[ other_pct_rept_long other_pct_rept_short other_pct_nrept_long other_pct_nrept_short] +
+      %i[ std_traders_oi_all std_traders_ncom_long std_traders_ncom_short std_traders_ncom_spread std_traders_com_long std_traders_com_short] +
+      %i[ std_traders_rept_long std_traders_rept_short] + 
+      %i[ old_traders_oi_all old_traders_ncom_long old_traders_ncom_short old_traders_ncom_spread old_traders_com_long old_traders_com_short] +
+      %i[ old_traders_rept_long old_traders_rept_short] +
+      %i[ other_traders_oi_all other_traders_ncom_long other_traders_ncom_short other_traders_ncom_spread other_traders_com_long] +
+      %i[ other_traders_com_short other_traders_rept_long other_traders_rept_short] +
+      %i[ std_conc_gross4_long std_conc_gross4_short std_conc_gross8_long std_conc_gross8_short std_conc_net4_long std_conc_net4_short] +
+      %i[ std_conc_net8_long std_conc_net8_short old_conc_gross4_long old_conc_gross4_short old_conc_gross8_long old_conc_gross8_short] +
+      %i[ old_conc_net4_long old_conc_net4_short old_conc_net8_long old_conc_net8_short other_conc_gross4_long other_conc_gross4_short] +
+      %i[ other_conc_gross8_long other_conc_gross8_short other_conc_net4_long other_conc_net4_short other_conc_net8_long other_conc_net8_short] +
+      %i[ units CFTCContractMarketCode(Quotes) CFTCMarketCodeinInitials(Quotes) CFTCCommodityCode(Quotes) supplement],
+
+      disagg: %i[name date date2 cftcid cftcid2 cftcid3 cftcid3] +
+      %i[ std_count_oi_all std_count_prod_long std_count_prod_short std_count_swap_long std_count_swap_short std_count_swap_spread] + 
+      %i[ std_count_money_long std_count_money_short std_count_money_spread std_count_other_long std_count_other_short std_count_other_spread] + 
+      %i[ std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short] +
+      %i[ old_count_oi_all old_count_prod_long old_count_prod_short old_count_swap_long old_count_swap_short old_count_swap_spread] +
+      %i[ old_count_money_long old_count_money_short old_count_money_spread old_count_other_long old_count_other_short old_count_other_spread] +
+      %i[ old_count_rept_long old_count_rept_short old_count_nrept_long old_count_nrept_short] +
+      %i[ other_count_oi_all other_count_prod_long other_count_prod_short other_count_swap_long other_count_swap_short other_count_swap_spread] +
+      %i[ other_count_money_long other_count_money_short other_count_money_spread other_count_other_long other_count_other_short other_count_other_spread] +
+      %i[ other_count_rept_long other_count_rept_short other_count_nrept_long other_count_nrept_short] +
+      %i[ std_change_oi_all std_change_prod_long std_change_prod_short std_change_swap_long std_change_swap_short std_change_swap_spread] +
+      %i[ std_change_money_long std_change_money_short std_change_money_spread std_change_other_long std_change_other_short std_change_other_spread] +
+      %i[ std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short] +
+      %i[ std_pct_oi_all std_pct_prod_long std_pct_prod_short std_pct_swap_long std_pct_swap_short std_pct_swap_spread] +
+      %i[ std_pct_money_long std_pct_money_short std_pct_money_spread std_pct_other_long std_pct_other_short std_pct_other_spread] +
+      %i[ std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short] +
+      %i[ old_pct_oi_all old_pct_prod_long old_pct_prod_short old_pct_swap_long old_pct_swap_short old_pct_swap_spread old_pct_money_long] +
+      %i[ old_pct_money_short old_pct_money_spread old_pct_other_long old_pct_other_short old_pct_other_spread] +
+      %i[ old_pct_rept_long old_pct_rept_short old_pct_nrept_long old_pct_nrept_short] + 
+      %i[ other_pct_oi_all other_pct_prod_long other_pct_prod_short other_pct_swap_long other_pct_swap_short other_pct_swap_spread] +
+      %i[ other_pct_money_long other_pct_money_short other_pct_money_spread other_pct_other_long other_pct_other_short other_pct_other_spread] +
+      %i[ other_pct_rept_long other_pct_rept_short other_pct_nrept_long other_pct_nrept_short] + 
+      %i[ std_traders_oi_all std_traders_prod_long std_traders_prod_short std_traders_swap_long std_traders_swap_short std_traders_swap_spread] +
+      %i[ std_traders_money_long std_traders_money_short std_traders_money_spread std_traders_other_long std_traders_other_short std_traders_other_spread] +
+      %i[ std_traders_rept_long std_traders_rept_short] +
+      %i[ old_traders_oi_all old_traders_prod_long old_traders_prod_short old_traders_swap_long old_traders_swap_short old_traders_swap_spread] +
+      %i[ old_traders_money_long old_traders_money_short old_traders_money_spread old_traders_other_long old_traders_other_short old_traders_other_spread] +
+      %i[ old_traders_rept_long old_traders_rept_short] +
+      %i[ other_traders_oi_all other_traders_prod_long other_traders_prod_short other_traders_swap_long other_traders_swap_short] +
+      %i[ other_traders_swap_spread other_traders_money_long other_traders_money_short other_traders_money_spread other_traders_other_long] +
+      %i[ other_traders_other_short other_traders_other_spread other_traders_rept_long other_traders_rept_short] + 
+      %i[ std_conc_gross4_long std_conc_gross4_short std_conc_gross8_long std_conc_gross8_short std_conc_net4_long std_conc_net4_short std_conc_net8_long] +
+      %i[ std_conc_net8_short old_conc_gross4_long old_conc_gross4_short old_conc_gross8_long old_conc_gross8_short old_conc_net4_long old_conc_net4_short] +
+      %i[ old_conc_net8_long old_conc_net8_short other_conc_gross4_long other_conc_gross4_short other_conc_gross8_long other_conc_gross8_short] +
+      %i[ other_conc_net4_long other_conc_net4_short other_conc_net8_long other_conc_net8_short] + 
+      %i[ units CFTC_Contract_Market_Code_Quotes CFTC_Market_Code_Quotes CFTC_Commodity_Code_Quotes CFTC_SubGroup_Code Fut_Combined supplement],
+
+      financial: %i[ name date date2 cftcid cftcid2 cftcid3 cftcid3] +
+      %i[ std_count_oi_all std_count_dealers_long std_count_dealers_short std_count_dealers_spread std_count_asset_long std_count_asset_short] +
+      %i[ std_count_asset_spread std_count_lev_long std_count_lev_short std_count_lev_spread std_count_other_long std_count_other_short] + 
+      %i[ std_count_other_spread std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short] +
+      %i[ std_change_oi_all std_change_dealers_long std_change_dealers_short std_change_dealers_spread std_change_asset_long std_change_asset_short] +
+      %i[ std_change_asset_spread std_change_lev_long std_change_lev_short std_change_lev_spread std_change_other_long std_change_other_short] +
+      %i[ std_change_other_spread std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short] +
+      %i[ std_pct_oi_all std_pct_dealers_long std_pct_dealers_short std_pct_dealers_spread std_pct_asset_long std_pct_asset_short] +
+      %i[ std_pct_asset_spread std_pct_lev_long std_pct_lev_short std_pct_lev_spread std_pct_other_long std_pct_other_short] +
+      %i[ std_pct_other_spread std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short] +
+      %i[ std_traders_oi_all std_traders_dealers_long std_traders_dealers_short std_traders_dealers_spread std_traders_asset_long std_traders_asset_short] +
+      %i[ std_traders_asset_spread std_traders_lev_long std_traders_lev_short std_traders_lev_spread std_traders_other_long std_traders_other_short std] +
+      %i[_traders_other_spread std_traders_rept_long std_traders_rept_short] +
+      %i[ std_conc_gross4_long std_conc_gross4_short std_conc_gross8_long std_conc_gross8_short std_conc_net4_long std_conc_net4_short] +
+      %i[ std_conc_net8_long std_conc_net8_short] +
+      %i[ units CFTC_Contract_Market_Code_Quotes CFTC_Market_Code_Quotes CFTC_Commodity_Code_Quotes CFTC_SubGroup_Code type supplement],
+
+
+      cit: %i[name date date2 cftcid cftcid2 cftcid3 cftcid3] + 
+      %i[std_count_oi_all std_count_ncom_long std_count_ncom_short std_count_ncom_spread std_count_com_long std_count_com_short] + 
+      %i[std_count_rept_long std_count_rept_short std_count_nrept_long std_count_nrept_short std_count_cit_long std_count_cit_short] + 
+      %i[std_change_oi_all std_change_ncom_long std_change_ncom_short std_change_ncom_spread std_change_com_long std_change_com_short] +
+      %i[std_change_rept_long std_change_rept_short std_change_nrept_long std_change_nrept_short std_change_cit_long std_change_cit_short] +
+      %i[std_pct_oi_all std_pct_ncom_long std_pct_ncom_short std_pct_ncom_spread std_pct_com_long std_pct_com_short] +
+      %i[std_pct_rept_long std_pct_rept_short std_pct_nrept_long std_pct_nrept_short std_pct_cit_long std_pct_cit_short] +
+      %i[std_traders_oi_all std_traders_ncom_long std_traders_ncom_short std_traders_ncom_spread std_traders_com_long std_traders_com_short] +
+      %i[std_traders_rept_long std_traders_rept_short std_traders_cit_long std_traders_cit_short] +
+      %i[units supplement]
+    }.freeze
+
+    CFTC_SYMBOL_EXAMPLES = [
+      { id: "13874U", symbol: "ET", ticksize: 0.25, power: 1.25, months: "HMUZ", bcf: 1.0, reports: "LF", name: "S&P 500 MICRO" },
+      { id: "209747", symbol: "NM", ticksize: 0.25, power: 0.5,  monhts: "HMUZ", bcf: 1.0, reports: "LF", name: "NASDAQ 100 MICRO" }
+    ].freeze
+
+
+  end
+end

data/lib/cotcube-cftcsource/dates.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+
+    # method to parse current years raw CFTC cot_files and return the latest available date
+    #
+    # @return [DateTime] DateTime object containing the date found
+    def date_of_last_report(config: init_config)
+      files = Dir["#{config[:data_path]}/raw/*#{Time.now.strftime('%Y')}.csv"]
+      DateTime.strptime(CSV.read(files[0]).tap { |x| x.shift if x[0][0] =~ /Market/ }.map { |x| x[2] }.max, '%Y-%m-%d') rescue nil
+    end
+
+    # method to extract the currently available most recent COT report from the according CFTC website
+    #
+    # @return [DateTime] a DateTime object containing the extracted date
+    def current_cot_date_on_website
+      uri = 'https://www.cftc.gov/MarketReports/CommitmentsofTraders/index.htm'
+      raw = HTTParty.get(uri).to_s
+      encoding_options = {
+        :invalid           => :replace,  # Replace invalid byte sequences
+        :undef             => :replace,  # Replace anything not defined in ASCII
+        :replace           => '',        # Use a blank for those replacements
+        :universal_newline => true       # Always break lines with \n
+      }
+      line = raw.each_line.select{|x| x if x =~ /Reports Dated/ }[0].encode(Encoding.find('ASCII'), encoding_options)
+      date_string = line.split('Dated ').last.split('- Current').first
+      DateTime.strptime(date_string, '%B %d,%Y')
+    end
+  end
+end

data/lib/cotcube-cftcsource/decommission.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+    # move expired
+    
+  def move_expired_reports
+    Dir['/var/cotcube/cftc/cot/*'].each do |dir|
+      target = "/var/cotcube/cftc/expired/#{dir.split('/').last}/"
+      Dir["#{dir}/*.csv"].each do |file|
+        next unless CSV.read(file, converters: :all).last[2] < Date.today - 30
+
+        xdebug "Moving #{file} to #{target}"
+        `mkdir -p #{target}` unless File.exist? target
+        `mv #{file} #{target}`
+      end
+      if Dir["#{dir}/*"].empty?
+        xdebug "Removing #{dir}"
+        `rm -rf #{dir}`
+      end
+    end
+  end
+
+  end
+end

data/lib/cotcube-cftcsource/distribute.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+
+    # Goes through CFTCLINKS to iterate over all types of reports
+    #
+    # @param [Hash] opts
+    # @option [Integer, String] Year to be processed, defaults to current year
+
+    def distribute(year: Time.now.year, debug: false, config: init)
+      raw_path = "#{config[:data_path]}/raw"
+      cot_path = "#{config[:data_path]}/cot"
+      CFTC_LINKS.each do |report, a|
+	a.each do |combined, _b|
+	  puts ("Processing #{report}\t#{combined}\t#{year}") if debug
+
+	  infile  = "#{raw_path}/#{report}_#{combined}_#{year}.csv"
+	  outfile = ->(symbol) { "#{cot_path}/#{symbol}/#{report}_#{combined}.csv" }
+
+	  last_report = lambda do |symbol|
+	    CSV.read("#{cot_path}/#{symbol}/#{report}_#{combined}.csv").last[1]
+	  rescue StandardError
+	    '1900-01-01'
+	  end
+
+	  cache = {}
+	  csv_data = CSV.read(infile).tap { |lines| lines.shift if lines[0][0] =~ /Market/ }
+	  csv_data.sort_by { |x| x[2] }.each do |line|
+	    next if line[3].length != 6
+
+	    sym = line[3]
+	    cache[sym] ||= last_report.call(sym)
+	    next if cache[sym] >= line[2]
+
+	    line << "#{report} #{combined}"
+	    line.map! { |x| x&.strip }
+	    begin
+	      CSV.open(outfile.call(sym), 'a+') { |f| f << line }
+	    rescue StandardError
+	      puts ("Found new id: #{sym}").colorize(:light_green)
+	      `mkdir #{cot_path}/#{sym}`
+	      CSV.open(outfile.call(sym), 'a+') { |f| f << line }
+	    end
+	  end
+	end
+      end
+    end
+
+    # print list 
+
+    def available_cftc_ids(config: init, print: true, sort: :exchange)
+      result = CSV.parse(`find #{init[:data_path]}/cot/* | grep legacy | xargs head -n1  | grep , | awk -v FS=, -v OFS=, '{print $5,$4,$1}' | sed 's/"//g' | sort | uniq`.chomp, headers: %i{ exchange id name }).map{|x| x.to_h} 
+      result = result.uniq{|row| row[:id] }
+      if print
+        result.sort_by{|x| x[sort]}.each {|row| puts "#{row[:exchange]}\t#{row[:id]}\t#{row[:name]}" }
+      end
+      result
+    end
+
+  end
+end

data/lib/cotcube-cftcsource/fetch.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+
+
+    # method to download all available historical cot report files
+    #
+    # @param [Integer, String] year
+    # @param [Boolean]         debug
+    # @param [Hash]            config
+    def fetch(year: Time.now.year, debug: false, silent: false, config: init)
+
+      CFTC_LINKS.each do |report, a|
+        a.each do |combined, _b|
+          puts ("====> working on #{report}\t#{combined}") if debug
+          raw_dir   = "#{config[:data_path]}/raw"
+
+          uri = "#{CFTC_LINKS[report][combined][:hist]}#{year}.zip"
+          file_data = nil
+          input = HTTParty.get(uri).body
+          Zip::InputStream.open(StringIO.new(input)) do |io|
+            while entry = io.get_next_entry
+              file_data = io.read
+            end
+          end
+          file = "#{raw_dir}/#{report}_#{combined}_#{year}.csv"
+          File.write(file, file_data)
+
+          puts "Contents have been written to '#{file}'." if debug
+          print "Check whether current date for #{report}_#{combined}_#{year} makes sense: " unless silent
+          puts `head -n2 #{file} | tail -n1 | cut -d, -f1,3`.chomp.light_white unless silent
+        end
+      end
+    end
+  end
+end

data/lib/cotcube-cftcsource/init.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+
+
+    def symbols(config: init)
+      if config[:symbols_file].nil?
+        CFTC_SYMBOL_EXAMPLES
+      else
+        CSV
+          .read(config[:symbols_file], headers: %i{ id symbol ticksize power months type bcf reports name})
+          .map{|row| row.to_h }
+          .map{|row| [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}; row }
+          .reject{|row| row[:id].nil? }
+      end
+    end
+
+    def config_prefix
+      os       = Gem::Platform.local.os
+      case os
+      when 'linux'
+        ''
+      when 'freebsd'
+        '/usr/local'
+      else
+        raise RuntimeError, 'unknown architecture'
+      end
+    end
+
+    def config_path
+      config_prefix + '/etc/cotcube'
+    end
+
+    def init(config_file_name: 'cftcsource.yml', debug: false)
+      name = 'cftcsource'
+      config_file = config_path + "/#{config_file_name}"
+
+      if File.exist?(config_file)
+        config      = YAML.load(File.read config_file).transform_keys(&:to_sym)
+      else
+        config      = {} 
+      end
+
+      defaults = { 
+        data_path: config_prefix + '/var/cotcube/' + name,
+      }
+
+
+      config = defaults.merge(config)
+
+
+      # part 2 of init process: Prepare directories
+
+      save_create_directory = lambda do |directory_name|
+        unless Dir.exist?(directory_name)
+          begin
+            `mkdir -p #{directory_name}`
+            unless $?.exitstatus.zero?
+              puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
+              exit 1 unless defined?(IRB)
+            end
+          rescue 
+            puts "Missing permissions to create or access '#{directory_name}', please clarify manually"
+            exit 1 unless defined?(IRB)
+          end
+        end
+      end
+      ['','raw','cot','series'].each do |path| 
+        dir = "#{config[:data_path]}#{path == '' ? '' : '/'}#{path}"
+        save_create_directory.call(dir)
+      end
+
+      # eventually return config
+      config
+    end
+
+  end
+end
+

data/lib/cotcube-cftcsource/provide.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+    def provide(symbol: nil,         # The barchart symbol of the contract
+                id: nil,             # The CFTC ID of the contract
+                report: :legacy,       # report type (legacy, disagg, financial, cit)
+                combined: :com,      # combined of fut_only
+                after: '1900-01-01', # start at given date
+                subset: true,        # set to false to return full data
+                measure: false, 
+                config: init,        # 
+                debug: false)        #
+
+      after = DateTime.strptime(after, '%Y-%m-%d') if after.is_a? String
+
+      unless symbol.nil? 
+        symbol_id = symbols.select{|s| s[:symbol] == symbol.to_s.upcase}.first[:id]
+        raise ArgumentError, "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}" if symbol_id.nil? 
+        raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if not id.nil? and symbol_id != id
+        id = symbol_id
+      end
+      raise ArgumentError, "Need either :symbol or :id as parameter" if id.nil? 
+
+      id_path    = "#{config[:data_path]}/cot/#{id}"
+      data_file  = "#{id_path}/#{report}_#{combined}.csv"
+
+      raise RuntimeError, "No data found for requested :id (#{id_path} does not exist)" unless Dir.exist?(id_path)
+      raise RuntimeError, "No data found for requested :report (#{report} #{combined} in #{id_path}" unless File.exist?(data_file)
+
+      puts "#{Time.now - measure}: PROVIDE: before CSV parse" if measure
+      raw        = `cat #{data_file} | sed -e 's/\r//'`
+      data       = CSV.parse(raw, row_sep: :auto, converters: :all, headers: CFTC_HEADERS[report]).map do |row|
+        d = row.to_h
+        d[:date] = d[:date2].strftime('%Y-%m-%d')
+        d
+      end
+      puts "#{Time.now - measure}: PROVIDE: after CSV parse" if measure
+      data.select! { |x| x[:date2] > after } if after
+      data.sort!   { |x| x[:date2] }.uniq!{ |x| x[:date2] }
+      data.reverse!
+
+      if subset 
+        data.map do |dataset|
+          dataset.map do |key,value|
+            dataset.delete(key) if       key.to_s =~ (/_traders_/)  || 
+              key.to_s =~ (/_pct_/)   || key.to_s =~ (/_conc_/)     || 
+              key.to_s =~ (/^other_/) || key.to_s =~ (/^old_/)      || key.to_s =~ (/_change_/) ||
+              [ :units, :"CFTCContractMarketCode(Quotes)", :"CFTCMarketCodeinInitials(Quotes)",
+                :"CFTCCommodityCode(Quotes)", :name,  :cftcid3, :date2 ].include?(key)
+          end
+        end
+      end
+      puts "#{Time.now - measure}: PROVIDE: after subset application, returning " if measure
+
+      data
+    end
+
+  end
+end

data/lib/cotcube-cftcsource/series.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Cotcube
+  module CftcSource
+
+    #   - there will be configfile, containing the indicators
+    # a call hence
+    #   1. will try to load from the series directory
+    #   2. will try to update the series already loaded (or starting from empty)
+    #   3. provides the series (containing only the lookback period requested)\q
+    #
+    def series(symbol: nil, 
+               report: :legacy, 
+               combined: :com,
+               config: init,
+               after: '1900-01-01',
+               lookback: nil,
+               set: nil,
+               force: false,
+               measure: false,
+               debug: false
+              )
+      raise ArgumentError, "Can only build series with given symbol" if symbol.nil?
+      symbol_config = symbols.select{|s| s[:symbol] == symbol.to_s.upcase }.first
+      raise ArgumentError, "Can only build series with known symbol, '#{symbol}' is unknown." if symbol_config.nil?
+      puts "Using symbol_config '#{symbol_config}'" if debug
+
+      lookback = config[:default_lookback] if lookback.nil? 
+      raise ArgumentError, "Lookback may not be nil, either provide option or set :default_lookback in configfile" if lookback.nil?
+
+      set      = config[:default_set]      if set.nil?
+      set      = 'default'                 if set.nil?              
+
+      series_dir  = "#{config[:data_path]}/series/#{symbol}"
+      series_file = "#{series_dir}/#{report}_#{combined}_#{set}_#{lookback}.csv"
+      puts "Guessing series_file '#{series_file}'" if debug
+
+      `mkdir -p #{series_dir}` unless Dir.exist?(series_dir)
+
+      header_keys = CFTC_HEADERS[report]
+        .select   { |key| key.to_s =~ /std_count/ }
+        .group_by { |key| key.to_s.split('_')[2] }
+        .keys
+        .tap      { |k|   k.delete('oi') }
+
+      puts "Using header_keys '#{header_keys}'" if debug
+
+
+      if set == 'default' or set.nil?
+	# load default indicatorset
+	indicators = {}
+	prefix     = "std_count"
+	header_keys.map do |key|
+	  indicators["#{prefix}_#{key}_net"] = Cotcube::Indicators.calc(a: "#{prefix}_#{key}_long", b: "#{prefix}_#{key}_short"){|a,b| a - b }
+	end
+      else
+	# load custom indicatorset
+	indicators_file = "#{config_path}/indicators-#{set}.rb"
+	raise ArgumentError, "Indicatorsfile #{indicators_file} does not exist" unless File.exist? indicators_file
+	load indicators_file
+	indicators = Cotcube::Indicators.build(lookback: lookback, keys: header_keys)
+      end
+
+      if File.exist?(series_file) and (Time.now - File.stat(series_file).mtime).to_i / 86400.0 < 5.0 and not force and not config[:always_force]
+	headers_file = "#{config[:headers_path]}/#{report}_#{combined}.json"
+	if File.exist? headers_file
+	  provide_headers = JSON.parse(File.read(headers_file)).map{|x| x.to_sym}
+	else
+	  provide_headers = (Cotcube::CftcSource.provide symbol: symbol, report: report, combined: combined, after: (Time.now - 20 * 86000).strftime("%Y-%m-%d")).first.keys
+	  File.open(headers_file, 'w'){|f| f << provide_headers.to_json }
+	end
+	headers         = provide_headers + indicators.keys
+	puts 'Returning saved data' if debug
+	puts "#{Time.now - measure}: SERIES: returning series from file" if measure
+
+	return CSV.read(series_file, headers: headers, converters: :all).map(&:to_h)
+      end
+
+      puts "#{Time.now - measure}: SERIES: before source" if measure
+
+      source = provide(symbol: symbol, report: report, combined: combined, config: config, after: after, measure: measure)
+
+      puts "#{Time.now - measure}: SERIES: after source" if measure
+
+      puts "First source: #{source.first}" if debug
+      puts "#{config}" if debug
+
+      source.each do |dataset| 
+	indicators.each do |key, lambada|
+	  dataset[key] = lambada.call(dataset)
+	end
+      end
+      CSV.open(series_file, 'w') { |csv| source.map{|x| csv << x.values } }
+
+      puts "#{Time.now - measure}: SERIES: returning series from source" if measure
+      source
+
+    end
+  end
+end

metadata

@@ -0,0 +1,193 @@
+--- !ruby/object:Gem::Specification
+name: cotcube-cftcsource
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Benjamin L. Tischendorf
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-12-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cotcube-indicators
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yaml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.6'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+description: Functions to download and process COT data, aggregated into a module
+email:
+- donkeybridge@jtown.eu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".irbrc.rb"
+- ".rspec"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- bin/console
+- bin/setup
+- bin/signals.rb
+- cotcube-cftcsource.gemspec
+- lib/cotcube-cftcsource.rb
+- lib/cotcube-cftcsource/constants.rb
+- lib/cotcube-cftcsource/dates.rb
+- lib/cotcube-cftcsource/decommission.rb
+- lib/cotcube-cftcsource/distribute.rb
+- lib/cotcube-cftcsource/fetch.rb
+- lib/cotcube-cftcsource/init.rb
+- lib/cotcube-cftcsource/provide.rb
+- lib/cotcube-cftcsource/series.rb
+homepage: https://github.com/donkeybridge/cotcube-cftcsource
+licenses:
+- BSD-4-Clause
+metadata:
+  homepage_uri: https://github.com/donkeybridge/cotcube-cftcsource
+  source_code_uri: https://github.com/donkeybridge/cotcube-cftcsource
+  changelog_uri: https://github.com/donkeybridge/cotcube-cftcsource/CHANGELOG.md
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.7'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Functions to download and process COT data, aggregated into a module
+test_files: []