cotcube-cftcsource 0.1.1 → 0.1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75a8631a91ea90a9fdf4505c9265508cfac8b06d385ab1b41f2e5683876e8617
-  data.tar.gz: db8caed963d6dcd7dc13d29faac945965bf1b55dfbeb36ce51a89ace235c149a
+  metadata.gz: '039cece5ef2151259826355643ab8152ff6d1c9f8a2b5d82cac3ace762116e1f'
+  data.tar.gz: 65fc1e380a478b512eb73d8a616eef82f1ab230364b9d1f6b32333220cb91661
 SHA512:
-  metadata.gz: 38aa716749ec46151657f0e503019ba05de75b3f26e461a7df850ba3b3580f7ece53b00ed9bc9fde3f667200dd74972060fdf7f71f8a1d13403b6238943fa060
-  data.tar.gz: d53d40b0b11e515b8cc405e915aca658e1fbafeb5a8b9cd6dc495bfb338bcd76daff3934a9c1e4674e3b147c28e74652cc733dd11914984a8534862e4a361d75
+  metadata.gz: 814b0ff7264211aad78993a3dcfbd382e56b5a33d1796bb1c9f1017ebab9221b98ee6bf1e2b4ff3a11b0a0a9304ebd486188a54619ce72354dd7c97c6960dfab
+  data.tar.gz: 402253ffafe44e028a9ef86046f6f206d7f50195a6f9e7dc779d0ddce6d3dce4836e630f76ebaa3e11d7b3d31a857de2322d73c801a2b25aaeef3ef58d0da70e

data/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.1.2.1 (July 25, 2021)
+  - series / provide: adopted proper Helpers.symbols usage
+  - bin/signals added :lookback parameter
+  - using .symbols from Helpers
+  - init.rb: added format support (due update symbols.csv)
+  - some minor bugs + enhanced debug output in .distribute
+  - minor enhancement in debug output
+  - added TODO: make cc::cftcs::private another gem
+  - added bounded versions to gemspec
+
+## 0.1.2 (December 30, 2020)
+  - minor change to README.md
+  - minor changes
+
 ## 0.1.1 (December 12, 2020)
 
 

data/README.md

@@ -70,6 +70,7 @@ Returns the requested series of raw COT data for given either :symbol or :id. Th
 Based on the data returned by CftcSource.provide and the given indicator set, a series is created, cached and returned. 
 
 The default indicator simply calculates the net positions for pairs of `_long` and `_short`. Other indicator sets can be created. A simple example is in `etc/cotcube/indicators-example.rb`.  
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2.1

data/bin/signals.rb

@@ -2,6 +2,13 @@
 
 require_relative '../lib/cotcube-cftcsource.rb'
 
-s = Cotcube::CftcSource.signals symbol:  ARGV[0], report: ARGV[1].downcase.to_sym, human: false, count: 800, quiet: true
-s.each {|x| puts x} 
+if ARGV[0].nil?
+  raise ArgumentError, "Need symbol to continue"
+end
+symbol = ARGV[0].upcase
+lookback = ARGV[1].nil? ? 55 : ARGV[1].to_i
+report = ARGV[2].nil? ? :legacy : ARGV[2].downcase
+filter = ARGV[3].nil? ? nil : /_#{ARGV[3].downcase}_/
 
+s = Cotcube::CftcSource.signals symbol: symbol, lookback: lookback, report: report.to_sym, human: false, count: 800, quiet: true, filter: filter
+s.each {|x| puts x} 

data/cotcube-cftcsource.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.email         = ['donkeybridge@jtown.eu']
 
   spec.summary       = 'Functions to download and process COT data, aggregated into a module'
-  spec.description   = 'Functions to download and process COT data, aggregated into a module'
+  spec.description   = 'Functions to download and process COT data, aggregated into a module '
 
   spec.homepage      = 'https://github.com/donkeybridge/'+ spec.name
   spec.license       = 'BSD-4-Clause'
@@ -26,15 +26,16 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'cotcube-indicators'
-  spec.add_dependency 'yaml'
-  spec.add_dependency 'activesupport'
-  spec.add_dependency 'colorize'
-  spec.add_dependency 'httparty'
-  spec.add_dependency 'rubyzip'
+  spec.add_dependency 'cotcube-indicators', '~> 0.1.10'
+  spec.add_dependency 'cotcube-helpers',    '~> 0.1'
+  spec.add_dependency 'yaml', '~> 0.1'
+  spec.add_dependency 'activesupport', '~> 6'
+  spec.add_dependency 'colorize', '~> 0.8'
+  spec.add_dependency 'httparty', '~> 0.16'
+  spec.add_dependency 'rubyzip',  '~> 2.3'
 
 
-  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rake', '~> 13'
   spec.add_development_dependency 'rspec', '~>3.6'
   spec.add_development_dependency 'yard', '~>0.9'
 end

data/lib/cotcube-cftcsource.rb

@@ -4,6 +4,7 @@
 # require 'rubygems/package'
 require 'active_support'
 require '../ci-dev/lib/cotcube-indicators'
+require 'cotcube-helpers'
 require 'colorize'
 require 'date' unless defined?(DateTime)
 require 'csv'  unless defined?(CSV)
@@ -20,18 +21,19 @@ require_relative 'cotcube-cftcsource/distribute'
 require_relative 'cotcube-cftcsource/provide'
 require_relative 'cotcube-cftcsource/series'
 
+# TODO: make these private files another gem, that finally requires _this_ gem to run
 private_files = Dir[__dir__ + '/cotcube-cftcsource/private/*.rb']
 private_files.each do |file| 
-  # puts 'Loading private module extension ' + file.chomp
+#  # puts 'Loading private module extension ' + file.chomp
   require file.chomp
 end
 
 module Cotcube
   module CftcSource
+    include Cotcube::Helpers
     module_function :config_path, # provides the path of configuration directory
       :config_prefix,             # provides the prefix of the configuration directory according to OS-specific FSH
       :init,                      # checks whether environment is prepared and returns the config hash
-      :symbols,                   # reads and provides the symbols file
       :date_of_last_report,       # returns the date of the currently last locally available report
       :current_cot_date_on_website, # returns the date of the current report officially available on cftc.gov
       :fetch,                     # downloads all (annual) reports, based on given year

data/lib/cotcube-cftcsource/distribute.rb

@@ -8,18 +8,18 @@ module Cotcube
     # @param [Hash] opts
     # @option [Integer, String] Year to be processed, defaults to current year
 
-    def distribute(year: Time.now.year, debug: false, config: init)
+    def distribute(year: Time.now.year, debug: false, config: init, quiet: false)
       raw_path = "#{config[:data_path]}/raw"
       cot_path = "#{config[:data_path]}/cot"
       CFTC_LINKS.each do |report, a|
 	a.each do |combined, _b|
-	  puts ("Processing #{report}\t#{combined}\t#{year}") if debug
+	  puts ("Processing #{report}\t#{combined}\t#{year}") unless quiet
 
 	  infile  = "#{raw_path}/#{report}_#{combined}_#{year}.csv"
 	  outfile = ->(symbol) { "#{cot_path}/#{symbol}/#{report}_#{combined}.csv" }
 
 	  last_report = lambda do |symbol|
-	    CSV.read("#{cot_path}/#{symbol}/#{report}_#{combined}.csv").last[1]
+	    CSV.parse(`tail -n1 #{cot_path}/#{symbol}/#{report}_#{combined}.csv`).last[1]
 	  rescue StandardError
 	    '1900-01-01'
 	  end
@@ -27,15 +27,18 @@ module Cotcube
 	  cache = {}
 	  csv_data = CSV.read(infile).tap { |lines| lines.shift if lines[0][0] =~ /Market/ }
 	  csv_data.sort_by { |x| x[2] }.each do |line|
+            puts "processing #{line.take(5)}" if debug
 	    next if line[3].length != 6
 
 	    sym = line[3]
 	    cache[sym] ||= last_report.call(sym)
-	    next if cache[sym] >= line[2]
+            puts "#{cache[sym]} >= #{line[2]}" if debug
+	    next if cache[sym] >= line[1]
 
 	    line << "#{report} #{combined}"
 	    line.map! { |x| x&.strip }
 	    begin
+              puts "Writing to #{outfile.call(sym)}: #{line.take(5)}" unless quiet
 	      CSV.open(outfile.call(sym), 'a+') { |f| f << line }
 	    rescue StandardError
 	      puts ("Found new id: #{sym}").colorize(:light_green)
@@ -50,7 +53,8 @@ module Cotcube
     # print list 
 
     def available_cftc_ids(config: init, print: true, sort: :exchange)
-      result = CSV.parse(`find #{init[:data_path]}/cot/* | grep legacy | xargs head -n1  | grep , | awk -v FS=, -v OFS=, '{print $5,$4,$1}' | sed 's/"//g' | sort | uniq`.chomp, headers: %i{ exchange id name }).map{|x| x.to_h} 
+      command = %Q[find #{init[:data_path]}/cot/* | grep legacy | xargs head -n1  | grep , | awk -v FS=, -v OFS=, '{print $5,$4,$1}' | sed 's/"//g' | sort | uniq]
+      result = CSV.parse(`#{command}`.chomp, headers: %i{ exchange id name }).map{|x| x.to_h}
       result = result.uniq{|row| row[:id] }
       if print
         result.sort_by{|x| x[sort]}.each {|row| puts "#{row[:exchange]}\t#{row[:id]}\t#{row[:name]}" }

data/lib/cotcube-cftcsource/fetch.rb

@@ -10,6 +10,7 @@ module Cotcube
     # @param [Boolean]         debug
     # @param [Hash]            config
     def fetch(year: Time.now.year, debug: false, silent: false, config: init)
+      recent_report_date = `tail -n1 #{config[:data_path]}/cot/13874A/legacy_com.csv  | cut -d ',' -f 3`.chomp
 
       CFTC_LINKS.each do |report, a|
         a.each do |combined, _b|
@@ -28,8 +29,10 @@ module Cotcube
           File.write(file, file_data)
 
           puts "Contents have been written to '#{file}'." if debug
-          print "Check whether current date for #{report}_#{combined}_#{year} makes sense: " unless silent
-          puts `head -n2 #{file} | tail -n1 | cut -d, -f1,3`.chomp.light_white unless silent
+          current_report_date = `head -n2 #{file} | tail -n1 | cut -d, -f3`.chomp
+          if recent_report_date == current_report_date || current_report_date==''
+            puts "#{report}:#{combined}\tLast date '#{current_report_date}' is same as recent_date '#{recent_report_date}', please re_run".colorize(:light_yellow)
+          end
         end
       end
     end

data/lib/cotcube-cftcsource/init.rb

@@ -4,17 +4,19 @@ module Cotcube
   module CftcSource
 
 
-    def symbols(config: init)
-      if config[:symbols_file].nil?
-        CFTC_SYMBOL_EXAMPLES
-      else
-        CSV
-          .read(config[:symbols_file], headers: %i{ id symbol ticksize power months type bcf reports name})
-          .map{|row| row.to_h }
-          .map{|row| [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}; row }
-          .reject{|row| row[:id].nil? }
-      end
-    end
+#    def symbols(config: init, type: nil, symbol: nil)
+#      type = [type] unless [NilClass, Array].include?(type.class)
+#      if config[:symbols_file].nil?
+#        SYMBOL_EXAMPLES
+#      else
+#        CSV.read(config[:symbols_file], headers: %i{ id symbol ticksize power months type bcf reports format name}).
+#          map{|row| row.to_h }.
+#          map{|row| [ :ticksize, :power, :bcf ].each {|z| row[z] = row[z].to_f}; row }.
+#          reject{|row| row[:id].nil? }.
+#          tap{|all| all.select!{|x| type.include?(x[:type])} unless type.nil? }.
+#          tap{|all| all.select!{|x| symbol.include?(x[:symbol])} unless symbol.nil? }
+#      end
+#    end
 
     def config_prefix
       os       = Gem::Platform.local.os

data/lib/cotcube-cftcsource/provide.rb

@@ -15,7 +15,8 @@ module Cotcube
       after = DateTime.strptime(after, '%Y-%m-%d') if after.is_a? String
 
       unless symbol.nil? 
-        symbol_id = symbols.select{|s| s[:symbol] == symbol.to_s.upcase}.first[:id]
+        symbol_config = Cotcube::Helpers.symbols(symbol: symbol).first
+        symbol_id     = symbol_config[:id]
         raise ArgumentError, "Could not find match in #{config[:symbols_file]} for given symbol #{symbol}" if symbol_id.nil? 
         raise ArgumentError, "Mismatching symbol #{symbol} and given id #{id}" if not id.nil? and symbol_id != id
         id = symbol_id

data/lib/cotcube-cftcsource/series.rb

@@ -21,7 +21,7 @@ module Cotcube
                debug: false
               )
       raise ArgumentError, "Can only build series with given symbol" if symbol.nil?
-      symbol_config = symbols.select{|s| s[:symbol] == symbol.to_s.upcase }.first
+      symbol_config = Cotcube::Helpers.symbols(symbol: symbol).first
       raise ArgumentError, "Can only build series with known symbol, '#{symbol}' is unknown." if symbol_config.nil?
       puts "Using symbol_config '#{symbol_config}'" if debug
 

metadata

@@ -1,113 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: cotcube-cftcsource
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2.1
 platform: ruby
 authors:
 - Benjamin L. Tischendorf
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-12 00:00:00.000000000 Z
+date: 2021-07-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: cotcube-indicators
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.1.10
+- !ruby/object:Gem::Dependency
+  name: cotcube-helpers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: yaml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.1'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '6'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.8'
 - !ruby/object:Gem::Dependency
   name: httparty
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.16'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.16'
 - !ruby/object:Gem::Dependency
   name: rubyzip
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '13'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -136,7 +150,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
-description: Functions to download and process COT data, aggregated into a module
+description: 'Functions to download and process COT data, aggregated into a module '
 email:
 - donkeybridge@jtown.eu
 executables: []