rvgp 0.3.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (97) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.rubocop.yml +23 -0
  4. data/LICENSE +504 -0
  5. data/README.md +223 -0
  6. data/Rakefile +32 -0
  7. data/bin/rvgp +8 -0
  8. data/lib/rvgp/application/config.rb +159 -0
  9. data/lib/rvgp/application/descendant_registry.rb +122 -0
  10. data/lib/rvgp/application/status_output.rb +139 -0
  11. data/lib/rvgp/application.rb +170 -0
  12. data/lib/rvgp/base/command.rb +457 -0
  13. data/lib/rvgp/base/grid.rb +531 -0
  14. data/lib/rvgp/base/reader.rb +29 -0
  15. data/lib/rvgp/base/reconciler.rb +434 -0
  16. data/lib/rvgp/base/validation.rb +261 -0
  17. data/lib/rvgp/commands/cashflow.rb +160 -0
  18. data/lib/rvgp/commands/grid.rb +70 -0
  19. data/lib/rvgp/commands/ireconcile.rb +95 -0
  20. data/lib/rvgp/commands/new_project.rb +296 -0
  21. data/lib/rvgp/commands/plot.rb +41 -0
  22. data/lib/rvgp/commands/publish_gsheets.rb +83 -0
  23. data/lib/rvgp/commands/reconcile.rb +58 -0
  24. data/lib/rvgp/commands/rotate_year.rb +202 -0
  25. data/lib/rvgp/commands/validate_journal.rb +59 -0
  26. data/lib/rvgp/commands/validate_system.rb +44 -0
  27. data/lib/rvgp/commands.rb +160 -0
  28. data/lib/rvgp/dashboard.rb +252 -0
  29. data/lib/rvgp/fakers/fake_feed.rb +245 -0
  30. data/lib/rvgp/fakers/fake_journal.rb +57 -0
  31. data/lib/rvgp/fakers/fake_reconciler.rb +88 -0
  32. data/lib/rvgp/fakers/faker_helpers.rb +25 -0
  33. data/lib/rvgp/gem.rb +80 -0
  34. data/lib/rvgp/journal/commodity.rb +453 -0
  35. data/lib/rvgp/journal/complex_commodity.rb +214 -0
  36. data/lib/rvgp/journal/currency.rb +101 -0
  37. data/lib/rvgp/journal/journal.rb +141 -0
  38. data/lib/rvgp/journal/posting.rb +156 -0
  39. data/lib/rvgp/journal/pricer.rb +267 -0
  40. data/lib/rvgp/journal.rb +24 -0
  41. data/lib/rvgp/plot/gnuplot.rb +478 -0
  42. data/lib/rvgp/plot/google-drive/output_csv.rb +44 -0
  43. data/lib/rvgp/plot/google-drive/output_google_sheets.rb +434 -0
  44. data/lib/rvgp/plot/google-drive/sheet.rb +67 -0
  45. data/lib/rvgp/plot.rb +293 -0
  46. data/lib/rvgp/pta/hledger.rb +237 -0
  47. data/lib/rvgp/pta/ledger.rb +308 -0
  48. data/lib/rvgp/pta.rb +311 -0
  49. data/lib/rvgp/reconcilers/csv_reconciler.rb +424 -0
  50. data/lib/rvgp/reconcilers/journal_reconciler.rb +41 -0
  51. data/lib/rvgp/reconcilers/shorthand/finance_gem_hacks.rb +48 -0
  52. data/lib/rvgp/reconcilers/shorthand/international_atm.rb +152 -0
  53. data/lib/rvgp/reconcilers/shorthand/investment.rb +144 -0
  54. data/lib/rvgp/reconcilers/shorthand/mortgage.rb +195 -0
  55. data/lib/rvgp/utilities/grid_query.rb +190 -0
  56. data/lib/rvgp/utilities/yaml.rb +131 -0
  57. data/lib/rvgp/utilities.rb +44 -0
  58. data/lib/rvgp/validations/balance_validation.rb +68 -0
  59. data/lib/rvgp/validations/duplicate_tags_validation.rb +48 -0
  60. data/lib/rvgp/validations/uncategorized_validation.rb +15 -0
  61. data/lib/rvgp.rb +66 -0
  62. data/resources/README.MD/2022-cashflow-google.png +0 -0
  63. data/resources/README.MD/2022-cashflow.png +0 -0
  64. data/resources/README.MD/all-wealth-growth-google.png +0 -0
  65. data/resources/README.MD/all-wealth-growth.png +0 -0
  66. data/resources/gnuplot/default.yml +80 -0
  67. data/resources/i18n/en.yml +192 -0
  68. data/resources/iso-4217-currencies.json +171 -0
  69. data/resources/skel/Rakefile +5 -0
  70. data/resources/skel/app/grids/cashflow_grid.rb +27 -0
  71. data/resources/skel/app/grids/monthly_income_and_expenses_grid.rb +25 -0
  72. data/resources/skel/app/grids/wealth_growth_grid.rb +35 -0
  73. data/resources/skel/app/plots/cashflow.yml +33 -0
  74. data/resources/skel/app/plots/monthly-income-and-expenses.yml +17 -0
  75. data/resources/skel/app/plots/wealth-growth.yml +20 -0
  76. data/resources/skel/config/csv-format-acme-checking.yml +9 -0
  77. data/resources/skel/config/google-secrets.yml +5 -0
  78. data/resources/skel/config/rvgp.yml +0 -0
  79. data/resources/skel/journals/prices.db +0 -0
  80. data/rvgp.gemspec +6 -0
  81. data/test/assets/ledger_total_monthly_liabilities_with_empty.xml +383 -0
  82. data/test/assets/ledger_total_monthly_liabilities_with_empty2.xml +428 -0
  83. data/test/test_command_base.rb +61 -0
  84. data/test/test_commodity.rb +270 -0
  85. data/test/test_csv_reconciler.rb +60 -0
  86. data/test/test_currency.rb +24 -0
  87. data/test/test_fake_feed.rb +228 -0
  88. data/test/test_fake_journal.rb +98 -0
  89. data/test/test_fake_reconciler.rb +60 -0
  90. data/test/test_journal_parse.rb +545 -0
  91. data/test/test_ledger.rb +102 -0
  92. data/test/test_plot.rb +133 -0
  93. data/test/test_posting.rb +50 -0
  94. data/test/test_pricer.rb +139 -0
  95. data/test/test_pta_adapter.rb +575 -0
  96. data/test/test_utilities.rb +45 -0
  97. metadata +268 -0
@@ -0,0 +1,144 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RVGP
4
+ module Reconcilers
5
+ # This module contains the built-in Shorthand classes that ship with RVGP. For more details on this
6
+ # feature, see the 'Shorthand' section of the {RVGP::Reconcilers} module.
7
+ module Shorthand
8
+ # This reconciler module will automatically allocate the proceeds (or losses) from a stock sale.
9
+ # This module will allocate capital gains or losses, given a symbol, amount, and price.
10
+ #
11
+ # The module parameters we support are:
12
+ # - **symbol** [String] - A commodity or currency code, that represents the purchased asset
13
+ # - **amount** [Integer] - The amount of :symbol that was purchased, or if negative, sold.
14
+ # - **price** [Commodity] - A unit price, for the symbol. This field should be delimited if :total is omitted.
15
+ # - **total** [Commodity] - A lot price, the symbol. This represents the net purchase price, which, would be
16
+ # divided by the amount, in order to arrive at a unit price. This field should be delimited if :price is
17
+ # omitted.
18
+ # - **capital_gains** [Commodity] - The amount of the total, to allocate to a capital gains account. Presumably
19
+ # for tax reporting.
20
+ # - **gains_account** [String] - The account name to allocate capital gains to.
21
+ #
22
+ # # Example
23
+ # Here's how this module might be used in your reconciler:
24
+ # ```
25
+ # ...
26
+ # - match: /Acme Stonk Exchange/
27
+ # to_shorthand: Investment
28
+ # shorthand_params:
29
+ # symbol: VOO
30
+ # price: "$ 400.00"
31
+ # amount: "-1000"
32
+ # capital_gains: "$ -100000.00"
33
+ # gains_account: Personal:Income:AcmeExchange:VOO
34
+ # ...
35
+ # ```
36
+ # And here's how that will reconcile, in your build:
37
+ # ```
38
+ # ...
39
+ # 2023-06-01 Acme Stonk Exchange ACH CREDIT 123456 Yukihiro Matsumoto
40
+ # Personal:Assets -1000 VOO @@ $ 400000.00
41
+ # Personal:Income:AcmeExchange:VOO $ 100000.00
42
+ # Personal:Assets:AcmeChecking
43
+ # ...
44
+ # ```
45
+ #
46
+ class Investment
47
+ # @!visibility private
48
+ attr_reader :tag, :symbol, :price, :amount, :total, :capital_gains,
49
+ :remainder_amount, :remainder_account, :targets, :is_sell,
50
+ :to, :gains_account
51
+
52
+ def initialize(rule)
53
+ @tag = rule[:tag]
54
+ @targets = rule[:targets]
55
+ @to = rule[:to] || 'Personal:Assets'
56
+
57
+ if rule.key? :shorthand_params
58
+ @symbol = rule[:shorthand_params][:symbol]
59
+ @amount = rule[:shorthand_params][:amount]
60
+ @gains_account = rule[:shorthand_params][:gains_account]
61
+
62
+ %w[price total capital_gains].each do |key|
63
+ if rule[:shorthand_params].key? key.to_sym
64
+ instance_variable_set "@#{key}".to_sym, rule[:shorthand_params][key.to_sym].to_commodity
65
+ end
66
+ end
67
+ end
68
+
69
+ unless [symbol, amount].all?
70
+ raise StandardError, format('Investment at line:%s missing fields', rule[:line].inspect)
71
+ end
72
+
73
+ @is_sell = (amount.to_f <= 0)
74
+
75
+ # I mostly just think this doesn't make any sense... I guess if we took a
76
+ # loss...
77
+ raise StandardError, format('Unimplemented %s', rule.inspect) if capital_gains && !is_sell
78
+
79
+ if (gains_account.nil? || capital_gains.nil?) && is_sell
80
+ raise StandardError, format('Investment at line:%s missing gains_account', rule.inspect)
81
+ end
82
+
83
+ unless total || price
84
+ raise StandardError, format('Investment at line:%s missing an price or total', rule[:line].inspect)
85
+ end
86
+
87
+ if total && price
88
+ raise StandardError, format('Investment at line:%s specified both price and total', rule[:line].inspect)
89
+ end
90
+ end
91
+
92
+ # @!visibility private
93
+ def to_tx(from_posting)
94
+ income_targets = []
95
+
96
+ # NOTE: I pulled most of this from: https://hledger.org/investments.html
97
+ if is_sell && capital_gains
98
+ # NOTE: I'm not positive about this .abs....
99
+ cost_basis = (total || (price * amount.to_f.abs)) - capital_gains
100
+
101
+ income_targets << { to: to,
102
+ complex_commodity: RVGP::Journal::ComplexCommodity.new(
103
+ left: [amount, symbol].join(' ').to_commodity,
104
+ operation: :per_lot,
105
+ right: cost_basis
106
+ ) }
107
+
108
+ income_targets << { to: gains_account, commodity: capital_gains.dup.invert! } if capital_gains
109
+ else
110
+ income_targets << { to: to,
111
+ complex_commodity: RVGP::Journal::ComplexCommodity.new(
112
+ left: [amount, symbol].join(' ').to_commodity,
113
+ operation: (price ? :per_unit : :per_lot),
114
+ right: price || total
115
+ ) }
116
+ end
117
+
118
+ if targets
119
+ income_targets += targets.map do |t|
120
+ ret = { to: t[:to] }
121
+ if t.key? :amount
122
+ # TODO: I think there's a bug here, in that amounts with commodities, won't parse...
123
+ ret[:commodity] = RVGP::Journal::Commodity.from_symbol_and_amount t[:currency] || '$', t[:amount].to_s
124
+ end
125
+
126
+ if t.key? :complex_commodity
127
+ ret[:complex_commodity] = RVGP::Journal::ComplexCommodity.from_s t[:complex_commodity]
128
+ end
129
+
130
+ ret
131
+ end
132
+ end
133
+
134
+ RVGP::Base::Reconciler::Posting.new from_posting.line_number,
135
+ date: from_posting.date,
136
+ description: from_posting.description,
137
+ from: from_posting.from,
138
+ tags: from_posting.tags,
139
+ targets: income_targets
140
+ end
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,195 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'finance'
4
+ require 'finance'
5
+ require_relative './finance_gem_hacks'
6
+
7
+ module RVGP
8
+ module Reconcilers
9
+ module Shorthand
10
+ # This reconciler module will automatically allocate the the escrow, principal, and interest components of a
11
+ # mortage debit, into constituent accounts. The amounts of each, are automatically calculated, based on the loan
12
+ # terms, and taking the residual leftover, into a escrow account, presumably for taxes and insurance to be paid by
13
+ # the mortgage provider.
14
+ #
15
+ # Its important to note, that a single reconciler rule will match every mortgage payment encountered. And, that
16
+ # each of these payments will generate four transactions in the output file:
17
+ # - The initial payment, which, will be transferred to an :intermediary_account
18
+ # - A principal payment, which, will be debited from the :intermediary account, to the :payee_principal
19
+ # - An interest payment, which, will be debited from the :intermediary account, to the :payee_interest
20
+ # - An escrow payment, which, will be debited from the :intermediary account, to the :escrow_account
21
+ # You can see details of this expansion under the example section below.
22
+ #
23
+ # With regards to the :escrow_account, It's likely that you'll want to either (choose one):
24
+ # - Manually transcribe debits from the escrow account, to escrow payees, in your project's
25
+ # ./journal/property-name-escrow.journal, based on when your mortgage provider alerts you to these payments.
26
+ # - Download a csv from your mortgage provider, of your escrow account (if they offer one), and define a
27
+ # reconciler to allocate escrow payments.
28
+ #
29
+ # The module parameters we support are:
30
+ # - **label** [String] - This is a prefix, used in the description of Principal, Interest, and Escrow transactions
31
+ # - **principal** [Commodity] - The mortgage principal
32
+ # - **rate** [Float] - The mortgage rate
33
+ # - **payee_principal** [String] - The account to ascribe principal payments to
34
+ # - **payee_interest** [String] - The account to ascribe interest payments to
35
+ # - **escrow_account** [String] - Te account to ascribe escrow payments to
36
+ # - **intermediary_account** [String] - The account to ascribe intermediary payments to, from the source account,
37
+ # before being assigned to principal, interest, and escrow break-outs.
38
+ # - **start_at_installment_number** [Integer] - The installment number, of the first matching transaction,
39
+ # encountered by this module. Year one of a mortgage would start at zero. Subsequent annual reconcilers would
40
+ # be expected to define an installment number from which calculations can automatically pick-up the work
41
+ # from years prior.
42
+ # - **additional_payments** [Array<Hash>] - Any additional payments, to apply to the principal, can be listed
43
+ # here. This field is expected to be an array of hashes, which, are composed of the following fields:
44
+ # - **before_installment** [Integer] - The payment number, before which, this :amount should apply
45
+ # - **amount** [Float] - A float that will be deducted from the principal. No commodity is necessary to
46
+ # delineate, as we assume the same commodity as the :principle.
47
+ # - **override_payments** [Array<Hash>] - I can't explain why this is necessary. But, it seems that the interest
48
+ # calculations used by some mortgage providers ... aren't accurate. This happened to me, at least. The
49
+ # calculation being used was off by a penny, on a single installment. And, I didn't care enough to call the
50
+ # institution and figure out why. So, I added this feature, to allow an override of the automatic calculation,
51
+ # with the amount provided. This field is expected to be an array of hashes, which are composed of the following
52
+ # fields:
53
+ # - **at_installment** [Integer] - The payment number to assert the :interest value.
54
+ # - **interest** [Float] - The amount of the interest calculation. No commodity is neccessary to delineate, as
55
+ # we assume the same commodity as the :principle.
56
+ #
57
+ # # Example
58
+ # Here's how this module might be used in your reconciler:
59
+ # ```
60
+ # ...
61
+ # - match: /AcmeFinance Servicing/
62
+ # to_shorthand: Mortgage
63
+ # shorthand_params:
64
+ # label: 1-8-1 Yurakucho Dori Mortgage
65
+ # intermediary_account: Personal:Expenses:Banking:MortgagePayment:181Yurakucho
66
+ # payee_principal: Personal:Liabilities:Mortgage:181Yurakucho
67
+ # payee_interest: Personal:Expenses:Banking:Interest:181Yurakucho
68
+ # escrow_account: Personal:Assets:AcmeFinance:181YurakuchoEscrow
69
+ # principal: 260000.00
70
+ # rate: 0.0499
71
+ # start_at_installment_number: 62
72
+ # ...
73
+ # ```
74
+ # And here's how that will reconcile one of your payments, in your build:
75
+ # ```
76
+ # ...
77
+ # 2023-01-03 AcmeFinance Servicing MTG PYMT 012345 Yukihiro Matsumoto
78
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho $ 3093.67
79
+ # Personal:Assets:AcmeBank:Checking
80
+ #
81
+ # 2023-01-03 1-8-1 Yurakucho Dori Mortgage (#61) Principal
82
+ # Personal:Liabilities:Mortgage:181Yurakucho $ 403.14
83
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho
84
+ #
85
+ # 2023-01-03 1-8-1 Yurakucho Dori Mortgage (#61) Interest
86
+ # Personal:Expenses:Banking:Interest:181Yurakucho $ 991.01
87
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho
88
+ #
89
+ # 2023-01-03 1-8-1 Yurakucho Dori Mortgage (#61) Escrow
90
+ # Personal:Assets:AcmeFinance:181YurakuchoEscrow $ 1699.52
91
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho
92
+ # ...
93
+ # ```
94
+ # Note that you'll have an automatically calculated reconcilation for each payment you
95
+ # make, during the year. A single reconciler rule, will take care of reconciling every
96
+ # payment, automatically.
97
+ class Mortgage
98
+ # @!visibility private
99
+ attr_accessor :principal, :rate, :start_at_installment_number,
100
+ :additional_payments, :amortization, :payee_principal, :payee_interest,
101
+ :intermediary_account, :currency, :label, :escrow_account, :override_payments
102
+
103
+ # @!visibility private
104
+ def initialize(rule)
105
+ @label = rule[:shorthand_params][:label]
106
+ @currency = rule[:currency] || '$'
107
+ @principal = rule[:shorthand_params][:principal].to_commodity
108
+ @rate = rule[:shorthand_params][:rate]
109
+ @payee_principal = rule[:shorthand_params][:payee_principal]
110
+ @payee_interest = rule[:shorthand_params][:payee_interest]
111
+ @intermediary_account = rule[:shorthand_params][:intermediary_account]
112
+ @escrow_account = rule[:shorthand_params][:escrow_account]
113
+ @start_at_installment_number = rule[:shorthand_params][:start_at_installment_number]
114
+ @additional_payments = rule[:shorthand_params][:additional_payments]
115
+ @override_payments = {}
116
+ if rule[:shorthand_params].key? :override_payments
117
+ rule[:shorthand_params][:override_payments].each do |override|
118
+ unless %i[at_installment interest].all? { |k| override.key? k }
119
+ raise StandardError, format('Invalid Payment Override : %s', override)
120
+ end
121
+
122
+ @override_payments[ override[:at_installment] ] = {
123
+ interest: RVGP::Journal::Commodity.from_symbol_and_amount(currency, override[:interest])
124
+ }
125
+ end
126
+ end
127
+
128
+ unless [principal, rate, payee_principal, payee_interest, intermediary_account, escrow_account, label].all?
129
+ raise StandardError, format('Mortgage at line:%d missing fields', rule[:line])
130
+ end
131
+
132
+ fr = Finance::Rate.new rate, :apr, duration: 360
133
+ @amortization = principal.to_f.amortize(fr) do |period, amortization|
134
+ additional_payments&.each do |ap|
135
+ if period == ap[:before_installment]
136
+ amortization.balance = amortization.balance - DecNum.new(ap[:amount].to_s)
137
+ end
138
+ end
139
+ end
140
+
141
+ @installment_i = start_at_installment_number ? (start_at_installment_number - 1) : 0
142
+ end
143
+
144
+ # @!visibility private
145
+ def to_tx(from_posting)
146
+ payment = RVGP::Journal::Commodity.from_symbol_and_amount(currency, amortization.payments[@installment_i]).abs
147
+ interest = RVGP::Journal::Commodity.from_symbol_and_amount(currency, amortization.interest[@installment_i])
148
+
149
+ interest = @override_payments[@installment_i][:interest] if @override_payments.key? @installment_i
150
+
151
+ principal = payment - interest
152
+ escrow = from_posting.commodity.abs - payment
153
+ total = principal + interest + escrow
154
+
155
+ @installment_i += 1
156
+
157
+ intermediary_opts = { date: from_posting.date, from: intermediary_account, tags: from_posting.tags }
158
+
159
+ [RVGP::Base::Reconciler::Posting.new(from_posting.line_number,
160
+ date: from_posting.date,
161
+ description: from_posting.description,
162
+ from: from_posting.from,
163
+ tags: from_posting.tags,
164
+ targets: [to: intermediary_account, commodity: total]),
165
+ # Principal:
166
+ RVGP::Base::Reconciler::Posting.new(
167
+ from_posting.line_number,
168
+ intermediary_opts.merge({ description: format('%<label>s (#%<num>d) Principal',
169
+ label: label,
170
+ num: @installment_i - 1),
171
+ targets: [{ to: payee_principal, commodity: principal }] })
172
+ ),
173
+
174
+ # Interest:
175
+ RVGP::Base::Reconciler::Posting.new(
176
+ from_posting.line_number,
177
+ intermediary_opts.merge({ description: format('%<label>s (#%<num>d) Interest',
178
+ label: label,
179
+ num: @installment_i - 1),
180
+ targets: [{ to: payee_interest, commodity: interest }] })
181
+ ),
182
+
183
+ # Escrow:
184
+ RVGP::Base::Reconciler::Posting.new(
185
+ from_posting.line_number,
186
+ intermediary_opts.merge({ description: format('%<label>s (#%<num>d) Escrow',
187
+ label: label,
188
+ num: @installment_i - 1),
189
+ targets: [{ to: escrow_account, commodity: escrow }] })
190
+ )]
191
+ end
192
+ end
193
+ end
194
+ end
195
+ end
@@ -0,0 +1,190 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'csv'
4
+
5
+ module RVGP
6
+ module Utilities
7
+ # This class provides a number of utility functions to query, and merge, grids
8
+ # that have been built. The primary purpose of these tools, is to decrease
9
+ # the overhead that would otherwise exist, if we were to operate directly on
10
+ # the pta output, every time we referenced this data. As well as to maintain
11
+ # auditability for this data, in the project build.
12
+ # @attr_reader [Array<String>] headers The first row of a grid, without the keystone included
13
+ # @attr_reader [Hash<String,Hash<String,Object>>] data A hash whose key is the series name, whose value is a hash of
14
+ # headername to value pairs
15
+ # @attr_reader [String] keystone The cell in the upper-left of the grid. Often this cell is used as a superordinate
16
+ # label for the series below it.
17
+ class GridQuery
18
+ # NOTE: I'm not exactly sure what this class wants to be just yet...
19
+ # Let's see if we end up using it for graphing... It might just be a 'Spreadsheet'
20
+ # and we may want/need to move the summary columns into here
21
+ attr_reader :headers, :data, :keystone
22
+
23
+ # Setup a GridQuery
24
+ # @param [Array<String>] from_files An array of paths, to grid files
25
+ # @param [Hash] options Additional options
26
+ # @option options [String] keystone see {RVGP::Utilities::GridQuery#keystone}
27
+ # @option options [Proc] store_cell This proc is provided one parameter, a data cell. The return value of this
28
+ # proc is then stored in :data, in lieu of the parameter.
29
+ # @option options [Proc] select_columns This proc is provided two parameters, a header (String), and the cells
30
+ # underneath it in the grid (Array<Object>). If this proc returns true,
31
+ # that column is stored in :data. If this proc returns false, that column
32
+ # is not retained in this query.
33
+ # @option options [Proc] select_rows This proc is provided two parameters, a series_name (String), and the cells
34
+ # to the right of it in the grid (Array<Object>). If this proc returns true,
35
+ # that column is stored in :data. If this proc returns false, that column
36
+ # is not retained in this query.
37
+ def initialize(from_files, options = {})
38
+ @headers = []
39
+ @data = {}
40
+
41
+ from_files.each do |file|
42
+ csv = CSV.open file, 'r', headers: true
43
+ rows = csv.read
44
+ headers = csv.headers
45
+
46
+ # We assume the first column of the row, is the series name
47
+ @keystone = headers.shift
48
+
49
+ if options[:select_columns]
50
+ selected_headers = headers.select do |header|
51
+ options[:select_columns].call(header, rows.map { |row| row[header] })
52
+ end
53
+ end
54
+ selected_headers ||= headers
55
+
56
+ add_columns selected_headers
57
+
58
+ rows.each do |row|
59
+ series_data = row.to_h
60
+ series_name = series_data.delete @keystone
61
+ if options.key? :store_cell
62
+ series_data.each do |col, cell|
63
+ next unless !options.key?(:select_columns) || selected_headers.include?(col)
64
+
65
+ series_data[col] = options[:store_cell].call cell
66
+ end
67
+ end
68
+
69
+ if !options.key?(:select_rows) || options[:select_rows].call(series_name, series_data)
70
+ add_data series_name, series_data
71
+ end
72
+ end
73
+ end
74
+
75
+ # This needs to be assigned after we've processed the csv's
76
+ @keystone = options[:keystone] if options.key? :keystone
77
+ end
78
+
79
+ # Returns the results of a query.
80
+ # @param [Hash] opts Additional options
81
+ # @option opts [Proc] sort_cols_by Columns are sorted by the value returned by this proc. This proc is provided
82
+ # the grid :headers, and this proc is handled according to the rules of
83
+ # Enumerable#sort_by.
84
+ # @option opts [Proc] sort_rows_by Rows are sorted by the value returned by this proc. This proc is provided
85
+ # each row in this grid, with the series_name of the row at position zero.
86
+ # This proc is handled according to the rules of Enumerable#sort_by.
87
+ # @option opts [Integer] truncate_columns Restrict the total number of columns to a maximum this number
88
+ # @option opts [Integer] truncate_rows Restrict the total number of rows to a maximum this number
89
+ # @option opts [String] truncate_remainder_row If the columns are truncated, the 'chopped' columns are summed into
90
+ # 'the last column'. The value of this option, is used as the label
91
+ # for that column. Candidly... this feature makes a lot of
92
+ # assumptions, and probably needs a bit more refinement.
93
+ # @option opts [TrueClass,FalseClass] switch_rows_columns Rotates the grid, so that series labels and headers are
94
+ # swapped. All data is preserved under/adjacent to the same series label and
95
+ # header, in the returned grid. Google offers this option in its GUI, but
96
+ # doesn't seem to support it via the API. So, we can just do that ourselves
97
+ # here.
98
+ # @return [Array<Array<Object>>] A grid of cells, composed of the provided grids, and transformed by the given
99
+ # parameters.
100
+ def to_grid(opts = {})
101
+ # First we'll collect the header row, possibly sorted :
102
+ if opts[:sort_cols_by]
103
+ # We collect the data under the column, and feed that into sort_cols_by
104
+ grid_columns = headers.map do |col|
105
+ [col] + data.map { |_, values| values[col] }
106
+ end.sort_by(&opts[:sort_cols_by]).map(&:first)
107
+ end
108
+ grid_columns ||= headers
109
+
110
+ # Then we collect the non-header rows:
111
+ grid = data.map { |series, values| [series] + grid_columns.map { |col| values[col] } }
112
+
113
+ # Sort those rows, if necessesary:
114
+ grid.sort_by!(&opts[:sort_rows_by]) if opts[:sort_rows_by]
115
+
116
+ # Affix the header row to the top of the grid. Now it's assembled.
117
+ grid.unshift [keystone] + grid_columns
118
+
119
+ # Do we Truncate Rows?
120
+ if opts[:truncate_rows] && grid.length > opts[:truncate_rows]
121
+ # We can only have about 26 columns on Google. And, since we're (sometimes)
122
+ # transposing rows to columns, we have to truncate the rows.
123
+
124
+ # NOTE: The 1 is for the truncate_remainder_row, the 'overflow' column
125
+ chop_length = grid.length - opts[:truncate_rows]
126
+
127
+ if chop_length.positive?
128
+ chopped_rows = grid.pop chop_length
129
+ truncate_row = chopped_rows.inject([]) do |sum, row|
130
+ # Starting at the second cell (the first is the series name) merge
131
+ # the contents of the current row, into the collection cell.
132
+ row[1...].each_with_index.map do |cell, i|
133
+ # This rigamarole is mostly to help prevent type issues...
134
+ if cell
135
+ sum[i] ? sum[i] + cell : cell
136
+ else
137
+ sum[i]
138
+ end
139
+ end
140
+ end
141
+
142
+ grid << ([(opts[:truncate_remainder_row]) || 'Other'] + truncate_row)
143
+ end
144
+ end
145
+
146
+ # Do we Truncate Columns?
147
+ if opts[:truncate_columns] && grid[0].length > opts[:truncate_columns]
148
+ # Go through each row, pop off the excess cells, and sum them onto the end
149
+ grid.each_with_index do |row, i|
150
+ # The plus one is to make room for the 'Other' column
151
+ chop_length = row.length - opts[:truncate_columns] + 1
152
+
153
+ chopped_cols = row.pop chop_length
154
+ truncate_cell = if i.zero?
155
+ opts[:truncate_remainder_row] || 'Other'
156
+ else
157
+ chopped_cols.all?(&:nil?) ? nil : chopped_cols.compact.sum
158
+ end
159
+
160
+ row << truncate_cell
161
+ end
162
+ end
163
+
164
+ # Google offers this option in its GUI, but doesn't seem to support it via
165
+ # the API. So, we can just do that ourselves:
166
+ grid = 0.upto(grid[0].length - 1).map { |i| grid.map { |row| row[i] } } if opts[:switch_rows_columns]
167
+
168
+ grid
169
+ end
170
+
171
+ private
172
+
173
+ def add_columns(headers)
174
+ @headers += headers.reject { |header| @headers.include? header }
175
+ end
176
+
177
+ # NOTE: that we're assuming value here, is always a commodity. Not sure about
178
+ # that over time.
179
+ def add_data(series_name, colname_to_value)
180
+ @data[series_name] ||= {}
181
+
182
+ colname_to_value.each do |colname, value|
183
+ raise StandardError, 'Unimplemented. How to merge?' if @data[series_name].key? colname
184
+
185
+ @data[series_name][colname] = value
186
+ end
187
+ end
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,131 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'psych'
4
+ require 'pathname'
5
+
6
+ Psych.add_builtin_type('proc') { |_, val| RVGP::Utilities::Yaml::PsychProc.new val }
7
+ Psych.add_builtin_type('include') { |_, val| RVGP::Utilities::Yaml::PsychInclude.new val }
8
+
9
+ module RVGP
10
+ module Utilities
11
+ # This class wraps the Psych library, and adds functionality we need, to parse
12
+ # yaml files.
13
+ # We mostly added this class because the Psych.add_builtin_type wasn't able to
14
+ # contain state outside it's return values. (which we need, in order to
15
+ # track include dependencies)
16
+ class Yaml
17
+ # This class is needed, in order to capture the dependencies needed to
18
+ # properly preserve uptodate status in rake builds
19
+ class PsychInclude
20
+ attr_reader :path
21
+
22
+ # Declare a proc, given the provided proc_as_string code
23
+ # @param [String] path A relative or absolute path, to include, in the place of this line
24
+ def initialize(path)
25
+ @path = path
26
+ end
27
+
28
+ # The contents of this include target.
29
+ # @param [Array<String>] include_paths A relative or absolute path, to include, in case our path is relative
30
+ # and we need to load it from a relative location. These paths are
31
+ # scanned for the include, in the relative order of their place in the
32
+ # array.
33
+ # @return [Hash] The contents of the target of this include. Keys are symbolized, and permitted_classes include
34
+ # Date, and Symbol
35
+ def contents(include_paths)
36
+ content_path = path if Pathname.new(path).absolute?
37
+ content_path ||= include_paths.map { |p| [p.chomp('/'), path].join('/') }.find do |p|
38
+ File.readable? p
39
+ end
40
+
41
+ unless content_path
42
+ raise StandardError, format('Unable to find %<path>s in any of the provided paths: %<paths>s',
43
+ path: path.inspect, paths: include_paths.inspect)
44
+ end
45
+
46
+ Psych.safe_load_file content_path, symbolize_names: true, permitted_classes: [Date, Symbol]
47
+ end
48
+ end
49
+
50
+ # This class wraps proc types, in the yaml, and offers us the ability to execute
51
+ # the code in these blocks.
52
+ class PsychProc
53
+ # Declare a proc, given the provided proc_as_string code
54
+ # @param [String] proc_as_string Ruby code, that this block will call.
55
+ def initialize(proc_as_string)
56
+ @block = eval(format('proc { %s }', proc_as_string)) # rubocop:disable Security/Eval
57
+ end
58
+
59
+ # Execute the block, using the provided params as locals
60
+ # NOTE: We expect symbol to value here
61
+ # @param params [Hash<Symbol, Object>] local values, available in the proc context. Note that keys
62
+ # are expected to be symbols.
63
+ # @return [Object] Whatever the proc returns
64
+ def call(params = {})
65
+ # params, here, act as instance variables, since we don't support named
66
+ # params in the provided string, the way you typically would.
67
+ @params = params
68
+ @block.call
69
+ end
70
+
71
+ # @!visibility private
72
+ def respond_to_missing?(name, _include_private = false)
73
+ @params&.key?(name.to_sym)
74
+ end
75
+
76
+ # @!visibility private
77
+ def method_missing(name)
78
+ respond_to_missing?(name) ? @params[name] : super(name)
79
+ end
80
+ end
81
+
82
+ attr_reader :path, :include_paths, :dependencies
83
+
84
+ # @param [String] path The full path to the yaml file you wish to parse
85
+ # @param [Array<String>] include_paths An array of directories, to search, when locating the target of
86
+ # a "!!include" line
87
+ def initialize(path, include_paths = nil)
88
+ @path = path
89
+ @dependencies = []
90
+ @include_paths = Array(include_paths || File.expand_path(File.dirname(path)))
91
+
92
+ vanilla_yaml = Psych.safe_load_file(path, symbolize_names: true, permitted_classes: [Date, Symbol])
93
+ @yaml = replace_each_in_yaml(vanilla_yaml, PsychInclude) do |psych_inc|
94
+ @dependencies << psych_inc.path
95
+ psych_inc.contents @include_paths
96
+ end
97
+ end
98
+
99
+ # Return the specified attribute, in this yaml file
100
+ # @param [String] attr The attribute you're looking for
101
+ # @return [Object] The value of the the provided attribute
102
+ def [](attr)
103
+ @yaml&.[](attr)
104
+ end
105
+
106
+ # Returns true or false, depending on whether the attribute you're looking for, exists in this
107
+ # yaml file.
108
+ # @param [String] attr The attribute you're looking for
109
+ # @return [TrueClass,FalseClass] Whether the key is defined in this file
110
+ def key?(attr)
111
+ @yaml&.key? attr
112
+ end
113
+
114
+ alias has_key? key?
115
+
116
+ private
117
+
118
+ # This is kind of a goofy function, but, it works
119
+ def replace_each_in_yaml(obj, of_class, &blk)
120
+ case obj
121
+ when Hash
122
+ obj.transform_values { |v| replace_each_in_yaml v, of_class, &blk }
123
+ when Array
124
+ obj.collect { |v| replace_each_in_yaml(v, of_class, &blk) }
125
+ else
126
+ obj.is_a?(of_class) ? blk.call(obj) : obj
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end