rvgp 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +8 -0
  3. data/.rubocop.yml +23 -0
  4. data/LICENSE +504 -0
  5. data/README.md +223 -0
  6. data/Rakefile +32 -0
  7. data/bin/rvgp +8 -0
  8. data/lib/rvgp/application/config.rb +159 -0
  9. data/lib/rvgp/application/descendant_registry.rb +122 -0
  10. data/lib/rvgp/application/status_output.rb +139 -0
  11. data/lib/rvgp/application.rb +170 -0
  12. data/lib/rvgp/base/command.rb +457 -0
  13. data/lib/rvgp/base/grid.rb +531 -0
  14. data/lib/rvgp/base/reader.rb +29 -0
  15. data/lib/rvgp/base/reconciler.rb +434 -0
  16. data/lib/rvgp/base/validation.rb +261 -0
  17. data/lib/rvgp/commands/cashflow.rb +160 -0
  18. data/lib/rvgp/commands/grid.rb +70 -0
  19. data/lib/rvgp/commands/ireconcile.rb +95 -0
  20. data/lib/rvgp/commands/new_project.rb +296 -0
  21. data/lib/rvgp/commands/plot.rb +41 -0
  22. data/lib/rvgp/commands/publish_gsheets.rb +83 -0
  23. data/lib/rvgp/commands/reconcile.rb +58 -0
  24. data/lib/rvgp/commands/rotate_year.rb +202 -0
  25. data/lib/rvgp/commands/validate_journal.rb +59 -0
  26. data/lib/rvgp/commands/validate_system.rb +44 -0
  27. data/lib/rvgp/commands.rb +160 -0
  28. data/lib/rvgp/dashboard.rb +252 -0
  29. data/lib/rvgp/fakers/fake_feed.rb +245 -0
  30. data/lib/rvgp/fakers/fake_journal.rb +57 -0
  31. data/lib/rvgp/fakers/fake_reconciler.rb +88 -0
  32. data/lib/rvgp/fakers/faker_helpers.rb +25 -0
  33. data/lib/rvgp/gem.rb +80 -0
  34. data/lib/rvgp/journal/commodity.rb +453 -0
  35. data/lib/rvgp/journal/complex_commodity.rb +214 -0
  36. data/lib/rvgp/journal/currency.rb +101 -0
  37. data/lib/rvgp/journal/journal.rb +141 -0
  38. data/lib/rvgp/journal/posting.rb +156 -0
  39. data/lib/rvgp/journal/pricer.rb +267 -0
  40. data/lib/rvgp/journal.rb +24 -0
  41. data/lib/rvgp/plot/gnuplot.rb +478 -0
  42. data/lib/rvgp/plot/google-drive/output_csv.rb +44 -0
  43. data/lib/rvgp/plot/google-drive/output_google_sheets.rb +434 -0
  44. data/lib/rvgp/plot/google-drive/sheet.rb +67 -0
  45. data/lib/rvgp/plot.rb +293 -0
  46. data/lib/rvgp/pta/hledger.rb +237 -0
  47. data/lib/rvgp/pta/ledger.rb +308 -0
  48. data/lib/rvgp/pta.rb +311 -0
  49. data/lib/rvgp/reconcilers/csv_reconciler.rb +424 -0
  50. data/lib/rvgp/reconcilers/journal_reconciler.rb +41 -0
  51. data/lib/rvgp/reconcilers/shorthand/finance_gem_hacks.rb +48 -0
  52. data/lib/rvgp/reconcilers/shorthand/international_atm.rb +152 -0
  53. data/lib/rvgp/reconcilers/shorthand/investment.rb +144 -0
  54. data/lib/rvgp/reconcilers/shorthand/mortgage.rb +195 -0
  55. data/lib/rvgp/utilities/grid_query.rb +190 -0
  56. data/lib/rvgp/utilities/yaml.rb +131 -0
  57. data/lib/rvgp/utilities.rb +44 -0
  58. data/lib/rvgp/validations/balance_validation.rb +68 -0
  59. data/lib/rvgp/validations/duplicate_tags_validation.rb +48 -0
  60. data/lib/rvgp/validations/uncategorized_validation.rb +15 -0
  61. data/lib/rvgp.rb +66 -0
  62. data/resources/README.MD/2022-cashflow-google.png +0 -0
  63. data/resources/README.MD/2022-cashflow.png +0 -0
  64. data/resources/README.MD/all-wealth-growth-google.png +0 -0
  65. data/resources/README.MD/all-wealth-growth.png +0 -0
  66. data/resources/gnuplot/default.yml +80 -0
  67. data/resources/i18n/en.yml +192 -0
  68. data/resources/iso-4217-currencies.json +171 -0
  69. data/resources/skel/Rakefile +5 -0
  70. data/resources/skel/app/grids/cashflow_grid.rb +27 -0
  71. data/resources/skel/app/grids/monthly_income_and_expenses_grid.rb +25 -0
  72. data/resources/skel/app/grids/wealth_growth_grid.rb +35 -0
  73. data/resources/skel/app/plots/cashflow.yml +33 -0
  74. data/resources/skel/app/plots/monthly-income-and-expenses.yml +17 -0
  75. data/resources/skel/app/plots/wealth-growth.yml +20 -0
  76. data/resources/skel/config/csv-format-acme-checking.yml +9 -0
  77. data/resources/skel/config/google-secrets.yml +5 -0
  78. data/resources/skel/config/rvgp.yml +0 -0
  79. data/resources/skel/journals/prices.db +0 -0
  80. data/rvgp.gemspec +6 -0
  81. data/test/assets/ledger_total_monthly_liabilities_with_empty.xml +383 -0
  82. data/test/assets/ledger_total_monthly_liabilities_with_empty2.xml +428 -0
  83. data/test/test_command_base.rb +61 -0
  84. data/test/test_commodity.rb +270 -0
  85. data/test/test_csv_reconciler.rb +60 -0
  86. data/test/test_currency.rb +24 -0
  87. data/test/test_fake_feed.rb +228 -0
  88. data/test/test_fake_journal.rb +98 -0
  89. data/test/test_fake_reconciler.rb +60 -0
  90. data/test/test_journal_parse.rb +545 -0
  91. data/test/test_ledger.rb +102 -0
  92. data/test/test_plot.rb +133 -0
  93. data/test/test_posting.rb +50 -0
  94. data/test/test_pricer.rb +139 -0
  95. data/test/test_pta_adapter.rb +575 -0
  96. data/test/test_utilities.rb +45 -0
  97. metadata +268 -0
@@ -0,0 +1,144 @@
1
+ # frozen_string_literal: true
2
+
3
+ module RVGP
4
+ module Reconcilers
5
+ # This module contains the built-in Shorthand classes that ship with RVGP. For more details on this
6
+ # feature, see the 'Shorthand' section of the {RVGP::Reconcilers} module.
7
+ module Shorthand
8
+ # This reconciler module will automatically allocate the proceeds (or losses) from a stock sale.
9
+ # This module will allocate capital gains or losses, given a symbol, amount, and price.
10
+ #
11
+ # The module parameters we support are:
12
+ # - **symbol** [String] - A commodity or currency code, that represents the purchased asset
13
+ # - **amount** [Integer] - The amount of :symbol that was purchased, or if negative, sold.
14
+ # - **price** [Commodity] - A unit price, for the symbol. This field should be delimited if :total is omitted.
15
+ # - **total** [Commodity] - A lot price, the symbol. This represents the net purchase price, which, would be
16
+ # divided by the amount, in order to arrive at a unit price. This field should be delimited if :price is
17
+ # omitted.
18
+ # - **capital_gains** [Commodity] - The amount of the total, to allocate to a capital gains account. Presumably
19
+ # for tax reporting.
20
+ # - **gains_account** [String] - The account name to allocate capital gains to.
21
+ #
22
+ # # Example
23
+ # Here's how this module might be used in your reconciler:
24
+ # ```
25
+ # ...
26
+ # - match: /Acme Stonk Exchange/
27
+ # to_shorthand: Investment
28
+ # shorthand_params:
29
+ # symbol: VOO
30
+ # price: "$ 400.00"
31
+ # amount: "-1000"
32
+ # capital_gains: "$ -100000.00"
33
+ # gains_account: Personal:Income:AcmeExchange:VOO
34
+ # ...
35
+ # ```
36
+ # And here's how that will reconcile, in your build:
37
+ # ```
38
+ # ...
39
+ # 2023-06-01 Acme Stonk Exchange ACH CREDIT 123456 Yukihiro Matsumoto
40
+ # Personal:Assets -1000 VOO @@ $ 400000.00
41
+ # Personal:Income:AcmeExchange:VOO $ 100000.00
42
+ # Personal:Assets:AcmeChecking
43
+ # ...
44
+ # ```
45
+ #
46
+ class Investment
47
+ # @!visibility private
48
+ attr_reader :tag, :symbol, :price, :amount, :total, :capital_gains,
49
+ :remainder_amount, :remainder_account, :targets, :is_sell,
50
+ :to, :gains_account
51
+
52
+ def initialize(rule)
53
+ @tag = rule[:tag]
54
+ @targets = rule[:targets]
55
+ @to = rule[:to] || 'Personal:Assets'
56
+
57
+ if rule.key? :shorthand_params
58
+ @symbol = rule[:shorthand_params][:symbol]
59
+ @amount = rule[:shorthand_params][:amount]
60
+ @gains_account = rule[:shorthand_params][:gains_account]
61
+
62
+ %w[price total capital_gains].each do |key|
63
+ if rule[:shorthand_params].key? key.to_sym
64
+ instance_variable_set "@#{key}".to_sym, rule[:shorthand_params][key.to_sym].to_commodity
65
+ end
66
+ end
67
+ end
68
+
69
+ unless [symbol, amount].all?
70
+ raise StandardError, format('Investment at line:%s missing fields', rule[:line].inspect)
71
+ end
72
+
73
+ @is_sell = (amount.to_f <= 0)
74
+
75
+ # I mostly just think this doesn't make any sense... I guess if we took a
76
+ # loss...
77
+ raise StandardError, format('Unimplemented %s', rule.inspect) if capital_gains && !is_sell
78
+
79
+ if (gains_account.nil? || capital_gains.nil?) && is_sell
80
+ raise StandardError, format('Investment at line:%s missing gains_account', rule.inspect)
81
+ end
82
+
83
+ unless total || price
84
+ raise StandardError, format('Investment at line:%s missing an price or total', rule[:line].inspect)
85
+ end
86
+
87
+ if total && price
88
+ raise StandardError, format('Investment at line:%s specified both price and total', rule[:line].inspect)
89
+ end
90
+ end
91
+
92
+ # @!visibility private
93
+ def to_tx(from_posting)
94
+ income_targets = []
95
+
96
+ # NOTE: I pulled most of this from: https://hledger.org/investments.html
97
+ if is_sell && capital_gains
98
+ # NOTE: I'm not positive about this .abs....
99
+ cost_basis = (total || (price * amount.to_f.abs)) - capital_gains
100
+
101
+ income_targets << { to: to,
102
+ complex_commodity: RVGP::Journal::ComplexCommodity.new(
103
+ left: [amount, symbol].join(' ').to_commodity,
104
+ operation: :per_lot,
105
+ right: cost_basis
106
+ ) }
107
+
108
+ income_targets << { to: gains_account, commodity: capital_gains.dup.invert! } if capital_gains
109
+ else
110
+ income_targets << { to: to,
111
+ complex_commodity: RVGP::Journal::ComplexCommodity.new(
112
+ left: [amount, symbol].join(' ').to_commodity,
113
+ operation: (price ? :per_unit : :per_lot),
114
+ right: price || total
115
+ ) }
116
+ end
117
+
118
+ if targets
119
+ income_targets += targets.map do |t|
120
+ ret = { to: t[:to] }
121
+ if t.key? :amount
122
+ # TODO: I think there's a bug here, in that amounts with commodities, won't parse...
123
+ ret[:commodity] = RVGP::Journal::Commodity.from_symbol_and_amount t[:currency] || '$', t[:amount].to_s
124
+ end
125
+
126
+ if t.key? :complex_commodity
127
+ ret[:complex_commodity] = RVGP::Journal::ComplexCommodity.from_s t[:complex_commodity]
128
+ end
129
+
130
+ ret
131
+ end
132
+ end
133
+
134
+ RVGP::Base::Reconciler::Posting.new from_posting.line_number,
135
+ date: from_posting.date,
136
+ description: from_posting.description,
137
+ from: from_posting.from,
138
+ tags: from_posting.tags,
139
+ targets: income_targets
140
+ end
141
+ end
142
+ end
143
+ end
144
+ end
@@ -0,0 +1,195 @@
1
+ # frozen_string_literal: true
2
+
3
+ gem 'finance'
4
+ require 'finance'
5
+ require_relative './finance_gem_hacks'
6
+
7
+ module RVGP
8
+ module Reconcilers
9
+ module Shorthand
10
+ # This reconciler module will automatically allocate the the escrow, principal, and interest components of a
11
+ # mortage debit, into constituent accounts. The amounts of each, are automatically calculated, based on the loan
12
+ # terms, and taking the residual leftover, into a escrow account, presumably for taxes and insurance to be paid by
13
+ # the mortgage provider.
14
+ #
15
+ # Its important to note, that a single reconciler rule will match every mortgage payment encountered. And, that
16
+ # each of these payments will generate four transactions in the output file:
17
+ # - The initial payment, which, will be transferred to an :intermediary_account
18
+ # - A principal payment, which, will be debited from the :intermediary account, to the :payee_principal
19
+ # - An interest payment, which, will be debited from the :intermediary account, to the :payee_interest
20
+ # - An escrow payment, which, will be debited from the :intermediary account, to the :escrow_account
21
+ # You can see details of this expansion under the example section below.
22
+ #
23
+ # With regards to the :escrow_account, It's likely that you'll want to either (choose one):
24
+ # - Manually transcribe debits from the escrow account, to escrow payees, in your project's
25
+ # ./journal/property-name-escrow.journal, based on when your mortgage provider alerts you to these payments.
26
+ # - Download a csv from your mortgage provider, of your escrow account (if they offer one), and define a
27
+ # reconciler to allocate escrow payments.
28
+ #
29
+ # The module parameters we support are:
30
+ # - **label** [String] - This is a prefix, used in the description of Principal, Interest, and Escrow transactions
31
+ # - **principal** [Commodity] - The mortgage principal
32
+ # - **rate** [Float] - The mortgage rate
33
+ # - **payee_principal** [String] - The account to ascribe principal payments to
34
+ # - **payee_interest** [String] - The account to ascribe interest payments to
35
+ # - **escrow_account** [String] - Te account to ascribe escrow payments to
36
+ # - **intermediary_account** [String] - The account to ascribe intermediary payments to, from the source account,
37
+ # before being assigned to principal, interest, and escrow break-outs.
38
+ # - **start_at_installment_number** [Integer] - The installment number, of the first matching transaction,
39
+ # encountered by this module. Year one of a mortgage would start at zero. Subsequent annual reconcilers would
40
+ # be expected to define an installment number from which calculations can automatically pick-up the work
41
+ # from years prior.
42
+ # - **additional_payments** [Array<Hash>] - Any additional payments, to apply to the principal, can be listed
43
+ # here. This field is expected to be an array of hashes, which, are composed of the following fields:
44
+ # - **before_installment** [Integer] - The payment number, before which, this :amount should apply
45
+ # - **amount** [Float] - A float that will be deducted from the principal. No commodity is necessary to
46
+ # delineate, as we assume the same commodity as the :principle.
47
+ # - **override_payments** [Array<Hash>] - I can't explain why this is necessary. But, it seems that the interest
48
+ # calculations used by some mortgage providers ... aren't accurate. This happened to me, at least. The
49
+ # calculation being used was off by a penny, on a single installment. And, I didn't care enough to call the
50
+ # institution and figure out why. So, I added this feature, to allow an override of the automatic calculation,
51
+ # with the amount provided. This field is expected to be an array of hashes, which are composed of the following
52
+ # fields:
53
+ # - **at_installment** [Integer] - The payment number to assert the :interest value.
54
+ # - **interest** [Float] - The amount of the interest calculation. No commodity is neccessary to delineate, as
55
+ # we assume the same commodity as the :principle.
56
+ #
57
+ # # Example
58
+ # Here's how this module might be used in your reconciler:
59
+ # ```
60
+ # ...
61
+ # - match: /AcmeFinance Servicing/
62
+ # to_shorthand: Mortgage
63
+ # shorthand_params:
64
+ # label: 1-8-1 Yurakucho Dori Mortgage
65
+ # intermediary_account: Personal:Expenses:Banking:MortgagePayment:181Yurakucho
66
+ # payee_principal: Personal:Liabilities:Mortgage:181Yurakucho
67
+ # payee_interest: Personal:Expenses:Banking:Interest:181Yurakucho
68
+ # escrow_account: Personal:Assets:AcmeFinance:181YurakuchoEscrow
69
+ # principal: 260000.00
70
+ # rate: 0.0499
71
+ # start_at_installment_number: 62
72
+ # ...
73
+ # ```
74
+ # And here's how that will reconcile one of your payments, in your build:
75
+ # ```
76
+ # ...
77
+ # 2023-01-03 AcmeFinance Servicing MTG PYMT 012345 Yukihiro Matsumoto
78
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho $ 3093.67
79
+ # Personal:Assets:AcmeBank:Checking
80
+ #
81
+ # 2023-01-03 1-8-1 Yurakucho Dori Mortgage (#61) Principal
82
+ # Personal:Liabilities:Mortgage:181Yurakucho $ 403.14
83
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho
84
+ #
85
+ # 2023-01-03 1-8-1 Yurakucho Dori Mortgage (#61) Interest
86
+ # Personal:Expenses:Banking:Interest:181Yurakucho $ 991.01
87
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho
88
+ #
89
+ # 2023-01-03 1-8-1 Yurakucho Dori Mortgage (#61) Escrow
90
+ # Personal:Assets:AcmeFinance:181YurakuchoEscrow $ 1699.52
91
+ # Personal:Expenses:Banking:MortgagePayment:181Yurakucho
92
+ # ...
93
+ # ```
94
+ # Note that you'll have an automatically calculated reconcilation for each payment you
95
+ # make, during the year. A single reconciler rule, will take care of reconciling every
96
+ # payment, automatically.
97
+ class Mortgage
98
+ # @!visibility private
99
+ attr_accessor :principal, :rate, :start_at_installment_number,
100
+ :additional_payments, :amortization, :payee_principal, :payee_interest,
101
+ :intermediary_account, :currency, :label, :escrow_account, :override_payments
102
+
103
+ # @!visibility private
104
+ def initialize(rule)
105
+ @label = rule[:shorthand_params][:label]
106
+ @currency = rule[:currency] || '$'
107
+ @principal = rule[:shorthand_params][:principal].to_commodity
108
+ @rate = rule[:shorthand_params][:rate]
109
+ @payee_principal = rule[:shorthand_params][:payee_principal]
110
+ @payee_interest = rule[:shorthand_params][:payee_interest]
111
+ @intermediary_account = rule[:shorthand_params][:intermediary_account]
112
+ @escrow_account = rule[:shorthand_params][:escrow_account]
113
+ @start_at_installment_number = rule[:shorthand_params][:start_at_installment_number]
114
+ @additional_payments = rule[:shorthand_params][:additional_payments]
115
+ @override_payments = {}
116
+ if rule[:shorthand_params].key? :override_payments
117
+ rule[:shorthand_params][:override_payments].each do |override|
118
+ unless %i[at_installment interest].all? { |k| override.key? k }
119
+ raise StandardError, format('Invalid Payment Override : %s', override)
120
+ end
121
+
122
+ @override_payments[ override[:at_installment] ] = {
123
+ interest: RVGP::Journal::Commodity.from_symbol_and_amount(currency, override[:interest])
124
+ }
125
+ end
126
+ end
127
+
128
+ unless [principal, rate, payee_principal, payee_interest, intermediary_account, escrow_account, label].all?
129
+ raise StandardError, format('Mortgage at line:%d missing fields', rule[:line])
130
+ end
131
+
132
+ fr = Finance::Rate.new rate, :apr, duration: 360
133
+ @amortization = principal.to_f.amortize(fr) do |period, amortization|
134
+ additional_payments&.each do |ap|
135
+ if period == ap[:before_installment]
136
+ amortization.balance = amortization.balance - DecNum.new(ap[:amount].to_s)
137
+ end
138
+ end
139
+ end
140
+
141
+ @installment_i = start_at_installment_number ? (start_at_installment_number - 1) : 0
142
+ end
143
+
144
+ # @!visibility private
145
+ def to_tx(from_posting)
146
+ payment = RVGP::Journal::Commodity.from_symbol_and_amount(currency, amortization.payments[@installment_i]).abs
147
+ interest = RVGP::Journal::Commodity.from_symbol_and_amount(currency, amortization.interest[@installment_i])
148
+
149
+ interest = @override_payments[@installment_i][:interest] if @override_payments.key? @installment_i
150
+
151
+ principal = payment - interest
152
+ escrow = from_posting.commodity.abs - payment
153
+ total = principal + interest + escrow
154
+
155
+ @installment_i += 1
156
+
157
+ intermediary_opts = { date: from_posting.date, from: intermediary_account, tags: from_posting.tags }
158
+
159
+ [RVGP::Base::Reconciler::Posting.new(from_posting.line_number,
160
+ date: from_posting.date,
161
+ description: from_posting.description,
162
+ from: from_posting.from,
163
+ tags: from_posting.tags,
164
+ targets: [to: intermediary_account, commodity: total]),
165
+ # Principal:
166
+ RVGP::Base::Reconciler::Posting.new(
167
+ from_posting.line_number,
168
+ intermediary_opts.merge({ description: format('%<label>s (#%<num>d) Principal',
169
+ label: label,
170
+ num: @installment_i - 1),
171
+ targets: [{ to: payee_principal, commodity: principal }] })
172
+ ),
173
+
174
+ # Interest:
175
+ RVGP::Base::Reconciler::Posting.new(
176
+ from_posting.line_number,
177
+ intermediary_opts.merge({ description: format('%<label>s (#%<num>d) Interest',
178
+ label: label,
179
+ num: @installment_i - 1),
180
+ targets: [{ to: payee_interest, commodity: interest }] })
181
+ ),
182
+
183
+ # Escrow:
184
+ RVGP::Base::Reconciler::Posting.new(
185
+ from_posting.line_number,
186
+ intermediary_opts.merge({ description: format('%<label>s (#%<num>d) Escrow',
187
+ label: label,
188
+ num: @installment_i - 1),
189
+ targets: [{ to: escrow_account, commodity: escrow }] })
190
+ )]
191
+ end
192
+ end
193
+ end
194
+ end
195
+ end
@@ -0,0 +1,190 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'csv'
4
+
5
+ module RVGP
6
+ module Utilities
7
+ # This class provides a number of utility functions to query, and merge, grids
8
+ # that have been built. The primary purpose of these tools, is to decrease
9
+ # the overhead that would otherwise exist, if we were to operate directly on
10
+ # the pta output, every time we referenced this data. As well as to maintain
11
+ # auditability for this data, in the project build.
12
+ # @attr_reader [Array<String>] headers The first row of a grid, without the keystone included
13
+ # @attr_reader [Hash<String,Hash<String,Object>>] data A hash whose key is the series name, whose value is a hash of
14
+ # headername to value pairs
15
+ # @attr_reader [String] keystone The cell in the upper-left of the grid. Often this cell is used as a superordinate
16
+ # label for the series below it.
17
+ class GridQuery
18
+ # NOTE: I'm not exactly sure what this class wants to be just yet...
19
+ # Let's see if we end up using it for graphing... It might just be a 'Spreadsheet'
20
+ # and we may want/need to move the summary columns into here
21
+ attr_reader :headers, :data, :keystone
22
+
23
+ # Setup a GridQuery
24
+ # @param [Array<String>] from_files An array of paths, to grid files
25
+ # @param [Hash] options Additional options
26
+ # @option options [String] keystone see {RVGP::Utilities::GridQuery#keystone}
27
+ # @option options [Proc] store_cell This proc is provided one parameter, a data cell. The return value of this
28
+ # proc is then stored in :data, in lieu of the parameter.
29
+ # @option options [Proc] select_columns This proc is provided two parameters, a header (String), and the cells
30
+ # underneath it in the grid (Array<Object>). If this proc returns true,
31
+ # that column is stored in :data. If this proc returns false, that column
32
+ # is not retained in this query.
33
+ # @option options [Proc] select_rows This proc is provided two parameters, a series_name (String), and the cells
34
+ # to the right of it in the grid (Array<Object>). If this proc returns true,
35
+ # that column is stored in :data. If this proc returns false, that column
36
+ # is not retained in this query.
37
+ def initialize(from_files, options = {})
38
+ @headers = []
39
+ @data = {}
40
+
41
+ from_files.each do |file|
42
+ csv = CSV.open file, 'r', headers: true
43
+ rows = csv.read
44
+ headers = csv.headers
45
+
46
+ # We assume the first column of the row, is the series name
47
+ @keystone = headers.shift
48
+
49
+ if options[:select_columns]
50
+ selected_headers = headers.select do |header|
51
+ options[:select_columns].call(header, rows.map { |row| row[header] })
52
+ end
53
+ end
54
+ selected_headers ||= headers
55
+
56
+ add_columns selected_headers
57
+
58
+ rows.each do |row|
59
+ series_data = row.to_h
60
+ series_name = series_data.delete @keystone
61
+ if options.key? :store_cell
62
+ series_data.each do |col, cell|
63
+ next unless !options.key?(:select_columns) || selected_headers.include?(col)
64
+
65
+ series_data[col] = options[:store_cell].call cell
66
+ end
67
+ end
68
+
69
+ if !options.key?(:select_rows) || options[:select_rows].call(series_name, series_data)
70
+ add_data series_name, series_data
71
+ end
72
+ end
73
+ end
74
+
75
+ # This needs to be assigned after we've processed the csv's
76
+ @keystone = options[:keystone] if options.key? :keystone
77
+ end
78
+
79
+ # Returns the results of a query.
80
+ # @param [Hash] opts Additional options
81
+ # @option opts [Proc] sort_cols_by Columns are sorted by the value returned by this proc. This proc is provided
82
+ # the grid :headers, and this proc is handled according to the rules of
83
+ # Enumerable#sort_by.
84
+ # @option opts [Proc] sort_rows_by Rows are sorted by the value returned by this proc. This proc is provided
85
+ # each row in this grid, with the series_name of the row at position zero.
86
+ # This proc is handled according to the rules of Enumerable#sort_by.
87
+ # @option opts [Integer] truncate_columns Restrict the total number of columns to a maximum this number
88
+ # @option opts [Integer] truncate_rows Restrict the total number of rows to a maximum this number
89
+ # @option opts [String] truncate_remainder_row If the columns are truncated, the 'chopped' columns are summed into
90
+ # 'the last column'. The value of this option, is used as the label
91
+ # for that column. Candidly... this feature makes a lot of
92
+ # assumptions, and probably needs a bit more refinement.
93
+ # @option opts [TrueClass,FalseClass] switch_rows_columns Rotates the grid, so that series labels and headers are
94
+ # swapped. All data is preserved under/adjacent to the same series label and
95
+ # header, in the returned grid. Google offers this option in its GUI, but
96
+ # doesn't seem to support it via the API. So, we can just do that ourselves
97
+ # here.
98
+ # @return [Array<Array<Object>>] A grid of cells, composed of the provided grids, and transformed by the given
99
+ # parameters.
100
+ def to_grid(opts = {})
101
+ # First we'll collect the header row, possibly sorted :
102
+ if opts[:sort_cols_by]
103
+ # We collect the data under the column, and feed that into sort_cols_by
104
+ grid_columns = headers.map do |col|
105
+ [col] + data.map { |_, values| values[col] }
106
+ end.sort_by(&opts[:sort_cols_by]).map(&:first)
107
+ end
108
+ grid_columns ||= headers
109
+
110
+ # Then we collect the non-header rows:
111
+ grid = data.map { |series, values| [series] + grid_columns.map { |col| values[col] } }
112
+
113
+ # Sort those rows, if necessesary:
114
+ grid.sort_by!(&opts[:sort_rows_by]) if opts[:sort_rows_by]
115
+
116
+ # Affix the header row to the top of the grid. Now it's assembled.
117
+ grid.unshift [keystone] + grid_columns
118
+
119
+ # Do we Truncate Rows?
120
+ if opts[:truncate_rows] && grid.length > opts[:truncate_rows]
121
+ # We can only have about 26 columns on Google. And, since we're (sometimes)
122
+ # transposing rows to columns, we have to truncate the rows.
123
+
124
+ # NOTE: The 1 is for the truncate_remainder_row, the 'overflow' column
125
+ chop_length = grid.length - opts[:truncate_rows]
126
+
127
+ if chop_length.positive?
128
+ chopped_rows = grid.pop chop_length
129
+ truncate_row = chopped_rows.inject([]) do |sum, row|
130
+ # Starting at the second cell (the first is the series name) merge
131
+ # the contents of the current row, into the collection cell.
132
+ row[1...].each_with_index.map do |cell, i|
133
+ # This rigamarole is mostly to help prevent type issues...
134
+ if cell
135
+ sum[i] ? sum[i] + cell : cell
136
+ else
137
+ sum[i]
138
+ end
139
+ end
140
+ end
141
+
142
+ grid << ([(opts[:truncate_remainder_row]) || 'Other'] + truncate_row)
143
+ end
144
+ end
145
+
146
+ # Do we Truncate Columns?
147
+ if opts[:truncate_columns] && grid[0].length > opts[:truncate_columns]
148
+ # Go through each row, pop off the excess cells, and sum them onto the end
149
+ grid.each_with_index do |row, i|
150
+ # The plus one is to make room for the 'Other' column
151
+ chop_length = row.length - opts[:truncate_columns] + 1
152
+
153
+ chopped_cols = row.pop chop_length
154
+ truncate_cell = if i.zero?
155
+ opts[:truncate_remainder_row] || 'Other'
156
+ else
157
+ chopped_cols.all?(&:nil?) ? nil : chopped_cols.compact.sum
158
+ end
159
+
160
+ row << truncate_cell
161
+ end
162
+ end
163
+
164
+ # Google offers this option in its GUI, but doesn't seem to support it via
165
+ # the API. So, we can just do that ourselves:
166
+ grid = 0.upto(grid[0].length - 1).map { |i| grid.map { |row| row[i] } } if opts[:switch_rows_columns]
167
+
168
+ grid
169
+ end
170
+
171
+ private
172
+
173
+ def add_columns(headers)
174
+ @headers += headers.reject { |header| @headers.include? header }
175
+ end
176
+
177
+ # NOTE: that we're assuming value here, is always a commodity. Not sure about
178
+ # that over time.
179
+ def add_data(series_name, colname_to_value)
180
+ @data[series_name] ||= {}
181
+
182
+ colname_to_value.each do |colname, value|
183
+ raise StandardError, 'Unimplemented. How to merge?' if @data[series_name].key? colname
184
+
185
+ @data[series_name][colname] = value
186
+ end
187
+ end
188
+ end
189
+ end
190
+ end
@@ -0,0 +1,131 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'psych'
4
+ require 'pathname'
5
+
6
+ Psych.add_builtin_type('proc') { |_, val| RVGP::Utilities::Yaml::PsychProc.new val }
7
+ Psych.add_builtin_type('include') { |_, val| RVGP::Utilities::Yaml::PsychInclude.new val }
8
+
9
+ module RVGP
10
+ module Utilities
11
+ # This class wraps the Psych library, and adds functionality we need, to parse
12
+ # yaml files.
13
+ # We mostly added this class because the Psych.add_builtin_type wasn't able to
14
+ # contain state outside it's return values. (which we need, in order to
15
+ # track include dependencies)
16
+ class Yaml
17
+ # This class is needed, in order to capture the dependencies needed to
18
+ # properly preserve uptodate status in rake builds
19
+ class PsychInclude
20
+ attr_reader :path
21
+
22
+ # Declare a proc, given the provided proc_as_string code
23
+ # @param [String] path A relative or absolute path, to include, in the place of this line
24
+ def initialize(path)
25
+ @path = path
26
+ end
27
+
28
+ # The contents of this include target.
29
+ # @param [Array<String>] include_paths A relative or absolute path, to include, in case our path is relative
30
+ # and we need to load it from a relative location. These paths are
31
+ # scanned for the include, in the relative order of their place in the
32
+ # array.
33
+ # @return [Hash] The contents of the target of this include. Keys are symbolized, and permitted_classes include
34
+ # Date, and Symbol
35
+ def contents(include_paths)
36
+ content_path = path if Pathname.new(path).absolute?
37
+ content_path ||= include_paths.map { |p| [p.chomp('/'), path].join('/') }.find do |p|
38
+ File.readable? p
39
+ end
40
+
41
+ unless content_path
42
+ raise StandardError, format('Unable to find %<path>s in any of the provided paths: %<paths>s',
43
+ path: path.inspect, paths: include_paths.inspect)
44
+ end
45
+
46
+ Psych.safe_load_file content_path, symbolize_names: true, permitted_classes: [Date, Symbol]
47
+ end
48
+ end
49
+
50
+ # This class wraps proc types, in the yaml, and offers us the ability to execute
51
+ # the code in these blocks.
52
+ class PsychProc
53
+ # Declare a proc, given the provided proc_as_string code
54
+ # @param [String] proc_as_string Ruby code, that this block will call.
55
+ def initialize(proc_as_string)
56
+ @block = eval(format('proc { %s }', proc_as_string)) # rubocop:disable Security/Eval
57
+ end
58
+
59
+ # Execute the block, using the provided params as locals
60
+ # NOTE: We expect symbol to value here
61
+ # @param params [Hash<Symbol, Object>] local values, available in the proc context. Note that keys
62
+ # are expected to be symbols.
63
+ # @return [Object] Whatever the proc returns
64
+ def call(params = {})
65
+ # params, here, act as instance variables, since we don't support named
66
+ # params in the provided string, the way you typically would.
67
+ @params = params
68
+ @block.call
69
+ end
70
+
71
+ # @!visibility private
72
+ def respond_to_missing?(name, _include_private = false)
73
+ @params&.key?(name.to_sym)
74
+ end
75
+
76
+ # @!visibility private
77
+ def method_missing(name)
78
+ respond_to_missing?(name) ? @params[name] : super(name)
79
+ end
80
+ end
81
+
82
+ attr_reader :path, :include_paths, :dependencies
83
+
84
+ # @param [String] path The full path to the yaml file you wish to parse
85
+ # @param [Array<String>] include_paths An array of directories, to search, when locating the target of
86
+ # a "!!include" line
87
+ def initialize(path, include_paths = nil)
88
+ @path = path
89
+ @dependencies = []
90
+ @include_paths = Array(include_paths || File.expand_path(File.dirname(path)))
91
+
92
+ vanilla_yaml = Psych.safe_load_file(path, symbolize_names: true, permitted_classes: [Date, Symbol])
93
+ @yaml = replace_each_in_yaml(vanilla_yaml, PsychInclude) do |psych_inc|
94
+ @dependencies << psych_inc.path
95
+ psych_inc.contents @include_paths
96
+ end
97
+ end
98
+
99
+ # Return the specified attribute, in this yaml file
100
+ # @param [String] attr The attribute you're looking for
101
+ # @return [Object] The value of the the provided attribute
102
+ def [](attr)
103
+ @yaml&.[](attr)
104
+ end
105
+
106
+ # Returns true or false, depending on whether the attribute you're looking for, exists in this
107
+ # yaml file.
108
+ # @param [String] attr The attribute you're looking for
109
+ # @return [TrueClass,FalseClass] Whether the key is defined in this file
110
+ def key?(attr)
111
+ @yaml&.key? attr
112
+ end
113
+
114
+ alias has_key? key?
115
+
116
+ private
117
+
118
+ # This is kind of a goofy function, but, it works
119
+ def replace_each_in_yaml(obj, of_class, &blk)
120
+ case obj
121
+ when Hash
122
+ obj.transform_values { |v| replace_each_in_yaml v, of_class, &blk }
123
+ when Array
124
+ obj.collect { |v| replace_each_in_yaml(v, of_class, &blk) }
125
+ else
126
+ obj.is_a?(of_class) ? blk.call(obj) : obj
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end