mfynab 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/exe/mfynab +6 -0
- data/lib/mfynab/cli.rb +89 -0
- data/lib/mfynab/money_forward.rb +95 -0
- data/lib/mfynab/money_forward_data.rb +58 -0
- data/lib/mfynab/version.rb +5 -0
- data/lib/mfynab/ynab_transaction_importer.rb +156 -0
- metadata +109 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: df7f640ee7b19163bf41b79a560d4de67f8892d797aaaa8b06ab6ba321b4f1b6
|
4
|
+
data.tar.gz: 99fcfc93e353bcc951e14505220dc626ed51230d1e0489cb0a90fc4f4d9fc5f1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7a7dc9f3cba490a05b552209b1b496398ab754f6f0d1ce289894780de4fb2bf7eab5bbd1d580e7bddc9230139a442d302a8849754a3f328860e7fa73339810a5
|
7
|
+
data.tar.gz: 790422bd20f6e40c6ddca4471b2bcdc9ba2e948a50229e5b93d3f28cf5f5b60bb2946335d9e3058af5846137dd9d58f40dfe2a6a14d395bdb526a34151f679a5
|
data/exe/mfynab
ADDED
data/lib/mfynab/cli.rb
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "yaml"
|
4
|
+
require "mfynab/money_forward"
|
5
|
+
require "mfynab/money_forward_data"
|
6
|
+
require "mfynab/ynab_transaction_importer"
|
7
|
+
|
8
|
+
class CLI
|
9
|
+
def self.start(argv)
|
10
|
+
new(argv).start
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(argv)
|
14
|
+
@argv = argv
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
logger.info("Running...")
|
19
|
+
|
20
|
+
Dir.mktmpdir("mfynab") do |save_path|
|
21
|
+
money_forward.download_csv(
|
22
|
+
session_id: session_id,
|
23
|
+
path: save_path,
|
24
|
+
months: config["months_to_sync"],
|
25
|
+
)
|
26
|
+
|
27
|
+
data = MFYNAB::MoneyForwardData.new(logger: logger)
|
28
|
+
data.read_all_csv(save_path)
|
29
|
+
ynab_transaction_importer.run(data.to_h)
|
30
|
+
end
|
31
|
+
|
32
|
+
logger.info("Done!")
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
attr_reader :argv
|
38
|
+
|
39
|
+
def session_id
|
40
|
+
@_session_id ||= money_forward.get_session_id(
|
41
|
+
username: config["moneyforward_username"],
|
42
|
+
password: config["moneyforward_password"],
|
43
|
+
)
|
44
|
+
end
|
45
|
+
|
46
|
+
def ynab_transaction_importer
|
47
|
+
@_ynab_transaction_importer ||= MFYNAB::YnabTransactionImporter.new(
|
48
|
+
config["ynab_access_token"],
|
49
|
+
config["ynab_budget"],
|
50
|
+
config["accounts"],
|
51
|
+
logger: logger,
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
def money_forward
|
56
|
+
@_money_forward ||= MFYNAB::MoneyForward.new(logger: logger)
|
57
|
+
end
|
58
|
+
|
59
|
+
def config_file
|
60
|
+
raise "You need to pass a config file" if argv.empty?
|
61
|
+
|
62
|
+
argv[0]
|
63
|
+
end
|
64
|
+
|
65
|
+
def config
|
66
|
+
@_config ||= YAML
|
67
|
+
.load_file(config_file)
|
68
|
+
.values
|
69
|
+
.first
|
70
|
+
.merge(
|
71
|
+
"months_to_sync" => 3,
|
72
|
+
"ynab_access_token" => ENV.fetch("YNAB_ACCESS_TOKEN"),
|
73
|
+
"moneyforward_username" => ENV.fetch("MONEYFORWARD_USERNAME"),
|
74
|
+
"moneyforward_password" => ENV.fetch("MONEYFORWARD_PASSWORD"),
|
75
|
+
)
|
76
|
+
end
|
77
|
+
|
78
|
+
def logger
|
79
|
+
@_logger ||= Logger.new($stdout, level: logger_level)
|
80
|
+
end
|
81
|
+
|
82
|
+
def logger_level
|
83
|
+
if ENV.fetch("DEBUG", nil)
|
84
|
+
Logger::DEBUG
|
85
|
+
else
|
86
|
+
Logger::INFO
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "ferrum"
|
4
|
+
|
5
|
+
module MFYNAB
|
6
|
+
class MoneyForward
|
7
|
+
DEFAULT_BASE_URL = "https://moneyforward.com"
|
8
|
+
SIGNIN_PATH = "/sign_in"
|
9
|
+
CSV_PATH = "/cf/csv"
|
10
|
+
SESSION_COOKIE_NAME = "_moneybook_session"
|
11
|
+
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
|
12
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
13
|
+
|
14
|
+
def initialize(logger:, base_url: DEFAULT_BASE_URL)
|
15
|
+
@base_url = URI(base_url)
|
16
|
+
@logger = logger
|
17
|
+
end
|
18
|
+
|
19
|
+
def get_session_id(username:, password:)
|
20
|
+
with_ferrum do |browser|
|
21
|
+
browser.goto("#{base_url}#{SIGNIN_PATH}")
|
22
|
+
browser.at_css("input[name='mfid_user[email]']").focus.type(username)
|
23
|
+
browser.at_css("input[name='mfid_user[password]']").focus.type(password)
|
24
|
+
browser.at_css("button#submitto").click
|
25
|
+
|
26
|
+
# FIXME: use custom error class
|
27
|
+
raise "Login failed" unless browser.cookies[SESSION_COOKIE_NAME]
|
28
|
+
|
29
|
+
browser.cookies[SESSION_COOKIE_NAME].value
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def download_csv(session_id:, path:, months:)
|
34
|
+
month = Date.today
|
35
|
+
month -= month.day - 1 # First day of month
|
36
|
+
|
37
|
+
Net::HTTP.start(base_url.host, use_ssl: true) do |http|
|
38
|
+
http.response_body_encoding = Encoding::SJIS
|
39
|
+
|
40
|
+
months.times do
|
41
|
+
date_string = month.strftime("%Y-%m")
|
42
|
+
|
43
|
+
logger.info("Downloading CSV for #{date_string}")
|
44
|
+
|
45
|
+
# FIXME: I don't really need to save the CSV files to disk anymore.
|
46
|
+
# Maybe just return parsed CSV data?
|
47
|
+
File.open(File.join(path, "#{date_string}.csv"), "wb") do |file|
|
48
|
+
file << download_csv_string(date: month, session_id: session_id)
|
49
|
+
end
|
50
|
+
|
51
|
+
month = month.prev_month
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def download_csv_string(date:, session_id:)
|
57
|
+
Net::HTTP.start(base_url.host, use_ssl: true) do |http|
|
58
|
+
http.response_body_encoding = Encoding::SJIS
|
59
|
+
|
60
|
+
request = Net::HTTP::Get.new(
|
61
|
+
"#{CSV_PATH}?from=#{date.strftime('%Y/%m/%d')}",
|
62
|
+
{
|
63
|
+
"Cookie" => "#{SESSION_COOKIE_NAME}=#{session_id}",
|
64
|
+
"User-Agent" => USER_AGENT,
|
65
|
+
},
|
66
|
+
)
|
67
|
+
|
68
|
+
result = http.request(request)
|
69
|
+
raise unless result.is_a?(Net::HTTPSuccess)
|
70
|
+
raise unless result.body.valid_encoding?
|
71
|
+
|
72
|
+
result.body.encode(Encoding::UTF_8)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
attr_reader :base_url, :logger
|
79
|
+
|
80
|
+
def with_ferrum
|
81
|
+
browser = Ferrum::Browser.new(timeout: 30, headless: !ENV.key?("NO_HEADLESS"))
|
82
|
+
browser.headers.add({
|
83
|
+
"Accept-Language" => "en-US,en",
|
84
|
+
"User-Agent" => USER_AGENT,
|
85
|
+
})
|
86
|
+
yield browser
|
87
|
+
rescue StandardError
|
88
|
+
browser.screenshot(path: "screenshot.png")
|
89
|
+
logger.error("An error occurred and a screenshot was saved to ./screenshot.png")
|
90
|
+
raise
|
91
|
+
ensure
|
92
|
+
browser&.quit
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "csv"
|
4
|
+
|
5
|
+
module MFYNAB
|
6
|
+
class MoneyForwardData
|
7
|
+
HEADERS = {
|
8
|
+
include: "計算対象",
|
9
|
+
date: "日付",
|
10
|
+
content: "内容",
|
11
|
+
amount: "金額(円)",
|
12
|
+
account: "保有金融機関",
|
13
|
+
category: "大項目",
|
14
|
+
subcategory: "中項目",
|
15
|
+
memo: "メモ",
|
16
|
+
transfer: "振替",
|
17
|
+
id: "ID",
|
18
|
+
}.freeze
|
19
|
+
|
20
|
+
def initialize(logger:)
|
21
|
+
@transactions = {}
|
22
|
+
@logger = logger
|
23
|
+
end
|
24
|
+
|
25
|
+
def read_all_csv(path)
|
26
|
+
Dir[File.join(path, "*.csv")].each do |file|
|
27
|
+
read_csv(file)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def read_csv(csv_file)
|
32
|
+
logger.info("Reading #{csv_file}")
|
33
|
+
CSV.foreach(
|
34
|
+
csv_file,
|
35
|
+
headers: true,
|
36
|
+
converters: :all,
|
37
|
+
header_converters: csv_header_converters,
|
38
|
+
) do |row|
|
39
|
+
transactions[row["account"]] ||= []
|
40
|
+
transactions[row["account"]] << row.to_h
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_h
|
45
|
+
transactions
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
attr_reader :transactions, :logger
|
51
|
+
|
52
|
+
def csv_header_converters
|
53
|
+
lambda do |header|
|
54
|
+
HEADERS.key(header)&.to_s || header
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "ynab"
|
4
|
+
|
5
|
+
module MFYNAB
|
6
|
+
class YnabTransactionImporter
|
7
|
+
def initialize(api_key, budget_name, account_mappings, logger:)
|
8
|
+
@api_key = api_key
|
9
|
+
@budget_name = budget_name
|
10
|
+
@account_mappings = account_mappings
|
11
|
+
@logger = logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def run(mf_transactions) # rubocop:disable Metrics/AbcSize
|
15
|
+
mf_transactions.map do |mf_account, mf_account_transactions|
|
16
|
+
account = ynab_account_for_mf_account(mf_account)
|
17
|
+
next unless account
|
18
|
+
|
19
|
+
transactions = mf_account_transactions.map do |row|
|
20
|
+
convert_mf_transaction(row, account)
|
21
|
+
end
|
22
|
+
|
23
|
+
begin
|
24
|
+
logger.info("Importing #{transactions.size} transactions for #{account.name}")
|
25
|
+
ynab_transactions_api.create_transaction(budget.id, transactions: transactions)
|
26
|
+
rescue StandardError => e
|
27
|
+
logger.error("Error importing transactions for #{budget.name}. #{e} : #{e.detail}")
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
attr_reader :api_key, :budget_name, :account_mappings, :logger
|
35
|
+
|
36
|
+
def convert_mf_transaction(row, account)
|
37
|
+
{
|
38
|
+
account_id: account.id,
|
39
|
+
amount: row["amount"] * 1_000,
|
40
|
+
payee_name: row["content"][0, 100],
|
41
|
+
date: Date.strptime(row["date"], "%Y/%m/%d").strftime("%Y-%m-%d"),
|
42
|
+
cleared: "cleared",
|
43
|
+
memo: generate_memo_for(row),
|
44
|
+
import_id: generate_import_id_for(row),
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
def generate_memo_for(row)
|
49
|
+
category = row
|
50
|
+
.values_at("category", "subcategory")
|
51
|
+
.delete_if { _1.nil? || _1.empty? || _1 == "未分類" }
|
52
|
+
.join("/")
|
53
|
+
|
54
|
+
memo_parts = [
|
55
|
+
row["memo"], # prioritize memo if present, since it's user input
|
56
|
+
row["content"],
|
57
|
+
category,
|
58
|
+
]
|
59
|
+
|
60
|
+
memo_parts
|
61
|
+
.delete_if { _1.nil? || _1.empty? }
|
62
|
+
.join(" - ")
|
63
|
+
.slice(0, 200) # YNAB's API currently limits memo to 200 characters,
|
64
|
+
# even though YNAB itself allows longer memos. See:
|
65
|
+
# https://github.com/ynab/ynab-sdk-ruby/issues/77
|
66
|
+
end
|
67
|
+
|
68
|
+
# ⚠️ Be very careful when changing this method!
|
69
|
+
#
|
70
|
+
# A different import_id can cause MFYNAB to create duplicate transactions.
|
71
|
+
#
|
72
|
+
# import_id is scoped to an account in a budget, this means that:
|
73
|
+
# - if 2 transactions have the same import_id, but are in different
|
74
|
+
# accounts then they will be imported as 2 unrelated transactions.
|
75
|
+
# - if 2 transactions in the same account have the same import_id,
|
76
|
+
# then the second transaction will be ignored,
|
77
|
+
# even if it has a different date and/or amount.
|
78
|
+
# Note that it might be useful for the import_id to stay consistent even if
|
79
|
+
# the transaction amount changes, since a transaction that originally
|
80
|
+
# appeared as a low-amount authorization might be updated to its final
|
81
|
+
# amount later.
|
82
|
+
# (I don't know what that means for cleared and reconciled transactions...)
|
83
|
+
def generate_import_id_for(row)
|
84
|
+
# Uniquely identify transactions to avoid conflicts with other potential import scripts
|
85
|
+
# Note: I don't remember why I named it MFBY (why not MFYNAB or MFY?),
|
86
|
+
# but changing it now would require a lot of work in preventing import
|
87
|
+
# duplicates due to inconsistent import_id.
|
88
|
+
prefix = "MFBY:v1:"
|
89
|
+
|
90
|
+
max_length = 36 # YNAB API limit
|
91
|
+
id_max_length = 28 # this leaves 8 characters for the prefix
|
92
|
+
|
93
|
+
id = row["id"]
|
94
|
+
|
95
|
+
# Only hash if the ID would exceed YNAB's limit.
|
96
|
+
# This improves backwards compatibility with old import_ids.
|
97
|
+
if prefix.length + id.length > max_length
|
98
|
+
id = Digest::SHA256.hexdigest(id)[0, id_max_length]
|
99
|
+
end
|
100
|
+
|
101
|
+
prefix + id
|
102
|
+
end
|
103
|
+
|
104
|
+
def ynab_account_for_mf_account(mf_account_name)
|
105
|
+
matching_mapping = account_mappings.find do |mapping|
|
106
|
+
mapping["money_forward_name"] == mf_account_name
|
107
|
+
end
|
108
|
+
|
109
|
+
unless matching_mapping
|
110
|
+
logger.debug("Debug: no mapping for MoneyForward account #{mf_account_name}. Skipping...")
|
111
|
+
return
|
112
|
+
end
|
113
|
+
|
114
|
+
ynab_account_name = matching_mapping["ynab_name"]
|
115
|
+
|
116
|
+
ynab_account = accounts.find do |account|
|
117
|
+
account.name.include?(ynab_account_name)
|
118
|
+
end
|
119
|
+
raise "No YNAB account found with name #{ynab_account_name}." unless ynab_account
|
120
|
+
|
121
|
+
ynab_account
|
122
|
+
end
|
123
|
+
|
124
|
+
def ynab_transactions_api
|
125
|
+
@_ynab_transactions_api ||= YNAB::TransactionsApi.new(ynab_api_client)
|
126
|
+
end
|
127
|
+
|
128
|
+
def accounts
|
129
|
+
@_accounts ||= YNAB::AccountsApi
|
130
|
+
.new(ynab_api_client)
|
131
|
+
.get_accounts(budget.id)
|
132
|
+
.data
|
133
|
+
.accounts
|
134
|
+
end
|
135
|
+
|
136
|
+
def budget
|
137
|
+
@_budget ||= YNAB::BudgetsApi
|
138
|
+
.new(ynab_api_client)
|
139
|
+
.get_budgets
|
140
|
+
.data
|
141
|
+
.budgets
|
142
|
+
.find { _1.name == budget_name }
|
143
|
+
end
|
144
|
+
|
145
|
+
def ynab_api_client
|
146
|
+
@_ynab_api_client ||= YNAB::ApiClient.new(ynab_api_config)
|
147
|
+
end
|
148
|
+
|
149
|
+
def ynab_api_config
|
150
|
+
@_ynab_api_config = YNAB::Configuration.new.tap do |config|
|
151
|
+
config.access_token = api_key
|
152
|
+
config.debugging = false
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
metadata
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mfynab
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- David Stosik
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-07-26 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: csv
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: ferrum
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.15'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.15'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: psych
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: ynab
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.4'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.4'
|
69
|
+
description: Sync transaction history from MoneyForward to YNAB.
|
70
|
+
email:
|
71
|
+
- david.stosik+git-noreply@gmail.com
|
72
|
+
executables:
|
73
|
+
- mfynab
|
74
|
+
extensions: []
|
75
|
+
extra_rdoc_files: []
|
76
|
+
files:
|
77
|
+
- exe/mfynab
|
78
|
+
- lib/mfynab/cli.rb
|
79
|
+
- lib/mfynab/money_forward.rb
|
80
|
+
- lib/mfynab/money_forward_data.rb
|
81
|
+
- lib/mfynab/version.rb
|
82
|
+
- lib/mfynab/ynab_transaction_importer.rb
|
83
|
+
homepage: https://github.com/davidstosik/moneyforward_ynab
|
84
|
+
licenses:
|
85
|
+
- MIT
|
86
|
+
metadata:
|
87
|
+
homepage_uri: https://github.com/davidstosik/moneyforward_ynab
|
88
|
+
source_code_uri: https://github.com/davidstosik/moneyforward_ynab
|
89
|
+
rubygems_mfa_required: 'true'
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
require_paths:
|
93
|
+
- lib
|
94
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - ">="
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: 3.3.0
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
requirements: []
|
105
|
+
rubygems_version: 3.5.11
|
106
|
+
signing_key:
|
107
|
+
specification_version: 4
|
108
|
+
summary: Sync transaction history from MoneyForward to YNAB.
|
109
|
+
test_files: []
|