mfynab 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/exe/mfynab +6 -0
- data/lib/mfynab/cli.rb +89 -0
- data/lib/mfynab/money_forward.rb +95 -0
- data/lib/mfynab/money_forward_data.rb +58 -0
- data/lib/mfynab/version.rb +5 -0
- data/lib/mfynab/ynab_transaction_importer.rb +156 -0
- metadata +109 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: df7f640ee7b19163bf41b79a560d4de67f8892d797aaaa8b06ab6ba321b4f1b6
|
4
|
+
data.tar.gz: 99fcfc93e353bcc951e14505220dc626ed51230d1e0489cb0a90fc4f4d9fc5f1
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7a7dc9f3cba490a05b552209b1b496398ab754f6f0d1ce289894780de4fb2bf7eab5bbd1d580e7bddc9230139a442d302a8849754a3f328860e7fa73339810a5
|
7
|
+
data.tar.gz: 790422bd20f6e40c6ddca4471b2bcdc9ba2e948a50229e5b93d3f28cf5f5b60bb2946335d9e3058af5846137dd9d58f40dfe2a6a14d395bdb526a34151f679a5
|
data/exe/mfynab
ADDED
data/lib/mfynab/cli.rb
ADDED
@@ -0,0 +1,89 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "yaml"
|
4
|
+
require "mfynab/money_forward"
|
5
|
+
require "mfynab/money_forward_data"
|
6
|
+
require "mfynab/ynab_transaction_importer"
|
7
|
+
|
8
|
+
class CLI
|
9
|
+
def self.start(argv)
|
10
|
+
new(argv).start
|
11
|
+
end
|
12
|
+
|
13
|
+
def initialize(argv)
|
14
|
+
@argv = argv
|
15
|
+
end
|
16
|
+
|
17
|
+
def start
|
18
|
+
logger.info("Running...")
|
19
|
+
|
20
|
+
Dir.mktmpdir("mfynab") do |save_path|
|
21
|
+
money_forward.download_csv(
|
22
|
+
session_id: session_id,
|
23
|
+
path: save_path,
|
24
|
+
months: config["months_to_sync"],
|
25
|
+
)
|
26
|
+
|
27
|
+
data = MFYNAB::MoneyForwardData.new(logger: logger)
|
28
|
+
data.read_all_csv(save_path)
|
29
|
+
ynab_transaction_importer.run(data.to_h)
|
30
|
+
end
|
31
|
+
|
32
|
+
logger.info("Done!")
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
attr_reader :argv
|
38
|
+
|
39
|
+
def session_id
|
40
|
+
@_session_id ||= money_forward.get_session_id(
|
41
|
+
username: config["moneyforward_username"],
|
42
|
+
password: config["moneyforward_password"],
|
43
|
+
)
|
44
|
+
end
|
45
|
+
|
46
|
+
def ynab_transaction_importer
|
47
|
+
@_ynab_transaction_importer ||= MFYNAB::YnabTransactionImporter.new(
|
48
|
+
config["ynab_access_token"],
|
49
|
+
config["ynab_budget"],
|
50
|
+
config["accounts"],
|
51
|
+
logger: logger,
|
52
|
+
)
|
53
|
+
end
|
54
|
+
|
55
|
+
def money_forward
|
56
|
+
@_money_forward ||= MFYNAB::MoneyForward.new(logger: logger)
|
57
|
+
end
|
58
|
+
|
59
|
+
def config_file
|
60
|
+
raise "You need to pass a config file" if argv.empty?
|
61
|
+
|
62
|
+
argv[0]
|
63
|
+
end
|
64
|
+
|
65
|
+
def config
|
66
|
+
@_config ||= YAML
|
67
|
+
.load_file(config_file)
|
68
|
+
.values
|
69
|
+
.first
|
70
|
+
.merge(
|
71
|
+
"months_to_sync" => 3,
|
72
|
+
"ynab_access_token" => ENV.fetch("YNAB_ACCESS_TOKEN"),
|
73
|
+
"moneyforward_username" => ENV.fetch("MONEYFORWARD_USERNAME"),
|
74
|
+
"moneyforward_password" => ENV.fetch("MONEYFORWARD_PASSWORD"),
|
75
|
+
)
|
76
|
+
end
|
77
|
+
|
78
|
+
def logger
|
79
|
+
@_logger ||= Logger.new($stdout, level: logger_level)
|
80
|
+
end
|
81
|
+
|
82
|
+
def logger_level
|
83
|
+
if ENV.fetch("DEBUG", nil)
|
84
|
+
Logger::DEBUG
|
85
|
+
else
|
86
|
+
Logger::INFO
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "ferrum"
|
4
|
+
|
5
|
+
module MFYNAB
|
6
|
+
class MoneyForward
|
7
|
+
DEFAULT_BASE_URL = "https://moneyforward.com"
|
8
|
+
SIGNIN_PATH = "/sign_in"
|
9
|
+
CSV_PATH = "/cf/csv"
|
10
|
+
SESSION_COOKIE_NAME = "_moneybook_session"
|
11
|
+
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
|
12
|
+
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
13
|
+
|
14
|
+
def initialize(logger:, base_url: DEFAULT_BASE_URL)
|
15
|
+
@base_url = URI(base_url)
|
16
|
+
@logger = logger
|
17
|
+
end
|
18
|
+
|
19
|
+
def get_session_id(username:, password:)
|
20
|
+
with_ferrum do |browser|
|
21
|
+
browser.goto("#{base_url}#{SIGNIN_PATH}")
|
22
|
+
browser.at_css("input[name='mfid_user[email]']").focus.type(username)
|
23
|
+
browser.at_css("input[name='mfid_user[password]']").focus.type(password)
|
24
|
+
browser.at_css("button#submitto").click
|
25
|
+
|
26
|
+
# FIXME: use custom error class
|
27
|
+
raise "Login failed" unless browser.cookies[SESSION_COOKIE_NAME]
|
28
|
+
|
29
|
+
browser.cookies[SESSION_COOKIE_NAME].value
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def download_csv(session_id:, path:, months:)
|
34
|
+
month = Date.today
|
35
|
+
month -= month.day - 1 # First day of month
|
36
|
+
|
37
|
+
Net::HTTP.start(base_url.host, use_ssl: true) do |http|
|
38
|
+
http.response_body_encoding = Encoding::SJIS
|
39
|
+
|
40
|
+
months.times do
|
41
|
+
date_string = month.strftime("%Y-%m")
|
42
|
+
|
43
|
+
logger.info("Downloading CSV for #{date_string}")
|
44
|
+
|
45
|
+
# FIXME: I don't really need to save the CSV files to disk anymore.
|
46
|
+
# Maybe just return parsed CSV data?
|
47
|
+
File.open(File.join(path, "#{date_string}.csv"), "wb") do |file|
|
48
|
+
file << download_csv_string(date: month, session_id: session_id)
|
49
|
+
end
|
50
|
+
|
51
|
+
month = month.prev_month
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def download_csv_string(date:, session_id:)
|
57
|
+
Net::HTTP.start(base_url.host, use_ssl: true) do |http|
|
58
|
+
http.response_body_encoding = Encoding::SJIS
|
59
|
+
|
60
|
+
request = Net::HTTP::Get.new(
|
61
|
+
"#{CSV_PATH}?from=#{date.strftime('%Y/%m/%d')}",
|
62
|
+
{
|
63
|
+
"Cookie" => "#{SESSION_COOKIE_NAME}=#{session_id}",
|
64
|
+
"User-Agent" => USER_AGENT,
|
65
|
+
},
|
66
|
+
)
|
67
|
+
|
68
|
+
result = http.request(request)
|
69
|
+
raise unless result.is_a?(Net::HTTPSuccess)
|
70
|
+
raise unless result.body.valid_encoding?
|
71
|
+
|
72
|
+
result.body.encode(Encoding::UTF_8)
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
private
|
77
|
+
|
78
|
+
attr_reader :base_url, :logger
|
79
|
+
|
80
|
+
def with_ferrum
|
81
|
+
browser = Ferrum::Browser.new(timeout: 30, headless: !ENV.key?("NO_HEADLESS"))
|
82
|
+
browser.headers.add({
|
83
|
+
"Accept-Language" => "en-US,en",
|
84
|
+
"User-Agent" => USER_AGENT,
|
85
|
+
})
|
86
|
+
yield browser
|
87
|
+
rescue StandardError
|
88
|
+
browser.screenshot(path: "screenshot.png")
|
89
|
+
logger.error("An error occurred and a screenshot was saved to ./screenshot.png")
|
90
|
+
raise
|
91
|
+
ensure
|
92
|
+
browser&.quit
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "csv"
|
4
|
+
|
5
|
+
module MFYNAB
|
6
|
+
class MoneyForwardData
|
7
|
+
HEADERS = {
|
8
|
+
include: "計算対象",
|
9
|
+
date: "日付",
|
10
|
+
content: "内容",
|
11
|
+
amount: "金額(円)",
|
12
|
+
account: "保有金融機関",
|
13
|
+
category: "大項目",
|
14
|
+
subcategory: "中項目",
|
15
|
+
memo: "メモ",
|
16
|
+
transfer: "振替",
|
17
|
+
id: "ID",
|
18
|
+
}.freeze
|
19
|
+
|
20
|
+
def initialize(logger:)
|
21
|
+
@transactions = {}
|
22
|
+
@logger = logger
|
23
|
+
end
|
24
|
+
|
25
|
+
def read_all_csv(path)
|
26
|
+
Dir[File.join(path, "*.csv")].each do |file|
|
27
|
+
read_csv(file)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def read_csv(csv_file)
|
32
|
+
logger.info("Reading #{csv_file}")
|
33
|
+
CSV.foreach(
|
34
|
+
csv_file,
|
35
|
+
headers: true,
|
36
|
+
converters: :all,
|
37
|
+
header_converters: csv_header_converters,
|
38
|
+
) do |row|
|
39
|
+
transactions[row["account"]] ||= []
|
40
|
+
transactions[row["account"]] << row.to_h
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def to_h
|
45
|
+
transactions
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
attr_reader :transactions, :logger
|
51
|
+
|
52
|
+
def csv_header_converters
|
53
|
+
lambda do |header|
|
54
|
+
HEADERS.key(header)&.to_s || header
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,156 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "ynab"
|
4
|
+
|
5
|
+
module MFYNAB
|
6
|
+
class YnabTransactionImporter
|
7
|
+
def initialize(api_key, budget_name, account_mappings, logger:)
|
8
|
+
@api_key = api_key
|
9
|
+
@budget_name = budget_name
|
10
|
+
@account_mappings = account_mappings
|
11
|
+
@logger = logger
|
12
|
+
end
|
13
|
+
|
14
|
+
def run(mf_transactions) # rubocop:disable Metrics/AbcSize
|
15
|
+
mf_transactions.map do |mf_account, mf_account_transactions|
|
16
|
+
account = ynab_account_for_mf_account(mf_account)
|
17
|
+
next unless account
|
18
|
+
|
19
|
+
transactions = mf_account_transactions.map do |row|
|
20
|
+
convert_mf_transaction(row, account)
|
21
|
+
end
|
22
|
+
|
23
|
+
begin
|
24
|
+
logger.info("Importing #{transactions.size} transactions for #{account.name}")
|
25
|
+
ynab_transactions_api.create_transaction(budget.id, transactions: transactions)
|
26
|
+
rescue StandardError => e
|
27
|
+
logger.error("Error importing transactions for #{budget.name}. #{e} : #{e.detail}")
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
private
|
33
|
+
|
34
|
+
attr_reader :api_key, :budget_name, :account_mappings, :logger
|
35
|
+
|
36
|
+
def convert_mf_transaction(row, account)
|
37
|
+
{
|
38
|
+
account_id: account.id,
|
39
|
+
amount: row["amount"] * 1_000,
|
40
|
+
payee_name: row["content"][0, 100],
|
41
|
+
date: Date.strptime(row["date"], "%Y/%m/%d").strftime("%Y-%m-%d"),
|
42
|
+
cleared: "cleared",
|
43
|
+
memo: generate_memo_for(row),
|
44
|
+
import_id: generate_import_id_for(row),
|
45
|
+
}
|
46
|
+
end
|
47
|
+
|
48
|
+
def generate_memo_for(row)
|
49
|
+
category = row
|
50
|
+
.values_at("category", "subcategory")
|
51
|
+
.delete_if { _1.nil? || _1.empty? || _1 == "未分類" }
|
52
|
+
.join("/")
|
53
|
+
|
54
|
+
memo_parts = [
|
55
|
+
row["memo"], # prioritize memo if present, since it's user input
|
56
|
+
row["content"],
|
57
|
+
category,
|
58
|
+
]
|
59
|
+
|
60
|
+
memo_parts
|
61
|
+
.delete_if { _1.nil? || _1.empty? }
|
62
|
+
.join(" - ")
|
63
|
+
.slice(0, 200) # YNAB's API currently limits memo to 200 characters,
|
64
|
+
# even though YNAB itself allows longer memos. See:
|
65
|
+
# https://github.com/ynab/ynab-sdk-ruby/issues/77
|
66
|
+
end
|
67
|
+
|
68
|
+
# ⚠️ Be very careful when changing this method!
|
69
|
+
#
|
70
|
+
# A different import_id can cause MFYNAB to create duplicate transactions.
|
71
|
+
#
|
72
|
+
# import_id is scoped to an account in a budget, this means that:
|
73
|
+
# - if 2 transactions have the same import_id, but are in different
|
74
|
+
# accounts then they will be imported as 2 unrelated transactions.
|
75
|
+
# - if 2 transactions in the same account have the same import_id,
|
76
|
+
# then the second transaction will be ignored,
|
77
|
+
# even if it has a different date and/or amount.
|
78
|
+
# Note that it might be useful for the import_id to stay consistent even if
|
79
|
+
# the transaction amount changes, since a transaction that originally
|
80
|
+
# appeared as a low-amount authorization might be updated to its final
|
81
|
+
# amount later.
|
82
|
+
# (I don't know what that means for cleared and reconciled transactions...)
|
83
|
+
def generate_import_id_for(row)
|
84
|
+
# Uniquely identify transactions to avoid conflicts with other potential import scripts
|
85
|
+
# Note: I don't remember why I named it MFBY (why not MFYNAB or MFY?),
|
86
|
+
# but changing it now would require a lot of work in preventing import
|
87
|
+
# duplicates due to inconsistent import_id.
|
88
|
+
prefix = "MFBY:v1:"
|
89
|
+
|
90
|
+
max_length = 36 # YNAB API limit
|
91
|
+
id_max_length = 28 # this leaves 8 characters for the prefix
|
92
|
+
|
93
|
+
id = row["id"]
|
94
|
+
|
95
|
+
# Only hash if the ID would exceed YNAB's limit.
|
96
|
+
# This improves backwards compatibility with old import_ids.
|
97
|
+
if prefix.length + id.length > max_length
|
98
|
+
id = Digest::SHA256.hexdigest(id)[0, id_max_length]
|
99
|
+
end
|
100
|
+
|
101
|
+
prefix + id
|
102
|
+
end
|
103
|
+
|
104
|
+
def ynab_account_for_mf_account(mf_account_name)
|
105
|
+
matching_mapping = account_mappings.find do |mapping|
|
106
|
+
mapping["money_forward_name"] == mf_account_name
|
107
|
+
end
|
108
|
+
|
109
|
+
unless matching_mapping
|
110
|
+
logger.debug("Debug: no mapping for MoneyForward account #{mf_account_name}. Skipping...")
|
111
|
+
return
|
112
|
+
end
|
113
|
+
|
114
|
+
ynab_account_name = matching_mapping["ynab_name"]
|
115
|
+
|
116
|
+
ynab_account = accounts.find do |account|
|
117
|
+
account.name.include?(ynab_account_name)
|
118
|
+
end
|
119
|
+
raise "No YNAB account found with name #{ynab_account_name}." unless ynab_account
|
120
|
+
|
121
|
+
ynab_account
|
122
|
+
end
|
123
|
+
|
124
|
+
def ynab_transactions_api
|
125
|
+
@_ynab_transactions_api ||= YNAB::TransactionsApi.new(ynab_api_client)
|
126
|
+
end
|
127
|
+
|
128
|
+
def accounts
|
129
|
+
@_accounts ||= YNAB::AccountsApi
|
130
|
+
.new(ynab_api_client)
|
131
|
+
.get_accounts(budget.id)
|
132
|
+
.data
|
133
|
+
.accounts
|
134
|
+
end
|
135
|
+
|
136
|
+
def budget
|
137
|
+
@_budget ||= YNAB::BudgetsApi
|
138
|
+
.new(ynab_api_client)
|
139
|
+
.get_budgets
|
140
|
+
.data
|
141
|
+
.budgets
|
142
|
+
.find { _1.name == budget_name }
|
143
|
+
end
|
144
|
+
|
145
|
+
def ynab_api_client
|
146
|
+
@_ynab_api_client ||= YNAB::ApiClient.new(ynab_api_config)
|
147
|
+
end
|
148
|
+
|
149
|
+
def ynab_api_config
|
150
|
+
@_ynab_api_config = YNAB::Configuration.new.tap do |config|
|
151
|
+
config.access_token = api_key
|
152
|
+
config.debugging = false
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
metadata
ADDED
@@ -0,0 +1,109 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: mfynab
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- David Stosik
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2024-07-26 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: csv
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ">="
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ">="
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: ferrum
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.15'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.15'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: psych
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: ynab
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.4'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.4'
|
69
|
+
description: Sync transaction history from MoneyForward to YNAB.
|
70
|
+
email:
|
71
|
+
- david.stosik+git-noreply@gmail.com
|
72
|
+
executables:
|
73
|
+
- mfynab
|
74
|
+
extensions: []
|
75
|
+
extra_rdoc_files: []
|
76
|
+
files:
|
77
|
+
- exe/mfynab
|
78
|
+
- lib/mfynab/cli.rb
|
79
|
+
- lib/mfynab/money_forward.rb
|
80
|
+
- lib/mfynab/money_forward_data.rb
|
81
|
+
- lib/mfynab/version.rb
|
82
|
+
- lib/mfynab/ynab_transaction_importer.rb
|
83
|
+
homepage: https://github.com/davidstosik/moneyforward_ynab
|
84
|
+
licenses:
|
85
|
+
- MIT
|
86
|
+
metadata:
|
87
|
+
homepage_uri: https://github.com/davidstosik/moneyforward_ynab
|
88
|
+
source_code_uri: https://github.com/davidstosik/moneyforward_ynab
|
89
|
+
rubygems_mfa_required: 'true'
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
require_paths:
|
93
|
+
- lib
|
94
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
95
|
+
requirements:
|
96
|
+
- - ">="
|
97
|
+
- !ruby/object:Gem::Version
|
98
|
+
version: 3.3.0
|
99
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
requirements: []
|
105
|
+
rubygems_version: 3.5.11
|
106
|
+
signing_key:
|
107
|
+
specification_version: 4
|
108
|
+
summary: Sync transaction history from MoneyForward to YNAB.
|
109
|
+
test_files: []
|