mfynab 0.1.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: df7f640ee7b19163bf41b79a560d4de67f8892d797aaaa8b06ab6ba321b4f1b6
+  data.tar.gz: 99fcfc93e353bcc951e14505220dc626ed51230d1e0489cb0a90fc4f4d9fc5f1
+SHA512:
+  metadata.gz: 7a7dc9f3cba490a05b552209b1b496398ab754f6f0d1ce289894780de4fb2bf7eab5bbd1d580e7bddc9230139a442d302a8849754a3f328860e7fa73339810a5
+  data.tar.gz: 790422bd20f6e40c6ddca4471b2bcdc9ba2e948a50229e5b93d3f28cf5f5b60bb2946335d9e3058af5846137dd9d58f40dfe2a6a14d395bdb526a34151f679a5

data/exe/mfynab

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "mfynab/cli"
+
+CLI.start(ARGV)

data/lib/mfynab/cli.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "mfynab/money_forward"
+require "mfynab/money_forward_data"
+require "mfynab/ynab_transaction_importer"
+
+class CLI
+  def self.start(argv)
+    new(argv).start
+  end
+
+  def initialize(argv)
+    @argv = argv
+  end
+
+  def start
+    logger.info("Running...")
+
+    Dir.mktmpdir("mfynab") do |save_path|
+      money_forward.download_csv(
+        session_id: session_id,
+        path: save_path,
+        months: config["months_to_sync"],
+      )
+
+      data = MFYNAB::MoneyForwardData.new(logger: logger)
+      data.read_all_csv(save_path)
+      ynab_transaction_importer.run(data.to_h)
+    end
+
+    logger.info("Done!")
+  end
+
+  private
+
+    attr_reader :argv
+
+    def session_id
+      @_session_id ||= money_forward.get_session_id(
+        username: config["moneyforward_username"],
+        password: config["moneyforward_password"],
+      )
+    end
+
+    def ynab_transaction_importer
+      @_ynab_transaction_importer ||= MFYNAB::YnabTransactionImporter.new(
+        config["ynab_access_token"],
+        config["ynab_budget"],
+        config["accounts"],
+        logger: logger,
+      )
+    end
+
+    def money_forward
+      @_money_forward ||= MFYNAB::MoneyForward.new(logger: logger)
+    end
+
+    def config_file
+      raise "You need to pass a config file" if argv.empty?
+
+      argv[0]
+    end
+
+    def config
+      @_config ||= YAML
+        .load_file(config_file)
+        .values
+        .first
+        .merge(
+          "months_to_sync" => 3,
+          "ynab_access_token" => ENV.fetch("YNAB_ACCESS_TOKEN"),
+          "moneyforward_username" => ENV.fetch("MONEYFORWARD_USERNAME"),
+          "moneyforward_password" => ENV.fetch("MONEYFORWARD_PASSWORD"),
+        )
+    end
+
+    def logger
+      @_logger ||= Logger.new($stdout, level: logger_level)
+    end
+
+    def logger_level
+      if ENV.fetch("DEBUG", nil)
+        Logger::DEBUG
+      else
+        Logger::INFO
+      end
+    end
+end

data/lib/mfynab/money_forward.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+require "ferrum"
+
+module MFYNAB
+  class MoneyForward
+    DEFAULT_BASE_URL = "https://moneyforward.com"
+    SIGNIN_PATH = "/sign_in"
+    CSV_PATH = "/cf/csv"
+    SESSION_COOKIE_NAME = "_moneybook_session"
+    USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
+                 "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
+
+    def initialize(logger:, base_url: DEFAULT_BASE_URL)
+      @base_url = URI(base_url)
+      @logger = logger
+    end
+
+    def get_session_id(username:, password:)
+      with_ferrum do |browser|
+        browser.goto("#{base_url}#{SIGNIN_PATH}")
+        browser.at_css("input[name='mfid_user[email]']").focus.type(username)
+        browser.at_css("input[name='mfid_user[password]']").focus.type(password)
+        browser.at_css("button#submitto").click
+
+        # FIXME: use custom error class
+        raise "Login failed" unless browser.cookies[SESSION_COOKIE_NAME]
+
+        browser.cookies[SESSION_COOKIE_NAME].value
+      end
+    end
+
+    def download_csv(session_id:, path:, months:)
+      month = Date.today
+      month -= month.day - 1 # First day of month
+
+      Net::HTTP.start(base_url.host, use_ssl: true) do |http|
+        http.response_body_encoding = Encoding::SJIS
+
+        months.times do
+          date_string = month.strftime("%Y-%m")
+
+          logger.info("Downloading CSV for #{date_string}")
+
+          # FIXME: I don't really need to save the CSV files to disk anymore.
+          # Maybe just return parsed CSV data?
+          File.open(File.join(path, "#{date_string}.csv"), "wb") do |file|
+            file << download_csv_string(date: month, session_id: session_id)
+          end
+
+          month = month.prev_month
+        end
+      end
+    end
+
+    def download_csv_string(date:, session_id:)
+      Net::HTTP.start(base_url.host, use_ssl: true) do |http|
+        http.response_body_encoding = Encoding::SJIS
+
+        request = Net::HTTP::Get.new(
+          "#{CSV_PATH}?from=#{date.strftime('%Y/%m/%d')}",
+          {
+            "Cookie" => "#{SESSION_COOKIE_NAME}=#{session_id}",
+            "User-Agent" => USER_AGENT,
+          },
+        )
+
+        result = http.request(request)
+        raise unless result.is_a?(Net::HTTPSuccess)
+        raise unless result.body.valid_encoding?
+
+        result.body.encode(Encoding::UTF_8)
+      end
+    end
+
+    private
+
+      attr_reader :base_url, :logger
+
+      def with_ferrum
+        browser = Ferrum::Browser.new(timeout: 30, headless: !ENV.key?("NO_HEADLESS"))
+        browser.headers.add({
+          "Accept-Language" => "en-US,en",
+          "User-Agent" => USER_AGENT,
+        })
+        yield browser
+      rescue StandardError
+        browser.screenshot(path: "screenshot.png")
+        logger.error("An error occurred and a screenshot was saved to ./screenshot.png")
+        raise
+      ensure
+        browser&.quit
+      end
+  end
+end

data/lib/mfynab/money_forward_data.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "csv"
+
+module MFYNAB
+  class MoneyForwardData
+    HEADERS = {
+      include: "計算対象",
+      date: "日付",
+      content: "内容",
+      amount: "金額（円）",
+      account: "保有金融機関",
+      category: "大項目",
+      subcategory: "中項目",
+      memo: "メモ",
+      transfer: "振替",
+      id: "ID",
+    }.freeze
+
+    def initialize(logger:)
+      @transactions = {}
+      @logger = logger
+    end
+
+    def read_all_csv(path)
+      Dir[File.join(path, "*.csv")].each do |file|
+        read_csv(file)
+      end
+    end
+
+    def read_csv(csv_file)
+      logger.info("Reading #{csv_file}")
+      CSV.foreach(
+        csv_file,
+        headers: true,
+        converters: :all,
+        header_converters: csv_header_converters,
+      ) do |row|
+        transactions[row["account"]] ||= []
+        transactions[row["account"]] << row.to_h
+      end
+    end
+
+    def to_h
+      transactions
+    end
+
+    private
+
+      attr_reader :transactions, :logger
+
+      def csv_header_converters
+        lambda do |header|
+          HEADERS.key(header)&.to_s || header
+        end
+      end
+  end
+end

data/lib/mfynab/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module MFYNAB
+  VERSION = "0.1.3"
+end

data/lib/mfynab/ynab_transaction_importer.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "ynab"
+
+module MFYNAB
+  class YnabTransactionImporter
+    def initialize(api_key, budget_name, account_mappings, logger:)
+      @api_key = api_key
+      @budget_name = budget_name
+      @account_mappings = account_mappings
+      @logger = logger
+    end
+
+    def run(mf_transactions) # rubocop:disable Metrics/AbcSize
+      mf_transactions.map do |mf_account, mf_account_transactions|
+        account = ynab_account_for_mf_account(mf_account)
+        next unless account
+
+        transactions = mf_account_transactions.map do |row|
+          convert_mf_transaction(row, account)
+        end
+
+        begin
+          logger.info("Importing #{transactions.size} transactions for #{account.name}")
+          ynab_transactions_api.create_transaction(budget.id, transactions: transactions)
+        rescue StandardError => e
+          logger.error("Error importing transactions for #{budget.name}. #{e} : #{e.detail}")
+        end
+      end
+    end
+
+    private
+
+      attr_reader :api_key, :budget_name, :account_mappings, :logger
+
+      def convert_mf_transaction(row, account)
+        {
+          account_id: account.id,
+          amount: row["amount"] * 1_000,
+          payee_name: row["content"][0, 100],
+          date: Date.strptime(row["date"], "%Y/%m/%d").strftime("%Y-%m-%d"),
+          cleared: "cleared",
+          memo: generate_memo_for(row),
+          import_id: generate_import_id_for(row),
+        }
+      end
+
+      def generate_memo_for(row)
+        category = row
+          .values_at("category", "subcategory")
+          .delete_if { _1.nil? || _1.empty? || _1 == "未分類" }
+          .join("/")
+
+        memo_parts = [
+          row["memo"], # prioritize memo if present, since it's user input
+          row["content"],
+          category,
+        ]
+
+        memo_parts
+          .delete_if { _1.nil? || _1.empty? }
+          .join(" - ")
+          .slice(0, 200) # YNAB's API currently limits memo to 200 characters,
+        # even though YNAB itself allows longer memos. See:
+        # https://github.com/ynab/ynab-sdk-ruby/issues/77
+      end
+
+      # ⚠️ Be very careful when changing this method!
+      #
+      # A different import_id can cause MFYNAB to create duplicate transactions.
+      #
+      # import_id is scoped to an account in a budget, this means that:
+      # - if 2 transactions have the same import_id, but are in different
+      #   accounts then they will be imported as 2 unrelated transactions.
+      # - if 2 transactions in the same account have the same import_id,
+      #   then the second transaction will be ignored,
+      #   even if it has a different date and/or amount.
+      # Note that it might be useful for the import_id to stay consistent even if
+      # the transaction amount changes, since a transaction that originally
+      # appeared as a low-amount authorization might be updated to its final
+      # amount later.
+      # (I don't know what that means for cleared and reconciled transactions...)
+      def generate_import_id_for(row)
+        # Uniquely identify transactions to avoid conflicts with other potential import scripts
+        # Note: I don't remember why I named it MFBY (why not MFYNAB or MFY?),
+        # but changing it now would require a lot of work in preventing import
+        # duplicates due to inconsistent import_id.
+        prefix = "MFBY:v1:"
+
+        max_length = 36     # YNAB API limit
+        id_max_length = 28  # this leaves 8 characters for the prefix
+
+        id = row["id"]
+
+        # Only hash if the ID would exceed YNAB's limit.
+        # This improves backwards compatibility with old import_ids.
+        if prefix.length + id.length > max_length
+          id = Digest::SHA256.hexdigest(id)[0, id_max_length]
+        end
+
+        prefix + id
+      end
+
+      def ynab_account_for_mf_account(mf_account_name)
+        matching_mapping = account_mappings.find do |mapping|
+          mapping["money_forward_name"] == mf_account_name
+        end
+
+        unless matching_mapping
+          logger.debug("Debug: no mapping for MoneyForward account #{mf_account_name}. Skipping...")
+          return
+        end
+
+        ynab_account_name = matching_mapping["ynab_name"]
+
+        ynab_account = accounts.find do |account|
+          account.name.include?(ynab_account_name)
+        end
+        raise "No YNAB account found with name #{ynab_account_name}." unless ynab_account
+
+        ynab_account
+      end
+
+      def ynab_transactions_api
+        @_ynab_transactions_api ||= YNAB::TransactionsApi.new(ynab_api_client)
+      end
+
+      def accounts
+        @_accounts ||= YNAB::AccountsApi
+          .new(ynab_api_client)
+          .get_accounts(budget.id)
+          .data
+          .accounts
+      end
+
+      def budget
+        @_budget ||= YNAB::BudgetsApi
+          .new(ynab_api_client)
+          .get_budgets
+          .data
+          .budgets
+          .find { _1.name == budget_name }
+      end
+
+      def ynab_api_client
+        @_ynab_api_client ||= YNAB::ApiClient.new(ynab_api_config)
+      end
+
+      def ynab_api_config
+        @_ynab_api_config = YNAB::Configuration.new.tap do |config|
+          config.access_token = api_key
+          config.debugging = false
+        end
+      end
+  end
+end

metadata

@@ -0,0 +1,109 @@
+--- !ruby/object:Gem::Specification
+name: mfynab
+version: !ruby/object:Gem::Version
+  version: 0.1.3
+platform: ruby
+authors:
+- David Stosik
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2024-07-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: csv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ferrum
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.15'
+- !ruby/object:Gem::Dependency
+  name: psych
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ynab
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+description: Sync transaction history from MoneyForward to YNAB.
+email:
+- david.stosik+git-noreply@gmail.com
+executables:
+- mfynab
+extensions: []
+extra_rdoc_files: []
+files:
+- exe/mfynab
+- lib/mfynab/cli.rb
+- lib/mfynab/money_forward.rb
+- lib/mfynab/money_forward_data.rb
+- lib/mfynab/version.rb
+- lib/mfynab/ynab_transaction_importer.rb
+homepage: https://github.com/davidstosik/moneyforward_ynab
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/davidstosik/moneyforward_ynab
+  source_code_uri: https://github.com/davidstosik/moneyforward_ynab
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.11
+signing_key:
+specification_version: 4
+summary: Sync transaction history from MoneyForward to YNAB.
+test_files: []