mfynab 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a71ca45747525d78d9d23d653968c82ce28a715645e229be2697de76aeffb474
-  data.tar.gz: 4eae7448677c630ec6ef0871ac9ec98253a356aebf7292a54059bc781d098ca4
+  metadata.gz: 1d0ef3ce6051c500eb5a269c3763ff18ae129fbbef15693eb8413fa6e3c788ef
+  data.tar.gz: d9788544bc9aaa518c8ad47d3abfe526fb8686fe676b49fe30358930e91dca75
 SHA512:
-  metadata.gz: 0bb90df5c776bf130ec6efcb26082b8bb9dfa94463fd96e92da4c319392060cfd9f20d555e6b5598025f6952893044e331030e86a8d4817afd8186c87775e301
-  data.tar.gz: 17500b0d34590b0fa4b6689d5fc9e30b56620c0a7fb4cd036c7cf995e6db90fd2b1bc0319d66e6ba1b12d07e97bd07af3d7160557eda047c9cade107d2841c66
+  metadata.gz: 7a6ae839f41a31c0615561ebf08c57df799ff576ef4cfde32e30eab5373e8289ca053887c272c8c2d0bc44c0643b29a41fe45a54e6f278514b6e05eecb6c8816
+  data.tar.gz: 7dd19ed30d13acaef9ed75649098611f36af8c19f314c4366e829e587c67a2f057ad88e9028c8058c66263b72384ff1e8cc481a48725a3e9924d7687b5edfe74

data/exe/mfynab

@@ -3,4 +3,4 @@
 
 require "mfynab/cli"
 
-CLI.start(ARGV)
+MFYNAB::CLI.start(ARGV)

data/lib/mfynab/cli.rb

@@ -2,91 +2,100 @@
 
 require "yaml"
 require "mfynab/money_forward"
+require "mfynab/money_forward/session"
 require "mfynab/money_forward_data"
 require "mfynab/ynab_transaction_importer"
 
-class CLI
-  def self.start(argv)
-    new(argv).start
-  end
+module MFYNAB
+  class CLI
+    def self.start(argv)
+      new(argv).start
+    end
 
-  def initialize(argv)
-    @argv = argv
-  end
+    def initialize(argv)
+      @argv = argv
+    end
 
-  def start
-    logger.info("Running...")
+    def start
+      logger.info("Running...")
 
-    Dir.mktmpdir("mfynab") do |save_path|
-      money_forward.download_csv(
-        session_id: session_id,
-        path: save_path,
-        months: months_to_sync,
-      )
+      money_forward.update_accounts(money_forward_account_names)
+
+      Dir.mktmpdir("mfynab") do |save_path|
+        money_forward.download_csv(
+          path: save_path,
+          months: months_to_sync,
+        )
+
+        data = MoneyForwardData.new(logger: logger)
+        data.read_all_csv(save_path)
+        ynab_transaction_importer.run(data.to_h)
+      end
 
-      data = MFYNAB::MoneyForwardData.new(logger: logger)
-      data.read_all_csv(save_path)
-      ynab_transaction_importer.run(data.to_h)
+      logger.info("Done!")
     end
 
-    logger.info("Done!")
-  end
+    private
 
-  private
+      attr_reader :argv
 
-    attr_reader :argv
+      def money_forward_account_names
+        @_money_forward_account_names = config["accounts"].map { _1["money_forward_name"] }
+      end
 
-    def session_id
-      @_session_id ||= money_forward.get_session_id(
-        username: config["moneyforward_username"],
-        password: config["moneyforward_password"],
-      )
-    end
+      def ynab_transaction_importer
+        @_ynab_transaction_importer ||= YnabTransactionImporter.new(
+          config["ynab_access_token"],
+          config["ynab_budget"],
+          config["accounts"],
+          logger: logger,
+        )
+      end
 
-    def ynab_transaction_importer
-      @_ynab_transaction_importer ||= MFYNAB::YnabTransactionImporter.new(
-        config["ynab_access_token"],
-        config["ynab_budget"],
-        config["accounts"],
-        logger: logger,
-      )
-    end
+      def months_to_sync
+        config.fetch("months_to_sync", 3)
+      end
 
-    def months_to_sync
-      config.fetch("months_to_sync", 3)
-    end
+      def money_forward
+        @_money_forward ||= MoneyForward.new(session, logger: logger)
+      end
 
-    def money_forward
-      @_money_forward ||= MFYNAB::MoneyForward.new(logger: logger)
-    end
+      def session
+        @_session ||= MoneyForward::Session.new(
+          username: config["moneyforward_username"],
+          password: config["moneyforward_password"],
+          logger: logger,
+        )
+      end
 
-    def config_file
-      raise "You need to pass a config file" if argv.empty?
+      def config_file
+        raise "You need to pass a config file" if argv.empty?
 
-      argv[0]
-    end
+        argv[0]
+      end
 
-    def config
-      @_config ||= YAML
-        .load_file(config_file)
-        .values
-        .first
-        .merge(
-          "ynab_access_token" => ENV.fetch("YNAB_ACCESS_TOKEN"),
-          "moneyforward_username" => ENV.fetch("MONEYFORWARD_USERNAME"),
-          "moneyforward_password" => ENV.fetch("MONEYFORWARD_PASSWORD"),
-        )
-    end
+      def config
+        @_config ||= YAML
+          .load_file(config_file)
+          .values
+          .first
+          .merge(
+            "ynab_access_token" => ENV.fetch("YNAB_ACCESS_TOKEN"),
+            "moneyforward_username" => ENV.fetch("MONEYFORWARD_USERNAME"),
+            "moneyforward_password" => ENV.fetch("MONEYFORWARD_PASSWORD"),
+          )
+      end
 
-    def logger
-      @_logger ||= Logger.new($stdout, level: logger_level)
-    end
+      def logger
+        @_logger ||= Logger.new($stdout, level: logger_level)
+      end
 
-    def logger_level
-      if ENV.fetch("DEBUG", nil)
-        Logger::DEBUG
-      else
-        Logger::INFO
+      def logger_level
+        if ENV.fetch("DEBUG", nil)
+          Logger::DEBUG
+        else
+          Logger::INFO
+        end
       end
-    end
+  end
 end

data/lib/mfynab/money_forward/account_status.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "ferrum"
+require "net/http"
+require "nokogiri"
+require "uri"
+
+module MFYNAB
+  class MoneyForward
+    class AccountStatus
+      class Fetcher
+        def initialize(session, logger:)
+          @logger = logger
+          @session = session
+        end
+
+        def fetch(account_names:)
+          logger.info("Checking Money Forward accounts status")
+          html = session.http_get("/accounts")
+          table_rows_selector = "section.accounts section.common-account-table-container > table > tr[id]"
+          Nokogiri::HTML(html).css(table_rows_selector).filter_map do |node|
+            account = parse_html_table_row(node)
+            next unless account_names.include?(account.name)
+
+            status_data = determine_status_data(account)
+            account.assign_status_data(**status_data)
+
+            account
+          end
+        end
+
+        private
+
+          attr_reader :session, :logger
+
+          def determine_status_data(account)
+            case account.raw_status
+            when "正常" then { key: :success }
+            when "更新中" then { key: :processing }
+            else
+              body = session.http_get("/accounts/polling/#{account.id_hash}")
+              # JSON format:
+              #  - loading: boolean
+              #  - message: Japanese text
+              #  - status: success, processing, additional_request, errors, important_announcement,
+              #            invalid_password, login, suspended
+              # FIXME: make more robust
+              queried_status = JSON.parse(body)
+              {
+                key: queried_status["status"].to_sym,
+                message: queried_status["message"],
+              }
+            end
+          end
+
+          def parse_html_table_row(node)
+            AccountStatus.new(
+              id_hash: node[:id],
+              name: node.xpath("td").first.text.lines[1].strip,
+              raw_status: node.css("td.account-status>span:not([id*='hidden'])").text.strip,
+              # FIXME: can this be a little more robust?
+              #  - this should not depend on the timezone the server is running in
+              #  - this should explicitly fail if parsing is not possible
+              #  - what if it's January 1st and last update was last year?
+              updated_at: Time.parse(node.css("td.created").text[/(?<=\().*?(?=\))/]),
+            )
+          end
+      end
+
+      FRESHNESS_LIMIT = 24 * 60 * 60 # 1 day
+
+      attr_reader :id_hash, :name, :raw_status, :updated_at, :key, :message
+
+      def initialize(id_hash:, name:, raw_status:, updated_at:)
+        @id_hash = id_hash
+        @name = name
+        @raw_status = raw_status
+        @updated_at = updated_at
+      end
+
+      def assign_status_data(key:, message: nil)
+        @key = key
+        @message = message
+      end
+
+      def invalid_state_warning
+        warning = "The Money Forward account named #{name} is in an invalid state: #{key}"
+        warning << " (#{message})" if message
+        warning << ". Please handle the issue manually to resume syncing."
+      end
+
+      def outdated?
+        # Require accounts to have been updated in the last day
+        # FIXME: make configurable?
+        updated_at < Time.now - FRESHNESS_LIMIT
+      end
+
+      def should_update?(update_invalid:)
+        (key == :success && outdated?) ||
+          (update_invalid && failed_state?)
+      end
+
+      def failed_state?
+        !%i[success processing].include?(key)
+      end
+
+      def trigger_update(session)
+        session.http_post(
+          "/faggregation_queue2/#{id_hash}",
+          URI.encode_www_form(commit: "更新"),
+          "X-CSRF-Token" => session.csrf_token,
+        )
+      end
+    end
+  end
+end

data/lib/mfynab/money_forward/session.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "ferrum"
+require "net/http"
+require "uri"
+
+module MFYNAB
+  class MoneyForward
+    class Session
+      COOKIE_NAME = "_moneybook_session"
+      DEFAULT_BASE_URL = "https://moneyforward.com"
+      SIGNIN_PATH = "/sign_in"
+
+      def initialize(username:, password:, logger:, base_url: DEFAULT_BASE_URL)
+        @username = username
+        @password = password
+        @logger = logger
+        @base_url = URI(base_url)
+      end
+
+      def login
+        logger.info("Logging in to Money Forward...")
+        with_ferrum do |browser|
+          submit_login_form(browser)
+
+          self.cookie = browser.cookies[COOKIE_NAME]
+        rescue Timeout::Error
+          # FIXME: use custom error class
+          raise "Login failed"
+        end
+      end
+
+      def cookie
+        @cookie || login
+      end
+
+      def http_get(path, params = {})
+        path = URI.join(base_url, path)
+        path.query = URI.encode_www_form(params) unless params.empty?
+        request = Net::HTTP::Get.new(path, "Cookie" => "#{COOKIE_NAME}=#{cookie.value}")
+        http_request(request)
+      end
+
+      def http_post(path, body, headers = {})
+        request = Net::HTTP::Post.new(
+          path,
+          "Cookie" => "#{COOKIE_NAME}=#{cookie.value}",
+          **headers,
+        )
+        request.body = body
+        http_request(request)
+      end
+
+      # FIXME: not sure a CSRF token is generic to the session
+      # Maybe it has different instances depending on the page it's on?
+      def csrf_token
+        @_csrf_token ||= Nokogiri::HTML(http_get("/accounts"))
+          .at_css("meta[name='csrf-token']")[:content]
+      end
+
+      private
+
+        attr_reader :username, :password, :logger, :base_url
+        attr_writer :cookie
+
+        def http_request(request)
+          # FIXME: switch to Faraday or another advanced HTTP library?
+          # (Better error handling, response parsing, cookies, possibly keep the connection open, etc.)
+          Net::HTTP.start(base_url.host, use_ssl: true) do |http|
+            result = http.request(request)
+            raise "Got unexpected result: #{result.inspect}" unless result.is_a?(Net::HTTPSuccess)
+
+            result.body
+          end
+        end
+
+        def with_ferrum
+          browser = Ferrum::Browser.new(
+            timeout: 30,
+            process_timeout: 20, # 10s was not always enough on CI (FIXME: make configurable per env)
+            headless: !ENV.key?("NO_HEADLESS"),
+          )
+          user_agent = browser.default_user_agent.sub("HeadlessChrome", "Chrome")
+          browser.headers.add({
+            "Accept-Language" => "en-US,en",
+            "User-Agent" => user_agent,
+          })
+          yield browser
+        rescue StandardError
+          # FIXME: add datetime to path
+          browser&.screenshot(path: "screenshot.png")
+          logger.error("An error occurred and a screenshot was saved to ./screenshot.png")
+          raise
+        ensure
+          browser&.quit
+        end
+
+        def submit_login_form(browser)
+          browser.goto("#{base_url}#{SIGNIN_PATH}")
+          browser.at_css("input[type='email']").focus.type(username)
+          browser.at_css("input[type='password']").focus.type(password, :Enter)
+          Timeout.timeout(5) do
+            sleep 0.1 until browser.body.include?("ログアウト")
+          end
+        end
+    end
+  end
+end

data/lib/mfynab/money_forward.rb

@@ -1,95 +1,96 @@
 # frozen_string_literal: true
 
-require "ferrum"
+require "nokogiri"
+require "mfynab/money_forward/account_status"
 
 module MFYNAB
   class MoneyForward
-    DEFAULT_BASE_URL = "https://moneyforward.com"
-    SIGNIN_PATH = "/sign_in"
     CSV_PATH = "/cf/csv"
-    SESSION_COOKIE_NAME = "_moneybook_session"
-    USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " \
-                 "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
 
-    def initialize(logger:, base_url: DEFAULT_BASE_URL)
-      @base_url = URI(base_url)
+    def initialize(session, logger:)
+      @session = session
       @logger = logger
     end
 
-    def get_session_id(username:, password:)
-      with_ferrum do |browser|
-        browser.goto("#{base_url}#{SIGNIN_PATH}")
-        browser.at_css("input[name='mfid_user[email]']").focus.type(username)
-        browser.at_css("input[name='mfid_user[password]']").focus.type(password)
-        browser.at_css("button#submitto").click
+    def update_accounts(account_names, update_invalid: true)
+      account_statuses = AccountStatus::Fetcher.new(session, logger: logger).fetch(account_names: account_names)
 
-        # FIXME: use custom error class
-        raise "Login failed" unless browser.cookies[SESSION_COOKIE_NAME]
+      finished = account_statuses.map do |account_status|
+        ensure_account_updated(account_status, update_invalid: update_invalid)
+      end.all?
 
-        browser.cookies[SESSION_COOKIE_NAME].value
-      end
+      return if finished
+
+      logger.info("Waiting for a while before checking status again...")
+      # FIXME: I'm never comfortable with using sleep().
+      # Do I want to implement a solution based on callbacks?
+      # For example, the script could say:
+      # > Accounts are out of date.
+      # > I triggered the updates, and will call myself again in X seconds/minutes.
+      # > When the accounts are updated, I'll proceed to the next step.
+      sleep(5)
+      update_accounts(account_names, update_invalid: false)
     end
 
-    def download_csv(session_id:, path:, months:)
+    def download_csv(path:, months:)
       month = Date.today
       month -= month.day - 1 # First day of month
+      months.times do
+        date_string = month.strftime("%Y-%m")
 
-      Net::HTTP.start(base_url.host, use_ssl: true) do |http|
-        http.response_body_encoding = Encoding::SJIS
-
-        months.times do
-          date_string = month.strftime("%Y-%m")
-
-          logger.info("Downloading CSV for #{date_string}")
+        logger.info("Downloading CSV for #{date_string}")
 
-          # FIXME: I don't really need to save the CSV files to disk anymore.
-          # Maybe just return parsed CSV data?
-          File.open(File.join(path, "#{date_string}.csv"), "wb") do |file|
-            file << download_csv_string(date: month, session_id: session_id)
-          end
-
-          month = month.prev_month
+        # FIXME: I don't really need to save the CSV files to disk anymore.
+        # Maybe just return parsed CSV data?
+        File.open(File.join(path, "#{date_string}.csv"), "wb") do |file|
+          file << download_csv_string(date: month)
         end
+
+        month = month.prev_month
       end
     end
 
-    def download_csv_string(date:, session_id:)
-      Net::HTTP.start(base_url.host, use_ssl: true) do |http|
-        http.response_body_encoding = Encoding::SJIS
-
-        request = Net::HTTP::Get.new(
-          "#{CSV_PATH}?from=#{date.strftime('%Y/%m/%d')}",
-          {
-            "Cookie" => "#{SESSION_COOKIE_NAME}=#{session_id}",
-            "User-Agent" => USER_AGENT,
-          },
-        )
-
-        result = http.request(request)
-        raise unless result.is_a?(Net::HTTPSuccess)
-        raise unless result.body.valid_encoding?
-
-        result.body.encode(Encoding::UTF_8)
-      end
+    # FIXME: make private or inline
+    def download_csv_string(date:)
+      # FIXME: handle errors/edge cases
+      session
+        .http_get(CSV_PATH, from: date.strftime("%Y/%m/%d"))
+        .force_encoding(Encoding::SJIS)
+        .encode(Encoding::UTF_8)
     end
 
     private
 
-      attr_reader :base_url, :logger
-
-      def with_ferrum
-        browser = Ferrum::Browser.new(timeout: 30, headless: !ENV.key?("NO_HEADLESS"))
-        browser.headers.add({
-          "Accept-Language" => "en-US,en",
-          "User-Agent" => USER_AGENT,
-        })
-        yield browser
-      rescue StandardError
-        browser.screenshot(path: "screenshot.png")
-        logger.error("An error occurred and a screenshot was saved to ./screenshot.png")
-        raise
-      ensure
-        browser&.quit
+      attr_reader :session, :logger
+
+      def ensure_account_updated(account_status, update_invalid:)
+        if account_status.should_update?(update_invalid: update_invalid)
+          suffix = "Will update"
+          account_status.trigger_update(session)
+          updated = false
+        elsif account_status.key == :processing
+          suffix = "Updating"
+          updated = false
+        elsif account_status.key == :success
+          suffix = "Up to date"
+          updated = true
+        else
+          # FIXME: we can probably handle :additional_request and/or :login right here in the script,
+          # if we can find the time.
+          # FIXME: sometimes, nothing can be done about the status, for example, I can currently see モバイルSUICA
+          # show an `errors` status with this message:
+          # ただいま大変混み合っております。今しばらくお待ちいただきますようお願いいたします。 失敗日時：02/22 14:03
+          # In other words: "please wait". Maybe improve messaging?
+          suffix = "Ignoring"
+          updated = true
+          logger.warn(account_status.invalid_state_warning)
+        end
+
+        # FIXME: I'll probably want to refresh the display instead of relogging everything every 5 seconds.
+        # FIXME: would be nice to tabulate the info better, for readability.
+        logger.info("#{account_status.name}:\t#{account_status.key} (#{account_status.updated_at}) - #{suffix}")
+
+        updated
       end
   end
 end

data/lib/mfynab/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MFYNAB
-  VERSION = "0.1.4"
+  VERSION = "0.2.0"
 end

data/lib/mfynab/ynab_transaction_importer.rb

@@ -4,6 +4,11 @@ require "ynab"
 
 module MFYNAB
   class YnabTransactionImporter
+    # See https://github.com/ynab/ynab-sdk-ruby/issues/77
+    MEMO_MAX_LENGTH = 500
+    PAYEE_MAX_LENGTH = 200
+    IMPORT_ID_MAX_LENGTH = 36
+
     def initialize(api_key, budget_name, account_mappings, logger:)
       @api_key = api_key
       @budget_name = budget_name
@@ -42,7 +47,7 @@ module MFYNAB
         {
           account_id: account.id,
           amount: row["amount"] * 1_000,
-          payee_name: row["content"][0, 100],
+          payee_name: row["content"][0, PAYEE_MAX_LENGTH],
           date: Date.strptime(row["date"], "%Y/%m/%d").strftime("%Y-%m-%d"),
           cleared: "cleared",
           memo: generate_memo_for(row),
@@ -65,9 +70,7 @@ module MFYNAB
         memo_parts
           .delete_if { _1.nil? || _1.empty? }
           .join(" - ")
-          .slice(0, 200) # YNAB's API currently limits memo to 200 characters,
-        # even though YNAB itself allows longer memos. See:
-        # https://github.com/ynab/ynab-sdk-ruby/issues/77
+          .slice(0, MEMO_MAX_LENGTH)
       end
 
       # ⚠️ Be very careful when changing this method!
@@ -91,16 +94,14 @@ module MFYNAB
         # but changing it now would require a lot of work in preventing import
         # duplicates due to inconsistent import_id.
         prefix = "MFBY:v1:"
-
-        max_length = 36     # YNAB API limit
-        id_max_length = 28  # this leaves 8 characters for the prefix
+        digest_max_length = IMPORT_ID_MAX_LENGTH - prefix.length
 
         id = row["id"]
 
         # Only hash if the ID would exceed YNAB's limit.
         # This improves backwards compatibility with old import_ids.
-        if prefix.length + id.length > max_length
-          id = Digest::SHA256.hexdigest(id)[0, id_max_length]
+        if prefix.length + id.length > IMPORT_ID_MAX_LENGTH
+          id = Digest::SHA256.hexdigest(id)[0, digest_max_length]
         end
 
         prefix + id
@@ -118,12 +119,17 @@ module MFYNAB
 
         ynab_account_name = matching_mapping["ynab_name"]
 
-        ynab_account = accounts.find do |account|
+        ynab_accounts = accounts.select do |account|
           account.name.include?(ynab_account_name)
         end
-        raise "No YNAB account found with name #{ynab_account_name}." unless ynab_account
 
-        ynab_account
+        if ynab_accounts.empty?
+          raise "No YNAB account found with name #{ynab_account_name}."
+        elsif ynab_accounts.size > 1
+          raise "Found multiple YNAB accounts matching name #{ynab_account_name}."
+        end
+
+        ynab_accounts.first
       end
 
       def ynab_transactions_api

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mfynab
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - David Stosik
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-25 00:00:00.000000000 Z
+date: 2025-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: csv
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.15'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.18'
 - !ruby/object:Gem::Dependency
   name: psych
   requirement: !ruby/object:Gem::Requirement
@@ -58,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.4'
+        version: '3.6'
 description: Sync transaction history from MoneyForward to YNAB.
 email:
 - david.stosik+git-noreply@gmail.com
@@ -77,6 +91,8 @@ files:
 - exe/mfynab
 - lib/mfynab/cli.rb
 - lib/mfynab/money_forward.rb
+- lib/mfynab/money_forward/account_status.rb
+- lib/mfynab/money_forward/session.rb
 - lib/mfynab/money_forward_data.rb
 - lib/mfynab/version.rb
 - lib/mfynab/ynab_transaction_importer.rb
@@ -103,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.11
+rubygems_version: 3.5.22
 signing_key:
 specification_version: 4
 summary: Sync transaction history from MoneyForward to YNAB.