ydim 1.1.4 → 1.1.6

data/lib/ydim/reconciler.rb

@@ -0,0 +1,223 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# YDIM::Reconciler -- ydim -- 09.08.2026 -- zdavatz@ywesee.com
+
+require 'ydim/camt'
+
+module YDIM
+  # Matches booked bank credits (YDIM::Camt::Entry) against invoices. Works on
+  # Invoice::Info structs rather than Invoice objects so that the result can
+  # travel over DRb, and so that it can be unit-tested without a database.
+  class Reconciler
+    # Words that say nothing about who paid, and would otherwise make two
+    # unrelated companies look like the same debitor.
+    NOISE = %w{ag gmbh sa sarl srl inc ltd co kg the und and dr med prof}
+    # Only these are safe to book without a human looking at them: the payer
+    # named the invoice and the money adds up to the cent.
+    APPLICABLE = [:exact, :split]
+
+    # The bit of an invoice reconciliation cares about. Results travel back to
+    # the client over DRb, and a client should not have to load the whole
+    # server just to unmarshal them, so matches carry these rather than
+    # Invoice::Info. Built from anything answering the same readers.
+    class InvoiceRef
+      KEYS = [:unique_id, :total_brutto, :currency, :date, :debitor_name,
+        :payment_received, :deleted]
+      attr_reader *KEYS
+      def initialize(invoice)
+        KEYS.each { |key|
+          instance_variable_set("@#{key}", invoice.send(key))
+        }
+      end
+      def payable?
+        !@payment_received && !@deleted
+      end
+      def total_cents
+        (@total_brutto.to_f * 100).round
+      end
+      def to_s
+        sprintf('%s %s %.2f %s', @unique_id, @currency, @total_brutto.to_f,
+          @debitor_name)
+      end
+    end
+
+    # One bank entry and the invoices it was matched to.
+    class Match
+      attr_reader :entry, :invoices, :state
+      attr_accessor :reason, :applied
+      def initialize(entry, invoices, state, reason = nil)
+        @entry = entry
+        @invoices = invoices
+        @state = state
+        @reason = reason
+        @applied = false
+      end
+      def applicable?
+        APPLICABLE.include?(@state)
+      end
+      # Matched, but not well enough to book automatically.
+      def review?
+        !applicable? && !@invoices.empty?
+      end
+      def applied?
+        @applied
+      end
+      def invoice_ids
+        @invoices.collect { |info| info.unique_id }
+      end
+    end
+
+    # What a reconciliation run found, including the entries that never made it
+    # as far as matching -- silently dropping those would make the report look
+    # like it had seen everything.
+    class Result
+      attr_reader :matches, :skipped
+      def initialize
+        @matches = []
+        @skipped = Hash.new(0)
+      end
+      def applicable
+        @matches.select { |match| match.applicable? }
+      end
+      def review
+        @matches.select { |match| match.review? }
+      end
+      def unmatched
+        @matches.select { |match| match.state == :unmatched }
+      end
+      def applied
+        @matches.select { |match| match.applied? }
+      end
+    end
+
+    attr_reader :accounts
+    # infos    - Invoice::Info structs of the invoices still awaiting payment;
+    #            these are the candidates for matching on amount alone.
+    # accounts - IBANs ydim owns; entries on any other account are ignored.
+    #            ydim only ever sees the ywesee business account, but the
+    #            e-banking download also carries the private ones.
+    # resolver - optional callable turning an invoice number into an Info, so
+    #            that a payment naming an invoice that is already settled can
+    #            be reported as such without loading the whole invoice book.
+    def initialize(infos, accounts = nil, resolver = nil)
+      @invoices = infos.collect { |info| InvoiceRef.new(info) }
+      @accounts = [accounts].flatten.compact
+      @resolver = resolver
+      @by_id = {}
+      @invoices.each { |ref| @by_id[ref.unique_id.to_s] = ref }
+    end
+    def reconcile(entries)
+      if @accounts.empty?
+        raise ArgumentError,
+          "no account given -- set camt_accounts to the IBAN of the ywesee " \
+          "business account, or the private accounts in the same download " \
+          "would be matched against invoices too"
+      end
+      result = Result.new
+      unique = YDIM::Camt.dedup(entries)
+      result.skipped[:duplicate] = entries.size - unique.size
+      unique.each { |entry|
+        if !entry.account?(@accounts)
+          result.skipped[:other_account] += 1
+        elsif !entry.credit?
+          result.skipped[:debit] += 1
+        elsif !entry.booked?
+          result.skipped[:not_booked] += 1
+        else
+          result.matches.push(match(entry))
+        end
+      }
+      result
+    end
+
+    private
+    def match(entry)
+      referenced = entry.numeric_tokens.collect { |token|
+        lookup(token.sub(/\A0+(?=\d)/, '')) || lookup(token)
+      }.compact.uniq
+      if referenced.empty?
+        match_by_amount(entry)
+      else
+        match_by_reference(entry, referenced)
+      end
+    end
+    def match_by_reference(entry, referenced)
+      open, paid = referenced.partition { |ref| ref.payable? }
+      if open.empty?
+        return Match.new(entry, paid, :not_open,
+          "invoice #{paid.collect { |ref| ref.unique_id }.join(', ')} " \
+          "is no longer open")
+      end
+      wrong = open.reject { |ref| currency?(entry, ref) }
+      unless wrong.empty?
+        return Match.new(entry, open, :currency_mismatch,
+          "paid in #{entry.currency}, invoiced in #{wrong.first.currency}")
+      end
+      total = open.inject(0) { |sum, ref| sum + ref.total_cents }
+      if total == entry.amount_cents
+        state = open.size == 1 ? :exact : :split
+        reason = open.size == 1 ? 'invoice number and amount match' \
+          : "invoice numbers found, amounts sum to #{money(entry)}"
+        reason += ", #{paid.size} already paid" unless paid.empty?
+        Match.new(entry, open, state, reason)
+      elsif entry.amount_cents < total
+        Match.new(entry, open, :underpaid,
+          "#{money(entry)} received, #{format_cents(total)} invoiced")
+      else
+        Match.new(entry, open, :overpaid,
+          "#{money(entry)} received, #{format_cents(total)} invoiced")
+      end
+    end
+    # Nothing in the remittance information named an invoice -- fall back to
+    # the amount, which is never good enough to book on its own.
+    def match_by_amount(entry)
+      candidates = @invoices.select { |ref|
+        ref.payable? && currency?(entry, ref) \
+          && ref.total_cents == entry.amount_cents \
+          && !later_than?(ref, entry)
+      }
+      named = candidates.select { |ref| same_party?(entry, ref) }
+      candidates = named unless named.empty?
+      case candidates.size
+      when 0
+        Match.new(entry, [], :unmatched, 'no invoice number, no amount match')
+      when 1
+        Match.new(entry, candidates, :amount_only,
+          named.empty? ? 'amount matches, no invoice number given' \
+            : 'amount and debitor name match, no invoice number given')
+      else
+        Match.new(entry, candidates, :ambiguous,
+          "#{candidates.size} open invoices over #{money(entry)}")
+      end
+    end
+    def lookup(id)
+      return @by_id[id] if @by_id.key?(id)
+      found = @resolver && @resolver.call(id)
+      @by_id[id] = found && InvoiceRef.new(found)
+    end
+    def currency?(entry, ref)
+      ref.currency.nil? || entry.currency.nil? \
+        || ref.currency.to_s.upcase == entry.currency.to_s.upcase
+    end
+    # An invoice cannot have been paid before it was written.
+    def later_than?(ref, entry)
+      ref.date && entry.booking_date && ref.date > entry.booking_date
+    end
+    def same_party?(entry, ref)
+      left = words(entry.counterparty)
+      right = words(ref.debitor_name)
+      !left.empty? && !right.empty? && !(left & right).empty?
+    end
+    def words(str)
+      str.to_s.downcase.split(/[^[[:alpha:]]]+/).reject { |word|
+        word.size < 3 || NOISE.include?(word)
+      }
+    end
+    def money(entry)
+      sprintf('%s %.2f', entry.currency, entry.amount.to_f)
+    end
+    def format_cents(total)
+      sprintf('%.2f', total / 100.0)
+    end
+  end
+end

data/lib/ydim/root_session.rb

@@ -7,6 +7,7 @@ require 'ydim/debitor'
 require 'ydim/invoice'
 require 'ydim/item'
 require 'ydim/mail'
+require 'ydim/reconciler'
 require 'odba'
 
 module YDIM
@@ -125,6 +126,51 @@ module YDIM
 			@serv.logger.debug(whoami) { "invoice_infos(#{status})" }
 			Invoice.search_by_status(status).collect { |inv| inv.info }
 		end
+    # The invoices a payment could still be settling. Both statuses are asked
+    # for because the status index is only refreshed once a day, so an invoice
+    # that has just fallen due may still be indexed as open.
+    def open_invoice_infos
+      @serv.logger.debug(whoami) { "open_invoice_infos" }
+      (Invoice.search_by_status('is_open') \
+        | Invoice.search_by_status('is_due')).collect { |inv| inv.info }
+    end
+    # Marks an invoice as paid. Stores the date the money was booked rather
+    # than a plain true, so that a later reconciliation run can tell an
+    # already-booked payment from a fresh one.
+    def mark_paid(invoice_id, date=Date.today)
+      @serv.logger.info(whoami) { "mark_paid(#{invoice_id}, #{date})" }
+      ODBA.transaction {
+        inv = invoice(invoice_id)
+        inv.payment_received = date
+        inv.odba_store
+        inv.info
+      }
+    end
+    # Matches booked bank credits against the invoice book. `entries` are
+    # YDIM::Camt::Entry objects, parsed client-side -- the daemon never needs
+    # to see the statement files. Read-only unless :apply is given, in which
+    # case only unambiguous matches (invoice number named *and* amount to the
+    # cent) are booked; everything else is left for a human.
+    def reconcile_camt(entries, opts={})
+      @serv.logger.info(whoami) {
+        "reconcile_camt(#{entries.size} entries, #{opts.inspect})" }
+      accounts = opts[:accounts] || @serv.config.camt_accounts
+      resolver = lambda { |id|
+        inv = Invoice.find_by_unique_id(id.to_s)
+        inv && inv.info
+      }
+      result = Reconciler.new(open_invoice_infos, accounts, resolver)\
+        .reconcile(entries)
+      if opts[:apply]
+        result.applicable.each { |match|
+          match.invoices.each { |info|
+            mark_paid(info.unique_id, match.entry.booking_date)
+          }
+          match.applied = true
+        }
+      end
+      result
+    end
 		def search_debitors(email_or_name)
 			@serv.logger.debug(whoami) { "search_debitors(#{email_or_name})" }
 			Debitor.search_by_exact_email(email_or_name) |

data/lib/ydim/server_config.rb

@@ -15,6 +15,11 @@ module YDIM
       'autoinvoice_hour'      => 1,
       'config'                => default_config_files,
       'conf_dir'              => File.join(ydim_default_dir, 'conf'),
+      # IBANs of the accounts ydim invoices are paid into. The e-banking camt
+      # download also carries the private accounts, whose entries must never be
+      # matched against invoices, so leaving this empty disables reconciliation
+      # rather than reconciling against everything.
+      'camt_accounts'         => [],
       'currencies'            => ['CHF', 'EUR', 'USD'],
       'currency_update_hour'  => 2,
       'data_dir'              => File.join(ydim_default_dir, 'data'),
@@ -46,7 +51,7 @@ module YDIM
       'smtp_port'             => 587,
       'smtp_server'           => 'localhost',
       'smtp_user'             => 'ydim@ywesee.com',
-      'vat_rate'              => 7.7,
+      'vat_rate'              => 8.1,
     }
     CONFIG =  RCLConf::RCLConf.new(ARGV, defaults)
     def Server.config

data/lib/ydim/version.rb

@@ -1,3 +1,3 @@
 module YDIM
-  VERSION = '1.1.4'
+  VERSION = '1.1.6'
 end

data/lib/ydim/ydim-camt

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+# ydim-camt -- ydim -- 09.08.2026 -- zdavatz@ywesee.com
+#
+# Reconciles bank statements (ISO-20022 camt.053, as downloaded from UBS
+# e-banking) against the open invoices. Reports by default; only books a
+# payment when told to with --apply, and even then only when the payer named
+# the invoice and the amount matches to the cent.
+#
+#   ydim-camt ~/Downloads/statements.zip
+#   ydim-camt --apply ~/Downloads/statements.zip
+#   ydim-camt --account CH87... /var/ydim/camt/
+
+require 'openssl'
+require 'optparse'
+require 'rrba/error'
+require 'ydim/camt'
+require 'ydim/client'
+require 'ydim/config'
+require 'ydim/reconciler'
+
+options = { :apply => false, :verbose => false, :accounts => [] }
+parser = OptionParser.new { |opt|
+  opt.banner = "Usage: ydim-camt [options] <statements.zip|directory|file.xml>"
+  opt.on('--apply', 'mark unambiguously matched invoices as paid') {
+    options[:apply] = true }
+  opt.on('--account IBAN', 'account to reconcile (repeatable); ',
+    'defaults to camt_accounts from the config') { |iban|
+    options[:accounts].push(iban) }
+  opt.on('-v', '--verbose', 'also list credits that matched nothing') {
+    options[:verbose] = true }
+  opt.on('-h', '--help') { puts opt; exit }
+}
+# Every ydim executable also takes its configuration on the command line, in
+# RCLConf's "key=value" form -- those are not statements to read, they were
+# already picked up from ARGV when ydim/config was required.
+paths = parser.parse(ARGV.dup).reject { |arg| arg =~ /\A[\w.]+=/ }
+if paths.empty?
+  warn parser
+  exit 1
+end
+missing = paths.reject { |path| File.exist?(path) }
+unless missing.empty?
+  warn "no such file: #{missing.join(', ')}"
+  exit 1
+end
+
+entries = paths.flat_map { |path| YDIM::Camt.entries(path) }
+if entries.empty?
+  warn "no camt entries found in #{paths.join(', ')}"
+  exit 1
+end
+
+config = YDIM::Client::CONFIG
+accounts = options[:accounts]
+accounts = [config.camt_accounts].flatten.compact if accounts.empty?
+if accounts.empty?
+  warn "no account to reconcile: pass --account IBAN, or set camt_accounts " \
+    "in the config to the IBAN of the ywesee business account. The private " \
+    "accounts in the same download must not be matched against invoices."
+  exit 1
+end
+
+def show(match, marker)
+  ids = match.invoice_ids
+  printf("  %-11s %10.2f %s  %s  %s\n", ids.first || '?',
+    match.entry.amount.to_f, match.entry.currency,
+    match.entry.booking_date, match.entry.counterparty.to_s[0, 30])
+  ids[1..-1].to_a.each { |id| printf("  %-11s\n", id) }
+  puts "  #{' ' * 11} #{marker} #{match.reason}"
+end
+
+server = DRb::DRbObject.new(nil, config.server_url)
+client = YDIM::Client.new(config)
+key = OpenSSL::PKey::DSA.new(File.read(config.private_key))
+
+DRb.start_service
+client.login(server, key)
+begin
+  result = client.reconcile_camt(entries,
+    :apply => options[:apply], :accounts => accounts)
+
+  applicable = result.applicable
+  puts(options[:apply] ? "BOOKED AS PAID" \
+    : "MATCHED (would be marked paid with --apply)")
+  if applicable.empty?
+    puts "  none"
+  else
+    applicable.each { |match| show(match, match.applied? ? '+' : '~') }
+  end
+
+  review = result.review
+  unless review.empty?
+    puts
+    puts "REVIEW (never booked automatically)"
+    review.each { |match| show(match, '?') }
+  end
+
+  unmatched = result.unmatched
+  if options[:verbose] && !unmatched.empty?
+    puts
+    puts "UNMATCHED CREDITS"
+    unmatched.each { |match| show(match, '-') }
+  end
+
+  puts
+  summary = ["#{applicable.size} matched", "#{review.size} to review",
+    "#{unmatched.size} unmatched"]
+  summary.push("#{result.applied.size} booked") if options[:apply]
+  result.skipped.sort_by { |reason, count| reason.to_s }.each { |reason, count|
+    summary.push("#{count} #{reason.to_s.tr('_', ' ')}") if count > 0
+  }
+  puts summary.join(', ')
+  unless options[:apply] || applicable.empty?
+    puts "run again with --apply to mark the #{applicable.size} matched " \
+      "invoice(s) paid"
+  end
+ensure
+  client.logout
+end

data/lib/ydim/ydim-edit

@@ -19,10 +19,9 @@ module YDIM
 	end
 end
 
-ydim_default_dir = File.join(ENV['HOME'], '.ydim')
+ydim_default_dir = '/etc/ydim'
 default_config_files = [
-	File.join(ydim_default_dir, 'ydimd.yml'),
-	'/etc/ydim/ydimd.yml',
+	File.join(ydim_default_dir, 'ydim.yml'),
 ]
 defaults = {
 	'autoinvoice_hour'			=> nil,
@@ -45,7 +44,7 @@ defaults = {
 	'root_key'							=> 'root_dsa',
 	'smtp_from'							=> '',
 	'smtp_server'						=> 'localhost',
-	'vat_rate'							=> 7.7,
+	'vat_rate'							=> 8.1,
 }
 config = RCLConf::RCLConf.new(ARGV, defaults)
 config.load(config.config)

data/readme.md

@@ -4,7 +4,11 @@
 
 ## DESCRIPTION:
 
-ywesee distributed invoice manager, Ruby
+ywesee distributed invoice manager, Ruby.
+
+A DRb daemon (`ydimd`) that keeps debitors, invoices and recurring auto-invoices in a
+PostgreSQL database via ODBA, renders invoices as PDF and mails them out. Clients talk to
+the daemon over `druby://` and authenticate with a DSA key.
 
 ## Install Ruby
 
@@ -12,6 +16,8 @@ ywesee distributed invoice manager, Ruby
 * echo 'eval "$(~/.rbenv/bin/rbenv init - bash)"' >> ~/.bashrc
 * git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build
 
+Tested against Ruby 3.0, 3.1 and 3.2 (see `.github/workflows/ruby.yml`).
+
 ## Install Postgresql
 ```
 * sudo apt-get install postgresql-10 postgresql-contrib-10
@@ -26,8 +32,12 @@ ywesee distributed invoice manager, Ruby
 * -> check DB
 * psql
 *  \l
-* bzcat 22:00-postgresql_database-ch_oddb-backup.bz2 | sudo -u postgres psql -p 5433 ydim
+* bzcat 22:00-postgresql_database-ydim-backup.bz2 | sudo -u postgres psql -p 5433 ydim
 ```
+
+`set_initial_ydim_db.sql` creates the ODBA schema for an empty database. The
+`get_db_ydim` script fetches the nightly dump from the backup host and restores it locally.
+
 ## INSTALL:
 
 * gem install ydim
@@ -41,12 +51,87 @@ Or if you are using bundler
 * bundle config build.pg --with-pg-config=/usr/local/pgsql-10.1/bin/pg_config
 * bundle install
 
+## CONFIGURATION:
+
+All server side configuration is read from `/etc/ydim` (since version 1.1.4/1.1.5):
+
+* `/etc/ydim/ydimd.yml` — the daemon (database, SMTP, VAT rate, invoice numbering, log level).
+  Defaults are in `lib/ydim/server_config.rb`.
+* `/etc/ydim/ydim.yml` — the client (`server_url`, `private_key`, currency, payment period).
+  Defaults are in `lib/ydim/config.rb`.
+* `camt_accounts` (in both files) — the IBANs ydim invoices are paid into, used by
+  `ydim-camt`. See *Checking payments* below.
+* `/etc/ydim/conf/` — key material, e.g. the `root_dsa` public key used to authenticate.
+* `~/.pdfinvoice/config.yml` or `/etc/pdfinvoice/config.yml` — creditor address, bank
+  details, logo and texts of the generated PDF. Defaults are in `lib/pdfinvoice/config.rb`,
+  and `test/data/config.yml` is a working example.
+
+Any default may also be overridden on the command line, e.g. `--log_level DEBUG`.
+
+## RUNNING:
+
+* Start the daemon: `bundle exec ruby lib/ydim/ydimd` (listens on `druby://localhost:12375`)
+* Interactive console against the live database: `bundle exec lib/ydim/ydim-edit`
+  (gives you `$server` and the Needle registry as `$needle` inside IRB)
+* Create and send an invoice from a YAML description: `bundle exec lib/ydim/ydim-inject < invoice.yml`
+
+The daemon additionally runs three daily jobs: generating and mailing due auto-invoices,
+updating currency conversion rates, and refreshing the stored invoice status.
+
+## Checking payments
+
+`ydim-camt` matches the credits on a bank statement against the open invoices. It reads the
+ISO-20022 camt.053 files banks provide — the zip downloaded from UBS e-banking, an unpacked
+directory, or a single xml:
+
+```
+bundle exec lib/ydim/ydim-camt ~/Downloads/statements.zip           # report only
+bundle exec lib/ydim/ydim-camt --apply ~/Downloads/statements.zip   # and book them
+```
+
+Set `camt_accounts` to the IBAN(s) ydim invoices are paid into, or pass `--account IBAN`.
+**This is required.** An e-banking download contains every account the login can see,
+including private ones, and a credit there can look exactly like a payment on an invoice.
+Rather than reconcile whatever it is given, `ydim-camt` refuses to run without being told
+which account is the business one.
+
+Without `--apply` nothing is modified. With it, only invoices whose number the payer wrote
+in the remittance information **and** whose amount matches to the cent are marked paid; one
+transfer settling several invoices counts when the amounts sum exactly. Everything else —
+a payment with no reference, a partial payment, two open invoices over the same amount — is
+listed under REVIEW and left to you. Duplicate deliveries of the same statement, debits and
+pending entries are skipped and counted in the summary.
+
+Marking a single invoice paid by hand, from `ydim-edit` or any client:
+
+```ruby
+$server.mark_paid(13363, Date.new(2026, 7, 10))
+```
+
+## DEVELOPMENT:
+
+```
+bundle install
+MT_COMPAT=1 bundle exec rake test    # run the whole suite
+```
+
+`MT_COMPAT=1` is required because the tests use `flexmock/test_unit`, which needs the
+minitest compatibility shim. A single test file or a single test case:
+
+```
+MT_COMPAT=1 bundle exec ruby -Ilib -Itest test/test_invoice.rb
+MT_COMPAT=1 bundle exec ruby -Ilib -Itest test/test_invoice.rb -n test_add_item
+```
+
+`bundle exec rake` (the default task) cleans, runs the tests and builds the gem into `pkg/`.
+The test suite needs no database — `test/stub/odba.rb` stubs out ODBA persistence.
+
 ## Migrating an old database
 
 An old database can be migrated to UTF-8 by calling
 
     bundle install --path vendor
-    bundle exec bin/migrate_to_utf_8
+    bundle exec ./ydim_migrate_to_utf_8
 
 ## DEVELOPERS: