unipump 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 47b3ded4adb8542b7c97b5163ad60d83b2415ded901121e205451977d2269fcf
+  data.tar.gz: 62853432f82b5cd7a64ec0ad16fb24efe8e71a2ab59e7aab7e0971dff6f09065
+SHA512:
+  metadata.gz: 00c5a85ec55dfb5e29fd2a01d7255a2fd08b421686bb2d23f5aa1f8c7ca3a455061f23e7940d9b0efc170ca337fcd02a3048f7cea526e98e2a6e690d39e8095a
+  data.tar.gz: 3d2350a002c0e4f4b569ee7fb39868d036cd896e04954143b9a774d766af48c36b398deb69c738efd4136745931a9865b107a97bd23648045a0d0d6c16706b0a

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1 @@
+ruby-3.1.2

data/README.md

@@ -0,0 +1,31 @@
+# Timepump
+
+TODO: Delete this and the text below, and describe your gem
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/unipump`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+## Installation
+
+TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install UPDATE_WITH_YOUR_GEM_NAME_IMMEDIATELY_AFTER_RELEASE_TO_RUBYGEMS_ORG
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/unipump.

data/Rakefile

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/TODO

@@ -0,0 +1,5 @@
+o Hvad med engelsk/dansk tekst?
+o Diskuter record-by-record update af TIMEREG med Hans: Kan det ske, at der er
+  records, som ikke kommer med i v_fakturalinie?
+
++ Log

data/exe/unipump

@@ -0,0 +1,103 @@
+#!/usr/bin/env ruby
+
+require 'yaml'
+require 'shellopts'
+require 'unipump.rb'
+
+SPEC = %(
+  @ Upload invoice lines to Uniconta
+
+  -- END-DATE [ACCOUNT...]
+
+  Uploads outstanding invoice lines to Uniconta and updates the
+  TIDSREG.FAKTURADATO field in the database. Note that END-DATE is inclusive
+
+  unipump will process the given orders or all available orders if ACCOUNTs are
+  absent 
+
+  -e,environment=ENV
+    Set the run-time environment to 'production' or 'test'. In the test
+    environment, data will be uploaded to the Uniconta test system and
+    TIMEREG_TEST is updated in the database instead of the TIMEREG table. The
+    default is 'test'
+
+    FIXME: Remove this. Only used to test sagsys update
+
+  -U,username=USERNAME
+    Uniconta username. Note that this is insecure. Use a credentials file for
+    better security
+
+  --password=PASSWORD
+    Uniconta password. Note that this is insecure. Use a credentials file for
+    better security
+
+  -c,credentials=FILE
+    Credentials file. Default is ./.unipump.environment
+
+  -l,log=FILE
+    Log file. The file is only appended to
+
+  -x,xlog=FILE
+    Transaction log file. It is used to save the affected timereg records so
+    that we can recover if the database crashes after orders have been
+    uploaded. It is a manual operation to fix the database, though. Note that
+    it is an error if the file exists because it indicates that the last run
+    failed. The default is 'unipump.transaction'
+
+  -j,jlog=FILE
+    Write generated JSON data to FILE
+
+  -q,quiet
+    Be quiet. Messages will still be written to the log file
+
+  -d,dry-run
+    Don't do any changes
+)
+
+opts, args = ShellOpts.process(SPEC, ARGV)
+
+# Get end date and optinally account numbers
+end_date, *accounts = args.extract(1..)
+end_date =~ /^\d\d\d\d-\d\d-\d\d$/ or ShellOpts.error "Illegal date: '#{end_date}'. Use YYYY-MM-DD"
+
+# Get username/password
+if opts.username
+  if !opts.password
+    ShellOpts.error "Password is required"
+  else
+    username = opts.username
+    password = opts.password
+  end
+elsif opts.password
+  ShellOpts.error "Username is required"
+else
+  credentials_file = opts.credentials || Pump::DEFAULT_CREDENTIALS_FILE
+  if !File.exist?(credentials_file)
+    ShellOpts.error "Can't find credentials file '#{credentials_file}'"
+  else
+    yaml = YAML.load(IO.read credentials_file)
+    username = yaml['username']
+    password = yaml['password']
+    username or ShellOpts.error "Can't find username in '#{credentials_file}'"
+    password or ShellOpts.error "Can't find password in '#{credentials_file}'"
+  end
+end
+
+# Get environment
+environment = opts.environment || Pump::DEFAULT_ENVIRONMENT
+%w(test prod).include?(environment) or ShellOpts.error "Illegal environment '#{environment}'"
+
+# Pump object
+pump = Pump.new(
+    username,
+    password,
+    environment,
+    log: opts.log, 
+    xlog: opts.xlog,
+    jlog: opts.jlog,
+    dry_run: opts.dry_run?,
+    quiet: opts.quiet?)
+
+# Pumping
+pump.upload(end_date, accounts)
+

data/lib/unipump/credentials.rb

@@ -0,0 +1,20 @@
+module PloneLogin
+  module Credentials
+    def self.load
+      credentials  = {
+        host: ENV['PLONE_LOGIN_HOST'],
+        port: ENV['PLONE_LOGIN_PORT'],
+        database: ENV['PLONE_LOGIN_DATABASE'],
+        username: ENV['PLONE_LOGIN_USERNAME'],
+        password: ENV['PLONE_LOGIN_PASSWORD']
+      }
+
+      credentials.each { |k,v| 
+        next if [:port, :password].include?(k)
+        !v.nil? or raise "Undefined connection parameter: #{k}. Please define PLONE_LOGIN_#{k.to_s.upcase}" 
+      }
+
+      credentials
+    end
+  end
+end

data/lib/unipump/mssql.rb

@@ -0,0 +1,58 @@
+require "tiny_tds"
+
+module UniPump
+  module MsSql
+    CREDENTIALS = {
+      host: "mary.danak.lan",
+      port: 1435,
+      database: "sagsys",
+      username: "sa",
+      password: "JHe1Mff!"
+    }
+
+    def self.connect(credentials = CREDENTIALS) TinyTds::Client.new(credentials) end
+    def self.active?(conn) conn.active? end
+    def self.close(conn) conn.close end
+    def self.exec(conn, sql) conn.execute(sql) end
+    def self.size(result) result.each.size end
+    def self.each(result, &block) result.each(&block) end
+  end
+
+  class Connection
+    def initialize(credentials = MsSql::CREDENTIALS)
+      @connection = MsSql.connect(credentials)
+      ObjectSpace.define_finalizer(self, self.class.finalize(@connection))
+    end
+
+    def connected?() return MsSql.active?(@connection) rescue MsSql::Error; false end
+    def close() MsSql.close(@connection) if connected? end
+    def exec(sql) do_exec(sql).each end
+    def structs(sql) do_exec(sql).each.map { |row| OpenStruct.new(row) } end
+    def records(sql) do_exec(sql).each.map end
+
+  private
+    def self.finalize(connection) proc { do_finalize(connection) } end
+#   def self.do_finalize(connection) MsSql.close(connection) end
+    def self.do_finalize(connection) MsSql.close(connection) if MsSql.active?(connection) end
+
+    def do_exec(sql) Result.new(MsSql.exec(@connection, sql)) end
+  end
+
+  class Result
+    attr_reader :result
+    def size() MsSql.size(@result) end
+    def each(&block) MsSql.each(@result, &block) end
+  private
+    def initialize(result) @result = result end
+  end
+
+  def self.connect(credentials, &block)
+    conn = Connection.new(credentials)
+    begin
+      yield(conn)
+    ensure
+      conn.close
+    end
+  end
+end
+

data/lib/unipump/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module UniPump
+  VERSION = "0.2.0"
+end

data/lib/unipump.rb

@@ -0,0 +1,435 @@
+# frozen_string_literal: true
+
+# NOTE
+#   Test cases
+#     100149 100780 100502 100707 100080 100082 101948 101864 100094
+#
+# TODO
+#   o Use v_fakturalinie_test
+#   o More filtering in v_fakturalinie
+#   o Hardcoded 24 mill.
+#   o Add a --reset-test option/command
+#   o Fixe typer på ordrenummer, kontonummer, (pris), timer. Både i
+#     v_fakturalinie og sag
+#   o Fixe curr_comma hack
+#   o Check for Net::ReadTimeout
+#
+#   + .unipump.environment
+#   + Move #emit out of update methods
+
+require 'json'
+require 'net/http'
+require 'uri'
+require 'base64'
+require 'fileutils'
+require 'indented_io'
+
+require_relative "unipump/version"
+require_relative "unipump/mssql"
+
+module UniPump; end
+
+class Pump
+  class Error < StandardError; end
+
+  DEFAULT_CREDENTIALS_FILE = ".unipump.environment"
+  DEFAULT_ENVIRONMENT = "test"
+
+  # Environment-dependent constants
+  TEST_SYSTEM = "81470" # Uniconta test system ID
+  PROD_SYSTEM = nil
+  TEST_TIMEREG_TABLE = "tidsreg_test"
+  PROD_TIMEREG_TABLE = "tidsreg"
+  TEST_FAKTURALINIE_VIEW = "v_fakturalinie_test"
+  PROD_FAKTURALINIE_VIEW = "v_fakturalinie"
+
+  # Transaction file. This contains IDs of the most recently uploaded TIDSREG
+  # records. It is cleared when the corresponding records in the database are
+  # updated
+  TRANSACTION_FILE = "unipump.transaction"
+
+  attr_reader :username
+  attr_reader :password
+  attr_reader :environment # Run-time environment. "prod" or "test"
+  attr_reader :log # Path to log file
+  attr_reader :xlog # Path to transaction log file
+  attr_reader :jlog # Path to JSON log file or nil
+  attr_reader :quiet, :dry_run # Standard options
+
+  attr_reader :log_file # Log file object
+  attr_reader :conn # Database connection
+
+  attr_reader :curr_comma # Hack FIXME
+
+  # Environment-dependent resources
+  def system = (environment == "prod" ? PROD_SYSTEM : TEST_SYSTEM)
+  def tidsreg_table = (environment == "prod" ? PROD_TIMEREG_TABLE : TEST_TIMEREG_TABLE)
+  def fakturalinie_view = (environment == "prod" ? PROD_FAKTURALINIE_VIEW : TEST_FAKTURALINIE_VIEW)
+  def root_url = "https://odata.uniconta.com/api/Entities/#{system}"
+  def order_list_url = "#{root_url}/DebtorOrderClient"
+  def insert_order_list_url = "#{root_url}/InsertList/DebtorOrderClient"
+  def insert_order_line_list_url = "#{root_url}/InsertList/DebtorOrderLineClient"
+
+  def initialize(
+      username, password, environment,
+      log: nil, xlog: nil, jlog: nil,
+      dry_run: false, quiet: false)
+    @environment = environment
+    @username = username
+    @password = password
+    @log = log || '/dev/null'
+    @xlog = xlog || TRANSACTION_FILE
+    @jlog = jlog
+    @dry_run = dry_run
+    @quiet = quiet
+    @log_file = File.open(@log, "a")
+    @conn = UniPump::Connection.new
+    @curr_comma = ""
+    !File.exist?(@xlog) or ShellOpts::error "Found transaction file: #{@xlog}. Please fix this"
+    FileUtils.rm_f(@jlog) if @jlog
+  end
+
+  def upload(end_date, accounts)
+    t0 = Time.now
+
+    emit("#{ShellOpts.instance.name} @ #{Time.now.strftime("%F %T")}")
+
+    # Select specific accounts. Mostly for debug
+    konto_constraint = (accounts.empty? ? "and 1 = 1" : "and kontonummer in (#{accounts.join(', ')})")
+
+    # Fetch existing Uniconta orders
+    uniconta_order_numbers = 
+        emit_exec("  Fetch orders") { get_uniconta_order_numbers(order_list_url) }
+    emit "  Found #{uniconta_order_numbers.size} orders in Uniconta"
+
+    # Compute orders that are does not exist in Uniconta
+    uniconta_order_numbers.reject! { |order_number| 
+      if order_number.to_s.size < 8
+        emit "    Illegal order number: '#{order_number}', ignored"
+      elsif order_number < 24000000
+        emit "    Outdated order number: '#{order_number}', ignored"
+      end
+    }
+
+    # Find order with lines that have not been pumped yet (lines with
+    # fakturadato equal to null)
+    orders = conn.structs %(
+      select distinct
+        sagsnummer,
+        ansvarlig, 
+        kontonummer,
+        ordrenummer
+      from #{fakturalinie_view}
+      where dato < '#{end_date}'
+      and fakturadato is null
+      #{konto_constraint}
+      order by ansvarlig, sagsnummer
+    )
+    emit "  Found #{orders.size} orders in Sagsys"
+
+    # Exclude orders with kontonummer equal to nil
+    excluded = []
+    orders.delete_if { |order|
+      if order.kontonummer.nil?
+        excluded << order
+      end
+    }
+    emit "  #{excluded.size} with absent kontonummer - ignored" if !excluded.empty?
+
+    # Compute order numbers that should be created. This is computed as a Set
+    # for fast lookup
+    missing_order_numbers = Set.new(orders.map(&:ordrenummer).map(&:to_i) - uniconta_order_numbers)
+
+    # create order in Uniconta
+    emit "  Create #{missing_order_numbers.size} orders"
+    for order in orders
+      if missing_order_numbers.include?(order.ordrenummer.to_i)
+        json = format_order(order)
+        emit_exec("    #{order.ordrenummer}") { upload_json(insert_order_list_url, json) }
+      end
+    end
+
+    # Process each order
+    emit "  Upload orders"
+    for order in orders
+      # We want to save a list of uploaded tidsreg records in case the
+      # database update fails after the invoice lines have been uploaded to
+      # Uniconta, but we also want the database to aggregate the tidsreg
+      # records for us so we end up with the two queries below. It is
+      # important the the two queries selects the same records so we use the
+      # common method #line_constraint to supply the where constraint
+      #
+      # The list is written to a FILE that is removed after the records have
+      # been updated sucessfully. It is an error if it exists already
+      #
+      line_constraint = %(
+        dato <= '#{end_date}'
+        and ordrenummer = '#{order.ordrenummer}'
+        and fakturadato is null
+        #{konto_constraint}
+      )
+      
+      # Begin transaction. We hope that the default isolation level is
+      # resonable in MsSql
+      conn.exec("begin transaction")
+
+      # Find keys for affected TIMEREG records. These are saved to disk so
+      # that we may recover from a database failure after the order lines
+      # have been uploaded
+      tidsreg_keys = conn.records %(
+        select 
+          meid,
+          said,
+          acid,
+          dato
+        from #{fakturalinie_view}
+        where #{line_constraint}
+        order by sagsnummer
+      )
+
+      # Save tidsreg keys to disk
+      File.open(xlog, "w") { |xfile| 
+        xfile.puts "# Ordrenummer: #{order.ordrenummer} (#{order.sagsnummer})"
+        xfile.puts tidsreg_keys.to_a 
+      }
+
+      # Find the order lines. These are aggregated tidsreg records and could
+      # be coalesced into the previous select but then we would have to do
+      # grouping and ordering in ruby. Note that the select clause in the pre
+      order_lines = conn.structs %(
+        select
+          ordrenummer,
+          medarbejderogrolle as "rolle",
+          maaned,
+          tekst + ': ' + md + ' ' + cast(aar as nvarchar) as "tekst",
+          ydelse,
+          sum(timer) as antal,
+          pris
+        from #{fakturalinie_view} f
+        where #{line_constraint}
+        group by ordrenummer, medarbejderogrolle, acid, tekst, md, maaned, aar, ydelse, pris
+        order by ordrenummer, maaned, medarbejderogrolle, acid
+      )
+      emit "    #{order.ordrenummer} (#{order.sagsnummer}) #{order_lines.size} items"
+
+      # Create json object for HTTP
+      json = format_order_line(order, order_lines)
+
+      # Transfer JSON document. Note that HTTP redirects are not supported
+      # (yet)
+      emit_exec("      Upload") { upload_json(insert_order_line_list_url, json) }
+
+      # Update Sagsys database
+      emit_exec("      Update") { update_tidsreg(tidsreg_keys, end_date) }
+
+      # The whole operation completed successfully so we remove the transaction file
+      FileUtils.rm_f(xlog)
+    end
+
+    emit "Done @ #{Time.now.strftime("%F %T")}"
+    emit ""
+  end
+
+private
+  def get_uniconta_order_numbers(url) # ChatGPT
+    # Prepare HTTP request
+    uri = URI(url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true if uri.scheme == 'https'
+    request = Net::HTTP::Get.new(uri, 'Content-Type' => 'application/json')
+
+    # Add authorization
+    credentials = Base64.strict_encode64("#{username}:#{password}")
+    request['Authorization'] = "Basic #{credentials}"
+
+    response = nil
+    begin
+      response = http.request(request)
+      case response
+        when Net::HTTPSuccess
+          ; # nop
+        when Net::HTTPRedirection
+          raise StandardError, "Redirects are not supported (yet)"
+      else
+        raise StandardError, "#{response.code} - #{response.message}"
+      end
+
+      JSON.load(response.body).map { |r| r["OrderNumber"] }
+
+    rescue StandardError => ex
+      msg = ex.message
+      log_file.puts "ERROR: #{msg}"
+      ShellOpts.failure "Get order list failed: #{msg}"
+    end
+  end
+
+  def upload_json(url, json) # ChatGPT
+    return
+    # Write to json log
+    File.open(jlog, "a") { |f| f.puts json } if jlog 
+
+    return if dry_run
+
+    # Prepare HTTP request
+    uri = URI(url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = true if uri.scheme == 'https'
+    request = Net::HTTP::Post.new(uri, 'Content-Type' => 'application/json')
+
+    # Add authorization
+    credentials = Base64.strict_encode64("#{username}:#{password}")
+    request['Authorization'] = "Basic #{credentials}"
+
+    # Add data
+    request.body = json
+    response = nil
+
+    # Transfer JSON document. Note that HTTP redirects are not supported
+    # (yet)
+    begin
+      response = http.request(request)
+      case response
+        when Net::HTTPSuccess
+          ; # nop
+        when Net::HTTPRedirection
+          raise StandardError, "Redirects are not supported (yet)"
+      else
+        raise StandardError, "#{response.code} - #{response.message}"
+      end
+
+    rescue StandardError => ex
+      FileUtils.rm_f(xlog) # Nothing happened so we delete the transaction
+      log_file.puts "ERROR: #{ex.message}"
+      ShellOpts.failure "Upload failed: #{ex.message}"
+    end
+  end
+
+  def update_tidsreg(keys, end_date)
+    return if dry_run
+
+    # Update Sagsys TIDSREG.FAKTURADATO
+    begin
+      keys.each { |key|
+        conn.exec %(
+          update #{tidsreg_table}
+          set fakturadato = '#{end_date}' 
+          where meid = '#{key["meid"]}'
+          and said = '#{key["said"]}' 
+          and acid = '#{key["acid"]}'
+          and dato = '#{key["dato"].strftime("%F")}'
+        )
+      }
+
+      # Commit this order
+      conn.exec("commit")
+
+    rescue StandardError => ex
+      msg = "Update failed: #{ex.message}\nPlease save the transaction file '#{xlog}'"
+      log_file.puts "ERROR: #{msg}"
+      ShellOpts.failure msg
+    end
+  end
+
+  def emit(msg)
+    puts msg if !quiet
+    log_file.puts msg
+    true
+  end
+
+  def emit_n(msg)
+    print msg if !quiet
+    log_file.print msg
+    true
+  end
+
+  def emit_exec(text, &block)
+    begin
+      emit_n text
+      result = yield
+      emit ", ok"
+    rescue 
+      emit ", failed"
+      raise
+    end
+    result
+  end
+
+  def format_fields(out, fields)
+    out.puts "{ " +
+      fields.map { |field, value| 
+        case value 
+          when Integer
+            if field == "OrderNumber"
+              "\"#{field}\":#{value}" 
+            else
+              "\"#{field}\":\"#{value}\"" 
+            end
+          when Float, BigDecimal; 
+            "\"#{field}\":#{'%.2f' % value}"
+          when String; "\"#{field}\":\"#{value}\"" 
+        else
+          raise ArgumentError, "Illegal type: #{value.class}"
+        end
+      }.join(", ") +
+    " }" + curr_comma
+  end
+
+  def format_blank_line(out, ordrenummer)
+    format_fields(
+        out,
+        "OrderNumber" => ordrenummer, 
+        "Text" => ""
+    )
+  end
+
+  def format_order(order)
+    @curr_comma = ""
+    out = StringIO.new
+    out.puts "["
+    format_fields(out, 
+        "Employee" => order.ansvarlig,
+        "Account" => order.kontonummer,
+        "OrderNumber" => order.ordrenummer.to_i,
+        "Dimension1" => order.sagsnummer,
+        "Remark" => order.sagsnummer
+    )
+    out.puts "]"
+    out.string
+  end
+
+  def format_order_line(order, order_lines)
+    @curr_comma = ","
+    out = StringIO.new
+    out.puts "[" 
+    out.indent { |out|
+      prev_rolle = nil
+      prev_maaned = nil
+      for order_line in order_lines
+        if prev_rolle != order_line.rolle || prev_maaned != order_line.maaned
+          format_blank_line(out, order_line.ordrenummer.to_i) if !prev_rolle.nil? && !prev_maaned.nil?
+          format_fields(out,
+              "OrderNumber" => order_line.ordrenummer.to_i, 
+              "Text" => order_line.rolle)
+          prev_rolle = order_line.rolle
+          prev_maaned = order_line.maaned
+        end
+        format_fields(out,
+            "OrderNumber" => order_line.ordrenummer.to_i, 
+            "Text" => "  #{order_line.tekst}",
+            "Item" => order_line.ydelse, 
+            "Qty" => order_line.antal, 
+            "Price" => order_line.pris)
+      end
+      @curr_comma = ""
+      format_blank_line(out, order_line.ordrenummer.to_i)
+    }
+    out.puts "]"
+    out.string
+  end
+end
+
+
+
+
+
+