datanorm 0.0.1

data/lib/datanorm/lines/v4/price.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V4
+      # A `Priceset` record has one or multiple `Price` records.
+      # A price represents one price for one product.
+      #
+      # Examples:
+      #   Q4058352208304;1;39000;1;0
+      #   RG601315U1;1;2550;1;5500
+      #   100033162;1;28500;;
+      #
+      # [0] Artikelnummer
+      #
+      # [1] Preiskennzeichen (analogous to `Datanorm::Lines::V5::Product`)
+      #     1=wholesale (higher end-customer price)
+      #     2=retail (lower bulk price)
+      #     9=ask-for-price (only V5)
+      #     Some documentation says this means 1=gross and 2=net but I cannot confirm that,
+      #     the prices are always net prices and the retail/wholesale fits the bill.
+      #
+      # [2] Preis (6 Vorkomma, 2 Nachkommastellen)
+      #     Price as Integer (6 digits before the comma, the last two digits represent the fraction)
+      #
+      # [3] Rabattkennzeichen (0=Rabattgruppe,1=Rabattsatz,2=Multi,3=Teuerungszuschlag)
+      #     Discount type
+      #     "0"=group [no change]
+      #     "1"=rate [price - price * (factor/10000)]
+      #     "2"=multiplier [price * (factor/1000)]
+      #     "3"=surcharge [price + price * (factor/100)]
+      #
+      # [4] Rabatt
+      class Price
+        def initialize(columns:)
+          @columns = columns
+        end
+
+        # ------
+        # Basics
+        # ------
+
+        def id
+          ::Datanorm::Helpers::Utf8.call columns[0]
+        end
+
+        # -----
+        # Price
+        # -----
+
+        def retail?
+          columns[1] == '2'
+        end
+
+        def wholesale?
+          columns[1] == '1'
+        end
+
+        def cents
+          columns[2].to_i
+        end
+
+        def price
+          BigDecimal(cents) / 100
+        end
+
+        # --------
+        # Discount
+        # --------
+
+        # If this is true, then `cents` represents the final price.
+        def no_discount?
+          return true if columns[3] == '2'
+
+          # Fallback: If not defined, assume no discount.
+          columns[3].nil? || columns[3].empty?
+        end
+
+        # If this is true, a discount should be applied to `cents`.
+        def percentage_discount?
+          columns[3] == '1'
+        end
+
+        # How much of a discount do we get?
+        def discount_percentage_integer
+          columns[4]&.to_i
+        end
+
+        # -------
+        # Helpers
+        # -------
+
+        def to_s
+          "<Price #{as_json}>"
+        end
+
+        # We don't need the Product ID here.
+        # Our "parent" Priceset has the ID and all `Price` instances refer to the same product.
+        def as_json
+          {
+            is_retail: retail?,
+            is_wholesale: wholesale?,
+            is_no_discount: no_discount?,
+            is_percentage_discount: percentage_discount?,
+            discount_percentage: discount_percentage_integer,
+            cents:
+          }
+        end
+
+        def to_json(...)
+          as_json.to_json(...)
+        end
+
+        private
+
+        attr_reader :columns
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v4/priceset.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V4
+      # rubocop:disable Layout/LineLength
+      # Represents one line of the Datanorm file that starts with "P".
+      # It contains price data for up to three individual products.
+      #
+      # Examples:
+      #   V4
+      #   P;A;Q4058352208304;1;39000;1;0;;;;;Q4058352208304;2;25521;;;;;;;Q4058352208403;1;42300;1;0;;;;;
+      #   P;A;RG601315U1;1;2550;1;5500;;;;;RG601215U1;1;2130;1;5500;;;;;RG6211420U1;1;3210;1;5500;;;;;
+      #   P;A;100033162;1;28500;;;;;;;;;;;;;;;;;;;;;;;;;
+      #
+      # rubocop:enable Layout/LineLength
+      class Priceset < ::Datanorm::Lines::Base
+        def to_s
+          "<Priceset with #{prices.size} prices>"
+        end
+
+        # Not available at this hierarchy level.
+        # The collection contains up to three product IDs.
+        def id
+          return prices.first.id if prices.size == 1
+
+          raise 'A Priceset with multiple products does not have one single ID'
+        end
+
+        def prices
+          [columns[2..6],
+           columns[11..15],
+           columns[20..24]].map do |set_columns|
+             next if set_columns[0].to_s == ''
+
+             ::Datanorm::Lines::V4::Price.new(columns: set_columns)
+           end.compact
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v4/product.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V4
+      # Represents one line containting a product main name, quantity unit and price.
+      #
+      # Example:
+      #   A;N;QBMK10208R;50;Brandmeldekabel rot;1;3;M;228313;2AED; ; ;
+      #
+      # [0] Satzartenkennzeichen "A"
+      # [1] Verarbeitungskennzeichen (N=Neuanlage, L=Löschung, A=Änderung, X=Artikelnummernänderung)
+      # [2] Artikelnummer
+      # [3] Textkennzeichen
+      # [4] Artikelbezeichnung 1 (Kurztextzeile 1)
+      # [5] Artikelbezeichnung 2 (Kurztextzeile 2)
+      #
+      # [6] Preiskennzeichen (see `Datanorm::Lines::V4::Price`)
+      #
+      # [7] Preiseinheit (0= per Mengeneinheit 1; 1=10, 2=100, 3=1000)
+      # [8] Mengeneinheit (Stk, m, lfm)
+      # [9] Preis (Wenn Hersteller die Preise mit Satzart "P" liefern, braucht hier kein Preis (0)
+      #     eingetragen werden)
+      # [10] Rabattgruppe (Zur Ermittlung des Netto-Artikelpreises über die Rabattmatrix)
+      # [11] Hauptwarengruppe
+      # [12] Langtextschlüssel (Mit dem Langtextschlüssel wird ein Text aus mehreren Zeilen
+      #      (Satzart T) an den Artikel gekettet.
+      class Product < ::Datanorm::Lines::Base
+        def to_s
+          "<Product #{as_json}>"
+        end
+
+        def id
+          ::Datanorm::Helpers::Utf8.call columns[2]
+        end
+
+        def text_id
+          ::Datanorm::Helpers::Utf8.call columns[12]
+        end
+
+        def retail_price?
+          columns[6] == '1'
+        end
+
+        def wholesale_price?
+          columns[6] == '2'
+        end
+
+        def cents
+          columns[9].to_i
+        end
+
+        def title
+          ::Datanorm::Helpers::Utf8.call columns[4..5].join(' ').strip
+        end
+
+        def quantity_unit
+          ::Datanorm::Helpers::Utf8.call columns[8]
+        end
+
+        def quantity
+          case columns[7].to_i
+          when 0 then 1
+          when 1 then 10
+          when 2 then 100
+          when 3 then 1000
+          end
+        end
+
+        def discount_group
+          ::Datanorm::Helpers::Utf8.call columns[9]
+        end
+
+        def as_json # rubocop:disable Metrics/MethodLength
+          {
+            id:,
+            text_id:,
+            is_retail_price: retail_price?,
+            is_wholesale_price: wholesale_price?,
+            cents:,
+            title:,
+            quantity_unit:,
+            quantity:,
+            discount_group:
+          }
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v4/text.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V4
+      # Wraps a line of a DATANORM file that represents one line of a product description.
+      # Also known as Langtextsatz.
+      class Text < ::Datanorm::Lines::Base
+        def to_s
+          "TEXT [#{id}] <#{line_number}> #{content.encode('UTF-8', 'CP850').gsub("\n", '⏎')}"
+        end
+
+        def id
+          columns[2]
+        end
+
+        def line_number
+          columns[4].to_i
+        end
+
+        def content
+          [columns[6], columns[9]].join("\n").strip
+        end
+
+        def <=>(other)
+          line_number <=> other.line_number
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v5/dimension.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V5
+      # Immediate product description texts. Should take precedence over Text records.
+      class Dimension < ::Datanorm::Lines::Base
+        def to_s
+          "[#{id}] DIMENSION-5 - #{columns}"
+        end
+
+        def id
+          encode columns[2]
+        end
+
+        def content
+          to_s
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v5/parse.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V5
+      # Converts one line of a Datanorm file to a Ruby object.
+      class Parse
+        include Calls
+
+        # Note that B-records in v4 contain data, but in V4, they are only DELETION notices.
+        CLASSES = {
+          'A' => Datanorm::Lines::V5::Product,
+          'T' => Datanorm::Lines::V5::Text,
+          'D' => Datanorm::Lines::V5::Dimension,
+          'P' => Datanorm::Lines::V5::Price
+          # 'C' => Datanorm::Lines::V5::Service
+        }.freeze
+
+        option :columns
+        option :source_line_number
+
+        def call
+          klass = CLASSES.fetch(columns.first[0], Datanorm::Lines::Base)
+          klass.new(columns:, source_line_number:)
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v5/price.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V5
+      # A price represents one price for one product.
+      #
+      # Examples:
+      #   P;100033152;1;1;28000;06;;;;;;;;
+      #   P;VSP-983-B;1;1;3410;BMT;
+      #
+      class Price
+        def to_s
+          "<Price id=#{id.inspect} #{discounts.join(' | ')}>"
+        end
+
+        def id
+          encode columns[2]
+        end
+
+        def cents
+          columns[4].to_i
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v5/product.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V5
+      # Ruby wrapper for one A line of a Datanorm file.
+      class Product < ::Datanorm::Lines::Base
+        def to_s
+          "[#{id}] Product-5 #{item_title} - EUR #{price}"
+        end
+
+        def id
+          columns[2]
+        end
+
+        def text_id
+          columns[23]
+        end
+
+        def cents
+          columns[8].to_i
+        end
+
+        def title
+          columns[3..4].join(' ').strip
+        end
+
+        def quantity_unit
+          columns[5]
+        end
+
+        def quantity
+          columns[6].to_i.nonzero?
+        end
+
+        def as_json
+          { id:, text_id:, cents:, title:, quantity_unit:, quantity: }
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v5/text.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V5
+      # One line of the Datanorm file starting with T
+      class Text < ::Datanorm::Lines::Base
+        def to_s
+          "[#{id}] TEXT-5 #{line_number} #{content}"
+        end
+
+        def id
+          columns[2]
+        end
+
+        def line_number
+          columns[4].to_i
+        end
+
+        def content
+          columns[5]
+        end
+
+        def <=>(other)
+          line_number <=> other.line_number
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/logger.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# A Rubygem to parse DATANORM files from the 90s.
+module Datanorm
+  # Helper to create a STDOUT logger
+  def self.logger
+    return @logger if defined?(@logger)
+
+    @logger = ::Logger.new($stdout)
+    @logger.formatter = proc do |severity, _, progname, message|
+      [severity.rjust(5), progname, '-', message, "\n"].join(' ')
+    end
+    @logger
+  end
+end

data/lib/datanorm/logging.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Datanorm
+  # Adds a convenience method for debug logging.
+  module Logging
+    def self.included(base)
+      base.extend ClassMethods
+    end
+
+    # Log helper on class level.
+    module ClassMethods
+      def log(&)
+        return unless ENV['DEBUG_DATANORM']
+
+        ::Datanorm.logger&.debug(to_s, &)
+      end
+    end
+
+    private
+
+    def log(&)
+      return unless ENV['DEBUG_DATANORM']
+
+      ::Datanorm.logger&.debug(self.class.to_s, &)
+    end
+  end
+end

data/lib/datanorm/progress.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Datanorm
+  # Represents how much processing elapsed and how much is left to be done.
+  class Progress
+    include Datanorm::Logging
+
+    attr_accessor :title, :current, :total
+
+    def increment!
+      self.current += 1
+    end
+
+    def to_s
+      "#{percentage}% (#{title} #{current}/#{total})"
+    end
+
+    def significant?
+      (current % 50_000).zero?
+    end
+
+    def percentage
+      ((current.to_f / total) * 100).round(1)
+    end
+  end
+end

data/lib/datanorm/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Datanorm
+  Version = ::Data.define(:number, :four?, :five?)
+end

data/lib/datanorm.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'bigdecimal'
+require 'calls'
+require 'csv'
+require 'json'
+require 'logger'
+require 'securerandom'
+require 'tmpdir'
+require 'tron'
+
+require 'datanorm/logger'
+require 'datanorm/logging'
+require 'datanorm/helpers/utf8'
+require 'datanorm/helpers/filename'
+
+require 'datanorm/headers/v4/date'
+require 'datanorm/headers/v4/version'
+require 'datanorm/headers/v5/date'
+require 'datanorm/headers/v5/version'
+
+require 'datanorm/lines/parse'
+require 'datanorm/lines/base'
+require 'datanorm/lines/v4/product'
+require 'datanorm/lines/v4/extra'
+require 'datanorm/lines/v4/priceset'
+require 'datanorm/lines/v4/price'
+require 'datanorm/lines/v4/dimension'
+require 'datanorm/lines/v4/text'
+require 'datanorm/lines/v5/product'
+require 'datanorm/lines/v5/price'
+require 'datanorm/lines/v5/text'
+require 'datanorm/lines/v5/dimension'
+
+require 'datanorm/lines/v4/parse'
+require 'datanorm/lines/v5/parse'
+
+require 'datanorm/documents/preprocess'
+require 'datanorm/documents/preprocesses/cache'
+require 'datanorm/documents/preprocesses/process'
+require 'datanorm/documents/assembles/product'
+require 'datanorm/documents/assembles/price'
+require 'datanorm/documents/assemble'
+require 'datanorm/progress'
+require 'datanorm/document'
+require 'datanorm/file'
+require 'datanorm/header'
+require 'datanorm/version'