datanorm 0.0.1

data/lib/datanorm/documents/preprocesses/process.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Documents
+    module Preprocesses
+      # Takes one record of a Datanorm file and saves it to disk in an organized way for later use.
+      class Process
+        include Calls
+        include ::Datanorm::Logging
+
+        option :workdir
+        option :record
+
+        # One common combo is [A], [B], [D] per product throughout the whole file.
+        # In that case, the ID is the same for all three.
+        #
+        # Another variant is all [T] at the beginning and then [A] etc. at the end of the file.
+        # In that case the IDs of [T] are separate and later referenced in [A].
+        def call
+          if record.kind_dimension? || record.kind_text?
+            cache_longtext
+
+          elsif record.kind_extra?
+            cache_json
+
+          elsif record.kind_priceset?
+            cache_priceset
+
+          elsif record.kind_product?
+            cache_product
+          end
+        end
+
+        private
+
+        # Record sets with their own unique IDs that can later be referenced by Products.
+        def cache_longtext
+          log { "Pre-Processing #{record.id}" }
+          ::Datanorm::Documents::Preprocesses::Cache.call(
+            workdir:,
+            namespace: record.record_kind,
+            id: record.id,
+            target_line_number: record.line_number,
+            content: record.content
+          )
+        end
+
+        def cache_json
+          ::Datanorm::Documents::Preprocesses::Cache.call(
+            workdir:,
+            namespace: record.record_kind,
+            id: record.id,
+            content: record.to_json
+          )
+        end
+
+        # One Product has many prices.
+        # We create one file per product that has one price per line for that product.
+        def cache_priceset
+          set_workdir = workdir.join('P')
+          FileUtils.mkdir_p(set_workdir)
+
+          record.prices.each do |price|
+            set_workdir.join(::Datanorm::Helpers::Filename.call(price.id)).open('a') do |file|
+              file.write("#{price.to_json}\n")
+            end
+          end
+        end
+
+        # When preprocessing is done, we'll need to loop throuth each product once.
+        # So let's append each product to one file, so that we can go through it later.
+        def cache_product
+          workdir.join('A.txt').open('a') do |file|
+            file.write("#{record.to_json}\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/file.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module Datanorm
+  # Parses a datanorm file line by line and wraps them in Ruby objects.
+  class File
+    include Datanorm::Logging
+    include Enumerable
+
+    def initialize(path:)
+      log { "Loading file `#{path}`" }
+      @path = path
+    end
+
+    def header
+      return @header if defined?(@header)
+
+      ::File.open(path, "r:#{Encoding::CP850}") do |file|
+        first_line = file.gets
+        log { 'Parsing header line...' }
+
+        @header = ::Datanorm::Header.new(line: first_line)
+      end
+    end
+
+    # Convenience shortcut.
+    def version
+      header.version
+    end
+
+    def each
+      line_number = 0
+
+      ::CSV.foreach(path, **options) do |columns|
+        line_number += 1
+        next if line_number == 1 # Skip header, it's parsed separately
+
+        yield ::Datanorm::Lines::Parse.call(version:, columns:, source_line_number: line_number)
+      end
+    end
+
+    # We want this, so that we can indicate how much progress has been done.
+    def lines_count
+      return @lines_count if defined?(@lines_count)
+
+      log { 'Scanning number of total lines... (this takes about 2 seconds per GB)' }
+      @lines_count = 0
+      # `foreach` doesn't load the entire file into memory.
+      ::File.foreach(path, encoding: Encoding::CP850) { @lines_count += 1 }
+      log { "Scan complete, counted #{@lines_count} lines." }
+      @lines_count
+    end
+
+    private
+
+    attr_reader :path
+
+    def options
+      {
+        encoding: Encoding::CP850,
+        col_sep: ';',
+        liberal_parsing: true
+      }
+    end
+  end
+end

data/lib/datanorm/header.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Datanorm
+  # Represents the first line of a DATANORM file.
+  class Header
+    def initialize(line:)
+      @line = line.to_s
+    end
+
+    def to_s
+      "HEADER <V#{version.number}> date <#{date}>"
+    end
+
+    def version
+      return @version if defined?(@version)
+
+      @version = parse_version
+    end
+
+    def date
+      return @date if defined?(@date)
+
+      @date = parse_date
+    end
+
+    private
+
+    attr_reader :line
+
+    def parse_version
+      # They are mutually exclusive in detecting their own version.
+      # I'm willing to bet that there is no canonical exact way to detect it anyway.
+      ::Datanorm::Headers::V5::Version.call(line:) ||
+        ::Datanorm::Headers::V4::Version.call(line:) ||
+        ::Datanorm::Version.new(number: -1, four?: false, five?: false) # Unknown version
+    end
+
+    def parse_date
+      if version.five?
+        ::Datanorm::Headers::V5::Date.call(line:)
+      elsif version.four?
+        ::Datanorm::Headers::V4::Date.call(line:)
+      end
+    end
+  end
+end

data/lib/datanorm/headers/v4/date.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Headers
+    module V4
+      # Parses a Date from the raw first line of a DATANORM file.
+      class Date
+        include Calls
+
+        option :line
+
+        def call
+          return unless ddmmyy.match?(/\A\d{6}\z/)
+
+          year = (yy < 50 ? 2000 : 1900) + yy
+          ::Date.new(year, mm, dd)
+        end
+
+        private
+
+        def dd
+          ddmmyy[0..1].to_i
+        end
+
+        def mm
+          ddmmyy[2..3].to_i
+        end
+
+        def yy
+          ddmmyy[4..5].to_i
+        end
+
+        def ddmmyy
+          line[2..7]
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/headers/v4/version.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Headers
+    module V4
+      # Parses the Version from the raw first line of a DATANORM version 4 file.
+      class Version
+        include Calls
+
+        option :line
+
+        def call
+          return if free_use_byte == ';' # Bail out if likely V5
+          return unless version_number == '04' # Bail out if not properly V4
+
+          ::Datanorm::Version.new(number: 4, four?: true, five?: false)
+        end
+
+        private
+
+        # V4 can have anything here.
+        # V5 is in 99.999% of cases a semicolon.
+        def free_use_byte
+          line[1]
+        end
+
+        # V3 is universally not supported.
+        # V4 has '04' at this position
+        # V5 doesn't use this and could have anything there.
+        def version_number
+          line[123..124]
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/headers/v5/date.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Headers
+    module V5
+      # Parses a Date from the raw first line of a DATANORM file.
+      class Date
+        include Calls
+
+        option :line
+
+        def call
+          # Date.parse(nil) always returns a valid date, so we need to catch that.
+          return unless columns[3]
+
+          ::Date.parse(columns[3])
+        end
+
+        def columns
+          line.split(';')
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/headers/v5/version.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Headers
+    module V5
+      # Parses the Version from the raw first line of a DATANORM version 5 file.
+      class Version
+        include Calls
+
+        option :line
+
+        def call
+          # This is an arbitrary number to see if there are enough semicolons
+          # to believe that this could be a v5 file.
+          return unless columns.size > 3
+          return unless version_number == '050'
+
+          ::Datanorm::Version.new(number: 5, four?: false, five?: true)
+        end
+
+        private
+
+        # V4 can have anything here.
+        # V5 definitely has "050" here.
+        # I'm pretty sure that stands for 0.5.0 and no newer version was ever released.
+        def version_number
+          columns[1]
+        end
+
+        def columns
+          line.split(';')
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/helpers/filename.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Helpers
+    # Converts a String to something suitable for a filename.
+    class Filename
+      include Calls
+
+      param :input
+
+      def call
+        raise "Should not write to file called `#{input.inspect}`" if input.nil?
+
+        # In case there are special characters in a product number.
+        utf8_encoded = ::Datanorm::Helpers::Utf8.call(input)
+        "#{Base64.urlsafe_encode64(utf8_encoded.to_s)}.txt"
+      end
+    end
+  end
+end

data/lib/datanorm/helpers/utf8.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Helpers
+    # Converts a String to UTF-8.
+    class Utf8
+      include Calls
+
+      param :input
+
+      # I sometimes encounter single spaces to indicate nil in Datanorm.
+      # So let's filter out those, too.
+      def call
+        return if input.nil? || input == ' ' || input.to_s.empty?
+
+        input.to_s.encode('UTF-8')
+      end
+    end
+  end
+end

data/lib/datanorm/lines/base.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    # Object that represents one line of a Datanorm file.
+    class Base
+      # Array that holds all attributes of one line.
+      # In the Datanorm file they are typically separated by seimcolons.
+      # Header rows may lack semicolons (in V4), in that case, this Array has only one long String.
+      attr_reader :columns
+
+      # Where in the originating Datanorm file this line is located.
+      attr_reader :source_line_number
+
+      # This class is subclassed by one type per row.
+      # Add convenient predicate methods to query the kind of record class.
+      # E.g. `Datanorm::Lines::V4::Extra` has `kind_extra?` to be true.
+      def self.inherited(subclass)
+        kind_method = "kind_#{subclass.name.split('::').last.downcase}?"
+
+        remove_method(kind_method) if method_defined?(kind_method) # Avoid warnings during tests
+        define_method(kind_method) do
+          self.class.name.split('::').last.downcase == subclass.name.split('::').last.downcase
+        end
+
+        super
+      end
+
+      def initialize(columns:, source_line_number:)
+        @columns = columns
+        @source_line_number = source_line_number
+      end
+
+      # Every row has a unique identifier. Most often a product number.
+      # Text records commonly have their own IDs, which are not equal to the product number.
+      # Multiple lines can have the same ID (e.g. one for price and several for description).
+      # Also known as "Satzartenkennzeichen".
+      def id
+        raise "Implement ##{__method__} in #{self.class}"
+      end
+
+      def as_json
+        raise "Implement ##{__method__} in #{self.class}"
+      end
+
+      # The first character in every line always represents the record type.
+      # E.g. "T", "A"
+      def record_kind
+        columns[0]
+      end
+
+      # Overridden in subclasses.
+      def to_s
+        to_json
+      end
+
+      # Convenience Shortcut to convert attributes from CP850 to UTF-8.
+      def encode(...)
+        ::Datanorm::Helpers::Utf8.call(...)
+      end
+
+      def to_json(...)
+        as_json.to_json(...)
+      end
+    end
+  end
+end

data/lib/datanorm/lines/parse.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    # Converts one line of a DATANORM file into a Ruby Object.
+    #
+    # V: Vorlaufsatz (identifies file metadata, often the first line).
+    # K: Kopfsatz (header with catalog or transaction details).
+    # A: Artikelsatz (product/product data).
+    # B: Zusatzsatz (additional product data, e.g., EAN, packaging).
+    # C: Leistungssatz/Konditionensatz (product installation time and public tender descriptions).
+    # D: Langtextsatz (long text descriptions).
+    # P: Preissatz (price data, often multiple products per line in V5).
+    # T: Textbausteinsatz (text modules for descriptions).
+    # S: Sonderbedingungssatz (special conditions, less common).
+    #
+    class Parse
+      include Calls
+
+      option :version
+      option :columns
+      option :source_line_number
+
+      def call
+        if version.four?
+          ::Datanorm::Lines::V4::Parse.call(columns:, source_line_number:)
+        elsif version.five?
+          ::Datanorm::Lines::V5::Parse.call(columns:, source_line_number:)
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v4/dimension.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V4
+      # Immediate product description texts. Should take precedence over Text records.
+      # Aufbau der „D“ Zeile (Dimensionstextsatz)
+      # 1 : Satzartenkennzeichen[1] : Buchstabe D für Dimensionstext
+      # 2 : Verarbeitungskennzeichen[1] : N=Neuanlage, L=Löschung, A=Änderung
+      # 3 : Artikelnummer[15] : Inhalt alphanumerische Zeichen
+      # 4 : Zeilennummer[2/0] : numerisch
+      # 5 : Unterkennzeichen[1] : alphanumerisch, F = freier Text,. T = Einfügen von Textblöcken,
+      #     E = Einfügungen von Textblöcken und Werten
+      # 6 : Frei[8] : alphanumerische Zeichen
+      # 7 : Zeilentext[40] : alphanumerische Zeichen
+      # 8 : Zeilennummer[2/0] : numerisch
+      # 9 : Unterkennzeichen[1] : alphanumerisch, F = freier Text,. T = Einfügen von Textblöcken,
+      #     E = Einfügungen von Textblöcken und Werten
+      # 10 : Frei[8] : alphanumerische Zeichen
+      # 11 : Zeilentext[40] : alphanumerische Zeichen
+      class Dimension < ::Datanorm::Lines::Base
+        def to_s
+          "<Dimension [#{id}] #{line_number.to_s.rjust(3)} #{content.gsub("\n", '⏎')}>"
+        end
+
+        def id
+          encode columns[2]
+        end
+
+        def line_number
+          columns[3].to_i
+        end
+
+        def content
+          "#{encode(columns[6])}\n#{encode(columns[10])}"
+        end
+
+        def <=>(other)
+          line_number <=> other.line_number
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v4/extra.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V4
+      # 1 : Satzartenkennzeichen[1] : Buchstabe B für Hauptsatz 2
+      # 2 : Verarbeitungskennzeichen[1] : N=Neuanlage, L=Löschung, A=Änderung
+      # 3 : Artikelnummer[15] : Inhalt alphanumerische Zeichen
+      # 4 : Matchcode[15] : alphanumerische Zeichen
+      # 5 : Alternativ-Artikelnummer[15] : alphanumerische Zeichen
+      # 6 : Katalogseite[8] : alphanumerische Zeichen
+      # 7 : Bereich für Kupferzuschlag
+      # 7a : Bereich für Kupferzuschlag
+      # 7b : Bereich für Kupferzuschlag
+      # 7c : Bereich für Kupferzuschlag
+      # 8 : EAN-Nummer[13] : alphanumerische Zeichen
+      # 9 : Anbindungsnummer[12] : alphanumerische Zeichen, zur Anbindung von Bildern
+      # 10 : Warengruppe[10] : alphanumerische Zeichen s.a. .WRG-Datei
+      # 11 : Kostenart[2/0] : numerisch
+      # 12 : Verpackungsmenge[5/0] : numerisch
+      # 13 : Referenznummer-Erstellerkürzel[4] : alphanumerische Zeichen
+      # 14 : Referenznummer[17] : alphanumerische Zeichen
+      class Extra < ::Datanorm::Lines::Base
+        def to_s
+          "EXTRA     [#{id}] #{"{#{matchcode}}" unless matchcode.empty?} EAN: #{ean}"
+        end
+
+        def id
+          encode columns[2]
+        end
+
+        # This is like a tag. E.g. a product category.
+        def matchcode
+          encode columns[3].to_s.strip
+        end
+
+        def alternative_id
+          encode columns[4]
+        end
+
+        def ean
+          encode columns[9]
+        end
+
+        def category_id
+          encode columns[11]
+        end
+
+        def as_json
+          { alternative_id:, matchcode:, ean:, category_id: }
+        end
+      end
+    end
+  end
+end

data/lib/datanorm/lines/v4/parse.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Datanorm
+  module Lines
+    module V4
+      # Converts a single DATANORM v4 line into a Ruby Object.
+      #
+      class Parse
+        include Calls
+
+        # Vorlaufsatz "V": keine Kennzeichen
+        # Kundenkontrollsatz "K": keine Kennzeichen
+        # Warengruppensatz "S": keine Kennzeichen
+        # Rabattsatz "R": keine Kennzeichen
+        # Hauptsatz 1 "A": N = Neuanlage; L = Löschung; A = Änderung; X = Artikelnummernänderung
+        # Hauptsatz 2 "B": N = Neuanlage; ; A = Änderung
+        # Dimensionssatz "D": N = Neuanlage; A = Änderung; L = Löschung
+        # Langtextsatz "T": N = Neuanlage; A = Änderung;        L = Löschung
+        # Einfügesatz "E": N = Neuanlage; A = Änderung; L = Löschung
+        # Staffelpreiszu-/-abschlagssatz "Z": N = Neuanlage; A = Änderung; L = Löschung
+        # Leistungssatz "C": N = Neuanlage; A = Änderung; L = Löschung
+        # Artikel-Set-Satz "J": N = Neuanlage; A = Änderung; L = Löschung
+        # Preisänderungssatz "P": A = Änderung; P = Preisänderung
+        CLASSES = {
+          'A' => Datanorm::Lines::V4::Product,
+          'B' => Datanorm::Lines::V4::Extra,
+          'D' => Datanorm::Lines::V4::Dimension,
+          'T' => Datanorm::Lines::V4::Text,
+          'P' => Datanorm::Lines::V4::Priceset
+        }.freeze
+
+        option :columns
+        option :source_line_number
+
+        def call
+          klass = CLASSES.fetch(columns.first[0], Datanorm::Lines::Base)
+          klass.new(columns:, source_line_number:)
+        end
+      end
+    end
+  end
+end