dcm 0.0.9

data/lib/cli.rb

@@ -0,0 +1,280 @@
+require "rubygems"
+require "bundler"
+
+require "ecu"
+require "thor"
+require "filesize"
+require "json"
+
+require_relative "file_reader"
+require_relative "codinginfo"
+require_relative "tempfile_handler"
+require_relative "diff_viewer"
+require_relative "label_selector"
+require_relative "core_ext"
+require_relative "version"
+
+module Dcm
+
+  class CLI < Thor
+
+    desc "show FILE", "Show all labels in FILE"
+    option :oneline, type: :boolean, aliases: ["-l"]
+    def show(file=nil)
+      display_list(list: parse_file(file), detail: options[:oneline] ? :onelinefull : true)
+    end
+
+    desc "new [SPEC]", "Create a new dcm from scratch"
+    option :indented,   aliases: ["-i"], type: :boolean, default: false
+    def new(*spec)
+      list = parse_spec(spec)
+      puts list.to_dcm(option[:indented])
+    rescue RuntimeError => e
+      puts e.message
+      exit 1
+    end
+
+    desc "liveview FILE", "Live search and view labels in FILE"
+    def liveview(file=nil)
+      list = parse_file(file)
+      loop { LabelSelector.choose_from(list) }
+    end
+
+    desc "creta2begu ID CODING FILE", "Convert a hierarchical DCM to a flat DCM"
+    def creta2begu(id, codingpath, file)
+      list = parse_file(file)
+
+      puts Codinginfo.new([id], codingpath)
+        .flatten_list(list)
+        .to_dcm
+    end
+
+    desc "begu2creta IDS CODING FILE", "Convert a flat DCM to a hierarchical DCM"
+    def begu2creta(ids, codingpath, file)
+      list = parse_file(file)
+
+      puts Codinginfo.new(ids.split(","), codingpath)
+        .unflatten_list(list)
+        .to_dcm
+    end
+
+    desc "list FILE", "List all labels in FILE"
+    option :lab, type: :boolean, aliases: ["-l"]
+    def list(file=nil)
+      display_list(list: parse_file(file),
+                   detail: false,
+                   format: (options[:lab] ? :lab : :tty))
+    end
+
+    desc "summary FILE", "Summarize labels in FILE"
+    option :verbose, type: :boolean, aliases: ["-v"], default: false
+    def summary(file=nil)
+      list = parse_file(file)
+      puts "DCM file with #{list.count.quantify("label")}"
+      if options[:verbose]
+        list.group_by(&:function).each do |function, labels|
+          puts "  #{function}: #{labels.count}"
+        end
+      end
+    end
+
+    desc "cat FILE", "Output (filtered) dcm content"
+    option :include,    aliases: ["-f"], default: ".*"
+    option :reject,     aliases: ["-F"], default: "(?!x)x"
+    option :inlab,      aliases: ["-l"], default: ""
+    option :notinlab,   aliases: ["-L"], default: ""
+    option :indented,   aliases: ["-i"], type: :boolean, default: false
+    option :headers,    aliases: ["-h"], default: [], type: :array
+    option :subheaders, aliases: ["-H"], default: [], type: :array
+    option :mfile, type: :boolean, aliases: ["-m"]
+    def cat(file=nil)
+      list = parse_file(file)
+      if options[:include] != ".*" || options[:reject] != "(?!x)x"
+        selector = Regexp.new(options[:include], true)
+        rejector = Regexp.new(options[:reject], true)
+        list = list.
+          select { |l| l.match(selector) }.
+          reject { |l| l.match(rejector) }
+      end
+      if options[:inlab].present?
+        labfile = Ecu::LabelList.from_lab(File.read(options[:inlab]))
+        list = list.select { |l| labfile.any? { |s| s.name == l.name } }
+      end
+      if options[:notinlab].present?
+        labfile = Ecu::LabelList.from_lab(File.read(options[:notinlab]))
+        list = list.reject { |l| labfile.any? { |s| s.name == l.name } }
+      end
+      list.headers    += options[:headers]
+      list.subheaders += options[:subheaders]
+      if options[:mfile]
+        puts list.to_mfile
+      else
+        puts list.to_dcm(options[:indented])
+      end
+    end
+
+    desc "rename MAPPING FILE", "Rename dcm content"
+    def rename(mapping=nil, file=nil)
+      list = parse_file(file)
+      mappings = JSON.parse(FileReader.read(mapping))
+      list = list.map_int do |label|
+        label.with \
+          name: mappings.filter_map { |m| m["new"] if m["old"] == label.name }&.first || label.name
+      end
+      puts list.to_dcm(options[:indented])
+    end
+
+    desc "validate FILE", "Validate DCM"
+    def validate(file=nil)
+      parse_file(file)
+    end
+
+    desc "compare FILE1 FILE2", "Compare contents of FILE1 and FILE2"
+    option :left,       type: :boolean, aliases: ["-l"]
+    option :right,      type: :boolean, aliases: ["-r"]
+    option :comparable, type: :boolean, aliases: ["-c"]
+    option :equal,      type: :boolean, aliases: ["-e"]
+    option :unequal,    type: :boolean, aliases: ["-u"]
+    option :precision,  type: :numeric, aliases: ["-p"], default: 3
+    def compare(file1, file2)
+      comparison = render_comparison(file1, file2, precision: options[:precision])
+
+      if options[:left]
+        puts to_lab(comparison.names(:left_exclusive))
+      elsif options[:right]
+        puts to_lab(comparison.names(:right_exclusive))
+      elsif options[:equal]
+        puts to_lab(comparison.names(:equal))
+      elsif options[:unequal]
+        puts to_lab(comparison.names(:nonequal))
+      elsif options[:comparable]
+        puts (
+          comparison.names(:equal).map { |l| "#{l}: equal".colorize(:green) } +
+          comparison.names(:nonequal).map { |l| "#{l}: unequal".colorize(:red) }
+        )
+      else
+        puts "#{comparison.names(:left_exclusive).size.quantify("label")} exclusive to #{file1}"
+        puts "#{comparison.names(:right_exclusive).size.quantify("label")} exclusive to #{file2}"
+        puts "#{comparison.names(:common).size.quantify("label")} comparable"
+        puts "  #{comparison.names(:equal).size.quantify("label")} equal".colorize(:green)
+        puts "  #{comparison.names(:nonequal).size.quantify("label")} unequal".colorize(:red)
+        puts "DCMs are identical!".colorize(:green) if comparison.identical?
+      end
+    end
+
+    desc "gitdiff [ARGS]", "Provide a wrapper for git diff"
+    def gitdiff(path, file1, hex1, mode1, file2, hex2, mode2)
+      if [file1, file2].all? { |f| File.extname(f).downcase == ".dcm" }
+        invoke(:diff, [file1, file2])
+      else
+        puts DiffViewer.new(File.read(file1), File.read(file2))
+      end
+    end
+
+    desc "diff FILE1 FILE2", "Diff contents between FILE1 and FILE2"
+    option :left,       type: :boolean, aliases: ["-l"]
+    option :right,      type: :boolean, aliases: ["-r"]
+    option :comparable, type: :boolean, aliases: ["-c"]
+    option :precision,  type: :numeric, aliases: ["-p"], default: 3
+    def diff(file1, file2)
+      comparison = render_comparison(file1, file2, precision: options[:precision])
+
+      showall = !(options[:left] || options[:comparable] || options[:right])
+
+      puts "-#{file1}".colorize(:red)
+      puts "+#{file2}".colorize(:green)
+      puts ""
+      if options[:left] || showall
+        comparison.left_exclusive.each do |label|
+          puts DiffViewer.new(label.to_s(detail: true), "")
+        end
+      end
+      if options[:comparable] || showall
+        comparison.differences.each do |left, right|
+          puts DiffViewer.new(left.to_s(detail: true), right.to_s(detail: true))
+        end
+      end
+      if options[:right] || showall
+        comparison.right_exclusive.each do |label|
+          puts DiffViewer.new("", label.to_s(detail: true))
+        end
+      end
+    end
+
+    desc "size FILE", "Guess the size of the labels contained in FILE"
+    def size(file)
+      list = parse_file(file)
+      puts Filesize.new(list.map(&:bytesize).reduce(:+)).pretty
+    end
+
+    desc "merge FILE1 FILE2 [FILE3 ...]", "Merge several files into a single DCM"
+    option :priority, aliases: ["-p"], default: "RIGHT"
+    def merge(*files)
+      baselist = parse_file(files.shift)
+      while not files.empty?
+        mergelist = parse_file(files.shift)
+        baselist = Ecu::LabelListComparison.new(baselist, mergelist).
+          merge(priority: options[:priority].downcase.to_sym)
+      end
+      puts baselist.to_dcm(options[:indented])
+    end
+
+    map %w[--version -v] => :__print_version
+
+    desc "--version, -v", "Print the version"
+    def __print_version
+      puts "v#{Dcm::VERSION}"
+    end
+
+    def self.exit_on_failure?
+      true
+    end
+
+    protected
+
+    def parse_file(file)
+      if file.nil? || file == "-"
+        file = TempfileHandler.create(STDIN.read, filename: ["stdin", ".dcm"])
+      end
+      Ecu::LabelList.from_dcm(FileReader.read(file))
+    rescue Ecu::MalformedDcmError => e
+      puts e.message
+      puts e.context
+      exit 1
+    end
+
+    def parse_spec(spec)
+      return Ecu::LabelList.new if spec.empty?
+      fail "Usage: dcm new TYPE NAME VALUE1 [VALUE2 ...]" if spec.length < 3
+      type, name, value = spec
+      value = Float(value) rescue value
+      label = Ecu::Festwert.new \
+        name: name,
+        value: value
+      Ecu::LabelList.new([label])
+    end
+
+    def display_list(list:, detail:, format: :tty)
+      case format
+      when :tty
+        puts list.map { ListColorizer.call(_1.to_s(detail: detail)) }.join("\n")
+      when :lab
+        puts list.to_lab
+      else
+        error "Unkown format #{format}"
+      end
+    end
+
+    def render_comparison(file1, file2, precision:)
+      lists = [file1, file2]
+        .map { FileReader.read(_1) }
+        .map { Ecu::LabelList.from_dcm(_1) }
+        .map { _1.map_int { |label| label.round_to(precision) } }
+      Ecu::LabelListComparison.new(*lists)
+    end
+
+    def to_lab(ary)
+      ary.unshift("[LABEL]").join("\n")
+    end
+  end
+end

data/lib/codinginfo.rb

@@ -0,0 +1,151 @@
+require "simple_xlsx_reader"
+
+class Codinginfo
+  CVAR_PREFIX_REGEXP = /^CVAR_[A-Za-z0-9_]+_\d+\./
+  PARAM_ASSIGNMENT_HEADERS = { name: /Parameter/, cvar: /Schalter/, package: /Komponente/ }
+
+  class Cvar
+
+    # The CVAR must have a name and _may_ have a package and value
+    # the package and value can be merged
+
+    attr_reader :name, :value, :package
+    def initialize(name:, package: nil, value: nil)
+      @name = name
+      @package = package
+      @value = value
+    end
+
+    def label_prefix
+      return /^#{name}_(\w+_)?#{value}\./ if package.nil?
+
+      [name, package, value].join("_") + "."
+    end
+
+    def to_s = "#{name}=#{value}"
+    def ==(other) = name == other.name
+
+    def merge(other)
+      fail "Cannot merge #{name} with #{other.name}" unless self == other
+
+      self.class.new \
+        name: name,
+        package: package || other.package,
+        value: value || other.value
+    end
+  end
+
+  class Variant
+    def initialize(headers, row)
+      @properties = headers.zip(row).to_h
+    end
+
+    def cvars
+      @properties
+        .select { |key, _| key.match?(/^CVAR_/) }
+        .map { Cvar.new(name: _1, value: _2.to_i) }
+    end
+  end
+
+  def initialize(ids, filepath)
+    @doc         = SimpleXlsxReader.open(filepath)
+    @headers     = []
+    @variants    = []
+    @assignments = {}
+
+    overview_sheet
+      .rows
+      .each do |row|
+        parse_headers(row) if @headers.empty?
+        next if @headers.empty?
+        next if row[1].nil? || row[3].nil?
+        next unless ids.any? { row[1].match?(/#{_1}/i) }
+        next unless row[3].match?(/CVAR/)
+
+        @variants << Variant.new(@headers, row)
+      end
+
+    param_assignment_sheet
+      .rows
+      .each(headers: PARAM_ASSIGNMENT_HEADERS) do |row|
+        next if row[:cvar].nil?
+
+        @assignments[row[:name]] = Cvar.new(name: row[:cvar], package: row[:package])
+      end
+  end
+
+  def overview_sheet
+    @doc
+      .sheets
+      .find { _1.name == "Gesamtübersicht" }
+      .tap { fail "Cannot find sheet Gesamtübersicht" if _1.nil? }
+  end
+
+  def param_assignment_sheet
+    @doc
+      .sheets
+      .find { _1.name == "Parameterdefinition" }
+      .tap { fail "Cannot find sheet Parameterdefinition" if _1.nil? }
+  end
+
+  def variant
+    fail "No variant matching #{id} found!" if @variants.empty?
+    fail "More than one variant matching #{id} found!" if @variants.size > 1
+
+    @variants.first
+  end
+
+  def unflatten_list(list)
+    Ecu::LabelList.new \
+      list.flat_map { |label|
+        if has_cvar_assignment?(label)
+          @variants.map { label.with(name: add_cvar_prefix(label, _1)) }
+        else
+          label
+        end
+      }
+  end
+
+  def flatten_list(list)
+    list
+      .select { !cvar_coded?(_1) || has_matching_cvar?(_1) }
+      .map_int { _1.with(name: remove_cvar_prefix(_1.name)) }
+  end
+
+  def cvar_coded?(label)
+    label.name.match?(CVAR_PREFIX_REGEXP)
+  end
+
+  def has_cvar_assignment?(label)
+    @assignments.key?(label.name)
+  end
+
+  def has_matching_cvar?(label)
+    variant.cvars.any? { label.name.match?(_1.label_prefix) }
+  end
+
+  def add_cvar_prefix(label, variant)
+    variant
+      .cvars
+      .find { _1 == @assignments[label.name] }
+      .then { _1.merge(@assignments[label.name]) }
+      .then { _1.label_prefix + label.name }
+  end
+
+  def remove_cvar_prefix(name)
+    name.sub(CVAR_PREFIX_REGEXP, "")
+  end
+
+  def parse_headers(row)
+    return unless row[1]&.match?(/SWFK-ID/)
+    return unless row[1]&.match?(/CVAR_BeguData/)
+    return unless row[2]&.match?(/Kommentar/)
+
+    @headers = row
+      .map(&:chomp)
+      .map(&:lstrip)
+      .map(&:strip)
+      .map { _1.sub(/\n.*/, "") }
+      .tap { _1[0] = "typestr" }
+  end
+end

data/lib/core_ext.rb

@@ -0,0 +1,40 @@
+require "pastel"
+
+class Object
+  def as
+    yield self
+  end
+end
+
+class Integer
+  def quantify(singular, plural="#{singular}s")
+    "#{to_s} #{self == 1 ? singular : plural}"
+  end
+end
+
+class String
+  def indent(spaces = 2)
+    lines.map { |line| " " * spaces + line }.join
+  end
+
+  def inline_dcm
+    "KONSERVIERUNG_FORMAT 2.0\n\n" + self
+  end
+
+  def enquote
+    "\"#{to_s}\""
+  end
+
+  def present?
+    !empty?
+  end
+
+  def colorize(color)
+    return self if false
+    pastel.decorate(self, color)
+  end
+
+  def pastel
+    @@pastel ||= Pastel.new
+  end
+end

data/lib/diff_viewer.rb

@@ -0,0 +1,14 @@
+require "diffy"
+
+module Dcm
+  class DiffViewer
+    def initialize(left, right)
+      Diffy::Diff.default_format = :color
+      @diff = Diffy::Diff.new(left, right)
+    end
+
+    def to_s
+      @diff.to_s + "\n"
+    end
+  end
+end

data/lib/file_reader.rb

@@ -0,0 +1,16 @@
+class FileReader
+  def self.read(file)
+    str = File.read(file)
+
+    # Try it as UTF-8 directly
+    cleaned = str.dup.force_encoding('UTF-8')
+    unless cleaned.valid_encoding?
+      # Some of it might be old Windows code page
+      cleaned = str.encode( 'UTF-8', 'Windows-1252' )
+    end
+    str = cleaned
+  rescue EncodingError
+    # Force it to UTF-8, throwing out invalid bits
+    str.encode!('UTF-8', invalid: :replace, undef: :replace)
+  end
+end

data/lib/label_selector.rb

@@ -0,0 +1,26 @@
+require_relative "core_ext"
+require_relative "selecta"
+require_relative "list_colorizer"
+
+class LabelSelector
+  def self.choose_from(list)
+    view = selecta(list)
+    print view
+    STDIN.gets
+  rescue Interrupt => e
+    clear_screen
+    exit 0
+  end
+
+  def self.selecta(list)
+    clear_screen
+    Selecta.new.main_api(keys: list.map(&:name),
+                         values: list.map { |e| ListColorizer.call(e.to_s(detail: true)) },
+                         options: { height: "full" })
+    clear_screen
+  end
+
+  def self.clear_screen
+    print "\e[2J\e[H"
+  end
+end

data/lib/list_colorizer.rb

@@ -0,0 +1,7 @@
+class ListColorizer
+  def self.call(str)
+    str.
+      gsub(/^([^ :]+)/) { $1.colorize(:green) }.
+      gsub(/^  (\w+)/) { "  " + $1.colorize(:bright_blue) }
+  end
+end