csvjoin 0.6.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 556d47fbf1bc4d3cb0821ec85ea557bddda5b2087e12d96657ca422b4d05e200
+  data.tar.gz: b5a0aeccd15f5f100917f1506460540b37f7c5e3bad513d3323cab249353bb21
+SHA512:
+  metadata.gz: cf7cd627f6ec3f8def14ed499d947bdace1115b275d77fc966259e812ac679108675a8a0b6f4cb3be02469d339d15f4ae072f9171635f3aa14aedb7f014e6f95
+  data.tar.gz: 548d2195ae23b5aac1bf4b6eeefec69c36ae33aa7fb1517d0c8c4628f2ec4828f30a3f5493857ed172c7aed7160da355495dad46f4b2ec22d6df2bc9f65d999d

data/bin/csvjoin

@@ -0,0 +1,19 @@
+#!/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/comparator.rb'
+
+if (ARGV.length < 2 || (ARGV.include?"-h") || (ARGV.include?"--help"))
+  puts "Usage: csvjoin FILE1 FILE2 [ColumnA1=ColumnA2,ColumnB1=ColumnB2]"
+  puts "Joins two CSV files looking for same values in specified columns. "
+  puts "If no columns specified by default it will use columns with the same name in both files"
+else
+  c = CSVJoin::Comparator.new
+  t1 = ARGV.shift
+  t2 = ARGV.shift
+  params = ARGV.shift
+  c.columns_to_compare(params) if params
+  c.input_col_sep = ";"
+  res = c.compare(t1, t2)
+  puts res
+end

data/lib/callbacks.rb

@@ -0,0 +1,33 @@
+# Callbacks for Diff::LCS with insert in left "<" + insert in right ">" instead of replace (!)
+module Diff
+  module LCS
+    class NoReplaceDiffCallbacks
+      # Returns the difference set collected during the diff process.
+      attr_reader :diffs
+
+      #:yields self:
+      def initialize
+        @diffs = []
+        yield self if block_given?
+      end
+
+      def match(event)
+        @diffs << Diff::LCS::ContextChange.simplify(event)
+      end
+
+      def discard_a(event)
+        @diffs << Diff::LCS::ContextChange.simplify(event)
+      end
+
+      def discard_b(event)
+        @diffs << Diff::LCS::ContextChange.simplify(event)
+      end
+
+      def change(event)
+        discard_a(Diff::LCS::ContextChange.new("<", event.old_position, event.old_element, nil, nil))
+        discard_b(Diff::LCS::ContextChange.new(">", nil, nil, event.new_position, event.new_element))
+        # @diffs << Diff::LCS::ContextChange.simplify(event)
+      end
+    end
+  end
+end

data/lib/comparator.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require_relative 'data_row'
+require_relative 'callbacks'
+require 'diff/lcs'
+
+
+module CSVJoin
+  LEFT = 1 # 'left'
+  RIGHT = 2 # 'right'
+
+  # Compare and join two tables
+  class Comparator
+    attr_accessor :columns, :weights
+    attr_accessor :headers, :data, :rows
+    attr_accessor :input_col_sep
+
+    def initialize
+      @data = []
+      @rows = []
+      @empty = []
+      @input_col_sep = ","
+    end
+
+    def intuit_col_sep(line)
+      return "," if line.nil?
+
+      [",", ";", "\t"].max_by { |char| line.count(char) }
+    end
+
+    def intuit_separator(file)
+      File.open(file, encoding: 'bom|utf-8').each do |line|
+        @input_col_sep = intuit_col_sep(line)
+        break
+      end
+      file
+    end
+
+    def parse(data)
+      if File.exist? data
+        intuit_separator(data)
+        csv = CSV.read(data, headers: true, col_sep: @input_col_sep)
+        raise "Wrong CSV" if csv == []
+      else
+        csv = CSV.parse(data, headers: true, col_sep: @input_col_sep)
+      end
+      csv
+    end
+
+    def csv_to_talimer_rows(csv, side: 'undef')
+      list = []
+      row_columns = columns.map { |c| side.eql?(LEFT) ? c.first : c.last }
+
+      csv.each do |row|
+        row2 = DataRow.new(row.headers, row.fields)
+        row2.columns = row_columns
+        row2.weights = weights
+        row2.side = side
+
+        list << row2
+      end
+
+      list
+    end
+
+    def parse_side(source, side: nil)
+      @data[side] = parse(source)
+      @empty[side] = [*[''] * @data[side].headers.size]
+    end
+
+    def prepare_rows(side: nil)
+      @rows[side] = csv_to_talimer_rows(@data[side], side: side)
+    end
+
+    def prepare(source1, source2)
+      parse_side(source1, side: LEFT)
+      parse_side(source2, side: RIGHT)
+
+      set_default_column_names
+
+      prepare_rows(side: LEFT)
+      prepare_rows(side: RIGHT)
+
+      @headers = [*@data[LEFT].headers, "diff", *@data[RIGHT].headers]
+    end
+
+    # by default use columns with same names in both tables
+    def set_default_column_names
+      return unless @columns.nil?
+
+      @columns = (@data[LEFT].headers & @data[RIGHT].headers).map { |a| [a, a] }
+      @weights = [*[1] * @columns.size]
+    end
+
+    def action_verbose(action)
+      repl = { "!": "!==", "-": "==>", "+": "<==", "=": "===" }
+      raise "wrong action #{action}" unless repl.has_key? action.to_sym
+
+      return repl[action.to_sym]
+    end
+
+    def compare(source1, source2)
+      prepare(source1, source2)
+
+      sdiff = Diff::LCS.sdiff(@rows[LEFT],
+                              @rows[RIGHT],
+                              Diff::LCS::NoReplaceDiffCallbacks)
+
+      CSV.generate(row_sep: "\n", col_sep: @input_col_sep) do |csv|
+        csv << @headers
+        sdiff.each do |change|
+          row = joined_row(change)
+          csv << row
+        end
+      end
+    end
+
+    def joined_row(change)
+      left_row = change.old_element.nil? ? @empty[LEFT] : change.old_element.fields
+      right_row = change.new_element.nil? ? @empty[RIGHT] : change.new_element.fields
+      [*left_row, action_verbose(change.action), *right_row]
+    end
+
+    def columns_to_compare(cols)
+      @columns = []
+
+      cols.scan(/([^,:=~]+)(?:[=~])([^,:=~]+)/).each do |from, to|
+        @columns << [from, to]
+      end
+
+      @weights = [1, *[0] * (@columns.size - 1)]
+    end
+  end
+end

data/lib/csvjoin/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module CSVJoin
+  VERSION = '0.6.1'
+end

data/lib/data_row.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'csv'
+
+module CSVJoin
+  # CSV::Row with specified important columns to compare
+  class DataRow < CSV::Row
+    attr_accessor :columns, :weights
+    attr_reader :side
+
+    attr_writer :side
+
+    def inspect
+      "#{side}:#{super}"
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      res = []
+      @weights.each_with_index do |_weight, index|
+        field = @columns[index]
+        warn("something wrong, #{inspect}, side #{side.inspect}, f'#{field}'==nil") if self[field].nil?
+        res << self[field]
+      end
+      return res.hash
+    end
+
+    #
+    # Returns +true+ if this row contains the same headers and fields in the
+    # same order as +other+.
+    #
+    def ==(other)
+      @columns.each_with_index do |from, index|
+        to = other.columns[index]
+        # warn "something wrong" if self[from].nil? || other[to].nil?
+        return false unless self[from].eql? other[to]
+      end
+      return true
+    end
+  end
+end

metadata

@@ -0,0 +1,148 @@
+--- !ruby/object:Gem::Specification
+name: csvjoin
+version: !ruby/object:Gem::Version
+  version: 0.6.1
+platform: ruby
+authors:
+- Sergey Evstegneiev
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: diff-lcs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.0
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: rspec-simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.49.1
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.16.1
+description: tool to align and merge two tables containing different parts of the
+  same data
+email:
+- serg123e+github@gmail.com
+executables:
+- csvjoin
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/csvjoin
+- lib/callbacks.rb
+- lib/comparator.rb
+- lib/csvjoin/version.rb
+- lib/data_row.rb
+homepage: https://www.github.com/serg123e/csvjoin
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Join 2 CSV tables by specified columns
+test_files: []