text_alignment 0.2.0

data/lib/text_alignment/lcs_min.rb

@@ -0,0 +1,160 @@
+#!/usr/bin/env ruby
+require 'diff-lcs'
+
+module TextAlignment; end unless defined? TextAlignment
+
+# change the class definition of ContextChange to allow update of the two instance variables
+class Diff::LCS::ContextChange
+  attr_accessor :old_position, :new_position
+end
+
+# It finds minimal lcs and sdiff of the given strings, str1 and str2.
+# It relies on the diff-lcs gem for the computation of lcs table.
+class TextAlignment::LCSMin
+  attr_reader :sdiff, :lcs, :m1_initial, :m1_final, :m2_initial, :m2_final
+
+  PLACEHOLDER_CHAR = '_'
+
+  def initialize (str1, str2)
+    raise ArgumentError, "nil string" if str1.nil? || str2.nil?
+    raise ArgumentError, "empty string" if str1.empty? || str2.empty?
+
+    # str1 is copied as it is.
+    # str2 is copied with w/s characters replaced with the placeholder characters,
+    # to avoid overfitting to w/s characters during LCS computation.
+    @str1 = str1
+    @str2 = str2.gsub(/\s/, PLACEHOLDER_CHAR)
+
+    # find the corresponding minimal range of the two strings
+    r = _find_min_range(0, @str1.length - 1, 0, @str2.length - 1)
+    @m1_initial, @m1_final, @m2_initial, @m2_final = r[:m1_initial], r[:m1_final], r[:m2_initial], r[:m2_final]
+
+    if @m1_initial.nil?
+      @sdiff = nil
+      @lcs = 0
+    else
+      # compute sdiff and lcs
+      # here the original str2 is used with all the w/s characters preserved.
+      @sdiff = Diff::LCS.sdiff(@str1[@m1_initial..@m1_final], str2[@m2_initial..@m2_final])
+      @lcs = @sdiff.count{|d| d.action == '='}
+
+      # adjust the position values of sdiff
+      @sdiff.each do |h|
+        h.old_position += @m1_initial unless h.old_position.nil?
+        h.new_position += @m2_initial unless h.new_position.nil?
+      end
+
+      (0 ... @m2_initial).reverse_each{|i| @sdiff.unshift(Diff::LCS::ContextChange.new('+', nil, nil, i, @str2[i]))}
+      (0 ... @m1_initial).reverse_each{|i| @sdiff.unshift(Diff::LCS::ContextChange.new('-', i, @str1[i], nil, nil))}
+      (@m1_final + 1 ... @str1.length).each{|i| @sdiff.push(Diff::LCS::ContextChange.new('-', i, @str1[i], nil, nil))}
+      (@m2_final + 1 ... @str2.length).each{|i| @sdiff.push(Diff::LCS::ContextChange.new('+', nil, nil, i, @str2[i]))}
+    end
+  end
+
+  def _find_min_range (m1_initial, m1_final, m2_initial, m2_final, clcs = 0)
+    return nil if (m1_final - m1_initial < 0) || (m2_final - m2_initial < 0)
+    sdiff = Diff::LCS.sdiff(@str1[m1_initial..m1_final], @str2[m2_initial..m2_final])
+    lcs = sdiff.count{|d| d.action == '='}
+
+    return nil if lcs == 0
+    return nil if lcs < clcs
+
+    match_last  = sdiff.rindex{|d| d.action == '='}
+    m1_final    = sdiff[match_last].old_position + m1_initial
+    m2_final    = sdiff[match_last].new_position + m2_initial
+
+    match_first = sdiff.index{|d| d.action == '='}
+    m1_initial  = sdiff[match_first].old_position + m1_initial
+    m2_initial  = sdiff[match_first].new_position + m2_initial
+
+    # attempt for shorter match
+    if ((m1_final - m1_initial) > (m2_final - m2_initial))
+      r = _find_min_range(m1_initial + 1, m1_final, m2_initial, m2_final, lcs)
+      return r unless r.nil?
+      r = _find_min_range(m1_initial, m1_final - 1, m2_initial, m2_final, lcs)
+      return r unless r.nil?
+    else
+      r = _find_min_range(m1_initial, m1_final, m2_initial + 1, m2_final, lcs)
+      return r unless r.nil?
+      r = _find_min_range(m1_initial, m1_final, m2_initial, m2_final - 1, lcs)
+      return r unless r.nil?
+    end
+
+    return {
+      m1_initial: m1_initial,
+      m1_final: m1_final,
+      m2_initial: m2_initial,
+      m2_final: m2_final
+    }
+  end
+
+  def num_big_gaps (sdiff, initial, last)
+    raise ArgumentError, "nil sdiff" if sdiff.nil?
+    raise ArgumentError, "invalid indice: #{initial}, #{last}" unless last >= initial
+
+    state1 = :initial
+    state2 = :initial
+    gaps1 = []
+    gaps2 = []
+
+    (initial .. last).each do |i|
+      case sdiff[i].action
+      when '='
+        state1 = :continue
+        state2 = :continue
+      when '!'
+        gaps1 << 1
+        state1 = :break
+
+        if state2 == :break
+          gaps2[-1] += 1
+        else
+          gaps2 << 1
+        end
+        state2 = :continue
+      when '+'
+        if state1 == :break
+          gaps1[-1] += 1
+        else
+          gaps1 << 1
+        end
+        state1 = :break
+      when '-'
+        if state2 == :break
+          gaps2[-1] += 1
+        else
+          gaps2 << 1
+        end
+        state2 = :break
+      end
+    end
+
+    num_big_gaps1 = gaps1.select{|g| g > MAX_LEN_BIG_GAP}.length
+    num_big_gaps2 = gaps2.select{|g| g > MAX_LEN_BIG_GAP}.length
+    num_big_gaps1 + num_big_gaps2
+  end
+
+end
+
+
+if __FILE__ == $0
+  require 'json'
+  require 'text_alignment/lcs_cdiff'
+
+  str2 = 'abcde'
+  str1 = 'naxbyzabcdexydzem'
+
+  str1 = "TI  - Identification of a region which directs the monocytic activity of the\n      colony-stimulating factor 1 (macrophage colony-stimulating factor) receptor\n      promoter and binds PEBP2/CBF (AML1)."
+  str2 = "Identification of a region which directs the monocytic activity of the colony-stimulating factor 1 (macrophage colony-stimulating factor) receptor promoter and binds PEBP2/CBF (AML1).\nThe receptor for the macrophage colony-stimulating factor (or colony-stimulating factor 1 [CSF-1]) is expressed from different promoters in monocytic cells and placental trophoblasts. We have demonstrated that the monocyte-specific expression of the CSF-1 receptor is regulated at the level of transcription by a tissue-specific promoter whose activity is stimulated by the monocyte/B-cell-specific transcription factor PU.1 (D.-E. Zhang, C.J. Hetherington, H.-M. Chen, and D.G. Tenen, Mol. Cell. Biol. 14:373-381, 1994). Here we report that the tissue specificity of this promoter is also mediated by sequences in a region II (bp -88 to -59), which lies 10 bp upstream from the PU.1-binding site. When analyzed by DNase footprinting, region II was protected preferentially in monocytic cells. Electrophoretic mobility shift assays confirmed that region II interacts specifically with nuclear proteins from monocytic cells. Two gel shift complexes (Mono A and Mono B) were formed with separate sequence elements within this region. Competition and supershift experiments indicate that Mono B contains a member of the polyomavirus enhancer-binding protein 2/core-binding factor (PEBP2/CBF) family, which includes the AML1 gene product, while Mono A is a distinct complex preferentially expressed in monocytic cells. Promoter constructs with mutations in these sequence elements were no longer expressed specifically in monocytes. Furthermore, multimerized region II sequence elements enhanced the activity of a heterologous thymidine kinase promoter in monocytic cells but not other cell types tested. These results indicate that the monocyte/B-cell-specific transcription factor PU.1 and the Mono A and Mono B protein complexes act in concert to regulate monocyte-specific transcription of the CSF-1 receptor."
+  # str2 = "Identification of a region which directs the monocytic activity of the colony-stimulating factor 1 (macrophage colony-stimulating factor) receptor promoter and binds PEBP2/CBF (AML1).\nThe receptor for the macrophage colony-stimulating factor (or colony-stimulating factor 1 [CSF-1]) is expressed from different promoters in monocytic cells and placental trophoblasts."
+
+  if ARGV.length == 2
+    str1 = JSON.parse(File.read(ARGV[0]).strip)["text"]
+    str2 = JSON.parse(File.read(ARGV[1]).strip)["text"]
+  end
+
+  lcsmin = TextAlignment::LCSMin.new(str1, str2)
+  # puts lcs
+  # sdiff.each {|h| p h}
+  puts TextAlignment.sdiff2cdiff(lcsmin.sdiff)
+end

data/lib/text_alignment/mappings.rb

@@ -0,0 +1,75 @@
+module TextAlignment
+
+  TextAlignment::MAPPINGS = [
+    ["©", "(c)"],   #U+00A9 (Copyright Sign)
+
+    ["α", "alpha"],   #U+03B1 (greek small letter alpha)
+    ["β", "beta"],    #U+03B2 (greek small letter beta)
+    ["γ", "gamma"],   #U+03B3 (greek small letter gamma)
+    ["δ", "delta"],   #U+03B4 (greek small letter delta)
+    ["ε", "epsilon"], #U+03B5 (greek small letter epsilon)
+    ["ζ", "zeta"],    #U+03B6 (greek small letter zeta)
+    ["η", "eta"],     #U+03B7 (greek small letter eta)
+    ["θ", "theta"],   #U+03B7 (greek small letter eta)
+    ["ι", "iota"],    #U+03B7 (greek small letter eta)
+    ["κ", "kappa"],   #U+03BA (greek small letter kappa)
+    ["λ", "lambda"],  #U+03BB (greek small letter lambda)
+    ["λ", "lamda"],  #U+03BB (greek small letter lambda)
+    ["μ", "mu"],      #U+03BC (greek small letter mu)
+    ["ν", "nu"],      #U+03BD (greek small letter nu)
+    ["ξ", "xi"],      #U+03BE (greek small letter xi)
+    ["ο", "omicron"], #U+03BF (greek small letter omicron)
+    ["π", "pi"],      #U+03C0 (greek small letter pi)
+    ["ρ", "rho"],     #U+03C1 (greek small letter rho)
+    ["σ", "sigma"],   #U+03C3 (greek small letter sigma)
+    ["τ", "tau"],     #U+03C4 (greek small letter tau)
+    ["υ", "upsilon"], #U+03C5 (greek small letter upsilon)
+    ["φ", "phi"],     #U+03C6 (greek small letter phi)
+    ["χ", "chi"],     #U+03C7 (greek small letter chi)
+    ["ψ", "psi"],     #U+03C8 (greek small letter psi)
+    ["ω", "omega"],   #U+03C9 (greek small letter omega)
+
+    ["Α", "Alpha"],   #U+0391 (greek capital letter alpha)
+    ["Β", "Beta"],    #U+0392 (greek capital letter beta)
+    ["Γ", "Gamma"],   #U+0393 (greek capital letter gamma)
+    ["Δ", "Delta"],   #U+0394 (greek capital letter delta)
+    ["Ε", "Epsilon"], #U+0395 (greek capital letter epsilon)
+    ["Ζ", "Zeta"],    #U+0396 (greek capital letter zeta)
+    ["Η", "Eta"],     #U+0397 (greek capital letter eta)
+    ["Θ", "Theta"],   #U+0398 (greek capital letter theta)
+    ["Ι", "Iota"],    #U+0399 (greek capital letter iota)
+    ["Κ", "Kappa"],   #U+039A (greek capital letter kappa)
+    ["Λ", "Lambda"],  #U+039B (greek capital letter lambda)
+    ["Λ", "Lamda"],  #U+039B (greek capital letter lambda)
+    ["Μ", "Mu"],      #U+039C (greek capital letter mu)
+    ["Ν", "Nu"],      #U+039D (greek capital letter nu)
+    ["Ξ", "Xi"],      #U+039E (greek capital letter xi)
+    ["Ο", "Omicron"], #U+039F (greek capital letter omicron)
+    ["Π", "Pi"],      #U+03A0 (greek capital letter pi)
+    ["Ρ", "Rho"],     #U+03A1 (greek capital letter rho)
+    ["Σ", "Sigma"],   #U+03A3 (greek capital letter sigma)
+    ["Τ", "Tau"],     #U+03A4 (greek capital letter tau)
+    ["Υ", "Upsilon"], #U+03A5 (greek capital letter upsilon)
+    ["Φ", "Phi"],     #U+03A6 (greek capital letter phi)
+    ["Χ", "Chi"],     #U+03A7 (greek capital letter chi)
+    ["Ψ", "Psi"],     #U+03A8 (greek capital letter Psi)
+    ["Ω", "Omega"],   #U+03A9 (greek capital letter omega)
+
+    ["ϕ", "phi"],     #U+03D5 (greek phi symbol)
+
+    ["×", "x"],       #U+00D7 (multiplication sign)
+    ["•", "*"],       #U+2022 (bullet)
+    [" ", " "],       #U+2009 (thin space)
+    [" ", " "],       #U+200A (hair space)
+    [" ", " "],       #U+00A0 (no-break space)
+    ["　", " "],       #U+3000 (ideographic space)
+    ["−", "-"],       #U+2212 (minus sign)
+    ["–", "-"],       #U+2013 (en dash)
+    ["′", "'"],       #U+2032 (prime)
+    ["‘", "'"],       #U+2018 (left single quotation mark)
+    ["’", "'"],       #U+2019 (right single quotation mark)
+    ["“", '"'],       #U+201C (left double quotation mark)
+    ["”", '"'],        #U+201D (right double quotation mark)
+    ['"', "''"]
+  ]
+end

data/lib/text_alignment/text_alignment.rb

@@ -0,0 +1,223 @@
+#!/usr/bin/env ruby
+require 'diff-lcs'
+require 'text_alignment/lcs_min'
+require 'text_alignment/find_divisions'
+require 'text_alignment/lcs_comparison'
+require 'text_alignment/lcs_alignment'
+require 'text_alignment/glcs_alignment'
+require 'text_alignment/mappings'
+
+module TextAlignment; end unless defined? TextAlignment
+
+TextAlignment::SIGNATURE_NGRAM = 5 unless defined? TextAlignment::SIGNATURE_NGRAM
+TextAlignment::NOMATCH_CHARS = "@^|#$%&_" unless defined? TextAlignment::NOMATCH_CHARS
+
+class TextAlignment::TextAlignment
+  attr_reader :sdiff
+  attr_reader :position_map_begin, :position_map_end
+  attr_reader :common_elements, :mapped_elements
+  attr_reader :similarity
+  attr_reader :str1_match_initial, :str1_match_final, :str2_match_initial, :str2_match_final
+
+  def initialize(str1, str2, mappings = [])
+    raise ArgumentError, "nil string" if str1.nil? || str2.nil?
+    raise ArgumentError, "nil mappings" if mappings.nil?
+
+    ## preprocessing
+    str1 = str1.dup
+    str2 = str2.dup
+
+    ## find the first nomatch character
+    TextAlignment::NOMATCH_CHARS.each_char do |c|
+      if str2.index(c).nil?
+        @nomatch_char1 = c
+        break
+      end
+    end
+    raise RuntimeError, "Cannot find nomatch character" if @nomatch_char1.nil? 
+
+    ## find the first nomatch character
+    TextAlignment::NOMATCH_CHARS.each_char do |c|
+      if c != @nomatch_char1 && str1.index(c).nil?
+        @nomatch_char2 = c
+        break
+      end
+    end
+    raise RuntimeError, "Cannot find nomatch character" if @nomatch_char2.nil? 
+
+    # single character mappings
+    character_mappings = mappings.select{|m| m[0].length == 1 && m[1].length == 1}
+    characters_from = character_mappings.collect{|m| m[0]}.join
+    characters_to   = character_mappings.collect{|m| m[1]}.join
+    characters_to.gsub!(/-/, '\-')
+
+    str1.tr!(characters_from, characters_to)
+    str2.tr!(characters_from, characters_to)
+
+    mappings.delete_if{|m| m[0].length == 1 && m[1].length == 1}
+
+    # ASCII foldings
+    # ascii_foldings = mappings.select{|m| m[0].length == 1 && m[1].length > 1}
+    # ascii_foldings.each do |f|
+    #   from = f[1]
+
+    #   if str2.index(f[0])
+    #     to   = f[0] + (@nomatch_char1 * (f[1].length - 1))
+    #     str1.gsub!(from, to)
+    #   end
+
+    #   if str1.index(f[0])
+    #     to   = f[0] + (@nomatch_char2 * (f[1].length - 1))
+    #     str2.gsub!(from, to)
+    #   end
+    # end
+
+    # mappings.delete_if{|m| m[0].length == 1 && m[1].length > 1}
+
+    _compute_mixed_alignment(str1, str2, mappings)
+  end
+
+  def transform_a_span(span)
+    {:begin=>@position_map_begin[span[:begin]], :end=>@position_map_end[span[:end]]}
+  end
+
+  def transform_spans(spans)
+    spans.map{|span| transform_a_span(span)}
+  end
+
+  def transform_denotations!(denotations)
+    denotations.map!{|d| d.begin = @position_map_begin[d.begin]; d.end = @position_map_end[d.end]; d} unless denotations.nil?
+  end
+
+  def transform_hdenotations(hdenotations)
+    unless hdenotations.nil?
+      hdenotations_new = Array.new(hdenotations)
+      (0...hdenotations.length).each {|i| hdenotations_new[i][:span] = transform_a_span(hdenotations[i][:span])}
+      hdenotations_new
+    end
+  end
+
+  private
+
+  def _compute_mixed_alignment(str1, str2, mappings = [])
+    lcsmin = TextAlignment::LCSMin.new(str1, str2)
+    lcs = lcsmin.lcs
+    @sdiff = lcsmin.sdiff
+
+    cmp = TextAlignment::LCSComparison.new(str1, str2, lcs, @sdiff)
+    @similarity         = cmp.similarity
+    @str1_match_initial = cmp.str1_match_initial
+    @str1_match_final   = cmp.str1_match_final
+    @str2_match_initial = cmp.str2_match_initial
+    @str2_match_final   = cmp.str2_match_final
+
+    posmap_begin, posmap_end = {}, {}
+    @common_elements, @mapped_elements = [], []
+
+    addition, deletion = [], []
+
+    @sdiff.each do |h|
+      case h.action
+      when '='
+        p1, p2 = h.old_position, h.new_position
+
+        @common_elements << [str1[p1], str2[p2]]
+        posmap_begin[p1], posmap_end[p1] = p2, p2
+
+        if !addition.empty? && deletion.empty?
+          posmap_end[p1] = p2 - addition.length unless p1 == 0
+        elsif addition.empty? && !deletion.empty?
+          deletion.each{|p| posmap_begin[p], posmap_end[p] = p2, p2}
+        elsif !addition.empty? && !deletion.empty?
+          if addition.length > 1 || deletion.length > 1
+            galign = TextAlignment::GLCSAlignment.new(str1[deletion[0] .. deletion[-1]], str2[addition[0] .. addition[-1]], mappings)
+            galign.position_map_begin.each {|k, v| posmap_begin[k + deletion[0]] = v.nil? ? nil : v + addition[0]}
+            galign.position_map_end.each   {|k, v|   posmap_end[k + deletion[0]] = v.nil? ? nil : v + addition[0]}
+            posmap_begin[p1], posmap_end[p1] = p2, p2
+            @common_elements += galign.common_elements
+            @mapped_elements += galign.mapped_elements
+          else
+            posmap_begin[deletion[0]], posmap_end[deletion[0]] = addition[0], addition[0]
+            deletion[1..-1].each{|p| posmap_begin[p], posmap_end[p] = nil, nil}
+            @mapped_elements << [str1[deletion[0], deletion.length], str2[addition[0], addition.length]]
+          end
+        end
+
+        addition.clear; deletion.clear
+
+      when '!'
+        deletion << h.old_position
+        addition << h.new_position
+      when '-'
+        deletion << h.old_position
+      when '+'
+        addition << h.new_position
+      end
+    end
+
+    p1, p2 = str1.length, str2.length
+    posmap_begin[p1], posmap_end[p1] = p2, p2
+
+    if !addition.empty? && deletion.empty?
+      posmap_end[p1] = p2 - addition.length unless p1 == 0
+    elsif addition.empty? && !deletion.empty?
+      deletion.each{|p| posmap_begin[p], posmap_end[p] = p2, p2}
+    elsif !addition.empty? && !deletion.empty?
+      if addition.length > 1 && deletion.length > 1
+        galign = TextAlignment::GLCSAlignment.new(str1[deletion[0] .. deletion[-1]], str2[addition[0] .. addition[-1]], mappings)
+        galign.position_map_begin.each {|k, v| posmap_begin[k + deletion[0]] = v.nil? ? nil : v + addition[0]}
+        galign.position_map_end.each   {|k, v|   posmap_end[k + deletion[0]] = v.nil? ? nil : v + addition[0]}
+        posmap_begin[p1], posmap_end[p1] = p2, p2
+        @common_elements += galign.common_elements
+        @mapped_elements += galign.mapped_elements
+      else
+        posmap_begin[deletion[0]], posmap_end[deletion[0]] = addition[0], addition[0]
+        deletion[1..-1].each{|p| posmap_begin[p], posmap_end[p] = nil, nil}
+        @mapped_elements << [str1[deletion[0], deletion.length], str2[addition[0], addition.length]]
+      end
+    end
+
+    @position_map_begin = posmap_begin.sort.to_h
+    @position_map_end = posmap_end.sort.to_h
+  end
+end
+
+if __FILE__ == $0
+  require 'json'
+  require 'text_alignment/lcs_cdiff'
+
+  str1 = "TI  - Identification of a region which directs the monocytic activity of the\n      colony-stimulating factor 1 (macrophage colony-stimulating factor) receptor\n      promoter and binds PEBP2/CBF (AML1)."
+  str2 = "Identification of a region which directs the monocytic activity of the colony-stimulating factor 1 (macrophage colony-stimulating factor) receptor promoter and binds PEBP2/CBF (AML1).\nThe receptor for the macrophage colony-stimulating factor (or colony-stimulating factor 1 [CSF-1]) is expressed from different promoters in monocytic cells and placental trophoblasts. We have demonstrated that the monocyte-specific expression of the CSF-1 receptor is regulated at the level of transcription by a tissue-specific promoter whose activity is stimulated by the monocyte/B-cell-specific transcription factor PU.1 (D.-E. Zhang, C.J. Hetherington, H.-M. Chen, and D.G. Tenen, Mol. Cell. Biol. 14:373-381, 1994). Here we report that the tissue specificity of this promoter is also mediated by sequences in a region II (bp -88 to -59), which lies 10 bp upstream from the PU.1-binding site. When analyzed by DNase footprinting, region II was protected preferentially in monocytic cells. Electrophoretic mobility shift assays confirmed that region II interacts specifically with nuclear proteins from monocytic cells. Two gel shift complexes (Mono A and Mono B) were formed with separate sequence elements within this region. Competition and supershift experiments indicate that Mono B contains a member of the polyomavirus enhancer-binding protein 2/core-binding factor (PEBP2/CBF) family, which includes the AML1 gene product, while Mono A is a distinct complex preferentially expressed in monocytic cells. Promoter constructs with mutations in these sequence elements were no longer expressed specifically in monocytes. Furthermore, multimerized region II sequence elements enhanced the activity of a heterologous thymidine kinase promoter in monocytic cells but not other cell types tested. These results indicate that the monocyte/B-cell-specific transcription factor PU.1 and the Mono A and Mono B protein complexes act in concert to regulate monocyte-specific transcription of the CSF-1 receptor."
+
+  # anns1 = JSON.parse File.read(ARGV[0]), :symbolize_names => true
+  # anns2 = JSON.parse File.read(ARGV[1]), :symbolize_names => true
+
+  if ARGV.length == 2
+    # str1  = JSON.parse(File.read(ARGV[0]).strip)["text"]
+    # denotations = JSON.parse(File.read(ARGV[0]).strip, symbolize_names:true)[:denotations]
+    # str2  = JSON.parse(File.read(ARGV[1]).strip)["text"]
+    str1 = File.read(ARGV[0])
+    str2 = File.read(ARGV[1])
+  end
+
+  # dictionary = [["β", "beta"]]
+  # align = TextAlignment::TextAlignment.new(str1, str2)
+  align = TextAlignment::TextAlignment.new(str1, str2, TextAlignment::MAPPINGS)
+
+  # p align.common_elements
+  # puts "---------------"
+  # p align.mapped_elements
+
+  puts TextAlignment::sdiff2cdiff(align.sdiff)
+  # p align
+  # puts "-----"
+
+  # p denotations
+  # puts "-----"
+
+  # new_denotations = align.transform_hdenotations(denotations)
+
+  # p new_denotations
+  # puts "-----"
+
+end