amatch 0.3.0 → 0.4.1

data/README.md

@@ -0,0 +1,124 @@
+# amatch - Approximate Matching Extension for Ruby
+
+## Description
+
+This is a collection of classes that can be used for Approximate
+matching, searching, and comparing of Strings. They implement algorithms
+that compute the Levenshtein edit distance, Sellers edit distance, the
+Hamming distance, the longest common subsequence length, the longest common
+substring length, the pair distance metric, the Jaro-Winkler metric.
+
+## Installation
+
+To install this extension as a gem type
+
+ # gem install amatch
+
+into the shell.
+
+## Download
+
+The homepage of this library is located at
+
+* https://github.com/flori/amatch
+
+## Examples
+
+    require 'amatch'
+    # => true
+    include Amatch
+    # => Object
+    
+    m = Sellers.new("pattern")
+    # => #<Amatch::Sellers:0x40366324>
+    m.match("pattren")
+    # => 2.0
+    m.substitution = m.insertion = 3
+    # => 3
+    m.match("pattren")
+    # => 4.0
+    m.reset_weights
+    # => #<Amatch::Sellers:0x40366324>
+    m.match(["pattren","parent"])
+    # => [2.0, 4.0]
+    m.search("abcpattrendef")
+    # => 2.0
+    
+    m = Levenshtein.new("pattern")
+    # => #<Amatch::Levenshtein:0x4035919c>
+    m.match("pattren")
+    # => 2
+    m.search("abcpattrendef")
+    # => 2
+    "pattern language".levenshtein_similar("language of patterns")
+    # => 0.2
+    
+    m = Amatch::DamerauLevenshtein.new("pattern")
+    # => #<Amatch::DamerauLevenshtein:0x007fc3483dd278>
+    m.match("pattren")
+    # => 1
+    "pattern language".damerau_levenshtein_similar("language of patterns")
+    # => 0.19999999999999996
+    
+    m = Hamming.new("pattern")
+    # => #<Amatch::Hamming:0x40350858>
+    m.match("pattren")
+    # => 2
+    "pattern language".hamming_similar("language of patterns")
+    # => 0.1
+    
+    m = PairDistance.new("pattern")
+    # => #<Amatch::PairDistance:0x40349be8>
+    m.match("pattr en")
+    # => 0.545454545454545
+    m.match("pattr en", nil)
+    # => 0.461538461538462
+    m.match("pattr en", /t+/)
+    # => 0.285714285714286
+    "pattern language".pair_distance_similar("language of patterns")
+    # => 0.928571428571429
+    
+    m = LongestSubsequence.new("pattern")
+    # => #<Amatch::LongestSubsequence:0x4033e900>
+    m.match("pattren")
+    # => 6
+    "pattern language".longest_subsequence_similar("language of patterns")
+    # => 0.4
+    
+    m = LongestSubstring.new("pattern")
+    # => #<Amatch::LongestSubstring:0x403378d0>
+    m.match("pattren")
+    # => 4
+    "pattern language".longest_substring_similar("language of patterns")
+    # => 0.4
+   
+    m = Jaro.new("pattern")
+    # => #<Amatch::Jaro:0x363b70>
+    m.match("paTTren")
+    # => 0.952380952380952
+    m.ignore_case = false
+    m.match("paTTren")
+    # => 0.742857142857143
+    "pattern language".jaro_similar("language of patterns")
+    # => 0.672222222222222
+   
+    m = JaroWinkler.new("pattern")
+    # #<Amatch::JaroWinkler:0x3530b8>
+    m.match("paTTren")
+    # => 0.971428571712403
+    m.ignore_case = false
+    m.match("paTTren")
+    # => 0.79428571505206
+    m.scaling_factor = 0.05
+    m.match("pattren")
+    # => 0.961904762046678
+    "pattern language".jarowinkler_similar("language of patterns")
+    # => 0.672222222222222
+
+## Author
+
+Florian Frank mailto:flori@ping.de
+
+## License
+
+Apache License, Version 2.0 – See the COPYING file in the source archive.

data/Rakefile

@@ -13,23 +13,18 @@ Amatch is a library for approximate string matching and searching in strings.
 Several algorithms can be used to do this, and it's also possible to compute a
 similarity metric number between 0.0 and 1.0 for two given strings.
 EOT
-  executables << 'agrep.rb'
+  executables << 'agrep' << 'dupfind'
   bindir      'bin'
   test_dir    'tests'
-  ignore      '.*.sw[pon]', 'pkg', 'Gemfile.lock', '.AppleDouble', '.rbx', '.bundle'
+  ignore      '.*.sw[pon]', 'pkg', 'Gemfile.lock', '.AppleDouble', '.rbx', 'Makefile'
+  package_ignore '.all_images.yml', '.gitignore', 'VERSION'
   title       "#{name.camelize} - Approximate Matching"
-  readme      'README.rdoc'
+  readme      'README.md'
   require_paths %w[lib ext]
   dependency             'tins',      '~>1.0'
-  development_dependency 'test-unit', '~>2.3'
-  development_dependency 'utils'
-  development_dependency 'rake',      '~>10', '<11.0'
-
-  install_library do
-    libdir = CONFIG["sitelibdir"]
-    src, = Dir['ext/amatch.*'].reject { |x| x =~ /\.[co]$/ }
-    install(src, File.join(libdir, File.basename(src)), :verbose => true)
-    mkdir_p dst = File.join(libdir, 'amatch')
-    install('lib/amatch/version.rb', File.join(dst, 'version.rb'), :verbose => true)
-  end
+  dependency             'mize'
+  development_dependency 'test-unit', '~>3.0'
+  development_dependency 'all_images'
+  required_ruby_version '>=2.4'
+  licenses << 'Apache-2.0'
 end

data/bin/{agrep.rb → agrep}

@@ -15,15 +15,21 @@ end
 
 class Amatch::Levenshtein
   def search_relative(strings)
-    search(strings).to_f / pattern.size
+    if Array === strings
+      search(strings).map { |s| s.to_f / pattern.size }
+    else
+      search(strings).to_f / pattern.size
+    end
   end
 end
 
+$algorithm = 'Levenshtein'
 $distance = 1
 $mode = :search
 begin
   parser = GetoptLong.new
   options = [
+    [ '--algorithm',  '-a',  GetoptLong::REQUIRED_ARGUMENT ],
     [ '--distance',   '-d',  GetoptLong::REQUIRED_ARGUMENT ],
     [ '--relative',   '-r',  GetoptLong::NO_ARGUMENT ],
     [ '--verbose',    '-v',  GetoptLong::NO_ARGUMENT ],
@@ -33,6 +39,8 @@ begin
   parser.each_option do |name, arg|
     name = name.sub(/^--/, '')
     case name
+    when 'algorithm'
+      $algorithm = arg
     when 'distance'
       $distance = arg.to_f
     when 'relative'
@@ -48,7 +56,7 @@ rescue
 end
 pattern = ARGV.shift or usage('Pattern needed!', options)
 
-matcher = Amatch::Levenshtein.new(pattern)
+matcher = Amatch.const_get($algorithm).new(pattern)
 size = 0
 start = Time.new
 if ARGV.size > 0 then
@@ -56,9 +64,12 @@ if ARGV.size > 0 then
     File.stat(filename).file? or next
     size += File.size(filename)
     begin
-      File.open(filename, 'r').each_line do |line|
-        if matcher.__send__($mode, line) <= $distance
-          puts "#{filename}:#{line}"
+      File.open(filename, 'r').each_line.each_slice(1000) do |lines|
+        results = matcher.__send__($mode, lines) 
+        lines.zip(results) do |line, r|
+          if r <= $distance
+            puts "#{filename}:#{line}"
+          end
         end
       end
     rescue
@@ -66,10 +77,13 @@ if ARGV.size > 0 then
     end
   end
 else
-  STDIN.each_line do |line|
-    size += line.size
-    if matcher.__send__($mode, line) <= $distance
-      puts line
+  STDIN.each_line.each_slice(1000) do |lines|
+    size += lines.size
+    results = matcher.__send__($mode, lines)
+    lines.zip(results) do |line, r|
+      if r <= $distance
+        puts line
+      end
     end
   end
 end

data/bin/dupfind

@@ -0,0 +1,153 @@
+#! /usr/bin/env ruby
+
+require 'tins/go'
+include Tins::GO
+require 'tins/minimize'
+class Array
+  include Tins::Minimize
+end
+require 'amatch'
+begin
+  require 'infobar'
+rescue LoadError
+  warn "Please install gem infobar to run this executable!"
+  exit 1
+end
+
+def usage
+  puts <<EOT
+Usage: #{File.basename($0)} [OPTIONS] FILE
+
+ -a ALGO     Amatch matching algorithm
+ -p LIMIT    more than p similarity to be a match
+ -R NUMBER   skip NUMBER mismatch for building ranges
+ -r NUMBER   minimum length to be counted as a range
+ -i          compute a PNG per file
+
+Repor$ bugs to <flori@ping.de>.
+EOT
+  exit 0
+end
+
+class FindDuplicates
+  def initialize(algo, p_lim, filename)
+    @algo, @p_lim, @filename = algo, p_lim, filename
+  end
+
+  attr_reader :filename
+
+  attr_reader :algo
+
+  attr_reader :p_lim
+
+  memoize method:
+  def lines
+    File.readlines(filename)
+  end
+
+  memoize method:
+  def matrix
+    result = lines.with_infobar(label: filename, output: STDERR).map do |l1|
+      +infobar
+      a = algo.new(l1)
+      r = a.similar(lines)
+      r.map! { |s| s >= p_lim ? ?1 : ?0 }
+      r.join
+    end
+    infobar.finish
+    infobar.newline
+    result
+  end
+
+  def pbm(output: $>)
+    output << <<HEADER
+P1
+#{matrix.size} #{matrix.size}
+HEADER
+    output << matrix.map { |line| line.each_char.to_a * ' ' } * ?\n
+    self
+  end
+
+  def png(output: $>)
+    IO.popen("pnmtopng", 'w+') do |conv|
+      pbm(output: conv)
+      conv.close_write
+      output.write(conv.read)
+    end
+    self
+  end
+
+  def create_image
+    suffix = Regexp.quote(File.extname(filename))
+    f = filename.sub(/(#{suffix}|)\z/, '.png')
+    File.open(f, 'wb') do |output|
+      png(output: output)
+      infobar.puts "Writing output to #{f.inspect}."
+    end
+    self
+  end
+
+  def similar_ranges(min_range: 3, skip_range: 0)
+    set = 0
+    ranges = { set => [] }
+    m = matrix
+    n = m.size
+    skip_count = 0
+    n.downto(1) do |h|
+      (n - h + 1).upto(n - 1) do |k|
+        i = k
+        j = k - (n - h + 1)
+        if m[i][j] == ?1
+          skip_count = 0
+          ranges[set] << [ i, j ]
+        elsif !ranges[set].empty? && skip_count < skip_range
+          skip_count += 1
+        else
+          skip_count = 0
+          ranges[set].empty? or ranges[set += 1] = []
+        end
+      end
+      skip_count = 0
+      ranges[set].empty? or ranges[set += 1] = []
+    end
+    ranges.each { |_, r|
+      r.flatten!
+      r.sort!
+      r.map! { |x| x + 1 }
+      r.minimize!
+      r.reject! { |s| s.size < min_range }
+    }.reject! { |_, r| r.empty? }
+    unions = []
+    while !ranges.empty?
+      _, r = ranges.first
+      equivalent = ranges.reject { |_, v| (v & r).empty? }
+      unions << equivalent.values.flatten.uniq
+      ranges.delete_if { |k, _| equivalent.keys.include?(k) }
+    end
+    unions.each do |r|
+      r.map! do |x|
+        "#{filename}:#{x.begin}-#{x.end}"
+      end
+    end
+    unions
+  end
+end
+
+opts = go 'a:p:R:r:ih'
+
+usage if opts[?h]
+algo       = Amatch.const_get(opts[?a] || 'Levenshtein')
+p_lim      = (opts[?p] || 0.95).to_f
+min_range  = (opts[?r] || 3).to_i
+skip_range = opts[?R].to_i
+ARGV.empty? and usage
+
+filenames = ARGV.inject([]) { |s, f| s.concat(Dir[f]) }
+for filename in filenames
+  finder = FindDuplicates.new(algo, p_lim, filename)
+  opts[?i] and finder.create_image
+  for s in finder.similar_ranges(min_range: min_range, skip_range: skip_range)
+    infobar.reset
+    puts s, ?\n
+  end
+end