similarity_tree 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzViYTMyMjQzZGU0ZTc2ZDc2OGI2NWI3NDYyNWYwZGM2MDFjMzc5MQ==
+  data.tar.gz: !binary |-
+    MDJmOWI2N2QxOTA2YTcwZjA4NThjZDhjZjRjMjAxOTMzY2Y5MTcyMw==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    Mzk0ZTllMGRiOGJmOTI3OTAwNWQyN2U2ZjhiNWNlNGExMDczODc2YjUwYzJh
+    MTI3N2ZjNDM1ZTEyY2MzODEzNmUwYjgyODZkOTRlOTg0ODU5NDZlMGM5YmFi
+    MTU4NmNiYjE0NDM0ZmEwNGE4ZTgzZGJhNGRiY2U1ZWQ3OTJlYTA=
+  data.tar.gz: !binary |-
+    MzE0YjkzOGI2NDQ5ODE4YmQwMmViMjQ2NjllZmZhOWQzNTU1MDVjNWVkMGY1
+    NTYzZGY3NjIwY2I2NDVhMjUyODllOTZkNWM5OTYwYWViMWIwMmU1OWUyM2Rh
+    YzMzNzE3ZmVhZGZiNDgxM2ExZDRkNDhjZDFjYTE4NmZkMDdjYjA=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in similarity_tree.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (c) 2013 Kent Mewhort
+Copyright (c) 2012 Open North Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,96 @@
+# SimilarityTree
+
+This library allows you to generate a tree representing branches/revisions to a set of text HTML files, without any
+prior knowledge of the timelines or change history necessary. You simply need to know the original source document and
+this library builds a tree based on the extent of differences between each document.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'similarity_tree'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install similarity_tree
+
+## Usage
+
+Build a "similarity matrix" of the diff scores between the different documents, then generate the tree from this matrix.
+First, build the "similarity matrix" of the diff scores between the different documents. You must input a set of HTML or
+text documents.  Then, to build the tree itself, you need to specify the document id or filename of the original/root
+document. Eg. for the set of different Creative Commons licences in the test dir:
+
+    documents = Dir.glob('../../similarity_tree/test/cc_licences/*.html')
+    tree = SimilarityTree::SimilarityMatrix.new(documents).build_tree("CC-BY-3.0.html")
+    put tree.to_s  # to_h and to_json are also available as other tree output formats
+
+Result:
+
+    CC-BY-3.0.html
+    -CC-BY-NC-3.0.html (0.9197574893009985)
+    --CC-BY-NC-SA-3.0.html (0.9503146737330241)
+    --CC-BY-NC-ND-3.0.html (0.9456402772710689)
+    -CC-BY-ND-3.0.html (0.9434472109631346)
+
+You can operate directly on **strings** rather than files (in this case, the node id's in the tree will be the file array indices):
+
+    documents = Dir.glob('../../similarity_tree/test/cc_licences/*.html').map { |f| File.read(f) }
+    tree = SimilarityTree::SimilarityMatrix.new(documents).build_tree("CC-BY-3.0.html")
+    put tree.to_s  # to_h and to_json are also available as other tree output formats
+
+    CC-BY-3.0.html
+    -CC-BY-NC-3.0.html (0.9197574893009985)
+    --CC-BY-NC-SA-3.0.html (0.9503146737330241)
+    --CC-BY-NC-ND-3.0.html (0.9456402772710689)
+    -CC-BY-ND-3.0.html (0.9434472109631346)
+
+Result:
+    0
+    -1 (0.9197574893009985)
+    --3 (0.9503146737330241)
+    --4 (0.9456402772710689)
+    -2 (0.9434472109631346)
+
+Or, you can use any **enumerable list of objects** (eg. ActiveRecords) as the inputs. Consider:
+
+    class Document < ActiveRecord::Base
+      attr_accessible :title, :text_filename
+      ...
+    end
+
+You can call:
+
+    tree = SimilarityTree::SimilarityMatrix.new(Document.all,
+        id_func: :title, content_func: :text_filename).build_tree(Document.first.title)
+
+## Additional Options
+
+### Calculation method
+
+You can use either the **term frequency–inverse document frequency** (:tf_idf, the default) or **Dice's coefficient** from a
+standard unix-style diff to calculate the diff scores. Tf-idf works much better where a document has a lot of translations
+(that is, "cut and pastes" of sections of text into different locations) and is also often faster.  However, if your intent
+is to show diffs of the text, the :diff option will correlate better to your diff rendering.
+
+    documents = Dir.glob('../../similarity_tree/test/cc_licences/*.html')
+    tf_idf_tree = SimilarityTree::SimilarityMatrix.new(documents,
+        calculation_method: :tf_idf).build_tree("CC-BY-3.0.html")
+    diff_tree = SimilarityTree::SimilarityMatrix.new(documents,
+        calculation_method: :diff).build_tree("CC-BY-3.0.html")
+
+### Progress output
+
+Performing all the diffs to build a similarity matrix can take a while for large document sets. If you're using this
+gem from a script or a console, you can add a progress bar:
+
+    tree = SimilarityTree::SimilarityMatrix.new(documents, show_progress: true).build_tree(id)
+
+## Licence and Credits
+
+(c) 2012-2013, Kent Mewhort (similarity tree) and Open North (original similarity_matrix implementation, see https://github.com/jpmckinney/clip-analysis),
+licensed under MIT. See LICENSE.txt for details.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/similarity_tree.rb

@@ -0,0 +1,4 @@
+require "similarity_tree/version"
+require "similarity_tree/node"
+require "similarity_tree/similarity_matrix"
+require "similarity_tree/similarity_tree"

data/lib/similarity_tree/node.rb

@@ -0,0 +1,53 @@
+require 'json'
+module SimilarityTree
+  class Node
+    attr_accessor :id, :diff_score, :parent, :children, :content
+
+    def initialize(id, diff_score, parent = nil, children = [], content = nil)
+      @id, @diff_score, @parent, @children, @content = id, diff_score, parent, children, content
+    end
+
+    # self and all descendents
+    def each_node
+      depth_first_recurse{|n, depth| yield n}
+    end
+
+    def to_s
+      str = ""
+      depth_first_recurse do |n, depth|
+        str += ("-" * depth) + n.id.to_s
+        str += ' (' + n.diff_score.to_s + ')' unless n.diff_score.nil?
+        str += "\n"
+      end
+      str
+    end
+
+    def to_h
+      result = {
+          id: id
+      }
+      result[:children] = children.map {|c| c.to_h} unless children.nil? || children.empty?
+      result[:diff_score] = diff_score unless diff_score.nil?
+
+      # if the content node has an as_json function, merge-in these attributes
+      if content.respond_to?(:as_json) && content.is_a?(Hash)
+        result = content.as_json.merge(result)
+      end
+      result
+    end
+
+    def to_json(opts = {})
+      JSON.generate to_h, opts
+    end
+
+    private
+    # helper for recursion into descendents
+    def depth_first_recurse(node = nil, depth = 0, &block)
+      node = self if node == nil
+      yield node, depth
+      node.children.each do |child|
+        depth_first_recurse(child, depth+1, &block)
+      end
+    end
+  end
+end

data/lib/similarity_tree/similarity_matrix.rb

@@ -0,0 +1,147 @@
+require 'matrix'
+require 'tf-idf-similarity'
+require 'fast_html_diff'
+
+module SimilarityTree
+  # Table of the diff/similarity scores between different text documents
+  class SimilarityMatrix
+
+    # Initialize a matrix for a set of documents
+    def initialize(sources, options = {})
+      @sources = sources
+      @config = default_options.merge(options)
+
+      @id = -1
+      @source_index = Hash.new
+      @matrix = nil
+    end
+
+    # calculate and output results as an array of arrays;
+    # optional block is run each comparison to help with any progress bars
+    def calculate
+      if @config[:calculation_method] == :tf_idf
+        @matrix = calculate_with_tf_idf
+      elsif @config[:calculation_method] == :diff
+        @matrix = calculate_with_diff
+      else
+        raise "Unknown calculation type"
+      end
+    end
+
+    def build_tree(root_id, score_threshold = 0)
+      # build the similarity tree
+      @matrix = self.calculate if @matrix.nil?
+      tree = SimilarityTree.new(root_id, @matrix, score_threshold).build
+
+      # populate the nodes with the sources for the compatibility matrix
+      tree.each_node {|n| n.content = @source_index[n.id] }
+      tree
+    end
+
+    private
+    def default_options
+      {
+          id_func: nil,
+          content_func: nil,
+          calculation_method: :tf_idf,
+          show_progress: false
+      }
+    end
+
+    def calculate_with_tf_idf
+      progress_bar = nil
+      if @config[:show_progress]
+        progress_bar = ProgressBar.create format: '%a |%B| %p%% %e', length: 80, smoothing: 0.5,
+                                         total: @sources.length
+      end
+
+      # iterate through the input texts and build the tf_idf corpus
+      corpus = []
+      ids = @sources.map do |source|
+        corpus << TfIdfSimilarity::Document.new(text_of(source))
+        progress_bar.increment unless progress_bar.nil?
+        id_of(source)
+      end
+      model = TfIdfSimilarity::TfIdfModel.new(corpus, function: :tf_idf)
+      similarity_matrix = model.similarity_matrix
+
+      # compile the results into an ordinary m*n array
+      matrix = {}
+      ids.each_with_index do |a,i|
+        matrix[a] = {}
+        ids.each_with_index do |b,j|
+          matrix[a][b] = similarity_matrix[i, j].round(6)
+        end
+      end
+      matrix
+    end
+
+    # Create a similarity matrix, using diff as the similarity measure, based on the difference of WORDS (not characters)
+    # (only counts insertions and deletions, not substitution and transposition).
+    def calculate_with_diff
+      progress_bar = nil
+      if @config[:show_progress]
+        progress_bar = ProgressBar.create format: '%a |%B| %p%% %e', length: 80, smoothing: 0.5,
+                                         total: @sources.length*(@sources.length-1)/2
+      end
+
+      matrix = {}
+      @sources.each_with_index do |a,i|
+        a_id = id_of(a)
+        a_text = text_of(a)
+
+        @sources[i + 1..-1].each do |b|
+          b_id   = id_of(b)
+          b_text = text_of(b)
+
+          stats = FastHtmlDiff::DiffBuilder.new(a_text, b_text).statistics
+
+          # http://en.wikipedia.org/wiki/Dice%27s_coefficient
+          total_count = 2 * stats[:matches][:words] + stats[:insertions][:words] + stats[:deletions][:words]
+          similarity = 2 * stats[:matches][:words] / total_count.to_f
+
+          # Build the similarity matrix,
+          matrix[a_id] ||= {a_id => 1}
+          matrix[a_id][b_id] = similarity
+          matrix[b_id] ||= {b_id => 1}
+          matrix[b_id][a_id] = similarity
+
+          progress_bar.increment unless progress_bar.nil?
+        end
+      end
+      matrix
+    end
+
+    def id_of(source)
+      id = nil
+      if !@config[:id_func].nil?
+        id = source.send @config[:id_func].to_s
+      else
+        if is_a_filename? source
+          id = File.basename(source)
+        else
+          id = @sources.find_index(source)
+        end
+      end
+
+      # maintain an index of id => source
+      @source_index[id] = source if @source_index[id].nil?
+      id
+    end
+
+    def text_of(source)
+      if !@config[:content_func].nil?
+        txt = source.send @config[:content_func].to_s
+      else
+        txt = source
+      end
+      txt = File.read(txt) if is_a_filename?(txt)
+      txt
+    end
+
+    # quick and dirty check on whether a string is a filename based on the string length and whether the file exists
+    def is_a_filename?(filename)
+      (filename.length < 512) && File.exists?(filename)
+    end
+  end
+end

data/lib/similarity_tree/similarity_tree.rb

@@ -0,0 +1,81 @@
+module SimilarityTree
+
+  # Constructs a hierarchy of nodes based on a specified root and the similarity "scores" between nodes. Each nodes is placed next
+  # to the node to which it is most similar; as between two nodes, the node most similar to the root is placed closest to the root.
+  class SimilarityTree
+    # initialize/build the tree hierarchy from an existing similarity matrix
+    def initialize(root_id, similarity_matrix, score_threshold = 0)
+      @nodes = similarity_matrix.map {|key, row| Node.new(key, 0)}
+      @root = @nodes.find {|n| n.id == root_id}
+      @root.diff_score = nil
+      @similarity_matrix = similarity_matrix
+      @score_threshold = score_threshold
+    end
+
+    # build the tree and return the root node
+    def build
+      build_tree
+      @root
+    end
+
+    private
+    def build_tree
+      tree = @root
+      flat = [@root]
+
+      # for each non-root node
+      @nodes.delete_if{|n| n == @root}.map do |n|
+        # find the best match to the nodes already in the tree
+        closest_diff_score = 0
+        closest = nil
+        flat.each do |m|
+          diff_score = @similarity_matrix[n.id][m.id]
+          if closest.nil? || (diff_score > closest_diff_score)
+            closest_diff_score = diff_score
+            closest = m
+          end
+        end
+
+        # if the closest match is the root node, or if the closest match's diff score with it's parent is stronger
+        # than between the present node and that parent, add as a child of the match
+        if (closest == @root) || (closest.diff_score >= @similarity_matrix[n.id][closest.parent.id])
+          n.parent = closest
+          closest.children << n
+          n.diff_score = @similarity_matrix[n.id][closest.id]
+          # else, if the new node is more similar to the parent, rotate so that the existing node becomes the child
+        else
+          # place children with the closest matching of the two
+          closest.children.dup.each do |child|
+            if @similarity_matrix[child.id][n.id] > child.diff_score
+              child.parent = n
+              closest.children.delete_if{|child_i| child_i == child }
+              n.children << child
+              child.diff_score = @similarity_matrix[child.id][n.id]
+            end
+          end
+
+          # connect the new node to the parent
+          n.parent = closest.parent
+          n.parent.children << n
+          n.diff_score = @similarity_matrix[n.id][n.parent.id]
+
+          # add the existing node as a child
+          closest.parent = n
+          n.parent.children.delete_if{|child_i| child_i == closest}
+          n.children << closest
+          closest.diff_score = @similarity_matrix[closest.id][n.id]
+        end
+
+        flat << n
+      end
+      prune(flat)
+    end
+
+    # prune away nodes that don't meet the configured score threshold
+    def prune(nodes)
+      nodes.each do |node|
+        node.parent.children.reject!{|n| n == node} if (node != @root) && (node.diff_score < @score_threshold)
+      end
+    end
+  end
+end