lorax 0.1.0

data/CHANGELOG.rdoc

@@ -0,0 +1,6 @@
+= Changelog
+
+== 0.1.0 (2010-03-09)
+
+* Happy Birthday!
+* Diffs and generates patches, and for trivial cases applies patches correctly.

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Mike Dalessio
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Manifest.txt

@@ -0,0 +1,37 @@
+CHANGELOG.rdoc
+LICENSE
+Manifest.txt
+README.rdoc
+Rakefile
+TODO
+bin/lorax
+lib/lorax.rb
+lib/lorax/delta.rb
+lib/lorax/delta/delete_delta.rb
+lib/lorax/delta/insert_delta.rb
+lib/lorax/delta/modify_delta.rb
+lib/lorax/delta_set.rb
+lib/lorax/delta_set_generator.rb
+lib/lorax/fast_matcher.rb
+lib/lorax/match.rb
+lib/lorax/match_set.rb
+lib/lorax/signature.rb
+spec/fast_matcher_spec.rb
+spec/files/Michael-Dalessio-200909.html
+spec/files/Michael-Dalessio-201001.html
+spec/files/slashdot-1.html
+spec/files/slashdot-2.html
+spec/files/slashdot-3.html
+spec/files/slashdot-4.html
+spec/integration/lorax_spec.rb
+spec/match_spec.rb
+spec/spec.opts
+spec/spec_helper.rb
+spec/unit/delta/delete_delta_spec.rb
+spec/unit/delta/insert_delta_spec.rb
+spec/unit/delta/modify_delta_spec.rb
+spec/unit/delta_set_generator_spec.rb
+spec/unit/delta_set_spec.rb
+spec/unit/lorax_spec.rb
+spec/unit/match_set_spec.rb
+spec/unit/signature_spec.rb

data/README.rdoc

@@ -0,0 +1,70 @@
+= Lorax
+
+* http://github.com/flavorjones/lorax
+
+== Description
+
+The Lorax is a full diff and patch library for XML/HTML documents, based on Nokogiri.
+
+It can tell you whether two XML/HTML documents are identical, or if
+they're not, tell you what's different. In trivial cases, it can even
+apply the patch.
+
+It's based loosely on Gregory Cobena's master's thesis paper, which
+generates deltas in less than O(n * log n) time, accepting some
+tradeoffs in the size of the delta set. You can find his paper at
+http://gregory.cobena.free.fr/www/Publications/thesis.html.
+
+== Features / Problems
+
+* Detect differences between documents, or tell whether two documents are the same.
+* Generate patches for the differences between documents.
+* Apply patches for trivial cases.
+* More work needs to be done to make sure patches apply cleanly.
+
+== Synopsis
+
+Imagine you have two Nokogiri::XML::Documents. You can tell if they're identical:
+
+    Lorax::Signature.new(doc1.root).signature == Lorax::Signature.new(doc2.root).signature
+
+You can generate a delta set (currently opaque (sorry kids)):
+
+    delta_set = Lorax.diff(doc1, doc2)
+
+and apply the delta set as a patch to the original document:
+
+    new_doc   = delta_set.apply(doc1)
+
+== Requirements
+
+* Nokogiri 1.4.0
+
+== Install
+
+* sudo gem install lorax
+
+== License
+
+(The MIT License)
+
+Copyright (c) 2010 Mike Dalessio
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,50 @@
+# -*- ruby -*-
+
+require 'rubygems'
+gem 'hoe', '>= 2.5.0'
+require 'hoe'
+
+Hoe.plugin :git
+
+Hoe.spec 'lorax' do
+  developer "Mike Dalessio", "mike.dalessio@gmail.com"
+
+  self.extra_rdoc_files = FileList["*.rdoc"]
+  self.history_file     = "CHANGELOG.rdoc"
+  self.readme_file      = "README.rdoc"
+
+  extra_deps << ["nokogiri", ">= 1.4.0"]
+  extra_dev_deps << ["rspec", ">= 1.2.9"]
+  extra_dev_deps << ["rr", ">= 0.10.4"]
+end
+
+task :redocs => :fix_css
+task :docs => :fix_css
+task :fix_css do
+  better_css = <<-EOT
+    .method-description pre {
+      margin                    : 1em 0 ;
+    }
+
+    .method-description ul {
+      padding                   : .5em 0 .5em 2em ;
+    }
+
+    .method-description p {
+      margin-top                : .5em ;
+    }
+
+    #main ul, div#documentation ul {
+      list-style-type           : disc ! IMPORTANT ;
+      list-style-position       : inside ! IMPORTANT ;
+    }
+
+    h2 + ul {
+      margin-top                : 1em;
+    }
+  EOT
+  puts "* fixing css"
+  File.open("doc/rdoc.css", "a") { |f| f.write better_css }
+end
+
+# vim: syntax=ruby

data/TODO

@@ -0,0 +1,40 @@
+# -*-org-*-
+Diffaroo TODO
+
+* gem
+*** gemspec
+*** license
+*** gemcutter
+* docs
+*** rdocs
+*** readme
+*** class description notes
+    - Signature: calculate and persist signatures and weights for nodes in a single document
+    - Match:     represents a match between two nodes
+    - MatchSet:  composed of Signatures and Matches.
+    - Matcher:   an algorithm that operates on a MatchSet statelessly to generate matches.
+    - Generator: generates a DeltaSet from a MatchSet
+    - Delta:     an atomic change step
+    - DeltaSet:  an ordered set of Deltas
+    - Apply:     f(doc1, DeltaSet) -> doc2
+*** algorithmic notes
+***** ignoring ID
+      - too many web sites fuck that up
+      - libxml2 allows duplicate ids
+      - algorithm would ignore changed content
+***** indexes (ascendant lookahead) needs to be implemented?
+***** if we do "phase 3" in weight-order, and recursively match parents, can't we avoid the "propagate to parent" step of phase 4?
+* core
+*** write integration test for MODIFY delta
+*** write integration test for DELETE delta
+*** write integration test for MODIFY delta with move
+*** change API to specify HTML or XML. or should we make user pass in Nokogirified docs?
+*** pick a hashing algorithm
+    - ruby hash / md5 / sha1
+    - benchmark? collision rate?
+* additional
+*** build an rspec matcher for xml
+*** build a test/unit assertion for xml
+*** try to make the code independent of the tree we're diffing
+    think about diffing any tree, e.g. AST, YAML
+*** benchmark suite so we can try different algorithms

data/bin/lorax

@@ -0,0 +1,15 @@
+#! /usr/bin/env ruby
+
+require "lorax"
+
+delta_set = Lorax.diff(File.read(ARGV[0]), File.read(ARGV[1]))
+summary = delta_set.deltas.map do |d|
+  d.descriptor
+end
+
+File.open("foo.yml",  "w+") {  |f| f.puts summary.to_yaml }
+File.open("foo.html", "w+") do |f|
+  doc = Nokogiri::HTML File.read(ARGV[0])
+  delta_set.apply!(doc)
+  f.puts doc.to_html
+end

data/lib/lorax.rb

@@ -0,0 +1,35 @@
+require 'nokogiri'
+
+module Lorax
+  VERSION = "0.1.0"
+  REQUIRED_NOKOGIRI_VERSION = "1.4.0"
+  raise LoadError, "lorax requires Nokogiri version #{REQUIRED_NOKOGIRI_VERSION} or higher" unless Nokogiri::VERSION >= REQUIRED_NOKOGIRI_VERSION
+end
+
+require "lorax/signature"
+require "lorax/match"
+require "lorax/match_set"
+require "lorax/fast_matcher"
+
+require "lorax/delta"
+require "lorax/delta_set_generator"
+require "lorax/delta_set"
+
+module Lorax
+  def Lorax.diff(string_or_io_or_nokogiridoc_1, string_or_io_or_nokogiridoc_2)
+    doc1      = documentize string_or_io_or_nokogiridoc_1
+    doc2      = documentize string_or_io_or_nokogiridoc_2
+
+    Lorax::FastMatcher.new(doc1, doc2).match.to_delta_set
+  end
+
+  private
+
+  def Lorax.documentize(string_or_io_or_nokogiridoc)
+    if string_or_io_or_nokogiridoc.is_a?(Nokogiri::XML::Document)
+      string_or_io_or_nokogiridoc
+    else
+      Nokogiri string_or_io_or_nokogiridoc
+    end
+  end
+end

data/lib/lorax/delta.rb

@@ -0,0 +1,28 @@
+module Lorax
+  class Delta
+    class NodeNotFoundError < RuntimeError ; end
+
+    def apply!(document)
+      raise NotImplementedError, self.class.to_s
+    end
+
+    def inspect
+      "#<#{self.class.name}:#{sprintf("0x%x", object_id)} #{descriptor.inspect}>"
+    end
+
+    private
+
+    def insert_node(node, parent, position)
+      children = parent.children
+      if children.empty? || position >= children.length
+        parent << node.dup
+      else
+        children[position].add_previous_sibling(node.dup)
+      end
+    end
+  end
+end
+
+require "lorax/delta/insert_delta"
+require "lorax/delta/modify_delta"
+require "lorax/delta/delete_delta"

data/lib/lorax/delta/delete_delta.rb

@@ -0,0 +1,19 @@
+module Lorax
+  class DeleteDelta < Delta
+    attr_accessor :node
+
+    def initialize(node)
+      @node = node
+    end
+
+    def apply!(document)
+      target = document.at_xpath(node.path)
+      raise NodeNotFoundError, xpath unless target
+      target.unlink
+    end
+
+    def descriptor
+      [:delete, {:xpath => node.path, :content => node.to_s}]
+    end
+  end
+end

data/lib/lorax/delta/insert_delta.rb

@@ -0,0 +1,22 @@
+module Lorax
+  class InsertDelta < Delta
+    attr_accessor :node, :xpath, :position
+
+    def initialize(node, xpath, position)
+      @node     = node
+      @xpath    = xpath
+      @position = position
+    end
+
+    def apply!(document)
+      # TODO: patch nokogiri to make inserting node copies efficient
+      parent = document.at_xpath(xpath)
+      raise NodeNotFoundError, xpath unless parent
+      insert_node(node.dup, parent, position)
+    end
+
+    def descriptor
+      [:insert, {:xpath => xpath, :position => position, :content => node.to_s}]
+    end
+  end
+end

data/lib/lorax/delta/modify_delta.rb

@@ -0,0 +1,51 @@
+module Lorax
+  class ModifyDelta < Delta
+    attr_accessor :node1, :node2
+
+    def initialize(node1, node2)
+      @node1 = node1
+      @node2 = node2
+    end
+
+    def apply!(doc)
+      node = doc.at_xpath(node1.path)
+      raise NodeNotFoundError, node1.path unless node
+
+      if node.text? || node.type == Nokogiri::XML::Node::CDATA_SECTION_NODE
+        node.content = node2.content
+      else
+        attributes = attributes_hash(node)
+        attributes2 = attributes_hash(node2)
+        if attributes != attributes2
+          attributes .each { |name, value| node.remove_attribute(name) }
+          attributes2.each { |name, value| node[name] = value }
+        end
+      end
+
+      if node1.path != node2.path
+        position = node2.parent.children.index(node2)
+        target_parent = doc.at_xpath(node2.parent.path)
+        raise NodeNotFoundError, node2.parent.path unless target_parent
+        node.unlink
+        insert_node(node, target_parent, position)
+      end
+    end
+
+    def descriptor
+      if node1.text? || node1.type == Nokogiri::XML::Node::CDATA_SECTION_NODE
+        [:modify, {:old => {:xpath => node1.path, :content => node1.to_s},
+                   :new => {:xpath => node2.path, :content => node2.to_s}}]
+      else
+        [:modify, {:old => {:xpath => node1.path, :name => node1.name, :attributes => node1.attributes.map{|k,v| [k, v.value]}},
+                   :new => {:xpath => node2.path, :name => node2.name, :attributes => node2.attributes.map{|k,v| [k, v.value]}}}]
+      end
+    end
+
+    private
+
+    def attributes_hash(node)
+      # lame.
+      node.attributes.inject({}) { |hash, attr| hash[attr.first] = attr.last.value ; hash }
+    end
+  end
+end

data/lib/lorax/delta_set.rb

@@ -0,0 +1,24 @@
+module Lorax
+  class DeltaSet
+    attr_accessor :deltas
+
+    def initialize
+      @deltas = []
+    end
+
+    def add(delta)
+      @deltas << delta
+    end
+
+    def apply(document)
+      apply! document.dup
+    end
+
+    def apply!(document)
+      deltas.each do |delta|
+        delta.apply! document
+      end
+      document
+    end
+  end
+end

data/lib/lorax/delta_set_generator.rb

@@ -0,0 +1,36 @@
+module Lorax
+  module DeltaSetGenerator
+    def self.generate_delta_set match_set
+      delta_set = DeltaSet.new
+      generate_inserts_and_moves_recursively delta_set, match_set, match_set.signature2.root
+      generate_deletes_recursively delta_set, match_set, match_set.signature1.root
+      delta_set
+    end
+
+    private
+
+    def self.generate_inserts_and_moves_recursively delta_set, match_set, node
+      match = match_set.match node
+      if match
+        if ! match.perfect?
+          if match_set.signature1.monogram(match.pair.first) != match_set.signature2.monogram(match.pair.last)
+            delta_set.add ModifyDelta.new(match.pair.first, match.pair.last)
+          end
+          node.children.each { |child| generate_inserts_and_moves_recursively delta_set, match_set, child }
+        end
+      else
+        delta_set.add InsertDelta.new(node, node.parent.path, node.parent.children.index(node)) # TODO: demeter violation
+      end
+    end
+
+    def self.generate_deletes_recursively delta_set, match_set, node
+      match = match_set.match(node)
+      if match
+        return if match.perfect?
+        node.children.each { |child| generate_deletes_recursively delta_set, match_set, child }
+      else
+        delta_set.add DeleteDelta.new(node)
+      end
+    end
+  end
+end