daff 1.2.3 → 1.2.4

data/README.md

@@ -3,6 +3,7 @@
 [![Gem Version](https://badge.fury.io/rb/daff.svg)](http://badge.fury.io/rb/daff)
 [![PyPI version](https://badge.fury.io/py/daff.svg)](http://badge.fury.io/py/daff)
 [![PHP version](https://badge.fury.io/ph/paulfitz%2Fdaff-php.svg)](http://badge.fury.io/ph/paulfitz%2Fdaff-php)
+[![tips](https://img.shields.io/gratipay/paulfitz.svg)](https://gratipay.com/paulfitz/)
 
 daff: data diff
 ===============
@@ -47,7 +48,7 @@ daff can produce and apply tabular diffs.
 Call as:
   daff [--output OUTPUT.csv] a.csv b.csv
   daff [--output OUTPUT.csv] parent.csv a.csv b.csv
-  daff [--output OUTPUT.jsonbook] a.jsonbook b.jsonbook
+  daff [--output OUTPUT.ndjson] a.ndjson b.ndjson
   daff patch [--inplace] [--output OUTPUT.csv] a.csv patch.csv
   daff merge [--inplace] [--output OUTPUT.csv] parent.csv a.csv b.csv
   daff trim [--output OUTPUT.csv] source.csv
@@ -72,6 +73,8 @@ If you need more control, here is the full list of flags:
      --plain:       do not use fancy utf8 characters to make arrows prettier
 ````
 
+Formats supported are CSV, TSV, and [ndjson](http://dataprotocols.org/ndjson/).
+
 Using with git
 --------------
 

data/lib/daff.rb

@@ -55,17 +55,25 @@ require_relative 'lib/coopy/index_item'
 require_relative 'lib/coopy/index_pair'
 require_relative 'lib/coopy/merger'
 require_relative 'lib/coopy/mover'
+require_relative 'lib/coopy/ndjson'
+require_relative 'lib/coopy/nested_cell_builder'
 require_relative 'lib/coopy/ordering'
 require_relative 'lib/coopy/table'
 require_relative 'lib/coopy/simple_table'
 require_relative 'lib/coopy/view'
 require_relative 'lib/coopy/simple_view'
 require_relative 'lib/coopy/sparse_sheet'
+require_relative 'lib/coopy/sql_column'
+require_relative 'lib/coopy/sql_compare'
+require_relative 'lib/coopy/sql_database'
+require_relative 'lib/coopy/sql_helper'
+require_relative 'lib/coopy/sql_table'
+require_relative 'lib/coopy/sql_table_name'
+require_relative 'lib/coopy/sqlite_helper'
 require_relative 'lib/coopy/table_comparison_state'
 require_relative 'lib/coopy/table_diff'
 require_relative 'lib/coopy/table_io'
 require_relative 'lib/coopy/table_modifier'
-require_relative 'lib/coopy/table_text'
 require_relative 'lib/coopy/terminal_diff_render'
 require_relative 'lib/coopy/unit'
 require_relative 'lib/coopy/viterbi'
@@ -96,16 +104,21 @@ def _hx_ord(s) return 0 if s.nil?; s.ord end
 $hx_exception_classes = {}
 def hx_exception_class(c)
   $hx_exception_classes[c.name] ||= Class.new(RuntimeError) do
-    Object.const_set((c.name.split(/::/).old_access(-1)||'') + 'HaxeException',self)
-    def initialize(target) @target = target; end
-    def method_missing(name, *args, &block)
-      @target.send(name, *args, &block)
-    end
+    Object.const_set((c.name.split(/::/)[-1]||'') + 'HaxeException',self)
+    attr_accessor :hx_exception_target
+    def initialize(target) @hx_exception_target = target; end
   end
 end
-def hx_exception(x)
+def hx_raise(x)
   hx_exception_class(x.class).new(x)
 end
+def hx_rescue(x)
+  hx_exception_class(x.class)
+end
+def hx_rescued(x)
+  return x.hx_exception_target if x.respond_to? :hx_exception_target
+  x
+end
 
 
 Daff = Coopy

data/lib/lib/coopy/alignment.rb

@@ -96,6 +96,12 @@ module Coopy
       return @order_cache
     end
     
+    def add_to_order(l,r,p = -2)
+      @order_cache = ::Coopy::Ordering.new if @order_cache == nil
+      @order_cache.add(l,r,p)
+      @order_cache_has_reference = p != -2
+    end
+    
     def get_source 
       return @ta
     end

data/lib/lib/coopy/cell_info.rb

@@ -7,6 +7,7 @@ module Coopy
     def initialize
     end
     
+    attr_accessor :raw
     attr_accessor :value
     attr_accessor :pretty_value
     attr_accessor :category

data/lib/lib/coopy/compare_flags.rb

@@ -16,6 +16,7 @@ module Coopy
       @acts = nil
       @ids = nil
       @columns_to_ignore = nil
+      @allow_nested_cells = false
     end
     
     attr_accessor :ordered
@@ -29,6 +30,7 @@ module Coopy
     attr_accessor :acts
     attr_accessor :ids
     attr_accessor :columns_to_ignore
+    attr_accessor :allow_nested_cells
     
     def filter(act,allow)
       if @acts == nil 

data/lib/lib/coopy/compare_table.rb

@@ -179,7 +179,7 @@ module Coopy
             return -1 if a1[1] > b1[1]
             return 0
           }
-          columns_eval.sort{|a,b| sorter.call(a,b)}
+          columns_eval.sort!{|a,b| sorter.call(a,b)}
           columns = Lambda.array(Lambda.map(columns_eval,lambda {|v|
             return v[0]
           }))

data/lib/lib/coopy/coopy.rb

@@ -9,6 +9,9 @@ module Coopy
       @format_preference = nil
       @delim_preference = nil
       @output_format = "copy"
+      @nested_output = false
+      @order_set = false
+      @order_preference = false
     end
     
     # protected - in ruby this doesn't play well with static/inline methods
@@ -17,6 +20,9 @@ module Coopy
     attr_accessor :delim_preference
     attr_accessor :extern_preference
     attr_accessor :output_format
+    attr_accessor :nested_output
+    attr_accessor :order_set
+    attr_accessor :order_preference
     attr_accessor :io
     attr_accessor :mv
     
@@ -25,10 +31,12 @@ module Coopy
       ext = ""
       pt = name.rindex(".",nil || 0) || -1
       if pt >= 0 
-        ext = name[pt + 1..-1].to_lower_case
+        ext = HxOverrides.substr(name,pt + 1,nil).to_lower_case
         case(ext)
         when "json"
           @format_preference = "json"
+        when "ndjson"
+          @format_preference = "ndjson"
         when "csv"
           @format_preference = "csv"
           @delim_preference = ","
@@ -38,10 +46,16 @@ module Coopy
         when "ssv"
           @format_preference = "csv"
           @delim_preference = ";"
+        when "sqlite3"
+          @format_preference = "sqlite"
+        when "sqlite"
+          @format_preference = "sqlite"
         else
           ext = ""
         end
       end
+      @nested_output = @format_preference == "json" || @format_preference == "ndjson"
+      @order_preference = !@nested_output
       return ext
     end
     
@@ -55,12 +69,17 @@ module Coopy
       self.set_format(@output_format) if @output_format != "copy"
       txt = ""
       self.check_format(name)
-      if @format_preference != "json" 
+      if @format_preference == "csv" 
         csv = ::Coopy::Csv.new(@delim_preference)
         txt = csv.render_table(t)
+      elsif @format_preference == "ndjson" 
+        txt = ::Coopy::Ndjson.new(t).render
+      elsif @format_preference == "sqlite" 
+        @io.write_stderr("! Cannot yet output to sqlite, aborting\n")
+        return false
       else 
         value = ::Coopy::Coopy.jsonify(t)
-        txt = ::Haxe::Format::JsonPrinter._print(value,nil,nil)
+        txt = ::Haxe::Format::JsonPrinter._print(value,nil,"  ")
       end
       return self.save_text(name,txt)
     end
@@ -77,14 +96,40 @@ module Coopy
     def load_table(name)
       txt = @io.get_content(name)
       ext = self.check_format(name)
+      if ext == "sqlite" 
+        sql = @io.open_sqlite_database(name)
+        if sql == nil 
+          @io.write_stderr("! Cannot open database, aborting\n")
+          return nil
+        end
+        helper = ::Coopy::SqliteHelper.new
+        names = helper.get_table_names(sql)
+        if names == nil 
+          @io.write_stderr("! Cannot find database tables, aborting\n")
+          return nil
+        end
+        if names.length == 0 
+          @io.write_stderr("! No tables in database, aborting\n")
+          return nil
+        end
+        tab = ::Coopy::SqlTable.new(sql,::Coopy::SqlTableName.new(names[0]),helper)
+        return tab
+      end
+      if ext == "ndjson" 
+        t = ::Coopy::SimpleTable.new(0,0)
+        ndjson = ::Coopy::Ndjson.new(t)
+        ndjson.parse(txt)
+        return t
+      end
       begin
         json = ::Haxe::Format::JsonParser.new(txt).parse_rec
         @format_preference = "json"
-        t = ::Coopy::Coopy.json_to_table(json)
-        raise "JSON failed" if t == nil
-        return t
+        t1 = ::Coopy::Coopy.json_to_table(json)
+        raise hx_raise("JSON failed") if t1 == nil
+        return t1
       rescue => e
-        raise e if ext == "json"
+        e = hx_rescued(e)
+        raise hx_raise(e) if ext == "json"
       end if ext == "json" || ext == ""
       @format_preference = "csv"
       csv = ::Coopy::Csv.new(@delim_preference)
@@ -344,6 +389,19 @@ module Coopy
               git = true
               args.slice!(i,1)
               break
+            elsif tag == "--unordered" 
+              more = true
+              flags.ordered = false
+              flags.unchanged_context = 0
+              @order_set = true
+              args.slice!(i,1)
+              break
+            elsif tag == "--ordered" 
+              more = true
+              flags.ordered = true
+              @order_set = true
+              args.slice!(i,1)
+              break
             elsif tag == "--color" 
               more = true
               color = true
@@ -415,7 +473,7 @@ module Coopy
         io.write_stderr("Call as:\n")
         io.write_stderr("  daff [--color] [--output OUTPUT.csv] a.csv b.csv\n")
         io.write_stderr("  daff [--output OUTPUT.csv] parent.csv a.csv b.csv\n")
-        io.write_stderr("  daff [--output OUTPUT.jsonbook] a.jsonbook b.jsonbook\n")
+        io.write_stderr("  daff [--output OUTPUT.ndjson] a.ndjson b.ndjson\n")
         io.write_stderr("  daff patch [--inplace] [--output OUTPUT.csv] a.csv patch.csv\n")
         io.write_stderr("  daff merge [--inplace] [--output OUTPUT.csv] parent.csv a.csv b.csv\n")
         io.write_stderr("  daff trim [--output OUTPUT.csv] source.csv\n")
@@ -428,14 +486,16 @@ module Coopy
         io.write_stderr("\n")
         io.write_stderr("If you need more control, here is the full list of flags:\n")
         io.write_stderr("  daff diff [--output OUTPUT.csv] [--context NUM] [--all] [--act ACT] a.csv b.csv\n")
-        io.write_stderr("     --id:          specify column to use as primary key (repeat for multi-column key)\n")
-        io.write_stderr("     --ignore:      specify column to ignore completely (can repeat)\n")
+        io.write_stderr("     --act ACT:     show only a certain kind of change (update, insert, delete)\n")
+        io.write_stderr("     --all:         do not prune unchanged rows\n")
         io.write_stderr("     --color:       highlight changes with terminal colors\n")
         io.write_stderr("     --context NUM: show NUM rows of context\n")
-        io.write_stderr("     --all:         do not prune unchanged rows\n")
-        io.write_stderr("     --act ACT:     show only a certain kind of change (update, insert, delete)\n")
+        io.write_stderr("     --id:          specify column to use as primary key (repeat for multi-column key)\n")
+        io.write_stderr("     --ignore:      specify column to ignore completely (can repeat)\n")
         io.write_stderr("     --input-format [csv|tsv|ssv|json]: set format to expect for input\n")
+        io.write_stderr("     --ordered:     assume row order is meaningful (default for CSV)\n")
         io.write_stderr("     --output-format [csv|tsv|ssv|json|copy]: set format for output\n")
+        io.write_stderr("     --unordered:   assume row order is meaningless (default for json formats)\n")
         io.write_stderr("\n")
         io.write_stderr("  daff diff --git path old-file old-hex old-mode new-file new-hex new-mode\n")
         io.write_stderr("     --git:         process arguments provided by git to diff drivers\n")
@@ -501,6 +561,11 @@ module Coopy
       output = "-" if output == nil
       ok = true
       if cmd1 == "diff" 
+        if !@order_set 
+          flags.ordered = @order_preference
+          flags.unchanged_context = 0 if !flags.ordered
+        end
+        flags.allow_nested_cells = @nested_output
         ct1 = ::Coopy::Coopy.compare_tables3(parent,a,b,flags)
         align = ct1.align
         td = ::Coopy::TableDiff.new(align,flags)
@@ -545,7 +610,7 @@ module Coopy
     class << self
     attr_accessor :version
     end
-    @version = "1.2.3"
+    @version = "1.2.4"
     
     def Coopy.compare_tables(local,remote,flags = nil)
       comp = ::Coopy::TableComparisonState.new
@@ -578,6 +643,7 @@ module Coopy
       hp = ::Coopy::HighlightPatch.new(nil,nil)
       csv = ::Coopy::Csv.new
       tm = ::Coopy::TableModifier.new(nil)
+      sc = ::Coopy::SqlCompare.new(nil,nil,nil)
       return 0
     end
     
@@ -697,11 +763,7 @@ module Coopy
               x = _g1
               _g1+=1
               v = t.get_cell(x,y)
-              if v != nil 
-                row.push(v[:to_s].call)
-              else 
-                row.push(nil)
-              end
+              row.push(v)
             end
           end
           sheet.push(row)

data/lib/lib/coopy/csv.rb

@@ -188,7 +188,7 @@ module Coopy
         return nil if result == "NULL"
         if first_non_underscore > start 
           del = first_non_underscore - start
-          return result[1..-1] if result[del..-1] == "NULL"
+          return HxOverrides.substr(result,1,nil) if HxOverrides.substr(result,del,nil) == "NULL"
         end
       end
       return result

data/lib/lib/coopy/diff_render.rb

@@ -17,6 +17,7 @@ module Coopy
     attr_accessor :td_close
     attr_accessor :open
     attr_accessor :pretty_arrows
+    attr_accessor :section
     
     public
     
@@ -32,6 +33,22 @@ module Coopy
     
     def begin_table 
       self.insert("<table>\n")
+      @section = nil
+    end
+    
+    def set_section(str)
+      return if str == @section
+      if @section != nil 
+        self.insert("</t")
+        self.insert(@section)
+        self.insert(">\n")
+      end
+      @section = str
+      if @section != nil 
+        self.insert("<t")
+        self.insert(@section)
+        self.insert(">\n")
+      end
     end
     
     def begin_row(mode)
@@ -41,9 +58,8 @@ module Coopy
       if mode == "header" 
         @td_open = "<th"
         @td_close = "</th>"
-      else 
-        row_class = mode
       end
+      row_class = mode
       tr = "<tr>"
       tr = "<tr class=\"" + _hx_str(row_class) + "\">" if row_class != ""
       self.insert(tr)
@@ -62,6 +78,7 @@ module Coopy
     end
     
     def end_table 
+      self.set_section(nil)
       self.insert("</table>\n")
     end
     
@@ -80,9 +97,9 @@ module Coopy
       render = self
       render.begin_table
       change_row = -1
-      tt = ::Coopy::TableText.new(tab)
       cell = ::Coopy::CellInfo.new
-      corner = tt.get_cell_text(0,0)
+      view = tab.get_cell_view
+      corner = view.to_s(tab.get_cell(0,0))
       off = nil
       if corner == "@:@" 
         off = 1
@@ -99,11 +116,16 @@ module Coopy
           row = _g1
           _g1+=1
           open = false
-          txt = tt.get_cell_text(off,row)
+          txt = view.to_s(tab.get_cell(off,row))
           txt = "" if txt == nil
-          ::Coopy::DiffRender.examine_cell(0,row,txt,"",txt,corner,cell)
+          ::Coopy::DiffRender.examine_cell(off,row,view,txt,"",txt,corner,cell,off)
           row_mode = cell.category
           change_row = row if row_mode == "spec"
+          if row_mode == "header" || row_mode == "spec" || row_mode == "index" 
+            self.set_section("head")
+          else 
+            self.set_section("body")
+          end
           render.begin_row(row_mode)
           begin
             _g3 = 0
@@ -111,7 +133,7 @@ module Coopy
             while(_g3 < _g2) 
               c = _g3
               _g3+=1
-              ::Coopy::DiffRender.examine_cell(c,row,tt.get_cell_text(c,row),((change_row >= 0) ? tt.get_cell_text(c,change_row) : ""),txt,corner,cell)
+              ::Coopy::DiffRender.examine_cell(c,row,view,tab.get_cell(c,row),((change_row >= 0) ? view.to_s(tab.get_cell(c,change_row)) : ""),txt,corner,cell,off)
               render.insert_cell(((@pretty_arrows) ? cell.pretty_value : cell.value),cell.category_given_tr)
             end
           end
@@ -123,17 +145,20 @@ module Coopy
     end
     
     def sample_css 
-      return ".highlighter .add { \n  background-color: #7fff7f;\n}\n\n.highlighter .remove { \n  background-color: #ff7f7f;\n}\n\n.highlighter td.modify { \n  background-color: #7f7fff;\n}\n\n.highlighter td.conflict { \n  background-color: #f00;\n}\n\n.highlighter .spec { \n  background-color: #aaa;\n}\n\n.highlighter .move { \n  background-color: #ffa;\n}\n\n.highlighter .null { \n  color: #888;\n}\n\n.highlighter table { \n  border-collapse:collapse;\n}\n\n.highlighter td, .highlighter th {\n  border: 1px solid #2D4068;\n  padding: 3px 7px 2px;\n}\n\n.highlighter th, .highlighter .header { \n  background-color: #aaf;\n  font-weight: bold;\n  padding-bottom: 4px;\n  padding-top: 5px;\n  text-align:left;\n}\n\n.highlighter tr:first-child td {\n  border-top: 1px solid #2D4068;\n}\n\n.highlighter td:first-child { \n  border-left: 1px solid #2D4068;\n}\n\n.highlighter td {\n  empty-cells: show;\n}\n"
+      return ".highlighter .add { \n  background-color: #7fff7f;\n}\n\n.highlighter .remove { \n  background-color: #ff7f7f;\n}\n\n.highlighter td.modify { \n  background-color: #7f7fff;\n}\n\n.highlighter td.conflict { \n  background-color: #f00;\n}\n\n.highlighter .spec { \n  background-color: #aaa;\n}\n\n.highlighter .move { \n  background-color: #ffa;\n}\n\n.highlighter .null { \n  color: #888;\n}\n\n.highlighter table { \n  border-collapse:collapse;\n}\n\n.highlighter td, .highlighter th {\n  border: 1px solid #2D4068;\n  padding: 3px 7px 2px;\n}\n\n.highlighter th, .highlighter .header { \n  background-color: #aaf;\n  font-weight: bold;\n  padding-bottom: 4px;\n  padding-top: 5px;\n  text-align:left;\n}\n\n.highlighter tr.header th {\n  border-bottom: 2px solid black;\n}\n\n.highlighter tr.index td, .highlighter .index, .highlighter tr.header th.index {\n  background-color: white;\n  border: none;\n}\n\n.highlighter .gap {\n  color: #888;\n}\n\n.highlighter td {\n  empty-cells: show;\n}\n"
     end
     
     def complete_html 
-      @text_to_insert.insert(0,"<html>\n<meta charset='utf-8'>\n<head>\n<style TYPE='text/css'>\n")
+      @text_to_insert.insert(0,"<!DOCTYPE html>\n<html>\n<head>\n<meta charset='utf-8'>\n<style TYPE='text/css'>\n")
       @text_to_insert.insert(1,self.sample_css)
       @text_to_insert.insert(2,"</style>\n</head>\n<body>\n<div class='highlighter'>\n")
       @text_to_insert.push("</div>\n</body>\n</html>\n")
     end
     
-    def DiffRender.examine_cell(x,y,value,vcol,vrow,vcorner,cell)
+    def DiffRender.examine_cell(x,y,view,raw,vcol,vrow,vcorner,cell,offset = 0)
+      nested = view.is_hash(raw)
+      value = nil
+      value = view.to_s(raw) if !nested
       cell.category = ""
       cell.category_given_tr = ""
       cell.separator = ""
@@ -148,6 +173,7 @@ module Coopy
       vcol = "" if vcol == nil
       removed_column = false
       cell.category = "move" if vrow == ":"
+      cell.category = "index" if vrow == "" && offset == 1 && y == 0
       if (vcol.index("+++",nil || 0) || -1) >= 0 
         cell.category_given_tr = cell.category = "add"
       elsif (vcol.index("---",nil || 0) || -1) >= 0 
@@ -158,6 +184,8 @@ module Coopy
         cell.category = "spec"
       elsif vrow == "@@" 
         cell.category = "header"
+      elsif vrow == "..." 
+        cell.category = "gap"
       elsif vrow == "+++" 
         cell.category = "add" if !removed_column
       elsif vrow == "---" 
@@ -168,20 +196,30 @@ module Coopy
           full = vrow
           part = tokens[1]
           part = full if part == nil
-          if (cell.value.index(part,nil || 0) || -1) >= 0 
+          if nested || (cell.value.index(part,nil || 0) || -1) >= 0 
             cat = "modify"
             div = part
             if part != full 
-              if (cell.value.index(full,nil || 0) || -1) >= 0 
+              if nested 
+                cell.conflicted = view.hash_exists(raw,"theirs")
+              else 
+                cell.conflicted = (cell.value.index(full,nil || 0) || -1) >= 0
+              end
+              if cell.conflicted 
                 div = full
                 cat = "conflict"
-                cell.conflicted = true
               end
             end
             cell.updated = true
             cell.separator = div
             cell.pretty_separator = div
-            if cell.pretty_value == div 
+            if nested 
+              if cell.conflicted 
+                tokens = [view.hash_get(raw,"before"),view.hash_get(raw,"ours"),view.hash_get(raw,"theirs")]
+              else 
+                tokens = [view.hash_get(raw,"before"),view.hash_get(raw,"after")]
+              end
+            elsif cell.pretty_value == div 
               tokens = ["",""]
             else 
               tokens = cell.pretty_value.split(div)
@@ -199,18 +237,19 @@ module Coopy
             cell.pretty_separator = [8594].pack("U")
             cell.pretty_value = pretty_tokens.join(cell.pretty_separator)
             cell.category_given_tr = cell.category = cat
-            offset = nil
+            offset1 = nil
             if cell.conflicted 
-              offset = 1
+              offset1 = 1
             else 
-              offset = 0
+              offset1 = 0
             end
-            cell.lvalue = tokens[offset]
-            cell.rvalue = tokens[offset + 1]
+            cell.lvalue = tokens[offset1]
+            cell.rvalue = tokens[offset1 + 1]
             cell.pvalue = tokens[0] if cell.conflicted
           end
         end
       end
+      cell.category_given_tr = cell.category = "index" if x == 0 && offset > 0
     end
     
     # protected - in ruby this doesn't play well with static/inline methods
@@ -244,16 +283,16 @@ module Coopy
     
     public
     
-    def DiffRender.render_cell(tt,x,y)
+    def DiffRender.render_cell(tab,view,x,y)
       cell = ::Coopy::CellInfo.new
-      corner = tt.get_cell_text(0,0)
+      corner = view.to_s(tab.get_cell(0,0))
       off = nil
       if corner == "@:@" 
         off = 1
       else 
         off = 0
       end
-      ::Coopy::DiffRender.examine_cell(x,y,tt.get_cell_text(x,y),tt.get_cell_text(x,off),tt.get_cell_text(off,y),corner,cell)
+      ::Coopy::DiffRender.examine_cell(x,y,view,tab.get_cell(x,y),view.to_s(tab.get_cell(x,off)),view.to_s(tab.get_cell(off,y)),corner,cell,off)
       return cell
     end