ipynbdiff 0.3.2 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e457c9e3908f4ebdf452ec01428e71ca866730a6e36d4af344cb3e703fe989cc
-  data.tar.gz: 4d12d8975526ad09c6f890af1283c7586773fbd8b2a43abda7d65772ac472818
+  metadata.gz: 10f77cf53513157724c6e1d8c8abc4659e3254dd77c3c14ebb2aaabd3155639c
+  data.tar.gz: 10815dc6a9cb76c73b4d6c9f712bdb28be9217a2b40efbcf076342e9f5d34811
 SHA512:
-  metadata.gz: 3da474c1bad0797b392f51cb33692216d2326822b8654a507849c05d52d4631799c88ccbcd92bc8df1b01f72352b8031e782384f605c9d0abf07dcce9dd4a176
-  data.tar.gz: ba36b69389d265d8ebd327a66d081688f242614938f51b0c3d644a07502aef1854ebb4804b20572ab9d81c64a7a45c4844b1bba6889ace7988ac85e95130a72d
+  metadata.gz: 06e895b990e7099d094b6e78c8a54fc6e5c618b28a9ae266cdb47a127189d67df7a7468f5a96fb2da21529f30911dfca761d7e4b0fc5452fc351e1878c697811
+  data.tar.gz: 54ee8fb62e05b130b304989787ee1032ffe41a2dc68dcdfde5bf7480588bdcd3a4f9a20c3b63eeb7cf21dd0ed89527b34d39a7ec15170f7cddda6114ea8fac0a

data/README.md

@@ -1,3 +1,58 @@
-# rb-ipynbdiff: Better Jupyter Notebook diffs, in Ruby
+# IpynbDiff: Better diff for Jupyter Notebooks
 
-This is a port of [ipynbdiff](https://gitlab.com/gitlab-org/incubation-engineering/mlops/ipynbdiff)
+This is a simple diff tool that cleans up jupyter notebooks, transforming each [notebook](example/1/from.ipynb)
+into a [readable markdown file](example/1/from_html.md), keeping the output of cells, and running the
+diff after. Markdowns are generated using an opinionated Jupyter to Markdown conversion. This means
+that the entire file is readable on the diff.
+
+The result are diffs that are much easier to read:
+
+| Diff | IpynbDiff - HTML | IpynbDiff - Percent |  
+| ------ | ------ | ------ |
+| [Here](example/diff.txt) | [Here](example/ipynbdiff.txt) | [Here](example/ipynbdiff_percent.txt) |
+| ![](example/img/diff.png) | ![](example/img/ipynbdiff_html.png) | ![](example/img/ipynbdiff_percent.png) | 
+
+
+This started as a port of This is a port of [ipynbdiff](https://gitlab.com/gitlab-org/incubation-engineering/mlops/ipynbdiff), 
+but now has extended functionality although not working as git driver.
+
+## Usage
+
+### Generating diffs
+
+```ruby
+IpynbDiff.diff(from_path, to_path, options)
+```
+
+Options:
+
+```ruby
+@default_transform_options = {
+  preprocess_input: TRUE, # Whether the input should be transformed
+  write_output_to: nil, # Pass a path to save the output to a file
+  format: :text, # These are the formats Diffy accepts https://github.com/samg/diffy
+  sources_are_files: FALSE, # Weather to use the from/to as string or path to a file 
+  raise_if_invalid_notebook: FALSE, # Raises an error if the notebooks are invalid, otherwise returns nil 
+  transform_options: @default_transform_options, # See below for transform options
+  diff_opts: {
+    include_diff_info: FALSE # These are passed to Diffy https://github.com/samg/diffy
+  }
+} 
+```
+
+### Transforming the notebooks
+
+It might be necessary to have the transformed files in addition to the diff.
+
+```ruby
+IpynbDiff.transform(notebook, options)
+```
+
+Options:
+
+```ruby
+@default_transform_options = {
+    include_metadata: FALSE, # Whether to include or not the notebook metadata (kernel, language, etc)
+    cell_decorator: :html # :html is useful to add styling with css, :percent is better for text format
+}
+```

data/ipynbdiff.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |s|
   s.name        = 'ipynbdiff'
-  s.version     = '0.3.2'
+  s.version     = '0.3.6'
   s.summary     = 'Human Readable diffs for Jupyter Notebooks'
   s.description = 'Better diff for Jupyter Notebooks by first preprocessing them and removing clutter'
   s.authors     = ['Eduardo Bonet']

data/lib/ipynbdiff.rb

@@ -2,42 +2,57 @@
 
 # Human Readable Jupyter Diffs
 module IpynbDiff
-  require 'transformer.rb'
+  require 'transformer'
   require 'diffy'
 
-  @default_options = {
+  @default_transform_options = {
+    include_metadata: FALSE,
+    cell_decorator: :html
+  }
+
+  @default_diff_options = {
     preprocess_input: TRUE,
     write_output_to: nil,
     format: :text,
     sources_are_files: FALSE,
-    include_metadata: TRUE,
+    raise_if_invalid_notebook: FALSE,
+    transform_options: @default_transform_options,
     diff_opts: {
       include_diff_info: FALSE
     }
   }.freeze
 
   def self.prepare_input(to_prepare, options)
-    prepared = to_prepare
-    prepared = File.read(prepared) if options[:sources_are_files]
-    prepared = Transformer.transform(prepared, include_metadata: options[:include_metadata]) if options[:preprocess_input]
+    return '' unless to_prepare
 
-    prepared
+    prep = to_prepare
+    prep = File.read(prep) if options[:sources_are_files]
+    prep = transform(prep, raise_errors: TRUE, options: options[:transform_options]) if options[:preprocess_input]
+    prep
   end
 
   def self.diff(
     from_notebook,
     to_notebook,
-    options = {}
+    options = @default_diff_options
   )
-    options = @default_options.merge(options)
+    options = @default_diff_options.merge(options)
 
-    from = from_notebook && prepare_input(from_notebook, options) || ''
-    to = to_notebook && prepare_input(to_notebook, options) || ''
+    from = prepare_input(from_notebook, options)
+    to = prepare_input(to_notebook, options)
 
     d = Diffy::Diff.new(from, to, **options[:diff_opts]).to_s(options[:format])
-
     File.write(options[:write_output_to], d) if options[:write_output_to]
-
     d
+  rescue InvalidNotebookError
+    raise if options[:raise_if_invalid_notebook]
+  end
+
+  def self.transform(notebook, raise_errors: FALSE, options: @default_transform_options)
+    options = @default_transform_options.merge(options)
+
+    Transformer.new(**options).transform(notebook)
+  rescue InvalidNotebookError
+    raise if raise_errors
   end
 end

data/lib/output_transformer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module IpynbDiff
+
+  # Transforms Jupyter output data into markdown
+  class OutputTransformer
+
+    ORDERED_KEYS = {
+      'execute_result' => %w[image/png image/svg+xml image/jpeg text/markdown text/latex text/plain],
+      'display_data' => %w[image/png image/svg+xml image/jpeg text/markdown text/latex]
+    }.freeze
+
+    def transform(output)
+      case (output_type = output['output_type'])
+      when 'error'
+        transform_error(output['traceback'])
+      when 'execute_result', 'display_data'
+        transform_non_error(ORDERED_KEYS[output_type], output['data'])
+      end
+    end
+
+    def transform_error(traceback)
+      traceback.map do |t|
+        t.split("\n").map do |line|
+          line.gsub(/\[[0-9][0-9;]*m/, '').sub("\u001B", '    ').gsub(/\u001B/, '').rstrip << "\n"
+        end
+      end
+    end
+
+    def transform_non_error(accepted_keys, elements)
+      accepted_keys.map do |key|
+        transform_element(key, elements[key]) if elements.key?(key)
+      end.flatten
+    end
+
+    def transform_element(output_type, output_element)
+      case output_type
+      when 'image/png', 'image/jpeg'
+        transform_image(output_type, output_element)
+      when 'image/svg+xml'
+        transform_svg(output_element)
+      when 'text/markdown', 'text/latex', 'text/plain'
+        transform_text(output_element)
+      end
+    end
+
+    def transform_image(image_type, image_content)
+      ["    ![](data:#{image_type};base64,#{image_content.gsub("\n", '')})", "\n"]
+    end
+
+    def transform_svg(image_content)
+      single_line = image_content.map(&:strip).join('').gsub(/\s+/, ' ')
+
+      ["    ![](data:image/svg+xml;utf8,#{single_line})", "\n"]
+    end
+
+    def transform_text(text_content)
+      text_content.map { |line| "    #{line}" }.append("\n")
+    end
+  end
+end

data/lib/transformer.rb

@@ -1,93 +1,92 @@
 # frozen_string_literal: true
 
 module IpynbDiff
+  class InvalidNotebookError < StandardError
+  end
+
   # Returns a markdown version of the Jupyter Notebook
   class Transformer
     require 'json'
     require 'yaml'
+    require 'output_transformer'
 
-    def self.transform(notebook, include_metadata: TRUE)
-      notebook_json = JSON.parse(notebook)
-      transformed_blocks = notebook_json['cells'].map { |cell| transform_cell(cell, notebook_json) }
-      transformed_blocks.prepend(transform_metadata(notebook_json)) if include_metadata
-      transformed_blocks.join("\n")
+    @cell_decorator = :html
+    @include_metadata = TRUE
+
+
+    def initialize(include_metadata: TRUE, cell_decorator: :html)
+      @include_metadata = include_metadata
+      @cell_decorator = cell_decorator
+      @output_transformer = OutputTransformer.new
     end
 
-    def self.transform_cell(cell, notebook)
-      cell['cell_type'] == 'code' ? transform_code_cell(cell, notebook) : transform_text_cell(cell)
+    def validate_notebook(notebook)
+      notebook_json = JSON.parse(notebook)
+
+      return notebook_json if notebook_json.key?('cells') && notebook_json.key?('metadata')
+
+      raise InvalidNotebookError
+    rescue JSON::ParserError
+      raise InvalidNotebookError
     end
 
-    def self.transform_code_cell(cell, notebook)
-      tags = cell['metadata'].fetch('tags', []).join(' ')
+    def transform(notebook)
+      notebook_json = validate_notebook(notebook)
+      transformed_blocks = notebook_json['cells'].map do |cell|
+        decorate_cell(transform_cell(cell, notebook_json), cell)
+      end
 
-      [
-        %(<div class="cell code" data-id="#{cell['id']}" data-tags="#{tags}">\n\n),
-        %(``` #{notebook['metadata']['kernelspec']['language']}\n),
-        *cell['source'],
-        "\n```\n",
-        *cell['outputs'].map { |output| transform_output(output) },
-        "\n</div>\n"
-      ].join('')
+      transformed_blocks.prepend(transform_metadata(notebook_json)) if @include_metadata
+      transformed_blocks.join("\n")
     end
 
-    def self.format_traceback(traceback)
-      traceback.map do |t|
-        t.split("\n").map do |line|
-          line.gsub(/\[[0-9][0-9;]*m/, '').sub("\u001B", '    ').gsub(/\u001B/, '').rstrip
-        end
-      end.join("\n")
+    def decorate_cell(rows, cell)
+      tags = cell['metadata']&.fetch('tags', [])
+      type = cell['cell_type'] || 'raw'
+
+      case @cell_decorator
+      when :html
+        rows.prepend(%(<div class="cell #{type}" data-id="#{cell['id']}" data-tags="#{tags&.join(' ')}">\n\n))
+            .append("\n</div>\n")
+      when :percent
+        rows.prepend(%(%% Cell type:#{type} id:#{cell['id']} tags:#{tags&.join(',')}\n\n))
+      else
+        rows
+      end.join('')
     end
 
-    def self.transform_execute_result(output)
-      [
-        %(\n<div class="output execute_result">\n\n),
-        *output['data']['text/plain'].map { |line| "    #{line}" },
-        "\n\n</div>\n"
-      ].join('')
+    def transform_cell(cell, notebook)
+      cell['cell_type'] == 'code' ? transform_code_cell(cell, notebook) : transform_text_cell(cell)
     end
 
-    def self.transform_image_result(output)
-      if output['data'].key?('image/png')
-        [
-          %(\n<div class="output display_data">\n\n),
-          "![](data:image/png;base64,#{output['data']['image/png'].gsub("\n", '')})",
-          "\n\n</div>\n"
-        ].join('')
+    def decorate_output(output_rows, output)
+      if @cell_decorator == :html
+        output_rows.prepend(%(\n<div class="output #{output['output_type']}">\n\n)).append("\n</div>\n")
+      else
+        output_rows.prepend(%(\n%%%% Output: #{output['output_type']}\n\n))
       end
     end
 
-    def self.transform_error_result(output)
+    def transform_code_cell(cell, notebook)
       [
-        %(\n<div class="output error">\n\n),
-        format_traceback(output['traceback']),
-        "\n\n</div>\n"
-      ].join('')
+        %(``` #{notebook['metadata']['kernelspec']['language']}\n),
+        *cell['source'],
+        "\n```\n",
+        *cell['outputs'].map { |output| transform_output(output) }
+      ]
     end
 
-    def self.transform_output(output)
-      case output['output_type']
-      when 'execute_result'
-        transform_execute_result(output)
-      when 'display_data'
-        transform_image_result(output)
-      when 'error'
-        transform_error_result(output)
-      end
-    end
+    def transform_output(output)
+      transformed = @output_transformer.transform(output)
 
-    def self.transform_text_cell(cell)
-      tags = cell['metadata'].fetch('tags', []).join(' ')
-      id = cell['id']
-      cell_type = cell['cell_type']
+      decorate_output(transformed, output).join('') if transformed
+    end
 
-      [
-        %(<div class="cell #{cell_type}" data-id="#{id}" data-tags="#{tags}">\n\n),
-        *cell['source'],
-        "\n\n</div>\n"
-      ].join('')
+    def transform_text_cell(cell)
+      cell['source'].append("\n")
     end
 
-    def self.transform_metadata(notebook_json)
+    def transform_metadata(notebook_json)
       {
         'jupyter' => {
           'kernelspec' => notebook_json['metadata']['kernelspec'],

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ipynbdiff
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.6
 platform: ruby
 authors:
 - Eduardo Bonet
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-13 00:00:00.000000000 Z
+date: 2021-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: diffy
@@ -121,6 +121,7 @@ files:
 - README.md
 - ipynbdiff.gemspec
 - lib/ipynbdiff.rb
+- lib/output_transformer.rb
 - lib/transformer.rb
 homepage: https://gitlab.com/gitlab-org/incubation-engineering/mlops/rb-ipynbdiff
 licenses: