erd_map 0.1.0

data/lib/erd_map/graph_renderer.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+module ErdMap
+  class GraphRenderer
+    extend Forwardable
+    def_delegators :@graph_renderer, :node_renderer
+
+    attr_reader :graph_renderer
+
+    VISIBLE = 1.0
+    TRANSLUCENT = 0.01
+    HIGHLIGHT_NODE_COLOR = "black"
+    HIGHLIGHT_EDGE_COLOR = "orange"
+    HIGHLIGHT_TEXT_COLOR = "white"
+    BASIC_COLOR = "darkslategray"
+    EMPTHASIS_NODE_SIZE = 80
+
+    def renderers
+      [circle_renderer, rect_renderer]
+    end
+
+    def cardinality_label
+      bokeh_models.LabelSet.new(
+        x: "x",
+        y: "y",
+        text: "text",
+        source: cardinality_data_source,
+        text_font_size: "12pt",
+        text_color: "text_color",
+        text_alpha: { field: "alpha" },
+      )
+    end
+
+    def js_args(plot)
+      {
+        graphRenderer: graph_renderer,
+        rectRenderer: rect_renderer,
+        circleRenderer: circle_renderer,
+        layoutProvider: layout_provider,
+        cardinalityDataSource: cardinality_data_source,
+        connectionsData: graph.connections.to_json,
+        layoutsByChunkData: graph.layouts_by_chunk.to_json,
+        chunkedNodesData: graph.chunked_nodes.to_json,
+        nodeWithCommunityIndexData: graph.node_with_community_index.to_json,
+        searchBox: plot.button_set[:search_box],
+        selectingNodeLabel: plot.button_set[:selecting_node_label],
+        zoomModeToggle: plot.button_set[:zoom_mode_toggle],
+        tapModeToggle: plot.button_set[:tap_mode_toggle],
+        displayTitleModeToggle: plot.button_set[:display_title_mode_toggle],
+        nodeLabels: {
+          titleModelLabel: title_model_label,
+          foreignModelLabel: foreign_model_label,
+          foreignColumnsLabel: foreign_columns_label,
+        },
+        plot: plot.plot,
+        VISIBLE: VISIBLE,
+        TRANSLUCENT: TRANSLUCENT,
+        HIGHLIGHT_NODE_COLOR: HIGHLIGHT_NODE_COLOR,
+        HIGHLIGHT_EDGE_COLOR: HIGHLIGHT_EDGE_COLOR,
+        HIGHLIGHT_TEXT_COLOR: HIGHLIGHT_TEXT_COLOR,
+        BASIC_COLOR: BASIC_COLOR,
+        EMPTHASIS_NODE_SIZE: EMPTHASIS_NODE_SIZE,
+      }
+    end
+
+    private
+
+    attr_reader :bokeh_models
+    attr_reader :graph
+
+    def initialize(graph)
+      import_modules = ErdMap.py_call_modules.imported_modules
+      @bokeh_models = import_modules[:bokeh_models]
+      @graph = graph
+      @graph_renderer = circle_renderer
+    end
+
+    def node_data_source
+      nodes_x, nodes_y = graph.node_names.map { |node| graph.initial_layout[node] ? graph.initial_layout[node] : graph.whole_layout[node] }.transpose
+      nodes_alpha = graph.node_names.map { |node| graph.initial_layout[node] ? VISIBLE : TRANSLUCENT }
+
+      columns_label = []
+      title_label = []
+      rect_heights = []
+      graph.node_names.map do |node_name|
+        title_text = format_text([node_name], title: true)
+        columns_text = [*title_text.scan("\n"), "\n", format_text(graph.association_columns[node_name])].join
+        columns_label << columns_text
+        title_label << [title_text, "\n", *columns_text.scan("\n")].join
+
+        padding = 36
+        line_count = columns_text.scan("\n").size + 1
+        rect_heights << line_count * 20 + padding
+      end
+
+      bokeh_models.ColumnDataSource.new(
+        data: {
+          index: graph.node_names,
+          alpha: nodes_alpha,
+          x: nodes_x,
+          y: nodes_y,
+          radius: graph.node_radius,
+          original_radius: graph.node_radius,
+          rect_height: rect_heights,
+          title_label: title_label,
+          columns_label: columns_label,
+          fill_color: graph.node_colors,
+          circle_original_color: graph.node_colors,
+          rect_original_color: graph.node_names.map { "white" },
+          text_color: graph.node_names.map { BASIC_COLOR },
+          text_outline_color: graph.node_names.map { nil },
+        }
+      )
+    end
+
+    def format_text(columns, title: false)
+      max_chars_size = title ? 18 : 20
+      columns.flat_map { |column| column.scan(/(\w{1,#{max_chars_size}})/) }.join("\n")
+    end
+
+    def circle_renderer
+      @circle_renderer ||= bokeh_models.GraphRenderer.new(
+        layout_provider: layout_provider,
+        visible: true,
+      ).tap do |renderer|
+        renderer.node_renderer.data_source = node_data_source
+        renderer.node_renderer.glyph = circle_glyph
+        renderer.node_renderer.selection_glyph = renderer.node_renderer.glyph
+        renderer.node_renderer.nonselection_glyph = renderer.node_renderer.glyph
+        renderer.edge_renderer.data_source = edge_data_source
+        renderer.edge_renderer.glyph = bokeh_models.MultiLine.new(
+          line_color: { field: "line_color" },
+          line_alpha: { field: "alpha" },
+          line_width: 1,
+        )
+      end
+    end
+
+    def rect_renderer
+      @rect_renderer ||= bokeh_models.GraphRenderer.new(
+        layout_provider: layout_provider,
+        visible: false,
+      ).tap do |renderer|
+        renderer.node_renderer.data_source = node_data_source
+        renderer.node_renderer.glyph = rect_glyph
+        renderer.node_renderer.selection_glyph = renderer.node_renderer.glyph
+        renderer.node_renderer.nonselection_glyph = renderer.node_renderer.glyph
+        renderer.edge_renderer.data_source = edge_data_source
+        renderer.edge_renderer.glyph = bokeh_models.MultiLine.new(
+          line_color: { field: "line_color" },
+          line_alpha: { field: "alpha" },
+          line_width: 1,
+        )
+      end
+    end
+
+    def circle_glyph
+      bokeh_models.Circle.new(
+        radius: "radius",
+        radius_units: "screen",
+        fill_color: { field: "fill_color" },
+        fill_alpha: { field: "alpha" },
+        line_alpha: { field: "alpha" },
+      )
+    end
+
+    def rect_glyph
+      bokeh_models.Rect.new(
+        width: 150,
+        height: { field: "rect_height" },
+        width_units: "screen",
+        height_units: "screen",
+        fill_color: { field: "fill_color" },
+        fill_alpha: { field: "alpha" },
+        line_color: BASIC_COLOR,
+        line_alpha: { field: "alpha" },
+      )
+    end
+
+    def title_model_label
+      bokeh_models.LabelSet.new(
+        x: "x",
+        y: "y",
+        text: "index",
+        source: graph_renderer.node_renderer.data_source,
+        text_font_size: "12pt",
+        text_color: { field: "text_color" },
+        text_outline_color: { field: "text_outline_color" },
+        text_align: "center",
+        text_baseline: "middle",
+        text_alpha: { field: "alpha" },
+        visible: true,
+      )
+    end
+
+    def foreign_model_label
+      bokeh_models.LabelSet.new(
+        x: "x",
+        y: "y",
+        text: { field: "title_label" },
+        source: graph_renderer.node_renderer.data_source,
+        text_font_size: "10pt",
+        text_font_style: "bold",
+        text_color: { field: "text_color" },
+        text_outline_color: { field: "text_outline_color" },
+        text_align: "center",
+        text_baseline: "middle",
+        text_alpha: { field: "alpha" },
+        # visible: false,
+      )
+    end
+
+    def foreign_columns_label
+      bokeh_models.LabelSet.new(
+        x: "x",
+        y: "y",
+        text: { field: "columns_label" },
+        source: graph_renderer.node_renderer.data_source,
+        text_font_size: "10pt",
+        text_font_style: "normal",
+        text_color: { field: "text_color" },
+        text_outline_color: { field: "text_outline_color" },
+        text_align: "center",
+        text_baseline: "middle",
+        text_alpha: { field: "alpha" },
+        # visible: false,
+      )
+    end
+
+    def edge_data_source
+      edge_start, edge_end = graph.edges.map { |edge| [edge[0], edge[1]] }.transpose
+      edges_alpha = graph.edges.map { |edge| graph.initial_nodes.include?(edge[0]) && graph.initial_nodes.include?(edge[1]) ? VISIBLE : TRANSLUCENT }
+      bokeh_models.ColumnDataSource.new(
+        data: {
+          start: edge_start,
+          end: edge_end,
+          alpha: edges_alpha,
+          line_color: graph.edges.map { BASIC_COLOR },
+        }
+      )
+    end
+
+    def cardinality_data_source
+      return @cardinality_data_source if @cardinality_data_source
+
+      @cardinality_data_source = bokeh_models.ColumnDataSource.new(
+        data: {
+          x: [],
+          y: [],
+          source: [],
+          target: [],
+          text: [],
+          alpha: [],
+          text_color: [],
+        }
+      )
+
+      graph.edges.each do |(source_node, target_node)|
+        next if source_node == target_node
+
+        label_alpha = graph.initial_nodes.include?(source_node) && graph.initial_nodes.include?(target_node) ?
+          VISIBLE : 0
+
+        x_offset = 0.2
+        y_offset = 0.3
+        source_x, source_y = graph.initial_layout[source_node] || graph.whole_layout[source_node]
+        target_x, target_y = graph.initial_layout[target_node] || graph.whole_layout[target_node]
+        vector_x = target_x - source_x
+        vector_y = target_y - source_y
+        length = Math.sqrt(vector_x**2 + vector_y**2)
+
+        @cardinality_data_source.data[:x] << source_x + (vector_x / length) * x_offset
+        @cardinality_data_source.data[:y] << source_y + (vector_y / length) * y_offset
+        @cardinality_data_source.data[:source] << source_node
+        @cardinality_data_source.data[:target] << target_node
+        @cardinality_data_source.data[:text] << "1"
+        @cardinality_data_source.data[:alpha] << label_alpha
+
+        @cardinality_data_source.data[:x] << target_x - (vector_x / length) * x_offset
+        @cardinality_data_source.data[:y] << target_y - (vector_y / length) * y_offset
+        @cardinality_data_source.data[:source] << source_node
+        @cardinality_data_source.data[:target] << target_node
+        @cardinality_data_source.data[:text] << "n" # FIXME: Show "1" when has_one association
+        @cardinality_data_source.data[:alpha] << label_alpha
+      end
+      @cardinality_data_source.data[:text_color] = Array.new(@cardinality_data_source.data[:x].to_a.size) { BASIC_COLOR }
+      @cardinality_data_source
+    end
+
+    def layout_provider
+      return @layout_provider if @layout_provider
+
+      nodes_x, nodes_y = graph.node_names.map { |node| graph.initial_layout[node] ? graph.initial_layout[node] : graph.whole_layout[node] }.transpose
+      graph_layout = graph.node_names.zip(nodes_x, nodes_y).map { |node, x, y| [node, [x, y]] }.to_h
+      @layout_provider = bokeh_models.StaticLayoutProvider.new(graph_layout: graph_layout)
+    end
+  end
+end

data/lib/erd_map/map_builder.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module ErdMap
+  class MapBuilder
+    def execute
+      import_modules
+      @graph = ErdMap::Graph.new
+      @graph_renderer = ErdMap::GraphRenderer.new(@graph)
+      save(build_layout)
+    end
+
+    private
+
+    attr_reader :bokeh_io, :bokeh_models, :bokeh_plotting
+    attr_reader :graph, :graph_renderer
+
+    def import_modules
+      import_modules = ErdMap.py_call_modules.imported_modules
+      @bokeh_io = import_modules[:bokeh_io]
+      @bokeh_models = import_modules[:bokeh_models]
+      @bokeh_plotting = import_modules[:bokeh_plotting]
+    end
+
+    def build_layout
+      plot = Plot.new(graph)
+      graph_renderer.renderers.each do |renderer|
+        plot.renderers.append(renderer)
+        renderer.node_renderer.data_source.selected.js_on_change("indices", toggle_tapped)
+      end
+      plot.add_layout(graph_renderer.cardinality_label)
+      bokeh_io.curdoc.js_on_event("document_ready", setup_graph_manager(plot))
+
+      bokeh_models.Column.new(
+        children: [
+          bokeh_models.Row.new(
+            children: [
+              plot.button_set[:left_spacer],
+              plot.button_set[:selecting_node_label],
+              plot.button_set[:search_box],
+              plot.button_set[:zoom_mode_toggle],
+              plot.button_set[:tap_mode_toggle],
+              plot.button_set[:display_title_mode_toggle],
+              plot.button_set[:re_layout_button],
+              plot.button_set[:zoom_in_button],
+              plot.button_set[:zoom_out_button],
+              plot.button_set[:re_compute_button],
+              plot.button_set[:right_spacer],
+            ],
+            sizing_mode: "stretch_width",
+          ),
+          plot.plot,
+        ],
+        sizing_mode: "stretch_both",
+      )
+    end
+
+    def save(layout)
+      tmp_dir = Rails.root.join("tmp", "erd_map")
+      FileUtils.makedirs(tmp_dir) unless Dir.exist?(tmp_dir)
+      output_path = File.join(tmp_dir, "map.html")
+
+      bokeh_io.output_file(output_path)
+      bokeh_io.save(layout)
+      puts output_path
+    end
+
+    def setup_graph_manager(plot)
+      bokeh_models.CustomJS.new(
+        args: graph_renderer.js_args(plot),
+        code: <<~JS
+          #{graph_manager}
+          window.graphManager = new GraphManager({
+            graphRenderer,
+            rectRenderer,
+            circleRenderer,
+            layoutProvider,
+            connectionsData,
+            layoutsByChunkData,
+            chunkedNodesData,
+            nodeWithCommunityIndexData,
+            selectingNodeLabel,
+            searchBox,
+            zoomModeToggle,
+            tapModeToggle,
+            displayTitleModeToggle,
+            nodeLabels,
+            plot,
+            windowObj: window,
+          })
+        JS
+      )
+    end
+
+    def toggle_tapped
+      bokeh_models.CustomJS.new(
+        code: <<~JS
+          window.graphManager.cbObj = cb_obj
+          window.graphManager.toggleTapped()
+        JS
+      )
+    end
+
+    def graph_manager
+      @graph_manager ||= File.read(__dir__ + "/graph_manager.js")
+    end
+
+    class << self
+      def build
+        Rails.logger.info "build start"
+        new.execute
+        Rails.logger.info "build completed"
+      end
+    end
+  end
+end

data/lib/erd_map/plot.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module ErdMap
+  class Plot
+    extend Forwardable
+    def_delegators :@plot, :renderers, :add_layout
+
+    def plot
+      return @plot if @plot
+
+      padding_ratio = 0.1
+      x_min, x_max, y_min, y_max = graph.initial_layout.values.transpose.map(&:minmax).flatten
+      x_padding, y_padding = [(x_max - x_min) * padding_ratio, (y_max - y_min) * padding_ratio]
+      @plot = bokeh_models.Plot.new(
+        sizing_mode: "stretch_both",
+        x_range: bokeh_models.Range1d.new(start: x_min - x_padding, end: x_max + x_padding),
+        y_range: bokeh_models.Range1d.new(start: y_min - y_padding, end: y_max + y_padding),
+        tools: [
+          wheel_zoom_tool = bokeh_models.WheelZoomTool.new,
+          bokeh_models.ResetTool.new,
+          bokeh_models.PanTool.new,
+          bokeh_models.TapTool.new,
+        ],
+      ).tap do |plot|
+        plot.toolbar.active_scroll = wheel_zoom_tool
+      end
+    end
+
+    def button_set
+      @button_set ||= ButtonSet.new
+    end
+
+    private
+
+    attr_reader :graph
+
+    def initialize(graph)
+      @graph = graph
+      register_callback
+    end
+
+    def register_callback
+      plot.x_range.js_on_change("start", custom_js("triggerZoom"))
+      plot.x_range.js_on_change("end", custom_js("triggerZoom"))
+      plot.js_on_event("mousemove", custom_js("toggleHovered"))
+      plot.js_on_event("mousemove", bokeh_models.CustomJS.new(code: save_mouse_position))
+      plot.js_on_event("reset", custom_js("resetPlot"))
+    end
+
+    def bokeh_models
+      @bokeh_models ||= ErdMap.py_call_modules.imported_modules[:bokeh_models]
+    end
+
+    def custom_js(function_name)
+      bokeh_models.CustomJS.new(
+        code: <<~JS
+          window.graphManager.cbObj = cb_obj
+          window.graphManager.#{function_name}()
+        JS
+      )
+    end
+
+    def save_mouse_position
+      <<~JS
+        if (window.saveMousePosition !== undefined) { clearTimeout(window.saveMousePosition) }
+        window.saveMousePosition = setTimeout(function() {
+          window.lastMouseX = cb_obj.x
+          window.lastMouseY = cb_obj.y
+        }, 100)
+      JS
+    end
+
+    class ButtonSet
+      extend Forwardable
+      def_delegators :@button_set, :[]
+
+      private
+
+      def initialize
+        @button_set = {
+          left_spacer: left_spacer,
+          selecting_node_label: selecting_node_label,
+          search_box: search_box,
+          zoom_mode_toggle: zoom_mode_toggle,
+          tap_mode_toggle: tap_mode_toggle,
+          display_title_mode_toggle: display_title_mode_toggle,
+          re_layout_button: re_layout_button,
+          zoom_in_button: zoom_in_button,
+          zoom_out_button: zoom_out_button,
+          re_compute_button: re_compute_button,
+          right_spacer: right_spacer,
+        }
+      end
+
+      def left_spacer
+        bokeh_models.Spacer.new(width: 0, sizing_mode: "stretch_width")
+      end
+
+      def right_spacer
+        bokeh_models.Spacer.new(width: 30, sizing_mode: "fixed")
+      end
+
+      def selecting_node_label
+        bokeh_models.Div.new(
+          text: "",
+          height: 28,
+          styles: { display: :flex, align_items: :center },
+        )
+      end
+
+      def search_box
+        bokeh_models.TextInput.new(placeholder: "🔍 Search model", width: 200).tap do |input|
+          input.js_on_change("value", custom_js("searchNodes"))
+        end
+      end
+
+      def zoom_mode_toggle
+        bokeh_models.Button.new(label: "Wheel mode: fix", button_type: "default").tap do |button|
+          button.js_on_click(custom_js("toggleZoomMode"))
+        end
+      end
+
+      def tap_mode_toggle
+        bokeh_models.Button.new(label: "Tap mode: association", button_type: "default").tap do |button|
+          button.js_on_click(custom_js("toggleTapMode"))
+        end
+      end
+
+      def display_title_mode_toggle
+        bokeh_models.Button.new(label: "Display mode: title", button_type: "default").tap do |button|
+          button.js_on_click(custom_js("toggleDisplayTitleMode"))
+        end
+      end
+
+      def re_layout_button
+        bokeh_models.Button.new(label: "Re-Layout", button_type: "default").tap do |button|
+          button.js_on_click(custom_js("reLayout"))
+        end
+      end
+
+      def zoom_in_button
+        bokeh_models.Button.new(label: "Zoom In", button_type: "primary").tap do |button|
+          button.js_on_click(custom_js("zoomIn",))
+        end
+      end
+
+      def zoom_out_button
+        bokeh_models.Button.new(label: "Zoom Out", button_type: "success").tap do |button|
+          button.js_on_click(custom_js("zoomOut"))
+        end
+      end
+
+      def re_compute_button
+        bokeh_models.Button.new(label: "Re-Compute", button_type: "default").tap do |button|
+          button.js_on_click(
+            bokeh_models.CustomJS.new(
+              args: { button: button },
+              code: <<~JS
+                button.disabled = true
+                button.label = "Computing ..."
+
+                fetch("/erd_map", { method: "PUT" })
+                  .then(response => {
+                    if (response.ok) { return response.text() }
+                    else { return response.json().then(json => { throw new Error(json.message) }) }
+                  })
+                  .then(data => { window.location.reload() })
+                  .catch(error => {
+                    alert(error.message)
+                    console.error(error)
+
+                    button.disabled = false
+                    button.label = "Re-Compute"
+                  })
+              JS
+            )
+          )
+        end
+      end
+
+      def bokeh_models
+        @bokeh_models ||= ErdMap.py_call_modules.imported_modules[:bokeh_models]
+      end
+
+      def custom_js(function_name)
+        bokeh_models.CustomJS.new(
+          code: <<~JS
+            window.graphManager.cbObj = cb_obj
+            window.graphManager.#{function_name}()
+          JS
+        )
+      end
+    end
+  end
+end

data/lib/erd_map/py_call_modules.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require "pycall"
+
+module ErdMap
+  class PyCallModules
+    def initialize
+      @nx = PyCall.import_module("networkx")
+      @bokeh_io = PyCall.import_module("bokeh.io")
+      @bokeh_models = PyCall.import_module("bokeh.models")
+      @bokeh_plotting = PyCall.import_module("bokeh.plotting")
+      @networkx_community = PyCall.import_module("networkx.algorithms.community")
+    end
+
+    def imported_modules
+      {
+        nx: @nx,
+        bokeh_io: @bokeh_io,
+        bokeh_models: @bokeh_models,
+        bokeh_plotting: @bokeh_plotting,
+        networkx_community: @networkx_community,
+      }
+    end
+  end
+end

data/lib/erd_map/version.rb

@@ -0,0 +1,3 @@
+module ErdMap
+  VERSION = "0.1.0"
+end

data/lib/erd_map.rb

@@ -0,0 +1,19 @@
+require "erd_map/version"
+require "erd_map/engine"
+require "erd_map/py_call_modules"
+require "erd_map/graph"
+require "erd_map/graph_renderer"
+require "erd_map/map_builder"
+require "erd_map/plot"
+
+module ErdMap
+  class << self
+    def py_call_modules
+      @py_call_modules
+    end
+
+    def py_call_modules=(py_call_modules)
+      @py_call_modules = py_call_modules
+    end
+  end
+end

data/lib/tasks/erd_map_tasks.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+desc "Compute erd_map"
+  task erd_map: :environment do
+  puts "Map computing start."
+  ErdMap::MapBuilder.build
+  puts "Map computing completed."
+end