diagrammatron 0.4.2 → 0.5.0

data/bin/diagrammatron-prune

@@ -1,14 +1,12 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2021 Ismo Kärkkäinen
+# Copyright © 2021-2023 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
 require_relative '../lib/common'
 require 'optparse'
-require 'yaml'
 require 'set'
-require 'pathname'
 
 
 def prune_array(original, matching, keep)
@@ -26,7 +24,12 @@ def prune(doc, patterns, keep)
   nodes = doc.fetch('nodes', [])
   matching = Array.new(nodes.size, false)
   nodes.each_index do |k|
-    label = nodes[k].fetch('label', '')
+    label = nodes[k].fetch('label', nil)
+    if label.nil?
+      # Schemas require 'label' to be present.
+      matching[k] = !keep
+      next
+    end
     patterns.each do |p|
       next unless p.match(label)
       matching[k] = true
@@ -40,10 +43,12 @@ def prune(doc, patterns, keep)
   edges.each_index do |k|
     between = edges[k].fetch('between', [])
     if between.size != 2
-      matching[k] = keep # Removing these suppresses later warnings.
+      matching[k] = !keep # Schemas require between size to be 2.
     elsif keep
+      # Both ends matched reversed remove rule, keep the edge.
       matching[k] = (labels.member?(between[0]) && labels.member?(between[1]))
     else
+      # If either end matched remove rule, remove the edge.
       matching[k] = (labels.member?(between[0]) || labels.member?(between[1]))
     end
   end
@@ -53,6 +58,7 @@ end
 def main
   input = nil
   output = nil
+  input_output_schema = 'nodes'
   keep = false
   ENV['POSIXLY_CORRECT'] = '1' # Leaves patterns as they are.
   parser = OptionParser.new do |opts|
@@ -75,6 +81,8 @@ def main
 Patterns are strings used to create Ruby Regexps.
 
 Input YAML file is expected to be dot_json2diagrammatron output.
+Input and output YAML file schema is returned by:
+diagrammatron-schema #{input_output_schema}
 
 Output is the input file with nodes that have labels that match patterns
 removed or kept depending on options. Edges to removed nodes are removed.
@@ -89,7 +97,7 @@ removed or kept depending on options. Edges to removed nodes are removed.
   rescue StandardError => e
     return aargh("Error creating Regexp: #{e}", 1)
   end
-  doc = load_source(input)
+  doc = load_verified(input, input_output_schema)
   return 2 if doc.nil?
 
   begin
@@ -98,7 +106,7 @@ removed or kept depending on options. Edges to removed nodes are removed.
     return aargh('Error processing input.', 3)
   end
 
-  dump_result(output, YAML.dump(doc, line_width: 1_000_000), 4)
+  save_verified(output, doc, 4, input_output_schema)
 end
 
 exit(main) if (defined? $unit_test).nil?

data/bin/diagrammatron-render

@@ -61,8 +61,7 @@ def separate_coordinates(doc)
     push_coords(xcoords, ycoords, ckd2count, node.clone, 1, 1, true)
   end
   doc['edges'].each do |edge|
-    path = edge.fetch('path', nil)
-    next if path.nil?
+    path = edge['path']
     xdirection, ydirection = end_directions(path[0], path[1])
     push_coords(xcoords, ycoords, ckd2count, path[0], xdirection, ydirection)
     (1...(path.size - 1)).each do |k|
@@ -80,9 +79,10 @@ class Styles
   def base_styles(m, styles, group)
     d = styles.dig(group, 'default')
     if d.nil?
-      d = m.fetch('default', {}) if d.nil?
+      d = m.fetch('default', {}) # No default in styles.
     else
       m['default'] = m.fetch('default', {}).merge(d)
+      d = m['default']
     end
     styles.fetch(group, {}).each_pair do |name, values|
       s = d.clone
@@ -98,7 +98,7 @@ class Styles
     @d = base_styles(base_styles({}, template_styles, 'diagram'), diagram_styles, 'diagram')
   end
 
-  def fill(mapping, type_name, item)
+  def fill(mapping, _type_name, item)
     styles = item.fetch('style', [ 'default' ])
     styles = [ styles ] unless styles.is_a?(Array)
     s = {}
@@ -111,7 +111,7 @@ class Styles
     end
     s.merge!(mapping['default']) unless found # Merge default at least.
     # Keep values specified explicitly.
-    item.merge!(s) { |key, existing, from_template| existing || from_template }
+    item.merge!(s) { |_key, existing, from_template| existing || from_template }
   end
 
   def apply_node_styles(node)
@@ -168,7 +168,7 @@ def estimate_sizes(doc, ckd2count)
   $render = SizeEstimation.new(ckd2count, doc)
   doc['nodes'].each do |node|
     $render.node = node
-    label = node.fetch('label', 'unnamed')
+    label = node['label']
     style = node.fetch('style', 'default')
     code = node.fetch('size_estimator',
       %(raise NotImplementedError, "No size estimator for style: #{style}"))
@@ -186,7 +186,7 @@ end
 def maxima(doc)
   xmax = Hash.new(0)
   ymax = Hash.new(0)
-  doc.fetch('nodes', []).each do |node|
+  doc['nodes'].each do |node|
     xmax[node['xo']] = [ node['w'], xmax[node['xo']] ].max
     ymax[node['yo']] = [ node['h'], ymax[node['yo']] ].max
   end
@@ -194,7 +194,7 @@ def maxima(doc)
 end
 
 def apply_maxima(doc, xmax, ymax)
-  doc.fetch('nodes', []).each do |node|
+  doc['nodes'].each do |node|
     node['w'] = xmax[node['xo']]
     node['h'] = ymax[node['yo']]
   end
@@ -257,13 +257,12 @@ class Render
   def dimensions
     w = 0
     h = 0
-    @doc.fetch('nodes', []).each do |node|
+    @doc['nodes'].each do |node|
       w = [ w, node['xo'] + node['w'] ].max
       h = [ h, node['yo'] + node['h'] ].max
     end
-    @doc.fetch('edges', []).each do |edge|
-      path = edge.fetch('path', nil)
-      next if path.nil?
+    @doc['edges'].each do |edge|
+      path = edge['path']
       path.each do |p|
         w = [ w, p['xo'] ].max
         h = [ h, p['yo'] ].max
@@ -280,9 +279,31 @@ def apply(doc, template)
   out
 end
 
+def reverse_depth_order(a, b)
+  d = b['depth'] <=> a['depth']
+  return d unless d.zero?
+  d = a['index'] <=> b['index']
+  return d unless d.zero?
+  a['kind'] <=> b['kind']
+end
+
+def reverse_depth_sort(items)
+  arr = []
+  items.each_index do |k|
+    arr.push({
+      'depth' => items[k].fetch('depth', 0),
+      'index' => k,
+      'kind' => items[k].fetch('kind', 'unknown')
+    })
+  end
+  arr.sort! { |a, b| reverse_depth_order(a, b) }
+  arr.map { |x| items[x['index']] }
+end
+
 def main
   template = nil
   input = nil
+  input_schema = 'render'
   output = nil
   styles = nil
   parser = OptionParser.new do |opts|
@@ -302,7 +323,10 @@ def main
     end
     opts.on('-h', '--help', 'Print this help and exit.') do
       $stdout.puts %(#{opts}
-Input YAML file is expected to be the output of diagrammatron-place.
+Input YAML file schema is returned by:
+diagrammatron-schema #{input_schema}
+
+There can be other fields present as needed by the template.
 
 Output is the file produced by the erb-template.
 )
@@ -325,15 +349,15 @@ Output is the file produced by the erb-template.
     end
   end
 
-  doc = load_source(input)
+  doc = load_verified(input, input_schema)
   return 2 if doc.nil?
 
   styles = Styles.new(template.fetch('styles', {}), doc.fetch('styles', {}))
-  doc.fetch('nodes', []).each do |node|
+  doc['nodes'].each do |node|
     styles.apply_node_styles(node)
-    node['text'] = node.fetch('text', node.fetch('label', '')).split("\n")
+    node['text'] = node.fetch('text', node['label']).split("\n")
   end
-  doc.fetch('edges', []).each { |edge| styles.apply_edge_styles(edge) }
+  doc['edges'].each { |edge| styles.apply_edge_styles(edge) }
   doc['diagram'] = {} unless doc.key? 'diagram'
   styles.apply_diagram_styles(doc['diagram'])
 
@@ -354,6 +378,30 @@ Output is the file produced by the erb-template.
   remap_coordinates(xcoords, xmax, x2min, doc.dig('diagram', 'edge_gap'))
   remap_coordinates(ycoords, ymax, y2min, doc.dig('diagram', 'edge_gap'))
 
+  doc['nodes'] = reverse_depth_sort(doc['nodes'])
+  doc['edges'] = reverse_depth_sort(doc['edges'])
+  all = doc['nodes'].map do |a|
+    {
+      'kind' => 'node',
+      'depth' => a.fetch('depth', 0),
+      'item' => a
+    }
+  end
+  all.concat(doc['edges'].map do |a|
+    {
+      'kind' => 'edge',
+      'depth' => a.fetch('depth', 0),
+      'item' => a
+    }
+  end)
+  all = reverse_depth_sort(all)
+  doc['all'] = all.map do |x|
+    {
+      'kind' => x['kind'],
+      'item' => x['item']
+    }
+  end
+
   dump_result(output, apply(doc, template), 5)
 end
 

data/bin/diagrammatron-schema

@@ -0,0 +1,107 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Copyright © 2023 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+require_relative '../lib/common'
+require 'optparse'
+require 'yaml'
+require 'json'
+
+def convert(result, format)
+  case format
+  when :json then JSON.generate(result)
+  when :yaml then YAML.dump(result)
+  when :prettyjson then JSON.pretty_generate(result)
+  end
+end
+
+def main
+  output = nil
+  format = :json
+  package = :bare
+  ENV['POSIXLY_CORRECT'] = '1' # Leaves patterns as they are.
+  parser = OptionParser.new do |opts|
+    opts.summary_indent = '  '
+    opts.summary_width = 20
+    opts.banner = 'Usage: diagrammatron-schema [options] schema-name ...'
+    opts.separator ''
+    opts.separator 'Options:'
+    opts.on('-f', '--format FMT', 'Output schema format (json, yaml, pretty).') do |fmt|
+      case fmt.upcase
+      when 'JSON' then format = :json
+      when 'YAML' then format = :yaml
+      when 'PRETTY' then format = :prettyjson
+      else
+        format = fmt
+      end
+    end
+    opts.on('-p', '--package PKG', 'Wrap schema(s) in given way.') do |pkg|
+      case pkg.upcase
+      when 'OBJECT' then package = :object
+      when 'ARRAY' then package = :array
+      when 'BARE' then package = :bare
+      else
+        package = pkg
+      end
+    end
+    opts.on('-o', '--output FILE', 'Output file name. Write to stdout if not given.') do |filename|
+      output = filename
+    end
+    opts.on('-l', '--list', 'List available schemas and exit.') do
+      $stdout.puts list_schemas.join("\n")
+      exit 0
+    end
+    opts.on('-h', '--help', 'Print this help and exit.') do
+      $stdout.puts %(#{opts}
+Outputs the given schema files.
+Formats are: JSON, YAML, pretty for prettified JSON.
+Packages are:
+  object: Output schemas as an object with name to schema mapping.
+  array: Output schemas as an array in the order given on the command-line
+  bare: Output schemas in the specified order as they are. The default.
+)
+      exit 0
+    end
+  end
+  parser.parse! ARGV
+
+  aargh("Unknown format: #{format}", 1) unless format.is_a? Symbol
+  case package
+  when :bare, :array then out = []
+  when :object then out = {}
+  else
+    return aargh("Unknown package: #{package}", 1) unless package.is_a? Symbol
+  end
+
+  ARGV.each do |name|
+    s = load_schema(name)
+    if out.is_a? Array
+      out.push(s)
+    else
+      out[name] = s
+    end
+  rescue Errno::ENOENT
+    aargh "No such schema: #{name}", 2
+  rescue StandardError => e
+    aargh "#{e}\nInternal error: #{name}", 3
+  end
+
+  if output.nil?
+    fp = $stdout
+  else
+    fp = File.new(output, 'w')
+  end
+  if package == :bare
+    out.each do |item|
+      fp.puts(convert(item, format))
+    end
+  else
+    fp.puts(convert(out, format))
+  end
+  fp.close
+  0
+end
+
+exit(main) if (defined? $unit_test).nil?

data/bin/diagrammatron-subset

@@ -0,0 +1,188 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Copyright © 2023 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+require_relative '../lib/common'
+require_relative '../lib/subsets'
+require 'optparse'
+require 'set'
+
+
+class Item
+  @@counter = 0
+  attr_reader :item, :ids
+
+  def initialize(item, label2id = {})
+    @item = item
+    @ids = [ @@counter ]
+    @@counter += 1
+    # This silently removes edges that have no valid end-point.
+    item.fetch('between', []).each { |x| @ids.push(label2id.fetch(x, -1)) }
+  end
+
+  def self.reset
+    @@counter = 0
+  end
+end
+
+def work_copy(src)
+  nodes = src.fetch('nodes', []).map { |n| Item.new(n) }
+  label2id = {}
+  nodes.each { |x| label2id[x.item['label']] = x.ids.first }
+  edges = src.fetch('edges', []).map { |e| Item.new(e, label2id) }
+  {
+    edges: edges,
+    nodes: nodes
+  }
+end
+
+def prepare_output(doc, work, chosen)
+  %i[edges nodes].each do |s|
+    work[s].select! do |x|
+      (x.ids.index { |id| !chosen.member?(id) }).nil?
+    end
+    doc[s.to_s] = work[s].map(&:item)
+  end
+end
+
+def gather_set(work, rules, current)
+  r = rules['sets'].fetch(current, nil)
+  return aargh("Undefined set: #{current}", 9) if r.nil?
+  result = Set.new
+  %i[nodes edges].each do |category|
+    work[category].each do |item|
+      if match_item(item.item, r.fetch(category, {}))
+        result.add(item.ids.first)
+      end
+    end
+  end
+  result
+end
+
+def gather(work, rules, current, above, set_cache)
+  e = rules['sets'].fetch(current, nil)
+  unless e.nil?
+    set = set_cache.fetch(current, nil)
+    return set unless set.nil?
+    # Evaluate needed set, store, and return it.
+    set = gather_set(work, rules, current)
+    set_cache[current] = set # Error may end up in cache, not a problem.
+    return set
+  end
+  return aargh("Loop in expressions for #{current}: #{above.reverse.join("\n")}", 8) if above.include?(current)
+  e = rules['expressions'].fetch(current, nil)
+  return aargh("Undefined expression: #{current}", 7) if e.nil?
+  above.push(current)
+  result = gather(work, rules, e.first, above, set_cache)
+  return result unless result.is_a?(Set)
+  (1...e.size).step(2) do |k|
+    op = e[k]
+    id = e[k + 1]
+    s = gather(work, rules, id, above, set_cache)
+    return s unless s.is_a?(Set)
+    case op
+    when :plus then result += s
+    when :minus then result -= s
+    else
+      return aargh("Unsupported operator: #{op}", 100)
+    end
+  end
+  above.pop
+  result
+end
+
+def main
+  input_output_schema = 'nodes'
+  input = nil
+  output = nil
+  selected = nil
+  ENV['POSIXLY_CORRECT'] = '1'
+  parser = OptionParser.new do |opts|
+    opts.summary_indent = '  '
+    opts.summary_width = 20
+    opts.banner = 'Usage: diagrammatron-subset [options] rule-file-names...'
+    opts.separator ''
+    opts.separator 'Options:'
+    opts.on('-i', '--input FILE', 'Input file name. Read from stdin if not given.') do |filename|
+      input = filename
+    end
+    opts.on('-o', '--output FILE', 'Output file name. Written to stdout if not given.') do |filename|
+      output = filename
+    end
+    opts.on('-s', '--select NAME', 'Select expression or subset NAME result.') do |name|
+      selected = name
+    end
+    opts.on('-h', '--help', 'Print this help and exit.') do
+      $stdout.puts %(#{opts}
+Rule files can specify named subset selection rules and named expressions on
+how to combine the subsets when determining which nodes and edges will form the
+diagram.
+
+Subset selection rules can apply to nodes only, edges only, or both. The rules
+give for a field name an array of strings or patterns. If a node or edge
+matches any of the rules, it belongs to the subset.
+
+Expressions can refer to the subset or expression names and use operations to
+form more complex subsets. In the simplest form, they just are a subset name.
+Operations are a + b to add elements of b and a - b to drop elements of b. If
+expression and subset have the same name, identifier refers to subset.
+
+The select option gives the name of the expression or subset to use.
+
+Edges to removed nodes are removed. Nodes with all edges removed are retained.
+Output is the input file with remaining nodes and edges.
+
+Input and output YAML file schema is returned by:
+diagrammatron-schema #{input_output_schema}
+
+Rule YAML file schema is returned by:
+diagrammatron-schema subset
+
+Note that input schema is minimal and does not restrict running this program
+at a later stage in the pipeline after other tools have added their fields.
+You can refer to fields outside the schema.
+)
+      exit 0
+    end
+  end
+  parser.parse! ARGV
+
+  if selected.nil?
+    return aargh('Must select an expression or subset to use for output.', 2)
+  end
+
+  rules = {}
+  ARGV.each do |filename|
+    r = load_verified(filename, 'subset')
+    return 5 if r.nil?
+    return 6 unless check_rules(r, filename)
+    merge_any(r) # The 'any' field exists to avoid duplicating rules by hand.
+    merge_rules(rules, r)
+  end
+
+  if rules.fetch('sets', {}).key? selected
+    # Refers directly to subset so other expressions do not matter.
+    rules['expressions'] = { selected => selected }
+  elsif !rules.fetch('expressions', {}).key? selected
+    return aargh("Selected #{selected} is not expression or subset name.", 2)
+  end
+
+  doc = load_verified(input, input_output_schema)
+  return 2 if doc.nil?
+
+  begin
+    work = work_copy(doc)
+    return 3 if work.nil?
+  rescue StandardError
+    return aargh('Error processing input.', 3)
+  end
+
+  chosen = gather(work, rules, selected, [], {})
+  return chosen unless chosen.is_a?(Set)
+  prepare_output(doc, work, chosen)
+  save_verified(output, doc, 4, input_output_schema)
+end
+
+exit(main) if (defined? $unit_test).nil?

data/bin/dot_json2diagrammatron

@@ -1,14 +1,11 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2021, 2022 Ismo Kärkkäinen
+# Copyright © 2021-2023 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
 require_relative '../lib/common'
 require 'optparse'
-require 'yaml'
-require 'json'
-require 'pathname'
 
 
 def convert(src)
@@ -63,10 +60,10 @@ def convert(src)
   { 'edges' => edges, 'nodes' => nodes }
 end
 
-$INPUT = nil
-$OUTPUT = nil
-
 def main
+  input = nil
+  output = nil
+  output_schema = 'nodes'
   parser = OptionParser.new do |opts|
     opts.summary_indent = '  '
     opts.summary_width = 20
@@ -74,24 +71,28 @@ def main
     opts.separator ''
     opts.separator 'Options:'
     opts.on('-i', '--input FILE', 'Input file name. Read from stdin if not given.') do |filename|
-      $INPUT = filename
+      input = filename
     end
     opts.on('-o', '--output FILE', 'Output file name. Write to stdout if not given.') do |filename|
-      $OUTPUT = filename
+      output = filename
     end
     opts.on('-h', '--help', 'Print this help and exit.') do
       $stdout.puts opts
       $stdout.puts %(
 Output is a YAML file with "name" used as "label" and "shape" used as is
-for nodes. For edges "tail" and "head" are used as the first and second element
-of "between" with the "name" as the identifier. Compound objects are ignored.
+for nodes. For edges "tail" and "head" are used as the first and second
+element of "between" with the "name" as the identifier. Compound objects
+are ignored.
+
+Output YAML file schema is returned by:
+diagrammatron-schema #{output_schema}
 )
       exit 0
     end
   end
   parser.parse! ARGV
 
-  doc = load_source
+  doc = load_source(input)
   exit(2) if doc.nil?
 
   begin
@@ -101,19 +102,7 @@ of "between" with the "name" as the identifier. Compound objects are ignored.
   end
   exit(3) if out.nil?
 
-  begin
-    if $OUTPUT.nil?
-      $stdout.puts YAML.dump(out)
-    else
-      fp = Pathname.new $OUTPUT
-      fp.open('w') do |f|
-        f.puts YAML.dump(out)
-      end
-    end
-  rescue StandardError => e
-    aargh("#{e}\nFailed to write output: #{$OUTPUT}")
-    exit 4
-  end
+  save_verified(output, pout, 4, output_schema)
 end
 
 main if (defined? $unit_test).nil?

data/lib/common.rb

@@ -1,8 +1,13 @@
 # frozen_string_literal: true
 
-# Copyright © 2021, 2022 Ismo Kärkkäinen
+# Copyright © 2021-2023 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
+require 'json_schemer'
+require 'yaml'
+require 'set'
+
+
 def aargh(message, return_value = nil)
   message = (message.map &(:to_s)).join("\n") if message.is_a? Array
   $stderr.puts message
@@ -20,9 +25,41 @@ end
 def load_source_hash(input)
   src = load_source(input)
   unless src.nil?
-    return aargh('Input is not a mapping.') unless src.is_a? Hash
+    return aargh("#{input} is not a simple mapping.") unless src.is_a? Hash
+  end
+  src
+end
+
+def list_schemas
+  (Dir.entries(File.dirname(__FILE__)).select { |name| name.upcase.end_with?('.YAML') }).map { |name| name[0, name.size - 5] }
+end
+
+def load_schema(schema_name)
+  YAML.safe_load_file(File.join(File.dirname(__FILE__), "#{schema_name}.yaml"))
+end
+
+def make_schemer(schema, reading)
+  JSONSchemer.schema(JSON.generate(schema),
+    meta_schema: 'https://json-schema.org/draft/2020-12/schema',
+    insert_property_defaults: reading)
+end
+
+def load_verified(input, schema_name)
+  src = load_source(input)
+  unless src.nil?
+    s = load_schema(schema_name)
+    schemer = make_schemer(s, true)
+    errs = schemer.validate(src).to_a
+    unless errs.empty?
+      aargh (errs.map { |e| e['error'] }).join("\n")
+      src = nil
+    end
   end
   src
+rescue Errno::ENOENT
+  aargh "Could not load #{schema_name}"
+rescue StandardError => e
+  aargh "Internal error: #{e}\n#{e.backtrace.join("\n")}\nFailed to read #{schema_name}"
 end
 
 def dump_result(output, doc, error_return)
@@ -38,3 +75,14 @@ def dump_result(output, doc, error_return)
 rescue StandardError => e
   aargh([ e, "Failed to write output: #{output || 'stdout'}" ], error_return)
 end
+
+def save_verified(output, doc, error_return, schema_name)
+  s = load_schema(schema_name)
+  schemer = make_schemer(s, false)
+  errs = schemer.validate(doc).to_a
+  unless errs.empty?
+    aargh (errs.map { |e| e['error'] }).join("\n")
+    return error_return
+  end
+  dump_result(output, YAML.dump(doc, line_width: 1_000_000), error_return)
+end