openapi-sourcetools 0.4.2 → 0.5.0

data/bin/openapi-merge

@@ -1,60 +1,14 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2021 Ismo Kärkkäinen
+# Copyright © 2021-2024 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
-require_relative '../lib/common.rb'
+require_relative '../lib/common'
 require 'optparse'
 require 'yaml'
-require 'json'
 require 'set'
 
-
-default_env(:out, '')
-default_env(:format, 'YAML')
-default_env(:prune, false);
-
-ENV['POSIXLY_CORRECT'] = '1'
-parser = OptionParser.new do |opts|
-  opts.summary_indent = '  '
-  opts.summary_width = 26
-  opts.banner = 'Usage: openapi-merge [options] sources...'
-  opts.separator ''
-  opts.separator 'Options (equivalent environment variable and value in parentheses):'
-  opts.on('-o', '--output FILE', 'Output result to FILE, not stdout (OUT=FILE).') do |f|
-    env(:out, f)
-  end
-  opts.on('-p', '--prune', 'Remove all unreferenced objects under components.') do
-    env(:prune, true)
-  end
-  opts.on('--yaml', 'Output format is YAML (default).') do
-    env(:format, 'YAML')
-  end
-  opts.on('--json', 'Output format is JSON.') do
-    env(:format, 'JSON')
-  end
-  opts.on('-h', '--help', 'Print this help and exit.') do
-    $stdout.puts %(#{opts}
-
-Source files are combined to form one API specification document. Sources are
-allowed only to append to the merged document, not re-define anything in it.
-)
-    exit 0
-  end
-end
-parser.parse!
-
-aargh("Format neither JSON nor YAML: #{env(:format)}", 1) unless %w[JSON YAML].include? env(:format)
-
-def read_source(filename)
-  YAML.safe_load(File.read(filename))
-rescue Errno::ENOENT => e
-  aargh("Could not read #{filename}", 2)
-rescue StandardError => e
-  aargh(e.to_s, 3)
-end
-
 def raise_se(message)
   raise StandardError, message
 end
@@ -74,12 +28,12 @@ end
 
 def add_undefined(merged, incoming, filename, path, max_depths)
   incoming.each_pair do |key, value|
-    unless merged.has_key? key
+    unless merged.key? key
       merged[key] = value
       next
     end
     m = merged[key]
-    raise_se "Path #{path_combo(path, false)} merged type #{m.class} differs from type #{value.class} in #{filename}" unless m.class == value.class
+    raise_se("Path #{path_combo(path, false)} merged type #{m.class} differs from type #{value.class} in #{filename}") unless m.instance_of?(value.class)
     raise_se("Re-definition of #{key} #{path_combo(path)} in #{filename}") if too_deep(path, max_depths)
     if m.is_a? Hash # paths or similar
       path.push key
@@ -94,22 +48,6 @@ def add_undefined(merged, incoming, filename, path, max_depths)
       raise_se "Re-definition of #{key} #{path_combo(path)} in #{filename}"
     end
   end
-rescue StandardError => e
-  aargh(e.to_s, 3)
-end
-
-max_depths = Hash.new(0)
-max_depths['openapi'] = 1
-max_depths['info'] = 1
-max_depths['servers'] = 1
-max_depths['paths'] = 2 # Allows get, post, options, etc. from different files.
-max_depths['webhooks'] = 2
-max_depths['components'] = 2
-max_depths['security'] = 1
-max_depths['tags'] = 1
-merged = {}
-ARGV.each do |filename|
-  add_undefined(merged, read_source(filename), filename, [], max_depths)
 end
 
 def gather_refs(doc, found)
@@ -126,11 +64,16 @@ def gather_refs(doc, found)
   end
 end
 
-if env(:prune)
+def prune(merged)
   prev_refs = Set.new
   loop do # May have references from deleted so repeat until nothing deleted.
     refs = Set.new
     gather_refs(merged, refs)
+    merged.fetch('security', []).each do |sec|
+      sec.each_key do |key|
+        refs.add("#/components/securitySchemes/#{key}")
+      end
+    end
     used = {}
     all = merged.fetch('components', {})
     refs.each do |ref|
@@ -145,7 +88,7 @@ if env(:prune)
       sub = used
       p.each_index do |k|
         if k + 1 < p.size
-          sub[p[k]] = {} unless sub.has_key? p[k]
+          sub[p[k]] = {} unless sub.key? p[k]
           sub = sub[p[k]]
         else
           sub[p[k]] = item
@@ -158,19 +101,58 @@ if env(:prune)
   end
 end
 
-output = env(:out)
-if output.empty?
-  output = $stdout
-else
-  begin
-    output = File.open(output, 'w')
-  rescue StandardError
-    aargh("Failed to open for writing: #{output}", 1)
+def main
+  output_file = nil
+  keep = false
+
+  ENV['POSIXLY_CORRECT'] = '1'
+  parser = OptionParser.new do |opts|
+    opts.summary_indent = '  '
+    opts.summary_width = 26
+    opts.banner = 'Usage: openapi-merge [options] sources...'
+    opts.separator ''
+    opts.separator 'Options:'
+    opts.on('-o', '--output FILE', 'Output result to FILE, not stdout.') do |f|
+      output_file = f
+    end
+    opts.on('-k', '--keep', 'Keep all unreferenced objects under components.') do
+      keep = true
+    end
+    opts.on('-p', '--prune', 'Prune all unreferenced objects under components (default).') do
+      keep = false
+    end
+    opts.on('-h', '--help', 'Print this help and exit.') do
+      $stdout.puts %(#{opts}
+
+Source files are combined to form one API specification document. Sources are
+allowed only to append to the merged document, not re-define anything in it.
+)
+      exit 0
+    end
+  end
+  parser.parse!
+
+  max_depths = Hash.new(0)
+  max_depths['openapi'] = 1
+  max_depths['info'] = 1
+  max_depths['servers'] = 1
+  max_depths['paths'] = 2 # Allows get, post, etc. from different files.
+  max_depths['webhooks'] = 2
+  max_depths['components'] = 2
+  max_depths['security'] = 1
+  max_depths['tags'] = 1
+  merged = {}
+  ARGV.each do |filename|
+    s = load_source(filename)
+    return 2 if s.nil?
+    add_undefined(merged, s, filename, [], max_depths)
+  rescue StandardError => e
+    return aargh(e.to_s, 4)
   end
-end
 
-case env(:format)
-when 'JSON' then output.puts JSON.generate(merged)
-when 'YAML' then output.puts YAML.dump(merged)
+  prune(merged) unless keep
+
+  dump_result(output_file, YAML.dump(merged), 3)
 end
-output.close
+
+exit(main) if (defined? $unit_test).nil?

data/bin/openapi-processpaths

@@ -1,111 +1,84 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2021 Ismo Kärkkäinen
+# Copyright © 2021-2024 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
-require_relative '../lib/common.rb'
+require_relative '../lib/common'
 require 'optparse'
 require 'yaml'
-require 'json'
 
 
-default_env(:in, '')
-default_env(:out, '')
-default_env(:format, 'YAML')
-default_env(:error, true)
+def main
+  input_name = nil
+  output_name = nil
+  error = true
 
-parser = OptionParser.new do |opts|
-  opts.summary_indent = '  '
-  opts.summary_width = 24
-  opts.banner = 'Usage: openapi-processpaths [options]'
-  opts.separator ''
-  opts.separator 'Options (equivalent environment variable and value in parentheses):'
-  opts.on('-i', '--input FILE', 'Read source from FILE, not stdin (IN=FILE).') do |f|
-    env(:in, f)
-  end
-  opts.on('-o', '--output FILE', 'Output result to FILE, not stdout (OUT=FILE).') do |f|
-    env(:out, f)
-  end
-  opts.on('--warn', 'Only warn of paths with same fixed parts (ERROR=0).') do
-    env(:error, false)
-  end
-  opts.on('--yaml', 'Output format is YAML (default) (FORMAT=YAML).') do
-    env(:format, 'YAML')
-  end
-  opts.on('--json', 'Output format is JSON. (FORMAT=JSON)') do
-    env(:format, 'JSON')
-  end
-  opts.on('-h', '--help', 'Print this help and exit.') do
-    $stdout.puts %(#{opts}
+  parser = OptionParser.new do |opts|
+    opts.summary_indent = '  '
+    opts.summary_width = 24
+    opts.banner = 'Usage: openapi-processpaths [options]'
+    opts.separator ''
+    opts.separator 'Options:'
+    opts.on('-i', '--input FILE', 'Read source from FILE, not stdin.') do |f|
+      input_name = f
+    end
+    opts.on('-o', '--output FILE', 'Output result to FILE, not stdout.') do |f|
+      output_name = f
+    end
+    opts.on('--warn', 'Only warn of paths with same fixed parts.') do
+      error = false
+    end
+    opts.on('-h', '--help', 'Print this help and exit.') do
+      $stdout.puts %(#{opts}
 
 Processes API specification document path objects into form that is expected by
 later stage tools. Checks for paths that may be ambiguous.
 )
-    exit 0
+      exit 0
+    end
   end
-end
-parser.parse!
+  parser.parse!
 
-aargh("Format neither JSON nor YAML: #{env(:format)}", 1) unless %w[JSON YAML].include? env(:format)
-
-def read_source(filename)
-  YAML.safe_load(filename.empty? ? $stdin : File.read(filename))
-rescue Errno::ENOENT => e
-  aargh("Could not read #{filename.empty? ? 'stdin' : filename}", 2)
-rescue StandardError => e
-  aargh(e.to_s, 3)
-end
-doc = read_source(env(:in))
+  doc = load_source(input_name)
+  return 2 if doc.nil?
 
-processed = {}
-doc.fetch('paths', {}).each_pair do |path, value|
-  parts = split_path(path, true)
-  processed[path] = {
-    'parts' => parts,
-    'orig' => value,
-    'lookalike' => [],
-    path: ServerPath.new(parts)
-  }
-end
-
-# Find lookalike sets.
-lookalikes = false
-paths = processed.keys.sort
-paths.each_index do |k|
-  pk = paths[k]
-  a = processed[pk]
-  (0...k).each do |n|
-    pn = paths[n]
-    b = processed[pn]
-    next unless (a[:path].compare(b[:path])) == 0
-    a['lookalike'].push pn
-    b['lookalike'].push pk
-    $stderr.puts("Similar: #{pn} #{pk}")
-    lookalikes = true
+  processed = {}
+  doc.fetch('paths', {}).each_pair do |path, value|
+    parts = split_path(path, true)
+    processed[path] = {
+      'parts' => parts,
+      'orig' => value,
+      'lookalike' => [],
+      path: ServerPath.new(parts)
+    }
   end
-end
-aargh('Similar paths found.', 7) if lookalikes && env(:error)
 
-# Clean-up temporary fields.
-processed.each_value do |v|
-  v.keys.each { |k| v.delete(k) if k.is_a? Symbol }
-end
-doc['paths'] = processed
+  # Find lookalike sets.
+  lookalikes = false
+  paths = processed.keys.sort!
+  paths.size.times do |k|
+    pk = paths[k]
+    a = processed[pk]
+    k.times do |n|
+      pn = paths[n]
+      b = processed[pn]
+      next unless a[:path].compare(b[:path]).zero?
+      a['lookalike'].push pn
+      b['lookalike'].push pk
+      $stderr.puts("Similar: #{pn} #{pk}")
+      lookalikes = true
+    end
+  end
+  return aargh('Similar paths found.', 4) if lookalikes && error
 
-output = env(:out)
-if output.empty?
-  output = $stdout
-else
-  begin
-    output = File.open(output, 'w')
-  rescue StandardError
-    aargh("Failed to open for writing: #{output}", 6)
+  # Remove temporary fields.
+  processed.each_value do |v|
+    v.keys.each { |k| v.delete(k) if k.is_a? Symbol }
   end
-end
+  doc['paths'] = processed
 
-case env(:format)
-when 'JSON' then output.puts JSON.generate(doc)
-when 'YAML' then output.puts YAML.dump(doc)
+  dump_result(output_name, YAML.dump(doc), 3)
 end
-output.close
+
+exit(main) if (defined? $unit_test).nil?

data/lib/apiobjects.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+require_relative './common'
+require 'set'
+
+def same(a, b, ignored_keys = Set.new(%w[summary description]))
+  return a == b unless a.is_a?(Hash) && b.is_a?(Hash)
+  keys = Set.new(a.keys + b.keys) - ignored_keys
+  keys.to_a.each do |k|
+    return false unless a.key?(k) && b.key?(k)
+    return false unless same(a[k], b[k], ignored_keys)
+  end
+  true
+end
+
+def ref_string(name, schema_path)
+  "#{schema_path}/#{name}"
+end
+
+def reference(obj, schemas, schema_path, ignored_keys = Set.new(%w[summary description]), prefix = 'Schema')
+  # Check if identical schema has been added and if so, return the $ref string.
+  schemas.keys.sort.each do |k|
+    return ref_string(k, schema_path) if same(obj, schemas[k], ignored_keys)
+  end
+  # One of the numbers will not match existing keys. More number than keys.
+  (schemas.size + 1).times do |n|
+    # 'x' is to simplify find and replace (Schema1x vs Schema1 and Schema10)
+    k = "#{prefix}#{n}x"
+    next if schemas.key?(k)
+    schemas[k] = obj.merge
+    return ref_string(k, schema_path)
+  end
+end
+
+class Components
+  attr_reader :path, :prefix
+  attr_accessor :items, :ignored_keys
+
+  def initialize(path, prefix, ignored_keys = %w[summary description examples example])
+    @items = {}
+    path = "#/#{path.join('/')}/" if path.is_a?(Array)
+    path = "#{path}/" unless path.end_with?('/')
+    @path = path
+    @prefix = prefix
+    @ignored_keys = Set.new(ignored_keys)
+  end
+
+  def add_options(opts)
+    opts.on('--use FIELD', 'Use FIELD in comparisons.') do |f|
+      @ignored_keys.delete(f)
+    end
+    opts.on('--ignore FIELD', 'Ignore FIELD in comparisons.') do |f|
+      @ignored_keys.add(f)
+    end
+  end
+
+  def help
+    %(All fields are used in object equality comparisons except:\n#{@ignored_keys.to_a.sort!.join("\n")})
+  end
+
+  def ref_string(name)
+    "#{@path}#{name}"
+  end
+
+  def reference(obj)
+    # Check if identical schema has been added. If so, return the $ref string.
+    @items.each do |k, v|
+      return ref_string(k) if same(obj, v, @ignored_keys)
+    end
+    # One of the numbers will not match existing keys. More number than keys.
+    (@items.size + 1).times do |n|
+      # 'x' is to simplify find and replace (Schema1x vs Schema1 and Schema10)
+      cand = "#{@prefix}#{n}x"
+      next if @items.key?(cand)
+      @items[cand] = obj.merge
+      return ref_string(cand)
+    end
+  end
+end
+
+
+class PathOperation
+  attr_accessor :path, :operation, :info, :parameters
+  attr_accessor :servers, :security, :tags
+  attr_accessor :summary, :description
+end
+
+# Should mapping JSON schema types to native types be a separate step?
+# Just add x-openapi-sourcetools-native: nativetype
+# Separate program that can pipe modified spec out to code generation?
+
+# JSON schema is way too complex to simplify here. One could have a convenience
+# method that determines how many bytes the value needs, and if it needs to be
+# signed.
+
+# When creating types for schemas or otherwise, the type name can be added
+# into the item and that way be used as an indicator that the type has been
+# declared or needs a declaration.
+
+def make_path_operations(apidoc)
+  # Check openapi
+  # Store info as is for reference
+  # Store servers as is for default value for PathOperation
+  # Process components. Lazy manner, only when referenced.
+  # Store security as is for default value for PathOperation.
+  # Store tags as mapping from name to object for use with PathOperation.
+  # Process paths:
+  # Store parameters as is for default value for PathOperation.
+  # All other fields, check if it looks like OperationObject and create a
+  # PathOperation using it. For others, store as is for default value.
+
+end
+
+
+class ServerPath
+  include Comparable
+
+  attr_accessor :parts
+
+  def initialize(parts)
+    @parts = parts
+  end
+
+  def <=>(other) # Variables are after fixed strings.
+    pp = other.is_a?(Array) ? other : p.parts
+    parts.each_index do |k|
+      return 1 if pp.size <= k # Longer comes after shorter.
+      pk = parts[k]
+      ppk = pp[k]
+      if pk.is_a? String
+        if ppk.is_a? String
+          c = pk <=> ppk
+        else
+          return -1
+        end
+      else
+        if ppk.is_a? String
+          return 1
+        else
+          c = pk.fetch('var', '') <=> ppk.fetch('var', '')
+        end
+      end
+      return c unless c.zero?
+    end
+    (parts.size < pp.size) ? -1 : 0
+  end
+
+  def compare(p, range = nil) # Not fit for sorting. Variable equals anything.
+    pp = p.is_a?(Array) ? p : p.parts
+    if range.nil?
+      range = 0...parts.size
+    elsif range.is_a? Number
+      range = range...(range + 1)
+    end
+    range.each do |k|
+      return 1 if pp.size <= k # Longer comes after shorter.
+      ppk = pp[k]
+      next unless ppk.is_a? String
+      pk = parts[k]
+      next unless pk.is_a? String
+      c = pk <=> ppk
+      return c unless c.zero?
+    end
+    (parts.size < pp.size) ? -1 : 0
+  end
+end
+
+# Adds all refs found in the array to refs with given required state.
+def gather_array_refs(refs, items, required)
+  items.each do |s|
+    r = s['$ref']
+    next if r.nil?
+    refs[r] = required || refs.fetch(r, false)
+  end
+end
+
+# For any key '$ref' adds to refs whether referred type is required.
+# Requires that there are no in-lined schemas, openapi-addschemas has been run.
+def gather_refs(refs, schema)
+  # This implies types mixed together according to examples. Needs mixed type.
+  # AND. Also, mixing may fail. Adds a new schema, do here.
+  items = schema['allOf']
+  return gather_array_refs(refs, items, true) unless items.nil?
+  # As long as one schema is fulfilled, it is ok. OR, first that fits.
+  items = schema['anyOf'] if items.nil?
+  # oneOf implies selection between different types. No multiple matches. XOR.
+  # Needs to ensure that later types do not match.
+  # Should check if there is enough difference to ensure single match.
+  # Use separate program run after addschemas to create allOf mixed schema
+  # and verify the others can be dealt with.
+  items = schema['oneOf'] if items.nil?
+  return gather_array_refs(refs, items, false) unless items.nil?
+  # Defaults below handle it if "type" is not "object".
+  reqs = schema.fetch('required', [])
+  schema.fetch('properties', {}).each do |name, spec|
+    r = spec['$ref']
+    next if r.nil?
+    refs[r] = reqs.include?(name) || refs.fetch(r, false)
+  end
+end
+
+class SchemaInfo
+  attr_accessor :ref, :schema, :direct_refs, :name, :post_refs
+
+  def initialize(ref, name, schema)
+    @ref = ref
+    @name = name
+    @schema = schema
+    @direct_refs = {}
+    gather_refs(@direct_refs, schema)
+  end
+
+  def set_post_refs(seen)
+    @post_refs = Set.new(@direct_refs.keys) - seen
+  end
+
+  def to_s
+    v = @direct_refs.keys.sort.map { |k| "#{k}:#{@direct_refs[k] ? 'req' : 'opt'}" }
+    "#{@ref}: #{v.join(' ')}"
+  end
+end
+
+def var_or_method_value(x, name)
+  if name.start_with?('@')
+    n = name
+  else
+    n = "@#{name}"
+  end
+  return x.instance_variable_get(n) if x.instance_variable_defined?(n)
+  return x.public_send(name) if x.respond_to?(name)
+  raise ArgumentError, "#{name} is not #{x.class} instance variable nor public method"
+end
+
+class SchemaOrderer
+  attr_accessor :schemas, :order, :orderer
+
+  def initialize(path, schema_specs)
+    @schemas = {}
+    schema_specs.each do |name, schema|
+      r = "#{path}#{name}"
+      @schemas[r] = SchemaInfo.new(r, name, schema)
+    end
+  end
+
+  def sort!(orderer = 'required_first')
+    case orderer
+    when 'required_first' then @order = required_first
+    when '<=>' then @order = @schemas.values.sort { |a, b| a <=> b }
+    else
+      @order = @schemas.values.sort do |a, b|
+        va = var_or_method_value(a, orderer)
+        vb = var_or_method_value(b, orderer)
+        va <=> vb
+      end
+    end
+    @orderer = orderer
+    seen = Set.new
+    @order.each do |si|
+      si.set_post_refs(seen)
+      seen.add(si.ref)
+    end
+    @order
+  end
+
+  def required_first
+    chosen = []
+    until chosen.size == @schemas.size
+      used = Set.new(chosen.map { |si| si.ref })
+      avail = @schemas.values.select { |si| !used.member?(si.ref) }
+      best = nil
+      avail.each do |si|
+        prereq = chosen.count { |x| x.direct_refs.fetch(si.ref, false) }
+        fulfilled = chosen.count { |x| si.direct_refs.fetch(x.ref, false) }
+        postreq = si.direct_refs.size - (prereq + fulfilled)
+        better = false
+        if best.nil?
+          better = true
+        else
+          # Minimize preceding types requiring this.
+          if prereq < best.first
+            better = true
+          elsif prereq == best.first
+            # Minimize remaining unfulfilled requires.
+            if postreq < best[1]
+              better = true
+            elsif postreq == best[1]
+              # Check mutual direct requirements.
+              best_req_si = best.last.direct_refs.fetch(si.ref, false)
+              si_req_best = si.direct_refs.fetch(best.last.ref, false)
+              if best_req_si
+                better = true unless si_req_best
+              end
+              # Order by name if no other difference.
+              better = si.ref < best.last.ref unless better
+            end
+          end
+        end
+        best = [ prereq, postreq, si ] if better
+      end
+      chosen.push(best.last)
+    end
+    chosen
+  end
+end