openapi-sourcetools 0.7.0 → 0.8.0

data/bin/openapi-merge

@@ -1,13 +1,13 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2021-2024 Ismo Kärkkäinen
+# Copyright © 2021-2025 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
-require_relative '../lib/common'
+require_relative '../lib/openapi/sourcetools/common'
 require 'optparse'
-require 'yaml'
 require 'set'
+include OpenAPISourceTools
 
 def raise_se(message)
   raise StandardError, message
@@ -52,7 +52,7 @@ end
 
 def gather_refs(doc, found)
   doc.each_pair do |key, value|
-    if key == '$ref' && value.is_a?(String) && value.start_with?('#/components/')
+    if key == '$ref' # Trust all refs to be valid.
       found.add(value)
     elsif value.is_a? Hash
       gather_refs(value, found)
@@ -64,6 +64,13 @@ def gather_refs(doc, found)
   end
 end
 
+def has_refd_anchor(item, refs)
+  return !((item.index { |v| has_refd_anchor(v, refs) }).nil?) if item.is_a?(Array)
+  return false unless item.is_a?(Hash)
+  return true if refs.member?('#' + item.fetch('$anchor', ''))
+  !((item.values.index { |v| has_refd_anchor(v, refs) }).nil?)
+end
+
 def prune(merged)
   prev_refs = Set.new
   loop do # May have references from deleted so repeat until nothing deleted.
@@ -74,16 +81,16 @@ def prune(merged)
         refs.add("#/components/securitySchemes/#{key}")
       end
     end
+    # Add schema ref for all schemas that have referenced anchor somewhere.
+    (merged.dig(*%w[components schemas]) || {}).each do |name, schema|
+      refs.add("#/components/Schemas/#{name}") if has_refd_anchor(schema, refs)
+    end
     used = {}
     all = merged.fetch('components', {})
     refs.each do |ref|
       p = ref.split('/')
       p.shift(2)
-      item = all
-      p.each do |key|
-        item = item.fetch(key, nil)
-        break if item.nil?
-      end
+      item = all.dig(*p)
       next if item.nil?
       sub = used
       p.each_index do |k|
@@ -105,7 +112,6 @@ def main
   output_file = nil
   keep = false
 
-  ENV['POSIXLY_CORRECT'] = '1'
   parser = OptionParser.new do |opts|
     opts.summary_indent = '  '
     opts.summary_width = 26
@@ -130,7 +136,7 @@ allowed only to append to the merged document, not re-define anything in it.
       exit 0
     end
   end
-  parser.parse!
+  parser.order!
 
   max_depths = Hash.new(0)
   max_depths['openapi'] = 1
@@ -143,16 +149,16 @@ allowed only to append to the merged document, not re-define anything in it.
   max_depths['tags'] = 1
   merged = {}
   ARGV.each do |filename|
-    s = load_source(filename)
+    s = Common.load_source(filename)
     return 2 if s.nil?
     add_undefined(merged, s, filename, [], max_depths)
   rescue StandardError => e
-    return aargh(e.to_s, 4)
+    return Common.aargh(e.to_s, 4)
   end
 
   prune(merged) unless keep
 
-  dump_result(output_file, YAML.dump(merged), 3)
+  Common.dump_result(output_file, merged, 3)
 end
 
-exit(main) if (defined? $unit_test).nil?
+exit(main) if defined?($unit_test).nil?

data/bin/openapi-modifypaths

@@ -1,15 +1,16 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2024 Ismo Kärkkäinen
+# Copyright © 2024-2025 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
-require_relative '../lib/common'
+require_relative '../lib/openapi/sourcetools/common'
 require 'optparse'
-require 'yaml'
+include OpenAPISourceTools
+
 
 def path2pieces(s)
-  s.split('/').reject { |p| p.empty? }
+  s.split('/').reject &:empty?
 end
 
 def pieces2path(p)
@@ -39,18 +40,18 @@ end
 
 def add_op(s)
   s = path2pieces(s)
-  Proc.new { |path| add(s, path) }
+  proc { |path| add(s, path) }
 end
 
 def remove_op(s)
   s = path2pieces(s)
-  Proc.new { |path| remove(s, path) }
+  proc { |path| remove(s, path) }
 end
 
 def replace_op(orig, s)
   o = path2pieces(orig)
   s = path2pieces(s)
-  Proc.new { |path| replace(o, s, path) }
+  proc { |path| replace(o, s, path) }
 end
 
 def perform_operations(paths, operations)
@@ -77,23 +78,23 @@ def main
     opts.separator ''
     opts.separator 'Options:'
     opts.on('-i', '--input FILE', 'Read API spec from FILE, not stdin.') do |f|
-      exit(aargh("Expected string to replace PREFIX.", 1)) unless orig.nil?
+      exit(Common.aargh('Expected string to replace PREFIX.', 1)) unless orig.nil?
       input_name = f
     end
     opts.on('-o', '--output FILE', 'Output to FILE, not stdout.') do |f|
-      exit(aargh("Expected string to replace PREFIX.", 1)) unless orig.nil?
+      exit(Common.aargh('Expected string to replace PREFIX.', 1)) unless orig.nil?
       output_name = f
     end
     opts.on('-a', '--add STR', 'Add prefix STR to all paths.') do |s|
-      exit(aargh("Expected string to replace PREFIX.", 1)) unless orig.nil?
+      exit(Common.aargh('Expected string to replace PREFIX.', 1)) unless orig.nil?
       operations.push(add_op(s))
     end
     opts.on('-d', '--delete PREFIX', 'Delete PREFIX when present.') do |s|
-      exit(aargh("Expected string to replace PREFIX.", 1)) unless orig.nil?
+      exit(Common.aargh('Expected string to replace PREFIX.', 1)) unless orig.nil?
       operations.push(remove_op(s))
     end
     opts.on('-r', '--replace PREFIX STR', 'Replace PREFIX with STR when present.') do |s|
-      exit(aargh('Empty string to replace.', 1)) if s.empty?
+      exit(Common.aargh('Empty string to replace.', 1)) if s.empty?
       orig = s
     end
     opts.on('-h', '--help', 'Print this help and exit.') do
@@ -106,18 +107,18 @@ STR and PREFIX are expected to be parts of a path surrounded by /.
     end
   end
   parser.order! do |s|
-    exit(aargh("String without option: #{s}", 1)) if orig.nil?
+    exit(Common.aargh("String without option: #{s}", 1)) if orig.nil?
     operations.push(replace_op(orig, s))
     orig = nil
   end
 
-  doc = load_source(input_name)
+  doc = Common.load_source(input_name)
   return 2 if doc.nil?
 
   p = perform_operations(doc.fetch('paths', {}), operations)
   doc['paths'] = p unless p.empty?
 
-  dump_result(output_name, doc, 3)
+  Common.dump_result(output_name, doc, 3)
 end
 
-main if defined?($unit_test).nil?
+exit(main) if defined?($unit_test).nil?

data/bin/openapi-processpaths

@@ -1,12 +1,13 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2021-2024 Ismo Kärkkäinen
+# Copyright © 2021-2025 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
-require_relative '../lib/common'
+require_relative '../lib/openapi/sourcetools/apiobjects'
+require_relative '../lib/openapi/sourcetools/common'
 require 'optparse'
-require 'yaml'
+include OpenAPISourceTools
 
 
 def main
@@ -32,26 +33,22 @@ def main
     opts.on('-h', '--help', 'Print this help and exit.') do
       $stdout.puts %(#{opts}
 
-Processes API specification document path objects into form that is expected by
-later stage tools. Checks for paths that may be ambiguous.
+Adds split path parts into API document path items under x-openapi-sourcetools
+key. Checks for paths that may be ambiguous.
 )
       exit 0
     end
   end
-  parser.parse!
+  parser.order!
 
-  doc = load_source(input_name)
+  doc = Common.load_source(input_name)
   return 2 if doc.nil?
 
   processed = {}
-  doc.fetch('paths', {}).each_pair do |path, value|
-    parts = split_path(path, true)
-    processed[path] = {
-      'parts' => parts,
-      'orig' => value,
-      'lookalike' => [],
-      path: ServerPath.new(parts)
-    }
+  doc.fetch('paths', {}).each do |path, item|
+    parts = Common.split_path(path, true)
+    item['x-openapi-sourcetools-parts'] = parts # Added to original path item.
+    processed[path] = ApiObjects::ServerPath.new(parts)
   end
 
   # Find lookalike sets.
@@ -63,22 +60,14 @@ later stage tools. Checks for paths that may be ambiguous.
     k.times do |n|
       pn = paths[n]
       b = processed[pn]
-      next unless a[:path].compare(b[:path]).zero?
-      a['lookalike'].push pn
-      b['lookalike'].push pk
+      next unless a.compare(b).zero?
       $stderr.puts("Similar: #{pn} #{pk}")
       lookalikes = true
     end
   end
-  return aargh('Similar paths found.', 4) if lookalikes && error
+  return Common.aargh('Similar paths found.', 4) if lookalikes && error
 
-  # Remove temporary fields.
-  processed.each_value do |v|
-    v.keys.each { |k| v.delete(k) if k.is_a? Symbol }
-  end
-  doc['paths'] = processed
-
-  dump_result(output_name, YAML.dump(doc), 3)
+  Common.dump_result(output_name, doc, 3)
 end
 
-exit(main) if (defined? $unit_test).nil?
+exit(main) unless defined?($unit_test)

data/lib/openapi/sourcetools/apiobjects.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+# Copyright © 2024-2025 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+module OpenAPISourceTools
+  # Various classes for handling objects in the API specification.
+  # Used in various programs.
+  module ApiObjects
+
+    def self.same(a, b, ignored_keys = Set.new(%w[summary description]))
+      return a == b unless a.is_a?(Hash) && b.is_a?(Hash)
+      keys = Set.new(a.keys + b.keys) - ignored_keys
+      keys.to_a.each do |k|
+        return false unless a.key?(k) && b.key?(k)
+        return false unless same(a[k], b[k], ignored_keys)
+      end
+      true
+    end
+
+    def self.ref_string(name, schema_path)
+      "#{schema_path}/#{name}"
+    end
+
+    def self.reference(obj, schemas, schema_path, ignored_keys = Set.new(%w[summary description]), prefix = 'Schema')
+      # Check if identical schema has been added and if so, return the $ref string.
+      schemas.keys.sort.each do |k|
+        return ref_string(k, schema_path) if same(obj, schemas[k], ignored_keys)
+      end
+      # One of the numbers will not match existing keys. More number than keys.
+      (schemas.size + 1).times do |n|
+        # 'x' is to simplify find and replace (Schema1x vs Schema1 and Schema10)
+        k = "#{prefix}#{n}x"
+        next if schemas.key?(k)
+        schemas[k] = obj.merge
+        return ref_string(k, schema_path)
+      end
+    end
+
+    # A component in the API specification for reference and anchor handling.
+    class Components
+      attr_reader :path, :prefix, :anchor2ref, :schema_names
+      attr_accessor :items, :ignored_keys
+
+      def initialize(path, prefix, ignored_keys = %w[summary description examples example $anchor])
+        path = "#/#{path.join('/')}/" if path.is_a?(Array)
+        path = "#{path}/" unless path.end_with?('/')
+        @path = path
+        @prefix = prefix
+        @anchor2ref = {}
+        @schema_names = Set.new
+        @items = {}
+        @ignored_keys = Set.new(ignored_keys)
+      end
+
+      def add_options(opts)
+        opts.on('--use FIELD', 'Use FIELD in comparisons.') do |f|
+          @ignored_keys.delete(f)
+        end
+        opts.on('--ignore FIELD', 'Ignore FIELD in comparisons.') do |f|
+          @ignored_keys.add(f)
+        end
+      end
+
+      def help
+        %(All fields are used in object equality comparisons except:\n#{@ignored_keys.to_a.sort!.join("\n")})
+      end
+
+      def add_schema_name(name)
+        @schema_names.add(name)
+      end
+
+      def ref_string(name)
+        return nil if name.nil?
+        "#{@path}#{name}"
+      end
+
+      def reference(obj)
+        # Check if identical schema has been added. If so, return the $ref string.
+        @items.each do |k, v|
+          return ref_string(k) if ApiObjects.same(obj, v, @ignored_keys)
+        end
+        # One of the numbers will not match existing keys. More number than keys.
+        (@items.size + 1).times do |n|
+          # 'x' is to simplify find and replace (Schema1x vs Schema1 and Schema10)
+          cand = "#{@prefix}#{n}x"
+          next if @items.key?(cand)
+          @items[cand] = obj.merge
+          @schema_names.add(cand)
+          return ref_string(cand)
+        end
+      end
+
+      def store_anchor(obj, ref = nil)
+        anchor_name = obj['$anchor']
+        return if anchor_name.nil?
+        ref = obj['$ref'] if ref.nil?
+        raise StandardError, 'ref is nil and no $ref or it is nil' if ref.nil?
+        @anchor2ref[anchor_name] = ref
+      end
+
+      def alter_anchors
+        replacements = {}
+        @anchor2ref.each_key do |a|
+          next if @schema_names.member?(a)
+          replacements[a] = ref_string(a)
+          @schema_names.add(a)
+        end
+        replacements.each do |a, r|
+          @anchor2ref[a] = r
+        end
+      end
+
+      def anchor_ref_replacement(ref)
+        @anchor2ref[ref[1...ref.size]] || ref
+      end
+    end
+
+    # Represents path with fixed parts and variables.
+    class ServerPath
+      # Probably moves to a separate file once processpaths and frequencies receive
+      # some attention.
+      include Comparable
+
+      attr_accessor :parts
+
+      def initialize(parts)
+        @parts = parts
+      end
+
+      # Parameters are after fixed strings.
+      def <=>(other)
+        pp = other.is_a?(Array) ? other : other.parts
+        @parts.size.times do |k|
+          return 1 if pp.size <= k # Longer comes after shorter.
+          pk = @parts[k]
+          ppk = pp[k]
+          if pk.key?('fixed')
+            if ppk.key?('fixed')
+              c = pk['fixed'] <=> ppk['fixed']
+            else
+              return -1
+            end
+          else
+            if ppk.key?('fixed')
+              return 1
+            else
+              c = pk.fetch('parameter', '') <=> ppk.fetch('parameter', '')
+            end
+          end
+          return c unless c.zero?
+        end
+        (@parts.size < pp.size) ? -1 : 0
+      end
+
+      # Not fit for sorting. Variable equals anything.
+      def compare(other, range = nil)
+        pp = other.is_a?(Array) ? other : other.parts
+        if range.nil?
+          range = 0...@parts.size
+        elsif range.is_a? Number
+          range = range...(range + 1)
+        end
+        range.each do |k|
+          return 1 if pp.size <= k # Longer comes after shorter.
+          ppk = pp[k]
+          next unless ppk.key?('fixed')
+          pk = parts[k]
+          next unless pk.key?('fixed')
+          c = pk['fixed'] <=> ppk['fixed']
+          return c unless c.zero?
+        end
+        (@parts.size < pp.size) ? -1 : 0
+      end
+    end
+
+    def self.operation_objects(path_item)
+      keys = %w[operationId requestBody responses callbacks]
+      out = {}
+      path_item.each do |method, op|
+        next unless op.is_a?(Hash)
+        keys.each do |key|
+          next unless op.key?(key)
+          out[method] = op
+          break
+        end
+      end
+      out
+    end
+  end
+end

data/lib/openapi/sourcetools/common.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# Copyright © 2021-2025 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+require 'pathname'
+require 'yaml'
+
+module OpenAPISourceTools
+  # Common methods used in programs and elsewhere gathered into one place.
+  module Common
+    def self.aargh(message, return_value = nil)
+      message = message.map(&:to_s).join("\n") if message.is_a? Array
+      $stderr.puts message
+      return_value
+    end
+
+    def self.yesno(boolean)
+      boolean ? 'yes' : 'no'
+    end
+
+    def self.bury(doc, path, value)
+      (path.size - 1).times do |k|
+        p = path[k]
+        doc[p] = {} unless doc.key?(p)
+        doc = doc[p]
+      end
+      doc[path.last] = value
+    end
+
+    module Out
+      attr_reader :count
+      module_function :count
+      attr_accessor :quiet
+      module_function :quiet
+      module_function :quiet=
+
+      def self.put(message)
+        Common.aargh(message) unless @quiet
+        @count = @count.nil? ? 1 : @count + 1
+      end
+    end
+
+    def self.split_path(p, spec = false)
+      parts = []
+      p = p.strip
+      unless spec
+        q = p.index('?')
+        p.slice!(0...q) unless q.nil?
+      end
+      p.split('/').each do |s|
+        next if s.empty?
+        s = { (spec && s.include?('{') ? 'parameter' : 'fixed') => s }
+        parts.push(s)
+      end
+      parts
+    end
+
+    def self.load_source(input)
+      YAML.safe_load(input.nil? ? $stdin : File.read(input))
+    rescue Errno::ENOENT
+      aargh "Could not load #{input || 'stdin'}"
+    rescue StandardError => e
+      aargh "#{e}\nFailed to read #{input || 'stdin'}"
+    end
+
+    def self.dump_result(output, doc, error_return)
+      doc = YAML.dump(doc, line_width: 1_000_000) unless doc.is_a?(String)
+      if output.nil?
+        $stdout.puts doc
+      else
+        fp = Pathname.new output
+        fp.open('w') do |f|
+          f.puts doc
+        end
+      end
+      0
+    rescue StandardError => e
+      aargh([ e, "Failed to write output: #{output || 'stdout'}" ], error_return)
+    end
+  end
+end

data/lib/openapi/sourcetools/config.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+# Copyright © 2024-2025 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+require_relative 'task'
+require_relative 'gen'
+require 'find'
+require 'yaml'
+
+module OpenAPISourceTools
+  # Configuration file find and load convenience functions.
+  # See the first 3 methods. The rest are intended to be internal helpers.
+  module ConfigLoader
+
+    # A function to find all files with a given prefix.
+    # Prefix is taken from Gen.config if nil.
+    # Returns an array of ConfigFileInfo objects.
+    def self.find_files(name_prefix:, extensions: [ '.*' ])
+      name_prefix = Gen.config if name_prefix.nil?
+      raise ArgumentError, 'name_prefix or config must be set' if name_prefix.nil?
+      root, name_prefix = prepare_prefix(name_prefix, Gen.wd)
+      file_paths = find_filenames(root, name_prefix)
+      splitter = path_splitter(Gen.separator)
+      out = file_paths.map { |fp| convert_path_end(fp, splitter, root.size + 1, extensions) }
+      out.sort!
+    end
+
+    # A function to read all YAML files in an array of ConfigFileInfo objects.
+    # Returns the same as contents_array.
+    def self.read_contents(config_file_infos)
+      config_file_infos.each do |cfi|
+        c = YAML.safe_load_file(cfi.path)
+        # Check allows e.g. copyright and license files be named with config prefix
+        # for clarity, but ignored during config loading.
+        next if cfi.keys.empty? && !c.is_a?(Hash)
+        cfi.bury_content(c)
+      rescue Psych::SyntaxError
+        next # Was not YAML. Other files can be named using config prefix.
+      end
+      contents_array(config_file_infos)
+    end
+
+    # Maps an array of ConfigFileInfo objects to an array of their contents.
+    def self.contents_array(config_file_infos)
+      config_file_infos.map(&:content).reject(&:nil?)
+    end
+
+    class ConfigFileInfo
+      include Comparable
+
+      attr_reader :root, :keys, :path, :content
+
+      def initialize(pieces, path)
+        @keys = []
+        @root = nil
+        pieces.each do |p|
+          if p.is_a?(String)
+            if @root.nil?
+              @root = p
+            else
+              @keys.push(p)
+            end
+          else
+            break if p == :extension
+          end
+        end
+        @path = path
+        @content = nil
+      end
+
+      def bury_content(content)
+        # Turns chain of keys into nested Hashes.
+        @keys.reverse.each do |key|
+          c = { key => content }
+          content = c
+        end
+        @content = content
+      end
+
+      def <=>(other)
+        d = @root <=> other.root
+        return d unless d.zero?
+        d = @keys.size <=> other.keys.size
+        return d unless d.zero?
+        d = @keys <=> other.keys
+        return d unless d.zero?
+        @path <=> other.path
+      end
+    end
+
+    def self.prepare_prefix(name_prefix, root)
+      name_prefix_dir = File.dirname(name_prefix)
+      root = File.realpath(name_prefix_dir, root) unless name_prefix_dir == '.'
+      name_prefix = File.basename(name_prefix)
+      if name_prefix == '.' # Just being nice. Daft argument.
+        name_prefix = File.basename(root)
+        root = File.dirname(root)
+      end
+      [root, name_prefix]
+    end
+
+    def self.find_filenames(root, name_prefix)
+      full_prefix = File.join(root, name_prefix)
+      file_paths = []
+      Find.find(root) do |path|
+        next if path.size < full_prefix.size
+        is_dir = File.directory?(path)
+        if path.start_with?(full_prefix)
+          file_paths.push(path) unless is_dir
+        elsif is_dir
+          Find.prune
+        end
+      end
+      file_paths
+    end
+
+    def self.path_splitter(separator)
+      parts = [ Regexp.quote('/') ]
+      parts.push(Regexp.quote(separator)) if separator.is_a?(String) && !separator.empty?
+      Regexp.new("(#{parts.join('|')})")
+    end
+
+    def self.remove_extension(file_path, extensions)
+      extensions.each do |e|
+        if e == '.*'
+          idx = file_path.rindex('.')
+          next if idx.nil? # No . anywhere.
+          ext = file_path[idx..]
+          next unless ext.index('/').nil? # Last . is before file name.
+          return [ file_path[0...idx], ext ]
+        elsif file_path.end_with?(e)
+          return [ file_path[0..-(1 + e.size)], e ]
+        end
+      end
+      [ file_path, nil ]
+    end
+
+    def self.convert_path_end(path, splitter, prefix_size, extensions)
+      relevant, ext = remove_extension(path[prefix_size..], extensions)
+      pieces = relevant.split(splitter).map do |piece|
+        case piece
+        when '' then nil
+        when '/' then :dir
+        when Gen.separator then :separator
+        else
+          piece
+        end
+      end
+      unless ext.nil?
+        pieces.push(:extension)
+        pieces.push(ext)
+      end
+      pieces.compact!
+      ConfigFileInfo.new(pieces, path)
+    end
+  end
+end