openapi-sourcetools 0.7.1 → 0.8.0

data/bin/openapi-processpaths

@@ -1,12 +1,13 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-# Copyright © 2021-2024 Ismo Kärkkäinen
+# Copyright © 2021-2025 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
-require_relative '../lib/common'
+require_relative '../lib/openapi/sourcetools/apiobjects'
+require_relative '../lib/openapi/sourcetools/common'
 require 'optparse'
-require 'yaml'
+include OpenAPISourceTools
 
 
 def main
@@ -32,26 +33,22 @@ def main
     opts.on('-h', '--help', 'Print this help and exit.') do
       $stdout.puts %(#{opts}
 
-Processes API specification document path objects into form that is expected by
-later stage tools. Checks for paths that may be ambiguous.
+Adds split path parts into API document path items under x-openapi-sourcetools
+key. Checks for paths that may be ambiguous.
 )
       exit 0
     end
   end
   parser.order!
 
-  doc = load_source(input_name)
+  doc = Common.load_source(input_name)
   return 2 if doc.nil?
 
   processed = {}
-  doc.fetch('paths', {}).each_pair do |path, value|
-    parts = split_path(path, true)
-    processed[path] = {
-      'parts' => parts,
-      'orig' => value,
-      'lookalike' => [],
-      path: ServerPath.new(parts)
-    }
+  doc.fetch('paths', {}).each do |path, item|
+    parts = Common.split_path(path, true)
+    item['x-openapi-sourcetools-parts'] = parts # Added to original path item.
+    processed[path] = ApiObjects::ServerPath.new(parts)
   end
 
   # Find lookalike sets.
@@ -63,22 +60,14 @@ later stage tools. Checks for paths that may be ambiguous.
     k.times do |n|
       pn = paths[n]
       b = processed[pn]
-      next unless a[:path].compare(b[:path]).zero?
-      a['lookalike'].push pn
-      b['lookalike'].push pk
+      next unless a.compare(b).zero?
       $stderr.puts("Similar: #{pn} #{pk}")
       lookalikes = true
     end
   end
-  return aargh('Similar paths found.', 4) if lookalikes && error
+  return Common.aargh('Similar paths found.', 4) if lookalikes && error
 
-  # Remove temporary fields.
-  processed.each_value do |v|
-    v.keys.each { |k| v.delete(k) if k.is_a? Symbol }
-  end
-  doc['paths'] = processed
-
-  dump_result(output_name, YAML.dump(doc), 3)
+  Common.dump_result(output_name, doc, 3)
 end
 
-exit(main) if defined?($unit_test).nil?
+exit(main) unless defined?($unit_test)

data/lib/openapi/sourcetools/apiobjects.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+# Copyright © 2024-2025 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+module OpenAPISourceTools
+  # Various classes for handling objects in the API specification.
+  # Used in various programs.
+  module ApiObjects
+
+    def self.same(a, b, ignored_keys = Set.new(%w[summary description]))
+      return a == b unless a.is_a?(Hash) && b.is_a?(Hash)
+      keys = Set.new(a.keys + b.keys) - ignored_keys
+      keys.to_a.each do |k|
+        return false unless a.key?(k) && b.key?(k)
+        return false unless same(a[k], b[k], ignored_keys)
+      end
+      true
+    end
+
+    def self.ref_string(name, schema_path)
+      "#{schema_path}/#{name}"
+    end
+
+    def self.reference(obj, schemas, schema_path, ignored_keys = Set.new(%w[summary description]), prefix = 'Schema')
+      # Check if identical schema has been added and if so, return the $ref string.
+      schemas.keys.sort.each do |k|
+        return ref_string(k, schema_path) if same(obj, schemas[k], ignored_keys)
+      end
+      # One of the numbers will not match existing keys. More number than keys.
+      (schemas.size + 1).times do |n|
+        # 'x' is to simplify find and replace (Schema1x vs Schema1 and Schema10)
+        k = "#{prefix}#{n}x"
+        next if schemas.key?(k)
+        schemas[k] = obj.merge
+        return ref_string(k, schema_path)
+      end
+    end
+
+    # A component in the API specification for reference and anchor handling.
+    class Components
+      attr_reader :path, :prefix, :anchor2ref, :schema_names
+      attr_accessor :items, :ignored_keys
+
+      def initialize(path, prefix, ignored_keys = %w[summary description examples example $anchor])
+        path = "#/#{path.join('/')}/" if path.is_a?(Array)
+        path = "#{path}/" unless path.end_with?('/')
+        @path = path
+        @prefix = prefix
+        @anchor2ref = {}
+        @schema_names = Set.new
+        @items = {}
+        @ignored_keys = Set.new(ignored_keys)
+      end
+
+      def add_options(opts)
+        opts.on('--use FIELD', 'Use FIELD in comparisons.') do |f|
+          @ignored_keys.delete(f)
+        end
+        opts.on('--ignore FIELD', 'Ignore FIELD in comparisons.') do |f|
+          @ignored_keys.add(f)
+        end
+      end
+
+      def help
+        %(All fields are used in object equality comparisons except:\n#{@ignored_keys.to_a.sort!.join("\n")})
+      end
+
+      def add_schema_name(name)
+        @schema_names.add(name)
+      end
+
+      def ref_string(name)
+        return nil if name.nil?
+        "#{@path}#{name}"
+      end
+
+      def reference(obj)
+        # Check if identical schema has been added. If so, return the $ref string.
+        @items.each do |k, v|
+          return ref_string(k) if ApiObjects.same(obj, v, @ignored_keys)
+        end
+        # One of the numbers will not match existing keys. More number than keys.
+        (@items.size + 1).times do |n|
+          # 'x' is to simplify find and replace (Schema1x vs Schema1 and Schema10)
+          cand = "#{@prefix}#{n}x"
+          next if @items.key?(cand)
+          @items[cand] = obj.merge
+          @schema_names.add(cand)
+          return ref_string(cand)
+        end
+      end
+
+      def store_anchor(obj, ref = nil)
+        anchor_name = obj['$anchor']
+        return if anchor_name.nil?
+        ref = obj['$ref'] if ref.nil?
+        raise StandardError, 'ref is nil and no $ref or it is nil' if ref.nil?
+        @anchor2ref[anchor_name] = ref
+      end
+
+      def alter_anchors
+        replacements = {}
+        @anchor2ref.each_key do |a|
+          next if @schema_names.member?(a)
+          replacements[a] = ref_string(a)
+          @schema_names.add(a)
+        end
+        replacements.each do |a, r|
+          @anchor2ref[a] = r
+        end
+      end
+
+      def anchor_ref_replacement(ref)
+        @anchor2ref[ref[1...ref.size]] || ref
+      end
+    end
+
+    # Represents path with fixed parts and variables.
+    class ServerPath
+      # Probably moves to a separate file once processpaths and frequencies receive
+      # some attention.
+      include Comparable
+
+      attr_accessor :parts
+
+      def initialize(parts)
+        @parts = parts
+      end
+
+      # Parameters are after fixed strings.
+      def <=>(other)
+        pp = other.is_a?(Array) ? other : other.parts
+        @parts.size.times do |k|
+          return 1 if pp.size <= k # Longer comes after shorter.
+          pk = @parts[k]
+          ppk = pp[k]
+          if pk.key?('fixed')
+            if ppk.key?('fixed')
+              c = pk['fixed'] <=> ppk['fixed']
+            else
+              return -1
+            end
+          else
+            if ppk.key?('fixed')
+              return 1
+            else
+              c = pk.fetch('parameter', '') <=> ppk.fetch('parameter', '')
+            end
+          end
+          return c unless c.zero?
+        end
+        (@parts.size < pp.size) ? -1 : 0
+      end
+
+      # Not fit for sorting. Variable equals anything.
+      def compare(other, range = nil)
+        pp = other.is_a?(Array) ? other : other.parts
+        if range.nil?
+          range = 0...@parts.size
+        elsif range.is_a? Number
+          range = range...(range + 1)
+        end
+        range.each do |k|
+          return 1 if pp.size <= k # Longer comes after shorter.
+          ppk = pp[k]
+          next unless ppk.key?('fixed')
+          pk = parts[k]
+          next unless pk.key?('fixed')
+          c = pk['fixed'] <=> ppk['fixed']
+          return c unless c.zero?
+        end
+        (@parts.size < pp.size) ? -1 : 0
+      end
+    end
+
+    def self.operation_objects(path_item)
+      keys = %w[operationId requestBody responses callbacks]
+      out = {}
+      path_item.each do |method, op|
+        next unless op.is_a?(Hash)
+        keys.each do |key|
+          next unless op.key?(key)
+          out[method] = op
+          break
+        end
+      end
+      out
+    end
+  end
+end

data/lib/openapi/sourcetools/common.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+# Copyright © 2021-2025 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+require 'pathname'
+require 'yaml'
+
+module OpenAPISourceTools
+  # Common methods used in programs and elsewhere gathered into one place.
+  module Common
+    def self.aargh(message, return_value = nil)
+      message = message.map(&:to_s).join("\n") if message.is_a? Array
+      $stderr.puts message
+      return_value
+    end
+
+    def self.yesno(boolean)
+      boolean ? 'yes' : 'no'
+    end
+
+    def self.bury(doc, path, value)
+      (path.size - 1).times do |k|
+        p = path[k]
+        doc[p] = {} unless doc.key?(p)
+        doc = doc[p]
+      end
+      doc[path.last] = value
+    end
+
+    module Out
+      attr_reader :count
+      module_function :count
+      attr_accessor :quiet
+      module_function :quiet
+      module_function :quiet=
+
+      def self.put(message)
+        Common.aargh(message) unless @quiet
+        @count = @count.nil? ? 1 : @count + 1
+      end
+    end
+
+    def self.split_path(p, spec = false)
+      parts = []
+      p = p.strip
+      unless spec
+        q = p.index('?')
+        p.slice!(0...q) unless q.nil?
+      end
+      p.split('/').each do |s|
+        next if s.empty?
+        s = { (spec && s.include?('{') ? 'parameter' : 'fixed') => s }
+        parts.push(s)
+      end
+      parts
+    end
+
+    def self.load_source(input)
+      YAML.safe_load(input.nil? ? $stdin : File.read(input))
+    rescue Errno::ENOENT
+      aargh "Could not load #{input || 'stdin'}"
+    rescue StandardError => e
+      aargh "#{e}\nFailed to read #{input || 'stdin'}"
+    end
+
+    def self.dump_result(output, doc, error_return)
+      doc = YAML.dump(doc, line_width: 1_000_000) unless doc.is_a?(String)
+      if output.nil?
+        $stdout.puts doc
+      else
+        fp = Pathname.new output
+        fp.open('w') do |f|
+          f.puts doc
+        end
+      end
+      0
+    rescue StandardError => e
+      aargh([ e, "Failed to write output: #{output || 'stdout'}" ], error_return)
+    end
+  end
+end

data/lib/openapi/sourcetools/config.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+# Copyright © 2024-2025 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+require_relative 'task'
+require_relative 'gen'
+require 'find'
+require 'yaml'
+
+module OpenAPISourceTools
+  # Configuration file find and load convenience functions.
+  # See the first 3 methods. The rest are intended to be internal helpers.
+  module ConfigLoader
+
+    # A function to find all files with a given prefix.
+    # Prefix is taken from Gen.config if nil.
+    # Returns an array of ConfigFileInfo objects.
+    def self.find_files(name_prefix:, extensions: [ '.*' ])
+      name_prefix = Gen.config if name_prefix.nil?
+      raise ArgumentError, 'name_prefix or config must be set' if name_prefix.nil?
+      root, name_prefix = prepare_prefix(name_prefix, Gen.wd)
+      file_paths = find_filenames(root, name_prefix)
+      splitter = path_splitter(Gen.separator)
+      out = file_paths.map { |fp| convert_path_end(fp, splitter, root.size + 1, extensions) }
+      out.sort!
+    end
+
+    # A function to read all YAML files in an array of ConfigFileInfo objects.
+    # Returns the same as contents_array.
+    def self.read_contents(config_file_infos)
+      config_file_infos.each do |cfi|
+        c = YAML.safe_load_file(cfi.path)
+        # Check allows e.g. copyright and license files be named with config prefix
+        # for clarity, but ignored during config loading.
+        next if cfi.keys.empty? && !c.is_a?(Hash)
+        cfi.bury_content(c)
+      rescue Psych::SyntaxError
+        next # Was not YAML. Other files can be named using config prefix.
+      end
+      contents_array(config_file_infos)
+    end
+
+    # Maps an array of ConfigFileInfo objects to an array of their contents.
+    def self.contents_array(config_file_infos)
+      config_file_infos.map(&:content).reject(&:nil?)
+    end
+
+    class ConfigFileInfo
+      include Comparable
+
+      attr_reader :root, :keys, :path, :content
+
+      def initialize(pieces, path)
+        @keys = []
+        @root = nil
+        pieces.each do |p|
+          if p.is_a?(String)
+            if @root.nil?
+              @root = p
+            else
+              @keys.push(p)
+            end
+          else
+            break if p == :extension
+          end
+        end
+        @path = path
+        @content = nil
+      end
+
+      def bury_content(content)
+        # Turns chain of keys into nested Hashes.
+        @keys.reverse.each do |key|
+          c = { key => content }
+          content = c
+        end
+        @content = content
+      end
+
+      def <=>(other)
+        d = @root <=> other.root
+        return d unless d.zero?
+        d = @keys.size <=> other.keys.size
+        return d unless d.zero?
+        d = @keys <=> other.keys
+        return d unless d.zero?
+        @path <=> other.path
+      end
+    end
+
+    def self.prepare_prefix(name_prefix, root)
+      name_prefix_dir = File.dirname(name_prefix)
+      root = File.realpath(name_prefix_dir, root) unless name_prefix_dir == '.'
+      name_prefix = File.basename(name_prefix)
+      if name_prefix == '.' # Just being nice. Daft argument.
+        name_prefix = File.basename(root)
+        root = File.dirname(root)
+      end
+      [root, name_prefix]
+    end
+
+    def self.find_filenames(root, name_prefix)
+      full_prefix = File.join(root, name_prefix)
+      file_paths = []
+      Find.find(root) do |path|
+        next if path.size < full_prefix.size
+        is_dir = File.directory?(path)
+        if path.start_with?(full_prefix)
+          file_paths.push(path) unless is_dir
+        elsif is_dir
+          Find.prune
+        end
+      end
+      file_paths
+    end
+
+    def self.path_splitter(separator)
+      parts = [ Regexp.quote('/') ]
+      parts.push(Regexp.quote(separator)) if separator.is_a?(String) && !separator.empty?
+      Regexp.new("(#{parts.join('|')})")
+    end
+
+    def self.remove_extension(file_path, extensions)
+      extensions.each do |e|
+        if e == '.*'
+          idx = file_path.rindex('.')
+          next if idx.nil? # No . anywhere.
+          ext = file_path[idx..]
+          next unless ext.index('/').nil? # Last . is before file name.
+          return [ file_path[0...idx], ext ]
+        elsif file_path.end_with?(e)
+          return [ file_path[0..-(1 + e.size)], e ]
+        end
+      end
+      [ file_path, nil ]
+    end
+
+    def self.convert_path_end(path, splitter, prefix_size, extensions)
+      relevant, ext = remove_extension(path[prefix_size..], extensions)
+      pieces = relevant.split(splitter).map do |piece|
+        case piece
+        when '' then nil
+        when '/' then :dir
+        when Gen.separator then :separator
+        else
+          piece
+        end
+      end
+      unless ext.nil?
+        pieces.push(:extension)
+        pieces.push(ext)
+      end
+      pieces.compact!
+      ConfigFileInfo.new(pieces, path)
+    end
+  end
+end

data/lib/openapi/sourcetools/docs.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Copyright © 2024-2025 Ismo Kärkkäinen
+# Licensed under Universal Permissive License. See LICENSE.txt.
+
+module OpenAPISourceTools
+  # To hold documents loaded via command-line.
+  # Provides attribute accessor methods for each document.
+  # Exposed via Gen.d to tasks.
+  class Docs
+    attr_reader :docs
+
+    def initialize
+      @docs = {}
+    end
+
+    def method_missing(method_name, *args)
+      name = method_name.to_s
+      if name.end_with?('=')
+        name = name[0...(name.size - 1)]
+        super unless @docs.key?(name)
+        @docs[name] = args.first
+        return args.first
+      end
+      super unless @docs.key?(name)
+      @docs[name]
+    end
+
+    def respond_to_missing?(method_name, *args)
+      name = method_name.to_s
+      name = name[0...(name.size - 1)] if name.end_with?('=')
+      @docs.key?(name) || super
+    end
+
+    def add(name, content)
+      return false if docs.key?(name)
+      @docs[name] = content
+      true
+    end
+  end
+end

data/lib/{gen.rb → openapi/sourcetools/gen.rb}

@@ -1,37 +1,44 @@
 # frozen_string_literal: true
 
-# Copyright © 2024 Ismo Kärkkäinen
+# Copyright © 2024-2025 Ismo Kärkkäinen
 # Licensed under Universal Permissive License. See LICENSE.txt.
 
 require_relative 'task'
 require_relative 'helper'
 require_relative 'docs'
 require_relative 'output'
+require_relative 'config'
+require 'deep_merge'
 
 
+# The generation module that contains things visible to tasks.
 module Gen
   def self.add_doc(symbol, docstr)
     return if docstr.nil?
     @docsrc = [] unless instance_variable_defined?('@docsrc')
-    @docsrc.push("- #{symbol.to_s} : #{docstr}")
+    @docsrc.push("- #{symbol} : #{docstr}")
   end
+  private_class_method :add_doc
 
   def self.read_attr(symbol, default)
     return if symbol.nil?
     attr_reader(symbol)
     module_function(symbol)
-    instance_variable_set("@#{symbol.to_s}", default)
+    instance_variable_set("@#{symbol}", default)
   end
+  private_class_method :read_attr
 
   def self.mod_attr2_reader(symbol, symbol2, docstr = nil, default = nil)
     read_attr(symbol, default)
     read_attr(symbol2, default)
     add_doc(symbol, docstr)
   end
+  private_class_method :mod_attr2_reader
 
   def self.mod_attr_reader(symbol, docstr = nil, default = nil)
     mod_attr2_reader(symbol, nil, docstr, default)
   end
+  private_class_method :mod_attr_reader
 
   def self.rw_attr(symbol, default)
     attr_accessor(symbol)
@@ -40,39 +47,56 @@ module Gen
     module_function((s + '=').to_sym)
     instance_variable_set("@#{s}", default)
   end
+  private_class_method :rw_attr
 
   def self.mod_attr2_accessor(symbol, symbol2, docstr = nil, default = nil)
     rw_attr(symbol, default)
     rw_attr(symbol2, default) unless symbol2.nil?
     add_doc(symbol, docstr)
   end
+  private_class_method :mod_attr2_accessor
 
   def self.mod_attr_accessor(symbol, docstr = nil, default = nil)
     mod_attr2_accessor(symbol, nil, docstr, default)
   end
+  private_class_method :mod_attr_accessor
 
   mod_attr_reader :doc, 'OpenAPI document.'
   mod_attr_reader :outdir, 'Output directory name.'
-  mod_attr_reader :d, 'Other documents object.', Docs.new
+  mod_attr_reader :d, 'Other documents object.', OpenAPISourceTools::Docs.new
+  mod_attr_reader :wd, 'Original working directory', Dir.pwd
+  mod_attr_reader :configuration, 'Generator internal configuration'
+  mod_attr_accessor :config, 'Configuration file name for next gem or Ruby file.'
+  mod_attr_accessor :separator, 'Key separator in config file names.', nil
   mod_attr_accessor :in_name, 'OpenAPI document name, nil if stdin.'
   mod_attr_accessor :in_basename, 'OpenAPI document basename, nil if stdin.'
-  mod_attr_accessor :tasks, 'Tasks array.', []
-  mod_attr_accessor :g, 'Hash for storing values visible to all tasks.', {}
-  mod_attr_accessor :a, 'Intended for instance with defined attributes.'
+  mod_attr_reader :g, 'Hash for storing values visible to all tasks.', {}
+  mod_attr_accessor :x, 'Hash for storing values visible to tasks from processor.', {}
   mod_attr_accessor :h, 'Instance of class with helper methods.'
+  mod_attr_accessor :tasks, 'Tasks array.', []
   mod_attr2_accessor :task, :t, 'Current task instance.'
   mod_attr_accessor :task_index, 'Current task index.'
-  mod_attr_accessor :loaders, 'Array of generator loader methods.', []
-  mod_attr2_accessor :output, :o, 'Output-related methods.', Output.new
+  mod_attr_accessor :loaders, 'Array of processor loader methods.', []
+  mod_attr_accessor :output, 'Output-formatting helper.', OpenAPISourceTools::Output.new
+
+  def self.load_config(config_prefix)
+    cfg = {}
+    cfgs = OpenAPISourceTools::ConfigLoader.find_files(name_prefix: config_prefix)
+    cfgs = OpenAPISourceTools::ConfigLoader.read_contents(cfgs)
+    cfgs.each { |c| cfg.deep_merge!(c) }
+    cfg
+  end
+  private_class_method :load_config
 
-  def self.setup(document_content, input_name, output_directory)
+  def self.setup(document_content, input_name, output_directory, config_prefix)
     @doc = document_content
     @outdir = output_directory
     unless input_name.nil?
       @in_name = File.basename(input_name)
       @in_basename = File.basename(input_name, '.*')
     end
-    add_task(task: HelperTask.new)
+    @configuration = load_config(config_prefix)
+    add_task(task: OpenAPISourceTools::HelperTask.new)
   end
 
   def self.add_task(task:, name: nil, executable: false, x: nil)
@@ -85,11 +109,12 @@ module Gen
   end
 
   def self.add_write_content(name:, content:, executable: false)
-    add_task(task: WriteTask.new(name, content, executable))
+    add_task(task: OpenAPISourceTools::WriteTask.new(name, content, executable))
   end
 
   def self.add(source:, template: nil, template_name: nil, name: nil, executable: false, x: nil)
-    add_task(task: Task.new(source, template, template_name), name: name, executable: executable, x: x)
+    add_task(task: OpenAPISourceTools::Task.new(source, template, template_name),
+      name:, executable:, x:)
   end
 
   def self.document