workato-connector-builder 0.0.1.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 23aae47128ed80aee07d7f2e12e811c2632b6b1f30137a24774b4f571415f7eb
+  data.tar.gz: 7d0fd63fc1d1d3274ea6036ea2d4569f59856196b51e0050eaf75ddf740e7711
+SHA512:
+  metadata.gz: e13f417404ee807284ef4deb3eb5ba02884f3378701b934f85113896629ff440f0fcb4f0e143adf58d8c002ac40c29f106b305b2fda434e8a8659bad8992155f
+  data.tar.gz: d04fe57dcaa85de89dff3961e1fa5ad41b88f3cbb81efb2fa2ec451169bde49fb04a0544c15cfd62508fb9c48d6a4ed0900c3bf29d90b65781fa864e1353f333

data/bin/workato-connector-builder

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'workato_connector_builder/cli/cli'
+WorkatoConnectorBuilder::CLI::CLI.start

data/lib/output.rb

@@ -0,0 +1,14 @@
+{
+  :test => 1,
+  :collision => 2,
+  'dup' => {
+    a: 1,
+    b: 2
+  },
+  :otherfield => 2,
+  dup: {
+    # some comment
+    a: 2,
+    c: 3 # another_comment
+  },
+}

data/lib/test.rb

@@ -0,0 +1,263 @@
+require 'parser/current'
+require 'unparser'
+
+# Emitter
+#
+#
+#
+# # level 2 will get us the hash keys we care about in the files like :action, etc
+# def look_for_type(ast, type, value_to_match, levels_to_traverse = 2)
+#   return unless Parser::AST::Node === ast
+#
+#   if ast.type == type # when we found block
+#     ast.children[0] == value_to_match
+#   else
+#     if levels_to_traverse >= 0
+#       match =
+#         ast.children.select do |child|
+#           # otherwise let's look for blocks
+#           # fail early, we don't want to traverse the whole tree
+#           match = look_for_type(child, type, value_to_match, levels_to_traverse - 1)
+#           if match
+#             break
+#           end
+#         end
+#       return match
+#     else
+#       return false
+#     end
+#   end
+# end
+#
+# def get_children_for_match(ast, type, value_to_match, levels_to_traverse = 2)
+#
+#   #
+#   # if ast.children.any? {|child| child.type == type  && child.children[0] == value_to_match}# when we found block
+#   #   ast.children
+#   # else
+#   #   if levels_to_traverse > 0
+#   #     match = ast.children.select do |child|
+#   #       # otherwise let's look for blocks
+#   #       # fail early, we don't want to traverse the whole tree
+#   #       match = get_children_for_match(child, type, value_to_match, levels_to_traverse - 1)
+#   #       if match
+#   #         break
+#   #       end
+#   #     end
+#   #     return match
+#   #   else
+#   #     return []
+#   #   end
+#   # end
+# end
+#
+# # combine the asts that are the same
+# $workato_hash_roots = {
+#   :title => [],
+#   :connection => [],
+#   :actions => [],
+#   :methods => [],
+#   :object_definitions => [],
+#   :pick_lists => [],
+#   :test => [],
+#   :triggers => [],
+# }
+
+require_relative 'workato_connector_builder/ast_generator'
+require_relative 'workato_connector_builder/hash_combiner'
+require 'rubocop-ast'
+include WorkatoConnectorBuilder
+
+# combine the source files
+builder = RuboCop::AST::Builder.new
+parser = Parser::CurrentRuby.new builder
+
+buffer1 = Parser::Source::Buffer.new('test1')
+buffer1.raw_source = %{{
+  :test => 1,
+  :collision => 1,
+  'dup' => {
+    a: 1,
+    b: 2
+  }
+}}
+buffer2 = Parser::Source::Buffer.new('test2')
+# buffer2.raw_source = '{ :otherfield => 2, :collision => 2, dup: { # some comment \n c: 3}}'
+buffer2.raw_source = %{{
+  :otherfield => 2,
+  :collision => 2,
+  dup: {
+    # some comment
+    a: 2,
+    c: 3 # another_comment
+  }
+}}
+
+require 'thor'
+combine_hashes = WorkatoConnectorBuilder::Utils::HashCombiner.new Thor::Shell::Color.new
+
+treeRewriter = Parser::Source::TreeRewriter.new buffer1
+
+node1 = parser.parse(buffer1)
+parser = Parser::CurrentRuby.new builder
+parser.reset
+node2 = parser.parse(buffer2)
+
+# class MyRule < Parser::AST::Processor
+#   include RuboCop::AST::Traversal
+#
+#   def on_sym(node)
+#     puts "I found a symbol! #{node.value}"
+#   end
+# end
+#
+#
+
+combine_hashes.combine node1, node2, treeRewriter
+#
+# source = RuboCop::ProcessedSource.new(Unparser.unparse(node3), 2.7)
+# rule = MyRule.new
+# source.ast.each_node { |n| rule.process(n) }
+
+File.write('./output.rb', treeRewriter.process, mode: 'w+')
+
+#   puts "Correcting output file"
+# Loutput = %x(rubocop --auto-correct -x ./output.rb)
+# asts = walk("../", %w[bin output test])
+#
+#
+#
+# split_asts =
+#   asts.map { |ast|
+#     ast.children.map { |child|
+#       children = child.children
+#       # first element is going to be the symbol for the hash, contained in this nested structure
+#       symbol = children[0].children[0]
+#       WORKATO_HASH_ROOTS[symbol].append children[1]
+#     }
+#   }
+#
+# def unparse(node, comment_array = [])
+#   return '' if node.nil?
+#
+#   Unparser::Buffer.new.tap do |buffer|
+#     Emitter.new(
+#       buffer,
+#       node,
+#       Unparser::Comments.new(comment_array)
+#     ).write_to_buffer
+#   end.content
+# end
+#
+# combined_asts =
+#   WORKATO_HASH_ROOTS.map { |symbol, matching_sets|
+#     combined_hashes = matching_sets[0]
+#     matching_sets.drop(1).each { |set|
+#       combined_hashes = combined_hashes.concat set
+#     }
+#
+#     s = Unparser.unparse(combined_hashes)
+#
+#     hash = "{#{symbol}: #{s}}"
+#     Unparser.parse(hash)
+#   }
+#
+# final_ast = Unparser.parse('{}')
+# combined_asts.each { |node|
+#   final_ast = final_ast.concat node
+# }
+# OUTPUT_PATH = 'output/core_connector.rb'
+# FULL_OUTPUT_FILE_PATH = File.join(File.dirname(__FILE__), '../', OUTPUT_PATH)
+#
+# output = Unparser.unparse(final_ast)
+# File.write(FULL_OUTPUT_FILE_PATH, output, nil, mode: 'w+')
+#
+# Dir.chdir('../') {
+#   puts %x(ls -ahl)
+#   value = %x(rubocop ./#{OUTPUT_PATH} --auto-correct -x - v)
+#   puts value
+# }
+# # command = Thread.new do
+# # end
+# # command.join
+#
+# # result = Array (79 elements)
+# #  [0] = :location
+# #  [1] = :assign_properties
+# #  [2] = :loc
+# #  [3] = :dup
+# #  [4] = :clone
+# #  [5] = :to_sexp
+# #  [6] = :updated
+# #  [7] = :fancy_type
+# #  [8] = :to_sexp_array
+# #  [9] = :<<
+# #  [10] = :to_a
+# #  [11] = :==
+# #  [12] = :to_s
+# #  [13] = :children
+# #  [14] = :eql?
+# #  [15] = :deconstruct
+# #  [16] = :+
+# #  [17] = :inspect
+# #  [18] = :append
+# #  [19] = :hash
+# #  [20] = :concat
+# #  [21] = :type
+# #  [22] = :to_ast
+# #  [23] = :pretty_print_inspect
+# #  [24] = :pretty_print_instance_variables
+# #  [25] = :pretty_print
+# #  [26] = :pretty_print_cycle
+# #  [27] = :itself
+# #  [28] = :yield_self
+# #  [29] = :then
+# #  [30] = :taint
+# #  [31] = :tainted?
+# #  [32] = :untaint
+# #  [33] = :untrust
+# #  [34] = :untrusted?
+# #  [35] = :trust
+# #  [36] = :frozen?
+# #  [37] = :methods
+# #  [38] = :singleton_methods
+# #  [39] = :protected_methods
+# #  [40] = :private_methods
+# #  [41] = :public_methods
+# #  [42] = :instance_variables
+# #  [43] = :instance_variable_get
+# #  [44] = :instance_variable_set
+# #  [45] = :instance_variable_defined?
+# #  [46] = :remove_instance_variable
+# #  [47] = :instance_of?
+# #  [48] = :kind_of?
+# #  [49] = :is_a?
+# #  [50] = :tap
+# #  [51] = :class
+# #  [52] = :display
+# #  [53] = :singleton_class
+# #  [54] = :public_send
+# #  [55] = :method
+# #  [56] = :public_method
+# #  [57] = :singleton_method
+# #  [58] = :define_singleton_method
+# #  [59] = :extend
+# #  [60] = :to_enum
+# #  [61] = :enum_for
+# #  [62] = :<=>
+# #  [63] = :===
+# #  [64] = :=~
+# #  [65] = :!~
+# #  [66] = :nil?
+# #  [67] = :respond_to?
+# #  [68] = :freeze
+# #  [69] = :object_id
+# #  [70] = :send
+# #  [71] = :pretty_inspect
+# #  [72] = :__send__
+# #  [73] = :!
+# #  [74] = :!=
+# #  [75] = :__id__
+# #  [76] = :equal?
+# #  [77] = :instance_eval
+# #  [78] = :instance_exec

data/lib/workato_connector_builder/ast_generator.rb

@@ -0,0 +1,57 @@
+module WorkatoConnectorBuilder
+  class ASTGenerator
+    attr_reader :search_path
+
+    def initialize(search_path, parser, ignored_paths = [], ignored_files = [])
+      @parser = parser
+      @search_path = search_path
+      @ignored_paths = ignored_paths
+      @ignored_files = ignored_files
+    end
+
+    def get_asts
+      walk(search_path)
+    end
+
+    def get_ignored_paths
+      @ignored_paths.clone
+    end
+
+    def get_ignored_files
+      @ignored_files.clone
+    end
+
+    private
+
+    attr_writer :search_path
+
+    def walk(start)
+      absolute_start = File.expand_path(start)
+      asts = []
+      ignored_paths = @ignored_paths
+      ignored_files = @ignored_files
+      Dir.foreach(absolute_start) do |f|
+        if ignored_paths.include?(f) || ignored_files.include?(f)
+          next
+        end
+
+        path = File.join(start, f)
+        absolute_path = File.join(absolute_start, f)
+        # ignore everything starts with . (dir references, hidden dirs/files)
+        if f.start_with?('.')
+          next
+        elsif File.directory?(absolute_path)
+          asts.merge! self.walk(path)
+        elsif f.end_with?('.rb') and !f.start_with?('.')
+          ps = RuboCop::AST::ProcessedSource.from_file(path, 2.4)
+          comments = ps.ast_with_comments
+          nodes = ps.ast
+
+          asts << { ast: nodes, comments: comments, path: path }
+        end
+      end
+
+      asts
+    end
+  end
+end

data/lib/workato_connector_builder/cli/build_command.rb

@@ -0,0 +1,139 @@
+require_relative '../ast_generator'
+require_relative '../hash_combiner'
+require_relative '../constants'
+require_relative '../validator'
+require_relative '../fix_newlines_rewriter'
+
+require 'parser/ruby24'
+require 'rubocop-ast'
+require 'unparser'
+
+module WorkatoConnectorBuilder
+  module CLI
+    class BuildCommand
+
+      def initialize(shell, source_path, output_file, ignored_files, ignored_dirs)
+        @shell = shell
+        @source_path = source_path
+        @output_file = output_file
+        @ignored_files = ignored_files
+        @ignored_dirs = ignored_dirs
+      end
+
+      def exec
+        unless @ignored_files.empty?
+          @shell.say "Ignoring files:\n"
+          @ignored_files.each do |f|
+            @shell.say "\t#{f}\n"
+          end
+        end
+        unless @ignored_dirs.empty?
+          @shell.say "Ignoring directories:\n"
+          @ignored_dirs.each do |f|
+            @shell.say "\t#{f}\n"
+          end
+        end
+
+        ast_generator = ASTGenerator.new(
+          @source_path,
+          create_parser,
+          @ignored_dirs,
+          @ignored_files,
+        )
+
+        hash_combiner = HashCombiner.new @shell
+        output = process ast_generator, hash_combiner
+
+        output_file = File.absolute_path(@output_file)
+
+        @shell.say "Writing output file to : #{output_file}\n"
+        File.open(output_file, 'w') do |file|
+          file.write output
+        end
+
+        @shell.say "Done\n"
+      end
+
+      private
+
+      attr_reader :shell, :ignored_files, :ignored_dirs, :output_file, :source_path
+
+      def create_parser
+        builder = RuboCop::AST::Builder.new
+        Parser::Ruby24.new builder
+      end
+
+      def create_buffer(file_name, source)
+        Parser::Source::Buffer.new file_name, **{ source: source }
+      end
+
+      def process(ast_generator, hash_combiner)
+        asts = ast_generator.get_asts
+        require 'pry-byebug'
+
+        first_node = asts.first
+        tree_rewriter = FixNewlinesRewriter.new
+
+        asts.drop(1).each do |node|
+          combined = hash_combiner.combine(first_node[:ast], node[:ast])
+          first_node[:ast] = combined
+        end
+
+        # Check for valid workato keys
+        validator = WorkatoConnectorBuilder::Validator.new(@shell)
+        final_node = validator.process(first_node[:ast])
+
+        output = write_hash(final_node, 0, 1)
+        buffer = create_buffer(@output_file, output)
+        parser = create_parser
+        tree_rewriter.rewrite(buffer, parser.parse(buffer))
+
+      end
+
+      def write_hash(hash_node, indent_level, child_indent_level)
+        indent = indent_string(indent_level)
+        child_indent = indent_string(child_indent_level)
+        # we never indent the start
+        hash_string = "{\n"
+        hash_node.pairs.each do |pair|
+          unless pair.left_sibling.nil?
+            hash_string << "\n"
+          end
+
+          hash_string << child_indent
+
+          if pair.key.str_type?
+            hash_string << "'#{pair.key.value}'"
+          elsif pair.key.sym_type?
+            if pair.hash_rocket?
+              hash_string << ":#{pair.key.value}"
+            else
+              hash_string << pair.key.value.to_s
+            end
+          else
+            hash_string << pair.key.value.to_s
+          end
+
+          key_string = pair.delimiter(with_spacing: true)
+          hash_string << key_string
+
+          if pair.value.type == :hash
+            hash_string << write_hash(pair.value, child_indent_level, child_indent_level + 1)
+          else
+            hash_string << pair.value.source
+          end
+          hash_string << ','
+
+          unless pair.right_sibling.nil?
+            hash_string << "\n"
+          end
+        end
+        hash_string << "\n#{indent}}"
+      end
+
+      def indent_string(indent_level)
+        ' ' * (indent_level * 2)
+      end
+    end
+  end
+end

data/lib/workato_connector_builder/cli/cli.rb

@@ -0,0 +1,38 @@
+require 'thor'
+require_relative './build_command'
+
+module WorkatoConnectorBuilder
+  module CLI
+    class CLI < Thor
+      package_name "WorkatoConnectorBuilder"
+      map "-L" => :list
+
+
+      desc 'build <source_dir> <output_file>', ''
+      method_option :ignored_file, type: :string, repeatable: true
+      method_option :ignored_dir, type: :string, repeatable: true
+      method_option :help, type: :boolean, aliases: ['-h']
+      def build(path = nil, output_file = nil)
+        if options[:help]
+          help('build')
+          return 0
+        end
+
+        say "Beginning build of source files from #{path}, output file #{output_file}"
+        ignored_files = options[:ignored_file] || []
+        ignored_dirs = options[:ignored_dir] || []
+        BuildCommand.new(
+          @shell,
+          path,
+          output_file,
+          ignored_files,
+          ignored_dirs
+        ).exec
+      end
+
+      def self.exit_on_failure?
+        true
+      end
+    end
+  end
+end

data/lib/workato_connector_builder/cli/config_store.rb

@@ -0,0 +1,17 @@
+module WorkatoConnectorBuilder
+  module CLI
+    class ConfigStore
+      def initialize
+
+      end
+
+      def load_config path
+
+      end
+
+      def load_from_args args
+
+      end
+    end
+  end
+end

data/lib/workato_connector_builder/constants.rb

@@ -0,0 +1,76 @@
+module WorkatoConnectorBuilder
+  module Constants
+    TOP_LEVEL_KEYS = [
+      :title,
+      :connection,
+      :test,
+      :actions,
+      :triggers,
+      :object_definitions,
+      :pick_lists,
+      :methods,
+      :secure_tunnel,
+      :webhook_keys,
+    ]
+
+    # These are the only child keys that the top level key supports
+    FIRST_CHILD_LEVEL_KEYS = {
+      connection: [
+        :fields,
+        :authorization,
+        :base_uri,
+      ],
+    }
+
+    # These are the only keys that every grandchild of these top level or first level keys support
+    SECOND_CHILD_LEVEL_KEYS = {
+      actions: [
+        :title,
+        :subtitle,
+        :description,
+        :help,
+        :config_fields,
+        :input_fields,
+        :execute,
+        :output_fields,
+        :sample_output,
+        :retry_on_response,
+        :retry_on_request,
+        :max_retries,
+        :summarize_input,
+        :summarize_output,
+      ],
+      triggers: [
+        :title,
+        :subtitle,
+        :description,
+        :help,
+        :config_fields,
+        :input_fields,
+        :webhook_key,
+        :webhook_payload_type,
+        :webhook_subscribe,
+        :webhook_unsubscribe,
+        :webhook_notification,
+        :poll,
+        :dedup,
+        :output_fields,
+        :sample_output,
+        :summarize_input,
+        :summarize_output
+      ],
+      authorization: [
+        :type,
+        :client_id,
+        :client_secret,
+        :authorization_url,
+        :token_url,
+        :acquire,
+        :apply,
+        :refresh_on,
+        :detect_on,
+        :refresh,
+      ],
+    }
+  end
+end

data/lib/workato_connector_builder/fix_newlines_rewriter.rb

@@ -0,0 +1,29 @@
+require 'parser'
+require 'byebug'
+
+class FixNewlinesRewriter < Parser::TreeRewriter
+
+  def on_pair(node)
+    range = node.source_range
+    add_trailing_comma(range, range.source.length)
+    super
+  end
+
+  private
+  def add_trailing_comma(range, source_length)
+    unless has_trailing_comma?(range, source_length)
+      insert_after range.end, ','
+    end
+  end
+
+  def remove_trailing_comma(range, source_length)
+    if has_trailing_comma?(range, source_length)
+      remove range.adjust(begin_pos: source_length, end_pos: 1)
+    end
+  end
+
+  def has_trailing_comma?(range, source_length)
+    next_char_range = range.adjust(begin_pos: source_length, end_pos: 1)
+    next_char_range.is?(',')
+  end
+end

data/lib/workato_connector_builder/hash_combiner.rb

@@ -0,0 +1,116 @@
+module WorkatoConnectorBuilder
+  class HashCombiner
+    def initialize(shell)
+      @shell = shell
+    end
+
+    def combine(first, second)
+      require 'pry-byebug'
+      if !first.hash_type? or !second.hash_type?
+        return
+      end
+
+      concat_pairs = []
+      first.pairs.each do |pair|
+        matching_pair = second.pairs.find { |sp| sp.key == pair.key }
+        if matching_pair.nil?
+          concat_pairs.push pair
+        else
+          if pair.value.hash_type? && matching_pair.value.hash_type?
+            combined_node = combine(pair.value, matching_pair.value)
+            concat_pairs.push pair.updated(:pair, [pair.key].push(combined_node))
+          else
+            chosen_pair = replace_pair pair, matching_pair
+
+            unless chosen_pair.nil?
+              concat_pairs.push chosen_pair.updated(:pair, chosen_pair.children)
+            end
+          end
+        end
+      end
+
+      concat_pairs.concat(second.pairs.select { |pair| !first.keys.include? pair.key } || [])
+
+      first.updated(:hash, concat_pairs)
+    end
+
+    private
+
+    attr_reader :shell
+
+    def replace_pair(first, second)
+      source = first
+
+      @shell.say_status(
+        "Error: Mismatch found for key '#{first.key.value}'",
+        "Occurs at '#{first.source_range.to_s}' and '#{second.source_range.to_s}'",
+        :red
+      )
+
+      choice = @shell.ask(
+        %{Select a choice:
+  1) #{first.source}
+  2) #{second.source}
+  3) Do not include key\n},
+        :limited_to => %w[1 2 3]
+      )
+
+      case choice
+      when '2'
+        source = second
+      when '3'
+        source = nil
+      end
+
+      source
+    end
+
+    def get_pair_source(pair, include_comma = false, remove_leading_newlines = false, rstrip = false)
+      parent_source = pair.parent.source
+      parent_source_range = pair.parent.source_range
+
+      # Let's get any formatting that was in place from the second node
+      # Assume we're starting at the first pair, so we'll make our start position
+      # after the opening { and the end pos is after the }
+      pair_begin_pos = 1
+      pair_end_pos = parent_source.length - 2
+      # We need to get the position in relation to the parent's source range,
+      # since we're not using any parents ancestors when getting this source
+      pair_left_sibling = pair.left_sibling
+      unless pair_left_sibling.nil?
+        # Start after the left sibling's comma
+        pair_begin_pos = pair_left_sibling.source_range.end_pos + 1 - parent_source_range.begin_pos
+      end
+
+      # If we have a right sibling, we'll get to the start of it so we can capture any
+      # comments
+      pair_right_sibling = pair.right_sibling
+      unless pair_right_sibling.nil?
+        pair_end_pos = pair_right_sibling.source_range.begin_pos - 1 - parent_source_range.begin_pos
+      end
+
+      range = Range.new pair_begin_pos, pair_end_pos
+      pair_source = parent_source[range]
+
+      # trim off any newlines at the start, since we'll be inserting
+      # this into the other hash which might have it's own ending whitespace
+      if remove_leading_newlines
+        pair_source.sub!(/^[\n\r]*/, '')
+      end
+
+      if rstrip
+        pair_source.rstrip!
+      end
+
+      if include_comma
+        unless pair_source.rstrip.end_with?(',')
+          pair_source[pair.source] = "#{pair.source},"
+        end
+      else
+        pair_source[pair_source.rindex(',')] = ''
+      end
+
+      pair_source
+    end
+  end
+end

data/lib/workato_connector_builder/validator.rb

@@ -0,0 +1,138 @@
+module WorkatoConnectorBuilder
+  require 'pry-byebug'
+  class Validator
+    def initialize(shell)
+      @shell = shell
+    end
+
+    def process(node)
+      validate_top_level_hash(node)
+    end
+
+    private
+
+    def validate_top_level_hash(hash_node)
+      valid_keys = WorkatoConnectorBuilder::Constants::TOP_LEVEL_KEYS
+      process_hash(
+        hash_node,
+        valid_keys,
+        :first
+      )
+    end
+
+    def validate_first_child_level(pair)
+      ancestor_keys = get_ancestor_keys(pair, include_self: true)
+      first_level_keys = WorkatoConnectorBuilder::Constants::FIRST_CHILD_LEVEL_KEYS[ancestor_keys.first] || []
+
+      processed_hash = process_hash(
+        pair.value,
+        first_level_keys,
+        :second
+      )
+
+      unless processed_hash.nil?
+        update_node(pair, children: [pair.key, processed_hash])
+      end
+    end
+
+    def validate_second_child_level(pair)
+      parent_keys = get_ancestor_keys(pair, include_self: true)
+      # check the current hash, then check the root one, this covers :connection.authorization and :triggers for example
+      second_level_keys = WorkatoConnectorBuilder::Constants::SECOND_CHILD_LEVEL_KEYS[parent_keys.last] ||
+        WorkatoConnectorBuilder::Constants::SECOND_CHILD_LEVEL_KEYS[parent_keys.first] || []
+
+      if second_level_keys.empty?
+        update_node(pair, copy_children: true)
+      end
+
+      processed_hash = process_hash(
+        pair.value,
+        second_level_keys,
+        nil
+      )
+
+      unless processed_hash.nil?
+        pair.updated(:pair, [pair.key, processed_hash])
+      end
+    end
+
+    def update_node(node, copy_children: false, children: nil)
+      children = copy_children ? node.children : children
+      if children.nil?
+        nil
+      else
+        node.updated(node.type, children)
+      end
+    end
+
+    def process_hash(hash_node, valid_hash_keys, next_level_func)
+      kept_pairs = []
+      kept_invalid_pairs = []
+      hash_node.pairs.each do |pair|
+        parent_keys = get_ancestor_keys(pair)
+        message = "Error: '#{pair.key.value}' is not a valid key for "
+        if parent_keys.nil? || parent_keys.empty?
+          message << 'the root level'
+        else
+          parent_keys_label = parent_keys.join('.')
+          message << "'#{parent_keys_label}'"
+        end
+        if !valid_hash_keys.empty? && !valid_hash_keys.include?(pair.key.value.to_s.to_sym)
+          @shell.say "Error: #{message}", :red
+          choice = @shell.ask(
+            'Do you wish to keep the key => value in the final output?',
+            :limited_to => %w[yes no]
+          )
+          # Only valid pairs need to have their children checked
+          if choice == 'yes'
+            # This will reset the sibling relationships
+            kept_invalid_pairs << update_node(pair, copy_children: true)
+          end
+
+          next
+        end
+
+        if pair.value.hash_type? && !next_level_func.nil?
+          processed_pair = self.send("validate_#{next_level_func}_child_level", pair)
+          unless processed_pair.nil?
+            kept_pairs << processed_pair
+          end
+        else
+          # This will reset the sibling relationships
+          kept_pairs << update_node(pair, copy_children: true)
+        end
+      end
+
+      kept_pairs.sort_by! { |p| sort_by_valid_keys(p, valid_hash_keys) }
+
+      kept_invalid_pairs.sort! { |a, b| a.key <=> b.key }
+
+      byebug
+      hash_node.updated(:hash, kept_pairs.concat(kept_invalid_pairs))
+    end
+
+    def sort_by_valid_keys(p, keys)
+      key = p.key.value.to_s.to_sym
+      keys.index(key) || key
+    end
+
+    def get_ancestor_keys(pair, include_self: false)
+      if pair.root?
+        return []
+      end
+
+      keys = include_self ? [pair.key.value.to_s.to_sym] : []
+      parent = pair.parent
+      while parent != nil
+        if parent.pair_type?
+          keys << parent.key.value.to_s.to_sym
+        end
+        parent = parent.parent
+      end
+
+      keys.reverse!
+
+      keys
+    end
+  end
+end

data/lib/workato_connector_builder.rb

@@ -0,0 +1,4 @@
+module WorkatoConnectorBuilder
+  require_relative 'workato_connector_builder/hash_combiner'
+  require_relative 'workato_connector_builder/ast_generator'
+end

data/workato-connector-builder.gemspec

@@ -0,0 +1,27 @@
+#!/usr/bin/env gem build
+# encoding: utf-8
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+  spec.name = 'workato-connector-builder'
+  spec.description = 'A tool for building Workato Connectors from multiple hash files'
+  spec.summary = spec.description
+  spec.authors = [ 'Steven Laroche', ]
+  spec.version = '0.0.1.pre'
+  spec.license = 'MIT'
+
+  spec.files = Dir['bin/*', 'lib/**/*.rb', 'workato-connector-builder.gemspec', 'LICENSE.txt']
+  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = [ 'lib' ]
+  spec.metadata = { 'issue_tracker' => 'https://www.github.com/johndoe/missing/issues' }
+
+
+  spec.required_ruby_version= '>= 2.0.0'
+
+  spec.add_runtime_dependency 'rubocop-ast', '~> 1.12'
+  spec.add_runtime_dependency 'thor', '~> 1.1'
+
+  spec.add_development_dependency 'pry-byebug', '~> 3.9'
+  spec.add_development_dependency 'rbs', '~> 1.6'
+  spec.add_development_dependency 'rspec', '~> 3.10'
+end

metadata

@@ -0,0 +1,127 @@
+--- !ruby/object:Gem::Specification
+name: workato-connector-builder
+version: !ruby/object:Gem::Version
+  version: 0.0.1.pre
+platform: ruby
+authors:
+- Steven Laroche
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2021-10-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rubocop-ast
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.9'
+- !ruby/object:Gem::Dependency
+  name: rbs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.10'
+description: A tool for building Workato Connectors from multiple hash files
+email: 
+executables:
+- workato-connector-builder
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/workato-connector-builder
+- lib/output.rb
+- lib/test.rb
+- lib/workato_connector_builder.rb
+- lib/workato_connector_builder/ast_generator.rb
+- lib/workato_connector_builder/cli/build_command.rb
+- lib/workato_connector_builder/cli/cli.rb
+- lib/workato_connector_builder/cli/config_store.rb
+- lib/workato_connector_builder/constants.rb
+- lib/workato_connector_builder/fix_newlines_rewriter.rb
+- lib/workato_connector_builder/hash_combiner.rb
+- lib/workato_connector_builder/validator.rb
+- workato-connector-builder.gemspec
+homepage: 
+licenses:
+- MIT
+metadata:
+  issue_tracker: https://www.github.com/johndoe/missing/issues
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.1.6
+signing_key: 
+specification_version: 4
+summary: A tool for building Workato Connectors from multiple hash files
+test_files: []