xapixctl 1.1.2 → 1.2.4

data/lib/xapixctl/phoenix_client/project_connection.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Xapixctl
+  module PhoenixClient
+    class ProjectConnection < OrganizationConnection
+      attr_reader :project
+
+      def initialize(connection, org, project)
+        super(connection, org)
+        @project = project
+      end
+
+      def project_resource(format: :hash, &block)
+        organization.resource('Project', @project, format: format, &block)
+      end
+
+      def organization
+        OrganizationConnection.new(@connection, @org)
+      end
+
+      def resource_types_for_export
+        @resource_types_for_export ||=
+          @connection.available_resource_types do |res|
+            res.on_success do |available_types|
+              prj_types = available_types.select { |desc| desc['context'] == 'Project' }
+              SUPPORTED_RESOURCE_TYPES & prj_types.map { |desc| desc['type'] }
+            end
+          end
+      end
+
+      # Notes on parameters:
+      # - Query parameters should be part of the URL
+      # - Path parameters should be marked with `{name}` in the URL, and values should be given in path_params hash
+      # - Headers should be given in headers hash
+      # - Cookies should be given in cookies hash
+      # - The body has to be given as a string
+      # - The required authentication schemes should be listed, referring to previously created schemes
+      #
+      # This returns a hash like the following:
+      #   "data_source" => { "id" => id, "resource_description" => resource_description }
+      #
+      # To successfully onboard a DB using the API, the following steps are needed:
+      #  1. setup the data source using add_rest_data_source.
+      #  2. retrieve a preview using preview_data_source using the id returned by previous step
+      #  3. confirm preview
+      #  4. call accept_data_source_preview to complete onboarding
+      #
+      def add_rest_data_source(http_method:, url:, path_params: {}, headers: {}, cookies: {}, body: nil, auth_schemes: [], &block)
+        data_source_details = {
+          data_source: {
+            http_method: http_method, url: url,
+            parameters: { path: path_params.to_query, header: headers.to_query, cookies: cookies.to_query, body: body },
+            auth_schemes: auth_schemes
+          }
+        }
+        result_handler(block).
+          run { @client[rest_data_source_path].post(data_source_details.to_json, content_type: :json) }
+      end
+
+      # Notes on parameters:
+      # - To call a data source which requires authentication, provide a hash with each required auth scheme as key and
+      #   as the value a reference to a previously created credential.
+      #   Example: { scheme_ref1 => credential_ref1, scheme_ref2 => credential_ref2 }
+      #
+      # This returns a hashified preview like the following:
+      #   { "preview" => {
+      #       "sample" => { "status" => integer, "body" => { ... }, "headers" => { ... }, "cookies" => { ... } },
+      #       "fetched_at" => Timestamp },
+      #     "data_source" => { "id" => id, "resource_description" => resource_description } }
+      #
+      def data_source_preview(data_source_id, authentications: {}, &block)
+        preview_data = {
+          authentications: authentications.map { |scheme, cred| { auth_scheme_id: scheme, auth_credential_id: cred } }
+        }
+        result_handler(block).
+          run { @client[data_source_preview_path(data_source_id)].post(preview_data.to_json, content_type: :json) }
+      end
+
+      def add_schema_import(spec_filename, &block)
+        spec_data = { schema_import: { file: File.new(spec_filename, 'r') } }
+        result_handler(block).
+          run { @client[schema_imports_path].post(spec_data) }
+      end
+
+      def update_schema_import(schema_import, spec_filename, &block)
+        spec_data = { schema_import: { file: File.new(spec_filename, 'r') } }
+        result_handler(block).
+          run { @client[schema_import_path(schema_import)].patch(spec_data) }
+      end
+
+      def pipeline_preview(pipeline_id, format: :hash, &block)
+        result_handler(block).
+          prepare_data(->(data) { data['pipeline_preview'] }).
+          formatter(PREVIEW_FORMATTERS[format]).
+          run { @client[pipeline_preview_path(pipeline_id)].get }
+      end
+
+      def endpoint_preview(endpoint_id, format: :hash, &block)
+        result_handler(block).
+          prepare_data(->(data) { data['endpoint_preview'] }).
+          formatter(PREVIEW_FORMATTERS[format]).
+          run { @client[endpoint_preview_path(endpoint_id)].get }
+      end
+
+      def stream_processor_preview(stream_processor_id, format: :hash, &block)
+        result_handler(block).
+          prepare_data(->(data) { data['stream_processor_preview'] }).
+          formatter(PREVIEW_FORMATTERS[format]).
+          run { @client[stream_processor_preview_path(stream_processor_id)].get }
+      end
+
+      def publish(&block)
+        result_handler(block).
+          run { @client[project_publications_path].post('') }
+      end
+
+      def logs(correlation_id, &block)
+        result_handler(block).
+          run { @client[project_logss_path(correlation_id)].get }
+      end
+
+      # This returns a hashified preview like the following:
+
+      def accept_data_source_preview(data_source_id, &block)
+        result_handler(block).
+          run { @client[data_source_preview_path(data_source_id)].patch('') }
+      end
+
+      def public_project_url
+        File.join(@connection.xapix_url, @org, @project)
+      end
+
+      private
+
+      def rest_data_source_path
+        "/projects/#{@org}/#{@project}/onboarding/data_sources/rest"
+      end
+
+      def data_source_preview_path(id)
+        "/projects/#{@org}/#{@project}/onboarding/data_sources/#{id}/preview"
+      end
+
+      def schema_imports_path
+        "/projects/#{@org}/#{@project}/onboarding/schema_imports"
+      end
+
+      def schema_import_path(schema_import)
+        "/projects/#{@org}/#{@project}/onboarding/schema_imports/#{schema_import}"
+      end
+
+      def resource_path(type, id)
+        "/projects/#{@org}/#{@project}/#{translate_type(type)}/#{id}"
+      end
+
+      def resources_path(type)
+        "/projects/#{@org}/#{@project}/#{translate_type(type)}"
+      end
+
+      def generic_resource_path
+        "projects/#{@org}/#{@project}/resource"
+      end
+
+      def pipeline_preview_path(pipeline)
+        "/projects/#{@org}/#{@project}/pipelines/#{pipeline}/preview"
+      end
+
+      def endpoint_preview_path(endpoint)
+        "/projects/#{@org}/#{@project}/endpoints/#{endpoint}/preview"
+      end
+
+      def stream_processor_preview_path(stream_processor)
+        "/projects/#{@org}/#{@project}/stream_processors/#{stream_processor}/preview"
+      end
+
+      def project_publications_path
+        "/projects/#{@org}/#{@project}/publications"
+      end
+
+      def project_logss_path(correlation_id)
+        "/projects/#{@org}/#{@project}/logs/#{correlation_id}"
+      end
+    end
+  end
+end

data/lib/xapixctl/phoenix_client/result_handler.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Xapixctl
+  module PhoenixClient
+    class ResultHandler
+      def initialize(default_success_handler:, default_error_handler:)
+        @success_handler = default_success_handler
+        @error_handler = default_error_handler
+        @result_handler = nil
+        yield self if block_given?
+      end
+
+      def on_success(&block); @success_handler = block; self; end
+
+      def on_error(&block); @error_handler = block; self; end
+
+      def prepare_data(proc); @result_handler = proc; self; end
+
+      def formatter(proc); @formatter = proc; self; end
+
+      def run
+        res = yield
+        res = res.present? ? JSON.parse(res) : res
+        res = @result_handler ? @result_handler.call(res) : res
+        res = @formatter ? @formatter.call(res) : res
+        @success_handler.call(res)
+      rescue RestClient::Exception => err
+        response = JSON.parse(err.response) rescue {}
+        @error_handler.call(err, response)
+      rescue SocketError, Errno::ECONNREFUSED => err
+        @error_handler.call(err, nil)
+      end
+    end
+  end
+end

data/lib/xapixctl/preview_cli.rb

@@ -3,9 +3,7 @@
 require 'xapixctl/base_cli'
 
 module Xapixctl
-  class Preview < BaseCli
-    option :org, aliases: "-o", desc: "Organization", required: true
-    option :project, aliases: "-p", desc: "Project", required: true
+  class PreviewCli < BaseCli
     option :format, aliases: "-f", default: 'text', enum: ['text', 'yaml', 'json'], desc: "Output format"
     desc "pipeline ID", "Preview a pipeline"
     long_desc <<-LONGDESC
@@ -17,16 +15,12 @@ module Xapixctl
 
       Examples:
       \x5> $ xapixctl preview pipeline -o xapix -p some-project pipeline
+      \x5> $ xapixctl preview pipeline -p xapix/some-project pipeline
     LONGDESC
     def pipeline(pipeline)
-      connection.pipeline_preview(pipeline, org: options[:org], project: options[:project], format: options[:format].to_sym) do |res|
-        res.on_success { |preview| puts preview }
-        res.on_error { |err, result| warn_api_error('could not fetch preview', err, result) }
-      end
+      say prj_connection.pipeline_preview(pipeline, format: options[:format].to_sym)
     end
 
-    option :org, aliases: "-o", desc: "Organization", required: true
-    option :project, aliases: "-p", desc: "Project", required: true
     option :format, aliases: "-f", default: 'text', enum: ['text', 'yaml', 'json'], desc: "Output format"
     desc "endpoint ID", "Preview an endpoint"
     long_desc <<-LONGDESC
@@ -36,16 +30,12 @@ module Xapixctl
 
       Examples:
       \x5> $ xapixctl preview endpoint -o xapix -p some-project endpoint
+      \x5> $ xapixctl preview endpoint -p xapix/some-project endpoint
     LONGDESC
     def endpoint(endpoint)
-      connection.endpoint_preview(endpoint, org: options[:org], project: options[:project], format: options[:format].to_sym) do |res|
-        res.on_success { |preview| puts preview }
-        res.on_error { |err, result| warn_api_error('could not fetch preview', err, result) }
-      end
+      say prj_connection.endpoint_preview(endpoint, format: options[:format].to_sym)
     end
 
-    option :org, aliases: "-o", desc: "Organization", required: true
-    option :project, aliases: "-p", desc: "Project", required: true
     option :format, aliases: "-f", default: 'text', enum: ['text', 'yaml', 'json'], desc: "Output format"
     desc "stream-processor ID", "Preview a stream processor"
     long_desc <<-LONGDESC
@@ -55,12 +45,10 @@ module Xapixctl
 
       Examples:
       \x5> $ xapixctl preview stream-processor -o xapix -p some-project processor
+      \x5> $ xapixctl preview stream-processor -p xapix/some-project processor
     LONGDESC
     def stream_processor(stream_processor)
-      connection.stream_processor_preview(stream_processor, org: options[:org], project: options[:project], format: options[:format].to_sym) do |res|
-        res.on_success { |preview| puts preview }
-        res.on_error { |err, result| warn_api_error('could not fetch preview', err, result) }
-      end
+      say prj_connection.stream_processor_preview(stream_processor, format: options[:format].to_sym)
     end
   end
 end

data/lib/xapixctl/sync_cli.rb

@@ -0,0 +1,241 @@
+# frozen_string_literal: true
+
+require 'xapixctl/base_cli'
+require 'pathname'
+require 'hashdiff'
+
+module Xapixctl
+  class SyncCli < BaseCli
+    class_option :credentials, desc: "Whether to include Credential resources in sync", type: :boolean, default: true
+    class_option :exclude_types, desc: "Resource types to exclude from sync", type: :array
+
+    desc "to-dir DIRECTORY", "Syncs resources in project to directory"
+    long_desc <<-LONGDESC
+      `xapixctl sync to-dir DIRECTORY -p org/prj` will export all resources of a given project and remove any additional resources from the directory.
+
+      With --no-credentials you can exclude all credentials from getting exported.
+
+      With --exclude-types you can specify any resource types besides Project you'd like to exclude.
+
+      When excluding types, the excluded types will be recorded in the sync directory in a file called .excluded_types, so that any future syncs will exclude those types.
+
+      Examples:
+      \x5> $ xapixctl sync to-dir ./project_dir -p xapix/some-project
+      \x5> $ xapixctl sync to-dir ./project_dir -p xapix/some-project --no-credentials
+      \x5> $ xapixctl sync to-dir ./project_dir -p xapix/some-project --exclude-types=ApiPublishing ApiPublishingRole Credential
+    LONGDESC
+    def to_dir(dir)
+      sync_path = SyncPath.new(shell, dir, prj_connection.resource_types_for_export, excluded_types)
+
+      res_details = prj_connection.project_resource
+      sync_path.write_file(generate_readme(res_details), 'README.md')
+      sync_path.write_resource_yaml(res_details, 'project')
+
+      sync_path.types_to_sync.each do |type|
+        res_path = sync_path.resource_path(type)
+        prj_connection.resource_ids(type).each do |res_id|
+          res_details = prj_connection.resource(type, res_id)
+          res_path.write_resource_yaml(res_details, res_id)
+        end
+        res_path.remove_outdated_resources
+      end
+      sync_path.update_excluded_types_file
+    end
+
+    desc "from-dir DIRECTORY", "Syncs resources in project from directory"
+    long_desc <<-LONGDESC
+      `xapixctl sync from-dir DIRECTORY -p org/prj` will import all resources into the given project from the directory and remove any additional resources which are not present in the directory.
+
+      With --no-credentials you can exclude all credentials from getting exported.
+
+      With --exclude-types you can specify any resource types besides Project you'd like to exclude.
+
+      Examples:
+      \x5> $ xapixctl sync from-dir ./project_dir -p xapix/some-project
+      \x5> $ xapixctl sync from-dir ./project_dir -p xapix/some-project --no-credentials
+      \x5> $ xapixctl sync from-dir ./project_dir -p xapix/some-project --exclude-types=ApiPublishing ApiPublishingRole Credential
+    LONGDESC
+    def from_dir(dir)
+      sync_path = SyncPath.new(shell, dir, prj_connection.resource_types_for_export, excluded_types)
+
+      sync_path.load_resource('project') do |desc|
+        say "applying #{desc['kind']} #{desc.dig('metadata', 'id')} to #{prj_connection.project}"
+        desc['metadata']['id'] = prj_connection.project
+        prj_connection.organization.apply(desc)
+      end
+
+      outdated_resources = {}
+      sync_path.types_to_sync.each do |type|
+        res_path = sync_path.resource_path(type)
+        updated_resource_ids = []
+        res_path.load_resources do |desc|
+          say "applying #{desc['kind']} #{desc.dig('metadata', 'id')}"
+          updated_resource_ids += prj_connection.apply(desc)
+        end
+        outdated_resources[type] = prj_connection.resource_ids(type) - updated_resource_ids
+      end
+
+      outdated_resources.each do |type, resource_ids|
+        resource_ids.each do |resource_id|
+          say "removing #{type} #{resource_id}"
+          prj_connection.delete(type, resource_id)
+        end
+      end
+    end
+
+    desc "diff DIRECTORY", "List resource which differ between project and directory"
+    long_desc <<-LONGDESC
+      `xapixctl sync diff DIRECTORY -p org/prj` will list the resources which are different between the given project and the given directory.
+
+      With --no-credentials you can exclude all credentials from getting exported.
+
+      With --exclude-types you can specify any resource types besides Project you'd like to exclude.
+
+      When only listing changed resources, the first character in a line indicates the status of the resource:
+      \x5 = - no changes
+      \x5 ~ - changed
+      \x5 ^ - in remote project
+      \x5 v - in directory
+
+      Examples:
+      \x5> $ xapixctl sync diff ./project_dir -p xapix/some-project
+      \x5> $ xapixctl sync diff ./project_dir -p xapix/some-project --no-credentials
+      \x5> $ xapixctl sync diff ./project_dir -p xapix/some-project --exclude-types=ApiPublishing ApiPublishingRole Credential
+    LONGDESC
+    option :details, desc: "Include detailed differences", type: :boolean, default: false
+    def diff(dir)
+      sync_path = SyncPath.new(shell, dir, prj_connection.resource_types_for_export, excluded_types)
+
+      sync_path.load_resource('project') do |desc|
+        desc['metadata']['id'] = prj_connection.project
+        res_details = prj_connection.project_resource
+        show_diff(desc, res_details)
+      end
+
+      sync_path.types_to_sync.each do |type|
+        res_path = sync_path.resource_path(type)
+        local_resource_ids = []
+        remote_resource_ids = prj_connection.resource_ids(type)
+        res_path.load_resources do |desc|
+          resource_id = desc['metadata']['id']
+          local_resource_ids << resource_id
+          if remote_resource_ids.include?(resource_id)
+            res_details = prj_connection.resource(type, desc['metadata']['id'])
+            show_diff(desc, res_details)
+          else
+            say "v #{type} #{resource_id}"
+          end
+        end
+        (remote_resource_ids - local_resource_ids).each do |resource_id|
+          say "^ #{type} #{resource_id}"
+        end
+      end
+    end
+
+    private
+
+    def show_diff(local, remote)
+      kind, id = local['kind'], local['metadata']['id']
+      local, remote = local.slice('definition'), remote.slice('definition')
+      changed = local != remote
+      status = changed ? "~" : "="
+      say "#{status} #{kind} #{id}"
+      return unless changed && options[:details]
+      shell.indent do
+        Hashdiff.diff(local, remote).each do |change|
+          status = change[0].tr('+-', '^v')
+          key = change[1]
+          say "#{status} #{key}"
+          shell.indent do
+            case status
+            when "~" then say "^ #{change[3]}"; say "v #{change[2]}"
+            else say "#{status} #{change[2]}" if change[2]
+            end
+          end
+        end
+      end
+    end
+
+    class ResourcePath
+      delegate :say, to: :@shell
+
+      def initialize(shell, path)
+        @shell = shell
+        @path = path
+        @resource_files = []
+      end
+
+      def write_file(content, filename)
+        @path.mkpath
+        unless @path.directory? && @path.writable?
+          warn "Cannot write to #{@path}, please check directory exists and is writable"
+          exit 1
+        end
+        file = @path.join(filename)
+        file.write(content)
+        say "updated #{file}..."
+        file
+      end
+
+      def write_resource_yaml(res_details, res_name)
+        file = write_file(res_details.to_yaml, "#{res_name}.yaml")
+        @resource_files << file
+        file
+      end
+
+      def load_resources(&block)
+        Util.resources_from_file(@path, ignore_missing: true, &block)
+      end
+
+      def load_resource(res_name, &block)
+        Util.resources_from_file(@path.join("#{res_name}.yaml"), ignore_missing: false, &block)
+      end
+
+      def remove_outdated_resources
+        (@path.glob('*.yaml') - @resource_files).each do |outdated_file|
+          outdated_file.delete
+          say "removed #{outdated_file}"
+        end
+      end
+    end
+
+    class SyncPath < ResourcePath
+      attr_reader :types_to_sync
+
+      def initialize(shell, dir, all_types, excluded_types)
+        super(shell, Pathname.new(dir))
+        @all_types = all_types
+        @excluded_types_file = @path.join('.excluded_types')
+        @excluded_types = excluded_types || []
+        @excluded_types += @excluded_types_file.read.split if @excluded_types_file.exist?
+        @excluded_types &= @all_types
+        @excluded_types.sort!
+        @types_to_sync = @all_types - @excluded_types
+        say "Resource types excluded from sync: #{@excluded_types.join(', ')}" if @excluded_types.any?
+      end
+
+      def resource_path(type)
+        ResourcePath.new(@shell, @path.join(type.underscore))
+      end
+
+      def update_excluded_types_file
+        @excluded_types_file.write(@excluded_types.join(" ") + "\n") if @excluded_types.any?
+      end
+    end
+
+    def excluded_types
+      excluded = options[:exclude_types] || []
+      excluded += ['Credential'] unless options[:credentials]
+      excluded
+    end
+
+    def generate_readme(res_details)
+      <<~EOREADME
+        # #{res_details.dig('definition', 'name')}
+        #{res_details.dig('definition', 'description')}
+
+        Project exported from #{File.join(prj_connection.public_project_url)} by xapixctl v#{Xapixctl::VERSION}.
+      EOREADME
+    end
+  end
+end