xapixctl 1.1.0 → 1.2.2

data/lib/xapixctl/phoenix_client/project_connection.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Xapixctl
+  module PhoenixClient
+    class ProjectConnection < OrganizationConnection
+      attr_reader :project
+
+      def initialize(connection, org, project)
+        super(connection, org)
+        @project = project
+      end
+
+      def project_resource(format: :hash, &block)
+        organization.resource('Project', @project, format: format, &block)
+      end
+
+      def organization
+        OrganizationConnection.new(@connection, @org)
+      end
+
+      def resource_types_for_export
+        @resource_types_for_export ||=
+          @connection.available_resource_types do |res|
+            res.on_success do |available_types|
+              prj_types = available_types.select { |desc| desc['context'] == 'Project' }
+              SUPPORTED_RESOURCE_TYPES & prj_types.map { |desc| desc['type'] }
+            end
+          end
+      end
+
+      # Notes on parameters:
+      # - Query parameters should be part of the URL
+      # - Path parameters should be marked with `{name}` in the URL, and values should be given in path_params hash
+      # - Headers should be given in headers hash
+      # - Cookies should be given in cookies hash
+      # - The body has to be given as a string
+      # - The required authentication schemes should be listed, referring to previously created schemes
+      #
+      # This returns a hash like the following:
+      #   "data_source" => { "id" => id, "resource_description" => resource_description }
+      #
+      # To successfully onboard a DB using the API, the following steps are needed:
+      #  1. setup the data source using add_rest_data_source.
+      #  2. retrieve a preview using preview_data_source using the id returned by previous step
+      #  3. confirm preview
+      #  4. call accept_data_source_preview to complete onboarding
+      #
+      def add_rest_data_source(http_method:, url:, path_params: {}, headers: {}, cookies: {}, body: nil, auth_schemes: [], &block)
+        data_source_details = {
+          data_source: {
+            http_method: http_method, url: url,
+            parameters: { path: path_params.to_query, header: headers.to_query, cookies: cookies.to_query, body: body },
+            auth_schemes: auth_schemes
+          }
+        }
+        result_handler(block).
+          run { @client[rest_data_source_path].post(data_source_details.to_json, content_type: :json) }
+      end
+
+      # Notes on parameters:
+      # - To call a data source which requires authentication, provide a hash with each required auth scheme as key and
+      #   as the value a reference to a previously created credential.
+      #   Example: { scheme_ref1 => credential_ref1, scheme_ref2 => credential_ref2 }
+      #
+      # This returns a hashified preview like the following:
+      #   { "preview" => {
+      #       "sample" => { "status" => integer, "body" => { ... }, "headers" => { ... }, "cookies" => { ... } },
+      #       "fetched_at" => Timestamp },
+      #     "data_source" => { "id" => id, "resource_description" => resource_description } }
+      #
+      def data_source_preview(data_source_id, authentications: {}, &block)
+        preview_data = {
+          authentications: authentications.map { |scheme, cred| { auth_scheme_id: scheme, auth_credential_id: cred } }
+        }
+        result_handler(block).
+          run { @client[data_source_preview_path(data_source_id)].post(preview_data.to_json, content_type: :json) }
+      end
+
+      def add_schema_import(spec_filename, &block)
+        spec_data = { schema_import: { file: File.new(spec_filename, 'r') } }
+        result_handler(block).
+          run { @client[schema_imports_path].post(spec_data) }
+      end
+
+      def update_schema_import(schema_import, spec_filename, &block)
+        spec_data = { schema_import: { file: File.new(spec_filename, 'r') } }
+        result_handler(block).
+          run { @client[schema_import_path(schema_import)].patch(spec_data) }
+      end
+
+      def pipeline_preview(pipeline_id, format: :hash, &block)
+        result_handler(block).
+          prepare_data(->(data) { data['pipeline_preview'] }).
+          formatter(PREVIEW_FORMATTERS[format]).
+          run { @client[pipeline_preview_path(pipeline_id)].get }
+      end
+
+      def endpoint_preview(endpoint_id, format: :hash, &block)
+        result_handler(block).
+          prepare_data(->(data) { data['endpoint_preview'] }).
+          formatter(PREVIEW_FORMATTERS[format]).
+          run { @client[endpoint_preview_path(endpoint_id)].get }
+      end
+
+      def stream_processor_preview(stream_processor_id, format: :hash, &block)
+        result_handler(block).
+          prepare_data(->(data) { data['stream_processor_preview'] }).
+          formatter(PREVIEW_FORMATTERS[format]).
+          run { @client[stream_processor_preview_path(stream_processor_id)].get }
+      end
+
+      def publish(&block)
+        result_handler(block).
+          run { @client[project_publications_path].post('') }
+      end
+
+      def logs(correlation_id, &block)
+        result_handler(block).
+          run { @client[project_logss_path(correlation_id)].get }
+      end
+
+      # This returns a hashified preview like the following:
+
+      def accept_data_source_preview(data_source_id, &block)
+        result_handler(block).
+          run { @client[data_source_preview_path(data_source_id)].patch('') }
+      end
+
+      def public_project_url
+        File.join(@connection.xapix_url, @org, @project)
+      end
+
+      private
+
+      def rest_data_source_path
+        "/projects/#{@org}/#{@project}/onboarding/data_sources/rest"
+      end
+
+      def data_source_preview_path(id)
+        "/projects/#{@org}/#{@project}/onboarding/data_sources/#{id}/preview"
+      end
+
+      def schema_imports_path
+        "/projects/#{@org}/#{@project}/onboarding/schema_imports"
+      end
+
+      def schema_import_path(schema_import)
+        "/projects/#{@org}/#{@project}/onboarding/schema_imports/#{schema_import}"
+      end
+
+      def resource_path(type, id)
+        "/projects/#{@org}/#{@project}/#{translate_type(type)}/#{id}"
+      end
+
+      def resources_path(type)
+        "/projects/#{@org}/#{@project}/#{translate_type(type)}"
+      end
+
+      def generic_resource_path
+        "projects/#{@org}/#{@project}/resource"
+      end
+
+      def pipeline_preview_path(pipeline)
+        "/projects/#{@org}/#{@project}/pipelines/#{pipeline}/preview"
+      end
+
+      def endpoint_preview_path(endpoint)
+        "/projects/#{@org}/#{@project}/endpoints/#{endpoint}/preview"
+      end
+
+      def stream_processor_preview_path(stream_processor)
+        "/projects/#{@org}/#{@project}/stream_processors/#{stream_processor}/preview"
+      end
+
+      def project_publications_path
+        "/projects/#{@org}/#{@project}/publications"
+      end
+
+      def project_logss_path(correlation_id)
+        "/projects/#{@org}/#{@project}/logs/#{correlation_id}"
+      end
+    end
+  end
+end

data/lib/xapixctl/phoenix_client/result_handler.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Xapixctl
+  module PhoenixClient
+    class ResultHandler
+      def initialize(default_success_handler:, default_error_handler:)
+        @success_handler = default_success_handler
+        @error_handler = default_error_handler
+        @result_handler = nil
+        yield self if block_given?
+      end
+
+      def on_success(&block); @success_handler = block; self; end
+
+      def on_error(&block); @error_handler = block; self; end
+
+      def prepare_data(proc); @result_handler = proc; self; end
+
+      def formatter(proc); @formatter = proc; self; end
+
+      def run
+        res = yield
+        res = res.present? ? JSON.parse(res) : res
+        res = @result_handler ? @result_handler.call(res) : res
+        res = @formatter ? @formatter.call(res) : res
+        @success_handler.call(res)
+      rescue RestClient::Exception => err
+        response = JSON.parse(err.response) rescue {}
+        @error_handler.call(err, response)
+      rescue SocketError, Errno::ECONNREFUSED => err
+        @error_handler.call(err, nil)
+      end
+    end
+  end
+end

data/lib/xapixctl/preview_cli.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'xapixctl/base_cli'
+
+module Xapixctl
+  class PreviewCli < BaseCli
+    option :format, aliases: "-f", default: 'text', enum: ['text', 'yaml', 'json'], desc: "Output format"
+    desc "pipeline ID", "Preview a pipeline"
+    long_desc <<-LONGDESC
+      `xapixctl preview pipeline` will return a preview of the given pipeline.
+
+      The preview function will not call any external data sources but calculate a preview based on the provided sample data.
+
+      To preview a pipeline attached to an endpoint, please use `xapixctl preview endpoint` to see the correct preview.
+
+      Examples:
+      \x5> $ xapixctl preview pipeline -o xapix -p some-project pipeline
+      \x5> $ xapixctl preview pipeline -p xapix/some-project pipeline
+    LONGDESC
+    def pipeline(pipeline)
+      puts prj_connection.pipeline_preview(pipeline, format: options[:format].to_sym)
+    end
+
+    option :format, aliases: "-f", default: 'text', enum: ['text', 'yaml', 'json'], desc: "Output format"
+    desc "endpoint ID", "Preview an endpoint"
+    long_desc <<-LONGDESC
+      `xapixctl preview endpoint` will return a preview of the given endpoint.
+
+      The preview function will not call any external data sources but calculate a preview based on the provided sample data.
+
+      Examples:
+      \x5> $ xapixctl preview endpoint -o xapix -p some-project endpoint
+      \x5> $ xapixctl preview endpoint -p xapix/some-project endpoint
+    LONGDESC
+    def endpoint(endpoint)
+      puts prj_connection.endpoint_preview(endpoint, format: options[:format].to_sym)
+    end
+
+    option :format, aliases: "-f", default: 'text', enum: ['text', 'yaml', 'json'], desc: "Output format"
+    desc "stream-processor ID", "Preview a stream processor"
+    long_desc <<-LONGDESC
+      `xapixctl preview stream-processor` will return a preview of the given stream processor.
+
+      The preview function will not call any external data sources but calculate a preview based on the provided sample data.
+
+      Examples:
+      \x5> $ xapixctl preview stream-processor -o xapix -p some-project processor
+      \x5> $ xapixctl preview stream-processor -p xapix/some-project processor
+    LONGDESC
+    def stream_processor(stream_processor)
+      puts prj_connection.stream_processor_preview(stream_processor, format: options[:format].to_sym)
+    end
+  end
+end

data/lib/xapixctl/sync_cli.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require 'xapixctl/base_cli'
+require 'pathname'
+
+module Xapixctl
+  class SyncCli < BaseCli
+    class_option :credentials, desc: "Whether to include Credential resources in sync", type: :boolean, default: true
+    class_option :exclude_types, desc: "Resource types to exclude from sync", type: :array
+
+    desc "to-dir DIRECTORY", "Syncs resources in project to directory"
+    long_desc <<-LONGDESC
+      `xapixctl sync to-dir DIRECTORY` will export all resources of a given project and remove any additional resources from the directory.
+
+      With --no-credentials you can exclude all credentials from getting exported.
+
+      With --exclude-types you can specify any resource types besides Project you'd like to exclude.
+
+      When excluding types, the excluded types will be recorded in the sync directory in a file called .excluded_types, so that any future syncs will exclude those types.
+
+      Examples:
+      \x5> $ xapixctl sync to-dir ./project_dir -p xapix/some-project
+      \x5> $ xapixctl sync to-dir ./project_dir -p xapix/some-project --no-credentials
+      \x5> $ xapixctl sync to-dir ./project_dir -p xapix/some-project --exclude-types=ApiPublishing ApiPublishingRole Credential
+    LONGDESC
+    def to_dir(dir)
+      sync_path = SyncPath.new(dir, prj_connection.resource_types_for_export, excluded_types)
+
+      res_details = prj_connection.project_resource
+      sync_path.write_file(generate_readme(res_details), 'README.md')
+      sync_path.write_resource_yaml(res_details, 'project')
+
+      sync_path.types_to_sync.each do |type|
+        res_path = sync_path.resource_path(type)
+        prj_connection.resource_ids(type).each do |res_id|
+          res_details = prj_connection.resource(type, res_id)
+          res_path.write_resource_yaml(res_details, res_id)
+        end
+        res_path.remove_outdated_resources
+      end
+      sync_path.update_excluded_types_file
+    end
+
+    desc "from-dir DIRECTORY", "Syncs resources in project from directory"
+    long_desc <<-LONGDESC
+      `xapixctl sync from-dir project dir` will import all resources into the given project from the directory and remove any additional resources which are not present in the directory.
+
+      With --no-credentials you can exclude all credentials from getting exported.
+
+      With --exclude-types you can specify any resource types besides Project you'd like to exclude.
+
+      Examples:
+      \x5> $ xapixctl sync from-dir ./project_dir -p xapix/some-project
+      \x5> $ xapixctl sync from-dir ./project_dir -p xapix/some-project --no-credentials
+      \x5> $ xapixctl sync from-dir ./project_dir -p xapix/some-project --exclude-types=ApiPublishing ApiPublishingRole Credential
+    LONGDESC
+    def from_dir(dir)
+      sync_path = SyncPath.new(dir, prj_connection.resource_types_for_export, excluded_types)
+
+      sync_path.load_resource('project') do |desc|
+        puts "applying #{desc['kind']} #{desc.dig('metadata', 'id')} to #{prj_connection.project}"
+        desc['metadata']['id'] = prj_connection.project
+        prj_connection.organization.apply(desc)
+      end
+
+      outdated_resources = {}
+      sync_path.types_to_sync.each do |type|
+        res_path = sync_path.resource_path(type)
+        updated_resource_ids = []
+        res_path.load_resources do |desc|
+          puts "applying #{desc['kind']} #{desc.dig('metadata', 'id')}"
+          updated_resource_ids += prj_connection.apply(desc)
+        end
+        outdated_resources[type] = prj_connection.resource_ids(type) - updated_resource_ids
+      end
+
+      outdated_resources.each do |type, resource_ids|
+        resource_ids.each do |resource_id|
+          puts "removing #{type} #{resource_id}"
+          prj_connection.delete(type, resource_id)
+        end
+      end
+    end
+
+    private
+
+    class ResourcePath
+      def initialize(path)
+        @path = path
+        @resource_files = []
+      end
+
+      def write_file(content, filename)
+        @path.mkpath
+        unless @path.directory? && @path.writable?
+          warn "Cannot write to #{@path}, please check directory exists and is writable"
+          exit 1
+        end
+        file = @path.join(filename)
+        file.write(content)
+        puts "updated #{file}..."
+        file
+      end
+
+      def write_resource_yaml(res_details, res_name)
+        file = write_file(res_details.to_yaml, "#{res_name}.yaml")
+        @resource_files << file
+        file
+      end
+
+      def load_resources(&block)
+        Util.resources_from_file(@path, ignore_missing: true, &block)
+      end
+
+      def load_resource(res_name, &block)
+        Util.resources_from_file(@path.join("#{res_name}.yaml"), ignore_missing: false, &block)
+      end
+
+      def remove_outdated_resources
+        (@path.glob('*.yaml') - @resource_files).each do |outdated_file|
+          outdated_file.delete
+          puts "removed #{outdated_file}"
+        end
+      end
+    end
+
+    class SyncPath < ResourcePath
+      attr_reader :types_to_sync
+
+      def initialize(dir, all_types, excluded_types)
+        super(Pathname.new(dir))
+        @all_types = all_types
+        @excluded_types_file = @path.join('.excluded_types')
+        @excluded_types = excluded_types || []
+        @excluded_types += @excluded_types_file.read.split if @excluded_types_file.exist?
+        @excluded_types &= @all_types
+        @excluded_types.sort!
+        @types_to_sync = @all_types - @excluded_types
+        puts "Resource types excluded from sync: #{@excluded_types.join(', ')}" if @excluded_types.any?
+      end
+
+      def resource_path(type)
+        ResourcePath.new(@path.join(type.underscore))
+      end
+
+      def update_excluded_types_file
+        @excluded_types_file.write(@excluded_types.join(" ") + "\n") if @excluded_types.any?
+      end
+    end
+
+    def excluded_types
+      excluded = options[:exclude_types]
+      excluded += ['Credential'] unless options[:credentials]
+      excluded
+    end
+
+    def generate_readme(res_details)
+      <<~EOREADME
+        # #{res_details.dig('definition', 'name')}
+        #{res_details.dig('definition', 'description')}
+
+        Project exported from #{File.join(prj_connection.public_project_url)} by xapixctl v#{Xapixctl::VERSION}.
+      EOREADME
+    end
+  end
+end

data/lib/xapixctl/titan_cli.rb

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+
+require 'xapixctl/base_cli'
+
+module Xapixctl
+  class TitanCli < BaseCli
+    DEFAULT_METHODS = {
+      "predict" => :predict,
+      "performance" => :performance,
+    }.freeze
+
+    option :schema_import, desc: "Resource id of an existing Schema import"
+    option :data, desc: "JSON encoded data for predict API", default: "[[1,2,3]]"
+    desc "service URL", "build a service for the deployed model"
+    long_desc <<-LONGDESC
+      `xapixctl titan service URL` will build a ML service around a deployed ML model.
+
+      We expect the following of the deployed ML model:
+
+      - There should be a "POST /predict" endpoint. Use --data="JSON" to specify an example dataset the model expects.
+
+      - If there is a "GET /performance" endpoint, it'll be made available. It's expected to return a JSON object with the properties 'accuracy', 'precision', 'recall', and 'min_acc_threshold'.
+
+      Examples:
+      \x5> $ xapixctl titan service https://services.demo.akoios.com/ai-model-name -p xapix/ml-project
+    LONGDESC
+    def service(akoios_url)
+      url = URI.parse(akoios_url)
+      schema = JSON.parse(RestClient.get(File.join(url.to_s, 'spec'), params: { host: url.hostname }, headers: { accept: :json }))
+      patch_schema(schema)
+      connector_refs = import_swagger(File.basename(url.path), schema)
+      say "\n== Onboarding Connectors", :bold
+      connectors = match_connectors_to_action(connector_refs)
+      say "\n== Building Service", :bold
+      service_doc = build_service(schema.dig('info', 'title'), connectors)
+      res = prj_connection.apply(service_doc)
+      say "\ncreated / updated service #{res.first}"
+    end
+
+    private
+
+    def patch_schema(schema)
+      predict_schema = schema.dig('paths', '/predict', 'post')
+      if predict_schema
+        predict_data = JSON.parse(options[:data]) rescue {}
+        predict_schema['operationId'] = 'predict'
+        predict_schema['parameters'].each do |param|
+          if param['name'] == 'json' && param['in'] == 'body'
+            param['schema']['properties'] = { "data" => extract_schema(predict_data) }
+            param['schema']['example'] = { "data" => predict_data }
+          end
+        end
+      end
+
+      performane_schema = schema.dig('paths', '/performance', 'get')
+      if performane_schema
+        performane_schema['operationId'] = 'performance'
+      end
+    end
+
+    def import_swagger(filename, schema)
+      Tempfile.create([filename, '.json']) do |f|
+        f.write(schema.to_json)
+        f.rewind
+
+        if options[:schema_import]
+          result = prj_connection.update_schema_import(options[:schema_import], f)
+          say "updated #{result.dig('resource', 'kind')} #{result.dig('resource', 'id')}"
+        else
+          result = prj_connection.add_schema_import(f)
+          say "created #{result.dig('resource', 'kind')} #{result.dig('resource', 'id')}"
+        end
+        result.dig('schema_import', 'updated_resources')
+      end
+    end
+
+    def match_connectors_to_action(connector_refs)
+      connector_refs.map do |connector_ref|
+        connector = prj_connection.resource(connector_ref['kind'], connector_ref['id'])
+        action = DEFAULT_METHODS[connector.dig('definition', 'name')]
+        say "\n#{connector['kind']} #{connector.dig('definition', 'name')} -> "
+        if action
+          say "#{action} action"
+          [action, update_connector_with_preview(connector)]
+        else
+          say "no action type detected, ignoring"
+          nil
+        end
+      end.compact
+    end
+
+    def update_connector_with_preview(connector)
+      say "fetching preview for #{connector['kind']} #{connector.dig('definition', 'name')}..."
+      preview_details = prj_connection.data_source_preview(connector.dig('metadata', 'id'))
+      preview = preview_details.dig('preview', 'sample')
+      say "got a #{preview['status']} response: #{preview['body']}"
+      if preview['status'] != 200
+        say "unexpected status, please check data or model"
+      elsif yes?("Does this look alright?", :bold)
+        res = prj_connection.accept_data_source_preview(connector.dig('metadata', 'id'))
+        return res.dig('data_source', 'resource_description')
+      end
+      connector
+    end
+
+    def extract_schema(data_sample)
+      case data_sample
+      when Array
+        { type: 'array', items: extract_schema(data_sample[0]) }
+      when Hash
+        { type: 'object', properties: data_sample.transform_values { |v| extract_schema(v) } }
+      when Numeric
+        { type: 'number' }
+      else
+        {}
+      end
+    end
+
+    def build_service(title, connectors)
+      {
+        version: 'v1',
+        kind: 'Service',
+        metadata: { id: title.parameterize },
+        definition: {
+          name: title.humanize,
+          actions: connectors.map { |action, connector| build_service_action(action, connector) }
+        }
+      }
+    end
+
+    def build_service_action(action_type, connector)
+      {
+        name: action_type,
+        parameter_schema: parameter_schema(action_type, connector),
+        result_schema: result_schema(action_type),
+        pipeline: { units: pipeline_units(action_type, connector) }
+      }
+    end
+
+    def parameter_schema(action_type, connector)
+      case action_type
+      when :predict
+        { type: 'object', properties: {
+          data: connector.dig('definition', 'parameter_schema', 'properties', 'body', 'properties', 'data'),
+        } }
+      when :performance
+        { type: 'object', properties: {} }
+      end
+    end
+
+    def result_schema(action_type)
+      case action_type
+      when :predict
+        { type: 'object', properties: {
+          prediction: { type: 'object', properties: { percent: { type: 'number' }, raw: { type: 'number' } } },
+          success: { type: 'boolean' },
+          error: { type: 'string' }
+        } }
+      when :performance
+        { type: 'object', properties: {
+          performance: {
+            type: 'object', properties: {
+              accuracy: { type: 'number' },
+              precision: { type: 'number' },
+              recall: { type: 'number' },
+              min_acc_threshold: { type: 'number' }
+            }
+          }
+        } }
+      end
+    end
+
+    def pipeline_units(action_type, connector)
+      case action_type
+      when :predict
+        [entry_unit, predict_unit(connector), predict_result_unit]
+      when :performance
+        [entry_unit, performance_unit(connector), performance_result_unit]
+      end
+    end
+
+    def entry_unit
+      { version: 'v2', kind: 'Unit/Entry',
+        metadata: { id: 'entry' },
+        definition: { name: 'Entry' } }
+    end
+
+    def predict_unit(connector)
+      leafs = extract_leafs(connector.dig('definition', 'parameter_schema', 'properties', 'body'))
+      { version: 'v2', kind: 'Unit/DataSource',
+        metadata: { id: 'predict' },
+        definition: {
+          name: 'Predict',
+          inputs: ['entry'],
+          data_source: connector.dig('metadata', 'id'),
+          formulas: leafs.map { |leaf|
+            { ref: leaf[:node]['$id'], formula: (['', 'entry'] + leaf[:path]).join('.') }
+          }
+        } }
+    end
+
+    def predict_result_unit
+      { version: 'v2', kind: 'Unit/Result',
+        metadata: { id: 'result' },
+        definition: {
+          name: 'Result',
+          inputs: ['predict'],
+          formulas: [{
+            ref: "#prediction.raw",
+            formula: "IF(.predict.status = 200, ROUND(100*coerce.to-float(.predict.body), 2))"
+          }, {
+            ref: "#prediction.percent",
+            formula: "IF(.predict.status = 200, coerce.to-float(.predict.body))"
+          }, {
+            ref: "#success",
+            formula: ".predict.status = 200"
+          }, {
+            ref: "#error",
+            formula: "IF(.predict.status <> 200, 'Model not trained!')"
+          }],
+          parameter_sample: {
+            "prediction" => { "percent" => 51, "raw" => 0.5112131 },
+            "success" => true,
+            "error" => nil
+          }
+        } }
+    end
+
+    def performance_unit(connector)
+      { version: 'v2', kind: 'Unit/DataSource',
+        metadata: { id: 'performance' },
+        definition: {
+          name: 'Performance',
+          inputs: ['entry'],
+          data_source: connector.dig('metadata', 'id')
+        } }
+    end
+
+    def performance_result_unit
+      { version: 'v2', kind: 'Unit/Result',
+        metadata: { id: 'result' },
+        definition: {
+          name: 'Result',
+          inputs: ['performance'],
+          formulas: [{
+            ref: "#performance.recall",
+            formula: "WITH(data, JSON.DECODE(REGEXREPLACE(performance.body, \"'\", \"\\\"\")), .data.recall)"
+          }, {
+            ref: "#performance.accuracy",
+            formula: "WITH(data, JSON.DECODE(REGEXREPLACE(performance.body, \"'\", \"\\\"\")), .data.accuracy)"
+          }, {
+            ref: "#performance.precision",
+            formula: "WITH(data, JSON.DECODE(REGEXREPLACE(performance.body, \"'\", \"\\\"\")), .data.precision)"
+          }, {
+            ref: "#performance.min_acc_threshold",
+            formula: "WITH(data, JSON.DECODE(REGEXREPLACE(performance.body, \"'\", \"\\\"\")), .data.min_acc_threshold)"
+          }],
+          parameter_sample: {
+            "performance" => {
+              "recall" => 0.8184713375796179,
+              "accuracy" => 0.9807847896440129,
+              "precision" => 0.8711864406779661,
+              "min_acc_threshold" => 0.84,
+            }
+          }
+        } }
+    end
+
+    def extract_leafs(schema, current_path = [])
+      return unless schema
+      case schema['type']
+      when 'object'
+        schema['properties'].flat_map { |key, sub_schema| extract_leafs(sub_schema, current_path + [key]) }.compact
+      when 'array'
+        extract_leafs(schema['items'], current_path + [:[]])
+      else
+        { path: current_path, node: schema }
+      end
+    end
+  end
+end