nose-cli 0.1.0pre

data/lib/nose_cli/proxy.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add a command to run a proxy server capable of executing queries
+    class NoSECLI < Thor
+      desc 'proxy PLAN_FILE', 'start a proxy with the given PLAN_FILE'
+
+      long_desc <<-LONGDESC
+        `nose proxy` loads the schema and execution plans from the given file.
+        It then starts the configured proxy server which is able to execute
+        the given set of plans. Available proxy servers can be seen in the
+        `lib/nose/proxy` subdirectory.
+      LONGDESC
+
+      def proxy(plan_file)
+        result = load_results plan_file
+        backend = get_backend(options, result)
+
+        # Create a new instance of the proxy class
+        proxy_class = get_class 'proxy', options
+        proxy = proxy_class.new options[:proxy], result, backend
+
+        # Start the proxy server
+        trap 'INT' do
+          proxy.stop
+        end
+        proxy.start
+      end
+    end
+  end
+end

data/lib/nose_cli/random_plans.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add a command to generate a graphic of the schema from a workload
+    class NoSECLI < Thor
+      desc 'random-plans WORKLOAD TAG',
+           'output random plans for the statement with TAG in WORKLOAD'
+
+      long_desc <<-LONGDESC
+        `nose random-plans` produces a set of randomly chosen plans for a given
+        workload. This is useful when evaluating the cost model for a given
+        system as this should give a good range of different execution plans.
+      LONGDESC
+
+      shared_option :format
+      shared_option :output
+
+      option :count, type: :numeric, default: 5, aliases: '-n',
+                     desc: 'the number of random plans to produce'
+      option :all, type: :boolean, default: false, aliases: '-a',
+                   desc: 'whether to include all possible plans, if given ' \
+                         'then --count is ignored'
+
+      def random_plans(workload_name, tag)
+        # Find the statement with the given tag
+        workload = Workload.load workload_name
+        statement = workload.find_with_tag tag
+
+        # Generate a random set of plans
+        indexes = IndexEnumerator.new(workload).indexes_for_workload
+        cost_model = get_class('cost', options[:cost_model][:name]) \
+                     .new(**options[:cost_model])
+        plans = find_random_plans statement, workload, indexes, cost_model,
+                                  options
+        results = random_plan_results workload, indexes, plans, cost_model
+        output_random_plans results, options[:output], options[:format]
+      end
+
+      private
+
+      # Generate random plans for a givne statement
+      # @return [Array]
+      def find_random_plans(statement, workload, indexes, cost_model, options)
+        planner = Plans::QueryPlanner.new workload, indexes, cost_model
+        plans = planner.find_plans_for_query(statement).to_a
+        plans = plans.sample options[:count] unless options[:all]
+
+        plans
+      end
+
+      # Build a results structure for these plans
+      # @return [OpenStruct]
+      def random_plan_results(workload, indexes, plans, cost_model)
+        results = OpenStruct.new
+        results.workload = workload
+        results.model = workload.model
+        results.enumerated_indexes = indexes
+        results.indexes = plans.map(&:indexes).flatten(1).to_set
+        results.plans = plans
+        results.cost_model = cost_model
+
+        results
+      end
+
+      # Output the results in the specified format
+      # @return [void]
+      def output_random_plans(results, output, format)
+        if output.nil?
+          file = $stdout
+        else
+          File.open(output, 'w')
+        end
+        begin
+          send(('output_' + format).to_sym, results, file, true)
+        ensure
+          file.close unless output.nil?
+        end
+      end
+    end
+  end
+end

data/lib/nose_cli/recost.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add a command to recalculate costs with a new model
+    class NoSECLI < Thor
+      desc 'recost PLAN_FILE COST_MODEL',
+           'recost the workload in PLAN_FILE with COST_MODEL'
+
+      long_desc <<-LONGDESC
+        `nose recost` takes a given plan file as input and then produces a new
+        JSON file with new values for the expected query cost from the named
+        cost model. This only works for cost models which don't require any
+        parameters such as `Cost::RequestCountCost`.
+      LONGDESC
+
+      def recost(plan_file, cost_model)
+        result = load_results plan_file
+
+        # Get the new cost model and recost all the queries
+        new_cost_model = get_class('cost', cost_model).new
+        result.plans.each { |plan| plan.cost_model = new_cost_model }
+
+        # Update the cost values
+        result.cost_model = new_cost_model
+        result.total_cost = total_cost result.workload, result.plans
+
+        output_json result
+      end
+
+      private
+
+      # Get the total cost for a set of plans in a given workload
+      # @return [Fixnum]
+      def total_cost(workload, plans)
+        plan_hash = Hash[plans.map { |plan| [plan.query, plan] }]
+
+        workload.statement_weights.map do |statement, weight|
+          next 0 unless statement.is_a? Query
+          weight * plan_hash[statement].cost
+        end.inject(0, &:+)
+      end
+    end
+  end
+end

data/lib/nose_cli/reformat.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add a command to reformat a plan file
+    class NoSECLI < Thor
+      desc 'reformat PLAN_FILE',
+           'reformat the data in PLAN_FILE'
+
+      shared_option :format
+      shared_option :mix
+
+      def reformat(plan_file)
+        result = load_results plan_file, options[:mix]
+        result.recalculate_cost
+
+        # Output the results in the specified format
+        send(('output_' + options[:format]).to_sym, result)
+      end
+    end
+  end
+end

data/lib/nose_cli/repl.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require 'formatador'
+
+begin
+  require 'readline'
+rescue LoadError
+  nil
+end
+
+module NoSE
+  module CLI
+    # Add a command to run a REPL which evaluates queries
+    class NoSECLI < Thor
+      desc 'repl PLAN_FILE', 'start the REPL with the given PLAN_FILE'
+
+      long_desc <<-LONGDESC
+        `nose repl` starts a terminal interface which loads execution plans
+        from the given file. It then loops continuously and accepts statements
+        from the workload and executes plans against the database. The result
+        of any query will be printed to the terminal.
+      LONGDESC
+
+      def repl(plan_file)
+        result = load_results plan_file
+        backend = get_backend(options, result)
+
+        load_history
+        loop do
+          begin
+            line = read_line
+          rescue Interrupt
+            line = nil
+          end
+          break if line.nil?
+
+          next if line.empty?
+
+          execute_statement line, result, backend
+        end
+      end
+
+      private
+
+      # Load the history file
+      # @return [void]
+      def load_history
+        return unless Object.const_defined? 'Readline'
+
+        history_file = File.expand_path '~/.nose_history'
+        begin
+          File.foreach(history_file) do |line|
+            Readline::HISTORY.push line.chomp
+          end
+        rescue Errno::ENOENT
+          nil
+        end
+      end
+
+      # Save the history file
+      # @return [void]
+      def save_history
+        return unless Object.const_defined? 'Readline'
+
+        history_file = File.expand_path '~/.nose_history'
+        File.open(history_file, 'w') do |f|
+          Readline::HISTORY.each do |line|
+            f.puts line
+          end
+        end
+      end
+
+      # Get the next inputted line in the REPL
+      # @return [String]
+      def read_line
+        prefix = '>> '
+
+        if Object.const_defined? 'Readline'
+          line = Readline.readline prefix
+          return if line.nil?
+
+          Readline::HISTORY.push line unless line.chomp.empty?
+        else
+          print prefix
+          line = gets
+        end
+
+        line.chomp
+      end
+
+      # Parse a statement from a given string of text
+      # @return [Statement]
+      def parse_statement(text, workload)
+        begin
+          statement = Statement.parse text, workload.model
+        rescue ParseFailed => e
+          puts '! ' + e.message
+          statement = nil
+        end
+
+        statement
+      end
+
+      # Try to execute a statement read from the REPL
+      # @return [void]
+      def execute_statement(line, result, backend)
+        # Parse the statement
+        statement = parse_statement line, result.workload
+        return if statement.nil?
+
+        begin
+          results, elapsed = backend_execute_statement backend, statement
+        rescue NotImplementedError, Backend::PlanNotFound => e
+          puts '! ' + e.message
+        else
+          display_statement_result results, elapsed
+        end
+      end
+
+      # Execute the statement on the provided backend and measure the runtime
+      def backend_execute_statement(backend, statement)
+        start_time = Time.now.utc
+
+        if statement.is_a? Query
+          results = backend.query statement
+        else
+          backend.update statement
+          results = []
+        end
+
+        elapsed = Time.now.utc - start_time
+
+        [results, elapsed]
+      end
+
+      # Show the results of executing a statement
+      # @return [void]
+      def display_statement_result(results, elapsed)
+        Formatador.display_compact_table results unless results.empty?
+        puts format('(%d rows in %.2fs)', results.length, elapsed)
+      end
+    end
+  end
+end

data/lib/nose_cli/search.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'formatador'
+require 'ostruct'
+require 'json'
+
+module NoSE
+  module CLI
+    # Add a command to run the advisor for a given workload
+    class NoSECLI < Thor
+      desc 'search NAME', 'run the workload NAME'
+
+      long_desc <<-LONGDESC
+        `nose search` is the primary command to run the NoSE advisor. It will
+        load the given workload, enumerate indexes for each statement, and
+        construct and solve an ILP to produce statement execution plans.
+      LONGDESC
+
+      shared_option :mix
+      shared_option :format
+      shared_option :output
+
+      option :max_space, type: :numeric, default: Float::INFINITY,
+                         aliases: '-s',
+                         desc: 'maximum space allocated to indexes'
+      option :enumerated, type: :boolean, default: false, aliases: '-e',
+                          desc: 'whether enumerated indexes should be output'
+      option :read_only, type: :boolean, default: false,
+                         desc: 'whether to ignore update statements'
+      option :objective, type: :string, default: 'cost',
+                         enum: %w(cost space indexes),
+                         desc: 'the objective function to use in the ILP'
+      option :by_id_graph, type: :boolean, default: false,
+                           desc: 'whether to group generated indexes in' \
+                                 'graphs by ID',
+                           aliases: '-i'
+
+      def search(name)
+        # Get the workload and cost model
+        workload = Workload.load name
+        workload.mix = options[:mix].to_sym \
+          unless options[:mix] == 'default' && workload.mix != :default
+        workload.remove_updates if options[:read_only]
+        cost_model = get_class_from_config options, 'cost', :cost_model
+
+        # Execute the advisor
+        objective = Search::Objective.const_get options[:objective].upcase
+        result = search_result workload, cost_model, options[:max_space],
+                               objective, options[:by_id_graph]
+        output_search_result result, options unless result.nil?
+      end
+
+      private
+
+      # Output results from the search procedure
+      # @return [void]
+      def output_search_result(result, options)
+        # Output the results in the specified format
+        file = if options[:output].nil?
+                 $stdout
+               else
+                 File.open(options[:output], 'w')
+               end
+
+        begin
+          backend = get_backend options, result
+          send(('output_' + options[:format]).to_sym,
+               result, file, options[:enumerated], backend)
+        rescue
+          nil
+        ensure
+          file.close unless options[:output].nil?
+        end
+      end
+    end
+  end
+end

data/lib/nose_cli/search_all.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module NoSE
+  module CLI
+    # Add a command to reformat a plan file
+    class NoSECLI < Thor
+      desc 'search-all NAME DIRECTORY',
+           'output all possible schemas for the workload NAME under ' \
+           'different storage constraints to DIRECTORY'
+
+      long_desc <<-LONGDESC
+        `nose search-all` is a convenience for executing `nose search` with a
+        variety of different storage constraints. It will start with the
+        smallest and largest possible schemas and then perform a binary search
+        throughout the search space to discover schemas of various sizes.
+      LONGDESC
+
+      shared_option :format
+      shared_option :mix
+
+      option :enumerated, type: :boolean, aliases: '-e',
+                          desc: 'whether enumerated indexes should be output'
+      option :read_only, type: :boolean, default: false,
+                         desc: 'whether to ignore update statements'
+      option :max_results, type: :numeric, default: Float::INFINITY,
+                           aliases: '-n',
+                           desc: 'the maximum number of results to produce'
+      def search_all(name, directory)
+        # Load the workload and cost model and create the output directory
+        workload = Workload.load name
+        workload.mix = options[:mix].to_sym \
+          unless options[:mix] == 'default' && workload.mix != :default
+        workload.remove_updates if options[:read_only]
+        cost_model = get_class_from_config options, 'cost', :cost_model
+        FileUtils.mkdir_p(directory) unless Dir.exist?(directory)
+
+        # Run the search and output the results
+        results = search_results workload, cost_model, options[:max_results]
+        output_results results, directory, options
+      end
+
+      private
+
+      # Get a list of all possible search results
+      # @return [Array<Search::Results>]
+      def search_results(workload, cost_model, max_results)
+        # Start with the maximum possible size and divide in two
+        max_result = search_result workload, cost_model
+        max_size = max_result.total_size
+        min_result = search_result workload, cost_model,
+                                   Float::INFINITY, Search::Objective::SPACE
+        min_size = min_result.total_size
+        min_result = search_result workload, cost_model, min_size
+
+        # If we only have one result, return
+        return [max_result] if max_size == min_size
+
+        results = [max_result, min_result]
+        num_results = 2
+        sizes = Set.new [min_size, max_size]
+        size_queue = [(max_size - min_size) / 2.0 + min_size]
+        until size_queue.empty?
+          # Stop if we found the appropriate number of results
+          return results if num_results >= max_results
+
+          # Find a new size to examine
+          size = size_queue.pop
+
+          # Continue dividing the range of examined sizes
+          next_size = sizes.sort.detect { |n| n > size }
+          next_size = (next_size - size) / 2.0 + size
+
+          prev_size = sizes.sort.reverse_each.detect { |n| n < size }
+          prev_size = (size - prev_size) / 2.0 + prev_size
+
+          begin
+            @logger.info "Running search with size #{size}"
+
+            result = search_result workload, cost_model, size
+            next if sizes.include?(result.total_size) || result.nil?
+          rescue Search::NoSolutionException, Plans::NoPlanException
+            # No result was found, so only explore the larger side
+            @logger.info "No solution for size #{size}"
+          else
+            # Add the smaller size to the queue and save the result
+            size_queue.push prev_size unless sizes.include? prev_size
+            results.push result
+            num_results += 1
+          end
+
+          # Add the larger size to the queue
+          size_queue.push(next_size) unless sizes.include? next_size
+
+          # Note that we visited this size (and the result size)
+          sizes.add size
+          sizes.add result.total_size unless result.nil?
+        end
+
+        results
+      end
+
+      # Output all results to file
+      # @return [void]
+      def output_results(results, directory, options)
+        results.sort_by!(&:total_size)
+        results.each_with_index do |result, i|
+          file = File.open File.join(directory, "#{i}.#{options[:format]}"), 'w'
+          begin
+            send(('output_' + options[:format]).to_sym,
+                 result, file, options[:enumerated])
+          ensure
+            file.close
+          end
+        end
+      end
+    end
+  end
+end