nose-cli 0.1.0pre

data/lib/nose_cli/search_bench.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add a command to run the advisor and benchmarks for a given workload
+    class NoSECLI < Thor
+      desc 'search-bench NAME', 'run the workload NAME and benchmarks'
+
+      long_desc <<-LONGDESC
+        `nose search-bench` is a convenient way to quickly test many different
+        schemas. It will run `nose search`, `nose create`, `nose load`, and
+        `nose benchmark` to produce final benchmark results. It also accepts
+        all of the command-line options for each of those commands.
+      LONGDESC
+
+      def search_bench(name)
+        # Open a tempfile which will be used for advisor output
+        filename = Tempfile.new('workload').path
+
+        # Set some default options for various commands
+        opts = options.to_h
+        opts[:output] = filename
+        opts[:format] = 'json'
+        opts[:skip_existing] = true
+
+        o = filter_command_options opts, 'search'
+        $stderr.puts "Running advisor #{o}..."
+        invoke self.class, :search, [name], o
+
+        invoke self.class, :reformat, [filename], {}
+
+        o = filter_command_options opts, 'create'
+        $stderr.puts "Creating indexes #{o}..."
+        invoke self.class, :create, [filename], o
+
+        o = filter_command_options opts, 'load'
+        $stderr.puts "Loading data #{o}..."
+        invoke self.class, :load, [filename], o
+
+        o = filter_command_options opts, 'benchmark'
+        $stderr.puts "Running benchmark #{o}..."
+        invoke self.class, :benchmark, [filename], o
+      end
+
+      # Allow this command to accept the options for all commands it calls
+      commands['search_bench'].options.merge! commands['create'].options
+      commands['search_bench'].options.merge! commands['benchmark'].options
+      commands['search_bench'].options.merge! commands['load'].options
+      commands['search_bench'].options.merge! commands['search'].options
+    end
+  end
+end

data/lib/nose_cli/shared_options.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add the use of shared options
+    class NoSECLI < Thor
+      # Add a new option to those which can be potentially shared
+      def self.share_option(name, options = {})
+        @options ||= {}
+        @options[name] = options
+      end
+
+      # Use a shared option for the current command
+      # @return [void]
+      def self.shared_option(name)
+        method_option name, @options[name]
+      end
+
+      share_option :mix, type: :string, default: 'default',
+                         desc: 'the name of the mix for weighting queries'
+      share_option :format, type: :string, default: 'txt',
+                            enum: %w(txt json yml html), aliases: '-f',
+                            desc: 'the format of the produced plans'
+      share_option :output, type: :string, default: nil, aliases: '-o',
+                            banner: 'FILE',
+                            desc: 'a file where produced plans ' \
+                                  'should be stored'
+    end
+  end
+end

data/lib/nose_cli/texify.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add a command to reformat a plan file
+    class NoSECLI < Thor
+      desc 'texify PLAN_FILE',
+           'print the results from PLAN_FILE in LaTeX format'
+
+      long_desc <<-LONGDESC
+        `nose texify` loads the generated schema from the given file and prints
+        the generated schema in LaTeX format.
+      LONGDESC
+
+      shared_option :mix
+
+      def texify(plan_file)
+        # Load the indexes from the file
+        result, = load_plans plan_file, options
+
+        # If these are manually generated plans, load them separately
+        if result.plans.nil?
+          plans = Plans::ExecutionPlans.load(plan_file) \
+                  .groups.values.flatten(1)
+          result.plans = plans.select { |p| p.update_steps.empty? }
+          result.update_plans = plans.reject { |p| p.update_steps.empty? }
+        end
+
+        # Print document header
+        puts "\\documentclass{article}\n\\begin{document}\n\\begin{flushleft}"
+
+        # Print the LaTeX for all indexes and plans
+        texify_indexes result.indexes
+        texify_plans result.plans + result.update_plans
+
+        # End the document
+        puts "\\end{flushleft}\n\\end{document}"
+      end
+
+      private
+
+      # Escape values for latex output
+      # @return [String]
+      def tex_escape(str)
+        str.gsub '_', '\\_'
+      end
+
+      # Print the LaTeX for all query plans
+      # @return [void]
+      def texify_plans(plans)
+        puts '\\bigskip\\textbf{Plans} \\\\\\bigskip'
+
+        plans.group_by(&:group).each do |group, grouped_plans|
+          group = group.nil? ? '' : tex_escape(group)
+          texify_plan_group group, grouped_plans
+        end
+      end
+
+      # Print the LaTeX from a group of query plans
+      # @return [void]
+      def texify_plan_group(group, grouped_plans)
+        puts "\\textbf{#{group}} \\\\" unless group.empty?
+
+        grouped_plans.each do |plan|
+          if plan.is_a?(Plans::QueryPlan) ||
+             (plan.is_a?(Plans::QueryExecutionPlan) &&
+              plan.update_steps.empty?)
+            puts texify_plan_steps plan.steps
+          else
+            puts texify_plan_steps plan.query_plans.flat_map(&:to_a) + \
+                                   plan.update_steps
+          end
+
+          puts ' \\\\'
+        end
+
+        puts '\\medskip'
+      end
+
+      # Print the LaTeX from a set of plan steps
+      # @return [void]
+      def texify_plan_steps(steps)
+        steps.map do |step|
+          case step
+          when Plans::IndexLookupPlanStep
+            "Request \\textbf{#{tex_escape step.index.key}}"
+          when Plans::FilterPlanStep
+            "Filter by #{texify_fields((step.eq + [step.range]).compact)}"
+          when Plans::SortPlanStep
+            "Sort by #{texify_fields step.sort_fields}"
+          when Plans::LimitPlanStep
+            "Limit #{step.limit}"
+          when Plans::DeletePlanStep
+            "Delete from \\textbf{#{tex_escape step.index.key}}"
+          when Plans::InsertPlanStep
+            "Insert into \\textbf{#{tex_escape step.index.key}}"
+          end
+        end.join(', ')
+      end
+
+      # Print all LaTeX for a given index
+      # @return [void]
+      def texify_indexes(indexes)
+        puts '\\bigskip\\textbf{Indexes} \\\\\\bigskip'
+
+        indexes.each do |index|
+          # Print the key of the index
+          puts "\\textbf{#{tex_escape index.key}} \\\\"
+
+          fields = index.hash_fields.map do |field|
+            texify_field(field, true)
+          end
+
+          fields += index.order_fields.map do |field|
+            texify_field(field, true, true)
+          end
+
+          fields += index.extra.map { |field| texify_field(field) }
+
+          puts fields.join(', ') + ' \\\\\\medskip'
+        end
+      end
+
+      # Produce the LaTex for an array of fields
+      # @return [String]
+      def texify_fields(fields)
+        fields.map { |field| texify_field field }.join ', '
+      end
+
+      # Produce the LaTeX for a given index field
+      # @return [String]
+      def texify_field(field, underline = false, italic = false)
+        tex = tex_escape field.to_s
+        tex = "\\textit{#{tex}}" if italic
+        tex = "\\underline{#{tex}}" if underline
+
+        tex
+      end
+    end
+  end
+end

data/lib/nose_cli/why.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module NoSE
+  module CLI
+    # Add a command to generate a graphic of the schema from a workload
+    class NoSECLI < Thor
+      desc 'why PLAN_FILE',
+           'output the reason for including each index in PLAN_FILE'
+
+      long_desc <<-LONGDESC
+        `nose why` is used to better understand why NoSE included a particular
+        index in a schema. This is especially helpful when comparing with
+        manually-defined execution plans.
+      LONGDESC
+
+      def why(plan_file)
+        result = load_results plan_file
+        indexes_usage = Hash.new { |h, k| h[k] = [] }
+
+        # Count the indexes used in queries
+        query_count = Set.new
+        update_index_usage result.plans, indexes_usage, query_count
+
+        # Count the indexes used in support queries
+        # (ignoring those used in queries)
+        support_count = Set.new
+        result.update_plans.each do |plan|
+          update_index_usage plan.query_plans, indexes_usage,
+                             support_count, query_count
+        end
+
+        # Produce the final output of index usage
+        print_index_usage indexes_usage, query_count, support_count
+      end
+
+      private
+
+      # Track usage of indexes in the set of query plans updating both
+      # a dictionary of statements relevant to each index and a set
+      # of unique statements used (optionally ignoring some)
+      # @return [void]
+      def update_index_usage(plans, indexes_usage, statement_usage,
+                             ignore = Set.new)
+        plans.each do |plan|
+          plan.indexes.each do |index|
+            indexes_usage[index] << if plan.respond_to?(:statement)
+                                      plan.statement
+                                    else
+                                      plan.query
+                                    end
+            statement_usage.add index unless ignore.include? index
+          end
+        end
+      end
+
+      # Print out the statements each index is used for
+      # @return [void]
+      def print_index_usage(indexes_usage, query_count, support_count)
+        indexes_usage.each do |index, statements|
+          p index
+          statements.each { |s| p s }
+          puts
+        end
+
+        puts "        Queries: #{query_count.length}"
+        puts "Support queries: #{support_count.length}"
+      end
+    end
+  end
+end

data/templates/completions.erb

@@ -0,0 +1,56 @@
+_nose_complete() {
+  case $1 in
+  1) _nose_commands;;
+  2)
+    case "$2" in
+      help) _nose_commands;;
+      *) _nose_option;;
+    esac;;
+  *) _nose_option;;
+  esac
+}
+
+if type compdef 1>/dev/null 2>/dev/null; then
+  compdef _nose nose
+  _nose()          { _nose_complete $((${#words} - 1)) "${words[2]}"; }
+  _nose_commands() { list=(<%= commands.map { |name, c| "#{name}:\"#{Shellwords.escape(c.description)}\"" }.join(' ') %>) _describe -t common-commands 'common commands' list; }
+  _nose_option()   {
+    case "${words[2]}" in
+<%= commands.map do |name, c|
+"      #{name})  _arguments -s -S " + c.options.values.flat_map do |opt|
+    (opt.aliases << opt.switch_name).map do |switch|
+      "\"#{switch}[#{Shellwords.escape(opt.description)}]\""
+    end
+  end.uniq.join(' ') + ' \'*:file:_files\' && return 0;;'
+end.join("\n") %>
+    esac
+  }
+elif type compctl 1>/dev/null 2>/dev/null; then
+  compctl -K _nose nose
+  _nose()          { read -cA words && _nose_complete $((${#words} - 1)) "${words[2]}"; }
+  _nose_commands() { reply=(<%= commands.map { |name, _| "\"#{name}\"" }.join(' ') %>); }
+  _nose_option()   {
+    case "${words[2]}" in
+<%= commands.map do |name, c|
+"      #{name})  reply=(" + c.options.values.flat_map do |opt|
+    (opt.aliases << opt.switch_name).map { |s| "\"#{s}\"" }
+  end.uniq.join(' ') + ');;'
+end.join("\n") %>
+    esac
+  }
+elif type complete 1>/dev/null 2>/dev/null; then
+  complete -F _nose nose
+  _nose()          { _nose_complete "$COMP_CWORD" "${COMP_WORDS[1]}"; }
+  _nose_commands() { COMPREPLY=( $(compgen -W "<%= commands.map(&:first).join(' ') %>" -- "${COMP_WORDS[COMP_CWORD]}") ); }
+  _nose_option()   {
+    local options
+    case "${COMP_WORDS[1]}" in
+<%= commands.map do |name, c|
+"      #{name})  options=\"" + c.options.values.flat_map do |opt|
+    (opt.aliases << opt.switch_name)
+  end.uniq.join(' ') + '";;'
+end.join("\n") %>
+    esac
+    COMPREPLY=( $(compgen -W "$options" -- "${COMP_WORDS[COMP_CWORD]}") )
+  }
+fi

data/templates/man.erb

@@ -0,0 +1,33 @@
+nose(1) -- NoSQL Schema Evaluator
+========================================
+
+## SYNOPSIS
+
+`nose` <%= options.each_value.map do |option|
+  '[' + (option.aliases + ["--#{option.name}"]).join('|') + ']'
+end.join ', ' %>
+
+## DESCRIPTION
+
+NoSE is a tool for schema design in NoSQL databases.
+
+## OPTIONS
+
+<% options.each do |name, option| %>
+  * <%= option.usage %>
+  <%= option.description  %>
+
+<% end %>
+
+## SUBCOMMANDS
+
+<% commands.each do |name, command| %>
+  `nose-<%= name %>`(1)
+  <%= command.description %>
+
+
+<% end %>
+
+## SEE ALSO
+
+<%= commands.map { |name, _| "`nose-#{name}`(1)" }.join ', ' %>

data/templates/report.erb

@@ -0,0 +1,138 @@
+<% require 'ansi-to-html' %>
+
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/jquery/3.1.0/jquery.min.js"></script>
+    <link href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
+    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
+    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.4.0/styles/default.min.css">
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.4.0/highlight.min.js"></script>
+    <script>hljs.initHighlightingOnLoad();</script>
+    <title>NoSE Schema Recommendation</title>
+  </head>
+  <body style="margin: 1em">
+    <h1>NoSE Schema Recommendation</h1>
+
+    <h2>Model</h2>
+    <%= svg %>
+
+    <% unless workload.model.source_code.nil? %>
+      <a class="btn btn-primary" style="display: block" data-toggle="collapse" data-target="#model-code">Model code
+        <span class="glyphicon glyphicon-menu-hamburger" style="float:right"></span></a>
+      <pre id="model-code" class="collapse"><code class="lang-ruby"><%= workload.model.source_code %></pre></code>
+    <% end %>
+
+    <h2>Queries</h2>
+    <% workload.queries.each do |query| %>
+      <% unless query.label.nil? %><strong><%= query.label %></strong><% end %>
+      <pre><code class="lang-sql"><%= query.text %></code></pre>
+    <% end %>
+
+    <% unless workload.updates.empty? %><h2>Updates</h2><% end %>
+    <% workload.updates.each do |update| %>
+      <% unless update.label.nil? %><strong><%= update.label %></strong><% end %>
+      <pre><code class="lang-sql"><%= update.text %></code></pre>
+    <% end %>
+
+    <% unless workload.source_code.nil? %>
+      <a class="btn btn-primary" style="display: block" data-toggle="collapse" data-target="#workload-code">Workload code
+        <span class="glyphicon glyphicon-menu-hamburger" style="float:right"></span></a>
+      <pre id="workload-code" class="collapse"><code class="lang-ruby"><%= workload.source_code %></pre></code>
+    <% end %>
+
+    <h2>Indexes</h2>
+    <% ddl = backend ? backend.indexes_ddl.to_a : nil %>
+    <% indexes.each_with_index do |index, i| %>
+      <h3><%= index.key %></h3>
+      <% if ddl %>
+        <p><tt><%= ddl[i] %></tt></p>
+      <% end %>
+      <p><%= Ansi::To::Html.new(index.inspect).to_html %></p>
+    <% end %>
+
+    <h2>Query plans</h2>
+    <% plans.each do |plan| %>
+      <% unless plan.query.label.nil? %><strong><%= plan.query.label %></strong><% end %>
+      <pre><code class="lang-sql"><%= Ansi::To::Html.new(plan.query.inspect).to_html %></code></pre>
+      <strong>Cost: <%= plan.cost %></strong>
+
+      <% if plan.steps.length > 1 %>
+        <ol>
+        <% plan.each do |step| %>
+          <li><%= Ansi::To::Html.new(step.inspect).to_html %></li>
+        <% end %>
+        </ol>
+      <% else %>
+        <p><%= Ansi::To::Html.new(plan.steps.first.inspect).to_html %></p>
+      <% end %>
+      <hr>
+    <% end %>
+
+    <h2>Update plans</h2>
+    <% update_plans.group_by(&:statement).each do |statement, plans| %>
+      <% unless statement.label.nil? %><strong><%= statement.label %></strong><% end %>
+      <pre><code class="lang-sql"><%= statement.text %></code></pre>
+      <strong>Total cost: <%= plans.map(&:cost).inject(0, &:+) %></strong>
+
+      <% plans.each do |plan| %>
+        <h4><%= plan.index.key %></h4>
+        <strong>Cost: <%= plan.cost %></strong>
+
+        <% unless plan.query_plans.empty? %>
+          <h5>Support queries</h5>
+        <% end %>
+
+        <% plan.query_plans.each do |query_plan| %>
+          <% unless query_plan.query.label.nil? %><strong><%= query_plan.query.label %></strong><% end %>
+          <pre><code class="lang-sql"><%= query_plan.query.text %></code></pre>
+          <% if query_plan.steps.length > 1 %>
+            <ol>
+            <% query_plan.each do |step| %>
+              <li><%= Ansi::To::Html.new(step.inspect).to_html %></li>
+            <% end %>
+            </ol>
+          <% else %>
+            <p><%= Ansi::To::Html.new(query_plan.steps.first.inspect).to_html %></p>
+          <% end %>
+        <% end %>
+
+        <h5>Updates</h5>
+
+        <% if plan.update_steps.length > 1 %>
+          <ol>
+          <% plan.update_steps.each do |step| %>
+            <li><%= Ansi::To::Html.new(step.inspect).to_html %></li>
+          <% end %>
+          </ol>
+        <% else %>
+          <p><%= Ansi::To::Html.new(plan.update_steps.first.inspect).to_html %></p>
+        <% end %>
+      <% end %>
+      <hr>
+    <% end %>
+
+    <% if enumerated_indexes %>
+      <h2>Enumerated Indexes</h2>
+      <% enumerated_indexes.each_with_index do |index, i| %>
+        <h3><%= index.key %></h3>
+        <% if ddl %>
+          <p><tt><%= ddl[i] %></tt></p>
+        <% end %>
+        <p><%= Ansi::To::Html.new(index.inspect).to_html %></p>
+      <% end %>
+    <% end %>
+
+    <h2>Summary</h2>
+    <dl>
+      <dt>Total size</dt>
+      <dd><%= total_size %></dd>
+
+      <dt>Total cost</dt>
+      <dd><%= total_cost %></dd>
+    </dl>
+  </body>
+</html>