elasticsearch-rails 0.1.0 → 0.1.1

data/lib/rails/templates/index.html.erb

@@ -0,0 +1,157 @@
+<div class="col-md-12">
+  <h1 class="text-right"><%= link_to 'Search New York Times articles', root_path %></h1>
+
+  <%= form_tag search_path, method: 'get', role: 'search' do %>
+    <div class="input-group">
+      <%= text_field_tag :q, params[:q], class: 'form-control', placeholder: 'Search...' %>
+
+      <span class="input-group-btn">
+        <button type="submit" class="btn btn-default">
+          <span class="glyphicon glyphicon-search"></span>
+        </button>
+      </span>
+    </div>
+
+    <div id="form-options" class="clearfix">
+      <div class="btn-group pull-left">
+        <label class="checkbox-inline">
+          <%= check_box_tag 'comments', 'y', params[:comments] == 'y', onclick: "$(this).closest('form').submit()" %>
+          Search in comments?
+        </label>
+        <% params.slice(:a, :c, :s).each do |name, value| %>
+          <%= hidden_field_tag name, value %>
+        <% end %>
+      </div>
+
+      <div class="btn-group pull-right">
+        <p style="float: left; margin: 0.1em 0 0 0"><small>Displaying <%= (params[:page] || 1).to_i.ordinalize %> page with <%= @articles.size %> articles
+                   of <strong>total <%= @articles.total %></strong></small></p>
+
+        <button class="btn btn-default btn-xs dropdown-toggle" type="button" data-toggle="dropdown" style="margin-left: 0.5em">
+          <% sort = case
+              when params[:s] then params[:s]
+              when params[:q].blank? then 'published_on'
+              else 'relevancy'
+             end
+          %>
+          sorted by <%= sort.humanize.downcase %> <span class="caret"></span>
+        </button>
+        <ul class="dropdown-menu" role="menu">
+          <li><%= link_to "Sort by published on", search_path(params.merge(s: 'published_on')), class: 'btn-xs' %></li>
+          <li><%= link_to "Sort by relevancy",    search_path(params.merge(s: nil)), class: 'btn-xs' %></li>
+        </ul>
+      </div>
+    </div>
+  <% end %>
+
+  <hr>
+</div>
+
+<% if @articles.size < 1 && (suggestions = @articles.response.response['suggest']) && suggestions.present?  %>
+<div class="col-md-12">
+  <p class="alert alert-warning">
+    No documents have been found. Maybe you mean
+    <%= suggestions.map { |k,v| v.first['options'] }.flatten.map {|v| v['text']}.uniq.map do |term|
+      link_to term, search_path(params.merge q: term)
+    end.to_sentence(last_word_connector: ' or ').html_safe %>?
+  </p>
+</div>
+<% end %>
+
+<div id="facets" class="col-md-3">
+  <% unless @articles.size < 1 %>
+
+    <div class="categories panel panel-default">
+      <p class="panel-heading"><%= link_to 'All Sections &rarr;'.html_safe, search_path(params.merge(c: nil))%></p>
+
+      <div class="list-group">
+        <% @articles.response.response['facets']['categories']['terms'].each do |c| %>
+        <%=
+          link_to search_path(params.merge(c: c['term'])),
+                  class: "list-group-item#{' active' if params[:c] == c['term']}" do
+                    c['term'].titleize.html_safe + content_tag(:small, c['count'], class: 'badge').html_safe
+          end
+        %>
+        <% end %>
+      </div>
+    </div>
+
+    <div class="authors panel panel-default">
+      <p class="panel-heading"><%= link_to 'All Authors &rarr;'.html_safe, search_path(params.merge(a: nil))%></p>
+
+      <div class="list-group">
+        <% @articles.response.response['facets']['authors']['terms'].each do |a| %>
+        <%=
+          link_to search_path(params.merge(a: a['term'])),
+                  class: "list-group-item#{' active' if params[:a] == a['term']}" do
+                    a['term'].titleize.html_safe + content_tag(:small, a['count'], class: 'badge').html_safe
+          end
+        %>
+        <% end %>
+      </div>
+    </div>
+
+    <div class="authors panel panel-default">
+      <p class="panel-heading"><%= link_to 'Any Date &rarr;'.html_safe, search_path(params.merge(w: nil))%></p>
+
+      <div class="list-group">
+        <% @articles.response.response['facets']['published']['entries'].each do |w| %>
+        <%=
+          __start = Time.at(w['time']/1000)
+          __end   = __start.end_of_week
+          __date  = __start.to_date.to_s(:iso)
+
+          link_to search_path(params.merge(w: __date)),
+                  class: "list-group-item#{' active' if params[:w] == __date}" do
+                    "#{__start.to_date.to_s(:short)} &mdash; #{__end.to_date.to_s(:short)}".html_safe + \
+                    content_tag(:small, w['count'], class: 'badge').html_safe
+          end
+        %>
+        <% end %>
+      </div>
+    </div>
+  <% end %>
+</div>
+
+<div class="col-md-9">
+  <div id="results">
+    <% @articles.each do |article|  %>
+      <div class="result">
+        <h3 class="title">
+          <%= (article.try(:highlight).try(:title) ? article.highlight.title.join.html_safe : article.title) %>
+          <small class="category"><%= article.categories.to_sentence %></small>
+        </h3>
+
+        <p class="body">
+          <% if article.try(:highlight).try(:abstract) %>
+            <%= article.highlight.abstract.join.html_safe %>
+          <% else  %>
+            <%= article.try(:highlight).try(:content) ? article.highlight.content.join('&hellip;').html_safe : article.abstract %>
+          <% end  %>
+        </p>
+
+        <% if comments = article.try(:highlight) && article.highlight['comments.body'] %>
+        <p class="comments">
+          Comments: <%= comments.join('&hellip;').html_safe %>
+        </p>
+        <% end  %>
+
+        <p class="text-muted">
+          <small>Authors: <%= article.authors.map(&:full_name).to_sentence %></small> |
+          <small>Published: <%= article.published_on %></small> |
+          <small>Score: <%= article._score %></small>
+        </p>
+      </div>
+    <% end %>
+  </div>
+
+  <ul class="pager">
+    <li class="previous"><%= link_to_previous_page @articles, 'Previous Page', params: params.slice(:q, :c, :a, :comments) %></li>
+    <li class="next"><%= link_to_next_page     @articles, 'Next Page', params: params.slice(:q, :c, :a, :comments) %></li>
+  </ul>
+
+</div>
+
+<div class="footer <%= @articles.size < 1 ? 'col-md-12' : 'col-md-9 col-md-offset-3' %>">
+  <p><small>Content provided by <a href="http://nytimes.com"><em>The New York Times</em></a>.</small></p>
+</div>

data/lib/rails/templates/indexer.rb

@@ -0,0 +1,27 @@
+# Indexer class for <http://sidekiq.org>
+#
+# Run me with:
+#
+#     $ bundle exec sidekiq --queue elasticsearch --verbose
+#
+class Indexer
+  include Sidekiq::Worker
+  sidekiq_options queue: 'elasticsearch', retry: false, backtrace: true
+
+  Logger = Sidekiq.logger.level == Logger::DEBUG ? Sidekiq.logger : nil
+  Client = Elasticsearch::Client.new host: (ENV['ELASTICSEARCH_URL'] || 'http://localhost:9200'), logger: Logger
+
+  def perform(operation, klass, record_id, options={})
+    logger.debug [operation, "#{klass}##{record_id} #{options.inspect}"]
+
+    case operation.to_s
+      when /index|update/
+        record = klass.constantize.find(record_id)
+        record.__elasticsearch__.client = Client
+        record.__elasticsearch__.__send__ "#{operation}_document"
+      when /delete/
+        Client.delete index: klass.constantize.index_name, type: klass.constantize.document_type, id: record_id
+      else raise ArgumentError, "Unknown operation '#{operation}'"
+    end
+  end
+end

data/lib/rails/templates/search.css

@@ -0,0 +1,72 @@
+h1 {
+  font-size: 28px !important;
+  color: #a3a3a3 !important;
+  text-transform: uppercase;
+  letter-spacing: -2px;
+}
+
+.label-highlight {
+  background: #f6fbfc !important;
+  box-shadow: 0px 1px 0px rgba(0,0,0,0.15);
+  padding: 0.2em 0.4em 0.2em 0.4em !important;
+}
+
+h3 .label-highlight {
+  background: transparent !important;
+  padding: 0.1em 0.4em 0px 0.4em !important;
+  border-bottom: 1px solid #999;
+  box-shadow: 0px 2px 1px rgba(0,0,0,0.15);
+  border-radius: 0;
+}
+
+.comments .label-highlight {
+  background: #fcfdf0 !important;
+}
+
+small.badge {
+  font-size: 80% !important;
+  font-weight: normal !important;
+  display: inline-block;
+  float: right;
+}
+
+form #form-options {
+  color: #666;
+  font-size: 95%;
+  margin-top: 1.5em;
+  padding: 0 0.25em;
+}
+form #form-options input {
+  margin-top: 0.25em;
+}
+
+#facets .panel-heading {
+  margin-bottom: 0;
+}
+
+.result {
+  border-bottom: 1px solid #ccc;
+  margin: 2em 0 0 0;
+  padding: 0 0 1em 0;
+}
+.result:first-child {
+  margin-top: 0.25em;
+}
+
+.result h3.title {
+  font-family: 'Rokkitt', sans-serif;
+  margin-top: 0;
+}
+
+.result .body {
+  font-family: Georgia, serif;
+}
+
+.result .category {
+  font-family: 'Rokkitt', sans-serif;
+}
+
+.result .comments {
+  color: #666666;
+  font-size: 80%;
+}

data/lib/rails/templates/searchable.rb

@@ -0,0 +1,212 @@
+module Searchable
+  extend ActiveSupport::Concern
+
+  included do
+    include Elasticsearch::Model
+
+    # Customize the index name
+    #
+    index_name [Rails.application.engine_name, Rails.env].join('_')
+
+    # Set up index configuration and mapping
+    #
+    settings index: { number_of_shards: 1, number_of_replicas: 0 } do
+      mapping do
+        indexes :title, type: 'multi_field' do
+          indexes :title,     analyzer: 'snowball'
+          indexes :tokenized, analyzer: 'simple'
+        end
+
+        indexes :content, type: 'multi_field' do
+          indexes :content,   analyzer: 'snowball'
+          indexes :tokenized, analyzer: 'simple'
+        end
+
+        indexes :published_on, type: 'date'
+
+        indexes :authors do
+          indexes :full_name, type: 'multi_field' do
+            indexes :full_name
+            indexes :raw, analyzer: 'keyword'
+          end
+        end
+
+        indexes :categories, analyzer: 'keyword'
+
+        indexes :comments, type: 'nested' do
+          indexes :body, analyzer: 'snowball'
+          indexes :stars
+          indexes :pick
+          indexes :user, analyzer: 'keyword'
+          indexes :user_location, type: 'multi_field' do
+            indexes :user_location
+            indexes :raw, analyzer: 'keyword'
+          end
+        end
+      end
+    end
+
+    # Set up callbacks for updating the index on model changes
+    #
+    after_commit lambda { Indexer.perform_async(:index,  self.class.to_s, self.id) }, on: :create
+    after_commit lambda { Indexer.perform_async(:update, self.class.to_s, self.id) }, on: :update
+    after_commit lambda { Indexer.perform_async(:delete, self.class.to_s, self.id) }, on: :destroy
+    after_touch  lambda { Indexer.perform_async(:update, self.class.to_s, self.id) }
+
+    # Customize the JSON serialization for Elasticsearch
+    #
+    def as_indexed_json(options={})
+      hash = self.as_json(
+        include: { authors:    { methods: [:full_name], only: [:full_name] },
+                   comments:   { only: [:body, :stars, :pick, :user, :user_location] }
+                 })
+      hash['categories'] = self.categories.map(&:title)
+      hash
+    end
+
+    # Search in title and content fields for `query`, include highlights in response
+    #
+    # @param query [String] The user query
+    # @return [Elasticsearch::Model::Response::Response]
+    #
+    def self.search(query, options={})
+
+      # Prefill and set the filters (top-level `filter` and `facet_filter` elements)
+      #
+      __set_filters = lambda do |key, f|
+
+        @search_definition[:filter][:and] ||= []
+        @search_definition[:filter][:and]  |= [f]
+
+        @search_definition[:facets][key.to_sym][:facet_filter][:and] ||= []
+        @search_definition[:facets][key.to_sym][:facet_filter][:and]  |= [f]
+      end
+
+      @search_definition = {
+        query: {},
+
+        highlight: {
+          pre_tags: ['<em class="label label-highlight">'],
+          post_tags: ['</em>'],
+          fields: {
+            title:    { number_of_fragments: 0 },
+            abstract: { number_of_fragments: 0 },
+            content:  { fragment_size: 50 }
+          }
+        },
+
+        filter: {},
+
+        facets: {
+          categories: {
+            terms: {
+              field: 'categories'
+            },
+            facet_filter: {}
+          },
+          authors: {
+            terms: {
+              field: 'authors.full_name.raw'
+            },
+            facet_filter: {}
+          },
+          published: {
+            date_histogram: {
+              field: 'published_on',
+              interval: 'week'
+            },
+            facet_filter: {}
+          }
+        }
+      }
+
+      unless query.blank?
+        @search_definition[:query] = {
+          bool: {
+            should: [
+              { multi_match: {
+                  query: query,
+                  fields: ['title^10', 'abstract^2', 'content'],
+                  operator: 'and'
+                }
+              }
+            ]
+          }
+        }
+      else
+        @search_definition[:query] = { match_all: {} }
+        @search_definition[:sort]  = { published_on: 'desc' }
+      end
+
+      if options[:category]
+        f = { term: { categories: options[:category] } }
+
+        __set_filters.(:authors, f)
+        __set_filters.(:published, f)
+      end
+
+      if options[:author]
+        f = { term: { 'authors.full_name.raw' => options[:author] } }
+
+        __set_filters.(:categories, f)
+        __set_filters.(:published, f)
+      end
+
+      if options[:published_week]
+        f = {
+          range: {
+            published_on: {
+              gte: options[:published_week],
+              lte: "#{options[:published_week]}||+1w"
+            }
+          }
+        }
+
+        __set_filters.(:categories, f)
+        __set_filters.(:authors, f)
+      end
+
+      if query.present? && options[:comments]
+        @search_definition[:query][:bool][:should] ||= []
+        @search_definition[:query][:bool][:should] << {
+          nested: {
+            path: 'comments',
+            query: {
+              multi_match: {
+                query: query,
+                fields: ['body'],
+                operator: 'and'
+              }
+            }
+          }
+        }
+        @search_definition[:highlight][:fields].update 'comments.body' => { fragment_size: 50 }
+      end
+
+      if options[:sort]
+        @search_definition[:sort]  = { options[:sort] => 'desc' }
+        @search_definition[:track_scores] = true
+      end
+
+      unless query.blank?
+        @search_definition[:suggest] = {
+          text: query,
+          suggest_title: {
+            term: {
+              field: 'title.tokenized',
+              suggest_mode: 'always'
+            }
+          },
+          suggest_body: {
+            term: {
+              field: 'content.tokenized',
+              suggest_mode: 'always'
+            }
+          }
+        }
+      end
+
+      __elasticsearch__.search(@search_definition)
+    end
+  end
+end