middleman-search-gds 0.11.0a

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f7a50a523cabca4a87a73bb6a1a87b765f1e0e3b
+  data.tar.gz: e62e0a792758b31fd98a00acf6bbe2091c6593be
+SHA512:
+  metadata.gz: a51080682bef38d99ea3a27958c34e30768454fc18e04432ca3ae26e6bf79325a7bcdc1b8b5ad6cd27787c183ef4499e3ad672ce2dc63ce2dae283cce18644f8
+  data.tar.gz: 6fa42ea890e06662dd5b876437a2b9b61cb8f6e57e0623fcedbd7870968130d8cc277baf4b815a751409b4dd9f0040cb1543c983fbd8a33aacad583107cc3aa5

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - 2.1.0
+  - 2.0.0
+  - 1.9.3

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in middleman-search.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Matías García Isaía
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,191 @@
+# Middleman::Search
+
+LunrJS-based search for Middleman.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'middleman-search'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install middleman-search
+
+## Usage
+
+You need to activate the module in your `config.rb`, telling the extension how to index your resources:
+
+```ruby
+activate :search do |search|
+
+  search.resources = ['blog/', 'index.html', 'contactus/index.html']
+
+  search.index_path = 'search/lunr-index.json' # defaults to `search.json`
+
+  search.lunr_dirs = ['source/vendor/lunr-custom/'] # optional alternate paths where to look for lunr js files
+
+  search.language = 'es' # defaults to 'en'
+
+  search.fields = {
+    title:   {boost: 100, store: true, required: true},
+    content: {boost: 50},
+    url:     {index: false, store: true},
+    author:  {boost: 30}
+  }
+end
+```
+
+Where `resources` is a list of the beginning of the URL of the resources to index (tested with `String#start_with?`), `index_path` is the relative path of the generated index file in your site, and `fields` is a hash with one entry for each field to be indexed, with a hash of options associated:
+
+- `boost` Specifies lunr relevance boost when searching this field
+- `store` Whether to store this field in the document map (see below), defaults to false
+- `index` Whether to index this field, defaults to true
+- `required` The resource will not be indexed if a field marked as required has an empty or null value
+
+Note that a special field `id` is included automatically, with an autogenerated identifier to be used as the `ref` for the document.
+
+All fields values are retrieved from the resource `data` (i.e. its frontmatter), or from the `options` in the `resource.metadata` (i.e. any options specified in a `proxy` page), except for:
+- `url` which is the actual resource url
+- `content` the text extracted from the rendered resource, without including its layout
+
+You can then query the index from Javascript via the `lunrIndex` object (see [Index file](#index-file) for more info):
+
+```javascript
+var max_search_entries = 50;
+
+var result = []; //initialize empty array
+
+lunrIndex.search(request.term).forEach( function (item, index) {
+  if ( index < max_search_entries ) {
+    result.push(lunrData.docs[item.ref]);
+  }
+});
+```
+
+(Thanks [@Jeepler](https://github.com/Jeepler) [for adapting](https://github.com/manastech/middleman-search/issues/11#issuecomment-220262546) the lodash v3 code [we used to use at Manas](https://manas.tech/blog/2015/10/22/middleman-search-client-side-search-in-your-middleman-site.html))
+
+### i18n
+
+This gem includes assets for alternate languages as provided by [MihaiValentin/lunr-languages](https://github.com/MihaiValentin/lunr-languages). Please refer to that repository for a list of the languages available.
+
+If you want to work with a language that is not included, set up a `lunr.yourlang.js` file in a folder in your project, and add that folder to `lunr_dirs` so the gem knows where to look for it.
+
+### Manual index manipulation
+
+You can fully customise the content to be indexed and stored per resource by defining a `before_index` callback:
+
+```ruby
+activate :search do |search|
+  search.before_index = Proc.new do |to_index, to_store, resource|
+    if author = resource.data.author
+      to_index[:author] = data.authors[author].name
+    end
+  end
+end
+```
+
+This option accepts a callback that will be executed for each resource, and will be executed with the document to be indexed and the map to be stored, in the `index` and `docs` objects of the output respectively (see below), as well as the resource being processed. You can use this callback to modify either of those, or `throw(:skip)` to skip the resource in question.
+
+### Lunr pipeline configuration
+
+In some cases, you may want to add new function to the lunr pipeline, both for creating the indexing and then for searching. You can do this by providing a `pipeline` hash with function names and body, for example:
+
+```ruby
+activate :search do |search|
+  search.pipeline = {
+    tildes: <<-JS
+      function(token, tokenIndex, tokens) {
+        return token
+          .replace('á', 'a')
+          .replace('é', 'e')
+          .replace('í', 'i')
+          .replace('ó', 'o')
+          .replace('ú', 'u');
+      }
+    JS
+  }
+end
+```
+
+This will register the `tildes` function in the lunr pipeline and add it when building the index. From the Lunr documentation:
+
+> Functions in the pipeline are called with three arguments: the current token being processed; the index of that token in the array of tokens, and the whole list of tokens part of the document being processed. This enables simple unigram processing of tokens as well as more sophisticated n-gram processing.
+>
+> The function should return the processed version of the text, which will in turn be passed to the next function in the pipeline. Returning undefined will prevent any further processing of the token, and that token will not make it to the index.
+
+Note that if you add a function to the pipeline, it will also be loaded when de-serialising the index, and lunr will fail with an `Cannot load un-registered function: tildes` error if it has not been re-registered. You can either register them manually, or simply include the following in a `.js.erb` file to be executed __before__ loading the index:
+```erb
+<%= search_lunr_js_pipeline %>
+```
+
+You can also remove pipeline functions that Lunr.js enables by default: trimmer, stemmer, and stopWordFilter.
+
+```ruby
+search.pipeline_remove = [
+  'stopWordFilter'
+]
+```
+
+## Index file
+
+The generated index file contains a JSON object with two properties:
+- `index` contains the serialised lunr.js index, which you can load via `lunr.Index.load(lunrData.index)`
+- `docs` is a map from the autogenerated document ids to an object that contains the attributes configured for storage
+
+You will typically load the `index` into a lunr index instance, and then use the `docs` map to look up the returned value and present it to the user.
+
+You should also `require` the `lunr.min.js` file in your main sprockets javascript file (if using the asset pipeline) to be able to actually load the index:
+
+```javascript
+//= require lunr.min
+```
+
+If you're using lunr's i18n capabilities, you should also load the Stemmer support and language files (in that order) here:
+
+```javascript
+//= require lunr.min
+//= require lunr.stemmer.support
+//= require lunr.es
+```
+
+### Asset pipeline
+
+The Middleman pipeline (if enabled) does not include `json` files by default, but you can easily modify this by adding `.json` to the `exts` option of the corresponding extensions, such as `gzip` and `asset_hash`:
+
+```ruby
+activate :asset_hash do |asset_hash|
+  asset_hash.exts << '.json'
+end
+```
+
+Note that if you run the index json file through the asset hash extension, you will need to retrieve the actual destination URL when loading the file in the browser for searching, using the `search_index_path` view helper:
+
+```javascript
+var lunrIndex = null;
+var lunrData  = null;
+
+// Download index data
+$.ajax({
+  url: "<%= search_index_path %>",
+  cache: true,
+  method: 'GET',
+  success: function(data) {
+    lunrData = data;
+    lunrIndex = lunr.Index.load(lunrData.index);
+  }
+});
+```
+
+## Acknowledgments
+
+A big thank you to:
+- [Octo-Labs](https://github.com/Octo-Labs)'s [jagthedrummer](https://github.com/jagthedrummer) for his [`middleman-alias`](https://github.com/Octo-Labs/middleman-alias) extension, in which we based for developing this one.
+- [jnovos](https://github.com/jnovos) and [256dpi](https://github.com/256dpi), for their [`middleman-lunrjs`](https://github.com/jnovos/middleman-lunrjs) and [`middleman-lunr`](https://github.com/256dpi/middleman-lunr) extensions, which served as inspirations for making this one.
+- [olivernn](https://github.com/olivernn) and all [`lunr.js`](http://lunrjs.com/) [contributors](https://github.com/olivernn/lunr.js/graphs/contributors)
+- [MihaiValentin](https://github.com/MihaiValentin) for the support for 10+ languages in [lunr-languages](https://github.com/MihaiValentin/lunr-languages).
+- [The Middleman](https://middlemanapp.com/) [team](https://github.com/orgs/middleman/people) and [contributors](https://github.com/middleman/middleman/graphs/contributors)

data/lib/middleman-search.rb

@@ -0,0 +1,7 @@
+require 'middleman-core'
+require "middleman-search/version"
+
+::Middleman::Extensions.register(:search) do
+  require 'middleman-search/extension'
+  ::Middleman::SearchExtension
+end

data/lib/middleman-search/extension.rb

@@ -0,0 +1,35 @@
+require 'middleman-core'
+require 'middleman-search/search-index-resource'
+
+module Middleman
+  class SearchExtension < Middleman::Extension
+    option :resources, [], 'Paths of resources to index'
+    option :fields, {}, 'Fields to index, with their options'
+    option :before_index, nil, 'Callback to execute before indexing a document'
+    option :index_path, 'search.json', 'Index file path'
+    option :pipeline, {}, 'Javascript pipeline functions to use in lunr index'
+    option :pipeline_remove, {}, 'Default pipeline functions to remove'
+    option :tokenizer_separator, nil, 'Replace the default tokizer separator'
+    option :cache, false, 'Avoid the cache to be rebuilt on every request in development mode'
+    option :language, 'en', 'Language code ("es", "fr") to use when indexing site\'s content'
+    option :lunr_dirs, [], 'Directories in which to look for custom lunr.js files'
+
+    def manipulate_resource_list(resources)
+      resources.push Middleman::Sitemap::SearchIndexResource.new(@app.sitemap, @options[:index_path], @options)
+      resources
+    end
+
+    helpers do
+      def search_lunr_js_pipeline
+        # Thanks http://stackoverflow.com/a/20187415/12791
+        extensions[:search].options[:pipeline].map do |name, function|
+          "lunr.Pipeline.registerFunction(#{function}, '#{name}');"
+        end.join("\n")
+      end
+
+      def search_index_path
+        (config || app.config)[:http_prefix] + sitemap.find_resource_by_path(extensions[:search].options[:index_path]).destination_path
+      end
+    end
+  end
+end

data/lib/middleman-search/search-index-resource.rb

@@ -0,0 +1,157 @@
+# encoding: UTF-8
+
+module Middleman
+  module Sitemap
+    class SearchIndexResource < ::Middleman::Sitemap::Resource
+      def initialize(store, path, options)
+        @resources_to_index = options[:resources]
+        @fields = options[:fields]
+        @callback = options[:before_index]
+        @pipeline = options[:pipeline]
+        @pipeline_remove = options[:pipeline_remove]
+        @tokenizer_separator = options[:tokenizer_separator]
+        @cache_index = options[:cache]
+        @language = options[:language]
+        @lunr_dirs = options[:lunr_dirs] + [File.expand_path("../../../vendor/assets/javascripts/", __FILE__)]
+        super(store, path)
+      end
+
+      def template?
+        false
+      end
+
+      def get_source_file
+        path
+      end
+
+      def render(opts={}, locs={})
+        if @cache_index
+          @index ||= build_index
+        else
+          build_index
+        end
+      end
+
+      def build_index
+        # Build js context
+        libs = []
+        libs << lunr_resource('lunr.js')
+        if @language != 'en' # English is the default
+          libs << lunr_resource("lunr.stemmer.support.js")
+          libs << lunr_resource("lunr.#{@language}.js")
+        end
+
+        source = libs.map { |lib| File.read(lib, mode: "rb:UTF-8") }
+        source << "lunr.Index.prototype.indexJson = function () {return JSON.stringify(this.toJSON());};"
+
+        @pipeline.each do |name, function|
+          source << "lunr.Pipeline.registerFunction((#{function}), '#{name}');"
+        end
+
+        # Build lunr based on config
+        source << "lunr.middlemanSearchIndex = lunr(function () {"
+
+        # Use autogenerated id field as reference
+        source << "this.ref('id');"
+
+        # Remove default pipeline filters
+        @pipeline_remove.each do |name|
+          source << "this.pipeline.remove(lunr.#{name});"
+        end
+
+        # Add functions to pipeline (just registering them isn't enough)
+        @pipeline.each do |name, function|
+          source << "this.pipeline.add(lunr.Pipeline.registeredFunctions.#{name});"
+        end
+
+        if @tokenizer_separator.present?
+          source << "this.tokenizerFn.seperator = #{@tokenizer_separator}"
+        end
+
+        # Use language if set
+        source << "this.use(lunr.#{@language});" if @language != 'en'
+
+        # Define fields with boost
+        @fields.each do |field, opts|
+          next if opts[:index] == false
+          source << "this.field('#{field}', { boost: #{opts[:boost]}});"
+        end
+
+        source << "});"
+
+
+        # Ref to resource map
+        store = Hash.new
+
+        # Iterate over all resources and build index
+        @app.sitemap.resources.each_with_index do |resource, id|
+          begin
+            catch(:skip) do
+              next if resource.data['index'] == false
+              next unless @resources_to_index.any? {|whitelisted| resource.path.start_with? whitelisted }
+
+              to_index = Hash.new
+              to_store = Hash.new
+
+              @fields.each do |field, opts|
+                value = value_for(resource, field, opts)
+                throw(:skip) if value.blank? && opts[:required]
+                to_index[field] = value unless opts[:index] == false
+                to_store[field] = value if opts[:store]
+              end
+
+              @callback.call(to_index, to_store, resource) if @callback
+
+              source << "lunr.middlemanSearchIndex.add(#{to_index.merge(id: id).to_json});"
+
+              store[id] = to_store
+            end
+          rescue => ex
+            @app.logger.warn "Error processing resource for index: #{resource.path}\n#{ex}\n #{ex.backtrace.join("\n ")}"
+          end
+        end
+
+        # Generate JSON output
+        context = ExecJS.compile(source.join("\n"))
+        json = context.eval('lunr.middlemanSearchIndex.indexJson()')
+        "{\"index\": #{json}, \"docs\": #{store.to_json}}"
+      end
+
+      def binary?
+        false
+      end
+
+      def ignored?
+        false
+      end
+
+      def value_for(resource, field, opts={})
+        case field.to_s
+        when 'content'
+
+          html = resource.render( { :layout => false }, { :current_path => resource.path } )
+          Nokogiri::HTML(html).xpath("//text()").text
+        when 'url'
+          resource.url
+        else
+          value = resource.data.send(field) || resource.metadata.fetch(:options, {}).fetch(field, nil)
+          value ? Array(value).compact.join(" ") : nil
+        end
+      end
+
+      private
+
+      def minified_path(resource_name)
+        return resource_name if resource_name.end_with? '.min.js'
+        return resource_name unless resource_name.end_with? '.js'
+        resource_name.sub(/(.*)\.js$/,'\1.min.js')
+      end
+
+      def lunr_resource(resource_name)
+        @lunr_dirs.flat_map do |dir|
+          [File.join(dir, minified_path(resource_name)), File.join(dir, resource_name)]
+        end.detect { |file| File.exists? file } or raise "Couldn't find #{resource_name} nor #{minified_path(resource_name)} in #{@lunr_dirs.map {|dir| File.absolute_path dir }.join File::PATH_SEPARATOR}"
+      end
+    end
+  end
+end