birds 0.0.0

data/README

@@ -0,0 +1,41 @@
+= Birds - Bibliographic information retrieval & document search
+
+== VERSION
+
+This documentation refers to Birds version 0.0.0
+
+
+== DESCRIPTION
+
+Experimental information retrieval system for bibliographic data.
+
+
+== LINKS
+
+Demo::          http://ixtrieve.fh-koeln.de/birds/litie
+Documentation:: https://blackwinter.github.com/birds
+Source code::   https://github.com/blackwinter/birds
+RubyGem::       https://rubygems.org/gems/birds
+
+
+== AUTHORS
+
+* Jens Wille <mailto:jens.wille@gmail.com>
+
+
+== LICENSE AND COPYRIGHT
+
+Copyright (C) 2014-2015 Jens Wille
+
+Birds is free software: you can redistribute it and/or modify it under the
+terms of the GNU Affero General Public License as published by the Free
+Software Foundation, either version 3 of the License, or (at your option)
+any later version.
+
+Birds is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for
+more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with Birds. If not, see <http://www.gnu.org/licenses/>.

data/Rakefile

@@ -0,0 +1,46 @@
+require File.expand_path(%q{../lib/birds/version}, __FILE__)
+
+begin
+  require 'hen'
+
+  Hen.lay! {{
+    gem: {
+      name:         %q{birds},
+      version:      Birds::VERSION,
+      summary:      %q{Bibliographic information retrieval & document search.},
+      description:  %q{Experimental information retrieval system for bibliographic data.},
+      author:       %q{Jens Wille},
+      email:        %q{jens.wille@gmail.com},
+      license:      %q{AGPL-3.0},
+      homepage:     :blackwinter,
+      extra_files:  FileList['*.sample', 'lib/**/{public,views}/*'].to_a,
+      dependencies: {
+        'sinatra-bells' => ['~> 0.0', '>= 0.0.2'],
+        'solr4r'        => ['~> 0.0', '>= 0.0.4'],
+        'unicode'       => '~> 0.4'
+      },
+
+      required_ruby_version: '>= 1.9.3'
+    }
+  }}
+rescue LoadError => err
+  warn "Please install the `hen' gem. (#{err})"
+end
+
+task c: :_ do
+  require 'irb'; IRB.start
+end
+
+task r: :_ do
+  load Gem.bin_path('rack', 'rackup')
+end
+
+task :_ do
+  ARGV.clear
+
+  b = File.expand_path('~/devel')  # XXX
+  $:.unshift(*%w[birds solr4r sinatra-bells].map { |i| File.join(b, i, 'lib') })
+
+  require 'birds'
+  require 'birds/app'
+end

data/config.ru.sample

@@ -0,0 +1,4 @@
+require 'birds'
+require 'birds/app'
+
+run Birds::App.set(solr_core: 'birds')

data/lib/birds/app/controllers.rb

@@ -0,0 +1,115 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Birds -- Bibliographic information retrieval & document search              #
+#                                                                             #
+# Copyright (C) 2014-2015 Jens Wille                                          #
+#                                                                             #
+# Birds is free software: you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation, either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Birds is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Birds. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require 'unicode'
+
+class Birds::App
+
+  get '/', render: :index do
+    result = settings.solr.count
+    @page_title, @page_title_extra = 'Home', "#{result.to_i} documents"
+  end
+
+  get '/search', render: :index do
+    @page_title = 'Search'
+
+    @query, @filter = params[:q] || params[:qq], Array(params[:fq])
+
+    paginate_query(@query, :results,
+      facet_params(debug: 'results', fq: @filter))
+
+    @explain, @facets = explain_result(@result), facet_counts(@result)
+  end
+
+  get '/browse', render: :browse do
+    @page_title, @fields = 'Browse', settings.browse_fields
+  end
+
+  get '/browse/:field', render: :browse do
+    @page_title, @field = 'Browse', params[:field]
+    bad_request unless labels = settings.browse_fields[@field]
+
+    @hierarchy, @page_title_extra = terms, labels.last
+  end
+
+  get '/browse/:field/*', render: :browse do
+    @page_title, @field = 'Browse', params[:field]
+    bad_request unless labels = settings.browse_fields[@field]
+
+    category = params[:splat].join('/')
+    bad_request if category.empty?
+
+    @page_title_extra = "#{labels.first}: #{category}"
+
+    @result = search_query(@field => category)
+    not_found if @result.empty?
+  end
+
+  get '/scroll', render: :scroll do
+    @page_title, @fields = 'Scroll', settings.scroll_fields
+  end
+
+  get '/scroll/:field', render: :scroll do
+    @page_title, @field = 'Scroll', params[:field]
+    bad_request unless label = settings.scroll_fields[@field]
+
+    @letters = terms.group_by { |term,|
+      Unicode.upcase(term[0])
+    }.map { |letter, values|
+      [letter, values.map(&:last).inject(:+)] if letter =~ /\p{Letter}/
+    }.compact
+
+    @page_title_extra = label
+  end
+
+  get '/scroll/:field/:letter', render: :scroll do
+    @page_title, @field = 'Scroll', params[:field]
+    bad_request unless label = settings.scroll_fields[@field]
+
+    @letter = params[:letter]
+    paginate_query({ @field => "#{@letter}*" }, :documents)
+
+    @page_title_extra = "#{label}: #{@letter}"
+  end
+
+  get '/document/*', render: :document do
+    id = params[:splat].join('/')
+
+    bad_request if id.empty?
+    not_found unless @document = search_document(id)
+
+    @similar = @document.more_like_this(settings.mlt_fields,
+      debugQuery: true, mlt: { boost: true, mintf: 1, minwl: 4 })
+
+    @explain = explain_result(@similar)
+
+    @page_title, @page_title_extra = 'Document', "##{id}"
+  end
+
+  get 'schema.xml' do
+    erb :schema
+  end
+
+end

data/lib/birds/app/helpers/controller.rb

@@ -0,0 +1,110 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Birds -- Bibliographic information retrieval & document search              #
+#                                                                             #
+# Copyright (C) 2014-2015 Jens Wille                                          #
+#                                                                             #
+# Birds is free software: you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation, either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Birds is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Birds. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Birds::App
+
+  module Helpers
+
+    module Controller
+
+      def search(params)
+        settings.solr.json_query(params)
+      end
+
+      def search_query(query, params = {})
+        search(params.merge(q: query, fl: '*,score', defType: 'edismax'))
+      end
+
+      def search_document(id)
+        search(q: { id: id }).first
+      end
+
+      def paginate_query(query, what, query_params = {}, per_page = 20)
+        return unless query
+
+        page = params[:page].to_i
+        page = 1 if page < 1
+
+        @prev_page, @next_page = page - 1, page + 1
+
+        @result = search_query(query, query_params.merge(
+          rows: per_page, start: @offset = @prev_page * per_page))
+
+        @page_title_extra = '%d %s, page %d of %d' % [
+          @result, what, page, @total_pages = (@result.to_i / per_page.to_f).ceil]
+
+        @prev_page = nil if @prev_page < 1
+        @next_page = nil if @next_page > @total_pages
+      end
+
+      def facet_params(params = {})
+        params.merge(f: f = params[:f] || {}, facet: {
+          field: fields = [], range: ranges = [], mincount: 1
+        }).tap { settings.facet_fields.each { |facet, (_, options)|
+          options.nil? ? fields << facet : begin ranges << facet
+            ((f[facet] ||= {})[:facet] ||= {})[:range] = options
+          end
+        } }
+      end
+
+      def terms(f = @field)
+        settings.solr.json('terms', terms: { fl: f, limit: -1 }).to_h[f].sort
+      end
+
+      def explain_result(result)
+        result % %w[debug explain] if result
+      end
+
+      def facet_counts(result)
+        return {} unless result && result.to_i > 1
+
+        exclude, prepare = [@query, *@filter], lambda { |facet_hash, &block|
+          facet_hash.delete_if { |key, hash|
+            gap = block[hash] if block
+
+            hash.delete_if { |term,|
+              exclude.include?(facet_query(key, term, *gap)) }.size < 2
+          }
+        }
+
+        prepare.(result.facet_fields.to_h).merge(
+        prepare.(result.facet_ranges.to_h) { |hash|
+          counts, before, start, gap = hash
+            .values_at(*%w[counts before start gap])
+
+          hash.clear
+          hash[-start] = before if before > 1
+          counts.each { |value, count| hash[value.to_i] = count }
+
+          hash.singleton_class.send(:define_method, :gap) { gap }
+          gap
+        })
+      end
+
+    end
+
+  end
+
+end

data/lib/birds/app/helpers/view.rb

@@ -0,0 +1,131 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Birds -- Bibliographic information retrieval & document search              #
+#                                                                             #
+# Copyright (C) 2014-2015 Jens Wille                                          #
+#                                                                             #
+# Birds is free software: you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation, either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Birds is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Birds. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Birds::App
+
+  module Helpers
+
+    module View
+
+      def nav_item(path, name, title = nil)
+        li_(link_to(name, path, title: title), class: active?(path))
+      end
+
+      def link_to_document(document)
+        label = format_label(settings.document_label, document)
+        link_to(h(label), :document, document['id'])
+      end
+
+      def link_to_search(text, query, *filters)
+        options = filters.last.is_a?(Hash) ? filters.pop : {}
+
+        link_to(text, :search, options.merge(params:
+          query ? query_params(query, filters) : { q: filters.first }))
+      end
+
+      def link_to_filter(text, query, filter)
+        link_to_search(h(text), query, filter, *@filter)
+      end
+
+      def link_to_field(field, value, query = nil)
+        link_to_filter(value, query, field_query(field, value))
+      end
+
+      def link_to_facet(field, value, gap = nil)
+        link_to_filter(
+          gap ? range_label(value, gap) : value,
+          @query, facet_query(field, value, gap))
+      end
+
+      def facet_query(field, value, gap = nil)
+        gap ? range_query(field, value, gap) : field_query(field, value)
+      end
+
+      def field_query(field, value)
+        %Q{#{field}:"#{value}"}
+      end
+
+      def range_query(field, value, gap)
+        value < 0 ?
+          %Q|#{field}:[* TO #{-value}}| :
+          %Q|#{field}:[#{value} TO #{value + gap}}|
+      end
+
+      def range_label(value, gap)
+        value < 0 ? "before #{-value}" : "#{value}–#{value + gap - 1}"
+      end
+
+      def query_params(q = @query, fq = @filter)
+        { q: q, 'fq[]' => fq }
+      end
+
+      def pagination_for(*args)
+        params = args.last.is_a?(Hash) ? args.pop.reject { |_, v| v.nil? } : {}
+
+        ul_([
+          [@prev_page, :first, 1],
+          [@prev_page, :prev,  @prev_page],
+          [@next_page, :next,  @next_page],
+          [@next_page, :last,  @total_pages]
+        ], class: 'pagination') { |condition, type, page|
+          li_(link_to_if(condition, pagination_icon(type),
+            *args, params: params.merge(page: page)),
+            class: disabled?(condition))
+        }
+      end
+
+      def pagination_icon(type)
+        tag_(:span, glyphicon(*{
+          first: [:fast_backward, 'First page'],
+          prev:  [:backward,      'Previous page'],
+          next:  [:forward,       'Next page'],
+          last:  [:fast_forward,  'Last page']
+        }[type]))
+      end
+
+      def glyphicon(name, title = nil)
+        name = name.to_s.tr('_', '-')
+        tag_(:span, class: "glyphicon glyphicon-#{name}", title: title)
+      end
+
+      def values_for(key, document = @document)
+        Array(document[key = key.to_s]).dup.tap { |values|
+          return if values.empty?
+
+          if settings.browse_fields.include?(key)
+            values.map! { |v| link_to(v = h(v), :browse, key, v) }
+          elsif field = settings.linkable_fields[key]
+            values.map! { |v| link_to_field(field, v) }
+          else
+            values.map!(&method(:h))
+          end
+        }
+      end
+
+    end
+
+  end
+
+end

data/lib/birds/app/helpers.rb

@@ -0,0 +1,31 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Birds -- Bibliographic information retrieval & document search              #
+#                                                                             #
+# Copyright (C) 2014-2015 Jens Wille                                          #
+#                                                                             #
+# Birds is free software: you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation, either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Birds is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Birds. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+require_relative 'helpers/controller'
+require_relative 'helpers/view'
+
+class Birds::App
+  helpers *Helpers.constants.map { |mod| Helpers.const_get(mod) }
+end

data/lib/birds/app/settings.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+
+#--
+###############################################################################
+#                                                                             #
+# Birds -- Bibliographic information retrieval & document search              #
+#                                                                             #
+# Copyright (C) 2014-2015 Jens Wille                                          #
+#                                                                             #
+# Birds is free software: you can redistribute it and/or modify it under the  #
+# terms of the GNU Affero General Public License as published by the Free     #
+# Software Foundation, either version 3 of the License, or (at your option)   #
+# any later version.                                                          #
+#                                                                             #
+# Birds is distributed in the hope that it will be useful, but WITHOUT ANY    #
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS   #
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for     #
+# more details.                                                               #
+#                                                                             #
+# You should have received a copy of the GNU Affero General Public License    #
+# along with Birds. If not, see <http://www.gnu.org/licenses/>.               #
+#                                                                             #
+###############################################################################
+#++
+
+class Birds::App
+
+  set :site_title,
+    'Birds – Bibliographic information retrieval & document search'
+
+  set :bootstrap_url,
+    '//netdna.bootstrapcdn.com/bootstrap/3.1.1'
+
+  set :solr_client do
+    require 'solr4r'
+    Solr4R::Client
+  end
+
+  solr_opts = Hash[%w[host port path core].map { |key| [key, "solr_#{key}"] }]
+
+  set :solr_opts do
+    Hash[solr_opts.map { |key, opt| [key.to_sym, settings.send(opt)] }]
+  end
+
+  solr_opts.each { |key, opt| set(opt) {
+    settings.solr_client.const_get("DEFAULT_#{key.upcase}") } }
+
+  set :solr do
+    settings.solr_client.new(settings.solr_opts)
+  end
+
+  set_hash :display_fields, %w[
+    title author year abstract language theme subject
+  ], %w[_txt] do |f, l| [f, l.capitalize] end
+
+  set_hash :linkable_fields, %w[
+    author theme subject
+  ], %w[_txt _ss]
+
+  set_hash :facet_fields, %w[
+    author language theme subject
+  ].insert(2,
+    [:year, start: 1900, end: 2100, gap: 10, other: 'before']
+  ) do |f, o| ["#{f}_#{o ? :i : :ss}", ["#{f}s".capitalize, o]] end
+
+  set :mlt_fields, %w[
+    author_txt title_txt abstract_txt subject_txt
+  ]
+
+  set :browse_fields, {
+#     'cat'        => %w[Category     Categories],
+#     'manu_exact' => %w[Manufacturer Manufacturers]
+  }
+
+  set :scroll_fields, {
+#     'author_s' => 'Author'
+  }
+
+  set :sample_queries, {
+    'Sample query 1' => 'author_txt:fugmann',
+    'Sample query 2' => 'fugmann -author_txt:fugmann'
+  }
+
+  set :document_label, '{author_ss:%s: }{title_ss( : )}{year_ss: (%s)}'
+
+end

data/lib/birds/app/views/_documents.erb

@@ -0,0 +1,13 @@
+<ol start="<%= defined?(start) && start || 1 %>" style="margin-top: 1em"><!-- XXX -->
+<% for document in documents; id, score = document['id'], h('%.2f' % document['score']); next if @document && id == @document['id'] %>
+  <li>
+    <%= link_to_document(document) %>
+  <% unless defined?(explain) && explain %>
+    <span class="badge"><%= score %></span>
+  <% else; eid = h("explain-#{id}") %>
+    <span class="badge toggle-explain" data-toggle="collapse" data-target="#<%= eid %>" title="Toggle score explanation"><%= score %></span>
+    <pre id="<%= eid %>" class="collapse small"><%=h explain[id] %></pre>
+  <% end %>
+  </li>
+<% end %>
+</ol>

data/lib/birds/app/views/_facets.erb

@@ -0,0 +1,30 @@
+<% position = 0; for facet, (label, _) in settings.facet_fields; counts = @facets[facet] or next; collapse = max = nil %>
+  <div class="panel panel-default">
+    <div class="panel-heading">
+      <h4 class="panel-title toggle-facet" data-toggle="collapse" data-target="#<%= fid = "facet-#{h(facet)}" %>" title="Toggle facet list"><%=h label %></h4>
+    </div>
+    <ul class="list-group compact-list-group collapse<%= ' in' if (position += 1) < 4 %>" id="<%= fid %>">
+  <% if counts.respond_to?(:gap) %>
+    <% for (value, count), index in counts.reverse_each.with_index %>
+      <li class="list-group-item clearfix<%= ' facet-collapse collapse' if collapse ||= index > 2 %>">
+        <%= link_to_facet(facet, value, counts.gap) %>
+        <span class="badge"><%= count %></span>
+      </li>
+    <% end %>
+  <% else %>
+    <% for (value, count), index in counts.each_with_index; max ||= count %>
+      <li class="list-group-item clearfix<%= ' facet-collapse collapse' if collapse ||= index > 5 || count < max / 10 %>">
+        <%= link_to_facet(facet, value) %>
+        <span class="badge"><%= count %></span>
+      </li>
+    <% end %>
+  <% end %>
+    <% if collapse %>
+      <li class="list-group-item toggle-facet" data-toggle="collapse" data-target="#<%= fid %> .facet-collapse.collapse">
+        <span class="facet-collapse collapse in">More…</span>
+        <span class="facet-collapse collapse">Less…</span>
+      </li>
+    <% end %>
+    </ul>
+  </div>
+<% end %>

data/lib/birds/app/views/browse.erb

@@ -0,0 +1,22 @@
+<% if @result %>
+  <ol>
+  <% for document in @result %>
+    <li><%= link_to_document(document) %></li>
+  <% end %>
+  </ol>
+<% elsif @fields %>
+  <ul>
+  <% for field, labels in @fields %>
+    <li><%= link_to(h(labels.last), :browse, field) %></li>
+  <% end %>
+  </ul>
+<% else %>
+  <ul>
+  <% for category, count in @hierarchy %>
+    <li>
+      <%= link_to(h(category), :browse, @field, category) %>
+      <span class="badge"><%=h count %></span>
+    </li>
+  <% end %>
+  </ul>
+<% end %>

data/lib/birds/app/views/document.erb

@@ -0,0 +1,12 @@
+<dl class="dl-horizontal">
+<% for key, label in settings.display_fields; values = values_for(key) or next %>
+  <dt class="text-muted" title="<%=h key %>:"><%=h label || key %></dt>
+  <dd><%= values.join('<br />') %></dd>
+<% end %>
+</dl>
+
+<% unless @similar.empty? %>
+<h2>Similar documents</h2>
+
+<%= erb :_documents, locals: { documents: @similar, explain: @explain } %>
+<% end %>