iqvoc_similar_terms 0.8.0 → 2.11.3

data/README.md

@@ -1,21 +1,16 @@
-Similar Terms for iQvoc
+# Similar terms for iQvoc
 
+![CI](https://github.com/innoq/iqvoc_similar_terms/workflows/CI/badge.svg)
 
-Getting Started
-===============
+## Getting Started
 
-1. Initialize database:
-
-    $ bundle exec rake db:create
-    $ bundle exec rake db:migrate iqvoc:db:seed_all
 
-2. Generate secret token:
-
-    $ rake iqvoc:setup:generate_secret_token
+1. Initialize database:
 
+        $ bundle exec rake db:create
+        $ bundle exec rake db:migrate iqvoc:db:seed_all
 
-Extensions
-==========
+## Extensions
 
 In addition to regular SKOS, Similar Terms supports
 [SKOS-XL](https://github.com/innoq/iqvoc_skosxl) and

data/app/assets/javascripts/manifest.js

@@ -0,0 +1,2 @@
+import '../../../node_modules/iqvoc/app/assets/javascripts/framework';
+import '../../../node_modules/iqvoc/app/assets/javascripts/iqvoc/manifest';

data/app/assets/stylesheets/manifest.scss

@@ -0,0 +1,3 @@
+@import 'iqvoc/app/assets/stylesheets/iqvoc/settings/manifest';
+@import 'iqvoc/app/assets/stylesheets/framework';
+@import 'iqvoc/app/assets/stylesheets/iqvoc/_manifest';

data/app/controllers/similar_terms_controller.rb

@@ -1,28 +1,61 @@
 # encoding: UTF-8
 
-require 'iqvoc/similar_terms' # XXX: should not be necessary!?
-
 class SimilarTermsController < ApplicationController
+  resource_description do
+    name 'Similar concepts'
+  end
 
-  def show
-    authorize! :read, Iqvoc::Concept.base_class
+  api :GET, ':lang/similar', <<-DOC
+    Returns labels which are semantically (not orthographically) similar,
+    as determined by concept relations.
+  DOC
+  formats [:html, :ttl, :rdf, :xml, :json]
+  param :terms, String, :required => true, :desc => <<-DOC
+    One or more terms (comma-separated, CSV-style) to be processed.
+  DOC
+  example <<-DOC
+    GET /de/similar.ttl?terms=wald,baum
 
-    unless params[:terms]
-      head 400 unless request.format.html? # non-GUI
-      return
-    end
+    # omitted namespace definitions
+    query:top skos:altLabel "Dünenwald"@de;
+              skos:altLabel "Baumart"@de.
+  DOC
+
+  def create
+    authorize! :read, Iqvoc::Concept.base_class
 
-    @terms = Iqvoc::InlineDataHelper.parse_inline_values(params[:terms])
+    @terms = InlineDataHelper.parse_inline_values(similar_terms_params)
     lang = params[:lang]
+    options = {
+      synonyms_only: ['1', 'true'].include?(params[:synonyms_only])
+    }
 
     respond_to do |format|
       format.html do
-        @results = Iqvoc::SimilarTerms.ranked(lang, *@terms)
+        @results = Services::SimilarTermsService.ranked(lang, options, *@terms)
+        render :show
       end
-      format.any(:rdf, :ttl) do
-        @results = Iqvoc::SimilarTerms.alphabetical(lang, *@terms)
+      format.any(:rdf, :ttl, :nt, :xml) do
+        @results = Services::SimilarTermsService.alphabetical(lang, options, *@terms)
+        render :show
       end
+      format.json {
+        results = Services::SimilarTermsService.alphabetical(lang, options, *@terms)
+        render json: {
+          "url": request.original_url,
+          "total_results": results.length,
+          "results": results.map {|c| c.value }
+        }.to_json
+      }
     end
   end
 
+  def new
+  end
+
+  private
+
+  def similar_terms_params
+    params.require(:terms)
+  end
 end

data/app/models/services/similar_terms_service.rb

@@ -0,0 +1,141 @@
+module Services
+  module SimilarTermsService # TODO: make language constraints optional
+
+    @@weightings = Iqvoc::SimilarTerms.weightings
+
+    # returns an array of label/concepts pairs, sorted descendingly by weighting -- XXX: unused/deprecated
+    # TODO: rename
+    def self.ranked(lang, options, *terms)
+      weighted(lang, options, *terms).sort_by { |label, _data| label }
+                            .sort_by { |_label, data| data[0] }
+                            .reverse
+                            .map { |label, data| [label] + data[1..-1] } # drop weighting
+    end
+
+    # returns a list of labels, sorted alphabetically
+    def self.alphabetical(lang, options, *terms)
+      concepts = base_query(lang, *terms)
+
+      results = concepts.map { |c| c.labelings.map { |ln| ln.target } }
+
+      unless options[:synonyms_only]
+        results << find_related_and_narrower_concepts(concepts, lang, *terms).map { |c| c.pref_labels }
+      end
+
+      results.flatten.sort_by { |l| l.value }
+    end
+
+    def self.base_query(lang, *terms)
+      # FIXME: move to initializer to run on start/boot. Could be a problem with intializer load
+      # order of host app because further_labeling classes are initialised after this check
+      Iqvoc::Concept.labeling_class_names.each do |klass_name, langs|
+        unless @@weightings.keys.include? klass_name
+          raise "#{klass_name} has no registered weighting. Please configure one using Iqvoc::SimilarTerms.register_weighting('MyLabelingClass', 1.0)"
+        end
+      end
+
+      # use only labelings with weighting > 0
+      used_weightings = @@weightings.select { |klass_name, weight| weight > 0 }.keys
+      terms_to_concepts(lang, *terms)
+        .includes(:labelings => [:owner, :target])
+        .where('labels.language' => lang)
+        .where('labelings.type IN (?)', used_weightings)
+    end
+
+    # returns a hash of label/weighting+concepts pairs -- XXX: unused/deprecated
+    def self.weighted(lang, options, *terms) # TODO: rename
+      concepts = base_query(lang, *terms)
+
+      return terms.inject({}) do |memo, term|
+        concepts.each do |concept|
+          concept.labelings.each do |ln|
+            concept = ln.owner
+            label = ln.target
+            weight = @@weightings[ln.class.name]
+
+            memo[label] ||= []
+            # weighting
+            memo[label][0] ||= 0
+            memo[label][0] += weight
+            # associated concepts
+            memo[label] << concept unless memo[label].include? concept
+          end
+        end
+
+        unless options[:synonyms_only]
+          find_related_and_narrower_concepts(concepts, lang, *terms).each do |c|
+            memo[c.pref_label] ||= []
+            # weighting
+            memo[c.pref_label][0] ||= 0
+            memo[c.pref_label][0] += 1
+            # associated concepts
+            memo[c.pref_label] << c unless memo[c.pref_label].include? c
+          end
+        end
+
+        memo
+      end
+    end
+
+    def self.terms_to_concepts(lang, *terms)
+      concept_ids = terms_to_labels(lang, *terms)
+                    .includes(:labelings)
+                    .map { |label| label.labelings.map(&:owner_id) }.flatten.uniq
+
+      return Iqvoc::Concept.base_class.published.where(id: concept_ids)
+    end
+
+    def self.find_related_and_narrower_concepts(concepts, lang, *terms)
+      results = []
+      concepts.each do |c|
+        results << c.narrower_relations.published.map(&:target_id)
+        results << c.concept_relation_skos_relateds.published.map(&:target_id)
+      end
+
+      # evaluate only if iqvoc_compound_forms engine is loaded
+      if Iqvoc.const_defined?(:CompoundForms) && results.empty?
+        terms.each do |term|
+          label = if Iqvoc.const_defined?(:Inflectionals)
+                    hash = Inflectional::Base.normalize(term)
+                    label_id = Inflectional::Base.select([:label_id])
+                                                 .where(:normal_hash => hash)
+                                                 .map(&:label_id)
+                                                 .first
+
+                    Iqvoc::XLLabel.base_class.where(:language => lang, :id => label_id).first
+                  else
+                    Iqvoc::XLLabel.base_class.where('LOWER(value) = ?', term.downcase).first
+                  end
+
+          results << label.compound_in.map { |ci| ci.concepts.map { |c| c.id } } if label.present?
+        end
+      end
+      Iqvoc::Concept.base_class.where(id: results.flatten.uniq)
+    end
+
+    # NB: case-insensitive only when inflectionals are available
+    def self.terms_to_labels(lang, *terms)
+      # efficiency enhancement to turn `IN` into `=` queries where possible
+      reduce = lambda { |arr| arr.length < 2 ? arr[0] : arr }
+
+      if Iqvoc.const_defined?(:Inflectionals)
+        # use normalized form for case-insensitivity (and performance)
+        hashes = terms.map { |term| Inflectional::Base.normalize(term) }
+        label_ids = Inflectional::Base.select([:label_id])
+                                      .where(normal_hash: reduce.call(hashes))
+                                      .map(&:label_id)
+
+        return Iqvoc::XLLabel.base_class.where(language: lang, id: reduce.call(label_ids))
+      elsif Iqvoc.const_defined?(:XLLabel)
+        return Iqvoc::XLLabel.base_class
+                             .where(language: lang)
+                             .where('LOWER(value) IN (?)', terms.map(&:downcase))
+      else
+        return Iqvoc::Label.base_class
+                           .where(language: lang)
+                           .where('LOWER(value) IN (?)', terms.map(&:downcase))
+      end
+    end
+
+  end
+end

data/app/views/similar_terms/_form.html.erb

@@ -0,0 +1,21 @@
+<%= bootstrap_form_tag(url: similar_url, method: :get) do |f| %>
+  <div class="p-3 row no-gutters align-items-center bg-white">
+    <div class="col-auto">
+      <i class="fa fa-search fa-lg"></i>
+    </div>
+    <div class="col">
+      <%= f.text_field :terms, hide_label: true, autofocus: (@terms ? false : true), wrapper_class: 'mb-0',
+                       class: 'form-control-lg form-control-borderless border-0', autocomplete: 'off',
+                       value: (@terms ? InlineDataHelper.generate_inline_values(@terms) : params[:terms])
+      %>
+    </div>
+    <div class="col-auto">
+      <%= f.submit t('txt.views.search_results.submit'), class: 'btn btn-block btn-lg btn-primary p-2' %>
+    </div>
+  </div>
+  <div class="py-3 row">
+    <div class="col-auto">
+      <%= f.check_box :synonyms_only, label: t('txt.views.similar_terms.synonyms_only'), checked: ['1', 'true'].include?(params[:synonyms_only]) %>
+    </div>
+  </div>
+<% end %>

data/app/views/similar_terms/new.html.erb

@@ -0,0 +1,21 @@
+<div class="text-center my-5">
+  <h1 class="mb-5">
+    <%= t("txt.views.similar_terms.title") %>
+  </h1>
+
+  <div class="row justify-content-center">
+    <div class="col-12 col-md-10 col-lg-8">
+      <%= render 'form' %>
+    </div>
+  </div>
+</div>
+
+<div class="my-5">
+  <h2 class="text-center my-4">
+    <%= t("txt.views.similar_terms.about.title") %>
+  </h2>
+
+  <div class="row col-md-8 offset-md-2">
+    <%= simple_format t("txt.views.similar_terms.about.description") %>
+  </div>
+</div>

data/app/views/similar_terms/show.html.erb

@@ -1,32 +1,36 @@
-<%= page_header :title => "Similar Terms" %> <%# TODO: i18n %>
+<% if @results.any? %>
+  <h2 class="text-center my-5">
+    <%= raw t('txt.views.similar_terms.results', terms: @terms.join(', '), count: @results.size) %>
+  </h2>
 
-<% if @terms %>
-  <%= sidebar do %>
-    <%= sidebar_header t('txt.common.representations') %>
-    <%= sidebar_item :icon => 'share', :text => 'HTML', :path => similar_path(:terms => params[:terms], :format => :html) %>
-    <%= sidebar_item :icon => 'share', :text => 'RDF/XML', :path => similar_path(:terms => params[:terms], :format => :rdf) %>
-    <%= sidebar_item :icon => 'share', :text => 'RDF/Turtle', :path => similar_path(:terms => params[:terms], :format => :ttl) %>
-  <% end %>
-<% end %>
-
-<%= form_tag(similar_path, :method => :get, :class => 'form-horizontal') do %>
-  <%= text_field_tag :terms, (@terms ?
-          Iqvoc::InlineDataHelper.generate_inline_values(@terms) :
-          params[:terms]) %>
-  <%= submit_tag t('txt.views.search_results.submit'), :class => 'btn btn-primary' %>
-<% end %>
-
-<ol>
-  <% (@results || []).each do |data|
-    label = data[0]
-    concepts = data[1..-1]
-  %>
-    <li>
-      <%= label.value %>
-      (<%= concepts.map do |concept|
+  <ol>
+    <% (@results || []).each do |data|
+      label = data[0]
+      concepts = data[1..-1]
+    %>
+      <li>
+        <%= label.value %>
+        (<%= concepts.map do |concept|
         origin = concept.origin
         link_to origin, rdf_url(origin, :format => nil, :lang => nil)
       end.join(", ").html_safe %>)
-    </li>
-  <% end %>
-</ol>
+      </li>
+    <% end %>
+  </ol>
+<% else %>
+  <h2 class="text-center my-5">
+    <%= t('txt.views.similar_terms.no_results') %>
+  </h2>
+<% end %>
+
+<div class="text-center my-5">
+  <h1 class="mb-5">
+    <%= t("txt.views.similar_terms.title") %>
+  </h1>
+
+  <div class="row justify-content-center">
+    <div class="col-12 col-md-10 col-lg-8">
+      <%= render 'form' %>
+    </div>
+  </div>
+</div>

data/app/views/similar_terms/show.xml.builder

@@ -0,0 +1,17 @@
+xml = Builder::XmlMarkup.new(:indent => 2)
+xml.instruct!
+
+xml.OneBoxResults "xmlns:xlink" => "http://www.w3.org/1999/xlink" do
+  xml.resultCode "success"
+  xml.totalResults @results.length
+  xml.urlText t("txt.views.similar_terms.title")
+  xml.urlLink url_for(request.query_parameters.
+      merge(:only_path => false, :anchor => ""))
+  xml.MODULE_RESULT do
+    xml.title t("txt.views.similar_terms.results_heading",
+        :terms => params[:terms])
+    @results.each do |label|
+      xml.Field label.value, "name" => label.value
+    end
+  end if @results.length > 0
+end

data/bin/bundle

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
+load Gem.bin_path('bundler', 'bundle')

data/bin/rails

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+APP_PATH = File.expand_path("../config/application", __dir__)
+require_relative "../config/boot"
+require "rails/commands"

data/bin/rake

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require_relative "../config/boot"
+require "rake"
+Rake.application.run

data/bin/setup

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require "fileutils"
+
+# path to your application root.
+APP_ROOT = File.expand_path("..", __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+FileUtils.chdir APP_ROOT do
+  # This script is a way to set up or update your development environment automatically.
+  # This script is idempotent, so that you can run it at any time and get an expectable outcome.
+  # Add necessary setup steps to this file.
+
+  puts "== Installing dependencies =="
+  system! "gem install bundler --conservative"
+  system("bundle check") || system!("bundle install")
+
+  # puts "\n== Copying sample files =="
+  # unless File.exist?("config/database.yml")
+  #   FileUtils.cp "config/database.yml.sample", "config/database.yml"
+  # end
+
+  puts "\n== Preparing database =="
+  system! "bin/rails db:prepare"
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! "bin/rails log:clear tmp:clear"
+
+  puts "\n== Restarting application server =="
+  system! "bin/rails restart"
+end

data/bin/update

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+require 'fileutils'
+include FileUtils
+
+# path to your application root.
+APP_ROOT = File.expand_path('..', __dir__)
+
+def system!(*args)
+  system(*args) || abort("\n== Command #{args} failed ==")
+end
+
+chdir APP_ROOT do
+  # This script is a way to update your development environment automatically.
+  # Add necessary update steps to this file.
+
+  puts '== Installing dependencies =='
+  system! 'gem install bundler --conservative'
+  system('bundle check') || system!('bundle install')
+
+  # Install JavaScript dependencies if using Yarn
+  # system('bin/yarn')
+
+  puts "\n== Updating database =="
+  system! 'bin/rails db:migrate'
+
+  puts "\n== Removing old logs and tempfiles =="
+  system! 'bin/rails log:clear tmp:clear'
+
+  puts "\n== Restarting application server =="
+  system! 'bin/rails restart'
+end

data/bin/yarn

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+APP_ROOT = File.expand_path('..', __dir__)
+Dir.chdir(APP_ROOT) do
+  yarn = ENV["PATH"].split(File::PATH_SEPARATOR).
+    select { |dir| File.expand_path(dir) != __dir__ }.
+    product(["yarn", "yarn.cmd", "yarn.ps1"]).
+    map { |dir, file| File.expand_path(file, dir) }.
+    find { |file| File.executable?(file) }
+
+  if yarn
+    exec yarn, *ARGV
+  else
+    $stderr.puts "Yarn executable was not detected in the system."
+    $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install"
+    exit 1
+  end
+end

data/config/application.rb

@@ -1,67 +1,38 @@
-require File.expand_path('../boot', __FILE__)
-
-require 'rails/all'
-
-if defined?(Bundler)
-  # If you precompile assets before deploying to production, use this line
-  Bundler.require *Rails.groups(:assets => %w(development test))
-  # If you want your assets lazily compiled in production, use this line
-  # Bundler.require(:default, :assets, Rails.env)
-end
+require_relative "boot"
+
+# Pick the frameworks you want:
+require "active_model/railtie"
+require "active_job/railtie"
+require "active_record/railtie"
+require "active_storage/engine"
+require "action_controller/railtie"
+require "action_mailer/railtie"
+require "action_mailbox/engine"
+require "action_text/engine"
+require "action_view/railtie"
+require "action_cable/engine"
+# require "sprockets/railtie"
+require "rails/test_unit/railtie"
+
+# Require the gems listed in Gemfile, including any gems
+# you've limited to :test, :development, or :production.
+Bundler.require(*Rails.groups)
 
 module Iqvoc::SimilarTerms
   class Application < Rails::Application
+    # Initialize configuration defaults for originally generated Rails version.
+    config.load_defaults 7.0
+
+    # Configuration for the application, engines, and railties goes here.
+    #
+    # These settings can be overridden in specific environments using the files
+    # in config/environments, which are processed later.
+    #
+    # config.time_zone = "Central Time (US & Canada)"
+    # config.eager_load_paths << Rails.root.join("extras")
+    #
 
     # This tells iqvoc_similar_terms.rb whether it it runs as app or as engine
     require 'iqvoc_similar_terms'
-
-    # Settings in config/environments/* take precedence over those specified here.
-    # Application configuration should go into files in config/initializers
-    # -- all .rb files in that directory are automatically loaded.
-
-    # Custom directories with classes and modules you want to be autoloadable.
-    # config.autoload_paths += %W(#{config.root}/extras)
-
-    # Only load the plugins named here, in the order given (default is alphabetical).
-    # :all can be used as a placeholder for all plugins not explicitly named.
-    # config.plugins = [ :exception_notification, :ssl_requirement, :all ]
-
-    # Activate observers that should always be running.
-    # config.active_record.observers = :cacher, :garbage_collector, :forum_observer
-
-    # Set Time.zone default to the specified zone and make Active Record auto-convert to this zone.
-    # Run "rake -D time" for a list of tasks for finding time zone names. Default is UTC.
-    # config.time_zone = 'Central Time (US & Canada)'
-
-    # The default locale is :en and all translations from config/locales/*.rb,yml are auto loaded.
-    # config.i18n.load_path += Dir[Rails.root.join('my', 'locales', '*.{rb,yml}').to_s]
-    config.i18n.default_locale = :de
-    # config.i18n.available_locales = [:de, :en]
-
-    # Configure the default encoding used in templates for Ruby 1.9.
-    config.encoding = "utf-8"
-
-    # Configure sensitive parameters which will be filtered from the log file.
-    config.filter_parameters += [:password, :password_confirmation]
-
-    # Enable escaping HTML in JSON.
-    config.active_support.escape_html_entities_in_json = true
-
-    # Use SQL instead of Active Record's schema dumper when creating the database.
-    # This is necessary if your schema can't be completely dumped by the schema dumper,
-    # like if you have constraints or database-specific column types
-    # config.active_record.schema_format = :sql
-
-    # Enforce whitelist mode for mass assignment.
-    # This will create an empty whitelist of attributes available for mass-assignment for all models
-    # in your app. As such, your models will need to explicitly whitelist or blacklist accessible
-    # parameters by using an attr_accessible or attr_protected declaration.
-    config.active_record.whitelist_attributes = true
-
-    # Enable the asset pipeline
-    config.assets.enabled = true
-
-    # Version of your assets, change this if you want to expire all your assets
-    config.assets.version = "1.0.0"
   end
 end

data/config/boot.rb

@@ -1,6 +1,4 @@
-require 'rubygems'
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__)
 
-# Set up gems listed in the Gemfile.
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__)
-
-require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE'])
+require "bundler/setup" # Set up gems listed in the Gemfile.
+require "bootsnap/setup" # Speed up boot time by caching expensive operations.

data/config/cable.yml

@@ -0,0 +1,10 @@
+development:
+  adapter: async
+
+test:
+  adapter: test
+
+production:
+  adapter: redis
+  url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
+  channel_prefix: iqvoc/similar_terms_production

data/config/database.yml.mysql

@@ -6,7 +6,7 @@ development:
   pool: 5
   username: root
   password:
-  hostname: 127.0.0.1
+  host: 127.0.0.1
 
 test:
   adapter: mysql2
@@ -16,7 +16,7 @@ test:
   pool: 5
   username: root
   password:
-  hostname: 127.0.0.1
+  host: 127.0.0.1
 
 production:
   adapter: mysql2
@@ -26,4 +26,4 @@ production:
   pool: 5
   username: root
   password:
-  hostname: 127.0.0.1
+  host: 127.0.0.1

data/config/database.yml.postgresql

@@ -0,0 +1,26 @@
+development:
+  adapter: postgresql
+  host: localhost
+  encoding: unicode
+  database: <%= ENV["POSTGRES_DB"] || 'iqvoc_similar_terms_development' %>
+  pool: 5
+  username: <%= ENV["POSTGRES_USER"] || 'postgres' %>
+  password: <%= ENV["POSTGRES_PASSWORD"] || 'postgres' %>
+
+test:
+  adapter: postgresql
+  host: localhost
+  encoding: unicode
+  database: <%= ENV["POSTGRES_DB"] || 'iqvoc_similar_terms_test' %>
+  pool: 5
+  username: <%= ENV["POSTGRES_USER"] || 'postgres' %>
+  password: <%= ENV["POSTGRES_PASSWORD"] || 'postgres' %>
+
+production:
+  adapter: postgresql
+  host: localhost
+  encoding: unicode
+  database: <%= ENV["POSTGRES_DB"] || 'iqvoc_similar_terms_production' %>
+  pool: 5
+  username: <%= ENV["POSTGRES_USER"] || 'postgres' %>
+  password: <%= ENV["POSTGRES_PASSWORD"] || 'postgres' %>

data/config/engine.rb

@@ -7,7 +7,7 @@ module Iqvoc
       paths["lib/tasks"] << "lib/engine_tasks"
 
       initializer "iqvoc_similar_terms.load_migrations" do |app|
-        app.config.paths['db/migrate'] += Iqvoc::SimilarTerms::Engine.paths['db/migrate'].existent
+        app.config.paths['db/migrate'].concat(Iqvoc::SimilarTerms::Engine.paths['db/migrate'].existent)
       end
     end