iqvoc 4.0.3 → 4.0.4

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 4.0.4
+
+  * Triplestore Sync
+  * Bugfixes
+
 ## 4.0.3
 
   * New frontpage with quick search

data/Gemfile

@@ -30,7 +30,8 @@ end
 gem 'kaminari'
 gem 'authlogic'
 gem 'cancan'
-gem 'iq_rdf', '~> 0.1.2'
+gem 'iq_rdf'
+gem 'iq_triplestorage'
 gem 'json'
 gem 'rails_autolink'
 gem 'jruby-openssl', :platforms => :jruby
@@ -67,6 +68,7 @@ group :test do
   gem 'spork-testunit'
   gem 'turn'
   gem 'minitest'
+  gem 'webmock'
 end
 
 group :production do

data/Gemfile.lock

@@ -60,6 +60,7 @@ GEM
       json
     childprocess (0.3.5)
       ffi (~> 1.0, >= 1.0.6)
+    crack (0.3.1)
     database_cleaner (0.8.0)
     erubis (2.7.0)
     excon (0.16.2)
@@ -83,9 +84,11 @@ GEM
       excon (~> 0.16.1)
     hike (1.2.1)
     i18n (0.6.1)
-    iq_rdf (0.1.5)
+    iq_rdf (0.1.8)
       builder
       bundler
+    iq_triplestorage (0.1.2)
+      typhoeus
     jdbc-mysql (5.1.13)
     jdbc-sqlite3 (3.7.2)
     journey (1.0.4)
@@ -179,12 +182,18 @@ GEM
       polyglot (>= 0.3.1)
     turn (0.9.6)
       ansi
+    typhoeus (0.4.2)
+      ffi (~> 1.0)
+      mime-types (~> 1.18)
     tzinfo (0.3.33)
     uglifier (1.3.0)
       execjs (>= 0.3.0)
       multi_json (~> 1.0, >= 1.0.2)
     view_marker (1.0.0)
       rails
+    webmock (1.8.11)
+      addressable (>= 2.2.7)
+      crack (>= 0.1.7)
     xpath (0.1.4)
       nokogiri (~> 1.3)
 
@@ -206,7 +215,8 @@ DEPENDENCIES
   factory_girl_rails
   fastercsv
   heroku
-  iq_rdf (~> 0.1.2)
+  iq_rdf
+  iq_triplestorage
   jruby-openssl
   json
   kaminari
@@ -225,3 +235,4 @@ DEPENDENCIES
   turn
   uglifier (>= 1.0.3)
   view_marker
+  webmock

data/app/controllers/concepts/versions_controller.rb

@@ -14,7 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require 'iqvoc/rdf_sync'
+
 class Concepts::VersionsController < ApplicationController
+  include Iqvoc::RDFSync::Helper
 
   def merge
     current_concept = Iqvoc::Concept.base_class.by_origin(params[:origin]).published.last
@@ -25,13 +28,17 @@ class Concepts::VersionsController < ApplicationController
 
     ActiveRecord::Base.transaction do
       if current_concept.blank? || current_concept.destroy
+        new_version.rdf_updated_at = nil
         new_version.publish
         new_version.unlock
         if new_version.valid_with_full_validation?
           new_version.save
-          if RdfStore.update(rdf_url(:id => new_version, :format => :ttl), concept_url(:id => new_version, :format => :ttl))
-            new_version.update_attribute(:rdf_updated_at, 1.seconds.since)
+
+          if Iqvoc.config["triplestore.autosync"]
+           synced = triplestore_syncer.sync([new_version]) # XXX: blocking
+           flash[:warning] = "triplestore synchronization failed" unless synced # TODO: i18n
           end
+
           flash[:success] = t("txt.controllers.versioning.published")
           redirect_to concept_path(:id => new_version)
         else

data/app/controllers/rdf_controller.rb

@@ -27,4 +27,25 @@ class RdfController < ApplicationController
     end
   end
 
+  def show
+    scope = if params[:published] == "0"
+      Iqvoc::Concept.base_class.unpublished
+    else
+      Iqvoc::Concept.base_class.published
+    end
+    if @concept = scope.by_origin(params[:id]).with_associations.last
+      respond_to do |format|
+        format.html do
+          redirect_to concept_url(:id => @concept.origin, :published => params[:published])
+        end
+        format.any do
+          authorize! :read, @concept
+          render "concepts/show"
+        end
+      end
+    else
+      raise ActiveRecord::RecordNotFound.new("Concept '#{params[:id]}' not found.")
+    end
+  end
+
 end

data/app/controllers/triplestore_sync_controller.rb

@@ -0,0 +1,61 @@
+# encoding: UTF-8
+
+# Copyright 2011 innoQ Deutschland GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'iqvoc/rdf_sync'
+
+class TriplestoreSyncController < ApplicationController
+  include Iqvoc::RDFSync::Helper
+
+  def index
+    authorize! :use, :dashboard
+
+    if Iqvoc.config["triplestore.url"] == Iqvoc.config.defaults["triplestore.url"]
+      flash.now[:warning] = I18n.t("txt.controllers.triplestore_sync.config_warning")
+    else
+      host = Iqvoc.config["triplestore.url"]
+      username = Iqvoc.config["triplestore.username"].presence
+      password = Iqvoc.config["triplestore.password"].presence
+      target_info = host
+      if username && password
+        target_info = "#{target_info} (as #{username} with password)" # XXX: i18n
+      elsif username
+        target_info = "#{target_info} (as #{username})" # XXX: i18n
+      end
+      flash.now[:info] = I18n.t("txt.controllers.triplestore_sync.config_info",
+          :target_info => target_info)
+    end
+
+    # per-class pagination
+    @candidates = Iqvoc::RDFSync.candidates.map do |records|
+      records.page(params[:page])
+    end
+  end
+
+  def sync
+    authorize! :use, :dashboard
+
+    success = triplestore_syncer.all # XXX: long-running
+
+    if success
+      flash[:success] = I18n.t("txt.controllers.triplestore_sync.success")
+    else
+      flash[:error] = I18n.t("txt.controllers.triplestore_sync.error")
+    end
+
+    redirect_to :action => "index"
+  end
+
+end

data/app/views/dashboard/_sidebar.html.erb

@@ -11,4 +11,6 @@
   <%= sidebar_header 'Extras' %>
   <%= sidebar_item(:icon => :upload, :text => t('txt.views.dashboard.import'),
     :path => import_url, :active => params[:controller] == 'import', :perms => [:import, Concept::Base]) %>
+  <%= sidebar_item(:icon => :repeat, :text => t('txt.views.triplestore_sync.caption'),
+    :path => triplestore_sync_path, :active => params[:controller] == "triplestore_sync") %>
 <% end %>

data/app/views/triplestore_sync/index.html.erb

@@ -0,0 +1,24 @@
+<%= page_header :title => t("txt.views.triplestore_sync.caption") %>
+<%= render "dashboard/sidebar" %>
+
+<%= form_tag triplestore_sync_path(:format => nil, :lang => nil), :method => :post do %>
+  <%= submit_tag t("txt.views.triplestore_sync.start"), :class => "btn btn-primary" %>
+<% end %>
+
+<% @candidates.each do |records| %>
+  <% klass = records.table.engine %>
+  <h3><%= klass.model_name.human(:count => 2) %></h3>
+  <ul>
+    <% records.each do |record| %>
+    <% # XXX: does not belong here -- also heavily over-engineered
+      doc = IqRdf::Document.new(root_url(:lang => nil))
+      Iqvoc.default_rdf_namespace_helper_methods.each do |meth|
+        doc.namespaces(self.send(meth))
+      end
+      uri = record.build_rdf_subject(doc, controller).full_uri # XXX: we actually just want path, plus HTML format
+    %>
+    <li><%= link_to record.to_s, uri %></li>
+    <% end %>
+  </ul>
+  <%= paginate records %>
+<% end %>

data/config/application.rb

@@ -68,27 +68,5 @@ module Iqvoc
 
     # Version of your assets, change this if you want to expire all your assets
     config.assets.version = '1.0'
-
-    # TODO This must be refactored!
-
-    # The JDBC driver url for the coinnection to the virtuoso triple store.
-    # Login crdentials have to be stored here too. See
-    # http://docs.openlinksw.com/virtuoso/VirtuosoDriverJDBC.html#jdbcurl4mat for
-    # more details.
-    # Example: "jdbc:virtuoso://localhost:1111/UID=dba/PWD=dba"
-    # Use nil to disable virtuoso triple synchronization
-    config.virtuoso_jdbc_driver_url = nil
-
-    # Set up the virtuoso synchronization (which is a triggered pull from the
-    # virtuoso server) to be run in a new thread.
-    # This is needed in environments where the webserver only runs in a single
-    # process/thread (mostly in development environments).
-    # When a synchronizaion would be triggered e.g. from a running
-    # update action in the UPB, the update would trigger virtuoso to do a HTTP GET
-    # back to the UPB to fetch the RDF data. But the only process in the UPB would be
-    # blocked by the update... => Deadlock. You can avoid this by using the threaded
-    # mode.
-    config.virtuoso_sync_threaded = false
-
   end
 end

data/config/locales/de.yml

@@ -92,6 +92,10 @@ de:
          available_languages: verfügbare Sprachen
          languages_pref_labeling: Sprachen für bevorzugte Labels
          languages_further_labelings_Labeling::SKOS::AltLabel: Sprachen für alternative Labels
+         triplestore_url: Triplestore-URL
+         triplestore_username: Benutzername für Triplestore
+         triplestore_password: Passwort für Triplestore
+         triplestore_autosync: automatische Synchronisation mit Triplestore
      alphabetical_concepts:
      untranslated_concepts:
        caption: "Fehlende Übersetzungen"
@@ -242,6 +246,9 @@ de:
        import: Import
      collections:
        new: "Neue Kollektion"
+     triplestore_sync:
+       caption: "Sync mit Triplestore"
+       start: "Synchronisierung starten"
 
    partials:
      note:
@@ -289,7 +296,6 @@ de:
        consistency_check_error: "Instanz ist inkonsistent!"
        to_review_success: "Wurde in den Freigabeprozess aufgenommen"
        to_review_error: "Konnte nicht in den Freigabeprozess aufgenommen werden"
-       virtuoso_exception: "Das Objekt wurde veröffentlicht, konnte aber nicht mit Virtuoso synchronisiert werden. Fehler: "
      labelings:
        save:
          success: "Das Labeling wurde erfolgreich gespeichert."
@@ -318,8 +324,6 @@ de:
        label_error: "Ungültige Label-Zuordnungen:\n%s"
      narrowers:
        error: "Die Assoziation wurde nicht gefunden. Evtl. wurde diese von der anderen Seite bearbeitet. Bitte das Fenster neu laden!"
-     triple_store_syncs:
-       success: "Es wurden %{concept_count} freigegebene Concepts und %{label_count} freigegebene Labels synchronisiert."
      collections:
        save:
          success: "Die Kollektion wurde erfolgreich gespeichert."
@@ -328,6 +332,11 @@ de:
          success: "Die Kollektion wurde erfolgreich gelöscht."
          error: "Die Kollektion konnte nicht gelöscht werden."
        circular_error: "Ungültige Subkollektions-Zuordnungen aufgrund von Zirkelbezügen:\n%s"
+     triplestore_sync:
+       success: "Synchronisierung erfolgreich."
+       error: "Bei der Synchronisierung traten Fehler auf - einige Einträge wurden nicht synchronisiert."
+       config_info: "Synchronisierung mit Triplestore: %{target_info}"
+       config_warning: "Bisher wurde kein Triplestore konfiguriert."
 
    models:
      label:

data/config/locales/en.yml

@@ -92,6 +92,10 @@ en:
          available_languages: available languages
          languages_pref_labeling: languages for preferred labels
          languages_further_labelings_Labeling::SKOS::AltLabel: languages for alternative labels
+         triplestore_url: triplestore URL
+         triplestore_username: username for triplestore
+         triplestore_password: password for triplestore
+         triplestore_autosync: automatic triplestore synchronization
      alphabetical_concepts:
      untranslated_concepts:
        caption: "Missing Translations"
@@ -249,6 +253,9 @@ en:
        import: Import
      collections:
        new: "New collection"
+     triplestore_sync:
+       caption: "Sync with Triplestore"
+       start: "Start synchronization"
 
    partials:
      note:
@@ -296,7 +303,6 @@ en:
        consistency_check_success: "Instance is consistent."
        consistency_check_success_with_warning: "Instance is consistent but has no relations."
        consistency_check_error: "Instance is inconsistent."
-       virtuoso_exception: "The object has been published, but could not by synced with Virtuoso. Error: "
      labelings:
        save:
          success: "The labeling has been saved."
@@ -325,8 +331,6 @@ en:
        label_error: "Invalid label assignments:\n%s"
      narrowers:
        error: "Relation not found. Probably removed from the target side. Please refresh your browser window."
-     triple_store_syncs:
-       success: "%{concept_count} published concepts and %{label_count} published labels have been synced."
      collections:
        save:
          success: "The collection has been successfully created."
@@ -335,6 +339,11 @@ en:
          success: "The collection has been successfully deleted."
          error: "The collection could not be deleted."
        circular_error: "Invalid sub-collection assignments due to circular references:\n%s"
+     triplestore_sync:
+       success: "Synchronization successful."
+       error: "An error occurred during the synchronization process - some entities have not been synchronized."
+       config_info: "Synchronizing with triplestore: %{target_info}"
+       config_warning: "Triplestore has not been configured yet."
 
    models:
      label:

data/config/routes.rb

@@ -28,8 +28,6 @@ Rails.application.routes.draw do
     resources :concepts
     resources :collections
 
-    resources :triple_store_syncs, :only => [:new, :create]
-
     match "/" => "frontpage#index"
 
     match "concepts/:origin/branch(.:format)"      => "concepts/versions#branch",    :as => "concept_versions_branch"
@@ -58,8 +56,11 @@ Rails.application.routes.draw do
     get "help" => "pages#help", :as => "help"
 
     root :to => 'frontpage#index', :format => nil
+
+    get "triplestore_sync" => "triplestore_sync#index"
+    post "triplestore_sync" => "triplestore_sync#sync"
   end
 
   match '/scheme(.:format)' => 'rdf#scheme', :as => 'scheme'
-  match '/:id(.:format)' => 'concepts#show', :as => 'rdf'
+  match '/:id(.:format)' => 'rdf#show', :as => 'rdf'
 end

data/lib/iqvoc.rb

@@ -20,6 +20,7 @@ require 'iqvoc/configuration/core'
 require 'iqvoc/configuration/concept'
 require 'iqvoc/configuration/collection'
 require 'iqvoc/configuration/label'
+require 'iqvoc/configuration/sync'
 
 module Iqvoc
   unless Iqvoc.const_defined?(:Application)
@@ -40,6 +41,10 @@ module Iqvoc
     include Iqvoc::Configuration::Label
   end
 
+  module Sync
+    include Iqvoc::Configuration::Sync
+  end
+
 end
 
 # FIXME: For reasons yet unknown, the load hook is executed twice

data/lib/iqvoc/configuration/sync.rb

@@ -0,0 +1,28 @@
+require 'active_support/concern'
+
+module Iqvoc
+  module Configuration
+    module Sync
+      extend ActiveSupport::Concern
+
+      included do
+        mattr_accessor :syncable_class_names
+        self.syncable_class_names = [Iqvoc::Concept.base_class_name]
+
+        Iqvoc.config.register_settings({
+          "triplestore.url" => "http://example.org:8080",
+          "triplestore.username" => "",
+          "triplestore.password" => "",
+          "triplestore.autosync" => false
+        })
+      end
+
+      module ClassMethods
+        def syncable_classes
+          self.syncable_class_names.map { |name| name.constantize }
+        end
+      end
+
+    end
+  end
+end

data/lib/iqvoc/environments/development.rb

@@ -82,26 +82,6 @@ module Iqvoc::Environments
 
     # The default URI prefix for RDF data. This will be followed by a document
     # specific shnippet like (specimenType) and the id.
-
-    # The JDBC driver URL for the connection to the virtuoso triple store.
-    # Login credentials have to be stored here too. See
-    # http://docs.openlinksw.com/virtuoso/VirtuosoDriverJDBC.html#jdbcurl4mat for
-    # more details.
-    # Example: "jdbc:virtuoso://localhost:1111/UID=dba/PWD=dba"
-    # Use nil to disable virtuoso triple synchronization
-    # Rails.application.config.virtuoso_jdbc_driver_url = "jdbc:virtuoso://virtuoso.dyndns.org:1111/UID=iqvoc/PWD=vocpass!/charset=UTF-8"
-    config.virtuoso_jdbc_driver_url = nil
-
-    # Set up the virtuoso synchronization (which is a triggered pull from the
-    # virtuoso server) to be run in a new thread.
-    # This is needed in environments where the web server only runs in a single
-    # process/thread (mostly in development environments).
-    # When a synchronization would be triggered e.g. from a running
-    # update action in the UPB, the update would trigger virtuoso to do a HTTP GET
-    # back to the UPB to fetch the RDF data. But the only process in the UPB would be
-    # blocked by the update... => Deadlock. You can avoid this by using the threaded
-    # mode.
-    config.virtuoso_sync_threaded = false
   end
 
 end

data/lib/iqvoc/environments/production.rb

@@ -67,26 +67,6 @@ module Iqvoc::Environments
     # in your app. As such, your models will need to explicitly whitelist or blacklist accessible
     # parameters by using an attr_accessible or attr_protected declaration.
     config.active_record.whitelist_attributes = false
-
-    # The JDBC driver URL for the connection to the virtuoso triple store.
-    # Login credentials have to be stored here too. See
-    # http://docs.openlinksw.com/virtuoso/VirtuosoDriverJDBC.html#jdbcurl4mat for
-    # more details.
-    # Example: "jdbc:virtuoso://localhost:1111/UID=dba/PWD=dba"
-    # Use nil to disable virtuoso triple synchronization
-    # Rails.application.config.virtuoso_jdbc_driver_url = "jdbc:virtuoso://virtuoso.dyndns.org:1111/UID=iqvoc/PWD=vocpass!/charset=UTF-8"
-    config.virtuoso_jdbc_driver_url = nil
-
-    # Set up the virtuoso synchronization (which is a triggered pull from the
-    # virtuoso server) to be run in a new thread.
-    # This is needed in environments where the web server only runs in a single
-    # process/thread (mostly in development environments).
-    # When a synchronization would be triggered e.g. from a running
-    # update action in the UPB, the update would trigger virtuoso to do a HTTP GET
-    # back to the UPB to fetch the RDF data. But the only process in the UPB would be
-    # blocked by the update... => Deadlock. You can avoid this by using the threaded
-    # mode.
-    config.virtuoso_sync_threaded = false
   end
 
 end

data/lib/iqvoc/rdf_sync.rb

@@ -0,0 +1,111 @@
+# encoding: UTF-8
+
+require 'iq_triplestorage/virtuoso_adaptor'
+
+class Iqvoc::RDFSync
+  delegate :url_helpers, :to => "Rails.application.routes"
+
+  def initialize(base_url, target_host, *args)
+    @base_url = base_url
+    @target_host = target_host
+    options = args.extract_options!
+    @target_port = options[:port]
+    @username = options[:username]
+    @password = options[:password]
+    @batch_size = options[:batch_size]
+    @view_context = options[:view_context] # XXX: not actually optional
+    raise(ArgumentError, "missing view context") unless @view_context # XXX: smell (see above)
+  end
+
+  def all # TODO: rename
+    timestamp = Time.now
+    errors = false
+
+    gather_candidates do |records|
+       success = sync(records, timestamp)
+       errors = true unless success
+    end
+
+    return !errors
+  end
+
+  def sync(records, timestamp=nil)
+    timestamp ||= Time.now
+
+    success = push(records)
+    if success
+      records.each do |record|
+        record.update_attribute(:rdf_updated_at, timestamp)
+      end
+    end
+
+    return success
+  end
+
+  def push(records)
+    data = records.inject({}) do |memo, record|
+      graph_uri = url_helpers.rdf_url(record.origin,
+          :host => URI.parse(@base_url).host, :format => nil, :lang => nil)
+      memo[graph_uri] = serialize(record)
+      memo
+    end
+
+    adaptor = IqTriplestorage::VirtuosoAdaptor.new(@target_host, @target_port,
+        @username, @password)
+    return adaptor.batch_update(data)
+  end
+
+  def serialize(record)
+    # while this method is really fugly, iQvoc essentially requires us to mock a
+    # view in order to get to the RDF serialization
+
+    doc = IqRdf::Document.new(@base_url)
+    Iqvoc.default_rdf_namespace_helper_methods.each do |meth|
+      doc.namespaces(@view_context.send(meth))
+    end
+
+    rdf_helper = Object.new.extend(RdfHelper)
+    if record.is_a? Iqvoc::Concept.base_class
+      rdf_helper.render_concept(doc, record)
+    else # XXX: must be extensible
+      raise NotImplementedError, "unable to render RDF for #{record.class}"
+    end
+
+    return doc.to_ntriples
+  end
+
+  # yields batches of candidates for synchronization
+  def gather_candidates # TODO: rename
+    Iqvoc::Sync.syncable_classes.each do |klass|
+      self.class.candidates(klass).find_in_batches(:batch_size => @batch_size) do |records|
+        yield records
+      end
+    end
+  end
+
+  # returns one or multiple ActiveRecord::RelationS, depending on whether
+  # `klass` is provided
+  def self.candidates(klass=nil)
+    return klass ? klass.published.unsynced :
+        Iqvoc::Sync.syncable_classes.map { |klass| candidates(klass) }
+  end
+
+end
+
+module Iqvoc::RDFSync::Helper # TODO: rename -- XXX: does not belong here!?
+
+  def triplestore_syncer
+    base_url = root_url(:lang => nil) # XXX: brittle in the face of future changes?
+
+    host = URI.parse(Iqvoc.config["triplestore.url"])
+    port = host.port
+    host.port = 80 # XXX: hack to remove port from serialization
+    host = host.to_s
+
+    return Iqvoc::RDFSync.new(base_url, host, :port => port,
+        :username => Iqvoc.config["triplestore.username"].presence,
+        :password => Iqvoc.config["triplestore.password"].presence,
+        :view_context => view_context) # fugly, but necessary; cf. RDFSync#serialize
+  end
+
+end

data/lib/iqvoc/version.rb

@@ -15,5 +15,5 @@
 # limitations under the License.
 
 module Iqvoc
-  VERSION = "4.0.3"
+  VERSION = "4.0.4"
 end

data/test/test_helper.rb

@@ -21,6 +21,8 @@ Spork.prefork do
   ENV["RAILS_ENV"] = "test"
   require File.expand_path('../../config/environment', __FILE__)
   require 'rails/test_help'
+  require 'webmock'
+  WebMock.allow_net_connect! # required for integration tests
 end
 
 Spork.each_run do

data/test/unit/rdf_sync_test.rb

@@ -0,0 +1,123 @@
+# encoding: UTF-8
+
+# Copyright 2011 innoQ Deutschland GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require File.join(File.expand_path(File.dirname(__FILE__)), '../test_helper')
+
+require 'iqvoc/rdf_sync'
+
+class RDFSyncTest < ActiveSupport::TestCase
+
+  setup do
+    @rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    @skos = "http://www.w3.org/2004/02/skos/core#"
+
+    @base_url = "http://example.com/"
+    @target_host = "http://example.org"
+    @username = "johndoe"
+
+    class FakeViewContext # XXX: does not belong here
+      def iqvoc_default_rdf_namespaces
+        return Iqvoc.rdf_namespaces
+      end
+    end
+    @view_context = FakeViewContext.new
+
+    @sync = Iqvoc::RDFSync.new(@base_url, @target_host, :username => @username,
+        :view_context => @view_context)
+
+    @concepts = 15.times.map do
+      FactoryGirl.create(:concept, :narrower_relations => [])
+    end
+
+    # HTTP request mocking
+    @observers = [] # one per request
+    WebMock.disable_net_connect!
+    WebMock.stub_request(:any, /.*example.org.*/).with do |req|
+      # not using WebMock's custom assertions as those didn't seem to provide
+      # sufficient flexibility
+      fn = @observers.shift
+      raise(TypeError, "missing request observer: #{req.inspect}") unless fn
+      fn.call(req)
+      true
+    end.to_return do |req|
+      { :status => 201 }
+    end
+  end
+
+  teardown do
+    WebMock.reset!
+    WebMock.allow_net_connect!
+    raise(TypeError, "unhandled request observer") unless @observers.length == 0
+  end
+
+  test "serialization" do
+    concept = @concepts[0]
+
+    assert @sync.serialize(concept).include?(<<-EOS.strip)
+<#{@base_url}#{concept.origin}> <#{@rdf}type> <#{@skos}Concept> .
+    EOS
+  end
+
+  test "single-batch syncronization" do
+    concepts = @concepts[0..4]
+
+    @observers << lambda do |req|
+      concepts.each do |concept|
+        uri = @base_url + concept.origin
+        assert req.body.include?("CLEAR GRAPH <#{uri}>")
+      end
+    end
+    @observers << lambda do |req|
+      concepts.each do |concept|
+        path = req.uri.path
+        assert path.start_with?("/DAV/home/#{@username}/")
+        assert_equal 4, path.count("/")
+        uri = @base_url + concept.origin
+        assert req.body.include?("INSERT IN GRAPH <#{uri}> {")
+      end
+    end
+    res = @sync.sync(concepts)
+  end
+
+  test "full synchronization" do
+    concepts = Iqvoc::Concept.base_class.published.unsynced
+
+    assert_not_equal 0, concepts.count
+    assert_not_equal 0, concepts.where(:rdf_updated_at => nil).count
+
+    2.times do # 3 * 2 requests (reset + insert per batch)
+      @observers << lambda { |req| } # no need to check details here
+    end
+    assert @sync.all
+    assert_equal 0, concepts.where(:rdf_updated_at => nil).count
+  end
+
+  test "request batches" do
+    concepts = Iqvoc::Concept.base_class.published.unsynced
+    concept_count = concepts.count
+    batch_count = 3
+
+    sync = Iqvoc::RDFSync.new(@base_url, @target_host, :username => @username,
+        :batch_size => (concept_count / batch_count).ceil,
+        :view_context => @view_context)
+
+    (batch_count * 2).times do # two requests (reset + insert) per batch
+      @observers << lambda { |req| } # no need to check details here
+    end
+    assert sync.all
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: iqvoc
 version: !ruby/object:Gem::Version
-  version: 4.0.3
+  version: 4.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -11,11 +11,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-02 00:00:00.000000000 Z
+date: 2012-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
-  requirement: &70342874186600 !ruby/object:Gem::Requirement
+  requirement: &70162871907980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -23,10 +23,10 @@ dependencies:
         version: 3.2.1
   type: :runtime
   prerelease: false
-  version_requirements: *70342874186600
+  version_requirements: *70162871907980
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70342874186200 !ruby/object:Gem::Requirement
+  requirement: &70162871905160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -34,10 +34,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342874186200
+  version_requirements: *70162871905160
 - !ruby/object:Gem::Dependency
   name: kaminari
-  requirement: &70342874185740 !ruby/object:Gem::Requirement
+  requirement: &70162871915900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -45,10 +45,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342874185740
+  version_requirements: *70162871915900
 - !ruby/object:Gem::Dependency
   name: authlogic
-  requirement: &70342874185320 !ruby/object:Gem::Requirement
+  requirement: &70162871913820 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -56,10 +56,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342874185320
+  version_requirements: *70162871913820
 - !ruby/object:Gem::Dependency
   name: cancan
-  requirement: &70342874184900 !ruby/object:Gem::Requirement
+  requirement: &70162871911780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -67,10 +67,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342874184900
+  version_requirements: *70162871911780
 - !ruby/object:Gem::Dependency
   name: iq_rdf
-  requirement: &70342874184400 !ruby/object:Gem::Requirement
+  requirement: &70162871922380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -78,10 +78,10 @@ dependencies:
         version: 0.1.2
   type: :runtime
   prerelease: false
-  version_requirements: *70342874184400
+  version_requirements: *70162871922380
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70342874183980 !ruby/object:Gem::Requirement
+  requirement: &70162871933860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -89,10 +89,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342874183980
+  version_requirements: *70162871933860
 - !ruby/object:Gem::Dependency
   name: rails_autolink
-  requirement: &70342891454560 !ruby/object:Gem::Requirement
+  requirement: &70162871942060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -100,10 +100,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342891454560
+  version_requirements: *70162871942060
 - !ruby/object:Gem::Dependency
   name: fastercsv
-  requirement: &70342891454100 !ruby/object:Gem::Requirement
+  requirement: &70162871941320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -111,10 +111,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342891454100
+  version_requirements: *70162871941320
 - !ruby/object:Gem::Dependency
   name: simple_form
-  requirement: &70342891453680 !ruby/object:Gem::Requirement
+  requirement: &70162871940440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -122,10 +122,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70342891453680
+  version_requirements: *70162871940440
 - !ruby/object:Gem::Dependency
   name: sass-rails
-  requirement: &70342891453180 !ruby/object:Gem::Requirement
+  requirement: &70162871939660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -133,7 +133,7 @@ dependencies:
         version: 3.2.5
   type: :runtime
   prerelease: false
-  version_requirements: *70342891453180
+  version_requirements: *70162871939660
 description: iQvoc - a SKOS(-XL) vocabulary management system built on the Semantic
   Web
 email:
@@ -196,7 +196,7 @@ files:
 - app/controllers/pages_controller.rb
 - app/controllers/rdf_controller.rb
 - app/controllers/search_results_controller.rb
-- app/controllers/triple_store_syncs_controller.rb
+- app/controllers/triplestore_sync_controller.rb
 - app/controllers/user_sessions_controller.rb
 - app/controllers/users_controller.rb
 - app/helpers/application_helper.rb
@@ -250,7 +250,6 @@ files:
 - app/models/note/skos/example.rb
 - app/models/note/skos/history_note.rb
 - app/models/note/skos/scope_note.rb
-- app/models/rdf_store.rb
 - app/models/search_extension.rb
 - app/models/user.rb
 - app/models/user_session.rb
@@ -328,7 +327,7 @@ files:
 - app/views/search_results/_sidebar.html.erb
 - app/views/search_results/index.html.erb
 - app/views/search_results/index.iqrdf
-- app/views/triple_store_syncs/new.html.erb
+- app/views/triplestore_sync/index.html.erb
 - app/views/user_sessions/_form.html.erb
 - app/views/user_sessions/new.html.erb
 - app/views/users/_form.html.erb
@@ -410,6 +409,7 @@ files:
 - lib/iqvoc/configuration/concept.rb
 - lib/iqvoc/configuration/core.rb
 - lib/iqvoc/configuration/label.rb
+- lib/iqvoc/configuration/sync.rb
 - lib/iqvoc/controller_extensions.rb
 - lib/iqvoc/deep_cloning.rb
 - lib/iqvoc/environments/development.rb
@@ -420,6 +420,7 @@ files:
 - lib/iqvoc/maker.rb
 - lib/iqvoc/origin.rb
 - lib/iqvoc/rankable.rb
+- lib/iqvoc/rdf_sync.rb
 - lib/iqvoc/skos_importer.rb
 - lib/iqvoc/version.rb
 - lib/iqvoc/versioning.rb
@@ -452,6 +453,7 @@ files:
 - test/unit/instance_configuration_test.rb
 - test/unit/note_test.rb
 - test/unit/origin_test.rb
+- test/unit/rdf_sync_test.rb
 - test/unit/skos_import_test.rb
 - vendor/assets/images/ajax-loader.gif
 - vendor/assets/images/bootstrap/glyphicons-halflings-white.png
@@ -548,4 +550,5 @@ test_files:
 - test/unit/instance_configuration_test.rb
 - test/unit/note_test.rb
 - test/unit/origin_test.rb
+- test/unit/rdf_sync_test.rb
 - test/unit/skos_import_test.rb

data/app/controllers/triple_store_syncs_controller.rb

@@ -1,45 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2011 innoQ Deutschland GmbH
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-class TripleStoreSyncsController < ApplicationController
-
-  # TODO: This must definitely be rebuild!
-
-  def new
-    authorize! :use, :dashboard # TODO
-  end
-
-  def create
-    authorize! :use, :dashboard # TODO
-
-    time = Time.now
-
-    rdf_helper = Object.new.extend(RdfHelper)
-
-    Concept::Base.published.unsynced.find_each do |concept|
-      concept.update_attribute(:rdf_updated_at, time) if RdfStore.mass_import(concept.rdf_uri, rdf_helper.render_ttl_for_concept(concept))
-    end
-
-    Label::Base.published.unsynced.find_each do |label|
-      label.update_attribute(:rdf_updated_at, time) if RdfStore.mass_import(label.rdf_uri, rdf_helper.render_ttl_for_label(label))
-    end
-
-    flash.now[:notice] = I18n.t("txt.controllers.triple_store_syncs.success")
-
-    render :action => "new"
-  end
-
-end

data/app/models/rdf_store.rb

@@ -1,96 +0,0 @@
-# encoding: UTF-8
-
-# Copyright 2011 innoQ Deutschland GmbH
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-class RdfStore
-  # TODO This must be modularized and refactored and the thread should be
-  # replaced by a direct rack call or usage of helpers
-  #
-  # * Modulization: The RdfStore should be more abstract. A special adapter
-  #   (e.g. for virtuoso or sesame) should be implemented in a extending
-  #   class.
-  # * No threads: throw out the JRuby stuff and replace it with:
-  #     * Rack calls to the particluar action or
-  #     * With the help of the RdfHelper#render_concept method
-
-  include Java rescue nil
-  include java.lang.Runnable rescue nil
-
-  @@conn = nil
-
-  def initialize(graph_uri, ttl_url, do_delete, ttl_content = nil)
-    if Rails.application.config.virtuoso_jdbc_driver_url
-      @graph_uri = graph_uri
-      @ttl_url = ttl_url
-      @ttl_content = ttl_content
-      @do_delete = do_delete
-
-      Rails.logger.info("** [RdfStore] Beginning virtuoso sync. Insert turtle into graph <#{@graph_uri}> from url: #{@ttl_url}; Clear graph first: #{@do_delete}.")
-
-      unless @@conn
-        # import "virtuoso.jdbc3.Driver"
-        java.lang.Class.forName("virtuoso.jdbc3.Driver", true, JRuby.runtime.jruby_class_loader)
-        @@conn = java.sql.DriverManager.getConnection(Rails.application.config.virtuoso_jdbc_driver_url)
-        Rails.logger.debug("** [RdfStore] JDBC connection is up.")
-      end
-
-      if Rails.application.config.virtuoso_sync_threaded
-        Rails.logger.debug("** [RdfStore] Starting sync thread.")
-        java.lang.Thread.new(self).start
-      else
-        self.run
-      end
-    end
-  end
-
-  def self.insert(graph_uri, ttl_url)
-    return false unless Rails.application.config.virtuoso_jdbc_driver_url
-    RdfStore.new(graph_uri, ttl_url, false)
-    true
-  end
-
-  def self.update(graph_uri, ttl_url)
-    return false unless Rails.application.config.virtuoso_jdbc_driver_url
-    RdfStore.new(graph_uri, ttl_url, true)
-    true
-  end
-
-  def self.delete(graph_uri)
-    return false unless Rails.application.config.virtuoso_jdbc_driver_url
-    RdfStore.new(graph_uri, nil, true)
-    true
-  end
-
-  def self.mass_import(graph_uri, turtle_content)
-    return false unless Rails.application.config.virtuoso_jdbc_driver_url
-    RdfStore.new(graph_uri, nil, true, turtle_content)
-    true
-  end
-
-  def run
-    if (@do_delete)
-      Rails.logger.debug("** [RdfStore] Executing SPARQL DELETE.")
-      @@conn.createStatement().execute("SPARQL CLEAR GRAPH <#{@graph_uri}>")
-    end
-    if (@ttl_url)
-      Rails.logger.debug("** [RdfStore] Executing turtle load from url.")
-      @@conn.createStatement().execute("DB.DBA.TTLP(HTTP_GET('#{@ttl_url}'), '', '#{@graph_uri}')")
-    end
-    if (@ttl_content)
-      @@conn.createStatement().execute("DB.DBA.TTLP('#{@ttl_content.gsub("\\", "\\\\\\").gsub("'", "\\\\'")}', '', '#{@graph_uri}')")
-    end
-    Rails.logger.info("** [RdfStore] Virtuoso sync done.")
-  end
-end

data/app/views/triple_store_syncs/new.html.erb

@@ -1,7 +0,0 @@
-<%= flash[:notice] %>
-
-<h2>Triple Store Sync</h2>
-
-<%= form_tag triple_store_syncs_path, :method => :post do %>
-  <%= submit_tag "Sync" %>
-<% end %>