supplejack_api 1

data/spec/dummy/Rakefile

@@ -0,0 +1,7 @@
+#!/usr/bin/env rake
+# Add your own tasks in files placed in lib/tasks ending in .rake,
+# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
+
+require File.expand_path('../config/application', __FILE__)
+
+Dummy::Application.load_tasks

data/spec/dummy/app/assets/javascripts/application.js

@@ -0,0 +1,23 @@
+/*
+ *The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+ * and is licensed under the GNU General Public License, version 3.
+ * One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+ * Supplejack was created by DigitalNZ at the National Library of NZ and 
+ * the Department of Internal Affairs. http://digitalnz.org/supplejack
+ */
+
+// This is a manifest file that'll be compiled into application.js, which will include all the files
+// listed below.
+//
+// Any JavaScript/Coffee file within this directory, lib/assets/javascripts, vendor/assets/javascripts,
+// or vendor/assets/javascripts of plugins, if any, can be referenced here using a relative path.
+//
+// It's not advisable to add code directly here, but if you do, it'll appear at the bottom of the
+// the compiled file.
+//
+// WARNING: THE FIRST BLANK LINE MARKS THE END OF WHAT'S TO BE PROCESSED, ANY BLANK LINE SHOULD
+// GO AFTER THE REQUIRES BELOW.
+//
+//= require jquery
+//= require jquery_ujs
+//= require_tree .

data/spec/dummy/app/assets/stylesheets/application.css

@@ -0,0 +1,22 @@
+/*
+ *The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+ * and is licensed under the GNU General Public License, version 3.
+ * One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+ * Supplejack was created by DigitalNZ at the National Library of NZ and 
+ * the Department of Internal Affairs. http://digitalnz.org/supplejack
+ */
+
+/*
+ * This is a manifest file that'll be compiled into application.css, which will include all the files
+ * listed below.
+ *
+ * Any CSS and SCSS file within this directory, lib/assets/stylesheets, vendor/assets/stylesheets,
+ * or vendor/assets/stylesheets of plugins, if any, can be referenced here using a relative path.
+ *
+ * You're free to add application-wide styles to this file and they'll appear at the top of the
+ * compiled file, but it's generally better to create a new file per style scope.
+ *
+ *= require_self
+ *= require_tree .
+ *= require supplejack_api/application
+ */

data/spec/dummy/app/controllers/application_controller.rb

@@ -0,0 +1,10 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+class ApplicationController < ActionController::Base
+  protect_from_forgery
+end

data/spec/dummy/app/helpers/application_helper.rb

@@ -0,0 +1,9 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module ApplicationHelper
+end

data/spec/dummy/app/supplejack_api/concept_schema.rb

@@ -0,0 +1,83 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+class ConceptSchema
+  include SupplejackApi::SchemaDefinition
+
+  # Namespaces
+  namespace :dcterms,     url: 'http://purl.org/dc/terms/'
+  namespace :edm,         url: 'http://www.europeana.eu/schemas/edm/'
+  namespace :foaf,        url: 'http://xmlns.com/foaf/0.1/'
+  namespace :owl,         url: 'http://www.w3.org/2002/07/owl#'
+  namespace :rdaGr2,      url: 'http://rdvocab.info/ElementsGr2/'
+  namespace :rdf,         url: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'
+  namespace :rdfs,        url: 'http://www.w3.org/2000/01/rdf-schema#'
+  namespace :skos,        url: 'http://www.w3.org/2004/02/skos/core#'
+  namespace :xsd,         url: 'http://www.w3.org/2001/XMLSchema#'
+  namespace :dc,          url: 'http://purl.org/dc/elements/1.1/'
+
+  # Fields (SourceAuthority fields)
+  string      :name
+  string      :prefLabel
+  string      :altLabel,                  multi_value: true
+  datetime    :dateOfBirth
+  datetime    :dateOfDeath
+  string      :biographicalInformation
+  string      :sameAs,                    multi_value: true
+  string      :givenName
+  string      :familyName
+  integer     :birthYear
+  integer     :deathYear
+
+  group :source_authorities
+  group :reverse
+
+  group :default do
+    fields [
+      :name,
+      :title
+    ]
+  end
+
+  group :verbose do
+    fields [
+      :name,
+      :prefLabel,
+      :altLabel,
+      :dateOfBirth,
+      :dateOfDeath,
+      :biographicalInformation,
+      :sameAs,
+      :title
+    ]
+  end
+
+  # Concept
+  model_field :name, field_options: { type: String }, search_as: [:fulltext], search_boost: 6, namespace: :foaf
+  model_field :prefLabel, field_options: { type: String }, namespace: :skos
+  model_field :altLabel, field_options: { type: Array }, search_as: [:fulltext], search_boost: 2, namespace: :skos
+  model_field :dateOfBirth, field_options: { type: Date }, namespace: :rdaGr2
+  model_field :dateOfDeath, field_options: { type: Date }, namespace: :rdaGr2
+  model_field :biographicalInformation, field_options: { type: String }, search_as: [:fulltext], search_boost: 1,  namespace: :rdaGr2
+  model_field :sameAs, field_options: { type: Array }, namespace: :owl
+
+  # Use store: false to display the fields in the /schema
+  model_field :title, store: false, namespace: :dc
+  model_field :date, store: false, namespace: :dc
+  model_field :description, store: false, namespace: :dc
+  model_field :agents, store: false, namespace: :edm
+  model_field :source_authority do
+    search_as [:filter]
+    store false
+    type :string
+    multi_value true
+    namespace :foaf
+    search_value do |concept|
+      concept.source_authorities.map(&:internal_identifier)
+    end
+  end
+end

data/spec/dummy/app/supplejack_api/record_schema.rb

@@ -0,0 +1,64 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+class RecordSchema
+  include SupplejackApi::SupplejackSchema
+
+  # Namespaces
+  namespace :dc, url: 'http://purl.org/dc/elements/1.1/'
+
+  # Fields
+  string    :record_id,    store: false
+  string    :name,         search_boost: 10,      search_as: [:filter, :fulltext], namespace: :dc
+  string    :address,      search_boost: 2,       search_as: [:filter, :fulltext]
+  string    :email,        multi_value: true,     search_as: [:filter]
+  string    :children,     multi_value: true
+  string    :contact,      multi_value: true
+  integer   :age
+  datetime  :birth_date
+  boolean   :nz_citizen,                          search_as: [:filter]
+  string    :display_collection,                                search_as: [:filter, :fulltext],  namespace: :sj
+
+  # facets
+  string    :category,     multi_value: true,     search_as: [:filter]
+  string    :copyright,    multi_value: true,     search_as: [:filter]
+
+  # Groups
+  group :default do
+    fields [
+      :name,
+      :address
+    ]
+  end
+
+  group :all do
+    includes [:default]
+    fields [
+      :email,
+      :children,
+      :nz_citizen,
+      :birth_date,
+      :age
+    ]
+  end
+
+  group :core do
+    fields [:record_id]
+  end
+
+  group :sets do
+    fields [
+      :name,
+      :address
+    ]
+  end
+
+   # Roles
+  role :developer, default: true
+  role :admin, admin: true
+
+end

data/spec/dummy/app/supplejack_api/record_schema.txt

@@ -0,0 +1,66 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government,
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details.
+#
+# Supplejack was created by DigitalNZ at the National Library of NZ and
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+class RecordSchema
+  include SupplejackApi::SupplejackSchema
+
+  # Namespaces
+  namespace :dc,        url: 'http://purl.org/dc/elements/1.1/'
+  namespace :sj,        url: ''
+
+  # Fields
+  string    :record_id,                   store: false,                                           namespace: :sj
+  string    :concept_ids,                                                                         namespace: :sj
+  string    :title,                       search_boost: 10,     search_as: [:filter, :fulltext],  namespace: :dc
+  string    :description,                 search_boost: 2,      search_as: [:filter, :fulltext],  namespace: :dc
+        
+  string    :display_content_partner,                           search_as: [:filter, :fulltext],  namespace: :sj
+  string    :display_collection,                                search_as: [:filter, :fulltext],  namespace: :sj
+  string    :source_url,                                                                          namespace: :sj
+  string    :thumbnail_url
+  string    :subject,                     multi_value: true,                                      namespace: :dc
+  string    :display_date,                                                                        namespace: :sj
+  string    :creator,                     multi_value: true,                                      namespace: :dc
+  string    :category,                    multi_value: true,    search_as: [:filter],             namespace: :sj
+
+  # Groups
+  group :default do
+    fields [
+      :title,
+      :description
+    ]
+  end
+
+  group :all do
+    includes [:default]
+    fields [
+      :record_id,
+      :concept_ids,
+      :subject,
+      :display_content_partner,
+      :display_collection,
+      :source_url,
+      :thumbnail_url,
+      :display_date,
+      :creator,
+      :category
+    ]
+  end
+
+  group :core do
+    fields [:record_id]
+  end
+
+   # Roles
+  role :developer do
+    default true
+  end
+  role :admin
+
+  mongo_index :source_url,         fields: [{source_url: 1}]
+
+end

data/spec/dummy/app/views/layouts/application.html.erb

@@ -0,0 +1,23 @@
+<%#   
+The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+and is licensed under the GNU General Public License, version 3.
+One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+
+Supplejack was created by DigitalNZ at the National Library of NZ and 
+the Department of Internal Affairs. http://digitalnz.org/supplejack 
+%>
+
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Dummy</title>
+  <%= stylesheet_link_tag    "application", :media => "all" %>
+  <%= javascript_include_tag "application" %>
+  <%= csrf_meta_tags %>
+</head>
+<body>
+
+<%= yield %>
+
+</body>
+</html>

data/spec/dummy/app/workers/supplejack_api/clear_index_buffer.rb

@@ -0,0 +1,32 @@
+# This is a worker class which gets executed every 5 minutes that pulls any
+# records_ids stored in Redis that should be indexed or removed from the 
+# index.
+#
+# Resque Scheduler is used to execute this task.
+
+module SupplejackApi
+  class ClearIndexBuffer
+    @queue = :solr_index
+
+    def self.perform
+      Rails.logger.level = 1 unless Rails.env.development?
+
+      session = Sunspot.session
+      Sunspot.session = Sunspot::Rails.build_session
+
+      buffer = SupplejackApi::IndexBuffer.new
+
+      index_record_ids = buffer.records_to_index
+      if index_record_ids.any?
+        Sunspot.index(index_record_ids)
+      end
+
+      remove_record_ids = buffer.records_to_remove
+      if remove_record_ids.any?
+        Sunspot.remove(remove_record_ids)
+      end
+
+      Sunspot.commit
+    end
+  end
+end

data/spec/dummy/app/workers/supplejack_api/flush_old_records_worker.rb

@@ -0,0 +1,17 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+ module SupplejackApi
+  class FlushOldRecordsWorker
+
+    @queue = :flush_records
+
+    def self.perform(source_id, job_id)
+      SupplejackApi::Record.flush_old_records(source_id, job_id)
+    end
+  end
+end

data/spec/dummy/app/workers/supplejack_api/index_buffer.rb

@@ -0,0 +1,49 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+# Handles the logic for storing and retreiving record_ids from Redis
+# which should be indexed or removed from Solr.
+
+module SupplejackApi
+  class IndexBuffer
+
+    def initialize
+      @redis = Resque.redis
+    end
+
+    def pop_record_ids(method=:index, num=1000)
+      # get all the entries from the list. Need to check what happens if the list doesn't exist
+      num ||= 100000
+      ids = []
+      while ids.count < num and id = @redis.lpop("#{method}_buffer_record_ids")
+        ids << id
+      end
+      ids
+    end
+
+    def records_to_index
+      @records_to_index ||= SupplejackApi::Record.where(:id.in => self.pop_record_ids(:index)).to_a
+      @records_to_index.keep_if {|r| r.should_index? }
+      @records_to_index
+    end
+
+    def records_to_remove
+      @records_to_remove ||= SupplejackApi::Record.where(:id.in => self.pop_record_ids(:remove)).to_a
+      @records_to_remove.delete_if {|r| r.should_index?}
+      @records_to_remove
+    end
+
+    [:index, :remove].each do |method|
+      define_method("#{method}_record_ids=") do |ids|
+        ids.each do |id|
+          # push each id
+          @redis.rpush("#{method}_buffer_record_ids", id) 
+        end
+      end
+    end
+  end
+end

data/spec/dummy/app/workers/supplejack_api/index_source_worker.rb

@@ -0,0 +1,39 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+module SupplejackApi
+  class IndexSourceWorker
+    @queue = :index_source
+
+    def self.perform(source_id, date=nil)
+      if date.present?
+        cursor = Record.where(:'fragments.source_id' => source_id, :updated_at.gt => Time.parse(date))
+      else
+        cursor = Record.where(:'fragments.source_id' => source_id)
+      end
+
+      in_chunks(cursor.where(status: "active")) do |records|
+        Sunspot.index(records)
+      end
+
+      in_chunks(cursor.where(status: "deleted")) do |records|
+        Sunspot.remove(records)
+      end
+    end
+
+    def self.in_chunks(cursor, &block)
+      total = cursor.count
+      start = 0
+      chunk_size = 10000
+      while start < total
+        records = cursor.limit(chunk_size).skip(start)
+        yield records
+        start += chunk_size
+      end
+    end
+  end
+end