supplejack_api 1

data/spec/support/api/schemas/metrics/daily_metrics_metadata.json

@@ -0,0 +1,24 @@
+{
+  "description": "Contains metrics about a day, metrics that do not fall into a specific subcategory",
+  "type": "object",
+  "required": [
+    "day",
+    "total_active_records",
+    "total_new_records"
+  ],
+  "properties": {
+    "day": {
+      "description": "Day this metadata refers to",
+      "type": "string",
+      "format": "date"
+    },
+    "total_active_records": {
+      "description": "Number of active records in the system on this date",
+      "type": "number"
+    },
+    "total_new_records": {
+      "description": "Number of records harvested on this date",
+      "type": "number"
+    }
+  }
+}

data/spec/support/api/schemas/metrics/extended_metrics.json

@@ -0,0 +1,26 @@
+{
+  "description": "container object for all extended metrics objects",
+  "type": "object",
+  "required": ["date"],
+  "properties": {
+    "date": {
+      "description": "dayDate this metric represents",
+      "type": "string",
+      "format": "date"          
+    },
+    "record": {
+      "description": "Record metadata metric",
+      "type": "array",
+      "items": {
+        "$ref": "record_metadata.json"
+      }
+    },
+    "view": {
+      "description": "Record view metadata metric",
+      "type": "array",
+      "items": {
+        "$ref": "view_metadata.json"
+      }
+    }
+  }
+}

data/spec/support/api/schemas/metrics/extended_response.json

@@ -0,0 +1,6 @@
+{
+  "type": "array",
+  "items": {
+    "$ref": "extended_metrics.json" 
+  }
+}

data/spec/support/api/schemas/metrics/facets_response.json

@@ -0,0 +1,6 @@
+{
+  "type": "array",
+  "items": {
+    "type": "string"
+  }
+}

data/spec/support/api/schemas/metrics/record_metadata.json

@@ -0,0 +1,33 @@
+{
+  "description": "Contains metadata about the records grouped by a facet on a specific day",
+  "type": "object",
+  "required": [
+    "id",
+    "total_active_records",
+    "total_new_records",
+    "category_counts",
+    "copyright_counts"
+  ],
+  "properties": {
+    "id": {
+      "description": "Facet name",
+      "type": "string"
+    },
+    "total_active_records": {
+      "description": "Number of active records for this facet",
+      "type": "number"
+    },
+    "total_new_records": {
+      "description": "Number of new records for this facet",
+      "type": "number"
+    },
+    "category_counts": {
+      "description": "Mapping of categories to counts, object shape is dynamic so it has no schema",
+      "type": "object"
+    },
+    "copyright_counts": {
+      "description": "Mapping of copyrights to counts, object shape is dynamic so it has no schema",
+      "type": "object"
+    }
+  }
+}

data/spec/support/api/schemas/metrics/top_level_response.json

@@ -0,0 +1,6 @@
+{
+  "type": "array",
+  "items": {
+    "$ref": "daily_metrics_metadata.json" 
+  }
+}

data/spec/support/api/schemas/metrics/view_metadata.json

@@ -0,0 +1,33 @@
+{
+  "description": "Contains metadata about how records in a display_collection were viewed on a specific day",
+  "type": "object",
+  "required": [
+    "id",
+    "searches",
+    "record_page_views",
+    "user_set_views",
+    "total_views"
+  ],
+  "properties": {
+    "id": {
+      "description": "Facet name",
+      "type": "string"
+    },
+    "searches": {
+      "description": "Number of times a record from this collection was part of a search result",
+      "type": "number"
+    },
+    "record_page_views": {
+      "description": "Number of times a record page for a record from this collection was viewed",
+      "type": "number"
+    },
+    "user_set_views": {
+      "description": "Number of times a record from this collection was viewed in a user set",
+      "type": "number"
+    },
+    "total_views": {
+      "description": "Cumulative count of all record views for this collection",
+      "type": "number"
+    }
+  }
+}

data/spec/support/api_schema_matcher.rb

@@ -0,0 +1,14 @@
+require 'json-schema'
+
+RSpec::Matchers.define :match_response_schema do |schema|
+  match do |json|
+    schema_directory = "#{Dir.pwd}/spec/support/api/schemas"
+    schema_path = "#{schema_directory}/#{schema}.json"
+
+    errors = JSON::Validator.fully_validate(schema_path, json)
+
+    fail Exception.new(errors.join(' ----- ')) unless errors.empty?
+
+    true
+  end
+end

data/spec/support/devise.rb

@@ -0,0 +1,10 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+RSpec.configure do |config|
+  config.include Devise::TestHelpers, type: :controller
+end

data/spec/workers/supplejack_api/daily_metrics_worker_spec.rb

@@ -0,0 +1,140 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+require "spec_helper"
+
+module SupplejackApi
+  describe DailyMetricsWorker, search: true, slow: true do
+
+    before :each do
+      Sunspot.session = Sunspot.session.original_session
+    end
+
+    after :each do
+      Sunspot.remove_all
+    end
+
+    describe "#call" do
+      def build_records
+        10.times do
+          create(:record_with_fragment, display_collection: 'pc1', copyright: ['0'],      category: ['0'], created_at: Date.current.midday)
+          create(:record_with_fragment, display_collection: 'pc2', copyright: ['1'],      category: ['1'], created_at: Date.current.midday)
+          create(:record_with_fragment, display_collection: 'pc1', copyright: ['0', '1'], category: ['0', '1'], created_at: Date.yesterday.midday)
+        end
+
+        Sunspot.commit
+      end
+
+      def facet_metrics_query(result, expected_results, &accessor_block)
+        result.each_with_index do |facet, index|
+          expect(accessor_block.call(facet, index)).to eq(expected_results[index])
+        end
+      end
+
+      it "handles records with missing categories/copyrights" do
+        10.times do |n|
+          c = n % 2 == 0 ? nil : ['0']
+          create(:record_with_fragment, created_at: Date.current.midday, copyright: c)
+        end
+        Sunspot.commit
+
+        DailyMetricsWorker.new.call
+        facet = SupplejackApi::FacetedMetrics.where(name: 'all').first
+
+        expect(facet.total_active_records).to eq(10)
+      end
+
+      it "handles copyrights with periods in the name" do
+        create(:record_with_fragment, copyright: ['1.0'], display_collection: 'test')
+        Sunspot.commit
+
+        DailyMetricsWorker.new.call
+        expect(FacetedMetrics.created_on(Date.current).first.copyright_counts.first.first).to eq("1.0")
+      end
+
+      it "correctly paginates the facet list when there are more than 150 facets" do
+        200.times do |n|
+          create(:record_with_fragment, display_collection: n.to_s)
+        end
+        Sunspot.commit
+
+        DailyMetricsWorker.new.call
+
+        expect(FacetedMetrics.count).to eq(201)
+      end
+
+      context "faceted metrics" do
+        before do
+          create(:record_with_fragment, status: "deleted") # inactive, should never show in counts
+          build_records
+
+          DailyMetricsWorker.new.call
+        end
+        let(:faceted_metrics)  {SupplejackApi::FacetedMetrics.all.where(:name.ne => 'all').to_a}
+
+        it "has metrics for each facet" do
+          expect(faceted_metrics.length).to eq(2)
+        end
+
+        it "has active record counts for each facet" do
+          facet_metrics_query(faceted_metrics, [20, 10]) {|facet| facet.total_active_records}
+        end
+
+        it "has new record counts for each display_collection" do
+          facet_metrics_query(faceted_metrics, [10, 10]) {|facet| facet.total_new_records}
+        end
+
+        it "has a count of records per category in each display_collection" do
+          facet_metrics_query(faceted_metrics, [20, 10]) {|facet, index| facet.category_counts[index.to_s]}
+        end
+
+        it "has a count of records per usage type in each display_collection" do
+          facet_metrics_query(faceted_metrics, [20, 10]) {|facet, index| facet.copyright_counts[index.to_s]}
+        end
+
+        it "creates an All facet that contains the summed metrics of for all the individual facets" do
+          all_metric = SupplejackApi::FacetedMetrics.where(name: 'all').first
+          
+          expect(all_metric.total_active_records).to eq(30)
+          expect(all_metric.total_new_records).to eq(20)
+          expect(all_metric.copyright_counts['0']).to eq(20)
+        end
+      end
+
+      context "daily metrics" do
+        let(:metric) do
+          # Lazy generate metrics
+          DailyMetricsWorker.new.call
+
+          SupplejackApi::DailyMetrics.first
+        end
+
+        it 'creates a DailyMetrics model' do
+          expect(metric).to be_present
+        end
+
+        it 'sets the date field to the current day' do
+          expect(metric.date).to eq(Date.current)
+        end
+
+        context "set metrics" do
+          it 'counts the number of sets in the system' do
+            create(:user_set)
+
+            expect(metric.total_public_sets).to eq(1)
+          end
+
+          it 'does not count non-public sets' do
+            create(:user_set, privacy: 'private')
+
+            expect(metric.total_public_sets).to eq(0)
+          end
+        end
+      end
+    end
+  end
+end

data/spec/workers/supplejack_api/flush_old_records_worker_spec.rb

@@ -0,0 +1,20 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+require "spec_helper"
+
+module SupplejackApi
+  describe FlushOldRecordsWorker do
+
+    describe "perform" do
+      it "should call 'flush_old_records' on Record with the given source_id & job_id" do
+        expect(Record).to receive(:flush_old_records).with("abc-123", "abc123")
+        FlushOldRecordsWorker.perform("abc-123", "abc123")
+      end
+    end
+  end
+end

data/spec/workers/supplejack_api/index_worker_spec.rb

@@ -0,0 +1,52 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+require "spec_helper"
+
+module SupplejackApi
+  describe IndexWorker do
+  
+    describe "#find_all" do
+      let(:record1) { FactoryGirl.create(:record, record_id: 12345) }
+      let(:record2) { FactoryGirl.create(:record, record_id: 67890) }
+  
+      it "finds all records by id" do
+        expect(IndexWorker.find_all("Record", [record1.id, record2.id])).to eq [record1, record2]
+      end
+  
+      it "returns records even when some where not found" do
+        expect(IndexWorker.find_all("Record", ["504d333aa9b6ad1860000056", record1.id])).to eq [record1]
+      end
+  
+      it "handles individual records" do
+        expect(IndexWorker.find_all("Record", record1.id)).to eq [record1]
+      end
+    end
+  
+    describe "perform" do
+      let(:record) { double(:record).as_null_object }
+  
+      it "indexes all given record id's" do
+        expect(IndexWorker).to receive(:find_all).with("Record", ["123"]) { [record] }
+        expect(IndexWorker).to receive(:index).with([record])
+        IndexWorker.perform(:index, {class: "Record", id: ["123"]})
+      end
+  
+      it "removes all given record id's" do
+        expect(IndexWorker).to receive(:find_all).with("Record", ["123"]) { [record] }
+        expect(IndexWorker).to receive(:remove).with([record])
+        IndexWorker.perform(:remove, {class: "Record", id: ["123"]})
+      end
+  
+      it "rescues from a RSolr::Error::Http errors when commiting SOLR" do
+       allow(Sunspot).to receive(:commit).and_raise(RSolr::Error::Http.new({}, {}))
+        IndexWorker.perform(:commit)
+      end
+    end
+  end
+
+end

data/spec/workers/supplejack_api/source_index_worker_spec.rb

@@ -0,0 +1,61 @@
+# The majority of the Supplejack API code is Crown copyright (C) 2014, New Zealand Government, 
+# and is licensed under the GNU General Public License, version 3.
+# One component is a third party component. See https://github.com/DigitalNZ/supplejack_api for details. 
+# 
+# Supplejack was created by DigitalNZ at the National Library of NZ and 
+# the Department of Internal Affairs. http://digitalnz.org/supplejack
+
+require "spec_helper"
+
+module SupplejackApi
+  describe IndexSourceWorker do
+    describe ".perform" do
+      let(:records) { [FactoryGirl.create(:record_with_fragment), FactoryGirl.create(:record_with_fragment)] }
+
+      before do
+        records.each do |r|
+          r.primary_fragment.source_id = 'tapuhi'
+          r.save
+        end
+
+        allow(Record).to receive(:where).with(hash_including(:'fragments.source_id' => 'tapuhi')).and_call_original
+      end
+
+      it "finds all active records and indexes them" do
+        expect(Record).to receive(:where).with(:'fragments.source_id' => 'tapuhi').and_call_original
+        expect(Sunspot).to receive(:index).with(records)
+        IndexSourceWorker.perform('tapuhi')
+      end
+      
+
+      it "finds all deleted records and removes them from solr" do
+        records.each { |r| r.update_attribute(:status, 'deleted') }
+
+        expect(Record).to receive(:where).with(:'fragments.source_id' => 'tapuhi').and_call_original
+        expect(Sunspot).to receive(:remove).with(records)
+        IndexSourceWorker.perform('tapuhi')
+      end
+
+      it "finds records updated more recently than the date given" do
+        date = Time.now
+        allow(Time).to receive(:parse) { date }
+        expect(Record).to receive(:where).with(:'fragments.source_id' => 'tapuhi', :updated_at.gt => date)
+        IndexSourceWorker.perform('tapuhi', date.to_s)
+      end
+    end
+
+    describe ".in_chunks" do
+      let(:records) { [double(:record), double(:record)] }
+
+      it "gets a chunk of records and yields to block" do
+        mock_cursor = double(:cursor)
+        mock_cursor_2 = double(:cursor)
+        expect(mock_cursor).to receive(:limit).with(10000) {mock_cursor_2}
+        expect(mock_cursor).to receive(:count) {2}
+        expect(mock_cursor_2).to receive(:skip).with(0) { records }
+
+        expect{|b| IndexSourceWorker.in_chunks(mock_cursor, &b) }.to yield_with_args(records)
+      end
+    end
+  end
+end