krikri 0.3.3 → 0.4.0

data/spec/lib/krikri/enrichments/dedup_values_spec.rb~

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::DedupValues do
+  it_behaves_like 'a field enrichment'
+
+  it do
+    binding.pry
+    require 'pry'
+  end
+end

data/spec/lib/krikri/enrichments/genre_filter_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::GenreFilter do
+  it_behaves_like 'a field enrichment'
+
+  values = [{ :string => 'removes non-genre terms',
+              :start => 'Not A Term',
+              :end => nil
+            },
+            { :string => 'retains genre terms',
+              :start => 'Book',
+              :end => 'Book'
+            },
+            { :string => 'normalizes string form',
+              :start => "\n bO  O-k!   \t",
+              :end => 'Book'
+            }]
+
+  it_behaves_like 'a string enrichment', values
+  include_examples 'deletes non-strings'
+end

data/spec/lib/krikri/enrichments/remove_empty_fields_spec.rb

@@ -21,4 +21,5 @@ describe Krikri::Enrichments::RemoveEmptyFields do
             }]
 
   it_behaves_like 'a string enrichment', values
+  include_examples 'skips non-strings'
 end

data/spec/lib/krikri/enrichments/strip_ending_punctuation_spec.rb

@@ -0,0 +1,28 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::StripEndingPunctuation do
+  it_behaves_like 'a field enrichment'
+
+  values = [{ :string => 'removes punctuation from end of field',
+              :start => "moomin!...!;,.",
+              :end => "moomin"
+            },
+            { :string => 'keeps initials',
+              :start => "Smith, Smithy Q.",
+              :end => "Smith, Smithy Q."
+            },
+            { :string => 'keeps closing parentheses',
+              :start => "(Smith)",
+              :end => "(Smith)"
+            },
+            { :string => 'keeps two letter abbreviations',
+              :start => "66 cm.",
+              :end => "66 cm."
+            },
+            { :string => 'leaves other fields unaltered',
+              :start => "moominpapa;:;:; moominmama",
+              :end => "moominpapa;:;:; moominmama"
+            }]
+
+  it_behaves_like 'a string enrichment', values
+end

data/spec/lib/krikri/enrichments/strip_ending_punctuation_spec.rb~

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::StripLeadingPunctuation do
+  it_behaves_like 'a field enrichment'
+
+  values = [{ :string => 'removes punctuation from beginning of field',
+              :start => "([!.;:\tmoominpapa;:;:; moominmama! ...\n",
+              :end => "\tmoominpapa;:;:; moominmama! ...\n"
+            },
+            { :string => 'leaves other fields unaltered',
+              :start => "'moominpapa;:;:; moominmama! ...\n'",
+              :end => "'moominpapa;:;:; moominmama! ...\n'"
+            }]
+
+  it_behaves_like 'a string enrichment', values
+end

data/spec/lib/krikri/enrichments/strip_html_spec.rb

@@ -18,4 +18,5 @@ describe Krikri::Enrichments::StripHtml do
             }]
 
   it_behaves_like 'a string enrichment', values
+  include_examples 'skips non-strings'
 end

data/spec/lib/krikri/enrichments/strip_leading_colons_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::StripLeadingColons do
+  it_behaves_like 'a field enrichment'
+
+  values = [{ :string => 'removes (semi)colons from beginning of field',
+              :start => ";:\tmoominpa()pa;;;",
+              :end => "\tmoominpa()pa;;;"
+            },
+            { :string => 'leaves other fields unaltered',
+              :start => ";:\tmoominpapa;:;:; moominmama! ...\n",
+              :end => "\tmoominpapa;:;:; moominmama! ...\n"
+            }]
+
+  it_behaves_like 'a string enrichment', values
+end

data/spec/lib/krikri/enrichments/strip_leading_colons_spec.rb~

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::StripLeadingColons do
+  it_behaves_like 'a field enrichment'
+
+  values = [{ :string => 'removes (semi)colons from beginning of field',
+              :start => "\tmoominpapa;... !@#$ moominmama  ",
+              :end => "\tmoominpapa  moominmama  "
+            },
+            { :string => 'leaves other fields unaltered',
+              :start => "\tmoominpapa;:;:; moominmama",
+              :end => "\tmoominpapa;:;:; moominmama"
+            }]
+
+  it_behaves_like 'a string enrichment', values
+end

data/spec/lib/krikri/enrichments/strip_leading_punctuation_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::StripLeadingPunctuation do
+  it_behaves_like 'a field enrichment'
+
+  values = [{ :string => 'removes punctuation from beginning of field',
+              :start => "([!.;:\tmoominpapa;:;:; moominmama! ...\n",
+              :end => "\tmoominpapa;:;:; moominmama! ...\n"
+            },
+            { :string => 'leaves other fields unaltered',
+              :start => "'moominpapa;:;:; moominmama! ...\n'",
+              :end => "'moominpapa;:;:; moominmama! ...\n'"
+            }]
+
+  it_behaves_like 'a string enrichment', values
+end

data/spec/lib/krikri/enrichments/strip_leading_punctuation_spec.rb~

@@ -0,0 +1,16 @@
+require 'spec_helper'
+
+describe Krikri::Enrichments::StripLeadingPunctuation do
+  it_behaves_like 'a field enrichment'
+
+  values = [{ :string => 'removes punctuation from beginning of field',
+              :start => ";:\tmoominpa()pa;;;",
+              :end => "\tmoominpa()pa;;;"
+            },
+            { :string => 'leaves other fields unaltered',
+              :start => ";:\tmoominpapa;:;:; moominmama! ...\n",
+              :end => "\tmoominpapa;:;:; moominmama! ...\n"
+            }]
+
+  it_behaves_like 'a string enrichment', values
+end

data/spec/lib/krikri/enrichments/strip_punctuation_spec.rb

@@ -12,4 +12,5 @@ describe Krikri::Enrichments::StripPunctuation do
               :end => "\tmoominpapa moominmama"
             }]
   it_behaves_like 'a string enrichment', values
+  include_examples 'skips non-strings'
 end

data/spec/lib/krikri/enrichments/strip_whitespace_spec.rb

@@ -13,4 +13,5 @@ describe Krikri::Enrichments::StripWhitespace do
             }]
 
   it_behaves_like 'a string enrichment', values
+  include_examples 'skips non-strings'
 end

data/spec/lib/krikri/enrichments/timespan_split_spec.rb

@@ -87,6 +87,21 @@ describe Krikri::Enrichments::TimespanSplit do
           end
         end
       end
+
+      context 'with precision' do
+        let(:label) { '2019' }
+
+        it 'sets begin to correct precision' do
+          expect(subject.enrich_value(timespan).begin.first.to_s)
+            .to eql Date.parse('2019-01-01').to_s
+        end
+
+        it 'sets end to correct precision' do
+          expect(subject.enrich_value(timespan).end.first.to_s)
+            .to eql Date.parse('2019-12-31').to_s
+
+        end
+      end
     end
   end
 end

data/spec/lib/krikri/harvesters/api_harvester_spec.rb

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Krikri::Harvesters::ApiHarvester do
+  let(:args) { { uri: 'http://example.org/endpoint', api: query_opts } }
+  let(:query_opts) { {} }
+  subject { described_class.new(args) }
+
+  describe '#new' do
+    let(:query_opts) { { 'params' => { 'q' => :abc } } }
+
+    it 'accepts api parameters' do
+      expect(described_class.new(args)).to have_attributes(:opts => query_opts)
+    end
+  end
+
+  context 'with responses' do
+    let(:query_opts) { { 'params' => { 'q' => 'tags_ssim:dpla' } } }
+    let(:response_string) do
+      <<EOM
+{ "response": {
+    "numFound": 4,
+    "start": 0,
+    "docs": [{"record_id": "123"}]
+  }
+}
+EOM
+    end
+
+    let(:second_response) do
+      <<EOM
+{ "response": {
+    "numFound": 4,
+    "start": 1,
+    "docs": [{"record_id": "345"},
+             {"record_id": "678"},
+             {"record_id": "910"}]
+  }
+}
+EOM
+    end
+
+    let(:empty_response) do
+      <<EOM
+{ "response": {
+    "numFound": 4,
+    "start": 1,
+    "docs": []
+  }
+}
+EOM
+    end
+
+    let(:single_record) do
+      <<EOM
+{ "response": {
+    "numFound": 1,
+    "start": 0,
+    "docs": [{"record_id": "123"}]
+  }
+}
+EOM
+    end
+
+    before do
+      stub_request(:get, "http://example.org/endpoint?q=tags_ssim:dpla")
+        .to_return(:status => 200, :body => response_string, :headers => {})
+      stub_request(:get, "http://example.org/endpoint?q=tags_ssim:dpla&start=1")
+        .to_return(:status => 200, :body => second_response, :headers => {})
+      stub_request(:get, "http://example.org/endpoint?q=tags_ssim:dpla&start=4")
+        .to_return(:status => 200, :body => empty_response, :headers => {})
+
+      stub_request(:get, "http://example.org/endpoint?q=id:123")
+        .to_return(:status => 200, :body => single_record, :headers => {})
+
+    end
+
+    it_behaves_like 'a harvester'
+
+    describe '#count' do
+      it 'returns a count' do
+        expect(subject.count).to eq 4
+      end
+    end
+
+    describe '#records' do
+      it 'returns records lazily' do
+        expect(subject).to receive(:request).once.with(args[:api])
+                            .and_return(JSON.parse(response_string))
+        subject.records.first
+      end
+
+      it 'gets all records' do
+        expect(subject.records.count).to eq 4
+      end
+    end
+  end
+end

data/spec/lib/krikri/harvesters/api_harvester_spec.rb~

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Krikri::Harvesters::ApiHarvester do
+  it_behaves_like 'a harvester'
+
+  let(:args) { { uri: 'http://example.org/endpoint' } }
+  subject { described_class.new(args) }
+
+  shared_context 'with responses' do
+    let(:query_opts) { { :q => 'tags_ssim:dpla' } }
+    let(:response_string) do
+      <<EOM
+{ "response": {
+    "numFound": 467354,
+    "start": 0,
+    "docs": []
+  }
+}
+EOM
+    end
+
+    before do
+      allow(RestClient).to recieve(:get).with(args[:uri], query_opts)
+                            .and_return(response_string)
+    end
+  end
+
+  describe '#new' do
+    it 'accepts api parameters' do
+      described_class.new()
+      require 'pry'
+      binding.pry
+    end
+  end
+
+  describe '.expected_opts' do
+    it 'returns a hash with key and options' do
+    end
+  end
+
+  describe '#count' do
+    it 'returns a count' do
+      expect(subject.count).to eq 100
+    end
+  end
+end

data/spec/lib/krikri/harvesters/mdl_api_harvester_spec.rb~

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+require 'spec_helper'
+
+describe Krikri::Harvesters::MdlApiHarvester do
+  it_behaves_like 'a harvester'
+
+  let(:args) { { uri: 'http://example.org/endpoint' } }
+  subject { described_class.new(args) }
+
+
+end

data/spec/lib/krikri/harvesters/mdl_harvester_spec.rb~

@@ -0,0 +1,8 @@
+require 'spec_helper'
+
+describe Krikri::Harvesters::MdlHarvester do
+  let(:args) { { uri: 'http://example.org/endpoint', api: query_opts } }
+  let(:query_opts) { {} }
+  subject { described_class.new(args) }
+
+end

data/spec/lib/krikri/harvesters/oai_harvester_spec.rb

@@ -361,21 +361,28 @@ EOM
     end
 
     describe '#request_with_sets' do
+      let(:sets) { [double('first'), double('second')] }
+      let(:opts) { { :set => sets } }
+
       it 'is lazy' do
-        sets = [double('first'), double('second')]
-        opts = { :set => sets }
         expect { |b| subject.send(:request_with_sets, opts, &b) }
           .not_to yield_control
       end
 
       it 'sends requests' do
-        sets = [double('first'), double('second')]
-        opts = { :set => sets }
         expect { |b| subject.send(:request_with_sets, opts, &b) }
           .not_to yield_successive_args({ :set => opts[:set][0] },
                                         { :set => opts[:set][1] })
       end
 
+      it 'sends requests lazily' do
+        expect(subject).to receive(:give_results).once
+                            .and_return([1,2,3])
+        enum = subject.send(:request_with_sets, opts) do |b|
+          subject.give_results
+        end
+        3.times { enum.next }
+      end
     end
 
     describe '#enqueue' do
@@ -416,7 +423,6 @@ EOM
     end
 
     describe 'concat_enum' do
-
       it 'concatenates enums' do
         expect(subject.concat_enum([(1..10), (100..110)]).to_a)
           .to eq (1..10).to_a.concat((100..110).to_a)
@@ -425,7 +431,7 @@ EOM
       it 'works with lazy' do
         enum = double
         expect(enum).not_to receive :each
-        subject.concat_enum([(1..10), enum]).lazy.take(10)
+        subject.concat_enum([(1..10), enum]).lazy.take(10).each(&:inspect)
       end
     end
 

data/spec/lib/krikri/job_spec.rb

@@ -0,0 +1,40 @@
+require 'spec_helper'
+
+describe Krikri::Job do
+  let(:agent) { double('agent') }
+
+  describe '.perform' do
+    let(:activity_id) { 1 }
+    let(:activity) { double('activity') }
+
+    before do
+      allow(Krikri::Activity).to receive(:find).with(activity_id)
+                                  .and_return(activity)
+    end
+
+    it 'calls #run on activity by id' do
+      expect(activity).to receive(:run).and_return(true)
+      described_class.perform(activity_id)
+    end
+
+    it 'passes its run method to block' do
+      uri = double('uri')
+      expect(activity).to receive(:run).and_yield(agent, uri)
+      expect(described_class).to receive(:run).with(agent, uri)
+      described_class.perform(activity_id)
+    end
+  end
+
+  describe '.run' do
+    it 'calls #run on agent' do
+      activity_uri = double('uri')
+      expect(agent).to receive(:run).with(activity_uri).and_return(true)
+      described_class.run(agent, activity_uri)
+    end
+
+    it 'defaults activity to nil' do
+      expect(agent).to receive(:run).with(nil).and_return(true)
+      described_class.run(agent)
+    end
+  end
+end