daengine 0.6.10 → 0.6.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0a7ec04edac11e46187d3dd6b978c9841ca73832
-  data.tar.gz: 69f6f64db73d5729782c85214e93691f0a0c8709
+  metadata.gz: 4a2ff236a90f4e3e9fd22ca6586e641cd968e5f9
+  data.tar.gz: 36a66024ce2ac659e8cdcea149c9b855607969eb
 SHA512:
-  metadata.gz: 1fea624f3a6f85860cf36c471486cb6bfd1884eedff6b38fbfba34b4603d0a92145560db2b1f07c270fcb8e64fc654014449b7659dfe009af43736b4f00d17ed
-  data.tar.gz: 83ef062a84f8d35612ec03b4a09c079f98e29bd7f815cb1c8e7743b212b5b43be64a8db8ef65b4609f822ae022cf694993c743e072b187dc783158f38add2b56
+  metadata.gz: 9a2191f074c80c664e0f56dbf370015d6c1549c0900732ff8fea277374b99653673a527d87519b8a15aaca790f5428af6b6ff5ec9d4ee9576a6df81334f88cff
+  data.tar.gz: 6d58d4df8d1577e312a0c84dc135ae53d50292bcbbc084419fe25072ae82c04fcb695fd9d42f020fd80d602687875c4d407a0569a2a06b995db3baf1f72f157e

data/app/models/service_digital_asset.rb

@@ -89,9 +89,13 @@ class ServiceDigitalAsset < ActiveRecord::Base
     {'digital_asset' => hash}
   end
 
-  def default_blank_time(time_field)
+  def default_blank_time(time_field, default_time = 10.years.ago)
     time = send(time_field.to_sym)
-    time.blank? ? 10.years.ago : time
+    time.blank? ? default_time : time
+  end
+
+  def changed_since(date)
+    !changed_at.blank? && changed_at.to_i >= date.to_i
   end
 
   def self.boolean_field?(field_name)

data/lib/daengine/teamsite_metadata_processor.rb

@@ -10,7 +10,7 @@ module Daengine
 
       time do
         assets = parse_file(file)
-        assets = select_1_asset_per_id(assets)
+        assets = select_1_asset_per_id(assets, last_read)
         assets = add_file_attributes(assets, last_read) unless Daengine.config[:disable_file_check]
         assets = add_fund_codes(assets)
         summary = call_service(assets)
@@ -30,19 +30,21 @@ module Daengine
     end
 
     def self.parse_file(file)
-puts "----- Parsing the file -----"
+      puts "----- Parsing the file -----"
       document = Document.new
       Nokogiri::XML::SAX::Parser.new(document).parse(file)
       Daengine.log("Nokogiri Parser complete...", "info")
       document.assets
     end
 
-    def self.select_1_asset_per_id(assets)
+    def self.select_1_asset_per_id(assets, last_read)
+      comparison_date = last_read.nil? ? 10.years.ago : last_read - 5.minutes
       results = {}
       assets.each do |key, values|
-puts "----- select_1 #{key} -----"
+        puts "----- select_1 #{key} -----"
         list = values.find_all { |v| v.effective? }
         asset = most_recent_non_expired(list)
+        asset = (asset.changed_since(comparison_date) ? asset : nil) unless asset.nil?
         asset.finra_path = finra_path(list) unless asset.nil?
         results[key] = asset unless asset.nil?
       end
@@ -53,7 +55,7 @@ puts "----- select_1 #{key} -----"
       results = {}
       assets.each do |key, asset|
         begin
-puts "----- add_file #{key} -----"
+          puts "----- add_file #{key} -----"
           file_name = asset_file_name(asset)
           asset.mark_for_deletion unless File.file?(file_name)
 
@@ -70,7 +72,7 @@ puts "----- add_file #{key} -----"
     def self.add_fund_codes(assets)
       results = {}
       assets.each do |key, asset|
-puts "----- add_fund_codes #{key} -----"
+        puts "----- add_fund_codes #{key} -----"
         fund_codes = []
         asset.product_ids.each do |product_id|
           fund_code = DigitalAssetLookupService.fund_code_from_id(product_id)
@@ -86,7 +88,7 @@ puts "----- add_fund_codes #{key} -----"
       results = {:errors => 0, :updated => 0, :deleted => 0}
       assets.each_value do |asset|
         begin
-puts "----- call_service #{asset.digital_asset_id} -----"
+          puts "----- call_service #{asset.digital_asset_id} -----"
           if asset.delete?
             path = "#{service_uri}/#{asset.digital_asset_id}"
             options = {:method => :delete, :headers => header}

data/lib/daengine/version.rb

@@ -1,3 +1,3 @@
 module Daengine
-  VERSION = "0.6.10"
+  VERSION = "0.6.11"
 end

data/spec/dummy/log/test.log

@@ -9200,3 +9200,103 @@ Completed 406 Not Acceptable in 2.0ms
 Processing by DigitalAssetsController#search as HTML
   Parameters: {"digital_asset_id"=>"blargh-blargh-blargh", "title"=>"Doc Title"}
 Completed 406 Not Acceptable in 1.0ms
+Started GET "/digital_assets" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#index as JSON
+Completed 200 OK in 6.0ms (Views: 4.0ms)
+Started GET "/digital_assets?digital_asset_id=id-foobar-permanent-1" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#index as JSON
+  Parameters: {"digital_asset_id"=>"id-foobar-permanent-1"}
+Completed 200 OK in 31.0ms (Views: 29.0ms)
+Started GET "/digital_assets?fund=00200" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#index as JSON
+  Parameters: {"fund"=>"00200"}
+Completed 200 OK in 27.0ms (Views: 25.0ms)
+Started GET "/digital_assets?digital_asset_id=id-foobar-permanent-4" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#index as JSON
+  Parameters: {"digital_asset_id"=>"id-foobar-permanent-4"}
+Completed 200 OK in 27.0ms (Views: 21.0ms)
+Started GET "/digital_assets?path=%2Fsome%2Fother%2Fpath.pdf" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#index as JSON
+  Parameters: {"path"=>"/some/other/path.pdf"}
+Completed 200 OK in 22.0ms (Views: 20.0ms)
+Started GET "/digital_assets?digital_asset_id=id-foobar-permanent-6" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#index as JSON
+  Parameters: {"digital_asset_id"=>"id-foobar-permanent-6"}
+Completed 200 OK in 33.0ms (Views: 29.0ms)
+Started GET "/digital_assets?published=2014-02-12+16%3A20%3A27+UTC" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#index as JSON
+  Parameters: {"published"=>"2014-02-12 16:20:27 UTC"}
+Completed 200 OK in 78.0ms (Views: 25.0ms)
+Started GET "/digital_assets/foo-bar-id" for 127.0.0.1 at 2014-02-12 11:20:27 -0700
+Processing by DigitalAssetsController#show as JSON
+  Parameters: {"id"=>"foo-bar-id"}
+Completed 200 OK in 46.0ms (Views: 12.0ms)
+Started GET "/digital_assets/not-found-doc" for 127.0.0.1 at 2014-02-12 11:20:28 -0700
+Processing by DigitalAssetsController#show as JSON
+  Parameters: {"id"=>"not-found-doc"}
+Completed 404 Not Found in 80.0ms (Views: 1.0ms)
+Started PUT "/digital_assets/foo-bar-digital-asset-id" for 127.0.0.1 at 2014-02-12 11:20:28 -0700
+Processing by DigitalAssetsController#update as JSON
+  Parameters: {"digital_asset"=>{"audiences"=>["investor"], "omniture_codes"=>["KOW2MY24D"], "orderable"=>"false", "pages"=>"1", "fund_codes"=>["00200", "00190", "00210"], "_id"=>"foo-bar-digital-asset-id", "title"=>"another title", "changed_at"=>"2014-02-10 18:20:26 UTC", "digital_asset_id"=>"foo-bar-digital-asset-id", "published_at"=>"2014-02-02 18:20:26 UTC", "expires_at"=>"2014-04-12 18:20:26 UTC", "sami_code"=>"IL1111.077", "business_owner"=>"biz owner", "path"=>"/9/foo/bar.txt", "finra_path"=>"/path/finradoc/10/foo/bar.txt", "legacy_path"=>"/oldpath/10/foo/bar.txt", "doc_changed_at"=>"2014-02-10 18:20:26 UTC", "content_type"=>"fact_sheet", "updated_at"=>"2014-02-12 18:20:28 UTC", "created_at"=>"2014-02-12 18:20:28 UTC"}, "id"=>"foo-bar-digital-asset-id"}
+Completed 200 OK in 30.0ms
+Started POST "/digital_assets" for 127.0.0.1 at 2014-02-12 11:20:28 -0700
+Processing by DigitalAssetsController#create as JSON
+  Parameters: {"digital_asset"=>{"audiences"=>["investor"], "omniture_codes"=>["KOW2MY24D"], "orderable"=>"false", "pages"=>"1", "fund_codes"=>["00200", "00190", "00210"], "_id"=>"52fbbb6cf2e6a4e84c4c68dd", "title"=>"Doc Title", "changed_at"=>"2014-02-10 18:20:26 UTC", "digital_asset_id"=>"new-digital-asset-id", "published_at"=>"2014-02-02 18:20:26 UTC", "expires_at"=>"2014-04-12 18:20:26 UTC", "sami_code"=>"IL1111.077", "business_owner"=>"biz owner", "path"=>"/10/foo/bar.txt", "finra_path"=>"/path/finradoc/11/foo/bar.txt", "legacy_path"=>"/oldpath/11/foo/bar.txt", "doc_changed_at"=>"2014-02-10 18:20:26 UTC", "content_type"=>"fact_sheet"}}
+Completed 201 Created in 80.0ms
+Started DELETE "/digital_assets/foo-bar-digital-asset-id" for 127.0.0.1 at 2014-02-12 11:20:28 -0700
+Processing by DigitalAssetsController#destroy as JSON
+  Parameters: {"id"=>"foo-bar-digital-asset-id"}
+Completed 200 OK in 6.0ms
+Started POST "/digital_assets/updated_time" for 127.0.0.1 at 2014-02-12 11:20:28 -0700
+Processing by DigitalAssetsController#updated_time as JSON
+  Parameters: {"ids"=>["id-1", "id-2", "id-3", "id-4"]}
+Completed 200 OK in 42.0ms (Views: 0.0ms)
+Processing by DigitalAssetsController#index as HTML
+Completed 200 OK in 4.0ms (Views: 2.0ms)
+Processing by DigitalAssetsController#show as HTML
+  Parameters: {"id"=>"id-foobar-permanent-9"}
+Completed 406 Not Acceptable in 5.0ms
+Processing by DigitalAssetsController#show as HTML
+  Parameters: {"id"=>"IL1111.077"}
+Completed 406 Not Acceptable in 9.0ms
+Processing by DigitalAssetsController#show as HTML
+  Parameters: {"id"=>"IL1111.077"}
+Completed 406 Not Acceptable in 6.0ms
+Processing by DigitalAssetsController#show as HTML
+  Parameters: {"id"=>"id-foobar-permanent-13"}
+Completed 406 Not Acceptable in 5.0ms
+Processing by DigitalAssetsController#search as HTML
+Completed 200 OK in 5.0ms (Views: 4.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"path"=>"/one/off.path"}
+Completed 200 OK in 18.0ms (Views: 16.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"doctype"=>"fact_sheet"}
+Completed 200 OK in 8.0ms (Views: 3.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"doctype"=>["fact_sheet", "prospectus"]}
+Completed 200 OK in 18.0ms (Views: 16.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"sami"=>"SOMETHING.001"}
+Completed 200 OK in 33.0ms (Views: 32.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"audience"=>"492"}
+Completed 200 OK in 37.0ms (Views: 35.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"title"=>"Doc Title"}
+Completed 200 OK in 6.0ms (Views: 3.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"digital_asset_id"=>"id-foobar-permanent-35"}
+Completed 200 OK in 19.0ms (Views: 18.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"business_owner"=>"biz owner"}
+Completed 200 OK in 4.0ms (Views: 2.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"digital_asset_id"=>"id-foobar-permanent-42", "title"=>"Doc Title"}
+Completed 200 OK in 13.0ms (Views: 11.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"title"=>"Doc Title", "audiences"=>["investor"], "sami"=>"IL1111.077"}
+Completed 200 OK in 93.0ms (Views: 90.0ms)
+Processing by DigitalAssetsController#search as HTML
+  Parameters: {"digital_asset_id"=>"blargh-blargh-blargh", "title"=>"Doc Title"}
+Completed 200 OK in 4.0ms (Views: 2.0ms)

data/spec/lib/teamsite_metadata_processor_spec.rb

@@ -37,15 +37,16 @@ describe Daengine::TeamsiteMetadataProcessor do
     end
   end
 
-  context '#filter_assets' do
+  context '#select_1_asset_per_id' do
     let(:asset) { FactoryGirl.build :service_digital_asset }
     let(:other_asset) { FactoryGirl.build :service_digital_asset }
+    let(:last_read) { 1.year.ago }
 
     it 'should return assets that are effective' do
       id = asset.digital_asset_id
       assets = {id => [asset]}
 
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
 
       results[id].should == asset
     end
@@ -55,7 +56,7 @@ describe Daengine::TeamsiteMetadataProcessor do
       asset.expires_at = nil
       assets = {id => [asset]}
 
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
 
       results[id].should be_nil
     end
@@ -65,7 +66,7 @@ describe Daengine::TeamsiteMetadataProcessor do
       asset.display_on_website = false
       assets = {id => [asset]}
 
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
 
       results[id].should be_nil
     end
@@ -75,12 +76,12 @@ describe Daengine::TeamsiteMetadataProcessor do
       other_asset.expires_at = 1.day.ago
 
       assets = {id => [asset, other_asset]}
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
       results[id].should == asset
 
       # order in list shouldn't matter
       assets = {id => [other_asset, asset]}
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
       results[id].should == asset
     end
 
@@ -89,12 +90,12 @@ describe Daengine::TeamsiteMetadataProcessor do
       other_asset.path = '/foo/manifest' + other_asset.path
 
       assets = {id => [asset, other_asset]}
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
       results[id].should == asset
 
       # order in list shouldn't matter
       assets = {id => [other_asset, asset]}
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
       results[id].should == asset
     end
 
@@ -103,12 +104,12 @@ describe Daengine::TeamsiteMetadataProcessor do
       other_asset.content_type = DigitalAsset::ContentType::FINRA
 
       assets = {id => [asset, other_asset]}
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
       results[id].should == asset
 
       # order in list shouldn't matter
       assets = {id => [other_asset, asset]}
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
       results[id].should == asset
     end
 
@@ -118,11 +119,21 @@ describe Daengine::TeamsiteMetadataProcessor do
       finra_path = other_asset.path
 
       assets = {id => [asset, other_asset]}
-      results = subject.select_1_asset_per_id(assets)
+      results = subject.select_1_asset_per_id(assets, last_read)
       result = results[id]
       result.should == asset
       result.finra_path.should == finra_path
     end
+
+    it 'should remove assets that have not changed since the last_read time' do
+      read_time = 1.day.ago
+      id = asset.digital_asset_id
+      assets = {id => [asset]}
+
+      results = subject.select_1_asset_per_id(assets, read_time)
+
+      results[id].should be_nil
+    end
   end
 
   context '#add_file_attributes' do

data/spec/mock_data/daengine.yml

@@ -1 +1 @@
-last_read_time: 2014-02-07 08:20:30 -0700
+last_read_time: 2014-02-12 11:20:31 -0700

data/spec/mock_data/taxonomy/taxonomyengine.yml

@@ -1 +1 @@
-last_read_time: 2014-02-07 08:20:33 -0700
+last_read_time: 2014-02-12 11:20:35 -0700

data/spec/models/service_digital_asset_spec.rb

@@ -132,5 +132,17 @@ describe ServiceDigitalAsset do
 
       result.to_i.should == expected.to_i
     end
+
+    it 'returns true if the asset has been updated since the provided date' do
+      date = 7.days.ago
+
+      asset.changed_since(date).should == true
+    end
+
+    it 'returns false if the asset has not been updated since the provided date' do
+      date = Time.now
+
+      asset.changed_since(date).should == false
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: daengine
 version: !ruby/object:Gem::Version
-  version: 0.6.10
+  version: 0.6.11
 platform: ruby
 authors:
 - sbhatia
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-02-11 00:00:00.000000000 Z
+date: 2014-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails