daengine 0.5.13 → 0.6

data/app/models/teamsite_digital_asset_shim.rb

@@ -0,0 +1,253 @@
+require 'mongoid'
+
+# shim class to allow old teamsite parser to create DigitalAssets in the new format
+
+class TeamsiteDigitalAssetShim
+  include Mongoid::Document
+  include Mongoid::Timestamps
+
+  field :title, type: String
+  field :changed_at, type: Time
+  field :audiences, type: Array, default: []
+  field :sami_code, type: String
+  field :product_ids, type: Array, default: []
+  field :published_at, type: Time
+  field :unpublished_at, type: Time
+  field :expires_at, type: Time
+  field :guid, type: String
+  # field :fund_ids, type: Array, default: []
+  field :business_owner, type: String
+  field :summary, type: String
+  field :content_organization_ids, type: Array, default: []
+  field :program_ids, type: Array, default: []
+
+  field :omniture_codes, type: Array, default: []
+  field :orderable, :type => Boolean, default: false
+  key :guid
+
+  # field :documents, type: Hash
+
+  embeds_many :documents, :class_name => 'DigitalAsset::Document'
+
+  accepts_nested_attributes_for :documents
+
+  #Exclude XBRL documents from all queries
+  default_scope excludes(:'documents.content_type' => "LDJDCMAIK") #Had to use static value instead of a Constant
+
+  scope :title_is, ->(title) { where(:title => title)}
+  scope :business_owner_is, ->(business_owner) { where(:business_owner => business_owner)}
+  scope :guid_is, ->(guid) { where(:guid => guid)}
+  # scope :funds_in, ->(fund_id) { where(:fund_ids.in => fund_id)}
+  scope :audience_in, ->(audience_id) {where(:audiences.in => audience_id)}
+  scope :audience_investor_approved, -> {where(:audiences.in => [Audience::INVESTOR_APPROVED])}
+
+  scope :content_organization_in, ->(content_organization_id) {where(:content_organization_ids.in => content_organization_id)}
+  scope :program_id_in, ->(program_id) {where(:program_ids.in => program_id)}
+  scope :sami_is, ->(sami_code) {where(:sami_code => sami_code)}
+  scope :sami_in, ->(sami_codes) {where(:sami_code.in => sami_codes)}
+  scope :path_is, ->(path) {where(:'documents.path' => path)}
+  scope :doctype_in, ->(types) {where(:'documents.content_type'.in => types)}
+  scope :content_type_in, ->(types) {where(:'documents.content_type'.in => types)}
+  scope :product_in, ->(types) {where(:product_ids.in => types)}
+  scope :stale, -> {where(:updated_at.lte => 2.minutes.ago)}
+  scope :orderable_is, ->(orderable) {where(:orderable => orderable)}
+  scope :orderable, -> {where(orderable: true)}
+  scope :has_finra, -> {where(:'documents.content_type' => DigitalAsset::ContentType::FINRA)}
+  scope :audience_in, ->(audience) {where(:audiences.in => audience)}
+  scope :alphabetical, order_by(:title => :asc)
+  scope :not_xbrl, -> {excludes(:'documents.content_type' => DigitalAsset::ContentType::XBRL_DOCUMENT)}
+
+  #scope :order_by_fund, order_by[[:product_ids, :asc]]
+  #default_scope {not_in(:'documents.content_type' => ["LDJDCMAIK"])}
+
+  # validations
+  validates_presence_of :guid, :title, :changed_at, :published_at,
+      :expires_at, :audiences, :documents
+
+  validate :validate_future_expiration
+
+  # validates_uniqueness_of :guid
+
+
+  def self.find_or_initialize_by(attrs = {})
+    tda = nil
+    # get the base DA if it exists
+    da = DigitalAsset.find_or_initialize_by(attrs)
+    # if it exists copy over the attributes
+    if(da.updated_at.present?)
+      tda = TeamsiteDigitalAssetShim.new(da.attributes)
+    else
+      tda = TeamsiteDigitalAssetShim.new(attrs)
+    end
+    # move metadata from the new spots to the old spots
+    tda.documents.build(path: da.path, doc_changed_at: da.doc_changed_at, 
+      content_type: da.content_type, pages: da.pages, size: da.size, mime_type: da.mime_type,
+      subject: da.subject, keywords: da.keywords, author: da.author)
+    # make a finra if there is one
+    if(da.finra_path.present?)
+      tda.documents.build(content_type: DigitalAsset::ContentType::FINRA, path: da.finra_path)
+    end
+    tda
+  end
+
+  def update_attributes!(atts = {})
+    # deconvert metadata from old to new
+    self.attributes = atts
+    da = DigitalAsset.new(atts)
+    doc = first_non_finra
+    da.path = doc.try(:path)
+    da.finra_path = finra_document.try(:path)
+    da.doc_changed_at = doc.doc_changed_at
+    da.content_type = doc.content_type
+    da.pages = doc.pages
+    da.size = doc.size
+    da.mime_type = doc.mime_type
+    da.subject = doc.subject
+    da.keywords = doc.keywords
+    da.author = doc.author
+    da.digital_asset_id = guid
+    # save the new metadata format
+    da.save!
+  end
+
+
+  def as_json(opts = {})
+    super(opts).merge({:comp_fundcode => fund_code, :content_type_id => content_type_id, :latest_doc_changed_at => latest_doc_changed_at})
+  end
+
+  def self.purge!
+    # last_update = DigitalAsset.desc(:updated_at).try(:first).try :updated_at
+    DigitalAsset.stale.destroy_all if bulk_processed?
+  end
+
+  def latest_doc_changed_at
+    documents.reduce(nil) do |latest_date, d|
+      unless d.content_type == '549'
+        latest_date = d.doc_changed_at if (latest_date == nil || latest_date < d.doc_changed_at)
+      end
+      latest_date
+    end
+  end
+
+  def validate_future_expiration
+    errors.add(:expires_at, "Expiration date must be at least 1 minute from now") unless expires_at and expires_at > 1.minute.from_now
+  end
+
+  def self.bulk_processed?
+    (stale.count.to_f / self.count) <= 0.05
+  end
+
+  def path_is(path)
+    documents.where(path: path).first unless documents.blank?
+  end
+
+  def doc_changed_at(path)
+    path_is(path).try(:doc_changed_at)
+  end
+
+  def doc_size
+    first_non_finra.try(:size)
+  end
+
+  def content_type_ids
+    ids = []
+    documents.try(:each) do |d|
+      ids << d.content_type
+    end
+    ids
+  end
+  alias :doctype_ids :content_type_ids
+
+  def has_finra?
+    finra_document != nil
+  end
+
+  def expired?
+    expires_at < Time.now
+  end
+
+  def finra_document
+    finra_idx = documents.index {|d| d.content_type == DigitalAsset::ContentType::FINRA}
+    documents[finra_idx] if finra_idx
+  end
+
+  def is_investor_approved?
+    audiences.index(DigitalAsset::Audience::INVESTOR_APPROVED)
+  end
+  alias :investor_approved? :is_investor_approved?
+
+  def is_institutional_use?
+    audiences.index(DigitalAsset::Audience::INSTITUTIONAL_USE)
+  end
+  alias :institutional_use? :is_institutional_use?
+
+  def product
+    TaxonomyTerm.label_for_term(product_ids[0])
+  end
+  def program
+    TaxonomyTerm.label_for_term(program_ids[0])
+  end
+  def content_org
+    TaxonomyTerm.label_for_term(content_organization_ids[0])
+  end
+
+  def fund_code
+    pid = product_ids.find {|pid| TaxonomyTerm.term_id_is(pid)[0].try(:fund_code)}
+    pid and TaxonomyTerm.term_id_is(pid)[0].try(:fund_code).try(:rjust, 5, '0')
+  end
+
+  def content_type
+    TaxonomyTerm.label_for_term(content_type_id)
+  end
+  def content_type_id
+   first_non_finra.try(:content_type)
+  end
+
+  def pages; first_non_finra.pages end
+
+  def audience
+    TaxonomyTerm.label_for_term(audiences[0])
+  end
+
+  def primary_path
+    first_non_finra.try(:path)
+  end
+
+  def primary_extension
+    first_non_finra.try(:path).try(:split,'.').try(:last).try(:upcase)
+  end
+
+  private
+
+  def first_non_finra
+   documents.try(:detect) do |d|
+    d.content_type != DigitalAsset::ContentType::FINRA
+   end
+  end
+
+
+end
+
+class DigitalAsset::Document
+  include Mongoid::Document
+
+  field :path, type: String
+  field :doc_changed_at, type: Time
+  field :content_type, type: String
+  field :pages, type: Integer, default: 1
+  field :size, type: String
+  field :mime_type, type: String
+  field :subject, type: String
+  field :keywords, type: Array, default: []
+  field :author, type: String
+
+  embedded_in :digital_asset
+
+  key :path
+
+  validates_uniqueness_of :path
+
+  validates_presence_of :path #, :doc_changed_at, :content_type
+  validates_format_of :path, without: /\/manifest|archives\// # dont accept manifest files
+
+end

data/app/service/digital_asset_lookup_service.rb

@@ -1,57 +1,57 @@
 class DigitalAssetLookupService
 
-  def self.find_documents_by_fund_code(fund_codes)
-    ids = fund_codes.collect {|c| term_id_from_fund_code(c)}
-    DigitalAsset.unscoped.content_type_in(DigitalAsset::ContentType::FUND_DOC_TYPES).product_in ids
-  end
+  # def self.find_documents_by_fund_code(fund_codes)
+  #   ids = fund_codes.collect {|c| term_id_from_fund_code(c)}
+  #   DigitalAsset.unscoped.content_type_in(DigitalAsset::ContentType::FUND_DOC_TYPES).product_in ids
+  # end
 
-  def self.find_documents_by_fund_code_and_content_type(fund_codes,content_type)
-    assets = DigitalAssetLookupService.find_documents_by_fund_code(fund_codes)
-    found_asset = nil
-    assets.each {|da|
-      da.documents.each {|doc|
-        found_asset = da if doc.content_type == content_type
-      }
-    }
-    found_asset
-  end
+  # def self.find_documents_by_fund_code_and_content_type(fund_codes,content_type)
+  #   assets = DigitalAssetLookupService.find_documents_by_fund_code(fund_codes)
+  #   found_asset = nil
+  #   assets.each {|da|
+  #     da.documents.each {|doc|
+  #       found_asset = da if doc.content_type == content_type
+  #     }
+  #   }
+  #   found_asset
+  # end
 
-  def self.fund_code_from_id(taxonomy_id)
-    TaxonomyTerm.term_id_is(taxonomy_id).try(:term_type).try([], 'FUND_CODE')
-  end
+  # def self.fund_code_from_id(taxonomy_id)
+  #   TaxonomyTerm.term_id_is(taxonomy_id).try(:term_type).try([], 'FUND_CODE')
+  # end
 
-  def self.term_id_from_fund_code(code) 
-   term = TaxonomyTerm.fund_code_is(code)
-   term.try(:first).try(:term_id)
-  end
+  # def self.term_id_from_fund_code(code) 
+  #  term = TaxonomyTerm.fund_code_is(code)
+  #  term.try(:first).try(:term_id)
+  # end
 
-  def self.fund_code_from_asset_path(path)
-    da = DigitalAsset.path_is(item.path).first
-    fund_code = fund_for_digital_asset da
-  end
-  def self.find_documents_by_query(arg_product_ids=nil, arg_content_type_ids=nil, arg_audience_ids=nil, finra=nil)
-    product_ids = [] << arg_product_ids 
-    product_ids.compact!
-    product_ids.flatten!
-    content_type_ids = [] << arg_content_type_ids
-    content_type_ids.compact!
-    content_type_ids.flatten!
-    audience_ids = [] << arg_audience_ids
-    audience_ids.compact!
-    audience_ids.flatten!
-    finra = finra || false
-    query = {}
-    query[:product_ids.in] = product_ids if !product_ids.blank?
-    query[:'documents.content_type'.in] = content_type_ids if !content_type_ids.blank?
-    query[:audiences.in] = audience_ids if !audience_ids.blank?
+  # def self.fund_code_from_asset_path(path)
+  #   da = DigitalAsset.path_is(item.path).first
+  #   fund_code = fund_for_digital_asset da
+  # end
+  # def self.find_documents_by_query(arg_product_ids=nil, arg_content_type_ids=nil, arg_audience_ids=nil, finra=nil)
+  #   product_ids = [] << arg_product_ids 
+  #   product_ids.compact!
+  #   product_ids.flatten!
+  #   content_type_ids = [] << arg_content_type_ids
+  #   content_type_ids.compact!
+  #   content_type_ids.flatten!
+  #   audience_ids = [] << arg_audience_ids
+  #   audience_ids.compact!
+  #   audience_ids.flatten!
+  #   finra = finra || false
+  #   query = {}
+  #   query[:product_ids.in] = product_ids if !product_ids.blank?
+  #   query[:'documents.content_type'.in] = content_type_ids if !content_type_ids.blank?
+  #   query[:audiences.in] = audience_ids if !audience_ids.blank?
   
-    digital_assets = DigitalAsset.where(query)
-    digital_assets = digital_assets.select {|d| d.has_finra?} if finra
-    digital_assets
-  end
+  #   digital_assets = DigitalAsset.where(query)
+  #   digital_assets = digital_assets.select {|d| d.has_finra?} if finra
+  #   digital_assets
+  # end
 
-  def self.find_documents_by_sami_code(sami_codes)
-    DigitalAsset.sami_in sami_codes
-  end
+  # def self.find_documents_by_sami_code(sami_codes)
+  #   DigitalAsset.sami_in sami_codes
+  # end
 
 end

data/bin/process_assets

@@ -1,4 +1,4 @@
-#!/usr/bin/env jruby
+#!/usr/bin/env ruby
 #
 # This file was generated by Bundler.
 #
@@ -6,11 +6,10 @@
 # this file is here to facilitate running it.
 #
 
-require 'pathname'
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
-  Pathname.new(__FILE__).realpath)
+require 'daengine'
 
-require 'rubygems'
-require 'bundler/setup'
+config = YAML.load_file(ARGV[0])
 
-load Gem.bin_path('daengine', 'process_assets')
+t = Daengine.execute(config)
+
+puts t

data/bin/process_availability

@@ -1,16 +1,15 @@
-#!/usr/bin/env jruby
+#!/usr/bin/env ruby
 #
 # This file was generated by Bundler.
 #
-# The application 'process_availability' is installed as part of a gem, and
+# The application 'process_assets' is installed as part of a gem, and
 # this file is here to facilitate running it.
 #
 
-require 'pathname'
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
-  Pathname.new(__FILE__).realpath)
+require 'daengine'
 
-require 'rubygems'
-require 'bundler/setup'
+config = YAML.load_file(ARGV[0])
 
-load Gem.bin_path('daengine', 'process_availability')
+t = Daengine.execute_content_service(config)
+
+puts t

data/bin/process_taxonomy

@@ -1,16 +1,15 @@
-#!/usr/bin/env jruby
+#!/usr/bin/env ruby
 #
 # This file was generated by Bundler.
 #
-# The application 'process_taxonomy' is installed as part of a gem, and
+# The application 'process_assets' is installed as part of a gem, and
 # this file is here to facilitate running it.
 #
 
-require 'pathname'
-ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
-  Pathname.new(__FILE__).realpath)
+require 'daengine'
 
-require 'rubygems'
-require 'bundler/setup'
+config = YAML.load_file(ARGV[0])
 
-load Gem.bin_path('daengine', 'process_taxonomy')
+t = Daengine.execute_taxonomy(config)
+
+puts t

data/config/routes.rb

@@ -1,9 +1,9 @@
 Rails.application.routes.draw do
-  match 'digital_assets/search' => 'digital_assets#search'
-  match 'digital_assets/fund_docs' => 'digital_assets#fund_docs'
-  # post "digital_assets/sync" => 'digital_assets#sync_assets'
-  # match 'digital_assets/:id' => 'digital_assets#sami'  #, :id => /\w{4,8}.\d*/
-  resources :digital_assets, :only => [:index, :show]
-  #resources :digital_assets
-  #get 'digital_assets' => 'digital_asset#index'
+  resources :digital_assets, except: [:new, :edit], defaults: {format: 'json'} do 
+    collection do
+      get 'fund_docs'
+      get 'search', to: 'digital_assets#search'
+      post 'updated_time'
+    end
+  end
 end

data/lib/daengine.rb

@@ -2,6 +2,7 @@ require "daengine/version"
 require 'daengine/engine'
 require 'daengine/railtie' if defined?(Rails)
 require File.expand_path('../../app/models/digital_asset',__FILE__)
+require File.expand_path('../../app/models/teamsite_digital_asset_shim',__FILE__)
 require File.expand_path('../../app/models/taxonomy_term',__FILE__)
 require File.expand_path('../../app/models/content_service_resource',__FILE__)
 require 'mini_exiftool'
@@ -10,7 +11,6 @@ require 'daengine/digital_asset_processor'
 require 'daengine/taxonomy_processor'
 require 'daengine/content_service_processor'
 require 'daengine/taxonomy_parser'
-require 'daengine/digital_asset_extension_processor'
 require 'mongoid'
 require 'logger'
 
@@ -97,18 +97,6 @@ module Daengine
     end
   end
 
-  def self.execute_digital_asset_extension(config_options)
-    self.configure(config_options)
-    self.set_logger(config[:digital_asset_extension_logfile_location])
-    self.log("Daengine: ### Loading application config ###", "info")
-    config_options.each { |key, value| self.log("CONFIG #{key}\t#{value}", "info")}
-    begin
-      DigitalAssetExtensionProcessor.process_extension_file
-    rescue Exception => e
-      self.log(e.message, "error")
-    end
-  end
-
   def self.set_logger(logfile)
     @logger = Logger.new(logfile, 'daily')
   end