daengine 0.1.2 → 0.1.3

data/app/views/digital_assets/show.html.erb

@@ -1,58 +1,58 @@
-<p id="notice"><%= notice %></p>
-
-<p>
-  <b>Title:</b>
-  <%= @digital_asset.title %>
-</p>
-
-<p>
-  <b>Updated at:</b>
-  <%= @digital_asset.updated_at %>
-</p>
-
-<p>
-  <b>Audience:</b>
-  <%= @digital_asset.audiences %>
-</p>
-
-<p>
-  <b>Sami code:</b>
-  <%= @digital_asset.sami_code %>
-</p>
-
-<p>
-  <b>Program:</b>
-  <%= @digital_asset.product_ids %>
-</p>
-
-<p>
-  <b>Published at:</b>
-  <%= @digital_asset.published_at %>
-</p>
-
-<p>
-  <b>Expires at:</b>
-  <%= @digital_asset.expires_at %>
-</p>
-
-<p>
-  <b>Guid:</b>
-  <%= @digital_asset.guid %>
-</p>
-
-<% if @digital_asset.documents.size > 0 %>
-    <h2>Current Documents</h2>
-    <table>
-      <tr>
-        <th>Content Type</th>
-        <th>Path</th>
-      </tr>
-      <% for document in @digital_asset.documents %>
-          <tr>
-            <td><%= document.content_type %></td>
-            <td><%= document.path %></td>
-          </tr>
-      <% end %>
-    </table>
-<% end %>
-
+<p id="notice"><%= notice %></p>
+
+<p>
+  <b>Title:</b>
+  <%= @digital_asset.title %>
+</p>
+
+<p>
+  <b>Updated at:</b>
+  <%= @digital_asset.updated_at %>
+</p>
+
+<p>
+  <b>Audience:</b>
+  <%= @digital_asset.audiences %>
+</p>
+
+<p>
+  <b>Sami code:</b>
+  <%= @digital_asset.sami_code %>
+</p>
+
+<p>
+  <b>Program:</b>
+  <%= @digital_asset.product_ids %>
+</p>
+
+<p>
+  <b>Published at:</b>
+  <%= @digital_asset.published_at %>
+</p>
+
+<p>
+  <b>Expires at:</b>
+  <%= @digital_asset.expires_at %>
+</p>
+
+<p>
+  <b>Guid:</b>
+  <%= @digital_asset.guid %>
+</p>
+
+<% if @digital_asset.documents.size > 0 %>
+    <h2>Current Documents</h2>
+    <table>
+      <tr>
+        <th>Content Type</th>
+        <th>Path</th>
+      </tr>
+      <% for document in @digital_asset.documents %>
+          <tr>
+            <td><%= document.content_type %></td>
+            <td><%= document.path %></td>
+          </tr>
+      <% end %>
+    </table>
+<% end %>
+

data/config/routes.rb

@@ -1,8 +1,8 @@
-Rails.application.routes.draw do
-  match 'digital_assets/search' => 'digital_assets#search'
-  post "digital_assets/sync" => 'digital_assets#sync_assets'
-  # match 'digital_assets/:id' => 'digital_assets#sami'  #, :id => /\w{4,8}.\d*/
-  resources :digital_assets, :only => [:index, :show]
-  #resources :digital_assets
-  #get 'digital_assets' => 'digital_asset#index'
-end
+Rails.application.routes.draw do
+  match 'digital_assets/search' => 'digital_assets#search'
+  # post "digital_assets/sync" => 'digital_assets#sync_assets'
+  # match 'digital_assets/:id' => 'digital_assets#sami'  #, :id => /\w{4,8}.\d*/
+  resources :digital_assets, :only => [:index, :show]
+  #resources :digital_assets
+  #get 'digital_assets' => 'digital_asset#index'
+end

data/lib/daengine.rb

@@ -1,44 +1,72 @@
-require "daengine/version"
-require 'daengine/engine'
-require File.expand_path('../../app/models/digital_asset',__FILE__)
-require 'daengine/teamsite_metadata_parser'
-require 'daengine/digital_asset_processor'
-require 'mongoid'
-
-module Daengine
-  @config = {
-    :assets_path => '/digital-assets'
-  }
-
-  @mongoid_config = {
-    'database' => 'ssc_assets', # mongoid database name
-    'host' => nil, # mongoid server
-    'port' => nil, # mongodb server port
-    'hosts' => nil
-  }
-
-  @valid_mongoid_keys = @mongoid_config.keys
-
-  # yaml file config
-  def self.configure(config_options)
-    config_options.each {|k,v| @config[k.to_sym] = v} 
-    config_options.each {|k,v| 
-      @mongoid_config[k] = v if @valid_mongoid_keys.include? k
-    } 
-    @mongoid_config.delete_if { |k,v| ! v }
-    Mongoid.configure do |config|
-      config.from_hash(@mongoid_config)
-    end
-    p "configured with keys #{@config.keys}"
-  end
-
-  def self.config
-    @config
-  end
-
-  def self.execute(config_options)
-    self.configure(config_options)
-    return DigitalAssetProcessor.process_tuple_directory  # start the thread daemon
-  end
-
-end
+require "daengine/version"
+require 'daengine/engine'
+require File.expand_path('../../app/models/digital_asset',__FILE__)
+require 'daengine/teamsite_metadata_parser'
+require 'daengine/digital_asset_processor'
+require 'mongoid'
+require 'logger'
+
+module Daengine
+  @logger = "/logs/daengine.log"
+
+  @config = {
+    :assets_path => '/digital-assets'
+  }
+
+  @mongoid_config = {
+    'database' => 'ssc_assets', # mongoid database name
+    'host' => nil, # mongoid server
+    'port' => nil, # mongodb server port
+    'hosts' => nil
+  }
+
+  @valid_mongoid_keys = @mongoid_config.keys
+
+  def self.log(message, level)
+    message = "[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}"
+
+    puts "#{message}"
+    case level
+      when "warn"
+         @logger.warn "#{message}"
+      when "error"
+        @logger.error "#{message}"
+      else
+        @logger.info "#{message}"
+    end
+  end
+
+  # yaml file config
+  def self.configure(config_options)
+    config_options.each {|k,v| @config[k.to_sym] = v} 
+    config_options.each {|k,v| 
+      @mongoid_config[k] = v if @valid_mongoid_keys.include? k
+    }
+    @mongoid_config.delete_if { |k,v| ! v }
+    Mongoid.configure do |config|
+      config.from_hash(@mongoid_config)
+    end
+  end
+
+  def self.config
+    @config
+  end
+
+  def self.execute(config_options)
+    self.configure(config_options)
+    self.set_logger()
+    self.log("Daengine: ### Loading application config ###", "info")
+    config_options.each { |key, value| self.log("CONFIG #{key}\t#{value}", "info")}
+    return DigitalAssetProcessor.process_tuple_directory  # start the thread daemon
+  end
+
+  def self.set_logger
+    logfile = self.config[:logfile_location]
+    @logger = Logger.new(logfile, 'daily')
+  end
+
+  def self.logger
+    @logger
+  end
+
+end

data/lib/daengine/digital_asset_processor.rb

@@ -1,70 +1,103 @@
 module Daengine
-class DigitalAssetProcessor
-  
-  @@last_read_time = 2.days.ago
+  class DigitalAssetProcessor
 
-  def self.trap_signals
-    sigtrap = proc { 
-      puts "DigitalAssetProcessor: caught trapped signal, shutting down"
-      @@run = false 
-    }
-    signals = ["SIGTERM", "SIGINT"]
-    signals.push("SIGHUP") unless is_windows?
-    signals.each do |signal|
-      trap signal, sigtrap
+    @@last_read_time = 2.days.ago
+
+    def self.trap_signals
+      sigtrap = proc {
+        puts "DigitalAssetProcessor: caught trapped signal, shutting down"
+        @@run = false
+      }
+      signals = ["SIGTERM", "SIGINT"]
+      signals.push("SIGHUP") unless is_windows?
+      signals.each do |signal|
+        trap signal, sigtrap
+      end
     end
-  end
 
-  def self.is_windows?
-    processor, platform, *rest = RUBY_PLATFORM.split("-")
-    platform =~ /mswin/ || platform =~ /mingw/
-  end
+    def self.is_windows?
+      processor, platform, *rest = RUBY_PLATFORM.split("-")
+      platform =~ /mswin/ || platform =~ /mingw/
+    end
 
-  def self.execute
-    @@run = true
-    trap_signals
-    @@wthread = Thread.new { worker() }  
-    return @@wthread
-  end
-  
-  def self.worker
-    puts "DigitalAssetProcessor: start processing digital assets!"
-    while @@run
-      begin
-        self.process_tuple_directory
-        process_tuple_directory
-      rescue => e
-        puts e.message
-        puts e.backtrace
+    def self.execute
+      @@run = true
+      trap_signals
+      @@wthread = Thread.new { worker() }
+      return @@wthread
+    end
+
+    def self.worker
+      puts "DigitalAssetProcessor: start processing digital assets!"
+      while @@run
+        begin
+          self.process_tuple_directory
+          process_tuple_directory
+        rescue => e
+          puts e.message
+          puts e.backtrace
+        end
+        sleep(5)
       end
-      sleep(5)
+      @@wthread.exit
     end
-    @@wthread.exit
-  end
 
-  def self.process_tuple_directory
-    path = Daengine.config[:assets_path]
-    raise "DigitalAssetProcessor: unable to read from asset path #{path}" unless File::directory?(path)
-    # read the given directory, process each file in date order starting 2 days ago
-    deploy_files= []
+    def self.process_tuple_directory
+      path = Daengine.config[:assets_path]
+      raise "ERROR: DigitalAssetProcessor: Invalid assets_path provided. Unable to read from #{path}" unless File::directory?(path)
+      # read the given directory, process each file in date order starting 2 days ago if last_read_time is nil
+      time = self.read_last_process_time
+      if not time.blank?
+        @@last_read_time = time
+      end
+      Daengine.log("DigitalAssetProcessor: Last process time was #{@@last_read_time}", "info")
+      deploy_files= []
+
+      deploy_files = Dir.entries(path).select {
+          |f| File.file?("#{path}/#{f}") and File.mtime("#{path}/#{f}") > @@last_read_time
+      }.sort_by { |f| File.mtime("#{path}/#{f}") }
+
+      if deploy_files.empty?
+        Daengine.log("DigitalAssetProcessor: No digital asset deployment files found to process under #{path}", "warn")
+      else
+        Daengine.log("DigitalAssetProcessor: Reading digital asset deployment files from #{path}", "info")
+        deploy_files.each do |filename|
+          #parse the file and add metadata content to database.
+          file = File.expand_path(filename, path)
+          Daengine.log("DigitalAssetProcessor: Processing file #{filename} --- #{File.mtime(file)}", "info")
+          open_file = File.open(file, 'rb')
+          Daengine::TeamsiteMetadataParser.parse_tuple_file(open_file)
+          Daengine.log("DigitalAssetProcessor: Finished processing #{filename}", "info")
+          @@last_read_time = File.mtime(file)
+          self.save_last_read_time
+          Daengine.log("DigitalAssetProcessor: Last process time set to #{@@last_read_time}", "info")
+        end
+      end
+    end
 
-    puts "DigitalAssetProcessor: reading digital asset deployment files from #{path}"
-    deploy_files = Dir.entries(path).select {
-      |f| File.file?("#{path}/#{f}") and File.mtime("#{path}/#{f}") > @@last_read_time
-    }.sort_by{ |f| File.mtime("#{path}/#{f}") }
+    def self.save_last_read_time
+      begin
+        target = Daengine.config[:daengine_yml_file]
+        content = "last_read_time: #{@@last_read_time}"
+        File.open(target, "w+") do |f|
+          f.write(content)
+        end
+      rescue Exception => e
+        Daengine.log("DigitalAssetProcessor: Failed to write to #{target}.", "warn")
+      end
+    end
 
-    deploy_files.each do |filename|
-      #parse the file and add content to database.
-      file = File.expand_path(filename, path) 
-      p "DigitalAssetProcessor: processing file #{file}"
-      open_file = File.open(file, 'rb')
-      Daengine::TeamsiteMetadataParser.parse_tuple_file(open_file)
-      p "DigitalAssetProcessor: finished parsing #{filename}."
-      @@last_read_time = File.mtime(file)
-      p "DigitalAssetProcessor: set last read time to #{@@last_read_time}."
+    def self.read_last_process_time
+      time = ""
+      begin
+        target = Daengine.config[:daengine_yml_file]
+        property = YAML.load_file(target)
+        time = property['last_read_time']
+      rescue Exception => e
+        Daengine.log("DigitalAssetProcessor: Failed to read from #{target}", "warn")
+      end
+      time
     end
 
   end
-  
-end
 end

data/lib/daengine/engine.rb

@@ -1,6 +1,6 @@
-require 'rails'
-
-module Daengine
- class Engine < ::Rails::Engine
- end
-end
+require 'rails'
+
+module Daengine
+ class Engine < ::Rails::Engine
+ end
+end

data/lib/daengine/teamsite_metadata_parser.rb

@@ -40,6 +40,8 @@ module Daengine::TeamsiteMetadataParser
   end
 
   def self.parse_tuple_file(file)
+    error_files = []
+    file_count = 0
     time do
       asset = nil
       assets = {}
@@ -83,12 +85,19 @@ module Daengine::TeamsiteMetadataParser
             da = DigitalAsset.find_or_initialize_by(guid: key)
             da.documents = []
             da.update_attributes!(assets[key])
+            file_count = file_count + 1
           end
         rescue Exception => e
-          p "Unable to save/update DigitalAsset guid=#{da.try(:guid)}, #{da.try(:errors).try(:full_messages)}"
+          error_files << "#{da.try(:guid)}, #{da.try(:errors).try(:full_messages)}"
         end      
       end
+      log_txt = "TeamsiteMetadataParser: Failed to save/update following DigitalAssets in database:\n"
+      error_files.each do |asset_file|
+        log_txt = log_txt + "> #{asset_file}\n"
+      end
+      Daengine.log(log_txt, "warn")
       DigitalAsset.purge!  # if the purge criteria is met, purge anything not updated
+      Daengine.log("TeamsiteMetadataParser: #{file_count} records successfully added/updated", "info")
     end
   end