daengine 0.0.4 → 0.0.5

data/app/models/digital_asset.rb

@@ -1,3 +1,5 @@
+require 'mongoid'
+
 class DigitalAsset
   include Mongoid::Document
   include Mongoid::Timestamps

data/lib/daengine.rb

@@ -1,6 +1,38 @@
-require "daengine/engine"
-
+require "daengine/version"
+require 'daengine/engine'
+require File.expand_path('../../app/models/digital_asset',__FILE__)
+require 'daengine/teamsite_metadata_parser'
+require 'daengine/digital_asset_processor'
 require 'mongoid'
 
 module Daengine
+  @config = {
+  }
+
+  @mongoid_config = {
+    'database' => 'ssc_assets', # mongoid database name
+    'host' => 'localhost', # mongoid server
+    'port' => 27017, # mongodb server port
+  }
+
+  @valid_mongoid_keys = @mongoid_config.keys
+
+  # yaml file config
+  def self.configure(config_options)
+    config_options.each {|k,v| @config[k.to_sym] = v} 
+    config_options.each {|k,v| @mongoid_config[k.to_sym] = v if @valid_mongoid_keys.include? k.to_sym} 
+    Mongoid.configure do |config|
+      config.from_hash(@mongoid_config)
+    end
+  end
+
+  def self.config
+    @config
+  end
+
+  def self.execute(config_options)
+    self.configure(config_options)
+    return DigitalAssetProcessor.execute  # start the thread daemon
+  end
+
 end

data/lib/daengine/digital_asset_processor.rb

@@ -0,0 +1,70 @@
+module Daengine
+class DigitalAssetProcessor
+  
+  @@last_read_time = 2.days.ago
+
+  def self.trap_signals
+    sigtrap = proc { 
+      puts "DigitalAssetProcessor: caught trapped signal, shutting down"
+      @@run = false 
+    }
+    signals = ["SIGTERM", "SIGINT"]
+    signals.push("SIGHUP") unless is_windows?
+    signals.each do |signal|
+      trap signal, sigtrap
+    end
+  end
+
+  def self.is_windows?
+    processor, platform, *rest = RUBY_PLATFORM.split("-")
+    platform =~ /mswin/ || platform =~ /mingw/
+  end
+
+  def self.execute
+    @@run = true
+    trap_signals
+    @@wthread = Thread.new { worker() }  
+    return @@wthread
+  end
+  
+  def self.worker
+    puts "DigitalAssetProcessor: start processing digital assets!"
+    while @@run
+      begin
+        self.process_tuple_directory
+        process_tuple_directory
+      rescue => e
+        puts e.message
+        puts e.backtrace
+      end
+      sleep(5)
+    end
+    @@wthread.exit
+  end
+
+  def self.process_tuple_directory
+    path = Daengine.config[:assets_path]
+    raise "DigitalAssetProcessor: unable to read from asset path #{path}" unless File::directory?(path)
+    # read the given directory, process each file in date order starting 2 days ago
+    deploy_files= []
+
+    puts "DigitalAssetProcessor: reading digital asset deployment files from #{path}"
+    deploy_files = Dir.entries(path).select {
+      |f| File.file?("#{path}/#{f}") and File.mtime("#{path}/#{f}") > @@last_read_time
+    }.sort_by{ |f| File.mtime("#{path}/#{f}") }
+
+    deploy_files.each do |filename|
+      #parse the file and add content to database.
+      file = File.expand_path(filename, path) 
+      p "DigitalAssetProcessor: processing file #{file}"
+      open_file = File.open(file, 'rb')
+      Daengine::TeamsiteMetadataParser.parse_tuple_file(open_file)
+      p "DigitalAssetProcessor: finished parsing #{filename}."
+      @@last_read_time = File.mtime(file)
+      p "DigitalAssetProcessor: set last read time to #{@@last_read_time}."
+    end
+
+  end
+  
+end
+end

data/lib/daengine/engine.rb

@@ -1,4 +1,4 @@
 module Daengine
-  class Engine < ::Rails::Engine
-  end
+ class Engine < ::Rails::Engine
+ end
 end

data/lib/daengine/teamsite_metadata_parser.rb

@@ -0,0 +1,101 @@
+# https://github.com/AustinBlues/RSS-Speed-Reader/blob/master/lib/rss_speed_reader.rb
+# https://github.com/tenderlove/nokogiri/pull/524
+
+module Daengine::TeamsiteMetadataParser
+
+  @@translation = {
+      'TeamSite/Metadata/web_title' => 'title',
+      "TeamSite/Metadata/enterprise_last_updated_date"=> 'changed_at',
+      "TeamSite/Metadata/enterprise_audience_id"=> 'audiences',
+      "TeamSite/Metadata/enterprise_sami_desc"=> 'sami_code',
+      "TeamSite/Metadata/enterprise_last_publication_date"=> 'published_at',
+      "TeamSite/Metadata/enterprise_unpublish_date"=> 'unpublished_at',
+      "TeamSite/Metadata/enterprise_expiration_date"=> 'expires_at',
+      "TeamSite/Metadata/enterprise_guid" => 'guid',
+      "TeamSite/Metadata/shortSynopsis" => 'summary',
+      "TeamSite/Metadata/business_owner" => 'business_owner',
+      "TeamSite/Metadata/enterprise_product_id" => 'product_ids'
+  }
+  @@path_tuples = {
+      "path" => 'path',
+      "TeamSite/Metadata/enterprise_last_content_update_date" => 'doc_changed_at',
+      "TeamSite/Metadata/enterprise_content_type_id" => 'content_type'
+  }
+  @@validations = {
+      "TeamSite/Metadata/display_on_website" => lambda { |val| /Y|1/ =~ val },
+      "TeamSite/Metadata/enterprise_expiration_date" => lambda { |val| Time.parse(val) > 1.minute.from_now },
+      # "TeamSite/Metadata/enterprise_unpublish_date" => lambda {|val| val.blank? },
+      "path" => lambda {|val| !(/\/manifest\// =~ val) }
+  }
+
+  @@logger = nil
+
+  def self.logger=(some_logger)
+    @@logger = some_logger
+    self
+  end
+
+  def self.log(args)
+    @@logger.error(args) unless @@logger.blank?
+  end
+
+  def self.parse_tuple_file(file)
+    time do
+      asset = nil
+      assets = {}
+      docpath = {}
+      valid = true
+      while (line = file.gets)
+        case line
+        when /<\/?data-tuple>/
+          if (asset.blank?)
+            asset = DigitalAsset.new
+          elsif(valid)
+            assets[asset.guid] ||= asset.attributes # first tuple metadata wins
+            assets[asset.guid]['documents_attributes'] ||= []           
+            assets[asset.guid]['documents_attributes'] << docpath
+            # assets[asset.guid]['_id'] = asset.guid
+            asset = nil; docpath = {}; valid = true;
+          else
+            asset = nil; docpath = {}; valid = true;
+          end
+        when /<tuple-field name="([^"]+)">([^<]+)<\/tuple-field>/
+          if (valid)
+            if @@validations[$1]
+              valid = @@validations[$1].call($2)
+            end
+            if @@path_tuples[$1]
+              docpath[@@path_tuples[$1]] = $2 # if this is one of our keys, 'send' to the method
+            elsif (@@translation[$1])
+              val = asset.send("#{@@translation[$1]}").respond_to?(:[]) ? $2.split(',') : $2
+              asset.send("#{@@translation[$1]}=", val)
+            end
+          end
+        end
+      end
+      # loop thru each doc in the collection, either replace or delete it
+      assets.keys.each do |key|
+        da = nil
+        begin
+          if(!assets[key]['unpublished_at'].blank?)
+             DigitalAsset.where(guid: key).delete_all
+          else
+            da = DigitalAsset.find_or_initialize_by(guid: key)
+            da.documents = []
+            da.update_attributes!(assets[key])
+          end
+        rescue Exception => e
+          p "Unable to save/update DigitalAsset guid=#{da.try(:guid)}, #{da.try(:errors).try(:full_messages)}"
+        end      
+      end
+      DigitalAsset.purge!  # if the purge criteria is met, purge anything not updated
+    end
+  end
+
+  def self.time
+    start = Time.now
+    yield
+    self.log "elapsed time was #{Time.now - start}"
+  end
+
+end

data/lib/daengine/version.rb

@@ -1,3 +1,3 @@
 module Daengine
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

metadata

@@ -2,7 +2,7 @@
 name: daengine
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - sbhatia
@@ -55,17 +55,6 @@ dependencies:
   requirement: *2106
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: activerecord-jdbcsqlite3-adapter
-  version_requirements: &2124 !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-    none: false
-  requirement: *2124
-  prerelease: false
-  type: :development
 description: Daengine handles digital asset processing.
 email:
 - samcojava@yahoo.com
@@ -86,7 +75,9 @@ files:
 - app/views/digital_assets/_form.html.erb
 - config/routes.rb
 - lib/daengine.rb
+- lib/daengine/digital_asset_processor.rb
 - lib/daengine/engine.rb
+- lib/daengine/teamsite_metadata_parser.rb
 - lib/daengine/version.rb
 - lib/tasks/daengine_tasks.rake
 - MIT-LICENSE