daengine 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- data/app/models/digital_asset.rb +2 -0
- data/lib/daengine.rb +34 -2
- data/lib/daengine/digital_asset_processor.rb +70 -0
- data/lib/daengine/engine.rb +2 -2
- data/lib/daengine/teamsite_metadata_parser.rb +101 -0
- data/lib/daengine/version.rb +1 -1
- metadata +3 -12
data/app/models/digital_asset.rb
CHANGED
data/lib/daengine.rb
CHANGED
@@ -1,6 +1,38 @@
|
|
1
|
-
require "daengine/
|
2
|
-
|
1
|
+
require "daengine/version"
|
2
|
+
require 'daengine/engine'
|
3
|
+
require File.expand_path('../../app/models/digital_asset',__FILE__)
|
4
|
+
require 'daengine/teamsite_metadata_parser'
|
5
|
+
require 'daengine/digital_asset_processor'
|
3
6
|
require 'mongoid'
|
4
7
|
|
5
8
|
module Daengine
|
9
|
+
@config = {
|
10
|
+
}
|
11
|
+
|
12
|
+
@mongoid_config = {
|
13
|
+
'database' => 'ssc_assets', # mongoid database name
|
14
|
+
'host' => 'localhost', # mongoid server
|
15
|
+
'port' => 27017, # mongodb server port
|
16
|
+
}
|
17
|
+
|
18
|
+
@valid_mongoid_keys = @mongoid_config.keys
|
19
|
+
|
20
|
+
# yaml file config
|
21
|
+
def self.configure(config_options)
|
22
|
+
config_options.each {|k,v| @config[k.to_sym] = v}
|
23
|
+
config_options.each {|k,v| @mongoid_config[k.to_sym] = v if @valid_mongoid_keys.include? k.to_sym}
|
24
|
+
Mongoid.configure do |config|
|
25
|
+
config.from_hash(@mongoid_config)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.config
|
30
|
+
@config
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.execute(config_options)
|
34
|
+
self.configure(config_options)
|
35
|
+
return DigitalAssetProcessor.execute # start the thread daemon
|
36
|
+
end
|
37
|
+
|
6
38
|
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
module Daengine
|
2
|
+
class DigitalAssetProcessor
|
3
|
+
|
4
|
+
@@last_read_time = 2.days.ago
|
5
|
+
|
6
|
+
def self.trap_signals
|
7
|
+
sigtrap = proc {
|
8
|
+
puts "DigitalAssetProcessor: caught trapped signal, shutting down"
|
9
|
+
@@run = false
|
10
|
+
}
|
11
|
+
signals = ["SIGTERM", "SIGINT"]
|
12
|
+
signals.push("SIGHUP") unless is_windows?
|
13
|
+
signals.each do |signal|
|
14
|
+
trap signal, sigtrap
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def self.is_windows?
|
19
|
+
processor, platform, *rest = RUBY_PLATFORM.split("-")
|
20
|
+
platform =~ /mswin/ || platform =~ /mingw/
|
21
|
+
end
|
22
|
+
|
23
|
+
def self.execute
|
24
|
+
@@run = true
|
25
|
+
trap_signals
|
26
|
+
@@wthread = Thread.new { worker() }
|
27
|
+
return @@wthread
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.worker
|
31
|
+
puts "DigitalAssetProcessor: start processing digital assets!"
|
32
|
+
while @@run
|
33
|
+
begin
|
34
|
+
self.process_tuple_directory
|
35
|
+
process_tuple_directory
|
36
|
+
rescue => e
|
37
|
+
puts e.message
|
38
|
+
puts e.backtrace
|
39
|
+
end
|
40
|
+
sleep(5)
|
41
|
+
end
|
42
|
+
@@wthread.exit
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.process_tuple_directory
|
46
|
+
path = Daengine.config[:assets_path]
|
47
|
+
raise "DigitalAssetProcessor: unable to read from asset path #{path}" unless File::directory?(path)
|
48
|
+
# read the given directory, process each file in date order starting 2 days ago
|
49
|
+
deploy_files= []
|
50
|
+
|
51
|
+
puts "DigitalAssetProcessor: reading digital asset deployment files from #{path}"
|
52
|
+
deploy_files = Dir.entries(path).select {
|
53
|
+
|f| File.file?("#{path}/#{f}") and File.mtime("#{path}/#{f}") > @@last_read_time
|
54
|
+
}.sort_by{ |f| File.mtime("#{path}/#{f}") }
|
55
|
+
|
56
|
+
deploy_files.each do |filename|
|
57
|
+
#parse the file and add content to database.
|
58
|
+
file = File.expand_path(filename, path)
|
59
|
+
p "DigitalAssetProcessor: processing file #{file}"
|
60
|
+
open_file = File.open(file, 'rb')
|
61
|
+
Daengine::TeamsiteMetadataParser.parse_tuple_file(open_file)
|
62
|
+
p "DigitalAssetProcessor: finished parsing #{filename}."
|
63
|
+
@@last_read_time = File.mtime(file)
|
64
|
+
p "DigitalAssetProcessor: set last read time to #{@@last_read_time}."
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
68
|
+
|
69
|
+
end
|
70
|
+
end
|
data/lib/daengine/engine.rb
CHANGED
@@ -0,0 +1,101 @@
|
|
1
|
+
# https://github.com/AustinBlues/RSS-Speed-Reader/blob/master/lib/rss_speed_reader.rb
|
2
|
+
# https://github.com/tenderlove/nokogiri/pull/524
|
3
|
+
|
4
|
+
module Daengine::TeamsiteMetadataParser
|
5
|
+
|
6
|
+
@@translation = {
|
7
|
+
'TeamSite/Metadata/web_title' => 'title',
|
8
|
+
"TeamSite/Metadata/enterprise_last_updated_date"=> 'changed_at',
|
9
|
+
"TeamSite/Metadata/enterprise_audience_id"=> 'audiences',
|
10
|
+
"TeamSite/Metadata/enterprise_sami_desc"=> 'sami_code',
|
11
|
+
"TeamSite/Metadata/enterprise_last_publication_date"=> 'published_at',
|
12
|
+
"TeamSite/Metadata/enterprise_unpublish_date"=> 'unpublished_at',
|
13
|
+
"TeamSite/Metadata/enterprise_expiration_date"=> 'expires_at',
|
14
|
+
"TeamSite/Metadata/enterprise_guid" => 'guid',
|
15
|
+
"TeamSite/Metadata/shortSynopsis" => 'summary',
|
16
|
+
"TeamSite/Metadata/business_owner" => 'business_owner',
|
17
|
+
"TeamSite/Metadata/enterprise_product_id" => 'product_ids'
|
18
|
+
}
|
19
|
+
@@path_tuples = {
|
20
|
+
"path" => 'path',
|
21
|
+
"TeamSite/Metadata/enterprise_last_content_update_date" => 'doc_changed_at',
|
22
|
+
"TeamSite/Metadata/enterprise_content_type_id" => 'content_type'
|
23
|
+
}
|
24
|
+
@@validations = {
|
25
|
+
"TeamSite/Metadata/display_on_website" => lambda { |val| /Y|1/ =~ val },
|
26
|
+
"TeamSite/Metadata/enterprise_expiration_date" => lambda { |val| Time.parse(val) > 1.minute.from_now },
|
27
|
+
# "TeamSite/Metadata/enterprise_unpublish_date" => lambda {|val| val.blank? },
|
28
|
+
"path" => lambda {|val| !(/\/manifest\// =~ val) }
|
29
|
+
}
|
30
|
+
|
31
|
+
@@logger = nil
|
32
|
+
|
33
|
+
def self.logger=(some_logger)
|
34
|
+
@@logger = some_logger
|
35
|
+
self
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.log(args)
|
39
|
+
@@logger.error(args) unless @@logger.blank?
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.parse_tuple_file(file)
|
43
|
+
time do
|
44
|
+
asset = nil
|
45
|
+
assets = {}
|
46
|
+
docpath = {}
|
47
|
+
valid = true
|
48
|
+
while (line = file.gets)
|
49
|
+
case line
|
50
|
+
when /<\/?data-tuple>/
|
51
|
+
if (asset.blank?)
|
52
|
+
asset = DigitalAsset.new
|
53
|
+
elsif(valid)
|
54
|
+
assets[asset.guid] ||= asset.attributes # first tuple metadata wins
|
55
|
+
assets[asset.guid]['documents_attributes'] ||= []
|
56
|
+
assets[asset.guid]['documents_attributes'] << docpath
|
57
|
+
# assets[asset.guid]['_id'] = asset.guid
|
58
|
+
asset = nil; docpath = {}; valid = true;
|
59
|
+
else
|
60
|
+
asset = nil; docpath = {}; valid = true;
|
61
|
+
end
|
62
|
+
when /<tuple-field name="([^"]+)">([^<]+)<\/tuple-field>/
|
63
|
+
if (valid)
|
64
|
+
if @@validations[$1]
|
65
|
+
valid = @@validations[$1].call($2)
|
66
|
+
end
|
67
|
+
if @@path_tuples[$1]
|
68
|
+
docpath[@@path_tuples[$1]] = $2 # if this is one of our keys, 'send' to the method
|
69
|
+
elsif (@@translation[$1])
|
70
|
+
val = asset.send("#{@@translation[$1]}").respond_to?(:[]) ? $2.split(',') : $2
|
71
|
+
asset.send("#{@@translation[$1]}=", val)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
# loop thru each doc in the collection, either replace or delete it
|
77
|
+
assets.keys.each do |key|
|
78
|
+
da = nil
|
79
|
+
begin
|
80
|
+
if(!assets[key]['unpublished_at'].blank?)
|
81
|
+
DigitalAsset.where(guid: key).delete_all
|
82
|
+
else
|
83
|
+
da = DigitalAsset.find_or_initialize_by(guid: key)
|
84
|
+
da.documents = []
|
85
|
+
da.update_attributes!(assets[key])
|
86
|
+
end
|
87
|
+
rescue Exception => e
|
88
|
+
p "Unable to save/update DigitalAsset guid=#{da.try(:guid)}, #{da.try(:errors).try(:full_messages)}"
|
89
|
+
end
|
90
|
+
end
|
91
|
+
DigitalAsset.purge! # if the purge criteria is met, purge anything not updated
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def self.time
|
96
|
+
start = Time.now
|
97
|
+
yield
|
98
|
+
self.log "elapsed time was #{Time.now - start}"
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
data/lib/daengine/version.rb
CHANGED
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: daengine
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.0.
|
5
|
+
version: 0.0.5
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- sbhatia
|
@@ -55,17 +55,6 @@ dependencies:
|
|
55
55
|
requirement: *2106
|
56
56
|
prerelease: false
|
57
57
|
type: :development
|
58
|
-
- !ruby/object:Gem::Dependency
|
59
|
-
name: activerecord-jdbcsqlite3-adapter
|
60
|
-
version_requirements: &2124 !ruby/object:Gem::Requirement
|
61
|
-
requirements:
|
62
|
-
- - ! '>='
|
63
|
-
- !ruby/object:Gem::Version
|
64
|
-
version: '0'
|
65
|
-
none: false
|
66
|
-
requirement: *2124
|
67
|
-
prerelease: false
|
68
|
-
type: :development
|
69
58
|
description: Daengine handles digital asset processing.
|
70
59
|
email:
|
71
60
|
- samcojava@yahoo.com
|
@@ -86,7 +75,9 @@ files:
|
|
86
75
|
- app/views/digital_assets/_form.html.erb
|
87
76
|
- config/routes.rb
|
88
77
|
- lib/daengine.rb
|
78
|
+
- lib/daengine/digital_asset_processor.rb
|
89
79
|
- lib/daengine/engine.rb
|
80
|
+
- lib/daengine/teamsite_metadata_parser.rb
|
90
81
|
- lib/daengine/version.rb
|
91
82
|
- lib/tasks/daengine_tasks.rake
|
92
83
|
- MIT-LICENSE
|