daengine 0.6.5 → 0.6.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +2 -0
- data/app/controllers/digital_assets_controller.rb +4 -2
- data/app/models/digital_asset.rb +5 -6
- data/app/models/service_digital_asset.rb +112 -0
- data/app/service/digital_asset_lookup_service.rb +2 -2
- data/bin/process_assets +17 -15
- data/config/routes.rb +1 -0
- data/lib/daengine.rb +4 -2
- data/lib/daengine/digital_asset_processor.rb +8 -2
- data/lib/daengine/http_client.rb +99 -0
- data/lib/daengine/teamsite_metadata_processor.rb +315 -0
- data/lib/daengine/version.rb +1 -1
- data/spec/acceptance/digital_assets_spec.rb +1 -1
- data/spec/dummy/log/test.log +300 -0
- data/spec/factories.rb +25 -5
- data/spec/lib/digital_asset_processor_spec.rb +5 -3
- data/spec/lib/teamsite_metadata_processor_spec.rb +236 -0
- data/spec/mock_data/daengine.yml +1 -1
- data/spec/mock_data/taxonomy/taxonomyengine.yml +1 -1
- data/spec/models/service_digital_asset_spec.rb +136 -0
- metadata +39 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c01b062e993cb4a3d66b78c1d9455434ead7bfae
|
4
|
+
data.tar.gz: 08dbdaa04cd104f02a1640d1e1cac4ea6664d310
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f2230266e28c55ca21829cd95f033db67d9357dfe94aa816ff4c7f13ee3bc2ec20066d0384c9124707069b5dea236102a7d6bea52949606c9e9ca459241e67be
|
7
|
+
data.tar.gz: 1875ea1715d7f2959f533567dea207f11312c42f86c4e55dad64168c472969f47ba2516991f2fd8b75ac281c618e747b8f97a177ef5cc2dddca0e30af353c7f2
|
data/Rakefile
CHANGED
@@ -13,8 +13,10 @@ class DigitalAssetsController < ApplicationController
|
|
13
13
|
#/digital_assets/search/sami_code=92023
|
14
14
|
#/digital_assets/search/sami_code=NE00192&title=Fund%20Prospectus&fund_code=20293
|
15
15
|
def index
|
16
|
-
|
17
|
-
|
16
|
+
search_params = params[:search] || request.query_parameters
|
17
|
+
@digital_assets = search_params ? search_da(search_params) : DigitalAsset.all
|
18
|
+
# respond_with(@digital_assets)
|
19
|
+
render json: @digital_assets
|
18
20
|
end
|
19
21
|
alias :search :index
|
20
22
|
|
data/app/models/digital_asset.rb
CHANGED
@@ -23,6 +23,8 @@ class DigitalAsset
|
|
23
23
|
field :legacy_path, type: String
|
24
24
|
field :doc_changed_at, type: DateTime
|
25
25
|
field :content_type, type: String
|
26
|
+
alias :doctype :content_type
|
27
|
+
alias :doctype_id :content_type
|
26
28
|
field :pages, type: Integer, default: 1
|
27
29
|
field :size, type: String
|
28
30
|
field :mime_type, type: String
|
@@ -40,13 +42,13 @@ class DigitalAsset
|
|
40
42
|
|
41
43
|
|
42
44
|
#Exclude XBRL documents from all queries
|
43
|
-
|
45
|
+
default_scope excludes(:'content_type' => "LDJDCMAIK") #Had to use static value instead of a Constant
|
44
46
|
|
45
47
|
scope :title_is, ->(title) { where(:title => title)}
|
46
48
|
scope :business_owner_is, ->(business_owner) { where(:business_owner => business_owner)}
|
47
49
|
scope :guid_is, ->(guid) { where(:guid => guid)}
|
48
50
|
scope :digital_asset_id_is, ->(id) { where(:digital_asset_id => id)}
|
49
|
-
scope :fund_in, ->(
|
51
|
+
scope :fund_in, ->(fund_codes) {where(:fund_codes.in => fund_codes)}
|
50
52
|
scope :audience_in, ->(audience_id) {where(:audiences.in => audience_id)}
|
51
53
|
scope :audience_investor_approved, -> {where(:audiences.in => [Audience::INVESTOR_APPROVED])}
|
52
54
|
|
@@ -157,12 +159,9 @@ class DigitalAsset
|
|
157
159
|
# pid and TaxonomyTerm.term_id_is(pid)[0].try(:fund_code).try(:rjust, 5, '0')
|
158
160
|
# end
|
159
161
|
|
160
|
-
def
|
162
|
+
def content_type_label
|
161
163
|
TaxonomyTerm.label_for_term(content_type_id)
|
162
164
|
end
|
163
|
-
def content_type_id
|
164
|
-
:content_type
|
165
|
-
end
|
166
165
|
|
167
166
|
def audience
|
168
167
|
TaxonomyTerm.label_for_term(audiences[0])
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'time'
|
2
|
+
require 'active_record'
|
3
|
+
require 'activerecord-tableless'
|
4
|
+
|
5
|
+
class ServiceDigitalAsset < ActiveRecord::Base
|
6
|
+
has_no_table
|
7
|
+
|
8
|
+
column :title, :string
|
9
|
+
column :changed_at, :time
|
10
|
+
column :audiences, :array, []
|
11
|
+
column :sami_code, :string
|
12
|
+
column :product_ids, :array, []
|
13
|
+
column :published_at, :time
|
14
|
+
column :unpublished_at, :time
|
15
|
+
column :expires_at, :time
|
16
|
+
column :guid, :string
|
17
|
+
column :business_owner, :string
|
18
|
+
column :summary, :string
|
19
|
+
column :content_organization_ids, :array, []
|
20
|
+
column :program_ids, :array, []
|
21
|
+
|
22
|
+
column :omniture_codes, :array, []
|
23
|
+
column :orderable, :boolean, false
|
24
|
+
# refactor version:
|
25
|
+
column :path, :string
|
26
|
+
column :legacy_path, :string
|
27
|
+
column :doc_changed_at, :time
|
28
|
+
column :content_type, :string
|
29
|
+
column :pages, :integer, 0
|
30
|
+
column :size, :string
|
31
|
+
column :mime_type, :string
|
32
|
+
column :subject, :string
|
33
|
+
column :keywords, :array, []
|
34
|
+
column :author, :string
|
35
|
+
column :finra_path, :string
|
36
|
+
column :fund_codes, :array, []
|
37
|
+
column :display_on_website, :boolean
|
38
|
+
column :digital_asset_id, :string
|
39
|
+
|
40
|
+
validates_presence_of :digital_asset_id, :title, :changed_at, :published_at,
|
41
|
+
:expires_at, :audiences, :path
|
42
|
+
validate :validate_future_expiration
|
43
|
+
|
44
|
+
def validate_future_expiration
|
45
|
+
errors.add(:expires_at, 'Expiration date must be at least 1 minute from now') unless expires_at and expires_at > 1.minute.from_now
|
46
|
+
end
|
47
|
+
|
48
|
+
def effective?
|
49
|
+
display_on_website && !expires_at.blank?
|
50
|
+
end
|
51
|
+
|
52
|
+
def expired?
|
53
|
+
expires_at.nil? || expires_at < 1.minute.from_now
|
54
|
+
end
|
55
|
+
|
56
|
+
def finra?
|
57
|
+
DigitalAsset::ContentType::FINRA == content_type
|
58
|
+
end
|
59
|
+
|
60
|
+
def manifest_file?
|
61
|
+
path.match('\/manifest\/')
|
62
|
+
end
|
63
|
+
|
64
|
+
def file_name
|
65
|
+
path.split('/').last
|
66
|
+
end
|
67
|
+
|
68
|
+
def delete?
|
69
|
+
!(unpublished_at.nil?) || expired?
|
70
|
+
end
|
71
|
+
|
72
|
+
def mark_for_deletion
|
73
|
+
# not sure why unpublished_at = value didn't work in spec
|
74
|
+
write_attribute(:unpublished_at, DateTime.now)
|
75
|
+
end
|
76
|
+
|
77
|
+
def as_hash
|
78
|
+
hash = attributes
|
79
|
+
# remove the file attribute fields that have default values
|
80
|
+
# as the service won't update them if they aren't in the data being sent over
|
81
|
+
%w(pages size mime_type subject keywords author).each do |field_name|
|
82
|
+
hash.delete(field_name) if hash[field_name] == ServiceDigitalAsset.default_value(field_name)
|
83
|
+
end
|
84
|
+
# convert the times to strings unless they are nil
|
85
|
+
%w(changed_at published_at unpublished_at expires_at doc_changed_at).each do |field_name|
|
86
|
+
hash[field_name] = hash[field_name].to_s unless hash[field_name].nil?
|
87
|
+
end
|
88
|
+
# service expects 'digital_asset' not 'service_digital_asset'
|
89
|
+
{'digital_asset' => hash}
|
90
|
+
end
|
91
|
+
|
92
|
+
def default_blank_time(time_field)
|
93
|
+
time = send(time_field.to_sym)
|
94
|
+
time.blank? ? 10.years.ago : time
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.boolean_field?(field_name)
|
98
|
+
columns_hash[field_name].sql_type == 'boolean'
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.time_field?(field_name)
|
102
|
+
%w(time datetime).include?(columns_hash[field_name].sql_type)
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.array_field?(field_name)
|
106
|
+
columns_hash[field_name].sql_type == 'array'
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.default_value(field_name)
|
110
|
+
column_defaults[field_name]
|
111
|
+
end
|
112
|
+
end
|
@@ -17,7 +17,7 @@ class DigitalAssetLookupService
|
|
17
17
|
# end
|
18
18
|
|
19
19
|
def self.fund_code_from_id(taxonomy_id)
|
20
|
-
TaxonomyTerm.term_id_is(taxonomy_id).try(:term_type).try([], 'FUND_CODE')
|
20
|
+
TaxonomyTerm.term_id_is(taxonomy_id).first.try(:term_type).try(:[], 'FUND_CODE')
|
21
21
|
end
|
22
22
|
|
23
23
|
# def self.term_id_from_fund_code(code)
|
@@ -44,7 +44,7 @@ class DigitalAssetLookupService
|
|
44
44
|
# query[:product_ids.in] = product_ids if !product_ids.blank?
|
45
45
|
# query[:'documents.content_type'.in] = content_type_ids if !content_type_ids.blank?
|
46
46
|
# query[:audiences.in] = audience_ids if !audience_ids.blank?
|
47
|
-
|
47
|
+
|
48
48
|
# digital_assets = DigitalAsset.where(query)
|
49
49
|
# digital_assets = digital_assets.select {|d| d.has_finra?} if finra
|
50
50
|
# digital_assets
|
data/bin/process_assets
CHANGED
@@ -1,15 +1,17 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
#
|
3
|
-
# This file was generated by Bundler.
|
4
|
-
#
|
5
|
-
# The application 'process_assets' is installed as part of a gem, and
|
6
|
-
# this file is here to facilitate running it.
|
7
|
-
#
|
8
|
-
|
9
|
-
require 'daengine'
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# This file was generated by Bundler.
|
4
|
+
#
|
5
|
+
# The application 'process_assets' is installed as part of a gem, and
|
6
|
+
# this file is here to facilitate running it.
|
7
|
+
#
|
8
|
+
|
9
|
+
require 'daengine'
|
10
|
+
|
11
|
+
puts "***** #{RUBY_VERSION}"
|
12
|
+
|
13
|
+
config = YAML.load_file(ARGV[0])
|
14
|
+
|
15
|
+
t = Daengine.execute(config)
|
16
|
+
|
17
|
+
puts t
|
data/config/routes.rb
CHANGED
data/lib/daengine.rb
CHANGED
@@ -4,7 +4,8 @@ require 'daengine/railtie' if defined?(Rails)
|
|
4
4
|
require File.expand_path('../../app/models/digital_asset',__FILE__)
|
5
5
|
require File.expand_path('../../app/models/teamsite_digital_asset_shim',__FILE__)
|
6
6
|
require File.expand_path('../../app/models/taxonomy_term',__FILE__)
|
7
|
-
require File.expand_path('../../app/models/
|
7
|
+
require File.expand_path('../../app/models/service_digital_asset',__FILE__)
|
8
|
+
require File.expand_path('../../app/service/digital_asset_lookup_service',__FILE__)
|
8
9
|
require 'mini_exiftool'
|
9
10
|
require 'daengine/teamsite_metadata_parser'
|
10
11
|
require 'daengine/digital_asset_processor'
|
@@ -20,7 +21,8 @@ module Daengine
|
|
20
21
|
@config = {
|
21
22
|
:assets_path => '/digital-assets',
|
22
23
|
:taxonomy_xml_filepath => '/taxonomy.xml',
|
23
|
-
:content_service_url => 'http://ofiwsqa.den.ofi.com/'
|
24
|
+
:content_service_url => 'http://ofiwsqa.den.ofi.com/',
|
25
|
+
:disable_file_check => false
|
24
26
|
}
|
25
27
|
|
26
28
|
@mongoid_config = {
|
@@ -1,3 +1,6 @@
|
|
1
|
+
#require File.expand_path("../teamsite_metadata_processor", __FILE__)
|
2
|
+
require 'daengine/teamsite_metadata_processor'
|
3
|
+
|
1
4
|
module Daengine
|
2
5
|
class DigitalAssetProcessor
|
3
6
|
|
@@ -35,13 +38,16 @@ module Daengine
|
|
35
38
|
file = File.expand_path(filename, path)
|
36
39
|
Daengine.log("DigitalAssetProcessor: Processing file #{filename} --- #{File.mtime(file)}", "info")
|
37
40
|
open_file = File.open(file, 'rb')
|
38
|
-
Daengine::TeamsiteMetadataParser.parse_tuple_file(open_file, last_run_time)
|
39
|
-
Daengine.log("DigitalAssetProcessor: Finished processing #{filename}", "info")
|
41
|
+
#Daengine::TeamsiteMetadataParser.parse_tuple_file(open_file, last_run_time)
|
42
|
+
#Daengine.log("DigitalAssetProcessor: Finished processing #{filename}", "info")
|
43
|
+
Daengine::TeamsiteMetadataProcessor.process_tuple_file(open_file, last_run_time)
|
44
|
+
Daengine.log("TeamsiteMetadataProcessor: Finished processing #{filename}", "info")
|
40
45
|
@@last_read_time = File.mtime(file) + 1.second
|
41
46
|
self.save_last_read_time
|
42
47
|
Daengine.log("DigitalAssetProcessor: Last process time set to #{@@last_read_time}", "info")
|
43
48
|
rescue
|
44
49
|
Daengine.log("Unable to process file #{filename}, #{$!.message}", "error")
|
50
|
+
Daengine.log($!.backtrace.join, "error")
|
45
51
|
end
|
46
52
|
end
|
47
53
|
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
#
|
2
|
+
# Lifted from eDist Gateway::HTTP::Client
|
3
|
+
#
|
4
|
+
require 'httparty'
|
5
|
+
|
6
|
+
module Daengine
|
7
|
+
module HTTP
|
8
|
+
module Client
|
9
|
+
SLOW_CALL_THRESHOLD = 5
|
10
|
+
# Available options
|
11
|
+
# :timeout => Timeout is specified in seconds.
|
12
|
+
# :headers - contains string hash such has content-type: {'content-type' => 'application/xml'}
|
13
|
+
# :body => Body to post.
|
14
|
+
#
|
15
|
+
def self.call(path, options = {})
|
16
|
+
begin
|
17
|
+
method = options.delete(:method) || :get
|
18
|
+
|
19
|
+
htoptions = {}
|
20
|
+
htoptions[:timeout] = options.delete(:timeout) || 10
|
21
|
+
headers = options.delete(:headers) || {}
|
22
|
+
headers['rake_task_name'] = (Thread.current[:rake_task_name] || 'unknown').to_s
|
23
|
+
headers['visitor_id'] = (Thread.current[:visitor_id] || 'unknown').to_s
|
24
|
+
headers['uuid'] ||= (Thread.current[:uuid] || 'unknown').to_s
|
25
|
+
htoptions[:headers] = headers
|
26
|
+
|
27
|
+
case method
|
28
|
+
when :get
|
29
|
+
query = options[:query] || options[:parameters] || options
|
30
|
+
htoptions[:query] = query unless query.blank?
|
31
|
+
when :post, :put, :delete
|
32
|
+
htoptions[:query] = options[:query] if options[:query]
|
33
|
+
body = options[:body] || options[:parameters] || options
|
34
|
+
htoptions[:body] = body
|
35
|
+
else
|
36
|
+
raise ArgumentError.new('you must specify a method of either :get, :post, :put, or :delete')
|
37
|
+
end
|
38
|
+
|
39
|
+
start = Time.now
|
40
|
+
resp = HTTParty.send(method.to_sym, path, htoptions)
|
41
|
+
elapsed = Time.now - start
|
42
|
+
Daengine.log("***** Daengine::HTTP::Client.call(#{method}, #{path}, #{htoptions}) TOOK #{elapsed} seconds +++++++++++++++++++++",
|
43
|
+
elapsed > SLOW_CALL_THRESHOLD ? 'warn' : 'info')
|
44
|
+
|
45
|
+
rescue => ex
|
46
|
+
Daengine.log("Error calling service (#{path}) - #{ex.inspect}\n#{ex.backtrace.join("\n\t")}", 'error')
|
47
|
+
#raise ex
|
48
|
+
end
|
49
|
+
Response.new(resp)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
class Response
|
54
|
+
def initialize(response)
|
55
|
+
@resp = response
|
56
|
+
end
|
57
|
+
|
58
|
+
def headers
|
59
|
+
@resp.headers
|
60
|
+
end
|
61
|
+
|
62
|
+
def body
|
63
|
+
@resp.body
|
64
|
+
end
|
65
|
+
|
66
|
+
def status
|
67
|
+
@resp.code
|
68
|
+
end
|
69
|
+
|
70
|
+
def as_hash
|
71
|
+
@resp.parsed_response
|
72
|
+
end
|
73
|
+
|
74
|
+
alias :code :status
|
75
|
+
alias :status_code :status
|
76
|
+
|
77
|
+
def success?
|
78
|
+
!!(status.to_s =~ /^2/)
|
79
|
+
end
|
80
|
+
|
81
|
+
def failed?
|
82
|
+
!success?
|
83
|
+
end
|
84
|
+
|
85
|
+
def server_error?
|
86
|
+
!!(status.to_s =~ /^5/)
|
87
|
+
end
|
88
|
+
|
89
|
+
def client_error?
|
90
|
+
!!(status.to_s =~ /^4/)
|
91
|
+
end
|
92
|
+
|
93
|
+
def redirection?
|
94
|
+
!!(status.to_s =~ /^3/)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
end
|
99
|
+
end
|
@@ -0,0 +1,315 @@
|
|
1
|
+
require 'nokogiri'
|
2
|
+
require 'mini_exiftool'
|
3
|
+
require 'daengine/http_client'
|
4
|
+
|
5
|
+
module Daengine
|
6
|
+
module TeamsiteMetadataProcessor
|
7
|
+
|
8
|
+
def self.process_tuple_file(file, last_read = nil)
|
9
|
+
verify_pre_conditions
|
10
|
+
|
11
|
+
time do
|
12
|
+
assets = parse_file(file)
|
13
|
+
assets = select_1_asset_per_id(assets)
|
14
|
+
assets = add_file_attributes(assets, last_read) unless Daengine.config[:disable_file_check]
|
15
|
+
assets = add_fund_codes(assets)
|
16
|
+
summary = call_service(assets)
|
17
|
+
Daengine.log("***** Summary = #{summary.inspect}", 'info')
|
18
|
+
end
|
19
|
+
rescue => ex
|
20
|
+
Daengine.log("Error processing XML file - #{ex.inspect}", 'error')
|
21
|
+
Daengine.log(ex.backtrace.join, 'error')
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.verify_pre_conditions
|
25
|
+
path = Daengine.config[:digital_assets_file_directory]
|
26
|
+
raise "Unable to locate digital assets at #{path}" unless File.directory?(path) || Daengine.config[:disable_file_check]
|
27
|
+
|
28
|
+
service_uri = Daengine.config[:digital_asset_service_url]
|
29
|
+
raise 'No digital asset service URL set' if service_uri.blank?
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.parse_file(file)
|
33
|
+
puts "----- Parsing the file -----"
|
34
|
+
document = Document.new
|
35
|
+
Nokogiri::XML::SAX::Parser.new(document).parse(file)
|
36
|
+
Daengine.log("Nokogiri Parser complete...", "info")
|
37
|
+
document.assets
|
38
|
+
end
|
39
|
+
|
40
|
+
def self.select_1_asset_per_id(assets)
|
41
|
+
results = {}
|
42
|
+
assets.each do |key, values|
|
43
|
+
puts "----- select_1 #{key} -----"
|
44
|
+
list = values.find_all { |v| v.effective? }
|
45
|
+
asset = most_recent_non_expired(list)
|
46
|
+
asset.finra_path = finra_path(list) unless asset.nil?
|
47
|
+
results[key] = asset unless asset.nil?
|
48
|
+
end
|
49
|
+
results
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.add_file_attributes(assets, last_read)
|
53
|
+
results = {}
|
54
|
+
assets.each do |key, asset|
|
55
|
+
begin
|
56
|
+
puts "----- add_file #{key} -----"
|
57
|
+
file_name = asset_file_name(asset)
|
58
|
+
asset.mark_for_deletion unless File.file?(file_name)
|
59
|
+
|
60
|
+
set_asset_file_attributes(file_name, asset, last_read) unless asset.delete?
|
61
|
+
|
62
|
+
results[key] = asset
|
63
|
+
rescue => ex
|
64
|
+
Daengine.log("***** Error processing asset with file name = #{asset.path} - #{ex.inspect}", 'error')
|
65
|
+
end
|
66
|
+
end
|
67
|
+
results
|
68
|
+
end
|
69
|
+
|
70
|
+
def self.add_fund_codes(assets)
|
71
|
+
results = {}
|
72
|
+
assets.each do |key, asset|
|
73
|
+
puts "----- add_fund_codes #{key} -----"
|
74
|
+
fund_codes = []
|
75
|
+
asset.product_ids.each do |product_id|
|
76
|
+
fund_code = DigitalAssetLookupService.fund_code_from_id(product_id)
|
77
|
+
fund_codes << fund_code.strip.rjust(5, '0') unless fund_code.blank?
|
78
|
+
end
|
79
|
+
asset.fund_codes = fund_codes unless fund_codes.empty?
|
80
|
+
results[key] = asset
|
81
|
+
end
|
82
|
+
results
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.call_service(assets)
|
86
|
+
results = {:errors => 0, :updated => 0, :deleted => 0}
|
87
|
+
assets.each_value do |asset|
|
88
|
+
begin
|
89
|
+
puts "----- call_service #{asset.digital_asset_id} -----"
|
90
|
+
if asset.delete?
|
91
|
+
path = "#{service_uri}/#{asset.digital_asset_id}"
|
92
|
+
options = {:method => :delete, :headers => header}
|
93
|
+
operation = :deleted
|
94
|
+
else
|
95
|
+
path = "#{service_uri}"
|
96
|
+
options = {:method => :post,
|
97
|
+
:query => asset.as_hash,
|
98
|
+
:headers => header}
|
99
|
+
operation = :updated
|
100
|
+
end
|
101
|
+
|
102
|
+
response = Daengine::HTTP::Client.call(path, options)
|
103
|
+
results[operation] += 1 if response.success?
|
104
|
+
results[:errors] += 1 unless response.success?
|
105
|
+
rescue => ex
|
106
|
+
Daengine.log("***** Error calling service for #{asset.inspect} - #{ex.inspect}", 'error')
|
107
|
+
results[:errors] += 1
|
108
|
+
end
|
109
|
+
end
|
110
|
+
results
|
111
|
+
end
|
112
|
+
|
113
|
+
def self.time
|
114
|
+
start = Time.now
|
115
|
+
yield
|
116
|
+
Daengine.log("***** Elapsed time was #{Time.now - start}", 'info')
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
def self.most_recent_non_expired(list)
|
121
|
+
list.inject do |previous, current|
|
122
|
+
prev_published_at = previous.default_blank_time(:published_at)
|
123
|
+
current_published_at = current.default_blank_time(:published_at)
|
124
|
+
!current.expired? &&
|
125
|
+
!current.manifest_file? &&
|
126
|
+
!current.finra? &&
|
127
|
+
current_published_at >= prev_published_at ? current : previous
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
def self.finra_path(list)
|
132
|
+
finra = list.find { |value| value.finra? }
|
133
|
+
finra.try(:path)
|
134
|
+
end
|
135
|
+
|
136
|
+
def self.asset_file_name(asset)
|
137
|
+
name = File.join(file_directory, asset.path)
|
138
|
+
name = File.join(file_directory, asset.file_name) unless File.exists?(name)
|
139
|
+
name
|
140
|
+
end
|
141
|
+
|
142
|
+
def self.set_asset_file_attributes(file_name, asset, last_read)
|
143
|
+
if File.mtime(file_name) > last_read
|
144
|
+
update_asset_file_attributes(file_name, asset)
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def self.update_asset_file_attributes(file_name, asset)
|
149
|
+
exifdata = ::MiniExiftool.new(file_name)
|
150
|
+
pages = exifdata.pagecount
|
151
|
+
pages = exifdata.pages if pages.blank?
|
152
|
+
pages = exifdata.slides if pages.blank?
|
153
|
+
asset.pages = pages
|
154
|
+
asset.size = exifdata.filesize
|
155
|
+
asset.mime_type = exifdata.mimetype
|
156
|
+
asset.author = exifdata.author
|
157
|
+
if exifdata.keywords.is_a? Enumerable
|
158
|
+
asset.keywords = exifdata.keywords
|
159
|
+
else
|
160
|
+
asset.keywords = exifdata.keywords.gsub(';', ',').gsub(':', ',').split(',') unless exifdata.keywords.nil?
|
161
|
+
end
|
162
|
+
if exifdata.description.is_a? Enumerable
|
163
|
+
asset.subject = exifdata.description.join(' ') unless exifdata.description.nil?
|
164
|
+
else
|
165
|
+
asset.subject = exifdata.description.gsub(':', '') unless exifdata.description.nil?
|
166
|
+
end
|
167
|
+
end
|
168
|
+
|
169
|
+
def self.header
|
170
|
+
{'Accept' => 'application/json',
|
171
|
+
'Accept-Encoding' => 'gzip;q=1.0,deflate;q=0.6,identity;q=0.3',
|
172
|
+
'User-Agent' => 'Ruby'}
|
173
|
+
end
|
174
|
+
|
175
|
+
def self.service_uri
|
176
|
+
Daengine.config[:digital_asset_service_url]
|
177
|
+
end
|
178
|
+
|
179
|
+
def self.file_directory
|
180
|
+
Daengine.config[:digital_assets_file_directory]
|
181
|
+
end
|
182
|
+
end
|
183
|
+
|
184
|
+
|
185
|
+
class Document < Nokogiri::XML::SAX::Document
|
186
|
+
DATA_TUPLE_ELEMENT = 'data-tuple'
|
187
|
+
TUPLE_FIELD_ELEMENT = 'tuple-field'
|
188
|
+
NAME_ATTRIBUTE = 'name'
|
189
|
+
|
190
|
+
# Teamsite Date example: Dec 29 2008 12:00:00:000AM
|
191
|
+
TIME_FORMAT = '%b %d %Y %k:%M:%S:%L%p'
|
192
|
+
|
193
|
+
TRANSLATION = {
|
194
|
+
'TeamSite/Metadata/rttTitle' => 'title',
|
195
|
+
'TeamSite/Metadata/enterprise_last_updated_date' => 'changed_at',
|
196
|
+
'TeamSite/Metadata/enterprise_audience_id' => 'audiences',
|
197
|
+
'TeamSite/Metadata/enterprise_sami_desc' => 'sami_code',
|
198
|
+
'TeamSite/Metadata/enterprise_last_publication_date' => 'published_at',
|
199
|
+
'TeamSite/Metadata/enterprise_unpublish_date' => 'unpublished_at',
|
200
|
+
'TeamSite/Metadata/enterprise_expiration_date' => 'expires_at',
|
201
|
+
#'TeamSite/Metadata/enterprise_guid' => 'guid',
|
202
|
+
'TeamSite/Metadata/enterprise_guid' => 'digital_asset_id',
|
203
|
+
'TeamSite/Metadata/shortSynopsis' => 'summary',
|
204
|
+
'TeamSite/Metadata/business_owner' => 'business_owner',
|
205
|
+
'TeamSite/Metadata/enterprise_product_id' => 'product_ids',
|
206
|
+
'TeamSite/Metadata/enterprise_content_organization_id' => 'content_organization_ids',
|
207
|
+
'TeamSite/Metadata/enterprise_program_id' => 'program_ids',
|
208
|
+
'TeamSite/Metadata/omnitureSiteSection_codes' => 'omniture_codes',
|
209
|
+
'TeamSite/Metadata/display_on_website' => 'display_on_website',
|
210
|
+
'path' => 'path',
|
211
|
+
'TeamSite/Metadata/enterprise_last_content_update_date' => 'doc_changed_at',
|
212
|
+
'TeamSite/Metadata/enterprise_content_type_id' => 'content_type'
|
213
|
+
}
|
214
|
+
|
215
|
+
def initialize
|
216
|
+
@asset = nil
|
217
|
+
@field_name = nil
|
218
|
+
@text = ''
|
219
|
+
@assets = {}
|
220
|
+
end
|
221
|
+
|
222
|
+
def assets
|
223
|
+
@assets
|
224
|
+
end
|
225
|
+
|
226
|
+
def start_element(name, attrs = [])
|
227
|
+
case name
|
228
|
+
when DATA_TUPLE_ELEMENT
|
229
|
+
@asset = ::ServiceDigitalAsset.new
|
230
|
+
@field_name = nil
|
231
|
+
when TUPLE_FIELD_ELEMENT
|
232
|
+
@field_name = translate_field_name(name_attr_value(attrs))
|
233
|
+
end
|
234
|
+
|
235
|
+
@text = ''
|
236
|
+
end
|
237
|
+
|
238
|
+
def characters(string)
|
239
|
+
@text << string
|
240
|
+
end
|
241
|
+
|
242
|
+
def end_element(name)
|
243
|
+
case name
|
244
|
+
when DATA_TUPLE_ELEMENT
|
245
|
+
@assets[@asset.digital_asset_id] = [] if @assets[@asset.digital_asset_id].nil?
|
246
|
+
@assets[@asset.digital_asset_id] << @asset
|
247
|
+
@asset = nil
|
248
|
+
when TUPLE_FIELD_ELEMENT
|
249
|
+
unless @field_name.nil?
|
250
|
+
value = convert_value(@field_name, @text)
|
251
|
+
setter_name = "#{@field_name}=".to_sym
|
252
|
+
@asset.send(setter_name, value) if @asset.respond_to?(setter_name)
|
253
|
+
end
|
254
|
+
end
|
255
|
+
|
256
|
+
@field_name, @text = nil, ''
|
257
|
+
end
|
258
|
+
|
259
|
+
def error(string)
|
260
|
+
Daengine.log("***** Parse error - #{string} *****", 'error')
|
261
|
+
end
|
262
|
+
|
263
|
+
def warning(string)
|
264
|
+
Daengine.log("***** Parse warning - #{string} *****", 'warn')
|
265
|
+
end
|
266
|
+
|
267
|
+
|
268
|
+
def convert_value(field_name, string)
|
269
|
+
begin
|
270
|
+
return convert_to_boolean(string) if boolean_field?(field_name)
|
271
|
+
return convert_to_datetime(string) if datetime_field?(field_name)
|
272
|
+
return convert_to_array(string) if array_field?(field_name)
|
273
|
+
rescue => ex
|
274
|
+
Daengine.log("***** convert-value('#{field_name}', '#{string}') - #{ex.inspect}", 'error')
|
275
|
+
end
|
276
|
+
|
277
|
+
string
|
278
|
+
end
|
279
|
+
|
280
|
+
def boolean_field?(field_name)
|
281
|
+
::ServiceDigitalAsset.boolean_field?(field_name)
|
282
|
+
end
|
283
|
+
|
284
|
+
def datetime_field?(field_name)
|
285
|
+
::ServiceDigitalAsset.time_field?(field_name)
|
286
|
+
end
|
287
|
+
|
288
|
+
def array_field?(field_name)
|
289
|
+
::ServiceDigitalAsset.array_field?(field_name)
|
290
|
+
end
|
291
|
+
|
292
|
+
def convert_to_boolean(string)
|
293
|
+
!!(string =~ /Y|1|true/)
|
294
|
+
end
|
295
|
+
|
296
|
+
def convert_to_datetime(string)
|
297
|
+
Time.strptime(string, TIME_FORMAT) unless string.blank?
|
298
|
+
end
|
299
|
+
|
300
|
+
def convert_to_array(string)
|
301
|
+
string.try(:split, ',')
|
302
|
+
end
|
303
|
+
|
304
|
+
def name_attr_value(attrs)
|
305
|
+
unless attrs.nil?
|
306
|
+
hash = Hash[*attrs.flatten]
|
307
|
+
hash[NAME_ATTRIBUTE]
|
308
|
+
end
|
309
|
+
end
|
310
|
+
|
311
|
+
def translate_field_name(name)
|
312
|
+
TRANSLATION[name]
|
313
|
+
end
|
314
|
+
end
|
315
|
+
end
|