sufia-models 6.2.0 → 6.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/app/actors/sufia/generic_file/actor.rb +4 -7
- data/app/jobs/active_fedora_id_based_job.rb +1 -1
- data/app/jobs/active_fedora_pid_based_job.rb +1 -2
- data/app/jobs/audit_job.rb +1 -1
- data/app/jobs/batch_update_job.rb +6 -6
- data/app/jobs/create_derivatives_job.rb +2 -3
- data/app/jobs/import_url_job.rb +8 -5
- data/app/jobs/ingest_local_file_job.rb +1 -1
- data/app/models/batch.rb +15 -21
- data/app/models/checksum_audit_log.rb +0 -1
- data/app/models/concerns/sufia/ability.rb +5 -0
- data/app/models/concerns/sufia/collection_behavior.rb +3 -2
- data/app/models/concerns/sufia/file_stat_utils.rb +19 -21
- data/app/models/concerns/sufia/generic_file/batches.rb +1 -3
- data/app/models/concerns/sufia/generic_file/characterization.rb +11 -16
- data/app/models/concerns/sufia/generic_file/content.rb +0 -1
- data/app/models/concerns/sufia/generic_file/derivatives.rb +2 -2
- data/app/models/concerns/sufia/generic_file/export.rb +50 -59
- data/app/models/concerns/sufia/generic_file/full_text_indexing.rb +15 -18
- data/app/models/concerns/sufia/generic_file/metadata.rb +0 -2
- data/app/models/concerns/sufia/generic_file/mime_types.rb +9 -9
- data/app/models/concerns/sufia/generic_file/permissions.rb +0 -1
- data/app/models/concerns/sufia/generic_file/proxy_deposit.rb +0 -1
- data/app/models/concerns/sufia/generic_file/querying.rb +9 -5
- data/app/models/concerns/sufia/generic_file/trophies.rb +1 -1
- data/app/models/concerns/sufia/generic_file/versions.rb +0 -4
- data/app/models/concerns/sufia/model_methods.rb +0 -1
- data/app/models/concerns/sufia/user.rb +15 -15
- data/app/models/concerns/sufia/user_usage_stats.rb +0 -2
- data/app/models/datastreams/fits_datastream.rb +25 -25
- data/app/models/domain_term.rb +2 -3
- data/app/models/featured_work.rb +3 -5
- data/app/models/file_download_stat.rb +3 -4
- data/app/models/file_usage.rb +10 -11
- data/app/models/file_view_stat.rb +3 -3
- data/app/models/follow.rb +1 -1
- data/app/models/geo_names_resource.rb +3 -3
- data/app/models/group.rb +1 -3
- data/app/models/local_authority.rb +26 -28
- data/app/models/proxy_deposit_request.rb +9 -9
- data/app/models/single_use_link.rb +10 -18
- data/app/models/sufia/download.rb +2 -2
- data/app/models/sufia/pageview.rb +1 -1
- data/app/models/trophy.rb +2 -4
- data/app/services/sufia/analytics.rb +10 -11
- data/app/services/sufia/generic_file_audit_service.rb +11 -12
- data/app/services/sufia/repository_audit_service.rb +1 -1
- data/config/locales/sufia.en.yml +2 -0
- data/lib/generators/sufia/models/abstract_migration_generator.rb +7 -6
- data/lib/generators/sufia/models/install_generator.rb +3 -3
- data/lib/generators/sufia/models/templates/config/arkivo_constraint.rb +1 -1
- data/lib/generators/sufia/models/templates/config/clamav.rb +1 -1
- data/lib/generators/sufia/models/templates/config/redis_config.rb +13 -5
- data/lib/generators/sufia/models/templates/config/resque_admin.rb +2 -2
- data/lib/generators/sufia/models/templates/config/resque_config.rb +1 -1
- data/lib/generators/sufia/models/templates/config/sufia.rb +10 -4
- data/lib/generators/sufia/models/templates/migrations/create_checksum_audit_logs.rb +1 -1
- data/lib/generators/sufia/models/templates/migrations/create_file_download_stats.rb +1 -1
- data/lib/generators/sufia/models/templates/migrations/create_file_view_stats.rb +1 -1
- data/lib/generators/sufia/models/templates/migrations/create_local_authorities.rb +1 -1
- data/lib/generators/sufia/models/update_content_blocks_generator.rb +0 -1
- data/lib/generators/sufia/models/upgrade600_generator.rb +0 -1
- data/lib/generators/sufia/models/user_stats_generator.rb +2 -2
- data/lib/sufia/messages.rb +17 -17
- data/lib/sufia/models.rb +1 -1
- data/lib/sufia/models/active_fedora/redis.rb +1 -4
- data/lib/sufia/models/active_record/redis.rb +2 -3
- data/lib/sufia/models/engine.rb +12 -7
- data/lib/sufia/models/file_content/versions.rb +0 -1
- data/lib/sufia/models/resque.rb +2 -2
- data/lib/sufia/models/stats/user_stat_importer.rb +65 -67
- data/lib/sufia/models/user_local_directory_behavior.rb +9 -13
- data/lib/sufia/models/utils.rb +1 -2
- data/lib/sufia/models/version.rb +1 -1
- data/lib/sufia/permissions.rb +0 -1
- data/lib/sufia/permissions/readable.rb +0 -1
- data/lib/sufia/permissions/writable.rb +20 -23
- data/lib/tasks/sufia-models_tasks.rake +18 -0
- data/sufia-models.gemspec +1 -1
- metadata +5 -5
@@ -1,9 +1,8 @@
|
|
1
1
|
module Sufia
|
2
2
|
class UserStatImporter
|
3
|
-
|
4
3
|
UserRecord = Struct.new("UserRecord", :id, :user_key, :last_stats_update)
|
5
4
|
|
6
|
-
def initialize(options={})
|
5
|
+
def initialize(options = {})
|
7
6
|
@verbose = options[:verbose]
|
8
7
|
@logging = options[:logging]
|
9
8
|
@delay_secs = options[:delay_secs].to_f
|
@@ -35,94 +34,93 @@ module Sufia
|
|
35
34
|
end
|
36
35
|
|
37
36
|
# Returns an array of users sorted by the date of their last
|
38
|
-
# stats update. Users that have not been recently updated
|
37
|
+
# stats update. Users that have not been recently updated
|
39
38
|
# will be at the top of the array.
|
40
39
|
def sorted_users
|
41
40
|
users = []
|
42
41
|
::User.find_each do |user|
|
43
42
|
users.push(UserRecord.new(user.id, user.user_key, date_since_last_cache(user)))
|
44
43
|
end
|
45
|
-
users.sort {|a, b| a.last_stats_update <=> b.last_stats_update}
|
44
|
+
users.sort { |a, b| a.last_stats_update <=> b.last_stats_update }
|
46
45
|
end
|
47
46
|
|
48
|
-
private
|
47
|
+
private
|
49
48
|
|
50
|
-
|
51
|
-
|
52
|
-
|
49
|
+
def delay
|
50
|
+
sleep @delay_secs
|
51
|
+
end
|
53
52
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
53
|
+
def rescue_and_retry(fail_message)
|
54
|
+
retry_count = 0
|
55
|
+
begin
|
56
|
+
return yield
|
57
|
+
rescue StandardError => e
|
58
|
+
retry_count += 1
|
59
|
+
if retry_count < @number_of_retries
|
60
|
+
delay
|
61
|
+
retry
|
62
|
+
else
|
63
|
+
log_message fail_message
|
64
|
+
log_message "Last exception #{e}"
|
65
|
+
end
|
66
66
|
end
|
67
67
|
end
|
68
|
-
end
|
69
68
|
|
70
|
-
|
71
|
-
|
69
|
+
def date_since_last_cache(user)
|
70
|
+
last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last
|
72
71
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
72
|
+
if last_cached_stat
|
73
|
+
last_cached_stat.date + 1.day
|
74
|
+
else
|
75
|
+
Sufia.config.analytic_start_date
|
76
|
+
end
|
77
77
|
end
|
78
|
-
end
|
79
78
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
79
|
+
def file_ids_for_user(user)
|
80
|
+
ids = []
|
81
|
+
::GenericFile.find_in_batches("#{Solrizer.solr_name('depositor', :symbol)}:\"#{user.user_key}\"", fl: "id") do |group|
|
82
|
+
ids.concat(group.map { |doc| doc["id"] })
|
83
|
+
end
|
84
|
+
ids
|
84
85
|
end
|
85
|
-
ids
|
86
|
-
end
|
87
86
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
87
|
+
# For each date, add the view and download counts for this
|
88
|
+
# file to the view & download sub-totals for that day.
|
89
|
+
# The resulting hash will look something like this:
|
90
|
+
# {"2014-11-30 00:00:00 UTC" => {:views=>2, :downloads=>5},
|
91
|
+
# "2014-12-01 00:00:00 UTC" => {:views=>4, :downloads=>4}}
|
92
|
+
def tally_results(file_stats, stat_name, total_stats)
|
93
|
+
file_stats.each do |stats|
|
94
|
+
# Exclude the stats from today since it will only be a partial day's worth of data
|
95
|
+
break if stats.date == Date.today
|
96
|
+
|
97
|
+
date_key = stats.date.to_s
|
98
|
+
old_count = total_stats[date_key] ? total_stats[date_key].fetch(stat_name) { 0 } : 0
|
99
|
+
new_count = old_count + stats.method(stat_name).call
|
100
|
+
|
101
|
+
old_values = total_stats[date_key] || {}
|
102
|
+
total_stats.store(date_key, old_values)
|
103
|
+
total_stats[date_key].store(stat_name, new_count)
|
104
|
+
end
|
105
|
+
total_stats
|
105
106
|
end
|
106
|
-
total_stats
|
107
|
-
end
|
108
107
|
|
109
|
-
|
110
|
-
|
111
|
-
|
108
|
+
def create_or_update_user_stats(stats, user)
|
109
|
+
stats.each do |date_string, data|
|
110
|
+
date = Time.zone.parse(date_string)
|
112
111
|
|
113
|
-
|
114
|
-
|
112
|
+
user_stat = UserStat.where(user_id: user.id).where(date: date).first
|
113
|
+
user_stat ||= UserStat.new(user_id: user.id, date: date)
|
115
114
|
|
116
|
-
|
117
|
-
|
118
|
-
|
115
|
+
user_stat.file_views = data[:views] || 0
|
116
|
+
user_stat.file_downloads = data[:downloads] || 0
|
117
|
+
user_stat.save!
|
118
|
+
end
|
119
119
|
end
|
120
|
-
end
|
121
|
-
|
122
|
-
def log_message(message)
|
123
|
-
puts message if @verbose
|
124
|
-
Rails.logger.info "#{self.class}: #{message}" if @logging
|
125
|
-
end
|
126
120
|
|
121
|
+
def log_message(message)
|
122
|
+
puts message if @verbose
|
123
|
+
Rails.logger.info "#{self.class}: #{message}" if @logging
|
124
|
+
end
|
127
125
|
end
|
128
126
|
end
|
@@ -1,29 +1,25 @@
|
|
1
|
-
# To enable local file ingest,
|
2
|
-
# - Make User model define .directory method that returns a String corresponding to the User's personal import directory on the server. This can be a simple ActiveRecord attribute on the User model, or it can be something more elaborate.
|
1
|
+
# To enable local file ingest,
|
2
|
+
# - Make User model define .directory method that returns a String corresponding to the User's personal import directory on the server. This can be a simple ActiveRecord attribute on the User model, or it can be something more elaborate.
|
3
3
|
# - Include this module in your User model, or define a .files() method that behaves the same
|
4
4
|
# - Set Sufia.config.enable_local_ingest to true
|
5
|
-
#
|
5
|
+
#
|
6
6
|
module Sufia::UserLocalDirectoryBehavior
|
7
|
-
|
8
7
|
# You can use this validator in your User model.
|
9
|
-
# Ensures that a string defining the path to the user's directory has been provided
|
8
|
+
# Ensures that a string defining the path to the user's directory has been provided
|
10
9
|
# and corresponds to a real directory on the server.
|
11
10
|
# @example
|
12
11
|
# validate :directory_must_exist
|
13
12
|
def directory_must_exist
|
14
|
-
|
15
|
-
|
16
|
-
end
|
13
|
+
return if directory.blank? || File.directory?(directory)
|
14
|
+
errors.add(:directory, "must be an existing directory")
|
17
15
|
end
|
18
16
|
|
19
17
|
# List the contents of the user's directory on the server
|
20
18
|
# Indicates whether each item is a directory or not.
|
21
19
|
def files
|
22
20
|
return [] unless directory.present? && File.directory?(directory)
|
23
|
-
Dir[File.join(directory, '*')].
|
24
|
-
accum << { name: File.basename(val), directory: File.directory?(val)}
|
25
|
-
accum
|
21
|
+
Dir[File.join(directory, '*')].each_with_object([]) do |val, accum|
|
22
|
+
accum << { name: File.basename(val), directory: File.directory?(val) }
|
26
23
|
end
|
27
24
|
end
|
28
|
-
|
29
|
-
end
|
25
|
+
end
|
data/lib/sufia/models/utils.rb
CHANGED
@@ -15,7 +15,6 @@ module Sufia
|
|
15
15
|
end
|
16
16
|
|
17
17
|
module ClassMethods
|
18
|
-
|
19
18
|
# retry the block if the conditional call is true unless we hit the maximum tries
|
20
19
|
#
|
21
20
|
# @param number_of_tries [enumerator] maximum number of times to retry the block
|
@@ -32,7 +31,7 @@ module Sufia
|
|
32
31
|
return result unless condition.call
|
33
32
|
sleep(Sufia.config.retry_unless_sleep) if Sufia.config.retry_unless_sleep > 0
|
34
33
|
end
|
35
|
-
raise
|
34
|
+
raise "retry_unless could not complete successfully. Try upping the # of tries?"
|
36
35
|
end
|
37
36
|
end
|
38
37
|
end
|
data/lib/sufia/models/version.rb
CHANGED
data/lib/sufia/permissions.rb
CHANGED
@@ -3,7 +3,7 @@ module Sufia
|
|
3
3
|
module Writable
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
6
|
-
#we're overriding the permissions= method which is in Hydra::AccessControls::Permissions
|
6
|
+
# we're overriding the permissions= method which is in Hydra::AccessControls::Permissions
|
7
7
|
include Hydra::AccessControls::Permissions
|
8
8
|
include Hydra::AccessControls::Visibility
|
9
9
|
|
@@ -14,20 +14,19 @@ module Sufia
|
|
14
14
|
def paranoid_permissions
|
15
15
|
valid = true
|
16
16
|
paranoid_edit_permissions.each do |validation|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
17
|
+
next unless validation[:condition].call(self)
|
18
|
+
errors[validation[:key]] ||= []
|
19
|
+
errors[validation[:key]] << validation[:message]
|
20
|
+
valid = false
|
22
21
|
end
|
23
|
-
|
22
|
+
valid
|
24
23
|
end
|
25
24
|
|
26
25
|
def paranoid_edit_permissions
|
27
26
|
[
|
28
|
-
{key: :edit_users, message: 'Depositor must have edit access', condition:
|
29
|
-
{key: :edit_groups, message: 'Public cannot have edit access', condition:
|
30
|
-
{key: :edit_groups, message: 'Registered cannot have edit access', condition:
|
27
|
+
{ key: :edit_users, message: 'Depositor must have edit access', condition: ->(obj) { !obj.edit_users.include?(obj.depositor) } },
|
28
|
+
{ key: :edit_groups, message: 'Public cannot have edit access', condition: ->(obj) { obj.edit_groups.include?('public') } },
|
29
|
+
{ key: :edit_groups, message: 'Registered cannot have edit access', condition: ->(obj) { obj.edit_groups.include?('registered') } }
|
31
30
|
]
|
32
31
|
end
|
33
32
|
|
@@ -56,20 +55,18 @@ module Sufia
|
|
56
55
|
|
57
56
|
private
|
58
57
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
58
|
+
def permission_hash
|
59
|
+
old_perms = permissions
|
60
|
+
user_perms = {}
|
61
|
+
old_perms.select { |r| r[:type] == 'user' }.each do |r|
|
62
|
+
user_perms[r[:name]] = r[:access]
|
63
|
+
end
|
64
|
+
group_perms = {}
|
65
|
+
old_perms.select { |r| r[:type] == 'group' }.each do |r|
|
66
|
+
group_perms[r[:name]] = r[:access]
|
67
|
+
end
|
68
|
+
{ 'person' => user_perms, 'group' => group_perms }
|
69
69
|
end
|
70
|
-
{'person'=>user_perms, 'group'=>group_perms}
|
71
|
-
end
|
72
|
-
|
73
70
|
end
|
74
71
|
end
|
75
72
|
end
|
@@ -1,5 +1,9 @@
|
|
1
1
|
require 'net/http'
|
2
2
|
|
3
|
+
# Pull in tasks from AF::Noid
|
4
|
+
af_noid = Gem::Specification.find_by_name 'active_fedora-noid'
|
5
|
+
load "#{af_noid.gem_dir}/lib/tasks/noid_tasks.rake"
|
6
|
+
|
3
7
|
namespace :solr do
|
4
8
|
desc "Enqueue a job to resolrize the repository objects"
|
5
9
|
task reindex: :environment do
|
@@ -8,6 +12,14 @@ namespace :solr do
|
|
8
12
|
end
|
9
13
|
|
10
14
|
namespace :sufia do
|
15
|
+
namespace :noid do
|
16
|
+
desc 'Migrate minter state file'
|
17
|
+
task migrate_statefile: :environment do
|
18
|
+
ENV['AFNOID_STATEFILE'] = Sufia.config.minter_statefile
|
19
|
+
Rake::Task['active_fedora:noid:migrate_statefile'].invoke if needs_migration?(Sufia.config.minter_statefile)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
11
23
|
namespace :user do
|
12
24
|
desc 'Populate user tokens'
|
13
25
|
task tokens: :environment do
|
@@ -94,3 +106,9 @@ def download_from_maven url, dst
|
|
94
106
|
end
|
95
107
|
end
|
96
108
|
end
|
109
|
+
|
110
|
+
def needs_migration?(statefile)
|
111
|
+
!!YAML.load(File.open(statefile).read)
|
112
|
+
rescue Psych::SyntaxError, Errno::ENOENT
|
113
|
+
false
|
114
|
+
end
|
data/sufia-models.gemspec
CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.add_dependency "active-fedora", "~> 9.1", ">= 9.1.1"
|
34
34
|
spec.add_dependency "hydra-collections", [">= 5.0.2", "< 6.0"]
|
35
35
|
spec.add_dependency 'hydra-derivatives', '~> 1.0'
|
36
|
-
spec.add_dependency 'active_fedora-noid', '~> 0
|
36
|
+
spec.add_dependency 'active_fedora-noid', '~> 1.0'
|
37
37
|
spec.add_dependency 'nest', '~> 1.1'
|
38
38
|
spec.add_dependency 'resque', '~> 1.23'
|
39
39
|
spec.add_dependency 'resque-pool', '~> 0.3'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sufia-models
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Friesen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -140,14 +140,14 @@ dependencies:
|
|
140
140
|
requirements:
|
141
141
|
- - "~>"
|
142
142
|
- !ruby/object:Gem::Version
|
143
|
-
version: '0
|
143
|
+
version: '1.0'
|
144
144
|
type: :runtime
|
145
145
|
prerelease: false
|
146
146
|
version_requirements: !ruby/object:Gem::Requirement
|
147
147
|
requirements:
|
148
148
|
- - "~>"
|
149
149
|
- !ruby/object:Gem::Version
|
150
|
-
version: '0
|
150
|
+
version: '1.0'
|
151
151
|
- !ruby/object:Gem::Dependency
|
152
152
|
name: nest
|
153
153
|
requirement: !ruby/object:Gem::Requirement
|
@@ -482,7 +482,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
482
482
|
version: '0'
|
483
483
|
requirements: []
|
484
484
|
rubyforge_project:
|
485
|
-
rubygems_version: 2.4.
|
485
|
+
rubygems_version: 2.4.8
|
486
486
|
signing_key:
|
487
487
|
specification_version: 4
|
488
488
|
summary: Models and services for sufia
|