sufia-models 6.2.0 → 6.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/actors/sufia/generic_file/actor.rb +4 -7
- data/app/jobs/active_fedora_id_based_job.rb +1 -1
- data/app/jobs/active_fedora_pid_based_job.rb +1 -2
- data/app/jobs/audit_job.rb +1 -1
- data/app/jobs/batch_update_job.rb +6 -6
- data/app/jobs/create_derivatives_job.rb +2 -3
- data/app/jobs/import_url_job.rb +8 -5
- data/app/jobs/ingest_local_file_job.rb +1 -1
- data/app/models/batch.rb +15 -21
- data/app/models/checksum_audit_log.rb +0 -1
- data/app/models/concerns/sufia/ability.rb +5 -0
- data/app/models/concerns/sufia/collection_behavior.rb +3 -2
- data/app/models/concerns/sufia/file_stat_utils.rb +19 -21
- data/app/models/concerns/sufia/generic_file/batches.rb +1 -3
- data/app/models/concerns/sufia/generic_file/characterization.rb +11 -16
- data/app/models/concerns/sufia/generic_file/content.rb +0 -1
- data/app/models/concerns/sufia/generic_file/derivatives.rb +2 -2
- data/app/models/concerns/sufia/generic_file/export.rb +50 -59
- data/app/models/concerns/sufia/generic_file/full_text_indexing.rb +15 -18
- data/app/models/concerns/sufia/generic_file/metadata.rb +0 -2
- data/app/models/concerns/sufia/generic_file/mime_types.rb +9 -9
- data/app/models/concerns/sufia/generic_file/permissions.rb +0 -1
- data/app/models/concerns/sufia/generic_file/proxy_deposit.rb +0 -1
- data/app/models/concerns/sufia/generic_file/querying.rb +9 -5
- data/app/models/concerns/sufia/generic_file/trophies.rb +1 -1
- data/app/models/concerns/sufia/generic_file/versions.rb +0 -4
- data/app/models/concerns/sufia/model_methods.rb +0 -1
- data/app/models/concerns/sufia/user.rb +15 -15
- data/app/models/concerns/sufia/user_usage_stats.rb +0 -2
- data/app/models/datastreams/fits_datastream.rb +25 -25
- data/app/models/domain_term.rb +2 -3
- data/app/models/featured_work.rb +3 -5
- data/app/models/file_download_stat.rb +3 -4
- data/app/models/file_usage.rb +10 -11
- data/app/models/file_view_stat.rb +3 -3
- data/app/models/follow.rb +1 -1
- data/app/models/geo_names_resource.rb +3 -3
- data/app/models/group.rb +1 -3
- data/app/models/local_authority.rb +26 -28
- data/app/models/proxy_deposit_request.rb +9 -9
- data/app/models/single_use_link.rb +10 -18
- data/app/models/sufia/download.rb +2 -2
- data/app/models/sufia/pageview.rb +1 -1
- data/app/models/trophy.rb +2 -4
- data/app/services/sufia/analytics.rb +10 -11
- data/app/services/sufia/generic_file_audit_service.rb +11 -12
- data/app/services/sufia/repository_audit_service.rb +1 -1
- data/config/locales/sufia.en.yml +2 -0
- data/lib/generators/sufia/models/abstract_migration_generator.rb +7 -6
- data/lib/generators/sufia/models/install_generator.rb +3 -3
- data/lib/generators/sufia/models/templates/config/arkivo_constraint.rb +1 -1
- data/lib/generators/sufia/models/templates/config/clamav.rb +1 -1
- data/lib/generators/sufia/models/templates/config/redis_config.rb +13 -5
- data/lib/generators/sufia/models/templates/config/resque_admin.rb +2 -2
- data/lib/generators/sufia/models/templates/config/resque_config.rb +1 -1
- data/lib/generators/sufia/models/templates/config/sufia.rb +10 -4
- data/lib/generators/sufia/models/templates/migrations/create_checksum_audit_logs.rb +1 -1
- data/lib/generators/sufia/models/templates/migrations/create_file_download_stats.rb +1 -1
- data/lib/generators/sufia/models/templates/migrations/create_file_view_stats.rb +1 -1
- data/lib/generators/sufia/models/templates/migrations/create_local_authorities.rb +1 -1
- data/lib/generators/sufia/models/update_content_blocks_generator.rb +0 -1
- data/lib/generators/sufia/models/upgrade600_generator.rb +0 -1
- data/lib/generators/sufia/models/user_stats_generator.rb +2 -2
- data/lib/sufia/messages.rb +17 -17
- data/lib/sufia/models.rb +1 -1
- data/lib/sufia/models/active_fedora/redis.rb +1 -4
- data/lib/sufia/models/active_record/redis.rb +2 -3
- data/lib/sufia/models/engine.rb +12 -7
- data/lib/sufia/models/file_content/versions.rb +0 -1
- data/lib/sufia/models/resque.rb +2 -2
- data/lib/sufia/models/stats/user_stat_importer.rb +65 -67
- data/lib/sufia/models/user_local_directory_behavior.rb +9 -13
- data/lib/sufia/models/utils.rb +1 -2
- data/lib/sufia/models/version.rb +1 -1
- data/lib/sufia/permissions.rb +0 -1
- data/lib/sufia/permissions/readable.rb +0 -1
- data/lib/sufia/permissions/writable.rb +20 -23
- data/lib/tasks/sufia-models_tasks.rake +18 -0
- data/sufia-models.gemspec +1 -1
- metadata +5 -5
@@ -1,9 +1,8 @@
|
|
1
1
|
module Sufia
|
2
2
|
class UserStatImporter
|
3
|
-
|
4
3
|
UserRecord = Struct.new("UserRecord", :id, :user_key, :last_stats_update)
|
5
4
|
|
6
|
-
def initialize(options={})
|
5
|
+
def initialize(options = {})
|
7
6
|
@verbose = options[:verbose]
|
8
7
|
@logging = options[:logging]
|
9
8
|
@delay_secs = options[:delay_secs].to_f
|
@@ -35,94 +34,93 @@ module Sufia
|
|
35
34
|
end
|
36
35
|
|
37
36
|
# Returns an array of users sorted by the date of their last
|
38
|
-
# stats update. Users that have not been recently updated
|
37
|
+
# stats update. Users that have not been recently updated
|
39
38
|
# will be at the top of the array.
|
40
39
|
def sorted_users
|
41
40
|
users = []
|
42
41
|
::User.find_each do |user|
|
43
42
|
users.push(UserRecord.new(user.id, user.user_key, date_since_last_cache(user)))
|
44
43
|
end
|
45
|
-
users.sort {|a, b| a.last_stats_update <=> b.last_stats_update}
|
44
|
+
users.sort { |a, b| a.last_stats_update <=> b.last_stats_update }
|
46
45
|
end
|
47
46
|
|
48
|
-
private
|
47
|
+
private
|
49
48
|
|
50
|
-
|
51
|
-
|
52
|
-
|
49
|
+
def delay
|
50
|
+
sleep @delay_secs
|
51
|
+
end
|
53
52
|
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
53
|
+
def rescue_and_retry(fail_message)
|
54
|
+
retry_count = 0
|
55
|
+
begin
|
56
|
+
return yield
|
57
|
+
rescue StandardError => e
|
58
|
+
retry_count += 1
|
59
|
+
if retry_count < @number_of_retries
|
60
|
+
delay
|
61
|
+
retry
|
62
|
+
else
|
63
|
+
log_message fail_message
|
64
|
+
log_message "Last exception #{e}"
|
65
|
+
end
|
66
66
|
end
|
67
67
|
end
|
68
|
-
end
|
69
68
|
|
70
|
-
|
71
|
-
|
69
|
+
def date_since_last_cache(user)
|
70
|
+
last_cached_stat = UserStat.where(user_id: user.id).order(date: :asc).last
|
72
71
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
72
|
+
if last_cached_stat
|
73
|
+
last_cached_stat.date + 1.day
|
74
|
+
else
|
75
|
+
Sufia.config.analytic_start_date
|
76
|
+
end
|
77
77
|
end
|
78
|
-
end
|
79
78
|
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
79
|
+
def file_ids_for_user(user)
|
80
|
+
ids = []
|
81
|
+
::GenericFile.find_in_batches("#{Solrizer.solr_name('depositor', :symbol)}:\"#{user.user_key}\"", fl: "id") do |group|
|
82
|
+
ids.concat(group.map { |doc| doc["id"] })
|
83
|
+
end
|
84
|
+
ids
|
84
85
|
end
|
85
|
-
ids
|
86
|
-
end
|
87
86
|
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
87
|
+
# For each date, add the view and download counts for this
|
88
|
+
# file to the view & download sub-totals for that day.
|
89
|
+
# The resulting hash will look something like this:
|
90
|
+
# {"2014-11-30 00:00:00 UTC" => {:views=>2, :downloads=>5},
|
91
|
+
# "2014-12-01 00:00:00 UTC" => {:views=>4, :downloads=>4}}
|
92
|
+
def tally_results(file_stats, stat_name, total_stats)
|
93
|
+
file_stats.each do |stats|
|
94
|
+
# Exclude the stats from today since it will only be a partial day's worth of data
|
95
|
+
break if stats.date == Date.today
|
96
|
+
|
97
|
+
date_key = stats.date.to_s
|
98
|
+
old_count = total_stats[date_key] ? total_stats[date_key].fetch(stat_name) { 0 } : 0
|
99
|
+
new_count = old_count + stats.method(stat_name).call
|
100
|
+
|
101
|
+
old_values = total_stats[date_key] || {}
|
102
|
+
total_stats.store(date_key, old_values)
|
103
|
+
total_stats[date_key].store(stat_name, new_count)
|
104
|
+
end
|
105
|
+
total_stats
|
105
106
|
end
|
106
|
-
total_stats
|
107
|
-
end
|
108
107
|
|
109
|
-
|
110
|
-
|
111
|
-
|
108
|
+
def create_or_update_user_stats(stats, user)
|
109
|
+
stats.each do |date_string, data|
|
110
|
+
date = Time.zone.parse(date_string)
|
112
111
|
|
113
|
-
|
114
|
-
|
112
|
+
user_stat = UserStat.where(user_id: user.id).where(date: date).first
|
113
|
+
user_stat ||= UserStat.new(user_id: user.id, date: date)
|
115
114
|
|
116
|
-
|
117
|
-
|
118
|
-
|
115
|
+
user_stat.file_views = data[:views] || 0
|
116
|
+
user_stat.file_downloads = data[:downloads] || 0
|
117
|
+
user_stat.save!
|
118
|
+
end
|
119
119
|
end
|
120
|
-
end
|
121
|
-
|
122
|
-
def log_message(message)
|
123
|
-
puts message if @verbose
|
124
|
-
Rails.logger.info "#{self.class}: #{message}" if @logging
|
125
|
-
end
|
126
120
|
|
121
|
+
def log_message(message)
|
122
|
+
puts message if @verbose
|
123
|
+
Rails.logger.info "#{self.class}: #{message}" if @logging
|
124
|
+
end
|
127
125
|
end
|
128
126
|
end
|
@@ -1,29 +1,25 @@
|
|
1
|
-
# To enable local file ingest,
|
2
|
-
# - Make User model define .directory method that returns a String corresponding to the User's personal import directory on the server. This can be a simple ActiveRecord attribute on the User model, or it can be something more elaborate.
|
1
|
+
# To enable local file ingest,
|
2
|
+
# - Make User model define .directory method that returns a String corresponding to the User's personal import directory on the server. This can be a simple ActiveRecord attribute on the User model, or it can be something more elaborate.
|
3
3
|
# - Include this module in your User model, or define a .files() method that behaves the same
|
4
4
|
# - Set Sufia.config.enable_local_ingest to true
|
5
|
-
#
|
5
|
+
#
|
6
6
|
module Sufia::UserLocalDirectoryBehavior
|
7
|
-
|
8
7
|
# You can use this validator in your User model.
|
9
|
-
# Ensures that a string defining the path to the user's directory has been provided
|
8
|
+
# Ensures that a string defining the path to the user's directory has been provided
|
10
9
|
# and corresponds to a real directory on the server.
|
11
10
|
# @example
|
12
11
|
# validate :directory_must_exist
|
13
12
|
def directory_must_exist
|
14
|
-
|
15
|
-
|
16
|
-
end
|
13
|
+
return if directory.blank? || File.directory?(directory)
|
14
|
+
errors.add(:directory, "must be an existing directory")
|
17
15
|
end
|
18
16
|
|
19
17
|
# List the contents of the user's directory on the server
|
20
18
|
# Indicates whether each item is a directory or not.
|
21
19
|
def files
|
22
20
|
return [] unless directory.present? && File.directory?(directory)
|
23
|
-
Dir[File.join(directory, '*')].
|
24
|
-
accum << { name: File.basename(val), directory: File.directory?(val)}
|
25
|
-
accum
|
21
|
+
Dir[File.join(directory, '*')].each_with_object([]) do |val, accum|
|
22
|
+
accum << { name: File.basename(val), directory: File.directory?(val) }
|
26
23
|
end
|
27
24
|
end
|
28
|
-
|
29
|
-
end
|
25
|
+
end
|
data/lib/sufia/models/utils.rb
CHANGED
@@ -15,7 +15,6 @@ module Sufia
|
|
15
15
|
end
|
16
16
|
|
17
17
|
module ClassMethods
|
18
|
-
|
19
18
|
# retry the block if the conditional call is true unless we hit the maximum tries
|
20
19
|
#
|
21
20
|
# @param number_of_tries [enumerator] maximum number of times to retry the block
|
@@ -32,7 +31,7 @@ module Sufia
|
|
32
31
|
return result unless condition.call
|
33
32
|
sleep(Sufia.config.retry_unless_sleep) if Sufia.config.retry_unless_sleep > 0
|
34
33
|
end
|
35
|
-
raise
|
34
|
+
raise "retry_unless could not complete successfully. Try upping the # of tries?"
|
36
35
|
end
|
37
36
|
end
|
38
37
|
end
|
data/lib/sufia/models/version.rb
CHANGED
data/lib/sufia/permissions.rb
CHANGED
@@ -3,7 +3,7 @@ module Sufia
|
|
3
3
|
module Writable
|
4
4
|
extend ActiveSupport::Concern
|
5
5
|
|
6
|
-
#we're overriding the permissions= method which is in Hydra::AccessControls::Permissions
|
6
|
+
# we're overriding the permissions= method which is in Hydra::AccessControls::Permissions
|
7
7
|
include Hydra::AccessControls::Permissions
|
8
8
|
include Hydra::AccessControls::Visibility
|
9
9
|
|
@@ -14,20 +14,19 @@ module Sufia
|
|
14
14
|
def paranoid_permissions
|
15
15
|
valid = true
|
16
16
|
paranoid_edit_permissions.each do |validation|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
end
|
17
|
+
next unless validation[:condition].call(self)
|
18
|
+
errors[validation[:key]] ||= []
|
19
|
+
errors[validation[:key]] << validation[:message]
|
20
|
+
valid = false
|
22
21
|
end
|
23
|
-
|
22
|
+
valid
|
24
23
|
end
|
25
24
|
|
26
25
|
def paranoid_edit_permissions
|
27
26
|
[
|
28
|
-
{key: :edit_users, message: 'Depositor must have edit access', condition:
|
29
|
-
{key: :edit_groups, message: 'Public cannot have edit access', condition:
|
30
|
-
{key: :edit_groups, message: 'Registered cannot have edit access', condition:
|
27
|
+
{ key: :edit_users, message: 'Depositor must have edit access', condition: ->(obj) { !obj.edit_users.include?(obj.depositor) } },
|
28
|
+
{ key: :edit_groups, message: 'Public cannot have edit access', condition: ->(obj) { obj.edit_groups.include?('public') } },
|
29
|
+
{ key: :edit_groups, message: 'Registered cannot have edit access', condition: ->(obj) { obj.edit_groups.include?('registered') } }
|
31
30
|
]
|
32
31
|
end
|
33
32
|
|
@@ -56,20 +55,18 @@ module Sufia
|
|
56
55
|
|
57
56
|
private
|
58
57
|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
58
|
+
def permission_hash
|
59
|
+
old_perms = permissions
|
60
|
+
user_perms = {}
|
61
|
+
old_perms.select { |r| r[:type] == 'user' }.each do |r|
|
62
|
+
user_perms[r[:name]] = r[:access]
|
63
|
+
end
|
64
|
+
group_perms = {}
|
65
|
+
old_perms.select { |r| r[:type] == 'group' }.each do |r|
|
66
|
+
group_perms[r[:name]] = r[:access]
|
67
|
+
end
|
68
|
+
{ 'person' => user_perms, 'group' => group_perms }
|
69
69
|
end
|
70
|
-
{'person'=>user_perms, 'group'=>group_perms}
|
71
|
-
end
|
72
|
-
|
73
70
|
end
|
74
71
|
end
|
75
72
|
end
|
@@ -1,5 +1,9 @@
|
|
1
1
|
require 'net/http'
|
2
2
|
|
3
|
+
# Pull in tasks from AF::Noid
|
4
|
+
af_noid = Gem::Specification.find_by_name 'active_fedora-noid'
|
5
|
+
load "#{af_noid.gem_dir}/lib/tasks/noid_tasks.rake"
|
6
|
+
|
3
7
|
namespace :solr do
|
4
8
|
desc "Enqueue a job to resolrize the repository objects"
|
5
9
|
task reindex: :environment do
|
@@ -8,6 +12,14 @@ namespace :solr do
|
|
8
12
|
end
|
9
13
|
|
10
14
|
namespace :sufia do
|
15
|
+
namespace :noid do
|
16
|
+
desc 'Migrate minter state file'
|
17
|
+
task migrate_statefile: :environment do
|
18
|
+
ENV['AFNOID_STATEFILE'] = Sufia.config.minter_statefile
|
19
|
+
Rake::Task['active_fedora:noid:migrate_statefile'].invoke if needs_migration?(Sufia.config.minter_statefile)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
11
23
|
namespace :user do
|
12
24
|
desc 'Populate user tokens'
|
13
25
|
task tokens: :environment do
|
@@ -94,3 +106,9 @@ def download_from_maven url, dst
|
|
94
106
|
end
|
95
107
|
end
|
96
108
|
end
|
109
|
+
|
110
|
+
def needs_migration?(statefile)
|
111
|
+
!!YAML.load(File.open(statefile).read)
|
112
|
+
rescue Psych::SyntaxError, Errno::ENOENT
|
113
|
+
false
|
114
|
+
end
|
data/sufia-models.gemspec
CHANGED
@@ -33,7 +33,7 @@ Gem::Specification.new do |spec|
|
|
33
33
|
spec.add_dependency "active-fedora", "~> 9.1", ">= 9.1.1"
|
34
34
|
spec.add_dependency "hydra-collections", [">= 5.0.2", "< 6.0"]
|
35
35
|
spec.add_dependency 'hydra-derivatives', '~> 1.0'
|
36
|
-
spec.add_dependency 'active_fedora-noid', '~> 0
|
36
|
+
spec.add_dependency 'active_fedora-noid', '~> 1.0'
|
37
37
|
spec.add_dependency 'nest', '~> 1.1'
|
38
38
|
spec.add_dependency 'resque', '~> 1.23'
|
39
39
|
spec.add_dependency 'resque-pool', '~> 0.3'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sufia-models
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 6.
|
4
|
+
version: 6.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Friesen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -140,14 +140,14 @@ dependencies:
|
|
140
140
|
requirements:
|
141
141
|
- - "~>"
|
142
142
|
- !ruby/object:Gem::Version
|
143
|
-
version: '0
|
143
|
+
version: '1.0'
|
144
144
|
type: :runtime
|
145
145
|
prerelease: false
|
146
146
|
version_requirements: !ruby/object:Gem::Requirement
|
147
147
|
requirements:
|
148
148
|
- - "~>"
|
149
149
|
- !ruby/object:Gem::Version
|
150
|
-
version: '0
|
150
|
+
version: '1.0'
|
151
151
|
- !ruby/object:Gem::Dependency
|
152
152
|
name: nest
|
153
153
|
requirement: !ruby/object:Gem::Requirement
|
@@ -482,7 +482,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
482
482
|
version: '0'
|
483
483
|
requirements: []
|
484
484
|
rubyforge_project:
|
485
|
-
rubygems_version: 2.4.
|
485
|
+
rubygems_version: 2.4.8
|
486
486
|
signing_key:
|
487
487
|
specification_version: 4
|
488
488
|
summary: Models and services for sufia
|