jgrouper 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +0 -2
- data/HISTORY.md +10 -0
- data/README.rdoc +26 -2
- data/bin/jgrouper-audit-archiver +4 -1
- data/jgrouper.gemspec +2 -0
- data/lib/jgrouper/audit_archiver.rb +71 -58
- data/lib/jgrouper/version.rb +1 -1
- data/lib/jgrouper.rb +26 -2
- metadata +33 -9
- checksums.yaml +0 -7
data/Gemfile
CHANGED
data/HISTORY.md
CHANGED
@@ -1,6 +1,16 @@
|
|
1
1
|
JGrouper History
|
2
2
|
================
|
3
3
|
|
4
|
+
2013-07-05 JGrouper v0.4.0
|
5
|
+
--------------------------
|
6
|
+
* Add "--batch-size NUMBER" to "jgrouper-audit-archiver"
|
7
|
+
* Add more code examples
|
8
|
+
* Update "JGrouper::AuditArchiver" to append to output files
|
9
|
+
* Update "JGrouper::AuditArchiver" to lock output file
|
10
|
+
* Update "JGrouper::AuditArchiver" to prune from database in batches to better support large datasets
|
11
|
+
* Update "JGrouper::AuditArchiver" earliest entry detection
|
12
|
+
|
13
|
+
|
4
14
|
2013-06-27 JGrouper v0.3.0
|
5
15
|
--------------------------
|
6
16
|
* Add "bin/jgrouper-grant"
|
data/README.rdoc
CHANGED
@@ -7,10 +7,26 @@
|
|
7
7
|
require 'jgrouper'
|
8
8
|
JGrouper.home! '/path/to/your/grouper/api/installation/directory'
|
9
9
|
|
10
|
+
==== Groups
|
11
|
+
|
12
|
+
# Find Group by name
|
13
|
+
g = JGrouper::Group.find $name
|
14
|
+
|
15
|
+
# Grant privileges
|
16
|
+
g.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('admin')
|
17
|
+
|
10
18
|
==== Members
|
11
19
|
|
12
20
|
# Find Member by UUID
|
13
|
-
m = JGrouper::Member.find uuid
|
21
|
+
m = JGrouper::Member.find $uuid
|
22
|
+
|
23
|
+
==== Privileges
|
24
|
+
|
25
|
+
# Find Privilege by name
|
26
|
+
p = JGrouper::Privilege.find 'admin'
|
27
|
+
p.access? # True
|
28
|
+
p.naming? # False
|
29
|
+
p.type # :access
|
14
30
|
|
15
31
|
==== Stems
|
16
32
|
|
@@ -24,7 +40,15 @@
|
|
24
40
|
root.groups
|
25
41
|
|
26
42
|
# Find stem by name
|
27
|
-
stem = JGrouper::Stem.find name
|
43
|
+
stem = JGrouper::Stem.find $name
|
44
|
+
|
45
|
+
# Grant privileges
|
46
|
+
stem.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('stem')
|
47
|
+
|
48
|
+
==== Subjects
|
49
|
+
|
50
|
+
# Find by id-or-identifier
|
51
|
+
subj = JGrouper::Subject.find $subj
|
28
52
|
|
29
53
|
==== Archiver
|
30
54
|
|
data/bin/jgrouper-audit-archiver
CHANGED
@@ -10,10 +10,13 @@ JGrouper::AuditArchiver.new do |archiver|
|
|
10
10
|
opts = OptionParser.new do |opts|
|
11
11
|
opts.banner = "USAGE: #{ File.basename(__FILE__) } [options]"
|
12
12
|
|
13
|
+
opts.on( '-b', '--batch-size NUMBER', Integer, "Delete this many entries at a time [DEFAULT: #{ JGrouper::AuditArchiver::BATCH_SIZE }]" ) do |batch_size|
|
14
|
+
archiver.batch_size = batch_size
|
15
|
+
end
|
13
16
|
opts.on( '-d', '--directory DIR', 'Write output to this directory [DEFAULT: .]' ) do |directory|
|
14
17
|
archiver.directory = directory
|
15
18
|
end
|
16
|
-
opts.on( '-n', '--number DAYS', Integer,
|
19
|
+
opts.on( '-n', '--number DAYS', Integer, "Archive this many days [DEFAULT: #{ JGrouper::AuditArchiver::NUMBER_OF_DAYS }]" ) do |number|
|
17
20
|
archiver.number_of_days = number
|
18
21
|
end
|
19
22
|
opts.on( '-s', '--skip COLUMNS', Array, 'Exclude these GROUPER_AUDIT_ENTRY comma-separated column names from archive [DEFAULT: none]' ) do |columns|
|
data/jgrouper.gemspec
CHANGED
@@ -18,6 +18,8 @@ Gem::Specification.new do |spec|
|
|
18
18
|
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
19
19
|
spec.require_paths = ['lib']
|
20
20
|
|
21
|
+
spec.add_dependency 'jruby-jars', '>= 1.7.3'
|
22
|
+
|
21
23
|
spec.add_development_dependency 'bundler', '~> 1.3'
|
22
24
|
spec.add_development_dependency 'rake'
|
23
25
|
spec.add_development_dependency 'rdoc-readme', '~> 0.1.2'
|
@@ -29,19 +29,22 @@ module JGrouper # :nodoc:
|
|
29
29
|
#
|
30
30
|
class AuditArchiver
|
31
31
|
|
32
|
+
BATCH_SIZE = 1000
|
32
33
|
GROUPER_AUDIT_ENTRY = 'grouper_audit_entry'
|
33
34
|
GROUPER_AUDIT_TYPE = 'grouper_audit_type'
|
35
|
+
NUMBER_OF_DAYS = 1
|
34
36
|
|
35
|
-
attr_writer :directory, :number_of_days, :verbose
|
37
|
+
attr_writer :batch_size, :directory, :number_of_days, :verbose
|
36
38
|
|
37
39
|
def initialize
|
40
|
+
@batch_size = BATCH_SIZE
|
38
41
|
@config = {}
|
39
42
|
@conn = nil
|
40
43
|
@date = nil
|
41
44
|
@directory = Dir.pwd
|
42
45
|
@fh = nil
|
43
46
|
@mappings = {}
|
44
|
-
@number_of_days =
|
47
|
+
@number_of_days = NUMBER_OF_DAYS
|
45
48
|
@skip_columns = []
|
46
49
|
@stop_date = nil
|
47
50
|
@verbose = false
|
@@ -58,16 +61,19 @@ module JGrouper # :nodoc:
|
|
58
61
|
connect do
|
59
62
|
mappings do
|
60
63
|
1.upto @number_of_days do
|
61
|
-
@date
|
64
|
+
@date = oldest_entry # Date of oldest entry
|
62
65
|
break if stop?
|
63
66
|
|
64
67
|
start_at = time_to_microseconds @date.to_time # @date start-of-day
|
65
68
|
stop_at = time_to_microseconds ( @date + 1 ).to_time - 1 # @date end-of-day
|
66
|
-
entries = []
|
67
69
|
|
68
|
-
|
69
|
-
|
70
|
+
num_entries = 0
|
71
|
+
entries = []
|
72
|
+
total_entries = entries_within_interval start_at, stop_at
|
73
|
+
|
74
|
+
log "archiving #{ @date } (#{total_entries} entries) ..."
|
70
75
|
|
76
|
+
filehandle @directory, "grouper-audit-entries-#{ @date }.csv" do
|
71
77
|
# TODO Extract!
|
72
78
|
# TODO Use prepared statement when I get JRuby, JDBC & Oracle to better cooperate on BigDecimal-ish data types
|
73
79
|
qry = "SELECT * FROM #{GROUPER_AUDIT_ENTRY} WHERE created_on BETWEEN #{start_at} AND #{stop_at} ORDER BY created_on"
|
@@ -82,22 +88,31 @@ module JGrouper # :nodoc:
|
|
82
88
|
v = rs.get_object(n)
|
83
89
|
|
84
90
|
next if v.nil?
|
85
|
-
|
91
|
+
entries << v if :id == k
|
92
|
+
entry << k << v.to_s.gsub(/\n/, ', ')
|
86
93
|
end
|
87
94
|
|
88
|
-
entry
|
89
|
-
|
95
|
+
entry = transform(entry)
|
96
|
+
entry = entry.each_slice(2).reject { |slice| skip? slice.first }.collect { |slice| "#{slice.first}=#{slice.last}" }
|
97
|
+
num_entries = num_entries + 1
|
90
98
|
|
91
|
-
@fh.puts CSV.generate_line(
|
92
|
-
|
99
|
+
@fh.puts CSV.generate_line( entry, col_sep: "\t" )
|
100
|
+
|
101
|
+
if entries.size > 0 && ( entries.size % @batch_size ) == 0
|
102
|
+
prune entries, num_entries, total_entries
|
103
|
+
entries.clear
|
104
|
+
@fh.fsync
|
105
|
+
end
|
106
|
+
|
107
|
+
yield entry if block_given?
|
93
108
|
end
|
94
109
|
rs.close
|
95
110
|
stmt.close
|
96
|
-
|
97
|
-
log "archiving #{ @date } (#{entries.size} entries) - done"
|
98
111
|
end
|
99
112
|
|
100
|
-
prune(entries
|
113
|
+
prune(entries, num_entries, total_entries) unless entries.empty?
|
114
|
+
log "archiving #{ @date } (#{num_entries}/#{total_entries} entries) - done"
|
115
|
+
|
101
116
|
end
|
102
117
|
end
|
103
118
|
end
|
@@ -159,15 +174,36 @@ module JGrouper # :nodoc:
|
|
159
174
|
end
|
160
175
|
end
|
161
176
|
|
177
|
+
#
|
178
|
+
# Return number of entries for time interval.
|
179
|
+
#
|
180
|
+
def entries_within_interval(start_at, stop_at)
|
181
|
+
num_entries = 0
|
182
|
+
|
183
|
+
stmt = @conn.create_statement
|
184
|
+
rs = stmt.execute_query "SELECT COUNT(*) FROM #{GROUPER_AUDIT_ENTRY} WHERE created_on BETWEEN #{start_at} AND #{stop_at}"
|
185
|
+
while rs.next
|
186
|
+
num_entries = rs.get_object(1).to_s.to_i # TODO Ugly
|
187
|
+
break
|
188
|
+
end
|
189
|
+
rs.close
|
190
|
+
stmt.close
|
191
|
+
return num_entries
|
192
|
+
end
|
193
|
+
|
162
194
|
#
|
163
|
-
# Open filehandle for
|
164
|
-
# TODO DRY
|
195
|
+
# Open filehandle for appending or raise exception if directory does not exist.
|
165
196
|
#
|
166
197
|
def filehandle(directory, file)
|
167
198
|
raise "ERROR: directory does not exist - #{directory}" unless File.directory?(directory)
|
168
199
|
fn = File.join directory, file
|
169
|
-
|
170
|
-
|
200
|
+
File.open(fn, 'a') do |fh|
|
201
|
+
@fh = fh
|
202
|
+
@fh.flock File::LOCK_EX
|
203
|
+
yield self
|
204
|
+
@fh.flock File::LOCK_UN
|
205
|
+
@fh = nil
|
206
|
+
end
|
171
207
|
end
|
172
208
|
|
173
209
|
# TODO Use Logger...
|
@@ -221,24 +257,6 @@ module JGrouper # :nodoc:
|
|
221
257
|
microseconds / 1000
|
222
258
|
end
|
223
259
|
|
224
|
-
#
|
225
|
-
# Return number of entries for time interval.
|
226
|
-
#
|
227
|
-
def num_entries(start_at, stop_at)
|
228
|
-
num_entries = 0
|
229
|
-
|
230
|
-
stmt = @conn.create_statement
|
231
|
-
rs = stmt.execute_query "SELECT COUNT(*) FROM #{GROUPER_AUDIT_ENTRY} WHERE created_on BETWEEN #{start_at} AND #{stop_at}"
|
232
|
-
while rs.next
|
233
|
-
num_entries = rs.get_object(1).to_s.to_i # TODO Ugly
|
234
|
-
break
|
235
|
-
end
|
236
|
-
rs.close
|
237
|
-
stmt.close
|
238
|
-
|
239
|
-
return num_entries
|
240
|
-
end
|
241
|
-
|
242
260
|
#
|
243
261
|
# Calculate archive date based on MIN(created_on)
|
244
262
|
#
|
@@ -246,10 +264,10 @@ module JGrouper # :nodoc:
|
|
246
264
|
d = Date.new
|
247
265
|
|
248
266
|
stmt = @conn.create_statement
|
249
|
-
rs = stmt.execute_query "SELECT MIN(created_on) FROM #{GROUPER_AUDIT_ENTRY}"
|
267
|
+
rs = stmt.execute_query "SELECT MIN(created_on) FROM #{GROUPER_AUDIT_ENTRY} ORDER BY created_on"
|
250
268
|
while rs.next
|
251
269
|
# Microseconds -> Seconds -> Time -> Date
|
252
|
-
d = Time.at( microseconds_to_seconds( rs.
|
270
|
+
d = Time.at( microseconds_to_seconds( rs.get_object(1).to_s.to_f ) ).to_date
|
253
271
|
break
|
254
272
|
end
|
255
273
|
rs.close
|
@@ -259,28 +277,23 @@ module JGrouper # :nodoc:
|
|
259
277
|
end
|
260
278
|
|
261
279
|
#
|
262
|
-
# Prune entries from database.
|
280
|
+
# Prune specified entries from database.
|
263
281
|
#
|
264
|
-
def prune(
|
265
|
-
log "pruning #{ @date } (#{
|
282
|
+
def prune(entries, num_entries, total_entries)
|
283
|
+
log "pruning #{ @date } (#{entries.size} entries, #{num_entries}/#{total_entries}) ..."
|
266
284
|
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
rv
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
unless expected == rv
|
280
|
-
raise "ERROR: not committing delete as number of deleted entries does not match expected number - #{rv} != #{expected}"
|
281
|
-
end
|
282
|
-
else
|
283
|
-
raise "ERROR: not deleting as number of found entries does not match expected number - #{found} != #{expected}"
|
285
|
+
@conn.auto_commit = false
|
286
|
+
qry = "DELETE FROM #{GROUPER_AUDIT_ENTRY} WHERE id IN ( " + entries.map { |_| "'#{_}'" }.join(',') + " )"
|
287
|
+
stmt = @conn.create_statement
|
288
|
+
rv = stmt.execute_update qry
|
289
|
+
if entries.size == rv
|
290
|
+
@conn.commit
|
291
|
+
log "pruning #{ @date } (#{rv} entries, #{num_entries}/#{total_entries}) - done"
|
292
|
+
end
|
293
|
+
stmt.close
|
294
|
+
unless entries.size == rv
|
295
|
+
log "ERROR: #{ entries.size } != #{rv} for query: #{qry}"
|
296
|
+
raise "ERROR: not committing delete as number of deleted entries does not match expected number - #{rv} != #{expected}"
|
284
297
|
end
|
285
298
|
end
|
286
299
|
|
data/lib/jgrouper/version.rb
CHANGED
data/lib/jgrouper.rb
CHANGED
@@ -17,10 +17,26 @@ require 'jgrouper/version'
|
|
17
17
|
# require 'jgrouper'
|
18
18
|
# JGrouper.home! '/path/to/your/grouper/api/installation/directory'
|
19
19
|
#
|
20
|
+
# ==== Groups
|
21
|
+
#
|
22
|
+
# # Find Group by name
|
23
|
+
# g = JGrouper::Group.find $name
|
24
|
+
#
|
25
|
+
# # Grant privileges
|
26
|
+
# g.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('admin')
|
27
|
+
#
|
20
28
|
# ==== Members
|
21
29
|
#
|
22
30
|
# # Find Member by UUID
|
23
|
-
# m = JGrouper::Member.find uuid
|
31
|
+
# m = JGrouper::Member.find $uuid
|
32
|
+
#
|
33
|
+
# ==== Privileges
|
34
|
+
#
|
35
|
+
# # Find Privilege by name
|
36
|
+
# p = JGrouper::Privilege.find 'admin'
|
37
|
+
# p.access? # True
|
38
|
+
# p.naming? # False
|
39
|
+
# p.type # :access
|
24
40
|
#
|
25
41
|
# ==== Stems
|
26
42
|
#
|
@@ -34,7 +50,15 @@ require 'jgrouper/version'
|
|
34
50
|
# root.groups
|
35
51
|
#
|
36
52
|
# # Find stem by name
|
37
|
-
# stem = JGrouper::Stem.find name
|
53
|
+
# stem = JGrouper::Stem.find $name
|
54
|
+
#
|
55
|
+
# # Grant privileges
|
56
|
+
# stem.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('stem')
|
57
|
+
#
|
58
|
+
# ==== Subjects
|
59
|
+
#
|
60
|
+
# # Find by id-or-identifier
|
61
|
+
# subj = JGrouper::Subject.find $subj
|
38
62
|
#
|
39
63
|
# ==== Archiver
|
40
64
|
#
|
metadata
CHANGED
@@ -1,18 +1,36 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: jgrouper
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
|
+
prerelease:
|
5
6
|
platform: ruby
|
6
7
|
authors:
|
7
8
|
- blair christensen
|
8
9
|
autorequire:
|
9
10
|
bindir: bin
|
10
11
|
cert_chain: []
|
11
|
-
date: 2013-
|
12
|
+
date: 2013-07-05 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: jruby-jars
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: 1.7.3
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: 1.7.3
|
13
30
|
- !ruby/object:Gem::Dependency
|
14
31
|
name: bundler
|
15
32
|
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
16
34
|
requirements:
|
17
35
|
- - ~>
|
18
36
|
- !ruby/object:Gem::Version
|
@@ -20,6 +38,7 @@ dependencies:
|
|
20
38
|
type: :development
|
21
39
|
prerelease: false
|
22
40
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
23
42
|
requirements:
|
24
43
|
- - ~>
|
25
44
|
- !ruby/object:Gem::Version
|
@@ -27,20 +46,23 @@ dependencies:
|
|
27
46
|
- !ruby/object:Gem::Dependency
|
28
47
|
name: rake
|
29
48
|
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
30
50
|
requirements:
|
31
|
-
- - '>='
|
51
|
+
- - ! '>='
|
32
52
|
- !ruby/object:Gem::Version
|
33
53
|
version: '0'
|
34
54
|
type: :development
|
35
55
|
prerelease: false
|
36
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
37
58
|
requirements:
|
38
|
-
- - '>='
|
59
|
+
- - ! '>='
|
39
60
|
- !ruby/object:Gem::Version
|
40
61
|
version: '0'
|
41
62
|
- !ruby/object:Gem::Dependency
|
42
63
|
name: rdoc-readme
|
43
64
|
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
44
66
|
requirements:
|
45
67
|
- - ~>
|
46
68
|
- !ruby/object:Gem::Version
|
@@ -48,6 +70,7 @@ dependencies:
|
|
48
70
|
type: :development
|
49
71
|
prerelease: false
|
50
72
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
51
74
|
requirements:
|
52
75
|
- - ~>
|
53
76
|
- !ruby/object:Gem::Version
|
@@ -86,25 +109,26 @@ files:
|
|
86
109
|
homepage: https://github.com/blairc/jgrouper/
|
87
110
|
licenses:
|
88
111
|
- MIT
|
89
|
-
metadata: {}
|
90
112
|
post_install_message:
|
91
113
|
rdoc_options: []
|
92
114
|
require_paths:
|
93
115
|
- lib
|
94
116
|
required_ruby_version: !ruby/object:Gem::Requirement
|
117
|
+
none: false
|
95
118
|
requirements:
|
96
|
-
- - '>='
|
119
|
+
- - ! '>='
|
97
120
|
- !ruby/object:Gem::Version
|
98
121
|
version: '0'
|
99
122
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
123
|
+
none: false
|
100
124
|
requirements:
|
101
|
-
- - '>='
|
125
|
+
- - ! '>='
|
102
126
|
- !ruby/object:Gem::Version
|
103
127
|
version: '0'
|
104
128
|
requirements: []
|
105
129
|
rubyforge_project:
|
106
|
-
rubygems_version:
|
130
|
+
rubygems_version: 1.8.23
|
107
131
|
signing_key:
|
108
|
-
specification_version:
|
132
|
+
specification_version: 3
|
109
133
|
summary: JRuby wrapper around the Internet2 Grouper API
|
110
134
|
test_files: []
|
checksums.yaml
DELETED
@@ -1,7 +0,0 @@
|
|
1
|
-
---
|
2
|
-
SHA1:
|
3
|
-
metadata.gz: 1d34d0e3926d5850e95ca673912825fae7f2d4d7
|
4
|
-
data.tar.gz: ef0be60e432fcddbf3245cc20988aa5fe2f48c86
|
5
|
-
SHA512:
|
6
|
-
metadata.gz: 8f05bcf26ee9937fbcb84390de299457cff24f82071308d78c0ab2c600c1c4c0a2ceca64b154f861b7e2d4633ef3a63096461b0299c9ee2e04aa8a490a0cfee3
|
7
|
-
data.tar.gz: dc8d6104cb439831e5550c7829b386044d0c1aebf48db5ee7594627302510637b3700097b36563511284d14e4ae0c916d64bad7c4299e11f75047d350fc9bf90
|