jgrouper 0.3.0 → 0.4.0

data/Gemfile

@@ -1,7 +1,5 @@
 source 'https://rubygems.org'
 
-ruby '1.9.3' # TODO , engine: 'jruby', engine_version: '1.7.3'
-
 # Specify your gem's dependencies in jgrouper.gemspec
 gemspec
 

data/HISTORY.md

@@ -1,6 +1,16 @@
 JGrouper History
 ================
 
+2013-07-05 JGrouper v0.4.0
+--------------------------
+* Add "--batch-size NUMBER" to "jgrouper-audit-archiver"
+* Add more code examples
+* Update "JGrouper::AuditArchiver" to append to output files
+* Update "JGrouper::AuditArchiver" to lock output file
+* Update "JGrouper::AuditArchiver" to prune from database in batches to better support large datasets
+* Update "JGrouper::AuditArchiver" earliest entry detection
+
+
 2013-06-27 JGrouper v0.3.0
 --------------------------
 * Add "bin/jgrouper-grant"

data/README.rdoc

@@ -7,10 +7,26 @@
   require 'jgrouper'
   JGrouper.home! '/path/to/your/grouper/api/installation/directory'
 
+==== Groups
+
+  # Find Group by name
+  g = JGrouper::Group.find $name
+
+  # Grant privileges
+  g.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('admin')
+
 ==== Members
 
   # Find Member by UUID
-  m = JGrouper::Member.find uuid
+  m = JGrouper::Member.find $uuid
+
+==== Privileges
+
+  # Find Privilege by name
+  p = JGrouper::Privilege.find 'admin'
+  p.access? # True
+  p.naming? # False
+  p.type    # :access
 
 ==== Stems
 
@@ -24,7 +40,15 @@
   root.groups
 
   # Find stem by name
-  stem = JGrouper::Stem.find name
+  stem = JGrouper::Stem.find $name
+
+  # Grant privileges
+  stem.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('stem')
+
+==== Subjects
+
+  # Find by id-or-identifier
+  subj = JGrouper::Subject.find $subj
 
 ==== Archiver
 

data/bin/jgrouper-audit-archiver

@@ -10,10 +10,13 @@ JGrouper::AuditArchiver.new do |archiver|
   opts = OptionParser.new do |opts|
     opts.banner = "USAGE: #{ File.basename(__FILE__) } [options]"
 
+    opts.on( '-b', '--batch-size NUMBER', Integer, "Delete this many entries at a time [DEFAULT: #{ JGrouper::AuditArchiver::BATCH_SIZE }]" ) do |batch_size|
+      archiver.batch_size = batch_size  
+    end   
     opts.on( '-d', '--directory DIR', 'Write output to this directory [DEFAULT: .]' ) do |directory|
       archiver.directory = directory  
     end   
-    opts.on( '-n', '--number DAYS', Integer, 'Archive this many days [DEFAULT: 1]' ) do |number|
+    opts.on( '-n', '--number DAYS', Integer, "Archive this many days [DEFAULT: #{ JGrouper::AuditArchiver::NUMBER_OF_DAYS }]" ) do |number|
       archiver.number_of_days = number
     end
     opts.on( '-s', '--skip COLUMNS', Array, 'Exclude these GROUPER_AUDIT_ENTRY comma-separated column names from archive [DEFAULT: none]' ) do |columns|

data/jgrouper.gemspec

@@ -18,6 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
+  spec.add_dependency 'jruby-jars', '>= 1.7.3'
+
   spec.add_development_dependency 'bundler', '~> 1.3'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rdoc-readme', '~> 0.1.2'

data/lib/jgrouper/audit_archiver.rb

@@ -29,19 +29,22 @@ module JGrouper # :nodoc:
   #
   class AuditArchiver
 
+    BATCH_SIZE          = 1000
     GROUPER_AUDIT_ENTRY = 'grouper_audit_entry' 
     GROUPER_AUDIT_TYPE  = 'grouper_audit_type'
+    NUMBER_OF_DAYS      = 1
 
-    attr_writer :directory, :number_of_days, :verbose
+    attr_writer :batch_size, :directory, :number_of_days, :verbose
 
     def initialize
+      @batch_size     = BATCH_SIZE
       @config         = {}
       @conn           = nil
       @date           = nil
       @directory      = Dir.pwd
       @fh             = nil
       @mappings       = {}
-      @number_of_days = 1
+      @number_of_days = NUMBER_OF_DAYS
       @skip_columns   = []
       @stop_date      = nil
       @verbose        = false
@@ -58,16 +61,19 @@ module JGrouper # :nodoc:
       connect do
         mappings do
           1.upto @number_of_days do
-            @date    = oldest_entry                                   # Date of oldest entry
+            @date = oldest_entry                                      # Date of oldest entry
             break if stop?
 
             start_at = time_to_microseconds @date.to_time             # @date start-of-day
             stop_at  = time_to_microseconds ( @date + 1 ).to_time - 1 # @date end-of-day
-            entries  = []
 
-            filehandle @directory, "grouper-audit-entries-#{ @date }.csv" do
-              log "archiving #{ @date } ..."
+            num_entries   = 0
+            entries       = []
+            total_entries = entries_within_interval start_at, stop_at
+
+            log "archiving #{ @date } (#{total_entries} entries) ..."
 
+            filehandle @directory, "grouper-audit-entries-#{ @date }.csv" do
               # TODO Extract!
               # TODO Use prepared statement when I get JRuby, JDBC & Oracle to better cooperate on BigDecimal-ish data types
               qry      = "SELECT * FROM #{GROUPER_AUDIT_ENTRY} WHERE created_on BETWEEN #{start_at} AND #{stop_at} ORDER BY created_on"
@@ -82,22 +88,31 @@ module JGrouper # :nodoc:
                   v = rs.get_object(n)
 
                   next if v.nil?
-                  entry << k << v.to_s.gsub(/\n/, ', ')
+                  entries << v if :id == k
+                  entry   << k << v.to_s.gsub(/\n/, ', ')
                 end
 
-                entry = transform(entry)
-                entries << entry.each_slice(2).reject { |slice| skip? slice.first }.collect { |slice| "#{slice.first}=#{slice.last}" }
+                entry       =  transform(entry)
+                entry       = entry.each_slice(2).reject { |slice| skip? slice.first }.collect { |slice| "#{slice.first}=#{slice.last}" }
+                num_entries = num_entries + 1
 
-                @fh.puts CSV.generate_line( entries.last, col_sep: "\t" )
-                yield entries.last if block_given?
+                @fh.puts CSV.generate_line( entry, col_sep: "\t" )
+
+                if entries.size > 0 && ( entries.size % @batch_size ) == 0
+                  prune entries, num_entries, total_entries
+                  entries.clear
+                  @fh.fsync
+                end
+                
+                yield entry if block_given?
               end
               rs.close
               stmt.close
-              
-              log "archiving #{ @date } (#{entries.size} entries) - done"
             end
 
-            prune(entries.size, start_at, stop_at) unless entries.empty?
+            prune(entries, num_entries, total_entries) unless entries.empty? 
+            log "archiving #{ @date } (#{num_entries}/#{total_entries} entries) - done"
+         
           end
         end
       end
@@ -159,15 +174,36 @@ module JGrouper # :nodoc:
         end
       end
 
+       #
+       # Return number of entries for time interval.
+       #
+       def entries_within_interval(start_at, stop_at)
+        num_entries = 0
+
+        stmt = @conn.create_statement
+        rs   = stmt.execute_query "SELECT COUNT(*) FROM #{GROUPER_AUDIT_ENTRY} WHERE created_on BETWEEN #{start_at} AND #{stop_at}"
+        while rs.next
+          num_entries = rs.get_object(1).to_s.to_i # TODO Ugly
+          break
+        end
+        rs.close
+        stmt.close
+        return num_entries
+       end
+
       #
-      # Open filehandle for writing or raise exception if a) directory does not exist or b) file does exist.
-      # TODO DRY
+      # Open filehandle for appending or raise exception if directory does not exist.
       #
       def filehandle(directory, file)
         raise "ERROR: directory does not exist - #{directory}" unless File.directory?(directory)
         fn = File.join directory, file
-        raise "ERROR: file already exists - #{fn}" if File.exists?(fn)
-        File.open(fn, 'w') { |fh| @fh = fh ; yield self ; @fh = nil }
+        File.open(fn, 'a') do |fh|
+          @fh = fh
+          @fh.flock File::LOCK_EX
+          yield self
+          @fh.flock File::LOCK_UN
+          @fh = nil
+        end
       end
 
       # TODO Use Logger...
@@ -221,24 +257,6 @@ module JGrouper # :nodoc:
         microseconds / 1000
       end
 
-      #
-      # Return number of entries for time interval.
-      #
-      def num_entries(start_at, stop_at)
-        num_entries = 0
-        
-        stmt = @conn.create_statement
-        rs   = stmt.execute_query "SELECT COUNT(*) FROM #{GROUPER_AUDIT_ENTRY} WHERE created_on BETWEEN #{start_at} AND #{stop_at}"
-        while rs.next
-          num_entries = rs.get_object(1).to_s.to_i # TODO Ugly
-          break
-        end
-        rs.close
-        stmt.close
-
-        return num_entries
-      end
-
       #
       # Calculate archive date based on MIN(created_on)
       #
@@ -246,10 +264,10 @@ module JGrouper # :nodoc:
         d = Date.new
 
         stmt = @conn.create_statement
-        rs   = stmt.execute_query "SELECT MIN(created_on) FROM #{GROUPER_AUDIT_ENTRY}"
+        rs   = stmt.execute_query "SELECT MIN(created_on) FROM #{GROUPER_AUDIT_ENTRY} ORDER BY created_on"
         while rs.next
           # Microseconds -> Seconds -> Time -> Date
-          d = Time.at( microseconds_to_seconds( rs.float(1) ) ).to_date
+          d = Time.at( microseconds_to_seconds( rs.get_object(1).to_s.to_f ) ).to_date
           break
         end
         rs.close
@@ -259,28 +277,23 @@ module JGrouper # :nodoc:
       end
 
       #
-      # Prune entries from database.
+      # Prune specified entries from database.
       #
-      def prune(expected, start_at, stop_at)
-        log "pruning #{ @date } (#{expected} entries) ..."
+      def prune(entries, num_entries, total_entries)
+        log "pruning #{ @date } (#{entries.size} entries, #{num_entries}/#{total_entries}) ..."
 
-        found = num_entries start_at, stop_at
-        if expected == found
-
-          @conn.auto_commit = false
-          qry  = "DELETE FROM #{GROUPER_AUDIT_ENTRY} WHERE created_on BETWEEN #{start_at} AND #{stop_at}" 
-          stmt = @conn.create_statement
-          rv   = stmt.execute_update qry
-          if expected == rv
-            @conn.commit
-            log "pruning #{ @date } (#{rv} entries) - done"
-          end
-          stmt.close
-          unless expected == rv
-            raise "ERROR: not committing delete as number of deleted entries does not match expected number - #{rv} != #{expected}"
-          end
-        else
-          raise "ERROR: not deleting as number of found entries does not match expected number - #{found} != #{expected}"
+        @conn.auto_commit = false
+        qry  = "DELETE FROM #{GROUPER_AUDIT_ENTRY} WHERE id IN ( " + entries.map { |_| "'#{_}'" }.join(',') + " )"
+        stmt = @conn.create_statement
+        rv   = stmt.execute_update qry
+        if entries.size == rv
+          @conn.commit
+          log "pruning #{ @date } (#{rv} entries, #{num_entries}/#{total_entries}) - done"
+        end
+        stmt.close
+        unless entries.size == rv
+          log "ERROR: #{ entries.size } != #{rv} for query: #{qry}"
+          raise "ERROR: not committing delete as number of deleted entries does not match expected number - #{rv} != #{expected}"
         end
       end
 

data/lib/jgrouper/version.rb

@@ -1,4 +1,4 @@
 module JGrouper
-  VERSION = '0.3.0'
+  VERSION = '0.4.0'
 end
 

data/lib/jgrouper.rb

@@ -17,10 +17,26 @@ require 'jgrouper/version'
 #   require 'jgrouper'
 #   JGrouper.home! '/path/to/your/grouper/api/installation/directory'
 #
+# ==== Groups
+#
+#   # Find Group by name
+#   g = JGrouper::Group.find $name
+#
+#   # Grant privileges
+#   g.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('admin')
+#
 # ==== Members
 #
 #   # Find Member by UUID
-#   m = JGrouper::Member.find uuid
+#   m = JGrouper::Member.find $uuid
+#
+# ==== Privileges
+#
+#   # Find Privilege by name
+#   p = JGrouper::Privilege.find 'admin'
+#   p.access? # True
+#   p.naming? # False
+#   p.type    # :access
 #
 # ==== Stems
 #
@@ -34,7 +50,15 @@ require 'jgrouper/version'
 #   root.groups
 #
 #   # Find stem by name
-#   stem = JGrouper::Stem.find name
+#   stem = JGrouper::Stem.find $name
+#
+#   # Grant privileges
+#   stem.grant JGrouper::Subject.find($subj), JGrouper::Privilege.find('stem')
+#
+# ==== Subjects
+#
+#   # Find by id-or-identifier
+#   subj = JGrouper::Subject.find $subj
 #
 # ==== Archiver
 #

metadata

@@ -1,18 +1,36 @@
 --- !ruby/object:Gem::Specification
 name: jgrouper
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
+  prerelease: 
 platform: ruby
 authors:
 - blair christensen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-27 00:00:00.000000000 Z
+date: 2013-07-05 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: jruby-jars
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.7.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.7.3
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -20,6 +38,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -27,20 +46,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rdoc-readme
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -48,6 +70,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,25 +109,26 @@ files:
 homepage: https://github.com/blairc/jgrouper/
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.2
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: JRuby wrapper around the Internet2 Grouper API
 test_files: []

checksums.yaml

@@ -1,7 +0,0 @@
----
-SHA1:
-  metadata.gz: 1d34d0e3926d5850e95ca673912825fae7f2d4d7
-  data.tar.gz: ef0be60e432fcddbf3245cc20988aa5fe2f48c86
-SHA512:
-  metadata.gz: 8f05bcf26ee9937fbcb84390de299457cff24f82071308d78c0ab2c600c1c4c0a2ceca64b154f861b7e2d4633ef3a63096461b0299c9ee2e04aa8a490a0cfee3
-  data.tar.gz: dc8d6104cb439831e5550c7829b386044d0c1aebf48db5ee7594627302510637b3700097b36563511284d14e4ae0c916d64bad7c4299e11f75047d350fc9bf90