workflow-archiver 1.3.2 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a0db4761d9140127396c162359225c3facbdff59
-  data.tar.gz: f87202e1789fdb753487445d382c98cb29e8a870
+  metadata.gz: 2e07be8143cac430dfa7166ad246e81d64c85558
+  data.tar.gz: 23612c1949726b163ad89534727679fbb6da3968
 SHA512:
-  metadata.gz: c41b25566683e1dc64120ace58c89db480de9ec57a87f7d58751362c012c7f02279c7de75bfcd62418970086bfb1718dc027b9e3b6fc023dec951a9fecceec2b
-  data.tar.gz: 1dda723f38294bc01ed7b8d67beb61b0b08943a5d516de9e19924cce26dd78ff116d9175bf2b3b5f71634f8bcf143350fa76867031e0e2f946c5dfa0180e6365
+  metadata.gz: 779182bd0fd4cdabb6c4480aa7ceb5aa102bfe33c60a00317ea2b98ec4d8e62e2caef69c4cdc784ccd5cea3e91960fe2d077b076bccdb47c02c73af796755fd6
+  data.tar.gz: 679c197f47b3c409a0f61f3350eae177bfc230e9ae9b9723b090a328f2a87be711c4b26cafbb7680ce280f2b42a45d1e4ed708db48be7f1e1f4aaed9641ac0c0

data/VERSION

@@ -1 +1 @@
-1.3.2
+2.0.0

data/lib/dor/archiver_version.rb

@@ -3,9 +3,6 @@ module Dor
     def self.archiver_version
       @archiver_version ||= File.read(File.join(File.dirname(__FILE__), '..', '..', 'VERSION')).chomp
     end
-
-   ARCHIVER_VERSION = self.archiver_version
-   end
-
-
+    ARCHIVER_VERSION = archiver_version
+  end
 end

data/lib/dor/workflow_archiver.rb

@@ -1,18 +1,16 @@
 require 'rest_client'
 require 'confstruct'
 require 'lyber_core'
-require 'oci8'
+require 'sequel'
 
 module Dor
-
   # Holds the paramaters about the workflow rows that need to be deleted
   ArchiveCriteria = Struct.new(:repository, :druid, :datastream, :version) do
     # @param [Array<Hash>] List of objects returned from {WorkflowArchiver#find_completed_objects}.  It expects the following keys in the hash
-    #  "REPOSITORY", "DRUID", "DATASTREAM".  Note they are all caps strings, not symbols
     def setup_from_query(row_hash)
-      self.repository = row_hash["REPOSITORY"]
-      self.druid = row_hash["DRUID"]
-      self.datastream = row_hash["DATASTREAM"]
+      self.repository = row_hash[:repository]
+      self.druid = row_hash[:druid]
+      self.datastream = row_hash[:datastream]
       set_current_version
       self
     end
@@ -21,29 +19,27 @@ module Dor
     # @return [Hash] Maps column names (in ALL caps) to non-nil column values
     def to_bind_hash
       h = {}
-      members.reject{|mem| mem =~ /version/}.each do |m|
-        h[m.swapcase] = self.send(m) if(self.send(m))
+      members.reject { |mem| mem =~ /version/ }.each do |m|
+        h[m] = send(m) if send(m)
       end
       h
     end
 
     def set_current_version
-      begin
-        self.version = RestClient.get WorkflowArchiver.config.dor_service_uri + "/dor/v1/objects/#{self.druid}/versions/current"
-      rescue RestClient::InternalServerError => ise
-        raise unless(ise.inspect =~ /Unable to find.*in fedora/)
-        LyberCore::Log.warn "#{ise.inspect}"
-        LyberCore::Log.warn "Moving workflow rows with version set to '1'"
-        self.version = '1'
-      end
+      self.version = RestClient.get WorkflowArchiver.config.dor_service_uri + "/dor/v1/objects/#{druid}/versions/current"
+    rescue RestClient::InternalServerError => ise
+      raise unless ise.inspect =~ /Unable to find.*in fedora/
+      LyberCore::Log.warn ise.inspect.to_s
+      LyberCore::Log.warn "Moving workflow rows with version set to '1'"
+      self.version = '1'
     end
   end
 
   class WorkflowArchiver
-    WF_COLUMNS = %w(ID DRUID DATASTREAM PROCESS STATUS ERROR_MSG ERROR_TXT DATETIME ATTEMPTS LIFECYCLE ELAPSED REPOSITORY NOTE PRIORITY LANE_ID)
+    WF_COLUMNS = %w(id druid datastream process status error_msg error_txt datetime attempts lifecycle elapsed repository note priority lane_id)
 
     # These attributes mostly used for testing
-    attr_reader :conn, :errors
+    attr_reader :errors
 
     def self.config
       @@conf ||= Confstruct::Configuration.new
@@ -52,63 +48,33 @@ module Dor
     # Sets up logging and connects to the database.  By default it reads values from constants:
     #  WORKFLOW_DB_LOGIN, WORKFLOW_DB_PASSWORD, WORKFLOW_DB_URI, DOR_SERVICE_URI but can be overriden with the opts Hash
     # @param [Hash] opts Options to override database parameters
-    # @option opts [String] :login ('WORKFLOW_DB_LOGIN') Database login id
-    # @option opts [String] :password ('WORKFLOW_DB_PASSWORD') Database password
     # @option opts [String] :db_uri ('WORKFLOW_DB_URI') Database uri
     # @option opts [String] :wf_table ('workflow') Name of the active workflow table
     # @option opts [String] :wfa_table ('workflow_archive') Name of the workflow archive table
-    # @option opts [String] :dor_service_uri ('DOR_SERVICE_URI') URI of the DOR Rest service
     # @option opts [Integer] :retry_delay (5) Number of seconds to sleep between retries of database operations
-    def initialize(opts={})
-      @login = (opts.include?(:login) ? opts[:login] : WorkflowArchiver.config.db_login)
-      @password = (opts.include?(:password) ? opts[:password] : WorkflowArchiver.config.db_password)
-      @db_uri = (opts.include?(:db_uri) ? opts[:db_uri] : WorkflowArchiver.config.db_uri)
-      @dor_service_uri = (opts.include?(:dor_service_uri) ? opts[:dor_service_uri] : WorkflowArchiver.config.dor_service_uri)
-      @workflow_table = (opts.include?(:wf_table) ? opts[:wf_table] : "workflow")
-      @workflow_archive_table = (opts.include?(:wfa_table) ? opts[:wfa_table] : "workflow_archive")
-      @retry_delay = (opts.include?(:retry_delay) ? opts[:retry_delay] : 5)
-
+    def initialize(opts = {})
+      @conn = opts[:db_connection]
+      @db_uri                 = opts.fetch(:db_uri, WorkflowArchiver.config.db_uri).freeze
+      @workflow_table         = opts.include?(:wf_table)    ? opts[:wf_table]    : 'workflow'
+      @workflow_archive_table = opts.include?(:wfa_table)   ? opts[:wfa_table]   : 'workflow_archive'
+      @retry_delay            = opts.include?(:retry_delay) ? opts[:retry_delay] : 5
       # initialize some counters
       @errors = 0
       @archived = 0
     end
 
-    def connect_to_db
-      $odb_pool ||= OCI8::ConnectionPool.new(1, 5, 2, @login, @password, @db_uri)
-      @conn = OCI8.new(@login, @password, $odb_pool)
-      @conn.autocommit = false
+    def conn
+      @conn ||= Sequel.connect(@db_uri)
     end
 
-    def destroy_pool
-      $odb_pool.destroy if($odb_pool)
-    end
-
-    def bind_and_exec_sql(sql, workflow_info)
-      # LyberCore::Log.debug("Executing: #{sql}")
-      cursor = @conn.parse(sql)
-
-      workflow_info.to_bind_hash.each do |k, v|
-        param = ":#{k}"
-        #LyberCore::Log.debug("Setting: #{param} #{v}")
-        cursor.bind_param(param, v)
-      end
-
-      num_rows = cursor.exec
-      unless num_rows > 0
-        raise "Expected more than 0 rows to be updated"
-      end
-    ensure
-      cursor.close
-    end
-
-    # @return String The columns appended with comma and newline
+    # @return [String] The columns appended with comma and newline
     def wf_column_string
-      WF_COLUMNS.inject('') { |str, col| str << col << ",\n"}
+      WF_COLUMNS.join(",\n")
     end
 
-    # @return String The columns prepended with 'w.' and appended with comma and newline
+    # @return [String] The columns prepended with 'w.' and appended with comma and newline
     def wf_archive_column_string
-      WF_COLUMNS.inject('') { |str, col| str << 'w.' << col << ",\n"}
+      WF_COLUMNS.map { |col| "#{@workflow_table}.#{col}" }.join(",\n")
     end
 
     # Use this as a one-shot method to archive all the steps of an object's particular datastream
@@ -120,17 +86,14 @@ module Dor
     # @param [String] version
     def archive_one_datastream(repository, druid, datastream, version)
       criteria = [ArchiveCriteria.new(repository, druid, datastream, version)]
-      connect_to_db
       archive_rows criteria
-    ensure
-      @conn.logoff if(@conn)
     end
 
     # Copies rows from the workflow table to the workflow_archive table, then deletes the rows from workflow
     # Both operations must complete, or they get rolled back
     # @param [Array<ArchiveCriteria>] objs List of objects returned from {#find_completed_objects} and mapped to an array of ArchiveCriteria objects.
     def archive_rows(objs)
-      Array(objs).each do |obj|
+      objs.each do |obj|
         tries = 0
         begin
           tries += 1
@@ -138,11 +101,8 @@ module Dor
           @archived += 1
         rescue => e
           LyberCore::Log.error "Rolling back transaction due to: #{e.inspect}\n" << e.backtrace.join("\n") << "\n!!!!!!!!!!!!!!!!!!"
-          @conn.rollback
-
-          # Retry this druid up to 3 times
-          if tries < 3
-            LyberCore::Log.error "  Retrying archive operation in #{@retry_delay.to_s} seconds..."
+          if tries < 3 # Retry this druid up to 3 times
+            LyberCore::Log.error "  Retrying archive operation in #{@retry_delay} seconds..."
             sleep @retry_delay
             retry
           end
@@ -150,55 +110,55 @@ module Dor
 
           @errors += 1
           if @errors >= 3
-            LyberCore::Log.fatal("Too many errors. Archiving halted")
+            LyberCore::Log.fatal('Too many errors. Archiving halted')
             break
           end
         end
-
       end # druids.each
     end
 
     # @param [ArchiveCriteria] workflow_info contains paramaters on the workflow rows to archive
     def do_one_archive(workflow_info)
       LyberCore::Log.info "Archiving #{workflow_info.inspect}"
-
-
-      copy_sql =<<-EOSQL
+      copy_sql = <<-EOSQL
         insert into #{@workflow_archive_table} (
-          #{wf_column_string}
-          VERSION
+          #{wf_column_string},
+          version
         )
         select
-          #{wf_archive_column_string}
-          #{workflow_info.version} as VERSION
-        from #{@workflow_table} w
-        where w.druid =    :DRUID
-        and w.datastream = :DATASTREAM
+          #{wf_archive_column_string},
+          #{workflow_info.version} as version
+        from #{@workflow_table}
+        where #{@workflow_table}.druid =    :druid
+        and #{@workflow_table}.datastream = :datastream
       EOSQL
 
-      delete_sql = "delete #{@workflow_table} where druid = :DRUID and datastream = :DATASTREAM "
+      delete_sql = "delete from #{@workflow_table} where druid = :druid and datastream = :datastream "
 
       if(workflow_info.repository)
-        copy_sql << "and w.repository = :REPOSITORY"
-        delete_sql << "and repository = :REPOSITORY"
+        copy_sql += "and #{@workflow_table}.repository = :repository"
+        delete_sql += 'and repository = :repository'
       else
-        copy_sql << "and w.repository IS NULL"
-        delete_sql << "and repository IS NULL"
+        copy_sql += "and #{@workflow_table}.repository IS NULL"
+        delete_sql += 'and repository IS NULL'
       end
 
-      bind_and_exec_sql(copy_sql, workflow_info)
+      conn.transaction do
+        conn.run Sequel::SQL::PlaceholderLiteralString.new(copy_sql, workflow_info.to_bind_hash)
 
-      LyberCore::Log.debug "  Removing old workflow rows"
-      bind_and_exec_sql(delete_sql, workflow_info)
+        LyberCore::Log.debug '  Removing old workflow rows'
 
-      @conn.commit
+        conn.run Sequel::SQL::PlaceholderLiteralString.new(delete_sql, workflow_info.to_bind_hash)
+      end
     end
 
     # Finds objects where all workflow steps are complete
-    # Returns an array of hashes, each hash having the following keys:
-    # {"REPOSITORY"=>"dor", "DRUID"=>"druid:345", "DATASTREAM"=>"googleScannedBookWF"}
+    # @return [Array<Hash{String=>String}>] each hash returned has the following keys:
+    #   {"REPOSITORY"=>"dor", "DRUID"=>"druid:345", "DATASTREAM"=>"googleScannedBookWF"}
     def find_completed_objects
-      completed_query =<<-EOSQL
+      return to_enum(:find_completed_objects) unless block_given?
+
+      completed_query = <<-EOSQL
        select distinct repository, datastream, druid
        from workflow w1
        where w1.status in ('completed', 'skipped')
@@ -213,18 +173,15 @@ module Dor
        )
       EOSQL
 
-      rows = []
-      cursor = @conn.exec(completed_query)
-      while r = cursor.fetch_hash
-        rows << r
+      conn.fetch(completed_query) do |row|
+        yield row
       end
-      rows
     end
 
     # @param [Array<Hash>] rows result from #find_completed_objects
     # @return [Array<ArchiveCriteria>] each result mapped to an ArchiveCriteria object
     def map_result_to_criteria(rows)
-      criteria = rows.map do |r|
+      rows.lazy.map do |r|
         begin
           ArchiveCriteria.new.setup_from_query(r)
         rescue => e
@@ -232,47 +189,22 @@ module Dor
           LyberCore::Log.error("#{e.inspect}\n" + e.backtrace.join("\n"))
           nil
         end
-      end
-      criteria.reject {|c| c.nil?}
-    end
-
-    def simple_sql_exec(sql)
-      @conn.exec(sql)
-    rescue Exception => e
-      LyberCore::Log.warn "Ignoring error: #{e.message}\n  while trying to execute: " << sql
-    end
-
-    def with_indexing_disabled(&block)
-      simple_sql_exec("drop index ds_wf_ar_bitmap_idx")
-      simple_sql_exec("drop index repo_wf_ar_bitmap_idx")
-      yield
-    ensure
-      simple_sql_exec("create bitmap index ds_wf_ar_bitmap_idx on workflow_archive (datastream)")
-      simple_sql_exec("create bitmap index repo_wf_ar_bitmap_idx on workflow_archive (repository)")
+      end.reject { |r| r.nil? }
     end
 
     # Does the work of finding completed objects and archiving the rows
     def archive
-      connect_to_db
       objs = find_completed_objects
 
-      if objs.size == 0
-        LyberCore::Log.info "Nothing to archive"
+      if objs.none?
+        LyberCore::Log.info 'Nothing to archive'
         exit true
       end
-
-      LyberCore::Log.info "Found #{objs.size.to_s} completed workflows"
-      objs = objs.first(600) # FIXME: temporarily limit objs processed until we fix cron not to fire if job still running
-
+      LyberCore::Log.info "Found #{objs.size} completed workflows"
       archiving_criteria = map_result_to_criteria(objs)
-      with_indexing_disabled { archive_rows(archiving_criteria) }
+      archive_rows(archiving_criteria)
 
-      LyberCore::Log.info "DONE! Processed #{@archived.to_s} objects with #{@errors.to_s} errors" if(@errors < 3 )
-    ensure
-      @conn.logoff
-      destroy_pool
+      LyberCore::Log.info "DONE! Processed #{@archived.to_s} objects with #{@errors.to_s} errors" if @errors < 3
     end
-
   end
-
 end

data/lib/workflow-archiver.rb

@@ -0,0 +1 @@
+require 'dor/workflow_archiver'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: workflow-archiver
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 2.0.0
 platform: ruby
 authors:
 - Willy Mene
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-30 00:00:00.000000000 Z
+date: 2016-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: lyber-core
@@ -39,7 +39,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-oci8
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -95,21 +95,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: coveralls
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: active-fedora
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -132,6 +118,7 @@ files:
 - VERSION
 - lib/dor/archiver_version.rb
 - lib/dor/workflow_archiver.rb
+- lib/workflow-archiver.rb
 homepage: 
 licenses: []
 metadata: {}
@@ -151,7 +138,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Enables archiving of DOR workflows